Support -fsanitize=address with --enable-asan

When --enable-asan is provided to configure then build all user
space components with fsanitize=address.  For kernel support
use the Linux KASAN feature instead.

https://github.com/google/sanitizers/wiki/AddressSanitizer

When using gcc version 4.8 any test case which intentionally
generates a core dump will fail when using --enable-asan.
The default behavior is to disable core dumps and only newer
versions allow this behavior to be controled at run time with
the ASAN_OPTIONS environment variable.

Additionally, this patch includes some build system cleanup.

* Rules.am updated to set the minimum AM_CFLAGS, AM_CPPFLAGS,
  and AM_LDFLAGS.  Any additional flags should be added on a
  per-Makefile basic.  The --enable-debug and --enable-asan
  options apply to all user space binaries and libraries.

* Compiler checks consolidated in always-compiler-options.m4
  and renamed for consistency.

* -fstack-check compiler flag was removed, this functionality
  is provided by asan when configured with --enable-asan.

* Split DEBUG_CFLAGS in to DEBUG_CFLAGS, DEBUG_CPPFLAGS, and
  DEBUG_LDFLAGS.

* Moved default kernel build flags in to module/Makefile.in and
  split in to ZFS_MODULE_CFLAGS and ZFS_MODULE_CPPFLAGS.  These
  flags are set with the standard ccflags-y kbuild mechanism.

* -Wframe-larger-than checks applied only to binaries or
  libraries which include source files which are built in
  both user space and kernel space.  This restriction is
  relaxed for user space only utilities.

* -Wno-unused-but-set-variable applied only to libzfs and
  libzpool.  The remaining warnings are the result of an
  ASSERT using a variable when is always declared.

* -D_POSIX_PTHREAD_SEMANTICS and -D__EXTENSIONS__ dropped
  because they are Solaris specific and thus not needed.

* Ensure $GDB is defined as gdb by default in zloop.sh.

Signed-off-by: DHE <[email protected]>
Signed-off-by: Brian Behlendorf <[email protected]>
Closes #7027 

12 files changed, 25 insertions, 42 deletions

diff --git a/lib/libavl/Makefile.am b/lib/libavl/Makefile.am
index 6a42649b8..82b30bd80 100644
--- a/lib/libavl/Makefile.am
+++ b/lib/libavl/Makefile.am
@@ -2,7 +2,8 @@ include $(top_srcdir)/config/Rules.am
 
 VPATH = $(top_srcdir)/module/avl/
 
-AM_CFLAGS += $(DEBUG_STACKFLAGS) $(FRAME_LARGER_THAN)
+# Includes kernel code, generate warnings for large stack frames
+AM_CFLAGS += $(FRAME_LARGER_THAN)
 
 DEFAULT_INCLUDES += \
 	-I$(top_srcdir)/include \
diff --git a/lib/libefi/Makefile.am b/lib/libefi/Makefile.am
index f0c05ee6c..9f69e4601 100644
--- a/lib/libefi/Makefile.am
+++ b/lib/libefi/Makefile.am
@@ -1,7 +1,5 @@
 include $(top_srcdir)/config/Rules.am
 
-AM_CFLAGS += $(DEBUG_STACKFLAGS) $(FRAME_LARGER_THAN)
-
 DEFAULT_INCLUDES += \
 	-I$(top_srcdir)/include \
 	-I$(top_srcdir)/lib/libspl/include
@@ -11,11 +9,7 @@ noinst_LTLIBRARIES = libefi.la
 USER_C = \
 	rdwr_efi.c
 
-KERNEL_C =
-
-nodist_libefi_la_SOURCES = \
-	$(USER_C) \
-	$(KERNEL_C)
+nodist_libefi_la_SOURCES = $(USER_C)
 
 libefi_la_LIBADD = $(LIBUUID)
 
diff --git a/lib/libicp/Makefile.am b/lib/libicp/Makefile.am
index e1f08c8dd..d04a99e94 100644
--- a/lib/libicp/Makefile.am
+++ b/lib/libicp/Makefile.am
@@ -4,7 +4,8 @@ VPATH = \
 	$(top_srcdir)/module/icp \
 	$(top_srcdir)/lib/libicp
 
-AM_CFLAGS += $(DEBUG_STACKFLAGS) $(FRAME_LARGER_THAN)
+# Includes kernel code, generate warnings for large stack frames
+AM_CFLAGS += $(FRAME_LARGER_THAN)
 
 DEFAULT_INCLUDES += \
 	-I$(top_srcdir)/include \
diff --git a/lib/libnvpair/Makefile.am b/lib/libnvpair/Makefile.am
index 6da679fd5..d6ba6f89a 100644
--- a/lib/libnvpair/Makefile.am
+++ b/lib/libnvpair/Makefile.am
@@ -4,7 +4,11 @@ VPATH = \
 	$(top_srcdir)/module/nvpair \
 	$(top_srcdir)/lib/libnvpair
 
-AM_CFLAGS += $(DEBUG_STACKFLAGS) $(FRAME_LARGER_THAN) $(LIBTIRPC_CFLAGS)
+# Required CFLAGS for libtirpc
+AM_CFLAGS += $(LIBTIRPC_CFLAGS)
+
+# Includes kernel code, generate warnings for large stack frames
+AM_CFLAGS += $(FRAME_LARGER_THAN)
 
 DEFAULT_INCLUDES += \
 	-I$(top_srcdir)/include \
diff --git a/lib/libshare/Makefile.am b/lib/libshare/Makefile.am
index 42bd207be..462e333ff 100644
--- a/lib/libshare/Makefile.am
+++ b/lib/libshare/Makefile.am
@@ -14,10 +14,6 @@ USER_C = \
 	smb.c \
 	smb.h
 
-KERNEL_C =
-
-nodist_libshare_la_SOURCES = \
-	$(USER_C)
-	$(KERNEL_C)
+nodist_libshare_la_SOURCES = $(USER_C)
 
 EXTRA_DIST = $(USER_C)
diff --git a/lib/libspl/Makefile.am b/lib/libspl/Makefile.am
index 395723af3..e4512700a 100644
--- a/lib/libspl/Makefile.am
+++ b/lib/libspl/Makefile.am
@@ -4,8 +4,6 @@ VPATH = \
 	$(top_srcdir)/lib/libspl \
 	$(top_srcdir)/lib/libspl/$(TARGET_ASM_DIR)
 
-AM_CFLAGS += $(DEBUG_STACKFLAGS) $(FRAME_LARGER_THAN)
-
 SUBDIRS = include $(TARGET_ASM_DIR)
 DIST_SUBDIRS = include asm-generic asm-i386 asm-x86_64
 
@@ -36,12 +34,9 @@ USER_C = \
 
 USER_ASM = atomic.S
 
-KERNEL_C =
-
 nodist_libspl_la_SOURCES = \
 	$(USER_C) \
-	$(USER_ASM) \
-	$(KERNEL_C)
+	$(USER_ASM)
 
 libspl_la_LIBADD = -lrt
 
diff --git a/lib/libtpool/Makefile.am b/lib/libtpool/Makefile.am
index adbaee6c3..586eec2ec 100644
--- a/lib/libtpool/Makefile.am
+++ b/lib/libtpool/Makefile.am
@@ -1,7 +1,5 @@
 include $(top_srcdir)/config/Rules.am
 
-AM_CFLAGS += $(DEBUG_STACKFLAGS)
-
 DEFAULT_INCLUDES += \
 	-I$(top_srcdir)/include \
 	-I$(top_srcdir)/lib/libspl/include
@@ -12,11 +10,7 @@ USER_C = \
 	thread_pool.c \
 	thread_pool_impl.h
 
-KERNEL_C =
-
-nodist_libtpool_la_SOURCES = \
-	$(USER_C) \
-	$(KERNEL_C)
+nodist_libtpool_la_SOURCES = $(USER_C)
 
 libtpool_la_LIBADD = \
 	$(top_builddir)/lib/libspl/libspl.la
diff --git a/lib/libunicode/Makefile.am b/lib/libunicode/Makefile.am
index 9bacae251..0a4734c03 100644
--- a/lib/libunicode/Makefile.am
+++ b/lib/libunicode/Makefile.am
@@ -2,7 +2,8 @@ include $(top_srcdir)/config/Rules.am
 
 VPATH = $(top_srcdir)/module/unicode
 
-AM_CFLAGS += $(DEBUG_STACKFLAGS) $(FRAME_LARGER_THAN)
+# Includes kernel code, generate warnings for large stack frames
+AM_CFLAGS += $(FRAME_LARGER_THAN)
 
 DEFAULT_INCLUDES += \
 	-I$(top_srcdir)/include \
diff --git a/lib/libuutil/Makefile.am b/lib/libuutil/Makefile.am
index ed4a30de7..09eef792a 100644
--- a/lib/libuutil/Makefile.am
+++ b/lib/libuutil/Makefile.am
@@ -1,7 +1,5 @@
 include $(top_srcdir)/config/Rules.am
 
-AM_CFLAGS += $(DEBUG_STACKFLAGS) $(FRAME_LARGER_THAN)
-
 DEFAULT_INCLUDES += \
 	-I$(top_srcdir)/include \
 	-I$(top_srcdir)/lib/libspl/include
@@ -19,11 +17,7 @@ USER_C = \
 	uu_pname.c \
 	uu_string.c
 
-KERNEL_C =
-
-nodist_libuutil_la_SOURCES = \
-	$(USER_C) \
-	$(KERNEL_C)
+nodist_libuutil_la_SOURCES = $(USER_C)
 
 libuutil_la_LIBADD = \
 	$(top_builddir)/lib/libavl/libavl.la \
diff --git a/lib/libzfs/Makefile.am b/lib/libzfs/Makefile.am
index e5b2ce765..da40c96ce 100644
--- a/lib/libzfs/Makefile.am
+++ b/lib/libzfs/Makefile.am
@@ -5,6 +5,9 @@ VPATH = \
 	$(top_srcdir)/module/zcommon \
 	$(top_srcdir)/lib/libzfs
 
+# Suppress unused but set variable warnings often due to ASSERTs
+AM_CFLAGS += $(NO_UNUSED_BUT_SET_VARIABLE)
+
 libzfs_pcdir = $(datarootdir)/pkgconfig
 libzfs_pc_DATA = libzfs.pc libzfs_core.pc
 
diff --git a/lib/libzfs_core/Makefile.am b/lib/libzfs_core/Makefile.am
index 5eafc25c0..8abaebe73 100644
--- a/lib/libzfs_core/Makefile.am
+++ b/lib/libzfs_core/Makefile.am
@@ -9,11 +9,7 @@ lib_LTLIBRARIES = libzfs_core.la
 USER_C = \
 	libzfs_core.c
 
-KERNEL_C =
-
-nodist_libzfs_core_la_SOURCES = \
-	$(USER_C) \
-	$(KERNEL_C)
+nodist_libzfs_core_la_SOURCES = $(USER_C)
 
 libzfs_core_la_LIBADD = \
 	$(top_builddir)/lib/libnvpair/libnvpair.la
diff --git a/lib/libzpool/Makefile.am b/lib/libzpool/Makefile.am
index 95e249342..06219168b 100644
--- a/lib/libzpool/Makefile.am
+++ b/lib/libzpool/Makefile.am
@@ -5,7 +5,11 @@ VPATH = \
 	$(top_srcdir)/module/zcommon \
 	$(top_srcdir)/lib/libzpool
 
-AM_CFLAGS += $(DEBUG_STACKFLAGS) $(FRAME_LARGER_THAN)
+# Suppress unused but set variable warnings often due to ASSERTs
+AM_CFLAGS += $(NO_UNUSED_BUT_SET_VARIABLE)
+
+# Includes kernel code generate warnings for large stack frames
+AM_CFLAGS += $(FRAME_LARGER_THAN)
 
 DEFAULT_INCLUDES += \
 	-I$(top_srcdir)/include \