Support -fsanitize=address with --enable-asan

When --enable-asan is provided to configure then build all user
space components with fsanitize=address.  For kernel support
use the Linux KASAN feature instead.

https://github.com/google/sanitizers/wiki/AddressSanitizer

When using gcc version 4.8 any test case which intentionally
generates a core dump will fail when using --enable-asan.
The default behavior is to disable core dumps and only newer
versions allow this behavior to be controled at run time with
the ASAN_OPTIONS environment variable.

Additionally, this patch includes some build system cleanup.

* Rules.am updated to set the minimum AM_CFLAGS, AM_CPPFLAGS,
  and AM_LDFLAGS.  Any additional flags should be added on a
  per-Makefile basic.  The --enable-debug and --enable-asan
  options apply to all user space binaries and libraries.

* Compiler checks consolidated in always-compiler-options.m4
  and renamed for consistency.

* -fstack-check compiler flag was removed, this functionality
  is provided by asan when configured with --enable-asan.

* Split DEBUG_CFLAGS in to DEBUG_CFLAGS, DEBUG_CPPFLAGS, and
  DEBUG_LDFLAGS.

* Moved default kernel build flags in to module/Makefile.in and
  split in to ZFS_MODULE_CFLAGS and ZFS_MODULE_CPPFLAGS.  These
  flags are set with the standard ccflags-y kbuild mechanism.

* -Wframe-larger-than checks applied only to binaries or
  libraries which include source files which are built in
  both user space and kernel space.  This restriction is
  relaxed for user space only utilities.

* -Wno-unused-but-set-variable applied only to libzfs and
  libzpool.  The remaining warnings are the result of an
  ASSERT using a variable when is always declared.

* -D_POSIX_PTHREAD_SEMANTICS and -D__EXTENSIONS__ dropped
  because they are Solaris specific and thus not needed.

* Ensure $GDB is defined as gdb by default in zloop.sh.

Signed-off-by: DHE <[email protected]>
Signed-off-by: Brian Behlendorf <[email protected]>
Closes #7027 

7 files changed, 27 insertions, 20 deletions

diff --git a/module/Makefile.in b/module/Makefile.in
index 60973ec7d..1ca979f9d 100644
--- a/module/Makefile.in
+++ b/module/Makefile.in
@@ -1,18 +1,25 @@
 subdir-m += avl
+subdir-m += icp
 subdir-m += nvpair
 subdir-m += unicode
 subdir-m += zcommon
 subdir-m += zfs
-subdir-m += icp
 
 INSTALL_MOD_DIR ?= extra
 
+ZFS_MODULE_CFLAGS += -std=gnu99 -Wno-declaration-after-statement
+ZFS_MODULE_CFLAGS += @KERNEL_DEBUG_CFLAGS@
 ZFS_MODULE_CFLAGS += -include @SPL_OBJ@/spl_config.h
 ZFS_MODULE_CFLAGS += -include @abs_top_builddir@/zfs_config.h
 ZFS_MODULE_CFLAGS += -I@abs_top_srcdir@/include -I@SPL@/include -I@SPL@
+
+ZFS_MODULE_CPPFLAGS += -DHAVE_SPL -D_KERNEL
+ZFS_MODULE_CPPFLAGS += @KERNEL_DEBUG_CPPFLAGS@
+
 @CONFIG_QAT_TRUE@ZFS_MODULE_CFLAGS += -I@QAT_SRC@/include
 @CONFIG_QAT_TRUE@KBUILD_EXTRA_SYMBOLS += @QAT_SYMBOLS@
-export ZFS_MODULE_CFLAGS
+
+export ZFS_MODULE_CFLAGS ZFS_MODULE_CPPFLAGS
 
 SUBDIR_TARGETS = icp
 
@@ -35,12 +42,12 @@ modules:
 	list='$(SUBDIR_TARGETS)'; for targetdir in $$list; do \
 		$(MAKE) -C $$targetdir; \
 	done
-	$(MAKE) -C @LINUX_OBJ@ SUBDIRS=`pwd` @KERNELMAKE_PARAMS@ CONFIG_ZFS=m $@
+	$(MAKE) -C @LINUX_OBJ@ SUBDIRS=`pwd` @KERNEL_MAKE@ CONFIG_ZFS=m $@
 
 clean:
 	@# Only cleanup the kernel build directories when CONFIG_KERNEL
 	@# is defined.  This indicates that kernel modules should be built.
-@CONFIG_KERNEL_TRUE@	$(MAKE) -C @LINUX_OBJ@ SUBDIRS=`pwd` @KERNELMAKE_PARAMS@ $@
+@CONFIG_KERNEL_TRUE@	$(MAKE) -C @LINUX_OBJ@ SUBDIRS=`pwd` @KERNEL_MAKE@ $@
 
 	if [ -f @SPL_SYMBOLS@ ]; then $(RM) @SPL_SYMBOLS@; fi
 	if [ -f @LINUX_SYMBOLS@ ]; then $(RM) @LINUX_SYMBOLS@; fi
diff --git a/module/avl/Makefile.in b/module/avl/Makefile.in
index 98c011e8a..217fa3ca5 100644
--- a/module/avl/Makefile.in
+++ b/module/avl/Makefile.in
@@ -3,8 +3,8 @@ obj = @abs_builddir@
 
 MODULE := zavl
 
-EXTRA_CFLAGS = $(ZFS_MODULE_CFLAGS) @KERNELCPPFLAGS@
-
 obj-$(CONFIG_ZFS) := $(MODULE).o
 
+ccflags-y := $(ZFS_MODULE_CFLAGS) $(ZFS_MODULE_CPPFLAGS)
+
 $(MODULE)-objs += avl.o
diff --git a/module/icp/Makefile.in b/module/icp/Makefile.in
index 77b2ec1b5..2eb9e6f1f 100644
--- a/module/icp/Makefile.in
+++ b/module/icp/Makefile.in
@@ -23,14 +23,11 @@ ifeq ($(TARGET_ASM_DIR), asm-generic)
 ASM_SOURCES :=
 endif
 
-EXTRA_CFLAGS = $(ZFS_MODULE_CFLAGS) @KERNELCPPFLAGS@
-EXTRA_AFLAGS = $(ZFS_MODULE_CFLAGS) @KERNELCPPFLAGS@
-
 obj-$(CONFIG_ZFS) := $(MODULE).o
 
-ccflags-y += -I$(src)/include
-asflags-y += -I$(src)/include
-asflags-y += $(ZFS_MODULE_CFLAGS)
+asflags-y := -I$(src)/include
+ccflags-y := -I$(src)/include
+ccflags-y += $(ZFS_MODULE_CFLAGS) $(ZFS_MODULE_CPPFLAGS)
 
 $(MODULE)-objs += illumos-crypto.o
 $(MODULE)-objs += api/kcf_cipher.o
diff --git a/module/nvpair/Makefile.in b/module/nvpair/Makefile.in
index a8144452a..f420ef98b 100644
--- a/module/nvpair/Makefile.in
+++ b/module/nvpair/Makefile.in
@@ -3,10 +3,10 @@ obj = @abs_builddir@
 
 MODULE := znvpair
 
-EXTRA_CFLAGS = $(ZFS_MODULE_CFLAGS) @KERNELCPPFLAGS@
-
 obj-$(CONFIG_ZFS) := $(MODULE).o
 
+ccflags-y := $(ZFS_MODULE_CFLAGS) $(ZFS_MODULE_CPPFLAGS)
+
 $(MODULE)-objs += nvpair.o
 $(MODULE)-objs += fnvpair.o
 $(MODULE)-objs += nvpair_alloc_spl.o
diff --git a/module/unicode/Makefile.in b/module/unicode/Makefile.in
index b26e66927..82c90373a 100644
--- a/module/unicode/Makefile.in
+++ b/module/unicode/Makefile.in
@@ -3,9 +3,9 @@ obj = @abs_builddir@
 
 MODULE := zunicode
 
-EXTRA_CFLAGS = $(ZFS_MODULE_CFLAGS) @KERNELCPPFLAGS@
-
 obj-$(CONFIG_ZFS) := $(MODULE).o
 
+ccflags-y := $(ZFS_MODULE_CFLAGS) $(ZFS_MODULE_CPPFLAGS)
+
 $(MODULE)-objs += u8_textprep.o
 $(MODULE)-objs += uconv.o
diff --git a/module/zcommon/Makefile.in b/module/zcommon/Makefile.in
index 86eb8ad39..501fb24e1 100644
--- a/module/zcommon/Makefile.in
+++ b/module/zcommon/Makefile.in
@@ -3,10 +3,10 @@ obj = @abs_builddir@
 
 MODULE := zcommon
 
-EXTRA_CFLAGS = $(ZFS_MODULE_CFLAGS) @KERNELCPPFLAGS@
-
 obj-$(CONFIG_ZFS) := $(MODULE).o
 
+ccflags-y := $(ZFS_MODULE_CFLAGS) $(ZFS_MODULE_CPPFLAGS)
+
 $(MODULE)-objs += zfeature_common.o
 $(MODULE)-objs += zfs_comutil.o
 $(MODULE)-objs += zfs_deleg.o
diff --git a/module/zfs/Makefile.in b/module/zfs/Makefile.in
index 66acc536d..cb352bf91 100644
--- a/module/zfs/Makefile.in
+++ b/module/zfs/Makefile.in
@@ -3,10 +3,13 @@ obj = @abs_builddir@
 
 MODULE := zfs
 
-EXTRA_CFLAGS = $(ZFS_MODULE_CFLAGS) @KERNELCPPFLAGS@
-
 obj-$(CONFIG_ZFS) := $(MODULE).o
 
+ccflags-y := $(ZFS_MODULE_CFLAGS) $(ZFS_MODULE_CPPFLAGS)
+
+# Suppress unused but set variable warnings often due to ASSERTs
+ccflags-y += $(NO_UNUSED_BUT_SET_VARIABLE)
+
 $(MODULE)-objs += abd.o
 $(MODULE)-objs += arc.o
 $(MODULE)-objs += blkptr.o