Handle possible null pointers from malloc/strdup/strndup()

GCC 12.1.1_p20220625's static analyzer caught these.

Of the two in the btree test, one had previously been caught by Coverity
and Smatch, but GCC flagged it as a false positive. Upon examining how
other test cases handle this, the solution was changed from
`ASSERT3P(node, !=, NULL);` to using `perror()` to be consistent with
the fixes to the other fixes done to the ZTS code.

That approach was also used in ZED since I did not see a better way of
handling this there. Also, upon inspection, additional unchecked
pointers from malloc()/calloc()/strdup() were found in ZED, so those
were handled too.

In other parts of the code, the existing methods to avoid issues from
memory allocators returning NULL were used, such as using
`umem_alloc(size, UMEM_NOFAIL)` or returning `ENOMEM`.

Reviewed-by: Brian Behlendorf <[email protected]>
Reviewed-by: Tony Hutter <[email protected]>
Signed-off-by: Richard Yao <[email protected]>
Closes #13979

4 files changed, 19 insertions, 1 deletions

diff --git a/tests/zfs-tests/cmd/badsend.c b/tests/zfs-tests/cmd/badsend.c
index 48d05bd3f..7a54532fb 100644
--- a/tests/zfs-tests/cmd/badsend.c
+++ b/tests/zfs-tests/cmd/badsend.c
@@ -76,6 +76,10 @@ main(int argc, char const * const argv[])
 	tosnap = p + 1;
 
 	fsname = strndup(tofull, p - tofull);
+	if (fsname == NULL) {
+		perror("strndup");
+		exit(EXIT_FAILURE);
+	}
 	if (strncmp(fsname, fromfull, p - tofull) != 0)
 		usage(argv[0]);
 
diff --git a/tests/zfs-tests/cmd/btree_test.c b/tests/zfs-tests/cmd/btree_test.c
index 4e2023003..ab8967b22 100644
--- a/tests/zfs-tests/cmd/btree_test.c
+++ b/tests/zfs-tests/cmd/btree_test.c
@@ -242,7 +242,10 @@ drain_tree(zfs_btree_t *bt, char *why)
 		zfs_btree_add_idx(bt, &randval, &bt_idx);
 
 		node = malloc(sizeof (int_node_t));
-		ASSERT3P(node, !=, NULL);
+		if (node == NULL) {
+			perror("malloc");
+			exit(EXIT_FAILURE);
+		}
 
 		node->data = randval;
 		if ((ret = avl_find(&avl, node, &avl_idx)) != NULL) {
@@ -319,6 +322,10 @@ stress_tree(zfs_btree_t *bt, char *why)
 
 		uint64_t randval = random() % tree_limit;
 		node = malloc(sizeof (*node));
+		if (node == NULL) {
+			perror("malloc");
+			exit(EXIT_FAILURE);
+		}
 		node->data = randval;
 
 		max = randval > max ? randval : max;
diff --git a/tests/zfs-tests/cmd/file/largest_file.c b/tests/zfs-tests/cmd/file/largest_file.c
index 8545bb7ee..d7252556b 100644
--- a/tests/zfs-tests/cmd/file/largest_file.c
+++ b/tests/zfs-tests/cmd/file/largest_file.c
@@ -78,6 +78,8 @@ main(int argc, char **argv)
 		return (errno);
 
 	testfile = strdup(argv[1]);
+	if (testfile == NULL)
+		return (errno);
 
 	fd = open(testfile, O_CREAT | O_RDWR, mode);
 	if (fd < 0) {
diff --git a/tests/zfs-tests/cmd/nvlist_to_lua.c b/tests/zfs-tests/cmd/nvlist_to_lua.c
index b65b3fd26..3da69397a 100644
--- a/tests/zfs-tests/cmd/nvlist_to_lua.c
+++ b/tests/zfs-tests/cmd/nvlist_to_lua.c
@@ -129,6 +129,11 @@ run_tests(void)
 	/* Note: maximum nvlist key length is 32KB */
 	int len = 1024 * 31;
 	char *bigstring = malloc(len);
+	if (bigstring == NULL) {
+		perror("malloc");
+		exit(EXIT_FAILURE);
+	}
+
 	for (int i = 0; i < len; i++)
 		bigstring[i] = 'a' + i % 26;
 	bigstring[len - 1] = '\0';