Remove enable_extended_FILE_stdio()

Even on Illumos it's only available in the 32-bit programming
environment, and, quoth enable_extended_FILE_stdio(3C):
> Historically, 32-bit Solaris applications have been limited to using
> only the file descriptors 0 through 255 with the standard I/O
> functions (see stdio(3C)) in the C library. The extended FILE
> facility allows well-behaved 32-bit applications to use any
> valid file descriptor with the standard I/O functions.
where "well-behaved" means that it
> does not directly access any fields in the FILE structure pointed
> to by the FILE pointer associated with any standard I/O stream,

And the stdio/flush.c implementation reads:
  /*
   * if this is not an internal extended FILE then check
   * if _file is being changed from underneath us.
   * It should not be because if
   * it is then then we lose our ability to guard against
   * silent data corruption.
   */
  if (!iop->__xf_nocheck && bad_fd > -1 && iop->_magic != bad_fd) {
      (void) fprintf(stderr,
          "Application violated extended FILE safety mechanism.\n"
          "Please read the man page for extendedFILE.\nAborting\n");
      abort();
  }

This appears to be an insane workaround for broken implementation with
exposed FILE internals and _file being an u8, both only on non-LP64;
it's shimmed out on all LP64 targets in Illumos,
and we shim it out as well: just get rid of it

This appears to've been originally fixed in illumos-gate
a5f69788de7ac07553de47f7fec8c05a9a94c105 ("PSARC 2006/162 Extended FILE
space for 32-bit Solaris processes", "1085341 32-bit stdio routines
should support file descriptors >255"), which also bears extendedFILE
and enable_extended_FILE_stdio(3C):
  -       unsigned char   _file;  /* UNIX System file descriptor */
  +       unsigned char   _magic; /* Old home of the file descriptor */
  +                               /* Only fileno(3C) can retrieve the
  				value now */
and
  +/*
  + * Macros to aid the extended fd FILE work.
  + * This helps isolate the changes to only the 32-bit code
  + * since 64-bit Solaris is not affected by this.
  + */
  +#ifdef  _LP64
  +#define        GET_FD(iop)             ((iop)->_file)
  +#define        SET_FILE(iop, fd)       ((iop)->_file = (fd))
  +#else
  +#define        GET_FD(iop)             \
  +               (((iop)->__extendedfd) ? _file_get(iop) : (iop)->_magic)
  +#define        SET_FILE(iop, fd)       (iop)->_magic = (fd); (iop)->__extendedfd = 0
  +#endif

Also remove the 1k setrlimit(NOFILE) calls: that's the default on Linux,
with 64k on Illumos and 171k on FreeBSD

Reviewed-by: Brian Behlendorf <[email protected]>
Signed-off-by: Ahelenia Ziemiańska <[email protected]>
Closes #13411

4 files changed, 1 insertions, 45 deletions

diff --git a/cmd/zdb/zdb.c b/cmd/zdb/zdb.c
index 6b418dd31..3d4d956e5 100644
--- a/cmd/zdb/zdb.c
+++ b/cmd/zdb/zdb.c
@@ -8405,7 +8405,6 @@ int
 main(int argc, char **argv)
 {
 	int c;
-	struct rlimit rl = { 1024, 1024 };
 	spa_t *spa = NULL;
 	objset_t *os = NULL;
 	int dump_all = 1;
@@ -8424,9 +8423,6 @@ main(int argc, char **argv)
 	boolean_t target_is_spa = B_TRUE, dataset_lookup = B_FALSE;
 	nvlist_t *cfg = NULL;
 
-	(void) setrlimit(RLIMIT_NOFILE, &rl);
-	(void) enable_extended_FILE_stdio(-1, -1);
-
 	dprintf_setup(&argc, argv);
 
 	/*
diff --git a/cmd/ztest.c b/cmd/ztest.c
index 2dae34e97..134a72c9d 100644
--- a/cmd/ztest.c
+++ b/cmd/ztest.c
@@ -7888,21 +7888,16 @@ exec_child(char *cmd, char *libpath, boolean_t ignorekill, int *statusp)
 		fatal(B_TRUE, "fork failed");
 
 	if (pid == 0) {	/* child */
-		char *emptyargv[2] = { cmd, NULL };
 		char fd_data_str[12];
 
-		struct rlimit rl = { 1024, 1024 };
-		(void) setrlimit(RLIMIT_NOFILE, &rl);
-
 		(void) close(ztest_fd_rand);
 		VERIFY3S(11, >=,
 		    snprintf(fd_data_str, 12, "%d", ztest_fd_data));
 		VERIFY0(setenv("ZTEST_FD_DATA", fd_data_str, 1));
 
-		(void) enable_extended_FILE_stdio(-1, -1);
 		if (libpath != NULL)
 			VERIFY0(setenv("LD_LIBRARY_PATH", libpath, 1));
-		(void) execv(cmd, emptyargv);
+		(void) execl(cmd, cmd, (char *)NULL);
 		ztest_dump_core = B_FALSE;
 		fatal(B_TRUE, "exec failed: %s", cmd);
 	}
diff --git a/lib/libspl/include/Makefile.am b/lib/libspl/include/Makefile.am
index e995ddf66..2c0284310 100644
--- a/lib/libspl/include/Makefile.am
+++ b/lib/libspl/include/Makefile.am
@@ -8,7 +8,6 @@ libspl_HEADERS = \
 	%D%/limits.h \
 	%D%/locale.h \
 	%D%/statcommon.h \
-	%D%/stdio.h \
 	%D%/stdlib.h \
 	%D%/string.h \
 	%D%/stropts.h \
diff --git a/lib/libspl/include/stdio.h b/lib/libspl/include/stdio.h
deleted file mode 100644
index 6152b09f1..000000000
--- a/lib/libspl/include/stdio.h
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * CDDL HEADER START
- *
- * The contents of this file are subject to the terms of the
- * Common Development and Distribution License, Version 1.0 only
- * (the "License").  You may not use this file except in compliance
- * with the License.
- *
- * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
- * or http://www.opensolaris.org/os/licensing.
- * See the License for the specific language governing permissions
- * and limitations under the License.
- *
- * When distributing Covered Code, include this CDDL HEADER in each
- * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
- * If applicable, add the following below this CDDL HEADER, with the
- * fields enclosed by brackets "[]" replaced with your own identifying
- * information: Portions Copyright [yyyy] [name of copyright owner]
- *
- * CDDL HEADER END
- */
-/*
- * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
- * Use is subject to license terms.
- */
-
-#include_next <stdio.h>
-
-#ifndef _LIBSPL_STDIO_H
-#define	_LIBSPL_STDIO_H
-
-#define	enable_extended_FILE_stdio(fd, sig)	((void) 0)
-
-#endif