Pass arguments via a resonse file if executing a child fails

Command line length limit of OS/2 is 32KiB. If the total length of
all arguments is larger than this limit, it's needed to use a
response file.

Previously, the total length calculation was simply to add length of
all arguments. However, this result was not match with real length of
arguments, which are passed to child processes with OS/2 APIs.

Because conversion methods of arguments from libc to OS/2 APIs are
different depending on libc.

For example, kLIBC inserts its signature to an argument list. In
addition, it passes arguments with a leading space like:

    arg0
     kLIBC signature
     arg1
     arg2
     ...

Whereas, EMX just distinguishes arg0 and others like:

    arg0
    arg1 arg2 arg3 ...

After all, simple sum of a length of arguments are not correct.

The better way is to try to execute a child process, and to retry with
a response file if it fails due to arguments-too-long.

This has been found while doing 'bootstrap', especially 'autoreconf'
in coreutils git repo. It stops with:

    autom4te: /usr/local/bin/m4: Invalid argument

From: KO Myung-Hun <komh@chollian.net>

os2.c

@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2015
+ * Copyright (c) 2015, 2017
  *	KO Myung-Hun <komh@chollian.net>
  * Copyright (c) 2017
  *	mirabilos <m@mirbsd.org>
@@ -31,7 +31,7 @@
 #include <unistd.h>
 #include <process.h>
 
-__RCSID("$MirOS: src/bin/mksh/os2.c,v 1.7 2017/12/22 16:37:14 tg Exp $");
+__RCSID("$MirOS: src/bin/mksh/os2.c,v 1.8 2017/12/22 16:41:42 tg Exp $");
 
 static char *remove_trailing_dots(char *);
 static int access_stat_ex(int (*)(), const char *, void *);
@@ -329,30 +329,13 @@ real_exec_name(const char *name)
 	return (real_name);
 }
 
-/* OS/2 can process a command line up to 32 KiB */
-#define MAX_CMD_LINE_LEN 32768
-
 /* make a response file to pass a very long command line */
 static char *
 make_response_file(char * const *argv)
 {
 	char rsp_name_arg[] = "@mksh-rsp-XXXXXX";
 	char *rsp_name = &rsp_name_arg[1];
-	int arg_len = 0;
 	int i;
-
-	for (i = 0; argv[i]; i++)
-		arg_len += strlen(argv[i]) + 1;
-
-	/*
-	 * If a length of command line is longer than MAX_CMD_LINE_LEN, then
-	 * use a response file. OS/2 cannot process a command line longer
-	 * than 32K. Of course, a response file cannot be recognised by a
-	 * normal OS/2 program, that is, neither non-EMX or non-kLIBC. But
-	 * it cannot accept a command line longer than 32K in itself. So
-	 * using a response file in this case, is an acceptable solution.
-	 */
-	if (arg_len > MAX_CMD_LINE_LEN) {
 	int fd;
 	char *result;
 
@@ -368,10 +351,8 @@ make_response_file(char * const *argv)
 	close(fd);
 	add_temp(rsp_name);
 	strdupx(result, rsp_name_arg, ATEMP);
-		return (result);
-	}
 
-	return (NULL);
+	return (result);
 }
 
 /* alias of execve() */
@@ -384,13 +365,12 @@ execve(const char *name, char * const *argv, char * const *envp)
 	const char *exec_name;
 	FILE *fp;
 	char sign[2];
-	char *rsp_argv[3];
-	char *rsp_name_arg;
 	int pid;
 	int status;
 	int fd;
 	int rc;
 	int saved_mode;
+	int saved_errno;
 
 	/*
 	 * #! /bin/sh : append .exe
@@ -430,16 +410,6 @@ execve(const char *name, char * const *argv, char * const *envp)
 	if (errno == ENOEXEC)
 		return (-1);
 
-	rsp_name_arg = make_response_file(argv);
-
-	if (rsp_name_arg) {
-		rsp_argv[0] = argv[0];
-		rsp_argv[1] = rsp_name_arg;
-		rsp_argv[2] = NULL;
-
-		argv = rsp_argv;
-	}
-
 	/*
 	 * Normal OS/2 programs expect that standard IOs, especially stdin,
 	 * are opened in text mode at the startup. By the way, on OS/2 kLIBC
@@ -451,15 +421,30 @@ execve(const char *name, char * const *argv, char * const *envp)
 	saved_mode = setmode(STDIN_FILENO, O_TEXT);
 
 	pid = spawnve(P_NOWAIT, exec_name, argv, envp);
+	saved_errno = errno;
+
+	/* arguments too long? */
+	if (pid == -1 && saved_errno == EINVAL) {
+		/* retry with a response file */
+		char *rsp_name_arg = make_response_file(argv);
+
+		if (rsp_name_arg) {
+			char *rsp_argv[3] = { argv[0], rsp_name_arg, NULL };
+
+			pid = spawnve(P_NOWAIT, exec_name, rsp_argv, envp);
+			saved_errno = errno;
+
+			afree(rsp_name_arg, ATEMP);
+		}
+	}
 
 	/* restore translation mode of stdin */
 	setmode(STDIN_FILENO, saved_mode);
 
-	afree(rsp_name_arg, ATEMP);
-
 	if (pid == -1) {
 		cleanup_temps();
 
+		errno = saved_errno;
 		return (-1);
 	}