• x=$(cat <<-EOF

EOF) # works again now, plugging a regression
• rewrite the here document parsing code to be *much* more efficient
  (and a bit more readable too!) using goto, while here (no kidding)

check.t

@@ -1,4 +1,4 @@
-# $MirOS: src/bin/mksh/check.t,v 1.416 2011/03/06 17:08:10 tg Exp $
+# $MirOS: src/bin/mksh/check.t,v 1.417 2011/03/07 20:32:48 tg Exp $
 # $OpenBSD: bksl-nl.t,v 1.2 2001/01/28 23:04:56 niklas Exp $
 # $OpenBSD: history.t,v 1.5 2001/01/28 23:04:56 niklas Exp $
 # $OpenBSD: read.t,v 1.3 2003/03/10 03:48:16 david Exp $
@@ -25,7 +25,7 @@
 # http://www.research.att.com/~gsf/public/ifs.sh
 
 expected-stdout:
-	@(#)MIRBSD KSH R39 2011/03/06
+	@(#)MIRBSD KSH R39 2011/03/07
 description:
 	Check version of shell.
 stdin:
@@ -6971,7 +6971,7 @@ stdin:
 		if [[ $line = '#1' ]]; then
 			lastf=0
 			continue
-		elif [[ $line = EOFN ]]; then
+		elif [[ $line = EOFN* ]]; then
 			fbody=$fbody$'\n'$line
 			continue
 		elif [[ $line != '#'* ]]; then
@@ -7037,6 +7037,29 @@ stdin:
 	echo 'fo\ob\"a\`r'\''b\$az'
 	#OSUBST_CSUBST_OPAT_SPAT_CPAT
 	[[ ${foo#blub} = @(bar|baz) ]]
+	#heredoc_closed
+	x=$(cat <<EOFN
+	note there must be no space between EOFN and )
+	EOFN); echo $x
+	#heredoc_space
+	x=$(cat <<EOFN\ 
+	note the space between EOFN and ) is actually part of the here document marker
+	EOFN ); echo $x
+	#patch_motd
+	x=$(sysctl -n kern.version | sed 1q)
+	[[ -s /etc/motd && "$([[ "$(head -1 /etc/motd)" != $x ]] && \
+	    ed -s /etc/motd 2>&1 <<-EOF
+		1,/^\$/d
+		0a
+			$x
+	
+		.
+		wq
+	EOF)" = @(?) ]] && rm -f /etc/motd
+	if [[ ! -s /etc/motd ]]; then
+		install -c -o root -g wheel -m 664 /dev/null /etc/motd
+		print -- "$x\n" >/etc/motd
+	fi
 	#0
 	EOD
 expected-stdout:
@@ -7447,6 +7470,161 @@ expected-stdout:
 	function reread_OSUBST_CSUBST_OPAT_SPAT_CPAT {
 		x=$(( [[ ${foo#blub} = @(bar|baz) ]] ) | tr u x ) 
 	} 
+	inline_heredoc_closed() {
+		x=$(cat <<EOFN
+		note there must be no space between EOFN and )
+	EOFN); echo $x
+	}
+	inline_heredoc_closed() {
+		x=$(cat <<EOFN 
+		note there must be no space between EOFN and )
+	EOFN
+	) 
+		echo $x 
+	} 
+	function comsub_heredoc_closed { x=$(
+		x=$(cat <<EOFN
+		note there must be no space between EOFN and )
+	EOFN); echo $x
+	); }
+	function comsub_heredoc_closed {
+		x=$(x=$(cat <<EOFN 
+		note there must be no space between EOFN and )
+	EOFN
+	) ; echo $x ) 
+	} 
+	function reread_heredoc_closed { x=$((
+		x=$(cat <<EOFN
+		note there must be no space between EOFN and )
+	EOFN); echo $x
+	)|tr u x); }
+	function reread_heredoc_closed {
+		x=$(( x=$(cat <<EOFN 
+		note there must be no space between EOFN and )
+	EOFN
+	) ; echo $x ) | tr u x ) 
+	} 
+	inline_heredoc_space() {
+		x=$(cat <<EOFN\ 
+		note the space between EOFN and ) is actually part of the here document marker
+	EOFN ); echo $x
+	}
+	inline_heredoc_space() {
+		x=$(cat <<EOFN\  
+		note the space between EOFN and ) is actually part of the here document marker
+	EOFN 
+	) 
+		echo $x 
+	} 
+	function comsub_heredoc_space { x=$(
+		x=$(cat <<EOFN\ 
+		note the space between EOFN and ) is actually part of the here document marker
+	EOFN ); echo $x
+	); }
+	function comsub_heredoc_space {
+		x=$(x=$(cat <<EOFN\  
+		note the space between EOFN and ) is actually part of the here document marker
+	EOFN 
+	) ; echo $x ) 
+	} 
+	function reread_heredoc_space { x=$((
+		x=$(cat <<EOFN\ 
+		note the space between EOFN and ) is actually part of the here document marker
+	EOFN ); echo $x
+	)|tr u x); }
+	function reread_heredoc_space {
+		x=$(( x=$(cat <<EOFN\  
+		note the space between EOFN and ) is actually part of the here document marker
+	EOFN 
+	) ; echo $x ) | tr u x ) 
+	} 
+	inline_patch_motd() {
+		x=$(sysctl -n kern.version | sed 1q)
+		[[ -s /etc/motd && "$([[ "$(head -1 /etc/motd)" != $x ]] && \
+		    ed -s /etc/motd 2>&1 <<-EOF
+			1,/^\$/d
+			0a
+				$x
+		
+			.
+			wq
+		EOF)" = @(?) ]] && rm -f /etc/motd
+		if [[ ! -s /etc/motd ]]; then
+			install -c -o root -g wheel -m 664 /dev/null /etc/motd
+			print -- "$x\n" >/etc/motd
+		fi
+	}
+	inline_patch_motd() {
+		x=$(sysctl -n kern.version | sed 1q ) 
+		[[ -s /etc/motd && "$([[ "$(head -1 /etc/motd )" != $x ]] && ed -s /etc/motd 2>&1 <<-EOF 
+	1,/^\$/d
+	0a
+	$x
+	
+	.
+	wq
+	EOF
+	)" = @(?) ]] && rm -f /etc/motd 
+		if [[ ! -s /etc/motd ]] 
+		then
+			install -c -o root -g wheel -m 664 /dev/null /etc/motd 
+			print -- "$x\n" >/etc/motd 
+		fi 
+	} 
+	function comsub_patch_motd { x=$(
+		x=$(sysctl -n kern.version | sed 1q)
+		[[ -s /etc/motd && "$([[ "$(head -1 /etc/motd)" != $x ]] && \
+		    ed -s /etc/motd 2>&1 <<-EOF
+			1,/^\$/d
+			0a
+				$x
+		
+			.
+			wq
+		EOF)" = @(?) ]] && rm -f /etc/motd
+		if [[ ! -s /etc/motd ]]; then
+			install -c -o root -g wheel -m 664 /dev/null /etc/motd
+			print -- "$x\n" >/etc/motd
+		fi
+	); }
+	function comsub_patch_motd {
+		x=$(x=$(sysctl -n kern.version | sed 1q ) ; [[ -s /etc/motd && "$([[ "$(head -1 /etc/motd )" != $x ]] && ed -s /etc/motd 2>&1 <<-EOF 
+	1,/^\$/d
+	0a
+	$x
+	
+	.
+	wq
+	EOF
+	)" = @(?) ]] && rm -f /etc/motd ; if [[ ! -s /etc/motd ]] ; then install -c -o root -g wheel -m 664 /dev/null /etc/motd ; print -- "$x\n" >/etc/motd ; fi ) 
+	} 
+	function reread_patch_motd { x=$((
+		x=$(sysctl -n kern.version | sed 1q)
+		[[ -s /etc/motd && "$([[ "$(head -1 /etc/motd)" != $x ]] && \
+		    ed -s /etc/motd 2>&1 <<-EOF
+			1,/^\$/d
+			0a
+				$x
+		
+			.
+			wq
+		EOF)" = @(?) ]] && rm -f /etc/motd
+		if [[ ! -s /etc/motd ]]; then
+			install -c -o root -g wheel -m 664 /dev/null /etc/motd
+			print -- "$x\n" >/etc/motd
+		fi
+	)|tr u x); }
+	function reread_patch_motd {
+		x=$(( x=$(sysctl -n kern.version | sed 1q ) ; [[ -s /etc/motd && "$([[ "$(head -1 /etc/motd )" != $x ]] && ed -s /etc/motd 2>&1 <<-EOF 
+	1,/^\$/d
+	0a
+	$x
+	
+	.
+	wq
+	EOF
+	)" = @(?) ]] && rm -f /etc/motd ; if [[ ! -s /etc/motd ]] ; then install -c -o root -g wheel -m 664 /dev/null /etc/motd ; print -- "$x\n" >/etc/motd ; fi ) | tr u x ) 
+	} 
 ---
 name: test-stnze-1
 description:

lex.c

@@ -23,7 +23,7 @@
 
 #include "sh.h"
 
-__RCSID("$MirOS: src/bin/mksh/lex.c,v 1.128 2011/03/07 20:30:39 tg Exp $");
+__RCSID("$MirOS: src/bin/mksh/lex.c,v 1.129 2011/03/07 20:32:49 tg Exp $");
 
 /*
  * states while lexing word
@@ -100,6 +100,7 @@ static int dopprompt(const char *, int, bool);
 
 static int backslash_skip;
 static int ignore_backslash_newline;
+static int comsub_nesting_level;
 
 /* optimised getsc_bn() */
 #define _getsc()	(*source->str != '\0' && *source->str != '\\' \
@@ -149,7 +150,7 @@ getsc_(void)
  *
  * tokens are not regular expressions, they are LL(1).
  * for example, "${var:-${PWD}}", and "$(size $(whence ksh))".
- * hence the state stack.
+ * hence the state stack. Note "$(...)" are now parsed recursively.
  */
 
 int
@@ -359,7 +360,9 @@ yylex(int cf)
 					} else {
 						ungetsc(c);
  subst_command:
+						++comsub_nesting_level;
 						sp = yyrecursive();
+						--comsub_nesting_level;
 						c2 = strlen(sp) + 1;
 						XcheckN(ws, wp, c2);
 						*wp++ = COMSUB;
@@ -1114,7 +1117,6 @@ readhere(struct ioword *iop)
 	int c;
 	char *volatile eof;
 	char *eofp;
-	int skiptabs;
 	XString xs;
 	char *xp;
 	int xpos;
@@ -1135,41 +1137,66 @@ readhere(struct ioword *iop)
 
 	Xinit(xs, xp, 256, ATEMP);
 
-	for (;;) {
+ heredoc_read_line:
-		eofp = eof;
+	/* beginning of line */
-		skiptabs = iop->flag & IOSKIP;
+	eofp = eof;
-		xpos = Xsavepos(xs, xp);
+	xpos = Xsavepos(xs, xp);
-		while ((c = getsc()) != 0) {
+	if (iop->flag & IOSKIP) {
-			if (skiptabs) {
+		/* skip over leading tabs */
-				if (c == '\t')
+		while ((c = getsc()) == '\t')
-					continue;
+			/* nothing */;
-				skiptabs = 0;
+		goto heredoc_parse_char;
-			}
+	}
-			if (c != *eofp)
+ heredoc_read_char:
+	c = getsc();
+ heredoc_parse_char:
+	/* compare with here document marker */
+	if (!*eofp) {
+		/* end of here document marker, what to do? */
+		switch (c) {
+		case /*(*/ ')':
+			if (!comsub_nesting_level)
+				/* not allowed outside $(...) => mismatch */
 				break;
-			Xcheck(xs, xp);
+			/* Allow $(...) to close here */
-			Xput(xs, xp, c);
+			ungetsc(/*(*/ ')');
-			eofp++;
+			/* FALLTHROUGH */
-		}
+		case 0:
-		/*
+			/*
-		 * Allow EOF here so commands with out trailing newlines
+			 * Allow EOF here to commands without trailing
-		 * will work (eg, ksh -c '...', $(...), etc).
+			 * newlines (mksh -c '...') will work as well.
-		 */
+			 */
-		if (*eofp == '\0' && (c == 0 || c == '\n')) {
+		case '\n':
-			xp = Xrestpos(xs, xp, xpos);
+			/* Newline terminates here document marker */
-			break;
+			goto heredoc_found_terminator;
-		}
-		ungetsc(c);
-		while ((c = getsc()) != '\n') {
-			if (c == 0)
-				yyerror("%s '%s' unclosed\n", "here document",
-				    eof);
-			Xcheck(xs, xp);
-			Xput(xs, xp, c);
 		}
+	} else if (c == *eofp++)
+		/* store; then read and compare next character */
+		goto heredoc_store_and_loop;
+	/* nope, mismatch; read until end of line */
+	while (c != '\n') {
+		if (!c)
+			/* oops, reached EOF */
+			yyerror("%s '%s' unclosed\n", "here document", eof);
+		/* store character */
 		Xcheck(xs, xp);
 		Xput(xs, xp, c);
+		/* read next character */
+		c = getsc();
 	}
+	/* we read a newline as last character */
+ heredoc_store_and_loop:
+	/* store character */
+	Xcheck(xs, xp);
+	Xput(xs, xp, c);
+	if (c == '\n')
+		goto heredoc_read_line;
+	goto heredoc_read_char;
+
+ heredoc_found_terminator:
+	/* jump back to saved beginning of line */
+	xp = Xrestpos(xs, xp, xpos);
+	/* terminate, close and store */
 	Xput(xs, xp, '\0');
 	iop->heredoc = Xclose(xs, xp);
 

sh.h

@@ -154,9 +154,9 @@
 #endif
 
 #ifdef EXTERN
-__RCSID("$MirOS: src/bin/mksh/sh.h,v 1.439 2011/03/07 20:30:40 tg Exp $");
+__RCSID("$MirOS: src/bin/mksh/sh.h,v 1.440 2011/03/07 20:32:50 tg Exp $");
 #endif
-#define MKSH_VERSION "R39 2011/03/06"
+#define MKSH_VERSION "R39 2011/03/07"
 
 #ifndef MKSH_INCLUDES_ONLY