• use a flag for determining here strings, don’t parse delimiter every time

• don’t leak memory parsing possible I/O redirection tokens
• get rid of volatile by using more const (also helps codegen, methinks)
• support empty here document markers (mksh extension)
• pimp the manpage

check.t

@@ -1,4 +1,4 @@
-# $MirOS: src/bin/mksh/check.t,v 1.449 2011/05/04 23:16:00 tg Exp $
+# $MirOS: src/bin/mksh/check.t,v 1.450 2011/05/05 00:04:55 tg Exp $
 # $OpenBSD: bksl-nl.t,v 1.2 2001/01/28 23:04:56 niklas Exp $
 # $OpenBSD: history.t,v 1.5 2001/01/28 23:04:56 niklas Exp $
 # $OpenBSD: read.t,v 1.3 2003/03/10 03:48:16 david Exp $
@@ -25,7 +25,7 @@
 # http://www.research.att.com/~gsf/public/ifs.sh
 
 expected-stdout:
-	@(#)MIRBSD KSH R39 2011/05/02
+	@(#)MIRBSD KSH R39 2011/05/04
 description:
 	Check version of shell.
 stdin:
@@ -2129,6 +2129,45 @@ expected-stdout:
 	} vf={=f $x @=
 	} |
 ---
+name: heredoc-11
+description:
+	Check here documents with no or empty delimiter
+stdin:
+	x=u
+	va=<<
+	=a $x \x40=
+	<<
+	vb=<<''
+	=b $x \x40=
+	
+	function foo {
+		vc=<<-
+			=c $x \x40=
+		<<
+		vd=<<-''
+			=d $x \x40=
+	
+	}
+	typeset -f foo
+	foo
+	print -r -- "| va={$va} vb={$vb} vc={$vc} vd={$vd} |"
+expected-stdout:
+	function foo {
+		vc= <<- 
+	=c $x \x40=
+	<<
+	
+		vd= <<-"" 
+	=d $x \x40=
+	
+	
+	} 
+	| va={=a u \x40=
+	} vb={=b $x \x40=
+	} vc={=c u \x40=
+	} vd={=d $x \x40=
+	} |
+---
 name: heredoc-comsub-1
 description:
 	Tests for here documents in COMSUB, taken from Austin ML

lex.c

@@ -22,7 +22,7 @@
 
 #include "sh.h"
 
-__RCSID("$MirOS: src/bin/mksh/lex.c,v 1.147 2011/05/02 22:52:51 tg Exp $");
+__RCSID("$MirOS: src/bin/mksh/lex.c,v 1.148 2011/05/05 00:04:57 tg Exp $");
 
 /*
  * states while lexing word
@@ -980,10 +980,14 @@ yylex(int cf)
 			iop->flag |= c == c2 ?
 			    (c == '>' ? IOCAT : IOHERE) : IORDWR;
 			if (iop->flag == IOHERE) {
-				if ((c2 = getsc()) == '-')
+				if ((c2 = getsc()) == '-') {
 					iop->flag |= IOSKIP;
-				else
+					c2 = getsc();
+				} else if (c2 == '<')
+					iop->flag |= IOHERESTR;
 				ungetsc(c2);
+				if (c2 == '\n')
+					iop->flag |= IONDELIM;
 			}
 		} else if (c2 == '&')
 			iop->flag |= IODUP | (c == '<' ? IORDUP : 0);
@@ -1003,7 +1007,7 @@ yylex(int cf)
 		yylval.iop = iop;
 		return (REDIR);
  no_iop:
-		;
+		afree(iop, ATEMP);
 	}
 
 	if (wp == dp && state == SBASE) {
@@ -1142,7 +1146,7 @@ gethere(bool iseof)
 	struct ioword **p;
 
 	for (p = heres; p < herep; p++)
-		if (iseof && (*p)->delim[1] != '<')
+		if (iseof && !((*p)->flag & IOHERESTR))
 			/* only here strings at EOF */
 			return;
 		else
@@ -1158,22 +1162,21 @@ static void
 readhere(struct ioword *iop)
 {
 	int c;
-	char *volatile eof;
+	const char *eof, *eofp;
-	char *eofp;
 	XString xs;
 	char *xp;
 	int xpos;
 
-	if (iop->delim[1] == '<') {
+	if (iop->flag & IOHERESTR) {
 		/* process the here string */
-		xp = iop->heredoc = evalstr(iop->delim, DOBLANK);
+		iop->heredoc = xp = evalstr(iop->delim, DOBLANK);
 		c = strlen(xp) - 1;
 		memmove(xp, xp + 1, c);
 		xp[c] = '\n';
 		return;
 	}
 
-	eof = evalstr(iop->delim, 0);
+	eof = iop->flag & IONDELIM ? "<<" : evalstr(iop->delim, 0);
 
 	if (!(iop->flag & IOEVAL))
 		ignore_backslash_newline++;

mksh.1

@@ -1,4 +1,4 @@
-.\" $MirOS: src/bin/mksh/mksh.1,v 1.258 2011/05/04 23:16:03 tg Exp $
+.\" $MirOS: src/bin/mksh/mksh.1,v 1.259 2011/05/05 00:04:58 tg Exp $
 .\" $OpenBSD: ksh.1,v 1.140 2011/04/23 10:14:59 sobrado Exp $
 .\"-
 .\" Copyright © 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009,
@@ -72,7 +72,7 @@
 .\" with -mandoc, it might implement .Mx itself, but we want to
 .\" use our own definition. And .Dd must come *first*, always.
 .\"
-.Dd $Mdocdate: May 4 2011 $
+.Dd $Mdocdate: May 5 2011 $
 .\"
 .\" Check which macro package we use
 .\"
@@ -1461,7 +1461,9 @@ on it; if
 .Ar word
 is not needed, it is not evaluated.
 .Pp
-The following forms of parameter substitution can also be used:
+The following forms of parameter substitution can also be used (if
+.Ar name
+is an array, its element #0 will be substituted in a scalar context):
 .Pp
 .Bl -tag -width Ds -compact
 .It Pf ${# Ns Ar name Ns \&}
@@ -1525,6 +1527,8 @@ A single
 .Ql #
 results in the shortest match, and two
 of them result in the longest match.
+Cannot be applied to a vector
+.Pq ${*} or ${@} or ${array[*]} or ${array[@]} .
 .Pp
 .Sm off
 .It Xo
@@ -1537,6 +1541,7 @@ of them result in the longest match.
 .Xc
 .Sm on
 Like ${..#..} substitution, but it deletes from the end of the value.
+Cannot be applied to a vector.
 .Pp
 .Sm off
 .It Xo
@@ -1569,6 +1574,8 @@ If
 is omitted, the
 .Ar pattern
 is replaced by the empty string, i.e. deleted.
+Cannot be applied to a vector.
+Inefficiently implemented.
 .Pp
 .Sm off
 .It Xo
@@ -1604,6 +1611,8 @@ are evaluated as arithmetic expressions.
 Currently,
 .Ar pos
 must start with a space, opening parenthesis or digit to be recognised.
+Cannot be applied to a vector.
+.Pp
 .It Pf ${ Ns Ar name Ns @#}
 The internal hash of the expansion of
 .Ar name .
@@ -1614,12 +1623,15 @@ This is the hash the shell uses internally for its associative arrays.
 .Pp
 Note that
 .Ar pattern
-may need to be escaped as an extended globbing pattern
+may need extended globbing pattern
 .Pq @(...) ,
-with single quotes
+single
 .Pq \&\*(aq...\&\*(aq
-or double quotes
+or double
-.Pq \&"...\&" .
+.Pq \&"...\&"
+quote escaping unless
+.Fl o Ic sh
+is set.
 .Pp
 The following special parameters are implicitly set by the shell and cannot be
 set directly using assignments:
@@ -1692,8 +1704,9 @@ arguments or splitting arguments with spaces.
 .El
 .Pp
 The following parameters are set and/or used by the shell:
-.Bl -tag -width "PIPESTATUS"
+.Bl -tag -width "KSH_VERSION"
-.It Ev _ No (underscore)
+.It Ev _
+.Pq underscore
 When an external command is executed by the shell, this parameter is set in the
 environment of the new process to the path of the executed command.
 In interactive use, this parameter is also set in the parent shell to the last
@@ -2217,7 +2230,7 @@ asynchronous commands created when job control is disabled, for which standard
 input is initially set to be from
 .Pa /dev/null ,
 and commands for which any of the following redirections have been specified:
-.Bl -tag -width Ds
+.Bl -tag -width XXxxmarker
 .It \*(Gt Ar file
 Standard output is redirected to
 .Ar file .
@@ -2283,6 +2296,20 @@ but not for
 .Ql \&" .
 If multiple here documents are used on the same command line, they are saved in
 order.
+.Pp
+If no
+.Ar marker
+is given, the here document ends at the next
+.Ic \*(Lt\*(Lt
+and substitution will be performed.
+If
+.Ar marker
+is only a set of either single
+.Dq \*(aq\*(aq
+or double
+.Sq \&""
+quotes with nothing in between, the here document ends at the next empty line
+and substitution will not be performed.
 .It \*(Lt\*(Lt\- Ar marker
 Same as
 .Ic \*(Lt\*(Lt ,
@@ -2373,13 +2400,9 @@ statements, etc.
 any redirections must appear at the end.
 Redirections are processed after
 pipelines are created and in the order they are given, so the following
-will print an error with a line number prepended to it, if you have a
+will print an error with a line number prepended to it:
-.Xr cat 1
-that supports the
-.Fl n
-option:
 .Pp
-.D1 $ cat /foo/bar 2\*(Gt&1 \*(Gt/dev/null \*(Ba cat \-n
+.D1 $ cat /foo/bar 2\*(Gt&1 \*(Gt/dev/null \*(Ba pr \-n \-t
 .Pp
 File descriptors created by input/output redirections are private to the
 Korn shell, but passed to sub-processes if

sh.h

@@ -151,9 +151,9 @@
 #endif
 
 #ifdef EXTERN
-__RCSID("$MirOS: src/bin/mksh/sh.h,v 1.467 2011/05/02 22:52:52 tg Exp $");
+__RCSID("$MirOS: src/bin/mksh/sh.h,v 1.468 2011/05/05 00:04:59 tg Exp $");
 #endif
-#define MKSH_VERSION "R39 2011/05/02"
+#define MKSH_VERSION "R39 2011/05/04"
 
 #ifndef MKSH_INCLUDES_ONLY
 
@@ -1185,6 +1185,8 @@ struct ioword {
 #define IORDUP		BIT(7)	/* x<&y (as opposed to x>&y) */
 #define IONAMEXP	BIT(8)	/* name has been expanded */
 #define IOBASH		BIT(9)	/* &> etc. */
+#define IOHERESTR	BIT(10)	/* <<< (here string) */
+#define IONDELIM	BIT(11)	/* null delimiter (<<) */
 
 /* execute/exchild flags */
 #define XEXEC	BIT(0)		/* execute without forking */

syn.c

@@ -22,7 +22,7 @@
 
 #include "sh.h"
 
-__RCSID("$MirOS: src/bin/mksh/syn.c,v 1.62 2011/05/02 22:52:53 tg Exp $");
+__RCSID("$MirOS: src/bin/mksh/syn.c,v 1.63 2011/05/05 00:05:01 tg Exp $");
 
 extern short subshell_nesting_level;
 
@@ -179,12 +179,15 @@ synio(int cf)
 		return (NULL);
 	ACCEPT;
 	iop = yylval.iop;
+	if (iop->flag & IONDELIM)
+		goto gotnulldelim;
 	ishere = (iop->flag & IOTYPE) == IOHERE;
 	musthave(LWORD, ishere ? HEREDELIM : 0);
 	if (ishere) {
 		iop->delim = yylval.cp;
 		if (*ident != 0)
 			/* unquoted */
+ gotnulldelim:
 			iop->flag |= IOEVAL;
 		if (herep > &heres[HERES - 1])
 			yyerror("too many %ss\n", "<<");

tree.c

@@ -22,7 +22,7 @@
 
 #include "sh.h"
 
-__RCSID("$MirOS: src/bin/mksh/tree.c,v 1.46 2011/05/04 22:38:27 tg Exp $");
+__RCSID("$MirOS: src/bin/mksh/tree.c,v 1.47 2011/05/05 00:05:01 tg Exp $");
 
 #define INDENT	8
 
@@ -198,12 +198,12 @@ ptree(struct op *t, int indent, struct shf *shf)
 			struct ioword *iop = *ioact++;
 
 			/* heredoc is 0 when tracing (set -x) */
-			if ((iop->flag & IOTYPE) == IOHERE && iop->heredoc &&
+			if ((iop->flag & (IOTYPE | IOHERESTR)) == IOHERE &&
-			    /* iop->delim[1] == '<' means here string */
+			    iop->heredoc) {
-			    (!iop->delim || iop->delim[1] != '<')) {
 				shf_putc('\n', shf);
 				shf_puts(iop->heredoc, shf);
 				fptreef(shf, indent, "%s",
+				    iop->flag & IONDELIM ? "<<" :
 				    evalstr(iop->delim, 0));
 				need_nl = true;
 			}