replace the code related to << changes and <<< with cleaner code,

fixing a few bugs also (including new testcases, except x=<< issue)
2015-09-06 19:47:01 +00:00
parent 0f6aa7eaab
commit ed5cb56849
7 changed files with 63 additions and 97 deletions
--- a/check.t
+++ b/check.t
@@ -1,4 +1,4 @@
-# $MirOS: src/bin/mksh/check.t,v 1.706 2015/09/05 20:20:42 tg Exp $
+# $MirOS: src/bin/mksh/check.t,v 1.707 2015/09/06 19:46:56 tg Exp $
 # -*- mode: sh -*-
 #-
 # Copyright © 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010,
@@ -30,7 +30,7 @@
 # (2013/12/02 20:39:44) http://openbsd.cs.toronto.edu/cgi-bin/cvsweb/src/regress/bin/ksh/?sortby=date

 expected-stdout:
-	@(#)MIRBSD KSH R51 2015/09/05
+	@(#)MIRBSD KSH R51 2015/09/06
 description:
 	Check version of shell.
 stdin:
@@ -39,7 +39,7 @@ name: KSH_VERSION
 category: shell:legacy-no
 ---
 expected-stdout:
-	@(#)LEGACY KSH R51 2015/09/05
+	@(#)LEGACY KSH R51 2015/09/06
 description:
 	Check version of legacy shell.
 stdin:
@@ -2325,6 +2325,8 @@ stdin:
 	tr abcdefghijklmnopqrstuvwxyz nopqrstuvwxyzabcdefghijklm <<<\$bar
 	tr abcdefghijklmnopqrstuvwxyz nopqrstuvwxyzabcdefghijklm <<<-foo
 	tr abcdefghijklmnopqrstuvwxyz nopqrstuvwxyzabcdefghijklm <<<"$(echo "foo bar")"
+	tr abcdefghijklmnopqrstuvwxyz nopqrstuvwxyzabcdefghijklm <<<"A $(echo "foo bar") B"
+	tr abcdefghijklmnopqrstuvwxyz nopqrstuvwxyzabcdefghijklm <<<\$b\$b$bar
 expected-stdout:
 	sbb
 	sbb
@@ -2334,6 +2336,9 @@ expected-stdout:
 	$one
 	-sbb
 	sbb one
+	A sbb one B
+	$o$oone
+		onm
 ---
 name: heredoc-9b
 description:
--- a/eval.c
+++ b/eval.c
@@ -23,7 +23,7 @@

 #include "sh.h"

-__RCSID("$MirOS: src/bin/mksh/eval.c,v 1.171 2015/09/05 19:19:02 tg Exp $");
+__RCSID("$MirOS: src/bin/mksh/eval.c,v 1.172 2015/09/06 19:46:59 tg Exp $");

 /*
 * string expansion
@@ -919,6 +919,8 @@ expand(
 			    (word == IFS_IWS || word == IFS_NWS) &&
 			    !ctype(c, C_IFSWS))) {
 emit_word:
+				if (f & DOHERESTR)
+					*dp++ = '\n';
 				*dp++ = '\0';
 				cp = Xclose(ds, dp);
 				if (fdo & DOBRACE)
--- a/exec.c
+++ b/exec.c
@@ -23,7 +23,7 @@

 #include "sh.h"

-__RCSID("$MirOS: src/bin/mksh/exec.c,v 1.162 2015/09/05 19:19:03 tg Exp $");
+__RCSID("$MirOS: src/bin/mksh/exec.c,v 1.163 2015/09/06 19:46:59 tg Exp $");

 #ifndef MKSH_DEFAULT_EXECSHELL
 #define MKSH_DEFAULT_EXECSHELL	MKSH_UNIXROOT "/bin/sh"
@@ -1518,9 +1518,9 @@ iosetup(struct ioword *iop, struct tbl *tp)
 * unquoted, the string is expanded first.
 */
 static int
-hereinval(const char *content, int sub, char **resbuf, struct shf *shf)
+hereinval(struct ioword *iop, int sub, char **resbuf, struct shf *shf)
 {
-	const char * volatile ccp = content;
+	const char * volatile ccp = iop->heredoc;
 	struct source *s, *osource;

 	osource = source;
@@ -1531,7 +1531,9 @@ hereinval(const char *content, int sub, char **resbuf, struct shf *shf)
 		/* special to iosetup(): don't print error */
 		return (-2);
 	}
-	if (sub) {
+	if (iop->ioflag & IOHERESTR) {
+		ccp = evalstr(iop->delim, DOHERESTR | DOSCALAR | DOHEREDOC);
+	} else if (sub) {
 		/* do substitutions on the content of heredoc */
 		s = pushs(SSTRING, ATEMP);
 		s->start = s->str = ccp;
@@ -1560,7 +1562,7 @@ herein(struct ioword *iop, char **resbuf)
 	int i;

 	/* ksh -c 'cat <<EOF' can cause this... */
-	if (iop->heredoc == NULL) {
+	if (iop->heredoc == NULL && !(iop->ioflag & IOHERESTR)) {
 		warningf(true, "%s missing", "here document");
 		/* special to iosetup(): don't print error */
 		return (-2);
@@ -1571,7 +1573,7 @@ herein(struct ioword *iop, char **resbuf)

 	/* skip all the fd setup if we just want the value */
 	if (resbuf != NULL)
-		return (hereinval(iop->heredoc, i, resbuf, NULL));
+		return (hereinval(iop, i, resbuf, NULL));

 	/*
 	 * Create temp file to hold content (done before newenv
@@ -1588,7 +1590,7 @@ herein(struct ioword *iop, char **resbuf)
 		return (-2);
 	}

-	if (hereinval(iop->heredoc, i, NULL, shf) == -2) {
+	if (hereinval(iop, i, NULL, shf) == -2) {
 		close(fd);
 		/* special to iosetup(): don't print error */
 		return (-2);
--- a/lex.c
+++ b/lex.c
@@ -23,7 +23,7 @@

 #include "sh.h"

-__RCSID("$MirOS: src/bin/mksh/lex.c,v 1.208 2015/09/06 13:10:48 tg Exp $");
+__RCSID("$MirOS: src/bin/mksh/lex.c,v 1.209 2015/09/06 19:47:00 tg Exp $");

 /*
 * states while lexing word
@@ -38,8 +38,8 @@ __RCSID("$MirOS: src/bin/mksh/lex.c,v 1.208 2015/09/06 13:10:48 tg Exp $");
 #define SQBRACE		7	/* inside "${}" */
 #define SBQUOTE		8	/* inside `` */
 #define SASPAREN	9	/* inside $(( )) */
-#define SHEREDELIM	10	/* parsing <<,<<-,<<< delimiter */
-#define SHEREDQUOTE	11	/* parsing " in <<,<<-,<<< delimiter */
+#define SHEREDELIM	10	/* parsing << or <<- delimiter */
+#define SHEREDQUOTE	11	/* parsing " in << or <<- delimiter */
 #define SPATTERN	12	/* parsing *(...|...) pattern (*+?@!) */
 #define SADELIM		13	/* like SBASE, looking for delimiter */
 #define STBRACEKORN	14	/* parsing ${...[#%]...} !FSH */
@@ -98,7 +98,7 @@ static int s_get(void);
 static void s_put(int);
 static char *get_brace_var(XString *, char *);
 static bool arraysub(char **);
-static void gethere(bool);
+static void gethere(void);
 static Lex_state *push_state_i(State_info *, Lex_state *);
 static Lex_state *pop_state_i(State_info *, Lex_state *);

@@ -240,20 +240,6 @@ yylex(int cf)
 	/* Initial state: one of SWORD SLETPAREN SHEREDELIM SBASE */
 	statep->type = state;

-	/* check for here string */
-	if (state == SHEREDELIM) {
-		c = getsc();
-		if (c == '<') {
-			state = SHEREDELIM;
-			while ((c = getsc()) == ' ' || c == '\t')
-				;
-			ungetsc(c);
-			c = '<';
-			goto accept_nonword;
-		}
-		ungetsc(c);
-	}
-
 	/* collect non-special or quoted characters to form word */
 	while (!((c = getsc()) == 0 ||
 	    ((state == SBASE || state == SHEREDELIM) && ctype(c, C_LEX1)))) {
@@ -262,7 +248,6 @@ yylex(int cf)
 		    c == /*{*/ '}')
 			/* possibly end ${ :;} */
 			break;
- accept_nonword:
 		Xcheck(ws, wp);
 		switch (state) {
 		case SADELIM:
@@ -806,7 +791,7 @@ yylex(int cf)
 				++statep->nparen;
 			goto Sbase2;

-		/* <<, <<-, <<< delimiter */
+		/* << or <<- delimiter */
 		case SHEREDELIM:
 			/*
 			 * here delimiters need a special case since
@@ -844,7 +829,7 @@ yylex(int cf)
 			}
 			break;

-		/* " in <<, <<-, <<< delimiter */
+		/* " in << or <<- delimiter */
 		case SHEREDQUOTE:
 			if (c != '"')
 				goto Subst;
@@ -941,22 +926,12 @@ yylex(int cf)
 			iop->ioflag |= c == c2 ?
 			    (c == '>' ? IOCAT : IOHERE) : IORDWR;
 			if (iop->ioflag == IOHERE) {
-				if ((c2 = getsc()) == '-') {
+				if ((c2 = getsc()) == '-')
 					iop->ioflag |= IOSKIP;
-					c2 = getsc();
-				} else if (c2 == '<')
+				else if (c2 == '<')
 					iop->ioflag |= IOHERESTR;
-				if (c2 == ' ') {
-					/*XXX reentrancy hack IONDELIM */
-					c2 = getsc();
-					if (c2 != '\n') {
-						ungetsc(c2);
-						c2 = ' ';
-					}
-				}
-				ungetsc(c2);
-				if (c2 == '\n')
-					iop->ioflag |= IONDELIM;
+				else
+					ungetsc(c2);
 			}
 		} else if (c2 == '&')
 			iop->ioflag |= IODUP | (c == '<' ? IORDUP : 0);
@@ -1006,12 +981,14 @@ yylex(int cf)
 			}
 #endif
 		} else if (c == '\n') {
-			gethere(false);
-			if (cf & CONTIN)
-				goto Again;
-		} else if (c == '\0')
-			/* need here strings at EOF */
-			gethere(true);
+			if (cf & HEREDELIM)
+				ungetsc(c);
+			else {
+				gethere();
+				if (cf & CONTIN)
+					goto Again;
+			}
+		}
 		return (c);
 	}

@@ -1034,23 +1011,8 @@ yylex(int cf)
 	/* copy word to unprefixed string ident */
 	sp = yylval.cp;
 	dp = ident;
-	if ((cf & HEREDELIM) && (sp[1] == '<')) {
- herestringloop:
-		switch ((c = *sp++)) {
-		case CHAR:
-			++sp;
-			/* FALLTHROUGH */
-		case OQUOTE:
-		case CQUOTE:
-			goto herestringloop;
-		default:
-			break;
-		}
-		/* dummy value */
-		*dp++ = 'x';
-	} else
-		while ((dp - ident) < IDENT && (c = *sp++) == CHAR)
-			*dp++ = *sp++;
+	while ((dp - ident) < IDENT && (c = *sp++) == CHAR)
+		*dp++ = *sp++;
 	if (c != EOS)
 		/* word is not unquoted */
 		dp = ident;
@@ -1126,15 +1088,12 @@ yylex(int cf)
 }

 static void
-gethere(bool iseof)
+gethere(void)
 {
 	struct ioword **p;

 	for (p = heres; p < herep; p++)
-		if (iseof && !((*p)->ioflag & IOHERESTR))
-			/* only here strings at EOF */
-			return;
-		else
+		if (!((*p)->ioflag & IOHERESTR))
 			readhere(*p);
 	herep = heres;
 }
@@ -1152,16 +1111,7 @@ readhere(struct ioword *iop)
 	char *xp;
 	size_t xpos;

-	if (iop->ioflag & IOHERESTR) {
-		/* process the here string */
-		iop->heredoc = xp = evalstr(iop->delim, DOBLANK);
-		xpos = strlen(xp) - 1;
-		memmove(xp, xp + 1, xpos);
-		xp[xpos] = '\n';
-		return;
-	}
-
-	eof = iop->ioflag & IONDELIM ? "<<" : evalstr(iop->delim, 0);
+	eof = evalstr(iop->delim, 0);

 	if (!(iop->ioflag & IOEVAL))
 		ignore_backslash_newline++;
--- a/sh.h
+++ b/sh.h
@@ -172,9 +172,9 @@
 #endif

 #ifdef EXTERN
-__RCSID("$MirOS: src/bin/mksh/sh.h,v 1.743 2015/09/05 20:20:46 tg Exp $");
+__RCSID("$MirOS: src/bin/mksh/sh.h,v 1.744 2015/09/06 19:47:00 tg Exp $");
 #endif
-#define MKSH_VERSION "R51 2015/09/05"
+#define MKSH_VERSION "R51 2015/09/06"

 /* arithmetic types: C implementation */
 #if !HAVE_CAN_INTTYPES
@@ -1454,6 +1454,7 @@ struct ioword {
 #define DOTCOMEXEC BIT(11)	/* not an eval flag, used by sh -c hack */
 #define DOSCALAR BIT(12)	/* change field handling to non-list context */
 #define DOHEREDOC BIT(13)	/* change scalar handling to heredoc body */
+#define DOHERESTR BIT(14)	/* append a newline char */

 #define X_EXTRA	20	/* this many extra bytes in X string */

--- a/syn.c
+++ b/syn.c
@@ -23,7 +23,7 @@

 #include "sh.h"

-__RCSID("$MirOS: src/bin/mksh/syn.c,v 1.103 2015/09/05 19:19:11 tg Exp $");
+__RCSID("$MirOS: src/bin/mksh/syn.c,v 1.104 2015/09/06 19:47:01 tg Exp $");

 struct nesting_state {
 	int start_token;	/* token than began nesting (eg, FOR) */
@@ -172,6 +172,8 @@ c_list(bool multi)
 	return (t);
 }

+static const char IONDELIM_delim[] = { CHAR, '<', CHAR, '<', EOS };
+
 static struct ioword *
 synio(int cf)
 {
@@ -189,15 +191,19 @@ synio(int cf)
 		return (NULL);
 	ACCEPT;
 	iop = yylval.iop;
-	if (iop->ioflag & IONDELIM)
-		goto gotnulldelim;
 	ishere = (iop->ioflag & IOTYPE) == IOHERE;
-	musthave(LWORD, ishere ? HEREDELIM : 0);
+	if (iop->ioflag & IOHERESTR) {
+		musthave(LWORD, 0);
+	} else if (ishere && tpeek(HEREDELIM) == '\n') {
+		ACCEPT;
+		yylval.cp = wdcopy(IONDELIM_delim, ATEMP);
+		iop->ioflag |= IOEVAL | IONDELIM;
+	} else
+		musthave(LWORD, ishere ? HEREDELIM : 0);
 	if (ishere) {
 		iop->delim = yylval.cp;
-		if (*ident != 0) {
+		if (*ident != 0 && !(iop->ioflag & IOHERESTR)) {
 			/* unquoted */
- gotnulldelim:
 			iop->ioflag |= IOEVAL;
 		}
 		if (herep > &heres[HERES - 1])
--- a/tree.c
+++ b/tree.c
@@ -23,7 +23,7 @@

 #include "sh.h"

-__RCSID("$MirOS: src/bin/mksh/tree.c,v 1.76 2015/09/05 20:20:48 tg Exp $");
+__RCSID("$MirOS: src/bin/mksh/tree.c,v 1.77 2015/09/06 19:47:01 tg Exp $");

 #define INDENT	8

@@ -226,7 +226,6 @@ ptree(struct op *t, int indent, struct shf *shf)
 				shf_putc('\n', shf);
 				shf_puts(iop->heredoc, shf);
 				fptreef(shf, indent, "%s",
-				    iop->ioflag & IONDELIM ? "<<" :
 				    evalstr(iop->delim, 0));
 				need_nl = true;
 			}
@@ -265,6 +264,8 @@ pioact(struct shf *shf, struct ioword *iop)
 		shf_puts("<<", shf);
 		if (flag & IOSKIP)
 			shf_putc('-', shf);
+		else if (flag & IOHERESTR)
+			shf_putc('<', shf);
 		break;
 	case IOCAT:
 		shf_puts(">>", shf);
@@ -283,9 +284,8 @@ pioact(struct shf *shf, struct ioword *iop)
 	}
 	/* name/delim are NULL when printing syntax errors */
 	if (type == IOHERE) {
-		if (iop->delim)
+		if (iop->delim && !(iop->ioflag & IONDELIM))
 			wdvarput(shf, iop->delim, 0, WDS_TPUTS);
-		/*XXX see IONDELIM hack */
 	} else if (iop->name) {
 		if (flag & IONAMEXP)
 			print_value_quoted(shf, iop->name);
@@ -931,7 +931,7 @@ dumpioact(struct shf *shf, struct op *t)
 		DB(IOHERESTR)
 		DB(IONDELIM)
 		shf_fprintf(shf, ",unit=%d", (int)iop->unit);
-		if (iop->delim) {
+		if (iop->delim && !(iop->ioflag & IONDELIM)) {
 			shf_puts(",delim<", shf);
 			dumpwdvar(shf, iop->delim);
 			shf_putc('>', shf);