(hopefully) fix the substitution thing for good

check.t

@@ -1,4 +1,4 @@
-# $MirOS: src/bin/mksh/check.t,v 1.364 2010/02/25 11:47:33 tg Exp $
+# $MirOS: src/bin/mksh/check.t,v 1.365 2010/02/25 20:18:14 tg Exp $
 # $OpenBSD: bksl-nl.t,v 1.2 2001/01/28 23:04:56 niklas Exp $
 # $OpenBSD: history.t,v 1.5 2001/01/28 23:04:56 niklas Exp $
 # $OpenBSD: read.t,v 1.3 2003/03/10 03:48:16 david Exp $
@@ -25,7 +25,7 @@
 # http://www.research.att.com/~gsf/public/ifs.sh
 
 expected-stdout:
-	@(#)MIRBSD KSH R39 2010/02/24
+	@(#)MIRBSD KSH R39 2010/02/25
 description:
 	Check version of shell.
 stdin:
@@ -1002,8 +1002,7 @@ expected-stderr: !
 ---
 name: expand-ugly
 description:
-	Check that ${foo+bar} constructs are parsed correctly
+	Check that weird ${foo+bar} constructs are parsed correctly
-	XXX pretty sure that 28 is wrong, but still under discussion
 stdin:
 	(echo 1 ${IFS+'}'z}) 2>&- || echo failed in 1
 	(echo 2 "${IFS+'}'z}") 2>&- || echo failed in 2
@@ -1019,7 +1018,7 @@ stdin:
 	(echo 12 "$(echo ${IFS+'}'z})") 2>&- || echo failed in 12
 	(echo 13 ${IFS+\}z}) 2>&- || echo failed in 13
 	(echo 14 "${IFS+\}z}") 2>&- || echo failed in 14
-	u=x; (echo 15 "foo ${IFS+a$u{{{\}b} c ${IFS+d{}} bar" ${IFS-e{}} baz) 2>&- || echo failed in 15
+	u=x; (echo -n '15 '; printf '<%s> ' "foo ${IFS+a"b$u{ {"{{\}b} c ${IFS+d{}} bar" ${IFS-e{}} baz; echo .) 2>&- || echo failed in 15
 	l=t; (echo 16 ${IFS+h`echo -n i ${IFS+$l}h`ere}) 2>&- || echo failed in 16
 	l=t; (echo 17 ${IFS+h$(echo -n i ${IFS+$l}h)ere}) 2>&- || echo failed in 17
 	l=t; (echo 18 "${IFS+h`echo -n i ${IFS+$l}h`ere}") 2>&- || echo failed in 18
@@ -1033,6 +1032,15 @@ stdin:
 	key=value; (echo -n '26 '; printf '%s\n' ${IFS+'$key'}) 2>&- || echo failed in 26
 	key=value; (echo -n '27 '; printf '%s\n' ${IFS+"'$key'"}) 2>&- || echo failed in 27
 	(echo -n '28 '; printf '%s\n' "${IFS+"'"x ~ x'}'x"'}"x}" #') 2>&- || echo failed in 28
+	u=x; (echo -n '29 '; printf '<%s> ' foo ${IFS+a"b$u{ {"{ {\}b} c ${IFS+d{}} bar ${IFS-e{}} baz; echo .) 2>&- || echo failed in 29
+	(echo -n '30 '; printf '<%s> ' ${IFS+foo 'b\
+	ar' baz}; echo .) 2>&- || (echo failed in 30; echo failed in 31)
+	(echo -n '32 '; printf '<%s> ' ${IFS+foo "b\
+	ar" baz}; echo .) 2>&- || (echo failed in 32; echo failed in 33)
+	(echo -n '34 '; printf '<%s> ' "${IFS+foo 'b\
+	ar' baz}"; echo .) 2>&- || (echo failed in 34; echo failed in 35)
+	(echo -n '36 '; printf '<%s> ' "${IFS+foo "b\
+	ar" baz}"; echo .) 2>&- || (echo failed in 36; echo failed in 37)
 expected-stdout:
 	1 }z
 	2 ''z}
@@ -1048,7 +1056,7 @@ expected-stdout:
 	12 }z
 	13 }z
 	14 }z
-	15 foo ax{{{}b c d{} bar } baz
+	15 <foo abx{ {{{}b c d{} bar> <}> <baz> .
 	16 hi there
 	17 hi there
 	18 hi there
@@ -1060,14 +1068,18 @@ expected-stdout:
 	24 'value'
 	25 'value'
 	26 $key
-	27 '$key'
+	27 'value'
-	28 'x
+	28 'x ~ x''x}"x}" #
-	~
+	29 <foo> <abx{ {{> <{}b> <c> <d{}> <bar> <}> <baz> .
-	x''x}"x}" #
+	30 <foo> <b\
+	ar> <baz> .
+	32 <foo> <bar> <baz> .
+	34 <foo 'bar' baz> .
+	36 <foo bar baz> .
 ---
 name: expand-unglob-dblq
 description:
-	Check that ${foo+bar} constructs are parsed correctly
+	Check that regular "${foo+bar}" constructs are parsed correctly
 stdin:
 	u=x
 	tl_norm() {
@@ -1197,8 +1209,7 @@ expected-stdout:
 ---
 name: expand-unglob-unq
 description:
-	Check that ${foo+bar} constructs are parsed correctly
+	Check that regular ${foo+bar} constructs are parsed correctly
-	AT&T ksh93 slightly fails this
 stdin:
 	u=x
 	tl_norm() {

eval.c

@@ -22,7 +22,7 @@
 
 #include "sh.h"
 
-__RCSID("$MirOS: src/bin/mksh/eval.c,v 1.82 2010/01/29 09:34:27 tg Exp $");
+__RCSID("$MirOS: src/bin/mksh/eval.c,v 1.83 2010/02/25 20:18:15 tg Exp $");
 
 /*
  * string expansion
@@ -263,7 +263,7 @@ expand(const char *cp,	/* input word */
 				quote = 1;
 				continue;
 			case CQUOTE:
-				quote = 0;
+				quote = st->quote;	/* XXX correct? */
 				continue;
 			case COMSUB:
 				tilde_ok = 0;

lex.c

@@ -22,7 +22,7 @@
 
 #include "sh.h"
 
-__RCSID("$MirOS: src/bin/mksh/lex.c,v 1.109 2010/02/23 21:51:49 tg Exp $");
+__RCSID("$MirOS: src/bin/mksh/lex.c,v 1.110 2010/02/25 20:18:16 tg Exp $");
 
 /*
  * states while lexing word
@@ -321,20 +321,21 @@ yylex(int cf)
  Sbase2:		/* doesn't include *(...|...) pattern (*+?@!) */
 			switch (c) {
 			case '\\':
-				c = getsc();
+ getsc_qchar:
-				if (c) {
+				if ((c = getsc())) {
 					/* trailing \ is lost */
 					*wp++ = QCHAR;
 					*wp++ = c;
 				}
 				break;
 			case '\'':
+ open_ssquote:
 				*wp++ = OQUOTE;
 				ignore_backslash_newline++;
 				PUSH_STATE(SSQUOTE);
 				break;
 			case '"':
- open_dquote:
+ open_sdquote:
 				*wp++ = OQUOTE;
 				PUSH_STATE(SDQUOTE);
 				break;
@@ -371,6 +372,7 @@ yylex(int cf)
 				}
 				break;
 			case '$':
+ subst_dollar:
 				c = getsc();
 				if (c == '(') /*)*/ {
 					c = getsc();
@@ -489,6 +491,7 @@ yylex(int cf)
 				}
 				break;
 			case '`':
+ subst_gravis:
 				PUSH_STATE(SBQUOTE);
 				*wp++ = COMSUB;
 				/* Need to know if we are inside double quotes
@@ -536,6 +539,7 @@ yylex(int cf)
 				}
 				/* FALLTHROUGH */
 			default:
+ store_char:
 				*wp++ = CHAR;
 				*wp++ = c;
 			}
@@ -684,37 +688,23 @@ yylex(int cf)
 			break;
 
 		case SBRACE:
+			if (c == '\'')
+				goto open_ssquote;
+			/* FALLTHROUGH */
 		case SQBRACE:
-			if (c == /*{*/ '}') {
+			if (c == '"')
+				goto open_sdquote;
+			else if (c == '\\')
+				goto getsc_qchar;
+			else if (c == '$')
+				goto subst_dollar;
+			else if (c == '`')
+				goto subst_gravis;
+			else if (c != /*{*/ '}')
+				goto store_char;
 			POP_STATE();
 			*wp++ = CSUBST;
 			*wp++ = /*{*/ '}';
-			} else if (c == '\\') {
-				if ((c = getsc())) {
-					*wp++ = QCHAR;
-					*wp++ = c;
-				}
-			} else if (c == '"') {
-				if (state == SQBRACE)
-					goto open_dquote;
- sbrace_quote:
-				c2 = c;
-				for (;;) {
-					if (!(c = getsc()))
-						goto Done;
-					if (c == c2)
-						break;
-					*wp++ = QCHAR;
-					*wp++ = c;
-				}
-			} else if (state == SBRACE && c == '\'') {
-				goto sbrace_quote;
-			} else if (c == '$' || c == '`') {
-				goto Subst;
-			} else {
-				*wp++ = CHAR;
-				*wp++ = c;
-			}
 			break;
 
 		case STBRACE:

mksh.1

@@ -1,4 +1,4 @@
-.\" $MirOS: src/bin/mksh/mksh.1,v 1.215 2010/01/28 20:58:33 tg Exp $
+.\" $MirOS: src/bin/mksh/mksh.1,v 1.216 2010/02/25 20:18:16 tg Exp $
 .\" $OpenBSD: ksh.1,v 1.129 2009/05/28 06:09:06 jmc Exp $
 .\"-
 .\" Copyright © 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010
@@ -71,7 +71,7 @@
 .\" with -mandoc, it might implement .Mx itself, but we want to
 .\" use our own definition. And .Dd must come *first*, always.
 .\"
-.Dd $Mdocdate: January 28 2010 $
+.Dd $Mdocdate: February 25 2010 $
 .\"
 .\" Check which macro package we use
 .\"
@@ -1416,6 +1416,13 @@ is a variable which expands to the null string, the
 error message is also printed.
 .El
 .Pp
+Note that, for all of the above,
+.Ar word
+is actually considered quoted, and special parsing rules apply.
+The parsing rules also differ on whether the expression is
+double-quoted with respect to the single quote
+.Pq Dq \*(aq .
+.Pp
 In the above modifiers, the
 .Ql \&:
 can be omitted, in which case the conditions only depend on
@@ -5992,7 +5999,7 @@ foo \*(Ba bar \*(Ba read baz    # will not change $baz
 .Ed
 .Pp
 This document attempts to describe
-.Nm mksh\ R39b
+.Nm mksh\ R39c
 and up,
 compiled without any options impacting functionality, such as
 .Dv MKSH_SMALL ,

sh.h

@@ -150,9 +150,9 @@
 #endif
 
 #ifdef EXTERN
-__RCSID("$MirOS: src/bin/mksh/sh.h,v 1.382 2010/02/23 18:13:04 tg Exp $");
+__RCSID("$MirOS: src/bin/mksh/sh.h,v 1.383 2010/02/25 20:18:17 tg Exp $");
 #endif
-#define MKSH_VERSION "R39 2010/02/23"
+#define MKSH_VERSION "R39 2010/02/25"
 
 #ifndef MKSH_INCLUDES_ONLY
 

tree.c

@@ -22,7 +22,7 @@
 
 #include "sh.h"
 
-__RCSID("$MirOS: src/bin/mksh/tree.c,v 1.29 2010/01/29 09:34:31 tg Exp $");
+__RCSID("$MirOS: src/bin/mksh/tree.c,v 1.30 2010/02/25 20:18:19 tg Exp $");
 
 #define INDENT	4
 
@@ -271,8 +271,7 @@ tputC(int c, struct shf *shf)
 static void
 tputS(char *wp, struct shf *shf)
 {
-	int c;
+	int c, quotelevel = 0;
-	bool quoted = false;
 
 	/* problems:
 	 *	`...` -> $(...)
@@ -291,7 +290,7 @@ tputS(char *wp, struct shf *shf)
 			break;
 		case QCHAR:
 			c = *wp++;
-			if (!quoted || (c == '"' || c == '`' || c == '$'))
+			if (!quotelevel || (c == '"' || c == '`' || c == '$'))
 				tputc('\\', shf);
 			tputC(c, shf);
 			break;
@@ -310,11 +309,12 @@ tputS(char *wp, struct shf *shf)
 			wp++;
 			break;
 		case OQUOTE:
-			quoted = true;
+			quotelevel++;
 			tputc('"', shf);
 			break;
 		case CQUOTE:
-			quoted = false;
+			if (quotelevel)
+				quotelevel--;
 			tputc('"', shf);
 			break;
 		case OSUBST: