make ${foo#'bar'} in here document behave like ksh93

reported by Martijn Dekker <martijn@inlv.org>

check.t

@@ -1,8 +1,8 @@
-# $MirOS: src/bin/mksh/check.t,v 1.801 2018/01/14 01:47:33 tg Exp $
+# $MirOS: src/bin/mksh/check.t,v 1.802 2018/03/09 01:29:08 tg Exp $
 # -*- mode: sh -*-
 #-
 # Copyright © 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010,
-#	      2011, 2012, 2013, 2014, 2015, 2016, 2017
+#	      2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018
 #	mirabilos <m@mirbsd.org>
 #
 # Provided that these terms and disclaimer and all copyright notices
@@ -30,7 +30,7 @@
 # (2013/12/02 20:39:44) http://cvsweb.openbsd.org/cgi-bin/cvsweb/src/regress/bin/ksh/?sortby=date
 
 expected-stdout:
-	@(#)MIRBSD KSH R56 2018/01/14
+	@(#)MIRBSD KSH R56 2018/03/09
 description:
 	Check base version of full shell
 stdin:
@@ -39,7 +39,7 @@ name: KSH_VERSION
 category: !shell:legacy-yes
 ---
 expected-stdout:
-	@(#)LEGACY KSH R56 2018/01/14
+	@(#)LEGACY KSH R56 2018/03/09
 description:
 	Check base version of legacy shell
 stdin:
@@ -3526,6 +3526,25 @@ stdin:
 expected-stdout:
 	'blah  1'
 ---
+name: single-quotes-in-heredoc-trim
+description:
+	In some cases, single quotes inside {} in heredoc are not normal
+stdin:
+	x=notOK
+	cat <<EOF
+	1: ${x#not} ${x:+${x#not}}
+	2: ${x#\n\o\t} ${x:+${x#\n\o\t}}
+	3: ${x#"not"} ${x:+${x#"not"}}
+	4: ${x#'not'} ${x:+${x#'not'}}
+	5: ${x#$'not'} ${x:+${x#$'not'}}
+	EOF
+expected-stdout:
+	1: OK OK
+	2: OK OK
+	3: OK OK
+	4: OK OK
+	5: OK OK
+---
 name: history-basic
 description:
 	See if we can test history at all

lex.c

@@ -23,7 +23,7 @@
 
 #include "sh.h"
 
-__RCSID("$MirOS: src/bin/mksh/lex.c,v 1.247 2018/01/14 01:44:01 tg Exp $");
+__RCSID("$MirOS: src/bin/mksh/lex.c,v 1.248 2018/03/09 01:29:11 tg Exp $");
 
 /*
  * states while lexing word
@@ -77,12 +77,17 @@ typedef struct lex_state {
 	short nparen;
 	/* type of this state */
 	uint8_t type;
+	/* extra flags */
+	uint8_t ls_flags;
 } Lex_state;
 #define ls_base		u.base
 #define ls_start	u.start
 #define ls_bool		u.abool
 #define ls_adelim	u.adelim
 
+/* ls_flags */
+#define LS_HEREDOC	BIT(0)
+
 typedef struct {
 	Lex_state *base;
 	Lex_state *end;
@@ -147,9 +152,11 @@ getsc_r(int c)
 #define STATE_BSIZE	8
 
 #define PUSH_STATE(s)	do {					\
+	uint8_t state_flags = statep->ls_flags;			\
 	if (++statep == state_info.end)				\
 		statep = push_state_i(&state_info, statep);	\
 	state = statep->type = (s);				\
+	statep->ls_flags = state_flags;				\
 } while (/* CONSTCOND */ 0)
 
 #define POP_STATE()	do {					\
@@ -242,6 +249,7 @@ yylex(int cf)
 
 	/* Initial state: one of SWORD SLETPAREN SHEREDELIM SBASE */
 	statep->type = state;
+	statep->ls_flags = (cf & HEREDOC) ? LS_HEREDOC : 0;
 
 	/* collect non-special or quoted characters to form word */
 	while (!((c = getsc()) == 0 ||
@@ -319,7 +327,7 @@ yylex(int cf)
 				break;
 			case ORD('\''):
  open_ssquote_unless_heredoc:
-				if ((cf & HEREDOC))
+				if ((statep->ls_flags & LS_HEREDOC))
 					goto store_char;
 				*wp++ = OQUOTE;
 				ignore_backslash_newline++;
@@ -357,7 +365,7 @@ yylex(int cf)
 				c = getsc();
 				switch (c) {
 				case ORD('"'):
-					if ((cf & HEREDOC))
+					if ((statep->ls_flags & LS_HEREDOC))
 						goto heredocquote;
 					/* FALLTHROUGH */
 				case ORD('\\'):
@@ -388,6 +396,8 @@ yylex(int cf)
 					if ((unsigned int)c == ORD('(' /*)*/)) {
 						*wp++ = EXPRSUB;
 						PUSH_SRETRACE(SASPAREN);
+						/* unneeded? */
+						/*statep->ls_flags &= ~LS_HEREDOC;*/
 						statep->nparen = 2;
 						*retrace_info->xp++ = '(';
 					} else {
@@ -434,6 +444,8 @@ yylex(int cf)
 							*wp++ = ADELIM;
 							*wp++ = ':';
 							PUSH_STATE(SBRACE);
+							/* perhaps unneeded? */
+							statep->ls_flags &= ~LS_HEREDOC;
 							PUSH_STATE(SADELIM);
 							statep->ls_adelim.delimiter = ':';
 							statep->ls_adelim.num = 1;
@@ -449,6 +461,8 @@ yylex(int cf)
 							}
 							ungetsc(c);
 							PUSH_STATE(SBRACE);
+							/* perhaps unneeded? */
+							statep->ls_flags &= ~LS_HEREDOC;
 							PUSH_STATE(SADELIM);
 							statep->ls_adelim.delimiter = ':';
 							statep->ls_adelim.num = 2;
@@ -466,6 +480,8 @@ yylex(int cf)
 						} else
 							ungetsc(c);
 						PUSH_STATE(SBRACE);
+						/* perhaps unneeded? */
+						statep->ls_flags &= ~LS_HEREDOC;
 						PUSH_STATE(SADELIM);
 						statep->ls_adelim.delimiter = '/';
 						statep->ls_adelim.num = 1;
@@ -489,6 +505,8 @@ yylex(int cf)
 							PUSH_STATE(STBRACEBOURNE);
 						else
 							PUSH_STATE(STBRACEKORN);
+						/* single-quotes-in-heredoc-trim */
+						statep->ls_flags &= ~LS_HEREDOC;
 					} else {
 						ungetsc(c);
 						if (state == SDQUOTE ||
@@ -496,6 +514,8 @@ yylex(int cf)
 							PUSH_STATE(SQBRACE);
 						else
 							PUSH_STATE(SBRACE);
+						/* here no LS_HEREDOC removal */
+						/* single-quotes-in-heredoc-braces */
 					}
 				} else if (ctype(c, C_ALPHX)) {
 					*wp++ = OSUBST;
@@ -601,7 +621,8 @@ yylex(int cf)
 		case SSQUOTE:
 			if ((unsigned int)c == ORD('\'')) {
 				POP_STATE();
-				if ((cf & HEREDOC) || state == SQBRACE)
+				if ((statep->ls_flags & LS_HEREDOC) ||
+				    state == SQBRACE)
 					goto store_char;
 				*wp++ = CQUOTE;
 				ignore_backslash_newline--;

sh.h

@@ -182,9 +182,9 @@
 #endif
 
 #ifdef EXTERN
-__RCSID("$MirOS: src/bin/mksh/sh.h,v 1.858 2018/01/14 01:47:36 tg Exp $");
+__RCSID("$MirOS: src/bin/mksh/sh.h,v 1.859 2018/03/09 01:29:11 tg Exp $");
 #endif
-#define MKSH_VERSION "R56 2018/01/14"
+#define MKSH_VERSION "R56 2018/03/09"
 
 /* arithmetic types: C implementation */
 #if !HAVE_CAN_INTTYPES