Merge remote-tracking branch 'mksh/master'

check.t

@@ -1,4 +1,4 @@
-# $MirOS: src/bin/mksh/check.t,v 1.728 2016/03/05 15:39:36 tg Exp $
+# $MirOS: src/bin/mksh/check.t,v 1.731 2016/05/05 22:58:19 tg Exp $
 # -*- mode: sh -*-
 #-
 # Copyright © 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010,
@@ -30,7 +30,7 @@
 # (2013/12/02 20:39:44) http://openbsd.cs.toronto.edu/cgi-bin/cvsweb/src/regress/bin/ksh/?sortby=date
 
 expected-stdout:
-	@(#)MIRBSD KSH R52 2016/03/04
+	@(#)MIRBSD KSH R52 2016/05/05
 description:
 	Check version of shell.
 stdin:
@@ -39,7 +39,7 @@ name: KSH_VERSION
 category: shell:legacy-no
 ---
 expected-stdout:
-	@(#)LEGACY KSH R52 2016/03/04
+	@(#)LEGACY KSH R52 2016/05/05
 description:
 	Check version of legacy shell.
 stdin:
@@ -1687,6 +1687,50 @@ stdin:
 expected-stdout:
 	<~/x> </x> <~> <\~> <~><~> <~/x> <~//etc> <~/~>
 ---
+name: expand-bang-1
+description:
+	Check corner case of ${!?} with ! being var vs. op
+stdin:
+	echo ${!?}
+expected-exit: 1
+expected-stderr-pattern: /not set/
+---
+name: expand-bang-2
+description:
+	Check corner case of ${!var} vs. ${var op} with var=!
+stdin:
+	echo 1 $! .
+	echo 2 ${!#} .
+	echo 3 ${!#[0-9]} .
+	echo 4 ${!-foo} .
+	# get an at least three-digit bg pid
+	while :; do
+		:&
+		x=$!
+		if [[ $x != +([0-9]) ]]; then
+			echo >&2 "cannot test, pid '$x' not numeric"
+			echo >&2 report this with as many details as possible
+			exit 1
+		fi
+		[[ $x = [0-9][0-9][0-9]* ]] && break
+	done
+	y=${x#?}
+	t=$!; [[ $t = $x ]]; echo 5 $? .
+	t=${!#}; [[ $t = $x ]]; echo 6 $? .
+	t=${!#[0-9]}; [[ $t = $y ]]; echo 7 $? .
+	t=${!-foo}; [[ $t = $x ]]; echo 8 $? .
+	t=${!?bar}; [[ $t = $x ]]; echo 9 $? .
+expected-stdout:
+	1 .
+	2 .
+	3 .
+	4 foo .
+	5 0 .
+	6 0 .
+	7 0 .
+	8 0 .
+	9 0 .
+---
 name: expand-number-1
 description:
 	Check that positional arguments do not overflow
@@ -3173,6 +3217,37 @@ expected-stdout:
 expected-stderr-pattern:
 	/(.*can't unlink HISTFILE.*\n)?X*$/
 ---
+name: history-multiline
+description:
+	Check correct multiline history, Debian #783978
+need-ctty: yes
+arguments: !-i!
+env-setup: !ENV=./Env!
+file-setup: file 644 "Env"
+	PS1=X
+	PS2=Y
+stdin:
+	for i in A B C
+	do
+	   print $i
+	   print $i
+	done
+	fc -l
+expected-stdout:
+	A
+	A
+	B
+	B
+	C
+	C
+	1	for i in A B C
+		do
+		   print $i
+		   print $i
+		done
+expected-stderr-pattern:
+	/^XYYYYXX$/
+---
 name: history-e-minus-1
 description:
 	Check if more recent command is executed
@@ -7775,6 +7850,27 @@ expected-stdout:
 	2 off
 	3 done
 ---
+name: utf8bug-1
+description:
+	Ensure trailing combining characters are not lost
+stdin:
+	set -U
+	a=a
+	b=$'\u0301'
+	x=$a$b
+	print -r -- "<e$x>"
+	x=$a
+	x+=$b
+	print -r -- "<e$x>"
+	b=$'\u0301'b
+	x=$a
+	x+=$b
+	print -r -- "<e$x>"
+expected-stdout:
+	<eá>
+	<eá>
+	<eáb>
+---
 name: aliases-1
 description:
 	Check if built-in shell aliases are okay

dot.mkshrc

@@ -1,8 +1,8 @@
 # $Id$
-# $MirOS: src/bin/mksh/dot.mkshrc,v 1.104 2015/12/31 21:00:12 tg Exp $
+# $MirOS: src/bin/mksh/dot.mkshrc,v 1.105 2016/04/09 16:33:23 tg Exp $
 #-
 # Copyright (c) 2002, 2003, 2004, 2006, 2007, 2008, 2009, 2010,
-#		2011, 2012, 2013, 2014, 2015
+#		2011, 2012, 2013, 2014, 2015, 2016
 #	mirabilos <m@mirbsd.org>
 #
 # Provided that these terms and disclaimer and all copyright notices
@@ -250,25 +250,23 @@ function pushd {
 }
 
 # pager (not control character safe)
-function smores {
+smores() (
-	(
+	\set +m
-		\set +m
+	\cat "$@" |&
-		\cat "$@" |&
+	\trap "rv=\$?; 'kill' $! >/dev/null 2>&1; 'exit' \$rv" EXIT
-		\trap "rv=\$?; 'kill' $! >/dev/null 2>&1; 'exit' \$rv" EXIT
+	while IFS= \read -pr line; do
-		while IFS= \read -pr line; do
+		llen=${%line}
-			llen=${%line}
+		(( llen == -1 )) && llen=${#line}
-			(( llen == -1 )) && llen=${#line}
+		(( llen = llen ? (llen + COLUMNS - 1) / COLUMNS : 1 ))
-			(( llen = llen ? (llen + COLUMNS - 1) / COLUMNS : 1 ))
+		if (( (curlin += llen) >= LINES )); then
-			if (( (curlin += llen) >= LINES )); then
+			\builtin print -n -- '\e[7m--more--\e[0m'
-				\builtin print -n -- '\e[7m--more--\e[0m'
+			\read -u1 || \exit $?
-				\read -u1 || \exit $?
+			[[ $REPLY = [Qq]* ]] && \exit 0
-				[[ $REPLY = [Qq]* ]] && \exit 0
+			curlin=$llen
-				curlin=$llen
+		fi
-			fi
+		\builtin print -r -- "$line"
-			\builtin print -r -- "$line"
+	done
-		done
+)
-	)
-}
 
 # base64 encoder and decoder, RFC compliant, NUL safe, not EBCDIC safe
 function Lb64decode {

edit.c

@@ -28,7 +28,7 @@
 
 #ifndef MKSH_NO_CMDLINE_EDITING
 
-__RCSID("$MirOS: src/bin/mksh/edit.c,v 1.294 2016/03/04 14:26:12 tg Exp $");
+__RCSID("$MirOS: src/bin/mksh/edit.c,v 1.296 2016/05/05 22:56:12 tg Exp $");
 
 /*
  * in later versions we might use libtermcap for this, but since external
@@ -1580,7 +1580,7 @@ static void
 x_goto(char *cp)
 {
 	cp = cp >= xep ? xep : x_bs0(cp, xbuf);
-	if (cp < xbp || cp >= utf_skipcols(xbp, x_displen)) {
+	if (cp < xbp || cp >= utf_skipcols(xbp, x_displen, NULL)) {
 		/* we are heading off screen */
 		xcp = cp;
 		x_adjust();
@@ -4922,16 +4922,16 @@ forwword(int argcnt)
 	ncursor = es->cursor;
 	while (ncursor < es->linelen && argcnt--) {
 		if (ksh_isalnux(es->cbuf[ncursor]))
-			while (ksh_isalnux(es->cbuf[ncursor]) &&
+			while (ncursor < es->linelen &&
-			    ncursor < es->linelen)
+			    ksh_isalnux(es->cbuf[ncursor]))
 				ncursor++;
 		else if (!ksh_isspace(es->cbuf[ncursor]))
-			while (!ksh_isalnux(es->cbuf[ncursor]) &&
+			while (ncursor < es->linelen &&
-			    !ksh_isspace(es->cbuf[ncursor]) &&
+			    !ksh_isalnux(es->cbuf[ncursor]) &&
-			    ncursor < es->linelen)
+			    !ksh_isspace(es->cbuf[ncursor]))
 				ncursor++;
-		while (ksh_isspace(es->cbuf[ncursor]) &&
+		while (ncursor < es->linelen &&
-		    ncursor < es->linelen)
+		    ksh_isspace(es->cbuf[ncursor]))
 			ncursor++;
 	}
 	return (ncursor);
@@ -4995,11 +4995,11 @@ Forwword(int argcnt)
 
 	ncursor = es->cursor;
 	while (ncursor < es->linelen && argcnt--) {
-		while (!ksh_isspace(es->cbuf[ncursor]) &&
+		while (ncursor < es->linelen &&
-		    ncursor < es->linelen)
+		    !ksh_isspace(es->cbuf[ncursor]))
 			ncursor++;
-		while (ksh_isspace(es->cbuf[ncursor]) &&
+		while (ncursor < es->linelen &&
-		    ncursor < es->linelen)
+		    ksh_isspace(es->cbuf[ncursor]))
 			ncursor++;
 	}
 	return (ncursor);

eval.c

@@ -23,7 +23,7 @@
 
 #include "sh.h"
 
-__RCSID("$MirOS: src/bin/mksh/eval.c,v 1.185 2016/02/26 19:05:21 tg Exp $");
+__RCSID("$MirOS: src/bin/mksh/eval.c,v 1.187 2016/05/05 22:45:57 tg Exp $");
 
 /*
  * string expansion
@@ -1213,67 +1213,65 @@ varsub(Expand *xp, const char *sp, const char *word,
 		}
 		/* POSIX 2009? */
 		zero_ok = true;
-	} else {
+	} else if ((p = cstrchr(sp, '[')) && (p[1] == '*' || p[1] == '@') &&
-		if ((p = cstrchr(sp, '[')) && (p[1] == '*' || p[1] == '@') &&
+	    p[2] == ']') {
-		    p[2] == ']') {
+		XPtrV wv;
-			XPtrV wv;
 
-			switch (stype & 0x17F) {
+		switch (stype & 0x17F) {
-			/* can't assign to a vector */
+		/* can't assign to a vector */
-			case '=':
+		case '=':
-			/* can't trim a vector (yet) */
+		/* can't trim a vector (yet) */
-			case '%':
+		case '%':
-			case '#':
+		case '#':
-			case '?':
+		case '?':
-			case '0':
+		case '0':
-			case '/':
+		case '/':
-			case 0x100 | '#':
+		case 0x100 | '#':
-			case 0x100 | 'Q':
+		case 0x100 | 'Q':
-				return (-1);
+			return (-1);
-			}
-			XPinit(wv, 32);
-			if ((c = sp[0]) == '!')
-				++sp;
-			vp = global(arrayname(sp));
-			for (; vp; vp = vp->u.array) {
-				if (!(vp->flag&ISSET))
-					continue;
-				XPput(wv, c == '!' ? shf_smprintf("%lu",
-				    arrayindex(vp)) :
-				    str_val(vp));
-			}
-			if (XPsize(wv) == 0) {
-				xp->str = null;
-				state = p[1] == '@' ? XNULLSUB : XSUB;
-				XPfree(wv);
-			} else {
-				XPput(wv, 0);
-				xp->u.strv = (const char **)XPptrv(wv);
-				xp->str = *xp->u.strv++;
-				/* ${foo[@]} */
-				xp->split = tobool(p[1] == '@');
-				state = XARG;
-			}
-		} else {
-			/* Can't assign things like $! or $1 */
-			if ((stype & 0x17F) == '=' &&
-			    ctype(*sp, C_VAR1 | C_DIGIT))
-				return (-1);
-			if (*sp == '!' && sp[1]) {
-				++sp;
-				xp->var = global(sp);
-				if (vstrchr(sp, '['))
-					xp->str = shf_smprintf("%s[%lu]",
-					    xp->var->name,
-					    arrayindex(xp->var));
-				else
-					xp->str = xp->var->name;
-			} else {
-				xp->var = global(sp);
-				xp->str = str_val(xp->var);
-			}
-			state = XSUB;
 		}
+		XPinit(wv, 32);
+		if ((c = sp[0]) == '!')
+			++sp;
+		vp = global(arrayname(sp));
+		for (; vp; vp = vp->u.array) {
+			if (!(vp->flag&ISSET))
+				continue;
+			XPput(wv, c == '!' ? shf_smprintf("%lu",
+			    arrayindex(vp)) :
+			    str_val(vp));
+		}
+		if (XPsize(wv) == 0) {
+			xp->str = null;
+			state = p[1] == '@' ? XNULLSUB : XSUB;
+			XPfree(wv);
+		} else {
+			XPput(wv, 0);
+			xp->u.strv = (const char **)XPptrv(wv);
+			xp->str = *xp->u.strv++;
+			/* ${foo[@]} */
+			xp->split = tobool(p[1] == '@');
+			state = XARG;
+		}
+	} else {
+		/* Can't assign things like $! or $1 */
+		if ((stype & 0x17F) == '=' &&
+		    ctype(*sp, C_VAR1 | C_DIGIT))
+			return (-1);
+		if (*sp == '!' && sp[1] && !ctype(sp[1], C_VAR1)) {
+			++sp;
+			xp->var = global(sp);
+			if (vstrchr(sp, '['))
+				xp->str = shf_smprintf("%s[%lu]",
+				    xp->var->name,
+				    arrayindex(xp->var));
+			else
+				xp->str = xp->var->name;
+		} else {
+			xp->var = global(sp);
+			xp->str = str_val(xp->var);
+		}
+		state = XSUB;
 	}
 
 	c = stype & 0x7F;

exec.c

@@ -23,7 +23,7 @@
 
 #include "sh.h"
 
-__RCSID("$MirOS: src/bin/mksh/exec.c,v 1.172 2016/03/01 18:30:04 tg Exp $");
+__RCSID("$MirOS: src/bin/mksh/exec.c,v 1.173 2016/04/09 16:41:07 tg Exp $");
 
 #ifndef MKSH_DEFAULT_EXECSHELL
 #define MKSH_DEFAULT_EXECSHELL	MKSH_UNIXROOT "/bin/sh"
@@ -1111,7 +1111,7 @@ define(const char *name, struct op *t)
 	}
 
 	if (tp->flag & ALLOC) {
-		tp->flag &= ~(ISSET|ALLOC);
+		tp->flag &= ~(ISSET|ALLOC|FKSH);
 		tfree(tp->val.t, tp->areap);
 	}
 

expr.c

@@ -23,7 +23,7 @@
 
 #include "sh.h"
 
-__RCSID("$MirOS: src/bin/mksh/expr.c,v 1.83 2016/03/01 18:29:38 tg Exp $");
+__RCSID("$MirOS: src/bin/mksh/expr.c,v 1.85 2016/05/05 22:56:13 tg Exp $");
 
 /* the order of these enums is constrained by the order of opinfo[] */
 enum token {
@@ -809,15 +809,26 @@ utf_mbswidth(const char *s)
 }
 
 const char *
-utf_skipcols(const char *p, int cols)
+utf_skipcols(const char *p, int cols, int *colp)
 {
 	int c = 0;
+	const char *q;
 
 	while (c < cols) {
-		if (!*p)
+		if (!*p) {
-			return (p + cols - c);
+			/* end of input; special handling for edit.c */
+			if (!colp)
+				return (p + cols - c);
+			*colp = c;
+			return (p);
+		}
 		c += utf_widthadj(p, &p);
 	}
+	if (UTFMODE)
+		while (utf_widthadj(p, &q) == 0)
+			p = q;
+	if (colp)
+		*colp = c;
 	return (p);
 }
 

lex.c

@@ -23,7 +23,7 @@
 
 #include "sh.h"
 
-__RCSID("$MirOS: src/bin/mksh/lex.c,v 1.222 2016/03/01 19:22:31 tg Exp $");
+__RCSID("$MirOS: src/bin/mksh/lex.c,v 1.224 2016/05/05 22:45:58 tg Exp $");
 
 /*
  * states while lexing word
@@ -1350,7 +1350,8 @@ getsc_line(Source *s)
 		alarm(ksh_tmout);
 	}
 	if (interactive) {
-		histsave(&s->line, NULL, HIST_FLUSH, true);
+		if (cur_prompt == PS1)
+			histsave(&s->line, NULL, HIST_FLUSH, true);
 		change_winsz();
 	}
 #ifndef MKSH_NO_CMDLINE_EDITING
@@ -1565,20 +1566,34 @@ get_brace_var(XString *wsp, char *wp)
 {
 	char c;
 	enum parse_state {
-		PS_INITIAL, PS_SAW_HASH, PS_IDENT,
+		PS_INITIAL, PS_SAW_PERCENT, PS_SAW_HASH, PS_SAW_BANG,
-		PS_NUMBER, PS_VAR1
+		PS_IDENT, PS_NUMBER, PS_VAR1
 	} state = PS_INITIAL;
 
 	while (/* CONSTCOND */ 1) {
 		c = getsc();
 		/* State machine to figure out where the variable part ends. */
 		switch (state) {
+		case PS_SAW_BANG:
+			if (ctype(c, C_VAR1))
+				goto out;
+
+			if (0)
+				/* FALLTHROUGH */
 		case PS_INITIAL:
-			if (c == '#' || c == '!' || c == '%') {
+			  switch (c) {
+			case '%':
+				state = PS_SAW_PERCENT;
+				goto next;
+			case '#':
 				state = PS_SAW_HASH;
-				break;
+				goto next;
+			case '!':
+				state = PS_SAW_BANG;
+				goto next;
 			}
 			/* FALLTHROUGH */
+		case PS_SAW_PERCENT:
 		case PS_SAW_HASH:
 			if (ksh_isalphx(c))
 				state = PS_IDENT;
@@ -1619,6 +1634,7 @@ get_brace_var(XString *wsp, char *wp)
 				}
 				goto out;
 			}
+ next:
 			break;
 		case PS_NUMBER:
 			if (!ksh_isdigit(c))

mksh.1

@@ -1,4 +1,4 @@
-.\" $MirOS: src/bin/mksh/mksh.1,v 1.393 2016/03/06 21:01:28 tg Exp $
+.\" $MirOS: src/bin/mksh/mksh.1,v 1.395 2016/05/05 22:45:58 tg Exp $
 .\" $OpenBSD: ksh.1,v 1.160 2015/07/04 13:27:04 feinerer Exp $
 .\"-
 .\" Copyright © 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009,
@@ -76,7 +76,7 @@
 .\" with -mandoc, it might implement .Mx itself, but we want to
 .\" use our own definition. And .Dd must come *first*, always.
 .\"
-.Dd $Mdocdate: March 6 2016 $
+.Dd $Mdocdate: May 5 2016 $
 .\"
 .\" Check which macro package we use, and do other -mdoc setup.
 .\"
@@ -1001,9 +1001,9 @@ Third, a double quote
 .Pq Sq \&"
 quotes all characters, except
 .Ql $ ,
-.Ql \`
-and
 .Ql \e ,
+and
+.Ql \` ,
 up to the next unescaped double quote.
 .Ql $
 and
@@ -1015,13 +1015,15 @@ substitution has backslash-quoting for double quotes enabled.
 If a
 .Ql \e
 inside a double-quoted string is followed by
-.Ql \e ,
-.Ql $ ,
-.Ql \` ,
-or
 .Ql \&" ,
-it is replaced by the second character; if it is followed by a newline, both
+.Ql $ ,
-the
+.Ql \e ,
+or
+.Ql \` ,
+only the
+.Ql \e
+is removed, i.e. the combination is replaced by the second character;
+if it is followed by a newline, both the
 .Ql \e
 and the newline are stripped; otherwise, both the
 .Ql \e
@@ -1591,6 +1593,8 @@ is a name reference (bound variable), created by the
 .Ic nameref
 command (which is an alias for
 .Ic typeset Fl n ) .
+.Ar name
+cannot be one of most special parameters (see below).
 .Pp
 .It Pf ${! Ns Ar name Ns \&[*]}
 .It Pf ${! Ns Ar name Ns \&[@]}
@@ -6585,7 +6589,7 @@ for the in-memory portion of the history is slow, should use
 .Xr memmove 3 .
 .Pp
 This document attempts to describe
-.Nm mksh\ R52c
+.Nm mksh\ R52c+CVS
 and up,
 .\" with vendor patches from insert-your-name-here,
 compiled without any options impacting functionality, such as

sh.h

@@ -175,9 +175,9 @@
 #endif
 
 #ifdef EXTERN
-__RCSID("$MirOS: src/bin/mksh/sh.h,v 1.768 2016/03/04 18:28:42 tg Exp $");
+__RCSID("$MirOS: src/bin/mksh/sh.h,v 1.771 2016/05/05 22:56:14 tg Exp $");
 #endif
-#define MKSH_VERSION "R52 2016/03/04"
+#define MKSH_VERSION "R52 2016/05/05"
 
 /* arithmetic types: C implementation */
 #if !HAVE_CAN_INTTYPES
@@ -1756,7 +1756,7 @@ size_t utf_mbtowc(unsigned int *, const char *);
 size_t utf_wctomb(char *, unsigned int);
 int utf_widthadj(const char *, const char **);
 size_t utf_mbswidth(const char *) MKSH_A_PURE;
-const char *utf_skipcols(const char *, int) MKSH_A_PURE;
+const char *utf_skipcols(const char *, int, int *);
 size_t utf_ptradj(const char *) MKSH_A_PURE;
 #ifdef MIRBSD_BOOTFLOPPY
 #define utf_wcwidth(i) wcwidth((wchar_t)(i))

shf.c

@@ -25,7 +25,7 @@
 
 #include "sh.h"
 
-__RCSID("$MirOS: src/bin/mksh/shf.c,v 1.70 2016/03/04 14:26:16 tg Exp $");
+__RCSID("$MirOS: src/bin/mksh/shf.c,v 1.73 2016/05/05 22:56:15 tg Exp $");
 
 /* flags to shf_emptybuf() */
 #define EB_READSW	0x01	/* about to switch to reading */
@@ -1040,15 +1040,11 @@ shf_vfprintf(struct shf *shf, const char *fmt, va_list args)
 		} else
 			field = 0;
 
-		if (precision) {
+		nwritten += precision;
-			const char *q;
+		precision = utf_skipcols(s, precision, &tmp) - s;
+		while (precision--)
+			shf_putc(*s++, shf);
 
-			nwritten += precision;
-			q = utf_skipcols(s, precision);
-			do {
-				shf_putc(*s, shf);
-			} while (++s < q);
-		}
 		nwritten += field;
 		while (field--)
 			shf_putc(c, shf);

var.c

@@ -28,7 +28,7 @@
 #include <sys/sysctl.h>
 #endif
 
-__RCSID("$MirOS: src/bin/mksh/var.c,v 1.201 2016/03/01 20:28:33 tg Exp $");
+__RCSID("$MirOS: src/bin/mksh/var.c,v 1.202 2016/05/05 22:56:15 tg Exp $");
 
 /*-
  * Variables
@@ -649,14 +649,14 @@ formatstr(struct tbl *vp, const char *s)
 
 	p = alloc((psiz = nlen * /* MB_LEN_MAX */ 3 + 1), ATEMP);
 	if (vp->flag & (RJUST|LJUST)) {
-		int slen = olen, i = 0;
+		int slen = olen;
 
 		if (vp->flag & RJUST) {
-			const char *qq = s;
+			const char *qq;
 			int n = 0;
 
-			while (i < slen)
+			qq = utf_skipcols(s, slen, &slen);
-				i += utf_widthadj(qq, &qq);
+
 			/* strip trailing spaces (AT&T uses qq[-1] == ' ') */
 			while (qq > s && ksh_isspace(qq[-1])) {
 				--qq;