Implement the roff(7) .shift and .return requests,

for example used by groff_hdtbl(7) and groff_mom(7).

Also correctly interpolate arguments during nested macro execution
even after .shift and .return, implemented using a stack of argument
arrays.

Note that only read.c, but not roff.c can detect the end of a macro
execution, and the existence of .shift implies that arguments cannot
be interpolated up front, so unfortunately, this includes a partial
revert of roff.c rev. 1.337, moving argument interpolation back into
the function roff_res().

21 files changed, 459 insertions, 205 deletions

diff --git a/TODO b/TODO
index 9543c806..52ea999c 100644
--- a/TODO
+++ b/TODO
@@ -1,6 +1,6 @@
 ************************************************************************
 * Official mandoc TODO.
-* $Id: TODO,v 1.267 2018/08/19 17:46:14 schwarze Exp $
+* $Id: TODO,v 1.268 2018/08/23 14:29:38 schwarze Exp $
 ************************************************************************
 
 Many issues are annotated for difficulty as follows:
@@ -57,7 +57,7 @@ are mere guesses, and some may be wrong.
   reported by brad@  Sat, 15 Jan 2011 15:45:23 -0500
   loc ***  exist ***  algo ***  size **  imp *
 
-- .while and .shift
+- .while
   found by jca@ in ratpoison(1)  Sun, 30 Jun 2013 12:01:09 +0200
   loc *  exist **  algo **  size **  imp **
 
diff --git a/libmandoc.h b/libmandoc.h
index 9cc8cce4..05f149fb 100644
--- a/libmandoc.h
+++ b/libmandoc.h
@@ -1,7 +1,7 @@
-/*	$Id: libmandoc.h,v 1.71 2018/04/09 22:27:04 schwarze Exp $ */
+/*	$Id: libmandoc.h,v 1.72 2018/08/23 14:29:38 schwarze Exp $ */
 /*
  * Copyright (c) 2009, 2010, 2011, 2012 Kristaps Dzonsons <kristaps@bsd.lv>
- * Copyright (c) 2013, 2014, 2015, 2017 Ingo Schwarze <schwarze@openbsd.org>
+ * Copyright (c) 2013,2014,2015,2017,2018 Ingo Schwarze <schwarze@openbsd.org>
  *
  * Permission to use, copy, modify, and distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -21,6 +21,8 @@ enum	rofferr {
 	ROFF_RERUN, /* re-run roff interpreter with offset */
 	ROFF_APPEND, /* re-run main parser, appending next line */
 	ROFF_REPARSE, /* re-run main parser on the result */
+	ROFF_USERCALL, /* dto., calling a user-defined macro */
+	ROFF_USERRET, /* abort parsing of user-defined macro */
 	ROFF_SO, /* include another file */
 	ROFF_IGN, /* ignore current line */
 };
@@ -64,6 +66,7 @@ struct roff_man	*roff_man_alloc(struct roff *, struct mparse *,
 			const char *, int);
 void		 roff_man_reset(struct roff_man *);
 enum rofferr	 roff_parseln(struct roff *, int, struct buf *, int *);
+void		 roff_userret(struct roff *);
 void		 roff_endparse(struct roff *);
 void		 roff_setreg(struct roff *, const char *, int, char sign);
 int		 roff_getreg(struct roff *, const char *);
diff --git a/mandoc.1 b/mandoc.1
index 76b3cda5..a93d1d43 100644
--- a/mandoc.1
+++ b/mandoc.1
@@ -1,4 +1,4 @@
-.\"	$Id: mandoc.1,v 1.226 2018/07/28 18:34:15 schwarze Exp $
+.\"	$Id: mandoc.1,v 1.227 2018/08/23 14:29:38 schwarze Exp $
 .\"
 .\" Copyright (c) 2009, 2010, 2011 Kristaps Dzonsons <kristaps@bsd.lv>
 .\" Copyright (c) 2012, 2014-2018 Ingo Schwarze <schwarze@openbsd.org>
@@ -15,7 +15,7 @@
 .\" ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 .\" OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 .\"
-.Dd $Mdocdate: July 28 2018 $
+.Dd $Mdocdate: August 23 2018 $
 .Dt MANDOC 1
 .Os
 .Sh NAME
@@ -1807,6 +1807,13 @@ or
 macro.
 It may be mistyped or unsupported.
 The request or macro is discarded including its arguments.
+.It Sy "skipping request outside macro"
+.Pq roff
+A
+.Ic shift
+or
+.Ic return
+request occurs outside any macro definition and has no effect.
 .It Sy "skipping insecure request"
 .Pq roff
 An input file attempted to run a shell command
@@ -1916,6 +1923,14 @@ When parsing for a request or a user-defined macro name to be called,
 only the escape sequence is discarded.
 The characters preceding it are used as the request or macro name,
 the characters following it are used as the arguments to the request or macro.
+.It Sy "using macro argument outside macro"
+.Pq roff
+The escape sequence \e$ occurs outside any macro definition
+and expands to the empty string.
+.It Sy "argument number is not numeric"
+.Pq roff
+The argument of the escape sequence \e$ is not a digit;
+the escape sequence expands to the empty string.
 .It Sy "NOT IMPLEMENTED: Bd -file"
 .Pq mdoc
 For security reasons, the
@@ -1978,6 +1993,13 @@ or
 .Ic \&gsize
 statement has a non-numeric or negative argument or no argument at all.
 The invalid request or statement is ignored.
+.It Sy "excessive shift"
+.Pq roff
+The argument of a
+.Ic shift
+request is larger than the number of arguments of the macro that is
+currently being executed.
+All macro arguments are deleted and \en(.$ is set to zero.
 .It Sy "NOT IMPLEMENTED: .so with absolute path or \(dq..\(dq"
 .Pq roff
 For security reasons,
diff --git a/mandoc.h b/mandoc.h
index 4136f20c..d328e942 100644
--- a/mandoc.h
+++ b/mandoc.h
@@ -1,4 +1,4 @@
-/*	$Id: mandoc.h,v 1.249 2018/08/16 13:54:06 schwarze Exp $ */
+/*	$Id: mandoc.h,v 1.250 2018/08/23 14:29:38 schwarze Exp $ */
 /*
  * Copyright (c) 2010, 2011, 2014 Kristaps Dzonsons <kristaps@bsd.lv>
  * Copyright (c) 2010-2018 Ingo Schwarze <schwarze@openbsd.org>
@@ -195,6 +195,7 @@ enum	mandocerr {
 	MANDOCERR_ROFFLOOP, /* input stack limit exceeded, infinite loop? */
 	MANDOCERR_CHAR_BAD, /* skipping bad character: number */
 	MANDOCERR_MACRO, /* skipping unknown macro: macro */
+	MANDOCERR_REQ_NOMAC, /* skipping request outside macro: ... */
 	MANDOCERR_REQ_INSEC, /* skipping insecure request: request */
 	MANDOCERR_IT_STRAY, /* skipping item outside list: It ... */
 	MANDOCERR_TA_STRAY, /* skipping column outside column list: Ta */
@@ -205,6 +206,8 @@ enum	mandocerr {
 
 	/* related to request and macro arguments */
 	MANDOCERR_NAMESC, /* escaped character not allowed in a name: name */
+	MANDOCERR_ARG_UNDEF, /* using macro argument outside macro */
+	MANDOCERR_ARG_NONUM, /* argument number is not numeric */
 	MANDOCERR_BD_FILE, /* NOT IMPLEMENTED: Bd -file */
 	MANDOCERR_BD_NOARG, /* skipping display without arguments: Bd */
 	MANDOCERR_BL_NOTYPE, /* missing list type, using -item: Bl */
@@ -213,6 +216,7 @@ enum	mandocerr {
 	MANDOCERR_OS_UNAME, /* uname(3) system call failed, using UNKNOWN */
 	MANDOCERR_ST_BAD, /* unknown standard specifier: St standard */
 	MANDOCERR_IT_NONUM, /* skipping request without numeric argument */
+	MANDOCERR_SHIFT, /* excessive shift: ..., but max is ... */
 	MANDOCERR_SO_PATH, /* NOT IMPLEMENTED: .so with absolute path or ".." */
 	MANDOCERR_SO_FAIL, /* .so request failed */
 	MANDOCERR_ARG_SKIP, /* skipping all arguments: macro args */
diff --git a/read.c b/read.c
index 0a583445..12901bd3 100644
--- a/read.c
+++ b/read.c
@@ -1,4 +1,4 @@
-/*	$Id: read.c,v 1.196 2018/07/28 18:34:15 schwarze Exp $ */
+/*	$Id: read.c,v 1.197 2018/08/23 14:29:38 schwarze Exp $ */
 /*
  * Copyright (c) 2008, 2009, 2010, 2011 Kristaps Dzonsons <kristaps@bsd.lv>
  * Copyright (c) 2010-2018 Ingo Schwarze <schwarze@openbsd.org>
@@ -62,7 +62,7 @@ struct	mparse {
 
 static	void	  choose_parser(struct mparse *);
 static	void	  resize_buf(struct buf *, size_t);
-static	int	  mparse_buf_r(struct mparse *, struct buf, size_t, int);
+static	enum rofferr mparse_buf_r(struct mparse *, struct buf, size_t, int);
 static	int	  read_whole_file(struct mparse *, const char *, int,
 				struct buf *, int *);
 static	void	  mparse_end(struct mparse *);
@@ -233,6 +233,7 @@ static	const char * const	mandocerrs[MANDOCERR_MAX] = {
 	"input stack limit exceeded, infinite loop?",
 	"skipping bad character",
 	"skipping unknown macro",
+	"ignoring request outside macro",
 	"skipping insecure request",
 	"skipping item outside list",
 	"skipping column outside column list",
@@ -243,6 +244,8 @@ static	const char * const	mandocerrs[MANDOCERR_MAX] = {
 
 	/* related to request and macro arguments */
 	"escaped character not allowed in a name",
+	"using macro argument outside macro",
+	"argument number is not numeric",
 	"NOT IMPLEMENTED: Bd -file",
 	"skipping display without arguments",
 	"missing list type, using -item",
@@ -251,6 +254,7 @@ static	const char * const	mandocerrs[MANDOCERR_MAX] = {
 	"uname(3) system call failed, using UNKNOWN",
 	"unknown standard specifier",
 	"skipping request without numeric argument",
+	"excessive shift",
 	"NOT IMPLEMENTED: .so with absolute path or \"..\"",
 	".so request failed",
 	"skipping all arguments",
@@ -338,14 +342,14 @@ choose_parser(struct mparse *curp)
  * macros, inline equations, and input line traps)
  * and indirectly (for .so file inclusion).
  */
-static int
+static enum rofferr
 mparse_buf_r(struct mparse *curp, struct buf blk, size_t i, int start)
 {
 	struct buf	 ln;
 	const char	*save_file;
 	char		*cp;
 	size_t		 pos; /* byte number in the ln buffer */
-	enum rofferr	 rr;
+	enum rofferr	 line_result, sub_result;
 	int		 of;
 	int		 lnn; /* line number in the real file */
 	int		 fd;
@@ -468,20 +472,36 @@ mparse_buf_r(struct mparse *curp, struct buf blk, size_t i, int start)
 				[curp->secondary->sz] = '\0';
 		}
 rerun:
-		rr = roff_parseln(curp->roff, curp->line, &ln, &of);
+		line_result = roff_parseln(curp->roff, curp->line, &ln, &of);
 
-		switch (rr) {
+		switch (line_result) {
 		case ROFF_REPARSE:
-			if (++curp->reparse_count > REPARSE_LIMIT)
+		case ROFF_USERCALL:
+			if (++curp->reparse_count > REPARSE_LIMIT) {
+				sub_result = ROFF_IGN;
 				mandoc_msg(MANDOCERR_ROFFLOOP, curp,
 				    curp->line, pos, NULL);
-			else if (mparse_buf_r(curp, ln, of, 0) == 1 ||
-			    start == 1) {
+			} else {
+				sub_result = mparse_buf_r(curp, ln, of, 0);
+				if (line_result == ROFF_USERCALL) {
+					if (sub_result == ROFF_USERRET)
+						sub_result = ROFF_CONT;
+					roff_userret(curp->roff);
+				}
+				if (start || sub_result == ROFF_CONT) {
+					pos = 0;
+					continue;
+				}
+			}
+			free(ln.buf);
+			return sub_result;
+		case ROFF_USERRET:
+			if (start) {
 				pos = 0;
 				continue;
 			}
 			free(ln.buf);
-			return 0;
+			return ROFF_USERRET;
 		case ROFF_APPEND:
 			pos = strlen(ln.buf);
 			continue;
@@ -495,7 +515,7 @@ rerun:
 			    (i >= blk.sz || blk.buf[i] == '\0')) {
 				curp->sodest = mandoc_strdup(ln.buf + of);
 				free(ln.buf);
-				return 1;
+				return ROFF_CONT;
 			}
 			/*
 			 * We remove `so' clauses from our lookaside
@@ -547,7 +567,7 @@ rerun:
 	}
 
 	free(ln.buf);
-	return 1;
+	return ROFF_CONT;
 }
 
 static int
diff --git a/regress/roff/Makefile b/regress/roff/Makefile
index 61501033..7c6f5e16 100644
--- a/regress/roff/Makefile
+++ b/regress/roff/Makefile
@@ -1,7 +1,7 @@
-# $OpenBSD: Makefile,v 1.20 2015/02/06 16:05:51 schwarze Exp $
+# $OpenBSD: Makefile,v 1.25 2018/08/23 14:16:12 schwarze Exp $
 
 SUBDIR  = args cond esc scale string
-SUBDIR += br cc de ds ft ig it ll na nr po ps rm rn sp ta ti tr
+SUBDIR += br cc de ds ft ig it ll na nr po ps return rm rn shift sp ta ti tr
 
 .include "../Makefile.sub"
 .include <bsd.subdir.mk>
diff --git a/regress/roff/de/infinite.in b/regress/roff/de/infinite.in
index 00931a06..dd17885f 100644
--- a/regress/roff/de/infinite.in
+++ b/regress/roff/de/infinite.in
@@ -1,5 +1,5 @@
-.\" $OpenBSD: infinite.in,v 1.3 2017/07/04 14:53:27 schwarze Exp $
-.Dd $Mdocdate: July 4 2017 $
+.\" $OpenBSD: infinite.in,v 1.4 2018/08/23 14:16:12 schwarze Exp $
+.Dd $Mdocdate: August 23 2018 $
 .Dt DE-INFINITE 1
 .Os
 .Sh NAME
@@ -10,8 +10,8 @@ initial text
 .de mym
 .Op \\$1 \\$2
 ..
-.mym $1 \$1
-.mym \$1 nothing
+.mym $1 \$1 end
+.mym \$1 middle end
 middle text
 .de mym
 .mym
diff --git a/regress/roff/de/infinite.out_ascii b/regress/roff/de/infinite.out_ascii
index 452af775..074362da 100644
--- a/regress/roff/de/infinite.out_ascii
+++ b/regress/roff/de/infinite.out_ascii
@@ -4,6 +4,6 @@ NNAAMMEE
      ddee--iinnffiinniittee - inifinte recursion in a user-defined macro
 
 DDEESSCCRRIIPPTTIIOONN
-     initial text [$1 $1] middle text final text
+     initial text [$1 end] [middle end] middle text final text
 
-OpenBSD                          July 4, 2017                          OpenBSD
+OpenBSD                         August 23, 2018                        OpenBSD
diff --git a/regress/roff/de/infinite.out_lint b/regress/roff/de/infinite.out_lint
index e7beb090..06254be5 100644
--- a/regress/roff/de/infinite.out_lint
+++ b/regress/roff/de/infinite.out_lint
@@ -1,2 +1,3 @@
-mandoc: infinite.in:14:5: ERROR: input stack limit exceeded, infinite loop?
+mandoc: infinite.in:13:9: ERROR: using macro argument outside macro: \$1
+mandoc: infinite.in:14:6: ERROR: using macro argument outside macro: \$1
 mandoc: infinite.in:20:5: ERROR: input stack limit exceeded, infinite loop?
diff --git a/regress/roff/return/Makefile b/regress/roff/return/Makefile
new file mode 100644
index 00000000..f7d510f7
--- /dev/null
+++ b/regress/roff/return/Makefile
@@ -0,0 +1,6 @@
+# $OpenBSD: Makefile,v 1.1 2018/08/23 14:16:12 schwarze Exp $
+
+REGRESS_TARGETS	= basic
+LINT_TARGETS	= basic
+
+.include <bsd.regress.mk>
diff --git a/regress/roff/return/basic.in b/regress/roff/return/basic.in
new file mode 100644
index 00000000..f0a777c1
--- /dev/null
+++ b/regress/roff/return/basic.in
@@ -0,0 +1,23 @@
+.\" $OpenBSD: basic.in,v 1.1 2018/08/23 14:16:12 schwarze Exp $
+.Dd $Mdocdate: August 23 2018 $
+.Dt RETURN-BASIC 1
+.Os
+.Sh NAME
+.Nm return-basic
+.Nd the return request
+.Sh DESCRIPTION
+return before macro
+.return
+.Pp
+.de mymacro
+text from macro (\\n(.$ argument: "\\$1"),
+.return
+not printed,
+..
+.mymacro myarg
+\n(.$ arguments after return: "\$1",
+.Pp
+return after macro
+.return
+.Pp
+final text
diff --git a/regress/roff/return/basic.out_ascii b/regress/roff/return/basic.out_ascii
new file mode 100644
index 00000000..25f66744
--- /dev/null
+++ b/regress/roff/return/basic.out_ascii
@@ -0,0 +1,15 @@
+RETURN-BASIC(1)             General Commands Manual            RETURN-BASIC(1)
+
+NNAAMMEE
+     rreettuurrnn--bbaassiicc - the return request
+
+DDEESSCCRRIIPPTTIIOONN
+     return before macro
+
+     text from macro (1 argument: "myarg"), 0 arguments after return: "",
+
+     return after macro
+
+     final text
+
+OpenBSD                         August 23, 2018                        OpenBSD
diff --git a/regress/roff/return/basic.out_lint b/regress/roff/return/basic.out_lint
new file mode 100644
index 00000000..f4b15bac
--- /dev/null
+++ b/regress/roff/return/basic.out_lint
@@ -0,0 +1,3 @@
+mandoc: basic.in:10:2: ERROR: ignoring request outside macro: return
+mandoc: basic.in:18:32: ERROR: using macro argument outside macro: \$1
+mandoc: basic.in:21:2: ERROR: ignoring request outside macro: return
diff --git a/regress/roff/shift/Makefile b/regress/roff/shift/Makefile
new file mode 100644
index 00000000..d356b963
--- /dev/null
+++ b/regress/roff/shift/Makefile
@@ -0,0 +1,6 @@
+# $OpenBSD: Makefile,v 1.1 2018/08/23 14:16:12 schwarze Exp $
+
+REGRESS_TARGETS	= basic bad
+LINT_TARGETS	= bad
+
+.include <bsd.regress.mk>
diff --git a/regress/roff/shift/bad.in b/regress/roff/shift/bad.in
new file mode 100644
index 00000000..809832de
--- /dev/null
+++ b/regress/roff/shift/bad.in
@@ -0,0 +1,30 @@
+.\" $OpenBSD: bad.in,v 1.1 2018/08/23 14:16:12 schwarze Exp $
+.TH SHIFT_BAD 1 "August 23, 2018"
+.SH NAME
+.B shift-bad
+\(en wrong usage of macro arguments
+.SH DESCRIPTION
+initial text
+.de mym
+in macro: "\\$1"
+.PP
+invalid argument number 'x': "\\$x"
+..
+.PP
+argument used before call: "\$1"
+.shift
+.PP
+.mym argument
+.PP
+argument used after call: "\$1"
+.shift 2
+.PP
+.de mym
+.shift badarg
+after shift badarg: "\\$1"
+.shift 2
+after excessive shift: \\n(.$ "\\$1"
+..
+.mym arg1 arg2
+.PP
+final text
diff --git a/regress/roff/shift/bad.out_ascii b/regress/roff/shift/bad.out_ascii
new file mode 100644
index 00000000..0b21f574
--- /dev/null
+++ b/regress/roff/shift/bad.out_ascii
@@ -0,0 +1,25 @@
+SHIFT_BAD(1)                General Commands Manual               SHIFT_BAD(1)
+
+
+
+NNAAMMEE
+       sshhiifftt--bbaadd - wrong usage of macro arguments
+
+DDEESSCCRRIIPPTTIIOONN
+       initial text
+
+       argument used before call: ""
+
+       in macro: "argument"
+
+       invalid argument number 'x': ""
+
+       argument used after call: ""
+
+       after shift badarg: "arg2" after excessive shift: 0 ""
+
+       final text
+
+
+
+OpenBSD                         August 23, 2018                   SHIFT_BAD(1)
diff --git a/regress/roff/shift/bad.out_lint b/regress/roff/shift/bad.out_lint
new file mode 100644
index 00000000..1f696fc8
--- /dev/null
+++ b/regress/roff/shift/bad.out_lint
@@ -0,0 +1,7 @@
+mandoc: bad.in:14:29: ERROR: using macro argument outside macro: \$1
+mandoc: bad.in:15:2: ERROR: ignoring request outside macro: shift
+mandoc: bad.in:17:31: ERROR: argument number is not numeric: \$x
+mandoc: bad.in:19:28: ERROR: using macro argument outside macro: \$1
+mandoc: bad.in:20:2: ERROR: ignoring request outside macro: shift
+mandoc: bad.in:28:8: ERROR: argument is not numeric, using 1: shift badarg
+mandoc: bad.in:28:9: ERROR: excessive shift: 2, but max is 1
diff --git a/regress/roff/shift/basic.in b/regress/roff/shift/basic.in
new file mode 100644
index 00000000..982042db
--- /dev/null
+++ b/regress/roff/shift/basic.in
@@ -0,0 +1,35 @@
+.\" $OpenBSD: basic.in,v 1.1 2018/08/23 14:16:12 schwarze Exp $
+.TH SHIFT_BASIC 1 "August 23, 2018"
+.SH NAME
+.B shift-basic
+\(en the shift request
+.SH DESCRIPTION
+.de showargs
+original arguments:
+.BI \\$@
+.PP
+.shift 2
+after shift 2:
+.BI \\$@
+.PP
+.shift
+after shift without argument:
+.BI \\$@
+.PP
+.shift 0
+after shift 0:
+.BI \\$@
+..
+.de useargs
+<\\$*>
+..
+.showargs one two three four five
+.PP
+expand to less than three bytes:
+.useargs 1
+.PP
+expand to exactly three bytes:
+.useargs x y
+.PP
+expand to more than three bytes:
+.useargs "a longer argument..." "and another"
diff --git a/regress/roff/shift/basic.out_ascii b/regress/roff/shift/basic.out_ascii
new file mode 100644
index 00000000..40675c67
--- /dev/null
+++ b/regress/roff/shift/basic.out_ascii
@@ -0,0 +1,25 @@
+SHIFT_BASIC(1)              General Commands Manual             SHIFT_BASIC(1)
+
+
+
+NNAAMMEE
+       sshhiifftt--bbaassiicc - the shift request
+
+DDEESSCCRRIIPPTTIIOONN
+       original arguments: oonnee_t_w_otthhrreeee_f_o_u_rffiivvee
+
+       after shift 2: tthhrreeee_f_o_u_rffiivvee
+
+       after shift without argument: ffoouurr_f_i_v_e
+
+       after shift 0: ffoouurr_f_i_v_e
+
+       expand to less than three bytes: <1>
+
+       expand to exactly three bytes: <x y>
+
+       expand to more than three bytes: <a longer argument... and another>
+
+
+
+OpenBSD                         August 23, 2018                 SHIFT_BASIC(1)
diff --git a/roff.7 b/roff.7
index 3c403604..c78edfd1 100644
--- a/roff.7
+++ b/roff.7
@@ -1,4 +1,4 @@
-.\"	$Id: roff.7,v 1.102 2018/08/21 18:15:22 schwarze Exp $
+.\"	$Id: roff.7,v 1.103 2018/08/23 14:29:38 schwarze Exp $
 .\"
 .\" Copyright (c) 2010, 2011, 2012 Kristaps Dzonsons <kristaps@bsd.lv>
 .\" Copyright (c) 2010-2018 Ingo Schwarze <schwarze@openbsd.org>
@@ -15,7 +15,7 @@
 .\" ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 .\" OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 .\"
-.Dd $Mdocdate: August 21 2018 $
+.Dd $Mdocdate: August 23 2018 $
 .Dt ROFF 7
 .Os
 .Sh NAME
@@ -1472,8 +1472,8 @@ Currently ignored.
 Set the maximum stack depth for recursive macros.
 This is a Heirloom extension and currently ignored.
 .It Ic \&return Op Ar twice
-Exit a macro and return to the caller.
-Currently unsupported.
+Exit the presently executed macro and return to the caller.
+The argument is currently ignored.
 .It Ic \&rfschar Ar font glyph ...
 Remove font-specific fallback glyph definitions.
 Currently unsupported.
@@ -1522,8 +1522,11 @@ This is a Heirloom extension and currently ignored.
 Change the soft hyphen character.
 Currently ignored.
 .It Ic \&shift Op Ar number
-Shift macro arguments.
-Currently unsupported.
+Shift macro arguments
+.Ar number
+times, by default once: \e\e$i becomes what \e\e$i+number was.
+Also decrement \en(.$ by
+.Ar number .
 .It Ic \&sizes Ar size ...
 Define permissible point sizes.
 This is a groff extension and currently ignored.
diff --git a/roff.c b/roff.c
index bef86e8c..3b31b97c 100644
--- a/roff.c
+++ b/roff.c
@@ -1,4 +1,4 @@
-/*	$Id: roff.c,v 1.338 2018/08/21 18:15:22 schwarze Exp $ */
+/*	$Id: roff.c,v 1.339 2018/08/23 14:29:39 schwarze Exp $ */
 /*
  * Copyright (c) 2008-2012, 2014 Kristaps Dzonsons <kristaps@bsd.lv>
  * Copyright (c) 2010-2015, 2017, 2018 Ingo Schwarze <schwarze@openbsd.org>
@@ -85,10 +85,21 @@ struct	roffreq {
 	char		 name[];
 };
 
+/*
+ * A macro processing context.
+ * More than one is needed when macro calls are nested.
+ */
+struct	mctx {
+	char		**argv;
+	int		 argc;
+	int		 argsz;
+};
+
 struct	roff {
 	struct mparse	*parse; /* parse point */
 	struct roff_man	*man; /* mdoc or man parser */
 	struct roffnode	*last; /* leaf of stack */
+	struct mctx	*mstack; /* stack of macro contexts */
 	int		*rstack; /* stack of inverted `ie' values */
 	struct ohash	*reqtab; /* request lookup table */
 	struct roffreg	*regtab; /* number registers */
@@ -104,6 +115,8 @@ struct	roff {
 	struct eqn_node	*eqn; /* active equation parser */
 	int		 eqn_inline; /* current equation is inline */
 	int		 options; /* parse options */
+	int		 mstacksz; /* current size of mstack */
+	int		 mstackpos; /* position in mstack */
 	int		 rstacksz; /* current size limit of rstack */
 	int		 rstackpos; /* position in rstack */
 	int		 format; /* current file in mdoc or man format */
@@ -205,6 +218,7 @@ static	enum rofferr	 roff_parsetext(struct roff *, struct buf *,
 				int, int *);
 static	enum rofferr	 roff_renamed(ROFF_ARGS);
 static	enum rofferr	 roff_res(struct roff *, struct buf *, int, int);
+static	enum rofferr	 roff_return(ROFF_ARGS);
 static	enum rofferr	 roff_rm(ROFF_ARGS);
 static	enum rofferr	 roff_rn(ROFF_ARGS);
 static	enum rofferr	 roff_rr(ROFF_ARGS);
@@ -214,6 +228,7 @@ static	void		 roff_setstr(struct roff *,
 				const char *, const char *, int);
 static	void		 roff_setstrn(struct roffkv **, const char *,
 				size_t, const char *, size_t, int);
+static	enum rofferr	 roff_shift(ROFF_ARGS);
 static	enum rofferr	 roff_so(ROFF_ARGS);
 static	enum rofferr	 roff_tr(ROFF_ARGS);
 static	enum rofferr	 roff_Dd(ROFF_ARGS);
@@ -521,7 +536,7 @@ static	struct roffmac	 roffs[TOKEN_NONE] = {
 	{ roff_unsupp, NULL, NULL, 0 },  /* rchar */
 	{ roff_line_ignore, NULL, NULL, 0 },  /* rd */
 	{ roff_line_ignore, NULL, NULL, 0 },  /* recursionlimit */
-	{ roff_unsupp, NULL, NULL, 0 },  /* return */
+	{ roff_return, NULL, NULL, 0 },  /* return */
 	{ roff_unsupp, NULL, NULL, 0 },  /* rfschar */
 	{ roff_line_ignore, NULL, NULL, 0 },  /* rhang */
 	{ roff_rm, NULL, NULL, 0 },  /* rm */
@@ -533,7 +548,7 @@ static	struct roffmac	 roffs[TOKEN_NONE] = {
 	{ roff_unsupp, NULL, NULL, 0 },  /* schar */
 	{ roff_line_ignore, NULL, NULL, 0 },  /* sentchar */
 	{ roff_line_ignore, NULL, NULL, 0 },  /* shc */
-	{ roff_unsupp, NULL, NULL, 0 },  /* shift */
+	{ roff_shift, NULL, NULL, 0 },  /* shift */
 	{ roff_line_ignore, NULL, NULL, 0 },  /* sizes */
 	{ roff_so, NULL, NULL, 0 },  /* so */
 	{ roff_line_ignore, NULL, NULL, 0 },  /* spacewidth */
@@ -713,6 +728,9 @@ roff_free1(struct roff *r)
 		eqn_free(r->last_eqn);
 	r->last_eqn = r->eqn = NULL;
 
+	while (r->mstackpos >= 0)
+		roff_userret(r);
+
 	while (r->last)
 		roffnode_pop(r);
 
@@ -752,7 +770,12 @@ roff_reset(struct roff *r)
 void
 roff_free(struct roff *r)
 {
+	int	 	 i;
+
 	roff_free1(r);
+	for (i = 0; i < r->mstacksz; i++)
+		free(r->mstack[i].argv);
+	free(r->mstack);
 	roffhash_free(r->reqtab);
 	free(r);
 }
@@ -767,6 +790,7 @@ roff_alloc(struct mparse *parse, int options)
 	r->reqtab = roffhash_alloc(0, ROFF_RENAMED);
 	r->options = options;
 	r->format = options & (MPARSE_MDOC | MPARSE_MAN);
+	r->mstackpos = -1;
 	r->rstackpos = -1;
 	r->escape = '\\';
 	return r;
@@ -1123,6 +1147,7 @@ deroff(char **dest, const struct roff_node *n)
 static enum rofferr
 roff_res(struct roff *r, struct buf *buf, int ln, int pos)
 {
+	struct mctx	*ctx;	/* current macro call context */
 	char		 ubuf[24]; /* buffer to print the number */
 	struct roff_node *n;	/* used for header comments */
 	const char	*start;	/* start of the string to process */
@@ -1134,11 +1159,14 @@ roff_res(struct roff *r, struct buf *buf, int ln, int pos)
 	char		*nbuf;	/* new buffer to copy buf->buf to */
 	size_t		 maxl;  /* expected length of the escape name */
 	size_t		 naml;	/* actual length of the escape name */
+	size_t		 asz;	/* length of the replacement */
+	size_t		 rsz;	/* length of the rest of the string */
 	enum mandoc_esc	 esc;	/* type of the escape sequence */
 	int		 inaml;	/* length returned from mandoc_escape() */
 	int		 expand_count;	/* to avoid infinite loops */
 	int		 npos;	/* position in numeric expression */
 	int		 arg_complete; /* argument not interrupted by eol */
+	int		 quote_args; /* true for \\$@, false for \\$* */
 	int		 done;	/* no more input available */
 	int		 deftype; /* type of definition to paste */
 	int		 rcsid;	/* kind of RCS id seen */
@@ -1275,6 +1303,7 @@ roff_res(struct roff *r, struct buf *buf, int ln, int pos)
 		cp = stesc + 1;
 		switch (*cp) {
 		case '*':
+		case '$':
 			res = NULL;
 			break;
 		case 'B':
@@ -1391,6 +1420,62 @@ roff_res(struct roff *r, struct buf *buf, int ln, int pos)
 				}
 			}
 			break;
+		case '$':
+			if (r->mstackpos < 0) {
+				mandoc_vmsg(MANDOCERR_ARG_UNDEF,
+				    r->parse, ln, (int)(stesc - buf->buf),
+				    "%.3s", stesc);
+				break;
+			}
+			ctx = r->mstack + r->mstackpos;
+			npos = stesc[2] - '1';
+			if (npos >= 0 && npos <= 8) {
+				res = npos < ctx->argc ?
+				    ctx->argv[npos] : "";
+				break;
+			}
+			if (stesc[2] == '*')
+				quote_args = 0;
+			else if (stesc[2] == '@')
+				quote_args = 1;
+			else {
+				mandoc_vmsg(MANDOCERR_ARG_NONUM,
+				    r->parse, ln, (int)(stesc - buf->buf),
+				    "%.3s", stesc);
+				break;
+			}
+			asz = 0;
+			for (npos = 0; npos < ctx->argc; npos++) {
+				if (npos)
+					asz++;  /* blank */
+				if (quote_args)
+					asz += 2;  /* quotes */
+				asz += strlen(ctx->argv[npos]);
+			}
+			if (asz != 3) {
+				rsz = buf->sz - (stesc - buf->buf) - 3;
+				if (asz < 3)
+					memmove(stesc + asz, stesc + 3, rsz);
+				buf->sz += asz - 3;
+				nbuf = mandoc_realloc(buf->buf, buf->sz);
+				start = nbuf + pos;
+				stesc = nbuf + (stesc - buf->buf);
+				buf->buf = nbuf;
+				if (asz > 3)
+					memmove(stesc + asz, stesc + 3, rsz);
+			}
+			for (npos = 0; npos < ctx->argc; npos++) {
+				if (npos)
+					*stesc++ = ' ';
+				if (quote_args)
+					*stesc++ = '"';
+				cp = ctx->argv[npos];
+				while (*cp != '\0')
+					*stesc++ = *cp++;
+				if (quote_args)
+					*stesc++ = '"';
+			}
+			continue;
 		case 'B':
 			npos = 0;
 			ubuf[0] = arg_complete &&
@@ -1414,9 +1499,10 @@ roff_res(struct roff *r, struct buf *buf, int ln, int pos)
 		}
 
 		if (res == NULL) {
-			mandoc_vmsg(MANDOCERR_STR_UNDEF,
-			    r->parse, ln, (int)(stesc - buf->buf),
-			    "%.*s", (int)naml, stnam);
+			if (stesc[1] == '*')
+				mandoc_vmsg(MANDOCERR_STR_UNDEF,
+				    r->parse, ln, (int)(stesc - buf->buf),
+				    "%.*s", (int)naml, stnam);
 			res = "";
 		} else if (buf->sz + strlen(res) > SHRT_MAX) {
 			mandoc_msg(MANDOCERR_ROFFLOOP, r->parse,
@@ -1634,6 +1720,25 @@ roff_parseln(struct roff *r, int ln, struct buf *buf, int *offs)
 	return (*roffs[t].proc)(r, t, buf, ln, spos, pos, offs);
 }
 
+/*
+ * Internal interface function to tell the roff parser that execution
+ * of the current macro ended.  This is required because macro
+ * definitions usually do not end with a .return request.
+ */
+void
+roff_userret(struct roff *r)
+{
+	struct mctx	*ctx;
+	int		 i;
+
+	assert(r->mstackpos >= 0);
+	ctx = r->mstack + r->mstackpos;
+	for (i = 0; i < ctx->argc; i++)
+		free(ctx->argv[i]);
+	ctx->argc = 0;
+	r->mstackpos--;
+}
+
 void
 roff_endparse(struct roff *r)
 {
@@ -2662,7 +2767,7 @@ roff_getregro(const struct roff *r, const char *name)
 
 	switch (*name) {
 	case '$':  /* Number of arguments of the last macro evaluated. */
-		return 0;
+		return r->mstackpos < 0 ? 0 : r->mstack[r->mstackpos].argc;
 	case 'A':  /* ASCII approximation mode is always off. */
 		return 0;
 	case 'g':  /* Groff compatibility mode is always on. */
@@ -3293,6 +3398,22 @@ roff_tr(ROFF_ARGS)
 	return ROFF_IGN;
 }
 
+/*
+ * Implementation of the .return request.
+ * There is no need to call roff_userret() from here.
+ * The read module will call that after rewinding the reader stack
+ * to the place from where the current macro was called.
+ */
+static enum rofferr
+roff_return(ROFF_ARGS)
+{
+	if (r->mstackpos >= 0)
+		return ROFF_USERRET;
+
+	mandoc_msg(MANDOCERR_REQ_NOMAC, r->parse, ln, ppos, "return");
+	return ROFF_IGN;
+}
+
 static enum rofferr
 roff_rn(ROFF_ARGS)
 {
@@ -3344,6 +3465,39 @@ roff_rn(ROFF_ARGS)
 }
 
 static enum rofferr
+roff_shift(ROFF_ARGS)
+{
+	struct mctx	*ctx;
+	int		 levels, i;
+
+	levels = 1;
+	if (buf->buf[pos] != '\0' &&
+	    roff_evalnum(r, ln, buf->buf, &pos, &levels, 0) == 0) {
+		mandoc_vmsg(MANDOCERR_CE_NONUM, r->parse,
+		    ln, pos, "shift %s", buf->buf + pos);
+		levels = 1;
+	}
+	if (r->mstackpos < 0) {
+		mandoc_msg(MANDOCERR_REQ_NOMAC, r->parse, ln, ppos, "shift");
+		return ROFF_IGN;
+	}
+	ctx = r->mstack + r->mstackpos;
+	if (levels > ctx->argc) {
+		mandoc_vmsg(MANDOCERR_SHIFT, r->parse,
+		    ln, pos, "%d, but max is %d", levels, ctx->argc);
+		levels = ctx->argc;
+	}
+	if (levels == 0)
+		return ROFF_IGN;
+	for (i = 0; i < levels; i++)
+		free(ctx->argv[i]);
+	ctx->argc -= levels;
+	for (i = 0; i < ctx->argc; i++)
+		ctx->argv[i] = ctx->argv[i + levels];
+	return ROFF_IGN;
+}
+
+static enum rofferr
 roff_so(ROFF_ARGS)
 {
 	char *name, *cp;
@@ -3378,186 +3532,58 @@ roff_so(ROFF_ARGS)
 static enum rofferr
 roff_userdef(ROFF_ARGS)
 {
-	const char	 *arg[16], *ap;
-	char		 *cp, *n1, *n2;
-	int		  argc, expand_count, i, ib, ie, quote_args;
-	size_t		  asz, esz, rsz;
+	struct mctx	 *ctx;
+	char		 *arg, *ap, *dst, *src;
+	size_t		  sz;
 
-	/*
-	 * Collect pointers to macro argument strings
-	 * and NUL-terminate them.
-	 */
+	/* Initialize a new macro stack context. */
 
-	argc = 0;
-	cp = buf->buf + pos;
-	for (i = 0; i < 16; i++) {
-		if (*cp == '\0')
-			arg[i] = "";
-		else {
-			arg[i] = mandoc_getarg(r->parse, &cp, ln, &pos);
-			argc = i + 1;
-		}
+	if (++r->mstackpos == r->mstacksz) {
+		r->mstack = mandoc_recallocarray(r->mstack,
+		    r->mstacksz, r->mstacksz + 8, sizeof(*r->mstack));
+		r->mstacksz += 8;
 	}
+	ctx = r->mstack + r->mstackpos;
+	ctx->argsz = 0;
+	ctx->argc = 0;
+	ctx->argv = NULL;
 
 	/*
-	 * Expand macro arguments.
+	 * Collect pointers to macro argument strings,
+	 * NUL-terminating them and escaping quotes.
 	 */
 
-	buf->sz = strlen(r->current_string) + 1;
-	n1 = n2 = cp = mandoc_malloc(buf->sz);
-	memcpy(n1, r->current_string, buf->sz);
-	expand_count = 0;
-	while (*cp != '\0') {
-
-		/* Scan ahead for the next argument invocation. */
-
-		if (*cp++ != '\\')
-			continue;
-		if (*cp++ != '$')
-			continue;
-
-		quote_args = 0;
-		switch (*cp) {
-		case '@':  /* \\$@ inserts all arguments, quoted */
-			quote_args = 1;
-			/* FALLTHROUGH */
-		case '*':  /* \\$* inserts all arguments, unquoted */
-			ib = 0;
-			ie = argc - 1;
-			break;
-		default:  /* \\$1 .. \\$9 insert one argument */
-			ib = ie = *cp - '1';
-			if (ib < 0 || ib > 8)
-				continue;
-			break;
-		}
-		cp -= 2;
-
-		/*
-		 * Prevent infinite recursion.
-		 */
-
-		if (cp >= n2)
-			expand_count = 1;
-		else if (++expand_count > EXPAND_LIMIT) {
-			mandoc_msg(MANDOCERR_ROFFLOOP, r->parse,
-			    ln, (int)(cp - n1), NULL);
-			free(buf->buf);
-			buf->buf = n1;
-			*offs = 0;
-			return ROFF_IGN;
-		}
-
-		/*
-		 * Determine the size of the expanded argument,
-		 * taking escaping of quotes into account.
-		 */
-
-		asz = ie > ib ? ie - ib : 0;  /* for blanks */
-		for (i = ib; i <= ie; i++) {
-			if (quote_args)
-				asz += 2;
-			for (ap = arg[i]; *ap != '\0'; ap++) {
-				asz++;
-				if (*ap == '"')
-					asz += 3;
-			}
-		}
-		if (asz != 3) {
-
-			/*
-			 * Determine the size of the rest of the
-			 * unexpanded macro, including the NUL.
-			 */
-
-			rsz = buf->sz - (cp - n1) - 3;
-
-			/*
-			 * When shrinking, move before
-			 * releasing the storage.
-			 */
-
-			if (asz < 3)
-				memmove(cp + asz, cp + 3, rsz);
-
-			/*
-			 * Resize the storage for the macro
-			 * and readjust the parse pointer.
-			 */
-
-			buf->sz += asz - 3;
-			n2 = mandoc_realloc(n1, buf->sz);
-			cp = n2 + (cp - n1);
-			n1 = n2;
-
-			/*
-			 * When growing, make room
-			 * for the expanded argument.
-			 */
-
-			if (asz > 3)
-				memmove(cp + asz, cp + 3, rsz);
+	src = buf->buf + pos;
+	while (*src != '\0') {
+		if (ctx->argc == ctx->argsz) {
+			ctx->argsz += 8;
+			ctx->argv = mandoc_reallocarray(ctx->argv,
+			    ctx->argsz, sizeof(*ctx->argv));
 		}
-
-		/* Copy the expanded argument, escaping quotes. */
-
-		n2 = cp;
-		for (i = ib; i <= ie; i++) {
-			if (quote_args)
-				*n2++ = '"';
-			for (ap = arg[i]; *ap != '\0'; ap++) {
-				if (*ap == '"') {
-					memcpy(n2, "\\(dq", 4);
-					n2 += 4;
-				} else
-					*n2++ = *ap;
-			}
-			if (quote_args)
-				*n2++ = '"';
-			if (i < ie)
-				*n2++ = ' ';
-		}
-	}
-
-	/*
-	 * Expand the number of arguments, if it is used.
-	 * This never makes the expanded macro longer.
-	 */
-
-	for (cp = n1; *cp != '\0'; cp++) {
-		if (cp[0] != '\\')
-			continue;
-		if (cp[1] == '\\') {
-			cp++;
-			continue;
+		arg = mandoc_getarg(r->parse, &src, ln, &pos);
+		sz = 1;  /* For the terminating NUL. */
+		for (ap = arg; *ap != '\0'; ap++)
+			sz += *ap == '"' ? 4 : 1;
+		ctx->argv[ctx->argc++] = dst = mandoc_malloc(sz);
+		for (ap = arg; *ap != '\0'; ap++) {
+			if (*ap == '"') {
+				memcpy(dst, "\\(dq", 4);
+				dst += 4;
+			} else
+				*dst++ = *ap;
 		}
-		if (strncmp(cp + 1, "n(.$", 4) == 0)
-			esz = 5;
-		else if (strncmp(cp + 1, "n[.$]", 5) == 0)
-			esz = 6;
-		else
-			continue;
-		asz = snprintf(cp, esz, "%d", argc);
-		assert(asz < esz);
-		rsz = buf->sz - (cp - n1) - esz;
-		memmove(cp + asz, cp + esz, rsz);
-		buf->sz -= esz - asz;
-		n2 = mandoc_realloc(n1, buf->sz);
-		cp = n2 + (cp - n1) + asz;
-		n1 = n2;
+		*dst = '\0';
 	}
 
-	/*
-	 * Replace the macro invocation
-	 * by the expanded macro.
-	 */
+	/* Replace the macro invocation by the macro definition. */
 
 	free(buf->buf);
-	buf->buf = n1;
+	buf->buf = mandoc_strdup(r->current_string);
+	buf->sz = strlen(buf->buf) + 1;
 	*offs = 0;
 
 	return buf->sz > 1 && buf->buf[buf->sz - 2] == '\n' ?
-	   ROFF_REPARSE : ROFF_APPEND;
+	   ROFF_USERCALL : ROFF_APPEND;
 }
 
 /*