Rename mandoc_getarg() to roff_getarg() and pass it the roff parser

struct as an argument such that after copy-in, it can call roff_expand()
once again, which used to be called roff_res() before this.  This
fixes a subtle low-level roff(7) parsing bug reported by Fabio
Scotoni <fabio at esse dot ch> in the 4.4BSD-Lite2 mdoc.samples(7)
manual page, because that page used an escaped escape sequence in
a macro argument.

To expand escaped escape sequences in quoted mdoc(7) arguments, too,
stop bypassing the call to roff_getarg() in mdoc_argv.c, function args()
for this case.  This does not solve the case of escaped escape sequences
in quoted .Bl -column phrases yet.

Because roff_expand() can make the string longer, roff_getarg() can no
longer operate in-place but needs to malloc(3) the returned string.
In the high-level parsers, free(3) that string after processing it.

25 files changed, 341 insertions, 12 deletions

diff --git a/regress/mdoc/Bl/Makefile b/regress/mdoc/Bl/Makefile
index aeb6bd56..8fde5d9e 100644
--- a/regress/mdoc/Bl/Makefile
+++ b/regress/mdoc/Bl/Makefile
@@ -1,8 +1,8 @@
-# $OpenBSD: Makefile,v 1.31 2016/10/17 19:00:16 schwarze Exp $
+# $OpenBSD: Makefile,v 1.40 2018/12/21 16:58:49 schwarze Exp $
 
 REGRESS_TARGETS  = item inset diag ohang bullet dash enum hang tag
 REGRESS_TARGETS += column column_nogroff colNoIt
-REGRESS_TARGETS += extend nested offset secstart
+REGRESS_TARGETS += esc extend nested offset secstart
 
 REGRESS_TARGETS += notype multitype badargs
 REGRESS_TARGETS += empty noIt emptyhead emptytag emptyitem multitag
diff --git a/regress/mdoc/Bl/esc.in b/regress/mdoc/Bl/esc.in
new file mode 100644
index 00000000..343ccc7d
--- /dev/null
+++ b/regress/mdoc/Bl/esc.in
@@ -0,0 +1,19 @@
+.\" $OpenBSD: esc.in,v 1.1 2018/12/21 16:58:49 schwarze Exp $
+.Dd $Mdocdate: December 21 2018 $
+.Dt BL-ESC 1
+.Os
+.Sh NAME
+.Nm Bl-esc
+.Nd escape sequences in full block macro heads
+.Sh DESCRIPTION
+.ds a \(at
+.Bl -tag -width 2n
+.It \*a
+unquoted unescaped
+.It "\*a"
+quoted unescaped
+.It \\*a
+unquoted escaped
+.It "\\*a"
+quoted escaped
+.El
diff --git a/regress/mdoc/Bl/esc.out_ascii b/regress/mdoc/Bl/esc.out_ascii
new file mode 100644
index 00000000..e73c929f
--- /dev/null
+++ b/regress/mdoc/Bl/esc.out_ascii
@@ -0,0 +1,15 @@
+BL-ESC(1)                   General Commands Manual                  BL-ESC(1)
+
+NNAAMMEE
+     BBll--eesscc - escape sequences in full block macro heads
+
+DDEESSCCRRIIPPTTIIOONN
+     @   unquoted unescaped
+
+     @   quoted unescaped
+
+     @   unquoted escaped
+
+     @   quoted escaped
+
+OpenBSD                        December 21, 2018                       OpenBSD
diff --git a/regress/mdoc/Bl/esc.out_markdown b/regress/mdoc/Bl/esc.out_markdown
new file mode 100644
index 00000000..780a61b8
--- /dev/null
+++ b/regress/mdoc/Bl/esc.out_markdown
@@ -0,0 +1,25 @@
+BL-ESC(1) - General Commands Manual
+
+# NAME
+
+**Bl-esc** - escape sequences in full block macro heads
+
+# DESCRIPTION
+
+@
+
+> unquoted unescaped
+
+@
+
+> quoted unescaped
+
+@
+
+> unquoted escaped
+
+@
+
+> quoted escaped
+
+OpenBSD - December 21, 2018
diff --git a/regress/mdoc/Eo/Makefile b/regress/mdoc/Eo/Makefile
index 0034e271..1e101ace 100644
--- a/regress/mdoc/Eo/Makefile
+++ b/regress/mdoc/Eo/Makefile
@@ -1,6 +1,7 @@
-# $OpenBSD: Makefile,v 1.4 2015/02/11 14:14:53 schwarze Exp $
+# $OpenBSD: Makefile,v 1.8 2018/12/21 16:58:49 schwarze Exp $
 
-REGRESS_TARGETS	 = break empty obsolete unclosed
+REGRESS_TARGETS	 = arg break empty obsolete unclosed
+UTF8_TARGETS	 = arg
 LINT_TARGETS	 = break obsolete unclosed
 
 # groff-1.22.3 defect:
diff --git a/regress/mdoc/Eo/arg.in b/regress/mdoc/Eo/arg.in
new file mode 100644
index 00000000..dd7928a7
--- /dev/null
+++ b/regress/mdoc/Eo/arg.in
@@ -0,0 +1,25 @@
+.\" $OpenBSD: arg.in,v 1.1 2018/12/21 16:58:49 schwarze Exp $
+.Dd $Mdocdate: December 21 2018 $
+.Dt EO-ARG 1
+.Os
+.Sh NAME
+.Nm Eo-arg
+.Nd escapes in arguments of enclosure macros
+.Sh DESCRIPTION
+.ds o \(Fo
+.ds c \(Fc
+.Eo \*o
+unquoted unescaped
+.Ec \*c
+.Pp
+.Eo "\*o"
+quoted unescaped
+.Ec "\*c"
+.Pp
+.Eo \\*o
+unquoted escaped
+.Ec \\*c
+.Pp
+.Eo "\\*o"
+quoted escaped
+.Ec "\\*c"
diff --git a/regress/mdoc/Eo/arg.out_ascii b/regress/mdoc/Eo/arg.out_ascii
new file mode 100644
index 00000000..004c3fe5
--- /dev/null
+++ b/regress/mdoc/Eo/arg.out_ascii
@@ -0,0 +1,15 @@
+EO-ARG(1)                   General Commands Manual                  EO-ARG(1)
+
+NNAAMMEE
+     EEoo--aarrgg - escapes in arguments of enclosure macros
+
+DDEESSCCRRIIPPTTIIOONN
+     <<unquoted unescaped>>
+
+     <<quoted unescaped>>
+
+     <<unquoted escaped>>
+
+     <<quoted escaped>>
+
+OpenBSD                        December 21, 2018                       OpenBSD
diff --git a/regress/mdoc/Eo/arg.out_markdown b/regress/mdoc/Eo/arg.out_markdown
new file mode 100644
index 00000000..9da6ff4f
--- /dev/null
+++ b/regress/mdoc/Eo/arg.out_markdown
@@ -0,0 +1,17 @@
+EO-ARG(1) - General Commands Manual
+
+# NAME
+
+**Eo-arg** - escapes in arguments of enclosure macros
+
+# DESCRIPTION
+
+&#171;unquoted unescaped&#187;
+
+&#171;quoted unescaped&#187;
+
+&#171;unquoted escaped&#187;
+
+&#171;quoted escaped&#187;
+
+OpenBSD - December 21, 2018
diff --git a/regress/mdoc/Eo/arg.out_utf8 b/regress/mdoc/Eo/arg.out_utf8
new file mode 100644
index 00000000..3b4a30bd
--- /dev/null
+++ b/regress/mdoc/Eo/arg.out_utf8
@@ -0,0 +1,15 @@
+EO-ARG(1)                   General Commands Manual                  EO-ARG(1)
+
+NNAAMMEE
+     EEoo--aarrgg – escapes in arguments of enclosure macros
+
+DDEESSCCRRIIPPTTIIOONN
+     «unquoted unescaped»
+
+     «quoted unescaped»
+
+     «unquoted escaped»
+
+     «quoted escaped»
+
+OpenBSD                        December 21, 2018                       OpenBSD
diff --git a/regress/mdoc/Fd/Makefile b/regress/mdoc/Fd/Makefile
index 7cd78c65..39b07f1d 100644
--- a/regress/mdoc/Fd/Makefile
+++ b/regress/mdoc/Fd/Makefile
@@ -1,6 +1,6 @@
-# $OpenBSD: Makefile,v 1.3 2015/02/06 03:31:11 schwarze Exp $
+# $OpenBSD: Makefile,v 1.7 2018/12/21 16:58:49 schwarze Exp $
 
-REGRESS_TARGETS	 = break empty eos font
+REGRESS_TARGETS	 = arg break empty eos font
 LINT_TARGETS	 = empty
 SKIP_TMAN	 = eos
 
diff --git a/regress/mdoc/Fd/arg.in b/regress/mdoc/Fd/arg.in
new file mode 100644
index 00000000..659d03c9
--- /dev/null
+++ b/regress/mdoc/Fd/arg.in
@@ -0,0 +1,16 @@
+.\" $OpenBSD: arg.in,v 1.1 2018/12/21 16:58:49 schwarze Exp $
+.Dd $Mdocdate: December 21 2018 $
+.Dt FD-ARG 2
+.Os
+.Sh NAME
+.Nm Fd-arg
+.Nd escape sequences in the arguments of in-line EOL macros
+.Sh DESCRIPTION
+.ds s \(sh
+.Fd \*sunquoted unescaped
+.Pp
+.Fd "\*squoted" unescaped
+.Pp
+.Fd \\*sunquoted escaped
+.Pp
+.Fd "\\*squoted" escaped
diff --git a/regress/mdoc/Fd/arg.out_ascii b/regress/mdoc/Fd/arg.out_ascii
new file mode 100644
index 00000000..42836e61
--- /dev/null
+++ b/regress/mdoc/Fd/arg.out_ascii
@@ -0,0 +1,15 @@
+FD-ARG(2)                     System Calls Manual                    FD-ARG(2)
+
+NNAAMMEE
+     FFdd--aarrgg - escape sequences in the arguments of in-line EOL macros
+
+DDEESSCCRRIIPPTTIIOONN
+     ##uunnqquuootteedd uunneessccaappeedd
+
+     ##qquuootteedd uunneessccaappeedd
+
+     ##uunnqquuootteedd eessccaappeedd
+
+     ##qquuootteedd eessccaappeedd
+
+OpenBSD                        December 21, 2018                       OpenBSD
diff --git a/regress/mdoc/Fd/arg.out_markdown b/regress/mdoc/Fd/arg.out_markdown
new file mode 100644
index 00000000..14dba6fc
--- /dev/null
+++ b/regress/mdoc/Fd/arg.out_markdown
@@ -0,0 +1,17 @@
+FD-ARG(2) - System Calls Manual
+
+# NAME
+
+**Fd-arg** - escape sequences in the arguments of in-line EOL macros
+
+# DESCRIPTION
+
+**&#35;unquoted unescaped**
+
+**&#35;quoted unescaped**
+
+**&#35;unquoted escaped**
+
+**&#35;quoted escaped**
+
+OpenBSD - December 21, 2018
diff --git a/regress/mdoc/Li/Makefile b/regress/mdoc/Li/Makefile
index f5afd199..7b2a3313 100644
--- a/regress/mdoc/Li/Makefile
+++ b/regress/mdoc/Li/Makefile
@@ -1,6 +1,6 @@
-# $OpenBSD: Makefile,v 1.4 2014/07/02 20:18:42 schwarze Exp $
+# $OpenBSD: Makefile,v 1.7 2018/12/21 16:58:49 schwarze Exp $
 
-REGRESS_TARGETS = punct font
+REGRESS_TARGETS = arg punct font
 LINT_TARGETS	= punct
 
 .include <bsd.regress.mk>
diff --git a/regress/mdoc/Li/arg.in b/regress/mdoc/Li/arg.in
new file mode 100644
index 00000000..d6c80672
--- /dev/null
+++ b/regress/mdoc/Li/arg.in
@@ -0,0 +1,20 @@
+.\" $OpenBSD: arg.in,v 1.1 2018/12/21 16:58:49 schwarze Exp $
+.Dd $Mdocdate: December 21 2018 $
+.Dt LI-ARG 1
+.Os
+.Sh NAME
+.Nm Li-arg
+.Nd escape sequences in arguments of in-line macros
+.Sh DESCRIPTION
+.ds a \(at
+unquoted unescaped:
+.Li \*a
+.Pp
+quoted unescaped:
+.Li "\*a"
+.Pp
+unquoted escaped:
+.Li \\*a
+.Pp
+quoted escaped:
+.Li "\\*a"
diff --git a/regress/mdoc/Li/arg.out_ascii b/regress/mdoc/Li/arg.out_ascii
new file mode 100644
index 00000000..f0a69f18
--- /dev/null
+++ b/regress/mdoc/Li/arg.out_ascii
@@ -0,0 +1,15 @@
+LI-ARG(1)                   General Commands Manual                  LI-ARG(1)
+
+NNAAMMEE
+     LLii--aarrgg - escape sequences in arguments of in-line macros
+
+DDEESSCCRRIIPPTTIIOONN
+     unquoted unescaped: @
+
+     quoted unescaped: @
+
+     unquoted escaped: @
+
+     quoted escaped: @
+
+OpenBSD                        December 21, 2018                       OpenBSD
diff --git a/regress/mdoc/Li/arg.out_markdown b/regress/mdoc/Li/arg.out_markdown
new file mode 100644
index 00000000..1b0ba54e
--- /dev/null
+++ b/regress/mdoc/Li/arg.out_markdown
@@ -0,0 +1,21 @@
+LI-ARG(1) - General Commands Manual
+
+# NAME
+
+**Li-arg** - escape sequences in arguments of in-line macros
+
+# DESCRIPTION
+
+unquoted unescaped:
+`@`
+
+quoted unescaped:
+`@`
+
+unquoted escaped:
+`@`
+
+quoted escaped:
+`@`
+
+OpenBSD - December 21, 2018
diff --git a/regress/mdoc/Ns/Makefile b/regress/mdoc/Ns/Makefile
index 45db66cc..b2c409ab 100644
--- a/regress/mdoc/Ns/Makefile
+++ b/regress/mdoc/Ns/Makefile
@@ -1,6 +1,6 @@
-# $OpenBSD: Makefile,v 1.5 2014/07/02 11:42:56 schwarze Exp $
+# $OpenBSD: Makefile,v 1.8 2018/12/21 16:58:49 schwarze Exp $
 
-REGRESS_TARGETS	= position punct
+REGRESS_TARGETS	= arg position punct
 LINT_TARGETS	= position
 
 .include <bsd.regress.mk>
diff --git a/regress/mdoc/Ns/arg.in b/regress/mdoc/Ns/arg.in
new file mode 100644
index 00000000..b83a1f4a
--- /dev/null
+++ b/regress/mdoc/Ns/arg.in
@@ -0,0 +1,13 @@
+.\" $OpenBSD: arg.in,v 1.1 2018/12/21 16:58:49 schwarze Exp $
+.Dd $Mdocdate: December 21 2018 $
+.Dt NS-ARG 1
+.Os
+.Sh NAME
+.Nm Ns-arg
+.Nd escape sequences in the arguments of in-line macros with an argument limit
+.Sh DESCRIPTION
+.ds a \(at
+.No unquoted unescaped Ns \*a
+.No quoted unescaped Ns "\*a"
+.No unquoted escaped Ns \\*a
+.No quoted escaped Ns "\\*a"
diff --git a/regress/mdoc/Ns/arg.out_ascii b/regress/mdoc/Ns/arg.out_ascii
new file mode 100644
index 00000000..2b8a5104
--- /dev/null
+++ b/regress/mdoc/Ns/arg.out_ascii
@@ -0,0 +1,10 @@
+NS-ARG(1)                   General Commands Manual                  NS-ARG(1)
+
+NNAAMMEE
+     NNss--aarrgg - escape sequences in the arguments of in-line macros with an
+     argument limit
+
+DDEESSCCRRIIPPTTIIOONN
+     unquoted unescaped@ quoted unescaped@ unquoted escaped@ quoted escaped@
+
+OpenBSD                        December 21, 2018                       OpenBSD
diff --git a/regress/mdoc/Ns/arg.out_markdown b/regress/mdoc/Ns/arg.out_markdown
new file mode 100644
index 00000000..7d600590
--- /dev/null
+++ b/regress/mdoc/Ns/arg.out_markdown
@@ -0,0 +1,14 @@
+NS-ARG(1) - General Commands Manual
+
+# NAME
+
+**Ns-arg** - escape sequences in the arguments of in-line macros with an argument limit
+
+# DESCRIPTION
+
+unquoted unescaped&#64;
+quoted unescaped&#64;
+unquoted escaped&#64;
+quoted escaped&#64;
+
+OpenBSD - December 21, 2018
diff --git a/regress/mdoc/Op/Makefile b/regress/mdoc/Op/Makefile
index 4a2d6fda..b2a79566 100644
--- a/regress/mdoc/Op/Makefile
+++ b/regress/mdoc/Op/Makefile
@@ -1,6 +1,6 @@
-# $OpenBSD: Makefile,v 1.3 2014/07/02 11:42:56 schwarze Exp $
+# $OpenBSD: Makefile,v 1.7 2018/12/21 16:58:49 schwarze Exp $
 
-REGRESS_TARGETS	= break broken punct
+REGRESS_TARGETS	= arg break broken punct
 LINT_TARGETS	= break broken punct
 
 SKIP_GROFF	= break
diff --git a/regress/mdoc/Op/arg.in b/regress/mdoc/Op/arg.in
new file mode 100644
index 00000000..2043b01e
--- /dev/null
+++ b/regress/mdoc/Op/arg.in
@@ -0,0 +1,20 @@
+.\" $OpenBSD: arg.in,v 1.1 2018/12/21 16:58:49 schwarze Exp $
+.Dd $Mdocdate: December 21 2018 $
+.Dt OP-ARG 1
+.Os
+.Sh NAME
+.Nm Op-arg
+.Nd escape sequences in the arguments of partial implicit macros
+.Sh DESCRIPTION
+.ds a \(at
+unquoted unescaped:
+.Op \*a
+.Pp
+quoted unescaped:
+.Op "\*a"
+.Pp
+unquoted escaped:
+.Op \\*a
+.Pp
+quoted escaped:
+.Op "\\*a"
diff --git a/regress/mdoc/Op/arg.out_ascii b/regress/mdoc/Op/arg.out_ascii
new file mode 100644
index 00000000..534917cb
--- /dev/null
+++ b/regress/mdoc/Op/arg.out_ascii
@@ -0,0 +1,15 @@
+OP-ARG(1)                   General Commands Manual                  OP-ARG(1)
+
+NNAAMMEE
+     OOpp--aarrgg - escape sequences in the arguments of partial implicit macros
+
+DDEESSCCRRIIPPTTIIOONN
+     unquoted unescaped: [@]
+
+     quoted unescaped: [@]
+
+     unquoted escaped: [@]
+
+     quoted escaped: [@]
+
+OpenBSD                        December 21, 2018                       OpenBSD
diff --git a/regress/mdoc/Op/arg.out_markdown b/regress/mdoc/Op/arg.out_markdown
new file mode 100644
index 00000000..5ad86f7d
--- /dev/null
+++ b/regress/mdoc/Op/arg.out_markdown
@@ -0,0 +1,21 @@
+OP-ARG(1) - General Commands Manual
+
+# NAME
+
+**Op-arg** - escape sequences in the arguments of partial implicit macros
+
+# DESCRIPTION
+
+unquoted unescaped:
+\[&#64;]
+
+quoted unescaped:
+\[&#64;]
+
+unquoted escaped:
+\[&#64;]
+
+quoted escaped:
+\[&#64;]
+
+OpenBSD - December 21, 2018