Split tagging into a validation part including prioritization

in tag.{h,c} and {mdoc,man}_validate.c
and into a formatting part including command line argument checking
in term_tag.{h,c}, html.c, and {mdoc|man}_{term|html}.c.

Immediate functional benefits include:
* Improved prioritization of automatic tags for .Em and .Sy.
* Avoiding bogus automatic tags when .Em, .Fn, or .Sy are explicitly tagged.
* Explicit tagging of .Er and .Fl now works in HTML output.
* Automatic tagging of .IP and .TP now works in HTML output.
But mainly, this patch provides clean earth to build further improvements on.

Technical changes:
* Main program: Write a tag file for ASCII and UTF-8 output only.
* All formatters: There is no more need to delay writing the tags.
* mdoc(7)+man(7) formatters: No more need for elaborate syntax tree inspection.
* HTML formatter: If available, use the "string" attribute as the tag.
* HTML formatter: New function to write permalinks, to reduce code duplication.

Style cleanup in the vicinity while here:
* mdoc(7) terminal formatter: To set up bold font for children,
defer to termp_bold_pre() rather than calling term_fontpush() manually.
* mdoc(7) terminal formatter: Garbage collect some duplicate functions.
* mdoc(7) HTML formatter: Unify <code> handling, delete redundant functions.
* Where possible, use switch statements rather than if cascades.
* Get rid of some more Yoda notation.

The necessity for such changes was first discussed with kn@, but i didn't
bother him with a request to review the resulting -673/+782 line patch.

1 files changed, 61 insertions, 29 deletions

diff --git a/regress/regress.pl b/regress/regress.pl
index 5e7927cb..3c6e64df 100755
--- a/regress/regress.pl
+++ b/regress/regress.pl
@@ -1,8 +1,8 @@
 #!/usr/bin/env perl
 #
-# $Id: regress.pl,v 1.13 2020/01/08 10:37:53 schwarze Exp $
+# $Id: regress.pl,v 1.14 2020/03/13 15:32:31 schwarze Exp $
 #
-# Copyright (c) 2017 Ingo Schwarze <schwarze@openbsd.org>
+# Copyright (c) 2017, 2018, 2019, 2020 Ingo Schwarze <schwarze@openbsd.org>
 #
 # Permission to use, copy, modify, and distribute this software for any
 # purpose with or without fee is hereby granted, provided that the above
@@ -120,16 +120,17 @@ sub fail ($$) {
 
 my $onlytest = shift // '';
 for (@ARGV) {
-	/^(all|ascii|utf8|man|html|markdown|lint|clean|verbose)$/
+	/^(all|ascii|tag|man|utf8|html|markdown|lint|clean|verbose)$/
 	    or usage "$_: invalid modifier";
 	$targets{$_} = 1;
 }
 $targets{all} = 1
-    unless $targets{ascii} || $targets{utf8} || $targets{man} ||
-      $targets{html} || $targets{markdown} ||
+    unless $targets{ascii} || $targets{tag} || $targets{man} ||
+      $targets{utf8} || $targets{html} || $targets{markdown} ||
       $targets{lint} || $targets{clean};
-$targets{ascii} = $targets{utf8} = $targets{man} = $targets{html} =
-    $targets{markdown} = $targets{lint} = 1 if $targets{all};
+$targets{ascii} = $targets{tag} = $targets{man} = $targets{utf8} =
+    $targets{html} = $targets{markdown} = $targets{lint} = 1
+    if $targets{all};
 
 
 # --- parse Makefiles --------------------------------------------------
@@ -157,7 +158,7 @@ sub parse_makefile ($%) {
 }
 
 my (@regress_tests, @utf8_tests, @lint_tests, @html_tests);
-my (%skip_ascii, %skip_man, %skip_markdown);
+my (%tag_tests, %skip_ascii, %skip_man, %skip_markdown);
 foreach my $module (qw(roff char mdoc man tbl eqn)) {
 	my %modvars;
 	parse_makefile "$module/Makefile", \%modvars;
@@ -168,36 +169,41 @@ foreach my $module (qw(roff char mdoc man tbl eqn)) {
 		delete $subvars{GOPTS};
 		delete $subvars{SKIP_GROFF};
 		delete $subvars{SKIP_GROFF_ASCII};
-		my @mandoc = ('../mandoc', split ' ', $subvars{MOPTS});
+		my @mopts = split ' ', $subvars{MOPTS};
 		delete $subvars{MOPTS};
 		my @regress_testnames;
+		if (defined $subvars{TAG_TARGETS}) {
+			$tag_tests{"$module/$subdir/$_"} = 1
+			    for split ' ', $subvars{TAG_TARGETS};
+			delete $subvars{TAG_TARGETS};
+		}
 		if (defined $subvars{REGRESS_TARGETS}) {
 			push @regress_testnames,
 			    split ' ', $subvars{REGRESS_TARGETS};
 			push @regress_tests, {
 			    NAME => "$module/$subdir/$_",
-			    MANDOC => \@mandoc,
+			    MOPTS => \@mopts,
 			} foreach @regress_testnames;
 			delete $subvars{REGRESS_TARGETS};
 		}
 		if (defined $subvars{UTF8_TARGETS}) {
 			push @utf8_tests, {
 			    NAME => "$module/$subdir/$_",
-			    MANDOC => \@mandoc,
+			    MOPTS => \@mopts,
 			} foreach split ' ', $subvars{UTF8_TARGETS};
 			delete $subvars{UTF8_TARGETS};
 		}
 		if (defined $subvars{HTML_TARGETS}) {
 			push @html_tests, {
 			    NAME => "$module/$subdir/$_",
-			    MANDOC => \@mandoc,
+			    MOPTS => \@mopts,
 			} foreach split ' ', $subvars{HTML_TARGETS};
 			delete $subvars{HTML_TARGETS};
 		}
 		if (defined $subvars{LINT_TARGETS}) {
 			push @lint_tests, {
 			    NAME => "$module/$subdir/$_",
-			    MANDOC => \@mandoc,
+			    MOPTS => \@mopts,
 			} foreach split ' ', $subvars{LINT_TARGETS};
 			delete $subvars{LINT_TARGETS};
 		}
@@ -243,22 +249,44 @@ foreach my $module (qw(roff char mdoc man tbl eqn)) {
 
 my $count_total = 0;
 my $count_ascii = 0;
+my $count_tag = 0;
 my $count_man = 0;
 my $count_rm = 0;
-if ($targets{ascii} || $targets{man}) {
-	print "Running ascii and man tests ";
+if ($targets{ascii} || $targets{tag} || $targets{man}) {
+	print "Running ascii, tag, and man tests ";
 	print "...\n" if $targets{verbose};
 }
 for my $test (@regress_tests) {
 	my $i = "$test->{NAME}.in";
 	my $o = "$test->{NAME}.mandoc_ascii";
 	my $w = "$test->{NAME}.out_ascii";
-	if ($targets{ascii} && !$skip_ascii{$test->{NAME}} &&
+	my $to = "$test->{NAME}.mandoc_tag";
+	my $tw = "$test->{NAME}.out_tag";
+	my $diff_ascii;
+	if ($targets{tag} && $tag_tests{$test->{NAME}} &&
 	    $test->{NAME} =~ /^$onlytest/) {
-		$count_ascii++;
+		$count_tag++;
 		$count_total++;
-		sysout $o, @{$test->{MANDOC}}, qw(-I os=OpenBSD -T ascii), $i
+		local $ENV{MANPAGER} = "./copyless $test->{NAME}";
+		my @cmd = (qw(../man -l), @{$test->{MOPTS}},
+		    qw(-I os=OpenBSD -T ascii), $i);
+		print "@cmd\n" if $targets{verbose};
+		system @cmd
+		    and fail $test->{NAME}, 'tag:man';
+		system @diff, $tw, $to
+		    and fail $test->{NAME}, 'tag:diff';
+		print "." unless $targets{verbose};
+		$diff_ascii = $targets{ascii};
+	} elsif ($targets{ascii} && !$skip_ascii{$test->{NAME}} &&
+	    $test->{NAME} =~ /^$onlytest/) {
+		sysout $o, '../mandoc', @{$test->{MOPTS}},
+		    qw(-I os=OpenBSD -T ascii), $i
 		    and fail $test->{NAME}, 'ascii:mandoc';
+		$diff_ascii = 1;
+	}
+	if ($diff_ascii) {
+		$count_ascii++;
+		$count_total++;
 		system @diff, $w, $o
 		    and fail $test->{NAME}, 'ascii:diff';
 		print "." unless $targets{verbose};
@@ -269,9 +297,10 @@ for my $test (@regress_tests) {
 	    $test->{NAME} =~ /^$onlytest/) {
 		$count_man++;
 		$count_total++;
-		sysout $m, @{$test->{MANDOC}}, qw(-I os=OpenBSD -T man), $i
+		sysout $m, '../mandoc', @{$test->{MOPTS}},
+		    qw(-I os=OpenBSD -T man), $i
 		    and fail $test->{NAME}, 'man:man';
-		sysout $mo, @{$test->{MANDOC}},
+		sysout $mo, '../mandoc', @{$test->{MOPTS}},
 		    qw(-man -I os=OpenBSD -T ascii -O mdoc), $m
 		    and fail $test->{NAME}, 'man:mandoc';
 		system @diff, $w, $mo
@@ -279,13 +308,13 @@ for my $test (@regress_tests) {
 		print "." unless $targets{verbose};
 	}
 	if ($targets{clean}) {
-		print "rm $o $m $mo\n" if $targets{verbose};
-		$count_rm += unlink $o, $m, $mo;
+		print "rm $o $to $m $mo\n" if $targets{verbose};
+		$count_rm += unlink $o, $to, $m, $mo;
 	}
 }
-if ($targets{ascii} || $targets{man}) {
-	print "Number of ascii and man tests:" if $targets{verbose};
-	print " $count_ascii + $count_man tests run.\n";
+if ($targets{ascii} || $targets{tag} || $targets{man}) {
+	print "Number of ascii, tag, and man tests:" if $targets{verbose};
+	print " $count_ascii + $count_tag + $count_man tests run.\n";
 }
 
 my $count_utf8 = 0;
@@ -300,7 +329,8 @@ for my $test (@utf8_tests) {
 	if ($targets{utf8} && $test->{NAME} =~ /^$onlytest/o) {
 		$count_utf8++;
 		$count_total++;
-		sysout $o, @{$test->{MANDOC}}, qw(-I os=OpenBSD -T utf8), $i
+		sysout $o, '../mandoc', @{$test->{MOPTS}},
+		    qw(-I os=OpenBSD -T utf8), $i
 		    and fail $test->{NAME}, 'utf8:mandoc';
 		system @diff, $w, $o
 		    and fail $test->{NAME}, 'utf8:diff';
@@ -328,7 +358,8 @@ for my $test (@html_tests) {
 	if ($targets{html} && $test->{NAME} =~ /^$onlytest/) {
 		$count_html++;
 		$count_total++;
-		syshtml $o, @{$test->{MANDOC}}, qw(-T html), $i
+		syshtml $o, '../mandoc', @{$test->{MOPTS}},
+		    qw(-T html), $i
 		    and fail $test->{NAME}, 'html:mandoc';
 		system @diff, $w, $o
 		    and fail $test->{NAME}, 'html:diff';
@@ -357,7 +388,7 @@ for my $test (@regress_tests) {
 	    $test->{NAME} =~ /^$onlytest/) {
 		$count_markdown++;
 		$count_total++;
-		sysout $o, @{$test->{MANDOC}},
+		sysout $o, '../mandoc', @{$test->{MOPTS}},
 		    qw(-I os=OpenBSD -T markdown), $i
 		    and fail $test->{NAME}, 'markdown:mandoc';
 		system @diff, $w, $o
@@ -386,7 +417,7 @@ for my $test (@lint_tests) {
 	if ($targets{lint} && $test->{NAME} =~ /^$onlytest/) {
 		$count_lint++;
 		$count_total++;
-		syslint $o, @{$test->{MANDOC}},
+		syslint $o, '../mandoc', @{$test->{MOPTS}},
 		    qw(-I os=OpenBSD -T lint -W all), $i
 		    and fail $test->{NAME}, 'lint:mandoc';
 		system @diff, $w, $o
@@ -418,6 +449,7 @@ if ($count_total == 1) {
 } elsif ($count_total) {
 	print "All $count_total tests OK:";
 	print " $count_ascii ascii" if $count_ascii;
+	print " $count_tag tag" if $count_tag;
 	print " $count_man man" if $count_man;
 	print " $count_utf8 utf8" if $count_utf8;
 	print " $count_html html" if $count_html;