Add an option -Q (quick) to mandocdb(8)

for accelerated generation of reduced-size databases.

Implement this by allowing the parsers to optionally
abort the parse sequence after the NAME section.

While here, garbage collect the unused void *arg attribute of
struct mparse and mparse_alloc() and fix some errors in mandoc(3).

This reduces the processing time of mandocdb(8) on /usr/share/man
by a factor of 2 and the database size by a factor of 4.
However, it still takes 5 times the time and 6 times the space
of makewhatis(8), so more work is clearly needed.

diff --git a/libman.h b/libman.h
index f2ba6a125630abe718920b1cc7438233d188941d..7d6de38232bb039dc039d4826ecde42e705b2ded 100644 (file)
--- a/libman.h
+++ b/libman.h
@@ -1,4 +1,4 @@
-/*     $Id: libman.h,v 1.56 2012/11/17 00:26:33 schwarze Exp $ */
+/*     $Id: libman.h,v 1.57 2014/01/05 20:26:36 schwarze Exp $ */
 /*
  * Copyright (c) 2009, 2010, 2011 Kristaps Dzonsons <kristaps@bsd.lv>
  *
@@ -24,6 +24,7 @@ enum  man_next {
 
 struct man {
        struct mparse   *parse; /* parse pointer */
+       int              quick; /* abort parse early */
        int              flags; /* parse flags */
 #define        MAN_HALT        (1 << 0) /* badness happened: die */
 #define        MAN_ELINE       (1 << 1) /* Next-line element scope. */

diff --git a/libmandoc.h b/libmandoc.h
index ab243ff978a4d6c45450b4007c3860fbacda52f9..7a1b4219cad3e52df0a327912ecfa45a87113a05 100644 (file)
--- a/libmandoc.h
+++ b/libmandoc.h
@@ -1,4 +1,4 @@
-/*     $Id: libmandoc.h,v 1.37 2014/01/05 19:10:56 joerg Exp $ */
+/*     $Id: libmandoc.h,v 1.38 2014/01/05 20:26:36 schwarze Exp $ */
 /*
  * Copyright (c) 2009, 2010, 2011, 2012 Kristaps Dzonsons <kristaps@bsd.lv>
  * Copyright (c) 2013 Ingo Schwarze <schwarze@openbsd.org>
@@ -50,7 +50,7 @@ int            mandoc_strntoi(const char *, size_t, int);
 const char     *mandoc_a2msec(const char*);
 
 void            mdoc_free(struct mdoc *);
-struct mdoc    *mdoc_alloc(struct roff *, struct mparse *, char *);
+struct mdoc    *mdoc_alloc(struct roff *, struct mparse *, char *, int);
 void            mdoc_reset(struct mdoc *);
 int             mdoc_parseln(struct mdoc *, int, char *, int);
 int             mdoc_endparse(struct mdoc *);
@@ -58,7 +58,7 @@ int            mdoc_addspan(struct mdoc *, const struct tbl_span *);
 int             mdoc_addeqn(struct mdoc *, const struct eqn *);
 
 void            man_free(struct man *);
-struct man     *man_alloc(struct roff *, struct mparse *);
+struct man     *man_alloc(struct roff *, struct mparse *, int);
 void            man_reset(struct man *);
 int             man_parseln(struct man *, int, char *, int);
 int             man_endparse(struct man *);

diff --git a/libmdoc.h b/libmdoc.h
index 3f14519d3b404358bd1c87e2f0475042b120e6c4..cd0ce9d302d692a358b94f081fbb852dda7df68b 100644 (file)
--- a/libmdoc.h
+++ b/libmdoc.h
@@ -1,4 +1,4 @@
-/*     $Id: libmdoc.h,v 1.82 2013/10/21 23:47:58 schwarze Exp $ */
+/*     $Id: libmdoc.h,v 1.83 2014/01/05 20:26:36 schwarze Exp $ */
 /*
  * Copyright (c) 2008, 2009, 2010, 2011 Kristaps Dzonsons <kristaps@bsd.lv>
  * Copyright (c) 2013 Ingo Schwarze <schwarze@openbsd.org>
@@ -26,6 +26,7 @@ enum  mdoc_next {
 struct mdoc {
        struct mparse    *parse; /* parse pointer */
        char             *defos; /* default argument for .Os */
+       int               quick; /* abort parse early */
        int               flags; /* parse flags */
 #define        MDOC_HALT        (1 << 0) /* error in parse: halt */
 #define        MDOC_LITERAL     (1 << 1) /* in a literal scope */

diff --git a/main.c b/main.c
index 7e5c7a98aef285cdefcc75c3ea3e5779c4f2222c..5038b63989114a327b7b189401fbf240a2564ae5 100644 (file)
--- a/main.c
+++ b/main.c
@@ -1,4 +1,4 @@
-/*     $Id: main.c,v 1.167 2012/11/19 17:22:26 schwarze Exp $ */
+/*     $Id: main.c,v 1.168 2014/01/05 20:26:36 schwarze Exp $ */
 /*
  * Copyright (c) 2008, 2009, 2010, 2011 Kristaps Dzonsons <kristaps@bsd.lv>
  * Copyright (c) 2010, 2011, 2012 Ingo Schwarze <schwarze@openbsd.org>
@@ -140,7 +140,7 @@ main(int argc, char *argv[])
                        /* NOTREACHED */
                }
 
-       curp.mp = mparse_alloc(type, curp.wlevel, mmsg, &curp, defos);
+       curp.mp = mparse_alloc(type, curp.wlevel, mmsg, defos, 0);
 
        /*
         * Conditionally start up the lookaside buffer before parsing.

diff --git a/man.c b/man.c
index 72c6afab6f80d6c40302736faf8dc3dab2627e3f..b810ac7b353071c14b429e7430658ac24d7d65f1 100644 (file)
--- a/man.c
+++ b/man.c
@@ -1,6 +1,7 @@
-/*     $Id: man.c,v 1.122 2013/12/31 23:23:10 schwarze Exp $ */
+/*     $Id: man.c,v 1.123 2014/01/05 20:26:36 schwarze Exp $ */
 /*
  * Copyright (c) 2008, 2009, 2010, 2011 Kristaps Dzonsons <kristaps@bsd.lv>
+ * Copyright (c) 2013, 2014 Ingo Schwarze <schwarze@openbsd.org>
  *
  * Permission to use, copy, modify, and distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -97,7 +98,7 @@ man_free(struct man *man)
 
 
 struct man *
-man_alloc(struct roff *roff, struct mparse *parse)
+man_alloc(struct roff *roff, struct mparse *parse, int quick)
 {
        struct man      *p;
 
@@ -105,6 +106,7 @@ man_alloc(struct roff *roff, struct mparse *parse)
 
        man_hash_init();
        p->parse = parse;
+       p->quick = quick;
        p->roff = roff;
 
        man_alloc1(p);
@@ -604,6 +606,12 @@ man_pmacro(struct man *man, int ln, char *buf, int offs)
        if ( ! (*man_macros[tok].fp)(man, tok, ln, ppos, &offs, buf))
                goto err;
 
+       /* In quick mode (for mandocdb), abort after the NAME section. */
+
+       if (man->quick && MAN_SH == tok &&
+           strcmp(man->last->prev->child->string, "NAME"))
+               return(2);
+
        /* 
         * We weren't in a block-line scope when entering the
         * above-parsed macro, so return.

diff --git a/mandoc.3 b/mandoc.3
index fe6503d5477ef050ef9c371f4be86b7b0dc2b9d7..db7637491491057123e9b52905392f1e00dccace 100644 (file)
--- a/mandoc.3
+++ b/mandoc.3
@@ -1,4 +1,4 @@
-.\"    $Id: mandoc.3,v 1.22 2013/10/06 17:01:52 schwarze Exp $
+.\"    $Id: mandoc.3,v 1.23 2014/01/05 20:26:36 schwarze Exp $
 .\"
 .\" Copyright (c) 2009, 2010, 2011 Kristaps Dzonsons <kristaps@bsd.lv>
 .\" Copyright (c) 2010 Ingo Schwarze <schwarze@openbsd.org>
@@ -15,7 +15,7 @@
 .\" ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 .\" OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 .\"
-.Dd $Mdocdate: October 6 2013 $
+.Dd $Mdocdate: January 5 2014 $
 .Dt MANDOC 3
 .Os
 .Sh NAME
@@ -50,8 +50,8 @@
 .In mandoc.h
 .Ft "enum mandoc_esc"
 .Fo mandoc_escape
-.Fa "const char const **end"
-.Fa "const char const **start"
+.Fa "const char **end"
+.Fa "const char **start"
 .Fa "int *sz"
 .Fc
 .Ft "const struct man_meta *"
@@ -97,16 +97,17 @@
 .Fc
 .Ft void
 .Fo mparse_alloc
-.Fa "enum mparset type"
+.Fa "enum mparset inttype"
 .Fa "enum mandoclevel wlevel"
-.Fa "mandocmsg msg"
-.Fa "void *msgarg"
+.Fa "mandocmsg mmsg"
+.Fa "char *defos"
+.Fa "int quick"
 .Fc
 .Ft void
 .Fo mparse_free
 .Fa "struct mparse *parse"
 .Fc
-.Ft void
+.Ft const char *
 .Fo mparse_getkeep
 .Fa "const struct mparse *parse"
 .Fc
@@ -203,7 +204,7 @@ An escape sequence classification.
 A fatal error, error, or warning message during parsing.
 .It Vt "enum mandoclevel"
 A classification of an
-.Vt "enum mandoclevel"
+.Vt "enum mandocerr"
 as regards system operation.
 .It Vt "struct mchars"
 An opaque pointer to an object allowing for translation between
@@ -352,6 +353,42 @@ implemented in
 .Pa mdoc.c .
 .It Fn mparse_alloc
 Allocate a parser.
+The arguments have the following effect:
+.Bl -tag -offset 5n -width inttype
+.It Ar inttype
+When set to
+.Dv MPARSE_MDOC
+or
+.Dv MPARSE_MAN ,
+only that parser will be used.
+With
+.Dv MPARSE_AUTO ,
+the document type will be automatically detected.
+.It Ar wlevel
+Can be set to
+.Dv MANDOCLEVEL_FATAL ,
+.Dv MANDOCLEVEL_ERROR ,
+or
+.Dv MANDOCLEVEL_WARNING .
+Messages below the selected level will be suppressed.
+.It Ar mmsg
+A callback function to handle errors and warnings.
+See
+.Pa main.c
+for an example.
+.It Ar defos
+A default string for the
+.Xr mdoc 7
+.Sq \&Os
+macro, overriding the
+.Dv OSNAME
+preprocessor definition and the results of
+.Xr uname 3 .
+.It Ar quick
+When set, parsing is aborted after the NAME section.
+This is for example useful to quickly build minimal databases.
+.El
+.Pp
 The same parser may be used for multiple files so long as
 .Fn mparse_reset
 is called between parses.

diff --git a/mandoc.h b/mandoc.h
index 3f1f118a2e77e0919b72abcbe329d854152ffcbb..9c209ee3874e304e39e9bdac26edcfee151d0367 100644 (file)
--- a/mandoc.h
+++ b/mandoc.h
@@ -1,4 +1,4 @@
-/*     $Id: mandoc.h,v 1.113 2014/01/02 16:29:55 schwarze Exp $ */
+/*     $Id: mandoc.h,v 1.114 2014/01/05 20:26:36 schwarze Exp $ */
 /*
  * Copyright (c) 2010, 2011 Kristaps Dzonsons <kristaps@bsd.lv>
  * Copyright (c) 2012, 2013, 2014 Ingo Schwarze <schwarze@openbsd.org>
@@ -422,7 +422,7 @@ int           mchars_spec2cp(const struct mchars *,
 const char      *mchars_spec2str(const struct mchars *, 
                        const char *, size_t, size_t *);
 struct mparse   *mparse_alloc(enum mparset, enum mandoclevel,
-                       mandocmsg, void *, char *);
+                       mandocmsg, char *, int);
 void             mparse_free(struct mparse *);
 void             mparse_keep(struct mparse *);
 enum mandoclevel  mparse_readfd(struct mparse *, int, const char *);

diff --git a/mandocdb.c b/mandocdb.c
index 0effafbc14c82376ef5c1ea9d5cd5f22ba772848..7068ddbc5008bea98905722d834e0a3bfd9d763e 100644 (file)
--- a/mandocdb.c
+++ b/mandocdb.c
@@ -1,4 +1,4 @@
-/*     $Id: mandocdb.c,v 1.101 2014/01/05 04:48:40 schwarze Exp $ */
+/*     $Id: mandocdb.c,v 1.102 2014/01/05 20:26:36 schwarze Exp $ */
 /*
  * Copyright (c) 2011, 2012 Kristaps Dzonsons <kristaps@bsd.lv>
  * Copyright (c) 2011, 2012, 2013, 2014 Ingo Schwarze <schwarze@openbsd.org>
@@ -167,8 +167,9 @@ static      int      treescan(void);
 static size_t   utf8(unsigned int, char [7]);
 
 static char            *progname;
-static int              use_all; /* use all found files */
 static int              nodb; /* no database changes */
+static int              quick; /* abort the parse early */
+static int              use_all; /* use all found files */
 static int              verb; /* print what we're doing */
 static int              warnings; /* warn about crap */
 static int              write_utf8; /* write UTF-8 output; else ASCII */
@@ -347,7 +348,7 @@ main(int argc, char *argv[])
        path_arg = NULL;
        op = OP_DEFAULT;
 
-       while (-1 != (ch = getopt(argc, argv, "aC:d:nT:tu:vW")))
+       while (-1 != (ch = getopt(argc, argv, "aC:d:nQT:tu:vW")))
                switch (ch) {
                case ('a'):
                        use_all = 1;
@@ -365,6 +366,9 @@ main(int argc, char *argv[])
                case ('n'):
                        nodb = 1;
                        break;
+               case ('Q'):
+                       quick = 1;
+                       break;
                case ('T'):
                        if (strcmp(optarg, "utf8")) {
                                fprintf(stderr, "-T%s: Unsupported "
@@ -404,7 +408,7 @@ main(int argc, char *argv[])
 
        exitcode = (int)MANDOCLEVEL_OK;
        mp = mparse_alloc(MPARSE_AUTO, 
-               MANDOCLEVEL_FATAL, NULL, NULL, NULL);
+               MANDOCLEVEL_FATAL, NULL, NULL, quick);
        mc = mchars_alloc();
 
        ohash_init(&mpages, 6, &mpages_info);
@@ -494,11 +498,11 @@ out:
        ohash_delete(&mlinks);
        return(exitcode);
 usage:
-       fprintf(stderr, "usage: %s [-anvW] [-C file] [-Tutf8]\n"
-                       "       %s [-anvW] [-Tutf8] dir ...\n"
-                       "       %s [-nvW] [-Tutf8] -d dir [file ...]\n"
+       fprintf(stderr, "usage: %s [-anQvW] [-C file] [-Tutf8]\n"
+                       "       %s [-anQvW] [-Tutf8] dir ...\n"
+                       "       %s [-nQvW] [-Tutf8] -d dir [file ...]\n"
                        "       %s [-nvW] -u dir [file ...]\n"
-                       "       %s -t file ...\n",
+                       "       %s [-Q] -t file ...\n",
                       progname, progname, progname, 
                       progname, progname);
 

diff --git a/mdoc.c b/mdoc.c
index 581002770c6d41df190bb2fe7229dfd780773655..a586e11c5540d7c011ab07083008e54d8572923d 100644 (file)
--- a/mdoc.c
+++ b/mdoc.c
@@ -1,7 +1,7 @@
-/*     $Id: mdoc.c,v 1.207 2013/12/31 23:23:11 schwarze Exp $ */
+/*     $Id: mdoc.c,v 1.208 2014/01/05 20:26:36 schwarze Exp $ */
 /*
  * Copyright (c) 2008, 2009, 2010, 2011 Kristaps Dzonsons <kristaps@bsd.lv>
- * Copyright (c) 2010, 2012, 2013 Ingo Schwarze <schwarze@openbsd.org>
+ * Copyright (c) 2010, 2012, 2013, 2014 Ingo Schwarze <schwarze@openbsd.org>
  *
  * Permission to use, copy, modify, and distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -197,7 +197,8 @@ mdoc_free(struct mdoc *mdoc)
  * Allocate volatile and non-volatile parse resources.  
  */
 struct mdoc *
-mdoc_alloc(struct roff *roff, struct mparse *parse, char *defos)
+mdoc_alloc(struct roff *roff, struct mparse *parse,
+       char *defos, int quick)
 {
        struct mdoc     *p;
 
@@ -205,6 +206,7 @@ mdoc_alloc(struct roff *roff, struct mparse *parse, char *defos)
 
        p->parse = parse;
        p->defos = defos;
+       p->quick = quick;
        p->roff = roff;
 
        mdoc_hash_init();
@@ -961,6 +963,12 @@ mdoc_pmacro(struct mdoc *mdoc, int ln, char *buf, int offs)
        if ( ! mdoc_macro(mdoc, tok, ln, sv, &offs, buf)) 
                goto err;
 
+       /* In quick mode (for mandocdb), abort after the NAME section. */
+
+       if (mdoc->quick && MDOC_Sh == tok &&
+           SEC_NAME != mdoc->last->sec)
+               return(2);
+
        return(1);
 
 err:   /* Error out. */

diff --git a/read.c b/read.c
index 7106c52d3ca3d75440944a796317acf784b8b4e3..417200e70eee9a03e7e34c0f3153a73c9dceafc7 100644 (file)
--- a/read.c
+++ b/read.c
@@ -1,4 +1,4 @@
-/*     $Id: read.c,v 1.40 2014/01/02 16:29:55 schwarze Exp $ */
+/*     $Id: read.c,v 1.41 2014/01/05 20:26:36 schwarze Exp $ */
 /*
  * Copyright (c) 2008, 2009, 2010, 2011 Kristaps Dzonsons <kristaps@bsd.lv>
  * Copyright (c) 2010-2014 Ingo Schwarze <schwarze@openbsd.org>
@@ -60,10 +60,10 @@ struct      mparse {
        struct roff      *roff; /* roff parser (!NULL) */
        int               reparse_count; /* finite interp. stack */
        mandocmsg         mmsg; /* warning/error message handler */
-       void             *arg; /* argument to mmsg */
        const char       *file; 
        struct buf       *secondary;
        char             *defos; /* default operating system */
+       int               quick; /* abort the parse early */
 };
 
 static void      resize_buf(struct buf *, size_t);
@@ -258,13 +258,14 @@ pset(const char *buf, int pos, struct mparse *curp)
        case (MPARSE_MDOC):
                if (NULL == curp->pmdoc) 
                        curp->pmdoc = mdoc_alloc(curp->roff, curp,
-                                       curp->defos);
+                                       curp->defos, curp->quick);
                assert(curp->pmdoc);
                curp->mdoc = curp->pmdoc;
                return;
        case (MPARSE_MAN):
                if (NULL == curp->pman) 
-                       curp->pman = man_alloc(curp->roff, curp);
+                       curp->pman = man_alloc(curp->roff, curp,
+                                       curp->quick);
                assert(curp->pman);
                curp->man = curp->pman;
                return;
@@ -275,14 +276,14 @@ pset(const char *buf, int pos, struct mparse *curp)
        if (pos >= 3 && 0 == memcmp(buf, ".Dd", 3))  {
                if (NULL == curp->pmdoc) 
                        curp->pmdoc = mdoc_alloc(curp->roff, curp,
-                                       curp->defos);
+                                       curp->defos, curp->quick);
                assert(curp->pmdoc);
                curp->mdoc = curp->pmdoc;
                return;
        } 
 
        if (NULL == curp->pman) 
-               curp->pman = man_alloc(curp->roff, curp);
+               curp->pman = man_alloc(curp->roff, curp, curp->quick);
        assert(curp->pman);
        curp->man = curp->pman;
 }
@@ -560,7 +561,8 @@ rerun:
                if (0 == rc) {
                        assert(MANDOCLEVEL_FATAL <= curp->file_status);
                        break;
-               }
+               } else if (2 == rc)
+                       break;
 
                /* Temporary buffers typically are not full. */
 
@@ -763,7 +765,7 @@ out:
 
 struct mparse *
 mparse_alloc(enum mparset inttype, enum mandoclevel wlevel,
-               mandocmsg mmsg, void *arg, char *defos)
+               mandocmsg mmsg, char *defos, int quick)
 {
        struct mparse   *curp;
 
@@ -773,9 +775,9 @@ mparse_alloc(enum mparset inttype, enum mandoclevel wlevel,
 
        curp->wlevel = wlevel;
        curp->mmsg = mmsg;
-       curp->arg = arg;
        curp->inttype = inttype;
        curp->defos = defos;
+       curp->quick = quick;
 
        curp->roff = roff_alloc(inttype, curp);
        return(curp);