Generalize the mparse_alloc() and roff_alloc() functions by giving

[mandoc.git] / mandocdb.c

diff --git a/mandocdb.c b/mandocdb.c
index 2fd918c18a528d9f264981f6fd023ddea37bac79..48d3deb3d13643368e356b9cc6f53397103e5e4b 100644 (file)
--- a/mandocdb.c
+++ b/mandocdb.c
@@ -1,4 +1,4 @@
-/*     $Id: mandocdb.c,v 1.106 2014/01/06 15:32:47 schwarze Exp $ */
+/*     $Id: mandocdb.c,v 1.116 2014/03/19 21:51:20 schwarze Exp $ */

 /*
  * Copyright (c) 2011, 2012 Kristaps Dzonsons <kristaps@bsd.lv>
  * Copyright (c) 2011, 2012, 2013, 2014 Ingo Schwarze <schwarze@openbsd.org>
 /*
  * Copyright (c) 2011, 2012 Kristaps Dzonsons <kristaps@bsd.lv>
  * Copyright (c) 2011, 2012, 2013, 2014 Ingo Schwarze <schwarze@openbsd.org>


@@ -20,6 +20,7 @@
 #endif
 
 #include <sys/stat.h>
 #endif
 
 #include <sys/stat.h>

+#include <sys/wait.h>

 
 #include <assert.h>
 #include <ctype.h>
 
 #include <assert.h>
 #include <ctype.h>


@@ -48,6 +49,9 @@
 #include "manpath.h"
 #include "mansearch.h"
 
 #include "manpath.h"
 #include "mansearch.h"
 

+extern int mansearch_keymax;
+extern const char *const mansearch_keynames[];
+

 #define        SQL_EXEC(_v) \
        if (SQLITE_OK != sqlite3_exec(db, (_v), NULL, NULL, NULL)) \
                fprintf(stderr, "%s\n", sqlite3_errmsg(db))
 #define        SQL_EXEC(_v) \
        if (SQLITE_OK != sqlite3_exec(db, (_v), NULL, NULL, NULL)) \
                fprintf(stderr, "%s\n", sqlite3_errmsg(db))


@@ -154,8 +158,7 @@ static      int      parse_mdoc_Nd(struct mpage *, const struct mdoc_node *);
 static int      parse_mdoc_Nm(struct mpage *, const struct mdoc_node *);
 static int      parse_mdoc_Sh(struct mpage *, const struct mdoc_node *);
 static int      parse_mdoc_Xr(struct mpage *, const struct mdoc_node *);
 static int      parse_mdoc_Nm(struct mpage *, const struct mdoc_node *);
 static int      parse_mdoc_Sh(struct mpage *, const struct mdoc_node *);
 static int      parse_mdoc_Xr(struct mpage *, const struct mdoc_node *);

-static void     putkey(const struct mpage *,
-                       const char *, uint64_t);
+static void     putkey(const struct mpage *, char *, uint64_t);

 static void     putkeys(const struct mpage *,
                        const char *, size_t, uint64_t);
 static void     putmdockey(const struct mpage *,
 static void     putkeys(const struct mpage *,
                        const char *, size_t, uint64_t);
 static void     putmdockey(const struct mpage *,


@@ -166,9 +169,10 @@ static     int      set_basedir(const char *);
 static int      treescan(void);
 static size_t   utf8(unsigned int, char [7]);
 
 static int      treescan(void);
 static size_t   utf8(unsigned int, char [7]);
 

+static char             tempfilename[32];

 static char            *progname;
 static int              nodb; /* no database changes */
 static char            *progname;
 static int              nodb; /* no database changes */

-static int              quick; /* abort the parse early */
+static int              mparse_options; /* abort the parse early */

 static int              use_all; /* use all found files */
 static int              verb; /* print what we're doing */
 static int              warnings; /* warn about crap */
 static int              use_all; /* use all found files */
 static int              verb; /* print what we're doing */
 static int              warnings; /* warn about crap */


@@ -347,6 +351,7 @@ main(int argc, char *argv[])
 
        path_arg = NULL;
        op = OP_DEFAULT;
 
        path_arg = NULL;
        op = OP_DEFAULT;

+       mparse_options = MPARSE_SO;

 
        while (-1 != (ch = getopt(argc, argv, "aC:d:nQT:tu:vW")))
                switch (ch) {
 
        while (-1 != (ch = getopt(argc, argv, "aC:d:nQT:tu:vW")))
                switch (ch) {


@@ -367,7 +372,7 @@ main(int argc, char *argv[])
                        nodb = 1;
                        break;
                case ('Q'):
                        nodb = 1;
                        break;
                case ('Q'):

-                       quick = 1;
+                       mparse_options |= MPARSE_QUICK;

                        break;
                case ('T'):
                        if (strcmp(optarg, "utf8")) {
                        break;
                case ('T'):
                        if (strcmp(optarg, "utf8")) {


@@ -407,8 +412,7 @@ main(int argc, char *argv[])
        }
 
        exitcode = (int)MANDOCLEVEL_OK;
        }
 
        exitcode = (int)MANDOCLEVEL_OK;

-       mp = mparse_alloc(MPARSE_AUTO, 
-               MANDOCLEVEL_FATAL, NULL, NULL, quick);
+       mp = mparse_alloc(mparse_options, MANDOCLEVEL_FATAL, NULL, NULL);

        mc = mchars_alloc();
 
        ohash_init(&mpages, 6, &mpages_info);
        mc = mchars_alloc();
 
        ohash_init(&mpages, 6, &mpages_info);


@@ -954,12 +958,13 @@ mlink_check(struct mpage *mpage, struct mlink *mlink)
 static void
 mpages_merge(struct mchars *mc, struct mparse *mp)
 {
 static void
 mpages_merge(struct mchars *mc, struct mparse *mp)
 {

+       char                     any[] = "any";

        struct ohash_info        str_info;
        struct mpage            *mpage;
        struct mlink            *mlink;
        struct mdoc             *mdoc;
        struct man              *man;
        struct ohash_info        str_info;
        struct mpage            *mpage;
        struct mlink            *mlink;
        struct mdoc             *mdoc;
        struct man              *man;

-       const char              *cp;
+       char                    *cp;

        int                      match;
        unsigned int             pslot;
        enum mandoclevel         lvl;
        int                      match;
        unsigned int             pslot;
        enum mandoclevel         lvl;


@@ -969,6 +974,9 @@ mpages_merge(struct mchars *mc, struct mparse *mp)
        str_info.hfree = hash_free;
        str_info.key_offset = offsetof(struct str, key);
 
        str_info.hfree = hash_free;
        str_info.key_offset = offsetof(struct str, key);
 

+       if (0 == nodb)
+               SQL_EXEC("BEGIN TRANSACTION");
+

        mpage = ohash_first(&mpages, &pslot);
        while (NULL != mpage) {
                mlinks_undupe(mpage);
        mpage = ohash_first(&mpages, &pslot);
        while (NULL != mpage) {
                mlinks_undupe(mpage);


@@ -1022,7 +1030,7 @@ mpages_merge(struct mchars *mc, struct mparse *mp)
                }
                putkey(mpage, mpage->sec, TYPE_sec);
                putkey(mpage, '\0' == *mpage->arch ?
                }
                putkey(mpage, mpage->sec, TYPE_sec);
                putkey(mpage, '\0' == *mpage->arch ?

-                   "any" : mpage->arch, TYPE_arch);
+                   any : mpage->arch, TYPE_arch);

 
                for (mlink = mpage->mlinks; mlink; mlink = mlink->next) {
                        if ('\0' != *mlink->dsec)
 
                for (mlink = mpage->mlinks; mlink; mlink = mlink->next) {
                        if ('\0' != *mlink->dsec)


@@ -1030,7 +1038,7 @@ mpages_merge(struct mchars *mc, struct mparse *mp)
                        if ('\0' != *mlink->fsec)
                                putkey(mpage, mlink->fsec, TYPE_sec);
                        putkey(mpage, '\0' == *mlink->arch ?
                        if ('\0' != *mlink->fsec)
                                putkey(mpage, mlink->fsec, TYPE_sec);
                        putkey(mpage, '\0' == *mlink->arch ?

-                           "any" : mlink->arch, TYPE_arch);
+                           any : mlink->arch, TYPE_arch);

                        putkey(mpage, mlink->name, TYPE_Nm);
                }
 
                        putkey(mpage, mlink->name, TYPE_Nm);
                }
 


@@ -1059,6 +1067,9 @@ mpages_merge(struct mchars *mc, struct mparse *mp)
                ohash_delete(&strings);
                mpage = ohash_next(&mpages, &pslot);
        }
                ohash_delete(&strings);
                mpage = ohash_next(&mpages, &pslot);
        }

+
+       if (0 == nodb)
+               SQL_EXEC("END TRANSACTION");

 }
 
 static void
 }
 
 static void


@@ -1177,10 +1188,15 @@ parse_cat(struct mpage *mpage)
  * Put a type/word pair into the word database for this particular file.
  */
 static void
  * Put a type/word pair into the word database for this particular file.
  */
 static void

-putkey(const struct mpage *mpage, const char *value, uint64_t type)
+putkey(const struct mpage *mpage, char *value, uint64_t type)

 {
 {

+       char     *cp;

 
        assert(NULL != value);
 
        assert(NULL != value);

+       if (TYPE_arch == type)
+               for (cp = value; *cp; cp++)
+                       if (isupper((unsigned char)*cp))
+                               *cp = _tolower((unsigned char)*cp);

        putkeys(mpage, value, strlen(value), type);
 }
 
        putkeys(mpage, value, strlen(value), type);
 }
 


@@ -1296,6 +1312,15 @@ parse_man(struct mpage *mpage, const struct man_node *n)
                                byte = start[sz];
                                start[sz] = '\0';
 
                                byte = start[sz];
                                start[sz] = '\0';
 

+                               /*
+                                * Assume a stray trailing comma in the
+                                * name list if a name begins with a dash.
+                                */
+
+                               if ('-' == start[0] ||
+                                   ('\\' == start[0] && '-' == start[1]))
+                                       break;
+

                                putkey(mpage, start, TYPE_Nm);
 
                                if (' ' == byte) {
                                putkey(mpage, start, TYPE_Nm);
 
                                if (' ' == byte) {


@@ -1425,7 +1450,7 @@ parse_mdoc_Fd(struct mpage *mpage, const struct mdoc_node *n)
 static int
 parse_mdoc_Fn(struct mpage *mpage, const struct mdoc_node *n)
 {
 static int
 parse_mdoc_Fn(struct mpage *mpage, const struct mdoc_node *n)
 {

-       const char      *cp;
+       char    *cp;

 
        if (NULL == (n = n->child) || MDOC_TEXT != n->type)
                return(0);
 
        if (NULL == (n = n->child) || MDOC_TEXT != n->type)
                return(0);


@@ -1547,12 +1572,24 @@ putkeys(const struct mpage *mpage,
        const char *cp, size_t sz, uint64_t v)
 {
        struct str      *s;
        const char *cp, size_t sz, uint64_t v)
 {
        struct str      *s;

-       unsigned int     slot;

        const char      *end;
        const char      *end;

+       uint64_t         mask;
+       unsigned int     slot;
+       int              i;

 
        if (0 == sz)
                return;
 
 
        if (0 == sz)
                return;
 

+       if (verb > 1) {
+               for (i = 0, mask = 1;
+                    i < mansearch_keymax;
+                    i++, mask <<= 1)
+                       if (mask & v)
+                               break;
+               say(mpage->mlinks->file, "Adding key %s=%*s",
+                   mansearch_keynames[i], sz, cp);
+       }
+

        end = cp + sz;
        slot = ohash_qlookupi(&strings, cp, &end);
        s = ohash_find(&strings, slot);
        end = cp + sz;
        slot = ohash_qlookupi(&strings, cp, &end);
        s = ohash_find(&strings, slot);


@@ -1629,7 +1666,7 @@ static void
 render_key(struct mchars *mc, struct str *key)
 {
        size_t           sz, bsz, pos;
 render_key(struct mchars *mc, struct str *key)
 {
        size_t           sz, bsz, pos;

-       char             utfbuf[7], res[5];
+       char             utfbuf[7], res[6];

        char            *buf;
        const char      *seq, *cpp, *val;
        int              len, u;
        char            *buf;
        const char      *seq, *cpp, *val;
        int              len, u;


@@ -1641,7 +1678,8 @@ render_key(struct mchars *mc, struct str *key)
        res[1] = '\t';
        res[2] = ASCII_NBRSP;
        res[3] = ASCII_HYPH;
        res[1] = '\t';
        res[2] = ASCII_NBRSP;
        res[3] = ASCII_HYPH;

-       res[4] = '\0';
+       res[4] = ASCII_BREAK;
+       res[5] = '\0';

 
        val = key->key;
        bsz = strlen(val);
 
        val = key->key;
        bsz = strlen(val);


@@ -1672,15 +1710,23 @@ render_key(struct mchars *mc, struct str *key)
                        val += sz;
                }
 
                        val += sz;
                }
 

-               if (ASCII_HYPH == *val) {
+               switch (*val) {
+               case (ASCII_HYPH):

                        buf[pos++] = '-';
                        val++;
                        continue;
                        buf[pos++] = '-';
                        val++;
                        continue;

-               } else if ('\t' == *val || ASCII_NBRSP == *val) {
+               case ('\t'):
+                       /* FALLTHROUGH */
+               case (ASCII_NBRSP):

                        buf[pos++] = ' ';
                        val++;
                        buf[pos++] = ' ';
                        val++;

+                       /* FALLTHROUGH */
+               case (ASCII_BREAK):

                        continue;
                        continue;

-               } else if ('\\' != *val)
+               default:
+                       break;
+               }
+               if ('\\' != *val)

                        break;
 
                /* Read past the slash. */
                        break;
 
                /* Read past the slash. */


@@ -1754,8 +1800,6 @@ dbadd(const struct mpage *mpage, struct mchars *mc)
        if (nodb)
                return;
 
        if (nodb)
                return;
 

-       SQL_EXEC("BEGIN TRANSACTION");
-

        i = 1;
        SQL_BIND_INT(stmts[STMT_INSERT_PAGE], i, FORM_SRC == mpage->form);
        SQL_STEP(stmts[STMT_INSERT_PAGE]);
        i = 1;
        SQL_BIND_INT(stmts[STMT_INSERT_PAGE], i, FORM_SRC == mpage->form);
        SQL_STEP(stmts[STMT_INSERT_PAGE]);


@@ -1787,8 +1831,6 @@ dbadd(const struct mpage *mpage, struct mchars *mc)
                        free(key->rendered);
                free(key);
        }
                        free(key->rendered);
                free(key);
        }

-
-       SQL_EXEC("END TRANSACTION");

 }
 
 static void
 }
 
 static void


@@ -1834,6 +1876,8 @@ static void
 dbclose(int real)
 {
        size_t           i;
 dbclose(int real)
 {
        size_t           i;

+       int              status;
+       pid_t            child;

 
        if (nodb)
                return;
 
        if (nodb)
                return;


@@ -1849,9 +1893,59 @@ dbclose(int real)
        if (real)
                return;
 
        if (real)
                return;
 

-       if (-1 == rename(MANDOC_DB "~", MANDOC_DB)) {
+       if ('\0' == *tempfilename) {
+               if (-1 == rename(MANDOC_DB "~", MANDOC_DB)) {
+                       exitcode = (int)MANDOCLEVEL_SYSERR;
+                       say(MANDOC_DB, "%s", strerror(errno));
+               }
+               return;
+       }
+
+       switch (child = fork()) {
+       case (-1):
+               exitcode = (int)MANDOCLEVEL_SYSERR;
+               say("fork cmp", "%s", strerror(errno));
+               return;
+       case (0):
+               execlp("cmp", "cmp", "-s",
+                   tempfilename, MANDOC_DB, NULL);
+               say("exec cmp", "%s", strerror(errno));
+               exit(0);
+       default:
+               break;
+       }
+       if (-1 == waitpid(child, &status, 0)) {
+               exitcode = (int)MANDOCLEVEL_SYSERR;
+               say("wait cmp", "%s", strerror(errno));
+       } else if (WIFSIGNALED(status)) {
+               exitcode = (int)MANDOCLEVEL_SYSERR;
+               say("cmp", "Died from a signal");
+       } else if (WEXITSTATUS(status)) {

                exitcode = (int)MANDOCLEVEL_SYSERR;
                exitcode = (int)MANDOCLEVEL_SYSERR;

-               say(MANDOC_DB, NULL);
+               say(MANDOC_DB,
+                   "Data changed, but cannot replace database");
+       }
+
+       *strrchr(tempfilename, '/') = '\0';
+       switch (child = fork()) {
+       case (-1):
+               exitcode = (int)MANDOCLEVEL_SYSERR;
+               say("fork rm", "%s", strerror(errno));
+               return;
+       case (0):
+               execlp("rm", "rm", "-rf", tempfilename, NULL);
+               say("exec rm", "%s", strerror(errno));
+               exit((int)MANDOCLEVEL_SYSERR);
+       default:
+               break;
+       }
+       if (-1 == waitpid(child, &status, 0)) {
+               exitcode = (int)MANDOCLEVEL_SYSERR;
+               say("wait rm", "%s", strerror(errno));
+       } else if (WIFSIGNALED(status) || WEXITSTATUS(status)) {
+               exitcode = (int)MANDOCLEVEL_SYSERR;
+               say(tempfilename,
+                   "Cannot remove temporary directory");

        }
 }
 
        }
 }
 


@@ -1866,42 +1960,62 @@ dbclose(int real)
 static int
 dbopen(int real)
 {
 static int
 dbopen(int real)
 {

-       const char      *file, *sql;
+       const char      *sql;

        int              rc, ofl;
 
        if (nodb) 
                return(1);
 
        int              rc, ofl;
 
        if (nodb) 
                return(1);
 

+       *tempfilename = '\0';

        ofl = SQLITE_OPEN_READWRITE;
        ofl = SQLITE_OPEN_READWRITE;

-       if (0 == real) {
-               file = MANDOC_DB "~";
-               if (-1 == remove(file) && ENOENT != errno) {
+
+       if (real) {
+               rc = sqlite3_open_v2(MANDOC_DB, &db, ofl, NULL);
+               if (SQLITE_OK != rc) {

                        exitcode = (int)MANDOCLEVEL_SYSERR;
                        exitcode = (int)MANDOCLEVEL_SYSERR;

-                       say(file, NULL);
+                       say(MANDOC_DB, "%s", sqlite3_errmsg(db));

                        return(0);
                }
                        return(0);
                }

-               ofl |= SQLITE_OPEN_EXCLUSIVE;
-       } else
-               file = MANDOC_DB;
+               goto prepare_statements;
+       }

 
 

-       rc = sqlite3_open_v2(file, &db, ofl, NULL);
+       ofl |= SQLITE_OPEN_CREATE | SQLITE_OPEN_EXCLUSIVE;
+
+       remove(MANDOC_DB "~");
+       rc = sqlite3_open_v2(MANDOC_DB "~", &db, ofl, NULL);

        if (SQLITE_OK == rc) 
        if (SQLITE_OK == rc) 

-               goto prepare_statements;
-       if (SQLITE_CANTOPEN != rc) {
+               goto create_tables;
+       if (MPARSE_QUICK & mparse_options) {

                exitcode = (int)MANDOCLEVEL_SYSERR;
                exitcode = (int)MANDOCLEVEL_SYSERR;

-               say(file, NULL);
+               say(MANDOC_DB "~", "%s", sqlite3_errmsg(db));

                return(0);
        }
 
                return(0);
        }
 

-       sqlite3_close(db);
-       db = NULL;
-
-       if (SQLITE_OK != (rc = sqlite3_open(file, &db))) {
+       if (strlcpy(tempfilename, "/tmp/mandocdb.XXXXXX",
+           sizeof(tempfilename)) >= sizeof(tempfilename)) {

                exitcode = (int)MANDOCLEVEL_SYSERR;
                exitcode = (int)MANDOCLEVEL_SYSERR;

-               say(file, NULL);
+               say("/tmp/mandocdb.XXXXXX", "Filename too long");
+               return(0);
+       }
+       if (NULL == mkdtemp(tempfilename)) {
+               exitcode = (int)MANDOCLEVEL_SYSERR;
+               say(tempfilename, "%s", strerror(errno));
+               return(0);
+       }
+       if (strlcat(tempfilename, "/" MANDOC_DB,
+           sizeof(tempfilename)) >= sizeof(tempfilename)) {
+               exitcode = (int)MANDOCLEVEL_SYSERR;
+               say(tempfilename, "Filename too long");
+               return(0);
+       }
+       rc = sqlite3_open_v2(tempfilename, &db, ofl, NULL);
+       if (SQLITE_OK != rc) {
+               exitcode = (int)MANDOCLEVEL_SYSERR;
+               say(tempfilename, "%s", sqlite3_errmsg(db));

                return(0);
        }
 
                return(0);
        }
 

+create_tables:

        sql = "CREATE TABLE \"mpages\" (\n"
              " \"form\" INTEGER NOT NULL,\n"
              " \"id\" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL\n"
        sql = "CREATE TABLE \"mpages\" (\n"
              " \"form\" INTEGER NOT NULL,\n"
              " \"id\" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL\n"


@@ -1912,23 +2026,19 @@ dbopen(int real)
              " \"arch\" TEXT NOT NULL,\n"
              " \"name\" TEXT NOT NULL,\n"
              " \"pageid\" INTEGER NOT NULL REFERENCES mpages(id) "
              " \"arch\" TEXT NOT NULL,\n"
              " \"name\" TEXT NOT NULL,\n"
              " \"pageid\" INTEGER NOT NULL REFERENCES mpages(id) "

-               "ON DELETE CASCADE,\n"
-             " \"id\" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL\n"
+               "ON DELETE CASCADE\n"

              ");\n"
              "\n"
              "CREATE TABLE \"keys\" (\n"
              " \"bits\" INTEGER NOT NULL,\n"
              " \"key\" TEXT NOT NULL,\n"
              " \"pageid\" INTEGER NOT NULL REFERENCES mpages(id) "
              ");\n"
              "\n"
              "CREATE TABLE \"keys\" (\n"
              " \"bits\" INTEGER NOT NULL,\n"
              " \"key\" TEXT NOT NULL,\n"
              " \"pageid\" INTEGER NOT NULL REFERENCES mpages(id) "

-               "ON DELETE CASCADE,\n"
-             " \"id\" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL\n"
-             ");\n"
-             "\n"
-             "CREATE INDEX \"key_index\" ON keys (key);\n";
+               "ON DELETE CASCADE\n"
+             ");\n";

 
        if (SQLITE_OK != sqlite3_exec(db, sql, NULL, NULL, NULL)) {
                exitcode = (int)MANDOCLEVEL_SYSERR;
 
        if (SQLITE_OK != sqlite3_exec(db, sql, NULL, NULL, NULL)) {
                exitcode = (int)MANDOCLEVEL_SYSERR;

-               say(file, "%s", sqlite3_errmsg(db));
+               say(MANDOC_DB, "%s", sqlite3_errmsg(db));

                return(0);
        }
 
                return(0);
        }