Add configurations (`Cd') to mandoc-db mining.

[mandoc.git] / mandoc-db.c

diff --git a/mandoc-db.c b/mandoc-db.c
index e41b7bd052fb1140b4ad29e91081623029c75795..85d5036e380c6c06bc0d671a14be643dd624ed92 100644 (file)
--- a/mandoc-db.c
+++ b/mandoc-db.c
@@ -1,4 +1,4 @@
-/*     $Id: mandoc-db.c,v 1.10 2011/04/11 21:36:44 kristaps Exp $ */
+/*     $Id: mandoc-db.c,v 1.18 2011/05/04 08:21:17 kristaps Exp $ */

 /*
  * Copyright (c) 2011 Kristaps Dzonsons <kristaps@bsd.lv>
  *
 /*
  * Copyright (c) 2011 Kristaps Dzonsons <kristaps@bsd.lv>
  *


@@ -48,18 +48,23 @@ enum        type {
        MANDOC_FUNCTION,
        MANDOC_UTILITY,
        MANDOC_INCLUDES,
        MANDOC_FUNCTION,
        MANDOC_UTILITY,
        MANDOC_INCLUDES,

-       MANDOC_VARIABLE
+       MANDOC_VARIABLE,
+       MANDOC_STANDARD,
+       MANDOC_AUTHOR,
+       MANDOC_CONFIG

 };
 
 #define        MAN_ARGS          DB *db, \
                          const char *dbn, \
                          DBT *key, size_t *ksz, \
                          DBT *val, \
 };
 
 #define        MAN_ARGS          DB *db, \
                          const char *dbn, \
                          DBT *key, size_t *ksz, \
                          DBT *val, \

+                         DBT *rval, size_t *rsz, \

                          const struct man_node *n
 #define        MDOC_ARGS         DB *db, \
                          const char *dbn, \
                          DBT *key, size_t *ksz, \
                          DBT *val, \
                          const struct man_node *n
 #define        MDOC_ARGS         DB *db, \
                          const char *dbn, \
                          DBT *key, size_t *ksz, \
                          DBT *val, \

+                         DBT *rval, size_t *rsz, \

                          const struct mdoc_node *n
 
 static void              dbt_append(DBT *, size_t *, const char *);
                          const struct mdoc_node *n
 
 static void              dbt_append(DBT *, size_t *, const char *);


@@ -68,17 +73,21 @@ static      void              dbt_appendb(DBT *, size_t *,
 static void              dbt_init(DBT *, size_t *);
 static void              dbt_put(DB *, const char *, DBT *, DBT *);
 static void              usage(void);
 static void              dbt_init(DBT *, size_t *);
 static void              dbt_put(DB *, const char *, DBT *, DBT *);
 static void              usage(void);

-static void              pman(DB *, const char *, DBT *, 
-                               size_t *, DBT *, struct man *);
+static void              pman(DB *, const char *, DBT *, size_t *, 
+                               DBT *, DBT *, size_t *, struct man *);

 static int               pman_node(MAN_ARGS);
 static int               pman_node(MAN_ARGS);

-static void              pmdoc(DB *, const char *, DBT *, 
-                               size_t *, DBT *, struct mdoc *);
+static void              pmdoc(DB *, const char *, DBT *, size_t *, 
+                               DBT *, DBT *, size_t *, struct mdoc *);

 static void              pmdoc_node(MDOC_ARGS);
 static void              pmdoc_node(MDOC_ARGS);

+static void              pmdoc_An(MDOC_ARGS);
+static void              pmdoc_Cd(MDOC_ARGS);

 static void              pmdoc_Fd(MDOC_ARGS);
 static void              pmdoc_In(MDOC_ARGS);
 static void              pmdoc_Fn(MDOC_ARGS);
 static void              pmdoc_Fo(MDOC_ARGS);
 static void              pmdoc_Fd(MDOC_ARGS);
 static void              pmdoc_In(MDOC_ARGS);
 static void              pmdoc_Fn(MDOC_ARGS);
 static void              pmdoc_Fo(MDOC_ARGS);

+static void              pmdoc_Nd(MDOC_ARGS);

 static void              pmdoc_Nm(MDOC_ARGS);
 static void              pmdoc_Nm(MDOC_ARGS);

+static void              pmdoc_St(MDOC_ARGS);

 static void              pmdoc_Vt(MDOC_ARGS);
 
 typedef        void            (*pmdoc_nf)(MDOC_ARGS);
 static void              pmdoc_Vt(MDOC_ARGS);
 
 typedef        void            (*pmdoc_nf)(MDOC_ARGS);


@@ -101,9 +110,9 @@ static      const pmdoc_nf    mdocs[MDOC_MAX] = {
        NULL, /* El */
        NULL, /* It */
        NULL, /* Ad */ 
        NULL, /* El */
        NULL, /* It */
        NULL, /* Ad */ 

-       NULL, /* An */ 
+       pmdoc_An, /* An */ 

        NULL, /* Ar */
        NULL, /* Ar */

-       NULL, /* Cd */ 
+       pmdoc_Cd, /* Cd */ 

        NULL, /* Cm */
        NULL, /* Dv */ 
        NULL, /* Er */ 
        NULL, /* Cm */
        NULL, /* Dv */ 
        NULL, /* Er */ 


@@ -117,13 +126,13 @@ static    const pmdoc_nf    mdocs[MDOC_MAX] = {
        NULL, /* Ic */ 
        pmdoc_In, /* In */ 
        NULL, /* Li */
        NULL, /* Ic */ 
        pmdoc_In, /* In */ 
        NULL, /* Li */

-       NULL, /* Nd */
+       pmdoc_Nd, /* Nd */

        pmdoc_Nm, /* Nm */
        NULL, /* Op */
        NULL, /* Ot */
        NULL, /* Pa */
        NULL, /* Rv */
        pmdoc_Nm, /* Nm */
        NULL, /* Op */
        NULL, /* Ot */
        NULL, /* Pa */
        NULL, /* Rv */

-       NULL, /* St */ 
+       pmdoc_St, /* St */ 

        pmdoc_Vt, /* Va */
        pmdoc_Vt, /* Vt */ 
        NULL, /* Xr */ 
        pmdoc_Vt, /* Va */
        pmdoc_Vt, /* Vt */ 
        NULL, /* Xr */ 


@@ -216,22 +225,24 @@ main(int argc, char *argv[])
        struct mparse   *mp; /* parse sequence */
        struct mdoc     *mdoc; /* resulting mdoc */
        struct man      *man; /* resulting man */
        struct mparse   *mp; /* parse sequence */
        struct mdoc     *mdoc; /* resulting mdoc */
        struct man      *man; /* resulting man */

-       char            *fn;
-       const char      *msec,
+       char            *fn; /* current file being parsed */
+       const char      *msec, /* manual section */
+                       *mtitle, /* manual title */

                        *dir; /* result dir (default: cwd) */
        char             ibuf[MAXPATHLEN], /* index fname */
                         ibbuf[MAXPATHLEN], /* index backup fname */
                         fbuf[MAXPATHLEN],  /* btree fname */
                         fbbuf[MAXPATHLEN]; /* btree backup fname */
                        *dir; /* result dir (default: cwd) */
        char             ibuf[MAXPATHLEN], /* index fname */
                         ibbuf[MAXPATHLEN], /* index backup fname */
                         fbuf[MAXPATHLEN],  /* btree fname */
                         fbbuf[MAXPATHLEN]; /* btree backup fname */

-       int              c;
+       int              ch;

        DB              *idx, /* index database */
                        *db; /* keyword database */
        DBT              rkey, rval, /* recno entries */
                         key, val; /* persistent keyword entries */
        DB              *idx, /* index database */
                        *db; /* keyword database */
        DBT              rkey, rval, /* recno entries */
                         key, val; /* persistent keyword entries */

-       size_t           ksz, rsz; /* entry buffer size */
-       char             vbuf[8];
+       size_t           sv,
+                        ksz, rsz; /* entry buffer size */
+       char             vbuf[8]; /* stringified record number */

        BTREEINFO        info; /* btree configuration */
        BTREEINFO        info; /* btree configuration */

-       recno_t          rec;
+       recno_t          rec; /* current record number */

        extern int       optind;
        extern char     *optarg;
 
        extern int       optind;
        extern char     *optarg;
 


@@ -243,8 +254,8 @@ main(int argc, char *argv[])
 
        dir = "";
 
 
        dir = "";
 

-       while (-1 != (c = getopt(argc, argv, "d:")))
-               switch (c) {
+       while (-1 != (ch = getopt(argc, argv, "d:")))
+               switch (ch) {

                case ('d'):
                        dir = optarg;
                        break;
                case ('d'):
                        dir = optarg;
                        break;


@@ -335,34 +346,66 @@ main(int argc, char *argv[])
        while (NULL != (fn = *argv++)) {
                mparse_reset(mp);
 
        while (NULL != (fn = *argv++)) {
                mparse_reset(mp);
 

+               /* Parse and get (non-empty) AST. */
+

                if (mparse_readfd(mp, -1, fn) >= MANDOCLEVEL_FATAL) {
                        fprintf(stderr, "%s: Parse failure\n", fn);
                        continue;
                }
                if (mparse_readfd(mp, -1, fn) >= MANDOCLEVEL_FATAL) {
                        fprintf(stderr, "%s: Parse failure\n", fn);
                        continue;
                }

-

                mparse_result(mp, &mdoc, &man);
                if (NULL == mdoc && NULL == man)
                        continue;
 
                mparse_result(mp, &mdoc, &man);
                if (NULL == mdoc && NULL == man)
                        continue;
 

+               /* Manual section: can be empty string. */
+

                msec = NULL != mdoc ? 
                        mdoc_meta(mdoc)->msec :
                        man_meta(man)->msec;
                msec = NULL != mdoc ? 
                        mdoc_meta(mdoc)->msec :
                        man_meta(man)->msec;

+               mtitle = NULL != mdoc ? 
+                       mdoc_meta(mdoc)->title :
+                       man_meta(man)->title;

 
 

-               rkey.data = &rec;
+               assert(msec);
+               assert(mtitle);
+
+               /* 
+                * The index record value consists of a nil-terminated
+                * filename, a nil-terminated manual section, and a
+                * nil-terminated description.  Since the description
+                * may not be set, we set a sentinel to see if we're
+                * going to write a nil byte in its place.
+                */

 
                dbt_init(&rval, &rsz);
                dbt_appendb(&rval, &rsz, fn, strlen(fn) + 1);
                dbt_appendb(&rval, &rsz, msec, strlen(msec) + 1);
 
                dbt_init(&rval, &rsz);
                dbt_appendb(&rval, &rsz, fn, strlen(fn) + 1);
                dbt_appendb(&rval, &rsz, msec, strlen(msec) + 1);

+               dbt_appendb(&rval, &rsz, mtitle, strlen(mtitle) + 1);
+               sv = rval.size;

 
 

-               dbt_put(idx, ibbuf, &rkey, &rval);
+               /* Fix the record number in the btree value. */

 
                memset(val.data, 0, sizeof(uint32_t));
                memcpy(val.data + 4, &rec, sizeof(uint32_t));
 
                if (mdoc)
 
                memset(val.data, 0, sizeof(uint32_t));
                memcpy(val.data + 4, &rec, sizeof(uint32_t));
 
                if (mdoc)

-                       pmdoc(db, fbbuf, &key, &ksz, &val, mdoc);
+                       pmdoc(db, fbbuf, &key, &ksz, 
+                               &val, &rval, &rsz, mdoc);

                else 
                else 

-                       pman(db, fbbuf, &key, &ksz, &val, man);
+                       pman(db, fbbuf, &key, &ksz, 
+                               &val, &rval, &rsz, man);
+               
+               /*
+                * Apply this to the index.  If we haven't had a
+                * description set, put an empty one in now.
+                */
+
+               if (rval.size == sv)
+                       dbt_appendb(&rval, &rsz, "", 1);
+
+               rkey.data = &rec;
+               dbt_put(idx, ibbuf, &rkey, &rval);
+
+               printf("Indexed: %s\n", fn);

                rec++;
        }
 
                rec++;
        }
 


@@ -420,6 +463,48 @@ dbt_appendb(DBT *key, size_t *ksz, const void *cp, size_t sz)
                key->data = mandoc_realloc(key->data, *ksz);
        }
 
                key->data = mandoc_realloc(key->data, *ksz);
        }
 

+#if 0
+       dstp = key->data + (int)key->size;
+
+       while (NULL != (endp = memchr(cp, '\\', sz))) {
+               ssz = endp - cp;
+               memcpy(dstp, cp, ssz);
+
+               dstp += ssz;
+               key->size += ssz;
+               sz -= ssz;
+
+               cp = endp++;
+               /* FIXME: expects nil-terminated string! */
+               esc = mandoc_escape((const char **)&endp, NULL, NULL);
+
+               switch (esc) {
+               case (ESCAPE_ERROR):
+                       /* Nil-terminate this point. */
+                       memcpy(dstp, "", 1);
+                       key->size++;
+                       return;
+               case (ESCAPE_PREDEF):
+                       /* FALLTHROUGH */
+               case (ESCAPE_SPECIAL):
+                       break;
+               default:
+                       sz -= endp - cp;
+                       cp = endp;
+                       continue;
+               }
+
+               ssz = endp - cp;
+               memcpy(dstp, cp, ssz);
+
+               dstp += ssz;
+               key->size += ssz;
+               sz -= ssz;
+
+               cp = endp;
+       }
+#endif
+

        memcpy(key->data + (int)key->size, cp, sz);
        key->size += sz;
 }
        memcpy(key->data + (int)key->size, cp, sz);
        key->size += sz;
 }


@@ -445,6 +530,23 @@ dbt_append(DBT *key, size_t *ksz, const char *cp)
        dbt_appendb(key, ksz, cp, sz + 1);
 }
 
        dbt_appendb(key, ksz, cp, sz + 1);
 }
 

+/* ARGSUSED */
+static void
+pmdoc_An(MDOC_ARGS)
+{
+       uint32_t         fl;
+       
+       if (SEC_AUTHORS != n->sec)
+               return;
+
+       for (n = n->child; n; n = n->next)
+               if (MDOC_TEXT == n->type)
+                       dbt_append(key, ksz, n->string);
+
+       fl = MANDOC_AUTHOR;
+       memcpy(val->data, &fl, 4);
+}
+

 /* ARGSUSED */
 static void
 pmdoc_Fd(MDOC_ARGS)
 /* ARGSUSED */
 static void
 pmdoc_Fd(MDOC_ARGS)


@@ -452,7 +554,6 @@ pmdoc_Fd(MDOC_ARGS)
        uint32_t         fl;
        const char      *start, *end;
        size_t           sz;
        uint32_t         fl;
        const char      *start, *end;
        size_t           sz;

-       const char       nil = '\0';

        
        if (SEC_SYNOPSIS != n->sec)
                return;
        
        if (SEC_SYNOPSIS != n->sec)
                return;


@@ -486,12 +587,29 @@ pmdoc_Fd(MDOC_ARGS)
                end--;
 
        dbt_appendb(key, ksz, start, end - start + 1);
                end--;
 
        dbt_appendb(key, ksz, start, end - start + 1);

-       dbt_appendb(key, ksz, &nil, 1);
+       dbt_appendb(key, ksz, "", 1);

 
        fl = MANDOC_INCLUDES;
        memcpy(val->data, &fl, 4);
 }
 
 
        fl = MANDOC_INCLUDES;
        memcpy(val->data, &fl, 4);
 }
 

+/* ARGSUSED */
+static void
+pmdoc_Cd(MDOC_ARGS)
+{
+       uint32_t         fl;
+       
+       if (SEC_SYNOPSIS != n->sec)
+               return;
+
+       for (n = n->child; n; n = n->next)
+               if (MDOC_TEXT == n->type)
+                       dbt_append(key, ksz, n->string);
+
+       fl = MANDOC_CONFIG;
+       memcpy(val->data, &fl, 4);
+}
+

 /* ARGSUSED */
 static void
 pmdoc_In(MDOC_ARGS)
 /* ARGSUSED */
 static void
 pmdoc_In(MDOC_ARGS)


@@ -536,14 +654,29 @@ pmdoc_Fn(MDOC_ARGS)
        memcpy(val->data, &fl, 4);
 }
 
        memcpy(val->data, &fl, 4);
 }
 

+/* ARGSUSED */
+static void
+pmdoc_St(MDOC_ARGS)
+{
+       uint32_t         fl;
+       
+       if (SEC_STANDARDS != n->sec)
+               return;
+       if (NULL == n->child || MDOC_TEXT != n->child->type)
+               return;
+
+       dbt_append(key, ksz, n->child->string);
+       fl = MANDOC_STANDARD;
+       memcpy(val->data, &fl, 4);
+}
+

 /* ARGSUSED */
 static void
 pmdoc_Vt(MDOC_ARGS)
 {
        uint32_t         fl;
 /* ARGSUSED */
 static void
 pmdoc_Vt(MDOC_ARGS)
 {
        uint32_t         fl;

-       const char      *start, *end;
+       const char      *start;

        size_t           sz;
        size_t           sz;

-       const char       nil = '\0';

        
        if (SEC_SYNOPSIS != n->sec)
                return;
        
        if (SEC_SYNOPSIS != n->sec)
                return;


@@ -564,15 +697,15 @@ pmdoc_Vt(MDOC_ARGS)
        if (0 == (sz = strlen(start)))
                return;
 
        if (0 == (sz = strlen(start)))
                return;
 

-       end = &start[sz - 1];
-       while (end > start && ';' == *end)
-               end--;
+       if (';' == start[sz - 1])
+               sz--;

 
 

-       if (end == start)
+       if (0 == sz)

                return;
 
                return;
 

-       dbt_appendb(key, ksz, start, end - start + 1);
-       dbt_appendb(key, ksz, &nil, 1);
+       dbt_appendb(key, ksz, start, sz);
+       dbt_appendb(key, ksz, "", 1);
+

        fl = MANDOC_VARIABLE;
        memcpy(val->data, &fl, 4);
 }
        fl = MANDOC_VARIABLE;
        memcpy(val->data, &fl, 4);
 }


@@ -593,6 +726,24 @@ pmdoc_Fo(MDOC_ARGS)
        memcpy(val->data, &fl, 4);
 }
 
        memcpy(val->data, &fl, 4);
 }
 

+
+/* ARGSUSED */
+static void
+pmdoc_Nd(MDOC_ARGS)
+{
+       int              first;
+       
+       for (first = 1, n = n->child; n; n = n->next) {
+               if (MDOC_TEXT != n->type)
+                       continue;
+               if (first) 
+                       dbt_appendb(rval, rsz, n->string, strlen(n->string) + 1);
+               else
+                       dbt_append(rval, rsz, n->string);
+               first = 0;
+       }
+}
+

 /* ARGSUSED */
 static void
 pmdoc_Nm(MDOC_ARGS)
 /* ARGSUSED */
 static void
 pmdoc_Nm(MDOC_ARGS)


@@ -665,24 +816,23 @@ pmdoc_node(MDOC_ARGS)
                        break;
 
                dbt_init(key, ksz);
                        break;
 
                dbt_init(key, ksz);

-               (*mdocs[n->tok])(db, dbn, key, ksz, val, n);

 
 

+               (*mdocs[n->tok])(db, dbn, key, ksz, val, rval, rsz, n);

                dbt_put(db, dbn, key, val);
                break;
        default:
                break;
        }
 
                dbt_put(db, dbn, key, val);
                break;
        default:
                break;
        }
 

-       pmdoc_node(db, dbn, key, ksz, val, n->child);
-       pmdoc_node(db, dbn, key, ksz, val, n->next);
+       pmdoc_node(db, dbn, key, ksz, val, rval, rsz, n->child);
+       pmdoc_node(db, dbn, key, ksz, val, rval, rsz, n->next);

 }
 
 static int
 pman_node(MAN_ARGS)
 {
        const struct man_node *head, *body;
 }
 
 static int
 pman_node(MAN_ARGS)
 {
        const struct man_node *head, *body;

-       const char      *start;
-       const char       nil = '\0';
+       const char      *start, *sv;

        size_t           sz;
        uint32_t         fl;
 
        size_t           sz;
        uint32_t         fl;
 


@@ -710,7 +860,8 @@ pman_node(MAN_ARGS)
                        fl = MANDOC_NAME;
                        memcpy(val->data, &fl, 4);
 
                        fl = MANDOC_NAME;
                        memcpy(val->data, &fl, 4);
 

-                       start = body->string;
+                       assert(body->string);
+                       start = sv = body->string;

 
                        /* 
                         * Go through a special heuristic dance here.
 
                        /* 
                         * Go through a special heuristic dance here.


@@ -721,19 +872,21 @@ pman_node(MAN_ARGS)
                         * the name parts here.
                         */
 
                         * the name parts here.
                         */
 

-                       while (start) {
+                       for ( ;; ) {

                                sz = strcspn(start, " ,");
                                if ('\0' == start[(int)sz])
                                        break;
 
                                dbt_init(key, ksz);
                                dbt_appendb(key, ksz, start, sz);
                                sz = strcspn(start, " ,");
                                if ('\0' == start[(int)sz])
                                        break;
 
                                dbt_init(key, ksz);
                                dbt_appendb(key, ksz, start, sz);

-                               dbt_appendb(key, ksz, &nil, 1);
+                               dbt_appendb(key, ksz, "", 1);

 
                                dbt_put(db, dbn, key, val);
 
 
                                dbt_put(db, dbn, key, val);
 

-                               if (' ' == start[(int)sz])
+                               if (' ' == start[(int)sz]) {
+                                       start += (int)sz + 1;

                                        break;
                                        break;

+                               }

 
                                assert(',' == start[(int)sz]);
                                start += (int)sz + 1;
 
                                assert(',' == start[(int)sz]);
                                start += (int)sz + 1;


@@ -741,33 +894,50 @@ pman_node(MAN_ARGS)
                                        start++;
                        }
 
                                        start++;
                        }
 

-                       return(1);
+                       if (sv == start) {
+                               dbt_init(key, ksz);
+                               dbt_append(key, ksz, start);
+                               return(1);
+                       }
+
+                       while (' ' == *start)
+                               start++;
+
+                       if ('\\' == *start && '-' == *(start + 1))
+                               start += 2;
+                       else if ('-' == *start)
+                               start++;
+
+                       while (' ' == *start)
+                               start++;
+
+                       dbt_appendb(rval, rsz, start, strlen(start) + 1);

                }
        }
 
                }
        }
 

-       if (pman_node(db, dbn, key, ksz, val, n->child))
+       if (pman_node(db, dbn, key, ksz, val, rval, rsz, n->child))

                return(1);
                return(1);

-       if (pman_node(db, dbn, key, ksz, val, n->next))
+       if (pman_node(db, dbn, key, ksz, val, rval, rsz, n->next))

                return(1);
 
        return(0);
 }
 
 static void
                return(1);
 
        return(0);
 }
 
 static void

-pman(DB *db, const char *dbn, DBT *key, 
-               size_t *ksz, DBT *val, struct man *m)
+pman(DB *db, const char *dbn, DBT *key, size_t *ksz, 
+               DBT *val, DBT *rval, size_t *rsz, struct man *m)

 {
 
 {
 

-       pman_node(db, dbn, key, ksz, val, man_node(m));
+       pman_node(db, dbn, key, ksz, val, rval, rsz, man_node(m));

 }
 
 
 static void
 }
 
 
 static void

-pmdoc(DB *db, const char *dbn, DBT *key, 
-               size_t *ksz, DBT *val, struct mdoc *m)
+pmdoc(DB *db, const char *dbn, DBT *key, size_t *ksz, 
+               DBT *val, DBT *rval, size_t *rsz, struct mdoc *m)

 {
 
 {
 

-       pmdoc_node(db, dbn, key, ksz, val, mdoc_node(m));
+       pmdoc_node(db, dbn, key, ksz, val, rval, rsz, mdoc_node(m));

 }
 
 static void
 }
 
 static void