mandocdb.c

   1 /*      $Id: mandocdb.c,v 1.22 2011/12/03 12:09:07 kristaps Exp $ */
   2 /*
   3  * Copyright (c) 2011 Kristaps Dzonsons <kristaps@bsd.lv>
   4  * Copyright (c) 2011 Ingo Schwarze <schwarze@openbsd.org>
   5  *
   6  * Permission to use, copy, modify, and distribute this software for any
   7  * purpose with or without fee is hereby granted, provided that the above
   8  * copyright notice and this permission notice appear in all copies.
   9  *
  10  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
  11  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
  12  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
  13  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
  14  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
  15  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
  16  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  17  */
  18 #ifdef HAVE_CONFIG_H
  19 #include "config.h"
  20 #endif
  21
  22 #include <sys/param.h>
  23 #include <sys/types.h>
  24 #include <sys/stat.h>
  25
  26 #include <assert.h>
  27 #include <dirent.h>
  28 #include <fcntl.h>
  29 #include <getopt.h>
  30 #include <stdio.h>
  31 #include <stdint.h>
  32 #include <stdlib.h>
  33 #include <string.h>
  34 #include <unistd.h>
  35
  36 #if defined(__linux__)
  37 # include <endian.h>
  38 # include <db_185.h>
  39 #elif defined(__APPLE__)
  40 # include <libkern/OSByteOrder.h>
  41 # include <db.h>
  42 #else
  43 # include <db.h>
  44 #endif
  45
  46 #include "man.h"
  47 #include "mdoc.h"
  48 #include "mandoc.h"
  49 #include "mandocdb.h"
  50 #include "manpath.h"
  51
  52 #define MANDOC_BUFSZ      BUFSIZ
  53 #define MANDOC_SLOP       1024
  54
  55 #define MANDOC_SRC        0x1
  56 #define MANDOC_FORM       0x2
  57
  58 /* Tiny list for files.  No need to bring in QUEUE. */
  59
  60 struct  of {
  61         char             *fname; /* heap-allocated */
  62         char             *sec;
  63         char             *arch;
  64         char             *title;
  65         int               src_form;
  66         struct of        *next; /* NULL for last one */
  67         struct of        *first; /* first in list */
  68 };
  69
  70 /* Buffer for storing growable data. */
  71
  72 struct  buf {
  73         char             *cp;
  74         size_t            len; /* current length */
  75         size_t            size; /* total buffer size */
  76 };
  77
  78 /* Operation we're going to perform. */
  79
  80 enum    op {
  81         OP_NEW = 0, /* new database */
  82         OP_UPDATE, /* delete/add entries in existing database */
  83         OP_DELETE /* delete entries from existing database */
  84 };
  85
  86 #define MAN_ARGS          DB *hash, \
  87                           struct buf *buf, \
  88                           struct buf *dbuf, \
  89                           const struct man_node *n
  90 #define MDOC_ARGS         DB *hash, \
  91                           struct buf *buf, \
  92                           struct buf *dbuf, \
  93                           const struct mdoc_node *n, \
  94                           const struct mdoc_meta *m
  95
  96 static  void              buf_appendmdoc(struct buf *,
  97                                 const struct mdoc_node *, int);
  98 static  void              buf_append(struct buf *, const char *);
  99 static  void              buf_appendb(struct buf *,
 100                                 const void *, size_t);
 101 static  void              dbt_put(DB *, const char *, DBT *, DBT *);
 102 static  void              hash_put(DB *, const struct buf *, uint64_t);
 103 static  void              hash_reset(DB **);
 104 static  void              index_merge(const struct of *, struct mparse *,
 105                                 struct buf *, struct buf *, DB *,
 106                                 DB *, const char *, DB *, const char *,
 107                                 recno_t, const recno_t *, size_t);
 108 static  void              index_prune(const struct of *, DB *,
 109                                 const char *, DB *, const char *,
 110                                 recno_t *, recno_t **, size_t *);
 111 static  void              ofile_argbuild(int, char *[], struct of **);
 112 static  int               ofile_dirbuild(const char *, const char *,
 113                                 const char *, int, struct of **);
 114 static  void              ofile_free(struct of *);
 115 static  void              pformatted(DB *, struct buf *, struct buf *,
 116                                 const struct of *);
 117 static  int               pman_node(MAN_ARGS);
 118 static  void              pmdoc_node(MDOC_ARGS);
 119 static  void              pmdoc_An(MDOC_ARGS);
 120 static  void              pmdoc_Cd(MDOC_ARGS);
 121 static  void              pmdoc_Er(MDOC_ARGS);
 122 static  void              pmdoc_Ev(MDOC_ARGS);
 123 static  void              pmdoc_Fd(MDOC_ARGS);
 124 static  void              pmdoc_In(MDOC_ARGS);
 125 static  void              pmdoc_Fn(MDOC_ARGS);
 126 static  void              pmdoc_Fo(MDOC_ARGS);
 127 static  void              pmdoc_Nd(MDOC_ARGS);
 128 static  void              pmdoc_Nm(MDOC_ARGS);
 129 static  void              pmdoc_Pa(MDOC_ARGS);
 130 static  void              pmdoc_St(MDOC_ARGS);
 131 static  void              pmdoc_Vt(MDOC_ARGS);
 132 static  void              pmdoc_Xr(MDOC_ARGS);
 133 static  void              usage(void);
 134
 135 typedef void            (*pmdoc_nf)(MDOC_ARGS);
 136
 137 static  const pmdoc_nf    mdocs[MDOC_MAX] = {
 138         NULL, /* Ap */
 139         NULL, /* Dd */
 140         NULL, /* Dt */
 141         NULL, /* Os */
 142         NULL, /* Sh */
 143         NULL, /* Ss */
 144         NULL, /* Pp */
 145         NULL, /* D1 */
 146         NULL, /* Dl */
 147         NULL, /* Bd */
 148         NULL, /* Ed */
 149         NULL, /* Bl */
 150         NULL, /* El */
 151         NULL, /* It */
 152         NULL, /* Ad */
 153         pmdoc_An, /* An */
 154         NULL, /* Ar */
 155         pmdoc_Cd, /* Cd */
 156         NULL, /* Cm */
 157         NULL, /* Dv */
 158         pmdoc_Er, /* Er */
 159         pmdoc_Ev, /* Ev */
 160         NULL, /* Ex */
 161         NULL, /* Fa */
 162         pmdoc_Fd, /* Fd */
 163         NULL, /* Fl */
 164         pmdoc_Fn, /* Fn */
 165         NULL, /* Ft */
 166         NULL, /* Ic */
 167         pmdoc_In, /* In */
 168         NULL, /* Li */
 169         pmdoc_Nd, /* Nd */
 170         pmdoc_Nm, /* Nm */
 171         NULL, /* Op */
 172         NULL, /* Ot */
 173         pmdoc_Pa, /* Pa */
 174         NULL, /* Rv */
 175         pmdoc_St, /* St */
 176         pmdoc_Vt, /* Va */
 177         pmdoc_Vt, /* Vt */
 178         pmdoc_Xr, /* Xr */
 179         NULL, /* %A */
 180         NULL, /* %B */
 181         NULL, /* %D */
 182         NULL, /* %I */
 183         NULL, /* %J */
 184         NULL, /* %N */
 185         NULL, /* %O */
 186         NULL, /* %P */
 187         NULL, /* %R */
 188         NULL, /* %T */
 189         NULL, /* %V */
 190         NULL, /* Ac */
 191         NULL, /* Ao */
 192         NULL, /* Aq */
 193         NULL, /* At */
 194         NULL, /* Bc */
 195         NULL, /* Bf */
 196         NULL, /* Bo */
 197         NULL, /* Bq */
 198         NULL, /* Bsx */
 199         NULL, /* Bx */
 200         NULL, /* Db */
 201         NULL, /* Dc */
 202         NULL, /* Do */
 203         NULL, /* Dq */
 204         NULL, /* Ec */
 205         NULL, /* Ef */
 206         NULL, /* Em */
 207         NULL, /* Eo */
 208         NULL, /* Fx */
 209         NULL, /* Ms */
 210         NULL, /* No */
 211         NULL, /* Ns */
 212         NULL, /* Nx */
 213         NULL, /* Ox */
 214         NULL, /* Pc */
 215         NULL, /* Pf */
 216         NULL, /* Po */
 217         NULL, /* Pq */
 218         NULL, /* Qc */
 219         NULL, /* Ql */
 220         NULL, /* Qo */
 221         NULL, /* Qq */
 222         NULL, /* Re */
 223         NULL, /* Rs */
 224         NULL, /* Sc */
 225         NULL, /* So */
 226         NULL, /* Sq */
 227         NULL, /* Sm */
 228         NULL, /* Sx */
 229         NULL, /* Sy */
 230         NULL, /* Tn */
 231         NULL, /* Ux */
 232         NULL, /* Xc */
 233         NULL, /* Xo */
 234         pmdoc_Fo, /* Fo */
 235         NULL, /* Fc */
 236         NULL, /* Oo */
 237         NULL, /* Oc */
 238         NULL, /* Bk */
 239         NULL, /* Ek */
 240         NULL, /* Bt */
 241         NULL, /* Hf */
 242         NULL, /* Fr */
 243         NULL, /* Ud */
 244         NULL, /* Lb */
 245         NULL, /* Lp */
 246         NULL, /* Lk */
 247         NULL, /* Mt */
 248         NULL, /* Brq */
 249         NULL, /* Bro */
 250         NULL, /* Brc */
 251         NULL, /* %C */
 252         NULL, /* Es */
 253         NULL, /* En */
 254         NULL, /* Dx */
 255         NULL, /* %Q */
 256         NULL, /* br */
 257         NULL, /* sp */
 258         NULL, /* %U */
 259         NULL, /* Ta */
 260 };
 261
 262 static  const char       *progname;
 263 static  int               use_all;  /* Use all directories and files. */
 264 static  int               verb;  /* Output verbosity level. */
 265
 266 int
 267 main(int argc, char *argv[])
 268 {
 269         struct mparse   *mp; /* parse sequence */
 270         struct manpaths  dirs;
 271         enum op          op; /* current operation */
 272         const char      *dir;
 273         char             ibuf[MAXPATHLEN], /* index fname */
 274                          fbuf[MAXPATHLEN];  /* btree fname */
 275         int              ch, i, flags;
 276         DB              *idx, /* index database */
 277                         *db, /* keyword database */
 278                         *hash; /* temporary keyword hashtable */
 279         BTREEINFO        info; /* btree configuration */
 280         recno_t          maxrec; /* last record number in the index */
 281         recno_t         *recs; /* the numbers of all empty records */
 282         size_t           sz1, sz2,
 283                          recsz, /* number of allocated slots in recs */
 284                          reccur; /* current number of empty records */
 285         struct buf       buf, /* keyword buffer */
 286                          dbuf; /* description buffer */
 287         struct of       *of; /* list of files for processing */
 288         extern int       optind;
 289         extern char     *optarg;
 290
 291         progname = strrchr(argv[0], '/');
 292         if (progname == NULL)
 293                 progname = argv[0];
 294         else
 295                 ++progname;
 296
 297         memset(&dirs, 0, sizeof(struct manpaths));
 298
 299         verb = 0;
 300         use_all = 0;
 301         of = NULL;
 302         db = idx = NULL;
 303         mp = NULL;
 304         hash = NULL;
 305         recs = NULL;
 306         recsz = reccur = 0;
 307         maxrec = 0;
 308         op = OP_NEW;
 309         dir = NULL;
 310
 311         while (-1 != (ch = getopt(argc, argv, "ad:u:v")))
 312                 switch (ch) {
 313                 case ('a'):
 314                         use_all = 1;
 315                         break;
 316                 case ('d'):
 317                         dir = optarg;
 318                         op = OP_UPDATE;
 319                         break;
 320                 case ('u'):
 321                         dir = optarg;
 322                         op = OP_DELETE;
 323                         break;
 324                 case ('v'):
 325                         verb++;
 326                         break;
 327                 default:
 328                         usage();
 329                         return((int)MANDOCLEVEL_BADARG);
 330                 }
 331
 332         argc -= optind;
 333         argv += optind;
 334
 335         memset(&info, 0, sizeof(BTREEINFO));
 336         info.lorder = 4321;
 337         info.flags = R_DUP;
 338
 339         mp = mparse_alloc(MPARSE_AUTO, MANDOCLEVEL_FATAL, NULL, NULL);
 340
 341         memset(&buf, 0, sizeof(struct buf));
 342         memset(&dbuf, 0, sizeof(struct buf));
 343
 344         buf.size = dbuf.size = MANDOC_BUFSZ;
 345
 346         buf.cp = mandoc_malloc(buf.size);
 347         dbuf.cp = mandoc_malloc(dbuf.size);
 348
 349         flags = OP_NEW == op ? O_CREAT|O_TRUNC|O_RDWR : O_CREAT|O_RDWR;
 350
 351         if (OP_UPDATE == op || OP_DELETE == op) {
 352                 ibuf[0] = fbuf[0] = '\0';
 353
 354                 strlcat(fbuf, dir, MAXPATHLEN);
 355                 strlcat(fbuf, "/", MAXPATHLEN);
 356                 sz1 = strlcat(fbuf, MANDOC_DB, MAXPATHLEN);
 357
 358                 strlcat(ibuf, dir, MAXPATHLEN);
 359                 strlcat(ibuf, "/", MAXPATHLEN);
 360                 sz2 = strlcat(ibuf, MANDOC_IDX, MAXPATHLEN);
 361
 362                 if (sz1 >= MAXPATHLEN || sz2 >= MAXPATHLEN) {
 363                         fprintf(stderr, "%s: Path too long\n", dir);
 364                         exit((int)MANDOCLEVEL_BADARG);
 365                 }
 366
 367                 db = dbopen(fbuf, flags, 0644, DB_BTREE, &info);
 368                 idx = dbopen(ibuf, flags, 0644, DB_RECNO, NULL);
 369
 370                 if (NULL == db) {
 371                         perror(fbuf);
 372                         exit((int)MANDOCLEVEL_SYSERR);
 373                 } else if (NULL == idx) {
 374                         perror(ibuf);
 375                         exit((int)MANDOCLEVEL_SYSERR);
 376                 }
 377
 378                 if (verb > 2) {
 379                         printf("%s: Opened\n", fbuf);
 380                         printf("%s: Opened\n", ibuf);
 381                 }
 382
 383                 ofile_argbuild(argc, argv, &of);
 384                 if (NULL == of)
 385                         goto out;
 386
 387                 of = of->first;
 388
 389                 index_prune(of, db, fbuf, idx, ibuf,
 390                                 &maxrec, &recs, &recsz);
 391
 392                 /*
 393                  * Go to the root of the respective manual tree
 394                  * such that .so links work.  In case of failure,
 395                  * just prod on, even though .so links won't work.
 396                  */
 397
 398                 if (OP_UPDATE == op) {
 399                         chdir(dir);
 400                         index_merge(of, mp, &dbuf, &buf, hash,
 401                                         db, fbuf, idx, ibuf,
 402                                         maxrec, recs, reccur);
 403                 }
 404
 405                 goto out;
 406         }
 407
 408         /*
 409          * Configure the directories we're going to scan.
 410          * If we have command-line arguments, use them.
 411          * If not, we use man(1)'s method (see mandocdb.8).
 412          */
 413
 414         if (argc > 0) {
 415                 dirs.paths = mandoc_malloc(argc * sizeof(char *));
 416                 dirs.sz = argc;
 417                 for (i = 0; i < argc; i++)
 418                         dirs.paths[i] = mandoc_strdup(argv[i]);
 419         } else
 420                 manpath_parse(&dirs, NULL, NULL);
 421
 422         for (i = 0; i < dirs.sz; i++) {
 423                 ibuf[0] = fbuf[0] = '\0';
 424
 425                 strlcat(fbuf, dirs.paths[i], MAXPATHLEN);
 426                 strlcat(fbuf, "/", MAXPATHLEN);
 427                 sz1 = strlcat(fbuf, MANDOC_DB, MAXPATHLEN);
 428
 429                 strlcat(ibuf, dirs.paths[i], MAXPATHLEN);
 430                 strlcat(ibuf, "/", MAXPATHLEN);
 431                 sz2 = strlcat(ibuf, MANDOC_IDX, MAXPATHLEN);
 432
 433                 if (sz1 >= MAXPATHLEN || sz2 >= MAXPATHLEN) {
 434                         fprintf(stderr, "%s: Path too long\n",
 435                                         dirs.paths[i]);
 436                         exit((int)MANDOCLEVEL_BADARG);
 437                 }
 438
 439                 if (db)
 440                         (*db->close)(db);
 441                 if (idx)
 442                         (*idx->close)(idx);
 443
 444                 db = dbopen(fbuf, flags, 0644, DB_BTREE, &info);
 445                 idx = dbopen(ibuf, flags, 0644, DB_RECNO, NULL);
 446
 447                 if (NULL == db) {
 448                         perror(fbuf);
 449                         exit((int)MANDOCLEVEL_SYSERR);
 450                 } else if (NULL == idx) {
 451                         perror(ibuf);
 452                         exit((int)MANDOCLEVEL_SYSERR);
 453                 }
 454
 455                 if (verb > 2) {
 456                         printf("%s: Truncated\n", fbuf);
 457                         printf("%s: Truncated\n", ibuf);
 458                 }
 459
 460                 ofile_free(of);
 461                 of = NULL;
 462
 463                 if ( ! ofile_dirbuild(dirs.paths[i], NULL, NULL,
 464                                         0, &of))
 465                         exit((int)MANDOCLEVEL_SYSERR);
 466
 467                 if (NULL == of)
 468                         continue;
 469
 470                 of = of->first;
 471
 472                 /*
 473                  * Go to the root of the respective manual tree
 474                  * such that .so links work.  In case of failure,
 475                  * just prod on, even though .so links won't work.
 476                  */
 477
 478                 chdir(dirs.paths[i]);
 479                 index_merge(of, mp, &dbuf, &buf, hash, db, fbuf,
 480                                 idx, ibuf, maxrec, recs, reccur);
 481         }
 482
 483 out:
 484         if (db)
 485                 (*db->close)(db);
 486         if (idx)
 487                 (*idx->close)(idx);
 488         if (hash)
 489                 (*hash->close)(hash);
 490         if (mp)
 491                 mparse_free(mp);
 492
 493         manpath_free(&dirs);
 494         ofile_free(of);
 495         free(buf.cp);
 496         free(dbuf.cp);
 497         free(recs);
 498
 499         return(MANDOCLEVEL_OK);
 500 }
 501
 502 void
 503 index_merge(const struct of *of, struct mparse *mp,
 504                 struct buf *dbuf, struct buf *buf, DB *hash,
 505                 DB *db, const char *dbf, DB *idx, const char *idxf,
 506                 recno_t maxrec, const recno_t *recs, size_t reccur)
 507 {
 508         recno_t          rec;
 509         int              ch;
 510         DBT              key, val;
 511         struct mdoc     *mdoc;
 512         struct man      *man;
 513         const char      *fn, *msec, *mtitle, *arch;
 514         size_t           sv;
 515         unsigned         seq;
 516         struct db_val    vbuf;
 517
 518         for (rec = 0; of; of = of->next) {
 519                 fn = of->fname;
 520
 521                 /*
 522                  * Reclaim an empty index record, if available.
 523                  */
 524
 525                 if (reccur > 0) {
 526                         --reccur;
 527                         rec = recs[(int)reccur];
 528                 } else if (maxrec > 0) {
 529                         rec = maxrec;
 530                         maxrec = 0;
 531                 } else
 532                         rec++;
 533
 534                 mparse_reset(mp);
 535                 hash_reset(&hash);
 536                 mdoc = NULL;
 537                 man = NULL;
 538
 539                 /*
 540                  * Try interpreting the file as mdoc(7) or man(7)
 541                  * source code, unless it is already known to be
 542                  * formatted.  Fall back to formatted mode.
 543                  */
 544
 545                 if ((MANDOC_SRC & of->src_form ||
 546                     ! (MANDOC_FORM & of->src_form)) &&
 547                     MANDOCLEVEL_FATAL > mparse_readfd(mp, -1, fn))
 548                         mparse_result(mp, &mdoc, &man);
 549
 550                 if (NULL != mdoc) {
 551                         msec = mdoc_meta(mdoc)->msec;
 552                         arch = mdoc_meta(mdoc)->arch;
 553                         mtitle = mdoc_meta(mdoc)->title;
 554                 } else if (NULL != man) {
 555                         msec = man_meta(man)->msec;
 556                         arch = NULL;
 557                         mtitle = man_meta(man)->title;
 558                 } else {
 559                         msec = of->sec;
 560                         arch = of->arch;
 561                         mtitle = of->title;
 562                 }
 563
 564                 /*
 565                  * By default, skip a file if the manual section
 566                  * and architecture given in the file disagree
 567                  * with the directory where the file is located.
 568                  */
 569
 570                 if (0 == use_all) {
 571                         assert(of->sec);
 572                         assert(msec);
 573                         if (strcmp(msec, of->sec))
 574                                 continue;
 575
 576                         if (NULL == arch) {
 577                                 if (NULL != of->arch)
 578                                         continue;
 579                         } else if (NULL == of->arch ||
 580                                         strcmp(arch, of->arch))
 581                                 continue;
 582                 }
 583
 584                 if (NULL == arch)
 585                         arch = "";
 586
 587                 /*
 588                  * By default, skip a file if the title given
 589                  * in the file disagrees with the file name.
 590                  * If both agree, use the file name as the title,
 591                  * because the one in the file usually is all caps.
 592                  */
 593
 594                 assert(of->title);
 595                 assert(mtitle);
 596
 597                 if (0 == strcasecmp(mtitle, of->title))
 598                         mtitle = of->title;
 599                 else if (0 == use_all)
 600                         continue;
 601
 602                 /*
 603                  * The index record value consists of a nil-terminated
 604                  * filename, a nil-terminated manual section, and a
 605                  * nil-terminated description.  Since the description
 606                  * may not be set, we set a sentinel to see if we're
 607                  * going to write a nil byte in its place.
 608                  */
 609
 610                 dbuf->len = 0;
 611                 buf_append(dbuf, mdoc ? "mdoc" : (man ? "man" : "cat"));
 612                 buf_appendb(dbuf, fn, strlen(fn) + 1);
 613                 buf_appendb(dbuf, msec, strlen(msec) + 1);
 614                 buf_appendb(dbuf, mtitle, strlen(mtitle) + 1);
 615                 buf_appendb(dbuf, arch, strlen(arch) + 1);
 616
 617                 sv = dbuf->len;
 618
 619                 /* Fix the record number in the btree value. */
 620
 621                 if (mdoc)
 622                         pmdoc_node(hash, buf, dbuf,
 623                                 mdoc_node(mdoc), mdoc_meta(mdoc));
 624                 else if (man)
 625                         pman_node(hash, buf, dbuf, man_node(man));
 626                 else
 627                         pformatted(hash, buf, dbuf, of);
 628
 629                 /*
 630                  * Copy from the in-memory hashtable of pending keywords
 631                  * into the database.
 632                  */
 633
 634                 vbuf.rec = htobe32(rec);
 635                 seq = R_FIRST;
 636                 while (0 == (ch = (*hash->seq)(hash, &key, &val, seq))) {
 637                         seq = R_NEXT;
 638                         vbuf.mask = htobe64(*(uint64_t *)val.data);
 639                         val.size = sizeof(struct db_val);
 640                         val.data = &vbuf;
 641                         dbt_put(db, dbf, &key, &val);
 642                 }
 643                 if (ch < 0) {
 644                         perror("hash");
 645                         exit((int)MANDOCLEVEL_SYSERR);
 646                 }
 647
 648                 /*
 649                  * Apply to the index.  If we haven't had a description
 650                  * set, put an empty one in now.
 651                  */
 652
 653                 if (dbuf->len == sv)
 654                         buf_appendb(dbuf, "", 1);
 655
 656                 key.data = &rec;
 657                 key.size = sizeof(recno_t);
 658
 659                 val.data = dbuf->cp;
 660                 val.size = dbuf->len;
 661
 662                 if (verb)
 663                         printf("%s: Added index\n", fn);
 664
 665                 dbt_put(idx, idxf, &key, &val);
 666         }
 667 }
 668
 669 /*
 670  * Scan through all entries in the index file `idx' and prune those
 671  * entries in `ofile'.
 672  * Pruning consists of removing from `db', then invalidating the entry
 673  * in `idx' (zeroing its value size).
 674  */
 675 static void
 676 index_prune(const struct of *ofile, DB *db, const char *dbf,
 677                 DB *idx, const char *idxf,
 678                 recno_t *maxrec, recno_t **recs, size_t *recsz)
 679 {
 680         const struct of *of;
 681         const char      *fn, *cp;
 682         struct db_val   *vbuf;
 683         unsigned         seq, sseq;
 684         DBT              key, val;
 685         size_t           reccur;
 686         int              ch;
 687
 688         reccur = 0;
 689         seq = R_FIRST;
 690         while (0 == (ch = (*idx->seq)(idx, &key, &val, seq))) {
 691                 seq = R_NEXT;
 692                 *maxrec = *(recno_t *)key.data;
 693                 cp = val.data;
 694
 695                 /* Deleted records are zero-sized.  Skip them. */
 696
 697                 if (0 == val.size)
 698                         goto cont;
 699
 700                 /*
 701                  * Make sure we're sane.
 702                  * Read past our mdoc/man/cat type to the next string,
 703                  * then make sure it's bounded by a NUL.
 704                  * Failing any of these, we go into our error handler.
 705                  */
 706
 707                 if (NULL == (fn = memchr(cp, '\0', val.size)))
 708                         break;
 709                 if (++fn - cp >= (int)val.size)
 710                         break;
 711                 if (NULL == memchr(fn, '\0', val.size - (fn - cp)))
 712                         break;
 713
 714                 /*
 715                  * Search for the file in those we care about.
 716                  * XXX: build this into a tree.  Too slow.
 717                  */
 718
 719                 for (of = ofile; of; of = of->next)
 720                         if (0 == strcmp(fn, of->fname))
 721                                 break;
 722
 723                 if (NULL == of)
 724                         continue;
 725
 726                 /*
 727                  * Search through the keyword database, throwing out all
 728                  * references to our file.
 729                  */
 730
 731                 sseq = R_FIRST;
 732                 while (0 == (ch = (*db->seq)(db, &key, &val, sseq))) {
 733                         sseq = R_NEXT;
 734                         if (sizeof(struct db_val) != val.size)
 735                                 break;
 736
 737                         vbuf = val.data;
 738                         if (*maxrec != betoh32(vbuf->rec))
 739                                 continue;
 740
 741                         if ((ch = (*db->del)(db, &key, R_CURSOR)) < 0)
 742                                 break;
 743                 }
 744
 745                 if (ch < 0) {
 746                         perror(dbf);
 747                         exit((int)MANDOCLEVEL_SYSERR);
 748                 } else if (1 != ch) {
 749                         fprintf(stderr, "%s: Corrupt database\n", dbf);
 750                         exit((int)MANDOCLEVEL_SYSERR);
 751                 }
 752
 753                 if (verb)
 754                         printf("%s: Deleted index\n", fn);
 755
 756                 val.size = 0;
 757                 ch = (*idx->put)(idx, &key, &val, R_CURSOR);
 758
 759                 if (ch < 0)
 760                         break;
 761 cont:
 762                 if (reccur >= *recsz) {
 763                         *recsz += MANDOC_SLOP;
 764                         *recs = mandoc_realloc
 765                                 (*recs, *recsz * sizeof(recno_t));
 766                 }
 767
 768                 (*recs)[(int)reccur] = *maxrec;
 769                 reccur++;
 770         }
 771
 772         if (ch < 0) {
 773                 perror(idxf);
 774                 exit((int)MANDOCLEVEL_SYSERR);
 775         } else if (1 != ch) {
 776                 fprintf(stderr, "%s: Corrupt index\n", idxf);
 777                 exit((int)MANDOCLEVEL_SYSERR);
 778         }
 779
 780         (*maxrec)++;
 781 }
 782
 783 /*
 784  * Grow the buffer (if necessary) and copy in a binary string.
 785  */
 786 static void
 787 buf_appendb(struct buf *buf, const void *cp, size_t sz)
 788 {
 789
 790         /* Overshoot by MANDOC_BUFSZ. */
 791
 792         while (buf->len + sz >= buf->size) {
 793                 buf->size = buf->len + sz + MANDOC_BUFSZ;
 794                 buf->cp = mandoc_realloc(buf->cp, buf->size);
 795         }
 796
 797         memcpy(buf->cp + (int)buf->len, cp, sz);
 798         buf->len += sz;
 799 }
 800
 801 /*
 802  * Append a nil-terminated string to the buffer.
 803  * This can be invoked multiple times.
 804  * The buffer string will be nil-terminated.
 805  * If invoked multiple times, a space is put between strings.
 806  */
 807 static void
 808 buf_append(struct buf *buf, const char *cp)
 809 {
 810         size_t           sz;
 811
 812         if (0 == (sz = strlen(cp)))
 813                 return;
 814
 815         if (buf->len)
 816                 buf->cp[(int)buf->len - 1] = ' ';
 817
 818         buf_appendb(buf, cp, sz + 1);
 819 }
 820
 821 /*
 822  * Recursively add all text from a given node.
 823  * This is optimised for general mdoc nodes in this context, which do
 824  * not consist of subexpressions and having a recursive call for n->next
 825  * would be wasteful.
 826  * The "f" variable should be 0 unless called from pmdoc_Nd for the
 827  * description buffer, which does not start at the beginning of the
 828  * buffer.
 829  */
 830 static void
 831 buf_appendmdoc(struct buf *buf, const struct mdoc_node *n, int f)
 832 {
 833
 834         for ( ; n; n = n->next) {
 835                 if (n->child)
 836                         buf_appendmdoc(buf, n->child, f);
 837
 838                 if (MDOC_TEXT == n->type && f) {
 839                         f = 0;
 840                         buf_appendb(buf, n->string,
 841                                         strlen(n->string) + 1);
 842                 } else if (MDOC_TEXT == n->type)
 843                         buf_append(buf, n->string);
 844
 845         }
 846 }
 847
 848 /* ARGSUSED */
 849 static void
 850 pmdoc_An(MDOC_ARGS)
 851 {
 852
 853         if (SEC_AUTHORS != n->sec)
 854                 return;
 855
 856         buf_appendmdoc(buf, n->child, 0);
 857         hash_put(hash, buf, TYPE_An);
 858 }
 859
 860 static void
 861 hash_reset(DB **db)
 862 {
 863         DB              *hash;
 864
 865         if (NULL != (hash = *db))
 866                 (*hash->close)(hash);
 867
 868         *db = dbopen(NULL, O_CREAT|O_RDWR, 0644, DB_HASH, NULL);
 869         if (NULL == *db) {
 870                 perror("hash");
 871                 exit((int)MANDOCLEVEL_SYSERR);
 872         }
 873 }
 874
 875 /* ARGSUSED */
 876 static void
 877 pmdoc_Fd(MDOC_ARGS)
 878 {
 879         const char      *start, *end;
 880         size_t           sz;
 881
 882         if (SEC_SYNOPSIS != n->sec)
 883                 return;
 884         if (NULL == (n = n->child) || MDOC_TEXT != n->type)
 885                 return;
 886
 887         /*
 888          * Only consider those `Fd' macro fields that begin with an
 889          * "inclusion" token (versus, e.g., #define).
 890          */
 891         if (strcmp("#include", n->string))
 892                 return;
 893
 894         if (NULL == (n = n->next) || MDOC_TEXT != n->type)
 895                 return;
 896
 897         /*
 898          * Strip away the enclosing angle brackets and make sure we're
 899          * not zero-length.
 900          */
 901
 902         start = n->string;
 903         if ('<' == *start || '"' == *start)
 904                 start++;
 905
 906         if (0 == (sz = strlen(start)))
 907                 return;
 908
 909         end = &start[(int)sz - 1];
 910         if ('>' == *end || '"' == *end)
 911                 end--;
 912
 913         assert(end >= start);
 914
 915         buf_appendb(buf, start, (size_t)(end - start + 1));
 916         buf_appendb(buf, "", 1);
 917
 918         hash_put(hash, buf, TYPE_In);
 919 }
 920
 921 /* ARGSUSED */
 922 static void
 923 pmdoc_Cd(MDOC_ARGS)
 924 {
 925
 926         if (SEC_SYNOPSIS != n->sec)
 927                 return;
 928
 929         buf_appendmdoc(buf, n->child, 0);
 930         hash_put(hash, buf, TYPE_Cd);
 931 }
 932
 933 /* ARGSUSED */
 934 static void
 935 pmdoc_In(MDOC_ARGS)
 936 {
 937
 938         if (SEC_SYNOPSIS != n->sec)
 939                 return;
 940         if (NULL == n->child || MDOC_TEXT != n->child->type)
 941                 return;
 942
 943         buf_append(buf, n->child->string);
 944         hash_put(hash, buf, TYPE_In);
 945 }
 946
 947 /* ARGSUSED */
 948 static void
 949 pmdoc_Fn(MDOC_ARGS)
 950 {
 951         const char      *cp;
 952
 953         if (SEC_SYNOPSIS != n->sec)
 954                 return;
 955         if (NULL == n->child || MDOC_TEXT != n->child->type)
 956                 return;
 957
 958         /* .Fn "struct type *arg" "foo" */
 959
 960         cp = strrchr(n->child->string, ' ');
 961         if (NULL == cp)
 962                 cp = n->child->string;
 963
 964         /* Strip away pointer symbol. */
 965
 966         while ('*' == *cp)
 967                 cp++;
 968
 969         buf_append(buf, cp);
 970         hash_put(hash, buf, TYPE_Fn);
 971 }
 972
 973 /* ARGSUSED */
 974 static void
 975 pmdoc_St(MDOC_ARGS)
 976 {
 977
 978         if (SEC_STANDARDS != n->sec)
 979                 return;
 980         if (NULL == n->child || MDOC_TEXT != n->child->type)
 981                 return;
 982
 983         buf_append(buf, n->child->string);
 984         hash_put(hash, buf, TYPE_St);
 985 }
 986
 987 /* ARGSUSED */
 988 static void
 989 pmdoc_Xr(MDOC_ARGS)
 990 {
 991
 992         if (NULL == (n = n->child))
 993                 return;
 994
 995         buf_appendb(buf, n->string, strlen(n->string));
 996
 997         if (NULL != (n = n->next)) {
 998                 buf_appendb(buf, ".", 1);
 999                 buf_appendb(buf, n->string, strlen(n->string) + 1);
1000         } else
1001                 buf_appendb(buf, ".", 2);
1002
1003         hash_put(hash, buf, TYPE_Xr);
1004 }
1005
1006 /* ARGSUSED */
1007 static void
1008 pmdoc_Vt(MDOC_ARGS)
1009 {
1010         const char      *start;
1011         size_t           sz;
1012
1013         if (SEC_SYNOPSIS != n->sec)
1014                 return;
1015         if (MDOC_Vt == n->tok && MDOC_BODY != n->type)
1016                 return;
1017         if (NULL == n->last || MDOC_TEXT != n->last->type)
1018                 return;
1019
1020         /*
1021          * Strip away leading pointer symbol '*' and trailing ';'.
1022          */
1023
1024         start = n->last->string;
1025
1026         while ('*' == *start)
1027                 start++;
1028
1029         if (0 == (sz = strlen(start)))
1030                 return;
1031
1032         if (';' == start[(int)sz - 1])
1033                 sz--;
1034
1035         if (0 == sz)
1036                 return;
1037
1038         buf_appendb(buf, start, sz);
1039         buf_appendb(buf, "", 1);
1040         hash_put(hash, buf, TYPE_Va);
1041 }
1042
1043 /* ARGSUSED */
1044 static void
1045 pmdoc_Fo(MDOC_ARGS)
1046 {
1047
1048         if (SEC_SYNOPSIS != n->sec || MDOC_HEAD != n->type)
1049                 return;
1050         if (NULL == n->child || MDOC_TEXT != n->child->type)
1051                 return;
1052
1053         buf_append(buf, n->child->string);
1054         hash_put(hash, buf, TYPE_Fn);
1055 }
1056
1057
1058 /* ARGSUSED */
1059 static void
1060 pmdoc_Nd(MDOC_ARGS)
1061 {
1062
1063         if (MDOC_BODY != n->type)
1064                 return;
1065
1066         buf_appendmdoc(dbuf, n->child, 1);
1067         buf_appendmdoc(buf, n->child, 0);
1068
1069         hash_put(hash, buf, TYPE_Nd);
1070 }
1071
1072 /* ARGSUSED */
1073 static void
1074 pmdoc_Er(MDOC_ARGS)
1075 {
1076
1077         if (SEC_ERRORS != n->sec)
1078                 return;
1079
1080         buf_appendmdoc(buf, n->child, 0);
1081         hash_put(hash, buf, TYPE_Er);
1082 }
1083
1084 /* ARGSUSED */
1085 static void
1086 pmdoc_Ev(MDOC_ARGS)
1087 {
1088
1089         if (SEC_ENVIRONMENT != n->sec)
1090                 return;
1091
1092         buf_appendmdoc(buf, n->child, 0);
1093         hash_put(hash, buf, TYPE_Ev);
1094 }
1095
1096 /* ARGSUSED */
1097 static void
1098 pmdoc_Pa(MDOC_ARGS)
1099 {
1100
1101         if (SEC_FILES != n->sec)
1102                 return;
1103
1104         buf_appendmdoc(buf, n->child, 0);
1105         hash_put(hash, buf, TYPE_Pa);
1106 }
1107
1108 /* ARGSUSED */
1109 static void
1110 pmdoc_Nm(MDOC_ARGS)
1111 {
1112
1113         if (SEC_NAME == n->sec) {
1114                 buf_appendmdoc(buf, n->child, 0);
1115                 hash_put(hash, buf, TYPE_Nm);
1116                 return;
1117         } else if (SEC_SYNOPSIS != n->sec || MDOC_HEAD != n->type)
1118                 return;
1119
1120         if (NULL == n->child)
1121                 buf_append(buf, m->name);
1122
1123         buf_appendmdoc(buf, n->child, 0);
1124         hash_put(hash, buf, TYPE_Nm);
1125 }
1126
1127 static void
1128 hash_put(DB *db, const struct buf *buf, uint64_t mask)
1129 {
1130         DBT              key, val;
1131         int              rc;
1132
1133         if (buf->len < 2)
1134                 return;
1135
1136         key.data = buf->cp;
1137         key.size = buf->len;
1138
1139         if ((rc = (*db->get)(db, &key, &val, 0)) < 0) {
1140                 perror("hash");
1141                 exit((int)MANDOCLEVEL_SYSERR);
1142         } else if (0 == rc)
1143                 mask |= *(uint64_t *)val.data;
1144
1145         val.data = &mask;
1146         val.size = sizeof(uint64_t);
1147
1148         if ((rc = (*db->put)(db, &key, &val, 0)) < 0) {
1149                 perror("hash");
1150                 exit((int)MANDOCLEVEL_SYSERR);
1151         }
1152 }
1153
1154 static void
1155 dbt_put(DB *db, const char *dbn, DBT *key, DBT *val)
1156 {
1157
1158         assert(key->size);
1159         assert(val->size);
1160
1161         if (0 == (*db->put)(db, key, val, 0))
1162                 return;
1163
1164         perror(dbn);
1165         exit((int)MANDOCLEVEL_SYSERR);
1166         /* NOTREACHED */
1167 }
1168
1169 /*
1170  * Call out to per-macro handlers after clearing the persistent database
1171  * key.  If the macro sets the database key, flush it to the database.
1172  */
1173 static void
1174 pmdoc_node(MDOC_ARGS)
1175 {
1176
1177         if (NULL == n)
1178                 return;
1179
1180         switch (n->type) {
1181         case (MDOC_HEAD):
1182                 /* FALLTHROUGH */
1183         case (MDOC_BODY):
1184                 /* FALLTHROUGH */
1185         case (MDOC_TAIL):
1186                 /* FALLTHROUGH */
1187         case (MDOC_BLOCK):
1188                 /* FALLTHROUGH */
1189         case (MDOC_ELEM):
1190                 if (NULL == mdocs[n->tok])
1191                         break;
1192
1193                 buf->len = 0;
1194                 (*mdocs[n->tok])(hash, buf, dbuf, n, m);
1195                 break;
1196         default:
1197                 break;
1198         }
1199
1200         pmdoc_node(hash, buf, dbuf, n->child, m);
1201         pmdoc_node(hash, buf, dbuf, n->next, m);
1202 }
1203
1204 static int
1205 pman_node(MAN_ARGS)
1206 {
1207         const struct man_node *head, *body;
1208         const char      *start, *sv;
1209         size_t           sz;
1210
1211         if (NULL == n)
1212                 return(0);
1213
1214         /*
1215          * We're only searching for one thing: the first text child in
1216          * the BODY of a NAME section.  Since we don't keep track of
1217          * sections in -man, run some hoops to find out whether we're in
1218          * the correct section or not.
1219          */
1220
1221         if (MAN_BODY == n->type && MAN_SH == n->tok) {
1222                 body = n;
1223                 assert(body->parent);
1224                 if (NULL != (head = body->parent->head) &&
1225                                 1 == head->nchild &&
1226                                 NULL != (head = (head->child)) &&
1227                                 MAN_TEXT == head->type &&
1228                                 0 == strcmp(head->string, "NAME") &&
1229                                 NULL != (body = body->child) &&
1230                                 MAN_TEXT == body->type) {
1231
1232                         assert(body->string);
1233                         start = sv = body->string;
1234
1235                         /*
1236                          * Go through a special heuristic dance here.
1237                          * This is why -man manuals are great!
1238                          * (I'm being sarcastic: my eyes are bleeding.)
1239                          * Conventionally, one or more manual names are
1240                          * comma-specified prior to a whitespace, then a
1241                          * dash, then a description.  Try to puzzle out
1242                          * the name parts here.
1243                          */
1244
1245                         for ( ;; ) {
1246                                 sz = strcspn(start, " ,");
1247                                 if ('\0' == start[(int)sz])
1248                                         break;
1249
1250                                 buf->len = 0;
1251                                 buf_appendb(buf, start, sz);
1252                                 buf_appendb(buf, "", 1);
1253
1254                                 hash_put(hash, buf, TYPE_Nm);
1255
1256                                 if (' ' == start[(int)sz]) {
1257                                         start += (int)sz + 1;
1258                                         break;
1259                                 }
1260
1261                                 assert(',' == start[(int)sz]);
1262                                 start += (int)sz + 1;
1263                                 while (' ' == *start)
1264                                         start++;
1265                         }
1266
1267                         buf->len = 0;
1268
1269                         if (sv == start) {
1270                                 buf_append(buf, start);
1271                                 return(1);
1272                         }
1273
1274                         while (' ' == *start)
1275                                 start++;
1276
1277                         if (0 == strncmp(start, "-", 1))
1278                                 start += 1;
1279                         else if (0 == strncmp(start, "\\-", 2))
1280                                 start += 2;
1281                         else if (0 == strncmp(start, "\\(en", 4))
1282                                 start += 4;
1283                         else if (0 == strncmp(start, "\\(em", 4))
1284                                 start += 4;
1285
1286                         while (' ' == *start)
1287                                 start++;
1288
1289                         sz = strlen(start) + 1;
1290                         buf_appendb(dbuf, start, sz);
1291                         buf_appendb(buf, start, sz);
1292
1293                         hash_put(hash, buf, TYPE_Nd);
1294                 }
1295         }
1296
1297         for (n = n->child; n; n = n->next)
1298                 if (pman_node(hash, buf, dbuf, n))
1299                         return(1);
1300
1301         return(0);
1302 }
1303
1304 /*
1305  * Parse a formatted manual page.
1306  * By necessity, this involves rather crude guesswork.
1307  */
1308 static void
1309 pformatted(DB *hash, struct buf *buf, struct buf *dbuf,
1310                  const struct of *of)
1311 {
1312         FILE            *stream;
1313         char            *line, *p;
1314         size_t           len, plen;
1315
1316         if (NULL == (stream = fopen(of->fname, "r"))) {
1317                 perror(of->fname);
1318                 return;
1319         }
1320
1321         /*
1322          * Always use the title derived from the filename up front,
1323          * do not even try to find it in the file.  This also makes
1324          * sure we don't end up with an orphan index record, even if
1325          * the file content turns out to be completely unintelligible.
1326          */
1327
1328         buf->len = 0;
1329         buf_append(buf, of->title);
1330         hash_put(hash, buf, TYPE_Nm);
1331
1332         while (NULL != (line = fgetln(stream, &len)) && '\n' != *line)
1333                 /* Skip to first blank line. */ ;
1334
1335         while (NULL != (line = fgetln(stream, &len)) &&
1336                         ('\n' == *line || ' ' == *line))
1337                 /* Skip to first section header. */ ;
1338
1339         /*
1340          * If no page content can be found,
1341          * reuse the page title as the page description.
1342          */
1343
1344         if (NULL == (line = fgetln(stream, &len))) {
1345                 buf_appendb(dbuf, buf->cp, buf->size);
1346                 hash_put(hash, buf, TYPE_Nd);
1347                 fclose(stream);
1348                 return;
1349         }
1350         fclose(stream);
1351
1352         /*
1353          * If there is a dash, skip to the text following it.
1354          */
1355
1356         for (p = line, plen = len; plen; p++, plen--)
1357                 if ('-' == *p)
1358                         break;
1359         for ( ; plen; p++, plen--)
1360                 if ('-' != *p && ' ' != *p && 8 != *p)
1361                         break;
1362         if (0 == plen) {
1363                 p = line;
1364                 plen = len;
1365         }
1366
1367         /*
1368          * Copy the rest of the line, but no more than 70 bytes.
1369          */
1370
1371         if (70 < plen)
1372                 plen = 70;
1373         p[plen-1] = '\0';
1374         buf_appendb(dbuf, p, plen);
1375         buf->len = 0;
1376         buf_appendb(buf, p, plen);
1377         hash_put(hash, buf, TYPE_Nd);
1378 }
1379
1380 static void
1381 ofile_argbuild(int argc, char *argv[], struct of **of)
1382 {
1383         char             buf[MAXPATHLEN];
1384         char            *sec, *arch, *title, *p;
1385         int              i, src_form;
1386         struct of       *nof;
1387
1388         for (i = 0; i < argc; i++) {
1389
1390                 /*
1391                  * Try to infer the manual section, architecture and
1392                  * page title from the path, assuming it looks like
1393                  *   man*[/<arch>]/<title>.<section>   or
1394                  *   cat<section>[/<arch>]/<title>.0
1395                  */
1396
1397                 if (strlcpy(buf, argv[i], sizeof(buf)) >= sizeof(buf)) {
1398                         fprintf(stderr, "%s: Path too long\n", argv[i]);
1399                         continue;
1400                 }
1401                 sec = arch = title = NULL;
1402                 src_form = 0;
1403                 p = strrchr(buf, '\0');
1404                 while (p-- > buf) {
1405                         if (NULL == sec && '.' == *p) {
1406                                 sec = p + 1;
1407                                 *p = '\0';
1408                                 if ('0' == *sec)
1409                                         src_form |= MANDOC_FORM;
1410                                 else if ('1' <= *sec && '9' >= *sec)
1411                                         src_form |= MANDOC_SRC;
1412                                 continue;
1413                         }
1414                         if ('/' != *p)
1415                                 continue;
1416                         if (NULL == title) {
1417                                 title = p + 1;
1418                                 *p = '\0';
1419                                 continue;
1420                         }
1421                         if (strncmp("man", p + 1, 3)) {
1422                                 src_form |= MANDOC_SRC;
1423                                 arch = p + 1;
1424                         } else if (strncmp("cat", p + 1, 3)) {
1425                                 src_form |= MANDOC_FORM;
1426                                 arch = p + 1;
1427                         }
1428                         break;
1429                 }
1430                 if (NULL == title)
1431                         title = buf;
1432
1433                 /*
1434                  * Build the file structure.
1435                  */
1436
1437                 nof = mandoc_calloc(1, sizeof(struct of));
1438                 nof->fname = mandoc_strdup(argv[i]);
1439                 if (NULL != sec)
1440                         nof->sec = mandoc_strdup(sec);
1441                 if (NULL != arch)
1442                         nof->arch = mandoc_strdup(arch);
1443                 nof->title = mandoc_strdup(title);
1444                 nof->src_form = src_form;
1445
1446                 /*
1447                  * Add the structure to the list.
1448                  */
1449
1450                 if (verb > 2)
1451                         printf("%s: Scheduling\n", argv[i]);
1452                 if (NULL == *of) {
1453                         *of = nof;
1454                         (*of)->first = nof;
1455                 } else {
1456                         nof->first = (*of)->first;
1457                         (*of)->next = nof;
1458                         *of = nof;
1459                 }
1460         }
1461 }
1462
1463 /*
1464  * Recursively build up a list of files to parse.
1465  * We use this instead of ftw() and so on because I don't want global
1466  * variables hanging around.
1467  * This ignores the mandoc.db and mandoc.index files, but assumes that
1468  * everything else is a manual.
1469  * Pass in a pointer to a NULL structure for the first invocation.
1470  */
1471 static int
1472 ofile_dirbuild(const char *dir, const char* psec, const char *parch,
1473                 int p_src_form, struct of **of)
1474 {
1475         char             buf[MAXPATHLEN];
1476         struct stat      sb;
1477         size_t           sz;
1478         DIR             *d;
1479         const char      *fn, *sec, *arch;
1480         char            *p, *q, *suffix;
1481         struct of       *nof;
1482         struct dirent   *dp;
1483         int              src_form;
1484
1485         if (NULL == (d = opendir(dir))) {
1486                 perror(dir);
1487                 return(0);
1488         }
1489
1490         while (NULL != (dp = readdir(d))) {
1491                 fn = dp->d_name;
1492
1493                 if ('.' == *fn)
1494                         continue;
1495
1496                 src_form = p_src_form;
1497
1498                 if (DT_DIR == dp->d_type) {
1499                         sec = psec;
1500                         arch = parch;
1501
1502                         /*
1503                          * By default, only use directories called:
1504                          *   man<section>/[<arch>/]   or
1505                          *   cat<section>/[<arch>/]
1506                          */
1507
1508                         if (NULL == sec) {
1509                                 if(0 == strncmp("man", fn, 3)) {
1510                                         src_form |= MANDOC_SRC;
1511                                         sec = fn + 3;
1512                                 } else if (0 == strncmp("cat", fn, 3)) {
1513                                         src_form |= MANDOC_FORM;
1514                                         sec = fn + 3;
1515                                 } else if (use_all)
1516                                         sec = fn;
1517                                 else
1518                                         continue;
1519                         } else if (NULL == arch && (use_all ||
1520                                         NULL == strchr(fn, '.')))
1521                                 arch = fn;
1522                         else if (0 == use_all)
1523                                 continue;
1524
1525                         buf[0] = '\0';
1526                         strlcat(buf, dir, MAXPATHLEN);
1527                         strlcat(buf, "/", MAXPATHLEN);
1528                         sz = strlcat(buf, fn, MAXPATHLEN);
1529
1530                         if (MAXPATHLEN <= sz) {
1531                                 fprintf(stderr, "%s: Path too long\n", dir);
1532                                 return(0);
1533                         }
1534
1535                         if (verb > 2)
1536                                 printf("%s: Scanning\n", buf);
1537
1538                         if ( ! ofile_dirbuild(buf, sec, arch,
1539                                         src_form, of))
1540                                 return(0);
1541                 }
1542                 if (DT_REG != dp->d_type ||
1543                     (NULL == psec && !use_all) ||
1544                     !strcmp(MANDOC_DB, fn) ||
1545                     !strcmp(MANDOC_IDX, fn))
1546                         continue;
1547
1548                 /*
1549                  * By default, skip files where the file name suffix
1550                  * does not agree with the section directory
1551                  * they are located in.
1552                  */
1553
1554                 suffix = strrchr(fn, '.');
1555                 if (0 == use_all) {
1556                         if (NULL == suffix)
1557                                 continue;
1558                         if ((MANDOC_SRC & src_form &&
1559                                          strcmp(suffix + 1, psec)) ||
1560                             (MANDOC_FORM & src_form &&
1561                                          strcmp(suffix + 1, "0")))
1562                                         continue;
1563                 }
1564                 if (NULL != suffix) {
1565                         if ('0' == suffix[1])
1566                                 src_form |= MANDOC_FORM;
1567                         else if ('1' <= suffix[1] && '9' >= suffix[1])
1568                                 src_form |= MANDOC_SRC;
1569                 }
1570
1571
1572                 /*
1573                  * Skip formatted manuals if a source version is
1574                  * available.  Ignore the age: it is very unlikely
1575                  * that people install newer formatted base manuals
1576                  * when they used to have source manuals before,
1577                  * and in ports, old manuals get removed on update.
1578                  */
1579                 if (0 == use_all && MANDOC_FORM & src_form &&
1580                                 NULL != psec) {
1581                         buf[0] = '\0';
1582                         strlcat(buf, dir, MAXPATHLEN);
1583                         p = strrchr(buf, '/');
1584                         if (NULL == p)
1585                                 p = buf;
1586                         else
1587                                 p++;
1588                         if (0 == strncmp("cat", p, 3))
1589                                 memcpy(p, "man", 3);
1590                         strlcat(buf, "/", MAXPATHLEN);
1591                         sz = strlcat(buf, fn, MAXPATHLEN);
1592                         if (sz >= MAXPATHLEN) {
1593                                 fprintf(stderr, "%s: Path too long\n", buf);
1594                                 continue;
1595                         }
1596                         q = strrchr(buf, '.');
1597                         if (NULL != q && p < q++) {
1598                                 *q = '\0';
1599                                 sz = strlcat(buf, psec, MAXPATHLEN);
1600                                 if (sz >= MAXPATHLEN) {
1601                                         fprintf(stderr,
1602                                             "%s: Path too long\n", buf);
1603                                         continue;
1604                                 }
1605                                 if (0 == stat(buf, &sb))
1606                                         continue;
1607                         }
1608                 }
1609
1610                 buf[0] = '\0';
1611                 strlcat(buf, dir, MAXPATHLEN);
1612                 strlcat(buf, "/", MAXPATHLEN);
1613                 sz = strlcat(buf, fn, MAXPATHLEN);
1614                 if (sz >= MAXPATHLEN) {
1615                         fprintf(stderr, "%s: Path too long\n", dir);
1616                         continue;
1617                 }
1618
1619                 nof = mandoc_calloc(1, sizeof(struct of));
1620                 nof->fname = mandoc_strdup(buf);
1621                 if (NULL != psec)
1622                         nof->sec = mandoc_strdup(psec);
1623                 if (NULL != parch)
1624                         nof->arch = mandoc_strdup(parch);
1625                 nof->src_form = src_form;
1626
1627                 /*
1628                  * Remember the file name without the extension,
1629                  * to be used as the page title in the database.
1630                  */
1631
1632                 if (NULL != suffix)
1633                         *suffix = '\0';
1634                 nof->title = mandoc_strdup(fn);
1635
1636                 /*
1637                  * Add the structure to the list.
1638                  */
1639
1640                 if (verb > 2)
1641                         printf("%s: Scheduling\n", buf);
1642                 if (NULL == *of) {
1643                         *of = nof;
1644                         (*of)->first = nof;
1645                 } else {
1646                         nof->first = (*of)->first;
1647                         (*of)->next = nof;
1648                         *of = nof;
1649                 }
1650         }
1651
1652         closedir(d);
1653         return(1);
1654 }
1655
1656 static void
1657 ofile_free(struct of *of)
1658 {
1659         struct of       *nof;
1660
1661         while (of) {
1662                 nof = of->next;
1663                 free(of->fname);
1664                 free(of->sec);
1665                 free(of->arch);
1666                 free(of->title);
1667                 free(of);
1668                 of = nof;
1669         }
1670 }
1671
1672 static void
1673 usage(void)
1674 {
1675
1676         fprintf(stderr, "usage: %s [-v] "
1677                         "[-d dir [files...] |"
1678                         " -u dir [files...] |"
1679                         " dir...]\n", progname);
1680 }