]> git.cameronkatri.com Git - mandoc.git/blob - mandocdb.c
git.ckatri.com / mandoc.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
Changing existing SQLite databases was utterly broken:
[mandoc.git] / mandocdb.c
1 /* $Id: mandocdb.c,v 1.57 2013/06/03 22:51:14 schwarze Exp $ */
2 /*
3 * Copyright (c) 2011, 2012 Kristaps Dzonsons <kristaps@bsd.lv>
4 * Copyright (c) 2011, 2012, 2013 Ingo Schwarze <schwarze@openbsd.org>
5 *
6 * Permission to use, copy, modify, and distribute this software for any
7 * purpose with or without fee is hereby granted, provided that the above
8 * copyright notice and this permission notice appear in all copies.
9 *
10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17 */
18 #ifdef HAVE_CONFIG_H
19 #include "config.h"
20 #endif
21
22 #include <sys/param.h>
23 #include <sys/stat.h>
24
25 #include <assert.h>
26 #include <ctype.h>
27 #include <errno.h>
28 #include <fcntl.h>
29 #include <fts.h>
30 #include <getopt.h>
31 #include <stddef.h>
32 #include <stdint.h>
33 #include <stdlib.h>
34 #include <string.h>
35 #include <unistd.h>
36
37 #ifdef HAVE_OHASH
38 #include <ohash.h>
39 #else
40 #include "compat_ohash.h"
41 #endif
42 #include <sqlite3.h>
43
44 #include "mdoc.h"
45 #include "man.h"
46 #include "mandoc.h"
47 #include "manpath.h"
48 #include "mansearch.h"
49
50 #define SQL_EXEC(_v) \
51 if (SQLITE_OK != sqlite3_exec(db, (_v), NULL, NULL, NULL)) \
52 fprintf(stderr, "%s\n", sqlite3_errmsg(db))
53 #define SQL_BIND_TEXT(_s, _i, _v) \
54 if (SQLITE_OK != sqlite3_bind_text \
55 ((_s), (_i)++, (_v), -1, SQLITE_STATIC)) \
56 fprintf(stderr, "%s\n", sqlite3_errmsg(db))
57 #define SQL_BIND_INT(_s, _i, _v) \
58 if (SQLITE_OK != sqlite3_bind_int \
59 ((_s), (_i)++, (_v))) \
60 fprintf(stderr, "%s\n", sqlite3_errmsg(db))
61 #define SQL_BIND_INT64(_s, _i, _v) \
62 if (SQLITE_OK != sqlite3_bind_int64 \
63 ((_s), (_i)++, (_v))) \
64 fprintf(stderr, "%s\n", sqlite3_errmsg(db))
65 #define SQL_STEP(_s) \
66 if (SQLITE_DONE != sqlite3_step((_s))) \
67 fprintf(stderr, "%s\n", sqlite3_errmsg(db))
68
69 enum op {
70 OP_DEFAULT = 0, /* new dbs from dir list or default config */
71 OP_CONFFILE, /* new databases from custom config file */
72 OP_UPDATE, /* delete/add entries in existing database */
73 OP_DELETE, /* delete entries from existing database */
74 OP_TEST /* change no databases, report potential problems */
75 };
76
77 enum form {
78 FORM_SRC, /* format is -man or -mdoc */
79 FORM_CAT, /* format is cat */
80 FORM_NONE /* format is unknown */
81 };
82
83 struct str {
84 char *utf8; /* key in UTF-8 form */
85 const struct of *of; /* if set, the owning parse */
86 struct str *next; /* next in owning parse sequence */
87 uint64_t mask; /* bitmask in sequence */
88 char key[]; /* the string itself */
89 };
90
91 struct id {
92 ino_t ino;
93 dev_t dev;
94 };
95
96 struct of {
97 struct id id; /* used for hashing routine */
98 struct of *next; /* next in ofs */
99 enum form dform; /* path-cued form */
100 enum form sform; /* suffix-cued form */
101 char file[MAXPATHLEN]; /* filename rel. to manpath */
102 const char *desc; /* parsed description */
103 const char *sec; /* suffix-cued section (or empty) */
104 const char *dsec; /* path-cued section (or empty) */
105 const char *arch; /* path-cued arch. (or empty) */
106 const char *name; /* name (from filename) (not empty) */
107 };
108
109 enum stmt {
110 STMT_DELETE = 0, /* delete manpage */
111 STMT_INSERT_DOC, /* insert manpage */
112 STMT_INSERT_KEY, /* insert parsed key */
113 STMT__MAX
114 };
115
116 typedef int (*mdoc_fp)(struct of *, const struct mdoc_node *);
117
118 struct mdoc_handler {
119 mdoc_fp fp; /* optional handler */
120 uint64_t mask; /* set unless handler returns 0 */
121 int flags; /* for use by pmdoc_node */
122 #define MDOCF_CHILD 0x01 /* automatically index child nodes */
123 };
124
125 static void dbclose(const char *, int);
126 static void dbindex(struct mchars *, int,
127 const struct of *, const char *);
128 static int dbopen(const char *, int);
129 static void dbprune(const char *);
130 static void fileadd(struct of *);
131 static int filecheck(const char *);
132 static void filescan(const char *, const char *);
133 static struct str *hashget(const char *, size_t);
134 static void *hash_alloc(size_t, void *);
135 static void hash_free(void *, size_t, void *);
136 static void *hash_halloc(size_t, void *);
137 static void inoadd(const struct stat *, struct of *);
138 static int inocheck(const struct stat *);
139 static void ofadd(const char *, int, const char *,
140 const char *, const char *, const char *,
141 const char *, const struct stat *);
142 static void offree(void);
143 static int ofmerge(struct mchars *, struct mparse *, const char *);
144 static void parse_catpage(struct of *, const char *);
145 static int parse_man(struct of *,
146 const struct man_node *);
147 static void parse_mdoc(struct of *, const struct mdoc_node *);
148 static int parse_mdoc_body(struct of *, const struct mdoc_node *);
149 static int parse_mdoc_head(struct of *, const struct mdoc_node *);
150 static int parse_mdoc_Fd(struct of *, const struct mdoc_node *);
151 static int parse_mdoc_Fn(struct of *, const struct mdoc_node *);
152 static int parse_mdoc_In(struct of *, const struct mdoc_node *);
153 static int parse_mdoc_Nd(struct of *, const struct mdoc_node *);
154 static int parse_mdoc_Nm(struct of *, const struct mdoc_node *);
155 static int parse_mdoc_Sh(struct of *, const struct mdoc_node *);
156 static int parse_mdoc_St(struct of *, const struct mdoc_node *);
157 static int parse_mdoc_Xr(struct of *, const struct mdoc_node *);
158 static int path_reset(const char *, int, const char *);
159 static void putkey(const struct of *,
160 const char *, uint64_t);
161 static void putkeys(const struct of *,
162 const char *, int, uint64_t);
163 static void putmdockey(const struct of *,
164 const struct mdoc_node *, uint64_t);
165 static void say(const char *, const char *, const char *, ...);
166 static char *stradd(const char *);
167 static char *straddbuf(const char *, size_t);
168 static int treescan(const char *);
169 static size_t utf8(unsigned int, char [7]);
170 static void utf8key(struct mchars *, struct str *);
171 static void wordaddbuf(const struct of *,
172 const char *, size_t, uint64_t);
173
174 static char *progname;
175 static int use_all; /* use all found files */
176 static int nodb; /* no database changes */
177 static int verb; /* print what we're doing */
178 static int warnings; /* warn about crap */
179 static enum op op; /* operational mode */
180 static struct ohash inos; /* table of inodes/devices */
181 static struct ohash filenames; /* table of filenames */
182 static struct ohash strings; /* table of all strings */
183 static struct of *ofs = NULL; /* vector of files to parse */
184 static struct str *words = NULL; /* word list in current parse */
185 static sqlite3 *db = NULL; /* current database */
186 static sqlite3_stmt *stmts[STMT__MAX]; /* current statements */
187
188 static const struct mdoc_handler mdocs[MDOC_MAX] = {
189 { NULL, 0, 0 }, /* Ap */
190 { NULL, 0, 0 }, /* Dd */
191 { NULL, 0, 0 }, /* Dt */
192 { NULL, 0, 0 }, /* Os */
193 { parse_mdoc_Sh, TYPE_Sh, MDOCF_CHILD }, /* Sh */
194 { parse_mdoc_head, TYPE_Ss, MDOCF_CHILD }, /* Ss */
195 { NULL, 0, 0 }, /* Pp */
196 { NULL, 0, 0 }, /* D1 */
197 { NULL, 0, 0 }, /* Dl */
198 { NULL, 0, 0 }, /* Bd */
199 { NULL, 0, 0 }, /* Ed */
200 { NULL, 0, 0 }, /* Bl */
201 { NULL, 0, 0 }, /* El */
202 { NULL, 0, 0 }, /* It */
203 { NULL, 0, 0 }, /* Ad */
204 { NULL, TYPE_An, MDOCF_CHILD }, /* An */
205 { NULL, TYPE_Ar, MDOCF_CHILD }, /* Ar */
206 { NULL, TYPE_Cd, MDOCF_CHILD }, /* Cd */
207 { NULL, TYPE_Cm, MDOCF_CHILD }, /* Cm */
208 { NULL, TYPE_Dv, MDOCF_CHILD }, /* Dv */
209 { NULL, TYPE_Er, MDOCF_CHILD }, /* Er */
210 { NULL, TYPE_Ev, MDOCF_CHILD }, /* Ev */
211 { NULL, 0, 0 }, /* Ex */
212 { NULL, TYPE_Fa, MDOCF_CHILD }, /* Fa */
213 { parse_mdoc_Fd, TYPE_In, 0 }, /* Fd */
214 { NULL, TYPE_Fl, MDOCF_CHILD }, /* Fl */
215 { parse_mdoc_Fn, 0, 0 }, /* Fn */
216 { NULL, TYPE_Ft, MDOCF_CHILD }, /* Ft */
217 { NULL, TYPE_Ic, MDOCF_CHILD }, /* Ic */
218 { parse_mdoc_In, TYPE_In, MDOCF_CHILD }, /* In */
219 { NULL, TYPE_Li, MDOCF_CHILD }, /* Li */
220 { parse_mdoc_Nd, TYPE_Nd, MDOCF_CHILD }, /* Nd */
221 { parse_mdoc_Nm, TYPE_Nm, MDOCF_CHILD }, /* Nm */
222 { NULL, 0, 0 }, /* Op */
223 { NULL, 0, 0 }, /* Ot */
224 { NULL, TYPE_Pa, MDOCF_CHILD }, /* Pa */
225 { NULL, 0, 0 }, /* Rv */
226 { parse_mdoc_St, TYPE_St, 0 }, /* St */
227 { NULL, TYPE_Va, MDOCF_CHILD }, /* Va */
228 { parse_mdoc_body, TYPE_Va, MDOCF_CHILD }, /* Vt */
229 { parse_mdoc_Xr, TYPE_Xr, 0 }, /* Xr */
230 { NULL, 0, 0 }, /* %A */
231 { NULL, 0, 0 }, /* %B */
232 { NULL, 0, 0 }, /* %D */
233 { NULL, 0, 0 }, /* %I */
234 { NULL, 0, 0 }, /* %J */
235 { NULL, 0, 0 }, /* %N */
236 { NULL, 0, 0 }, /* %O */
237 { NULL, 0, 0 }, /* %P */
238 { NULL, 0, 0 }, /* %R */
239 { NULL, 0, 0 }, /* %T */
240 { NULL, 0, 0 }, /* %V */
241 { NULL, 0, 0 }, /* Ac */
242 { NULL, 0, 0 }, /* Ao */
243 { NULL, 0, 0 }, /* Aq */
244 { NULL, TYPE_At, MDOCF_CHILD }, /* At */
245 { NULL, 0, 0 }, /* Bc */
246 { NULL, 0, 0 }, /* Bf */
247 { NULL, 0, 0 }, /* Bo */
248 { NULL, 0, 0 }, /* Bq */
249 { NULL, TYPE_Bsx, MDOCF_CHILD }, /* Bsx */
250 { NULL, TYPE_Bx, MDOCF_CHILD }, /* Bx */
251 { NULL, 0, 0 }, /* Db */
252 { NULL, 0, 0 }, /* Dc */
253 { NULL, 0, 0 }, /* Do */
254 { NULL, 0, 0 }, /* Dq */
255 { NULL, 0, 0 }, /* Ec */
256 { NULL, 0, 0 }, /* Ef */
257 { NULL, TYPE_Em, MDOCF_CHILD }, /* Em */
258 { NULL, 0, 0 }, /* Eo */
259 { NULL, TYPE_Fx, MDOCF_CHILD }, /* Fx */
260 { NULL, TYPE_Ms, MDOCF_CHILD }, /* Ms */
261 { NULL, 0, 0 }, /* No */
262 { NULL, 0, 0 }, /* Ns */
263 { NULL, TYPE_Nx, MDOCF_CHILD }, /* Nx */
264 { NULL, TYPE_Ox, MDOCF_CHILD }, /* Ox */
265 { NULL, 0, 0 }, /* Pc */
266 { NULL, 0, 0 }, /* Pf */
267 { NULL, 0, 0 }, /* Po */
268 { NULL, 0, 0 }, /* Pq */
269 { NULL, 0, 0 }, /* Qc */
270 { NULL, 0, 0 }, /* Ql */
271 { NULL, 0, 0 }, /* Qo */
272 { NULL, 0, 0 }, /* Qq */
273 { NULL, 0, 0 }, /* Re */
274 { NULL, 0, 0 }, /* Rs */
275 { NULL, 0, 0 }, /* Sc */
276 { NULL, 0, 0 }, /* So */
277 { NULL, 0, 0 }, /* Sq */
278 { NULL, 0, 0 }, /* Sm */
279 { NULL, 0, 0 }, /* Sx */
280 { NULL, TYPE_Sy, MDOCF_CHILD }, /* Sy */
281 { NULL, TYPE_Tn, MDOCF_CHILD }, /* Tn */
282 { NULL, 0, 0 }, /* Ux */
283 { NULL, 0, 0 }, /* Xc */
284 { NULL, 0, 0 }, /* Xo */
285 { parse_mdoc_head, TYPE_Fn, 0 }, /* Fo */
286 { NULL, 0, 0 }, /* Fc */
287 { NULL, 0, 0 }, /* Oo */
288 { NULL, 0, 0 }, /* Oc */
289 { NULL, 0, 0 }, /* Bk */
290 { NULL, 0, 0 }, /* Ek */
291 { NULL, 0, 0 }, /* Bt */
292 { NULL, 0, 0 }, /* Hf */
293 { NULL, 0, 0 }, /* Fr */
294 { NULL, 0, 0 }, /* Ud */
295 { NULL, TYPE_Lb, MDOCF_CHILD }, /* Lb */
296 { NULL, 0, 0 }, /* Lp */
297 { NULL, TYPE_Lk, MDOCF_CHILD }, /* Lk */
298 { NULL, TYPE_Mt, MDOCF_CHILD }, /* Mt */
299 { NULL, 0, 0 }, /* Brq */
300 { NULL, 0, 0 }, /* Bro */
301 { NULL, 0, 0 }, /* Brc */
302 { NULL, 0, 0 }, /* %C */
303 { NULL, 0, 0 }, /* Es */
304 { NULL, 0, 0 }, /* En */
305 { NULL, TYPE_Dx, MDOCF_CHILD }, /* Dx */
306 { NULL, 0, 0 }, /* %Q */
307 { NULL, 0, 0 }, /* br */
308 { NULL, 0, 0 }, /* sp */
309 { NULL, 0, 0 }, /* %U */
310 { NULL, 0, 0 }, /* Ta */
311 };
312
313 int
314 main(int argc, char *argv[])
315 {
316 char cwd[MAXPATHLEN];
317 int ch, rc, fd, i;
318 unsigned int index;
319 size_t j, sz;
320 const char *dir;
321 struct str *s;
322 struct mchars *mc;
323 struct manpaths dirs;
324 struct mparse *mp;
325 struct ohash_info ino_info, filename_info, str_info;
326
327 memset(stmts, 0, STMT__MAX * sizeof(sqlite3_stmt *));
328 memset(&dirs, 0, sizeof(struct manpaths));
329
330 ino_info.halloc = filename_info.halloc =
331 str_info.halloc = hash_halloc;
332 ino_info.hfree = filename_info.hfree =
333 str_info.hfree = hash_free;
334 ino_info.alloc = filename_info.alloc =
335 str_info.alloc = hash_alloc;
336
337 ino_info.key_offset = offsetof(struct of, id);
338 filename_info.key_offset = offsetof(struct of, file);
339 str_info.key_offset = offsetof(struct str, key);
340
341 progname = strrchr(argv[0], '/');
342 if (progname == NULL)
343 progname = argv[0];
344 else
345 ++progname;
346
347 /*
348 * Remember where we started by keeping a fd open to the origin
349 * path component: throughout this utility, we chdir() a lot to
350 * handle relative paths, and by doing this, we can return to
351 * the starting point.
352 */
353 if (NULL == getcwd(cwd, MAXPATHLEN)) {
354 perror(NULL);
355 return(EXIT_FAILURE);
356 } else if (-1 == (fd = open(cwd, O_RDONLY, 0))) {
357 perror(cwd);
358 return(EXIT_FAILURE);
359 }
360
361 /*
362 * We accept a few different invocations.
363 * The CHECKOP macro makes sure that invocation styles don't
364 * clobber each other.
365 */
366 #define CHECKOP(_op, _ch) do \
367 if (OP_DEFAULT != (_op)) { \
368 fprintf(stderr, "-%c: Conflicting option\n", (_ch)); \
369 goto usage; \
370 } while (/*CONSTCOND*/0)
371
372 dir = NULL;
373 op = OP_DEFAULT;
374
375 while (-1 != (ch = getopt(argc, argv, "aC:d:ntu:vW")))
376 switch (ch) {
377 case ('a'):
378 use_all = 1;
379 break;
380 case ('C'):
381 CHECKOP(op, ch);
382 dir = optarg;
383 op = OP_CONFFILE;
384 break;
385 case ('d'):
386 CHECKOP(op, ch);
387 dir = optarg;
388 op = OP_UPDATE;
389 break;
390 case ('n'):
391 nodb = 1;
392 break;
393 case ('t'):
394 CHECKOP(op, ch);
395 dup2(STDOUT_FILENO, STDERR_FILENO);
396 op = OP_TEST;
397 nodb = warnings = 1;
398 break;
399 case ('u'):
400 CHECKOP(op, ch);
401 dir = optarg;
402 op = OP_DELETE;
403 break;
404 case ('v'):
405 verb++;
406 break;
407 case ('W'):
408 warnings = 1;
409 break;
410 default:
411 goto usage;
412 }
413
414 argc -= optind;
415 argv += optind;
416
417 if (OP_CONFFILE == op && argc > 0) {
418 fprintf(stderr, "-C: Too many arguments\n");
419 goto usage;
420 }
421
422 rc = 1;
423 mp = mparse_alloc(MPARSE_AUTO,
424 MANDOCLEVEL_FATAL, NULL, NULL, NULL);
425 mc = mchars_alloc();
426
427 ohash_init(&strings, 6, &str_info);
428 ohash_init(&inos, 6, &ino_info);
429 ohash_init(&filenames, 6, &filename_info);
430
431 if (OP_UPDATE == op || OP_DELETE == op || OP_TEST == op) {
432 /*
433 * Force processing all files.
434 */
435 use_all = 1;
436 if (NULL == dir)
437 dir = cwd;
438 /*
439 * All of these deal with a specific directory.
440 * Jump into that directory then collect files specified
441 * on the command-line.
442 */
443 if (0 == path_reset(cwd, fd, dir))
444 goto out;
445 for (i = 0; i < argc; i++)
446 filescan(argv[i], dir);
447 if (0 == dbopen(dir, 1))
448 goto out;
449 if (OP_TEST != op)
450 dbprune(dir);
451 if (OP_DELETE != op)
452 rc = ofmerge(mc, mp, dir);
453 dbclose(dir, 1);
454 } else {
455 /*
456 * If we have arguments, use them as our manpaths.
457 * If we don't, grok from manpath(1) or however else
458 * manpath_parse() wants to do it.
459 */
460 if (argc > 0) {
461 dirs.paths = mandoc_calloc
462 (argc, sizeof(char *));
463 dirs.sz = (size_t)argc;
464 for (i = 0; i < argc; i++)
465 dirs.paths[i] = mandoc_strdup(argv[i]);
466 } else
467 manpath_parse(&dirs, dir, NULL, NULL);
468
469 /*
470 * First scan the tree rooted at a base directory.
471 * Then whak its database (if one exists), parse, and
472 * build up the database.
473 * Ignore zero-length directories and strip trailing
474 * slashes.
475 */
476 for (j = 0; j < dirs.sz; j++) {
477 sz = strlen(dirs.paths[j]);
478 if (sz && '/' == dirs.paths[j][sz - 1])
479 dirs.paths[j][--sz] = '\0';
480 if (0 == sz)
481 continue;
482 if (0 == path_reset(cwd, fd, dirs.paths[j]))
483 goto out;
484 if (0 == treescan(dirs.paths[j]))
485 goto out;
486 if (0 == path_reset(cwd, fd, dirs.paths[j]))
487 goto out;
488 if (0 == dbopen(dirs.paths[j], 0))
489 goto out;
490
491 /*
492 * Since we're opening up a new database, we can
493 * turn off synchronous mode for much better
494 * performance.
495 */
496 #ifndef __APPLE__
497 SQL_EXEC("PRAGMA synchronous = OFF");
498 #endif
499
500 if (0 == ofmerge(mc, mp, dirs.paths[j]))
501 goto out;
502 dbclose(dirs.paths[j], 0);
503 offree();
504 ohash_delete(&inos);
505 ohash_init(&inos, 6, &ino_info);
506 ohash_delete(&filenames);
507 ohash_init(&filenames, 6, &filename_info);
508 }
509 }
510 out:
511 close(fd);
512 manpath_free(&dirs);
513 mchars_free(mc);
514 mparse_free(mp);
515 for (s = ohash_first(&strings, &index);
516 NULL != s; s = ohash_next(&strings, &index)) {
517 if (s->utf8 != s->key)
518 free(s->utf8);
519 free(s);
520 }
521 ohash_delete(&strings);
522 ohash_delete(&inos);
523 ohash_delete(&filenames);
524 offree();
525 return(rc ? EXIT_SUCCESS : EXIT_FAILURE);
526 usage:
527 fprintf(stderr, "usage: %s [-anvW] [-C file]\n"
528 " %s [-anvW] dir ...\n"
529 " %s [-nvW] -d dir [file ...]\n"
530 " %s [-nvW] -u dir [file ...]\n"
531 " %s -t file ...\n",
532 progname, progname, progname,
533 progname, progname);
534
535 return(EXIT_FAILURE);
536 }
537
538 /*
539 * Scan a directory tree rooted at "base" for manpages.
540 * We use fts(), scanning directory parts along the way for clues to our
541 * section and architecture.
542 *
543 * If use_all has been specified, grok all files.
544 * If not, sanitise paths to the following:
545 *
546 * [./]man*[/<arch>]/<name>.<section>
547 * or
548 * [./]cat<section>[/<arch>]/<name>.0
549 *
550 * TODO: accomodate for multi-language directories.
551 */
552 static int
553 treescan(const char *base)
554 {
555 FTS *f;
556 FTSENT *ff;
557 int dform;
558 char *sec;
559 const char *dsec, *arch, *cp, *name, *path;
560 const char *argv[2];
561
562 argv[0] = ".";
563 argv[1] = (char *)NULL;
564
565 /*
566 * Walk through all components under the directory, using the
567 * logical descent of files.
568 */
569 f = fts_open((char * const *)argv, FTS_LOGICAL, NULL);
570 if (NULL == f) {
571 perror(base);
572 return(0);
573 }
574
575 dsec = arch = NULL;
576 dform = FORM_NONE;
577
578 while (NULL != (ff = fts_read(f))) {
579 path = ff->fts_path + 2;
580 /*
581 * If we're a regular file, add an "of" by using the
582 * stored directory data and handling the filename.
583 * Disallow duplicate (hard-linked) files.
584 */
585 if (FTS_F == ff->fts_info) {
586 if ( ! use_all && ff->fts_level < 2) {
587 if (warnings)
588 say(base, path, "Extraneous file");
589 continue;
590 } else if (inocheck(ff->fts_statp)) {
591 if (warnings)
592 say(base, path, "Duplicate file");
593 continue;
594 }
595
596 cp = ff->fts_name;
597
598 if (0 == strcmp(cp, "mandocdb.db")) {
599 if (warnings)
600 say(base, path, "Skip database");
601 continue;
602 } else if (NULL != (cp = strrchr(cp, '.'))) {
603 if (0 == strcmp(cp + 1, "html")) {
604 if (warnings)
605 say(base, path, "Skip html");
606 continue;
607 } else if (0 == strcmp(cp + 1, "gz")) {
608 if (warnings)
609 say(base, path, "Skip gz");
610 continue;
611 } else if (0 == strcmp(cp + 1, "ps")) {
612 if (warnings)
613 say(base, path, "Skip ps");
614 continue;
615 } else if (0 == strcmp(cp + 1, "pdf")) {
616 if (warnings)
617 say(base, path, "Skip pdf");
618 continue;
619 }
620 }
621
622 if (NULL != (sec = strrchr(ff->fts_name, '.'))) {
623 *sec = '\0';
624 sec = stradd(sec + 1);
625 }
626 name = stradd(ff->fts_name);
627 ofadd(base, dform, path,
628 name, dsec, sec, arch, ff->fts_statp);
629 continue;
630 } else if (FTS_D != ff->fts_info &&
631 FTS_DP != ff->fts_info)
632 continue;
633
634 switch (ff->fts_level) {
635 case (0):
636 /* Ignore the root directory. */
637 break;
638 case (1):
639 /*
640 * This might contain manX/ or catX/.
641 * Try to infer this from the name.
642 * If we're not in use_all, enforce it.
643 */
644 dsec = NULL;
645 dform = FORM_NONE;
646 cp = ff->fts_name;
647 if (FTS_DP == ff->fts_info)
648 break;
649
650 if (0 == strncmp(cp, "man", 3)) {
651 dform = FORM_SRC;
652 dsec = stradd(cp + 3);
653 } else if (0 == strncmp(cp, "cat", 3)) {
654 dform = FORM_CAT;
655 dsec = stradd(cp + 3);
656 }
657
658 if (NULL != dsec || use_all)
659 break;
660
661 if (warnings)
662 say(base, path, "Unknown directory part");
663 fts_set(f, ff, FTS_SKIP);
664 break;
665 case (2):
666 /*
667 * Possibly our architecture.
668 * If we're descending, keep tabs on it.
669 */
670 arch = NULL;
671 if (FTS_DP != ff->fts_info && NULL != dsec)
672 arch = stradd(ff->fts_name);
673 break;
674 default:
675 if (FTS_DP == ff->fts_info || use_all)
676 break;
677 if (warnings)
678 say(base, path, "Extraneous directory part");
679 fts_set(f, ff, FTS_SKIP);
680 break;
681 }
682 }
683
684 fts_close(f);
685 return(1);
686 }
687
688 /*
689 * Add a file to the file vector.
690 * Do not verify that it's a "valid" looking manpage (we'll do that
691 * later).
692 *
693 * Try to infer the manual section, architecture, and page name from the
694 * path, assuming it looks like
695 *
696 * [./]man*[/<arch>]/<name>.<section>
697 * or
698 * [./]cat<section>[/<arch>]/<name>.0
699 *
700 * Stuff this information directly into the "of" vector.
701 * See treescan() for the fts(3) version of this.
702 */
703 static void
704 filescan(const char *file, const char *base)
705 {
706 const char *sec, *arch, *name, *dsec;
707 char *p, *start, *buf;
708 int dform;
709 struct stat st;
710
711 assert(use_all);
712
713 if (0 == strncmp(file, "./", 2))
714 file += 2;
715
716 if (-1 == stat(file, &st)) {
717 if (warnings)
718 say(base, file, "%s", strerror(errno));
719 return;
720 } else if ( ! (S_IFREG & st.st_mode)) {
721 if (warnings)
722 say(base, file, "Not a regular file");
723 return;
724 } else if (inocheck(&st)) {
725 if (warnings)
726 say(base, file, "Duplicate file");
727 return;
728 }
729
730 buf = mandoc_strdup(file);
731 start = buf;
732 sec = arch = name = dsec = NULL;
733 dform = FORM_NONE;
734
735 /*
736 * First try to guess our directory structure.
737 * If we find a separator, try to look for man* or cat*.
738 * If we find one of these and what's underneath is a directory,
739 * assume it's an architecture.
740 */
741 if (NULL != (p = strchr(start, '/'))) {
742 *p++ = '\0';
743 if (0 == strncmp(start, "man", 3)) {
744 dform = FORM_SRC;
745 dsec = start + 3;
746 } else if (0 == strncmp(start, "cat", 3)) {
747 dform = FORM_CAT;
748 dsec = start + 3;
749 }
750
751 start = p;
752 if (NULL != dsec && NULL != (p = strchr(start, '/'))) {
753 *p++ = '\0';
754 arch = start;
755 start = p;
756 }
757 }
758
759 /*
760 * Now check the file suffix.
761 * Suffix of `.0' indicates a catpage, `.1-9' is a manpage.
762 */
763 p = strrchr(start, '\0');
764 while (p-- > start && '/' != *p && '.' != *p)
765 /* Loop. */ ;
766
767 if ('.' == *p) {
768 *p++ = '\0';
769 sec = p;
770 }
771
772 /*
773 * Now try to parse the name.
774 * Use the filename portion of the path.
775 */
776 name = start;
777 if (NULL != (p = strrchr(start, '/'))) {
778 name = p + 1;
779 *p = '\0';
780 }
781
782 ofadd(base, dform, file, name, dsec, sec, arch, &st);
783 free(buf);
784 }
785
786 /*
787 * See fileadd().
788 */
789 static int
790 filecheck(const char *name)
791 {
792 unsigned int index;
793
794 index = ohash_qlookup(&filenames, name);
795 return(NULL != ohash_find(&filenames, index));
796 }
797
798 /*
799 * Use the standard hashing mechanism (K&R) to see if the given filename
800 * already exists.
801 */
802 static void
803 fileadd(struct of *of)
804 {
805 unsigned int index;
806
807 index = ohash_qlookup(&filenames, of->file);
808 assert(NULL == ohash_find(&filenames, index));
809 ohash_insert(&filenames, index, of);
810 }
811
812 /*
813 * See inoadd().
814 */
815 static int
816 inocheck(const struct stat *st)
817 {
818 struct id id;
819 uint32_t hash;
820 unsigned int index;
821
822 memset(&id, 0, sizeof(id));
823 id.ino = hash = st->st_ino;
824 id.dev = st->st_dev;
825 index = ohash_lookup_memory
826 (&inos, (char *)&id, sizeof(id), hash);
827
828 return(NULL != ohash_find(&inos, index));
829 }
830
831 /*
832 * The hashing function used here is quite simple: simply take the inode
833 * and use uint32_t of its bits.
834 * Then when we do the lookup, use both the inode and device identifier.
835 */
836 static void
837 inoadd(const struct stat *st, struct of *of)
838 {
839 uint32_t hash;
840 unsigned int index;
841
842 of->id.ino = hash = st->st_ino;
843 of->id.dev = st->st_dev;
844 index = ohash_lookup_memory
845 (&inos, (char *)&of->id, sizeof(of->id), hash);
846
847 assert(NULL == ohash_find(&inos, index));
848 ohash_insert(&inos, index, of);
849 }
850
851 static void
852 ofadd(const char *base, int dform, const char *file,
853 const char *name, const char *dsec, const char *sec,
854 const char *arch, const struct stat *st)
855 {
856 struct of *of;
857 int sform;
858
859 assert(NULL != file);
860
861 if (NULL == name)
862 name = "";
863 if (NULL == sec)
864 sec = "";
865 if (NULL == dsec)
866 dsec = "";
867 if (NULL == arch)
868 arch = "";
869
870 sform = FORM_NONE;
871 if (NULL != sec && *sec <= '9' && *sec >= '1')
872 sform = FORM_SRC;
873 else if (NULL != sec && *sec == '0') {
874 sec = dsec;
875 sform = FORM_CAT;
876 }
877
878 of = mandoc_calloc(1, sizeof(struct of));
879 strlcpy(of->file, file, MAXPATHLEN);
880 of->name = name;
881 of->sec = sec;
882 of->dsec = dsec;
883 of->arch = arch;
884 of->sform = sform;
885 of->dform = dform;
886 of->next = ofs;
887 ofs = of;
888
889 /*
890 * Add to unique identifier hash.
891 * Then if it's a source manual and we're going to use source in
892 * favour of catpages, add it to that hash.
893 */
894 inoadd(st, of);
895 fileadd(of);
896 }
897
898 static void
899 offree(void)
900 {
901 struct of *of;
902
903 while (NULL != (of = ofs)) {
904 ofs = of->next;
905 free(of);
906 }
907 }
908
909 /*
910 * Run through the files in the global vector "ofs" and add them to the
911 * database specified in "base".
912 *
913 * This handles the parsing scheme itself, using the cues of directory
914 * and filename to determine whether the file is parsable or not.
915 */
916 static int
917 ofmerge(struct mchars *mc, struct mparse *mp, const char *base)
918 {
919 int form;
920 size_t sz;
921 struct mdoc *mdoc;
922 struct man *man;
923 char buf[MAXPATHLEN];
924 char *bufp;
925 const char *msec, *march, *mtitle, *cp;
926 struct of *of;
927 enum mandoclevel lvl;
928
929 for (of = ofs; NULL != of; of = of->next) {
930 /*
931 * If we're a catpage (as defined by our path), then see
932 * if a manpage exists by the same name (ignoring the
933 * suffix).
934 * If it does, then we want to use it instead of our
935 * own.
936 */
937 if ( ! use_all && FORM_CAT == of->dform) {
938 sz = strlcpy(buf, of->file, MAXPATHLEN);
939 if (sz >= MAXPATHLEN) {
940 if (warnings)
941 say(base, of->file,
942 "Filename too long");
943 continue;
944 }
945 bufp = strstr(buf, "cat");
946 assert(NULL != bufp);
947 memcpy(bufp, "man", 3);
948 if (NULL != (bufp = strrchr(buf, '.')))
949 *++bufp = '\0';
950 strlcat(buf, of->dsec, MAXPATHLEN);
951 if (filecheck(buf)) {
952 if (warnings)
953 say(base, of->file, "Man "
954 "source exists: %s", buf);
955 continue;
956 }
957 }
958
959 words = NULL;
960 mparse_reset(mp);
961 mdoc = NULL;
962 man = NULL;
963 form = 0;
964 msec = of->dsec;
965 march = of->arch;
966 mtitle = of->name;
967
968 /*
969 * Try interpreting the file as mdoc(7) or man(7)
970 * source code, unless it is already known to be
971 * formatted. Fall back to formatted mode.
972 */
973 if (FORM_SRC == of->dform || FORM_SRC == of->sform) {
974 lvl = mparse_readfd(mp, -1, of->file);
975 if (lvl < MANDOCLEVEL_FATAL)
976 mparse_result(mp, &mdoc, &man);
977 }
978
979 if (NULL != mdoc) {
980 form = 1;
981 msec = mdoc_meta(mdoc)->msec;
982 march = mdoc_meta(mdoc)->arch;
983 mtitle = mdoc_meta(mdoc)->title;
984 } else if (NULL != man) {
985 form = 1;
986 msec = man_meta(man)->msec;
987 march = "";
988 mtitle = man_meta(man)->title;
989 }
990
991 if (NULL == msec)
992 msec = "";
993 if (NULL == march)
994 march = "";
995 if (NULL == mtitle)
996 mtitle = "";
997
998 /*
999 * Check whether the manual section given in a file
1000 * agrees with the directory where the file is located.
1001 * Some manuals have suffixes like (3p) on their
1002 * section number either inside the file or in the
1003 * directory name, some are linked into more than one
1004 * section, like encrypt(1) = makekey(8). Do not skip
1005 * manuals for such reasons.
1006 */
1007 if (warnings && !use_all && form &&
1008 strcasecmp(msec, of->dsec))
1009 say(base, of->file, "Section \"%s\" "
1010 "manual in %s directory",
1011 msec, of->dsec);
1012
1013 /*
1014 * Manual page directories exist for each kernel
1015 * architecture as returned by machine(1).
1016 * However, many manuals only depend on the
1017 * application architecture as returned by arch(1).
1018 * For example, some (2/ARM) manuals are shared
1019 * across the "armish" and "zaurus" kernel
1020 * architectures.
1021 * A few manuals are even shared across completely
1022 * different architectures, for example fdformat(1)
1023 * on amd64, i386, sparc, and sparc64.
1024 * Thus, warn about architecture mismatches,
1025 * but don't skip manuals for this reason.
1026 */
1027 if (warnings && !use_all && strcasecmp(march, of->arch))
1028 say(base, of->file, "Architecture \"%s\" "
1029 "manual in \"%s\" directory",
1030 march, of->arch);
1031
1032 putkey(of, of->name, TYPE_Nm);
1033
1034 if (NULL != mdoc) {
1035 if (NULL != (cp = mdoc_meta(mdoc)->name))
1036 putkey(of, cp, TYPE_Nm);
1037 parse_mdoc(of, mdoc_node(mdoc));
1038 } else if (NULL != man)
1039 parse_man(of, man_node(man));
1040 else
1041 parse_catpage(of, base);
1042
1043 dbindex(mc, form, of, base);
1044 }
1045
1046 return(1);
1047 }
1048
1049 static void
1050 parse_catpage(struct of *of, const char *base)
1051 {
1052 FILE *stream;
1053 char *line, *p, *title;
1054 size_t len, plen, titlesz;
1055
1056 if (NULL == (stream = fopen(of->file, "r"))) {
1057 if (warnings)
1058 say(base, of->file, "%s", strerror(errno));
1059 return;
1060 }
1061
1062 /* Skip to first blank line. */
1063
1064 while (NULL != (line = fgetln(stream, &len)))
1065 if ('\n' == *line)
1066 break;
1067
1068 /*
1069 * Assume the first line that is not indented
1070 * is the first section header. Skip to it.
1071 */
1072
1073 while (NULL != (line = fgetln(stream, &len)))
1074 if ('\n' != *line && ' ' != *line)
1075 break;
1076
1077 /*
1078 * Read up until the next section into a buffer.
1079 * Strip the leading and trailing newline from each read line,
1080 * appending a trailing space.
1081 * Ignore empty (whitespace-only) lines.
1082 */
1083
1084 titlesz = 0;
1085 title = NULL;
1086
1087 while (NULL != (line = fgetln(stream, &len))) {
1088 if (' ' != *line || '\n' != line[len - 1])
1089 break;
1090 while (len > 0 && isspace((unsigned char)*line)) {
1091 line++;
1092 len--;
1093 }
1094 if (1 == len)
1095 continue;
1096 title = mandoc_realloc(title, titlesz + len);
1097 memcpy(title + titlesz, line, len);
1098 titlesz += len;
1099 title[titlesz - 1] = ' ';
1100 }
1101
1102 /*
1103 * If no page content can be found, or the input line
1104 * is already the next section header, or there is no
1105 * trailing newline, reuse the page title as the page
1106 * description.
1107 */
1108
1109 if (NULL == title || '\0' == *title) {
1110 if (warnings)
1111 say(base, of->file, "Cannot find NAME section");
1112 fclose(stream);
1113 free(title);
1114 return;
1115 }
1116
1117 title = mandoc_realloc(title, titlesz + 1);
1118 title[titlesz] = '\0';
1119
1120 /*
1121 * Skip to the first dash.
1122 * Use the remaining line as the description (no more than 70
1123 * bytes).
1124 */
1125
1126 if (NULL != (p = strstr(title, "- "))) {
1127 for (p += 2; ' ' == *p || '\b' == *p; p++)
1128 /* Skip to next word. */ ;
1129 } else {
1130 if (warnings)
1131 say(base, of->file, "No dash in title line");
1132 p = title;
1133 }
1134
1135 plen = strlen(p);
1136
1137 /* Strip backspace-encoding from line. */
1138
1139 while (NULL != (line = memchr(p, '\b', plen))) {
1140 len = line - p;
1141 if (0 == len) {
1142 memmove(line, line + 1, plen--);
1143 continue;
1144 }
1145 memmove(line - 1, line + 1, plen - len);
1146 plen -= 2;
1147 }
1148
1149 of->desc = stradd(p);
1150 putkey(of, p, TYPE_Nd);
1151 fclose(stream);
1152 free(title);
1153 }
1154
1155 /*
1156 * Put a type/word pair into the word database for this particular file.
1157 */
1158 static void
1159 putkey(const struct of *of, const char *value, uint64_t type)
1160 {
1161
1162 assert(NULL != value);
1163 wordaddbuf(of, value, strlen(value), type);
1164 }
1165
1166 /*
1167 * Like putkey() but for unterminated strings.
1168 */
1169 static void
1170 putkeys(const struct of *of, const char *value, int sz, uint64_t type)
1171 {
1172
1173 wordaddbuf(of, value, sz, type);
1174 }
1175
1176 /*
1177 * Grok all nodes at or below a certain mdoc node into putkey().
1178 */
1179 static void
1180 putmdockey(const struct of *of, const struct mdoc_node *n, uint64_t m)
1181 {
1182
1183 for ( ; NULL != n; n = n->next) {
1184 if (NULL != n->child)
1185 putmdockey(of, n->child, m);
1186 if (MDOC_TEXT == n->type)
1187 putkey(of, n->string, m);
1188 }
1189 }
1190
1191 static int
1192 parse_man(struct of *of, const struct man_node *n)
1193 {
1194 const struct man_node *head, *body;
1195 char *start, *sv, *title;
1196 char byte;
1197 size_t sz, titlesz;
1198
1199 if (NULL == n)
1200 return(0);
1201
1202 /*
1203 * We're only searching for one thing: the first text child in
1204 * the BODY of a NAME section. Since we don't keep track of
1205 * sections in -man, run some hoops to find out whether we're in
1206 * the correct section or not.
1207 */
1208
1209 if (MAN_BODY == n->type && MAN_SH == n->tok) {
1210 body = n;
1211 assert(body->parent);
1212 if (NULL != (head = body->parent->head) &&
1213 1 == head->nchild &&
1214 NULL != (head = (head->child)) &&
1215 MAN_TEXT == head->type &&
1216 0 == strcmp(head->string, "NAME") &&
1217 NULL != (body = body->child) &&
1218 MAN_TEXT == body->type) {
1219
1220 title = NULL;
1221 titlesz = 0;
1222
1223 /*
1224 * Suck the entire NAME section into memory.
1225 * Yes, we might run away.
1226 * But too many manuals have big, spread-out
1227 * NAME sections over many lines.
1228 */
1229
1230 for ( ; NULL != body; body = body->next) {
1231 if (MAN_TEXT != body->type)
1232 break;
1233 if (0 == (sz = strlen(body->string)))
1234 continue;
1235 title = mandoc_realloc
1236 (title, titlesz + sz + 1);
1237 memcpy(title + titlesz, body->string, sz);
1238 titlesz += sz + 1;
1239 title[titlesz - 1] = ' ';
1240 }
1241 if (NULL == title)
1242 return(1);
1243
1244 title = mandoc_realloc(title, titlesz + 1);
1245 title[titlesz] = '\0';
1246
1247 /* Skip leading space. */
1248
1249 sv = title;
1250 while (isspace((unsigned char)*sv))
1251 sv++;
1252
1253 if (0 == (sz = strlen(sv))) {
1254 free(title);
1255 return(1);
1256 }
1257
1258 /* Erase trailing space. */
1259
1260 start = &sv[sz - 1];
1261 while (start > sv && isspace((unsigned char)*start))
1262 *start-- = '\0';
1263
1264 if (start == sv) {
1265 free(title);
1266 return(1);
1267 }
1268
1269 start = sv;
1270
1271 /*
1272 * Go through a special heuristic dance here.
1273 * Conventionally, one or more manual names are
1274 * comma-specified prior to a whitespace, then a
1275 * dash, then a description. Try to puzzle out
1276 * the name parts here.
1277 */
1278
1279 for ( ;; ) {
1280 sz = strcspn(start, " ,");
1281 if ('\0' == start[sz])
1282 break;
1283
1284 byte = start[sz];
1285 start[sz] = '\0';
1286
1287 putkey(of, start, TYPE_Nm);
1288
1289 if (' ' == byte) {
1290 start += sz + 1;
1291 break;
1292 }
1293
1294 assert(',' == byte);
1295 start += sz + 1;
1296 while (' ' == *start)
1297 start++;
1298 }
1299
1300 if (sv == start) {
1301 putkey(of, start, TYPE_Nm);
1302 free(title);
1303 return(1);
1304 }
1305
1306 while (isspace((unsigned char)*start))
1307 start++;
1308
1309 if (0 == strncmp(start, "-", 1))
1310 start += 1;
1311 else if (0 == strncmp(start, "\\-\\-", 4))
1312 start += 4;
1313 else if (0 == strncmp(start, "\\-", 2))
1314 start += 2;
1315 else if (0 == strncmp(start, "\\(en", 4))
1316 start += 4;
1317 else if (0 == strncmp(start, "\\(em", 4))
1318 start += 4;
1319
1320 while (' ' == *start)
1321 start++;
1322
1323 assert(NULL == of->desc);
1324 of->desc = stradd(start);
1325 putkey(of, start, TYPE_Nd);
1326 free(title);
1327 return(1);
1328 }
1329 }
1330
1331 for (n = n->child; n; n = n->next)
1332 if (parse_man(of, n))
1333 return(1);
1334
1335 return(0);
1336 }
1337
1338 static void
1339 parse_mdoc(struct of *of, const struct mdoc_node *n)
1340 {
1341
1342 assert(NULL != n);
1343 for (n = n->child; NULL != n; n = n->next) {
1344 switch (n->type) {
1345 case (MDOC_ELEM):
1346 /* FALLTHROUGH */
1347 case (MDOC_BLOCK):
1348 /* FALLTHROUGH */
1349 case (MDOC_HEAD):
1350 /* FALLTHROUGH */
1351 case (MDOC_BODY):
1352 /* FALLTHROUGH */
1353 case (MDOC_TAIL):
1354 if (NULL != mdocs[n->tok].fp)
1355 if (0 == (*mdocs[n->tok].fp)(of, n))
1356 break;
1357
1358 if (MDOCF_CHILD & mdocs[n->tok].flags)
1359 putmdockey(of, n->child, mdocs[n->tok].mask);
1360 break;
1361 default:
1362 assert(MDOC_ROOT != n->type);
1363 continue;
1364 }
1365 if (NULL != n->child)
1366 parse_mdoc(of, n);
1367 }
1368 }
1369
1370 static int
1371 parse_mdoc_Fd(struct of *of, const struct mdoc_node *n)
1372 {
1373 const char *start, *end;
1374 size_t sz;
1375
1376 if (SEC_SYNOPSIS != n->sec ||
1377 NULL == (n = n->child) ||
1378 MDOC_TEXT != n->type)
1379 return(0);
1380
1381 /*
1382 * Only consider those `Fd' macro fields that begin with an
1383 * "inclusion" token (versus, e.g., #define).
1384 */
1385
1386 if (strcmp("#include", n->string))
1387 return(0);
1388
1389 if (NULL == (n = n->next) || MDOC_TEXT != n->type)
1390 return(0);
1391
1392 /*
1393 * Strip away the enclosing angle brackets and make sure we're
1394 * not zero-length.
1395 */
1396
1397 start = n->string;
1398 if ('<' == *start || '"' == *start)
1399 start++;
1400
1401 if (0 == (sz = strlen(start)))
1402 return(0);
1403
1404 end = &start[(int)sz - 1];
1405 if ('>' == *end || '"' == *end)
1406 end--;
1407
1408 if (end > start)
1409 putkeys(of, start, end - start + 1, TYPE_In);
1410 return(1);
1411 }
1412
1413 static int
1414 parse_mdoc_In(struct of *of, const struct mdoc_node *n)
1415 {
1416
1417 if (NULL != n->child && MDOC_TEXT == n->child->type)
1418 return(0);
1419
1420 putkey(of, n->child->string, TYPE_In);
1421 return(1);
1422 }
1423
1424 static int
1425 parse_mdoc_Fn(struct of *of, const struct mdoc_node *n)
1426 {
1427 const char *cp;
1428
1429 if (NULL == (n = n->child) || MDOC_TEXT != n->type)
1430 return(0);
1431
1432 /*
1433 * Parse: .Fn "struct type *name" "char *arg".
1434 * First strip away pointer symbol.
1435 * Then store the function name, then type.
1436 * Finally, store the arguments.
1437 */
1438
1439 if (NULL == (cp = strrchr(n->string, ' ')))
1440 cp = n->string;
1441
1442 while ('*' == *cp)
1443 cp++;
1444
1445 putkey(of, cp, TYPE_Fn);
1446
1447 if (n->string < cp)
1448 putkeys(of, n->string, cp - n->string, TYPE_Ft);
1449
1450 for (n = n->next; NULL != n; n = n->next)
1451 if (MDOC_TEXT == n->type)
1452 putkey(of, n->string, TYPE_Fa);
1453
1454 return(0);
1455 }
1456
1457 static int
1458 parse_mdoc_St(struct of *of, const struct mdoc_node *n)
1459 {
1460
1461 if (NULL == n->child || MDOC_TEXT != n->child->type)
1462 return(0);
1463
1464 putkey(of, n->child->string, TYPE_St);
1465 return(1);
1466 }
1467
1468 static int
1469 parse_mdoc_Xr(struct of *of, const struct mdoc_node *n)
1470 {
1471
1472 if (NULL == (n = n->child))
1473 return(0);
1474
1475 putkey(of, n->string, TYPE_Xr);
1476 return(1);
1477 }
1478
1479 static int
1480 parse_mdoc_Nd(struct of *of, const struct mdoc_node *n)
1481 {
1482 size_t sz;
1483 char *sv, *desc;
1484
1485 if (MDOC_BODY != n->type)
1486 return(0);
1487
1488 /*
1489 * Special-case the `Nd' because we need to put the description
1490 * into the document table.
1491 */
1492
1493 desc = NULL;
1494 for (n = n->child; NULL != n; n = n->next) {
1495 if (MDOC_TEXT == n->type) {
1496 sz = strlen(n->string) + 1;
1497 if (NULL != (sv = desc))
1498 sz += strlen(desc) + 1;
1499 desc = mandoc_realloc(desc, sz);
1500 if (NULL != sv)
1501 strlcat(desc, " ", sz);
1502 else
1503 *desc = '\0';
1504 strlcat(desc, n->string, sz);
1505 }
1506 if (NULL != n->child)
1507 parse_mdoc_Nd(of, n);
1508 }
1509
1510 of->desc = NULL != desc ? stradd(desc) : NULL;
1511 free(desc);
1512 return(1);
1513 }
1514
1515 static int
1516 parse_mdoc_Nm(struct of *of, const struct mdoc_node *n)
1517 {
1518
1519 if (SEC_NAME == n->sec)
1520 return(1);
1521 else if (SEC_SYNOPSIS != n->sec || MDOC_HEAD != n->type)
1522 return(0);
1523
1524 return(1);
1525 }
1526
1527 static int
1528 parse_mdoc_Sh(struct of *of, const struct mdoc_node *n)
1529 {
1530
1531 return(SEC_CUSTOM == n->sec && MDOC_HEAD == n->type);
1532 }
1533
1534 static int
1535 parse_mdoc_head(struct of *of, const struct mdoc_node *n)
1536 {
1537
1538 return(MDOC_HEAD == n->type);
1539 }
1540
1541 static int
1542 parse_mdoc_body(struct of *of, const struct mdoc_node *n)
1543 {
1544
1545 return(MDOC_BODY == n->type);
1546 }
1547
1548 /*
1549 * See straddbuf().
1550 */
1551 static char *
1552 stradd(const char *cp)
1553 {
1554
1555 return(straddbuf(cp, strlen(cp)));
1556 }
1557
1558 /*
1559 * This looks up or adds a string to the string table.
1560 * The string table is a table of all strings encountered during parse
1561 * or file scan.
1562 * In using it, we avoid having thousands of (e.g.) "cat1" string
1563 * allocations for the "of" table.
1564 * We also have a layer atop the string table for keeping track of words
1565 * in a parse sequence (see wordaddbuf()).
1566 */
1567 static char *
1568 straddbuf(const char *cp, size_t sz)
1569 {
1570 struct str *s;
1571 unsigned int index;
1572 const char *end;
1573
1574 if (NULL != (s = hashget(cp, sz)))
1575 return(s->key);
1576
1577 s = mandoc_calloc(sizeof(struct str) + sz + 1, 1);
1578 memcpy(s->key, cp, sz);
1579
1580 end = cp + sz;
1581 index = ohash_qlookupi(&strings, cp, &end);
1582 assert(NULL == ohash_find(&strings, index));
1583 ohash_insert(&strings, index, s);
1584 return(s->key);
1585 }
1586
1587 static struct str *
1588 hashget(const char *cp, size_t sz)
1589 {
1590 unsigned int index;
1591 const char *end;
1592
1593 end = cp + sz;
1594 index = ohash_qlookupi(&strings, cp, &end);
1595 return(ohash_find(&strings, index));
1596 }
1597
1598 /*
1599 * Add a word to the current parse sequence.
1600 * Within the hashtable of strings, we maintain a list of strings that
1601 * are currently indexed.
1602 * Each of these ("words") has a bitmask modified within the parse.
1603 * When we finish a parse, we'll dump the list, then remove the head
1604 * entry -- since the next parse will have a new "of", it can keep track
1605 * of its entries without conflict.
1606 */
1607 static void
1608 wordaddbuf(const struct of *of,
1609 const char *cp, size_t sz, uint64_t v)
1610 {
1611 struct str *s;
1612 unsigned int index;
1613 const char *end;
1614
1615 if (0 == sz)
1616 return;
1617
1618 s = hashget(cp, sz);
1619
1620 if (NULL != s && of == s->of) {
1621 s->mask |= v;
1622 return;
1623 } else if (NULL == s) {
1624 s = mandoc_calloc(sizeof(struct str) + sz + 1, 1);
1625 memcpy(s->key, cp, sz);
1626 end = cp + sz;
1627 index = ohash_qlookupi(&strings, cp, &end);
1628 assert(NULL == ohash_find(&strings, index));
1629 ohash_insert(&strings, index, s);
1630 }
1631
1632 s->next = words;
1633 s->of = of;
1634 s->mask = v;
1635 words = s;
1636 }
1637
1638 /*
1639 * Take a Unicode codepoint and produce its UTF-8 encoding.
1640 * This isn't the best way to do this, but it works.
1641 * The magic numbers are from the UTF-8 packaging.
1642 * They're not as scary as they seem: read the UTF-8 spec for details.
1643 */
1644 static size_t
1645 utf8(unsigned int cp, char out[7])
1646 {
1647 size_t rc;
1648
1649 rc = 0;
1650 if (cp <= 0x0000007F) {
1651 rc = 1;
1652 out[0] = (char)cp;
1653 } else if (cp <= 0x000007FF) {
1654 rc = 2;
1655 out[0] = (cp >> 6 & 31) | 192;
1656 out[1] = (cp & 63) | 128;
1657 } else if (cp <= 0x0000FFFF) {
1658 rc = 3;
1659 out[0] = (cp >> 12 & 15) | 224;
1660 out[1] = (cp >> 6 & 63) | 128;
1661 out[2] = (cp & 63) | 128;
1662 } else if (cp <= 0x001FFFFF) {
1663 rc = 4;
1664 out[0] = (cp >> 18 & 7) | 240;
1665 out[1] = (cp >> 12 & 63) | 128;
1666 out[2] = (cp >> 6 & 63) | 128;
1667 out[3] = (cp & 63) | 128;
1668 } else if (cp <= 0x03FFFFFF) {
1669 rc = 5;
1670 out[0] = (cp >> 24 & 3) | 248;
1671 out[1] = (cp >> 18 & 63) | 128;
1672 out[2] = (cp >> 12 & 63) | 128;
1673 out[3] = (cp >> 6 & 63) | 128;
1674 out[4] = (cp & 63) | 128;
1675 } else if (cp <= 0x7FFFFFFF) {
1676 rc = 6;
1677 out[0] = (cp >> 30 & 1) | 252;
1678 out[1] = (cp >> 24 & 63) | 128;
1679 out[2] = (cp >> 18 & 63) | 128;
1680 out[3] = (cp >> 12 & 63) | 128;
1681 out[4] = (cp >> 6 & 63) | 128;
1682 out[5] = (cp & 63) | 128;
1683 } else
1684 return(0);
1685
1686 out[rc] = '\0';
1687 return(rc);
1688 }
1689
1690 /*
1691 * Store the UTF-8 version of a key, or alias the pointer if the key has
1692 * no UTF-8 transcription marks in it.
1693 */
1694 static void
1695 utf8key(struct mchars *mc, struct str *key)
1696 {
1697 size_t sz, bsz, pos;
1698 char utfbuf[7], res[5];
1699 char *buf;
1700 const char *seq, *cpp, *val;
1701 int len, u;
1702 enum mandoc_esc esc;
1703
1704 assert(NULL == key->utf8);
1705
1706 res[0] = '\\';
1707 res[1] = '\t';
1708 res[2] = ASCII_NBRSP;
1709 res[3] = ASCII_HYPH;
1710 res[4] = '\0';
1711
1712 val = key->key;
1713 bsz = strlen(val);
1714
1715 /*
1716 * Pre-check: if we have no stop-characters, then set the
1717 * pointer as ourselvse and get out of here.
1718 */
1719 if (strcspn(val, res) == bsz) {
1720 key->utf8 = key->key;
1721 return;
1722 }
1723
1724 /* Pre-allocate by the length of the input */
1725
1726 buf = mandoc_malloc(++bsz);
1727 pos = 0;
1728
1729 while ('\0' != *val) {
1730 /*
1731 * Halt on the first escape sequence.
1732 * This also halts on the end of string, in which case
1733 * we just copy, fallthrough, and exit the loop.
1734 */
1735 if ((sz = strcspn(val, res)) > 0) {
1736 memcpy(&buf[pos], val, sz);
1737 pos += sz;
1738 val += sz;
1739 }
1740
1741 if (ASCII_HYPH == *val) {
1742 buf[pos++] = '-';
1743 val++;
1744 continue;
1745 } else if ('\t' == *val || ASCII_NBRSP == *val) {
1746 buf[pos++] = ' ';
1747 val++;
1748 continue;
1749 } else if ('\\' != *val)
1750 break;
1751
1752 /* Read past the slash. */
1753
1754 val++;
1755 u = 0;
1756
1757 /*
1758 * Parse the escape sequence and see if it's a
1759 * predefined character or special character.
1760 */
1761 esc = mandoc_escape
1762 ((const char **)&val, &seq, &len);
1763 if (ESCAPE_ERROR == esc)
1764 break;
1765
1766 if (ESCAPE_SPECIAL != esc)
1767 continue;
1768 if (0 == (u = mchars_spec2cp(mc, seq, len)))
1769 continue;
1770
1771 /*
1772 * If we have a Unicode codepoint, try to convert that
1773 * to a UTF-8 byte string.
1774 */
1775 cpp = utfbuf;
1776 if (0 == (sz = utf8(u, utfbuf)))
1777 continue;
1778
1779 /* Copy the rendered glyph into the stream. */
1780
1781 sz = strlen(cpp);
1782 bsz += sz;
1783
1784 buf = mandoc_realloc(buf, bsz);
1785
1786 memcpy(&buf[pos], cpp, sz);
1787 pos += sz;
1788 }
1789
1790 buf[pos] = '\0';
1791 key->utf8 = buf;
1792 }
1793
1794 /*
1795 * Flush the current page's terms (and their bits) into the database.
1796 * Wrap the entire set of additions in a transaction to make sqlite be a
1797 * little faster.
1798 * Also, UTF-8-encode the description at the last possible moment.
1799 */
1800 static void
1801 dbindex(struct mchars *mc, int form,
1802 const struct of *of, const char *base)
1803 {
1804 struct str *key;
1805 const char *desc;
1806 int64_t recno;
1807 size_t i;
1808
1809 if (verb)
1810 say(base, of->file, "Adding to index");
1811
1812 if (nodb)
1813 return;
1814
1815 desc = "";
1816 if (NULL != of->desc) {
1817 key = hashget(of->desc, strlen(of->desc));
1818 assert(NULL != key);
1819 if (NULL == key->utf8)
1820 utf8key(mc, key);
1821 desc = key->utf8;
1822 }
1823
1824 SQL_EXEC("BEGIN TRANSACTION");
1825
1826 i = 1;
1827 SQL_BIND_TEXT(stmts[STMT_INSERT_DOC], i, of->file);
1828 SQL_BIND_TEXT(stmts[STMT_INSERT_DOC], i, of->sec);
1829 SQL_BIND_TEXT(stmts[STMT_INSERT_DOC], i, of->arch);
1830 SQL_BIND_TEXT(stmts[STMT_INSERT_DOC], i, desc);
1831 SQL_BIND_INT(stmts[STMT_INSERT_DOC], i, form);
1832 SQL_STEP(stmts[STMT_INSERT_DOC]);
1833 recno = sqlite3_last_insert_rowid(db);
1834 sqlite3_reset(stmts[STMT_INSERT_DOC]);
1835
1836 for (key = words; NULL != key; key = key->next) {
1837 assert(key->of == of);
1838 if (NULL == key->utf8)
1839 utf8key(mc, key);
1840 i = 1;
1841 SQL_BIND_INT64(stmts[STMT_INSERT_KEY], i, key->mask);
1842 SQL_BIND_TEXT(stmts[STMT_INSERT_KEY], i, key->utf8);
1843 SQL_BIND_INT64(stmts[STMT_INSERT_KEY], i, recno);
1844 SQL_STEP(stmts[STMT_INSERT_KEY]);
1845 sqlite3_reset(stmts[STMT_INSERT_KEY]);
1846 }
1847
1848 SQL_EXEC("END TRANSACTION");
1849 }
1850
1851 static void
1852 dbprune(const char *base)
1853 {
1854 struct of *of;
1855 size_t i;
1856
1857 if (nodb)
1858 return;
1859
1860 for (of = ofs; NULL != of; of = of->next) {
1861 i = 1;
1862 SQL_BIND_TEXT(stmts[STMT_DELETE], i, of->file);
1863 SQL_STEP(stmts[STMT_DELETE]);
1864 sqlite3_reset(stmts[STMT_DELETE]);
1865 if (verb)
1866 say(base, of->file, "Deleted from index");
1867 }
1868 }
1869
1870 /*
1871 * Close an existing database and its prepared statements.
1872 * If "real" is not set, rename the temporary file into the real one.
1873 */
1874 static void
1875 dbclose(const char *base, int real)
1876 {
1877 size_t i;
1878 char file[MAXPATHLEN];
1879
1880 if (nodb)
1881 return;
1882
1883 for (i = 0; i < STMT__MAX; i++) {
1884 sqlite3_finalize(stmts[i]);
1885 stmts[i] = NULL;
1886 }
1887
1888 sqlite3_close(db);
1889 db = NULL;
1890
1891 if (real)
1892 return;
1893
1894 strlcpy(file, MANDOC_DB, MAXPATHLEN);
1895 strlcat(file, "~", MAXPATHLEN);
1896 if (-1 == rename(file, MANDOC_DB))
1897 perror(MANDOC_DB);
1898 }
1899
1900 /*
1901 * This is straightforward stuff.
1902 * Open a database connection to a "temporary" database, then open a set
1903 * of prepared statements we'll use over and over again.
1904 * If "real" is set, we use the existing database; if not, we truncate a
1905 * temporary one.
1906 * Must be matched by dbclose().
1907 */
1908 static int
1909 dbopen(const char *base, int real)
1910 {
1911 char file[MAXPATHLEN];
1912 const char *sql;
1913 int rc, ofl;
1914 size_t sz;
1915
1916 if (nodb)
1917 return(1);
1918
1919 sz = strlcpy(file, MANDOC_DB, MAXPATHLEN);
1920 if ( ! real)
1921 sz = strlcat(file, "~", MAXPATHLEN);
1922
1923 if (sz >= MAXPATHLEN) {
1924 fprintf(stderr, "%s: Path too long\n", file);
1925 return(0);
1926 }
1927
1928 if ( ! real)
1929 remove(file);
1930
1931 ofl = SQLITE_OPEN_READWRITE |
1932 (0 == real ? SQLITE_OPEN_EXCLUSIVE : 0);
1933
1934 rc = sqlite3_open_v2(file, &db, ofl, NULL);
1935 if (SQLITE_OK == rc)
1936 goto prepare_statements;
1937 if (SQLITE_CANTOPEN != rc) {
1938 perror(file);
1939 return(0);
1940 }
1941
1942 sqlite3_close(db);
1943 db = NULL;
1944
1945 if (SQLITE_OK != (rc = sqlite3_open(file, &db))) {
1946 perror(file);
1947 return(0);
1948 }
1949
1950 sql = "CREATE TABLE \"docs\" (\n"
1951 " \"file\" TEXT NOT NULL,\n"
1952 " \"sec\" TEXT NOT NULL,\n"
1953 " \"arch\" TEXT NOT NULL,\n"
1954 " \"desc\" TEXT NOT NULL,\n"
1955 " \"form\" INTEGER NOT NULL,\n"
1956 " \"id\" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL\n"
1957 ");\n"
1958 "\n"
1959 "CREATE TABLE \"keys\" (\n"
1960 " \"bits\" INTEGER NOT NULL,\n"
1961 " \"key\" TEXT NOT NULL,\n"
1962 " \"docid\" INTEGER NOT NULL REFERENCES docs(id) "
1963 "ON DELETE CASCADE,\n"
1964 " \"id\" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL\n"
1965 ");\n"
1966 "\n"
1967 "CREATE INDEX \"key_index\" ON keys (key);\n";
1968
1969 if (SQLITE_OK != sqlite3_exec(db, sql, NULL, NULL, NULL)) {
1970 perror(sqlite3_errmsg(db));
1971 return(0);
1972 }
1973
1974 prepare_statements:
1975 SQL_EXEC("PRAGMA foreign_keys = ON");
1976 sql = "DELETE FROM docs where file=?";
1977 sqlite3_prepare_v2(db, sql, -1, &stmts[STMT_DELETE], NULL);
1978 sql = "INSERT INTO docs "
1979 "(file,sec,arch,desc,form) VALUES (?,?,?,?,?)";
1980 sqlite3_prepare_v2(db, sql, -1, &stmts[STMT_INSERT_DOC], NULL);
1981 sql = "INSERT INTO keys "
1982 "(bits,key,docid) VALUES (?,?,?)";
1983 sqlite3_prepare_v2(db, sql, -1, &stmts[STMT_INSERT_KEY], NULL);
1984 return(1);
1985 }
1986
1987 static void *
1988 hash_halloc(size_t sz, void *arg)
1989 {
1990
1991 return(mandoc_calloc(sz, 1));
1992 }
1993
1994 static void *
1995 hash_alloc(size_t sz, void *arg)
1996 {
1997
1998 return(mandoc_malloc(sz));
1999 }
2000
2001 static void
2002 hash_free(void *p, size_t sz, void *arg)
2003 {
2004
2005 free(p);
2006 }
2007
2008 static int
2009 path_reset(const char *cwd, int fd, const char *base)
2010 {
2011
2012 if (-1 == fchdir(fd)) {
2013 perror(cwd);
2014 return(0);
2015 } else if (-1 == chdir(base)) {
2016 perror(base);
2017 return(0);
2018 }
2019 return(1);
2020 }
2021
2022 static void
2023 say(const char *dir, const char *file, const char *format, ...)
2024 {
2025 va_list ap;
2026
2027 fprintf(stderr, "%s", dir);
2028 if ('\0' != *file)
2029 fprintf(stderr, "//%s", file);
2030 fputs(": ", stderr);
2031
2032 va_start(ap, format);
2033 vfprintf(stderr, format, ap);
2034 va_end(ap);
2035
2036 fputc('\n', stderr);
2037 }
A git conversion mirror of mandoc.bsd.lv