mansearch.c

   1 /*      $Id: mansearch.c,v 1.38 2014/07/12 14:00:25 schwarze Exp $ */
   2 /*
   3  * Copyright (c) 2012 Kristaps Dzonsons <kristaps@bsd.lv>
   4  * Copyright (c) 2013, 2014 Ingo Schwarze <schwarze@openbsd.org>
   5  *
   6  * Permission to use, copy, modify, and distribute this software for any
   7  * purpose with or without fee is hereby granted, provided that the above
   8  * copyright notice and this permission notice appear in all copies.
   9  *
  10  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
  11  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
  12  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
  13  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
  14  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
  15  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
  16  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  17  */
  18 #ifdef HAVE_CONFIG_H
  19 #include "config.h"
  20 #endif
  21
  22 #include <sys/mman.h>
  23 #include <assert.h>
  24 #include <fcntl.h>
  25 #include <getopt.h>
  26 #include <limits.h>
  27 #include <regex.h>
  28 #include <stdio.h>
  29 #include <stdint.h>
  30 #include <stddef.h>
  31 #include <stdlib.h>
  32 #include <string.h>
  33 #include <unistd.h>
  34
  35 #ifdef HAVE_OHASH
  36 #include <ohash.h>
  37 #else
  38 #include "compat_ohash.h"
  39 #endif
  40 #include <sqlite3.h>
  41
  42 #include "mandoc.h"
  43 #include "mandoc_aux.h"
  44 #include "manpath.h"
  45 #include "mansearch.h"
  46
  47 extern int mansearch_keymax;
  48 extern const char *const mansearch_keynames[];
  49
  50 #define SQL_BIND_TEXT(_db, _s, _i, _v) \
  51         do { if (SQLITE_OK != sqlite3_bind_text \
  52                 ((_s), (_i)++, (_v), -1, SQLITE_STATIC)) \
  53                 fprintf(stderr, "%s\n", sqlite3_errmsg((_db))); \
  54         } while (0)
  55 #define SQL_BIND_INT64(_db, _s, _i, _v) \
  56         do { if (SQLITE_OK != sqlite3_bind_int64 \
  57                 ((_s), (_i)++, (_v))) \
  58                 fprintf(stderr, "%s\n", sqlite3_errmsg((_db))); \
  59         } while (0)
  60 #define SQL_BIND_BLOB(_db, _s, _i, _v) \
  61         do { if (SQLITE_OK != sqlite3_bind_blob \
  62                 ((_s), (_i)++, (&_v), sizeof(_v), SQLITE_STATIC)) \
  63                 fprintf(stderr, "%s\n", sqlite3_errmsg((_db))); \
  64         } while (0)
  65
  66 struct  expr {
  67         regex_t          regexp;  /* compiled regexp, if applicable */
  68         const char      *substr;  /* to search for, if applicable */
  69         struct expr     *next;    /* next in sequence */
  70         uint64_t         bits;    /* type-mask */
  71         int              equal;   /* equality, not subsring match */
  72         int              open;    /* opening parentheses before */
  73         int              and;     /* logical AND before */
  74         int              close;   /* closing parentheses after */
  75 };
  76
  77 struct  match {
  78         uint64_t         pageid; /* identifier in database */
  79         char            *desc; /* manual page description */
  80         int              form; /* 0 == catpage */
  81 };
  82
  83 static  void             buildnames(struct manpage *, sqlite3 *,
  84                                 sqlite3_stmt *, uint64_t,
  85                                 const char *, int form);
  86 static  char            *buildoutput(sqlite3 *, sqlite3_stmt *,
  87                                  uint64_t, uint64_t);
  88 static  void            *hash_alloc(size_t, void *);
  89 static  void             hash_free(void *, void *);
  90 static  void            *hash_calloc(size_t, size_t, void *);
  91 static  struct expr     *exprcomp(const struct mansearch *,
  92                                 int, char *[]);
  93 static  void             exprfree(struct expr *);
  94 static  struct expr     *exprspec(struct expr *, uint64_t,
  95                                  const char *, const char *);
  96 static  struct expr     *exprterm(const struct mansearch *, char *, int);
  97 static  void             sql_append(char **sql, size_t *sz,
  98                                 const char *newstr, int count);
  99 static  void             sql_match(sqlite3_context *context,
 100                                 int argc, sqlite3_value **argv);
 101 static  void             sql_regexp(sqlite3_context *context,
 102                                 int argc, sqlite3_value **argv);
 103 static  char            *sql_statement(const struct expr *);
 104
 105
 106 int
 107 mansearch_setup(int start)
 108 {
 109         static void     *pagecache;
 110         int              c;
 111
 112 #define PC_PAGESIZE     1280
 113 #define PC_NUMPAGES     256
 114
 115         if (start) {
 116                 if (NULL != pagecache) {
 117                         fprintf(stderr, "pagecache already enabled\n");
 118                         return((int)MANDOCLEVEL_BADARG);
 119                 }
 120
 121                 pagecache = mmap(NULL, PC_PAGESIZE * PC_NUMPAGES,
 122                     PROT_READ | PROT_WRITE, MAP_ANON, -1, 0);
 123
 124                 if (MAP_FAILED == pagecache) {
 125                         perror("mmap");
 126                         pagecache = NULL;
 127                         return((int)MANDOCLEVEL_SYSERR);
 128                 }
 129
 130                 c = sqlite3_config(SQLITE_CONFIG_PAGECACHE,
 131                     pagecache, PC_PAGESIZE, PC_NUMPAGES);
 132
 133                 if (SQLITE_OK == c)
 134                         return((int)MANDOCLEVEL_OK);
 135
 136                 fprintf(stderr, "pagecache: %s\n", sqlite3_errstr(c));
 137
 138         } else if (NULL == pagecache) {
 139                 fprintf(stderr, "pagecache missing\n");
 140                 return((int)MANDOCLEVEL_BADARG);
 141         }
 142
 143         if (-1 == munmap(pagecache, PC_PAGESIZE * PC_NUMPAGES)) {
 144                 perror("munmap");
 145                 pagecache = NULL;
 146                 return((int)MANDOCLEVEL_SYSERR);
 147         }
 148
 149         pagecache = NULL;
 150         return((int)MANDOCLEVEL_OK);
 151 }
 152
 153 int
 154 mansearch(const struct mansearch *search,
 155                 const struct manpaths *paths,
 156                 int argc, char *argv[],
 157                 const char *outkey,
 158                 struct manpage **res, size_t *sz)
 159 {
 160         int              fd, rc, c, indexbit;
 161         int64_t          pageid;
 162         uint64_t         outbit, iterbit;
 163         char             buf[PATH_MAX];
 164         char            *sql;
 165         struct manpage  *mpage;
 166         struct expr     *e, *ep;
 167         sqlite3         *db;
 168         sqlite3_stmt    *s, *s2;
 169         struct match    *mp;
 170         struct ohash_info info;
 171         struct ohash     htab;
 172         unsigned int     idx;
 173         size_t           i, j, cur, maxres;
 174
 175         info.calloc = hash_calloc;
 176         info.alloc = hash_alloc;
 177         info.free = hash_free;
 178         info.key_offset = offsetof(struct match, pageid);
 179
 180         *sz = cur = maxres = 0;
 181         sql = NULL;
 182         *res = NULL;
 183         fd = -1;
 184         e = NULL;
 185         rc = 0;
 186
 187         if (0 == argc)
 188                 goto out;
 189         if (NULL == (e = exprcomp(search, argc, argv)))
 190                 goto out;
 191
 192         outbit = 0;
 193         if (NULL != outkey) {
 194                 for (indexbit = 0, iterbit = 1;
 195                      indexbit < mansearch_keymax;
 196                      indexbit++, iterbit <<= 1) {
 197                         if (0 == strcasecmp(outkey,
 198                             mansearch_keynames[indexbit])) {
 199                                 outbit = iterbit;
 200                                 break;
 201                         }
 202                 }
 203         }
 204
 205         /*
 206          * Save a descriptor to the current working directory.
 207          * Since pathnames in the "paths" variable might be relative,
 208          * and we'll be chdir()ing into them, we need to keep a handle
 209          * on our current directory from which to start the chdir().
 210          */
 211
 212         if (NULL == getcwd(buf, PATH_MAX)) {
 213                 perror("getcwd");
 214                 goto out;
 215         } else if (-1 == (fd = open(buf, O_RDONLY, 0))) {
 216                 perror(buf);
 217                 goto out;
 218         }
 219
 220         sql = sql_statement(e);
 221
 222         /*
 223          * Loop over the directories (containing databases) for us to
 224          * search.
 225          * Don't let missing/bad databases/directories phase us.
 226          * In each, try to open the resident database and, if it opens,
 227          * scan it for our match expression.
 228          */
 229
 230         for (i = 0; i < paths->sz; i++) {
 231                 if (-1 == fchdir(fd)) {
 232                         perror(buf);
 233                         free(*res);
 234                         break;
 235                 } else if (-1 == chdir(paths->paths[i])) {
 236                         perror(paths->paths[i]);
 237                         continue;
 238                 }
 239
 240                 c = sqlite3_open_v2(MANDOC_DB, &db,
 241                     SQLITE_OPEN_READONLY, NULL);
 242
 243                 if (SQLITE_OK != c) {
 244                         perror(MANDOC_DB);
 245                         sqlite3_close(db);
 246                         continue;
 247                 }
 248
 249                 /*
 250                  * Define the SQL functions for substring
 251                  * and regular expression matching.
 252                  */
 253
 254                 c = sqlite3_create_function(db, "match", 2,
 255                     SQLITE_UTF8 | SQLITE_DETERMINISTIC,
 256                     NULL, sql_match, NULL, NULL);
 257                 assert(SQLITE_OK == c);
 258                 c = sqlite3_create_function(db, "regexp", 2,
 259                     SQLITE_UTF8 | SQLITE_DETERMINISTIC,
 260                     NULL, sql_regexp, NULL, NULL);
 261                 assert(SQLITE_OK == c);
 262
 263                 j = 1;
 264                 c = sqlite3_prepare_v2(db, sql, -1, &s, NULL);
 265                 if (SQLITE_OK != c)
 266                         fprintf(stderr, "%s\n", sqlite3_errmsg(db));
 267
 268                 for (ep = e; NULL != ep; ep = ep->next) {
 269                         if (NULL == ep->substr) {
 270                                 SQL_BIND_BLOB(db, s, j, ep->regexp);
 271                         } else
 272                                 SQL_BIND_TEXT(db, s, j, ep->substr);
 273                         if (0 == ((TYPE_Nd | TYPE_Nm) & ep->bits))
 274                                 SQL_BIND_INT64(db, s, j, ep->bits);
 275                 }
 276
 277                 memset(&htab, 0, sizeof(struct ohash));
 278                 ohash_init(&htab, 4, &info);
 279
 280                 /*
 281                  * Hash each entry on its [unique] document identifier.
 282                  * This is a uint64_t.
 283                  * Instead of using a hash function, simply convert the
 284                  * uint64_t to a uint32_t, the hash value's type.
 285                  * This gives good performance and preserves the
 286                  * distribution of buckets in the table.
 287                  */
 288                 while (SQLITE_ROW == (c = sqlite3_step(s))) {
 289                         pageid = sqlite3_column_int64(s, 2);
 290                         idx = ohash_lookup_memory(&htab,
 291                             (char *)&pageid, sizeof(uint64_t),
 292                             (uint32_t)pageid);
 293
 294                         if (NULL != ohash_find(&htab, idx))
 295                                 continue;
 296
 297                         mp = mandoc_calloc(1, sizeof(struct match));
 298                         mp->pageid = pageid;
 299                         mp->form = sqlite3_column_int(s, 1);
 300                         if (TYPE_Nd == outbit)
 301                                 mp->desc = mandoc_strdup(
 302                                     sqlite3_column_text(s, 0));
 303                         ohash_insert(&htab, idx, mp);
 304                 }
 305
 306                 if (SQLITE_DONE != c)
 307                         fprintf(stderr, "%s\n", sqlite3_errmsg(db));
 308
 309                 sqlite3_finalize(s);
 310
 311                 c = sqlite3_prepare_v2(db,
 312                     "SELECT sec, arch, name, pageid FROM mlinks "
 313                     "WHERE pageid=? ORDER BY sec, arch, name",
 314                     -1, &s, NULL);
 315                 if (SQLITE_OK != c)
 316                         fprintf(stderr, "%s\n", sqlite3_errmsg(db));
 317
 318                 c = sqlite3_prepare_v2(db,
 319                     "SELECT bits, key, pageid FROM keys "
 320                     "WHERE pageid=? AND bits & ?",
 321                     -1, &s2, NULL);
 322                 if (SQLITE_OK != c)
 323                         fprintf(stderr, "%s\n", sqlite3_errmsg(db));
 324
 325                 for (mp = ohash_first(&htab, &idx);
 326                                 NULL != mp;
 327                                 mp = ohash_next(&htab, &idx)) {
 328                         if (cur + 1 > maxres) {
 329                                 maxres += 1024;
 330                                 *res = mandoc_reallocarray(*res,
 331                                     maxres, sizeof(struct manpage));
 332                         }
 333                         mpage = *res + cur;
 334                         mpage->form = mp->form;
 335                         buildnames(mpage, db, s, mp->pageid,
 336                             paths->paths[i], mp->form);
 337                         mpage->output = TYPE_Nd & outbit ?
 338                             mp->desc : outbit ?
 339                             buildoutput(db, s2, mp->pageid, outbit) : NULL;
 340
 341                         free(mp);
 342                         cur++;
 343                 }
 344
 345                 sqlite3_finalize(s);
 346                 sqlite3_finalize(s2);
 347                 sqlite3_close(db);
 348                 ohash_delete(&htab);
 349         }
 350         rc = 1;
 351 out:
 352         if (-1 != fd) {
 353                 if (-1 == fchdir(fd))
 354                         perror(buf);
 355                 close(fd);
 356         }
 357         exprfree(e);
 358         free(sql);
 359         *sz = cur;
 360         return(rc);
 361 }
 362
 363 static void
 364 buildnames(struct manpage *mpage, sqlite3 *db, sqlite3_stmt *s,
 365                 uint64_t pageid, const char *path, int form)
 366 {
 367         char            *newnames, *prevsec, *prevarch;
 368         const char      *oldnames, *sep1, *name, *sec, *sep2, *arch, *fsec;
 369         size_t           i;
 370         int              c;
 371
 372         mpage->file = NULL;
 373         mpage->names = NULL;
 374         prevsec = prevarch = NULL;
 375         i = 1;
 376         SQL_BIND_INT64(db, s, i, pageid);
 377         while (SQLITE_ROW == (c = sqlite3_step(s))) {
 378
 379                 /* Decide whether we already have some names. */
 380
 381                 if (NULL == mpage->names) {
 382                         oldnames = "";
 383                         sep1 = "";
 384                 } else {
 385                         oldnames = mpage->names;
 386                         sep1 = ", ";
 387                 }
 388
 389                 /* Fetch the next name. */
 390
 391                 sec = sqlite3_column_text(s, 0);
 392                 arch = sqlite3_column_text(s, 1);
 393                 name = sqlite3_column_text(s, 2);
 394
 395                 /* If the section changed, append the old one. */
 396
 397                 if (NULL != prevsec &&
 398                     (strcmp(sec, prevsec) ||
 399                      strcmp(arch, prevarch))) {
 400                         sep2 = '\0' == *prevarch ? "" : "/";
 401                         mandoc_asprintf(&newnames, "%s(%s%s%s)",
 402                             oldnames, prevsec, sep2, prevarch);
 403                         free(mpage->names);
 404                         oldnames = mpage->names = newnames;
 405                         free(prevsec);
 406                         free(prevarch);
 407                         prevsec = prevarch = NULL;
 408                 }
 409
 410                 /* Save the new section, to append it later. */
 411
 412                 if (NULL == prevsec) {
 413                         prevsec = mandoc_strdup(sec);
 414                         prevarch = mandoc_strdup(arch);
 415                 }
 416
 417                 /* Append the new name. */
 418
 419                 mandoc_asprintf(&newnames, "%s%s%s",
 420                     oldnames, sep1, name);
 421                 free(mpage->names);
 422                 mpage->names = newnames;
 423
 424                 /* Also save the first file name encountered. */
 425
 426                 if (NULL != mpage->file)
 427                         continue;
 428
 429                 if (form) {
 430                         sep1 = "man";
 431                         fsec = sec;
 432                 } else {
 433                         sep1 = "cat";
 434                         fsec = "0";
 435                 }
 436                 sep2 = '\0' == *arch ? "" : "/";
 437                 mandoc_asprintf(&mpage->file, "%s/%s%s%s%s/%s.%s",
 438                     path, sep1, sec, sep2, arch, name, fsec);
 439         }
 440         if (SQLITE_DONE != c)
 441                 fprintf(stderr, "%s\n", sqlite3_errmsg(db));
 442         sqlite3_reset(s);
 443
 444         /* Append one final section to the names. */
 445
 446         if (NULL != prevsec) {
 447                 sep2 = '\0' == *prevarch ? "" : "/";
 448                 mandoc_asprintf(&newnames, "%s(%s%s%s)",
 449                     mpage->names, prevsec, sep2, prevarch);
 450                 free(mpage->names);
 451                 mpage->names = newnames;
 452                 free(prevsec);
 453                 free(prevarch);
 454         }
 455 }
 456
 457 static char *
 458 buildoutput(sqlite3 *db, sqlite3_stmt *s, uint64_t pageid, uint64_t outbit)
 459 {
 460         char            *output, *newoutput;
 461         const char      *oldoutput, *sep1, *data;
 462         size_t           i;
 463         int              c;
 464
 465         output = NULL;
 466         i = 1;
 467         SQL_BIND_INT64(db, s, i, pageid);
 468         SQL_BIND_INT64(db, s, i, outbit);
 469         while (SQLITE_ROW == (c = sqlite3_step(s))) {
 470                 if (NULL == output) {
 471                         oldoutput = "";
 472                         sep1 = "";
 473                 } else {
 474                         oldoutput = output;
 475                         sep1 = " # ";
 476                 }
 477                 data = sqlite3_column_text(s, 1);
 478                 mandoc_asprintf(&newoutput, "%s%s%s",
 479                     oldoutput, sep1, data);
 480                 free(output);
 481                 output = newoutput;
 482         }
 483         if (SQLITE_DONE != c)
 484                 fprintf(stderr, "%s\n", sqlite3_errmsg(db));
 485         sqlite3_reset(s);
 486         return(output);
 487 }
 488
 489 /*
 490  * Implement substring match as an application-defined SQL function.
 491  * Using the SQL LIKE or GLOB operators instead would be a bad idea
 492  * because that would require escaping metacharacters in the string
 493  * being searched for.
 494  */
 495 static void
 496 sql_match(sqlite3_context *context, int argc, sqlite3_value **argv)
 497 {
 498
 499         assert(2 == argc);
 500         sqlite3_result_int(context, NULL != strcasestr(
 501             (const char *)sqlite3_value_text(argv[1]),
 502             (const char *)sqlite3_value_text(argv[0])));
 503 }
 504
 505 /*
 506  * Implement regular expression match
 507  * as an application-defined SQL function.
 508  */
 509 static void
 510 sql_regexp(sqlite3_context *context, int argc, sqlite3_value **argv)
 511 {
 512
 513         assert(2 == argc);
 514         sqlite3_result_int(context, !regexec(
 515             (regex_t *)sqlite3_value_blob(argv[0]),
 516             (const char *)sqlite3_value_text(argv[1]),
 517             0, NULL, 0));
 518 }
 519
 520 static void
 521 sql_append(char **sql, size_t *sz, const char *newstr, int count)
 522 {
 523         size_t           newsz;
 524
 525         newsz = 1 < count ? (size_t)count : strlen(newstr);
 526         *sql = mandoc_realloc(*sql, *sz + newsz + 1);
 527         if (1 < count)
 528                 memset(*sql + *sz, *newstr, (size_t)count);
 529         else
 530                 memcpy(*sql + *sz, newstr, newsz);
 531         *sz += newsz;
 532         (*sql)[*sz] = '\0';
 533 }
 534
 535 /*
 536  * Prepare the search SQL statement.
 537  */
 538 static char *
 539 sql_statement(const struct expr *e)
 540 {
 541         char            *sql;
 542         size_t           sz;
 543         int              needop;
 544
 545         sql = mandoc_strdup(
 546             "SELECT desc, form, pageid FROM mpages WHERE ");
 547         sz = strlen(sql);
 548
 549         for (needop = 0; NULL != e; e = e->next) {
 550                 if (e->and)
 551                         sql_append(&sql, &sz, " AND ", 1);
 552                 else if (needop)
 553                         sql_append(&sql, &sz, " OR ", 1);
 554                 if (e->open)
 555                         sql_append(&sql, &sz, "(", e->open);
 556                 sql_append(&sql, &sz,
 557                     TYPE_Nd & e->bits
 558                     ? (NULL == e->substr
 559                         ? "desc REGEXP ?"
 560                         : "desc MATCH ?")
 561                     : TYPE_Nm == e->bits
 562                     ? (NULL == e->substr
 563                         ? "pageid IN (SELECT pageid FROM names "
 564                           "WHERE name REGEXP ?)"
 565                         : e->equal
 566                         ? "pageid IN (SELECT pageid FROM names "
 567                           "WHERE name = ?)"
 568                         : "pageid IN (SELECT pageid FROM names "
 569                           "WHERE name MATCH ?)")
 570                     : (NULL == e->substr
 571                         ? "pageid IN (SELECT pageid FROM keys "
 572                           "WHERE key REGEXP ? AND bits & ?)"
 573                         : "pageid IN (SELECT pageid FROM keys "
 574                           "WHERE key MATCH ? AND bits & ?)"), 1);
 575                 if (e->close)
 576                         sql_append(&sql, &sz, ")", e->close);
 577                 needop = 1;
 578         }
 579
 580         return(sql);
 581 }
 582
 583 /*
 584  * Compile a set of string tokens into an expression.
 585  * Tokens in "argv" are assumed to be individual expression atoms (e.g.,
 586  * "(", "foo=bar", etc.).
 587  */
 588 static struct expr *
 589 exprcomp(const struct mansearch *search, int argc, char *argv[])
 590 {
 591         uint64_t         mask;
 592         int              i, toopen, logic, igncase, toclose;
 593         struct expr     *first, *prev, *cur, *next;
 594
 595         first = cur = NULL;
 596         logic = igncase = toclose = 0;
 597         toopen = NULL != search->sec || NULL != search->arch;
 598
 599         for (i = 0; i < argc; i++) {
 600                 if (0 == strcmp("(", argv[i])) {
 601                         if (igncase)
 602                                 goto fail;
 603                         toopen++;
 604                         toclose++;
 605                         continue;
 606                 } else if (0 == strcmp(")", argv[i])) {
 607                         if (toopen || logic || igncase || NULL == cur)
 608                                 goto fail;
 609                         cur->close++;
 610                         if (0 > --toclose)
 611                                 goto fail;
 612                         continue;
 613                 } else if (0 == strcmp("-a", argv[i])) {
 614                         if (toopen || logic || igncase || NULL == cur)
 615                                 goto fail;
 616                         logic = 1;
 617                         continue;
 618                 } else if (0 == strcmp("-o", argv[i])) {
 619                         if (toopen || logic || igncase || NULL == cur)
 620                                 goto fail;
 621                         logic = 2;
 622                         continue;
 623                 } else if (0 == strcmp("-i", argv[i])) {
 624                         if (igncase)
 625                                 goto fail;
 626                         igncase = 1;
 627                         continue;
 628                 }
 629                 next = exprterm(search, argv[i], !igncase);
 630                 if (NULL == next)
 631                         goto fail;
 632                 if (NULL == first)
 633                         first = next;
 634                 else
 635                         cur->next = next;
 636                 prev = cur = next;
 637
 638                 /*
 639                  * Searching for descriptions must be split out
 640                  * because they are stored in the mpages table,
 641                  * not in the keys table.
 642                  */
 643
 644                 for (mask = TYPE_Nm; mask <= TYPE_Nd; mask <<= 1) {
 645                         if (mask & cur->bits && ~mask & cur->bits) {
 646                                 next = mandoc_calloc(1,
 647                                     sizeof(struct expr));
 648                                 memcpy(next, cur, sizeof(struct expr));
 649                                 prev->open = 1;
 650                                 cur->bits = mask;
 651                                 cur->next = next;
 652                                 cur = next;
 653                                 cur->bits &= ~mask;
 654                         }
 655                 }
 656                 prev->and = (1 == logic);
 657                 prev->open += toopen;
 658                 if (cur != prev)
 659                         cur->close = 1;
 660
 661                 toopen = logic = igncase = 0;
 662         }
 663         if (toopen || logic || igncase || toclose)
 664                 goto fail;
 665
 666         if (NULL != search->sec || NULL != search->arch)
 667                 cur->close++;
 668         if (NULL != search->arch)
 669                 cur = exprspec(cur, TYPE_arch, search->arch, "^(%s|any)$");
 670         if (NULL != search->sec)
 671                 exprspec(cur, TYPE_sec, search->sec, "^%s$");
 672
 673         return(first);
 674
 675 fail:
 676         if (NULL != first)
 677                 exprfree(first);
 678         return(NULL);
 679 }
 680
 681 static struct expr *
 682 exprspec(struct expr *cur, uint64_t key, const char *value,
 683                 const char *format)
 684 {
 685         char     errbuf[BUFSIZ];
 686         char    *cp;
 687         int      irc;
 688
 689         mandoc_asprintf(&cp, format, value);
 690         cur->next = mandoc_calloc(1, sizeof(struct expr));
 691         cur = cur->next;
 692         cur->and = 1;
 693         cur->bits = key;
 694         if (0 != (irc = regcomp(&cur->regexp, cp,
 695             REG_EXTENDED | REG_NOSUB | REG_ICASE))) {
 696                 regerror(irc, &cur->regexp, errbuf, sizeof(errbuf));
 697                 fprintf(stderr, "regcomp: %s\n", errbuf);
 698                 cur->substr = value;
 699         }
 700         free(cp);
 701         return(cur);
 702 }
 703
 704 static struct expr *
 705 exprterm(const struct mansearch *search, char *buf, int cs)
 706 {
 707         char             errbuf[BUFSIZ];
 708         struct expr     *e;
 709         char            *key, *val;
 710         uint64_t         iterbit;
 711         int              i, irc;
 712
 713         if ('\0' == *buf)
 714                 return(NULL);
 715
 716         e = mandoc_calloc(1, sizeof(struct expr));
 717
 718         if (MANSEARCH_MAN & search->flags) {
 719                 e->bits = search->deftype;
 720                 e->substr = buf;
 721                 e->equal = 1;
 722                 return(e);
 723         }
 724
 725         /*
 726          * Look for an '=' or '~' operator,
 727          * unless forced to some fixed macro keys.
 728          */
 729
 730         if (MANSEARCH_WHATIS & search->flags)
 731                 val = NULL;
 732         else
 733                 val = strpbrk(buf, "=~");
 734
 735         if (NULL == val) {
 736                 e->bits = search->deftype;
 737                 e->substr = buf;
 738
 739         /*
 740          * Found an operator.
 741          * Regexp search is requested by !e->substr.
 742          */
 743
 744         } else {
 745                 if (val == buf)
 746                         e->bits = search->deftype;
 747                 if ('=' == *val)
 748                         e->substr = val + 1;
 749                 *val++ = '\0';
 750                 if (NULL != strstr(buf, "arch"))
 751                         cs = 0;
 752         }
 753
 754         /* Compile regular expressions. */
 755
 756         if (MANSEARCH_WHATIS & search->flags) {
 757                 e->substr = NULL;
 758                 mandoc_asprintf(&val, "[[:<:]]%s[[:>:]]", buf);
 759         }
 760
 761         if (NULL == e->substr) {
 762                 irc = regcomp(&e->regexp, val,
 763                     REG_EXTENDED | REG_NOSUB | (cs ? 0 : REG_ICASE));
 764                 if (MANSEARCH_WHATIS & search->flags)
 765                         free(val);
 766                 if (irc) {
 767                         regerror(irc, &e->regexp, errbuf, sizeof(errbuf));
 768                         fprintf(stderr, "regcomp: %s\n", errbuf);
 769                         free(e);
 770                         return(NULL);
 771                 }
 772         }
 773
 774         if (e->bits)
 775                 return(e);
 776
 777         /*
 778          * Parse out all possible fields.
 779          * If the field doesn't resolve, bail.
 780          */
 781
 782         while (NULL != (key = strsep(&buf, ","))) {
 783                 if ('\0' == *key)
 784                         continue;
 785                 for (i = 0, iterbit = 1;
 786                      i < mansearch_keymax;
 787                      i++, iterbit <<= 1) {
 788                         if (0 == strcasecmp(key,
 789                             mansearch_keynames[i])) {
 790                                 e->bits |= iterbit;
 791                                 break;
 792                         }
 793                 }
 794                 if (i == mansearch_keymax) {
 795                         if (strcasecmp(key, "any")) {
 796                                 free(e);
 797                                 return(NULL);
 798                         }
 799                         e->bits |= ~0ULL;
 800                 }
 801         }
 802
 803         return(e);
 804 }
 805
 806 static void
 807 exprfree(struct expr *p)
 808 {
 809         struct expr     *pp;
 810
 811         while (NULL != p) {
 812                 pp = p->next;
 813                 free(p);
 814                 p = pp;
 815         }
 816 }
 817
 818 static void *
 819 hash_calloc(size_t nmemb, size_t sz, void *arg)
 820 {
 821
 822         return(mandoc_calloc(nmemb, sz));
 823 }
 824
 825 static void *
 826 hash_alloc(size_t sz, void *arg)
 827 {
 828
 829         return(mandoc_malloc(sz));
 830 }
 831
 832 static void
 833 hash_free(void *p, void *arg)
 834 {
 835
 836         free(p);
 837 }