mansearch.c

   1 /*      $Id: mansearch.c,v 1.60 2015/10/13 15:53:05 schwarze Exp $ */
   2 /*
   3  * Copyright (c) 2012 Kristaps Dzonsons <kristaps@bsd.lv>
   4  * Copyright (c) 2013, 2014, 2015 Ingo Schwarze <schwarze@openbsd.org>
   5  *
   6  * Permission to use, copy, modify, and distribute this software for any
   7  * purpose with or without fee is hereby granted, provided that the above
   8  * copyright notice and this permission notice appear in all copies.
   9  *
  10  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHORS DISCLAIM ALL WARRANTIES
  11  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
  12  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR
  13  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
  14  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
  15  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
  16  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  17  */
  18 #include "config.h"
  19
  20 #include <sys/mman.h>
  21 #include <sys/types.h>
  22
  23 #include <assert.h>
  24 #include <err.h>
  25 #include <errno.h>
  26 #include <fcntl.h>
  27 #include <getopt.h>
  28 #include <glob.h>
  29 #include <limits.h>
  30 #include <regex.h>
  31 #include <stdio.h>
  32 #include <stdint.h>
  33 #include <stddef.h>
  34 #include <stdlib.h>
  35 #include <string.h>
  36 #include <unistd.h>
  37
  38 #include <sqlite3.h>
  39 #ifndef SQLITE_DETERMINISTIC
  40 #define SQLITE_DETERMINISTIC 0
  41 #endif
  42
  43 #include "mandoc.h"
  44 #include "mandoc_aux.h"
  45 #include "mandoc_ohash.h"
  46 #include "manconf.h"
  47 #include "mansearch.h"
  48
  49 extern int mansearch_keymax;
  50 extern const char *const mansearch_keynames[];
  51
  52 #define SQL_BIND_TEXT(_db, _s, _i, _v) \
  53         do { if (SQLITE_OK != sqlite3_bind_text \
  54                 ((_s), (_i)++, (_v), -1, SQLITE_STATIC)) \
  55                 warnx("%s", sqlite3_errmsg((_db))); \
  56         } while (0)
  57 #define SQL_BIND_INT64(_db, _s, _i, _v) \
  58         do { if (SQLITE_OK != sqlite3_bind_int64 \
  59                 ((_s), (_i)++, (_v))) \
  60                 warnx("%s", sqlite3_errmsg((_db))); \
  61         } while (0)
  62 #define SQL_BIND_BLOB(_db, _s, _i, _v) \
  63         do { if (SQLITE_OK != sqlite3_bind_blob \
  64                 ((_s), (_i)++, (&_v), sizeof(_v), SQLITE_STATIC)) \
  65                 warnx("%s", sqlite3_errmsg((_db))); \
  66         } while (0)
  67
  68 struct  expr {
  69         regex_t          regexp;  /* compiled regexp, if applicable */
  70         const char      *substr;  /* to search for, if applicable */
  71         struct expr     *next;    /* next in sequence */
  72         uint64_t         bits;    /* type-mask */
  73         int              equal;   /* equality, not subsring match */
  74         int              open;    /* opening parentheses before */
  75         int              and;     /* logical AND before */
  76         int              close;   /* closing parentheses after */
  77 };
  78
  79 struct  match {
  80         uint64_t         pageid; /* identifier in database */
  81         uint64_t         bits; /* name type mask */
  82         char            *desc; /* manual page description */
  83         int              form; /* bit field: formatted, zipped? */
  84 };
  85
  86 static  void             buildnames(const struct mansearch *,
  87                                 struct manpage *, sqlite3 *,
  88                                 sqlite3_stmt *, uint64_t,
  89                                 const char *, int form);
  90 static  char            *buildoutput(sqlite3 *, sqlite3_stmt *,
  91                                  uint64_t, uint64_t);
  92 static  struct expr     *exprcomp(const struct mansearch *,
  93                                 int, char *[]);
  94 static  void             exprfree(struct expr *);
  95 static  struct expr     *exprterm(const struct mansearch *, char *, int);
  96 static  int              manpage_compare(const void *, const void *);
  97 static  void             sql_append(char **sql, size_t *sz,
  98                                 const char *newstr, int count);
  99 static  void             sql_match(sqlite3_context *context,
 100                                 int argc, sqlite3_value **argv);
 101 static  void             sql_regexp(sqlite3_context *context,
 102                                 int argc, sqlite3_value **argv);
 103 static  char            *sql_statement(const struct expr *);
 104
 105
 106 int
 107 mansearch_setup(int start)
 108 {
 109         static void     *pagecache;
 110         int              c;
 111
 112 #define PC_PAGESIZE     1280
 113 #define PC_NUMPAGES     256
 114
 115         if (start) {
 116                 if (NULL != pagecache) {
 117                         warnx("pagecache already enabled");
 118                         return (int)MANDOCLEVEL_BADARG;
 119                 }
 120
 121                 pagecache = mmap(NULL, PC_PAGESIZE * PC_NUMPAGES,
 122                     PROT_READ | PROT_WRITE,
 123                     MAP_SHARED | MAP_ANON, -1, 0);
 124
 125                 if (MAP_FAILED == pagecache) {
 126                         perror("mmap");
 127                         pagecache = NULL;
 128                         return (int)MANDOCLEVEL_SYSERR;
 129                 }
 130
 131                 c = sqlite3_config(SQLITE_CONFIG_PAGECACHE,
 132                     pagecache, PC_PAGESIZE, PC_NUMPAGES);
 133
 134                 if (SQLITE_OK == c)
 135                         return (int)MANDOCLEVEL_OK;
 136
 137                 warnx("pagecache: %s", sqlite3_errstr(c));
 138
 139         } else if (NULL == pagecache) {
 140                 warnx("pagecache missing");
 141                 return (int)MANDOCLEVEL_BADARG;
 142         }
 143
 144         if (-1 == munmap(pagecache, PC_PAGESIZE * PC_NUMPAGES)) {
 145                 perror("munmap");
 146                 pagecache = NULL;
 147                 return (int)MANDOCLEVEL_SYSERR;
 148         }
 149
 150         pagecache = NULL;
 151         return (int)MANDOCLEVEL_OK;
 152 }
 153
 154 int
 155 mansearch(const struct mansearch *search,
 156                 const struct manpaths *paths,
 157                 int argc, char *argv[],
 158                 struct manpage **res, size_t *sz)
 159 {
 160         int64_t          pageid;
 161         uint64_t         outbit, iterbit;
 162         char             buf[PATH_MAX];
 163         char            *sql;
 164         struct manpage  *mpage;
 165         struct expr     *e, *ep;
 166         sqlite3         *db;
 167         sqlite3_stmt    *s, *s2;
 168         struct match    *mp;
 169         struct ohash     htab;
 170         unsigned int     idx;
 171         size_t           i, j, cur, maxres;
 172         int              c, chdir_status, getcwd_status, indexbit;
 173
 174         if (argc == 0 || (e = exprcomp(search, argc, argv)) == NULL) {
 175                 *sz = 0;
 176                 return 0;
 177         }
 178
 179         cur = maxres = 0;
 180         *res = NULL;
 181
 182         if (NULL != search->outkey) {
 183                 outbit = TYPE_Nd;
 184                 for (indexbit = 0, iterbit = 1;
 185                      indexbit < mansearch_keymax;
 186                      indexbit++, iterbit <<= 1) {
 187                         if (0 == strcasecmp(search->outkey,
 188                             mansearch_keynames[indexbit])) {
 189                                 outbit = iterbit;
 190                                 break;
 191                         }
 192                 }
 193         } else
 194                 outbit = 0;
 195
 196         /*
 197          * Remember the original working directory, if possible.
 198          * This will be needed if the second or a later directory
 199          * is given as a relative path.
 200          * Do not error out if the current directory is not
 201          * searchable: Maybe it won't be needed after all.
 202          */
 203
 204         if (getcwd(buf, PATH_MAX) == NULL) {
 205                 getcwd_status = 0;
 206                 (void)strlcpy(buf, strerror(errno), sizeof(buf));
 207         } else
 208                 getcwd_status = 1;
 209
 210         sql = sql_statement(e);
 211
 212         /*
 213          * Loop over the directories (containing databases) for us to
 214          * search.
 215          * Don't let missing/bad databases/directories phase us.
 216          * In each, try to open the resident database and, if it opens,
 217          * scan it for our match expression.
 218          */
 219
 220         chdir_status = 0;
 221         for (i = 0; i < paths->sz; i++) {
 222                 if (chdir_status && paths->paths[i][0] != '/') {
 223                         if ( ! getcwd_status) {
 224                                 warnx("%s: getcwd: %s", paths->paths[i], buf);
 225                                 continue;
 226                         } else if (chdir(buf) == -1) {
 227                                 perror(buf);
 228                                 continue;
 229                         }
 230                 }
 231                 if (chdir(paths->paths[i]) == -1) {
 232                         perror(paths->paths[i]);
 233                         continue;
 234                 }
 235                 chdir_status = 1;
 236
 237                 c = sqlite3_open_v2(MANDOC_DB, &db,
 238                     SQLITE_OPEN_READONLY, NULL);
 239
 240                 if (SQLITE_OK != c) {
 241                         warn("%s/%s", paths->paths[i], MANDOC_DB);
 242                         sqlite3_close(db);
 243                         continue;
 244                 }
 245
 246                 /*
 247                  * Define the SQL functions for substring
 248                  * and regular expression matching.
 249                  */
 250
 251                 c = sqlite3_create_function(db, "match", 2,
 252                     SQLITE_UTF8 | SQLITE_DETERMINISTIC,
 253                     NULL, sql_match, NULL, NULL);
 254                 assert(SQLITE_OK == c);
 255                 c = sqlite3_create_function(db, "regexp", 2,
 256                     SQLITE_UTF8 | SQLITE_DETERMINISTIC,
 257                     NULL, sql_regexp, NULL, NULL);
 258                 assert(SQLITE_OK == c);
 259
 260                 j = 1;
 261                 c = sqlite3_prepare_v2(db, sql, -1, &s, NULL);
 262                 if (SQLITE_OK != c)
 263                         warnx("%s", sqlite3_errmsg(db));
 264
 265                 for (ep = e; NULL != ep; ep = ep->next) {
 266                         if (NULL == ep->substr) {
 267                                 SQL_BIND_BLOB(db, s, j, ep->regexp);
 268                         } else
 269                                 SQL_BIND_TEXT(db, s, j, ep->substr);
 270                         if (0 == ((TYPE_Nd | TYPE_Nm) & ep->bits))
 271                                 SQL_BIND_INT64(db, s, j, ep->bits);
 272                 }
 273
 274                 mandoc_ohash_init(&htab, 4, offsetof(struct match, pageid));
 275
 276                 /*
 277                  * Hash each entry on its [unique] document identifier.
 278                  * This is a uint64_t.
 279                  * Instead of using a hash function, simply convert the
 280                  * uint64_t to a uint32_t, the hash value's type.
 281                  * This gives good performance and preserves the
 282                  * distribution of buckets in the table.
 283                  */
 284                 while (SQLITE_ROW == (c = sqlite3_step(s))) {
 285                         pageid = sqlite3_column_int64(s, 2);
 286                         idx = ohash_lookup_memory(&htab,
 287                             (char *)&pageid, sizeof(uint64_t),
 288                             (uint32_t)pageid);
 289
 290                         if (NULL != ohash_find(&htab, idx))
 291                                 continue;
 292
 293                         mp = mandoc_calloc(1, sizeof(struct match));
 294                         mp->pageid = pageid;
 295                         mp->form = sqlite3_column_int(s, 1);
 296                         mp->bits = sqlite3_column_int64(s, 3);
 297                         if (TYPE_Nd == outbit)
 298                                 mp->desc = mandoc_strdup((const char *)
 299                                     sqlite3_column_text(s, 0));
 300                         ohash_insert(&htab, idx, mp);
 301                 }
 302
 303                 if (SQLITE_DONE != c)
 304                         warnx("%s", sqlite3_errmsg(db));
 305
 306                 sqlite3_finalize(s);
 307
 308                 c = sqlite3_prepare_v2(db,
 309                     "SELECT sec, arch, name, pageid FROM mlinks "
 310                     "WHERE pageid=? ORDER BY sec, arch, name",
 311                     -1, &s, NULL);
 312                 if (SQLITE_OK != c)
 313                         warnx("%s", sqlite3_errmsg(db));
 314
 315                 c = sqlite3_prepare_v2(db,
 316                     "SELECT bits, key, pageid FROM keys "
 317                     "WHERE pageid=? AND bits & ?",
 318                     -1, &s2, NULL);
 319                 if (SQLITE_OK != c)
 320                         warnx("%s", sqlite3_errmsg(db));
 321
 322                 for (mp = ohash_first(&htab, &idx);
 323                                 NULL != mp;
 324                                 mp = ohash_next(&htab, &idx)) {
 325                         if (cur + 1 > maxres) {
 326                                 maxres += 1024;
 327                                 *res = mandoc_reallocarray(*res,
 328                                     maxres, sizeof(struct manpage));
 329                         }
 330                         mpage = *res + cur;
 331                         mpage->ipath = i;
 332                         mpage->bits = mp->bits;
 333                         mpage->sec = 10;
 334                         mpage->form = mp->form;
 335                         buildnames(search, mpage, db, s, mp->pageid,
 336                             paths->paths[i], mp->form);
 337                         if (mpage->names != NULL) {
 338                                 mpage->output = TYPE_Nd & outbit ?
 339                                     mp->desc : outbit ?
 340                                     buildoutput(db, s2, mp->pageid, outbit) :
 341                                     NULL;
 342                                 cur++;
 343                         }
 344                         free(mp);
 345                 }
 346
 347                 sqlite3_finalize(s);
 348                 sqlite3_finalize(s2);
 349                 sqlite3_close(db);
 350                 ohash_delete(&htab);
 351
 352                 /*
 353                  * In man(1) mode, prefer matches in earlier trees
 354                  * over matches in later trees.
 355                  */
 356
 357                 if (cur && search->firstmatch)
 358                         break;
 359         }
 360         qsort(*res, cur, sizeof(struct manpage), manpage_compare);
 361         if (chdir_status && getcwd_status && chdir(buf) == -1)
 362                 perror(buf);
 363         exprfree(e);
 364         free(sql);
 365         *sz = cur;
 366         return 1;
 367 }
 368
 369 void
 370 mansearch_free(struct manpage *res, size_t sz)
 371 {
 372         size_t   i;
 373
 374         for (i = 0; i < sz; i++) {
 375                 free(res[i].file);
 376                 free(res[i].names);
 377                 free(res[i].output);
 378         }
 379         free(res);
 380 }
 381
 382 static int
 383 manpage_compare(const void *vp1, const void *vp2)
 384 {
 385         const struct manpage    *mp1, *mp2;
 386         int                      diff;
 387
 388         mp1 = vp1;
 389         mp2 = vp2;
 390         return (diff = mp2->bits - mp1->bits) ? diff :
 391             (diff = mp1->sec - mp2->sec) ? diff :
 392             strcasecmp(mp1->names, mp2->names);
 393 }
 394
 395 static void
 396 buildnames(const struct mansearch *search, struct manpage *mpage,
 397                 sqlite3 *db, sqlite3_stmt *s,
 398                 uint64_t pageid, const char *path, int form)
 399 {
 400         glob_t           globinfo;
 401         char            *firstname, *newnames, *prevsec, *prevarch;
 402         const char      *oldnames, *sep1, *name, *sec, *sep2, *arch, *fsec;
 403         size_t           i;
 404         int              c, globres;
 405
 406         mpage->file = NULL;
 407         mpage->names = NULL;
 408         firstname = prevsec = prevarch = NULL;
 409         i = 1;
 410         SQL_BIND_INT64(db, s, i, pageid);
 411         while (SQLITE_ROW == (c = sqlite3_step(s))) {
 412
 413                 /* Decide whether we already have some names. */
 414
 415                 if (NULL == mpage->names) {
 416                         oldnames = "";
 417                         sep1 = "";
 418                 } else {
 419                         oldnames = mpage->names;
 420                         sep1 = ", ";
 421                 }
 422
 423                 /* Fetch the next name, rejecting sec/arch mismatches. */
 424
 425                 sec = (const char *)sqlite3_column_text(s, 0);
 426                 if (search->sec != NULL && strcasecmp(sec, search->sec))
 427                         continue;
 428                 arch = (const char *)sqlite3_column_text(s, 1);
 429                 if (search->arch != NULL && *arch != '\0' &&
 430                     strcasecmp(arch, search->arch))
 431                         continue;
 432                 name = (const char *)sqlite3_column_text(s, 2);
 433
 434                 /* Remember the first section found. */
 435
 436                 if (9 < mpage->sec && '1' <= *sec && '9' >= *sec)
 437                         mpage->sec = (*sec - '1') + 1;
 438
 439                 /* If the section changed, append the old one. */
 440
 441                 if (NULL != prevsec &&
 442                     (strcmp(sec, prevsec) ||
 443                      strcmp(arch, prevarch))) {
 444                         sep2 = '\0' == *prevarch ? "" : "/";
 445                         mandoc_asprintf(&newnames, "%s(%s%s%s)",
 446                             oldnames, prevsec, sep2, prevarch);
 447                         free(mpage->names);
 448                         oldnames = mpage->names = newnames;
 449                         free(prevsec);
 450                         free(prevarch);
 451                         prevsec = prevarch = NULL;
 452                 }
 453
 454                 /* Save the new section, to append it later. */
 455
 456                 if (NULL == prevsec) {
 457                         prevsec = mandoc_strdup(sec);
 458                         prevarch = mandoc_strdup(arch);
 459                 }
 460
 461                 /* Append the new name. */
 462
 463                 mandoc_asprintf(&newnames, "%s%s%s",
 464                     oldnames, sep1, name);
 465                 free(mpage->names);
 466                 mpage->names = newnames;
 467
 468                 /* Also save the first file name encountered. */
 469
 470                 if (mpage->file != NULL)
 471                         continue;
 472
 473                 if (form & FORM_SRC) {
 474                         sep1 = "man";
 475                         fsec = sec;
 476                 } else {
 477                         sep1 = "cat";
 478                         fsec = "0";
 479                 }
 480                 sep2 = *arch == '\0' ? "" : "/";
 481                 mandoc_asprintf(&mpage->file, "%s/%s%s%s%s/%s.%s",
 482                     path, sep1, sec, sep2, arch, name, fsec);
 483                 if (access(mpage->file, R_OK) != -1)
 484                         continue;
 485
 486                 /* Handle unusual file name extensions. */
 487
 488                 if (firstname == NULL)
 489                         firstname = mpage->file;
 490                 else
 491                         free(mpage->file);
 492                 mandoc_asprintf(&mpage->file, "%s/%s%s%s%s/%s.*",
 493                     path, sep1, sec, sep2, arch, name);
 494                 globres = glob(mpage->file, 0, NULL, &globinfo);
 495                 free(mpage->file);
 496                 mpage->file = globres ? NULL :
 497                     mandoc_strdup(*globinfo.gl_pathv);
 498                 globfree(&globinfo);
 499         }
 500         if (c != SQLITE_DONE)
 501                 warnx("%s", sqlite3_errmsg(db));
 502         sqlite3_reset(s);
 503
 504         /* If none of the files is usable, use the first name. */
 505
 506         if (mpage->file == NULL)
 507                 mpage->file = firstname;
 508         else if (mpage->file != firstname)
 509                 free(firstname);
 510
 511         /* Append one final section to the names. */
 512
 513         if (prevsec != NULL) {
 514                 sep2 = *prevarch == '\0' ? "" : "/";
 515                 mandoc_asprintf(&newnames, "%s(%s%s%s)",
 516                     mpage->names, prevsec, sep2, prevarch);
 517                 free(mpage->names);
 518                 mpage->names = newnames;
 519                 free(prevsec);
 520                 free(prevarch);
 521         }
 522 }
 523
 524 static char *
 525 buildoutput(sqlite3 *db, sqlite3_stmt *s, uint64_t pageid, uint64_t outbit)
 526 {
 527         char            *output, *newoutput;
 528         const char      *oldoutput, *sep1, *data;
 529         size_t           i;
 530         int              c;
 531
 532         output = NULL;
 533         i = 1;
 534         SQL_BIND_INT64(db, s, i, pageid);
 535         SQL_BIND_INT64(db, s, i, outbit);
 536         while (SQLITE_ROW == (c = sqlite3_step(s))) {
 537                 if (NULL == output) {
 538                         oldoutput = "";
 539                         sep1 = "";
 540                 } else {
 541                         oldoutput = output;
 542                         sep1 = " # ";
 543                 }
 544                 data = (const char *)sqlite3_column_text(s, 1);
 545                 mandoc_asprintf(&newoutput, "%s%s%s",
 546                     oldoutput, sep1, data);
 547                 free(output);
 548                 output = newoutput;
 549         }
 550         if (SQLITE_DONE != c)
 551                 warnx("%s", sqlite3_errmsg(db));
 552         sqlite3_reset(s);
 553         return output;
 554 }
 555
 556 /*
 557  * Implement substring match as an application-defined SQL function.
 558  * Using the SQL LIKE or GLOB operators instead would be a bad idea
 559  * because that would require escaping metacharacters in the string
 560  * being searched for.
 561  */
 562 static void
 563 sql_match(sqlite3_context *context, int argc, sqlite3_value **argv)
 564 {
 565
 566         assert(2 == argc);
 567         sqlite3_result_int(context, NULL != strcasestr(
 568             (const char *)sqlite3_value_text(argv[1]),
 569             (const char *)sqlite3_value_text(argv[0])));
 570 }
 571
 572 /*
 573  * Implement regular expression match
 574  * as an application-defined SQL function.
 575  */
 576 static void
 577 sql_regexp(sqlite3_context *context, int argc, sqlite3_value **argv)
 578 {
 579
 580         assert(2 == argc);
 581         sqlite3_result_int(context, !regexec(
 582             (regex_t *)sqlite3_value_blob(argv[0]),
 583             (const char *)sqlite3_value_text(argv[1]),
 584             0, NULL, 0));
 585 }
 586
 587 static void
 588 sql_append(char **sql, size_t *sz, const char *newstr, int count)
 589 {
 590         size_t           newsz;
 591
 592         newsz = 1 < count ? (size_t)count : strlen(newstr);
 593         *sql = mandoc_realloc(*sql, *sz + newsz + 1);
 594         if (1 < count)
 595                 memset(*sql + *sz, *newstr, (size_t)count);
 596         else
 597                 memcpy(*sql + *sz, newstr, newsz);
 598         *sz += newsz;
 599         (*sql)[*sz] = '\0';
 600 }
 601
 602 /*
 603  * Prepare the search SQL statement.
 604  */
 605 static char *
 606 sql_statement(const struct expr *e)
 607 {
 608         char            *sql;
 609         size_t           sz;
 610         int              needop;
 611
 612         sql = mandoc_strdup(e->equal ?
 613             "SELECT desc, form, pageid, bits "
 614                 "FROM mpages NATURAL JOIN names WHERE " :
 615             "SELECT desc, form, pageid, 0 FROM mpages WHERE ");
 616         sz = strlen(sql);
 617
 618         for (needop = 0; NULL != e; e = e->next) {
 619                 if (e->and)
 620                         sql_append(&sql, &sz, " AND ", 1);
 621                 else if (needop)
 622                         sql_append(&sql, &sz, " OR ", 1);
 623                 if (e->open)
 624                         sql_append(&sql, &sz, "(", e->open);
 625                 sql_append(&sql, &sz,
 626                     TYPE_Nd & e->bits
 627                     ? (NULL == e->substr
 628                         ? "desc REGEXP ?"
 629                         : "desc MATCH ?")
 630                     : TYPE_Nm == e->bits
 631                     ? (NULL == e->substr
 632                         ? "pageid IN (SELECT pageid FROM names "
 633                           "WHERE name REGEXP ?)"
 634                         : e->equal
 635                         ? "name = ? "
 636                         : "pageid IN (SELECT pageid FROM names "
 637                           "WHERE name MATCH ?)")
 638                     : (NULL == e->substr
 639                         ? "pageid IN (SELECT pageid FROM keys "
 640                           "WHERE key REGEXP ? AND bits & ?)"
 641                         : "pageid IN (SELECT pageid FROM keys "
 642                           "WHERE key MATCH ? AND bits & ?)"), 1);
 643                 if (e->close)
 644                         sql_append(&sql, &sz, ")", e->close);
 645                 needop = 1;
 646         }
 647
 648         return sql;
 649 }
 650
 651 /*
 652  * Compile a set of string tokens into an expression.
 653  * Tokens in "argv" are assumed to be individual expression atoms (e.g.,
 654  * "(", "foo=bar", etc.).
 655  */
 656 static struct expr *
 657 exprcomp(const struct mansearch *search, int argc, char *argv[])
 658 {
 659         uint64_t         mask;
 660         int              i, toopen, logic, igncase, toclose;
 661         struct expr     *first, *prev, *cur, *next;
 662
 663         first = cur = NULL;
 664         logic = igncase = toopen = toclose = 0;
 665
 666         for (i = 0; i < argc; i++) {
 667                 if (0 == strcmp("(", argv[i])) {
 668                         if (igncase)
 669                                 goto fail;
 670                         toopen++;
 671                         toclose++;
 672                         continue;
 673                 } else if (0 == strcmp(")", argv[i])) {
 674                         if (toopen || logic || igncase || NULL == cur)
 675                                 goto fail;
 676                         cur->close++;
 677                         if (0 > --toclose)
 678                                 goto fail;
 679                         continue;
 680                 } else if (0 == strcmp("-a", argv[i])) {
 681                         if (toopen || logic || igncase || NULL == cur)
 682                                 goto fail;
 683                         logic = 1;
 684                         continue;
 685                 } else if (0 == strcmp("-o", argv[i])) {
 686                         if (toopen || logic || igncase || NULL == cur)
 687                                 goto fail;
 688                         logic = 2;
 689                         continue;
 690                 } else if (0 == strcmp("-i", argv[i])) {
 691                         if (igncase)
 692                                 goto fail;
 693                         igncase = 1;
 694                         continue;
 695                 }
 696                 next = exprterm(search, argv[i], !igncase);
 697                 if (NULL == next)
 698                         goto fail;
 699                 if (NULL == first)
 700                         first = next;
 701                 else
 702                         cur->next = next;
 703                 prev = cur = next;
 704
 705                 /*
 706                  * Searching for descriptions must be split out
 707                  * because they are stored in the mpages table,
 708                  * not in the keys table.
 709                  */
 710
 711                 for (mask = TYPE_Nm; mask <= TYPE_Nd; mask <<= 1) {
 712                         if (mask & cur->bits && ~mask & cur->bits) {
 713                                 next = mandoc_calloc(1,
 714                                     sizeof(struct expr));
 715                                 memcpy(next, cur, sizeof(struct expr));
 716                                 prev->open = 1;
 717                                 cur->bits = mask;
 718                                 cur->next = next;
 719                                 cur = next;
 720                                 cur->bits &= ~mask;
 721                         }
 722                 }
 723                 prev->and = (1 == logic);
 724                 prev->open += toopen;
 725                 if (cur != prev)
 726                         cur->close = 1;
 727
 728                 toopen = logic = igncase = 0;
 729         }
 730         if ( ! (toopen || logic || igncase || toclose))
 731                 return first;
 732
 733 fail:
 734         if (NULL != first)
 735                 exprfree(first);
 736         return NULL;
 737 }
 738
 739 static struct expr *
 740 exprterm(const struct mansearch *search, char *buf, int cs)
 741 {
 742         char             errbuf[BUFSIZ];
 743         struct expr     *e;
 744         char            *key, *val;
 745         uint64_t         iterbit;
 746         int              i, irc;
 747
 748         if ('\0' == *buf)
 749                 return NULL;
 750
 751         e = mandoc_calloc(1, sizeof(struct expr));
 752
 753         if (search->argmode == ARG_NAME) {
 754                 e->bits = TYPE_Nm;
 755                 e->substr = buf;
 756                 e->equal = 1;
 757                 return e;
 758         }
 759
 760         /*
 761          * Separate macro keys from search string.
 762          * If needed, request regular expression handling
 763          * by setting e->substr to NULL.
 764          */
 765
 766         if (search->argmode == ARG_WORD) {
 767                 e->bits = TYPE_Nm;
 768                 e->substr = NULL;
 769                 mandoc_asprintf(&val, "[[:<:]]%s[[:>:]]", buf);
 770                 cs = 0;
 771         } else if ((val = strpbrk(buf, "=~")) == NULL) {
 772                 e->bits = TYPE_Nm | TYPE_Nd;
 773                 e->substr = buf;
 774         } else {
 775                 if (val == buf)
 776                         e->bits = TYPE_Nm | TYPE_Nd;
 777                 if ('=' == *val)
 778                         e->substr = val + 1;
 779                 *val++ = '\0';
 780                 if (NULL != strstr(buf, "arch"))
 781                         cs = 0;
 782         }
 783
 784         /* Compile regular expressions. */
 785
 786         if (NULL == e->substr) {
 787                 irc = regcomp(&e->regexp, val,
 788                     REG_EXTENDED | REG_NOSUB | (cs ? 0 : REG_ICASE));
 789                 if (search->argmode == ARG_WORD)
 790                         free(val);
 791                 if (irc) {
 792                         regerror(irc, &e->regexp, errbuf, sizeof(errbuf));
 793                         warnx("regcomp: %s", errbuf);
 794                         free(e);
 795                         return NULL;
 796                 }
 797         }
 798
 799         if (e->bits)
 800                 return e;
 801
 802         /*
 803          * Parse out all possible fields.
 804          * If the field doesn't resolve, bail.
 805          */
 806
 807         while (NULL != (key = strsep(&buf, ","))) {
 808                 if ('\0' == *key)
 809                         continue;
 810                 for (i = 0, iterbit = 1;
 811                      i < mansearch_keymax;
 812                      i++, iterbit <<= 1) {
 813                         if (0 == strcasecmp(key,
 814                             mansearch_keynames[i])) {
 815                                 e->bits |= iterbit;
 816                                 break;
 817                         }
 818                 }
 819                 if (i == mansearch_keymax) {
 820                         if (strcasecmp(key, "any")) {
 821                                 free(e);
 822                                 return NULL;
 823                         }
 824                         e->bits |= ~0ULL;
 825                 }
 826         }
 827
 828         return e;
 829 }
 830
 831 static void
 832 exprfree(struct expr *p)
 833 {
 834         struct expr     *pp;
 835
 836         while (NULL != p) {
 837                 pp = p->next;
 838                 free(p);
 839                 p = pp;
 840         }
 841 }