]>
git.cameronkatri.com Git - mandoc.git/blob - mansearch.c
1 /* $Id: mansearch.c,v 1.58 2015/10/06 18:32:19 schwarze Exp $ */
3 * Copyright (c) 2012 Kristaps Dzonsons <kristaps@bsd.lv>
4 * Copyright (c) 2013, 2014, 2015 Ingo Schwarze <schwarze@openbsd.org>
6 * Permission to use, copy, modify, and distribute this software for any
7 * purpose with or without fee is hereby granted, provided that the above
8 * copyright notice and this permission notice appear in all copies.
10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHORS DISCLAIM ALL WARRANTIES
11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR
13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
21 #include <sys/types.h>
40 #include "compat_ohash.h"
43 #ifndef SQLITE_DETERMINISTIC
44 #define SQLITE_DETERMINISTIC 0
48 #include "mandoc_aux.h"
50 #include "mansearch.h"
52 extern int mansearch_keymax
;
53 extern const char *const mansearch_keynames
[];
55 #define SQL_BIND_TEXT(_db, _s, _i, _v) \
56 do { if (SQLITE_OK != sqlite3_bind_text \
57 ((_s), (_i)++, (_v), -1, SQLITE_STATIC)) \
58 fprintf(stderr, "%s\n", sqlite3_errmsg((_db))); \
60 #define SQL_BIND_INT64(_db, _s, _i, _v) \
61 do { if (SQLITE_OK != sqlite3_bind_int64 \
62 ((_s), (_i)++, (_v))) \
63 fprintf(stderr, "%s\n", sqlite3_errmsg((_db))); \
65 #define SQL_BIND_BLOB(_db, _s, _i, _v) \
66 do { if (SQLITE_OK != sqlite3_bind_blob \
67 ((_s), (_i)++, (&_v), sizeof(_v), SQLITE_STATIC)) \
68 fprintf(stderr, "%s\n", sqlite3_errmsg((_db))); \
72 regex_t regexp
; /* compiled regexp, if applicable */
73 const char *substr
; /* to search for, if applicable */
74 struct expr
*next
; /* next in sequence */
75 uint64_t bits
; /* type-mask */
76 int equal
; /* equality, not subsring match */
77 int open
; /* opening parentheses before */
78 int and; /* logical AND before */
79 int close
; /* closing parentheses after */
83 uint64_t pageid
; /* identifier in database */
84 uint64_t bits
; /* name type mask */
85 char *desc
; /* manual page description */
86 int form
; /* bit field: formatted, zipped? */
89 static void buildnames(const struct mansearch
*,
90 struct manpage
*, sqlite3
*,
91 sqlite3_stmt
*, uint64_t,
92 const char *, int form
);
93 static char *buildoutput(sqlite3
*, sqlite3_stmt
*,
95 static void *hash_alloc(size_t, void *);
96 static void hash_free(void *, void *);
97 static void *hash_calloc(size_t, size_t, void *);
98 static struct expr
*exprcomp(const struct mansearch
*,
100 static void exprfree(struct expr
*);
101 static struct expr
*exprterm(const struct mansearch
*, char *, int);
102 static int manpage_compare(const void *, const void *);
103 static void sql_append(char **sql
, size_t *sz
,
104 const char *newstr
, int count
);
105 static void sql_match(sqlite3_context
*context
,
106 int argc
, sqlite3_value
**argv
);
107 static void sql_regexp(sqlite3_context
*context
,
108 int argc
, sqlite3_value
**argv
);
109 static char *sql_statement(const struct expr
*);
113 mansearch_setup(int start
)
115 static void *pagecache
;
118 #define PC_PAGESIZE 1280
119 #define PC_NUMPAGES 256
122 if (NULL
!= pagecache
) {
123 fprintf(stderr
, "pagecache already enabled\n");
124 return (int)MANDOCLEVEL_BADARG
;
127 pagecache
= mmap(NULL
, PC_PAGESIZE
* PC_NUMPAGES
,
128 PROT_READ
| PROT_WRITE
,
129 MAP_SHARED
| MAP_ANON
, -1, 0);
131 if (MAP_FAILED
== pagecache
) {
134 return (int)MANDOCLEVEL_SYSERR
;
137 c
= sqlite3_config(SQLITE_CONFIG_PAGECACHE
,
138 pagecache
, PC_PAGESIZE
, PC_NUMPAGES
);
141 return (int)MANDOCLEVEL_OK
;
143 fprintf(stderr
, "pagecache: %s\n", sqlite3_errstr(c
));
145 } else if (NULL
== pagecache
) {
146 fprintf(stderr
, "pagecache missing\n");
147 return (int)MANDOCLEVEL_BADARG
;
150 if (-1 == munmap(pagecache
, PC_PAGESIZE
* PC_NUMPAGES
)) {
153 return (int)MANDOCLEVEL_SYSERR
;
157 return (int)MANDOCLEVEL_OK
;
161 mansearch(const struct mansearch
*search
,
162 const struct manpaths
*paths
,
163 int argc
, char *argv
[],
164 struct manpage
**res
, size_t *sz
)
167 uint64_t outbit
, iterbit
;
170 struct manpage
*mpage
;
173 sqlite3_stmt
*s
, *s2
;
175 struct ohash_info info
;
178 size_t i
, j
, cur
, maxres
;
179 int c
, chdir_status
, getcwd_status
, indexbit
;
181 if (argc
== 0 || (e
= exprcomp(search
, argc
, argv
)) == NULL
) {
186 info
.calloc
= hash_calloc
;
187 info
.alloc
= hash_alloc
;
188 info
.free
= hash_free
;
189 info
.key_offset
= offsetof(struct match
, pageid
);
194 if (NULL
!= search
->outkey
) {
196 for (indexbit
= 0, iterbit
= 1;
197 indexbit
< mansearch_keymax
;
198 indexbit
++, iterbit
<<= 1) {
199 if (0 == strcasecmp(search
->outkey
,
200 mansearch_keynames
[indexbit
])) {
209 * Remember the original working directory, if possible.
210 * This will be needed if the second or a later directory
211 * is given as a relative path.
212 * Do not error out if the current directory is not
213 * searchable: Maybe it won't be needed after all.
216 if (getcwd(buf
, PATH_MAX
) == NULL
) {
218 (void)strlcpy(buf
, strerror(errno
), sizeof(buf
));
222 sql
= sql_statement(e
);
225 * Loop over the directories (containing databases) for us to
227 * Don't let missing/bad databases/directories phase us.
228 * In each, try to open the resident database and, if it opens,
229 * scan it for our match expression.
233 for (i
= 0; i
< paths
->sz
; i
++) {
234 if (chdir_status
&& paths
->paths
[i
][0] != '/') {
235 if ( ! getcwd_status
) {
236 fprintf(stderr
, "%s: getcwd: %s\n",
237 paths
->paths
[i
], buf
);
239 } else if (chdir(buf
) == -1) {
244 if (chdir(paths
->paths
[i
]) == -1) {
245 perror(paths
->paths
[i
]);
250 c
= sqlite3_open_v2(MANDOC_DB
, &db
,
251 SQLITE_OPEN_READONLY
, NULL
);
253 if (SQLITE_OK
!= c
) {
254 fprintf(stderr
, "%s/%s: %s\n",
255 paths
->paths
[i
], MANDOC_DB
, strerror(errno
));
261 * Define the SQL functions for substring
262 * and regular expression matching.
265 c
= sqlite3_create_function(db
, "match", 2,
266 SQLITE_UTF8
| SQLITE_DETERMINISTIC
,
267 NULL
, sql_match
, NULL
, NULL
);
268 assert(SQLITE_OK
== c
);
269 c
= sqlite3_create_function(db
, "regexp", 2,
270 SQLITE_UTF8
| SQLITE_DETERMINISTIC
,
271 NULL
, sql_regexp
, NULL
, NULL
);
272 assert(SQLITE_OK
== c
);
275 c
= sqlite3_prepare_v2(db
, sql
, -1, &s
, NULL
);
277 fprintf(stderr
, "%s\n", sqlite3_errmsg(db
));
279 for (ep
= e
; NULL
!= ep
; ep
= ep
->next
) {
280 if (NULL
== ep
->substr
) {
281 SQL_BIND_BLOB(db
, s
, j
, ep
->regexp
);
283 SQL_BIND_TEXT(db
, s
, j
, ep
->substr
);
284 if (0 == ((TYPE_Nd
| TYPE_Nm
) & ep
->bits
))
285 SQL_BIND_INT64(db
, s
, j
, ep
->bits
);
288 memset(&htab
, 0, sizeof(struct ohash
));
289 ohash_init(&htab
, 4, &info
);
292 * Hash each entry on its [unique] document identifier.
293 * This is a uint64_t.
294 * Instead of using a hash function, simply convert the
295 * uint64_t to a uint32_t, the hash value's type.
296 * This gives good performance and preserves the
297 * distribution of buckets in the table.
299 while (SQLITE_ROW
== (c
= sqlite3_step(s
))) {
300 pageid
= sqlite3_column_int64(s
, 2);
301 idx
= ohash_lookup_memory(&htab
,
302 (char *)&pageid
, sizeof(uint64_t),
305 if (NULL
!= ohash_find(&htab
, idx
))
308 mp
= mandoc_calloc(1, sizeof(struct match
));
310 mp
->form
= sqlite3_column_int(s
, 1);
311 mp
->bits
= sqlite3_column_int64(s
, 3);
312 if (TYPE_Nd
== outbit
)
313 mp
->desc
= mandoc_strdup((const char *)
314 sqlite3_column_text(s
, 0));
315 ohash_insert(&htab
, idx
, mp
);
318 if (SQLITE_DONE
!= c
)
319 fprintf(stderr
, "%s\n", sqlite3_errmsg(db
));
323 c
= sqlite3_prepare_v2(db
,
324 "SELECT sec, arch, name, pageid FROM mlinks "
325 "WHERE pageid=? ORDER BY sec, arch, name",
328 fprintf(stderr
, "%s\n", sqlite3_errmsg(db
));
330 c
= sqlite3_prepare_v2(db
,
331 "SELECT bits, key, pageid FROM keys "
332 "WHERE pageid=? AND bits & ?",
335 fprintf(stderr
, "%s\n", sqlite3_errmsg(db
));
337 for (mp
= ohash_first(&htab
, &idx
);
339 mp
= ohash_next(&htab
, &idx
)) {
340 if (cur
+ 1 > maxres
) {
342 *res
= mandoc_reallocarray(*res
,
343 maxres
, sizeof(struct manpage
));
347 mpage
->bits
= mp
->bits
;
349 mpage
->form
= mp
->form
;
350 buildnames(search
, mpage
, db
, s
, mp
->pageid
,
351 paths
->paths
[i
], mp
->form
);
352 if (mpage
->names
!= NULL
) {
353 mpage
->output
= TYPE_Nd
& outbit
?
355 buildoutput(db
, s2
, mp
->pageid
, outbit
) :
363 sqlite3_finalize(s2
);
368 * In man(1) mode, prefer matches in earlier trees
369 * over matches in later trees.
372 if (cur
&& search
->firstmatch
)
375 qsort(*res
, cur
, sizeof(struct manpage
), manpage_compare
);
376 if (chdir_status
&& getcwd_status
&& chdir(buf
) == -1)
385 mansearch_free(struct manpage
*res
, size_t sz
)
389 for (i
= 0; i
< sz
; i
++) {
398 manpage_compare(const void *vp1
, const void *vp2
)
400 const struct manpage
*mp1
, *mp2
;
405 return (diff
= mp2
->bits
- mp1
->bits
) ? diff
:
406 (diff
= mp1
->sec
- mp2
->sec
) ? diff
:
407 strcasecmp(mp1
->names
, mp2
->names
);
411 buildnames(const struct mansearch
*search
, struct manpage
*mpage
,
412 sqlite3
*db
, sqlite3_stmt
*s
,
413 uint64_t pageid
, const char *path
, int form
)
416 char *firstname
, *newnames
, *prevsec
, *prevarch
;
417 const char *oldnames
, *sep1
, *name
, *sec
, *sep2
, *arch
, *fsec
;
423 firstname
= prevsec
= prevarch
= NULL
;
425 SQL_BIND_INT64(db
, s
, i
, pageid
);
426 while (SQLITE_ROW
== (c
= sqlite3_step(s
))) {
428 /* Decide whether we already have some names. */
430 if (NULL
== mpage
->names
) {
434 oldnames
= mpage
->names
;
438 /* Fetch the next name, rejecting sec/arch mismatches. */
440 sec
= (const char *)sqlite3_column_text(s
, 0);
441 if (search
->sec
!= NULL
&& strcasecmp(sec
, search
->sec
))
443 arch
= (const char *)sqlite3_column_text(s
, 1);
444 if (search
->arch
!= NULL
&& *arch
!= '\0' &&
445 strcasecmp(arch
, search
->arch
))
447 name
= (const char *)sqlite3_column_text(s
, 2);
449 /* Remember the first section found. */
451 if (9 < mpage
->sec
&& '1' <= *sec
&& '9' >= *sec
)
452 mpage
->sec
= (*sec
- '1') + 1;
454 /* If the section changed, append the old one. */
456 if (NULL
!= prevsec
&&
457 (strcmp(sec
, prevsec
) ||
458 strcmp(arch
, prevarch
))) {
459 sep2
= '\0' == *prevarch
? "" : "/";
460 mandoc_asprintf(&newnames
, "%s(%s%s%s)",
461 oldnames
, prevsec
, sep2
, prevarch
);
463 oldnames
= mpage
->names
= newnames
;
466 prevsec
= prevarch
= NULL
;
469 /* Save the new section, to append it later. */
471 if (NULL
== prevsec
) {
472 prevsec
= mandoc_strdup(sec
);
473 prevarch
= mandoc_strdup(arch
);
476 /* Append the new name. */
478 mandoc_asprintf(&newnames
, "%s%s%s",
479 oldnames
, sep1
, name
);
481 mpage
->names
= newnames
;
483 /* Also save the first file name encountered. */
485 if (mpage
->file
!= NULL
)
488 if (form
& FORM_SRC
) {
495 sep2
= *arch
== '\0' ? "" : "/";
496 mandoc_asprintf(&mpage
->file
, "%s/%s%s%s%s/%s.%s",
497 path
, sep1
, sec
, sep2
, arch
, name
, fsec
);
498 if (access(mpage
->file
, R_OK
) != -1)
501 /* Handle unusual file name extensions. */
503 if (firstname
== NULL
)
504 firstname
= mpage
->file
;
507 mandoc_asprintf(&mpage
->file
, "%s/%s%s%s%s/%s.*",
508 path
, sep1
, sec
, sep2
, arch
, name
);
509 globres
= glob(mpage
->file
, 0, NULL
, &globinfo
);
511 mpage
->file
= globres
? NULL
:
512 mandoc_strdup(*globinfo
.gl_pathv
);
515 if (c
!= SQLITE_DONE
)
516 fprintf(stderr
, "%s\n", sqlite3_errmsg(db
));
519 /* If none of the files is usable, use the first name. */
521 if (mpage
->file
== NULL
)
522 mpage
->file
= firstname
;
523 else if (mpage
->file
!= firstname
)
526 /* Append one final section to the names. */
528 if (prevsec
!= NULL
) {
529 sep2
= *prevarch
== '\0' ? "" : "/";
530 mandoc_asprintf(&newnames
, "%s(%s%s%s)",
531 mpage
->names
, prevsec
, sep2
, prevarch
);
533 mpage
->names
= newnames
;
540 buildoutput(sqlite3
*db
, sqlite3_stmt
*s
, uint64_t pageid
, uint64_t outbit
)
542 char *output
, *newoutput
;
543 const char *oldoutput
, *sep1
, *data
;
549 SQL_BIND_INT64(db
, s
, i
, pageid
);
550 SQL_BIND_INT64(db
, s
, i
, outbit
);
551 while (SQLITE_ROW
== (c
= sqlite3_step(s
))) {
552 if (NULL
== output
) {
559 data
= (const char *)sqlite3_column_text(s
, 1);
560 mandoc_asprintf(&newoutput
, "%s%s%s",
561 oldoutput
, sep1
, data
);
565 if (SQLITE_DONE
!= c
)
566 fprintf(stderr
, "%s\n", sqlite3_errmsg(db
));
572 * Implement substring match as an application-defined SQL function.
573 * Using the SQL LIKE or GLOB operators instead would be a bad idea
574 * because that would require escaping metacharacters in the string
575 * being searched for.
578 sql_match(sqlite3_context
*context
, int argc
, sqlite3_value
**argv
)
582 sqlite3_result_int(context
, NULL
!= strcasestr(
583 (const char *)sqlite3_value_text(argv
[1]),
584 (const char *)sqlite3_value_text(argv
[0])));
588 * Implement regular expression match
589 * as an application-defined SQL function.
592 sql_regexp(sqlite3_context
*context
, int argc
, sqlite3_value
**argv
)
596 sqlite3_result_int(context
, !regexec(
597 (regex_t
*)sqlite3_value_blob(argv
[0]),
598 (const char *)sqlite3_value_text(argv
[1]),
603 sql_append(char **sql
, size_t *sz
, const char *newstr
, int count
)
607 newsz
= 1 < count
? (size_t)count
: strlen(newstr
);
608 *sql
= mandoc_realloc(*sql
, *sz
+ newsz
+ 1);
610 memset(*sql
+ *sz
, *newstr
, (size_t)count
);
612 memcpy(*sql
+ *sz
, newstr
, newsz
);
618 * Prepare the search SQL statement.
621 sql_statement(const struct expr
*e
)
627 sql
= mandoc_strdup(e
->equal
?
628 "SELECT desc, form, pageid, bits "
629 "FROM mpages NATURAL JOIN names WHERE " :
630 "SELECT desc, form, pageid, 0 FROM mpages WHERE ");
633 for (needop
= 0; NULL
!= e
; e
= e
->next
) {
635 sql_append(&sql
, &sz
, " AND ", 1);
637 sql_append(&sql
, &sz
, " OR ", 1);
639 sql_append(&sql
, &sz
, "(", e
->open
);
640 sql_append(&sql
, &sz
,
647 ? "pageid IN (SELECT pageid FROM names "
648 "WHERE name REGEXP ?)"
651 : "pageid IN (SELECT pageid FROM names "
652 "WHERE name MATCH ?)")
654 ? "pageid IN (SELECT pageid FROM keys "
655 "WHERE key REGEXP ? AND bits & ?)"
656 : "pageid IN (SELECT pageid FROM keys "
657 "WHERE key MATCH ? AND bits & ?)"), 1);
659 sql_append(&sql
, &sz
, ")", e
->close
);
667 * Compile a set of string tokens into an expression.
668 * Tokens in "argv" are assumed to be individual expression atoms (e.g.,
669 * "(", "foo=bar", etc.).
672 exprcomp(const struct mansearch
*search
, int argc
, char *argv
[])
675 int i
, toopen
, logic
, igncase
, toclose
;
676 struct expr
*first
, *prev
, *cur
, *next
;
679 logic
= igncase
= toopen
= toclose
= 0;
681 for (i
= 0; i
< argc
; i
++) {
682 if (0 == strcmp("(", argv
[i
])) {
688 } else if (0 == strcmp(")", argv
[i
])) {
689 if (toopen
|| logic
|| igncase
|| NULL
== cur
)
695 } else if (0 == strcmp("-a", argv
[i
])) {
696 if (toopen
|| logic
|| igncase
|| NULL
== cur
)
700 } else if (0 == strcmp("-o", argv
[i
])) {
701 if (toopen
|| logic
|| igncase
|| NULL
== cur
)
705 } else if (0 == strcmp("-i", argv
[i
])) {
711 next
= exprterm(search
, argv
[i
], !igncase
);
721 * Searching for descriptions must be split out
722 * because they are stored in the mpages table,
723 * not in the keys table.
726 for (mask
= TYPE_Nm
; mask
<= TYPE_Nd
; mask
<<= 1) {
727 if (mask
& cur
->bits
&& ~mask
& cur
->bits
) {
728 next
= mandoc_calloc(1,
729 sizeof(struct expr
));
730 memcpy(next
, cur
, sizeof(struct expr
));
738 prev
->and = (1 == logic
);
739 prev
->open
+= toopen
;
743 toopen
= logic
= igncase
= 0;
745 if ( ! (toopen
|| logic
|| igncase
|| toclose
))
755 exprterm(const struct mansearch
*search
, char *buf
, int cs
)
766 e
= mandoc_calloc(1, sizeof(struct expr
));
768 if (search
->argmode
== ARG_NAME
) {
776 * Separate macro keys from search string.
777 * If needed, request regular expression handling
778 * by setting e->substr to NULL.
781 if (search
->argmode
== ARG_WORD
) {
784 mandoc_asprintf(&val
, "[[:<:]]%s[[:>:]]", buf
);
786 } else if ((val
= strpbrk(buf
, "=~")) == NULL
) {
787 e
->bits
= TYPE_Nm
| TYPE_Nd
;
791 e
->bits
= TYPE_Nm
| TYPE_Nd
;
795 if (NULL
!= strstr(buf
, "arch"))
799 /* Compile regular expressions. */
801 if (NULL
== e
->substr
) {
802 irc
= regcomp(&e
->regexp
, val
,
803 REG_EXTENDED
| REG_NOSUB
| (cs
? 0 : REG_ICASE
));
804 if (search
->argmode
== ARG_WORD
)
807 regerror(irc
, &e
->regexp
, errbuf
, sizeof(errbuf
));
808 fprintf(stderr
, "regcomp: %s\n", errbuf
);
818 * Parse out all possible fields.
819 * If the field doesn't resolve, bail.
822 while (NULL
!= (key
= strsep(&buf
, ","))) {
825 for (i
= 0, iterbit
= 1;
826 i
< mansearch_keymax
;
827 i
++, iterbit
<<= 1) {
828 if (0 == strcasecmp(key
,
829 mansearch_keynames
[i
])) {
834 if (i
== mansearch_keymax
) {
835 if (strcasecmp(key
, "any")) {
847 exprfree(struct expr
*p
)
859 hash_calloc(size_t nmemb
, size_t sz
, void *arg
)
862 return mandoc_calloc(nmemb
, sz
);
866 hash_alloc(size_t sz
, void *arg
)
869 return mandoc_malloc(sz
);
873 hash_free(void *p
, void *arg
)