read.c

   1 /*      $Id: read.c,v 1.141 2015/09/14 15:36:14 schwarze Exp $ */
   2 /*
   3  * Copyright (c) 2008, 2009, 2010, 2011 Kristaps Dzonsons <kristaps@bsd.lv>
   4  * Copyright (c) 2010-2015 Ingo Schwarze <schwarze@openbsd.org>
   5  * Copyright (c) 2010, 2012 Joerg Sonnenberger <joerg@netbsd.org>
   6  *
   7  * Permission to use, copy, modify, and distribute this software for any
   8  * purpose with or without fee is hereby granted, provided that the above
   9  * copyright notice and this permission notice appear in all copies.
  10  *
  11  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHORS DISCLAIM ALL WARRANTIES
  12  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
  13  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR
  14  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
  15  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
  16  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
  17  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  18  */
  19 #include "config.h"
  20
  21 #include <sys/types.h>
  22 #if HAVE_MMAP
  23 #include <sys/mman.h>
  24 #include <sys/stat.h>
  25 #endif
  26
  27 #include <assert.h>
  28 #include <ctype.h>
  29 #include <errno.h>
  30 #include <fcntl.h>
  31 #include <stdarg.h>
  32 #include <stdint.h>
  33 #include <stdio.h>
  34 #include <stdlib.h>
  35 #include <string.h>
  36 #include <unistd.h>
  37 #include <zlib.h>
  38
  39 #include "mandoc_aux.h"
  40 #include "mandoc.h"
  41 #include "roff.h"
  42 #include "mdoc.h"
  43 #include "man.h"
  44 #include "libmandoc.h"
  45 #include "roff_int.h"
  46
  47 #define REPARSE_LIMIT   1000
  48
  49 struct  mparse {
  50         struct roff_man  *man; /* man parser */
  51         struct roff      *roff; /* roff parser (!NULL) */
  52         const struct mchars *mchars; /* character table */
  53         char             *sodest; /* filename pointed to by .so */
  54         const char       *file; /* filename of current input file */
  55         struct buf       *primary; /* buffer currently being parsed */
  56         struct buf       *secondary; /* preprocessed copy of input */
  57         const char       *defos; /* default operating system */
  58         mandocmsg         mmsg; /* warning/error message handler */
  59         enum mandoclevel  file_status; /* status of current parse */
  60         enum mandoclevel  wlevel; /* ignore messages below this */
  61         int               options; /* parser options */
  62         int               gzip; /* current input file is gzipped */
  63         int               filenc; /* encoding of the current file */
  64         int               reparse_count; /* finite interp. stack */
  65         int               line; /* line number in the file */
  66 };
  67
  68 static  void      choose_parser(struct mparse *);
  69 static  void      resize_buf(struct buf *, size_t);
  70 static  void      mparse_buf_r(struct mparse *, struct buf, size_t, int);
  71 static  int       read_whole_file(struct mparse *, const char *, int,
  72                                 struct buf *, int *);
  73 static  void      mparse_end(struct mparse *);
  74 static  void      mparse_parse_buffer(struct mparse *, struct buf,
  75                         const char *);
  76
  77 static  const enum mandocerr    mandoclimits[MANDOCLEVEL_MAX] = {
  78         MANDOCERR_OK,
  79         MANDOCERR_WARNING,
  80         MANDOCERR_WARNING,
  81         MANDOCERR_ERROR,
  82         MANDOCERR_UNSUPP,
  83         MANDOCERR_MAX,
  84         MANDOCERR_MAX
  85 };
  86
  87 static  const char * const      mandocerrs[MANDOCERR_MAX] = {
  88         "ok",
  89
  90         "generic warning",
  91
  92         /* related to the prologue */
  93         "missing manual title, using UNTITLED",
  94         "missing manual title, using \"\"",
  95         "lower case character in document title",
  96         "missing manual section, using \"\"",
  97         "unknown manual section",
  98         "missing date, using today's date",
  99         "cannot parse date, using it verbatim",
 100         "missing Os macro, using \"\"",
 101         "duplicate prologue macro",
 102         "late prologue macro",
 103         "skipping late title macro",
 104         "prologue macros out of order",
 105
 106         /* related to document structure */
 107         ".so is fragile, better use ln(1)",
 108         "no document body",
 109         "content before first section header",
 110         "first section is not \"NAME\"",
 111         "NAME section without name",
 112         "NAME section without description",
 113         "description not at the end of NAME",
 114         "bad NAME section content",
 115         "missing description line, using \"\"",
 116         "sections out of conventional order",
 117         "duplicate section title",
 118         "unexpected section",
 119         "unusual Xr order",
 120         "unusual Xr punctuation",
 121         "AUTHORS section without An macro",
 122
 123         /* related to macros and nesting */
 124         "obsolete macro",
 125         "macro neither callable nor escaped",
 126         "skipping paragraph macro",
 127         "moving paragraph macro out of list",
 128         "skipping no-space macro",
 129         "blocks badly nested",
 130         "nested displays are not portable",
 131         "moving content out of list",
 132         "fill mode already enabled, skipping",
 133         "fill mode already disabled, skipping",
 134         "line scope broken",
 135
 136         /* related to missing macro arguments */
 137         "skipping empty request",
 138         "conditional request controls empty scope",
 139         "skipping empty macro",
 140         "empty block",
 141         "empty argument, using 0n",
 142         "missing display type, using -ragged",
 143         "list type is not the first argument",
 144         "missing -width in -tag list, using 8n",
 145         "missing utility name, using \"\"",
 146         "missing function name, using \"\"",
 147         "empty head in list item",
 148         "empty list item",
 149         "missing font type, using \\fR",
 150         "unknown font type, using \\fR",
 151         "nothing follows prefix",
 152         "empty reference block",
 153         "missing -std argument, adding it",
 154         "missing option string, using \"\"",
 155         "missing resource identifier, using \"\"",
 156         "missing eqn box, using \"\"",
 157
 158         /* related to bad macro arguments */
 159         "unterminated quoted argument",
 160         "duplicate argument",
 161         "skipping duplicate argument",
 162         "skipping duplicate display type",
 163         "skipping duplicate list type",
 164         "skipping -width argument",
 165         "wrong number of cells",
 166         "unknown AT&T UNIX version",
 167         "comma in function argument",
 168         "parenthesis in function name",
 169         "invalid content in Rs block",
 170         "invalid Boolean argument",
 171         "unknown font, skipping request",
 172         "odd number of characters in request",
 173
 174         /* related to plain text */
 175         "blank line in fill mode, using .sp",
 176         "tab in filled text",
 177         "whitespace at end of input line",
 178         "bad comment style",
 179         "invalid escape sequence",
 180         "undefined string, using \"\"",
 181
 182         /* related to tables */
 183         "tbl line starts with span",
 184         "tbl column starts with span",
 185         "skipping vertical bar in tbl layout",
 186
 187         "generic error",
 188
 189         /* related to tables */
 190         "non-alphabetic character in tbl options",
 191         "skipping unknown tbl option",
 192         "missing tbl option argument",
 193         "wrong tbl option argument size",
 194         "empty tbl layout",
 195         "invalid character in tbl layout",
 196         "unmatched parenthesis in tbl layout",
 197         "tbl without any data cells",
 198         "ignoring data in spanned tbl cell",
 199         "ignoring extra tbl data cells",
 200         "data block open at end of tbl",
 201
 202         /* related to document structure and macros */
 203         NULL,
 204         "input stack limit exceeded, infinite loop?",
 205         "skipping bad character",
 206         "skipping unknown macro",
 207         "skipping insecure request",
 208         "skipping item outside list",
 209         "skipping column outside column list",
 210         "skipping end of block that is not open",
 211         "fewer RS blocks open, skipping",
 212         "inserting missing end of block",
 213         "appending missing end of block",
 214
 215         /* related to request and macro arguments */
 216         "escaped character not allowed in a name",
 217         "NOT IMPLEMENTED: Bd -file",
 218         "missing list type, using -item",
 219         "missing manual name, using \"\"",
 220         "uname(3) system call failed, using UNKNOWN",
 221         "unknown standard specifier",
 222         "skipping request without numeric argument",
 223         "NOT IMPLEMENTED: .so with absolute path or \"..\"",
 224         ".so request failed",
 225         "skipping all arguments",
 226         "skipping excess arguments",
 227         "divide by zero",
 228
 229         "unsupported feature",
 230         "input too large",
 231         "unsupported control character",
 232         "unsupported roff request",
 233         "eqn delim option in tbl",
 234         "unsupported tbl layout modifier",
 235         "ignoring macro in table",
 236 };
 237
 238 static  const char * const      mandoclevels[MANDOCLEVEL_MAX] = {
 239         "SUCCESS",
 240         "RESERVED",
 241         "WARNING",
 242         "ERROR",
 243         "UNSUPP",
 244         "BADARG",
 245         "SYSERR"
 246 };
 247
 248
 249 static void
 250 resize_buf(struct buf *buf, size_t initial)
 251 {
 252
 253         buf->sz = buf->sz > initial/2 ? 2 * buf->sz : initial;
 254         buf->buf = mandoc_realloc(buf->buf, buf->sz);
 255 }
 256
 257 static void
 258 choose_parser(struct mparse *curp)
 259 {
 260         char            *cp, *ep;
 261         int              format;
 262
 263         /*
 264          * If neither command line arguments -mdoc or -man select
 265          * a parser nor the roff parser found a .Dd or .TH macro
 266          * yet, look ahead in the main input buffer.
 267          */
 268
 269         if ((format = roff_getformat(curp->roff)) == 0) {
 270                 cp = curp->primary->buf;
 271                 ep = cp + curp->primary->sz;
 272                 while (cp < ep) {
 273                         if (*cp == '.' || *cp == '\'') {
 274                                 cp++;
 275                                 if (cp[0] == 'D' && cp[1] == 'd') {
 276                                         format = MPARSE_MDOC;
 277                                         break;
 278                                 }
 279                                 if (cp[0] == 'T' && cp[1] == 'H') {
 280                                         format = MPARSE_MAN;
 281                                         break;
 282                                 }
 283                         }
 284                         cp = memchr(cp, '\n', ep - cp);
 285                         if (cp == NULL)
 286                                 break;
 287                         cp++;
 288                 }
 289         }
 290
 291         if (curp->man == NULL) {
 292                 curp->man = roff_man_alloc(curp->roff, curp, curp->defos,
 293                     curp->options & MPARSE_QUICK ? 1 : 0);
 294                 curp->man->macroset = MACROSET_MAN;
 295                 curp->man->first->tok = TOKEN_NONE;
 296         }
 297
 298         if (format == MPARSE_MDOC) {
 299                 mdoc_hash_init();
 300                 curp->man->macroset = MACROSET_MDOC;
 301                 curp->man->first->tok = TOKEN_NONE;
 302         } else {
 303                 man_hash_init();
 304                 curp->man->macroset = MACROSET_MAN;
 305                 curp->man->first->tok = TOKEN_NONE;
 306         }
 307 }
 308
 309 /*
 310  * Main parse routine for a buffer.
 311  * It assumes encoding and line numbering are already set up.
 312  * It can recurse directly (for invocations of user-defined
 313  * macros, inline equations, and input line traps)
 314  * and indirectly (for .so file inclusion).
 315  */
 316 static void
 317 mparse_buf_r(struct mparse *curp, struct buf blk, size_t i, int start)
 318 {
 319         const struct tbl_span   *span;
 320         struct buf       ln;
 321         const char      *save_file;
 322         char            *cp;
 323         size_t           pos; /* byte number in the ln buffer */
 324         enum rofferr     rr;
 325         int              of;
 326         int              lnn; /* line number in the real file */
 327         int              fd;
 328         unsigned char    c;
 329
 330         memset(&ln, 0, sizeof(ln));
 331
 332         lnn = curp->line;
 333         pos = 0;
 334
 335         while (i < blk.sz) {
 336                 if (0 == pos && '\0' == blk.buf[i])
 337                         break;
 338
 339                 if (start) {
 340                         curp->line = lnn;
 341                         curp->reparse_count = 0;
 342
 343                         if (lnn < 3 &&
 344                             curp->filenc & MPARSE_UTF8 &&
 345                             curp->filenc & MPARSE_LATIN1)
 346                                 curp->filenc = preconv_cue(&blk, i);
 347                 }
 348
 349                 while (i < blk.sz && (start || blk.buf[i] != '\0')) {
 350
 351                         /*
 352                          * When finding an unescaped newline character,
 353                          * leave the character loop to process the line.
 354                          * Skip a preceding carriage return, if any.
 355                          */
 356
 357                         if ('\r' == blk.buf[i] && i + 1 < blk.sz &&
 358                             '\n' == blk.buf[i + 1])
 359                                 ++i;
 360                         if ('\n' == blk.buf[i]) {
 361                                 ++i;
 362                                 ++lnn;
 363                                 break;
 364                         }
 365
 366                         /*
 367                          * Make sure we have space for the worst
 368                          * case of 11 bytes: "\\[u10ffff]\0"
 369                          */
 370
 371                         if (pos + 11 > ln.sz)
 372                                 resize_buf(&ln, 256);
 373
 374                         /*
 375                          * Encode 8-bit input.
 376                          */
 377
 378                         c = blk.buf[i];
 379                         if (c & 0x80) {
 380                                 if ( ! (curp->filenc && preconv_encode(
 381                                     &blk, &i, &ln, &pos, &curp->filenc))) {
 382                                         mandoc_vmsg(MANDOCERR_CHAR_BAD, curp,
 383                                             curp->line, pos, "0x%x", c);
 384                                         ln.buf[pos++] = '?';
 385                                         i++;
 386                                 }
 387                                 continue;
 388                         }
 389
 390                         /*
 391                          * Exclude control characters.
 392                          */
 393
 394                         if (c == 0x7f || (c < 0x20 && c != 0x09)) {
 395                                 mandoc_vmsg(c == 0x00 || c == 0x04 ||
 396                                     c > 0x0a ? MANDOCERR_CHAR_BAD :
 397                                     MANDOCERR_CHAR_UNSUPP,
 398                                     curp, curp->line, pos, "0x%x", c);
 399                                 i++;
 400                                 if (c != '\r')
 401                                         ln.buf[pos++] = '?';
 402                                 continue;
 403                         }
 404
 405                         /* Trailing backslash = a plain char. */
 406
 407                         if (blk.buf[i] != '\\' || i + 1 == blk.sz) {
 408                                 ln.buf[pos++] = blk.buf[i++];
 409                                 continue;
 410                         }
 411
 412                         /*
 413                          * Found escape and at least one other character.
 414                          * When it's a newline character, skip it.
 415                          * When there is a carriage return in between,
 416                          * skip that one as well.
 417                          */
 418
 419                         if ('\r' == blk.buf[i + 1] && i + 2 < blk.sz &&
 420                             '\n' == blk.buf[i + 2])
 421                                 ++i;
 422                         if ('\n' == blk.buf[i + 1]) {
 423                                 i += 2;
 424                                 ++lnn;
 425                                 continue;
 426                         }
 427
 428                         if ('"' == blk.buf[i + 1] || '#' == blk.buf[i + 1]) {
 429                                 i += 2;
 430                                 /* Comment, skip to end of line */
 431                                 for (; i < blk.sz; ++i) {
 432                                         if ('\n' == blk.buf[i]) {
 433                                                 ++i;
 434                                                 ++lnn;
 435                                                 break;
 436                                         }
 437                                 }
 438
 439                                 /* Backout trailing whitespaces */
 440                                 for (; pos > 0; --pos) {
 441                                         if (ln.buf[pos - 1] != ' ')
 442                                                 break;
 443                                         if (pos > 2 && ln.buf[pos - 2] == '\\')
 444                                                 break;
 445                                 }
 446                                 break;
 447                         }
 448
 449                         /* Catch escaped bogus characters. */
 450
 451                         c = (unsigned char) blk.buf[i+1];
 452
 453                         if ( ! (isascii(c) &&
 454                             (isgraph(c) || isblank(c)))) {
 455                                 mandoc_vmsg(MANDOCERR_CHAR_BAD, curp,
 456                                     curp->line, pos, "0x%x", c);
 457                                 i += 2;
 458                                 ln.buf[pos++] = '?';
 459                                 continue;
 460                         }
 461
 462                         /* Some other escape sequence, copy & cont. */
 463
 464                         ln.buf[pos++] = blk.buf[i++];
 465                         ln.buf[pos++] = blk.buf[i++];
 466                 }
 467
 468                 if (pos >= ln.sz)
 469                         resize_buf(&ln, 256);
 470
 471                 ln.buf[pos] = '\0';
 472
 473                 /*
 474                  * A significant amount of complexity is contained by
 475                  * the roff preprocessor.  It's line-oriented but can be
 476                  * expressed on one line, so we need at times to
 477                  * readjust our starting point and re-run it.  The roff
 478                  * preprocessor can also readjust the buffers with new
 479                  * data, so we pass them in wholesale.
 480                  */
 481
 482                 of = 0;
 483
 484                 /*
 485                  * Maintain a lookaside buffer of all parsed lines.  We
 486                  * only do this if mparse_keep() has been invoked (the
 487                  * buffer may be accessed with mparse_getkeep()).
 488                  */
 489
 490                 if (curp->secondary) {
 491                         curp->secondary->buf = mandoc_realloc(
 492                             curp->secondary->buf,
 493                             curp->secondary->sz + pos + 2);
 494                         memcpy(curp->secondary->buf +
 495                             curp->secondary->sz,
 496                             ln.buf, pos);
 497                         curp->secondary->sz += pos;
 498                         curp->secondary->buf
 499                                 [curp->secondary->sz] = '\n';
 500                         curp->secondary->sz++;
 501                         curp->secondary->buf
 502                                 [curp->secondary->sz] = '\0';
 503                 }
 504 rerun:
 505                 rr = roff_parseln(curp->roff, curp->line, &ln, &of);
 506
 507                 switch (rr) {
 508                 case ROFF_REPARSE:
 509                         if (REPARSE_LIMIT >= ++curp->reparse_count)
 510                                 mparse_buf_r(curp, ln, of, 0);
 511                         else
 512                                 mandoc_msg(MANDOCERR_ROFFLOOP, curp,
 513                                     curp->line, pos, NULL);
 514                         pos = 0;
 515                         continue;
 516                 case ROFF_APPEND:
 517                         pos = strlen(ln.buf);
 518                         continue;
 519                 case ROFF_RERUN:
 520                         goto rerun;
 521                 case ROFF_IGN:
 522                         pos = 0;
 523                         continue;
 524                 case ROFF_SO:
 525                         if ( ! (curp->options & MPARSE_SO) &&
 526                             (i >= blk.sz || blk.buf[i] == '\0')) {
 527                                 curp->sodest = mandoc_strdup(ln.buf + of);
 528                                 free(ln.buf);
 529                                 return;
 530                         }
 531                         /*
 532                          * We remove `so' clauses from our lookaside
 533                          * buffer because we're going to descend into
 534                          * the file recursively.
 535                          */
 536                         if (curp->secondary)
 537                                 curp->secondary->sz -= pos + 1;
 538                         save_file = curp->file;
 539                         if (mparse_open(curp, &fd, ln.buf + of) ==
 540                             MANDOCLEVEL_OK) {
 541                                 mparse_readfd(curp, fd, ln.buf + of);
 542                                 curp->file = save_file;
 543                         } else {
 544                                 curp->file = save_file;
 545                                 mandoc_vmsg(MANDOCERR_SO_FAIL,
 546                                     curp, curp->line, pos,
 547                                     ".so %s", ln.buf + of);
 548                                 ln.sz = mandoc_asprintf(&cp,
 549                                     ".sp\nSee the file %s.\n.sp",
 550                                     ln.buf + of);
 551                                 free(ln.buf);
 552                                 ln.buf = cp;
 553                                 of = 0;
 554                                 mparse_buf_r(curp, ln, of, 0);
 555                         }
 556                         pos = 0;
 557                         continue;
 558                 default:
 559                         break;
 560                 }
 561
 562                 /*
 563                  * If input parsers have not been allocated, do so now.
 564                  * We keep these instanced between parsers, but set them
 565                  * locally per parse routine since we can use different
 566                  * parsers with each one.
 567                  */
 568
 569                 if (curp->man == NULL ||
 570                     curp->man->macroset == MACROSET_NONE)
 571                         choose_parser(curp);
 572
 573                 /*
 574                  * Lastly, push down into the parsers themselves.
 575                  * If libroff returns ROFF_TBL, then add it to the
 576                  * currently open parse.  Since we only get here if
 577                  * there does exist data (see tbl_data.c), we're
 578                  * guaranteed that something's been allocated.
 579                  * Do the same for ROFF_EQN.
 580                  */
 581
 582                 if (rr == ROFF_TBL)
 583                         while ((span = roff_span(curp->roff)) != NULL)
 584                                 roff_addtbl(curp->man, span);
 585                 else if (rr == ROFF_EQN)
 586                         roff_addeqn(curp->man, roff_eqn(curp->roff));
 587                 else if ((curp->man->macroset == MACROSET_MDOC ?
 588                     mdoc_parseln(curp->man, curp->line, ln.buf, of) :
 589                     man_parseln(curp->man, curp->line, ln.buf, of)) == 2)
 590                                 break;
 591
 592                 /* Temporary buffers typically are not full. */
 593
 594                 if (0 == start && '\0' == blk.buf[i])
 595                         break;
 596
 597                 /* Start the next input line. */
 598
 599                 pos = 0;
 600         }
 601
 602         free(ln.buf);
 603 }
 604
 605 static int
 606 read_whole_file(struct mparse *curp, const char *file, int fd,
 607                 struct buf *fb, int *with_mmap)
 608 {
 609         gzFile           gz;
 610         size_t           off;
 611         ssize_t          ssz;
 612
 613 #if HAVE_MMAP
 614         struct stat      st;
 615         if (-1 == fstat(fd, &st)) {
 616                 perror(file);
 617                 exit((int)MANDOCLEVEL_SYSERR);
 618         }
 619
 620         /*
 621          * If we're a regular file, try just reading in the whole entry
 622          * via mmap().  This is faster than reading it into blocks, and
 623          * since each file is only a few bytes to begin with, I'm not
 624          * concerned that this is going to tank any machines.
 625          */
 626
 627         if (curp->gzip == 0 && S_ISREG(st.st_mode)) {
 628                 if (st.st_size > 0x7fffffff) {
 629                         mandoc_msg(MANDOCERR_TOOLARGE, curp, 0, 0, NULL);
 630                         return(0);
 631                 }
 632                 *with_mmap = 1;
 633                 fb->sz = (size_t)st.st_size;
 634                 fb->buf = mmap(NULL, fb->sz, PROT_READ, MAP_SHARED, fd, 0);
 635                 if (fb->buf != MAP_FAILED)
 636                         return(1);
 637         }
 638 #endif
 639
 640         if (curp->gzip) {
 641                 if ((gz = gzdopen(fd, "rb")) == NULL) {
 642                         perror(file);
 643                         exit((int)MANDOCLEVEL_SYSERR);
 644                 }
 645         } else
 646                 gz = NULL;
 647
 648         /*
 649          * If this isn't a regular file (like, say, stdin), then we must
 650          * go the old way and just read things in bit by bit.
 651          */
 652
 653         *with_mmap = 0;
 654         off = 0;
 655         fb->sz = 0;
 656         fb->buf = NULL;
 657         for (;;) {
 658                 if (off == fb->sz) {
 659                         if (fb->sz == (1U << 31)) {
 660                                 mandoc_msg(MANDOCERR_TOOLARGE, curp,
 661                                     0, 0, NULL);
 662                                 break;
 663                         }
 664                         resize_buf(fb, 65536);
 665                 }
 666                 ssz = curp->gzip ?
 667                     gzread(gz, fb->buf + (int)off, fb->sz - off) :
 668                     read(fd, fb->buf + (int)off, fb->sz - off);
 669                 if (ssz == 0) {
 670                         fb->sz = off;
 671                         return(1);
 672                 }
 673                 if (ssz == -1) {
 674                         perror(file);
 675                         exit((int)MANDOCLEVEL_SYSERR);
 676                 }
 677                 off += (size_t)ssz;
 678         }
 679
 680         free(fb->buf);
 681         fb->buf = NULL;
 682         return(0);
 683 }
 684
 685 static void
 686 mparse_end(struct mparse *curp)
 687 {
 688
 689         if (curp->man == NULL && curp->sodest == NULL)
 690                 curp->man = roff_man_alloc(curp->roff, curp, curp->defos,
 691                     curp->options & MPARSE_QUICK ? 1 : 0);
 692         if (curp->man->macroset == MACROSET_NONE)
 693                 curp->man->macroset = MACROSET_MAN;
 694         if (curp->man->macroset == MACROSET_MDOC)
 695                 mdoc_endparse(curp->man);
 696         else
 697                 man_endparse(curp->man);
 698         roff_endparse(curp->roff);
 699 }
 700
 701 static void
 702 mparse_parse_buffer(struct mparse *curp, struct buf blk, const char *file)
 703 {
 704         struct buf      *svprimary;
 705         const char      *svfile;
 706         size_t           offset;
 707         static int       recursion_depth;
 708
 709         if (64 < recursion_depth) {
 710                 mandoc_msg(MANDOCERR_ROFFLOOP, curp, curp->line, 0, NULL);
 711                 return;
 712         }
 713
 714         /* Line number is per-file. */
 715         svfile = curp->file;
 716         curp->file = file;
 717         svprimary = curp->primary;
 718         curp->primary = &blk;
 719         curp->line = 1;
 720         recursion_depth++;
 721
 722         /* Skip an UTF-8 byte order mark. */
 723         if (curp->filenc & MPARSE_UTF8 && blk.sz > 2 &&
 724             (unsigned char)blk.buf[0] == 0xef &&
 725             (unsigned char)blk.buf[1] == 0xbb &&
 726             (unsigned char)blk.buf[2] == 0xbf) {
 727                 offset = 3;
 728                 curp->filenc &= ~MPARSE_LATIN1;
 729         } else
 730                 offset = 0;
 731
 732         mparse_buf_r(curp, blk, offset, 1);
 733
 734         if (--recursion_depth == 0)
 735                 mparse_end(curp);
 736
 737         curp->primary = svprimary;
 738         curp->file = svfile;
 739 }
 740
 741 enum mandoclevel
 742 mparse_readmem(struct mparse *curp, void *buf, size_t len,
 743                 const char *file)
 744 {
 745         struct buf blk;
 746
 747         blk.buf = buf;
 748         blk.sz = len;
 749
 750         mparse_parse_buffer(curp, blk, file);
 751         return(curp->file_status);
 752 }
 753
 754 /*
 755  * Read the whole file into memory and call the parsers.
 756  * Called recursively when an .so request is encountered.
 757  */
 758 enum mandoclevel
 759 mparse_readfd(struct mparse *curp, int fd, const char *file)
 760 {
 761         struct buf       blk;
 762         int              with_mmap;
 763         int              save_filenc;
 764
 765         if (read_whole_file(curp, file, fd, &blk, &with_mmap)) {
 766                 save_filenc = curp->filenc;
 767                 curp->filenc = curp->options &
 768                     (MPARSE_UTF8 | MPARSE_LATIN1);
 769                 mparse_parse_buffer(curp, blk, file);
 770                 curp->filenc = save_filenc;
 771 #if HAVE_MMAP
 772                 if (with_mmap)
 773                         munmap(blk.buf, blk.sz);
 774                 else
 775 #endif
 776                         free(blk.buf);
 777         }
 778
 779         if (fd != STDIN_FILENO && close(fd) == -1)
 780                 perror(file);
 781
 782         return(curp->file_status);
 783 }
 784
 785 enum mandoclevel
 786 mparse_open(struct mparse *curp, int *fd, const char *file)
 787 {
 788         char             *cp;
 789
 790         curp->file = file;
 791         cp = strrchr(file, '.');
 792         curp->gzip = (cp != NULL && ! strcmp(cp + 1, "gz"));
 793
 794         /* First try to use the filename as it is. */
 795
 796         if ((*fd = open(file, O_RDONLY)) != -1)
 797                 return(MANDOCLEVEL_OK);
 798
 799         /*
 800          * If that doesn't work and the filename doesn't
 801          * already  end in .gz, try appending .gz.
 802          */
 803
 804         if ( ! curp->gzip) {
 805                 mandoc_asprintf(&cp, "%s.gz", file);
 806                 *fd = open(file, O_RDONLY);
 807                 free(cp);
 808                 if (*fd != -1) {
 809                         curp->gzip = 1;
 810                         return(MANDOCLEVEL_OK);
 811                 }
 812         }
 813
 814         /* Neither worked, give up. */
 815
 816         mandoc_msg(MANDOCERR_FILE, curp, 0, 0, strerror(errno));
 817         return(MANDOCLEVEL_ERROR);
 818 }
 819
 820 struct mparse *
 821 mparse_alloc(int options, enum mandoclevel wlevel, mandocmsg mmsg,
 822     const struct mchars *mchars, const char *defos)
 823 {
 824         struct mparse   *curp;
 825
 826         curp = mandoc_calloc(1, sizeof(struct mparse));
 827
 828         curp->options = options;
 829         curp->wlevel = wlevel;
 830         curp->mmsg = mmsg;
 831         curp->defos = defos;
 832
 833         curp->mchars = mchars;
 834         curp->roff = roff_alloc(curp, curp->mchars, options);
 835         curp->man = roff_man_alloc( curp->roff, curp, curp->defos,
 836                 curp->options & MPARSE_QUICK ? 1 : 0);
 837         if (curp->options & MPARSE_MDOC) {
 838                 mdoc_hash_init();
 839                 curp->man->macroset = MACROSET_MDOC;
 840         } else if (curp->options & MPARSE_MAN) {
 841                 man_hash_init();
 842                 curp->man->macroset = MACROSET_MAN;
 843         }
 844         curp->man->first->tok = TOKEN_NONE;
 845         return(curp);
 846 }
 847
 848 void
 849 mparse_reset(struct mparse *curp)
 850 {
 851
 852         roff_reset(curp->roff);
 853
 854         if (curp->man != NULL)
 855                 roff_man_reset(curp->man);
 856         if (curp->secondary)
 857                 curp->secondary->sz = 0;
 858
 859         curp->file_status = MANDOCLEVEL_OK;
 860
 861         free(curp->sodest);
 862         curp->sodest = NULL;
 863 }
 864
 865 void
 866 mparse_free(struct mparse *curp)
 867 {
 868
 869         roff_man_free(curp->man);
 870         if (curp->roff)
 871                 roff_free(curp->roff);
 872         if (curp->secondary)
 873                 free(curp->secondary->buf);
 874
 875         free(curp->secondary);
 876         free(curp->sodest);
 877         free(curp);
 878 }
 879
 880 void
 881 mparse_result(struct mparse *curp, struct roff_man **man,
 882         char **sodest)
 883 {
 884
 885         if (sodest && NULL != (*sodest = curp->sodest)) {
 886                 *man = NULL;
 887                 return;
 888         }
 889         if (man)
 890                 *man = curp->man;
 891 }
 892
 893 void
 894 mandoc_vmsg(enum mandocerr t, struct mparse *m,
 895                 int ln, int pos, const char *fmt, ...)
 896 {
 897         char             buf[256];
 898         va_list          ap;
 899
 900         va_start(ap, fmt);
 901         (void)vsnprintf(buf, sizeof(buf), fmt, ap);
 902         va_end(ap);
 903
 904         mandoc_msg(t, m, ln, pos, buf);
 905 }
 906
 907 void
 908 mandoc_msg(enum mandocerr er, struct mparse *m,
 909                 int ln, int col, const char *msg)
 910 {
 911         enum mandoclevel level;
 912
 913         level = MANDOCLEVEL_UNSUPP;
 914         while (er < mandoclimits[level])
 915                 level--;
 916
 917         if (level < m->wlevel && er != MANDOCERR_FILE)
 918                 return;
 919
 920         if (m->mmsg)
 921                 (*m->mmsg)(er, level, m->file, ln, col, msg);
 922
 923         if (m->file_status < level)
 924                 m->file_status = level;
 925 }
 926
 927 const char *
 928 mparse_strerror(enum mandocerr er)
 929 {
 930
 931         return(mandocerrs[er]);
 932 }
 933
 934 const char *
 935 mparse_strlevel(enum mandoclevel lvl)
 936 {
 937         return(mandoclevels[lvl]);
 938 }
 939
 940 void
 941 mparse_keep(struct mparse *p)
 942 {
 943
 944         assert(NULL == p->secondary);
 945         p->secondary = mandoc_calloc(1, sizeof(struct buf));
 946 }
 947
 948 const char *
 949 mparse_getkeep(const struct mparse *p)
 950 {
 951
 952         assert(p->secondary);
 953         return(p->secondary->sz ? p->secondary->buf : NULL);
 954 }