read.c

   1 /*      $Id: read.c,v 1.165 2017/05/16 19:06:30 schwarze Exp $ */
   2 /*
   3  * Copyright (c) 2008, 2009, 2010, 2011 Kristaps Dzonsons <kristaps@bsd.lv>
   4  * Copyright (c) 2010-2017 Ingo Schwarze <schwarze@openbsd.org>
   5  * Copyright (c) 2010, 2012 Joerg Sonnenberger <joerg@netbsd.org>
   6  *
   7  * Permission to use, copy, modify, and distribute this software for any
   8  * purpose with or without fee is hereby granted, provided that the above
   9  * copyright notice and this permission notice appear in all copies.
  10  *
  11  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHORS DISCLAIM ALL WARRANTIES
  12  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
  13  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR
  14  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
  15  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
  16  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
  17  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  18  */
  19 #include "config.h"
  20
  21 #include <sys/types.h>
  22 #include <sys/mman.h>
  23 #include <sys/stat.h>
  24
  25 #include <assert.h>
  26 #include <ctype.h>
  27 #if HAVE_ERR
  28 #include <err.h>
  29 #endif
  30 #include <errno.h>
  31 #include <fcntl.h>
  32 #include <stdarg.h>
  33 #include <stdio.h>
  34 #include <stdlib.h>
  35 #include <string.h>
  36 #include <unistd.h>
  37 #include <zlib.h>
  38
  39 #include "mandoc_aux.h"
  40 #include "mandoc.h"
  41 #include "roff.h"
  42 #include "mdoc.h"
  43 #include "man.h"
  44 #include "libmandoc.h"
  45 #include "roff_int.h"
  46
  47 #define REPARSE_LIMIT   1000
  48
  49 struct  mparse {
  50         struct roff      *roff; /* roff parser (!NULL) */
  51         struct roff_man  *man; /* man parser */
  52         char             *sodest; /* filename pointed to by .so */
  53         const char       *file; /* filename of current input file */
  54         struct buf       *primary; /* buffer currently being parsed */
  55         struct buf       *secondary; /* preprocessed copy of input */
  56         const char       *defos; /* default operating system */
  57         mandocmsg         mmsg; /* warning/error message handler */
  58         enum mandoclevel  file_status; /* status of current parse */
  59         enum mandoclevel  wlevel; /* ignore messages below this */
  60         int               options; /* parser options */
  61         int               gzip; /* current input file is gzipped */
  62         int               filenc; /* encoding of the current file */
  63         int               reparse_count; /* finite interp. stack */
  64         int               line; /* line number in the file */
  65 };
  66
  67 static  void      choose_parser(struct mparse *);
  68 static  void      resize_buf(struct buf *, size_t);
  69 static  int       mparse_buf_r(struct mparse *, struct buf, size_t, int);
  70 static  int       read_whole_file(struct mparse *, const char *, int,
  71                                 struct buf *, int *);
  72 static  void      mparse_end(struct mparse *);
  73 static  void      mparse_parse_buffer(struct mparse *, struct buf,
  74                         const char *);
  75
  76 static  const enum mandocerr    mandoclimits[MANDOCLEVEL_MAX] = {
  77         MANDOCERR_OK,
  78         MANDOCERR_STYLE,
  79         MANDOCERR_WARNING,
  80         MANDOCERR_ERROR,
  81         MANDOCERR_UNSUPP,
  82         MANDOCERR_MAX,
  83         MANDOCERR_MAX
  84 };
  85
  86 static  const char * const      mandocerrs[MANDOCERR_MAX] = {
  87         "ok",
  88
  89         "generic style suggestion",
  90         "generic warning",
  91
  92         /* related to the prologue */
  93         "missing manual title, using UNTITLED",
  94         "missing manual title, using \"\"",
  95         "lower case character in document title",
  96         "missing manual section, using \"\"",
  97         "unknown manual section",
  98         "missing date, using today's date",
  99         "cannot parse date, using it verbatim",
 100         "missing Os macro, using \"\"",
 101         "duplicate prologue macro",
 102         "late prologue macro",
 103         "skipping late title macro",
 104         "prologue macros out of order",
 105
 106         /* related to document structure */
 107         ".so is fragile, better use ln(1)",
 108         "no document body",
 109         "content before first section header",
 110         "first section is not \"NAME\"",
 111         "NAME section without Nm before Nd",
 112         "NAME section without description",
 113         "description not at the end of NAME",
 114         "bad NAME section content",
 115         "missing comma before name",
 116         "missing description line, using \"\"",
 117         "description line outside NAME section",
 118         "sections out of conventional order",
 119         "duplicate section title",
 120         "unexpected section",
 121         "unusual Xr order",
 122         "unusual Xr punctuation",
 123         "AUTHORS section without An macro",
 124
 125         /* related to macros and nesting */
 126         "obsolete macro",
 127         "macro neither callable nor escaped",
 128         "skipping paragraph macro",
 129         "moving paragraph macro out of list",
 130         "skipping no-space macro",
 131         "blocks badly nested",
 132         "nested displays are not portable",
 133         "moving content out of list",
 134         "fill mode already enabled, skipping",
 135         "fill mode already disabled, skipping",
 136         "line scope broken",
 137
 138         /* related to missing macro arguments */
 139         "skipping empty request",
 140         "conditional request controls empty scope",
 141         "skipping empty macro",
 142         "empty block",
 143         "empty argument, using 0n",
 144         "missing display type, using -ragged",
 145         "list type is not the first argument",
 146         "missing -width in -tag list, using 6n",
 147         "missing utility name, using \"\"",
 148         "missing function name, using \"\"",
 149         "empty head in list item",
 150         "empty list item",
 151         "missing font type, using \\fR",
 152         "unknown font type, using \\fR",
 153         "nothing follows prefix",
 154         "empty reference block",
 155         "missing section argument",
 156         "missing -std argument, adding it",
 157         "missing option string, using \"\"",
 158         "missing resource identifier, using \"\"",
 159         "missing eqn box, using \"\"",
 160
 161         /* related to bad macro arguments */
 162         "unterminated quoted argument",
 163         "duplicate argument",
 164         "skipping duplicate argument",
 165         "skipping duplicate display type",
 166         "skipping duplicate list type",
 167         "skipping -width argument",
 168         "wrong number of cells",
 169         "unknown AT&T UNIX version",
 170         "comma in function argument",
 171         "parenthesis in function name",
 172         "invalid content in Rs block",
 173         "invalid Boolean argument",
 174         "unknown font, skipping request",
 175         "odd number of characters in request",
 176
 177         /* related to plain text */
 178         "blank line in fill mode, using .sp",
 179         "tab in filled text",
 180         "whitespace at end of input line",
 181         "new sentence, new line",
 182         "bad comment style",
 183         "invalid escape sequence",
 184         "undefined string, using \"\"",
 185
 186         /* related to tables */
 187         "tbl line starts with span",
 188         "tbl column starts with span",
 189         "skipping vertical bar in tbl layout",
 190
 191         "generic error",
 192
 193         /* related to tables */
 194         "non-alphabetic character in tbl options",
 195         "skipping unknown tbl option",
 196         "missing tbl option argument",
 197         "wrong tbl option argument size",
 198         "empty tbl layout",
 199         "invalid character in tbl layout",
 200         "unmatched parenthesis in tbl layout",
 201         "tbl without any data cells",
 202         "ignoring data in spanned tbl cell",
 203         "ignoring extra tbl data cells",
 204         "data block open at end of tbl",
 205
 206         /* related to document structure and macros */
 207         NULL,
 208         "input stack limit exceeded, infinite loop?",
 209         "skipping bad character",
 210         "skipping unknown macro",
 211         "skipping insecure request",
 212         "skipping item outside list",
 213         "skipping column outside column list",
 214         "skipping end of block that is not open",
 215         "fewer RS blocks open, skipping",
 216         "inserting missing end of block",
 217         "appending missing end of block",
 218
 219         /* related to request and macro arguments */
 220         "escaped character not allowed in a name",
 221         "NOT IMPLEMENTED: Bd -file",
 222         "skipping display without arguments",
 223         "missing list type, using -item",
 224         "missing manual name, using \"\"",
 225         "uname(3) system call failed, using UNKNOWN",
 226         "unknown standard specifier",
 227         "skipping request without numeric argument",
 228         "NOT IMPLEMENTED: .so with absolute path or \"..\"",
 229         ".so request failed",
 230         "skipping all arguments",
 231         "skipping excess arguments",
 232         "divide by zero",
 233
 234         "unsupported feature",
 235         "input too large",
 236         "unsupported control character",
 237         "unsupported roff request",
 238         "eqn delim option in tbl",
 239         "unsupported tbl layout modifier",
 240         "ignoring macro in table",
 241 };
 242
 243 static  const char * const      mandoclevels[MANDOCLEVEL_MAX] = {
 244         "SUCCESS",
 245         "STYLE",
 246         "WARNING",
 247         "ERROR",
 248         "UNSUPP",
 249         "BADARG",
 250         "SYSERR"
 251 };
 252
 253
 254 static void
 255 resize_buf(struct buf *buf, size_t initial)
 256 {
 257
 258         buf->sz = buf->sz > initial/2 ? 2 * buf->sz : initial;
 259         buf->buf = mandoc_realloc(buf->buf, buf->sz);
 260 }
 261
 262 static void
 263 choose_parser(struct mparse *curp)
 264 {
 265         char            *cp, *ep;
 266         int              format;
 267
 268         /*
 269          * If neither command line arguments -mdoc or -man select
 270          * a parser nor the roff parser found a .Dd or .TH macro
 271          * yet, look ahead in the main input buffer.
 272          */
 273
 274         if ((format = roff_getformat(curp->roff)) == 0) {
 275                 cp = curp->primary->buf;
 276                 ep = cp + curp->primary->sz;
 277                 while (cp < ep) {
 278                         if (*cp == '.' || *cp == '\'') {
 279                                 cp++;
 280                                 if (cp[0] == 'D' && cp[1] == 'd') {
 281                                         format = MPARSE_MDOC;
 282                                         break;
 283                                 }
 284                                 if (cp[0] == 'T' && cp[1] == 'H') {
 285                                         format = MPARSE_MAN;
 286                                         break;
 287                                 }
 288                         }
 289                         cp = memchr(cp, '\n', ep - cp);
 290                         if (cp == NULL)
 291                                 break;
 292                         cp++;
 293                 }
 294         }
 295
 296         if (format == MPARSE_MDOC) {
 297                 curp->man->macroset = MACROSET_MDOC;
 298                 if (curp->man->mdocmac == NULL)
 299                         curp->man->mdocmac = roffhash_alloc(MDOC_Dd, MDOC_MAX);
 300         } else {
 301                 curp->man->macroset = MACROSET_MAN;
 302                 if (curp->man->manmac == NULL)
 303                         curp->man->manmac = roffhash_alloc(MAN_TH, MAN_MAX);
 304         }
 305         curp->man->first->tok = TOKEN_NONE;
 306 }
 307
 308 /*
 309  * Main parse routine for a buffer.
 310  * It assumes encoding and line numbering are already set up.
 311  * It can recurse directly (for invocations of user-defined
 312  * macros, inline equations, and input line traps)
 313  * and indirectly (for .so file inclusion).
 314  */
 315 static int
 316 mparse_buf_r(struct mparse *curp, struct buf blk, size_t i, int start)
 317 {
 318         const struct tbl_span   *span;
 319         struct buf       ln;
 320         const char      *save_file;
 321         char            *cp;
 322         size_t           pos; /* byte number in the ln buffer */
 323         size_t           j;  /* auxiliary byte number in the blk buffer */
 324         enum rofferr     rr;
 325         int              of;
 326         int              lnn; /* line number in the real file */
 327         int              fd;
 328         unsigned char    c;
 329
 330         memset(&ln, 0, sizeof(ln));
 331
 332         lnn = curp->line;
 333         pos = 0;
 334
 335         while (i < blk.sz) {
 336                 if (0 == pos && '\0' == blk.buf[i])
 337                         break;
 338
 339                 if (start) {
 340                         curp->line = lnn;
 341                         curp->reparse_count = 0;
 342
 343                         if (lnn < 3 &&
 344                             curp->filenc & MPARSE_UTF8 &&
 345                             curp->filenc & MPARSE_LATIN1)
 346                                 curp->filenc = preconv_cue(&blk, i);
 347                 }
 348
 349                 while (i < blk.sz && (start || blk.buf[i] != '\0')) {
 350
 351                         /*
 352                          * When finding an unescaped newline character,
 353                          * leave the character loop to process the line.
 354                          * Skip a preceding carriage return, if any.
 355                          */
 356
 357                         if ('\r' == blk.buf[i] && i + 1 < blk.sz &&
 358                             '\n' == blk.buf[i + 1])
 359                                 ++i;
 360                         if ('\n' == blk.buf[i]) {
 361                                 ++i;
 362                                 ++lnn;
 363                                 break;
 364                         }
 365
 366                         /*
 367                          * Make sure we have space for the worst
 368                          * case of 11 bytes: "\\[u10ffff]\0"
 369                          */
 370
 371                         if (pos + 11 > ln.sz)
 372                                 resize_buf(&ln, 256);
 373
 374                         /*
 375                          * Encode 8-bit input.
 376                          */
 377
 378                         c = blk.buf[i];
 379                         if (c & 0x80) {
 380                                 if ( ! (curp->filenc && preconv_encode(
 381                                     &blk, &i, &ln, &pos, &curp->filenc))) {
 382                                         mandoc_vmsg(MANDOCERR_CHAR_BAD, curp,
 383                                             curp->line, pos, "0x%x", c);
 384                                         ln.buf[pos++] = '?';
 385                                         i++;
 386                                 }
 387                                 continue;
 388                         }
 389
 390                         /*
 391                          * Exclude control characters.
 392                          */
 393
 394                         if (c == 0x7f || (c < 0x20 && c != 0x09)) {
 395                                 mandoc_vmsg(c == 0x00 || c == 0x04 ||
 396                                     c > 0x0a ? MANDOCERR_CHAR_BAD :
 397                                     MANDOCERR_CHAR_UNSUPP,
 398                                     curp, curp->line, pos, "0x%x", c);
 399                                 i++;
 400                                 if (c != '\r')
 401                                         ln.buf[pos++] = '?';
 402                                 continue;
 403                         }
 404
 405                         /* Trailing backslash = a plain char. */
 406
 407                         if (blk.buf[i] != '\\' || i + 1 == blk.sz) {
 408                                 ln.buf[pos++] = blk.buf[i++];
 409                                 continue;
 410                         }
 411
 412                         /*
 413                          * Found escape and at least one other character.
 414                          * When it's a newline character, skip it.
 415                          * When there is a carriage return in between,
 416                          * skip that one as well.
 417                          */
 418
 419                         if ('\r' == blk.buf[i + 1] && i + 2 < blk.sz &&
 420                             '\n' == blk.buf[i + 2])
 421                                 ++i;
 422                         if ('\n' == blk.buf[i + 1]) {
 423                                 i += 2;
 424                                 ++lnn;
 425                                 continue;
 426                         }
 427
 428                         if ('"' == blk.buf[i + 1] || '#' == blk.buf[i + 1]) {
 429                                 j = i;
 430                                 i += 2;
 431                                 /* Comment, skip to end of line */
 432                                 for (; i < blk.sz; ++i) {
 433                                         if (blk.buf[i] != '\n')
 434                                                 continue;
 435                                         if (blk.buf[i - 1] == ' ' ||
 436                                             blk.buf[i - 1] == '\t')
 437                                                 mandoc_msg(
 438                                                     MANDOCERR_SPACE_EOL,
 439                                                     curp, curp->line,
 440                                                     pos + i-1 - j, NULL);
 441                                         ++i;
 442                                         ++lnn;
 443                                         break;
 444                                 }
 445
 446                                 /* Backout trailing whitespaces */
 447                                 for (; pos > 0; --pos) {
 448                                         if (ln.buf[pos - 1] != ' ')
 449                                                 break;
 450                                         if (pos > 2 && ln.buf[pos - 2] == '\\')
 451                                                 break;
 452                                 }
 453                                 break;
 454                         }
 455
 456                         /* Catch escaped bogus characters. */
 457
 458                         c = (unsigned char) blk.buf[i+1];
 459
 460                         if ( ! (isascii(c) &&
 461                             (isgraph(c) || isblank(c)))) {
 462                                 mandoc_vmsg(MANDOCERR_CHAR_BAD, curp,
 463                                     curp->line, pos, "0x%x", c);
 464                                 i += 2;
 465                                 ln.buf[pos++] = '?';
 466                                 continue;
 467                         }
 468
 469                         /* Some other escape sequence, copy & cont. */
 470
 471                         ln.buf[pos++] = blk.buf[i++];
 472                         ln.buf[pos++] = blk.buf[i++];
 473                 }
 474
 475                 if (pos >= ln.sz)
 476                         resize_buf(&ln, 256);
 477
 478                 ln.buf[pos] = '\0';
 479
 480                 /*
 481                  * A significant amount of complexity is contained by
 482                  * the roff preprocessor.  It's line-oriented but can be
 483                  * expressed on one line, so we need at times to
 484                  * readjust our starting point and re-run it.  The roff
 485                  * preprocessor can also readjust the buffers with new
 486                  * data, so we pass them in wholesale.
 487                  */
 488
 489                 of = 0;
 490
 491                 /*
 492                  * Maintain a lookaside buffer of all parsed lines.  We
 493                  * only do this if mparse_keep() has been invoked (the
 494                  * buffer may be accessed with mparse_getkeep()).
 495                  */
 496
 497                 if (curp->secondary) {
 498                         curp->secondary->buf = mandoc_realloc(
 499                             curp->secondary->buf,
 500                             curp->secondary->sz + pos + 2);
 501                         memcpy(curp->secondary->buf +
 502                             curp->secondary->sz,
 503                             ln.buf, pos);
 504                         curp->secondary->sz += pos;
 505                         curp->secondary->buf
 506                                 [curp->secondary->sz] = '\n';
 507                         curp->secondary->sz++;
 508                         curp->secondary->buf
 509                                 [curp->secondary->sz] = '\0';
 510                 }
 511 rerun:
 512                 rr = roff_parseln(curp->roff, curp->line, &ln, &of);
 513
 514                 switch (rr) {
 515                 case ROFF_REPARSE:
 516                         if (++curp->reparse_count > REPARSE_LIMIT)
 517                                 mandoc_msg(MANDOCERR_ROFFLOOP, curp,
 518                                     curp->line, pos, NULL);
 519                         else if (mparse_buf_r(curp, ln, of, 0) == 1 ||
 520                             start == 1) {
 521                                 pos = 0;
 522                                 continue;
 523                         }
 524                         free(ln.buf);
 525                         return 0;
 526                 case ROFF_APPEND:
 527                         pos = strlen(ln.buf);
 528                         continue;
 529                 case ROFF_RERUN:
 530                         goto rerun;
 531                 case ROFF_IGN:
 532                         pos = 0;
 533                         continue;
 534                 case ROFF_SO:
 535                         if ( ! (curp->options & MPARSE_SO) &&
 536                             (i >= blk.sz || blk.buf[i] == '\0')) {
 537                                 curp->sodest = mandoc_strdup(ln.buf + of);
 538                                 free(ln.buf);
 539                                 return 1;
 540                         }
 541                         /*
 542                          * We remove `so' clauses from our lookaside
 543                          * buffer because we're going to descend into
 544                          * the file recursively.
 545                          */
 546                         if (curp->secondary)
 547                                 curp->secondary->sz -= pos + 1;
 548                         save_file = curp->file;
 549                         if ((fd = mparse_open(curp, ln.buf + of)) != -1) {
 550                                 mparse_readfd(curp, fd, ln.buf + of);
 551                                 close(fd);
 552                                 curp->file = save_file;
 553                         } else {
 554                                 curp->file = save_file;
 555                                 mandoc_vmsg(MANDOCERR_SO_FAIL,
 556                                     curp, curp->line, pos,
 557                                     ".so %s", ln.buf + of);
 558                                 ln.sz = mandoc_asprintf(&cp,
 559                                     ".sp\nSee the file %s.\n.sp",
 560                                     ln.buf + of);
 561                                 free(ln.buf);
 562                                 ln.buf = cp;
 563                                 of = 0;
 564                                 mparse_buf_r(curp, ln, of, 0);
 565                         }
 566                         pos = 0;
 567                         continue;
 568                 default:
 569                         break;
 570                 }
 571
 572                 if (curp->man->macroset == MACROSET_NONE)
 573                         choose_parser(curp);
 574
 575                 /*
 576                  * Lastly, push down into the parsers themselves.
 577                  * If libroff returns ROFF_TBL, then add it to the
 578                  * currently open parse.  Since we only get here if
 579                  * there does exist data (see tbl_data.c), we're
 580                  * guaranteed that something's been allocated.
 581                  * Do the same for ROFF_EQN.
 582                  */
 583
 584                 if (rr == ROFF_TBL)
 585                         while ((span = roff_span(curp->roff)) != NULL)
 586                                 roff_addtbl(curp->man, span);
 587                 else if (rr == ROFF_EQN)
 588                         roff_addeqn(curp->man, roff_eqn(curp->roff));
 589                 else if ((curp->man->macroset == MACROSET_MDOC ?
 590                     mdoc_parseln(curp->man, curp->line, ln.buf, of) :
 591                     man_parseln(curp->man, curp->line, ln.buf, of)) == 2)
 592                                 break;
 593
 594                 /* Temporary buffers typically are not full. */
 595
 596                 if (0 == start && '\0' == blk.buf[i])
 597                         break;
 598
 599                 /* Start the next input line. */
 600
 601                 pos = 0;
 602         }
 603
 604         free(ln.buf);
 605         return 1;
 606 }
 607
 608 static int
 609 read_whole_file(struct mparse *curp, const char *file, int fd,
 610                 struct buf *fb, int *with_mmap)
 611 {
 612         struct stat      st;
 613         gzFile           gz;
 614         size_t           off;
 615         ssize_t          ssz;
 616
 617         if (fstat(fd, &st) == -1)
 618                 err((int)MANDOCLEVEL_SYSERR, "%s", file);
 619
 620         /*
 621          * If we're a regular file, try just reading in the whole entry
 622          * via mmap().  This is faster than reading it into blocks, and
 623          * since each file is only a few bytes to begin with, I'm not
 624          * concerned that this is going to tank any machines.
 625          */
 626
 627         if (curp->gzip == 0 && S_ISREG(st.st_mode)) {
 628                 if (st.st_size > 0x7fffffff) {
 629                         mandoc_msg(MANDOCERR_TOOLARGE, curp, 0, 0, NULL);
 630                         return 0;
 631                 }
 632                 *with_mmap = 1;
 633                 fb->sz = (size_t)st.st_size;
 634                 fb->buf = mmap(NULL, fb->sz, PROT_READ, MAP_SHARED, fd, 0);
 635                 if (fb->buf != MAP_FAILED)
 636                         return 1;
 637         }
 638
 639         if (curp->gzip) {
 640                 if ((gz = gzdopen(fd, "rb")) == NULL)
 641                         err((int)MANDOCLEVEL_SYSERR, "%s", file);
 642         } else
 643                 gz = NULL;
 644
 645         /*
 646          * If this isn't a regular file (like, say, stdin), then we must
 647          * go the old way and just read things in bit by bit.
 648          */
 649
 650         *with_mmap = 0;
 651         off = 0;
 652         fb->sz = 0;
 653         fb->buf = NULL;
 654         for (;;) {
 655                 if (off == fb->sz) {
 656                         if (fb->sz == (1U << 31)) {
 657                                 mandoc_msg(MANDOCERR_TOOLARGE, curp,
 658                                     0, 0, NULL);
 659                                 break;
 660                         }
 661                         resize_buf(fb, 65536);
 662                 }
 663                 ssz = curp->gzip ?
 664                     gzread(gz, fb->buf + (int)off, fb->sz - off) :
 665                     read(fd, fb->buf + (int)off, fb->sz - off);
 666                 if (ssz == 0) {
 667                         fb->sz = off;
 668                         return 1;
 669                 }
 670                 if (ssz == -1)
 671                         err((int)MANDOCLEVEL_SYSERR, "%s", file);
 672                 off += (size_t)ssz;
 673         }
 674
 675         free(fb->buf);
 676         fb->buf = NULL;
 677         return 0;
 678 }
 679
 680 static void
 681 mparse_end(struct mparse *curp)
 682 {
 683         if (curp->man->macroset == MACROSET_NONE)
 684                 curp->man->macroset = MACROSET_MAN;
 685         if (curp->man->macroset == MACROSET_MDOC)
 686                 mdoc_endparse(curp->man);
 687         else
 688                 man_endparse(curp->man);
 689         roff_endparse(curp->roff);
 690 }
 691
 692 static void
 693 mparse_parse_buffer(struct mparse *curp, struct buf blk, const char *file)
 694 {
 695         struct buf      *svprimary;
 696         const char      *svfile;
 697         size_t           offset;
 698         static int       recursion_depth;
 699
 700         if (64 < recursion_depth) {
 701                 mandoc_msg(MANDOCERR_ROFFLOOP, curp, curp->line, 0, NULL);
 702                 return;
 703         }
 704
 705         /* Line number is per-file. */
 706         svfile = curp->file;
 707         curp->file = file;
 708         svprimary = curp->primary;
 709         curp->primary = &blk;
 710         curp->line = 1;
 711         recursion_depth++;
 712
 713         /* Skip an UTF-8 byte order mark. */
 714         if (curp->filenc & MPARSE_UTF8 && blk.sz > 2 &&
 715             (unsigned char)blk.buf[0] == 0xef &&
 716             (unsigned char)blk.buf[1] == 0xbb &&
 717             (unsigned char)blk.buf[2] == 0xbf) {
 718                 offset = 3;
 719                 curp->filenc &= ~MPARSE_LATIN1;
 720         } else
 721                 offset = 0;
 722
 723         mparse_buf_r(curp, blk, offset, 1);
 724
 725         if (--recursion_depth == 0)
 726                 mparse_end(curp);
 727
 728         curp->primary = svprimary;
 729         curp->file = svfile;
 730 }
 731
 732 enum mandoclevel
 733 mparse_readmem(struct mparse *curp, void *buf, size_t len,
 734                 const char *file)
 735 {
 736         struct buf blk;
 737
 738         blk.buf = buf;
 739         blk.sz = len;
 740
 741         mparse_parse_buffer(curp, blk, file);
 742         return curp->file_status;
 743 }
 744
 745 /*
 746  * Read the whole file into memory and call the parsers.
 747  * Called recursively when an .so request is encountered.
 748  */
 749 enum mandoclevel
 750 mparse_readfd(struct mparse *curp, int fd, const char *file)
 751 {
 752         struct buf       blk;
 753         int              with_mmap;
 754         int              save_filenc;
 755
 756         if (read_whole_file(curp, file, fd, &blk, &with_mmap)) {
 757                 save_filenc = curp->filenc;
 758                 curp->filenc = curp->options &
 759                     (MPARSE_UTF8 | MPARSE_LATIN1);
 760                 mparse_parse_buffer(curp, blk, file);
 761                 curp->filenc = save_filenc;
 762                 if (with_mmap)
 763                         munmap(blk.buf, blk.sz);
 764                 else
 765                         free(blk.buf);
 766         }
 767         return curp->file_status;
 768 }
 769
 770 int
 771 mparse_open(struct mparse *curp, const char *file)
 772 {
 773         char             *cp;
 774         int               fd;
 775
 776         curp->file = file;
 777         cp = strrchr(file, '.');
 778         curp->gzip = (cp != NULL && ! strcmp(cp + 1, "gz"));
 779
 780         /* First try to use the filename as it is. */
 781
 782         if ((fd = open(file, O_RDONLY)) != -1)
 783                 return fd;
 784
 785         /*
 786          * If that doesn't work and the filename doesn't
 787          * already  end in .gz, try appending .gz.
 788          */
 789
 790         if ( ! curp->gzip) {
 791                 mandoc_asprintf(&cp, "%s.gz", file);
 792                 fd = open(cp, O_RDONLY);
 793                 free(cp);
 794                 if (fd != -1) {
 795                         curp->gzip = 1;
 796                         return fd;
 797                 }
 798         }
 799
 800         /* Neither worked, give up. */
 801
 802         mandoc_msg(MANDOCERR_FILE, curp, 0, 0, strerror(errno));
 803         return -1;
 804 }
 805
 806 struct mparse *
 807 mparse_alloc(int options, enum mandoclevel wlevel, mandocmsg mmsg,
 808     const char *defos)
 809 {
 810         struct mparse   *curp;
 811
 812         curp = mandoc_calloc(1, sizeof(struct mparse));
 813
 814         curp->options = options;
 815         curp->wlevel = wlevel;
 816         curp->mmsg = mmsg;
 817         curp->defos = defos;
 818
 819         curp->roff = roff_alloc(curp, options);
 820         curp->man = roff_man_alloc( curp->roff, curp, curp->defos,
 821                 curp->options & MPARSE_QUICK ? 1 : 0);
 822         if (curp->options & MPARSE_MDOC) {
 823                 curp->man->macroset = MACROSET_MDOC;
 824                 if (curp->man->mdocmac == NULL)
 825                         curp->man->mdocmac = roffhash_alloc(MDOC_Dd, MDOC_MAX);
 826         } else if (curp->options & MPARSE_MAN) {
 827                 curp->man->macroset = MACROSET_MAN;
 828                 if (curp->man->manmac == NULL)
 829                         curp->man->manmac = roffhash_alloc(MAN_TH, MAN_MAX);
 830         }
 831         curp->man->first->tok = TOKEN_NONE;
 832         return curp;
 833 }
 834
 835 void
 836 mparse_reset(struct mparse *curp)
 837 {
 838         roff_reset(curp->roff);
 839         roff_man_reset(curp->man);
 840
 841         free(curp->sodest);
 842         curp->sodest = NULL;
 843
 844         if (curp->secondary)
 845                 curp->secondary->sz = 0;
 846
 847         curp->file_status = MANDOCLEVEL_OK;
 848         curp->gzip = 0;
 849 }
 850
 851 void
 852 mparse_free(struct mparse *curp)
 853 {
 854
 855         roffhash_free(curp->man->mdocmac);
 856         roffhash_free(curp->man->manmac);
 857         roff_man_free(curp->man);
 858         roff_free(curp->roff);
 859         if (curp->secondary)
 860                 free(curp->secondary->buf);
 861
 862         free(curp->secondary);
 863         free(curp->sodest);
 864         free(curp);
 865 }
 866
 867 void
 868 mparse_result(struct mparse *curp, struct roff_man **man,
 869         char **sodest)
 870 {
 871
 872         if (sodest && NULL != (*sodest = curp->sodest)) {
 873                 *man = NULL;
 874                 return;
 875         }
 876         if (man)
 877                 *man = curp->man;
 878 }
 879
 880 void
 881 mparse_updaterc(struct mparse *curp, enum mandoclevel *rc)
 882 {
 883         if (curp->file_status > *rc)
 884                 *rc = curp->file_status;
 885 }
 886
 887 void
 888 mandoc_vmsg(enum mandocerr t, struct mparse *m,
 889                 int ln, int pos, const char *fmt, ...)
 890 {
 891         char             buf[256];
 892         va_list          ap;
 893
 894         va_start(ap, fmt);
 895         (void)vsnprintf(buf, sizeof(buf), fmt, ap);
 896         va_end(ap);
 897
 898         mandoc_msg(t, m, ln, pos, buf);
 899 }
 900
 901 void
 902 mandoc_msg(enum mandocerr er, struct mparse *m,
 903                 int ln, int col, const char *msg)
 904 {
 905         enum mandoclevel level;
 906
 907         level = MANDOCLEVEL_UNSUPP;
 908         while (er < mandoclimits[level])
 909                 level--;
 910
 911         if (level < m->wlevel && er != MANDOCERR_FILE)
 912                 return;
 913
 914         if (m->mmsg)
 915                 (*m->mmsg)(er, level, m->file, ln, col, msg);
 916
 917         if (m->file_status < level)
 918                 m->file_status = level;
 919 }
 920
 921 const char *
 922 mparse_strerror(enum mandocerr er)
 923 {
 924
 925         return mandocerrs[er];
 926 }
 927
 928 const char *
 929 mparse_strlevel(enum mandoclevel lvl)
 930 {
 931         return mandoclevels[lvl];
 932 }
 933
 934 void
 935 mparse_keep(struct mparse *p)
 936 {
 937
 938         assert(NULL == p->secondary);
 939         p->secondary = mandoc_calloc(1, sizeof(struct buf));
 940 }
 941
 942 const char *
 943 mparse_getkeep(const struct mparse *p)
 944 {
 945
 946         assert(p->secondary);
 947         return p->secondary->sz ? p->secondary->buf : NULL;
 948 }