read.c

   1 /*      $Id: read.c,v 1.144 2015/10/13 22:59:54 schwarze Exp $ */
   2 /*
   3  * Copyright (c) 2008, 2009, 2010, 2011 Kristaps Dzonsons <kristaps@bsd.lv>
   4  * Copyright (c) 2010-2015 Ingo Schwarze <schwarze@openbsd.org>
   5  * Copyright (c) 2010, 2012 Joerg Sonnenberger <joerg@netbsd.org>
   6  *
   7  * Permission to use, copy, modify, and distribute this software for any
   8  * purpose with or without fee is hereby granted, provided that the above
   9  * copyright notice and this permission notice appear in all copies.
  10  *
  11  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHORS DISCLAIM ALL WARRANTIES
  12  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
  13  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR
  14  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
  15  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
  16  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
  17  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  18  */
  19 #include "config.h"
  20
  21 #include <sys/types.h>
  22 #if HAVE_MMAP
  23 #include <sys/mman.h>
  24 #include <sys/stat.h>
  25 #endif
  26
  27 #include <assert.h>
  28 #include <ctype.h>
  29 #include <err.h>
  30 #include <errno.h>
  31 #include <fcntl.h>
  32 #include <stdarg.h>
  33 #include <stdint.h>
  34 #include <stdio.h>
  35 #include <stdlib.h>
  36 #include <string.h>
  37 #include <unistd.h>
  38 #include <zlib.h>
  39
  40 #include "mandoc_aux.h"
  41 #include "mandoc.h"
  42 #include "roff.h"
  43 #include "mdoc.h"
  44 #include "man.h"
  45 #include "libmandoc.h"
  46 #include "roff_int.h"
  47
  48 #define REPARSE_LIMIT   1000
  49
  50 struct  mparse {
  51         struct roff_man  *man; /* man parser */
  52         struct roff      *roff; /* roff parser (!NULL) */
  53         char             *sodest; /* filename pointed to by .so */
  54         const char       *file; /* filename of current input file */
  55         struct buf       *primary; /* buffer currently being parsed */
  56         struct buf       *secondary; /* preprocessed copy of input */
  57         const char       *defos; /* default operating system */
  58         mandocmsg         mmsg; /* warning/error message handler */
  59         enum mandoclevel  file_status; /* status of current parse */
  60         enum mandoclevel  wlevel; /* ignore messages below this */
  61         int               options; /* parser options */
  62         int               gzip; /* current input file is gzipped */
  63         int               filenc; /* encoding of the current file */
  64         int               reparse_count; /* finite interp. stack */
  65         int               line; /* line number in the file */
  66 };
  67
  68 static  void      choose_parser(struct mparse *);
  69 static  void      resize_buf(struct buf *, size_t);
  70 static  void      mparse_buf_r(struct mparse *, struct buf, size_t, int);
  71 static  int       read_whole_file(struct mparse *, const char *, int,
  72                                 struct buf *, int *);
  73 static  void      mparse_end(struct mparse *);
  74 static  void      mparse_parse_buffer(struct mparse *, struct buf,
  75                         const char *);
  76
  77 static  const enum mandocerr    mandoclimits[MANDOCLEVEL_MAX] = {
  78         MANDOCERR_OK,
  79         MANDOCERR_WARNING,
  80         MANDOCERR_WARNING,
  81         MANDOCERR_ERROR,
  82         MANDOCERR_UNSUPP,
  83         MANDOCERR_MAX,
  84         MANDOCERR_MAX
  85 };
  86
  87 static  const char * const      mandocerrs[MANDOCERR_MAX] = {
  88         "ok",
  89
  90         "generic warning",
  91
  92         /* related to the prologue */
  93         "missing manual title, using UNTITLED",
  94         "missing manual title, using \"\"",
  95         "lower case character in document title",
  96         "missing manual section, using \"\"",
  97         "unknown manual section",
  98         "missing date, using today's date",
  99         "cannot parse date, using it verbatim",
 100         "missing Os macro, using \"\"",
 101         "duplicate prologue macro",
 102         "late prologue macro",
 103         "skipping late title macro",
 104         "prologue macros out of order",
 105
 106         /* related to document structure */
 107         ".so is fragile, better use ln(1)",
 108         "no document body",
 109         "content before first section header",
 110         "first section is not \"NAME\"",
 111         "NAME section without name",
 112         "NAME section without description",
 113         "description not at the end of NAME",
 114         "bad NAME section content",
 115         "missing description line, using \"\"",
 116         "sections out of conventional order",
 117         "duplicate section title",
 118         "unexpected section",
 119         "unusual Xr order",
 120         "unusual Xr punctuation",
 121         "AUTHORS section without An macro",
 122
 123         /* related to macros and nesting */
 124         "obsolete macro",
 125         "macro neither callable nor escaped",
 126         "skipping paragraph macro",
 127         "moving paragraph macro out of list",
 128         "skipping no-space macro",
 129         "blocks badly nested",
 130         "nested displays are not portable",
 131         "moving content out of list",
 132         "fill mode already enabled, skipping",
 133         "fill mode already disabled, skipping",
 134         "line scope broken",
 135
 136         /* related to missing macro arguments */
 137         "skipping empty request",
 138         "conditional request controls empty scope",
 139         "skipping empty macro",
 140         "empty block",
 141         "empty argument, using 0n",
 142         "missing display type, using -ragged",
 143         "list type is not the first argument",
 144         "missing -width in -tag list, using 8n",
 145         "missing utility name, using \"\"",
 146         "missing function name, using \"\"",
 147         "empty head in list item",
 148         "empty list item",
 149         "missing font type, using \\fR",
 150         "unknown font type, using \\fR",
 151         "nothing follows prefix",
 152         "empty reference block",
 153         "missing -std argument, adding it",
 154         "missing option string, using \"\"",
 155         "missing resource identifier, using \"\"",
 156         "missing eqn box, using \"\"",
 157
 158         /* related to bad macro arguments */
 159         "unterminated quoted argument",
 160         "duplicate argument",
 161         "skipping duplicate argument",
 162         "skipping duplicate display type",
 163         "skipping duplicate list type",
 164         "skipping -width argument",
 165         "wrong number of cells",
 166         "unknown AT&T UNIX version",
 167         "comma in function argument",
 168         "parenthesis in function name",
 169         "invalid content in Rs block",
 170         "invalid Boolean argument",
 171         "unknown font, skipping request",
 172         "odd number of characters in request",
 173
 174         /* related to plain text */
 175         "blank line in fill mode, using .sp",
 176         "tab in filled text",
 177         "whitespace at end of input line",
 178         "bad comment style",
 179         "invalid escape sequence",
 180         "undefined string, using \"\"",
 181
 182         /* related to tables */
 183         "tbl line starts with span",
 184         "tbl column starts with span",
 185         "skipping vertical bar in tbl layout",
 186
 187         "generic error",
 188
 189         /* related to tables */
 190         "non-alphabetic character in tbl options",
 191         "skipping unknown tbl option",
 192         "missing tbl option argument",
 193         "wrong tbl option argument size",
 194         "empty tbl layout",
 195         "invalid character in tbl layout",
 196         "unmatched parenthesis in tbl layout",
 197         "tbl without any data cells",
 198         "ignoring data in spanned tbl cell",
 199         "ignoring extra tbl data cells",
 200         "data block open at end of tbl",
 201
 202         /* related to document structure and macros */
 203         NULL,
 204         "input stack limit exceeded, infinite loop?",
 205         "skipping bad character",
 206         "skipping unknown macro",
 207         "skipping insecure request",
 208         "skipping item outside list",
 209         "skipping column outside column list",
 210         "skipping end of block that is not open",
 211         "fewer RS blocks open, skipping",
 212         "inserting missing end of block",
 213         "appending missing end of block",
 214
 215         /* related to request and macro arguments */
 216         "escaped character not allowed in a name",
 217         "NOT IMPLEMENTED: Bd -file",
 218         "missing list type, using -item",
 219         "missing manual name, using \"\"",
 220         "uname(3) system call failed, using UNKNOWN",
 221         "unknown standard specifier",
 222         "skipping request without numeric argument",
 223         "NOT IMPLEMENTED: .so with absolute path or \"..\"",
 224         ".so request failed",
 225         "skipping all arguments",
 226         "skipping excess arguments",
 227         "divide by zero",
 228
 229         "unsupported feature",
 230         "input too large",
 231         "unsupported control character",
 232         "unsupported roff request",
 233         "eqn delim option in tbl",
 234         "unsupported tbl layout modifier",
 235         "ignoring macro in table",
 236 };
 237
 238 static  const char * const      mandoclevels[MANDOCLEVEL_MAX] = {
 239         "SUCCESS",
 240         "RESERVED",
 241         "WARNING",
 242         "ERROR",
 243         "UNSUPP",
 244         "BADARG",
 245         "SYSERR"
 246 };
 247
 248
 249 static void
 250 resize_buf(struct buf *buf, size_t initial)
 251 {
 252
 253         buf->sz = buf->sz > initial/2 ? 2 * buf->sz : initial;
 254         buf->buf = mandoc_realloc(buf->buf, buf->sz);
 255 }
 256
 257 static void
 258 choose_parser(struct mparse *curp)
 259 {
 260         char            *cp, *ep;
 261         int              format;
 262
 263         /*
 264          * If neither command line arguments -mdoc or -man select
 265          * a parser nor the roff parser found a .Dd or .TH macro
 266          * yet, look ahead in the main input buffer.
 267          */
 268
 269         if ((format = roff_getformat(curp->roff)) == 0) {
 270                 cp = curp->primary->buf;
 271                 ep = cp + curp->primary->sz;
 272                 while (cp < ep) {
 273                         if (*cp == '.' || *cp == '\'') {
 274                                 cp++;
 275                                 if (cp[0] == 'D' && cp[1] == 'd') {
 276                                         format = MPARSE_MDOC;
 277                                         break;
 278                                 }
 279                                 if (cp[0] == 'T' && cp[1] == 'H') {
 280                                         format = MPARSE_MAN;
 281                                         break;
 282                                 }
 283                         }
 284                         cp = memchr(cp, '\n', ep - cp);
 285                         if (cp == NULL)
 286                                 break;
 287                         cp++;
 288                 }
 289         }
 290
 291         if (curp->man == NULL) {
 292                 curp->man = roff_man_alloc(curp->roff, curp, curp->defos,
 293                     curp->options & MPARSE_QUICK ? 1 : 0);
 294                 curp->man->macroset = MACROSET_MAN;
 295                 curp->man->first->tok = TOKEN_NONE;
 296         }
 297
 298         if (format == MPARSE_MDOC) {
 299                 mdoc_hash_init();
 300                 curp->man->macroset = MACROSET_MDOC;
 301                 curp->man->first->tok = TOKEN_NONE;
 302         } else {
 303                 man_hash_init();
 304                 curp->man->macroset = MACROSET_MAN;
 305                 curp->man->first->tok = TOKEN_NONE;
 306         }
 307 }
 308
 309 /*
 310  * Main parse routine for a buffer.
 311  * It assumes encoding and line numbering are already set up.
 312  * It can recurse directly (for invocations of user-defined
 313  * macros, inline equations, and input line traps)
 314  * and indirectly (for .so file inclusion).
 315  */
 316 static void
 317 mparse_buf_r(struct mparse *curp, struct buf blk, size_t i, int start)
 318 {
 319         const struct tbl_span   *span;
 320         struct buf       ln;
 321         const char      *save_file;
 322         char            *cp;
 323         size_t           pos; /* byte number in the ln buffer */
 324         enum rofferr     rr;
 325         int              of;
 326         int              lnn; /* line number in the real file */
 327         int              fd;
 328         unsigned char    c;
 329
 330         memset(&ln, 0, sizeof(ln));
 331
 332         lnn = curp->line;
 333         pos = 0;
 334
 335         while (i < blk.sz) {
 336                 if (0 == pos && '\0' == blk.buf[i])
 337                         break;
 338
 339                 if (start) {
 340                         curp->line = lnn;
 341                         curp->reparse_count = 0;
 342
 343                         if (lnn < 3 &&
 344                             curp->filenc & MPARSE_UTF8 &&
 345                             curp->filenc & MPARSE_LATIN1)
 346                                 curp->filenc = preconv_cue(&blk, i);
 347                 }
 348
 349                 while (i < blk.sz && (start || blk.buf[i] != '\0')) {
 350
 351                         /*
 352                          * When finding an unescaped newline character,
 353                          * leave the character loop to process the line.
 354                          * Skip a preceding carriage return, if any.
 355                          */
 356
 357                         if ('\r' == blk.buf[i] && i + 1 < blk.sz &&
 358                             '\n' == blk.buf[i + 1])
 359                                 ++i;
 360                         if ('\n' == blk.buf[i]) {
 361                                 ++i;
 362                                 ++lnn;
 363                                 break;
 364                         }
 365
 366                         /*
 367                          * Make sure we have space for the worst
 368                          * case of 11 bytes: "\\[u10ffff]\0"
 369                          */
 370
 371                         if (pos + 11 > ln.sz)
 372                                 resize_buf(&ln, 256);
 373
 374                         /*
 375                          * Encode 8-bit input.
 376                          */
 377
 378                         c = blk.buf[i];
 379                         if (c & 0x80) {
 380                                 if ( ! (curp->filenc && preconv_encode(
 381                                     &blk, &i, &ln, &pos, &curp->filenc))) {
 382                                         mandoc_vmsg(MANDOCERR_CHAR_BAD, curp,
 383                                             curp->line, pos, "0x%x", c);
 384                                         ln.buf[pos++] = '?';
 385                                         i++;
 386                                 }
 387                                 continue;
 388                         }
 389
 390                         /*
 391                          * Exclude control characters.
 392                          */
 393
 394                         if (c == 0x7f || (c < 0x20 && c != 0x09)) {
 395                                 mandoc_vmsg(c == 0x00 || c == 0x04 ||
 396                                     c > 0x0a ? MANDOCERR_CHAR_BAD :
 397                                     MANDOCERR_CHAR_UNSUPP,
 398                                     curp, curp->line, pos, "0x%x", c);
 399                                 i++;
 400                                 if (c != '\r')
 401                                         ln.buf[pos++] = '?';
 402                                 continue;
 403                         }
 404
 405                         /* Trailing backslash = a plain char. */
 406
 407                         if (blk.buf[i] != '\\' || i + 1 == blk.sz) {
 408                                 ln.buf[pos++] = blk.buf[i++];
 409                                 continue;
 410                         }
 411
 412                         /*
 413                          * Found escape and at least one other character.
 414                          * When it's a newline character, skip it.
 415                          * When there is a carriage return in between,
 416                          * skip that one as well.
 417                          */
 418
 419                         if ('\r' == blk.buf[i + 1] && i + 2 < blk.sz &&
 420                             '\n' == blk.buf[i + 2])
 421                                 ++i;
 422                         if ('\n' == blk.buf[i + 1]) {
 423                                 i += 2;
 424                                 ++lnn;
 425                                 continue;
 426                         }
 427
 428                         if ('"' == blk.buf[i + 1] || '#' == blk.buf[i + 1]) {
 429                                 i += 2;
 430                                 /* Comment, skip to end of line */
 431                                 for (; i < blk.sz; ++i) {
 432                                         if ('\n' == blk.buf[i]) {
 433                                                 ++i;
 434                                                 ++lnn;
 435                                                 break;
 436                                         }
 437                                 }
 438
 439                                 /* Backout trailing whitespaces */
 440                                 for (; pos > 0; --pos) {
 441                                         if (ln.buf[pos - 1] != ' ')
 442                                                 break;
 443                                         if (pos > 2 && ln.buf[pos - 2] == '\\')
 444                                                 break;
 445                                 }
 446                                 break;
 447                         }
 448
 449                         /* Catch escaped bogus characters. */
 450
 451                         c = (unsigned char) blk.buf[i+1];
 452
 453                         if ( ! (isascii(c) &&
 454                             (isgraph(c) || isblank(c)))) {
 455                                 mandoc_vmsg(MANDOCERR_CHAR_BAD, curp,
 456                                     curp->line, pos, "0x%x", c);
 457                                 i += 2;
 458                                 ln.buf[pos++] = '?';
 459                                 continue;
 460                         }
 461
 462                         /* Some other escape sequence, copy & cont. */
 463
 464                         ln.buf[pos++] = blk.buf[i++];
 465                         ln.buf[pos++] = blk.buf[i++];
 466                 }
 467
 468                 if (pos >= ln.sz)
 469                         resize_buf(&ln, 256);
 470
 471                 ln.buf[pos] = '\0';
 472
 473                 /*
 474                  * A significant amount of complexity is contained by
 475                  * the roff preprocessor.  It's line-oriented but can be
 476                  * expressed on one line, so we need at times to
 477                  * readjust our starting point and re-run it.  The roff
 478                  * preprocessor can also readjust the buffers with new
 479                  * data, so we pass them in wholesale.
 480                  */
 481
 482                 of = 0;
 483
 484                 /*
 485                  * Maintain a lookaside buffer of all parsed lines.  We
 486                  * only do this if mparse_keep() has been invoked (the
 487                  * buffer may be accessed with mparse_getkeep()).
 488                  */
 489
 490                 if (curp->secondary) {
 491                         curp->secondary->buf = mandoc_realloc(
 492                             curp->secondary->buf,
 493                             curp->secondary->sz + pos + 2);
 494                         memcpy(curp->secondary->buf +
 495                             curp->secondary->sz,
 496                             ln.buf, pos);
 497                         curp->secondary->sz += pos;
 498                         curp->secondary->buf
 499                                 [curp->secondary->sz] = '\n';
 500                         curp->secondary->sz++;
 501                         curp->secondary->buf
 502                                 [curp->secondary->sz] = '\0';
 503                 }
 504 rerun:
 505                 rr = roff_parseln(curp->roff, curp->line, &ln, &of);
 506
 507                 switch (rr) {
 508                 case ROFF_REPARSE:
 509                         if (REPARSE_LIMIT >= ++curp->reparse_count)
 510                                 mparse_buf_r(curp, ln, of, 0);
 511                         else
 512                                 mandoc_msg(MANDOCERR_ROFFLOOP, curp,
 513                                     curp->line, pos, NULL);
 514                         pos = 0;
 515                         continue;
 516                 case ROFF_APPEND:
 517                         pos = strlen(ln.buf);
 518                         continue;
 519                 case ROFF_RERUN:
 520                         goto rerun;
 521                 case ROFF_IGN:
 522                         pos = 0;
 523                         continue;
 524                 case ROFF_SO:
 525                         if ( ! (curp->options & MPARSE_SO) &&
 526                             (i >= blk.sz || blk.buf[i] == '\0')) {
 527                                 curp->sodest = mandoc_strdup(ln.buf + of);
 528                                 free(ln.buf);
 529                                 return;
 530                         }
 531                         /*
 532                          * We remove `so' clauses from our lookaside
 533                          * buffer because we're going to descend into
 534                          * the file recursively.
 535                          */
 536                         if (curp->secondary)
 537                                 curp->secondary->sz -= pos + 1;
 538                         save_file = curp->file;
 539                         if (mparse_open(curp, &fd, ln.buf + of) ==
 540                             MANDOCLEVEL_OK) {
 541                                 mparse_readfd(curp, fd, ln.buf + of);
 542                                 curp->file = save_file;
 543                         } else {
 544                                 curp->file = save_file;
 545                                 mandoc_vmsg(MANDOCERR_SO_FAIL,
 546                                     curp, curp->line, pos,
 547                                     ".so %s", ln.buf + of);
 548                                 ln.sz = mandoc_asprintf(&cp,
 549                                     ".sp\nSee the file %s.\n.sp",
 550                                     ln.buf + of);
 551                                 free(ln.buf);
 552                                 ln.buf = cp;
 553                                 of = 0;
 554                                 mparse_buf_r(curp, ln, of, 0);
 555                         }
 556                         pos = 0;
 557                         continue;
 558                 default:
 559                         break;
 560                 }
 561
 562                 /*
 563                  * If input parsers have not been allocated, do so now.
 564                  * We keep these instanced between parsers, but set them
 565                  * locally per parse routine since we can use different
 566                  * parsers with each one.
 567                  */
 568
 569                 if (curp->man == NULL ||
 570                     curp->man->macroset == MACROSET_NONE)
 571                         choose_parser(curp);
 572
 573                 /*
 574                  * Lastly, push down into the parsers themselves.
 575                  * If libroff returns ROFF_TBL, then add it to the
 576                  * currently open parse.  Since we only get here if
 577                  * there does exist data (see tbl_data.c), we're
 578                  * guaranteed that something's been allocated.
 579                  * Do the same for ROFF_EQN.
 580                  */
 581
 582                 if (rr == ROFF_TBL)
 583                         while ((span = roff_span(curp->roff)) != NULL)
 584                                 roff_addtbl(curp->man, span);
 585                 else if (rr == ROFF_EQN)
 586                         roff_addeqn(curp->man, roff_eqn(curp->roff));
 587                 else if ((curp->man->macroset == MACROSET_MDOC ?
 588                     mdoc_parseln(curp->man, curp->line, ln.buf, of) :
 589                     man_parseln(curp->man, curp->line, ln.buf, of)) == 2)
 590                                 break;
 591
 592                 /* Temporary buffers typically are not full. */
 593
 594                 if (0 == start && '\0' == blk.buf[i])
 595                         break;
 596
 597                 /* Start the next input line. */
 598
 599                 pos = 0;
 600         }
 601
 602         free(ln.buf);
 603 }
 604
 605 static int
 606 read_whole_file(struct mparse *curp, const char *file, int fd,
 607                 struct buf *fb, int *with_mmap)
 608 {
 609         gzFile           gz;
 610         size_t           off;
 611         ssize_t          ssz;
 612
 613 #if HAVE_MMAP
 614         struct stat      st;
 615
 616         if (fstat(fd, &st) == -1)
 617                 err((int)MANDOCLEVEL_SYSERR, "%s", file);
 618
 619         /*
 620          * If we're a regular file, try just reading in the whole entry
 621          * via mmap().  This is faster than reading it into blocks, and
 622          * since each file is only a few bytes to begin with, I'm not
 623          * concerned that this is going to tank any machines.
 624          */
 625
 626         if (curp->gzip == 0 && S_ISREG(st.st_mode)) {
 627                 if (st.st_size > 0x7fffffff) {
 628                         mandoc_msg(MANDOCERR_TOOLARGE, curp, 0, 0, NULL);
 629                         return 0;
 630                 }
 631                 *with_mmap = 1;
 632                 fb->sz = (size_t)st.st_size;
 633                 fb->buf = mmap(NULL, fb->sz, PROT_READ, MAP_SHARED, fd, 0);
 634                 if (fb->buf != MAP_FAILED)
 635                         return 1;
 636         }
 637 #endif
 638
 639         if (curp->gzip) {
 640                 if ((gz = gzdopen(fd, "rb")) == NULL)
 641                         err((int)MANDOCLEVEL_SYSERR, "%s", file);
 642         } else
 643                 gz = NULL;
 644
 645         /*
 646          * If this isn't a regular file (like, say, stdin), then we must
 647          * go the old way and just read things in bit by bit.
 648          */
 649
 650         *with_mmap = 0;
 651         off = 0;
 652         fb->sz = 0;
 653         fb->buf = NULL;
 654         for (;;) {
 655                 if (off == fb->sz) {
 656                         if (fb->sz == (1U << 31)) {
 657                                 mandoc_msg(MANDOCERR_TOOLARGE, curp,
 658                                     0, 0, NULL);
 659                                 break;
 660                         }
 661                         resize_buf(fb, 65536);
 662                 }
 663                 ssz = curp->gzip ?
 664                     gzread(gz, fb->buf + (int)off, fb->sz - off) :
 665                     read(fd, fb->buf + (int)off, fb->sz - off);
 666                 if (ssz == 0) {
 667                         fb->sz = off;
 668                         return 1;
 669                 }
 670                 if (ssz == -1)
 671                         err((int)MANDOCLEVEL_SYSERR, "%s", file);
 672                 off += (size_t)ssz;
 673         }
 674
 675         free(fb->buf);
 676         fb->buf = NULL;
 677         return 0;
 678 }
 679
 680 static void
 681 mparse_end(struct mparse *curp)
 682 {
 683
 684         if (curp->man == NULL && curp->sodest == NULL)
 685                 curp->man = roff_man_alloc(curp->roff, curp, curp->defos,
 686                     curp->options & MPARSE_QUICK ? 1 : 0);
 687         if (curp->man->macroset == MACROSET_NONE)
 688                 curp->man->macroset = MACROSET_MAN;
 689         if (curp->man->macroset == MACROSET_MDOC)
 690                 mdoc_endparse(curp->man);
 691         else
 692                 man_endparse(curp->man);
 693         roff_endparse(curp->roff);
 694 }
 695
 696 static void
 697 mparse_parse_buffer(struct mparse *curp, struct buf blk, const char *file)
 698 {
 699         struct buf      *svprimary;
 700         const char      *svfile;
 701         size_t           offset;
 702         static int       recursion_depth;
 703
 704         if (64 < recursion_depth) {
 705                 mandoc_msg(MANDOCERR_ROFFLOOP, curp, curp->line, 0, NULL);
 706                 return;
 707         }
 708
 709         /* Line number is per-file. */
 710         svfile = curp->file;
 711         curp->file = file;
 712         svprimary = curp->primary;
 713         curp->primary = &blk;
 714         curp->line = 1;
 715         recursion_depth++;
 716
 717         /* Skip an UTF-8 byte order mark. */
 718         if (curp->filenc & MPARSE_UTF8 && blk.sz > 2 &&
 719             (unsigned char)blk.buf[0] == 0xef &&
 720             (unsigned char)blk.buf[1] == 0xbb &&
 721             (unsigned char)blk.buf[2] == 0xbf) {
 722                 offset = 3;
 723                 curp->filenc &= ~MPARSE_LATIN1;
 724         } else
 725                 offset = 0;
 726
 727         mparse_buf_r(curp, blk, offset, 1);
 728
 729         if (--recursion_depth == 0)
 730                 mparse_end(curp);
 731
 732         curp->primary = svprimary;
 733         curp->file = svfile;
 734 }
 735
 736 enum mandoclevel
 737 mparse_readmem(struct mparse *curp, void *buf, size_t len,
 738                 const char *file)
 739 {
 740         struct buf blk;
 741
 742         blk.buf = buf;
 743         blk.sz = len;
 744
 745         mparse_parse_buffer(curp, blk, file);
 746         return curp->file_status;
 747 }
 748
 749 /*
 750  * Read the whole file into memory and call the parsers.
 751  * Called recursively when an .so request is encountered.
 752  */
 753 enum mandoclevel
 754 mparse_readfd(struct mparse *curp, int fd, const char *file)
 755 {
 756         struct buf       blk;
 757         int              with_mmap;
 758         int              save_filenc;
 759
 760         if (read_whole_file(curp, file, fd, &blk, &with_mmap)) {
 761                 save_filenc = curp->filenc;
 762                 curp->filenc = curp->options &
 763                     (MPARSE_UTF8 | MPARSE_LATIN1);
 764                 mparse_parse_buffer(curp, blk, file);
 765                 curp->filenc = save_filenc;
 766 #if HAVE_MMAP
 767                 if (with_mmap)
 768                         munmap(blk.buf, blk.sz);
 769                 else
 770 #endif
 771                         free(blk.buf);
 772         }
 773
 774         if (fd != STDIN_FILENO && close(fd) == -1)
 775                 perror(file);
 776
 777         return curp->file_status;
 778 }
 779
 780 enum mandoclevel
 781 mparse_open(struct mparse *curp, int *fd, const char *file)
 782 {
 783         char             *cp;
 784
 785         curp->file = file;
 786         cp = strrchr(file, '.');
 787         curp->gzip = (cp != NULL && ! strcmp(cp + 1, "gz"));
 788
 789         /* First try to use the filename as it is. */
 790
 791         if ((*fd = open(file, O_RDONLY)) != -1)
 792                 return MANDOCLEVEL_OK;
 793
 794         /*
 795          * If that doesn't work and the filename doesn't
 796          * already  end in .gz, try appending .gz.
 797          */
 798
 799         if ( ! curp->gzip) {
 800                 mandoc_asprintf(&cp, "%s.gz", file);
 801                 *fd = open(file, O_RDONLY);
 802                 free(cp);
 803                 if (*fd != -1) {
 804                         curp->gzip = 1;
 805                         return MANDOCLEVEL_OK;
 806                 }
 807         }
 808
 809         /* Neither worked, give up. */
 810
 811         mandoc_msg(MANDOCERR_FILE, curp, 0, 0, strerror(errno));
 812         return MANDOCLEVEL_ERROR;
 813 }
 814
 815 struct mparse *
 816 mparse_alloc(int options, enum mandoclevel wlevel, mandocmsg mmsg,
 817     const char *defos)
 818 {
 819         struct mparse   *curp;
 820
 821         curp = mandoc_calloc(1, sizeof(struct mparse));
 822
 823         curp->options = options;
 824         curp->wlevel = wlevel;
 825         curp->mmsg = mmsg;
 826         curp->defos = defos;
 827
 828         curp->roff = roff_alloc(curp, options);
 829         curp->man = roff_man_alloc( curp->roff, curp, curp->defos,
 830                 curp->options & MPARSE_QUICK ? 1 : 0);
 831         if (curp->options & MPARSE_MDOC) {
 832                 mdoc_hash_init();
 833                 curp->man->macroset = MACROSET_MDOC;
 834         } else if (curp->options & MPARSE_MAN) {
 835                 man_hash_init();
 836                 curp->man->macroset = MACROSET_MAN;
 837         }
 838         curp->man->first->tok = TOKEN_NONE;
 839         return curp;
 840 }
 841
 842 void
 843 mparse_reset(struct mparse *curp)
 844 {
 845
 846         roff_reset(curp->roff);
 847
 848         if (curp->man != NULL)
 849                 roff_man_reset(curp->man);
 850         if (curp->secondary)
 851                 curp->secondary->sz = 0;
 852
 853         curp->file_status = MANDOCLEVEL_OK;
 854
 855         free(curp->sodest);
 856         curp->sodest = NULL;
 857 }
 858
 859 void
 860 mparse_free(struct mparse *curp)
 861 {
 862
 863         roff_man_free(curp->man);
 864         if (curp->roff)
 865                 roff_free(curp->roff);
 866         if (curp->secondary)
 867                 free(curp->secondary->buf);
 868
 869         free(curp->secondary);
 870         free(curp->sodest);
 871         free(curp);
 872 }
 873
 874 void
 875 mparse_result(struct mparse *curp, struct roff_man **man,
 876         char **sodest)
 877 {
 878
 879         if (sodest && NULL != (*sodest = curp->sodest)) {
 880                 *man = NULL;
 881                 return;
 882         }
 883         if (man)
 884                 *man = curp->man;
 885 }
 886
 887 void
 888 mandoc_vmsg(enum mandocerr t, struct mparse *m,
 889                 int ln, int pos, const char *fmt, ...)
 890 {
 891         char             buf[256];
 892         va_list          ap;
 893
 894         va_start(ap, fmt);
 895         (void)vsnprintf(buf, sizeof(buf), fmt, ap);
 896         va_end(ap);
 897
 898         mandoc_msg(t, m, ln, pos, buf);
 899 }
 900
 901 void
 902 mandoc_msg(enum mandocerr er, struct mparse *m,
 903                 int ln, int col, const char *msg)
 904 {
 905         enum mandoclevel level;
 906
 907         level = MANDOCLEVEL_UNSUPP;
 908         while (er < mandoclimits[level])
 909                 level--;
 910
 911         if (level < m->wlevel && er != MANDOCERR_FILE)
 912                 return;
 913
 914         if (m->mmsg)
 915                 (*m->mmsg)(er, level, m->file, ln, col, msg);
 916
 917         if (m->file_status < level)
 918                 m->file_status = level;
 919 }
 920
 921 const char *
 922 mparse_strerror(enum mandocerr er)
 923 {
 924
 925         return mandocerrs[er];
 926 }
 927
 928 const char *
 929 mparse_strlevel(enum mandoclevel lvl)
 930 {
 931         return mandoclevels[lvl];
 932 }
 933
 934 void
 935 mparse_keep(struct mparse *p)
 936 {
 937
 938         assert(NULL == p->secondary);
 939         p->secondary = mandoc_calloc(1, sizeof(struct buf));
 940 }
 941
 942 const char *
 943 mparse_getkeep(const struct mparse *p)
 944 {
 945
 946         assert(p->secondary);
 947         return p->secondary->sz ? p->secondary->buf : NULL;
 948 }