read.c

   1 /*      $Id: read.c,v 1.145 2015/10/30 19:04:16 schwarze Exp $ */
   2 /*
   3  * Copyright (c) 2008, 2009, 2010, 2011 Kristaps Dzonsons <kristaps@bsd.lv>
   4  * Copyright (c) 2010-2015 Ingo Schwarze <schwarze@openbsd.org>
   5  * Copyright (c) 2010, 2012 Joerg Sonnenberger <joerg@netbsd.org>
   6  *
   7  * Permission to use, copy, modify, and distribute this software for any
   8  * purpose with or without fee is hereby granted, provided that the above
   9  * copyright notice and this permission notice appear in all copies.
  10  *
  11  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHORS DISCLAIM ALL WARRANTIES
  12  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
  13  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR
  14  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
  15  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
  16  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
  17  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  18  */
  19 #include "config.h"
  20
  21 #include <sys/types.h>
  22 #if HAVE_MMAP
  23 #include <sys/mman.h>
  24 #include <sys/stat.h>
  25 #endif
  26
  27 #include <assert.h>
  28 #include <ctype.h>
  29 #include <err.h>
  30 #include <errno.h>
  31 #include <fcntl.h>
  32 #include <stdarg.h>
  33 #include <stdint.h>
  34 #include <stdio.h>
  35 #include <stdlib.h>
  36 #include <string.h>
  37 #include <unistd.h>
  38 #include <zlib.h>
  39
  40 #include "mandoc_aux.h"
  41 #include "mandoc.h"
  42 #include "roff.h"
  43 #include "mdoc.h"
  44 #include "man.h"
  45 #include "libmandoc.h"
  46 #include "roff_int.h"
  47
  48 #define REPARSE_LIMIT   1000
  49
  50 struct  mparse {
  51         struct roff_man  *man; /* man parser */
  52         struct roff      *roff; /* roff parser (!NULL) */
  53         char             *sodest; /* filename pointed to by .so */
  54         const char       *file; /* filename of current input file */
  55         struct buf       *primary; /* buffer currently being parsed */
  56         struct buf       *secondary; /* preprocessed copy of input */
  57         const char       *defos; /* default operating system */
  58         mandocmsg         mmsg; /* warning/error message handler */
  59         enum mandoclevel  file_status; /* status of current parse */
  60         enum mandoclevel  wlevel; /* ignore messages below this */
  61         int               options; /* parser options */
  62         int               gzip; /* current input file is gzipped */
  63         int               filenc; /* encoding of the current file */
  64         int               reparse_count; /* finite interp. stack */
  65         int               line; /* line number in the file */
  66 };
  67
  68 static  void      choose_parser(struct mparse *);
  69 static  void      resize_buf(struct buf *, size_t);
  70 static  void      mparse_buf_r(struct mparse *, struct buf, size_t, int);
  71 static  int       read_whole_file(struct mparse *, const char *, int,
  72                                 struct buf *, int *);
  73 static  void      mparse_end(struct mparse *);
  74 static  void      mparse_parse_buffer(struct mparse *, struct buf,
  75                         const char *);
  76
  77 static  const enum mandocerr    mandoclimits[MANDOCLEVEL_MAX] = {
  78         MANDOCERR_OK,
  79         MANDOCERR_WARNING,
  80         MANDOCERR_WARNING,
  81         MANDOCERR_ERROR,
  82         MANDOCERR_UNSUPP,
  83         MANDOCERR_MAX,
  84         MANDOCERR_MAX
  85 };
  86
  87 static  const char * const      mandocerrs[MANDOCERR_MAX] = {
  88         "ok",
  89
  90         "generic warning",
  91
  92         /* related to the prologue */
  93         "missing manual title, using UNTITLED",
  94         "missing manual title, using \"\"",
  95         "lower case character in document title",
  96         "missing manual section, using \"\"",
  97         "unknown manual section",
  98         "missing date, using today's date",
  99         "cannot parse date, using it verbatim",
 100         "missing Os macro, using \"\"",
 101         "duplicate prologue macro",
 102         "late prologue macro",
 103         "skipping late title macro",
 104         "prologue macros out of order",
 105
 106         /* related to document structure */
 107         ".so is fragile, better use ln(1)",
 108         "no document body",
 109         "content before first section header",
 110         "first section is not \"NAME\"",
 111         "NAME section without name",
 112         "NAME section without description",
 113         "description not at the end of NAME",
 114         "bad NAME section content",
 115         "missing description line, using \"\"",
 116         "sections out of conventional order",
 117         "duplicate section title",
 118         "unexpected section",
 119         "unusual Xr order",
 120         "unusual Xr punctuation",
 121         "AUTHORS section without An macro",
 122
 123         /* related to macros and nesting */
 124         "obsolete macro",
 125         "macro neither callable nor escaped",
 126         "skipping paragraph macro",
 127         "moving paragraph macro out of list",
 128         "skipping no-space macro",
 129         "blocks badly nested",
 130         "nested displays are not portable",
 131         "moving content out of list",
 132         "fill mode already enabled, skipping",
 133         "fill mode already disabled, skipping",
 134         "line scope broken",
 135
 136         /* related to missing macro arguments */
 137         "skipping empty request",
 138         "conditional request controls empty scope",
 139         "skipping empty macro",
 140         "empty block",
 141         "empty argument, using 0n",
 142         "missing display type, using -ragged",
 143         "list type is not the first argument",
 144         "missing -width in -tag list, using 8n",
 145         "missing utility name, using \"\"",
 146         "missing function name, using \"\"",
 147         "empty head in list item",
 148         "empty list item",
 149         "missing font type, using \\fR",
 150         "unknown font type, using \\fR",
 151         "nothing follows prefix",
 152         "empty reference block",
 153         "missing -std argument, adding it",
 154         "missing option string, using \"\"",
 155         "missing resource identifier, using \"\"",
 156         "missing eqn box, using \"\"",
 157
 158         /* related to bad macro arguments */
 159         "unterminated quoted argument",
 160         "duplicate argument",
 161         "skipping duplicate argument",
 162         "skipping duplicate display type",
 163         "skipping duplicate list type",
 164         "skipping -width argument",
 165         "wrong number of cells",
 166         "unknown AT&T UNIX version",
 167         "comma in function argument",
 168         "parenthesis in function name",
 169         "invalid content in Rs block",
 170         "invalid Boolean argument",
 171         "unknown font, skipping request",
 172         "odd number of characters in request",
 173
 174         /* related to plain text */
 175         "blank line in fill mode, using .sp",
 176         "tab in filled text",
 177         "whitespace at end of input line",
 178         "bad comment style",
 179         "invalid escape sequence",
 180         "undefined string, using \"\"",
 181
 182         /* related to tables */
 183         "tbl line starts with span",
 184         "tbl column starts with span",
 185         "skipping vertical bar in tbl layout",
 186
 187         "generic error",
 188
 189         /* related to tables */
 190         "non-alphabetic character in tbl options",
 191         "skipping unknown tbl option",
 192         "missing tbl option argument",
 193         "wrong tbl option argument size",
 194         "empty tbl layout",
 195         "invalid character in tbl layout",
 196         "unmatched parenthesis in tbl layout",
 197         "tbl without any data cells",
 198         "ignoring data in spanned tbl cell",
 199         "ignoring extra tbl data cells",
 200         "data block open at end of tbl",
 201
 202         /* related to document structure and macros */
 203         NULL,
 204         "input stack limit exceeded, infinite loop?",
 205         "skipping bad character",
 206         "skipping unknown macro",
 207         "skipping insecure request",
 208         "skipping item outside list",
 209         "skipping column outside column list",
 210         "skipping end of block that is not open",
 211         "fewer RS blocks open, skipping",
 212         "inserting missing end of block",
 213         "appending missing end of block",
 214
 215         /* related to request and macro arguments */
 216         "escaped character not allowed in a name",
 217         "NOT IMPLEMENTED: Bd -file",
 218         "skipping display without arguments",
 219         "missing list type, using -item",
 220         "missing manual name, using \"\"",
 221         "uname(3) system call failed, using UNKNOWN",
 222         "unknown standard specifier",
 223         "skipping request without numeric argument",
 224         "NOT IMPLEMENTED: .so with absolute path or \"..\"",
 225         ".so request failed",
 226         "skipping all arguments",
 227         "skipping excess arguments",
 228         "divide by zero",
 229
 230         "unsupported feature",
 231         "input too large",
 232         "unsupported control character",
 233         "unsupported roff request",
 234         "eqn delim option in tbl",
 235         "unsupported tbl layout modifier",
 236         "ignoring macro in table",
 237 };
 238
 239 static  const char * const      mandoclevels[MANDOCLEVEL_MAX] = {
 240         "SUCCESS",
 241         "RESERVED",
 242         "WARNING",
 243         "ERROR",
 244         "UNSUPP",
 245         "BADARG",
 246         "SYSERR"
 247 };
 248
 249
 250 static void
 251 resize_buf(struct buf *buf, size_t initial)
 252 {
 253
 254         buf->sz = buf->sz > initial/2 ? 2 * buf->sz : initial;
 255         buf->buf = mandoc_realloc(buf->buf, buf->sz);
 256 }
 257
 258 static void
 259 choose_parser(struct mparse *curp)
 260 {
 261         char            *cp, *ep;
 262         int              format;
 263
 264         /*
 265          * If neither command line arguments -mdoc or -man select
 266          * a parser nor the roff parser found a .Dd or .TH macro
 267          * yet, look ahead in the main input buffer.
 268          */
 269
 270         if ((format = roff_getformat(curp->roff)) == 0) {
 271                 cp = curp->primary->buf;
 272                 ep = cp + curp->primary->sz;
 273                 while (cp < ep) {
 274                         if (*cp == '.' || *cp == '\'') {
 275                                 cp++;
 276                                 if (cp[0] == 'D' && cp[1] == 'd') {
 277                                         format = MPARSE_MDOC;
 278                                         break;
 279                                 }
 280                                 if (cp[0] == 'T' && cp[1] == 'H') {
 281                                         format = MPARSE_MAN;
 282                                         break;
 283                                 }
 284                         }
 285                         cp = memchr(cp, '\n', ep - cp);
 286                         if (cp == NULL)
 287                                 break;
 288                         cp++;
 289                 }
 290         }
 291
 292         if (curp->man == NULL) {
 293                 curp->man = roff_man_alloc(curp->roff, curp, curp->defos,
 294                     curp->options & MPARSE_QUICK ? 1 : 0);
 295                 curp->man->macroset = MACROSET_MAN;
 296                 curp->man->first->tok = TOKEN_NONE;
 297         }
 298
 299         if (format == MPARSE_MDOC) {
 300                 mdoc_hash_init();
 301                 curp->man->macroset = MACROSET_MDOC;
 302                 curp->man->first->tok = TOKEN_NONE;
 303         } else {
 304                 man_hash_init();
 305                 curp->man->macroset = MACROSET_MAN;
 306                 curp->man->first->tok = TOKEN_NONE;
 307         }
 308 }
 309
 310 /*
 311  * Main parse routine for a buffer.
 312  * It assumes encoding and line numbering are already set up.
 313  * It can recurse directly (for invocations of user-defined
 314  * macros, inline equations, and input line traps)
 315  * and indirectly (for .so file inclusion).
 316  */
 317 static void
 318 mparse_buf_r(struct mparse *curp, struct buf blk, size_t i, int start)
 319 {
 320         const struct tbl_span   *span;
 321         struct buf       ln;
 322         const char      *save_file;
 323         char            *cp;
 324         size_t           pos; /* byte number in the ln buffer */
 325         enum rofferr     rr;
 326         int              of;
 327         int              lnn; /* line number in the real file */
 328         int              fd;
 329         unsigned char    c;
 330
 331         memset(&ln, 0, sizeof(ln));
 332
 333         lnn = curp->line;
 334         pos = 0;
 335
 336         while (i < blk.sz) {
 337                 if (0 == pos && '\0' == blk.buf[i])
 338                         break;
 339
 340                 if (start) {
 341                         curp->line = lnn;
 342                         curp->reparse_count = 0;
 343
 344                         if (lnn < 3 &&
 345                             curp->filenc & MPARSE_UTF8 &&
 346                             curp->filenc & MPARSE_LATIN1)
 347                                 curp->filenc = preconv_cue(&blk, i);
 348                 }
 349
 350                 while (i < blk.sz && (start || blk.buf[i] != '\0')) {
 351
 352                         /*
 353                          * When finding an unescaped newline character,
 354                          * leave the character loop to process the line.
 355                          * Skip a preceding carriage return, if any.
 356                          */
 357
 358                         if ('\r' == blk.buf[i] && i + 1 < blk.sz &&
 359                             '\n' == blk.buf[i + 1])
 360                                 ++i;
 361                         if ('\n' == blk.buf[i]) {
 362                                 ++i;
 363                                 ++lnn;
 364                                 break;
 365                         }
 366
 367                         /*
 368                          * Make sure we have space for the worst
 369                          * case of 11 bytes: "\\[u10ffff]\0"
 370                          */
 371
 372                         if (pos + 11 > ln.sz)
 373                                 resize_buf(&ln, 256);
 374
 375                         /*
 376                          * Encode 8-bit input.
 377                          */
 378
 379                         c = blk.buf[i];
 380                         if (c & 0x80) {
 381                                 if ( ! (curp->filenc && preconv_encode(
 382                                     &blk, &i, &ln, &pos, &curp->filenc))) {
 383                                         mandoc_vmsg(MANDOCERR_CHAR_BAD, curp,
 384                                             curp->line, pos, "0x%x", c);
 385                                         ln.buf[pos++] = '?';
 386                                         i++;
 387                                 }
 388                                 continue;
 389                         }
 390
 391                         /*
 392                          * Exclude control characters.
 393                          */
 394
 395                         if (c == 0x7f || (c < 0x20 && c != 0x09)) {
 396                                 mandoc_vmsg(c == 0x00 || c == 0x04 ||
 397                                     c > 0x0a ? MANDOCERR_CHAR_BAD :
 398                                     MANDOCERR_CHAR_UNSUPP,
 399                                     curp, curp->line, pos, "0x%x", c);
 400                                 i++;
 401                                 if (c != '\r')
 402                                         ln.buf[pos++] = '?';
 403                                 continue;
 404                         }
 405
 406                         /* Trailing backslash = a plain char. */
 407
 408                         if (blk.buf[i] != '\\' || i + 1 == blk.sz) {
 409                                 ln.buf[pos++] = blk.buf[i++];
 410                                 continue;
 411                         }
 412
 413                         /*
 414                          * Found escape and at least one other character.
 415                          * When it's a newline character, skip it.
 416                          * When there is a carriage return in between,
 417                          * skip that one as well.
 418                          */
 419
 420                         if ('\r' == blk.buf[i + 1] && i + 2 < blk.sz &&
 421                             '\n' == blk.buf[i + 2])
 422                                 ++i;
 423                         if ('\n' == blk.buf[i + 1]) {
 424                                 i += 2;
 425                                 ++lnn;
 426                                 continue;
 427                         }
 428
 429                         if ('"' == blk.buf[i + 1] || '#' == blk.buf[i + 1]) {
 430                                 i += 2;
 431                                 /* Comment, skip to end of line */
 432                                 for (; i < blk.sz; ++i) {
 433                                         if ('\n' == blk.buf[i]) {
 434                                                 ++i;
 435                                                 ++lnn;
 436                                                 break;
 437                                         }
 438                                 }
 439
 440                                 /* Backout trailing whitespaces */
 441                                 for (; pos > 0; --pos) {
 442                                         if (ln.buf[pos - 1] != ' ')
 443                                                 break;
 444                                         if (pos > 2 && ln.buf[pos - 2] == '\\')
 445                                                 break;
 446                                 }
 447                                 break;
 448                         }
 449
 450                         /* Catch escaped bogus characters. */
 451
 452                         c = (unsigned char) blk.buf[i+1];
 453
 454                         if ( ! (isascii(c) &&
 455                             (isgraph(c) || isblank(c)))) {
 456                                 mandoc_vmsg(MANDOCERR_CHAR_BAD, curp,
 457                                     curp->line, pos, "0x%x", c);
 458                                 i += 2;
 459                                 ln.buf[pos++] = '?';
 460                                 continue;
 461                         }
 462
 463                         /* Some other escape sequence, copy & cont. */
 464
 465                         ln.buf[pos++] = blk.buf[i++];
 466                         ln.buf[pos++] = blk.buf[i++];
 467                 }
 468
 469                 if (pos >= ln.sz)
 470                         resize_buf(&ln, 256);
 471
 472                 ln.buf[pos] = '\0';
 473
 474                 /*
 475                  * A significant amount of complexity is contained by
 476                  * the roff preprocessor.  It's line-oriented but can be
 477                  * expressed on one line, so we need at times to
 478                  * readjust our starting point and re-run it.  The roff
 479                  * preprocessor can also readjust the buffers with new
 480                  * data, so we pass them in wholesale.
 481                  */
 482
 483                 of = 0;
 484
 485                 /*
 486                  * Maintain a lookaside buffer of all parsed lines.  We
 487                  * only do this if mparse_keep() has been invoked (the
 488                  * buffer may be accessed with mparse_getkeep()).
 489                  */
 490
 491                 if (curp->secondary) {
 492                         curp->secondary->buf = mandoc_realloc(
 493                             curp->secondary->buf,
 494                             curp->secondary->sz + pos + 2);
 495                         memcpy(curp->secondary->buf +
 496                             curp->secondary->sz,
 497                             ln.buf, pos);
 498                         curp->secondary->sz += pos;
 499                         curp->secondary->buf
 500                                 [curp->secondary->sz] = '\n';
 501                         curp->secondary->sz++;
 502                         curp->secondary->buf
 503                                 [curp->secondary->sz] = '\0';
 504                 }
 505 rerun:
 506                 rr = roff_parseln(curp->roff, curp->line, &ln, &of);
 507
 508                 switch (rr) {
 509                 case ROFF_REPARSE:
 510                         if (REPARSE_LIMIT >= ++curp->reparse_count)
 511                                 mparse_buf_r(curp, ln, of, 0);
 512                         else
 513                                 mandoc_msg(MANDOCERR_ROFFLOOP, curp,
 514                                     curp->line, pos, NULL);
 515                         pos = 0;
 516                         continue;
 517                 case ROFF_APPEND:
 518                         pos = strlen(ln.buf);
 519                         continue;
 520                 case ROFF_RERUN:
 521                         goto rerun;
 522                 case ROFF_IGN:
 523                         pos = 0;
 524                         continue;
 525                 case ROFF_SO:
 526                         if ( ! (curp->options & MPARSE_SO) &&
 527                             (i >= blk.sz || blk.buf[i] == '\0')) {
 528                                 curp->sodest = mandoc_strdup(ln.buf + of);
 529                                 free(ln.buf);
 530                                 return;
 531                         }
 532                         /*
 533                          * We remove `so' clauses from our lookaside
 534                          * buffer because we're going to descend into
 535                          * the file recursively.
 536                          */
 537                         if (curp->secondary)
 538                                 curp->secondary->sz -= pos + 1;
 539                         save_file = curp->file;
 540                         if (mparse_open(curp, &fd, ln.buf + of) ==
 541                             MANDOCLEVEL_OK) {
 542                                 mparse_readfd(curp, fd, ln.buf + of);
 543                                 curp->file = save_file;
 544                         } else {
 545                                 curp->file = save_file;
 546                                 mandoc_vmsg(MANDOCERR_SO_FAIL,
 547                                     curp, curp->line, pos,
 548                                     ".so %s", ln.buf + of);
 549                                 ln.sz = mandoc_asprintf(&cp,
 550                                     ".sp\nSee the file %s.\n.sp",
 551                                     ln.buf + of);
 552                                 free(ln.buf);
 553                                 ln.buf = cp;
 554                                 of = 0;
 555                                 mparse_buf_r(curp, ln, of, 0);
 556                         }
 557                         pos = 0;
 558                         continue;
 559                 default:
 560                         break;
 561                 }
 562
 563                 /*
 564                  * If input parsers have not been allocated, do so now.
 565                  * We keep these instanced between parsers, but set them
 566                  * locally per parse routine since we can use different
 567                  * parsers with each one.
 568                  */
 569
 570                 if (curp->man == NULL ||
 571                     curp->man->macroset == MACROSET_NONE)
 572                         choose_parser(curp);
 573
 574                 /*
 575                  * Lastly, push down into the parsers themselves.
 576                  * If libroff returns ROFF_TBL, then add it to the
 577                  * currently open parse.  Since we only get here if
 578                  * there does exist data (see tbl_data.c), we're
 579                  * guaranteed that something's been allocated.
 580                  * Do the same for ROFF_EQN.
 581                  */
 582
 583                 if (rr == ROFF_TBL)
 584                         while ((span = roff_span(curp->roff)) != NULL)
 585                                 roff_addtbl(curp->man, span);
 586                 else if (rr == ROFF_EQN)
 587                         roff_addeqn(curp->man, roff_eqn(curp->roff));
 588                 else if ((curp->man->macroset == MACROSET_MDOC ?
 589                     mdoc_parseln(curp->man, curp->line, ln.buf, of) :
 590                     man_parseln(curp->man, curp->line, ln.buf, of)) == 2)
 591                                 break;
 592
 593                 /* Temporary buffers typically are not full. */
 594
 595                 if (0 == start && '\0' == blk.buf[i])
 596                         break;
 597
 598                 /* Start the next input line. */
 599
 600                 pos = 0;
 601         }
 602
 603         free(ln.buf);
 604 }
 605
 606 static int
 607 read_whole_file(struct mparse *curp, const char *file, int fd,
 608                 struct buf *fb, int *with_mmap)
 609 {
 610         gzFile           gz;
 611         size_t           off;
 612         ssize_t          ssz;
 613
 614 #if HAVE_MMAP
 615         struct stat      st;
 616
 617         if (fstat(fd, &st) == -1)
 618                 err((int)MANDOCLEVEL_SYSERR, "%s", file);
 619
 620         /*
 621          * If we're a regular file, try just reading in the whole entry
 622          * via mmap().  This is faster than reading it into blocks, and
 623          * since each file is only a few bytes to begin with, I'm not
 624          * concerned that this is going to tank any machines.
 625          */
 626
 627         if (curp->gzip == 0 && S_ISREG(st.st_mode)) {
 628                 if (st.st_size > 0x7fffffff) {
 629                         mandoc_msg(MANDOCERR_TOOLARGE, curp, 0, 0, NULL);
 630                         return 0;
 631                 }
 632                 *with_mmap = 1;
 633                 fb->sz = (size_t)st.st_size;
 634                 fb->buf = mmap(NULL, fb->sz, PROT_READ, MAP_SHARED, fd, 0);
 635                 if (fb->buf != MAP_FAILED)
 636                         return 1;
 637         }
 638 #endif
 639
 640         if (curp->gzip) {
 641                 if ((gz = gzdopen(fd, "rb")) == NULL)
 642                         err((int)MANDOCLEVEL_SYSERR, "%s", file);
 643         } else
 644                 gz = NULL;
 645
 646         /*
 647          * If this isn't a regular file (like, say, stdin), then we must
 648          * go the old way and just read things in bit by bit.
 649          */
 650
 651         *with_mmap = 0;
 652         off = 0;
 653         fb->sz = 0;
 654         fb->buf = NULL;
 655         for (;;) {
 656                 if (off == fb->sz) {
 657                         if (fb->sz == (1U << 31)) {
 658                                 mandoc_msg(MANDOCERR_TOOLARGE, curp,
 659                                     0, 0, NULL);
 660                                 break;
 661                         }
 662                         resize_buf(fb, 65536);
 663                 }
 664                 ssz = curp->gzip ?
 665                     gzread(gz, fb->buf + (int)off, fb->sz - off) :
 666                     read(fd, fb->buf + (int)off, fb->sz - off);
 667                 if (ssz == 0) {
 668                         fb->sz = off;
 669                         return 1;
 670                 }
 671                 if (ssz == -1)
 672                         err((int)MANDOCLEVEL_SYSERR, "%s", file);
 673                 off += (size_t)ssz;
 674         }
 675
 676         free(fb->buf);
 677         fb->buf = NULL;
 678         return 0;
 679 }
 680
 681 static void
 682 mparse_end(struct mparse *curp)
 683 {
 684
 685         if (curp->man == NULL && curp->sodest == NULL)
 686                 curp->man = roff_man_alloc(curp->roff, curp, curp->defos,
 687                     curp->options & MPARSE_QUICK ? 1 : 0);
 688         if (curp->man->macroset == MACROSET_NONE)
 689                 curp->man->macroset = MACROSET_MAN;
 690         if (curp->man->macroset == MACROSET_MDOC)
 691                 mdoc_endparse(curp->man);
 692         else
 693                 man_endparse(curp->man);
 694         roff_endparse(curp->roff);
 695 }
 696
 697 static void
 698 mparse_parse_buffer(struct mparse *curp, struct buf blk, const char *file)
 699 {
 700         struct buf      *svprimary;
 701         const char      *svfile;
 702         size_t           offset;
 703         static int       recursion_depth;
 704
 705         if (64 < recursion_depth) {
 706                 mandoc_msg(MANDOCERR_ROFFLOOP, curp, curp->line, 0, NULL);
 707                 return;
 708         }
 709
 710         /* Line number is per-file. */
 711         svfile = curp->file;
 712         curp->file = file;
 713         svprimary = curp->primary;
 714         curp->primary = &blk;
 715         curp->line = 1;
 716         recursion_depth++;
 717
 718         /* Skip an UTF-8 byte order mark. */
 719         if (curp->filenc & MPARSE_UTF8 && blk.sz > 2 &&
 720             (unsigned char)blk.buf[0] == 0xef &&
 721             (unsigned char)blk.buf[1] == 0xbb &&
 722             (unsigned char)blk.buf[2] == 0xbf) {
 723                 offset = 3;
 724                 curp->filenc &= ~MPARSE_LATIN1;
 725         } else
 726                 offset = 0;
 727
 728         mparse_buf_r(curp, blk, offset, 1);
 729
 730         if (--recursion_depth == 0)
 731                 mparse_end(curp);
 732
 733         curp->primary = svprimary;
 734         curp->file = svfile;
 735 }
 736
 737 enum mandoclevel
 738 mparse_readmem(struct mparse *curp, void *buf, size_t len,
 739                 const char *file)
 740 {
 741         struct buf blk;
 742
 743         blk.buf = buf;
 744         blk.sz = len;
 745
 746         mparse_parse_buffer(curp, blk, file);
 747         return curp->file_status;
 748 }
 749
 750 /*
 751  * Read the whole file into memory and call the parsers.
 752  * Called recursively when an .so request is encountered.
 753  */
 754 enum mandoclevel
 755 mparse_readfd(struct mparse *curp, int fd, const char *file)
 756 {
 757         struct buf       blk;
 758         int              with_mmap;
 759         int              save_filenc;
 760
 761         if (read_whole_file(curp, file, fd, &blk, &with_mmap)) {
 762                 save_filenc = curp->filenc;
 763                 curp->filenc = curp->options &
 764                     (MPARSE_UTF8 | MPARSE_LATIN1);
 765                 mparse_parse_buffer(curp, blk, file);
 766                 curp->filenc = save_filenc;
 767 #if HAVE_MMAP
 768                 if (with_mmap)
 769                         munmap(blk.buf, blk.sz);
 770                 else
 771 #endif
 772                         free(blk.buf);
 773         }
 774
 775         if (fd != STDIN_FILENO && close(fd) == -1)
 776                 perror(file);
 777
 778         return curp->file_status;
 779 }
 780
 781 enum mandoclevel
 782 mparse_open(struct mparse *curp, int *fd, const char *file)
 783 {
 784         char             *cp;
 785
 786         curp->file = file;
 787         cp = strrchr(file, '.');
 788         curp->gzip = (cp != NULL && ! strcmp(cp + 1, "gz"));
 789
 790         /* First try to use the filename as it is. */
 791
 792         if ((*fd = open(file, O_RDONLY)) != -1)
 793                 return MANDOCLEVEL_OK;
 794
 795         /*
 796          * If that doesn't work and the filename doesn't
 797          * already  end in .gz, try appending .gz.
 798          */
 799
 800         if ( ! curp->gzip) {
 801                 mandoc_asprintf(&cp, "%s.gz", file);
 802                 *fd = open(file, O_RDONLY);
 803                 free(cp);
 804                 if (*fd != -1) {
 805                         curp->gzip = 1;
 806                         return MANDOCLEVEL_OK;
 807                 }
 808         }
 809
 810         /* Neither worked, give up. */
 811
 812         mandoc_msg(MANDOCERR_FILE, curp, 0, 0, strerror(errno));
 813         return MANDOCLEVEL_ERROR;
 814 }
 815
 816 struct mparse *
 817 mparse_alloc(int options, enum mandoclevel wlevel, mandocmsg mmsg,
 818     const char *defos)
 819 {
 820         struct mparse   *curp;
 821
 822         curp = mandoc_calloc(1, sizeof(struct mparse));
 823
 824         curp->options = options;
 825         curp->wlevel = wlevel;
 826         curp->mmsg = mmsg;
 827         curp->defos = defos;
 828
 829         curp->roff = roff_alloc(curp, options);
 830         curp->man = roff_man_alloc( curp->roff, curp, curp->defos,
 831                 curp->options & MPARSE_QUICK ? 1 : 0);
 832         if (curp->options & MPARSE_MDOC) {
 833                 mdoc_hash_init();
 834                 curp->man->macroset = MACROSET_MDOC;
 835         } else if (curp->options & MPARSE_MAN) {
 836                 man_hash_init();
 837                 curp->man->macroset = MACROSET_MAN;
 838         }
 839         curp->man->first->tok = TOKEN_NONE;
 840         return curp;
 841 }
 842
 843 void
 844 mparse_reset(struct mparse *curp)
 845 {
 846
 847         roff_reset(curp->roff);
 848
 849         if (curp->man != NULL)
 850                 roff_man_reset(curp->man);
 851         if (curp->secondary)
 852                 curp->secondary->sz = 0;
 853
 854         curp->file_status = MANDOCLEVEL_OK;
 855
 856         free(curp->sodest);
 857         curp->sodest = NULL;
 858 }
 859
 860 void
 861 mparse_free(struct mparse *curp)
 862 {
 863
 864         roff_man_free(curp->man);
 865         if (curp->roff)
 866                 roff_free(curp->roff);
 867         if (curp->secondary)
 868                 free(curp->secondary->buf);
 869
 870         free(curp->secondary);
 871         free(curp->sodest);
 872         free(curp);
 873 }
 874
 875 void
 876 mparse_result(struct mparse *curp, struct roff_man **man,
 877         char **sodest)
 878 {
 879
 880         if (sodest && NULL != (*sodest = curp->sodest)) {
 881                 *man = NULL;
 882                 return;
 883         }
 884         if (man)
 885                 *man = curp->man;
 886 }
 887
 888 void
 889 mandoc_vmsg(enum mandocerr t, struct mparse *m,
 890                 int ln, int pos, const char *fmt, ...)
 891 {
 892         char             buf[256];
 893         va_list          ap;
 894
 895         va_start(ap, fmt);
 896         (void)vsnprintf(buf, sizeof(buf), fmt, ap);
 897         va_end(ap);
 898
 899         mandoc_msg(t, m, ln, pos, buf);
 900 }
 901
 902 void
 903 mandoc_msg(enum mandocerr er, struct mparse *m,
 904                 int ln, int col, const char *msg)
 905 {
 906         enum mandoclevel level;
 907
 908         level = MANDOCLEVEL_UNSUPP;
 909         while (er < mandoclimits[level])
 910                 level--;
 911
 912         if (level < m->wlevel && er != MANDOCERR_FILE)
 913                 return;
 914
 915         if (m->mmsg)
 916                 (*m->mmsg)(er, level, m->file, ln, col, msg);
 917
 918         if (m->file_status < level)
 919                 m->file_status = level;
 920 }
 921
 922 const char *
 923 mparse_strerror(enum mandocerr er)
 924 {
 925
 926         return mandocerrs[er];
 927 }
 928
 929 const char *
 930 mparse_strlevel(enum mandoclevel lvl)
 931 {
 932         return mandoclevels[lvl];
 933 }
 934
 935 void
 936 mparse_keep(struct mparse *p)
 937 {
 938
 939         assert(NULL == p->secondary);
 940         p->secondary = mandoc_calloc(1, sizeof(struct buf));
 941 }
 942
 943 const char *
 944 mparse_getkeep(const struct mparse *p)
 945 {
 946
 947         assert(p->secondary);
 948         return p->secondary->sz ? p->secondary->buf : NULL;
 949 }