read.c

   1 /*      $Id: read.c,v 1.198 2018/08/23 19:33:27 schwarze Exp $ */
   2 /*
   3  * Copyright (c) 2008, 2009, 2010, 2011 Kristaps Dzonsons <kristaps@bsd.lv>
   4  * Copyright (c) 2010-2018 Ingo Schwarze <schwarze@openbsd.org>
   5  * Copyright (c) 2010, 2012 Joerg Sonnenberger <joerg@netbsd.org>
   6  *
   7  * Permission to use, copy, modify, and distribute this software for any
   8  * purpose with or without fee is hereby granted, provided that the above
   9  * copyright notice and this permission notice appear in all copies.
  10  *
  11  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHORS DISCLAIM ALL WARRANTIES
  12  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
  13  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR
  14  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
  15  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
  16  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
  17  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  18  */
  19 #include "config.h"
  20
  21 #include <sys/types.h>
  22 #include <sys/mman.h>
  23 #include <sys/stat.h>
  24
  25 #include <assert.h>
  26 #include <ctype.h>
  27 #include <errno.h>
  28 #include <fcntl.h>
  29 #include <stdarg.h>
  30 #include <stdio.h>
  31 #include <stdlib.h>
  32 #include <string.h>
  33 #include <unistd.h>
  34 #include <zlib.h>
  35
  36 #include "mandoc_aux.h"
  37 #include "mandoc.h"
  38 #include "roff.h"
  39 #include "mdoc.h"
  40 #include "man.h"
  41 #include "libmandoc.h"
  42
  43 #define REPARSE_LIMIT   1000
  44
  45 struct  mparse {
  46         struct roff      *roff; /* roff parser (!NULL) */
  47         struct roff_man  *man; /* man parser */
  48         char             *sodest; /* filename pointed to by .so */
  49         const char       *file; /* filename of current input file */
  50         struct buf       *primary; /* buffer currently being parsed */
  51         struct buf       *secondary; /* copy of top level input */
  52         const char       *os_s; /* default operating system */
  53         mandocmsg         mmsg; /* warning/error message handler */
  54         enum mandoclevel  file_status; /* status of current parse */
  55         enum mandocerr    mmin; /* ignore messages below this */
  56         int               options; /* parser options */
  57         int               gzip; /* current input file is gzipped */
  58         int               filenc; /* encoding of the current file */
  59         int               reparse_count; /* finite interp. stack */
  60         int               line; /* line number in the file */
  61 };
  62
  63 static  void      choose_parser(struct mparse *);
  64 static  void      free_buf_list(struct buf *);
  65 static  void      resize_buf(struct buf *, size_t);
  66 static  enum rofferr mparse_buf_r(struct mparse *, struct buf, size_t, int);
  67 static  int       read_whole_file(struct mparse *, const char *, int,
  68                                 struct buf *, int *);
  69 static  void      mparse_end(struct mparse *);
  70 static  void      mparse_parse_buffer(struct mparse *, struct buf,
  71                         const char *);
  72
  73 static  const enum mandocerr    mandoclimits[MANDOCLEVEL_MAX] = {
  74         MANDOCERR_OK,
  75         MANDOCERR_OK,
  76         MANDOCERR_WARNING,
  77         MANDOCERR_ERROR,
  78         MANDOCERR_UNSUPP,
  79         MANDOCERR_MAX,
  80         MANDOCERR_MAX
  81 };
  82
  83 static  const char * const      mandocerrs[MANDOCERR_MAX] = {
  84         "ok",
  85
  86         "base system convention",
  87
  88         "Mdocdate found",
  89         "Mdocdate missing",
  90         "unknown architecture",
  91         "operating system explicitly specified",
  92         "RCS id missing",
  93         "referenced manual not found",
  94
  95         "generic style suggestion",
  96
  97         "legacy man(7) date format",
  98         "normalizing date format to",
  99         "lower case character in document title",
 100         "duplicate RCS id",
 101         "possible typo in section name",
 102         "unterminated quoted argument",
 103         "useless macro",
 104         "consider using OS macro",
 105         "errnos out of order",
 106         "duplicate errno",
 107         "trailing delimiter",
 108         "no blank before trailing delimiter",
 109         "fill mode already enabled, skipping",
 110         "fill mode already disabled, skipping",
 111         "verbatim \"--\", maybe consider using \\(em",
 112         "function name without markup",
 113         "whitespace at end of input line",
 114         "bad comment style",
 115
 116         "generic warning",
 117
 118         /* related to the prologue */
 119         "missing manual title, using UNTITLED",
 120         "missing manual title, using \"\"",
 121         "missing manual section, using \"\"",
 122         "unknown manual section",
 123         "missing date, using today's date",
 124         "cannot parse date, using it verbatim",
 125         "date in the future, using it anyway",
 126         "missing Os macro, using \"\"",
 127         "late prologue macro",
 128         "prologue macros out of order",
 129
 130         /* related to document structure */
 131         ".so is fragile, better use ln(1)",
 132         "no document body",
 133         "content before first section header",
 134         "first section is not \"NAME\"",
 135         "NAME section without Nm before Nd",
 136         "NAME section without description",
 137         "description not at the end of NAME",
 138         "bad NAME section content",
 139         "missing comma before name",
 140         "missing description line, using \"\"",
 141         "description line outside NAME section",
 142         "sections out of conventional order",
 143         "duplicate section title",
 144         "unexpected section",
 145         "cross reference to self",
 146         "unusual Xr order",
 147         "unusual Xr punctuation",
 148         "AUTHORS section without An macro",
 149
 150         /* related to macros and nesting */
 151         "obsolete macro",
 152         "macro neither callable nor escaped",
 153         "skipping paragraph macro",
 154         "moving paragraph macro out of list",
 155         "skipping no-space macro",
 156         "blocks badly nested",
 157         "nested displays are not portable",
 158         "moving content out of list",
 159         "first macro on line",
 160         "line scope broken",
 161         "skipping blank line in line scope",
 162
 163         /* related to missing macro arguments */
 164         "skipping empty request",
 165         "conditional request controls empty scope",
 166         "skipping empty macro",
 167         "empty block",
 168         "empty argument, using 0n",
 169         "missing display type, using -ragged",
 170         "list type is not the first argument",
 171         "missing -width in -tag list, using 6n",
 172         "missing utility name, using \"\"",
 173         "missing function name, using \"\"",
 174         "empty head in list item",
 175         "empty list item",
 176         "missing argument, using next line",
 177         "missing font type, using \\fR",
 178         "unknown font type, using \\fR",
 179         "nothing follows prefix",
 180         "empty reference block",
 181         "missing section argument",
 182         "missing -std argument, adding it",
 183         "missing option string, using \"\"",
 184         "missing resource identifier, using \"\"",
 185         "missing eqn box, using \"\"",
 186
 187         /* related to bad macro arguments */
 188         "duplicate argument",
 189         "skipping duplicate argument",
 190         "skipping duplicate display type",
 191         "skipping duplicate list type",
 192         "skipping -width argument",
 193         "wrong number of cells",
 194         "unknown AT&T UNIX version",
 195         "comma in function argument",
 196         "parenthesis in function name",
 197         "unknown library name",
 198         "invalid content in Rs block",
 199         "invalid Boolean argument",
 200         "unknown font, skipping request",
 201         "odd number of characters in request",
 202
 203         /* related to plain text */
 204         "blank line in fill mode, using .sp",
 205         "tab in filled text",
 206         "new sentence, new line",
 207         "invalid escape sequence",
 208         "undefined string, using \"\"",
 209
 210         /* related to tables */
 211         "tbl line starts with span",
 212         "tbl column starts with span",
 213         "skipping vertical bar in tbl layout",
 214
 215         "generic error",
 216
 217         /* related to tables */
 218         "non-alphabetic character in tbl options",
 219         "skipping unknown tbl option",
 220         "missing tbl option argument",
 221         "wrong tbl option argument size",
 222         "empty tbl layout",
 223         "invalid character in tbl layout",
 224         "unmatched parenthesis in tbl layout",
 225         "tbl without any data cells",
 226         "ignoring data in spanned tbl cell",
 227         "ignoring extra tbl data cells",
 228         "data block open at end of tbl",
 229
 230         /* related to document structure and macros */
 231         NULL,
 232         "duplicate prologue macro",
 233         "skipping late title macro",
 234         "input stack limit exceeded, infinite loop?",
 235         "skipping bad character",
 236         "skipping unknown macro",
 237         "ignoring request outside macro",
 238         "skipping insecure request",
 239         "skipping item outside list",
 240         "skipping column outside column list",
 241         "skipping end of block that is not open",
 242         "fewer RS blocks open, skipping",
 243         "inserting missing end of block",
 244         "appending missing end of block",
 245
 246         /* related to request and macro arguments */
 247         "escaped character not allowed in a name",
 248         "using macro argument outside macro",
 249         "argument number is not numeric",
 250         "NOT IMPLEMENTED: Bd -file",
 251         "skipping display without arguments",
 252         "missing list type, using -item",
 253         "argument is not numeric, using 1",
 254         "missing manual name, using \"\"",
 255         "uname(3) system call failed, using UNKNOWN",
 256         "unknown standard specifier",
 257         "skipping request without numeric argument",
 258         "excessive shift",
 259         "NOT IMPLEMENTED: .so with absolute path or \"..\"",
 260         ".so request failed",
 261         "skipping all arguments",
 262         "skipping excess arguments",
 263         "divide by zero",
 264
 265         "unsupported feature",
 266         "input too large",
 267         "unsupported control character",
 268         "unsupported roff request",
 269         "eqn delim option in tbl",
 270         "unsupported tbl layout modifier",
 271         "ignoring macro in table",
 272 };
 273
 274 static  const char * const      mandoclevels[MANDOCLEVEL_MAX] = {
 275         "SUCCESS",
 276         "STYLE",
 277         "WARNING",
 278         "ERROR",
 279         "UNSUPP",
 280         "BADARG",
 281         "SYSERR"
 282 };
 283
 284
 285 static void
 286 resize_buf(struct buf *buf, size_t initial)
 287 {
 288
 289         buf->sz = buf->sz > initial/2 ? 2 * buf->sz : initial;
 290         buf->buf = mandoc_realloc(buf->buf, buf->sz);
 291 }
 292
 293 static void
 294 free_buf_list(struct buf *buf)
 295 {
 296         struct buf *tmp;
 297
 298         while (buf != NULL) {
 299                 tmp = buf;
 300                 buf = tmp->next;
 301                 free(tmp->buf);
 302                 free(tmp);
 303         }
 304 }
 305
 306 static void
 307 choose_parser(struct mparse *curp)
 308 {
 309         char            *cp, *ep;
 310         int              format;
 311
 312         /*
 313          * If neither command line arguments -mdoc or -man select
 314          * a parser nor the roff parser found a .Dd or .TH macro
 315          * yet, look ahead in the main input buffer.
 316          */
 317
 318         if ((format = roff_getformat(curp->roff)) == 0) {
 319                 cp = curp->primary->buf;
 320                 ep = cp + curp->primary->sz;
 321                 while (cp < ep) {
 322                         if (*cp == '.' || *cp == '\'') {
 323                                 cp++;
 324                                 if (cp[0] == 'D' && cp[1] == 'd') {
 325                                         format = MPARSE_MDOC;
 326                                         break;
 327                                 }
 328                                 if (cp[0] == 'T' && cp[1] == 'H') {
 329                                         format = MPARSE_MAN;
 330                                         break;
 331                                 }
 332                         }
 333                         cp = memchr(cp, '\n', ep - cp);
 334                         if (cp == NULL)
 335                                 break;
 336                         cp++;
 337                 }
 338         }
 339
 340         if (format == MPARSE_MDOC) {
 341                 curp->man->macroset = MACROSET_MDOC;
 342                 if (curp->man->mdocmac == NULL)
 343                         curp->man->mdocmac = roffhash_alloc(MDOC_Dd, MDOC_MAX);
 344         } else {
 345                 curp->man->macroset = MACROSET_MAN;
 346                 if (curp->man->manmac == NULL)
 347                         curp->man->manmac = roffhash_alloc(MAN_TH, MAN_MAX);
 348         }
 349         curp->man->first->tok = TOKEN_NONE;
 350 }
 351
 352 /*
 353  * Main parse routine for a buffer.
 354  * It assumes encoding and line numbering are already set up.
 355  * It can recurse directly (for invocations of user-defined
 356  * macros, inline equations, and input line traps)
 357  * and indirectly (for .so file inclusion).
 358  */
 359 static enum rofferr
 360 mparse_buf_r(struct mparse *curp, struct buf blk, size_t i, int start)
 361 {
 362         struct buf       ln;
 363         struct buf      *firstln, *lastln, *thisln;
 364         const char      *save_file;
 365         char            *cp;
 366         size_t           pos; /* byte number in the ln buffer */
 367         enum rofferr     line_result, result;
 368         int              of;
 369         int              lnn; /* line number in the real file */
 370         int              fd;
 371         unsigned char    c;
 372
 373         ln.sz = 256;
 374         ln.buf = mandoc_malloc(ln.sz);
 375         ln.next = NULL;
 376         firstln = NULL;
 377         lnn = curp->line;
 378         pos = 0;
 379         result = ROFF_CONT;
 380
 381         while (i < blk.sz) {
 382                 if (0 == pos && '\0' == blk.buf[i])
 383                         break;
 384
 385                 if (start) {
 386                         curp->line = lnn;
 387                         curp->reparse_count = 0;
 388
 389                         if (lnn < 3 &&
 390                             curp->filenc & MPARSE_UTF8 &&
 391                             curp->filenc & MPARSE_LATIN1)
 392                                 curp->filenc = preconv_cue(&blk, i);
 393                 }
 394
 395                 while (i < blk.sz && (start || blk.buf[i] != '\0')) {
 396
 397                         /*
 398                          * When finding an unescaped newline character,
 399                          * leave the character loop to process the line.
 400                          * Skip a preceding carriage return, if any.
 401                          */
 402
 403                         if ('\r' == blk.buf[i] && i + 1 < blk.sz &&
 404                             '\n' == blk.buf[i + 1])
 405                                 ++i;
 406                         if ('\n' == blk.buf[i]) {
 407                                 ++i;
 408                                 ++lnn;
 409                                 break;
 410                         }
 411
 412                         /*
 413                          * Make sure we have space for the worst
 414                          * case of 12 bytes: "\\[u10ffff]\n\0"
 415                          */
 416
 417                         if (pos + 12 > ln.sz)
 418                                 resize_buf(&ln, 256);
 419
 420                         /*
 421                          * Encode 8-bit input.
 422                          */
 423
 424                         c = blk.buf[i];
 425                         if (c & 0x80) {
 426                                 if ( ! (curp->filenc && preconv_encode(
 427                                     &blk, &i, &ln, &pos, &curp->filenc))) {
 428                                         mandoc_vmsg(MANDOCERR_CHAR_BAD, curp,
 429                                             curp->line, pos, "0x%x", c);
 430                                         ln.buf[pos++] = '?';
 431                                         i++;
 432                                 }
 433                                 continue;
 434                         }
 435
 436                         /*
 437                          * Exclude control characters.
 438                          */
 439
 440                         if (c == 0x7f || (c < 0x20 && c != 0x09)) {
 441                                 mandoc_vmsg(c == 0x00 || c == 0x04 ||
 442                                     c > 0x0a ? MANDOCERR_CHAR_BAD :
 443                                     MANDOCERR_CHAR_UNSUPP,
 444                                     curp, curp->line, pos, "0x%x", c);
 445                                 i++;
 446                                 if (c != '\r')
 447                                         ln.buf[pos++] = '?';
 448                                 continue;
 449                         }
 450
 451                         ln.buf[pos++] = blk.buf[i++];
 452                 }
 453                 ln.buf[pos] = '\0';
 454
 455                 /*
 456                  * Maintain a lookaside buffer of all lines.
 457                  * parsed from this input source.
 458                  */
 459
 460                 thisln = mandoc_malloc(sizeof(*thisln));
 461                 thisln->buf = mandoc_strdup(ln.buf);
 462                 thisln->sz = strlen(ln.buf) + 1;
 463                 thisln->next = NULL;
 464                 if (firstln == NULL) {
 465                         firstln = lastln = thisln;
 466                         if (curp->secondary == NULL)
 467                                 curp->secondary = firstln;
 468                 } else {
 469                         lastln->next = thisln;
 470                         lastln = thisln;
 471                 }
 472
 473                 /* XXX Ugly hack to mark the end of the input. */
 474
 475                 if (i == blk.sz || blk.buf[i] == '\0') {
 476                         ln.buf[pos++] = '\n';
 477                         ln.buf[pos] = '\0';
 478                 }
 479
 480                 /*
 481                  * A significant amount of complexity is contained by
 482                  * the roff preprocessor.  It's line-oriented but can be
 483                  * expressed on one line, so we need at times to
 484                  * readjust our starting point and re-run it.  The roff
 485                  * preprocessor can also readjust the buffers with new
 486                  * data, so we pass them in wholesale.
 487                  */
 488
 489                 of = 0;
 490 rerun:
 491                 line_result = roff_parseln(curp->roff, curp->line, &ln, &of);
 492
 493                 switch (line_result) {
 494                 case ROFF_REPARSE:
 495                 case ROFF_USERCALL:
 496                         if (++curp->reparse_count > REPARSE_LIMIT) {
 497                                 result = ROFF_IGN;
 498                                 mandoc_msg(MANDOCERR_ROFFLOOP, curp,
 499                                     curp->line, pos, NULL);
 500                         } else {
 501                                 result = mparse_buf_r(curp, ln, of, 0);
 502                                 if (line_result == ROFF_USERCALL) {
 503                                         if (result == ROFF_USERRET)
 504                                                 result = ROFF_CONT;
 505                                         roff_userret(curp->roff);
 506                                 }
 507                                 if (start || result == ROFF_CONT) {
 508                                         pos = 0;
 509                                         continue;
 510                                 }
 511                         }
 512                         goto out;
 513                 case ROFF_USERRET:
 514                         if (start) {
 515                                 pos = 0;
 516                                 continue;
 517                         }
 518                         result = ROFF_USERRET;
 519                         goto out;
 520                 case ROFF_APPEND:
 521                         pos = strlen(ln.buf);
 522                         continue;
 523                 case ROFF_RERUN:
 524                         goto rerun;
 525                 case ROFF_IGN:
 526                         pos = 0;
 527                         continue;
 528                 case ROFF_SO:
 529                         if ( ! (curp->options & MPARSE_SO) &&
 530                             (i >= blk.sz || blk.buf[i] == '\0')) {
 531                                 curp->sodest = mandoc_strdup(ln.buf + of);
 532                                 goto out;
 533                         }
 534                         save_file = curp->file;
 535                         if ((fd = mparse_open(curp, ln.buf + of)) != -1) {
 536                                 mparse_readfd(curp, fd, ln.buf + of);
 537                                 close(fd);
 538                                 curp->file = save_file;
 539                         } else {
 540                                 curp->file = save_file;
 541                                 mandoc_vmsg(MANDOCERR_SO_FAIL,
 542                                     curp, curp->line, pos,
 543                                     ".so %s", ln.buf + of);
 544                                 ln.sz = mandoc_asprintf(&cp,
 545                                     ".sp\nSee the file %s.\n.sp",
 546                                     ln.buf + of);
 547                                 free(ln.buf);
 548                                 ln.buf = cp;
 549                                 of = 0;
 550                                 mparse_buf_r(curp, ln, of, 0);
 551                         }
 552                         pos = 0;
 553                         continue;
 554                 default:
 555                         break;
 556                 }
 557
 558                 if (curp->man->macroset == MACROSET_NONE)
 559                         choose_parser(curp);
 560
 561                 if ((curp->man->macroset == MACROSET_MDOC ?
 562                     mdoc_parseln(curp->man, curp->line, ln.buf, of) :
 563                     man_parseln(curp->man, curp->line, ln.buf, of)) == 2)
 564                                 break;
 565
 566                 /* Temporary buffers typically are not full. */
 567
 568                 if (0 == start && '\0' == blk.buf[i])
 569                         break;
 570
 571                 /* Start the next input line. */
 572
 573                 pos = 0;
 574         }
 575 out:
 576         free(ln.buf);
 577         if (firstln != curp->secondary)
 578                 free_buf_list(firstln);
 579         return result;
 580 }
 581
 582 static int
 583 read_whole_file(struct mparse *curp, const char *file, int fd,
 584                 struct buf *fb, int *with_mmap)
 585 {
 586         struct stat      st;
 587         gzFile           gz;
 588         size_t           off;
 589         ssize_t          ssz;
 590         int              gzerrnum, retval;
 591
 592         if (fstat(fd, &st) == -1) {
 593                 mandoc_vmsg(MANDOCERR_FILE, curp, 0, 0,
 594                     "fstat: %s", strerror(errno));
 595                 return 0;
 596         }
 597
 598         /*
 599          * If we're a regular file, try just reading in the whole entry
 600          * via mmap().  This is faster than reading it into blocks, and
 601          * since each file is only a few bytes to begin with, I'm not
 602          * concerned that this is going to tank any machines.
 603          */
 604
 605         if (curp->gzip == 0 && S_ISREG(st.st_mode)) {
 606                 if (st.st_size > 0x7fffffff) {
 607                         mandoc_msg(MANDOCERR_TOOLARGE, curp, 0, 0, NULL);
 608                         return 0;
 609                 }
 610                 *with_mmap = 1;
 611                 fb->sz = (size_t)st.st_size;
 612                 fb->buf = mmap(NULL, fb->sz, PROT_READ, MAP_SHARED, fd, 0);
 613                 if (fb->buf != MAP_FAILED)
 614                         return 1;
 615         }
 616
 617         if (curp->gzip) {
 618                 /*
 619                  * Duplicating the file descriptor is required
 620                  * because we will have to call gzclose(3)
 621                  * to free memory used internally by zlib,
 622                  * but that will also close the file descriptor,
 623                  * which this function must not do.
 624                  */
 625                 if ((fd = dup(fd)) == -1) {
 626                         mandoc_vmsg(MANDOCERR_FILE, curp, 0, 0,
 627                             "dup: %s", strerror(errno));
 628                         return 0;
 629                 }
 630                 if ((gz = gzdopen(fd, "rb")) == NULL) {
 631                         mandoc_vmsg(MANDOCERR_FILE, curp, 0, 0,
 632                             "gzdopen: %s", strerror(errno));
 633                         close(fd);
 634                         return 0;
 635                 }
 636         } else
 637                 gz = NULL;
 638
 639         /*
 640          * If this isn't a regular file (like, say, stdin), then we must
 641          * go the old way and just read things in bit by bit.
 642          */
 643
 644         *with_mmap = 0;
 645         off = 0;
 646         retval = 0;
 647         fb->sz = 0;
 648         fb->buf = NULL;
 649         for (;;) {
 650                 if (off == fb->sz) {
 651                         if (fb->sz == (1U << 31)) {
 652                                 mandoc_msg(MANDOCERR_TOOLARGE, curp,
 653                                     0, 0, NULL);
 654                                 break;
 655                         }
 656                         resize_buf(fb, 65536);
 657                 }
 658                 ssz = curp->gzip ?
 659                     gzread(gz, fb->buf + (int)off, fb->sz - off) :
 660                     read(fd, fb->buf + (int)off, fb->sz - off);
 661                 if (ssz == 0) {
 662                         fb->sz = off;
 663                         retval = 1;
 664                         break;
 665                 }
 666                 if (ssz == -1) {
 667                         if (curp->gzip)
 668                                 (void)gzerror(gz, &gzerrnum);
 669                         mandoc_vmsg(MANDOCERR_FILE, curp, 0, 0, "read: %s",
 670                             curp->gzip && gzerrnum != Z_ERRNO ?
 671                             zError(gzerrnum) : strerror(errno));
 672                         break;
 673                 }
 674                 off += (size_t)ssz;
 675         }
 676
 677         if (curp->gzip && (gzerrnum = gzclose(gz)) != Z_OK)
 678                 mandoc_vmsg(MANDOCERR_FILE, curp, 0, 0, "gzclose: %s",
 679                     gzerrnum == Z_ERRNO ? strerror(errno) :
 680                     zError(gzerrnum));
 681         if (retval == 0) {
 682                 free(fb->buf);
 683                 fb->buf = NULL;
 684         }
 685         return retval;
 686 }
 687
 688 static void
 689 mparse_end(struct mparse *curp)
 690 {
 691         if (curp->man->macroset == MACROSET_NONE)
 692                 curp->man->macroset = MACROSET_MAN;
 693         if (curp->man->macroset == MACROSET_MDOC)
 694                 mdoc_endparse(curp->man);
 695         else
 696                 man_endparse(curp->man);
 697         roff_endparse(curp->roff);
 698 }
 699
 700 static void
 701 mparse_parse_buffer(struct mparse *curp, struct buf blk, const char *file)
 702 {
 703         struct buf      *svprimary;
 704         const char      *svfile;
 705         size_t           offset;
 706         static int       recursion_depth;
 707
 708         if (64 < recursion_depth) {
 709                 mandoc_msg(MANDOCERR_ROFFLOOP, curp, curp->line, 0, NULL);
 710                 return;
 711         }
 712
 713         /* Line number is per-file. */
 714         svfile = curp->file;
 715         curp->file = file;
 716         svprimary = curp->primary;
 717         curp->primary = &blk;
 718         curp->line = 1;
 719         recursion_depth++;
 720
 721         /* Skip an UTF-8 byte order mark. */
 722         if (curp->filenc & MPARSE_UTF8 && blk.sz > 2 &&
 723             (unsigned char)blk.buf[0] == 0xef &&
 724             (unsigned char)blk.buf[1] == 0xbb &&
 725             (unsigned char)blk.buf[2] == 0xbf) {
 726                 offset = 3;
 727                 curp->filenc &= ~MPARSE_LATIN1;
 728         } else
 729                 offset = 0;
 730
 731         mparse_buf_r(curp, blk, offset, 1);
 732
 733         if (--recursion_depth == 0)
 734                 mparse_end(curp);
 735
 736         curp->primary = svprimary;
 737         curp->file = svfile;
 738 }
 739
 740 enum mandoclevel
 741 mparse_readmem(struct mparse *curp, void *buf, size_t len,
 742                 const char *file)
 743 {
 744         struct buf blk;
 745
 746         blk.buf = buf;
 747         blk.sz = len;
 748
 749         mparse_parse_buffer(curp, blk, file);
 750         return curp->file_status;
 751 }
 752
 753 /*
 754  * Read the whole file into memory and call the parsers.
 755  * Called recursively when an .so request is encountered.
 756  */
 757 enum mandoclevel
 758 mparse_readfd(struct mparse *curp, int fd, const char *file)
 759 {
 760         struct buf       blk;
 761         int              with_mmap;
 762         int              save_filenc;
 763
 764         if (read_whole_file(curp, file, fd, &blk, &with_mmap)) {
 765                 save_filenc = curp->filenc;
 766                 curp->filenc = curp->options &
 767                     (MPARSE_UTF8 | MPARSE_LATIN1);
 768                 mparse_parse_buffer(curp, blk, file);
 769                 curp->filenc = save_filenc;
 770                 if (with_mmap)
 771                         munmap(blk.buf, blk.sz);
 772                 else
 773                         free(blk.buf);
 774         }
 775         return curp->file_status;
 776 }
 777
 778 int
 779 mparse_open(struct mparse *curp, const char *file)
 780 {
 781         char             *cp;
 782         int               fd;
 783
 784         curp->file = file;
 785         cp = strrchr(file, '.');
 786         curp->gzip = (cp != NULL && ! strcmp(cp + 1, "gz"));
 787
 788         /* First try to use the filename as it is. */
 789
 790         if ((fd = open(file, O_RDONLY)) != -1)
 791                 return fd;
 792
 793         /*
 794          * If that doesn't work and the filename doesn't
 795          * already  end in .gz, try appending .gz.
 796          */
 797
 798         if ( ! curp->gzip) {
 799                 mandoc_asprintf(&cp, "%s.gz", file);
 800                 fd = open(cp, O_RDONLY);
 801                 free(cp);
 802                 if (fd != -1) {
 803                         curp->gzip = 1;
 804                         return fd;
 805                 }
 806         }
 807
 808         /* Neither worked, give up. */
 809
 810         mandoc_msg(MANDOCERR_FILE, curp, 0, 0, strerror(errno));
 811         return -1;
 812 }
 813
 814 struct mparse *
 815 mparse_alloc(int options, enum mandocerr mmin, mandocmsg mmsg,
 816     enum mandoc_os os_e, const char *os_s)
 817 {
 818         struct mparse   *curp;
 819
 820         curp = mandoc_calloc(1, sizeof(struct mparse));
 821
 822         curp->options = options;
 823         curp->mmin = mmin;
 824         curp->mmsg = mmsg;
 825         curp->os_s = os_s;
 826
 827         curp->roff = roff_alloc(curp, options);
 828         curp->man = roff_man_alloc(curp->roff, curp, curp->os_s,
 829                 curp->options & MPARSE_QUICK ? 1 : 0);
 830         if (curp->options & MPARSE_MDOC) {
 831                 curp->man->macroset = MACROSET_MDOC;
 832                 if (curp->man->mdocmac == NULL)
 833                         curp->man->mdocmac = roffhash_alloc(MDOC_Dd, MDOC_MAX);
 834         } else if (curp->options & MPARSE_MAN) {
 835                 curp->man->macroset = MACROSET_MAN;
 836                 if (curp->man->manmac == NULL)
 837                         curp->man->manmac = roffhash_alloc(MAN_TH, MAN_MAX);
 838         }
 839         curp->man->first->tok = TOKEN_NONE;
 840         curp->man->meta.os_e = os_e;
 841         return curp;
 842 }
 843
 844 void
 845 mparse_reset(struct mparse *curp)
 846 {
 847         roff_reset(curp->roff);
 848         roff_man_reset(curp->man);
 849         free_buf_list(curp->secondary);
 850         curp->secondary = NULL;
 851
 852         free(curp->sodest);
 853         curp->sodest = NULL;
 854
 855         curp->file_status = MANDOCLEVEL_OK;
 856         curp->gzip = 0;
 857 }
 858
 859 void
 860 mparse_free(struct mparse *curp)
 861 {
 862         roffhash_free(curp->man->mdocmac);
 863         roffhash_free(curp->man->manmac);
 864         roff_man_free(curp->man);
 865         roff_free(curp->roff);
 866         free_buf_list(curp->secondary);
 867         free(curp->sodest);
 868         free(curp);
 869 }
 870
 871 void
 872 mparse_result(struct mparse *curp, struct roff_man **man,
 873         char **sodest)
 874 {
 875
 876         if (sodest && NULL != (*sodest = curp->sodest)) {
 877                 *man = NULL;
 878                 return;
 879         }
 880         if (man)
 881                 *man = curp->man;
 882 }
 883
 884 void
 885 mparse_updaterc(struct mparse *curp, enum mandoclevel *rc)
 886 {
 887         if (curp->file_status > *rc)
 888                 *rc = curp->file_status;
 889 }
 890
 891 void
 892 mandoc_vmsg(enum mandocerr t, struct mparse *m,
 893                 int ln, int pos, const char *fmt, ...)
 894 {
 895         char             buf[256];
 896         va_list          ap;
 897
 898         va_start(ap, fmt);
 899         (void)vsnprintf(buf, sizeof(buf), fmt, ap);
 900         va_end(ap);
 901
 902         mandoc_msg(t, m, ln, pos, buf);
 903 }
 904
 905 void
 906 mandoc_msg(enum mandocerr er, struct mparse *m,
 907                 int ln, int col, const char *msg)
 908 {
 909         enum mandoclevel level;
 910
 911         if (er < m->mmin && er != MANDOCERR_FILE)
 912                 return;
 913
 914         level = MANDOCLEVEL_UNSUPP;
 915         while (er < mandoclimits[level])
 916                 level--;
 917
 918         if (m->mmsg)
 919                 (*m->mmsg)(er, level, m->file, ln, col, msg);
 920
 921         if (m->file_status < level)
 922                 m->file_status = level;
 923 }
 924
 925 const char *
 926 mparse_strerror(enum mandocerr er)
 927 {
 928
 929         return mandocerrs[er];
 930 }
 931
 932 const char *
 933 mparse_strlevel(enum mandoclevel lvl)
 934 {
 935         return mandoclevels[lvl];
 936 }
 937
 938 void
 939 mparse_copy(const struct mparse *p)
 940 {
 941         struct buf      *buf;
 942
 943         for (buf = p->secondary; buf != NULL; buf = buf->next)
 944                 puts(buf->buf);
 945 }