-/* $Id: preconv.c,v 1.7 2014/08/10 23:54:41 schwarze Exp $ */
+/* $Id: preconv.c,v 1.17 2018/12/13 11:55:47 schwarze Exp $ */
/*
* Copyright (c) 2011 Kristaps Dzonsons <kristaps@bsd.lv>
+ * Copyright (c) 2014 Ingo Schwarze <schwarze@openbsd.org>
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
#include "config.h"
#include <sys/types.h>
-#ifdef HAVE_MMAP
-#include <sys/stat.h>
-#include <sys/mman.h>
-#endif
#include <assert.h>
-#include <fcntl.h>
#include <stdio.h>
-#include <stdlib.h>
#include <string.h>
-#include <unistd.h>
-/*
- * The read_whole_file() and resize_buf() functions are copied from
- * read.c, including all dependency code.
- */
-
-enum enc {
- ENC_UTF_8, /* UTF-8 */
- ENC_US_ASCII, /* US-ASCII */
- ENC_LATIN_1, /* Latin-1 */
- ENC__MAX
-};
-
-struct buf {
- char *buf; /* binary input buffer */
- size_t sz; /* size of binary buffer */
- size_t offs; /* starting buffer offset */
-};
-
-struct encode {
- const char *name;
- int (*conv)(const struct buf *);
-};
-
-static int cue_enc(const struct buf *, size_t *, enum enc *);
-static int conv_latin_1(const struct buf *);
-static int conv_us_ascii(const struct buf *);
-static int conv_utf_8(const struct buf *);
-static int read_whole_file(const char *, int,
- struct buf *, int *);
-static void resize_buf(struct buf *, size_t);
-static void usage(void);
-
-static const struct encode encs[ENC__MAX] = {
- { "utf-8", conv_utf_8 }, /* ENC_UTF_8 */
- { "us-ascii", conv_us_ascii }, /* ENC_US_ASCII */
- { "latin-1", conv_latin_1 }, /* ENC_LATIN_1 */
-};
-
-static const char *progname;
-
-static void
-usage(void)
-{
-
- fprintf(stderr, "usage: %s "
- "[-D enc] "
- "[-e ENC] "
- "[file]\n", progname);
-}
+#include "mandoc.h"
+#include "roff.h"
+#include "mandoc_parse.h"
+#include "libmandoc.h"
-static int
-conv_latin_1(const struct buf *b)
+int
+preconv_encode(const struct buf *ib, size_t *ii, struct buf *ob, size_t *oi,
+ int *filenc)
{
- size_t i;
- unsigned char cu;
- const char *cp;
-
- cp = b->buf + (int)b->offs;
-
- /*
- * Latin-1 falls into the first 256 code-points of Unicode, so
- * there's no need for any sort of translation. Just make the
- * 8-bit characters use the Unicode escape.
- * Note that binary values 128 < v < 160 are passed through
- * unmodified to mandoc.
- */
-
- for (i = b->offs; i < b->sz; i++) {
- cu = (unsigned char)*cp++;
- cu < 160U ? putchar(cu) : printf("\\[u%.4X]", cu);
+ const unsigned char *cu;
+ int nby;
+ unsigned int accum;
+
+ cu = (const unsigned char *)ib->buf + *ii;
+ assert(*cu & 0x80);
+
+ if ( ! (*filenc & MPARSE_UTF8))
+ goto latin;
+
+ nby = 1;
+ while (nby < 5 && *cu & (1 << (7 - nby)))
+ nby++;
+
+ switch (nby) {
+ case 2:
+ accum = *cu & 0x1f;
+ if (accum < 0x02) /* Obfuscated ASCII. */
+ goto latin;
+ break;
+ case 3:
+ accum = *cu & 0x0f;
+ break;
+ case 4:
+ accum = *cu & 0x07;
+ if (accum > 0x04) /* Beyond Unicode. */
+ goto latin;
+ break;
+ default: /* Bad sequence header. */
+ goto latin;
}
- return(1);
-}
-
-static int
-conv_us_ascii(const struct buf *b)
-{
-
- /*
- * US-ASCII has no conversion since it falls into the first 128
- * bytes of Unicode.
- */
-
- fwrite(b->buf, 1, b->sz, stdout);
- return(1);
-}
-
-static int
-conv_utf_8(const struct buf *b)
-{
- int state, be;
- unsigned int accum;
- size_t i;
- unsigned char cu;
- const char *cp;
- const long one = 1L;
-
- cp = b->buf + (int)b->offs;
- state = 0;
- accum = 0U;
- be = 0;
-
- /* Quick test for big-endian value. */
-
- if ( ! (*((const char *)(&one))))
- be = 1;
-
- for (i = b->offs; i < b->sz; i++) {
- cu = (unsigned char)*cp++;
- if (state) {
- if ( ! (cu & 128) || (cu & 64)) {
- /* Bad sequence header. */
- return(0);
- }
-
- /* Accept only legitimate bit patterns. */
-
- if (cu > 191 || cu < 128) {
- /* Bad in-sequence bits. */
- return(0);
- }
-
- accum |= (cu & 63) << --state * 6;
-
- /*
- * Accum is held in little-endian order as
- * stipulated by the UTF-8 sequence coding. We
- * need to convert to a native big-endian if our
- * architecture requires it.
- */
-
- if (0 == state && be)
- accum = (accum >> 24) |
- ((accum << 8) & 0x00FF0000) |
- ((accum >> 8) & 0x0000FF00) |
- (accum << 24);
-
- if (0 == state) {
- accum < 128U ? putchar(accum) :
- printf("\\[u%.4X]", accum);
- accum = 0U;
- }
- } else if (cu & (1 << 7)) {
- /*
- * Entering a UTF-8 state: if we encounter a
- * UTF-8 bitmask, calculate the expected UTF-8
- * state from it.
- */
- for (state = 0; state < 7; state++)
- if ( ! (cu & (1 << (7 - state))))
- break;
-
- /* Accept only legitimate bit patterns. */
-
- switch (state) {
- case (4):
- if (cu <= 244 && cu >= 240) {
- accum = (cu & 7) << 18;
- break;
- }
- /* Bad 4-sequence start bits. */
- return(0);
- case (3):
- if (cu <= 239 && cu >= 224) {
- accum = (cu & 15) << 12;
- break;
- }
- /* Bad 3-sequence start bits. */
- return(0);
- case (2):
- if (cu <= 223 && cu >= 194) {
- accum = (cu & 31) << 6;
- break;
- }
- /* Bad 2-sequence start bits. */
- return(0);
- default:
- /* Bad sequence bit mask. */
- return(0);
- }
- state--;
- } else
- putchar(cu);
+ cu++;
+ switch (nby) {
+ case 3:
+ if ((accum == 0x00 && ! (*cu & 0x20)) || /* Use 2-byte. */
+ (accum == 0x0d && *cu & 0x20)) /* Surrogates. */
+ goto latin;
+ break;
+ case 4:
+ if ((accum == 0x00 && ! (*cu & 0x30)) || /* Use 3-byte. */
+ (accum == 0x04 && *cu & 0x30)) /* Beyond Unicode. */
+ goto latin;
+ break;
+ default:
+ break;
}
- if (0 != state) {
- /* Bad trailing bits. */
- return(0);
+ while (--nby) {
+ if ((*cu & 0xc0) != 0x80) /* Invalid continuation. */
+ goto latin;
+ accum <<= 6;
+ accum += *cu & 0x3f;
+ cu++;
}
- return(1);
-}
+ assert(accum > 0x7f);
+ assert(accum < 0x110000);
+ assert(accum < 0xd800 || accum > 0xdfff);
-static void
-resize_buf(struct buf *buf, size_t initial)
-{
+ *oi += snprintf(ob->buf + *oi, 11, "\\[u%.4X]", accum);
+ *ii = (const char *)cu - ib->buf;
+ *filenc &= ~MPARSE_LATIN1;
+ return 1;
- buf->sz = buf->sz > initial / 2 ?
- 2 * buf->sz : initial;
+latin:
+ if ( ! (*filenc & MPARSE_LATIN1))
+ return 0;
- buf->buf = realloc(buf->buf, buf->sz);
- if (NULL == buf->buf) {
- perror(NULL);
- exit(EXIT_FAILURE);
- }
-}
-
-static int
-read_whole_file(const char *f, int fd,
- struct buf *fb, int *with_mmap)
-{
- size_t off;
- ssize_t ssz;
-
-#ifdef HAVE_MMAP
- struct stat st;
- if (-1 == fstat(fd, &st)) {
- perror(f);
- return(0);
- }
+ *oi += snprintf(ob->buf + *oi, 11,
+ "\\[u%.4X]", (unsigned char)ib->buf[(*ii)++]);
- /*
- * If we're a regular file, try just reading in the whole entry
- * via mmap(). This is faster than reading it into blocks, and
- * since each file is only a few bytes to begin with, I'm not
- * concerned that this is going to tank any machines.
- */
-
- if (S_ISREG(st.st_mode) && st.st_size >= (1U << 31)) {
- fprintf(stderr, "%s: input too large\n", f);
- return(0);
- }
-
- if (S_ISREG(st.st_mode)) {
- *with_mmap = 1;
- fb->sz = (size_t)st.st_size;
- fb->buf = mmap(NULL, fb->sz, PROT_READ, MAP_SHARED, fd, 0);
- if (fb->buf != MAP_FAILED)
- return(1);
- }
-#endif
-
- /*
- * If this isn't a regular file (like, say, stdin), then we must
- * go the old way and just read things in bit by bit.
- */
-
- *with_mmap = 0;
- off = 0;
- fb->sz = 0;
- fb->buf = NULL;
- for (;;) {
- if (off == fb->sz && fb->sz == (1U << 31)) {
- fprintf(stderr, "%s: input too large\n", f);
- break;
- }
-
- if (off == fb->sz)
- resize_buf(fb, 65536);
-
- ssz = read(fd, fb->buf + (int)off, fb->sz - off);
- if (ssz == 0) {
- fb->sz = off;
- return(1);
- }
- if (ssz == -1) {
- perror(f);
- break;
- }
- off += (size_t)ssz;
- }
-
- free(fb->buf);
- fb->buf = NULL;
- return(0);
+ *filenc &= ~MPARSE_UTF8;
+ return 1;
}
-static int
-cue_enc(const struct buf *b, size_t *offs, enum enc *enc)
+int
+preconv_cue(const struct buf *b, size_t offset)
{
const char *ln, *eoln, *eoph;
- size_t sz, phsz, nsz;
- int i;
+ size_t sz, phsz;
- ln = b->buf + (int)*offs;
- sz = b->sz - *offs;
+ ln = b->buf + offset;
+ sz = b->sz - offset;
/* Look for the end-of-line. */
if (NULL == (eoln = memchr(ln, '\n', sz)))
- return(-1);
-
- /* Set next-line marker. */
-
- *offs = (size_t)((eoln + 1) - b->buf);
+ eoln = ln + sz;
/* Check if we have the correct header/trailer. */
- if ((sz = (size_t)(eoln - ln)) < 10 ||
- memcmp(ln, ".\\\" -*-", 7) ||
- memcmp(eoln - 3, "-*-", 3))
- return(0);
+ if ((sz = (size_t)(eoln - ln)) < 10 ||
+ memcmp(ln, ".\\\" -*-", 7) || memcmp(eoln - 3, "-*-", 3))
+ return MPARSE_UTF8 | MPARSE_LATIN1;
/* Move after the header and adjust for the trailer. */
/* Only account for the "coding" phrase. */
- if ((phsz = (size_t)(eoph - ln)) < 7 ||
- strncasecmp(ln, "coding:", 7)) {
+ if ((phsz = eoph - ln) < 7 ||
+ strncasecmp(ln, "coding:", 7)) {
sz -= phsz;
ln += phsz;
continue;
- }
+ }
sz -= 7;
ln += 7;
sz--;
}
if (0 == sz)
- break;
+ return 0;
/* Check us against known encodings. */
- for (i = 0; i < (int)ENC__MAX; i++) {
- nsz = strlen(encs[i].name);
- if (phsz < nsz)
- continue;
- if (strncasecmp(ln, encs[i].name, nsz))
- continue;
-
- *enc = (enum enc)i;
- return(1);
- }
-
- /* Unknown encoding. */
-
- *enc = ENC__MAX;
- return(1);
- }
-
- return(0);
-}
-
-int
-main(int argc, char *argv[])
-{
- int i, ch, map, fd, rc;
- struct buf b;
- const char *fn;
- enum enc enc, def;
- unsigned char bom[3] = { 0xEF, 0xBB, 0xBF };
- size_t offs;
- extern int optind;
- extern char *optarg;
-
- progname = strrchr(argv[0], '/');
- if (progname == NULL)
- progname = argv[0];
- else
- ++progname;
-
- fn = "<stdin>";
- fd = STDIN_FILENO;
- rc = EXIT_FAILURE;
- enc = def = ENC__MAX;
- map = 0;
-
- memset(&b, 0, sizeof(struct buf));
-
- while (-1 != (ch = getopt(argc, argv, "D:e:rdvh")))
- switch (ch) {
- case ('D'):
- /* FALLTHROUGH */
- case ('e'):
- for (i = 0; i < (int)ENC__MAX; i++) {
- if (strcasecmp(optarg, encs[i].name))
- continue;
- break;
- }
- if (i < (int)ENC__MAX) {
- if ('D' == ch)
- def = (enum enc)i;
- else
- enc = (enum enc)i;
- break;
- }
-
- fprintf(stderr, "%s: Bad encoding\n", optarg);
- return(EXIT_FAILURE);
- case ('r'):
- /* FALLTHROUGH */
- case ('d'):
- /* FALLTHROUGH */
- case ('v'):
- /* Compatibility with GNU preconv. */
- break;
- case ('h'):
- /* Compatibility with GNU preconv. */
- /* FALLTHROUGH */
- default:
- usage();
- return(EXIT_FAILURE);
- }
-
- argc -= optind;
- argv += optind;
-
- /*
- * Open and read the first argument on the command-line.
- * If we don't have one, we default to stdin.
- */
-
- if (argc > 0) {
- fn = *argv;
- fd = open(fn, O_RDONLY, 0);
- if (-1 == fd) {
- perror(fn);
- return(EXIT_FAILURE);
- }
- }
-
- if ( ! read_whole_file(fn, fd, &b, &map))
- goto out;
-
- /* Try to read the UTF-8 BOM. */
-
- if (ENC__MAX == enc)
- if (b.sz > 3 && 0 == memcmp(b.buf, bom, 3)) {
- b.offs = 3;
- enc = ENC_UTF_8;
- }
-
- /* Try reading from the "-*-" cue. */
-
- if (ENC__MAX == enc) {
- offs = b.offs;
- ch = cue_enc(&b, &offs, &enc);
- if (0 == ch)
- ch = cue_enc(&b, &offs, &enc);
+ if (phsz > 4 && !strncasecmp(ln, "utf-8", 5))
+ return MPARSE_UTF8;
+ if (phsz > 10 && !strncasecmp(ln, "iso-latin-1", 11))
+ return MPARSE_LATIN1;
+ return 0;
}
-
- /*
- * No encoding has been detected.
- * Thus, we either fall into our default encoder, if specified,
- * or use Latin-1 if all else fails.
- */
-
- if (ENC__MAX == enc)
- enc = ENC__MAX == def ? ENC_LATIN_1 : def;
-
- if ( ! (*encs[(int)enc].conv)(&b)) {
- fprintf(stderr, "%s: Bad encoding\n", fn);
- goto out;
- }
-
- rc = EXIT_SUCCESS;
-out:
-#ifdef HAVE_MMAP
- if (map)
- munmap(b.buf, b.sz);
- else
-#endif
- free(b.buf);
-
- if (fd > STDIN_FILENO)
- close(fd);
-
- return(rc);
+ return MPARSE_UTF8 | MPARSE_LATIN1;
}
git.ckatri.com / mandoc.git / blobdiff