-/* $Id: preconv.c,v 1.1 2011/05/26 00:30:11 kristaps Exp $ */
+/* $Id: preconv.c,v 1.4 2011/05/26 21:13:07 kristaps Exp $ */
/*
* Copyright (c) 2011 Kristaps Dzonsons <kristaps@bsd.lv>
*
int (*conv)(const struct buf *);
};
+static int cue_enc(const struct buf *, size_t *, enum enc *);
static int conv_latin_1(const struct buf *);
static int conv_us_ascii(const struct buf *);
static int conv_utf_8(const struct buf *);
conv_latin_1(const struct buf *b)
{
size_t i;
- unsigned char c;
+ unsigned char cu;
const char *cp;
cp = b->buf + (int)b->offs;
* Latin-1 falls into the first 256 code-points of Unicode, so
* there's no need for any sort of translation. Just make the
* 8-bit characters use the Unicode escape.
+ * Note that binary values 128 < v < 160 are passed through
+ * unmodified to mandoc.
*/
for (i = b->offs; i < b->sz; i++) {
- c = (unsigned char)*cp++;
- c < 128 ? putchar(c) : printf("\\[u%.4X]", c);
+ cu = (unsigned char)*cp++;
+ cu < 160U ? putchar(cu) : printf("\\[u%.4X]", cu);
}
return(1);
static int
conv_utf_8(const struct buf *b)
{
+ int state, be;
+ unsigned int accum;
+ size_t i;
+ unsigned char cu;
+ const char *cp;
+ const long one = 1L;
+
+ cp = b->buf + (int)b->offs;
+ state = 0;
+ accum = 0U;
+ be = 0;
+
+ /* Quick test for big-endian value. */
+
+ if ( ! (*((const char *)(&one))))
+ be = 1;
+
+ for (i = b->offs; i < b->sz; i++) {
+ cu = (unsigned char)*cp++;
+ if (state) {
+ if ( ! (cu & 128) || (cu & 64)) {
+ /* Bad sequence header. */
+ return(0);
+ }
+
+ /* Accept only legitimate bit patterns. */
+
+ if (cu > 191 || cu < 128) {
+ /* Bad in-sequence bits. */
+ return(0);
+ }
+
+ accum |= (cu & 63) << --state * 6;
+
+ /*
+ * Accum is held in little-endian order as
+ * stipulated by the UTF-8 sequence coding. We
+ * need to convert to a native big-endian if our
+ * architecture requires it.
+ */
+
+ if (0 == state && be)
+ accum = (accum >> 24) |
+ ((accum << 8) & 0x00FF0000) |
+ ((accum >> 8) & 0x0000FF00) |
+ (accum << 24);
+
+ if (0 == state) {
+ accum < 128U ? putchar(accum) :
+ printf("\\[u%.4X]", accum);
+ accum = 0U;
+ }
+ } else if (cu & (1 << 7)) {
+ /*
+ * Entering a UTF-8 state: if we encounter a
+ * UTF-8 bitmask, calculate the expected UTF-8
+ * state from it.
+ */
+ for (state = 0; state < 7; state++)
+ if ( ! (cu & (1 << (7 - state))))
+ break;
+
+ /* Accept only legitimate bit patterns. */
+
+ switch (state) {
+ case (4):
+ if (cu <= 244 && cu >= 240) {
+ accum = (cu & 7) << 18;
+ break;
+ }
+ /* Bad 4-sequence start bits. */
+ return(0);
+ case (3):
+ if (cu <= 239 && cu >= 224) {
+ accum = (cu & 15) << 12;
+ break;
+ }
+ /* Bad 3-sequence start bits. */
+ return(0);
+ case (2):
+ if (cu <= 223 && cu >= 194) {
+ accum = (cu & 31) << 6;
+ break;
+ }
+ /* Bad 2-sequence start bits. */
+ return(0);
+ default:
+ /* Bad sequence bit mask. */
+ return(0);
+ }
+ state--;
+ } else
+ putchar(cu);
+ }
+
+ if (0 != state) {
+ /* Bad trailing bits. */
+ return(0);
+ }
return(1);
}
return(0);
}
+static int
+cue_enc(const struct buf *b, size_t *offs, enum enc *enc)
+{
+ const char *ln, *eoln, *eoph;
+ size_t sz, phsz, nsz;
+ int i;
+
+ ln = b->buf + (int)*offs;
+ sz = b->sz - *offs;
+
+ /* Look for the end-of-line. */
+
+ if (NULL == (eoln = memchr(ln, '\n', sz)))
+ return(-1);
+
+ /* Set next-line marker. */
+
+ *offs = (size_t)((eoln + 1) - b->buf);
+
+ /* Check if we have the correct header/trailer. */
+
+ if ((sz = (size_t)(eoln - ln)) < 10 ||
+ memcmp(ln, ".\\\" -*-", 7) ||
+ memcmp(eoln - 3, "-*-", 3))
+ return(0);
+
+ /* Move after the header and adjust for the trailer. */
+
+ ln += 7;
+ sz -= 10;
+
+ while (sz > 0) {
+ while (sz > 0 && ' ' == *ln) {
+ ln++;
+ sz--;
+ }
+ if (0 == sz)
+ break;
+
+ /* Find the end-of-phrase marker (or eoln). */
+
+ if (NULL == (eoph = memchr(ln, ';', sz)))
+ eoph = eoln - 3;
+ else
+ eoph++;
+
+ /* Only account for the "coding" phrase. */
+
+ if ((phsz = (size_t)(eoph - ln)) < 7 ||
+ strncasecmp(ln, "coding:", 7)) {
+ sz -= phsz;
+ ln += phsz;
+ continue;
+ }
+
+ sz -= 7;
+ ln += 7;
+
+ while (sz > 0 && ' ' == *ln) {
+ ln++;
+ sz--;
+ }
+ if (0 == sz)
+ break;
+
+ /* Check us against known encodings. */
+
+ for (i = 0; i < (int)ENC__MAX; i++) {
+ nsz = strlen(encs[i].name);
+ if (phsz < nsz)
+ continue;
+ if (strncasecmp(ln, encs[i].name, nsz))
+ continue;
+
+ *enc = (enum enc)i;
+ return(1);
+ }
+
+ /* Unknown encoding. */
+
+ *enc = ENC__MAX;
+ return(1);
+ }
+
+ return(0);
+}
+
int
main(int argc, char *argv[])
{
int i, ch, map, fd, rc;
- struct buf buf;
+ struct buf b;
const char *fn;
enum enc enc, def;
+ unsigned char bom[3] = { 0xEF, 0xBB, 0xBF };
+ size_t offs;
extern int optind;
extern char *optarg;
enc = def = ENC__MAX;
map = 0;
- memset(&buf, 0, sizeof(struct buf));
+ memset(&b, 0, sizeof(struct buf));
while (-1 != (ch = getopt(argc, argv, "D:e:rdvh")))
switch (ch) {
case ('D'):
/* FALLTHROUGH */
case ('e'):
- for (i = 0; i < ENC__MAX; i++) {
+ for (i = 0; i < (int)ENC__MAX; i++) {
if (strcasecmp(optarg, encs[i].name))
continue;
break;
}
- if (i < ENC__MAX) {
+ if (i < (int)ENC__MAX) {
if ('D' == ch)
def = (enum enc)i;
else
}
}
- if ( ! read_whole_file(fn, fd, &buf, &map))
+ if ( ! read_whole_file(fn, fd, &b, &map))
goto out;
+ /* Try to read the UTF-8 BOM. */
+
+ if (ENC__MAX == enc)
+ if (b.sz > 3 && 0 == memcmp(b.buf, bom, 3)) {
+ b.offs = 3;
+ enc = ENC_UTF_8;
+ }
+
+ /* Try reading from the "-*-" cue. */
+
if (ENC__MAX == enc) {
- /* TODO: search for BOM. */
+ offs = b.offs;
+ ch = cue_enc(&b, &offs, &enc);
+ if (0 == ch)
+ ch = cue_enc(&b, &offs, &enc);
}
/*
if (ENC__MAX == enc)
enc = ENC__MAX == def ? ENC_LATIN_1 : def;
- if ( ! (*encs[(int)enc].conv)(&buf))
+ if ( ! (*encs[(int)enc].conv)(&b)) {
+ fprintf(stderr, "%s: Bad encoding\n", fn);
goto out;
+ }
rc = EXIT_SUCCESS;
out:
if (map)
- munmap(buf.buf, buf.sz);
+ munmap(b.buf, b.sz);
else
- free(buf.buf);
+ free(b.buf);
if (fd > STDIN_FILENO)
close(fd);
git.ckatri.com / mandoc.git / blobdiff