Significantly improve preconv. Allow it to recode UTF-8 characters into

[mandoc.git] / preconv.c

diff --git a/preconv.c b/preconv.c
index ce091ec88afc126ed13cff45786f392b361f5c24..da5af9b4f5ead994acf16f12e0772c6749ccc55b 100644 (file)
--- a/preconv.c
+++ b/preconv.c
@@ -1,4 +1,4 @@
-/*     $Id: preconv.c,v 1.1 2011/05/26 00:30:11 kristaps Exp $ */
+/*     $Id: preconv.c,v 1.2 2011/05/26 12:01:14 kristaps Exp $ */

 /*
  * Copyright (c) 2011 Kristaps Dzonsons <kristaps@bsd.lv>
  *
 /*
  * Copyright (c) 2011 Kristaps Dzonsons <kristaps@bsd.lv>
  *


@@ -85,7 +85,7 @@ static int
 conv_latin_1(const struct buf *b)
 {
        size_t           i;
 conv_latin_1(const struct buf *b)
 {
        size_t           i;

-       unsigned char    c;
+       unsigned char    cu;

        const char      *cp;
 
        cp = b->buf + (int)b->offs;
        const char      *cp;
 
        cp = b->buf + (int)b->offs;


@@ -97,8 +97,8 @@ conv_latin_1(const struct buf *b)
         */
 
        for (i = b->offs; i < b->sz; i++) {
         */
 
        for (i = b->offs; i < b->sz; i++) {

-               c = (unsigned char)*cp++;
-               c < 128 ? putchar(c) : printf("\\[u%.4X]", c);
+               cu = (unsigned char)*cp++;
+               cu < 128U ? putchar(cu) : printf("\\[u%.4X]", cu);

        }
 
        return(1);
        }
 
        return(1);


@@ -120,6 +120,105 @@ conv_us_ascii(const struct buf *b)
 static int
 conv_utf_8(const struct buf *b)
 {
 static int
 conv_utf_8(const struct buf *b)
 {

+       int              state, be;
+       unsigned int     accum;
+       size_t           i;
+       unsigned char    cu;
+       const char      *cp;
+       const long       one = 1L;
+
+       cp = b->buf + (int)b->offs;
+       state = 0;
+       accum = 0U;
+       be = 0;
+
+       /* Quick test for big-endian value. */
+
+       if ( ! (*((char *)(&one))))
+               be = 1;
+
+       for (i = b->offs; i < b->sz; i++) {
+               cu = (unsigned char)*cp++;
+               if (state) {
+                       if ( ! (cu & 128) || (cu & 64)) {
+                               /* Bad sequence header. */
+                               return(0);
+                       }
+
+                       /* Accept only legitimate bit patterns. */
+
+                       if (cu > 191 || cu < 128) {
+                               /* Bad in-sequence bits. */
+                               return(0);
+                       }
+
+                       accum |= (cu & 63) << --state * 6;
+
+                       /*
+                        * Accum is held in little-endian order as
+                        * stipulated by the UTF-8 sequence coding.  We
+                        * need to convert to a native big-endian if our
+                        * architecture requires it.
+                        */
+
+                       if (0 == state && be) 
+                               accum = (accum >> 24) | 
+                                       ((accum << 8) & 0x00FF0000) |
+                                       ((accum >> 8) & 0x0000FF00) |
+                                       (accum << 24);
+
+                       if (0 == state) {
+                               accum < 128U ? putchar(accum) : 
+                                       printf("\\[u%.4X]", accum);
+                               accum = 0U;
+                       }
+               } else if (cu & (1 << 7)) {
+                       /*
+                        * Entering a UTF-8 state:  if we encounter a
+                        * UTF-8 bitmask, calculate the expected UTF-8
+                        * state from it.
+                        */
+                       for (state = 0; state < 7; state++) 
+                               if ( ! (cu & (1 << (7 - state))))
+                                       break;
+
+                       /* Accept only legitimate bit patterns. */
+
+                       switch (state) {
+                       case (4):
+                               if (cu <= 244 && cu >= 240) {
+                                       accum = (cu & 7) << 18;
+                                       break;
+                               }
+                               /* Bad 4-sequence start bits. */
+                               return(0);
+                       case (3):
+                               if (cu <= 239 && cu >= 224) {
+                                       accum = (cu & 15) << 12;
+                                       break;
+                               }
+                               /* Bad 3-sequence start bits. */
+                               return(0);
+                       case (2):
+                               if (cu <= 223 && cu >= 194) {
+                                       accum = (cu & 31) << 6;
+                                       break;
+                               }
+                               /* Bad 2-sequence start bits. */
+                               return(0);
+                       default:
+                               /* Bad sequence bit mask. */
+                               return(0);
+                       }
+                       state--;
+               } else
+                       putchar(cu);
+       }
+
+       if (0 != state) {
+               /* Bad trailing bits. */
+               return(0);
+       }

 
        return(1);
 }
 
        return(1);
 }


@@ -211,9 +310,10 @@ int
 main(int argc, char *argv[])
 {
        int              i, ch, map, fd, rc;
 main(int argc, char *argv[])
 {
        int              i, ch, map, fd, rc;

-       struct buf       buf;
+       struct buf       b;

        const char      *fn;
        enum enc         enc, def;
        const char      *fn;
        enum enc         enc, def;

+       const char       bom[3] = { 0xEF, 0xBB, 0xBF };

        extern int       optind;
        extern char     *optarg;
 
        extern int       optind;
        extern char     *optarg;
 


@@ -229,7 +329,7 @@ main(int argc, char *argv[])
        enc = def = ENC__MAX;
        map = 0;
 
        enc = def = ENC__MAX;
        map = 0;
 

-       memset(&buf, 0, sizeof(struct buf));
+       memset(&b, 0, sizeof(struct buf));

 
        while (-1 != (ch = getopt(argc, argv, "D:e:rdvh")))
                switch (ch) {
 
        while (-1 != (ch = getopt(argc, argv, "D:e:rdvh")))
                switch (ch) {


@@ -283,12 +383,16 @@ main(int argc, char *argv[])
                }
        }
 
                }
        }
 

-       if ( ! read_whole_file(fn, fd, &buf, &map))
+       if ( ! read_whole_file(fn, fd, &b, &map))

                goto out;
 
                goto out;
 

-       if (ENC__MAX == enc) {
-               /* TODO: search for BOM. */
-       }
+       /* Try to read the UTF-8 BOM. */
+
+       if (ENC__MAX == enc)
+               if (b.sz > 3 && 0 == memcmp(b.buf, bom, 3)) {
+                       b.offs = 3;
+                       enc = ENC_UTF_8;
+               }

 
        /*
         * No encoding has been detected.
 
        /*
         * No encoding has been detected.


@@ -299,15 +403,15 @@ main(int argc, char *argv[])
        if (ENC__MAX == enc) 
                enc = ENC__MAX == def ? ENC_LATIN_1 : def;
 
        if (ENC__MAX == enc) 
                enc = ENC__MAX == def ? ENC_LATIN_1 : def;
 

-       if ( ! (*encs[(int)enc].conv)(&buf))
+       if ( ! (*encs[(int)enc].conv)(&b)) 

                goto out;
 
        rc = EXIT_SUCCESS;
 out:
        if (map)
                goto out;
 
        rc = EXIT_SUCCESS;
 out:
        if (map)

-               munmap(buf.buf, buf.sz);
+               munmap(b.buf, b.sz);

        else 
        else 

-               free(buf.buf);
+               free(b.buf);

 
        if (fd > STDIN_FILENO)
                close(fd);
 
        if (fd > STDIN_FILENO)
                close(fd);