Significantly improve preconv. Allow it to recode UTF-8 characters into

the \[uNNNN] strings (taking into account big-endian archs).  Also allow
it to determine from the BOM whether it's a UTF-8 file.  Also add the
initial manual.  This has been tested over a random selection of UTF-8
documents, as

 % preconv -e utf-8 foo.1 | ./mandoc -Tlocale

where -Tlocale is allowed (-DUSE_WCHAR).

Note that we're still missing the "type" indicator that preconv accepts.

diff --git a/Makefile b/Makefile
index aa3b29b740525c2cda90afcf123452869464b2dd..5f139b8effcc769bbb57ef567d371842ee68e4d4 100644 (file)
--- a/Makefile
+++ b/Makefile
@@ -88,6 +88,8 @@ SRCS           = Makefile \
                   msec.in \
                   out.c \
                   out.h \
                   msec.in \
                   out.c \
                   out.h \

+                  preconv.1 \
+                  preconv.c \

                   predefs.in \
                   read.c \
                   roff.7 \
                   predefs.in \
                   read.c \
                   roff.7 \


@@ -268,6 +270,11 @@ INDEX_MANS  = makewhatis.1.html \
                   mdoc.7.ps \
                   mdoc.7.pdf \
                   mdoc.7.txt \
                   mdoc.7.ps \
                   mdoc.7.pdf \
                   mdoc.7.txt \

+                  preconv.1.html \
+                  preconv.1.xhtml \
+                  preconv.1.ps \
+                  preconv.1.pdf \
+                  preconv.1.txt \

                   roff.7.html \
                   roff.7.xhtml \
                   roff.7.ps \
                   roff.7.html \
                   roff.7.xhtml \
                   roff.7.ps \

diff --git a/preconv.1 b/preconv.1
new file mode 100644 (file)
index 0000000..7297006
--- /dev/null
+++ b/preconv.1
@@ -0,0 +1,137 @@
+.\"    $Id: preconv.1,v 1.1 2011/05/26 12:01:14 kristaps Exp $
+.\"
+.\" Copyright (c) 2011 Kristaps Dzonsons <kristaps@bsd.lv>
+.\"
+.\" Permission to use, copy, modify, and distribute this software for any
+.\" purpose with or without fee is hereby granted, provided that the above
+.\" copyright notice and this permission notice appear in all copies.
+.\"
+.\" THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+.\" WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+.\" MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+.\" ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+.\" WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+.\" ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+.\" OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+.\"
+.Dd $Mdocdate: May 26 2011 $
+.Dt PRECONV 1
+.Os
+.Sh NAME
+.Nm preconv
+.Nd recodes multibyte UNIX manuals as mandoc input
+.Sh SYNOPSIS
+.Nm preconv
+.Op Fl D Ar enc
+.Op Fl e Ar enc
+.Op Ar file
+.Sh DESCRIPTION
+The
+.Nm
+utility recodes multibyte
+.Ux
+manual files into
+.Xr mandoc 1
+input.
+Its arguments are as follows:
+.Bl -tag -width Ds
+.It Fl D Ar enc
+The default encoding.
+This is case-insensitive.
+See
+.Sx Algorithm
+and
+.Sx Encodings .
+.It Fl e Ar enc
+The document's encoding.
+This is case-insensitive.
+See
+.Sx Algorithm
+and
+.Sx Encodings .
+.It Ar file
+The input file.
+.El
+.Pp
+If
+.Ar file
+is not provided,
+.Nm
+accepts standard input.
+Output is written to standard output.
+Unicode characters in the ASCII range are printed as regular ASCII
+characters; those above this range are printed using the
+.Sq \e[uNNNN]
+format documented in
+.Xr mandoc_char 7 .
+.Pp
+If input bytes are improperly formed in the current encoding, they're
+passed unmodified to standard output.
+.Ss Encodings
+The
+.Nm
+utility accepts the
+.Ar utf\-8 ,
+.Ar us\-ascii ,
+and
+.Ar latin\-1
+encodings as arguments to
+.Fl D Ar enc
+or
+.Fl e Ar enc .
+.Ss Algorithm
+An encoding is chosen according to the following steps:
+.Bl -enum
+.It
+From the argument passed to
+.Fl e Ar enc .
+.It
+If a BOM exists, utf\-8 encoding is selected.
+.It
+From the argument passed to
+.Fl D Ar enc .
+.It
+If all else fails, Latin\-1 is used.
+.El
+.\" .Sh IMPLEMENTATION NOTES
+.\" Not used in OpenBSD.
+.\" .Sh RETURN VALUES
+.\" For sections 2, 3, & 9 only.
+.\" .Sh ENVIRONMENT
+.\" For sections 1, 6, 7, & 8 only.
+.\" .Sh FILES
+.Sh EXIT STATUS
+.Ex -std
+.\" .Sh EXAMPLES
+.\" .Sh DIAGNOSTICS
+.\" For sections 1, 4, 6, 7, & 8 only.
+.\" .Sh ERRORS
+.\" For sections 2, 3, & 9 only.
+.Sh SEE ALSO
+.Xr mandoc 1 ,
+.Xr mandoc_char 7
+.Sh STANDARDS
+The
+.Nm
+utility references the US-ASCII character set standard, ANSI_X3.4\-1968;
+the Latin\-1 character set standard, ISO/IEC 8859\-1:1998; the UTF\-8
+character set standard; and UCS (Unicode), ISO/IEC 10646.
+.Sh HISTORY
+The
+.Nm
+utility first appeared in the GNU troff
+.Pq Dq groff
+system in December 2005, authored by Tomohiro Kubota and Werner
+Lemberg.
+The implementation that is part of the
+.Xr mandoc 1
+utility appeared in May 2011.
+.Sh AUTHORS
+The
+.Nm
+utility was written by
+.An Kristaps Dzonsons Aq kristaps@bsd.lv .
+.\" .Sh CAVEATS
+.\" .Sh BUGS
+.\" .Sh SECURITY CONSIDERATIONS
+.\" Not used in OpenBSD.

diff --git a/preconv.c b/preconv.c
index ce091ec88afc126ed13cff45786f392b361f5c24..da5af9b4f5ead994acf16f12e0772c6749ccc55b 100644 (file)
--- a/preconv.c
+++ b/preconv.c
@@ -1,4 +1,4 @@
-/*     $Id: preconv.c,v 1.1 2011/05/26 00:30:11 kristaps Exp $ */
+/*     $Id: preconv.c,v 1.2 2011/05/26 12:01:14 kristaps Exp $ */

 /*
  * Copyright (c) 2011 Kristaps Dzonsons <kristaps@bsd.lv>
  *
 /*
  * Copyright (c) 2011 Kristaps Dzonsons <kristaps@bsd.lv>
  *


@@ -85,7 +85,7 @@ static int
 conv_latin_1(const struct buf *b)
 {
        size_t           i;
 conv_latin_1(const struct buf *b)
 {
        size_t           i;

-       unsigned char    c;
+       unsigned char    cu;

        const char      *cp;
 
        cp = b->buf + (int)b->offs;
        const char      *cp;
 
        cp = b->buf + (int)b->offs;


@@ -97,8 +97,8 @@ conv_latin_1(const struct buf *b)
         */
 
        for (i = b->offs; i < b->sz; i++) {
         */
 
        for (i = b->offs; i < b->sz; i++) {

-               c = (unsigned char)*cp++;
-               c < 128 ? putchar(c) : printf("\\[u%.4X]", c);
+               cu = (unsigned char)*cp++;
+               cu < 128U ? putchar(cu) : printf("\\[u%.4X]", cu);

        }
 
        return(1);
        }
 
        return(1);


@@ -120,6 +120,105 @@ conv_us_ascii(const struct buf *b)
 static int
 conv_utf_8(const struct buf *b)
 {
 static int
 conv_utf_8(const struct buf *b)
 {

+       int              state, be;
+       unsigned int     accum;
+       size_t           i;
+       unsigned char    cu;
+       const char      *cp;
+       const long       one = 1L;
+
+       cp = b->buf + (int)b->offs;
+       state = 0;
+       accum = 0U;
+       be = 0;
+
+       /* Quick test for big-endian value. */
+
+       if ( ! (*((char *)(&one))))
+               be = 1;
+
+       for (i = b->offs; i < b->sz; i++) {
+               cu = (unsigned char)*cp++;
+               if (state) {
+                       if ( ! (cu & 128) || (cu & 64)) {
+                               /* Bad sequence header. */
+                               return(0);
+                       }
+
+                       /* Accept only legitimate bit patterns. */
+
+                       if (cu > 191 || cu < 128) {
+                               /* Bad in-sequence bits. */
+                               return(0);
+                       }
+
+                       accum |= (cu & 63) << --state * 6;
+
+                       /*
+                        * Accum is held in little-endian order as
+                        * stipulated by the UTF-8 sequence coding.  We
+                        * need to convert to a native big-endian if our
+                        * architecture requires it.
+                        */
+
+                       if (0 == state && be) 
+                               accum = (accum >> 24) | 
+                                       ((accum << 8) & 0x00FF0000) |
+                                       ((accum >> 8) & 0x0000FF00) |
+                                       (accum << 24);
+
+                       if (0 == state) {
+                               accum < 128U ? putchar(accum) : 
+                                       printf("\\[u%.4X]", accum);
+                               accum = 0U;
+                       }
+               } else if (cu & (1 << 7)) {
+                       /*
+                        * Entering a UTF-8 state:  if we encounter a
+                        * UTF-8 bitmask, calculate the expected UTF-8
+                        * state from it.
+                        */
+                       for (state = 0; state < 7; state++) 
+                               if ( ! (cu & (1 << (7 - state))))
+                                       break;
+
+                       /* Accept only legitimate bit patterns. */
+
+                       switch (state) {
+                       case (4):
+                               if (cu <= 244 && cu >= 240) {
+                                       accum = (cu & 7) << 18;
+                                       break;
+                               }
+                               /* Bad 4-sequence start bits. */
+                               return(0);
+                       case (3):
+                               if (cu <= 239 && cu >= 224) {
+                                       accum = (cu & 15) << 12;
+                                       break;
+                               }
+                               /* Bad 3-sequence start bits. */
+                               return(0);
+                       case (2):
+                               if (cu <= 223 && cu >= 194) {
+                                       accum = (cu & 31) << 6;
+                                       break;
+                               }
+                               /* Bad 2-sequence start bits. */
+                               return(0);
+                       default:
+                               /* Bad sequence bit mask. */
+                               return(0);
+                       }
+                       state--;
+               } else
+                       putchar(cu);
+       }
+
+       if (0 != state) {
+               /* Bad trailing bits. */
+               return(0);
+       }

 
        return(1);
 }
 
        return(1);
 }


@@ -211,9 +310,10 @@ int
 main(int argc, char *argv[])
 {
        int              i, ch, map, fd, rc;
 main(int argc, char *argv[])
 {
        int              i, ch, map, fd, rc;

-       struct buf       buf;
+       struct buf       b;

        const char      *fn;
        enum enc         enc, def;
        const char      *fn;
        enum enc         enc, def;

+       const char       bom[3] = { 0xEF, 0xBB, 0xBF };

        extern int       optind;
        extern char     *optarg;
 
        extern int       optind;
        extern char     *optarg;
 


@@ -229,7 +329,7 @@ main(int argc, char *argv[])
        enc = def = ENC__MAX;
        map = 0;
 
        enc = def = ENC__MAX;
        map = 0;
 

-       memset(&buf, 0, sizeof(struct buf));
+       memset(&b, 0, sizeof(struct buf));

 
        while (-1 != (ch = getopt(argc, argv, "D:e:rdvh")))
                switch (ch) {
 
        while (-1 != (ch = getopt(argc, argv, "D:e:rdvh")))
                switch (ch) {


@@ -283,12 +383,16 @@ main(int argc, char *argv[])
                }
        }
 
                }
        }
 

-       if ( ! read_whole_file(fn, fd, &buf, &map))
+       if ( ! read_whole_file(fn, fd, &b, &map))

                goto out;
 
                goto out;
 

-       if (ENC__MAX == enc) {
-               /* TODO: search for BOM. */
-       }
+       /* Try to read the UTF-8 BOM. */
+
+       if (ENC__MAX == enc)
+               if (b.sz > 3 && 0 == memcmp(b.buf, bom, 3)) {
+                       b.offs = 3;
+                       enc = ENC_UTF_8;
+               }

 
        /*
         * No encoding has been detected.
 
        /*
         * No encoding has been detected.


@@ -299,15 +403,15 @@ main(int argc, char *argv[])
        if (ENC__MAX == enc) 
                enc = ENC__MAX == def ? ENC_LATIN_1 : def;
 
        if (ENC__MAX == enc) 
                enc = ENC__MAX == def ? ENC_LATIN_1 : def;
 

-       if ( ! (*encs[(int)enc].conv)(&buf))
+       if ( ! (*encs[(int)enc].conv)(&b)) 

                goto out;
 
        rc = EXIT_SUCCESS;
 out:
        if (map)
                goto out;
 
        rc = EXIT_SUCCESS;
 out:
        if (map)

-               munmap(buf.buf, buf.sz);
+               munmap(b.buf, b.sz);

        else 
        else 

-               free(buf.buf);
+               free(b.buf);

 
        if (fd > STDIN_FILENO)
                close(fd);
 
        if (fd > STDIN_FILENO)
                close(fd);