diff options
| -rw-r--r-- | Makefile | 7 | ||||
| -rw-r--r-- | preconv.1 | 137 | ||||
| -rw-r--r-- | preconv.c | 130 |
3 files changed, 261 insertions, 13 deletions
@@ -88,6 +88,8 @@ SRCS = Makefile \ msec.in \ out.c \ out.h \ + preconv.1 \ + preconv.c \ predefs.in \ read.c \ roff.7 \ @@ -268,6 +270,11 @@ INDEX_MANS = makewhatis.1.html \ mdoc.7.ps \ mdoc.7.pdf \ mdoc.7.txt \ + preconv.1.html \ + preconv.1.xhtml \ + preconv.1.ps \ + preconv.1.pdf \ + preconv.1.txt \ roff.7.html \ roff.7.xhtml \ roff.7.ps \ diff --git a/preconv.1 b/preconv.1 new file mode 100644 index 00000000..72970061 --- /dev/null +++ b/preconv.1 @@ -0,0 +1,137 @@ +.\" $Id: preconv.1,v 1.1 2011/05/26 12:01:14 kristaps Exp $ +.\" +.\" Copyright (c) 2011 Kristaps Dzonsons <kristaps@bsd.lv> +.\" +.\" Permission to use, copy, modify, and distribute this software for any +.\" purpose with or without fee is hereby granted, provided that the above +.\" copyright notice and this permission notice appear in all copies. +.\" +.\" THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +.\" WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +.\" MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +.\" ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +.\" WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +.\" ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +.\" OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +.\" +.Dd $Mdocdate: May 26 2011 $ +.Dt PRECONV 1 +.Os +.Sh NAME +.Nm preconv +.Nd recodes multibyte UNIX manuals as mandoc input +.Sh SYNOPSIS +.Nm preconv +.Op Fl D Ar enc +.Op Fl e Ar enc +.Op Ar file +.Sh DESCRIPTION +The +.Nm +utility recodes multibyte +.Ux +manual files into +.Xr mandoc 1 +input. +Its arguments are as follows: +.Bl -tag -width Ds +.It Fl D Ar enc +The default encoding. +This is case-insensitive. +See +.Sx Algorithm +and +.Sx Encodings . +.It Fl e Ar enc +The document's encoding. +This is case-insensitive. +See +.Sx Algorithm +and +.Sx Encodings . +.It Ar file +The input file. +.El +.Pp +If +.Ar file +is not provided, +.Nm +accepts standard input. +Output is written to standard output. +Unicode characters in the ASCII range are printed as regular ASCII +characters; those above this range are printed using the +.Sq \e[uNNNN] +format documented in +.Xr mandoc_char 7 . +.Pp +If input bytes are improperly formed in the current encoding, they're +passed unmodified to standard output. +.Ss Encodings +The +.Nm +utility accepts the +.Ar utf\-8 , +.Ar us\-ascii , +and +.Ar latin\-1 +encodings as arguments to +.Fl D Ar enc +or +.Fl e Ar enc . +.Ss Algorithm +An encoding is chosen according to the following steps: +.Bl -enum +.It +From the argument passed to +.Fl e Ar enc . +.It +If a BOM exists, utf\-8 encoding is selected. +.It +From the argument passed to +.Fl D Ar enc . +.It +If all else fails, Latin\-1 is used. +.El +.\" .Sh IMPLEMENTATION NOTES +.\" Not used in OpenBSD. +.\" .Sh RETURN VALUES +.\" For sections 2, 3, & 9 only. +.\" .Sh ENVIRONMENT +.\" For sections 1, 6, 7, & 8 only. +.\" .Sh FILES +.Sh EXIT STATUS +.Ex -std +.\" .Sh EXAMPLES +.\" .Sh DIAGNOSTICS +.\" For sections 1, 4, 6, 7, & 8 only. +.\" .Sh ERRORS +.\" For sections 2, 3, & 9 only. +.Sh SEE ALSO +.Xr mandoc 1 , +.Xr mandoc_char 7 +.Sh STANDARDS +The +.Nm +utility references the US-ASCII character set standard, ANSI_X3.4\-1968; +the Latin\-1 character set standard, ISO/IEC 8859\-1:1998; the UTF\-8 +character set standard; and UCS (Unicode), ISO/IEC 10646. +.Sh HISTORY +The +.Nm +utility first appeared in the GNU troff +.Pq Dq groff +system in December 2005, authored by Tomohiro Kubota and Werner +Lemberg. +The implementation that is part of the +.Xr mandoc 1 +utility appeared in May 2011. +.Sh AUTHORS +The +.Nm +utility was written by +.An Kristaps Dzonsons Aq kristaps@bsd.lv . +.\" .Sh CAVEATS +.\" .Sh BUGS +.\" .Sh SECURITY CONSIDERATIONS +.\" Not used in OpenBSD. @@ -1,4 +1,4 @@ -/* $Id: preconv.c,v 1.1 2011/05/26 00:30:11 kristaps Exp $ */ +/* $Id: preconv.c,v 1.2 2011/05/26 12:01:14 kristaps Exp $ */ /* * Copyright (c) 2011 Kristaps Dzonsons <kristaps@bsd.lv> * @@ -85,7 +85,7 @@ static int conv_latin_1(const struct buf *b) { size_t i; - unsigned char c; + unsigned char cu; const char *cp; cp = b->buf + (int)b->offs; @@ -97,8 +97,8 @@ conv_latin_1(const struct buf *b) */ for (i = b->offs; i < b->sz; i++) { - c = (unsigned char)*cp++; - c < 128 ? putchar(c) : printf("\\[u%.4X]", c); + cu = (unsigned char)*cp++; + cu < 128U ? putchar(cu) : printf("\\[u%.4X]", cu); } return(1); @@ -120,6 +120,105 @@ conv_us_ascii(const struct buf *b) static int conv_utf_8(const struct buf *b) { + int state, be; + unsigned int accum; + size_t i; + unsigned char cu; + const char *cp; + const long one = 1L; + + cp = b->buf + (int)b->offs; + state = 0; + accum = 0U; + be = 0; + + /* Quick test for big-endian value. */ + + if ( ! (*((char *)(&one)))) + be = 1; + + for (i = b->offs; i < b->sz; i++) { + cu = (unsigned char)*cp++; + if (state) { + if ( ! (cu & 128) || (cu & 64)) { + /* Bad sequence header. */ + return(0); + } + + /* Accept only legitimate bit patterns. */ + + if (cu > 191 || cu < 128) { + /* Bad in-sequence bits. */ + return(0); + } + + accum |= (cu & 63) << --state * 6; + + /* + * Accum is held in little-endian order as + * stipulated by the UTF-8 sequence coding. We + * need to convert to a native big-endian if our + * architecture requires it. + */ + + if (0 == state && be) + accum = (accum >> 24) | + ((accum << 8) & 0x00FF0000) | + ((accum >> 8) & 0x0000FF00) | + (accum << 24); + + if (0 == state) { + accum < 128U ? putchar(accum) : + printf("\\[u%.4X]", accum); + accum = 0U; + } + } else if (cu & (1 << 7)) { + /* + * Entering a UTF-8 state: if we encounter a + * UTF-8 bitmask, calculate the expected UTF-8 + * state from it. + */ + for (state = 0; state < 7; state++) + if ( ! (cu & (1 << (7 - state)))) + break; + + /* Accept only legitimate bit patterns. */ + + switch (state) { + case (4): + if (cu <= 244 && cu >= 240) { + accum = (cu & 7) << 18; + break; + } + /* Bad 4-sequence start bits. */ + return(0); + case (3): + if (cu <= 239 && cu >= 224) { + accum = (cu & 15) << 12; + break; + } + /* Bad 3-sequence start bits. */ + return(0); + case (2): + if (cu <= 223 && cu >= 194) { + accum = (cu & 31) << 6; + break; + } + /* Bad 2-sequence start bits. */ + return(0); + default: + /* Bad sequence bit mask. */ + return(0); + } + state--; + } else + putchar(cu); + } + + if (0 != state) { + /* Bad trailing bits. */ + return(0); + } return(1); } @@ -211,9 +310,10 @@ int main(int argc, char *argv[]) { int i, ch, map, fd, rc; - struct buf buf; + struct buf b; const char *fn; enum enc enc, def; + const char bom[3] = { 0xEF, 0xBB, 0xBF }; extern int optind; extern char *optarg; @@ -229,7 +329,7 @@ main(int argc, char *argv[]) enc = def = ENC__MAX; map = 0; - memset(&buf, 0, sizeof(struct buf)); + memset(&b, 0, sizeof(struct buf)); while (-1 != (ch = getopt(argc, argv, "D:e:rdvh"))) switch (ch) { @@ -283,12 +383,16 @@ main(int argc, char *argv[]) } } - if ( ! read_whole_file(fn, fd, &buf, &map)) + if ( ! read_whole_file(fn, fd, &b, &map)) goto out; - if (ENC__MAX == enc) { - /* TODO: search for BOM. */ - } + /* Try to read the UTF-8 BOM. */ + + if (ENC__MAX == enc) + if (b.sz > 3 && 0 == memcmp(b.buf, bom, 3)) { + b.offs = 3; + enc = ENC_UTF_8; + } /* * No encoding has been detected. @@ -299,15 +403,15 @@ main(int argc, char *argv[]) if (ENC__MAX == enc) enc = ENC__MAX == def ? ENC_LATIN_1 : def; - if ( ! (*encs[(int)enc].conv)(&buf)) + if ( ! (*encs[(int)enc].conv)(&b)) goto out; rc = EXIT_SUCCESS; out: if (map) - munmap(buf.buf, buf.sz); + munmap(b.buf, b.sz); else - free(buf.buf); + free(b.buf); if (fd > STDIN_FILENO) close(fd); |