-.\" $Id: preconv.1,v 1.2 2011/05/26 12:14:46 kristaps Exp $
+.\" $Id: preconv.1,v 1.3 2011/05/26 14:43:07 kristaps Exp $
.\"
.\" Copyright (c) 2011 Kristaps Dzonsons <kristaps@bsd.lv>
.\"
.Bl -tag -width Ds
.It Fl D Ar enc
The default encoding.
-This is case-insensitive.
-See
-.Sx Algorithm
-and
-.Sx Encodings .
.It Fl e Ar enc
The document's encoding.
-This is case-insensitive.
-See
-.Sx Algorithm
-and
-.Sx Encodings .
.It Ar file
The input file.
.El
is not provided,
.Nm
accepts standard input.
-Output is written to standard output.
-Unicode characters in the ASCII range are printed as regular ASCII
-characters; those above this range are printed using the
+See
+.Sx Algorithm
+for encoding choice.
+.Pp
+The recoded input is written to standard output: Unicode characters in
+the ASCII range are printed as regular ASCII characters, while those
+above this range are printed using the
.Sq \e[uNNNN]
format documented in
.Xr mandoc_char 7 .
.Pp
If input bytes are improperly formed in the current encoding, they're
passed unmodified to standard output.
-.Ss Encodings
-The
+For some encodings, such as UTF-8, unrecoverable input sequences will
+cause
.Nm
-utility accepts the
-.Ar utf\-8 ,
-.Ar us\-ascii ,
-and
-.Ar latin\-1
-encodings as arguments to
-.Fl D Ar enc
-or
-.Fl e Ar enc .
+to stop processing and exit.
.Ss Algorithm
An encoding is chosen according to the following steps:
.Bl -enum
From the argument passed to
.Fl e Ar enc .
.It
-If a BOM exists, utf\-8 encoding is selected.
+If a BOM exists, UTF\-8 encoding is selected.
+.It
+From the coding tags parsed from
+.Qq File Variables
+on the first two lines of input.
+A file variable is an input line of the form
+.Pp
+.Dl \%.\e\(dq -*- key: val [; key: val ]* -*-
+.Pp
+where
+.Cm key
+is
+.Qq coding
+and
+.Cm val
+is the name of the encoding.
+A typical usage may be
+.Pp
+.Dl \%.\e\(dq -*- mode: troff; coding: utf-8 -*-
.It
From the argument passed to
.Fl D Ar enc .
.It
If all else fails, Latin\-1 is used.
.El
+.Pp
+The
+.Nm
+utility recognises the UTF\-8, us\-ascii, and latin\-1 encodings as
+passed to the
+.Fl e
+and
+.Fl D
+arguments, or as coding tags.
+Encodings are matched case-insensitively.
.\" .Sh IMPLEMENTATION NOTES
.\" Not used in OpenBSD.
.\" .Sh RETURN VALUES
.\" .Sh FILES
.Sh EXIT STATUS
.Ex -std
-.\" .Sh EXAMPLES
+.Sh EXAMPLES
+Explicitly page a UTF\-8 manual
+.Pa foo.1
+in the current locale:
+.Pp
+.Dl $ preconv \-e utf\-8 foo.1 | mandoc -Tlocale | less
.\" .Sh DIAGNOSTICS
.\" For sections 1, 4, 6, 7, & 8 only.
.\" .Sh ERRORS
-/* $Id: preconv.c,v 1.2 2011/05/26 12:01:14 kristaps Exp $ */
+/* $Id: preconv.c,v 1.3 2011/05/26 14:43:07 kristaps Exp $ */
/*
* Copyright (c) 2011 Kristaps Dzonsons <kristaps@bsd.lv>
*
int (*conv)(const struct buf *);
};
+static int cue_enc(const struct buf *, size_t *, enum enc *);
static int conv_latin_1(const struct buf *);
static int conv_us_ascii(const struct buf *);
static int conv_utf_8(const struct buf *);
* Latin-1 falls into the first 256 code-points of Unicode, so
* there's no need for any sort of translation. Just make the
* 8-bit characters use the Unicode escape.
+ * Note that binary values 128 < v < 160 are passed through
+ * unmodified to mandoc.
*/
for (i = b->offs; i < b->sz; i++) {
cu = (unsigned char)*cp++;
- cu < 128U ? putchar(cu) : printf("\\[u%.4X]", cu);
+ cu < 160U ? putchar(cu) : printf("\\[u%.4X]", cu);
}
return(1);
return(0);
}
+static int
+cue_enc(const struct buf *b, size_t *offs, enum enc *enc)
+{
+ const char *ln, *eoln, *eoph;
+ size_t sz, phsz, nsz;
+ int i;
+
+ ln = b->buf + (int)*offs;
+ sz = b->sz - *offs;
+
+ /* Look for the end-of-line. */
+
+ if (NULL == (eoln = memchr(ln, '\n', sz)))
+ return(-1);
+
+ /* Set next-line marker. */
+
+ *offs = (size_t)((eoln + 1) - b->buf);
+
+ /* Check if we have the correct header/trailer. */
+
+ if ((sz = (size_t)(eoln - ln)) < 10 ||
+ memcmp(ln, ".\\\" -*-", 7) ||
+ memcmp(eoln - 3, "-*-", 3))
+ return(0);
+
+ /* Move after the header and adjust for the trailer. */
+
+ ln += 7;
+ sz -= 10;
+
+ while (sz > 0) {
+ while (sz > 0 && ' ' == *ln) {
+ ln++;
+ sz--;
+ }
+ if (0 == sz)
+ break;
+
+ /* Find the end-of-phrase marker (or eoln). */
+
+ if (NULL == (eoph = memchr(ln, ';', sz)))
+ eoph = eoln - 3;
+ else
+ eoph++;
+
+ /* Only account for the "coding" phrase. */
+
+ if ((phsz = (size_t)(eoph - ln)) < 7 ||
+ strncasecmp(ln, "coding:", 7)) {
+ sz -= phsz;
+ ln += phsz;
+ continue;
+ }
+
+ sz -= 7;
+ ln += 7;
+
+ while (sz > 0 && ' ' == *ln) {
+ ln++;
+ sz--;
+ }
+ if (0 == sz)
+ break;
+
+ /* Check us against known encodings. */
+
+ for (i = 0; i < ENC__MAX; i++) {
+ nsz = strlen(encs[i].name);
+ if (phsz < nsz)
+ continue;
+ if (strncasecmp(ln, encs[i].name, nsz))
+ continue;
+
+ *enc = (enum enc)i;
+ return(1);
+ }
+
+ /* Unknown encoding. */
+
+ *enc = ENC__MAX;
+ return(1);
+ }
+
+ return(0);
+}
+
int
main(int argc, char *argv[])
{
const char *fn;
enum enc enc, def;
const char bom[3] = { 0xEF, 0xBB, 0xBF };
+ size_t offs;
extern int optind;
extern char *optarg;
enc = ENC_UTF_8;
}
+ /* Try reading from the "-*-" cue. */
+
+ if (ENC__MAX == enc) {
+ offs = b.offs;
+ ch = cue_enc(&b, &offs, &enc);
+ if (0 == ch)
+ ch = cue_enc(&b, &offs, &enc);
+ }
+
/*
* No encoding has been detected.
* Thus, we either fall into our default encoder, if specified,
if (ENC__MAX == enc)
enc = ENC__MAX == def ? ENC_LATIN_1 : def;
- if ( ! (*encs[(int)enc].conv)(&b))
+ if ( ! (*encs[(int)enc].conv)(&b)) {
+ fprintf(stderr, "%s: Bad encoding\n", fn);
goto out;
+ }
rc = EXIT_SUCCESS;
out:
git.ckatri.com / mandoc.git / commitdiff