5
\$\begingroup\$

The following program takes an arbitrary byte sequence as input and outputs well-formed UTF-8. All UTF-8 sequences from the input are copied unmodified. All other bytes are assumed to be encoded in ISO 8859-1 or some variant, and their byte value is output as the corresponding code point in UTF-8.

#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>

static int buf[4];
static size_t buflen;

/* Reads from stdin until buf[index] can be accessed. */
static bool read_buf(size_t index) {
    while (!(index < buflen)) {
        buf[buflen] = getchar();
        if (buf[buflen] == EOF) {
            return false;
        }
        buflen++;
    }
    return buf[buflen - 1] != EOF;
}

static void write_buf(size_t n) {
    for (size_t i = 0; i < n; i++) {
        putchar(buf[i]);
    }
    buflen = 0;
}

static bool start_between(int min, int max) {
    return min <= buf[0] && buf[0] < max;
}

static bool is_cont(int index) {
    return 0x80 <= buf[index] && buf[index] < 0xC0;
}

static int code_point(int bits18, int bits12, int bits06, int bits00) {
    return   ((bits18 & 0x3F) << 18)
           | ((bits12 & 0x3F) << 12)
           | ((bits06 & 0x3F) <<  6)
           |  (bits00 & 0x3F);
}

int main(void) {
    while (read_buf(0) && !ferror(stdout)) {
        if (start_between(0x00, 0x80)) {
            write_buf(1);
        } else if (start_between(0xC2, 0xE0) && read_buf(1) && is_cont(1)
                   && code_point(0, 0, buf[0] & 0x1F, buf[1]) >= 0x0080) {
            write_buf(2);
        } else if (start_between(0xE0, 0xF0) && read_buf(2) && is_cont(1) && is_cont(2)
                   && code_point(0, buf[0] & 0x0F, buf[1], buf[2]) >= 0x0800) {
            write_buf(3);
        } else if (start_between(0xF0, 0xF8) && read_buf(3) && is_cont(1) && is_cont(2) && is_cont(3)
                   && code_point(buf[0] & 0x07, buf[1], buf[2], buf[3]) >= 0x010000
                   && code_point(buf[0] & 0x07, buf[1], buf[2], buf[3]) < 0x110000) {
            write_buf(4);
        } else {
            putchar(0xC0 + ((buf[0] >> 6) & 0x1F));
            putchar(0x80 + ((buf[0] >> 0) & 0x3F));
            buflen--;
            buf[0] = buf[1];
            buf[1] = buf[2];
            buf[2] = buf[3];
            buf[3] = 0;
        }
    }

    return ferror(stdin) || ferror(stdout) ? EXIT_FAILURE : EXIT_SUCCESS;
}

To test the code, I manually ran variants of the following command:

perl -e 'print pack("H*", "fcf09f9985e08080fc")' | ./latin1_to_utf8 | hexdump -C
\$\endgroup\$
2
\$\begingroup\$

A useful idea, and very well implemented.

I'd re-write this condition:

while (!(index < buflen))

as

while (buflen <= index)

Apart from that, I found the code very readable.

It's a good idea to store the received data as int rather than converting to char immediately (though I do worry somewhat about using a signed type in code_point - perhaps its arguments should be narrowed to unsigned char?).

Another minor nit: it's not necessary to assign to buf[3] in the else case, although I see it might aid debugging.

I was never able to get an error return from the program (I tried closing stdout using >& in Bash, and also tried redirecting to /dev/full), but always got an exit status of 0. That surprised me, as the code looks correct.

\$\endgroup\$

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Not the answer you're looking for? Browse other questions tagged or ask your own question.