Convert a mix of latin 1 and UTF-8 to proper UTF-8

Question

The following program takes an arbitrary byte sequence as input and outputs well-formed UTF-8. All UTF-8 sequences from the input are copied unmodified. All other bytes are assumed to be encoded in ISO 8859-1 or some variant, and their byte value is output as the corresponding code point in UTF-8.

#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>

static int buf[4];
static size_t buflen;

/* Reads from stdin until buf[index] can be accessed. */
static bool read_buf(size_t index) {
    while (!(index < buflen)) {
        buf[buflen] = getchar();
        if (buf[buflen] == EOF) {
            return false;
        }
        buflen++;
    }
    return buf[buflen - 1] != EOF;
}

static void write_buf(size_t n) {
    for (size_t i = 0; i < n; i++) {
        putchar(buf[i]);
    }
    buflen = 0;
}

static bool start_between(int min, int max) {
    return min <= buf[0] && buf[0] < max;
}

static bool is_cont(int index) {
    return 0x80 <= buf[index] && buf[index] < 0xC0;
}

static int code_point(int bits18, int bits12, int bits06, int bits00) {
    return   ((bits18 & 0x3F) << 18)
           | ((bits12 & 0x3F) << 12)
           | ((bits06 & 0x3F) <<  6)
           |  (bits00 & 0x3F);
}

int main(void) {
    while (read_buf(0) && !ferror(stdout)) {
        if (start_between(0x00, 0x80)) {
            write_buf(1);
        } else if (start_between(0xC2, 0xE0) && read_buf(1) && is_cont(1)
                   && code_point(0, 0, buf[0] & 0x1F, buf[1]) >= 0x0080) {
            write_buf(2);
        } else if (start_between(0xE0, 0xF0) && read_buf(2) && is_cont(1) && is_cont(2)
                   && code_point(0, buf[0] & 0x0F, buf[1], buf[2]) >= 0x0800) {
            write_buf(3);
        } else if (start_between(0xF0, 0xF8) && read_buf(3) && is_cont(1) && is_cont(2) && is_cont(3)
                   && code_point(buf[0] & 0x07, buf[1], buf[2], buf[3]) >= 0x010000
                   && code_point(buf[0] & 0x07, buf[1], buf[2], buf[3]) < 0x110000) {
            write_buf(4);
        } else {
            putchar(0xC0 + ((buf[0] >> 6) & 0x1F));
            putchar(0x80 + ((buf[0] >> 0) & 0x3F));
            buflen--;
            buf[0] = buf[1];
            buf[1] = buf[2];
            buf[2] = buf[3];
            buf[3] = 0;
        }
    }

    return ferror(stdin) || ferror(stdout) ? EXIT_FAILURE : EXIT_SUCCESS;
}

To test the code, I manually ran variants of the following command:

perl -e 'print pack("H*", "fcf09f9985e08080fc")' | ./latin1_to_utf8 | hexdump -C

Answer 1

A useful idea, and very well implemented.

I'd re-write this condition:

while (!(index < buflen))

as

while (buflen <= index)

Apart from that, I found the code very readable.

It's a good idea to store the received data as int rather than converting to char immediately (though I do worry somewhat about using a signed type in code_point - perhaps its arguments should be narrowed to unsigned char?).

Another minor nit: it's not necessary to assign to buf[3] in the else case, although I see it might aid debugging.

I was never able to get an error return from the program (I tried closing stdout using >& in Bash, and also tried redirecting to /dev/full), but always got an exit status of 0. That surprised me, as the code looks correct.