You may see full code here (note that the link points to the specific commit).
Language is "clean C" (that is, a subset of C89, C99 and C++98 — it is intended to compile under all of these standards). The code must be portable between x86 and x86_64.
UTF-8 handling implemented based on information here and tested with data in this file.
First and foremost I'm interested in correctness. But I'm a bit worried about the length and readability of the code and its speed (I did not profile this thing yet though, so that particular worry is not motivated). I will accept any comments, including constructive nitpicks.
The function itself:
/*
* *Increments* len_bytes by the number of bytes read.
* Fails on invalid UTF-8 characters.
*/
static int ltsLS_eatutf8char(lts_LoadState * ls, size_t * len_bytes)
{
unsigned char b = 0;
signed char expected_length = 0;
int i = 0;
const unsigned char * origin = ls->pos;
/* Check if we have any data in the buffer */
if (!ltsLS_good(ls) || ltsLS_unread(ls) < 1)
{
ls->unread = 0;
ls->pos = NULL;
return LUATEXTS_ECLIPPED;
}
/* We have at least one byte in the buffer, let's check it out. */
b = *ls->pos;
/* We did just eat a byte, no matter what happens next. */
++ls->pos;
--ls->unread;
/* Get an expected length of a character. */
expected_length = utf8_char_len[b];
/* Check if it was a valid first byte. */
if (expected_length < 1)
{
ls->unread = 0;
ls->pos = NULL;
return LUATEXTS_EBADUTF8;
}
/* If it was a single-byte ASCII character, return right away. */
if (expected_length == 1)
{
*len_bytes += expected_length;
return LUATEXTS_ESUCCESS;
}
/*
* It was a multi-byte character. Check if we have enough bytes unread.
* Note that we've eaten one byte already.
*/
if (ltsLS_unread(ls) + 1 < expected_length)
{
ls->unread = 0;
ls->pos = NULL;
return LUATEXTS_ECLIPPED; /* Assuming it is buffer's fault. */
}
/* Let's eat the rest of characters */
for (i = 1; i < expected_length; ++i)
{
b = *ls->pos;
/* We did just eat a byte, no matter what happens next. */
++ls->pos;
--ls->unread;
/* Check if it is a continuation byte */
if (utf8_char_len[b] != -1)
{
ls->unread = 0;
ls->pos = NULL;
return LUATEXTS_EBADUTF8;
}
}
/* All bytes are correct, let's check out for overlong forms */
if (
expected_length == 2 && (
(origin[0] & 0xFE) == 0xC0
)
)
{
ls->unread = 0;
ls->pos = NULL;
return LUATEXTS_EBADUTF8;
}
else if (
expected_length == 3 && (
origin[0] == 0xE0
&& (origin[1] & 0xE0) == 0x80
)
)
{
ls->unread = 0;
ls->pos = NULL;
return LUATEXTS_EBADUTF8;
}
else if (
expected_length == 4 && (
origin[0] == 0xF0
&& (origin[1] & 0xF0) == 0x80
)
)
{
ls->unread = 0;
ls->pos = NULL;
return LUATEXTS_EBADUTF8;
}
else if (
expected_length == 5 && (
origin[0] == 0xF8
&& (origin[1] & 0xF8) == 0x80
)
)
{
ls->unread = 0;
ls->pos = NULL;
return LUATEXTS_EBADUTF8;
}
else if (
expected_length == 6 && (
origin[0] == 0xFC
&& (origin[1] & 0xFC) == 0x80
)
)
{
ls->unread = 0;
ls->pos = NULL;
return LUATEXTS_EBADUTF8;
}
/* No overlongs, check for surrogates. */
if (
expected_length == 3 && (
origin[0] == 0xED
&& (origin[1] & 0xE0) == 0xA0
)
)
{
ls->unread = 0;
ls->pos = NULL;
return LUATEXTS_EBADUTF8;
}
/*
* Note: Not checking for U+FFFE or U+FFFF.
*
* Chapter 3 of version 3.2 of the Unicode standard, Paragraph C5 says
* "A process shall not interpret either U+FFFE or U+FFFF as an abstract
* character", but table 3.1B includes them among
* the "Legal UTF-8 Byte Sequences".
*
* We opt to pass them through.
*/
/* Phew. All done, the UTF-8 character is valid. */
*len_bytes += expected_length;
return LUATEXTS_ESUCCESS;
}
The function relies on this lookup table:
static const signed char utf8_char_len[256] =
{
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 0, 0
};
The function works on lts_LoadState buffer "stream iterator", which should be prepared by caller. Relevant pieces of code:
typedef struct lts_LoadState
{
const unsigned char * pos;
size_t unread;
} lts_LoadState;
static void ltsLS_init(
lts_LoadState * ls,
const unsigned char * data,
size_t len
)
{
ls->pos = (len > 0) ? data : NULL;
ls->unread = len;
}
#define ltsLS_good(ls) \
((ls)->pos != NULL)
#define ltsLS_unread(ls) \
((ls)->unread)
