myUTF-8 small lib (validate UTF-8, guess language, count chars)

Question

I'm new to C language and never got my self into the details of UTF-8, and after reading some articles about it, I wanted to try and play with UTF-8 with C language for both fun and practicing purposes. This is a little C library that is supposed to validates UTF-8 strings, counts chars, guesses the language of the string based on the Unicode code points blocks.

• What do you think about it ?

• this is my first time to design a C library, what do you think of it as a library?

• also what do you think about the way I organized the source files/protected the code/exposed the code (the design of the software) ?

Thank you very much

The source code consists of 3 files

libmyutf8.c

This is the entire code of the library

#include <stdio.h>
#include <stdlib.h>
#include <stddef.h>
#include "headerAll.h"


unsigned long buildCodePoint(unsigned char byte1,
                            unsigned char byte2,
                            unsigned char byte3,
                            unsigned char byte4,
                            int numberOfBytes );

int increseCodeBlock(  unsigned long codePoint,
                        int codeBlocksCount,
                        struct codePointBlock *codePointBlocks);

//macros
//https://stackoverflow.com/questions/523724/c-c-check-if-one-bit-is-set-in-i-e-int-variable
#define CHECK_BIT(var,pos) ((var) & (1<<(pos)) )





unsigned long scanUTF8(char *str, struct codePointBlock *codePointBlocks)
{

    int i = 0;
    unsigned long maxLength = 10485760; // 10MB
    unsigned char currentByte;
    unsigned char byte1 = 0, byte2 = 0, byte3 = 0, byte4 = 0;
    int firstByte = 1;
    int charBytesRemaining = 0;

    unsigned long codePoint = 0;
    int numberOfBytes = 0;
    int codeBlocksCount = 0;
    int charsFound = 0;

    while(codePointBlocks[codeBlocksCount].blockName) codeBlocksCount++;



    do {

        currentByte = *(str + i);
        //printf("iiiiiiiiiiiiiii%d\n", i);
        //printf("curentByteee%d\n", currentByte);
        if(currentByte == 0) break;//end of string

        if (firstByte) {
            firstByte = 0;
            //lastCodePoint is ready
            byte1 = currentByte;

            if ( !(CHECK_BIT(currentByte, 7)) ){
                // if 0XXXXXXX
                firstByte = 1;
                numberOfBytes = 1;
                charBytesRemaining = 0;
                codePoint = buildCodePoint(byte1, 0, 0, 0, numberOfBytes);
                increseCodeBlock(codePoint, codeBlocksCount, codePointBlocks);
                charsFound++;
            } else if (CHECK_BIT(currentByte, 7) && CHECK_BIT(currentByte, 6) &&
                      !CHECK_BIT(currentByte, 5)
            ){
                // if 110XXXXX
                charBytesRemaining = 1;
                numberOfBytes = 2;
            } else if (CHECK_BIT(currentByte, 7) && CHECK_BIT(currentByte, 6) &&
                       CHECK_BIT(currentByte, 5) && !CHECK_BIT(currentByte, 4)
            ){
                // if 1110XXXX
                charBytesRemaining = 2;
                numberOfBytes = 3;
            } else if (CHECK_BIT(currentByte, 7) && CHECK_BIT(currentByte, 6) &&
                       CHECK_BIT(currentByte, 5) && CHECK_BIT(currentByte, 4) &&
                      !CHECK_BIT(currentByte, 3)
            ){
              // if 11110XXX
                charBytesRemaining = 3;
                numberOfBytes = 4;
            } else {
                //not utf-8
                return -1;
            }

        } else {
            //not first byte in char
            if (CHECK_BIT(currentByte, 7) && !CHECK_BIT(currentByte, 6)){
                //must be 10XXXXXX
                // byte1    byte2    byte3    byte4
                if (charBytesRemaining == 3){
                    byte2 = currentByte;
                } else if(charBytesRemaining == 2){
                    if (numberOfBytes == 4){
                        byte3 = currentByte;
                    } else if (numberOfBytes == 3){
                        byte2 = currentByte;
                    }
                } else if (charBytesRemaining == 1){
                    if (numberOfBytes == 4){
                        byte4 = currentByte;
                    } else if (numberOfBytes == 3){
                        byte3 = currentByte;
                    } else if (numberOfBytes == 2){
                        byte2 = currentByte;
                    }
                }
                charBytesRemaining--;

                if (charBytesRemaining == 0){
                    //end of char (last byte)
                    firstByte = 1;
                    /*
                    // uncomment for debugging
                    printf("codePointCalled\n");
                    printf("byte1:%d\n", byte1);
                    printf("byte2:%d\n", byte2);
                    printf("byte3:%d\n", byte3);
                    printf("byte4:%d\n", byte4);
                    printf("numberOfBytes:%d\n", numberOfBytes);
                    */
                    codePoint = buildCodePoint(byte1, byte2, byte3, byte4, numberOfBytes);
                    increseCodeBlock(codePoint, codeBlocksCount, codePointBlocks);
                    byte1 = byte2 = byte3 = byte4 = numberOfBytes = 0;
                    charsFound++;
                }
            }else{
                //not utf-8
                return -1;
            }
        }


        i++;
    } while (i < maxLength);



    return charsFound;

}

int increseCodeBlock(  unsigned long codePoint,
                        int codeBlocksCount,
                        struct codePointBlock *codePointBlocks)
{
    for(int i = 0; i < codeBlocksCount; i++){
        if(codePoint >= codePointBlocks[i].start && codePoint <= codePointBlocks[i].end){
            codePointBlocks[i].count++;
            return 1;
        }
    }
    return 0;
}

unsigned long buildCodePoint(unsigned char byte1,
                            unsigned char byte2,
                            unsigned char byte3,
                            unsigned char byte4,
                            int numberOfBytes )
{
    //  codePoint = (Byte1) | (Byte2) | (Byte3) | (Byte4)

    if(numberOfBytes == 1){
        byte2 = 0;
        byte3 = 0;
        byte4 = 0;
    }else if(numberOfBytes == 2){
        byte1 &= 0b00011111;
        byte2 &= 0b00111111;
        byte3 = 0;
        byte4 = 0;
    }else if(numberOfBytes == 3){
        byte1 &= 0b00001111;
        byte2 &= 0b00111111;
        byte3 &= 0b00111111;
        byte4 = 0;
    }else if(numberOfBytes == 4){
        byte1 &= 0b00000111;
        byte2 &= 0b00111111;
        byte3 &= 0b00111111;
        byte4 &= 0b00111111;
    }else {
        perror("buildCodePoint number of bytes is not riht");
        return 0;
    }

    if(numberOfBytes == 1) return (unsigned long) byte1;

    unsigned long byte1L = (unsigned long) byte1;
    unsigned long byte2L = (unsigned long) byte2;
    unsigned long byte3L = (unsigned long) byte3;
    unsigned long byte4L = (unsigned long) byte4;

    int missedBits = (4 - numberOfBytes) * 8;// 0     4
    // 1(8)     3
    // 2(16)     2
    //  codePoint = (Byte1) | (Byte2) | (Byte3) | (Byte4)
    //
    unsigned long codePoint =   byte4L |
                                (byte3L << (8 -  (missedBits + 2 * (numberOfBytes - 3) ) ) )|
                                (byte2L << (16 - (missedBits + 2 * (numberOfBytes - 2) )) ) |
                                (byte1L << (24 - (missedBits + 2 * (numberOfBytes - 1) )));

    return codePoint;
}


//code blocks extracted from  https://www.utf8-chartable.de/unicode-utf8-table.pl

struct codePointBlock codePointBlocks[] = {

        //281 blocks
        {0x0000, 0x007F, "Basic Latin",0},
        {0x0080, 0x00FF, "Latin-1 Supplement",0},
        {0x0100, 0x017F, "Latin Extended-A",0},
        {0x0180, 0x024F, "Latin Extended-B",0},
        {0x0250, 0x02AF, "IPA Extensions",0},
        {0x02B0, 0x02FF, "Spacing Modifier Letters",0},
        {0x0300, 0x036F, "Combining Diacritical Marks",0},
        {0x0370, 0x03FF, "Greek and Coptic",0},
        {0x0400, 0x04FF, "Cyrillic",0},
        {0x0500, 0x052F, "Cyrillic Supplement",0},
        {0x0530, 0x058F, "Armenian",0},
        {0x0590, 0x05FF, "Hebrew",0},
        {0x0600, 0x06FF, "Arabic",0},
        {0x0700, 0x074F, "Syriac",0},
        {0x0750, 0x077F, "Arabic Supplement",0},
        {0x0780, 0x07BF, "Thaana",0},
        {0x07C0, 0x07FF, "NKo",0},
        {0x0800, 0x083F, "Samaritan",0},
        {0x0840, 0x085F, "Mandaic",0},
        {0x0860, 0x086F, "Syriac Supplement",0},
        {0x08A0, 0x08FF, "Arabic Extended-A",0},
        {0x0900, 0x097F, "Devanagari",0},
        {0x0980, 0x09FF, "Bengali",0},
        {0x0A00, 0x0A7F, "Gurmukhi",0},
        {0x0A80, 0x0AFF, "Gujarati",0},
        {0x0B00, 0x0B7F, "Oriya",0},
        {0x0B80, 0x0BFF, "Tamil",0},
        {0x0C00, 0x0C7F, "Telugu",0},
        {0x0C80, 0x0CFF, "Kannada",0},
        {0x0D00, 0x0D7F, "Malayalam",0},
        {0x0D80, 0x0DFF, "Sinhala",0},
        {0x0E00, 0x0E7F, "Thai",0},
        {0x0E80, 0x0EFF, "Lao",0},
        {0x0F00, 0x0FFF, "Tibetan",0},
        {0x1000, 0x109F, "Myanmar",0},
        {0x10A0, 0x10FF, "Georgian",0},
        {0x1100, 0x11FF, "Hangul Jamo",0},
        {0x1200, 0x137F, "Ethiopic",0},
        {0x1380, 0x139F, "Ethiopic Supplement",0},
        {0x13A0, 0x13FF, "Cherokee",0},
        {0x1400, 0x167F, "Unified Canadian Aboriginal Syllabics",0},
        {0x1680, 0x169F, "Ogham",0},
        {0x16A0, 0x16FF, "Runic",0},
        {0x1700, 0x171F, "Tagalog",0},
        {0x1720, 0x173F, "Hanunoo",0},
        {0x1740, 0x175F, "Buhid",0},
        {0x1760, 0x177F, "Tagbanwa",0},
        {0x1780, 0x17FF, "Khmer",0},
        {0x1800, 0x18AF, "Mongolian",0},
        {0x18B0, 0x18FF, "Unified Canadian Aboriginal Syllabics Extended",0},
        {0x1900, 0x194F, "Limbu",0},
        {0x1950, 0x197F, "Tai Le",0},
        {0x1980, 0x19DF, "New Tai Lue",0},
        {0x19E0, 0x19FF, "Khmer Symbols",0},
        {0x1A00, 0x1A1F, "Buginese",0},
        {0x1A20, 0x1AAF, "Tai Tham",0},
        {0x1AB0, 0x1AFF, "Combining Diacritical Marks Extended",0},
        {0x1B00, 0x1B7F, "Balinese",0},
        {0x1B80, 0x1BBF, "Sundanese",0},
        {0x1BC0, 0x1BFF, "Batak",0},
        {0x1C00, 0x1C4F, "Lepcha",0},
        {0x1C50, 0x1C7F, "Ol Chiki",0},
        {0x1C80, 0x1C8F, "Cyrillic Extended-C",0},
        {0x1CC0, 0x1CCF, "Sundanese Supplement",0},
        {0x1CD0, 0x1CFF, "Vedic Extensions",0},
        {0x1D00, 0x1D7F, "Phonetic Extensions",0},
        {0x1D80, 0x1DBF, "Phonetic Extensions Supplement",0},
        {0x1DC0, 0x1DFF, "Combining Diacritical Marks Supplement",0},
        {0x1E00, 0x1EFF, "Latin Extended Additional",0},
        {0x1F00, 0x1FFF, "Greek Extended",0},
        {0x2000, 0x206F, "General Punctuation",0},
        {0x2070, 0x209F, "Superscripts and Subscripts",0},
        {0x20A0, 0x20CF, "Currency Symbols",0},
        {0x20D0, 0x20FF, "Combining Diacritical Marks for Symbols",0},
        {0x2100, 0x214F, "Letterlike Symbols",0},
        {0x2150, 0x218F, "Number Forms",0},
        {0x2190, 0x21FF, "Arrows",0},
        {0x2200, 0x22FF, "Mathematical Operators",0},
        {0x2300, 0x23FF, "Miscellaneous Technical",0},
        {0x2400, 0x243F, "Control Pictures",0},
        {0x2440, 0x245F, "Optical Character Recognition",0},
        {0x2460, 0x24FF, "Enclosed Alphanumerics",0},
        {0x2500, 0x257F, "Box Drawing",0},
        {0x2580, 0x259F, "Block Elements",0},
        {0x25A0, 0x25FF, "Geometric Shapes",0},
        {0x2600, 0x26FF, "Miscellaneous Symbols",0},
        {0x2700, 0x27BF, "Dingbats",0},
        {0x27C0, 0x27EF, "Miscellaneous Mathematical Symbols-A",0},
        {0x27F0, 0x27FF, "Supplemental Arrows-A",0},
        {0x2800, 0x28FF, "Braille Patterns",0},
        {0x2900, 0x297F, "Supplemental Arrows-B",0},
        {0x2980, 0x29FF, "Miscellaneous Mathematical Symbols-B",0},
        {0x2A00, 0x2AFF, "Supplemental Mathematical Operators",0},
        {0x2B00, 0x2BFF, "Miscellaneous Symbols and Arrows",0},
        {0x2C00, 0x2C5F, "Glagolitic",0},
        {0x2C60, 0x2C7F, "Latin Extended-C",0},
        {0x2C80, 0x2CFF, "Coptic",0},
        {0x2D00, 0x2D2F, "Georgian Supplement",0},
        {0x2D30, 0x2D7F, "Tifinagh",0},
        {0x2D80, 0x2DDF, "Ethiopic Extended",0},
        {0x2DE0, 0x2DFF, "Cyrillic Extended-A",0},
        {0x2E00, 0x2E7F, "Supplemental Punctuation",0},
        {0x2E80, 0x2EFF, "CJK Radicals Supplement",0},
        {0x2F00, 0x2FDF, "Kangxi Radicals",0},
        {0x2FF0, 0x2FFF, "Ideographic Description Characters",0},
        {0x3000, 0x303F, "CJK Symbols and Punctuation",0},
        {0x3040, 0x309F, "Hiragana",0},
        {0x30A0, 0x30FF, "Katakana",0},
        {0x3100, 0x312F, "Bopomofo",0},
        {0x3130, 0x318F, "Hangul Compatibility Jamo",0},
        {0x3190, 0x319F, "Kanbun",0},
        {0x31A0, 0x31BF, "Bopomofo Extended",0},
        {0x31C0, 0x31EF, "CJK Strokes",0},
        {0x31F0, 0x31FF, "Katakana Phonetic Extensions",0},
        {0x3200, 0x32FF, "Enclosed CJK Letters and Months",0},
        {0x3300, 0x33FF, "CJK Compatibility",0},
        {0x3400, 0x4DBF, "CJK Unified Ideographs Extension A",0},
        {0x4DC0, 0x4DFF, "Yijing Hexagram Symbols",0},
        {0x4E00, 0x9FFF, "CJK Unified Ideographs",0},
        {0xA000, 0xA48F, "Yi Syllables",0},
        {0xA490, 0xA4CF, "Yi Radicals",0},
        {0xA4D0, 0xA4FF, "Lisu",0},
        {0xA500, 0xA63F, "Vai",0},
        {0xA640, 0xA69F, "Cyrillic Extended-B",0},
        {0xA6A0, 0xA6FF, "Bamum",0},
        {0xA700, 0xA71F, "Modifier Tone Letters",0},
        {0xA720, 0xA7FF, "Latin Extended-D",0},
        {0xA800, 0xA82F, "Syloti Nagri",0},
        {0xA830, 0xA83F, "Common Indic Number Forms",0},
        {0xA840, 0xA87F, "Phags-pa",0},
        {0xA880, 0xA8DF, "Saurashtra",0},
        {0xA8E0, 0xA8FF, "Devanagari Extended",0},
        {0xA900, 0xA92F, "Kayah Li",0},
        {0xA930, 0xA95F, "Rejang",0},
        {0xA960, 0xA97F, "Hangul Jamo Extended-A",0},
        {0xA980, 0xA9DF, "Javanese",0},
        {0xA9E0, 0xA9FF, "Myanmar Extended-B",0},
        {0xAA00, 0xAA5F, "Cham",0},
        {0xAA60, 0xAA7F, "Myanmar Extended-A",0},
        {0xAA80, 0xAADF, "Tai Viet",0},
        {0xAAE0, 0xAAFF, "Meetei Mayek Extensions",0},
        {0xAB00, 0xAB2F, "Ethiopic Extended-A",0},
        {0xAB30, 0xAB6F, "Latin Extended-E",0},
        {0xAB70, 0xABBF, "Cherokee Supplement",0},
        {0xABC0, 0xABFF, "Meetei Mayek",0},
        {0xAC00, 0xD7AF, "Hangul Syllables",0},
        {0xD7B0, 0xD7FF, "Hangul Jamo Extended-B",0},
        {0xD800, 0xDB7F, "High Surrogates",0},
        {0xDB80, 0xDBFF, "High Private Use Surrogates",0},
        {0xDC00, 0xDFFF, "Low Surrogates",0},
        {0xE000, 0xF8FF, "Private Use Area",0},
        {0xF900, 0xFAFF, "CJK Compatibility Ideographs",0},
        {0xFB00, 0xFB4F, "Alphabetic Presentation Forms",0},
        {0xFB50, 0xFDFF, "Arabic Presentation Forms-A",0},
        {0xFE00, 0xFE0F, "Variation Selectors",0},
        {0xFE10, 0xFE1F, "Vertical Forms",0},
        {0xFE20, 0xFE2F, "Combining Half Marks",0},
        {0xFE30, 0xFE4F, "CJK Compatibility Forms",0},
        {0xFE50, 0xFE6F, "Small Form Variants",0},
        {0xFE70, 0xFEFF, "Arabic Presentation Forms-B",0},
        {0xFF00, 0xFFEF, "Halfwidth and Fullwidth Forms",0},
        {0xFFF0, 0xFFFF, "Specials",0},
        {0x10000, 0x1007F, "Linear B Syllabary",0},
        {0x10080, 0x100FF, "Linear B Ideograms",0},
        {0x10100, 0x1013F, "Aegean Numbers",0},
        {0x10140, 0x1018F, "Ancient Greek Numbers",0},
        {0x10190, 0x101CF, "Ancient Symbols",0},
        {0x101D0, 0x101FF, "Phaistos Disc",0},
        {0x10280, 0x1029F, "Lycian",0},
        {0x102A0, 0x102DF, "Carian",0},
        {0x102E0, 0x102FF, "Coptic Epact Numbers",0},
        {0x10300, 0x1032F, "Old Italic",0},
        {0x10330, 0x1034F, "Gothic",0},
        {0x10350, 0x1037F, "Old Permic",0},
        {0x10380, 0x1039F, "Ugaritic",0},
        {0x103A0, 0x103DF, "Old Persian",0},
        {0x10400, 0x1044F, "Deseret",0},
        {0x10450, 0x1047F, "Shavian",0},
        {0x10480, 0x104AF, "Osmanya",0},
        {0x104B0, 0x104FF, "Osage",0},
        {0x10500, 0x1052F, "Elbasan",0},
        {0x10530, 0x1056F, "Caucasian Albanian",0},
        {0x10600, 0x1077F, "Linear A",0},
        {0x10800, 0x1083F, "Cypriot Syllabary",0},
        {0x10840, 0x1085F, "Imperial Aramaic",0},
        {0x10860, 0x1087F, "Palmyrene",0},
        {0x10880, 0x108AF, "Nabataean",0},
        {0x108E0, 0x108FF, "Hatran",0},
        {0x10900, 0x1091F, "Phoenician",0},
        {0x10920, 0x1093F, "Lydian",0},
        {0x10980, 0x1099F, "Meroitic Hieroglyphs",0},
        {0x109A0, 0x109FF, "Meroitic Cursive",0},
        {0x10A00, 0x10A5F, "Kharoshthi",0},
        {0x10A60, 0x10A7F, "Old South Arabian",0},
        {0x10A80, 0x10A9F, "Old North Arabian",0},
        {0x10AC0, 0x10AFF, "Manichaean",0},
        {0x10B00, 0x10B3F, "Avestan",0},
        {0x10B40, 0x10B5F, "Inscriptional Parthian",0},
        {0x10B60, 0x10B7F, "Inscriptional Pahlavi",0},
        {0x10B80, 0x10BAF, "Psalter Pahlavi",0},
        {0x10C00, 0x10C4F, "Old Turkic",0},
        {0x10C80, 0x10CFF, "Old Hungarian",0},
        {0x10E60, 0x10E7F, "Rumi Numeral Symbols",0},
        {0x11000, 0x1107F, "Brahmi",0},
        {0x11080, 0x110CF, "Kaithi",0},
        {0x110D0, 0x110FF, "Sora Sompeng",0},
        {0x11100, 0x1114F, "Chakma",0},
        {0x11150, 0x1117F, "Mahajani",0},
        {0x11180, 0x111DF, "Sharada",0},
        {0x111E0, 0x111FF, "Sinhala Archaic Numbers",0},
        {0x11200, 0x1124F, "Khojki",0},
        {0x11280, 0x112AF, "Multani",0},
        {0x112B0, 0x112FF, "Khudawadi",0},
        {0x11300, 0x1137F, "Grantha",0},
        {0x11400, 0x1147F, "Newa",0},
        {0x11480, 0x114DF, "Tirhuta",0},
        {0x11580, 0x115FF, "Siddham",0},
        {0x11600, 0x1165F, "Modi",0},
        {0x11660, 0x1167F, "Mongolian Supplement",0},
        {0x11680, 0x116CF, "Takri",0},
        {0x11700, 0x1173F, "Ahom",0},
        {0x118A0, 0x118FF, "Warang Citi",0},
        {0x11A00, 0x11A4F, "Zanabazar Square",0},
        {0x11A50, 0x11AAF, "Soyombo",0},
        {0x11AC0, 0x11AFF, "Pau Cin Hau",0},
        {0x11C00, 0x11C6F, "Bhaiksuki",0},
        {0x11C70, 0x11CBF, "Marchen",0},
        {0x11D00, 0x11D5F, "Masaram Gondi",0},
        {0x12000, 0x123FF, "Cuneiform",0},
        {0x12400, 0x1247F, "Cuneiform Numbers and Punctuation",0},
        {0x12480, 0x1254F, "Early Dynastic Cuneiform",0},
        {0x13000, 0x1342F, "Egyptian Hieroglyphs",0},
        {0x14400, 0x1467F, "Anatolian Hieroglyphs",0},
        {0x16800, 0x16A3F, "Bamum Supplement",0},
        {0x16A40, 0x16A6F, "Mro",0},
        {0x16AD0, 0x16AFF, "Bassa Vah",0},
        {0x16B00, 0x16B8F, "Pahawh Hmong",0},
        {0x16F00, 0x16F9F, "Miao",0},
        {0x16FE0, 0x16FFF, "Ideographic Symbols and Punctuation",0},
        {0x17000, 0x187FF, "Tangut",0},
        {0x18800, 0x18AFF, "Tangut Components",0},
        {0x1B000, 0x1B0FF, "Kana Supplement",0},
        {0x1B100, 0x1B12F, "Kana Extended-A",0},
        {0x1B170, 0x1B2FF, "Nushu",0},
        {0x1BC00, 0x1BC9F, "Duployan",0},
        {0x1BCA0, 0x1BCAF, "Shorthand Format Controls",0},
        {0x1D000, 0x1D0FF, "Byzantine Musical Symbols",0},
        {0x1D100, 0x1D1FF, "Musical Symbols",0},
        {0x1D200, 0x1D24F, "Ancient Greek Musical Notation",0},
        {0x1D300, 0x1D35F, "Tai Xuan Jing Symbols",0},
        {0x1D360, 0x1D37F, "Counting Rod Numerals",0},
        {0x1D400, 0x1D7FF, "Mathematical Alphanumeric Symbols",0},
        {0x1D800, 0x1DAAF, "Sutton SignWriting",0},
        {0x1E000, 0x1E02F, "Glagolitic Supplement",0},
        {0x1E800, 0x1E8DF, "Mende Kikakui",0},
        {0x1E900, 0x1E95F, "Adlam",0},
        {0x1EE00, 0x1EEFF, "Arabic Mathematical Alphabetic Symbols",0},
        {0x1F000, 0x1F02F, "Mahjong Tiles",0},
        {0x1F030, 0x1F09F, "Domino Tiles",0},
        {0x1F0A0, 0x1F0FF, "Playing Cards",0},
        {0x1F100, 0x1F1FF, "Enclosed Alphanumeric Supplement",0},
        {0x1F200, 0x1F2FF, "Enclosed Ideographic Supplement",0},
        {0x1F300, 0x1F5FF, "Miscellaneous Symbols and Pictographs",0},
        {0x1F600, 0x1F64F, "Emoticons",0},
        {0x1F650, 0x1F67F, "Ornamental Dingbats",0},
        {0x1F680, 0x1F6FF, "Transport and Map Symbols",0},
        {0x1F700, 0x1F77F, "Alchemical Symbols",0},
        {0x1F780, 0x1F7FF, "Geometric Shapes Extended",0},
        {0x1F800, 0x1F8FF, "Supplemental Arrows-C",0},
        {0x1F900, 0x1F9FF, "Supplemental Symbols and Pictographs",0},
        {0x20000, 0x2A6DF, "CJK Unified Ideographs Extension B",0},
        {0x2A700, 0x2B73F, "CJK Unified Ideographs Extension C",0},
        {0x2B740, 0x2B81F, "CJK Unified Ideographs Extension D",0},
        {0x2B820, 0x2CEAF, "CJK Unified Ideographs Extension E",0},
        {0x2CEB0, 0x2EBEF, "CJK Unified Ideographs Extension F",0},
        {0x2F800, 0x2FA1F, "CJK Compatibility Ideographs Supplement",0},
        {0xE0000, 0xE007F, "Tags",0},
        {0xE0100, 0xE01EF, "Variation Selectors Supplement",0},
        {0xF0000, 0xFFFFF, "Supplementary Private Use Area-A",0},
        {0x100000, 0x10FFFF, "Supplementary Private Use Area-B",0},
        {0, 0xFFFFFF, "Unknown",0},
        {0,0,NULL,0}
};

libmyutf8.h

This is only what the library user is going to include (the library interface) . Only one function

#ifndef HEADER_MYLIB
#define HEADER_MYLIB

#include "headerAll.h"


extern unsigned long scanUTF8(char *str, struct codePointBlock *codePointBlocks );
extern struct codePointBlock codePointBlocks;


#endif

headerAll.h

This header file will contain definitions needed for both the library files, and the files of the user

#ifndef HEADER_ALL
#define HEADER_ALL


struct codePointBlock{
    int start;
    int end;
    char *blockName;
    int count;
};

#endif

Testing

This is the user files

#include <stdio.h>
#include <stdlib.h>
#include "libmyutf8.h" // < the library



int main( int argc, char *argv[])
{

    FILE* pFile;
    char *buffer = 0;

    pFile = fopen(argv[1], "rb");
    if(pFile == NULL) return 1;


    fseek (pFile, 0, SEEK_END);
    long length = ftell (pFile);
    fseek (pFile, 0, SEEK_SET);
    buffer = malloc (length + 1);

    if (buffer) fread (buffer, 1, length, pFile);
    fclose (pFile);
    buffer[length] = '\0';

    struct codePointBlock *cpbPointer = &codePointBlocks;
    int charsCount = scanUTF8(buffer, cpbPointer);

    int i = 0;
    while(cpbPointer[i].blockName){
        if(cpbPointer[i].count) printf("%s: %d\n", cpbPointer[i].blockName, cpbPointer[i].count);
        i++;
    }
    printf("chars count:%d\n", charsCount);


    return 0;
}

I tested it like this

./libmyutf8 "utf8testingfile.txt"

And the program output was like this

Basic Latin: 49
Latin-1 Supplement: 18
Latin Extended-B: 9
Armenian: 9
Hebrew: 9
Arabic: 72
Enclosed Alphanumerics: 9
Old Persian: 9
chars count:184

Answer 1

libmyutf8.c should be including "libmyutf8.h" rather than "headerAll.h" - that ensures that the function definitions are consistent with the header's prototypes. With that change, there's no need for a separate "headerAll.h", so it can be inlined into "libmyutf8.h".

---

The non-public functions ought to be declared with static linkage, so that they don't pollute the namespace of user code. By adding static to the signature, we can prevent the problem where other code can use the same identifier but then find conflicts when linking the object files together.

---

Our functions should accept pointer to const, since we don't intend to modify the input string.

---

I recommend using an unsigned 1u rather than 1 here:

#define CHECK_BIT(var,pos) ((var) & (1<<(pos)) )

That ensures that all the terms of the calculation are unsigned, and there's no unexpected promotion to a signed type (I don't believe that's a problem anywhere this is used, but it makes it easier to reason about; always prefer unsigned types for bit operations where possible).

---

The URI in the comment can be shortened: https://stackoverflow.com/q/523724.

---

Testing a group of bits is simpler when done as a mask operation. So instead of:

        } else if (CHECK_BIT(currentByte, 7) && CHECK_BIT(currentByte, 6) &&
                   CHECK_BIT(currentByte, 5) && CHECK_BIT(currentByte, 4) &&
                  !CHECK_BIT(currentByte, 3))

We can write:

        } else if ((currentByte & 0xf8) == 0xf0)

---

Instead of a linear search in increseCodeBlock (is that a typo for increase?), we could use a binary search. An alternative would be to have a table of pointers using the high portion of the character to index to the start point in codePointBlocks. I haven't fully thought this through, but it would go something like this:

/* Instead of writing this by hand, we could initialise this using
   code to determine where each xx00 can be found */
static int blockIndex[] = {
                           0,   /* 00xx - Basic Latin and Latin-1 */
                           2,   /* 01xx - Latin Extended A and B */
                           3,   /* 02xx - Latin B, to Spacing Modifiers */
                           6,   /* 03xx - Combining Diacriticals, Greek, Coptic */
                           ... /* lots more... */
};
static const size_t blockIndexSize = sizeof blockIndex / sizeof *blockIndex;

unsigned long highPart = codePoint / 0x100;
if (highPart >= blockIndexSize) {
    highPart = blockIndexSize - 1;
}

for (int i = blockIndex[highPart]; i < codeBlocksCount; i++) {

The blockIndex table just serves to allow us to start i at a point nearer to the target, so saving us testing so many entries.

---

The big do/while loop in scanUtf8 looks like it could be a for loop (we have an initial i = 0, a test of i and an increment ++i, so would be clearer expressed that way). It looks like

int i = 0;
do {
    /* code */
    ++i;
} while (i < maxLength);

Most C programmers would expect to see that as

for (int i = 0;  i < maxLength;  ++i) {
    /* code */
}

(The equivalence does require 0 < maxLength, because this change moves the test from the end to the beginning of each loop)

---

Instead of four variables byte1, byte2, byte3, byte4, it may be better to build up the code point incrementally:

/* UNTESTED! */

/* Store next UTF-8 character into ch, and return next start position */
const char* scanUTF8(const char *s, wchar_t *ch)
{
    int remaining = 0;

    for (; *s;  ++s) {
        unsigned char c = *s;
        if (remaining) {
            /* check that it's a continuation byte */
            if (c & 0xc0 != 0x80) {
                *ch = BAD_UTF8;
                return s;
            }
            *ch = (*ch << 6) + (c & 0x3f);
            if (!--remaining) {
                return s;
            }
        } else if (c & 0x80 == 0) {
            /* single-byte (ASCII) */
            *ch = c;
            return s;
        } else {
            /* should be a start byte */
            for (remaining = 3;  remaining > 0;  --remaining) {
                if (~c >> (6 - remaining) == 1u) {
                    *ch = c & ((1u << (6 - remaining)) - 1);
                    break;
                }
            }
            if (!remaining) {
                /* not a valid start byte */
                *ch = BAD_UTF8;
                return ++s;
            }
        }
    }

    /* incomplete UTF-8 sequence */
    *ch = BAD_UTF8;
    return s;
}

---

The test program assumes that malloc() will succeed, without checking. Don't do this, not even in test programs. Actually, especially not in test programs - make it fail with a useful message, so we don't confuse runtime errors with code bugs.

---

A utility of this nature really deserves a decent unit-test suite. Whilst it's good that there is a test program included, we can do better. Making a set of minimal inputs and expected outputs has these benefits over the test program:

• It's self-contained, rather than needing a separate input file.
• Each test exercises a known subset of the code (making it easier to relate a failed test to its underlying cause).
• Tests can be run automatically every build (and fail the build if they don't all pass).

If we have access to a C++ compiler, that allows us to use one of the excellent test frameworks implemented in that language (it's easy to link our C function into a C++ program, using extern "C").

When writing unit tests, I usually start with the easy error cases first (null string, empty string). That will quickly get us into the testing mindset. After that, we can start adding the success cases and more complex errors (out-of-range characters, extra or missing continuation bytes, surrogate codepoints, overlong encoding, and so on). You might want to read Markus Kuhn's decoder capability tests for some ideas on what to test.