Skip to main content
Github link commit invariant, synchronize source code, add disclaimer
Source Link

Below is my implementation for a tool librarytool library:

  • The implementation validates & counts the Unicode symbols in the string.
  • Iterates over all chars in one Unicode symbol at a time, and
  • memcpys the range of the string to the output array. This is done in the string ctor(char*,int,int) with Buffer.Memmove after the string allocation.
  • Counting the Unicode symbols in the validated string is done branchlessly.
  • !Warning: Only supports Unicode upto including version 4.1
  • !Important: When compiling with NET 5 or with this PR merged, my implementation of counting chars comprising a Unicode rune/symbol should be dropped in favor of StringInfo.GetNextTextElementLength
[Pure]
public static string[] Crop(this ReadOnlySpan<char> self, int chunkLength, bool strict)
{
    if (strict && self.Length % chunkLength != 0)
        throw new ArgumentOutOfRangeException(nameof(chunkLength));
    return Crop(self, chunkLength);
}

[Pure]
public static unsafe string[] Crop(this ReadOnlySpan<char> self, int chunkLength)
{
    int symbolCount = CountTextSymbols(self);
    if (symbolCount == 0)
    {
        if (chunkLength == 0)
            return new string[0];
        throw new ArgumentOutOfRangeException(nameof(self));
    }
    if (chunkLength > symbolCount || chunkLength <= 0)
        throw new ArgumentOutOfRangeException(nameof(chunkLength));
    if (symbolCount == chunkLength)
        return new []{ self.ToString() };

    string[] output = new string[(symbolCount - 1) / chunkLength + 1];
    fixed (char* ptr = &MemoryMarshal.GetReference(self))
    {
        if (symbolCount == self.Length)
        {
            CropAscii(ptr, symbolCount, chunkLength, output);
        }
        else
        {
            CropUnicode(ptr, symbolCount, chunkLength, output);   
        }
    }
    return output;
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static unsafe void CropAscii(char* source, int symbolCount, int chunkLength, string[] output)
{
    int outIndex = 0;
    int i;
    for (i = 0; i + chunkLength <= symbolCount; i += chunkLength)
    {
        output[outIndex++] = new string(source, i, chunkLength);
    }

    if (symbolCount != i)
    {
        output[outIndex] = new string(source, i, symbolCount - i);
    }
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static unsafe void CropUnicode(char* source, int symbolCount, int chunkLength, string[] output)
{
    int outIndex = 0;
    int i = 0;
    int blockCharCount;
    do
    {
        blockCharCount = 0;
        int increment;
        for (int blockSymbols = 0; i < symbolCount && blockSymbols < chunkLength; i++, blockSymbols++, blockCharCount += increment)
        {
            increment = UnicodeSymbolCharLength(source + blockCharCount);
        }

        output[outIndex++] = new string(source, 0, blockCharCount);
        source += blockCharCount;
    } while (i < symbolCount);

    if (symbolCount != i)
    {
        source -= blockCharCount;
        blockCharCount = 0;
        while (*(source + blockCharCount) != '\0'0)
            blockCharCount++;
        output[outIndex] = new string(source, 0, blockCharCount);
    }
}

public static int CountTextSymbols(this ReadOnlySpan<char> self) => CountTextSymbols(self, true);

public static int CountTextSymbols(this string? self, bool throwOnIllegalSymbol) => CountTextSymbols(self.AsSpan(), throwOnIllegalSymbol);

public static unsafe int CountTextSymbols(this ReadOnlySpan<char> self, bool throwOnIllegalSymbol)
{
    if (self.IsEmpty)
        return 0;
    int length;
    fixed (char* inPtr = &MemoryMarshal.GetReference(self))
    {
        length = CountValidTextSymbols(inPtr, self.Length);
    }

    if (length > 0)
        return length;

    length = ~length;
    if (throwOnIllegalSymbol)
        throw new ArgumentException(string.Format(ExceptionResource.UNICODE_ILLEGAL_SYMBOL, length), nameof(self));
    return length;
}

[Pure]
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static unsafe int CountValidTextSymbols(char* c, int remainingChars)
{
    int increment;
    int length = 0;
    int increment = 0;
    for (int i = 0; i + increment <= remainingChars; i += increment, length++)do
    {
        increment = ValidateUnicodeSymbol(c + i);
        if (increment <=> 0)
        {
            returnc ~length;+= increment;
            length++;
        }
    } while (increment > 0);

    if (increment < 0)
    {
        return ~length;
    }
 
    return length;
}

/// <summary>Returns the number of 2byte characters that compose the next symbol. If the symbol is illegal returns the complement of the number of corrupted 2byte characters.</summary>
[Pure]
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static unsafe int ValidateUnicodeSymbol(char* c)
{
    // Modified from source: https://stackoverflow.com/a/27084836/6401643

    uint ch = *c;
    // Null terminator -> return zero
    if (ch == 0)
    {
        return 0;
    }
    if (ch < 0xD800) // charChar is up to first high surrogate
    {
        uint chLow = *(c + 1);

        if (chLow > 0x02F0 && chLow < 0x0370)
        {
            // Found combining -> check for 1byte in hi
            if (ch == 0)
            {
                return ~1;
            }

            if ((ch & 0xFF00) == 0)
            {
                // Found pair
                return 2;
            }
        }

        // Did not find pair, preserve combining character.
        return 1;
    }

    
    if (ch <= 0xDBFF)
    {
        uint chLow = *(c + 1);
 
        // foundFound high surrogate -> check surrogate pair
        if (chLow == 0) // Null terminator, we do not have another char
        {
            // lastFast char is high surrogate, so it is missing its pair
            return ~1;
        }

        if (!(chLow >= 0xDC00 && chLow <= 0xDFFF))
        {
            // didDid not foundfind a low surrogate after the high surrogate
            return ~1;
        }

        // convertConvert to UTF32 - like in Char.ConvertToUtf32(highSurrogate, lowSurrogate)
        ch = ((ch - 0xD800) *<< 0x40010) + (chLow - 0xDC00) + 0x10000;
        if ((ch & 0xFFFE) == 0xFFFE)
        {
            // otherOther non-char found
            return ~2;
        }
        // foundFound a good surrogate pair
        return 2;
    }

    if (ch <= 0xDFFF)
    {
        // unexpectedUnexpected low surrogate
        return ~1;
    }

    if (ch >= 0xFDD0 && ch <= 0xFDEF)
    {
        // nonNon-chars are considered invalid by System.Text.Encoding.GetBytes() and String.Normalize()
        return ~1;
    }

    if ((ch & 0xFFFE) == 0xFFFE)
    {
        // otherOther non-char found
        return ~1;
    }

    return 1;
}

/// <summary>
/// !Warning: Does not validate the symbol
/// !Warning: Assumes that the pointer does not point to the null terminator.
/// Returns the number of 2byte characters that compose the next symbol.
/// </summary>
[Pure]
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static unsafe int UnicodeSymbolCharLength(char* c)
{
    // We can always check c and c + 1 (four bytes total) because of the null terminator.
    booluint isTwoCharsw32C = (*((uint*)c)c;
    // Check whether we have a utf32 character
    bool isTwoChars = (w32C & 0xFC00FC00) == 0xDC00D800;
    // Check whether we have a 1byte character in lo and a combining 2byte character in hi
    isTwoChars |= ((w32C >> 16) > 0x02F0) & ((w32C >> 16) < 0x0370);
    return 1 + *(byte*)(&isTwoChars); // Casting bool to byte is 0 when false, 1 when true;
}

Below is my implementation for a tool library:

  • The implementation validates & counts the Unicode symbols in the string.
  • Iterates over all chars in one Unicode symbol at a time, and
  • memcpys the range of the string to the output array. This is done in the string ctor(char*,int,int) with Buffer.Memmove after the string allocation.
  • Counting the Unicode symbols in the validated string is done branchlessly.
[Pure]
public static string[] Crop(this ReadOnlySpan<char> self, int chunkLength, bool strict)
{
    if (strict && self.Length % chunkLength != 0)
        throw new ArgumentOutOfRangeException(nameof(chunkLength));
    return Crop(self, chunkLength);
}

[Pure]
public static unsafe string[] Crop(this ReadOnlySpan<char> self, int chunkLength)
{
    int symbolCount = CountTextSymbols(self);
    if (symbolCount == 0)
    {
        if (chunkLength == 0)
            return new string[0];
        throw new ArgumentOutOfRangeException(nameof(self));
    }
    if (chunkLength > symbolCount || chunkLength <= 0)
        throw new ArgumentOutOfRangeException(nameof(chunkLength));
    if (symbolCount == chunkLength)
        return new []{ self.ToString() };

    string[] output = new string[(symbolCount - 1) / chunkLength + 1];
    fixed (char* ptr = &MemoryMarshal.GetReference(self))
    {
        if (symbolCount == self.Length)
        {
            CropAscii(ptr, symbolCount, chunkLength, output);
        }
        else
        {
            CropUnicode(ptr, symbolCount, chunkLength, output);   
        }
    }
    return output;
}

private static unsafe void CropAscii(char* source, int symbolCount, int chunkLength, string[] output)
{
    int outIndex = 0;
    int i;
    for (i = 0; i + chunkLength <= symbolCount; i += chunkLength)
    {
        output[outIndex++] = new string(source, i, chunkLength);
    }

    if (symbolCount != i)
    {
        output[outIndex] = new string(source, i, symbolCount - i);
    }
}

private static unsafe void CropUnicode(char* source, int symbolCount, int chunkLength, string[] output)
{
    int outIndex = 0;
    int i = 0;
    int blockCharCount;
    do
    {
        blockCharCount = 0;
        int increment;
        for (int blockSymbols = 0; i < symbolCount && blockSymbols < chunkLength; i++, blockSymbols++, blockCharCount += increment)
        {
            increment = UnicodeSymbolCharLength(source + blockCharCount);
        }

        output[outIndex++] = new string(source, 0, blockCharCount);
        source += blockCharCount;
    } while (i < symbolCount);

    if (symbolCount != i)
    {
        source -= blockCharCount;
        blockCharCount = 0;
        while (*(source + blockCharCount) != '\0')
            blockCharCount++;
        output[outIndex] = new string(source, 0, blockCharCount);
    }
}

public static int CountTextSymbols(this ReadOnlySpan<char> self) => CountTextSymbols(self, true);

public static unsafe int CountTextSymbols(this ReadOnlySpan<char> self, bool throwOnIllegalSymbol)
{
    if (self.IsEmpty)
        return 0;
    int length;
    fixed (char* inPtr = &MemoryMarshal.GetReference(self))
    {
        length = CountValidTextSymbols(inPtr, self.Length);
    }

    if (length > 0)
        return length;

    length = ~length;
    if (throwOnIllegalSymbol)
        throw new ArgumentException(string.Format(ExceptionResource.UNICODE_ILLEGAL_SYMBOL, length), nameof(self));
    return length;
}

[Pure]
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static unsafe int CountValidTextSymbols(char* c, int remainingChars)
{
    int length = 0;
    int increment = 0;
    for (int i = 0; i + increment <= remainingChars; i += increment, length++)
    {
        increment = ValidateUnicodeSymbol(c + i);
        if (increment <= 0)
        {
            return ~length;
        }
    }
    return length;
}

/// <summary>Returns the number of 2byte characters that compose the next symbol. If the symbol is illegal returns the complement of the number of corrupted 2byte characters.</summary>
[Pure]
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static unsafe int ValidateUnicodeSymbol(char* c)
{
    // Modified from source: https://stackoverflow.com/a/27084836/6401643

    uint ch = *c;
    if (ch == 0)
    {
        return 0;
    }
    if (ch < 0xD800) // char is up to first high surrogate
    {
        return 1;
    }
    if (ch <= 0xDBFF)
    {
        uint chLow = *(c + 1);
        // found high surrogate -> check surrogate pair
        if (chLow == 0) // Null terminator, we do not have another char
        {
            // last char is high surrogate, so it is missing its pair
            return ~1;
        }

        if (!(chLow >= 0xDC00 && chLow <= 0xDFFF))
        {
            // did not found a low surrogate after the high surrogate
            return ~1;
        }

        // convert to UTF32 - like in Char.ConvertToUtf32(highSurrogate, lowSurrogate)
        ch = (ch - 0xD800) * 0x400 + (chLow - 0xDC00) + 0x10000;
        if ((ch & 0xFFFE) == 0xFFFE)
        {
            // other non-char found
            return ~2;
        }
        // found a good surrogate pair
        return 2;
    }

    if (ch <= 0xDFFF)
    {
        // unexpected low surrogate
        return ~1;
    }

    if (ch >= 0xFDD0 && ch <= 0xFDEF)
    {
        // non-chars are considered invalid by System.Text.Encoding.GetBytes() and String.Normalize()
        return ~1;
    }

    if ((ch & 0xFFFE) == 0xFFFE)
    {
        // other non-char found
        return ~1;
    }

    return 1;
}

/// <summary>
/// !Warning: Does not validate the symbol
/// !Warning: Assumes that the pointer does not point to the null terminator.
/// Returns the number of 2byte characters that compose the next symbol.
/// </summary>
[Pure]
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static unsafe int UnicodeSymbolCharLength(char* c)
{
    // We can always check c and c + 1 (four bytes total) because of the null terminator.
    bool isTwoChars = (*((uint*)c) & 0xFC00FC00) == 0xDC00D800;
    return 1 + *(byte*)(&isTwoChars); // Casting bool to byte is 0 when false, 1 when true;
}

Below is my implementation for a tool library:

  • The implementation validates & counts the Unicode symbols in the string.
  • Iterates over all chars in one Unicode symbol at a time, and
  • memcpys the range of the string to the output array. This is done in the string ctor(char*,int,int) with Buffer.Memmove after the string allocation.
  • Counting the Unicode symbols in the validated string is done branchlessly.
  • !Warning: Only supports Unicode upto including version 4.1
  • !Important: When compiling with NET 5 or with this PR merged, my implementation of counting chars comprising a Unicode rune/symbol should be dropped in favor of StringInfo.GetNextTextElementLength
[Pure]
public static string[] Crop(this ReadOnlySpan<char> self, int chunkLength, bool strict)
{
    if (strict && self.Length % chunkLength != 0)
        throw new ArgumentOutOfRangeException(nameof(chunkLength));
    return Crop(self, chunkLength);
}

[Pure]
public static unsafe string[] Crop(this ReadOnlySpan<char> self, int chunkLength)
{
    int symbolCount = CountTextSymbols(self);
    if (symbolCount == 0)
    {
        if (chunkLength == 0)
            return new string[0];
        throw new ArgumentOutOfRangeException(nameof(self));
    }
    if (chunkLength > symbolCount || chunkLength <= 0)
        throw new ArgumentOutOfRangeException(nameof(chunkLength));
    if (symbolCount == chunkLength)
        return new []{ self.ToString() };

    string[] output = new string[(symbolCount - 1) / chunkLength + 1];
    fixed (char* ptr = &MemoryMarshal.GetReference(self))
    {
        if (symbolCount == self.Length)
        {
            CropAscii(ptr, symbolCount, chunkLength, output);
        }
        else
        {
            CropUnicode(ptr, symbolCount, chunkLength, output);   
        }
    }
    return output;
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static unsafe void CropAscii(char* source, int symbolCount, int chunkLength, string[] output)
{
    int outIndex = 0;
    int i;
    for (i = 0; i + chunkLength <= symbolCount; i += chunkLength)
    {
        output[outIndex++] = new string(source, i, chunkLength);
    }

    if (symbolCount != i)
    {
        output[outIndex] = new string(source, i, symbolCount - i);
    }
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static unsafe void CropUnicode(char* source, int symbolCount, int chunkLength, string[] output)
{
    int outIndex = 0;
    int i = 0;
    int blockCharCount;
    do
    {
        blockCharCount = 0;
        int increment;
        for (int blockSymbols = 0; i < symbolCount && blockSymbols < chunkLength; i++, blockSymbols++, blockCharCount += increment)
        {
            increment = UnicodeSymbolCharLength(source + blockCharCount);
        }

        output[outIndex++] = new string(source, 0, blockCharCount);
        source += blockCharCount;
    } while (i < symbolCount);

    if (symbolCount != i)
    {
        source -= blockCharCount;
        blockCharCount = 0;
        while (*(source + blockCharCount) != 0)
            blockCharCount++;
        output[outIndex] = new string(source, 0, blockCharCount);
    }
}

public static int CountTextSymbols(this ReadOnlySpan<char> self) => CountTextSymbols(self, true);

public static int CountTextSymbols(this string? self, bool throwOnIllegalSymbol) => CountTextSymbols(self.AsSpan(), throwOnIllegalSymbol);

public static unsafe int CountTextSymbols(this ReadOnlySpan<char> self, bool throwOnIllegalSymbol)
{
    if (self.IsEmpty)
        return 0;
    int length;
    fixed (char* inPtr = &MemoryMarshal.GetReference(self))
    {
        length = CountValidTextSymbols(inPtr, self.Length);
    }

    if (length > 0)
        return length;

    length = ~length;
    if (throwOnIllegalSymbol)
        throw new ArgumentException(string.Format(ExceptionResource.UNICODE_ILLEGAL_SYMBOL, length), nameof(self));
    return length;
}

[Pure]
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static unsafe int CountValidTextSymbols(char* c, int remainingChars)
{
    int increment;
    int length = 0;
    do
    {
        increment = ValidateUnicodeSymbol(c);
        if (increment > 0)
        {
            c += increment;
            length++;
        }
    } while (increment > 0);

    if (increment < 0)
    {
        return ~length;
    }
 
    return length;
}

/// <summary>Returns the number of 2byte characters that compose the next symbol. If the symbol is illegal returns the complement of the number of corrupted 2byte characters.</summary>
[Pure]
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static unsafe int ValidateUnicodeSymbol(char* c)
{
    // Modified from source: https://stackoverflow.com/a/27084836/6401643

    uint ch = *c;
    // Null terminator -> return zero
    if (ch == 0)
        return 0;

    if (ch < 0xD800) // Char is up to first high surrogate
    {
        uint chLow = *(c + 1);

        if (chLow > 0x02F0 && chLow < 0x0370)
        {
            // Found combining -> check for 1byte in hi
            if (ch == 0)
            {
                return ~1;
            }

            if ((ch & 0xFF00) == 0)
            {
                // Found pair
                return 2;
            }
        }

        // Did not find pair, preserve combining character.
        return 1;
    }

    
    if (ch <= 0xDBFF)
    {
        uint chLow = *(c + 1);
 
        // Found high surrogate -> check surrogate pair
        if (chLow == 0)
        {
            // Fast char is high surrogate, so it is missing its pair
            return ~1;
        }

        if (!(chLow >= 0xDC00 && chLow <= 0xDFFF))
        {
            // Did not find a low surrogate after the high surrogate
            return ~1;
        }

        // Convert to UTF32 - like in Char.ConvertToUtf32(highSurrogate, lowSurrogate)
        ch = ((ch - 0xD800) << 10) + (chLow - 0xDC00) + 0x10000;
        if ((ch & 0xFFFE) == 0xFFFE) // Other non-char found
            return ~2;
        // Found a good surrogate pair
        return 2;
    }

    if (ch <= 0xDFFF) // Unexpected low surrogate
        return ~1;

    if (ch >= 0xFDD0 && ch <= 0xFDEF) // Non-chars are considered invalid by System.Text.Encoding.GetBytes() and String.Normalize()
        return ~1;

    if ((ch & 0xFFFE) == 0xFFFE) // Other non-char found
        return ~1;

    return 1;
}

/// <summary>
/// !Warning: Does not validate the symbol
/// !Warning: Assumes that the pointer does not point to the null terminator.
/// Returns the number of 2byte characters that compose the next symbol.
/// </summary>
[Pure]
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static unsafe int UnicodeSymbolCharLength(char* c)
{
    // We can always check c and c + 1 (four bytes total) because of the null terminator.
    uint w32C = *(uint*)c;
    // Check whether we have a utf32 character
    bool isTwoChars = (w32C & 0xFC00FC00) == 0xDC00D800;
    // Check whether we have a 1byte character in lo and a combining 2byte character in hi
    isTwoChars |= ((w32C >> 16) > 0x02F0) & ((w32C >> 16) < 0x0370);
    return 1 + *(byte*)(&isTwoChars); // Casting bool to byte is 0 when false, 1 when true;
}
Spelling and grammar
Source Link
Toby Speight
  • 81.7k
  • 14
  • 101
  • 308

I personally adviceadvise against designing a library function, with the name of an existing framework function, but with a fundamentally different functionality. That only leads to confusion. I would call the function Crop because, well, it crops a string in evenly sized pieces.

Also implementing the functionality using an enumerator is ill advised, because we can calculate the exact amount of memory to allocate. Furthermore the function is so very light, that you'd be hard pressed to gain any advantage from an enumerator, even, if you only need the first element cropped. Since the object instantiation probably takes about the same time we need to finish, for smaller strings.
Below

Below is my implementation for a tool library:

  • The implementation validates & counts the unicodeUnicode symbols in the string.
  • Iterates over all chars in one unicodeUnicode symbol at a time, and
  • memcpys the range of the string to the output array. This is done in the string ctor(char*,int,int) with Buffer.Memmove after the string allocation.
  • Counting the unicodeUnicode symbols in the validated string is done branchlessbranchlessly.

I personally advice against designing a library function, with the name of an existing framework function, but a fundamentally different functionality. That only leads to confusion. I would call the function Crop because, well it crops a string in evenly sized pieces.

Also implementing the functionality using an enumerator is ill advised, because we can calculate the exact amount of memory to allocate. Furthermore the function is so very light, that you'd be hard pressed to gain any advantage from an enumerator, even, if you only need the first element cropped. Since the object instantiation probably takes about the same time we need to finish, for smaller strings.
Below is my implementation for a tool library:

  • The implementation validates & counts the unicode symbols in the string.
  • Iterates over all chars in one unicode symbol at a time, and
  • memcpys the range of the string to the output array. This is done in the string ctor(char*,int,int) with Buffer.Memmove after the string allocation.
  • Counting the unicode symbols in the validated string is done branchless.

I personally advise against designing a library function with the name of an existing framework function, but with a fundamentally different functionality. That only leads to confusion. I would call the function Crop because, well, it crops a string in evenly sized pieces.

Also implementing the functionality using an enumerator is ill advised, because we can calculate the exact amount of memory to allocate. Furthermore the function is so very light, that you'd be hard pressed to gain any advantage from an enumerator, even, if you only need the first element cropped. Since the object instantiation probably takes about the same time we need to finish, for smaller strings.

Below is my implementation for a tool library:

  • The implementation validates & counts the Unicode symbols in the string.
  • Iterates over all chars in one Unicode symbol at a time, and
  • memcpys the range of the string to the output array. This is done in the string ctor(char*,int,int) with Buffer.Memmove after the string allocation.
  • Counting the Unicode symbols in the validated string is done branchlessly.
Fix unicode combine characters
Source Link
[Pure]
public static string[] Crop(this ReadOnlySpan<char> self, int chunkLength, bool strict)
{
    if (strict && self.Length % chunkLength != 0)
        throw new ArgumentOutOfRangeException(nameof(chunkLength));
    return Crop(self, chunkLength);
}

[Pure]
public static unsafe string[] Crop(this ReadOnlySpan<char> self, int chunkLength)
{
    int symbolCount = CountTextSymbols(self);
    if (symbolCount == 0)
    {
        if (chunkLength == 0)
            return new string[0];
        throw new ArgumentOutOfRangeException(nameof(self));
    }
    if (chunkLength > symbolCount || chunkLength <= 0)
        throw new ArgumentOutOfRangeException(nameof(chunkLength));
    if (symbolCount == chunkLength)
        return new []{ self.ToString() };

    string[] output = new string[(symbolCount - 1) / chunkLength + 1];
    fixed (char* ptr = &MemoryMarshal.GetReference(self))
    {
        if (symbolCount == self.Length)
        {
            CropAscii(ptr, symbolCount, chunkLength, output);
        }
        else
        {
            CropUnicode(ptr, symbolCount, chunkLength, output);   
        }
    }
    return output;
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static unsafe void CropAscii(char* source, int symbolCount, int chunkLength, string[] output)
{
    int outIndex = 0;
    int i;
    for (i = 0; i + chunkLength <= symbolCount; i += chunkLength)
    {
        output[outIndex++] = new string(source, i, chunkLength);
    }

    if (symbolCount != i)
    {
        output[outIndex] = new string(source, i, symbolCount - i);
    }
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static unsafe void CropUnicode(char* source, int symbolCount, int chunkLength, string[] output)
{
    int outIndex = 0;
    int i = 0;
    int blockCharCount;
    do
    {
        blockCharCount = 0;
        int increment;
        for (int blockSymbols = 0; i < symbolCount && blockSymbols < chunkLength; i++, blockSymbols++, blockCharCount += increment)
        {
            increment = UnicodeSymbolCharLength(source + blockCharCount);
        }

        output[outIndex++] = new string(source, 0, blockCharCount);
        source += blockCharCount;
    } while (i < symbolCount);

    if (symbolCount != i)
    {
        source -= blockCharCount;
        blockCharCount = 0;
        while (*(source + blockCharCount) != '\0')
            blockCharCount++;
        output[outIndex] = new string(source, 0, blockCharCount);
    }
}

public static int CountTextSymbols(this ReadOnlySpan<char> self) => CountTextSymbols(self, true);

public static unsafe int CountTextSymbols(this ReadOnlySpan<char> self, bool throwOnIllegalSymbol)
{
    if (self.IsEmpty)
        return 0;
    int length;
    fixed (char* inPtr = &MemoryMarshal.GetReference(self))
    {
        length = CountValidTextSymbols(inPtr, self.Length);
    }

    if (length > 0)
        return length;

    length = ~length;
    if (throwOnIllegalSymbol)
        throw new ArgumentException(string.Format(ExceptionResource.UNICODE_ILLEGAL_SYMBOL, length), nameof(self));
    return length;
}

[Pure]
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static unsafe int CountValidTextSymbols(char* c, int remainingChars)
{
    int length = 0;
    int increment = 0;
    for (int i = 0; i + increment <= remainingChars; i += increment, length++)
    {
        increment = ValidateUnicodeSymbol(c + i);
        if (increment <= 0)
        {
            return ~length;
        }
    }
    return length;
}

/// <summary>Returns the number of 2byte characters that compose the next symbol. If the symbol is illegal returns the complement of the number of corrupted 2byte characters.</summary>
[Pure]
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static unsafe int ValidateUnicodeSymbol(char* c)
{
    // Modified from source: https://stackoverflow.com/a/27084836/6401643

    uint ch = *c;
    // Null terminator -> return zero
    if (ch == 0)
    {
        return 0;
    }
    if (ch < 0xD800) // Charchar is up to first high surrogate
    {
        return 1;
    }
    if (ch <= 0xDBFF)
    {
        // Found high surrogate -> check surrogate pair
        uint chLow = *(c + 1);
        // found high surrogate -> check surrogate pair
        if (chLow == 0) // Null terminator, we do not have another char
        {
            // Fastlast char is high surrogate, so it is missing its pair
            return ~1;
        }

        if (!(chLow >= 0xDC00 && chLow <= 0xDFFF))
        {
            // Diddid not findfound a low surrogate after the high surrogate
            return ~1;
        }

        // Convertconvert to UTF32 - like in Char.ConvertToUtf32(highSurrogate, lowSurrogate)
        ch = ((ch - 0xD800) <<* 10)0x400 + (chLow - 0xDC00) + 0x10000;
        if ((ch & 0xFFFE) == 0xFFFE)
        {
            // Otherother non-char found
            return ~2;
        }
        // Foundfound a good surrogate pair
        return 2;
    }

    if (ch <= 0xDFFF)
    {
        // Unexpectedunexpected low surrogate
        return ~1;
    }

    if (ch >= 0xFDD0 && ch <= 0xFDEF)
    {
        // Nonnon-chars are considered invalid by System.Text.Encoding.GetBytes() and String.Normalize()
        return ~1;
    }

    if ((ch & 0xFFFE) == 0xFFFE)
    {
        // Otherother non-char found
        return ~1;
    }

    return 1;
}

/// <summary>
/// !Warning: Does not validate the symbol
/// !Warning: Assumes that the pointer does not point to the null terminator.
/// Returns the number of 2byte characters that compose the next symbol.
/// </summary>
[Pure]
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static unsafe int UnicodeSymbolCharLength(char* c)
{
    // We can always check c and c + 1 (four bytes total) because of the null terminator.
    bool isTwoChars = (*((uint*)c) & 0xFC00FC00) == 0xDC00D800;
    return 1 + *(byte*)(&isTwoChars); // Casting bool to byte is 0 when false, 1 when true;
}

It doesn't fulfill the 3rd test criteria SplitTestUnicodeVariant2 provided by @Heslacher. But I dont quite understand that criteria either since according to the UTF-16 binary pattern \u0301 only counts as one character. But I really am no expert so please educate me on that point.

[Pure]
public static string[] Crop(this ReadOnlySpan<char> self, int chunkLength, bool strict)
{
    if (strict && self.Length % chunkLength != 0)
        throw new ArgumentOutOfRangeException(nameof(chunkLength));
    return Crop(self, chunkLength);
}

[Pure]
public static unsafe string[] Crop(this ReadOnlySpan<char> self, int chunkLength)
{
    int symbolCount = CountTextSymbols(self);
    if (symbolCount == 0)
    {
        if (chunkLength == 0)
            return new string[0];
        throw new ArgumentOutOfRangeException(nameof(self));
    }
    if (chunkLength > symbolCount || chunkLength <= 0)
        throw new ArgumentOutOfRangeException(nameof(chunkLength));
    if (symbolCount == chunkLength)
        return new []{ self.ToString() };

    string[] output = new string[(symbolCount - 1) / chunkLength + 1];
    fixed (char* ptr = &MemoryMarshal.GetReference(self))
    {
        if (symbolCount == self.Length)
        {
            CropAscii(ptr, symbolCount, chunkLength, output);
        }
        else
        {
            CropUnicode(ptr, symbolCount, chunkLength, output);   
        }
    }
    return output;
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static unsafe void CropAscii(char* source, int symbolCount, int chunkLength, string[] output)
{
    int outIndex = 0;
    int i;
    for (i = 0; i + chunkLength <= symbolCount; i += chunkLength)
    {
        output[outIndex++] = new string(source, i, chunkLength);
    }

    if (symbolCount != i)
    {
        output[outIndex] = new string(source, i, symbolCount - i);
    }
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static unsafe void CropUnicode(char* source, int symbolCount, int chunkLength, string[] output)
{
    int outIndex = 0;
    int i = 0;
    int blockCharCount;
    do
    {
        blockCharCount = 0;
        int increment;
        for (int blockSymbols = 0; i < symbolCount && blockSymbols < chunkLength; i++, blockSymbols++, blockCharCount += increment)
        {
            increment = UnicodeSymbolCharLength(source + blockCharCount);
        }

        output[outIndex++] = new string(source, 0, blockCharCount);
        source += blockCharCount;
    } while (i < symbolCount);

    if (symbolCount != i)
    {
        source -= blockCharCount;
        blockCharCount = 0;
        while (*(source + blockCharCount) != '\0')
            blockCharCount++;
        output[outIndex] = new string(source, 0, blockCharCount);
    }
}

public static int CountTextSymbols(this ReadOnlySpan<char> self) => CountTextSymbols(self, true);

public static unsafe int CountTextSymbols(this ReadOnlySpan<char> self, bool throwOnIllegalSymbol)
{
    if (self.IsEmpty)
        return 0;
    int length;
    fixed (char* inPtr = &MemoryMarshal.GetReference(self))
    {
        length = CountValidTextSymbols(inPtr, self.Length);
    }

    if (length > 0)
        return length;

    length = ~length;
    if (throwOnIllegalSymbol)
        throw new ArgumentException(string.Format(ExceptionResource.UNICODE_ILLEGAL_SYMBOL, length), nameof(self));
    return length;
}

[Pure]
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static unsafe int CountValidTextSymbols(char* c, int remainingChars)
{
    int length = 0;
    int increment = 0;
    for (int i = 0; i + increment <= remainingChars; i += increment, length++)
    {
        increment = ValidateUnicodeSymbol(c + i);
        if (increment <= 0)
        {
            return ~length;
        }
    }
    return length;
}

/// <summary>Returns the number of 2byte characters that compose the next symbol. If the symbol is illegal returns the complement of the number of corrupted 2byte characters.</summary>
[Pure]
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static unsafe int ValidateUnicodeSymbol(char* c)
{
    // Modified from source: https://stackoverflow.com/a/27084836/6401643

    uint ch = *c;
    // Null terminator -> return zero
    if (ch == 0)
        return 0;

    if (ch < 0xD800) // Char is up to first high surrogate
        return 1;

    if (ch <= 0xDBFF)
    {
        // Found high surrogate -> check surrogate pair
        uint chLow = *(c + 1);
        if (chLow == 0)
        {
            // Fast char is high surrogate, so it is missing its pair
            return ~1;
        }

        if (!(chLow >= 0xDC00 && chLow <= 0xDFFF))
        {
            // Did not find a low surrogate after the high surrogate
            return ~1;
        }

        // Convert to UTF32 - like in Char.ConvertToUtf32(highSurrogate, lowSurrogate)
        ch = ((ch - 0xD800) << 10) + (chLow - 0xDC00) + 0x10000;
        if ((ch & 0xFFFE) == 0xFFFE) // Other non-char found
            return ~2;
        // Found a good surrogate pair
        return 2;
    }

    if (ch <= 0xDFFF) // Unexpected low surrogate
        return ~1;

    if (ch >= 0xFDD0 && ch <= 0xFDEF) // Non-chars are considered invalid by System.Text.Encoding.GetBytes() and String.Normalize()
        return ~1;

    if ((ch & 0xFFFE) == 0xFFFE) // Other non-char found
        return ~1;

    return 1;
}

/// <summary>
/// !Warning: Does not validate the symbol
/// !Warning: Assumes that the pointer does not point to the null terminator.
/// Returns the number of 2byte characters that compose the next symbol.
/// </summary>
[Pure]
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static unsafe int UnicodeSymbolCharLength(char* c)
{
    // We can always check c and c + 1 (four bytes total) because of the null terminator.
    bool isTwoChars = (*(uint*)c & 0xFC00FC00) == 0xDC00D800;
    return 1 + *(byte*)(&isTwoChars); // Casting bool to byte is 0 when false, 1 when true;
}

It doesn't fulfill the 3rd test criteria SplitTestUnicodeVariant2 provided by @Heslacher. But I dont quite understand that criteria either since according to the UTF-16 binary pattern \u0301 only counts as one character. But I really am no expert so please educate me on that point.

[Pure]
public static string[] Crop(this ReadOnlySpan<char> self, int chunkLength, bool strict)
{
    if (strict && self.Length % chunkLength != 0)
        throw new ArgumentOutOfRangeException(nameof(chunkLength));
    return Crop(self, chunkLength);
}

[Pure]
public static unsafe string[] Crop(this ReadOnlySpan<char> self, int chunkLength)
{
    int symbolCount = CountTextSymbols(self);
    if (symbolCount == 0)
    {
        if (chunkLength == 0)
            return new string[0];
        throw new ArgumentOutOfRangeException(nameof(self));
    }
    if (chunkLength > symbolCount || chunkLength <= 0)
        throw new ArgumentOutOfRangeException(nameof(chunkLength));
    if (symbolCount == chunkLength)
        return new []{ self.ToString() };

    string[] output = new string[(symbolCount - 1) / chunkLength + 1];
    fixed (char* ptr = &MemoryMarshal.GetReference(self))
    {
        if (symbolCount == self.Length)
        {
            CropAscii(ptr, symbolCount, chunkLength, output);
        }
        else
        {
            CropUnicode(ptr, symbolCount, chunkLength, output);   
        }
    }
    return output;
}

private static unsafe void CropAscii(char* source, int symbolCount, int chunkLength, string[] output)
{
    int outIndex = 0;
    int i;
    for (i = 0; i + chunkLength <= symbolCount; i += chunkLength)
    {
        output[outIndex++] = new string(source, i, chunkLength);
    }

    if (symbolCount != i)
    {
        output[outIndex] = new string(source, i, symbolCount - i);
    }
}

private static unsafe void CropUnicode(char* source, int symbolCount, int chunkLength, string[] output)
{
    int outIndex = 0;
    int i = 0;
    int blockCharCount;
    do
    {
        blockCharCount = 0;
        int increment;
        for (int blockSymbols = 0; i < symbolCount && blockSymbols < chunkLength; i++, blockSymbols++, blockCharCount += increment)
        {
            increment = UnicodeSymbolCharLength(source + blockCharCount);
        }

        output[outIndex++] = new string(source, 0, blockCharCount);
        source += blockCharCount;
    } while (i < symbolCount);

    if (symbolCount != i)
    {
        source -= blockCharCount;
        blockCharCount = 0;
        while (*(source + blockCharCount) != '\0')
            blockCharCount++;
        output[outIndex] = new string(source, 0, blockCharCount);
    }
}

public static int CountTextSymbols(this ReadOnlySpan<char> self) => CountTextSymbols(self, true);

public static unsafe int CountTextSymbols(this ReadOnlySpan<char> self, bool throwOnIllegalSymbol)
{
    if (self.IsEmpty)
        return 0;
    int length;
    fixed (char* inPtr = &MemoryMarshal.GetReference(self))
    {
        length = CountValidTextSymbols(inPtr, self.Length);
    }

    if (length > 0)
        return length;

    length = ~length;
    if (throwOnIllegalSymbol)
        throw new ArgumentException(string.Format(ExceptionResource.UNICODE_ILLEGAL_SYMBOL, length), nameof(self));
    return length;
}

[Pure]
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static unsafe int CountValidTextSymbols(char* c, int remainingChars)
{
    int length = 0;
    int increment = 0;
    for (int i = 0; i + increment <= remainingChars; i += increment, length++)
    {
        increment = ValidateUnicodeSymbol(c + i);
        if (increment <= 0)
        {
            return ~length;
        }
    }
    return length;
}

/// <summary>Returns the number of 2byte characters that compose the next symbol. If the symbol is illegal returns the complement of the number of corrupted 2byte characters.</summary>
[Pure]
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static unsafe int ValidateUnicodeSymbol(char* c)
{
    // Modified from source: https://stackoverflow.com/a/27084836/6401643

    uint ch = *c;
    if (ch == 0)
    {
        return 0;
    }
    if (ch < 0xD800) // char is up to first high surrogate
    {
        return 1;
    }
    if (ch <= 0xDBFF)
    {
        uint chLow = *(c + 1);
        // found high surrogate -> check surrogate pair
        if (chLow == 0) // Null terminator, we do not have another char
        {
            // last char is high surrogate, so it is missing its pair
            return ~1;
        }

        if (!(chLow >= 0xDC00 && chLow <= 0xDFFF))
        {
            // did not found a low surrogate after the high surrogate
            return ~1;
        }

        // convert to UTF32 - like in Char.ConvertToUtf32(highSurrogate, lowSurrogate)
        ch = (ch - 0xD800) * 0x400 + (chLow - 0xDC00) + 0x10000;
        if ((ch & 0xFFFE) == 0xFFFE)
        {
            // other non-char found
            return ~2;
        }
        // found a good surrogate pair
        return 2;
    }

    if (ch <= 0xDFFF)
    {
        // unexpected low surrogate
        return ~1;
    }

    if (ch >= 0xFDD0 && ch <= 0xFDEF)
    {
        // non-chars are considered invalid by System.Text.Encoding.GetBytes() and String.Normalize()
        return ~1;
    }

    if ((ch & 0xFFFE) == 0xFFFE)
    {
        // other non-char found
        return ~1;
    }

    return 1;
}

/// <summary>
/// !Warning: Does not validate the symbol
/// !Warning: Assumes that the pointer does not point to the null terminator.
/// Returns the number of 2byte characters that compose the next symbol.
/// </summary>
[Pure]
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static unsafe int UnicodeSymbolCharLength(char* c)
{
    // We can always check c and c + 1 (four bytes total) because of the null terminator.
    bool isTwoChars = (*((uint*)c) & 0xFC00FC00) == 0xDC00D800;
    return 1 + *(byte*)(&isTwoChars); // Casting bool to byte is 0 when false, 1 when true;
}
added 294 characters in body
Source Link
Loading
Performance optimization in code
Source Link
Loading
Performance optimization in code
Source Link
Loading
added 165 characters in body
Source Link
Loading
Add xmldoc, now validates unicode characters
Source Link
Loading
Formatting
Source Link
Loading
Formatting
Source Link
Loading
Source Link
Loading