Below is my implementation for a tool librarytool library:
- The implementation validates & counts the Unicode symbols in the string.
- Iterates over all chars in one Unicode symbol at a time, and
memcpy
s the range of the string to the output array. This is done in the stringctor(char*,int,int)
withBuffer.Memmove
after the string allocation.- Counting the Unicode symbols in the validated string is done branchlessly.
- !Warning: Only supports Unicode upto including version 4.1
- !Important: When compiling with NET 5 or with this PR merged, my implementation of counting chars comprising a Unicode rune/symbol should be dropped in favor of
StringInfo.GetNextTextElementLength
[Pure]
public static string[] Crop(this ReadOnlySpan<char> self, int chunkLength, bool strict)
{
if (strict && self.Length % chunkLength != 0)
throw new ArgumentOutOfRangeException(nameof(chunkLength));
return Crop(self, chunkLength);
}
[Pure]
public static unsafe string[] Crop(this ReadOnlySpan<char> self, int chunkLength)
{
int symbolCount = CountTextSymbols(self);
if (symbolCount == 0)
{
if (chunkLength == 0)
return new string[0];
throw new ArgumentOutOfRangeException(nameof(self));
}
if (chunkLength > symbolCount || chunkLength <= 0)
throw new ArgumentOutOfRangeException(nameof(chunkLength));
if (symbolCount == chunkLength)
return new []{ self.ToString() };
string[] output = new string[(symbolCount - 1) / chunkLength + 1];
fixed (char* ptr = &MemoryMarshal.GetReference(self))
{
if (symbolCount == self.Length)
{
CropAscii(ptr, symbolCount, chunkLength, output);
}
else
{
CropUnicode(ptr, symbolCount, chunkLength, output);
}
}
return output;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static unsafe void CropAscii(char* source, int symbolCount, int chunkLength, string[] output)
{
int outIndex = 0;
int i;
for (i = 0; i + chunkLength <= symbolCount; i += chunkLength)
{
output[outIndex++] = new string(source, i, chunkLength);
}
if (symbolCount != i)
{
output[outIndex] = new string(source, i, symbolCount - i);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static unsafe void CropUnicode(char* source, int symbolCount, int chunkLength, string[] output)
{
int outIndex = 0;
int i = 0;
int blockCharCount;
do
{
blockCharCount = 0;
int increment;
for (int blockSymbols = 0; i < symbolCount && blockSymbols < chunkLength; i++, blockSymbols++, blockCharCount += increment)
{
increment = UnicodeSymbolCharLength(source + blockCharCount);
}
output[outIndex++] = new string(source, 0, blockCharCount);
source += blockCharCount;
} while (i < symbolCount);
if (symbolCount != i)
{
source -= blockCharCount;
blockCharCount = 0;
while (*(source + blockCharCount) != '\0'0)
blockCharCount++;
output[outIndex] = new string(source, 0, blockCharCount);
}
}
public static int CountTextSymbols(this ReadOnlySpan<char> self) => CountTextSymbols(self, true);
public static int CountTextSymbols(this string? self, bool throwOnIllegalSymbol) => CountTextSymbols(self.AsSpan(), throwOnIllegalSymbol);
public static unsafe int CountTextSymbols(this ReadOnlySpan<char> self, bool throwOnIllegalSymbol)
{
if (self.IsEmpty)
return 0;
int length;
fixed (char* inPtr = &MemoryMarshal.GetReference(self))
{
length = CountValidTextSymbols(inPtr, self.Length);
}
if (length > 0)
return length;
length = ~length;
if (throwOnIllegalSymbol)
throw new ArgumentException(string.Format(ExceptionResource.UNICODE_ILLEGAL_SYMBOL, length), nameof(self));
return length;
}
[Pure]
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static unsafe int CountValidTextSymbols(char* c, int remainingChars)
{
int increment;
int length = 0;
int increment = 0;
for (int i = 0; i + increment <= remainingChars; i += increment, length++)do
{
increment = ValidateUnicodeSymbol(c + i);
if (increment <=> 0)
{
returnc ~length;+= increment;
length++;
}
} while (increment > 0);
if (increment < 0)
{
return ~length;
}
return length;
}
/// <summary>Returns the number of 2byte characters that compose the next symbol. If the symbol is illegal returns the complement of the number of corrupted 2byte characters.</summary>
[Pure]
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static unsafe int ValidateUnicodeSymbol(char* c)
{
// Modified from source: https://stackoverflow.com/a/27084836/6401643
uint ch = *c;
// Null terminator -> return zero
if (ch == 0)
{
return 0;
}
if (ch < 0xD800) // charChar is up to first high surrogate
{
uint chLow = *(c + 1);
if (chLow > 0x02F0 && chLow < 0x0370)
{
// Found combining -> check for 1byte in hi
if (ch == 0)
{
return ~1;
}
if ((ch & 0xFF00) == 0)
{
// Found pair
return 2;
}
}
// Did not find pair, preserve combining character.
return 1;
}
if (ch <= 0xDBFF)
{
uint chLow = *(c + 1);
// foundFound high surrogate -> check surrogate pair
if (chLow == 0) // Null terminator, we do not have another char
{
// lastFast char is high surrogate, so it is missing its pair
return ~1;
}
if (!(chLow >= 0xDC00 && chLow <= 0xDFFF))
{
// didDid not foundfind a low surrogate after the high surrogate
return ~1;
}
// convertConvert to UTF32 - like in Char.ConvertToUtf32(highSurrogate, lowSurrogate)
ch = ((ch - 0xD800) *<< 0x40010) + (chLow - 0xDC00) + 0x10000;
if ((ch & 0xFFFE) == 0xFFFE)
{
// otherOther non-char found
return ~2;
}
// foundFound a good surrogate pair
return 2;
}
if (ch <= 0xDFFF)
{
// unexpectedUnexpected low surrogate
return ~1;
}
if (ch >= 0xFDD0 && ch <= 0xFDEF)
{
// nonNon-chars are considered invalid by System.Text.Encoding.GetBytes() and String.Normalize()
return ~1;
}
if ((ch & 0xFFFE) == 0xFFFE)
{
// otherOther non-char found
return ~1;
}
return 1;
}
/// <summary>
/// !Warning: Does not validate the symbol
/// !Warning: Assumes that the pointer does not point to the null terminator.
/// Returns the number of 2byte characters that compose the next symbol.
/// </summary>
[Pure]
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static unsafe int UnicodeSymbolCharLength(char* c)
{
// We can always check c and c + 1 (four bytes total) because of the null terminator.
booluint isTwoCharsw32C = (*((uint*)c)c;
// Check whether we have a utf32 character
bool isTwoChars = (w32C & 0xFC00FC00) == 0xDC00D800;
// Check whether we have a 1byte character in lo and a combining 2byte character in hi
isTwoChars |= ((w32C >> 16) > 0x02F0) & ((w32C >> 16) < 0x0370);
return 1 + *(byte*)(&isTwoChars); // Casting bool to byte is 0 when false, 1 when true;
}