StringTokenizer for .NET Core

Question

Splitting a string in tokens is a more complex topic than String.Split() wants to make us believe. There are at least three common policies according to which a string might be interpreted and split in tokens.

Policy 1: Equivalent to String.Split()

There is not much to mention about this policy. Given a string s and a delimiter d, break s into segments delimited by d. The main drawback here is that if the delimiter is part of at least one of the tokens, reconstructing the desired tokens might be costly.

Policy 2: Escape special characters

A character is declared as the escape character e (commonly the backslash \) resulting in the character following it losing its special meaning. A token string then might look like this:

token_1 token_2 very\ long \ token

which would be equivalent to

{ "token_1", "token_2", "very long token" }

Policy 3: Place tokens in quotation marks

This approach is for example used in CSV files generated in MSExcel. Everything between quotation marks is considered as a token. If quotation marks " are part of the token, they are doubled "". A token string then might look like this:

token_1,token_2,"token2,5"

which would be equivalent to

{ "token_1", "token_2", "token2,5" }

Code

using System;
using System.Text;
using System.Text.RegularExpressions;
using System.Collections.Generic;

namespace Pillepalle1.ConsoleTelegramBot.Model.Misc
{
    public sealed class StringTokenizer
    {
        private string _sourceString = null;                            // Provided data to split

        #region Constructors
        /// <summary>
        /// Creates a new StringTokenizer
        /// </summary>
        /// <param name="dataInput">Data to be split into tokens</param>
        public StringTokenizer(string dataInput)
        {
            _sourceString = dataInput ?? string.Empty;
        }
        #endregion

        #region Interface
        /// <summary>
        /// Access tokens by index
        /// </summary>
        public string this[int index]
        {
            get
            {
                if (index >= this.Count)
                {
                    return String.Empty;
                }

                return _Tokens[index];
            }
        }

        /// <summary>
        /// How many tokens does the command consist of
        /// </summary>
        public int Count
        {
            get
            {
                return _Tokens.Count;
            }
        }

        /// <summary>
        /// Which strategy is used to split the string into tokens
        /// </summary>
        public StringTokenizerStrategy Strategy
        {
            get
            {
                return _strategy;
            }
            set
            {
                if (value != _strategy)
                {
                    _strategy = value;
                    _tokens = null;
                }
            }
        }
        private StringTokenizerStrategy _strategy = StringTokenizerStrategy.Split;

        /// <summary>
        /// Character used to delimit tokens
        /// </summary>
        public char Delimiter
        {
            get
            {
                return _delimiter;
            }
            set
            {
                if (value != _delimiter)
                {
                    _delimiter = value;
                    _tokens = null;
                }
            }
        }
        private char _delimiter = ' ';

        /// <summary>
        /// Character used to escape the following character
        /// </summary>
        public char Escape
        {
            get
            {
                return _escape;
            }
            set
            {
                if (value != _escape)
                {
                    _escape = value;

                    if (Strategy == StringTokenizerStrategy.Escaping)
                    {
                        _tokens = null;
                    }
                }
            }
        }
        private char _escape = '\\';

        /// <summary>
        /// Character used to surround tokens
        /// </summary>
        public char Quotes
        {
            get
            {
                return _quotes;
            }
            set
            {
                if (value != _quotes)
                {
                    _quotes = value;

                    if (Strategy == StringTokenizerStrategy.Quotation)
                    {
                        _tokens = null;
                    }
                }
            }
        }
        private char _quotes = '"';
        #endregion

        #region Predefined Regex
        private Regex Whitespaces
        {
            get
            {
                return new Regex("\\s+");
            }
        }
        #endregion

        #region Implementation Details
        /// <summary>
        /// Formats and splits the tokens by delimiter allowing to add delimiters by quoting
        /// </summary>
        private List<string> _SplitRespectingQuotation()
        {
            string data = _sourceString;

            // Doing some basic transformations
            data = Whitespaces.Replace(data, " ");

            // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 
            // Initialisation
            List<string> l = new List<string>();
            char[] record = data.ToCharArray();

            StringBuilder property = new StringBuilder();
            char c;

            bool quoting = false;

            // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 
            // Scan character by character
            for (int i = 0; i < record.Length; i++)
            {
                c = record[i];

                // Quotation-Character: Single -> Quote; Double -> Append
                if (c == Quotes)
                {
                    if (i == record.Length - 1)
                    {
                        quoting = !quoting;
                    }
                    else if (Quotes == record[1 + i])
                    {
                        property.Append(c);
                        i++;
                    }
                    else
                    {
                        quoting = !quoting;
                    }
                }

                // Delimiter: Escaping -> Append; Otherwise append
                else if (c == Delimiter)
                {
                    if (quoting)
                    {
                        property.Append(c);
                    }
                    else
                    {
                        l.Add(property.ToString());
                        property.Clear();
                    }
                }

                // Any other character: Append
                else
                {
                    property.Append(c);
                }
            }

            l.Add(property.ToString());                         // Add last token

            // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 
            // Checking consistency
            if (quoting) throw new FormatException();          // All open quotation marks closed

            return l;
        }

        /// <summary>
        /// Splits the string by declaring one character as escape
        /// </summary>
        private List<string> _SplitRespectingEscapes()
        {
            string data = _sourceString;

            // Doing some basic transformations
            data = Whitespaces.Replace(data, " ");

            // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 
            // Initialisation
            List<string> l = new List<string>();
            char[] record = data.ToCharArray();

            StringBuilder property = new StringBuilder();
            char c;

            bool escaping = false;

            // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 
            // Scan character by character
            for (int i = 0; i < record.Length; i++)
            {
                c = record[i];

                if (escaping)
                {
                    property.Append(c);
                    escaping = false;
                    continue;
                }

                if (c == Escape)
                {
                    escaping = true;
                }
                else if (c == Delimiter)
                {
                    l.Add(property.ToString());
                    property.Clear();
                }
                else
                {
                    property.Append(c);
                }
            }

            return l;
        }

        /// <summary>
        /// Splits the string by calling a simple String.Split
        /// </summary>
        private List<string> _SplitPlain()
        {
            return new List<string>(Whitespaces.Replace(_sourceString, " ").Split(Delimiter));
        }

        /// <summary>
        /// Backer for tokens
        /// </summary>
        private List<string> _Tokens
        {
            get
            {
                if (null == _tokens)
                {
                    switch (Strategy)
                    {
                        case (StringTokenizerStrategy.Quotation): _tokens = _SplitRespectingQuotation(); break;
                        case (StringTokenizerStrategy.Escaping): _tokens = _SplitRespectingEscapes(); break;

                        default: _tokens = _SplitPlain(); break;
                    }
                }

                return _tokens;
            }
        }
        private List<string> _tokens = null;
        #endregion
    }

    public enum StringTokenizerStrategy
    {
        Split,
        Quotation,
        Escaping
    }
}

Answer 1

I'm not so sure this belongs in a class - at least not a single one!

Take a step back and look at what unites and what separates each "strategy". They all need to transform an input string into a list of tokens based on a variable delimiter. However, there are properties that are only used by one of the three options, and the majority of the splitting logic is unique to its strategy.

Suggestion 1: Three "standalone" functions.

You'd really have to put them into a static class or do something special with delegates/lambdas, but ultimately there's little to gain from having one big class.

  public static IList<string> SplitRespectingQuotation(string sourceString, char delimiter = ' ', char quote = '"') { ... }
  public static IList<string> SplitRespectingEscapes(string sourceString, char delimiter = ' ', char escape = '\') { ... }
  public static IList<string> SplitPlain(string sourceString, char delimiter = ' ') { ... }

If you'd like the output to communicate the input parameters, you can make a much more lightweight class that does so. Its properties would be readonly; if you need to change them and re-compute, just call the function again. After all, that's essentially what you're doing on the inside of your current class!

Another plus: if and when you come up with a new strategy for splitting, you can just create a new function without affecting the others. They're all independently testable, editable, and delete-able.

Suggestion 2: Three concrete classes that extend an abstract base class.

I do like what you did with the _Tokens property: it allows you to defer the computation until you really need it, which is helpful in cases that you won't. Also, one use case it supports (that isn't supported by "standalone" functions) is to change e.g. the escape character and have the result automatically "invalidated".

To maintain that behavior, you could pull the common elements into an abstract base class, like the following:

public abstract class StringTokenizer
{
  public string SourceString { get; }

  public StringTokenizer(string dataInput)
  {
    SourceString = dataInput ?? string.Empty;
  }

  public string this[int index] => index >= this.Count ? String.Empty : Tokens[index];

  public int Count => Tokens.Count;

  public char Delimiter
  {
    get { return _delimiter; }
    set
    {
      if (value != _delimiter)
      {
         _delimiter = value;
         InvalidateResult();
      }
    }
  }
  private char _delimiter = ' ';

  public IEnumerable<string> Tokens
  {
    get
    {
      if (_tokens is null)
      {
        _tokens = ComputeTokens();
      }
      return _tokens;
    }
  }
  private List<string> _tokens = null;

  protected abstract List<string> ComputeTokens();

  protected void InvalidateResult()
  {
    _tokens = null;
  }
}

Notable changes:

1. The actual split logic is absent. Each strategy will provide its own.
2. Strategy-specific properties are absent. There's no need for an escape-based strategy to have a property for a quote character, and vice-versa.
3. Instead of directly setting _tokens = null, properties should call InvalidateResult. This allows _tokens to be made private which keeps the logic contained to the base class.
4. Tokens is public, and is an IEnumerable. This allows consumers to use foreach, but discourages direct modification.

A base class now has exactly one job: implement ComputeTokens. If it needs to create properties to do so, it may do so, based on its own, strategy-specific logic. If those properties need to invalidate previously-computed tokens when they change, they may call InvalidateResult.

Here's a rough example of what a strategy sub class would look like:

public sealed class EscapeStringTokenizer : StringTokenizer
{
  public EscapeStringTokenizer (string dataInput) : base(dataInput) { }

  public char Escape
  {
    get { return _escape; }
    set
    {
      if (value != _escape)
      {
         _escape = value;
         InvalidateResult();
      }
    }
  }

  protected override List<string> ComputeTokens()
  {
    // Actual logic omitted
  }
}

Other Observations

1. You allow the delimiter to be specified, but you always condense whitespace. If I split "a,a and b,b" with a delimiter of ",", I would expect to get {"a", "a and b", "b"} back - but would actually get {"a", "a and b", "b"}.
2. If the delimiter, etc., can be publicly read, why not expose the source string as well? See SourceString my abstract class above.
3. I find the (relatively new) expression-bodied property accessors to be all-around better for simple properties. See Count in my abstract class above.
4. I don't think it's possible to accidentally assign null to a variable as the condition of an if statement. This is because x = null evaluates to be the same type as x, which needs to be a bool (and thus, not nullable) in order to be a valid condition. If you'd still like to avoid x == null, you can say x is null.
5. As mentioned by others, you shouldn't prefix properties with _. It's not there to differentiate between public and private, but between local variables and class fields. Personally, though, I don't even use _ in that case, but instead prefer this. if needed. But overall you will need to be flexible about that and make sure to follow any pattern that's already established in an existing team or project.
6. Also as others have mentioned, use var when declaring variables whenever possible. Any good IDE will be able to tell you the type when you hover over the variable, and its name should tell you what it's for even without the type.
7. On that note, avoid names like c and l. i is fine because it's idiomatic as a loop/index variable, but the others require extra context to understand. Source code characters are cheap, so pay for some extra readability by using currentChar and finishedTokens.
8. You don't need to translate the source string into a char[]; you can already access characters in a string by index.

Answer 2

• You shouldn't have Whitespaces as a get-only property but as a private readonly field and you should have that regex compiled because you are using it quite often.

• Using region is considered an antypattern

• Use underscore-prefixing only for private fields. Don't use them for methods or properties.

• If the type of a variable is clear from the right hand side of an assignment you should use var instead of the concrete type.

• The code is doing a lot althought _sourceString may be string.Empty because the passed ctor-argument dataInput may be null or string.Empty. I would prefer to throw an exception in the ctor.

• Instead of assigning a variable to another and then manipulate the resulting variable you could just do it on one line like e.g

  string data = Whitespaces.Replace(_sourceString, " ");  
  

  instead of

  string data = _sourceString;
  
  // Doing some basic transformations
  data = Whitespaces.Replace(data, " ");  
  

• If you only need to access single items of an array and don't need to look ahead, you should prefer a foreach over a for loop.

Answer 3

• A single-letter l name seems bad to me.

• I think you should add a message to the exception that describes the reason for the error.

• By default, you remove all whitespaces from the data. But they may be needed inside tokens. You can make an additional option to specify this.

Answer 4

Thanks to everybody for the great feedback. I have adopted most of the changes to my code which is being hosted as FOS on https://github.com/pillepalle1/dotnet-pillepalle1 where it will receive further maintenance.

For now, I have packed the splitting logic into three static extension methods. Additionally I have built wrappers as suggested by therubberduck to optionally keep the comfort of automatic token invalidation

Suggestions I have implemented

• Variable naming Variable names such as l have been replaced by more descriptive names

• Exception messages Have been added

• Modification of token content Has been removed completely from the extension methods and made optionally available in the wrappers

• Regions have been completely removed

• Using var whenever reasonable/ possible

• Loops Preferring foreach over for loops and iterating over the sourceString instead of converting it to char[] first

• Inputstring Throwing ArgumentNullException instead of converting null to String.Empty

• CSV Splitting according to RFC4180

I would have adopted more changes but some suggestions (ie regarding Whitespaces and expression bodied properties) have become obsolete in the new implementation.

Suggestions I have not implemented

• The underscore naming for everything private/ protected seems more reasonable to me than just distinguishing between member- and local variables since when implementing concurrency robust data structures (which became a big thing since Tasks were implemented) it is incredibly valuable to see at first glance whether a method performs concurrency checks (public) or not (private).

Code

Static Tokenizer Methods

using System;
using System.Text;
using System.Collections.Immutable;

namespace pillepalle1.Text
{
    public static class StringTokenizer
    {
        private static FormatException _nonQuotedTokenMayNotContainQuotes =
            new FormatException("[RFC4180] If fields are not enclosed with double quotes, then double quotes may not appear inside the fields.");

        private static FormatException _quotesMustBeEscapedException =
            new FormatException("[RFC4180] If double-quotes are used to enclose fields, then a double-quote appearing inside a field must be escaped by preceding it with another double quote.");

        private static FormatException _tokenNotFullyEnclosed =
            new FormatException("[RFC4180] \"Each field may or may not be enclosed in double quotes\". However, for the final field the closing quotes are missing.");


        /// <summary>
        /// <para>
        /// Formats and splits the tokens by delimiter allowing to add delimiters by quoting 
        /// similar to https://tools.ietf.org/html/rfc4180
        /// </para>
        /// 
        /// <para>
        /// Each field may or may not be enclosed in double quotes (however some programs, such as 
        /// Microsoft Excel, do not use double quotes at all). If fields are not enclosed with 
        /// double quotes, then double quotes may not appear inside the fields.
        /// </para>
        /// 
        /// <para>
        /// Fields containing line breaks (CRLF), double quotes, and commas should be enclosed in 
        /// double-quotes.
        /// </para>
        /// 
        /// <para>
        /// If double-quotes are used to enclose fields, then a double-quote appearing inside a 
        /// field must be escaped by preceding it with another double quote.
        /// </para>
        /// 
        /// <para>
        /// The ABNF defines 
        /// 
        /// [field = (escaped / non-escaped)] ||  
        /// [non-escaped = *TEXTDATA] || 
        /// [TEXTDATA =  %x20-21 / %x23-2B / %x2D-7E]
        /// 
        /// specifically forbidding to include quotes in non-escaped fields, hardening the *SHOULD*
        /// requirement above.
        /// </para>
        /// </summary>
        public static ImmutableList<string> SplitRespectingQuotation(this string sourceString, char delimiter = ' ', char quotes = '"')
        {
            // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 
            // Initialisation
            var tokenList = ImmutableList<string>.Empty;
            var tokenBuilder = new StringBuilder();

            var expectingDelimiterOrQuotes = false;     // Next char must be Delimiter or Quotes
            var hasReadTokenChar = false;               // We are not between tokens (=> No quoting)
            var isQuoting = false;

            // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 
            // Scan character by character
            foreach (char c in sourceString)
            {
                if (expectingDelimiterOrQuotes)
                {
                    expectingDelimiterOrQuotes = false;

                    if (c == delimiter)
                    {
                        isQuoting = false;
                    }

                    else if (c == quotes)
                    {
                        tokenBuilder.Append(c);
                        hasReadTokenChar = true;
                        continue;
                    }

                    else
                    {
                        throw _quotesMustBeEscapedException;
                    }
                }

                // -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --

                if (c == quotes)
                {
                    if (isQuoting)
                    {
                        expectingDelimiterOrQuotes = true;
                    }

                    else
                    {
                        if (hasReadTokenChar)
                        {
                            throw _nonQuotedTokenMayNotContainQuotes;
                        }

                        isQuoting = true;
                    }
                }

                else if (c == delimiter)
                {
                    if (isQuoting)
                    {
                        tokenBuilder.Append(c);
                        hasReadTokenChar = true;
                    }
                    else
                    {
                        tokenList = tokenList.Add(tokenBuilder.ToString());
                        tokenBuilder.Clear();
                        hasReadTokenChar = false;
                    }
                }

                // Any other character is just being appended to
                else
                {
                    tokenBuilder.Append(c);
                    hasReadTokenChar = true;
                }
            }

            // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 
            // Tidy up open flags and checking consistency

            tokenList = tokenList.Add(tokenBuilder.ToString());

            if (isQuoting && !expectingDelimiterOrQuotes)
            {
                throw _tokenNotFullyEnclosed;
            }

            return tokenList;
        }

        /// <summary>
        /// Splits the string by declaring one character as escape
        /// </summary>
        public static ImmutableList<string> SplitRespectingEscapes(this string sourceString, char delimiter = ' ', char escapeChar = '\\')
        {
            // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 
            // Initialisation
            var tokenList = ImmutableList<string>.Empty;
            var tokenBuilder = new StringBuilder();

            var escapeNext = false;

            // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 
            // Scan character by character
            foreach (char c in sourceString)
            {
                if (escapeNext)
                {
                    tokenBuilder.Append(c);
                    escapeNext = false;
                    continue;
                }

                if (c == escapeChar)
                {
                    escapeNext = true;
                }
                else if (c == delimiter)
                {
                    tokenList = tokenList.Add(tokenBuilder.ToString());
                    tokenBuilder.Clear();
                }
                else
                {
                    tokenBuilder.Append(c);
                }
            }

            // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 
            // Tidy up open flags and checking consistency
            tokenList = tokenList.Add(tokenBuilder.ToString());

            if (escapeNext) throw new FormatException();            // Expecting additional char


            return tokenList;
        }

        /// <summary>
        /// Splits the string by calling a simple String.Split
        /// </summary>
        public static ImmutableList<string> SplitPlain(this string sourceString, char delimiter = ' ')
        {
            return ImmutableList<string>.Empty.AddRange(sourceString.Split(delimiter));
        }
    }
}

Abstract wrapper base class

using System;
using System.Collections.Immutable;

namespace pillepalle1.Text
{
    public abstract class AStringTokenizer
    {
        public AStringTokenizer()
        {

        }

        public AStringTokenizer(string sourceString)
        {
            SourceString = sourceString;
        }

        /// <summary>
        /// String that is supposed to be split in tokens
        /// </summary>
        public string SourceString
        {
            get
            {
                return _sourceString;
            }
            set
            {
                if (null == value)
                {
                    throw new ArgumentNullException("Cannot split null in tokens");
                }
                else if (_sourceString.Equals(value))
                {
                    // nop
                }
                else
                {
                    _sourceString = value;
                    _InvalidateTokens();
                }
            }
        }
        private string _sourceString = String.Empty;

        /// <summary>
        /// Character indicating how the source string is supposed to be split
        /// </summary>
        public char Delimiter
        {
            get
            {
                return _delimiter;
            }
            set
            {
                if (value != _delimiter)
                {
                    _delimiter = value;
                    _InvalidateTokens();
                }
            }
        }
        private char _delimiter = ' ';

        /// <summary>
        /// Flag indicating whether whitespaces should be removed from start and end of each token
        /// </summary>
        public bool TrimTokens
        {
            get
            {
                return _trimTokens;
            }
            set
            {
                if (value != _trimTokens)
                {
                    _trimTokens = value;
                    _InvalidateTokens();
                }
            }
        }
        private bool _trimTokens = false;

        /// <summary>
        /// Result of tokenization
        /// </summary>
        public ImmutableList<string> Tokens
        {
            get
            {
                if (null == _tokens)
                {
                    _tokens = Tokenize();

                    if (TrimTokens)
                    {
                        _tokens = _TrimTokens(_tokens);
                    }
                }

                return _tokens;
            }
        }
        private ImmutableList<string> _tokens = null;

        /// <summary>
        /// Split SourceString into tokens
        /// </summary>
        protected abstract ImmutableList<string> Tokenize();

        /// <summary>
        /// Trims whitespaces from tokens
        /// </summary>
        /// <param name="candidates">List of tokens</param>
        private ImmutableList<string> _TrimTokens(ImmutableList<string> candidates)
        {
            var trimmedTokens = ImmutableList<string>.Empty;

            foreach (var token in candidates)
            {
                trimmedTokens = trimmedTokens.Add(token.Trim());
            }

            return trimmedTokens;
        }

        /// <summary>
        /// Invalidate and recompute tokens if necessary
        /// </summary>
        protected void _InvalidateTokens()
        {
            _tokens = null;
        }
    }
}

Wrapper for plain tokenization

using System.Collections.Immutable;

namespace pillepalle1.Text
{
    public class PlainStringTokenizer : AStringTokenizer
    {
        protected override ImmutableList<string> Tokenize()
        {
            return SourceString.SplitPlain(Delimiter);
        }
    }
}

Wrapper for quotation tokenization

using System.Collections.Immutable;

namespace pillepalle1.Text
{
    public class QuotationStringTokenizer : AStringTokenizer
    {
        /// <summary>
        /// Indicates which character is used to encapsulate tokens
        /// </summary>
        public char Quotes
        {
            get
            {
                return _quotes;
            }
            set
            {
                if (value != _quotes)
                {
                    _quotes = value;
                    _InvalidateTokens();
                }
            }
        }
        private char _quotes = '"';

        protected override ImmutableList<string> Tokenize()
        {
            return SourceString.SplitRespectingQuotation(Delimiter, Quotes);
        }
    }
}

Wrapper for escape tokenization

using System.Collections.Immutable;

namespace pillepalle1.Text
{
    public class EscapedStringTokenizer : AStringTokenizer
    {
        /// <summary>
        /// Indicates which character is used to escape characters
        /// </summary>
        public char Escape
        {
            get
            {
                return _escape;
            }
            set
            {
                if (value != _escape)
                {
                    _escape = value;
                    _InvalidateTokens();
                }
            }
        }
        private char _escape = '"';

        protected override ImmutableList<string> Tokenize()
        {
            return SourceString.SplitRespectingEscapes(Delimiter, Escape);
        }
    }
}