I wanted to be able to tokenize a few different containers so I created a generic way to do it. I originally wrote this in Visual Studio 2012, but I had to modify it to get it to compile on Ideone.
Here is a brief description of my tokenizer functions:
Tokenize ()
: Tokenizes a container based on a single delimiter.
TokenizeIf ()
: Tokenizes a container based on a single delimiter and a condition.
BackInsertTokenize ()
and BackInsertTokenizeIf ()
: Wrapper for containers that can use a std::back_inserter
iterator.
A specialized BackInsertTokenize ()
for character types (char
, wchar
, etc).
Here are my generic tokenizer functions:
#include <algorithm>
#include <iterator>
#include <string>
#include <utility>
template <typename Token, typename Iter, typename OutIter, typename Condition>
auto TokenizeIf (Iter begin, Iter end, OutIter out, typename std::iterator_traits <Iter>::value_type delimiter, Condition condition) -> OutIter
{
if (begin == end) {
return out ;
}
auto current = begin ;
auto next = begin ;
do {
next = std::find (current, end, delimiter) ;
Token token (current, next) ;
if (condition (token) == true) {
*out++ = std::move (token) ;
}
current = next ;
} while (next != end && ++current != end) ;
if (next != end) {
Token token ;
if (condition (token) == true) {
*out++ = std::move (token) ;
}
}
return out ;
};
template <typename Token, typename Iter, typename OutIter>
auto Tokenize (Iter begin, Iter end, OutIter out, typename std::iterator_traits <Iter>::value_type delimiter) -> OutIter
{
if (begin == end) {
return out ;
}
auto current = begin ;
auto next = begin ;
do {
next = std::find (current, end, delimiter) ;
*out++ = Token (current, next) ;
current = next ;
} while (next != end && ++current != end) ;
if (next != end) {
*out++ = Token () ;
}
return out ;
};
template <class ContainerOut, class ContainerIn, class Condition>
auto BackInsertTokenizeIf (ContainerIn const &in, typename ContainerIn::value_type delimiter, Condition condition) -> ContainerOut
{
typedef typename ContainerOut::value_type Token ;
ContainerOut out ;
TokenizeIf <Token> (std::begin (in), std::end (in), std::back_inserter (out), delimiter, condition) ;
return out ;
}
template <class ContainerOut, class ContainerIn>
auto BackInsertTokenize (ContainerIn const &in, typename ContainerIn::value_type delimiter) -> ContainerOut
{
typedef typename ContainerOut::value_type Token ;
ContainerOut out ;
Tokenize <Token> (std::begin (in), std::end (in), std::back_inserter (out), delimiter) ;
return out ;
}
template <class ContainerOut, class CharT>
auto BackInsertTokenize (const CharT *in, CharT delimiter) -> ContainerOut
{
typedef typename ContainerOut::value_type Token ;
ContainerOut out ;
Tokenize <Token> (in, in + std::char_traits<CharT>::length (in), std::back_inserter (out), delimiter) ;
return out ;
}
These are the test cases that I was interested in:
#include <vector>
int main ()
{
std::string const s1 = ",hello,5,cat,192.3," ;
auto const t1 = BackInsertTokenize <std::vector <std::string>> (s1, ',') ;
std::vector <char> v1 = {'S', '1', '\0', 'S', '2', 'I', 'N', '\0', 'S', '3', '\0', '\0'} ;
auto const t2 = BackInsertTokenizeIf <std::vector <std::string>> (v1, '\0', [] (const std::string &s) {
return !s.empty () ;
}) ;
auto const t3 = BackInsertTokenize <std::vector <std::string>> ("C:\\Some\\Path\\To\\Nowhere", '\\') ;
return 0 ;
}