5
\$\begingroup\$

I am a hobbyist computer programmer trying to learn modern C++ (in this case C++17). I thought it might be an interesting challenge to write a tokenizer class for breaking down microsoft (and other) .CSV files into tokens. My tokens are simply a std::variant<int, double, std::string>, and I wrote an enum class to facilitate easy code reading. I have chosen to make functions camelCase, and variables snake_case.

I do have some questions:

  • Is my constructor "kosher"? That is, I know that it can throw an exception.
  • I also know that it could be rewritten as a bunch of overloads, would that be better?
  • Is my dependence on overloading the << and >> operators along with specialized functions considered good style or should I choose one paradigm and force the user to adapt?
  • Are there other stylistic or algorithmic problems throughout this code?
#include <iostream>
#include <iterator>
#include <algorithm>
#include <sstream>
#include <vector>
#include <deque>
#include <string>
#include <string_view>
#include <fstream>
#include <variant>
#include <functional>
#include <optional>

/*
123456789 123456789 123456789 123456789 123456789 123456789 123456789 1234567890
*/
using std::cout;
using namespace std::string_literals;

using token = std::variant<int, double, std::string>;

enum class token_type : std::size_t  {
    INT,
    DOUBLE,
    STRING
};

std::ostream& operator<<(std::ostream& out, const token& tok)
{
    std::visit([&out](const auto& content) {out << content; }, tok);
    return out;
}

class tokenizer
{
public:
    tokenizer(const char* delim = ",;:|\t ", const char* escape = "\"\'",
        bool unquote = true, bool remove_blanks = false, bool convert = true) :
        delim_{ std::string(escape).append(delim) }, escape_{ escape },
        unquote_{ unquote }, remove_blanks_{ remove_blanks }, 
        convert_{ convert }  {};
    tokenizer& operator<<(std::string_view s);
    tokenizer& operator<<(const char* line);
    tokenizer& operator>>(token& tok);
    tokenizer& operator>>(std::optional<token>& otok);
    void operator>>(std::vector<token>& toks);
    std::optional<token> getNext();
    std::vector<token> getVector();
    void clear() { toks_.clear(); }

private:
    std::vector<std::string> splitString(std::string_view s);
    void convertString(std::string s);
    std::deque<token> toks_;
    const std::string delim_{ };
    const std::string_view escape_{ };
    bool unquote_{ true };
    bool remove_blanks_{ false };
    bool convert_{ true };
};

tokenizer& tokenizer::operator<<(std::string_view s)
{
    if (s.length() == 0) {
        if (!remove_blanks_) toks_.emplace_back("");
        return *this;
    }
    std::vector<std::string> temp_vec = splitString(s);
    for (auto& value : temp_vec) {
        auto start = 0;
        auto count = value.length();
        if (remove_blanks_ && (count == 0 || (count == 2 && unquote_ &&
            escape_.find(value[0]) != escape_.npos))) {
            continue;
        }
        if (count == 0) {
            if (unquote_) toks_.emplace_back(value);  //  token_type::STRING
            else toks_.emplace_back(""s + *(escape_.cbegin()) + *(escape_.cbegin()));
            continue;
        }
        auto q_index = std::find(escape_.begin(), escape_.end(), value[0]);
        if (q_index != escape_.end() && value[0] == *q_index) {
            if (!unquote_) {
                toks_.emplace_back(value);  //  token_type::STRING
                continue;
            }
            ++start;
            count -= 2;
            std::string double_quote{ ""s + *q_index + *q_index };
            auto pos = value.find(double_quote, 1);  // "\"\""
            while (pos != std::string::npos && pos != value.length() - 2) {
                --count;
                value.erase(pos, 1);
                pos = value.find(double_quote, 1);  // "\"\""
            }
            value = value.substr(start, count);
        }

        if (convert_) convertString(value);
        else toks_.emplace_back(value);  //  token_type::STRING
    }
    return *this;
}

tokenizer& tokenizer::operator<<(const char* line)
{
    return operator<<(std::string_view(line));
}

tokenizer& tokenizer::operator>>(token& tok)
{
    tok = std::move(toks_.front());
    toks_.pop_front();
    return *this;
}

tokenizer& tokenizer::operator>>(std::optional<token>& otok)
{
    if (toks_.empty()) {
        otok = std::nullopt;
        return *this;
    }
    otok = std::move(toks_.front());
    toks_.pop_front();
    return *this;
}

void tokenizer::operator>>(std::vector<token>& toks)
{
    toks.reserve(toks.size()+toks_.size());
    while (!toks_.empty()) {
        toks.push_back(std::move(toks_.front()));
        toks_.pop_front();
    }
    toks_.clear();
}

std::optional<token> tokenizer::getNext()
{
    if (toks_.empty()) {
        return std::nullopt;
    }
    token ret(std::move(toks_.front()));
    toks_.pop_front();
    return ret;
}

std::vector<token> tokenizer::getVector()
{
    std::vector<token> ret;
    ret.reserve(toks_.size());
    while (!toks_.empty()) {
        ret.push_back(toks_.front());
        toks_.pop_front();
    }
    toks_.clear();
    return ret;
}

std::vector<std::string> tokenizer::splitString(std::string_view s)
{
    std::vector<std::string> ret;
    auto start = s.begin();
    while (*start == ' ') ++start;
    auto pos = std::find_first_of(start, s.end(), delim_.cbegin(),
        delim_.cend());
    while (pos != s.end()) {
        auto q_index = std::find(escape_.begin(), escape_.end(), *pos);
        if (q_index != escape_.end()) {
            pos = std::find(pos + 1, s.end(), *q_index);
            if ((pos + 1) != s.end() && *(pos + 1) == *q_index) {
                ++pos;
                continue;
            }
            pos = std::find_first_of(pos + 1, s.end(), delim_.cbegin(),
                delim_.cend());
        }
        ret.emplace_back(s, start - s.begin(), pos - start);
        if (pos == s.end()) {
            start = pos;
            break;
        }
        start = pos + 1;
        while (*start == ' ') ++start;
        pos = std::find_first_of(start, s.end(), delim_.begin(),
            delim_.end());
    }
    if (start != s.end()) ret.emplace_back(s, start - s.begin(),
        s.end() - start);
    return ret;
}

void tokenizer::convertString(std::string s)
{
    if (s.length() == 0) {
        toks_.emplace_back(s);
        return;
    }
    std::uint8_t offset = 0;
    if (s[0] == '-' || s[0] == '+') ++offset;
    auto iter = std::find_if(s.begin()+offset, s.end(),
        [](unsigned char c) noexcept -> bool {
            return !(isdigit(c) || c == '.');
        });
    if (iter != s.end()) {
        toks_.emplace_back(s);  // , token_type::STRING
        return;
    }
    iter = std::find_if(s.begin() + offset, s.end(), [](unsigned char c)
        {
            return c == '.';
        });
    std::stringstream iss{ s };
    if (iter != s.end()) {
        double dvalue;
        iss >> dvalue;
        toks_.emplace_back(dvalue);  // , token_type::DOUBLE
        return;
    }
    int ivalue;
    iss >> ivalue;
    toks_.emplace_back(ivalue);  // , token_type::INT
}

int main()
{
    constexpr auto typer = [](std::size_t index) -> const char*
    {
        switch (static_cast<token_type>(index)) {
        case token_type::STRING:
            return "string";
        case token_type::DOUBLE:
            return "double";
        case token_type::INT:
        default:
            return "int";
        }
    };
    tokenizer token_maker (",", "\""); // , true, false, true
    token_maker << "";
    token_maker << "\"help, this!\",32";
    token_maker << "gasoline" << "";
    std::vector<token> tokens{ token_maker.getVector() };
    for (auto t : tokens) cout << t << " : " << typer(t.index()) << "\n";
    cout << "\n";
    std::string test_string;
    std::istringstream iss("1,\"\"\"2,3\"\", a\",4,-1.1,,\"\", 5,\"asdf\"");
    std::getline(iss, test_string);
    cout << test_string << " unquote, !remove_blanks, convert\n";
    tokens.clear();
    token_maker << test_string >> tokens;
    for (auto t : tokens) cout << t << " : " <<
        typer(t.index()) << "\n"; //  t.index()
    cout << "\n";
    cout << test_string << " !unquote, !remove_blanks, convert\n";
    std::optional<token> next_tok;
    tokenizer token_maker2(",", "\"", false);
    token_maker2 << test_string;
    token_maker2 >> next_tok;
    while (next_tok) {
        cout << *next_tok << " : " << typer(next_tok->index()) << "\n";
        token_maker2 >> next_tok;
    }
    cout << "\n";
    cout << test_string << " !unquote, remove_blanks, convert\n";
    tokenizer token_maker3(",", "\"", false, true);
    token_maker3 << test_string;
    while (auto next_tok = token_maker3.getNext()) cout << *next_tok << " : " <<
        typer(next_tok->index()) << "\n";
    cout << "\n";
    cout << test_string << " !unquote, remove_blanks, !convert\n";
    tokenizer token_maker4(",", "\"", false, true, false);
    tokens.clear();
    token_maker4 << test_string >>tokens;
    for (auto t : tokens) cout << t << " : " << typer(t.index()) << "\n";
    cout << "\n";
    cout << test_string << " unquote, remove_blanks, convert\n";
    tokenizer token_maker5(",", "\"", true, true, true);
    tokens.clear();
    token_maker5 << test_string >> tokens;
    for (auto t : tokens) cout << t << " : " << typer(t.index()) << "\n";
    cout << "\nBreaking down typical .csv file: delimited by ,using \"";
    cout << " as escape characters, and unquote, !remove_blanks,";
    cout << "converting to type\n\n";
    std::vector<token> headings;
    headings.reserve(15);
    std::ifstream test_data("ExportedGridData.csv");
    std::getline(test_data, test_string);
    token_maker << test_string >> headings;
    int i=0, category_column=-1;
    for (auto t : headings) {
        std::string s = *std::get_if<std::string>(&t);
        if (s.find("Categories", 0)!=s.npos) category_column = i;
        ++i;
    }
    const auto last_column = headings.size() - 1;
    if (last_column != category_column) {
        headings.erase(headings.begin() + category_column);
        headings.emplace_back("Categories"s);
    }
    for (auto t : headings) cout << t << " ";
    cout << "\n";
    while (std::getline(test_data, test_string)) {
        token_maker << test_string;
        std::vector<token> data;
        token_maker >> data;
        token_maker << std::get<std::string>(data.at(category_column)) >> data;
        data.erase(data.begin()+category_column);
        std::sort(data.begin() + last_column, data.end(), 
            [](const token & a, const token & b) -> bool {
                return a < b;
            });
        for (auto s : data) cout << s << " ";
        cout << "\n";
    }
}

Here is the output:

 : string
help, this! : string
32 : int
gasoline : string
 : string

1,"""2,3"", a",4,-1.1,,"", 5,"asdf" unquote, !remove_blanks, convert
1 : int
"2,3", a : string
4 : int
-1.1 : double
 : string
 : string
5 : int
asdf : string

1,"""2,3"", a",4,-1.1,,"", 5,"asdf" !unquote, !remove_blanks, convert
1 : int
"""2,3"", a" : string
4 : int
-1.1 : double
"" : string
"" : string
5 : int
"asdf" : string

1,"""2,3"", a",4,-1.1,,"", 5,"asdf" !unquote, remove_blanks, convert
1 : int
"""2,3"", a" : string
4 : int
-1.1 : double
"" : string
5 : int
"asdf" : string

1,"""2,3"", a",4,-1.1,,"", 5,"asdf" !unquote, remove_blanks, !convert
1 : string
"""2,3"", a" : string
4 : string
-1.1 : string
"" : string
5 : string
"asdf" : string

1,"""2,3"", a",4,-1.1,,"", 5,"asdf" unquote, remove_blanks, convert
1 : int
"2,3", a : string
4 : int
-1.1 : double
5 : int
asdf : string

Breaking down typical .csv file: delimited by ,using " as escape characters, and unquote, !remove_blanks,converting to type

Order Question Title ID/Rev Type Status difficulty Weight Avg Answer Time  Group Last Editor Categories
1 Free Radical 66526 / 2 MChoice TRUE 0.623596 1 1:16  Harrison, D B01 Biochemistry BT01 - Recall (remembering and understanding) L01 - Introductory Parts
2 Phosphatitic acid 70264 / 1 MChoice TRUE 0.314286 1 1:44  Harrison, D B01 Biochemistry BT01 - Recall (remembering and understanding) L01 - Introductory Parts
3 Copy of Epimers 2 70231 / 1 MChoice TRUE 0.0714286 1 3:24  Harrison, D B01 Biochemistry BT01 - Recall (remembering and understanding) L01 - Introductory Parts
4 'Epimers 2' 70230 / 1 MChoice TRUE 0.457143 1 2:45  Harrison, D B01 Biochemistry BT01 - Recall (remembering and understanding) L01 - Introductory Parts
5 "Anabolism" 65576 / 4 MChoice TRUE 0.62605 1 1:05  Harrison, D B01 Biochemistry BT01 - Recall (remembering and understanding) Fed/Fast L01 - Introductory
6 dilution of hydroxide 70284 / 2 Fill in the Blank TRUE 0.185714 1 2:58  Harrison, D B01 Biochemistry BT02 - Interpretation (applying and analyzing) L01 - Introductory Water/pH/pKa
7 What Changes 68853 / 2 MChoice TRUE 0.414286 1 1:52  Harrison, D B01 Biochemistry BT01 - Recall (remembering and understanding) L01 - Introductory Protein Structure-Function
8 Secondary Structure 68854 / 1 MChoice TRUE 0.357143 1 1:33  Harrison, D B01 Biochemistry BT01 - Recall (remembering and understanding) L01 - Introductory Protein Structure-Function
9 Kinases 3358 / 2 MChoice TRUE 0 1 0  Harrison, D B01 Biochemistry B05 Biochemistry/Biotechnology B05.01 chemistry of biomacromolecules (proteins Protein Structure-Function and DNA) carbohydrates lipids
10 Competitive FITB 21116 / 4 Fill in the Blank TRUE 0.185714 1 5:53 NC Harrison, D B01 Biochemistry BT02 - Interpretation (applying and analyzing) Enzyme Regulation L01 - Introductory

Notice the double space after the time, indicating not removing a column when blank.

Responses will be appreciated.

Edit: improved std::ostream& operator<<(std::ostream& out, const token& tok)

Edit: ExportedGridData.csv:

Order,Question Title,ID/Rev,Type,Status,difficulty,Weight,Avg Answer Time ,Group,Last Editor,Categories
1,Free Radical,66526 / 2,MChoice,TRUE,0.623595506,1,1:16,,"Harrison, D"," L01 - Introductory, BT01 - Recall (remembering and understanding), B01 Biochemistry, Parts"
2,Phosphatitic acid,70264 / 1,MChoice,TRUE,0.314285714,1,1:44,,"Harrison, D"," L01 - Introductory, BT01 - Recall (remembering and understanding), B01 Biochemistry, Parts"
3,Copy of Epimers 2,70231 / 1,MChoice,TRUE,0.071428571,1,3:24,,"Harrison, D"," L01 - Introductory, BT01 - Recall (remembering and understanding), B01 Biochemistry, Parts"
4,'Epimers 2',70230 / 1,MChoice,TRUE,0.457142857,1,2:45,,"Harrison, D"," L01 - Introductory, BT01 - Recall (remembering and understanding), B01 Biochemistry, Parts"
5,"""Anabolism""",65576 / 4,MChoice,TRUE,0.62605042,1,1:05,,"Harrison, D"," L01 - Introductory, BT01 - Recall (remembering and understanding), Fed/Fast, B01 Biochemistry"
6,dilution of hydroxide,70284 / 2,Fill in the Blank,TRUE,0.185714286,1,2:58,,"Harrison, D"," L01 - Introductory, BT02 - Interpretation (applying and analyzing), B01 Biochemistry, Water/pH/pKa"
7,What Changes,68853 / 2,MChoice,TRUE,0.414285714,1,1:52,,"Harrison, D"," L01 - Introductory, BT01 - Recall (remembering and understanding), Protein Structure-Function, B01 Biochemistry"
8,Secondary Structure,68854 / 1,MChoice,TRUE,0.357142857,1,1:33,,"Harrison, D"," L01 - Introductory, BT01 - Recall (remembering and understanding), Protein Structure-Function, B01 Biochemistry"
9,Kinases,3358 / 2,MChoice,TRUE,0,1,-,,"Harrison, D"," B05 Biochemistry/Biotechnology, B05.01 chemistry of biomacromolecules (proteins, lipids, carbohydrates, and DNA), Protein Structure-Function, B01 Biochemistry"
10,Competitive FITB,21116 / 4,Fill in the Blank,TRUE,0.185714286,1,5:53,NC,"Harrison, D"," L01 - Introductory, BT02 - Interpretation (applying and analyzing), Enzyme Regulation,B01 Biochemistry"
\$\endgroup\$
  • 1
    \$\begingroup\$ Incorporating advice from an answer into the question violates the question-and-answer nature of this site. You could post improved code as a new question, as an answer, or as a link to an external site - as described in I improved my code based on the reviews. What next?. I have rolled back the edit, so the answers make sense again. \$\endgroup\$ – Toby Speight Sep 20 at 14:15
3
\$\begingroup\$

Firstly, thanks for providing the test program. That always makes code easier to review. Unfortunately, when I tried it, I found a null pointer dereference here:

    std::string s = *std::get_if<std::string>(&t);

I had to replace with

    auto p = std::get_if<std::string>(&t);
    if (!p) { continue; }
    std::string s = *p;

The test program really looks like it could benefit from being divided into individual tests within a unit-test framework (and the tests could become self-checking, rather than relying on a human to read the output). In fact, I normally recommend a test-first approach for code like this.

I'd recommend including the ExportedGridData.csv as a string stream, rather than relying on an external resource (which we reviewers don't have!). If you do keep it external, then at least check (e.g. if (test_data) that we've successfully read from it before assuming that test_string is valid.

In fact, this lack of checking of streaming is prevalent throughout the code - e.g. in convertString().


The token_type enum doesn't buy us much. It's only used in main() (so could be reduced in scope), and there only within a single switch. It might be simpler to just use the numeric values there, with comments. The one advantage to having it close to token definition is that we can see that it matches.

Given that it's intended to be used to convert from std::size_t values, perhaps it ought to be plain enum rather than enum class?


The streaming out operator could use the return value from std::visit() as its own return value. There's a slight wrinkle in that lambda expressions normally return values rather than references; to return a reference we need to either specify the return type explicitly:

std::ostream& operator<<(std::ostream& out, const token& tok)
{
    return std::visit([&out](const auto& content) -> std::ostream& { return out << content; }, tok);
}

or use a std::reference_wrapper (from <functional>):

std::ostream& operator<<(std::ostream& out, const token& tok)
{
    return std::visit([&out](const auto& content) { return std::ref(out << content); }, tok);
}

It's probably better to name the lambda, so it's more readable and a sane line length:

std::ostream& operator<<(std::ostream& out, const token& tok)
{
    auto const print = [&out](const auto& content) -> std::ostream&
                       { return out << content; };
    return std::visit(print, tok);
}

Moving on to tokenizer, its constructor ought to be explicit, since it can be invoked with a single argument. I'm not a fan of multiple bool arguments like that, as it's hard to see at the call site what each one means.

This overload of the << operator has no benefit, since std::string_view has a converting constructor from const char*:

tokenizer& tokenizer::operator<<(const char* line)
{
    return operator<<(std::string_view(line));
}

Just remove it and C-style strings work fine.

I think it's clearer to write (s.empty()) than if (s.length() == 0).

The actual tokenization is very hard to follow, particularly with the different options affecting its behaviour. I expected to see a simple state machine for this parsing. It might be simpler to deal with remove_blanks and strip_quotes (and possibly also convert) when streaming out rather than in.

A different approach to determining the type would be to attempt to convert to integer, else to float and finally keep as a string, and let the conversions tell us which was successful (e.g. using std::stoi() and std::stod(), or std::stringstream's input operators).


When writing to a vector, there's no need for a loop to build the vector an element at a time. std::deque is a standard container, so we can create the vector directly from its start and end iterators:

void tokenizer::operator>>(std::vector<token>& out)
{
    out.insert(out.end(), toks_.begin(), toks_.end());
    toks_.clear();
}

std::vector<token> tokenizer::getVector()
{
    std::vector<token> ret{toks_.begin(), toks_.end()};
    toks_.clear();
    return ret;
}

We might consider std::move() algorithm instead, to reduce string copying.


In the test program, there's an obvious candidate for a raw string:

std::istringstream iss("1,\"\"\"2,3\"\", a\",4,-1.1,,\"\", 5,\"asdf\"");

I think this is much easier to read (admittedly, the Stack Exchange syntax highlighter doesn't yet handle it properly, but good editors do):

std::istringstream iss(R"***(1,"""2,3"", a",4,-1.1,,"", 5,"asdf")***");

There's no need for using std::cout; at global scope. If you really feel it's worthwhile, this can be within main().

We're quite right to import the whole std::string_literals namespace - that's one of the few namespaces designed to be used like that.

\$\endgroup\$
  • \$\begingroup\$ Thank you for the review! How does one include a file like ExportedGridData.csv? I was trying to demonstrate a "practical example of how the tokenizer class might be used. I never saw a null pointer dereference error. Interestingly, your ostream code results in a runtime error in VS 2019 C++ "The thread 0x5950 has exited with code 0 (0x0)." I'm thinking that we are seeing differences in how different compilers are implemented.... I am not qualified to say which one conforms to the c++17 standard. \$\endgroup\$ – davidbear Sep 20 at 12:10
  • \$\begingroup\$ You should be able to include the file the same way that you included the other sample code, using a string-stream (you might want to use raw strings to compose it, so it's less tedious). You probably don't see the null pointer because you have a ExportedGridData.csv file to read. \$\endgroup\$ – Toby Speight Sep 20 at 13:07
1
\$\begingroup\$

After reading through @Toby_Speight's Review, I adopted a number of changes to (I hope) improve the code.

First, I separated the Tokenizer class into its own .hpp file so that it can now be readily used by a number of different projects.

Second, I changed the constructor to now only allow for the delimiters and the escape characters. The Tokenizer class uses defaults for handling of weather or not to remove escape characters (and turn double escape characters into a single escape character), remove empty tokens, convert from strings to ints or doubles, and to trim white spaces from the beginning and end of tokens. I decided that needing a new tokenizer for each of these was "safe", but highly burdensome. To give the user control over these defaults, I included removeQuotes, removeBlanks, typeConvertion, and trimWhiteSpace "setter" functions. Since, the Tokenizer class does all of its tokenizing during the input phase (operator<<()) I decided to clear any existing tokens when these "setters" were used. I can think of no use case where you would want to use to different switches for different input data without using the tokens in between input strings.

Third, I decided that trimming white spaces should be a user option and not hard wired trimWhiteSpace.

Fourth, I made some private functions to clean up redundancy in the code and make the main code easier to read. These new functions include empty_string, parse, and emplace_string. The function parse also represents my attempt to make use of the new c++17 function from_chars and clean up some of its clunky interface.

The tokenizer.hpp code is shown below:

// Tokenizer.hpp

#ifndef TOKENIZER_HPP
#define TOKENIZER_HPP

#include <algorithm>
#include <charconv>
#include <deque>
#include <iterator>
#include <optional>
#include <string>
#include <string_view>
#include <variant>
#include <vector>

using namespace std::string_literals;

using token = std::variant<int, double, std::string>;

enum class token_type : std::size_t {
    INT,
    DOUBLE,
    STRING
};

std::ostream& operator<<(std::ostream& out, const token& tok)
{

    std::visit([&out](auto&& content) {out << content; }, tok);
    return out;
}

using svciter = std::string_view::const_iterator;

class tokenizer
{
public:
    explicit tokenizer(const char* delim = ",;:|\t ",
        const char* escape = "\"\'") :
        delim_{ std::string(escape).append(delim) }, escape_{ escape } {}
    tokenizer& operator<<(std::string_view s);
    tokenizer& operator>>(token& tok);
    tokenizer& operator>>(std::optional<token>& otok);
    void operator>>(std::vector<token>& toks);
    std::optional<token> getNext();
    std::vector<token> getVector();
    void removeQuotes(bool remove_quotes);
    void removeBlanks(bool remove_blanks);
    void typeConvertion(bool convert);
    void trimWhiteSpace(bool trim);
    void clear() { toks_.clear(); }

private:
    std::vector<std::string_view> splitString(std::string_view s);
    void convertString(std::string_view s);
    std::string empty_string();
    template <class T>
    std::optional<T> parse(std::string_view s);
    void emplace_string(std::string_view s);
    std::deque<token> toks_;
    const std::string delim_{ };
    const std::string_view escape_{ };
    bool remove_quotes_{ true };
    bool remove_blanks_{ false };
    bool type_convert_{ true };
    bool trim_{ true };
};

std::string tokenizer::empty_string()
{
    if (remove_quotes_ || escape_.length() == 0) return "";
    return ""s + *(escape_.cbegin()) + *(escape_.cbegin());
}

tokenizer& tokenizer::operator<<(std::string_view s)
{
    if (s.empty()) {
        if (!remove_blanks_) toks_.emplace_back(empty_string());
        return *this;
    }
    std::vector<std::string_view> temp_vec = splitString(s);
    for (auto& value : temp_vec) {
        auto count = value.length();
        if (remove_blanks_ && (count == 0 || (count == 2 && remove_quotes_ &&
            escape_.find(value[0]) != escape_.npos))) {
            continue;
        }
        if (count == 0) {
            toks_.emplace_back(empty_string());
            continue;
        }
        auto q_index = std::find(escape_.begin(), escape_.end(), value[0]);
        std::string temp;
        if (q_index != escape_.end() && value[0] == *q_index) {
            if (!remove_quotes_) {
                std::copy(value.begin(), value.end(), std::back_inserter(temp));
                toks_.push_back(temp);   // token_type::STRING
                continue;
            }
            std::copy(value.begin() + 1, value.end() - 1, std::back_inserter(temp));
            const std::string double_quote{ ""s + *q_index + *q_index };
            for (auto pos = temp.find(double_quote, 0);
                pos != std::string::npos; pos = temp.find(double_quote, 0)) {
                temp.erase(pos, 1);
            }
            value = temp;
        }
        if (type_convert_) convertString(value);
        else {
            std::copy(value.begin(), value.end(), std::back_inserter(temp));
            toks_.push_back(temp);  //;    token_type::STRING
        }
    }
    return *this;
}

tokenizer& tokenizer::operator>>(token& tok)
{
    tok = std::move(toks_.front());
    toks_.pop_front();
    return *this;
}

tokenizer& tokenizer::operator>>(std::optional<token>& otok)
{
    if (toks_.empty()) {
        otok = std::nullopt;
        return *this;
    }
    otok = std::move(toks_.front());
    toks_.pop_front();
    return *this;
}

void tokenizer::operator>>(std::vector<token>& out)
{
    out.reserve(out.size() + toks_.size());
    out.insert(out.end(), toks_.begin(), toks_.end());
    toks_.clear();
}

std::optional<token> tokenizer::getNext()
{
    if (toks_.empty()) {
        return std::nullopt;
    }
    token ret(std::move(toks_.front()));
    toks_.pop_front();
    return ret;
}

std::vector<token> tokenizer::getVector()
{
    std::vector<token> ret;
    if (toks_.empty()) return ret;
    ret.reserve(toks_.size());
    std::move(toks_.begin(), toks_.end(), std::back_inserter(ret));
    toks_.clear();
    return ret;
}

void tokenizer::removeQuotes(bool remove_quotes)
{
    remove_quotes_ = remove_quotes;
    toks_.clear();
}

void tokenizer::removeBlanks(bool remove_blanks)
{
    remove_blanks_ = remove_blanks;
    toks_.clear();
}

void tokenizer::typeConvertion(bool convert)
{
    type_convert_ = convert;
    toks_.clear();
}

void tokenizer::trimWhiteSpace(bool trim)
{
    trim_ = trim;
    toks_.clear();
}

std::vector<std::string_view> tokenizer::splitString(std::string_view s)
{
    auto select_space = [&](char c) -> bool {
        if (!isspace(c)) return false;
        return delim_.find(c) == delim_.npos;
    };
    std::vector<std::string_view> ret;
    auto start = s.begin();
    if (trim_) while (select_space(*start)) ++start;
    auto pos_iter = std::find_first_of(start, s.end(), delim_.cbegin(),
        delim_.cend());
    while (pos_iter != s.end()) {
        auto q_index = std::find(escape_.begin(), escape_.end(), *pos_iter);
        if (q_index != escape_.end()) {
            pos_iter = std::find(pos_iter + 1, s.end(), *q_index);
            if ((pos_iter + 1) != s.end() && *(pos_iter + 1) == *q_index) {
                ++pos_iter;
                continue;
            }
            pos_iter = std::find_first_of(pos_iter + 1, s.end(),
                delim_.cbegin(), delim_.cend());
        }
        auto address = &s.at(start - s.begin());
        auto length = pos_iter - start;
        if (trim_) while (select_space(*(address + length - 1))) --length;
        ret.emplace_back(address, length);
        if (pos_iter == s.end()) return ret;
        if (pos_iter + 1 == s.end()) {
            ret.emplace_back("");
            return ret;
        }
        start = pos_iter + 1;
        if (trim_) while (select_space(*start)) ++start;
        pos_iter = std::find_first_of(start, s.end(), delim_.begin(),
            delim_.end());
    }
    auto address = &s.at(start - s.begin());
    auto length = s.end() - start;
    if (trim_) while (select_space(*(address + length - 1))) --length;
    ret.emplace_back(address, length);
    return ret;
}

template <typename T>
std::optional<T> tokenizer::parse(std::string_view s)
{
    T value;
    auto res = std::from_chars(s.data(), s.data() + s.size(), value);
    if (res.ec != std::errc{}) return std::nullopt;
    return value;
}

void tokenizer::emplace_string(std::string_view s)
{
    std::string temp;
    std::copy(s.begin(), s.end(), std::back_inserter(temp));
    toks_.emplace_back(temp);
}

void tokenizer::convertString(std::string_view s)
{
    if (s.empty()) {
        toks_.emplace_back("");
        return;
    }
    std::uint8_t offset = 0;
    if (s[0] == '-' || s[0] == '+') ++offset;
    auto iter = std::find_if(s.begin() + offset, s.end(),
        [](unsigned char c) noexcept -> bool {
            return !(isdigit(c) || c == '.');
        });
    if (iter != s.end() || s.begin() + offset == s.end()) {
        emplace_string(s);
        return;
    }
    if (s[0] == '+') {              // from_chars can't handle '+ddd'
        s = std::string_view(&s.at(1), s.size() - 1);
    }
    iter = std::find(s.begin(), s.end(), '.');
    if (iter != s.end()) {
        if (auto dvalue = parse<double>(s)) toks_.push_back(dvalue.value());
        else emplace_string(s);
        return;
    }
    if (auto ivalue = parse<int>(s)) toks_.push_back(ivalue.value());
    else emplace_string(s);
    return;
}

#endif // !TOKENIZER_HPP

As far as testing is concerned, I have included a assert statements to test that the output is as expected for eight different tests that demonstrate each of the approaches to get data into and out of the tokenizer and look at the effects of changing the "switches." I then show how the tokenizer class might be used in an application that takes apart a typical .csv file (see above).

This is the main.cpp code

#include <algorithm>
#include <cassert>
#include <fstream>
#include <iostream>
#include <optional>
#include <sstream>
#include <string>
#include <string_view>
#include <variant>
#include <vector>
#include "tokenizer.hpp"

using std::cout;
using namespace std::string_literals;

int main()
{
    constexpr auto typer = [](std::size_t index) -> const char*
    {
        switch (static_cast<token_type>(index)) {
        case token_type::STRING:
            return "string";
        case token_type::DOUBLE:
            return "double";
        case token_type::INT:
        default:
            return "int";
        }
    };
    tokenizer token_maker (",", "\""); // , true, false, true
    token_maker << "";
    token_maker << R"("""help, this!""" ,+32 )";
    token_maker << R"("gasoline",.,.0,)"; 
    std::vector<token> test_tokens{ "",
        R"("help, this!")",32,"gasoline",".",0.,"" };
    std::vector<token> tokens{ token_maker.getVector() };
    for (auto t : tokens) cout << t << " : " << typer(t.index()) << "\n";
    assert(test_tokens.size() == tokens.size() && 
        std::equal(test_tokens.begin(), test_tokens.end(), tokens.begin())
        && "tokens are unexpected\n");
    cout << "PASSED zeroth test\n\n";
    std::string test_string;
    std::istringstream iss(R"*^*(1, """2,3"", a",   4,-1.1,,"", 5,"asdf")*^*");
    std::getline(iss, test_string);
    cout << test_string << " remove_quotes, !remove_blanks, convert, trim\n";
    test_tokens = std::vector<token>{ 1, R"("2,3", a)", 4, -1.1, "", "", 5, 
        "asdf" };
    tokens.clear();
    token_maker << test_string >> tokens;
    assert(test_tokens.size() == tokens.size() && 
        std::equal(test_tokens.begin(), test_tokens.end(), tokens.begin()) &&
        "tokens are unexpected in first test\n");
    for (auto t : tokens) cout << t << " : " <<
        typer(t.index()) << "\n"; //  t.index()
    cout << "PASSED first test\n\n";
    token_maker.removeQuotes(false);
    cout << test_string << " !remove_quotes, !remove_blanks, convert, trim\n";
    token_maker << test_string;
    std::optional<token> next_tok;
    test_tokens = std::vector<token>{ 1, R"("""2,3"", a")", 4, -1.1, "\"\"", 
        "\"\"", 5, "\"asdf\"" };
    auto iter_test = test_tokens.begin();
    token_maker >> next_tok;
    while (next_tok) {
        assert(iter_test != test_tokens.end() && 
            *next_tok == *iter_test++ && "unexpected tokens in second test");
        cout << *next_tok << " : " << typer(next_tok->index()) << "\n";
        token_maker >> next_tok;
    }
    cout << "PASSED second test\n\n";
    token_maker.removeBlanks(true);
    cout << test_string << " !remove_quotes, remove_blanks, convert, trim\n";
    test_tokens.erase(test_tokens.begin()+4);
    token_maker << test_string;
    iter_test = test_tokens.begin();
    while (auto next_tok = token_maker.getNext()) {
        assert(iter_test != test_tokens.end() &&
            *next_tok == *iter_test++ && "unexpected tokens in third test");
        cout << *next_tok << " : " <<
            typer(next_tok->index()) << "\n";
    }
    cout << "PASSED third test\n\n";
    token_maker.typeConvertion(false);
    token_maker.trimWhiteSpace(false);
    cout << test_string << " !remove_quotes, remove_blanks, !convert, !trim\n";
    tokens.clear();
    token_maker << test_string >>tokens;
    for (auto t : tokens) {
        cout << t << " : " << typer(t.index()) << "\n";
        assert(static_cast<token_type>(t.index()) == token_type::STRING &&
        "tokens are unexpected in fourth test\n");
    }
    cout << "PASSED fourth test\n\n";
    token_maker.removeQuotes(true);
    token_maker.typeConvertion(true);
    token_maker.trimWhiteSpace(true);
    cout << test_string << " remove_quotes, remove_blanks, convert, trim\n";
    tokens.clear();
    test_tokens = std::vector<token>{ 1, R"("2,3", a)", 4, -1.1,  5, "asdf" };
    iter_test = test_tokens.begin();
    token_maker << test_string >> tokens;
    for (auto t : tokens) {
        assert(iter_test != test_tokens.end() && 
            t == *iter_test++ && "unexpected tokens in fifth test");
        cout << t << " : " << typer(t.index()) << "\n";
    }
    cout << "PASSED fifth test\n\n";
    cout << "space delimited 'this is a + text now  ' ";
    cout << " remove_quotes, !remove_blanks, convert, trim\n";
    tokenizer token_maker2{ " ","" };
    token_maker2 << "this is a +  text now  ";
    test_tokens = 
        std::vector<token>{ "this","is","a","+","","text","now","","" };
    iter_test = test_tokens.begin();
    while (auto next_tok = token_maker2.getNext()) {
        cout << '\"' << *next_tok << '\"' << "\n";
        assert(iter_test != test_tokens.end() && *next_tok == *iter_test++
            && "unexpected tokens in fifth test");
    }
    cout << "PASSED sixth test\n\n";
    token_maker.removeBlanks(false);
    cout << "Breaking down typical .csv file: delimited by , using \"";
    cout << " as escape characters, and removeQuotes = true, ";
    cout << "removeBlanks = false, typeConvertion = true\n\n";
    std::vector<token> headings;
    headings.reserve(15);
    std::ifstream test_data("ExportedGridData.csv");
    if (!test_data.is_open()) std::exit(EXIT_FAILURE);
    std::getline(test_data, test_string);
    token_maker << test_string >> headings;
    int i=0, category_column=-1;
    for (auto t : headings) {
        std::string s = *std::get_if<std::string>(&t);
        if (s.find("Categories", 0)!=s.npos) category_column = i;
        ++i;
    }
    const auto last_column = headings.size() - 1;
    if (last_column != category_column) {
        headings.erase(headings.begin() + category_column);
        headings.emplace_back("Categories"s);
    }
    for (auto t : headings) cout << t << " ";
    cout << "\n";
    while (std::getline(test_data, test_string)) {
        token_maker << test_string;
        std::vector<token> data;
        token_maker >> data;
        token_maker << std::get<std::string>(data.at(category_column)) >> data;
        data.erase(data.begin()+category_column);
        std::sort(data.begin() + last_column, data.end(), 
            [](const token & a, const token & b) -> bool {
                return a < b;
            });
        for (auto s : data) cout << s << " ";
        cout << "\n";
    }
}

This is the output:

 : string
"help, this!" : string
32 : int
gasoline : string
. : string
0 : double
 : string
PASSED zeroth test

1, """2,3"", a",        4,-1.1,,"", 5,"asdf" remove_quotes, !remove_blanks, convert, trim
1 : int
"2,3", a : string
4 : int
-1.1 : double
 : string
 : string
5 : int
asdf : string
PASSED first test

1, """2,3"", a",        4,-1.1,,"", 5,"asdf" !remove_quotes, !remove_blanks, convert, trim
1 : int
"""2,3"", a" : string
4 : int
-1.1 : double
"" : string
"" : string
5 : int
"asdf" : string
PASSED second test

1, """2,3"", a",        4,-1.1,,"", 5,"asdf" !remove_quotes, remove_blanks, convert, trim
1 : int
"""2,3"", a" : string
4 : int
-1.1 : double
"" : string
5 : int
"asdf" : string
PASSED third test

1, """2,3"", a",        4,-1.1,,"", 5,"asdf" !remove_quotes, remove_blanks, !convert, !trim
1 : string
 """2,3"", a" : string
        4 : string
-1.1 : string
"" : string
 5 : string
"asdf" : string
PASSED fourth test

1, """2,3"", a",        4,-1.1,,"", 5,"asdf" remove_quotes, remove_blanks, convert, trim
1 : int
"2,3", a : string
4 : int
-1.1 : double
5 : int
asdf : string
PASSED fifth test

space delimited 'this is a + text now  '  remove_quotes, !remove_blanks, convert, trim
"this"
"is"
"a"
"+"
""
"text"
"now"
""
""
PASSED sixth test

Breaking down typical .csv file: delimited by , using " as escape characters, and removeQuotes = true, removeBlanks = false, typeConvertion = true

Order Question Title ID/Rev Type Status difficulty Weight Avg Answer Time Group Last Editor Categories
1 Free Radical 66526 / 2 MChoice TRUE 0.623596 1 1:16  Harrison, D B01 Biochemistry BT01 - Recall (remembering and understanding) L01 - Introductory Parts
2 Phosphatitic acid 70264 / 1 MChoice TRUE 0.314286 1 1:44  Harrison, D B01 Biochemistry BT01 - Recall (remembering and understanding) L01 - Introductory Parts
3 Copy of Epimers 2 70231 / 1 MChoice TRUE 0.0714286 1 3:24  Harrison, D B01 Biochemistry BT01 - Recall (remembering and understanding) L01 - Introductory Parts
4 'Epimers 2' 70230 / 1 MChoice TRUE 0.457143 1 2:45  Harrison, D B01 Biochemistry BT01 - Recall (remembering and understanding) L01 - Introductory Parts
5 "Anabolism" 65576 / 4 MChoice TRUE 0.62605 1 1:05  Harrison, D B01 Biochemistry BT01 - Recall (remembering and understanding) Fed/Fast L01 - Introductory
6 dilution of hydroxide 70284 / 2 Fill in the Blank TRUE 0.185714 1 2:58  Harrison, D B01 Biochemistry BT02 - Interpretation (applying and analyzing) L01 - Introductory Water/pH/pKa
7 What Changes 68853 / 2 MChoice TRUE 0.414286 1 1:52  Harrison, D B01 Biochemistry BT01 - Recall (remembering and understanding) L01 - Introductory Protein Structure-Function
8 Secondary Structure 68854 / 1 MChoice TRUE 0.357143 1 1:33  Harrison, D B01 Biochemistry BT01 - Recall (remembering and understanding) L01 - Introductory Protein Structure-Function
9 Kinases 3358 / 2 MChoice TRUE 0 1 -  Harrison, D B01 Biochemistry B05 Biochemistry/Biotechnology B05.01 chemistry of biomacromolecules (proteins Protein Structure-Function and DNA) carbohydrates lipids
10 Competitive FITB 21116 / 4 Fill in the Blank TRUE 0.185714 1 5:53 NC Harrison, D B01 Biochemistry BT02 - Interpretation (applying and analyzing) Enzyme Regulation L01 - Introductory

I hope this helps. Additional comments welcome.

\$\endgroup\$

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Not the answer you're looking for? Browse other questions tagged or ask your own question.