UPDATE: I have refactored the code into a Gist using @Dmitry's answer as a guide. The update is much simpler to grok, implements IDisposable, and is roughly thirty lines shorter.
I wrote this over the weekend for fun and am looking for critique. Style and readability comments are welcome but what I truly need to know is:
- Does it function as advertised?
- Are there any lingering bugs that I've missed?
- Can you come up with a way to make it faster?
When I ask these of myself I get 1 = yes, 2 = no, and 3 = maaaaaybe. I'd like to add other features like skipping the header row, inferring data types, validating field counts, etc. but I'll be tackling that kind of thing via derivation or extension since such logic will be simpler to implement if based on an existing IEnumerable<IEnumerable<>> like this one.
FLAME ON;
Usage:
foreach (var row in DelimitedReader.Create(fileName)) {
foreach (var field in row) {
// do stuff
}
}
Features:
- Accurate: RFC4180 Compliant
- Efficient: memory usage is (roughly) equal to the size of the largest row
- Fast: average throughput of ~25 megabytes per second
- Flexible: the default encoding and separator/escape characters can be user-defined
- Lightweight: single 160 line class with no external dependencies
Code:
using System;
using System.Collections;
using System.Collections.Generic;
using System.IO;
using System.Text;
namespace ByteTerrace
{
public class DelimitedReader : IEnumerable<IEnumerable<string>>
{
private const int DEFAULT_CHUNK_SIZE = 128;
private const char DEFAULT_ESCAPE_CHAR = '"';
private const char DEFAULT_SEPARATOR_CHAR = ',';
private readonly char[] m_buffer;
private readonly Encoding m_encoding;
private readonly char m_escapeChar;
private readonly string m_fileName;
private readonly char m_separatorChar;
public char[] Buffer {
get {
return m_buffer;
}
}
public Encoding Encoding {
get {
return m_encoding;
}
}
public char EscapeChar {
get {
return m_escapeChar;
}
}
public string FileName {
get {
return m_fileName;
}
}
public char SeparatorChar {
get {
return m_separatorChar;
}
}
public DelimitedReader(string fileName, char separatorChar = DEFAULT_SEPARATOR_CHAR, char escapeChar = DEFAULT_ESCAPE_CHAR, Encoding encoding = null, int bufferSize = DEFAULT_CHUNK_SIZE) {
m_buffer = new char[bufferSize];
m_encoding = (encoding ?? Encoding.UTF8);
m_escapeChar = escapeChar;
m_fileName = fileName;
m_separatorChar = separatorChar;
}
public IEnumerator<IEnumerable<string>> GetEnumerator() {
return ReadFields().GetEnumerator();
}
IEnumerator IEnumerable.GetEnumerator() {
return GetEnumerator();
}
IEnumerable<IEnumerable<string>> ReadFields() {
return ReadFields(ReadAllChunks(FileName, Encoding, Buffer), SeparatorChar, EscapeChar);
}
public static DelimitedReader Create(string fileName, char separatorChar = DEFAULT_SEPARATOR_CHAR, char escapeChar = DEFAULT_ESCAPE_CHAR, Encoding encoding = null, int bufferSize = DEFAULT_CHUNK_SIZE) {
return new DelimitedReader(fileName, separatorChar, escapeChar, encoding, bufferSize);
}
public static IEnumerable<char[]> ReadAllChunks(TextReader reader, char[] buffer) {
var count = buffer.Length;
var numBytesRead = 0;
while ((numBytesRead = reader.ReadBlock(buffer, 0, count)) == count) {
yield return buffer;
}
if (numBytesRead > 0) {
Array.Resize(ref buffer, numBytesRead);
yield return buffer;
}
}
public static IEnumerable<char[]> ReadAllChunks(string fileName, Encoding encoding, char[] buffer) {
return ReadAllChunks(new StreamReader(new FileStream(fileName, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, FileOptions.SequentialScan), encoding), buffer);
}
public static string ReadField(StringBuilder buffer, int offset, int position, char escapeChar) {
if (buffer[offset] == escapeChar) {
if (position - offset != 2) {
return buffer.ToString(offset + 1, position - offset - 3);
}
else {
return string.Empty;
}
}
else {
return buffer.ToString(offset, position - offset - 1);
}
}
public static IEnumerable<IEnumerable<string>> ReadFields(IEnumerable<char[]> chunks, char separatorChar = DEFAULT_SEPARATOR_CHAR, char escapeChar = DEFAULT_ESCAPE_CHAR) {
var buffer = new StringBuilder();
var fields = new List<string>();
var endOfBuffer = 0;
var escaping = false;
var offset = 0;
var position = 0;
var head0 = '\0';
var head1 = head0;
foreach (var chunk in chunks) {
buffer.Append(chunk, 0, chunk.Length);
endOfBuffer = buffer.Length;
while (position < endOfBuffer) {
head1 = head0;
if ((head0 = buffer[position++]) == escapeChar) {
escaping = !escaping;
if ((head0 == escapeChar) && (head1 == escapeChar)) {
endOfBuffer--;
position--;
buffer.Remove(position, 1);
}
}
if (!escaping) {
if ((head0 == '\n') || (head0 == '\r')) {
if ((head1 != '\r') || (head0 == '\r')) {
fields.Add(ReadField(buffer, offset, position, escapeChar));
yield return fields;
buffer.Remove(0, position);
endOfBuffer = buffer.Length;
fields.Clear();
offset = 0;
position = 0;
}
else {
offset++;
}
}
else if (head0 == separatorChar) {
fields.Add(ReadField(buffer, offset, position, escapeChar));
offset = position;
}
}
}
}
if (buffer.Length > 0) {
fields.Add(buffer.ToString());
}
if (fields.Count > 0) {
yield return fields;
}
}
}
}
