Skip to main content
2 of 4
edited title and tags; edited tags
t3chb0t
  • 44.3k
  • 9
  • 84
  • 190

Remove control and non-ASCII characters rrom large files

I get huge incoming files (up to 6GB) and they are littered with Control and non-ASCII characters. I need to strip them and I made this routine (below). The problem is that it is insanely slow. I would love any thoughts or advice on how I can speed it up.

public void StripHighBitCharacters(string fn)
{
    string writeFile = fn + "B";
    using (var reader = new StreamReader(fn))
    using (var writer = new StreamWriter(writeFile))
    {
        while (!reader.EndOfStream)
        {
            string line = reader.ReadLine();
            if (line.Length > 0)
            {
                writer.WriteLine(BuildClearString(line));
            }
            else
            {
                writer.WriteLine(line);
            }
        }
    }
    File.Copy(writeFile, fn, true);
    File.Delete(writeFile);
}
public string BuildClearString(string line)
{

    StringBuilder sb = new StringBuilder();

    foreach (char c in line)
    {
        if (c >= 32 && c <= 175)
        {
            sb.Append(c);
        }
    }

    return (sb.ToString());
}
Missy
  • 157
  • 1
  • 6