I get huge incoming files ( upup to 6GB) and they are littered with Control and non-ASCII characters. I need to strip them and I made this routine (below). The problem is that it is insanely slow. I would love any thoughts or advice on how I can speed it up.
public void StripHighBitCharacters(string fn)
{
string writeFile = fn + "B";
using (var reader = new StreamReader(fn))
using (var writer = new StreamWriter(writeFile))
{
while (!reader.EndOfStream)
{
string line = reader.ReadLine();
if (line.Length > 0)
{
writer.WriteLine(BuildClearString(line));
}
else
{
writer.WriteLine(line);
}
}
}
File.Copy(writeFile, fn, true);
File.Delete(writeFile);
}
public string BuildClearString(string line)
{
StringBuilder sb = new StringBuilder();
foreach (char c in line)
{
if (c >= 32 && c <= 175)
{
sb.Append(c);
}
}
return (sb.ToString());
}