Finished Someone helped me see that i was framenting the output by writing to each file inside the for loop instead i needed to have one big for loop for each file so that the harddrive didnt have to move its head everytime
https://stackoverflow.com/questions/9913788/outputing-dictionary-optimally
I have 4 Dictionary
s that contain 800k strings with 200 to 6000 characters.
when i load it into memory it takes up about 11 gigs of memory.
it is taking me 2 minutes to parse the data and 2 minutes to output the data.
Is there anyway to output the data faster than what I am using below?
I am only getting 20-31 MB per second disk IO and I know the hard drive can do 800ish
var hash1 = new Dictionary<int, Dictionary<string, string>>(f.Count + 2);
var hash2 = new Dictionary<int, Dictionary<string, string>>(f.Count + 2);
var hash3 = new Dictionary<int, Dictionary<string, string>>(f.Count + 2);
var hash4 = new Dictionary<int, Dictionary<string, string>>(f.Count + 2);
...
foreach (var me in mswithfilenames)
{
filename = me.Key.ToString();
string filenamef = filename + "index1";
string filenameq = filename + "index2";
string filenamefq = filename + "index3";
string filenameqq = filename + "index4";
StreamWriter sw = File.AppendText(filenamef);
StreamWriter sw2 = File.AppendText(filenameq);
StreamWriter swq = File.AppendText(filenamefq);
StreamWriter sw2q = File.AppendText(filenameqq);
for (i = 0; i <= totalinhash; i++)
{
if (hashs1[i].ContainsKey(filenamef))
{
sw.Write(hashs1[i][filenamef]);
}
if (hashs2[i].ContainsKey(filenameq))
{
sw2.Write(hashs2[i][filenameq]);
}
if (hashs3[i].ContainsKey(filenamefastaq))
{
swq.Write(hash4[i][filenamefastaq]);
}
if (hash4[i].ContainsKey(filenameqq))
{
sw2q.Write(hash4[i][filenameqq]);
}
}
sw.Close();
sw2.Close();
sw3.Close();
sw4.Close();
swq.Close();
sw2q.Close();
}