I have to do a pairwise compare of a couple of text files. They have the following format:
id_1|errorcode_1|m|data_1_1|data_1_2|...|data_1_m
id_2|errorcode_2|m|data_2_1|data_2_2|...|data_2_m
.
.
.
id_n|errorcode_n|m|data_n_1|data_n_2|...|data_n_m
The columns are ordered, but the rows can be in an arbitrary order. And there are always a couple of data-fields that have to be ignored.
This is my method that I call twice (switching the target and source array). It takes around 20 minutes to execute with two files where n = 16000 and m = 100.
private static string[] GetDifferences( IEnumerable<string> source
, IEnumerable<string> target
, IEnumerable<int> ignoredIndices)
{
var uniqueInSource = new List<string>();
foreach (var sourceLine in source)
{
var found = false;
var sourceSplit = sourceLine.Split('|').ToList();
// filter out malformed lines
if (sourceSplit.Count > 2 && !int.TryParse(sourceSplit[2], out _))
{
continue;
}
foreach (var index in ignoredIndices)
{
sourceSplit.RemoveAt(index + 3);
}
foreach (var targetLine in source)
{
var targetSplit = targetLine.Split('|').ToList();
if (targetSplit.Count > 2 && !int.TryParse(targetSplit[2], out _))
{
continue;
}
foreach (var index in ignoredIndices)
{
targetSplit.RemoveAt(index + 3);
}
if (targetSplit.SequenceEqual(sourceSplit))
{
found = true;
break;
}
}
if (!found)
{
uniqueInSource.Add(string.Join("|", sourceLine));
}
}
return uniqueInSource.ToArray();
}
I feel that my approach might be too naive since I think it's somewhere around O(m*n2).