I need to read multiple CSV files that have the same number of rows, and compute some results for each row. I would like to be able to do so using the fast-csv parser that triggers events on each row reading. Mostly to avoid having to load the entire file into variables and then compute what I need.
I came up with the following solution, with a dummy example to sum the values from each file. It seems to be working, but because I'm working with events, I'm not sure if there might be a case when some data could go missing, or if there's any other issue. I would appreciate any help to improve performance, functionality, maintainability, readability, etc..
Because executing the code requires an external dependency and some files, here's a Repl.it.
Here's the code if you just want to read:
const csv = require("fast-csv");
const fs = require('fs');
const EventEmitter = require('events');
const files = ['csv/file1.csv', 'csv/file2.csv'];
// Create parsers for each file
const parsers = files.map(file => csv.parseStream(fs.createReadStream(file), { delimiter: ';', headers: false }));
// Variables to hold temporary data and definitive results
const data = parsers.map(() => []);
let result = [];
// Handler for a set of the same row from all the different files
var eventEmitter = new EventEmitter();
eventEmitter.on('allRows', (rows) => {
var sum = (r, a) => r.map((b, i) => Number(a[i]) + Number(b));
result.push(rows.reduce(sum));
});
// Handler to display the result once all files have been read
eventEmitter.on('end', (rows) => {
console.log("result", result);
});
// Handler for each row reading
const onDataHandler = function(row, idx) {
data[idx].push(row);
// When we have a row parsed for each file, we emit a signal
if (data.reduce((acc, el) => acc && el.length > 0, true)) {
if (data.reduce((acc, el) => acc && el[0] === null, true)) {
// 'end' if all results have been shifted
eventEmitter.emit('end');
} else {
// 'allRows' otherwise, with the first element of each file
eventEmitter.emit(
'allRows',
data.reduce((acc, el) => {
acc.push(el.shift());
return acc;
}, [])
);
}
}
}
// We use readable to use 'flowing' mode and make sure we don't miss the last rows
parsers.forEach((parser, idx) => parser.on('readable', () => {
onDataHandler(parser.read(), idx);
}));