Loading in two CSV files and storing them in two separate arrays

Question

As the title says, I'm loading (or at least trying to) two CSV files and store them in two separate arrays. What I have so far worked, but it isn't the most elegant, no efficient solution.

The end goal is to read in two CSV files and run comparisons on them both - that's why the separating them is important.

Note - I use the csv-parser library, but am open to different solutions.

const express = require("express");
const parse = require("csv-parser");
const fs = require("fs");

const app = express();

const port = 3000;

var CSVOne = [];
var CSVTwo = [];

fs.createReadStream("public/activity.csv")
  .pipe(parse())
  .on("data", data => CSVOne.push(data))
  .on("end", () => {
    sender = CSVOne.map(d => {
      return {
        email: d.Sender
      };
    });
    fs.createReadStream("public/groups.csv")
      .pipe(parse())
      .on("data", dataTwo => CSVTwo.push(dataTwo))
      .on("end", () => {
        one = CSVTwo.map(d => {
          return {
            clinic: d.one
          };
        });
        console.log(sender);
        console.log(one);
      });
  });

app.listen(port, function() {
  console.log("Server has started");
});

Answer 1

I don't know about elegant, but you could load the files in parallel for better performance. To do something when all CSV files are ready you can use promises or just a simple counter.

The example also shows how you can use objects as return value in your arrow functions without the extra { return {...} }.

Example of loading in parallel using the counter method:

const express = require("express");
const parse = require("csv-parser");
const fs = require("fs");

const app = express();

const port = 3000;

var CSVOne = [];
var CSVTwo = [];
var toGo = 2;

fs.createReadStream("public/activity.csv")
  .pipe(parse())
  .on("data", data => CSVOne.push(data))
  .on("end", () => {
    sender = CSVOne.map(d => ({
      email: d.Sender
    }));
    maybeDone();
  });

fs.createReadStream("public/groups.csv")
  .pipe(parse())
  .on("data", dataTwo => CSVTwo.push(dataTwo))
  .on("end", () => {
    one = CSVTwo.map(d => ({
      clinic: d.one
    }));
    maybeDone();
  });

function maybeDone() {
  toGo -= 1;
  if (toGo === 0)
    done();
}

function done() {
  console.log("Both CSV files are ready");
  console.log(CSVOne, CSVTwo);
}

app.listen(port, function() {
  console.log("Server has started");
});