I have a few brands that send me several thousand JSON documents a day - 2 types: CREATE
and REDEEM
. I originally created a massive script to dynamically pull, but once we agreed upon specific headers (they were random before), I decided to re-write it. Kind of simplified, but curious about any improvement/ES6-ification.
I stash the client as an object to dynamically associate each 'pick-up directory' with each drop-off so the client can freely name them.
I then glob
all files into 'require' that file one-by-one to pull the data. Then I push that file into the array that I iterate through to actually get the data.
The purpose of the if
wrapper is to avoid errors if no files exist.
Immediate reduction I know I can do is modifying the headers and just 'inserting' the different ones that create demands, but since I need them in this order exactly, I felt like re-iterating the array might be a better option.
var glob = require('glob')
, path = require('path')
, fs = require('fs')
, csvWriter = require('csv-write-stream')
, moment = require('moment')
, redeemHeaders =
[
"token", "campaignId", "campaignName", "campaignOfferId", "campaignOfferPromoId",
"event", "timestamp", "issuingPartner", "issuerName", "consumer/emailAddress",
"location", "GUID"
]
, createHeaders =
[
"token", "campaignId", "campaignName", "campaignOfferId", "campaignOfferPromoId",
"event", "timestamp", "meta/offerId", "meta/campaignId", "issuingPartner", "consumer/emailAddress",
"location"
]
, today = moment().format("YYYYMMDD")
, client_dir =
{
"Brand1": "brand1_output",
"Brand2": "brand2_output",
"Brand3": "brand3_output",
"Brand4": "brand4_output"
}
, redeemArr = []
, createArr = [];
for (var k in client_dir) {
var writer = csvWriter(
{ headers: redeemHeaders }
)
//Reset Arrays for next entries
redeemArr = [];
createArr = [];
fileArr = [];
//Update var with newest file list
glob.sync(`./${k}/REDEEM*-${today}*.json` ).forEach (function( file) {
redeemArr.push(file);
});
if (redeemArr.length > 0) {
writer.pipe(fs.createWriteStream(`./ddnOutput/REDEEM_${client_dir[k]}_${today}.csv`));
//${client_dir[k]}
redeemArr.forEach(function (file){
var tempRedeemFile = require(file);
writer.write([
tempRedeemFile['token'],
tempRedeemFile['campaignId'],
tempRedeemFile['campaignName'],
tempRedeemFile['campaignOfferId'],
tempRedeemFile['campaignOfferPromoId'],
tempRedeemFile['event'],
tempRedeemFile['timestamp'],
tempRedeemFile['issuingPartner'],
tempRedeemFile['consumer']['emailAddress'],
tempRedeemFile['location'],
tempRedeemFile['AztecAccountId']
]);
});
console.log(redeemArr.length)
writer.end();
}; // End redeemArr if
glob.sync(`./${k}/CREATE*-${today}*.json` ).forEach (function( file) {
createArr.push(file);
});
if (createArr.length > 0) {
//Create Section
var writer2 = csvWriter(
{ headers: createHeaders }
)
writer2.pipe(fs.createWriteStream(`./ddnOutput/CREATE_${client_dir[k]}_${today}.csv`));
//${client_dir[k]}
createArr.forEach(function (file){
var tempCreateFile = require(file);
writer2.write([
tempCreateFile['token'],
tempCreateFile['campaignId'],
tempCreateFile['campaignName'],
tempCreateFile['campaignOfferId'],
tempCreateFile['campaignOfferPromoId'],
tempCreateFile['event'],
tempCreateFile['timestamp'],
tempCreateFile['meta']['offerId'],
tempCreateFile['meta']['campaignId'],
tempCreateFile['issuingPartner'],
tempCreateFile['consumer']['emailAddress'],
tempCreateFile['location']
])
});
console.log(createArr.length)
writer2.end();
}; // End createArr if
};