1
\$\begingroup\$

I have a few brands that send me several thousand JSON documents a day - 2 types: CREATE and REDEEM. I originally created a massive script to dynamically pull, but once we agreed upon specific headers (they were random before), I decided to re-write it. Kind of simplified, but curious about any improvement/ES6-ification.

I stash the client as an object to dynamically associate each 'pick-up directory' with each drop-off so the client can freely name them.

I then glob all files into 'require' that file one-by-one to pull the data. Then I push that file into the array that I iterate through to actually get the data.

The purpose of the if wrapper is to avoid errors if no files exist.

Immediate reduction I know I can do is modifying the headers and just 'inserting' the different ones that create demands, but since I need them in this order exactly, I felt like re-iterating the array might be a better option.

var   glob = require('glob')
    , path = require('path')
    , fs = require('fs')
    , csvWriter = require('csv-write-stream')
    , moment = require('moment')
    , redeemHeaders = 
    [
        "token", "campaignId", "campaignName", "campaignOfferId", "campaignOfferPromoId", 
        "event", "timestamp", "issuingPartner", "issuerName", "consumer/emailAddress",
        "location", "GUID"
    ]
    , createHeaders = 
    [
        "token", "campaignId", "campaignName", "campaignOfferId", "campaignOfferPromoId", 
        "event", "timestamp", "meta/offerId", "meta/campaignId", "issuingPartner", "consumer/emailAddress",
        "location"
    ]
    , today = moment().format("YYYYMMDD")
    , client_dir = 
    {
        "Brand1": "brand1_output",
        "Brand2": "brand2_output",
        "Brand3": "brand3_output",
        "Brand4": "brand4_output"
    }
    , redeemArr = []
    , createArr = [];


for (var k in client_dir) {
    var writer = csvWriter(
        { headers: redeemHeaders }
    )
    //Reset Arrays for next entries
    redeemArr = [];
    createArr = [];
    fileArr = [];
    //Update var with newest file list
    glob.sync(`./${k}/REDEEM*-${today}*.json` ).forEach (function( file) {
        redeemArr.push(file);
    });
    if (redeemArr.length > 0) {
        writer.pipe(fs.createWriteStream(`./ddnOutput/REDEEM_${client_dir[k]}_${today}.csv`));
        //${client_dir[k]}

        redeemArr.forEach(function (file){
            var tempRedeemFile = require(file);
            writer.write([
                tempRedeemFile['token'],
                tempRedeemFile['campaignId'],
                tempRedeemFile['campaignName'],
                tempRedeemFile['campaignOfferId'],
                tempRedeemFile['campaignOfferPromoId'],
                tempRedeemFile['event'],
                tempRedeemFile['timestamp'],
                tempRedeemFile['issuingPartner'],
                tempRedeemFile['consumer']['emailAddress'],
                tempRedeemFile['location'],
                tempRedeemFile['AztecAccountId']
            ]);
        });
        console.log(redeemArr.length)
        writer.end();
    }; // End redeemArr if

    glob.sync(`./${k}/CREATE*-${today}*.json` ).forEach (function( file) {
        createArr.push(file);
    }); 
    if (createArr.length > 0) {
        //Create Section

        var writer2 = csvWriter(
            { headers: createHeaders }  
        )
        writer2.pipe(fs.createWriteStream(`./ddnOutput/CREATE_${client_dir[k]}_${today}.csv`));
        //${client_dir[k]}

        createArr.forEach(function (file){
            var tempCreateFile = require(file);
            writer2.write([
                tempCreateFile['token'],
                tempCreateFile['campaignId'],
                tempCreateFile['campaignName'],
                tempCreateFile['campaignOfferId'],
                tempCreateFile['campaignOfferPromoId'],
                tempCreateFile['event'],
                tempCreateFile['timestamp'],
                tempCreateFile['meta']['offerId'],
                tempCreateFile['meta']['campaignId'],
                tempCreateFile['issuingPartner'],
                tempCreateFile['consumer']['emailAddress'],
                tempCreateFile['location']
            ])
        });
        console.log(createArr.length)

        writer2.end();
    }; // End createArr if


};
\$\endgroup\$

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.