I have this working code, but would like your review:
sync.coffe
path = require "path"
Promise = require "bluebird"
DBFParser = require "node-dbf"
request = require "request"
config = require "config"
zlib = require "zlib"
gzipPr = Promise.promisify zlib.gzip, zlib
postPr = Promise.promisify request.post, request
parseAccountRecord = (dbfr) ->
# SNAME.SN_DORMANT = record no longer active - we don't care
if dbfr['@deleted'] or dbfr.SN_DORMANT is "T" then return null
{
# Account meta
snAccount: dbfr.SN_ACCOUNT
snStop: dbfr.SN_STOP
snTprfl: dbfr.SN_TPRFL #
snCprfl: dbfr.SN_CPRFL
snCustype: dbfr.SN_CUSTYPE
snTerrtry: dbfr.SN_TERRTRY
# Account invoice details
snVat: dbfr.SN_VRN
snContact: dbfr.SN_CONTACT
snEmail: dbfr.SN_EMAIL
snName: dbfr.SN_NAME
snAddr1: dbfr.SN_ADDR1
snAddr2: dbfr.SN_ADDR2
snAddr3: dbfr.SN_ADDR3
snAddr4: dbfr.SN_ADDR4
snPstcode: dbfr.SN_PSTCODE
snCtry: dbfr.SN_CTRY
snTelno: dbfr.SN_TELENO
snWwwpage: dbfr.SN_WWWPAGE
}
filesInfo =
"c_sname.dbf" : [ parseAccountRecord, "account"]
uploadRecords = (records, endPoint) ->
url = config.webapp.baseUrl + "/api/" + endPoint
process.stdout.write "...Pushing #{records.length} #{endPoint} to #{url}, "
gzipPr new Buffer(JSON.stringify(records))
.then (gzbuf) ->
postOptions =
url: url
body: gzbuf
headers:
"Content-type": "application/json"
"Content-Encoding": "gzip"
"Content-Length": gzbuf.length
postPr postOptions
.spread (response, body) ->
console.log "pushed", body
parseNPush = (basename, parseFunc, apiEndPoint) ->
console.log ">>>"
console.log " ...parsing #{basename}"
fullPath = path.join config.opera.dataDir , basename
parseDBFFile( fullPath, parseFunc )
.then( (records) -> uploadRecords(records, apiEndPoint) )
.catch( (err) -> console.error(err) )
onFileChanged = (fname) ->
fname = path.basename fname
fInfo = filesInfo[fname]
if not fInfo then return
parseNPush fname, fInfo[0], fInfo[1]
parseAll = ->
Promise.map Object.keys(filesInfo), onFileChanged, {concurrency: 1}
.then ->
console.log "All records parsed -- Done, re-load in 60s..."
setTimeout parseAll, config.parseInterval
then on the node.js application I have
api.coffee
express = require "express"
filters = require "../filters"
_ = require "lodash"
config = require "config"
redisClient = require("redis-url").connect(config.database.redis.url)
Promise = require "bluebird"
hmsetPr = Promise.promisify redisClient.hmset, redisClient
exports.router = () ->
express.Router()
.post "/account", account
.use (req,res) -> res.status(404).send("Invalid API Call")
account = (req, res, next) ->
snProfileMap = _(req.body).map( (r) ->
profileMap = {
snAccount: r.snAccount
snStop: r.snStop
snTprfl: r.snTprfl
snCprfl: r.snCprfl
snCustype: r.snCustype
snTerrtry: r.snTerrtry
}
[r.snAccount, JSON.stringify(profileMap)] ).zipObject().value()
snInvoiceMap = _(req.body).map( (r) ->
invoiceMap = {
snAccount: r.snAccount
snVat: r.snVat
snContact: r.snContact
snEmail: r.snEmail
snName: r.snName
snAddr1: r.snAddr1
snAddr2: r.snAddr2
snAddr3: r.snAddr3
snAddr4: r.snAddr4
snPstcode: r.snPstcode
snCtry: r.snCtry
snTelno: r.snTelno
snWwwpage: r.snWwwpage
}
[r.snAccount, JSON.stringify(invoiceMap)] ).zipObject().value()
hmsetPr "profile", snProfileMap
hmsetPr "invoice", snInvoiceMap
.then (result) -> res.status(200).send("Ok")
.catch next
Basically, I am sending one large json file, this is then split into two parts and stored each part inside a different Redis set, so in this case profile
and invoice
Is there a better way to re-write sync.coffee
, keeping in mind that the JSON file will grow over time?
I was thinking of parsing the record twice, but it does not make sense as this is done already and sending it to two different end-points!
Any advice much appreciated