Using a cronjob on node for storing a json file, parsing it and update a database

Question

I need to retrieve a file from a website every minute that contains boats location.

Then, I parse the file, delete all the boats from my database and re-create all of them at the new position indicated in the file.

Here's my code so far - in the server.js of my loopbackjs app:

var CronJob = require('cron').CronJob;
var exec = require("child_process").exec;
var reload = require('require-reload')(require);
var app = module.exports = loopback();

// Bootstrap the application, configure models, datasources and middleware.(loopbackjs mandatory code)
boot(app, __dirname, function (err) {
    if (err)
        throw err;

    var Boat = app.models.Boat;
    var ais_url_france = 'http://data.aishub.net/ws.php?username=XXXXXXXXX&format=1&output=json&compress=0&latmin=40&latmax=52&lonmin=-5&lonmax=10';

    //start the cronjob on a node env every min
    new CronJob('* * * * *', function () {
                // get the file and save as a .json
                exec('wget "' + ais_url_france + '" -O aishub_france.json', function (error, stdout, stderr) {
                    // I need to use reload here because the file is stored in memory otherwise and only read once after the first download...
                    var file = reload('../aishub_france.json');
                    if (file[0].ERROR === false) {
                        var boats = file[1];
                        if (boats.length > 0) {
                            Boat.destroyAll({typeId: 'aishub_boats'}, function (err, info) {
                                for (var i = 0; i < boats.length; i++) {
                                    Boat.create({
                                        name: boats[i].NAME,
                                        latlng: {
                                            lat: boats[i].LATITUDE,
                                            lng: boats[i].LONGITUDE
                                        },
                                        typeId: 'aishub_boats'
                                    });
                                }
                            });
                        }
                    }
                });
            }, null, true, null);
});

I'm not sure the code here is so great but here are my troubles:

• The downloaded file is huge (1,3Mo)
• I need to remove and re-add all the boats because they change location and some might appear/disappear (maybe there is a better solution)
• Not sure using require-reload is the best solution, but if I don't the file is read only once in the cronjob even if the donwloaded file changed each time...

• I think I might have some memory leak because I'm having strange stacks trace recently after I added this code. Here is the trace:

  33503717 ms: Mark-sweep 1288.0 (1403.6) -> 1288.0 (1404.6) MB, 672.5 / 0.0 ms [allocation failure] [GC in old space requested].

  33504750 ms: Mark-sweep 1288.0 (1404.6) -> 1288.0 (1404.6) MB, 847.3 / 0.0 ms [allocation failure] [GC in old space requested].

  33505597 ms: Mark-sweep 1288.0 (1404.6) -> 1288.0 (1403.6) MB, 846.4 / 0.0 ms [last resort gc].

  33506438 ms: Mark-sweep 1288.0 (1403.6) -> 1288.0 (1403.6) MB, 841.4 / 0.0 ms [last resort gc].

Answer 1

OK, thank you James!

I was able to get rid of require-reload using request, and use the compression with adm-zip on the fly:

var CronJob = require('cron').CronJob;
var request = require('request');
var AdmZip = require('adm-zip');

var ais_url_france = 'http://data.aishub.net/ws.php?username=XXX&format=1&output=json&compress=1&latmin=40&latmax=52&lonmin=-5&lonmax=10';
new CronJob('* * * * *', function () {
     request({url: ais_url_france, encoding: null}, function (error, response, data) {
            if (!error && response.statusCode === 200) {
                var zip = new AdmZip(data);
                var zipEntries = zip.getEntries();
                if (zipEntries && zipEntries.length) {
                    var file = JSON.parse(zip.readAsText(zipEntries[0]));
                    if (file[0].ERROR === false) {
                        var boats = file[1];
                        if (boats.length > 0) {
                            Boat.destroyAll(function (err, info) {
                                for (var i = 0; i < boats.length; i++) {
                                    Boat.create({
                                        name: boats[i].NAME,
                                        latlng: {
                                            lat: boats[i].LATITUDE,
                                            lng: boats[i].LONGITUDE
                                        }
                                    });
                                }
                            });
                        }
                    }
                }
            }
        });
    }, null, true, null);