1
\$\begingroup\$

Here's a few line written for web scraping using CasperJS. The code does what it should, but how can I improve it? For example, how can I make it more reusable? It could be also nice if I could remove the for loop?

It extracts some shop names, and their coordination points, from a page that keeps all the coordinates in a global JavaScript array, called ALLshops, and the shop names in h3 headers in a div called #shoplist, then puts them together in an array like string and saves it to hard disk.

    var shopNames = [];
    var casper = require('casper').create({
    verbose: false,
    logLevel: 'debug',
    pageSettings: {
        loadImages:  false,
        loadPlugins: false
        }
});
var fs = require('fs');
function getShopNames() {
    var shopNames = document.querySelectorAll('#shop-list h3');

    return Array.prototype.map.call(shopNames, function(e) {

        // return all the names
        return e.textContent.replace("Address ","");
    });
}

// I just removed the url 
casper.start('PAGE_URL_HERE_', function() {
});

casper.then(function() {
   var shopsCoords,
       i,
       j,
       shopsTotal,
       outputString,
       savePath, 
       fname = 'aleksi13.txt';
       finalList = [];

    var shopsCoords = casper.evaluate(function() {
        return ALLShops;
    });

    shopNames = this.evaluate(getShopNames);


  shopsTotal = shopsCoords.length;
  for(i = 0; i < shopsTotal; i++) {
          finalList.push('['+ shopsCoords[i].coordinates.lat,shopsCoords[i].coordinates.lng, JSON.stringify(shopNames[i]) + ']'); 

  }
    outputString = 'The Locations Are: \n ['+finalList+']'; 
    savePath = fs.pathJoin(fs.workingDirectory, 'output',fname);
    fs.write(savePath, outputString, 'w');
});


casper.run();
\$\endgroup\$

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.