Here's a few line written for web scraping using CasperJS. The code does what it should, but how can I improve it? For example, how can I make it more reusable? It could be also nice if I could remove the for
loop?
It extracts some shop names, and their coordination points, from a page that keeps all the coordinates in a global JavaScript array, called ALLshops
, and the shop names in h3
headers in a div
called #shoplist
, then puts them together in an array like string and saves it to hard disk.
var shopNames = [];
var casper = require('casper').create({
verbose: false,
logLevel: 'debug',
pageSettings: {
loadImages: false,
loadPlugins: false
}
});
var fs = require('fs');
function getShopNames() {
var shopNames = document.querySelectorAll('#shop-list h3');
return Array.prototype.map.call(shopNames, function(e) {
// return all the names
return e.textContent.replace("Address ","");
});
}
// I just removed the url
casper.start('PAGE_URL_HERE_', function() {
});
casper.then(function() {
var shopsCoords,
i,
j,
shopsTotal,
outputString,
savePath,
fname = 'aleksi13.txt';
finalList = [];
var shopsCoords = casper.evaluate(function() {
return ALLShops;
});
shopNames = this.evaluate(getShopNames);
shopsTotal = shopsCoords.length;
for(i = 0; i < shopsTotal; i++) {
finalList.push('['+ shopsCoords[i].coordinates.lat,shopsCoords[i].coordinates.lng, JSON.stringify(shopNames[i]) + ']');
}
outputString = 'The Locations Are: \n ['+finalList+']';
savePath = fs.pathJoin(fs.workingDirectory, 'output',fname);
fs.write(savePath, outputString, 'w');
});
casper.run();