I'm using HapiJS to create a little backend for my side-project. In grabbing RSS feeds, I want to loop through my list of website URLs, grab the correct RSS links in the <head>, then loop through the RSS feeds, pulling out the articles, to return an aggregated JSON response (via the reply() call in HapiJS), to have all these sites articles in one response for my API.
I'm fumbling with a couple of things:
- The async nature of Node/JavaScript and doing what I want here in the most efficient/canonical manner
- And getting my data within the innermost loop to the outer one to return (scope).
This code console.logs all the article data correctly, but I'm unsure how to properly place the callback() to get these arrays of articles from each site, into one larger array that I will eventually return. This is the one controller in my HapiJS backend that is handling this (suggestions on how to break this up better welcome as well).
var feed = require('feed-read');
var request = require('request');
var _ = require('lodash');
var finder = require('find-rss');
var async = require('async');
module.exports = blogs;
function blogs () {
var articles;
}
blogs.getAll = function (req, reply) {
//TODO: Handle facebook tech blog which probably requires an accept header properly set
//TODO: Store all proper web/rss feeds somewhere
//TODO: periodically loop through them and get new articles, and store them
//TODO: Clean up API instead of one monolithic endpoint
//iterate over all sites
var sites = [
'http://eng.joingrouper.com',
'http://eng.rightscale.com',
'https://engineering.linkedin.com/blog',
'http://blog.stackoverflow.com/',
'http://googledevelopers.blogspot.com/',
'http://yahooeng.tumblr.com/',
'http://code.flickr.net/',
'http://engineering.pinterest.com/',
'https://blog.twitter.com/engineering',
'http://blog.42floors.com/',
'http://engineering.flipboard.com/',
'http://codeascraft.com/',
'http://engineering.foursquare.com/',
'http://instagram-engineering.tumblr.com/',
'https://code.facebook.com/posts/',
'https://labs.spotify.com/',
'http://engineering.voxer.com/',
'http://www.ebaytechblog.com/',
'http://tech.gilt.com/',
'http://engineering.heroku.com/',
'http://www.thumbtack.com/engineering/',
'http://engineering.silk.co/',
'https://www.paypal-engineering.com/',
'http://code.zynga.com/',
'http://blog.risingstack.com/',
'https://tech.blog.box.com/',
'https://engineering.groupon.com/',
'http://dev.hubspot.com/blog',
'http://code.hootsuite.com/',
'http://www.buzzfeed.com/techblog',
'https://medium.com/medium-eng',
'http://code.mixpanel.com'
];
var feeds = [];
var articles = [];
var href;
//var results = blogs.getAllFeeds(sites, articles);
// 1st para in async.each() is the array of items
async.each(sites,
// 2nd param is the function that each item is passed to
function(site, callback){
// Call an asynchronous function, often a save() to DB
console.log('start first site..');
// Async call is done, alert via callback
//Discover feed
finder(site, function(error, response, body){
if (error) {
return error;
}
if(response[0]){
href = response[0]['href'];
}
//TODO: handle these conditions better
if(href === '/atom.xml' ||
href === '/rss/' ||
href === '/feed.xml'){
//Make full urls out of those sites that reference their feed relatively
href = site + href;
}
//Parse feed for articles
feed(href, function(err, articles) {
if (err) {
console.log("SITE: " + site);
console.log("Error: " + err);
console.log("---------------------------------------------");
} else {
var _articles = [];
_.forEach(articles, function(article) {
//console.log(article['title']);
_articles.push(article['title'], article['link']);
});
console.log(_articles);
}
});
});
blogs.checkDone(sites);
},
// 3rd param is the function to call when everything's done
function(err, _articles, articles){
if(err){
console.log(err);
}
console.log('done');
reply('ap');
// All tasks are done now
//checkDone(sites);
}
);
if(!articles){
reply(articles);
}
};
blogs.checkDone = function(object) {
_.after(object.length, function(){
console.log('done here');
});
};
