1
\$\begingroup\$

I usually do automation and integration testing with Selenium but have heard good things about some of the Node headless frameworks. So I put this project together to get some experience with Nightmare.js and various aspects of Node that I haven't worked with as much.

Project

At this point it is all basically working well and I think I did an alright job with it but I don't know any really good Node developers to tell me otherwise. I'm planning on consolidating and improving the debug/logging aspect as well as switching the model to a query builder (probably Knex) for ease of use and back-end support.

Questions / Feedback

  • One concrete question I have is for automated testing. I'm having a hard time seeing how I would go about testing this other than saving some of the relevant pages and using those as mocks.

  • Are there any things I've done obviously wrong from a more experienced perspective in regards to Node standards, performance, best practices, etc?

index.js

'use strict';

const ig       = require('./lib/indiegala');
const steam    = require('./lib/steam');
const priority = require('./lib/enter-giveaways');

steam.updateOwnedGames();

ig.parseGiveawaysList()
    .then(steam.scrape);

priority
    .prioritizeGiveaways()
    .then( ig.enterGiveaways )
    .then( ig.checkWins )
    .then( ig.close )
    .catch( (err) => {
        console.error(err);
        process.exit();
    });

lib/indiegala.js

'use strict';

// modules
const nconf       = require('./config');
const Nightmare   = require('nightmare');
const async       = require('async');
const request     = require('request');
const cheerio     = require('cheerio');
const model       = require('./model');
const fs          = require('fs');
const Q           = require('q');
const colors      = require('colors');

// config / globals
const baseUrl     = nconf.get('indiegala:baseUrl');

// IndieGala Constants
const IG_NOT_AUTHORIZED   = 'You are not authorized access for this giveaway.';
const IG_NOT_ENOUGH_COINS = 'Insufficient Indiegala Coins. Please choose a cheaper giveaway.';

/**
 * The number of giveaway listing pages to parse. There are 12 giveaways per page.
 *
 * @type {number}
 */
const pagesToParse = nconf.get('indiegala:pagesToParse');


// setup Nightmare instance
// require('nightmare-iframe-manager')(Nightmare);
const nmConfig  = nconf.get('nightmare');
nmConfig.show   = true; // show for manual login
const nmInst    = Nightmare(nmConfig);
nmInst.useragent(nmConfig.userAgent);


/**
 * Process giveaway pages using Request/Cheerio to grab data and store in DB.
 * @param {string[]} links  Relative links to individual giveaways
 */
function processGiveawayPages(links) {
    async.each(links, (link, next) => {
        const giveawayUrl = baseUrl + link;
        request(giveawayUrl, (err, resp, body) => {
            if (err) {
                console.error('Error loading giveaway url: \n' + err);
                next(err);
                return;
            }

            const $ = cheerio.load(body);

            // get end date from thier JS and convert to unix ts
            let endTime    = -1;
            const found    = body.match(/new Date\(Date\.UTC.*;/g);
            if (null !== found) {
                let getEndTime = new Function('return ' + found[0]);
                endTime        = Math.round(getEndTime().getTime() / 1000);
            }

            // get the steam url and remove trailing slashes
            let steamUrl = $('.ticket-info-cont .steam-link').attr('href');
            if (steamUrl.endsWith('/')) {
                steamUrl = steamUrl.slice(0, -1);
            }

            const steamId = steamUrl.substring(steamUrl.lastIndexOf('/') + 1);

            const giveaway = {
                'id':       $('.ticket-right .relative').attr('rel'),
                'name':     $('.ticket-info-cont h2').text(),
                'price':    $('.ticket-left .ticket-price strong').text(),
                'endDate':  endTime,
                'level':    $('.type-level-cont').text().trim(),
                steamUrl,
                steamId
            };

            model.insertGiveaway(giveaway);

            next();
        });
    }, (err) => {
        // all finished
        if (err) {
            console.error(err);
        }
    });
}


/////////////////////// Electron Context Functions ////////////////////

/**
 * Scrape the links to individual game links. This is run in Nightmare/Electron browser context.
 *
 * @return {string[]} Array of URLs to individual giveaway pages
 */
function parseGameLinks() {
    // eslint-disable-next-line no-var, no-undef
    var links = document.querySelectorAll('.ticket-info-cont h2 a');

    return Array.prototype.map.call(links, function(e) {
        return e.getAttribute('href');
    });
}

/**
 * Returns data about the attempted giveaway. This is run in Nightmare/Electron browser context.
 *
 * @return {Object}
 */
function getDataFromGiveaway() {
    return {
        entered: $('.giv-coupon').length === 0,
        title: document.title,
        coins: document.querySelector('.coins-amount').title,
        error: $('.warning-cover:visible span').text()
    };
}

////////////////////////////////////////////////////////////////////

/**
 * Parses pages of giveaways in series, count and level set in config.
 *
 * On each page, grab the URLs to each Giveaway details page. Then pass the
 * urls to {@link processGiveawayPages} for individual parsing.
 *
 * @returns {Promise}
 */
function parseGiveawaysList() {
    const deferred = Q.defer();
    // Setup local Nightmare and start processing pages
    const nmLocal = Nightmare(nconf.get('nightmare'));
    nmLocal.useragent(nconf.get('nightmare:userAgent'));

    async.timesSeries(pagesToParse + 1, (n, next) => {
        // timesSeries is zero-based, the links are 1 based so skip
        if (0 === n) {
            next();
        }
        else {
            const level = nconf.get('indiegala:level');
            const thisPage = baseUrl + '/giveaways/' + n + '/expiry/asc/level/' + level;
            nmLocal
                .goto(thisPage)
                .wait('.giveaways-main-page')
                .evaluate(parseGameLinks)
                .then((links) => {
                    // console.log(links);
                    processGiveawayPages(links);
                    console.log('Finished ' + thisPage);
                    next(null, links);
                })
                .catch((err) => {
                    console.error('Nightmare error loading giveaway list page: ');
                    console.error(colors.red(err));
                    next(err, links);
                });
        }

    }, (err, allLinks) => {
        if (err) {
            console.error('All Finished Error: ' + err);
            return;
        }


        setTimeout(() => {
            close(nmLocal);
            deferred.resolve();
        }, 3000);
    });

    return deferred.promise;
}

/**
 * Insert a random delay, between configurable milliseconds using nconf for defaults
 *
 * IG will log you out without this (1 to 3 seconds seems to be the min)
 */
function rndDelay(low, high) {
    low  = low  || nconf.get('delayMs:low');
    high = high || nconf.get('delayMs:high');

    return Math.floor(Math.random() * (high - low + 1) + low);
}


//////////////////// Enter Giveaway functions ////////////////

/**
 * Next item callback for async.each style
 *
 * @see {@link http://caolan.github.io/async/docs.html#.each | async.each Docs}
 * @callback asyncNextItem
 * @param {(string|Object)} [error]
 */

/**
 *
 * @param {Object} giveaway - Giveaway obj with url and id
 * @param {Object} data - Results obj returned from {@link getDataFromGiveaway}
 * @param {asyncNextItem} next - Callback to continue to next item
 * @return {bool}
 */
function shouldRetryGiveaway(giveaway, data, next) {
    console.log(data.coins + '\t' + data.title);

    if (data.entered) {
        model.markAsEntered(giveaway.id);
        next();

        return false;
    }

    if (data.coins === '0 Indiegala Coins') {
        next('No More Coins');

        return false;
    }

    if (data.error === IG_NOT_AUTHORIZED) {
        console.log('--- Level Not High Enough');
        next();

        return false;
    }

    if (data.error === IG_NOT_ENOUGH_COINS) {
        console.log('--- Not Enough Coins');
        next();

        return false;
    }


    return true;
}



/**
 * For Nightmare.use() - goes to the url, clicks enter and
 * runs {@link getDataFromGiveaway} in the Electron context.
 */
function gotoAndClickTicket(url) {
    return (nightmare) => {
        nightmare
            .goto(url)
            .wait(rndDelay())
            .click('.giv-coupon')
            .wait(rndDelay())
            .evaluate(getDataFromGiveaway);
    }
}

/**
 * Enter the supplied giveaways and mark them as such
 *
 * @returns {Promise}
 */
function enterGiveaways(giveaways) {
    console.log(colors.green('Giveaways In Queue: ' + giveaways.length));

    const deferred = Q.defer();

    async.eachSeries(giveaways, (giveaway, next) => {
        nmInst
            .use(gotoAndClickTicket(giveaway.url))
            .then( (data) => {
                // console.log(data);
                if (shouldRetryGiveaway(giveaway, data, next)) {
                    console.log('Could not dectect succsessful entery, retry'.yellow);
                    return nmInst
                        .use(gotoAndClickTicket(giveaway.url))
                        .then( (data) => {
                            if (shouldRetryGiveaway(giveaway, data, next)) {
                                next();
                            }
                        });
                }
            })
            .catch((err) => {
                if (err === 'Unable to find element by selector: .giv-coupon') {
                    model.markAsEntered(giveaway.id);
                    next();
                }
                else {
                    console.error(err);
                    next('Nightmare Error entering giveaway: ' + err.message);
                }
            });
    }, (err) => {
        if (err) {
            console.error(err);
            // deferred.reject(err);
        }
        // all giveaways entered
        console.log('All Giveaways Entered');
        deferred.resolve();
    });

    return deferred.promise;
}

/**
 * Clicks all the 'Check if you won!' buttons
 */
function clickAllCheckForWinButtons(buttonCount, deferred) {
    console.log('Found Buttons: ' + buttonCount);

    async.timesSeries(buttonCount, (n, next) => {
        nmInst
            .click('.btn-check-if-won')
            // without this wait it tries to click the same button
            .wait(rndDelay(4000, 6000))
            .then(() => {
                next();
            });
    }, err => {
        if (err) {
            console.error(err);
            deferred.reject(err);
        }
        // all giveaways entered
        console.log('All Completed Giveaways Checked');
        deferred.resolve();
    });
}

/**
 * Go to profile page and check any completed ones for wins
 */
function checkWins() {
    console.log('Check for Wins');
    const deferred = Q.defer();

    nmInst
        .goto('https://www.indiegala.com/profile')
        .wait('#open-giveaways-library')
        .click('#open-giveaways-library')
        .click('.giveaway-completed .open-library')
        .wait(rndDelay(4000, 6000))
        .evaluate(() => {
            return document.querySelectorAll('.btn-check-if-won').length;
        })
        .then((buttonCount) => {
            clickAllCheckForWinButtons(buttonCount, deferred);
        })
        .catch((err) => {
            console.error(err);
        });

    return deferred.promise;
}


/**
 * End the Nightmare/Electron session
 */
function close(aNightmareInst) {
    aNightmareInst = aNightmareInst || nmInst;
    aNightmareInst
        .goto(baseUrl)
        .end()
        .catch((err) => {
            console.error(err);
        });
}

module.exports = {
    login,
    checkWins,
    enterGiveaways,
    parseGiveawaysList,
    close
};

lib/model.js

'use strict';
const nconf       = require('./config');
const Q           = require('q');
const sqlite3     = require('sqlite3').verbose();
const db          = new sqlite3.Database(nconf.get('sqliteFile'));

sqlite3.verbose();

// setup tables
const createGamesTableSql = `
    CREATE TABLE IF NOT EXISTS games (
        steamId INTEGER,
        reviewText TEXT,
        reviewStats TEXT,
        genre TEXT,
        metascore INTEGER,
        tag1 TEXT,
        tag2 TEXT,
        tag3 TEXT,
        shortDesc TEXT,
        PRIMARY KEY(steamId)
    );
`;

const creatGiveawaysTableSql = `
    CREATE TABLE IF NOT EXISTS giveaways (
        id INTEGER,
        name TEXT,
        steamUrl TEXT,
        price NUMERIC,
        endDate INTEGER,
        steamId INTEGER,
        entered INTEGER DEFAULT 0,
        PRIMARY KEY(id)
    );
`;

const createGamesOwnedTableSql = `
    CREATE TABLE IF NOT EXISTS games_owned (
        steamId INTEGER,
        PRIMARY KEY(steamID)
    );
`;

// setup queries


const newGamesToDetailSql = `
    SELECT DISTINCT steamUrl, steamId
    FROM giveaways
    WHERE steamId IS NOT NULL
    AND steamId NOT IN (SELECT DISTINCT steamId FROM games)
`;

db.serialize();
db.run(createGamesTableSql);
db.run(creatGiveawaysTableSql);
db.run(createGamesOwnedTableSql);
db.parallelize();

// giveaways queries
const insertGiveaway = db.prepare(
    'INSERT INTO giveaways (id, name, steamUrl, price, endDate, steamId) ' +
    'VALUES (?, ?, ?, ?, ?, ?)');

const markAsEntered = db.prepare('UPDATE giveaways SET entered = 1 WHERE id = ?');

// games queries
const insertGame = db.prepare('INSERT INTO games VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)');

// games_owned queries
const insertOwnedGame = db.prepare('INSERT INTO games_owned VALUES (?)');


module.exports = {

    // direct interface to the db
    db,

    // set giveaways.entered = 1 for the provided id
    markAsEntered: (giveawayId) => {
        markAsEntered.run(giveawayId, (updateError) => {
            if (updateError) {
                console.error(updateError);
            }
            else {
                // console.log('Updated ' + giveawayId);
            }
        });
    },

    // insert new row to giveaways
    insertGiveaway: (giveaway) => {
        insertGiveaway.run(
            giveaway.id,
            giveaway.name,
            giveaway.steamUrl,
            giveaway.price,
            giveaway.endDate,
            giveaway.steamId,
            (insertErr) => {
                if (insertErr) {
                    // ignore PK insert errors
                    if (! insertErr.message.includes('UNIQUE constraint failed')) {
                        console.error(insertErr);
                    }
                }
            }
        );
    },

    // insert new row to games
    insertGame: (game) => {
        insertGame.run(
            game.steamId,
            game.reviewText,
            game.reviewStats,
            game.genre,
            game.metascore,
            game.tag1,
            game.tag2,
            game.tag3,
            game.shortDesc,
            (insertErr) => {
                if (insertErr) {
                    // ignore PK insert errors
                    if (! insertErr.message.includes('UNIQUE constraint failed')) {
                        console.error(insertErr);
                    }
                }
            }
        );
    },

    // insert new row to games
    insertOwnedGame: (steamId) => {
        insertOwnedGame.run(
            steamId,
            (insertErr) => {
                if (insertErr) {
                    // ignore PK insert errors
                    if (! insertErr.message.includes('UNIQUE constraint failed')) {
                        console.error(insertErr);
                    }
                }
            }
        );
    },

    // returns promise for array of objs with steamUrl and steamId
    getNewGamesToDetail: () => {
        return new Q.Promise((fulfill, reject) => {
            db.all(newGamesToDetailSql, (err, rows) => {
                if (err) {
                    console.error(err);
                    reject(err);
                }

                fulfill(rows);
            });
        });
    }
};
\$\endgroup\$

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.