Searching all new Reddit comments for comments matching regular expressions and notify of matches

Question

This program constantly sends an https request to Reddit's API for new comments.

Using a PostgreSQL Database containing Regular Expressions, find comments of interest and notify about them using faye.

When run locally, the information is sent to localhost:8080, otherwise data is sent to a heroku application.

Full code can be found here.

app.js - Entry point of program

// Local files
require('./DatabaseFetch.js')();
require('./CommonTools.js')();
const CommentSearchProcessor = require('./CommentFinder.js');
const RedditClientImport = require('./RedditClient.js');

const pg = require('pg');

const secondsTimeToWaitBetweenPostingSameCommentToASubreddit = 60 * 30;
const intervalToWaitInMillisecondsBetweenReadingComments = 1100;
const intervalToWaitBeforeSendingIdleMessage = 30;
const commentCacheSize = 2000;
const dissallowedSubreddits = ['suicidewatch', 'depression' ];
const userIgnoreList = ['agree-with-you'];

var lastMessageSentAt = new Date().getTime();

const clientConnection = isLocal() ? 'http://localhost:8000/' : 'http://reddit-agree-with-you.herokuapp.com/';

const faye = require('faye');
const client = new faye.Client(clientConnection);

var CommentFinder;
var redditClient = new RedditClientImport();

let commentHistory = GetUniqueArray(3000);
let subredditModsList = GetUniqueArray(3000);

console.log('is local?: ' + isLocal());
console.log('connecting to: ' + clientConnection);
console.log('Database URL: ' + process.env.DATABASE_URL);

if (!process.env.DATABASE_URL) {
  throw 'Please set process.env.DATABASE_URL! e.g SET DATABASE_URL=postgres://.....';
}

// Execute 
GetCommentSearchObjectsFromDatabase(pg, process.env.DATABASE_URL, function(commentSearchObjects) {
    CommentFinder = new CommentSearchProcessor(commentSearchObjects, commentCacheSize);
    console.log('starting...');
    start();
});

function start()
{
    client.publish('/messages', {message: 'starting up.'});

    setInterval(function() {
        redditClient.getCommentsFromSubreddit(RedditClientImport.MAX_NUM_POSTS, 'all', 'comments', function(comments) {
            comments.forEach(
                comment => {
                    var replyMessage = CommentFinder.searchComment(comment);

                    if (replyMessage)
                    {
                        // filter by disallowed subreddits
                        if (dissallowedSubreddits.includes(comment.subreddit.toLowerCase()))
                        {
                            console.log('Ignoring comment, disallowed subreddit found for comment: ');
                            console.log(comment);
                        }
                        else
                        {
                            processComment(comment, replyMessage);
                        }
                    }
                });
        });

        if (GetSecondsSinceTimeInSeconds(lastMessageSentAt) > intervalToWaitBeforeSendingIdleMessage)
        {
            console.log('sending inactive message');
            client.publish('/messages', {inactive: '1'});
            lastMessageSentAt = new Date().getTime();
        }
    }, intervalToWaitInMillisecondsBetweenReadingComments);
}

function processComment(comment, replyMessage)
{
    // So we don't spam a subreddit with the same message
    let timeThisReplyWasLastSubmittedOnThisSubreddit = {id: (comment.subreddit +  ':' + replyMessage), created: comment.created };
    let thisSubredditModList = {id: comment.subreddit};

    if (subredditModsList.includes(thisSubredditModList))
    {
        if (subredditModsList.get(thisSubredditModList).modList.includes(comment.author))
        {
            console.log('Modderator comment!!! :' + comment.author + ' comment: ' + comment.body);
            return;
        }
    }
    else
    {
        redditClient.getSubredditModList(thisSubredditModList.id, function(modList) { 
            thisSubredditModList.modList = modList;
            subredditModsList.push(thisSubredditModList);
            console.log('pushed: ' + thisSubredditModList.id);
            processComment(comment, replyMessage);
            return;
        });
    }

    if (userIgnoreList.includes(comment.author))
    {
        console.log('Skipping comment, is posted by: ' + comment.author + ' comment: ' + comment.body);
        return;
    }

    console.log('continue...');

    if (!commentHistory.includes(timeThisReplyWasLastSubmittedOnThisSubreddit))
    {
        publishComment(comment, replyMessage);
        commentHistory.push(timeThisReplyWasLastSubmittedOnThisSubreddit);
    }
    else
    {
        var existingComment = commentHistory.get(timeThisReplyWasLastSubmittedOnThisSubreddit);

        if (GetSecondsSinceUTCTimestamp(existingComment.created) > secondsTimeToWaitBetweenPostingSameCommentToASubreddit)
        {
            publishComment(comment, replyMessage);
            commentHistory.push(timeThisReplyWasLastSubmittedOnThisSubreddit);
        }
        else 
        {
            console.log('skipping comment, we\'ve already posted to this subreddit recently!');
            console.log(comment);
            console.log(commentHistory);
        }
    }
}

function publishComment(comment, replyMessage)
{
    console.log('posting comment');
    client.publish('/messages', {comment: comment, reply: replyMessage});
    lastMessageSentAt = new Date().getTime();
    console.log(comment);
    console.log('reply: ' + replyMessage);
}

/**
 * Checks if this program is currently running locally
 * We do this by checking if 'heroku' property is found in 'process.env._'
*/
function isLocal()
{
    return !(process.env._ && process.env._.indexOf("heroku"));
}

Table layout for "RegexpComment":

|subredditmatch|commentmatch|replymessage|id|
|--------------|------------|------------|--|

'SubredditMatch' - A regular expression containing Subreddits to find comments on. E.G '.*' for all
'commentMatch'   - A regular expression for the comment. For example '^hello (.*)' would find all comments starting with 'hello'
'replyMessage'   - The comment reply
'id'             - the primary key, auto incremented

Answer 1

Module inclusion

Some style guides recommend using import instead of require - e.g. the AirBnB style guide, and also recommend putting import statements above non-import statements (AirBnB, Google).

Why? Since imports are hoisted, keeping them all at the top prevents surprising behavior. ¹

Indentation Levels

The function start() contains a call to setInterval() that passes an anonymous function/closure, which calls a method on redditClient that also has a callback function parameter, which then calls the forEach() method on the comments argument with a callback function. That makes me tired just typing it out. This is what some would call Callback Hell - look at how many indentation levels exist. This can be remedied with a few techniques:

1. pulling out named functions
2. using ecmascript-8 keywords async / await if possible

For more tips see callbackhell.com

Variable declaration keywords

Some believe there is no use for var in modern JS, except if you intend to hoist variables. Some variables declared with let could be declared with const since they are not re-assigned - e.g. timeThisReplyWasLastSubmittedOnThisSubreddit, thisSubredditModList. Using const instead of let helps avoid accidental re-assignment and other bugs.

Constants in CONSTANT_CASE

Just as you do in rust and other languages, a common convention is to have true constants named in ALL_CAPS with underscores separating words (style guide rules: AirBnB, Google). This would apply to constants like secondsTimeToWaitBetweenPostingSameCommentToASubreddit, intervalToWaitInMillisecondsBetweenReadingComments, intervalToWaitBeforeSendingIdleMessage, commentCacheSize, etc.

Note that constants cannot be re-assigned but are immutable - so variables like dissallowedSubreddits can be modified via methods like push() unless they are frozen via Object.freeze().