Simple syntax highlighter - part 2

Question

I've completely revamped my old syntax highlighter, and made it more flexible and easy to use. It works the same way as the previous one, wrapping <span> elements around sections of text in an element with certain styles, but it now allows for string highlighting, and is more extensible by allowing the user to define their own highlighting styles.

/**
 * Define a new syntax for highlighting.
 * @param {Array} keywords - A list of strings representing the language keywords.
 * @param {Array} globals  - A list of strings representing the language globals.
 * @param {Array} literals - A list of strings used to match numbers, strings, etc.
 */
function syntax(keywords, globals, literals, styles) {
    return {
        keywords: keywords,
        globals:  globals,
        literals: literals,
        styles:   styles,
    };
}


/**
 * This array contains some built-in
 * language defaults. You can create
 * your own though!
 */
var DEFAULTS = {
    JavaScript: syntax(
        ["\\babstract\\b",     "\\barguments\\b",  "\\bboolean\\b",   "\\bbreak\\b",    
         "\\bbyte\\b",         "\\bcase\\b",       "\\bcatch\\b",     "\\bchar\\b",       
         "\\bclass\\b",        "\\bconst\\b",      "\\bcontinue\\b",  "\\bdebugger\\b",  
         "\\bdefault\\b",      "\\bdelete\\b",     "\\bdo\\b",        "\\bdouble\\b",    
         "\\belse\\b",         "\\benum\\b",       "\\beval\\b",      "\\bexport\\b",
         "\\bextends\\b",      "\\bfalse\\b",      "\\bfinal\\b",     "\\bfinally\\b",      
         "\\bfloat\\b",        "\\bfor\\b",        "\\bfunction\\b",  "\\bgoto\\b",     
         "\\bif\\b",           "\\bimplements\\b", "\\bimport\\b",    "\\bin\\b",         
         "\\binstanceof\\b",   "\\bint\\b",        "\\binterface\\b", "\\blet\\b",       
         "\\blong\\b",         "\\bnative\\b",     "\\bnew\\b",       "\\bpackage\\b",  
         "\\bprivate\\b",      "\\bprotected\\b",  "\\bpublic\\b",    "\\breturn\\b",       
         "\\bshort\\b",        "\\bstatic\\b",     "\\bsuper\\b",     "\\bswitch\\b",       
         "\\bsynchronized\\b", "\\bthrow\\b",      "\\bthrows\\b",    
         "\\btransient\\b",    "\\btrue\\b",       "\\btry\\b",       "\\btypeof\\b",    
         "\\bvar\\b",          "\\bvoid\\b",       "\\bvolatile\\b",  "\\bwhile\\b",    
         "\\bwith\\b",         "\\byield\\b"],

        ["\\bnull\\b",      "\\bundefined\\b", "\\btrue\\b", "\\bfalse\\b",
         "\\bprototype\\b", "\\bthis\\b"],

        ["\\b\\d+\\b", "(('.*')|(\".*\"))", "\/\/.*"],

        {
            keywords: "color:blue",
            globals:  "color:LightBlue",
            literals: "color:Gray",
        }
    ),
    Python: syntax(
        ["\\band\\b",     "\\bas\\b",       "\\bassert\\b", "\\bbreak\\b", 
         "\\bclass\\b",   "\\bcontinue\\b", "\\bdef\\b",    "\\bdel\\b", 
         "\\belif\\b",    "\\belse\\b",     "\\bexcept\\b", "\\bexec\\b", 
         "\\bfinally\\b", "\\bfor\\b",      "\\bfrom\\b",   "\\bglobal\\b", 
         "\\bif\\b",      "\\bimport\\b",   "\\bin\\b",     "\\bis\\b", 
         "\\blambda\\b",  "\\bnot\\b",      "\\bor\\b",     "\\bpass\\b", 
         "\\bprint\\b",   "\\braise\\b",    "\\breturn\\b", "\\btry\\b", 
         "\\bwhile\\b",   "\\bwith\\b",     "\\byield\\b",  "\\bself\\b"],

        ["\\babs\\b",        "\\bdict\\b",    "\\bhelp\\b",        "\\bmin\\b", 
         "\\bsetattr\\b",    "\\ball\\b",     "\\bdir\\b",         "\\bhex\\b",     
         "\\bnext\\b",       "\\bslice\\b",   "\\bany\\b",         "\\bdivmod\\b",     
         "\\bid\\b",         "\\bobject\\b",  "\\bsorted\\b",      "\\bascii\\b",   
         "\\benumerate\\b",  "\\binput\\b",   "\\boct\\b",         "\\bstaticmethod\\b",
         "\\bbin\\b",        "\\beval\\b",    "\\bint\\b",         "\\bopen\\b",    
         "\\bstr\\b",        "\\bbool\\b",    "\\bexec\\b",        "\\bisinstance\\b",  
         "\\bord\\b",        "\\bsum\\b",     "\\bbytearray\\b",   "\\bfilter\\b",  
         "\\bissubclass\\b", "\\bpow\\b",     "\\bsuper\\b",       "\\bbytes\\b",   
         "\\bfloat\\b",      "\\biter\\b",    "\\bprint\\b",       "\\btuple\\b",
         "\\bcallable\\b",   "\\bformat\\b",  "\\blen\\b",         "\\bproperty\\b",    
         "\\brange\\b",      "\\bvars\\b",    "\\bclassmethod\\b", "\\bgetattr\\b", 
         "\\blocals\\b",     "\\brepr\\b",    "\\bzip\\b",         "\\bcompile\\b", 
         "\\bglobals\\b",    "\\bmap\\b",     "\\breversed\\b",    "\\b__import__\\b",
         "\\bcomplex\\b",    "\\bhasattr\\b", "\\bmax\\b",         "\\bround\\b",    
         "\\bdelattr\\b",    "\\bhash\\b",    "\\bmemoryview\\b",  "\\bset\\b",
         "\\bTrue\\b",       "\\bFalse\\b",   "\\bNone\\b"],

        ["\\b\\d+\\b", "((\".*\")|('.*'))"],

        {
            keywords: "color:Gold",
            globals:  "color:Purple",
            literals: "color:blue",
        }
    ),
};


/**
 * Highlight an element with a set syntax.
 * @param element - The element to highlight.
 * @param syntax  - The syntax to use.
 */
function highlight(element, syntax) {
    var keywordsRegex = new RegExp(syntax.keywords.join("|"), "g");
    var globalsRegex  = new RegExp(syntax.globals.join("|"), "g");
    var literalsRegex = new RegExp(syntax.literals.join("|"), "g");

    var keywordStyles = syntax.styles.keywords;
    var globalStyles  = syntax.styles.globals;
    var literalStyles = syntax.styles.literals;

    $(element).html($(element).text().replace(
        keywordsRegex,
        function(stringToReplace) {
            return "<span style=" + keywordStyles + ">" + stringToReplace + "</span>";
    }).replace(
        globalsRegex,
        function(stringToReplace) {
            return "<span style=" + globalStyles + ">" + stringToReplace + "</span>"
    }).replace(
        literalsRegex,
        function(stringToReplace) {
            return "<span style=" + literalStyles + ">" + stringToReplace + "</span>"
    }));
}


/**
 * Run the highlight function on an
 * element here with it's specific
 * syntax.
 */
$(document).ready(function() {
    highlight('.code-js', DEFAULTS.JavaScript);
    highlight('.code-py', DEFAULTS.Python);
})

If you want to find a live demo, it can be found at this link here. Do note that I am still working on this, so the contents in this link are liable to change at any point.

Answer 1

It's certainly an improvement over the first iteration. However, all those repeated \\bs are begging to be abstracted somehow. I'd go crazy having to write those when defining the rules. My answer to your previous question gave an example of how that could be avoided.

I also wonder why it's limited to 3 categories: Keywords, literals, and globals. I know those are the same three I mentioned in my answer on your previous question, but that was partly just an example.

What will you do with comments, for example? Right now they seem to fall under the heading "literal". But comments aren't literals; they're comments. Similarly, what about, say, pre-processor statements (in languages that have those)? What if you want to highlight built-in language constructs differently (e.g. echo in PHP, which isn't really a function). Or maybe you want to highlight "invalid" code like return followed by a linebreak in JavaScript. Or operators like *, ++, and instanceof?

My point is, that there's no reason to limit yourself to 3 categories. Besides, the syntax function doesn't provide much functionality. In fact, it obscures the code somewhat.

Right now, you have syntax(arg1, arg2, arg3, arg4). Well, what are those parameters? You'll have to go look at the syntax function to find out. Even if you know what parameters there are, can you remember their order? Was it globals first, or keywords?

But if you instead just skip syntax and write your object inline, the code gets a lot more declarative:

{
  keywords: arg1,
  globals: arg2,
  literals: arg3
}

Now it's clear what the args mean, because they're named in-place. And you can add whatever categories you want; builtin, macro, comment, invalid, etc. etc..

Of course, you still need to style all that, but I'd again suggest using class names and a separate stylesheet for that. Colors are presentation, not information. Semantically, something might be a keyword, but that does not imply that it's always colored blue.

Right now, you'd have to change your highlighting engine whenever you want to change colors. But browsers already have a powerful means of separating information from presentation: Stylesheets.

It's doubly powerful, because you can apply multiple classes to an element. E.g. you may want the option of styling literal strings differently from literal numbers. But they're both literals. So you could do:

{
  "literal number": <pattern to match numbers>,
  "literal string": <pattern to match strings>
}

Presuming those create elements like <code class="literal number">...</code>, you can style both types of literals using .literal { color: something }, or, if you want to differentiate, you can add style like .literal.number { color: something-else }. It's still a literal, but it's a specific kind of literal. Same with operators: = and == are both operators, but one is an assignment operator, and the other is a comparison operator. Whether that distinction matters to the highlighter is up to you (I probably wouldn't bother), but it is a distinction.

That'll let you make high-level stylesheets, that color the "main" components of a syntax (say, .keyword, .comment, .literal or something), and fine-grained stylesheets, with rules for .comment.multi-line, or .keyword.function-definition, etc..

In fact, this is close to how editors like TextMate and Sublime define grammars (Sublime in fact just uses TextMate's language grammars): Multiple levels of specificity.

Last thing you may want to look into is how to resolve conflicts between highlighting rules. For instance, what about code examples inside multi-line comments? It's still a comment, not code, and should be colored as such. Or what about keywords likes if appearing inside a string? It's not a keyword anymore. Conversely, the double-slash in a URL string, could easily make the rest of the line look like a comment, but it's not.

I'll leave that as an exercise to the reader. Be warned, though, that it's not the simplest task; syntax highlighting can get very involved.