3
\$\begingroup\$

I am creating a text editor that will need to work in the browser, syntax highlight, and allow users to edit files up to 50MB in size (i.e. the space allowed by indexedDB) without an "out of memory" error and without the file taking hours to load. My grand solution is to simply create an efficient parser that renderers dynamically by swapping out the text as the user scrolls through the file based on buddy-paired horizontal panels and each line swapping up-and down (recycling) as you scroll up and down. To all my current knowledge of DOM, this should be wicked fast, however it is quite evidently brutally slow in Chrome.

EDIT: I don't know why, by my code has a bitter dislike of codeReview (it bricks the page for some reason when I try to run it in a code snippet). So, please go to jsfiddle.net to see my code. I apologize for the inconvenience. Below is a static copy of the code (I dare not let you run it on codereview because, of course, it would likely brick your page too).

"use strict";
function getCharWidth(blockcontainer){
    var str = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789`~!@#$%^&*()_+-=[]{};\':"|\\?,./ ';
    var ele = document.createElement("span");
    ele.style.display = "inline-block";
    ele.textContent = str;
    blockcontainer.appendChild(ele);
    var width = ele.getBoundingClientRect().width / str.length;
    blockcontainer.removeChild(ele);
    return width;
}
function renderText(ele, text){
    var charWidth = getCharWidth(ele);
    var charHeight = parseFloat(getComputedStyle(ele).fontSize);
    var textData = [], /*dataCur = textData[0],*/ i=0, Len = text.length, curStack="", /*curLine=0,*/ curChar="";
    var maxLineWidth = 0;
    while (i !== Len){
        curChar = text[i];
        if (curChar === "\n" || curChar === "\r") {
            textData.push(curStack);
            maxLineWidth = Math.max(maxLineWidth, curStack.length);
            /*++curLine;*/
            curStack = "";
        } else {
            curStack += curChar
        }
        ++i;
    }
    var horozontalBufferLen = 2;
    //var pageElements = []; // for the two horozontal panels
    var lineElements = []; // for vertically shifting lines
    var width = Math.ceil(window.innerWidth / charWidth);
    var widthInPX = width * charWidth;
    var widthOverPX = Math.floor(widthInPX - (window.innerWidth / charWidth) * charWidth);
    var height = Math.ceil(window.innerHeight / charHeight);
    //var heightInPX = width * charHeight;
    //var heightOverPX = Math.floor(heightInPX - (window.innerHeight / charHeight) * charWidth);
    var heightOverPX = Math.floor(window.innerWidth % charHeight);
    var j = 0, i = 0, v = 0;
    while (j !== horozontalBufferLen){
        console.log("j going");
        var jCurLineArray = [];
        lineElements.push(jCurLineArray);
        i = 0;
        while (i !== height){
            var curElement = document.createElement("div");
            curElement.style.position = "absolute";
            curElement.style.top = i * charHeight + "px";
            var curCharArray = [];
            v = 0;
            while (v !== width){
                var curLetter = document.createElement("div");
                curLetter.style.position = "absolute";
                curLetter.style.top = "0px";
                curLetter.style.left = v * charWidth + "px";
                curLetter.style.color = "#111";
                var curText = document.createTextNode("");
                curCharArray.push(curText);
                curLetter.appendChild(curText);
                curElement.appendChild(curLetter);
                curElement
                ++v;
            }
            curElement.style.position = "absolute";
            curElement.style.left = 0;
            curElement.style.top = charWidth * i;
            ele.appendChild(curElement);
            jCurLineArray.push([curCharArray, curElement]);
            ++i;
        }
        ++j;
    }
    ele.insertAdjacentHTML("beforeend", '<div style="width:' + (maxLineWidth*charWidth) + 'px;height:' + (textData.length*charHeight) + 'px;pointer-events:none"></div>'); // this is just the placeholder to preserve the scrolling space
    console.log(lineElements);
    j = 0;
    var currentScrollTop = -0x7fffffff, currentScrollLeft = -0x7fffffff;
    function refreshLine(arr, top, left){ // refreshes a line of text shown to the user
        var lineData = textData[top];
        if (lineData){
            var lineLen = lineData.length;
            arr.forEach(function(txtNode, index){
                var newVal = (index+left) < lineLen ? lineData[index+left] : " ";
                if (txtNode.nodeValue !== newVal) txtNode.nodeValue = newVal;
            });
        } else {
            arr.forEach(function(txtNode, index){
                if (txtNode.nodeValue !== " ") txtNode.nodeValue = " ";
            });
        }
        // refresh a quadrent of the screen
    }
    var isCurrentlyGoing = false;
    function whenScroll(){
        var newScrollTop = ele.scrollTop;
        var newScrollLeft = ele.scrollLeft;

        var oldScrollTopSect  = Math.floor(currentScrollTop  / charHeight );
        var newScrollTopSect  = Math.floor(newScrollTop      / charHeight );
        var oldScrollLeftSect = Math.floor(currentScrollLeft / widthInPX  );
        var newScrollLeftSect = Math.floor(newScrollLeft     / widthInPX  );

        if (oldScrollTopSect === newScrollTopSect && oldScrollLeftSect === newScrollLeftSect) return;

        // to avoid having to update if the update is not needed
        //var needsonlyOneTop = newScrollTop % charHeight <= heightOverPX;
        var needsonlyOneLeft = newScrollLeft % widthInPX <= widthOverPX;

        var scrollLeftSectDiff = newScrollLeftSect - oldScrollLeftSect;
        if (scrollLeftSectDiff > 0){
            // user scrolled to the right ===>

        } else if (scrollLeftSectDiff < 0) {
            // <=== user scrolled to the left

        }

        var scrollTopSectDiff = newScrollTopSect - oldScrollTopSect;
        if (scrollTopSectDiff < 0){
            // user scrolled up
            var changedElementsFirst = lineElements[0].splice(scrollTopSectDiff);
            var changedElementsSecnd = lineElements[1].splice(scrollTopSectDiff);
            Len = lineElements[0].length, i = 0;
            lineElements[0].unshift.apply(lineElements[0], changedElementsFirst);
            lineElements[1].unshift.apply(lineElements[1], changedElementsFirst);
            while (i !== Len) {
                refreshLine(lineElements[0][i][0], newScrollTopSect+i, 0);
                lineElements[0][i][1].style.top = (newScrollTopSect+i) * charHeight + "px";
                i++;
            }
        } else {
            // user scrolled down
            var changedElementsFirst = lineElements[0].splice(0, scrollTopSectDiff);
            var changedElementsSecnd = lineElements[1].splice(0, scrollTopSectDiff);
            Len = lineElements[0].length, i = Len;
            lineElements[0].push.apply(lineElements[0], changedElementsFirst);
            lineElements[1].push.apply(lineElements[1], changedElementsFirst);
            while (i !== height) {
                refreshLine(lineElements[0][i][0], newScrollTopSect+i, 0);
                lineElements[0][i][1].style.top = (newScrollTopSect+i) * charHeight + "px";
                i++;
            }
        }
        currentScrollTop, currentScrollLeft = newScrollTop, newScrollLeft;
        isCurrentlyGoing = false;
    }
    ele.addEventListener("scroll", function(){
        if (!isCurrentlyGoing){
            isCurrentlyGoing = true;
            requestAnimationFrame(whenScroll);
        }
    }, {passive: 1});
    requestAnimationFrame(whenScroll);
    //ele.appendChild( document.createTextNode(text) );
}
///////////////////////////////////////////////////////////
//// The code below generates the example text content ////
///////////////////////////////////////////////////////////
var numToWords = (function(){
    const num = x => Number(x) || 0;
    const isEmpty = xs => xs.length === 0;
    const reverse = xs => xs.slice(0).reverse();
    const comp = f => g => x => f (g (x));
    const not = x => !x;
    const chunk = n => xs =>
    isEmpty(xs) ? [] : [xs.slice(0, n), ...chunk (n) (xs.slice(n))];
    const a = [
        '', 'one', 'two', 'three', 'four',
        'five', 'six', 'seven', 'eight', 'nine',
        'ten', 'eleven', 'twelve', 'thirteen', 'fourteen',
        'fifteen', 'sixteen', 'seventeen', 'eighteen', 'nineteen'
    ];
    const b = [
        '', '', 'twenty', 'thirty', 'forty',
        'fifty', 'sixty', 'seventy', 'eighty', 'ninety'
    ];
    const g = [
        '', 'thousand', 'million', 'billion', 'trillion', 'quadrillion',
        'quintillion', 'sextillion', 'septillion', 'octillion', 'nonillion'
    ];
    // this part is really nasty still
    // it might edit this again later to show how Monoids could fix this up
    const makeGroup = ([ones,tens,huns]) => {
        return [
            num(huns) === 0 ? '' : a[huns] + ' hundred ',
            num(ones) === 0 ? b[tens] : b[tens] && b[tens] + '-' || '',
            a[tens+ones] || a[ones]
        ].join('');
    };
    const thousand = (group,i) => group === '' ? group : group + " " + g[i];
    return n => {
        // "thousands" constructor; no real good names for this, i guess
        // execute !
        if (typeof n === 'number') return numToWords(String(n));
        if (n === '0')             return 'zero';
        return comp (chunk(3)) (reverse) (Array.from(n))
        .map(makeGroup)
        .map(thousand)
        .filter(comp(not)(isEmpty))
        .reverse()
        .join(' ');
    };
})();
var textToBinary = (new TextEncoder()).encode;
var binaryToText = (new TextDecoder()).decode;
var textToRender = "", i=256;
while (i--) {
    textToRender += numToWords(i**4) + "\n";
}
var renderElement = document.getElementById("codingarea");
renderText(renderElement, textToRender);
html, body {
    width: 100%;
    height: 100%;
    margin: 0;
}
#codingarea {
    position: relative;
    top: 0; bottom: 0;
    left: 0; right: 0;
    width: 100%;
    height: 100%;
    overflow: auto;
    overflow-y: scroll;
    white-space: pre;
    font-family: monospace;
    line-height: 1em;
    font-family: Cousine, monospace;
    line-height: 1em;
}
<div id="codingarea"></div>

P.S. I apologize, but there is not really any practical way to condense my code down further without removing the meaning of it. If you are like me and you don't like to spend awhile reading through someone else's code then I would recommend opening up the console and observing how the elements change as you scroll through the page. It should be fairly self-explanatory in the console's DOM tree view.

A potential solution I have thought up is to simply render the whole thing on a canvas. This would definitely be even faster, however it would take some time to implement, so I am curious if there is a pure DOM way to do it.

* NOTE * Please note that my following code above only has the vertical scrolling worked out. All I am concerned about for the moment is performance before I continue with the rest of my coding. Another thing to note is that there is a reason I make each letter individual instead of writing the whole line out in one sweep: syntax highlighting. Each letter must be able to be colored independently. I am willing to try out other methods of having each letter colored independently.

\$\endgroup\$
  • 1
    \$\begingroup\$ Hey, welcome to Code Review! When you used the word "bug" in the title, you weren't meaning that the code wasn't working correctly, just that it was really slow, right? I removed it because that's what it sounds like, I don't want others to have the impression that it's broken code, which would be off-topic. \$\endgroup\$ – Phrancis Mar 27 '18 at 3:25
  • \$\begingroup\$ @Phrancis Thank you for correcting that typo. You are very helpful. \$\endgroup\$ – Jack Giffin Mar 27 '18 at 3:25
  • 1
    \$\begingroup\$ I don't know if it's intentional, but I noticed a weird behavior running it on jsfiddle. If I scroll down the whole body of text, and I keep scrolling with my mouse wheel, it keeps adding new blank lines at the bottom. \$\endgroup\$ – Phrancis Mar 27 '18 at 3:29
3
\$\begingroup\$

Restart

I was going to give a detailed review on the code, but it quickly became apparent that the whole design is fundamentally flawed.

As a developing app I would say stop and start again, any more work on it as it is now is just pushing the brown stuff up hill.

Example of bad.

  • The user interface is counter intuitive. Selecting lines truncates them on copy.

  • A DOM element for each and every character! OMDG what were you thinking???

  • Storing the document in you context's RAM, this is a second copy, one on disk/storage as DB. Then your app creates a copy of that, (50MB text document WOW!! a lot to read) and then the first thing you do in the function is double+ the RAM usage by making a copy of it again ('textData' array), and then not deleting the unneeded original. No eyes will ever be able to read all 25,000,000 million characters yet you maintain 3 copies.

Yes 50MB is nothing these days, that is unless you have a low end device running on near dead battery, or a priority app sharing resources.

Some pointers

  • You should not be storing the whole document.

    The problem of the large data set is solved at the source of the data. You need to organised the data so you can retrieve a view quickly (maybe indexed by line number) When the user changes the view (eg scroll) then fetch/query DB for the relevant lines only.

  • Highlight strings of characters not individual characters eg "my text has <span class="highlight">highlighted</span> text"

  • Let the browser handle the hard work like styles and the horizontal scroll.

Look and Learn

Look at other examples of similar applications and learn from them. This particular type of app is not new and there is a lot of open source code to learn from.

Some sources of code examples are Google search, Github, MDN (MDN's DB indexed API reference will have some data retrieval examples I am sure) and even the fiddle page you gave. The code windows are syntax highlighted.

Yay

I will say it is good to see you use requestAnimationFrame to trigger visual updates. This is seldom used and is best practice for any sort of visual DOM changes.

\$\endgroup\$
  • \$\begingroup\$ I love your solution, however a few things. I know that having each character separate is bad, but it's a necessary evil. I will soon be implementing code highlighting where each letter may need individual highlighting. Also fetching/DB querying is pointless. The user can load the full 50MB into memory at once. Sure, it would be effective if I was using a lower level programming language like C++ where there would be a significant performance difference, but there is simply not. IndexedDB was not necessarily designed for segmented reading, and reading everything in one sweep is more important \$\endgroup\$ – Jack Giffin Mar 28 '18 at 0:39
  • 1
    \$\begingroup\$ Even if each letter needs individual highlighting, you should contemplate highlighting strings in the case of several characters in a row with same color. And if you calculate the color most used, you can skip having tags for those and make it default color with css \$\endgroup\$ – juvian Mar 28 '18 at 14:28
  • 1
    \$\begingroup\$ Well they say that it works for files up to 4000000 lines, which assuming 20 chars per line would be 80mb \$\endgroup\$ – juvian Mar 28 '18 at 15:58
  • 1
    \$\begingroup\$ @lolzerywowzery I have played around with indexDB. I Added random lines of text ~170chars per line till the storage was near full. I then timed random access via ranged cursor to get a page of lines at a time. The average rate was approx 15000 lines per second or 42000 unicode chars (84000 8bit chars) per frame (50fps) DB access is async and there is plenty of time between reads to format the text. And if there was a slowdown you can offload it all to workers keeping the main thread smooth as silk And if that still taxes the device you can get more through put using compressed text. \$\endgroup\$ – Blindman67 Mar 28 '18 at 15:58
  • 1
    \$\begingroup\$ @lolzerywowzery The problem is that we do not know what the content is, how much entropy it contains, what the expected edits will be. A 50mb text file (UNicode) represents 25 million characters. If all words (and natural language) then with a fast simple two stage tokenized dictionary (16bit) you can reduce the size of the text to about 5mb (maintain random line access) . This gives you room to add edits to the DB in bulk form for lazy update in idle time. But you have to accept that without native code and direct access to the device storage there are limitations you can not get past \$\endgroup\$ – Blindman67 Mar 28 '18 at 18:17

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Not the answer you're looking for? Browse other questions tagged or ask your own question.