1
\$\begingroup\$

Essentially the AJAX component of this question, reimplemented in ES2017 style async/await. The code connects to a remote JSON API and scrapes it using a limited number of streams to avoid triggering rate limiting when the number of pages is large. Each stream retries on server-side errors and when rate limiting kicks in, up to 3 times by default. A rate limiting 429 response also results in a 5-second wait before further requests, increasing 50% each time up to 30 seconds. Other network errors result in a thrown error with a custom message.

const Module = {};

Module.HttpError = class HttpError extends TypeError {
    constructor(result = "Network Error", ...args) {
        super(result, ...args);
        this.message =
            "Network access failed with status code " +
            result.status + ": " + result.statusText +
            " at " + result.url + ". Check your network connection.";
    }
};

/**
 * Local copy of a remote API endpoint.
 */
Module.Remote = class Remote {
    constructor(path, query = {}) {
        this.remote = new URL(path, "https://online-go.com/api/v1/");
        this.remote.searchParams.set("format", "json");
        for (let i of Object.keys(query))
            this.remote.searchParams.set(i, query[i]);

        this.meta = fetch(new URL("?page_size=1", this.remote))
            .then(r=>{
                if(r.ok) {
                    r.json();
                } else {
                    throw new Module.HttpError(r);
                }
            });
    }

    /**
     * Fetches everything from the API path specified, using a number of streams.
     *
     * @param {number=} maxStreams - number of simultaneous streams.
     * @param {number=} pageSize - size of pages to fetch.
     *
     * @return {(Array<Object>|Object)} An array of pages
     */
    async scrape(maxStreams = 5, pageSize = 100) {
        const meta = await this.meta;
        if ("count" in meta) { // Check number of pages from API metadata.
            let target = this.remote;
            target.searchParams.set("page_size", pageSize);

            const numP = Math.ceil((meta.count)/pageSize);
            let q = Array.from({length: numP}, (v, i) => i+1);
            let pages = [];
            let wait = 0;

            /**
             * Creates a stream that fetches pages from the shared queue,
             * then saves the result to the array pages.
             *
             * @param {!URL} target - the URL and path of the remote.
             * @param {number} page - the page number to fetch.
             * @param {number=} retries - maximum number of retries before error.
             * @return {number} zero on sucessful completion.
             */
            async function stream (target, page, retries=3) {
                if(wait) await sleep(5000*1.5**wait);

                target.searchParams.set("page", page);
                let result = await fetch(target);

                if(result.ok) {
                    pages[page-1] = result.json();
                    // When queue is empty, return 0.
                    return q.length?stream(target, q.shift()):0;
                } else if(result.status === 429 && retries) {
                    wait>4||(wait+=1/maxStreams); // Max wait = 5 or ~ 30 secs
                    console.warn("Too many requests! Backing off.");
                    return stream(target, page, retries-1);
                } else if(500<=result.status && retries) {
                    console.warn("Server error, retrying");
                    return stream(target, page, retries-1);
                } else {
                    throw new dvt.HttpError(result);
                }
            }

            await Promise.all(Array.from({length: maxStreams}, v => stream(target, q.shift())));

            console.info("Final wait level: " + wait);

            this.pages = pages;
            return this.pages;
        } else {
            return this.meta;
        }
    }
};

export default Module;
\$\endgroup\$

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.