4
\$\begingroup\$

I recently started to take a look at the Go programming language and decided to write a small project without practical use.

The objectives were:

  • Regularly poll the latest funding rates from https://ftx.com/api/funding_rates (I chose them, because each JSON object is rather small and because new updates are available by the hour)
  • Maintain a duplicate free database of all dbRecords == jsonObjects seen

Database setup

Dockerfile

FROM postgres:13
COPY *.sql /docker-entrypoint-initdb.d/

Database init query

-- Create Database and mark it as active so that all
-- consecutive commands are applied to this table
CREATE DATABASE crypto_mining;
\connect crypto_mining;

CREATE TABLE funding_rates(
    id SERIAL PRIMARY KEY,
    exchange TEXT NOT NULL,
    future TEXT NOT NULL,
    time TIMESTAMP NOT NULL,
    rate DOUBLE PRECISION DEFAULT 0.0
);

REST Api Miner

Dockerfile

FROM golang
WORKDIR /src

RUN go get github.com/lib/pq

COPY . .
ENTRYPOINT [ "go", "run", "ftxminer.go" ]

Polling the API

package main

import "fmt"
import "os"
import "time"
import "sort"
import "net/http"
import "io/ioutil"
import "encoding/json"
import "database/sql"
import _ "github.com/lib/pq"

type t_fundingRates struct {
    Future string    `json:"future`
    Rate   float64   `json:"rate"`
    Time   time.Time `json:"time"`
}

type t_fundingRates_apiResult struct {
    Result  []t_fundingRates `json:"result"`
    Success bool             `json:"success"`
}

// implement logic to sort t_fundingRates by time
type rateByTime []t_fundingRates

func (r rateByTime) Len() int {
    return len(r)
}

func (r rateByTime) Swap(i, j int) {
    r[i], r[j] = r[j], r[i]
}

func (r rateByTime) Less(i, j int) bool {
    return r[i].Time.Before(r[j].Time)
}

func getFundingRates(minedRates chan<- []t_fundingRates) {
    ticker := time.NewTicker(30 * time.Second)
    for {
        <-ticker.C

        // get response object with status code, result, etc
        httpResponse, err := http.Get("https://ftx.com/api/funding_rates")
        if nil != err {
            fmt.Printf(err.Error())
            continue
        }

        // extract application data
        httpResponseText, err := ioutil.ReadAll(httpResponse.Body)
        if nil != err {
            fmt.Printf(err.Error())
            continue
        }

        // parse application data
        httpResponseJson := t_fundingRates_apiResult{}
        if err := json.Unmarshal(httpResponseText, &httpResponseJson); err != nil {
            fmt.Printf(err.Error())
            continue
        }

        // if the api responds success
        if httpResponseJson.Success {
            // send data to consumer
            minedRates <- httpResponseJson.Result
        } else {
            fmt.Printf("api did not send funding rates")
            continue
        }
    }
}

func getIndex(rates []t_fundingRates, future string) int {
    for i, rate := range rates {
        if future == rate.Future {
            return i
        }
    }

    return -1
}

func storeFundingRates(minedRates <-chan []t_fundingRates) {
    psqlInfo := os.Getenv("DATABASE_URL")
    knownRates := make([]t_fundingRates, 0)       // rates we have seen previously
    updateCandidates := make([]t_fundingRates, 0) // rates we maybe need to compare to database and maybe update
    updateRates := make([]t_fundingRates, 0)      // rates we need to update in database

    for {
        // wait for funding rates to arrive
        rates := <-minedRates
        fmt.Println("Successfully retrieved", len(rates), "funding rates from ftx.com")

        // evaluate which rates we need to update
        updateRates = make([]t_fundingRates, 0)      // clear updateRates and
        updateCandidates = make([]t_fundingRates, 0) // updateCandidates first
        for _, rate := range rates {
            knownRatesIndex := getIndex(knownRates, rate.Future)

            if -1 == knownRatesIndex {
                // if we have not seen the future before, then add it to collection
                updateCandidates = append(updateCandidates, rate) // mark as need to update in database
            } else {
                // if we have seen the future earlier, then compare dates and decide whether we
                // need to update
                knownRate := knownRates[knownRatesIndex]

                if knownRate.Time.Before(rate.Time) {
                    knownRates[knownRatesIndex] = rate
                    updateRates = append(updateRates, rate)
                }
            }
        }

        // only pass this point if there is data to be inserted into the database
        if 0 == len(updateRates) && 0 == len(updateCandidates) {
            fmt.Println("No outdated or new futures found")
            continue
        } else {
            fmt.Println("Found", len(updateRates)+len(updateCandidates), "outdated or potentially new futures")
        }

        // prepare the database connection
        db, err := sql.Open("postgres", psqlInfo)
        if nil != err {
            fmt.Println(err.Error())
            continue
        }
        defer db.Close()

        // establish the connection and send a ping
        err = db.Ping()
        if nil != err {
            fmt.Println(err.Error())
            continue
        }
        fmt.Println("Successfully connected")

        // check for update candidates and potentially mark for inserting
        sort.Sort(rateByTime(updateCandidates))
        for _, candidate := range updateCandidates {

            candidateRecord := t_fundingRates{}
            err := db.QueryRow(
                "SELECT DISTINCT ON (future,time) future,time,rate FROM funding_rates WHERE exchange='ftx' AND future=$1 ORDER BY future,time DESC;",
                candidate.Future).Scan(
                &candidateRecord.Future,
                &candidateRecord.Time,
                &candidateRecord.Rate)
            isCandidateRecordValid := true

            // did we get the result? if not I am just going to assume that
            if err != nil {
                if err.Error() == "sql: no rows in result set" {
                    isCandidateRecordValid = false  
                    knownRates = append(knownRates, candidate)
                    updateRates = append(updateRates, candidate)
                } else {
                    fmt.Println(err.Error())
                    continue
                }
            }

            // add the future to the list of known futures
            if isCandidateRecordValid {
                if candidateRecord.Time.Before(candidate.Time) {
                    updateRates = append(updateRates, candidate) // mark as definite update

                    knownRatesIndex := getIndex(knownRates, candidate.Future) // things get a bit messy here due to duplicates in source data
                    if -1 == knownRatesIndex {
                        knownRates = append(knownRates, candidate)
                    } else {
                        knownRate := knownRates[knownRatesIndex]

                        if knownRate.Time.Before(candidate.Time) {
                            knownRates[knownRatesIndex] = candidate
                        }
                    }
                } else {
                    knownRates = append(knownRates, candidateRecord)
                }
            }
        }

        // insert guaranteed updates into database
        for _, update := range updateRates {
            _, err = db.Exec("INSERT INTO funding_rates (exchange , future , time , rate) VALUES ('ftx' , $1 , $2 , $3);", update.Future, update.Time, update.Rate)
            if nil != err {
                fmt.Println(err.Error())
                continue
            }
        }
    }
}

func main() {
    fmt.Println("Starting FTX Miner")

    chan_fundingRates := make(chan []t_fundingRates)
    go storeFundingRates(chan_fundingRates)
    go getFundingRates(chan_fundingRates)
    fmt.Println("... Funding Rates [OK]")

    sleepForever := make(chan string)
    <-sleepForever

    os.Exit(0)
}

Putting it all together in docker-compose.yml

version: '2.0'

services:
    database:
        build:
            context: ./database/
            dockerfile: ./Dockerfile
        environment:
            POSTGRES_PASSWORD: test123
        ports:
            - 5432:5432
        volumes:
            - databasevol:/var/lib/postgresql/data
        networks:
            - net

    ftxminer:
        build:
            context: ./ftx/
            dockerfile: ./Dockerfile
        depends_on:
            - database
        environment:
            DATABASE_URL: host=database port=5432 user=postgres password=test123 dbname=crypto_mining sslmode=disable
        networks:
            - net

volumes:
    databasevol:
        driver: local

networks:
    net:
        driver: bridge

Points that I found clumsy during writing the go code or that I am particularly interested in improving:

  • Is it the proper way to keep all go logic in one file?
  • Is it common that go code is permanently interrupted by small error checks (especially as in storeFundingRates)?
  • Is it common to distinguish error types by their error message (such as if err.Error() == "sql: no rows in result set" { in storeFundingRates?
\$\endgroup\$
4
\$\begingroup\$

Code should be correct, maintainable, robust, reasonably efficient, and, most importantly, readable. Code should be useful and have real value.

I found your code hard to read. Your code does not appear to be correct.


In Go we write:

if err != nil {
    // error handling
    return // or continue, etc.
}
// normal code

You confuse everybody, for no obvious reason, by writing:

if nil != err {
    // error handling
    return // or continue, etc.
}
// normal code

Why?


Your comments largely duplicate the code.


In Go, read the package documentation.

You write:

// implement logic to sort t_fundingRates by time
type rateByTime []t_fundingRates

func (r rateByTime) Len() int {
    return len(r)
}

func (r rateByTime) Swap(i, j int) {
    r[i], r[j] = r[j], r[i]
}

func (r rateByTime) Less(i, j int) bool {
    return r[i].Time.Before(r[j].Time)
}

sort.Sort(rateByTime(updateCandidates))

It is more readable to write:

func ratesByTime(r []t_fundingRates) {
    sort.Slice(r, func(i, j int) bool {
        return r[i].Time.After(r[j].Time)
    })
}

ratesByTime(updateCandidates)

Your code is more than a little ugly, so the sort.Slice function was added in Go 1.8. An Example in the sort package documentation clearly illustrates the difference:

Go Playground: https://play.golang.org/p/LZgZi-s8IX-

Why did you choose the complex, hard-to-read version?


In Go, read the package documentation.

The net/http package documentation states:

The client must close the response body when finished with it:

resp, err := http.Get("http://example.com/")
if err != nil {
  // handle error
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
// ...

You do not Close httpResponse.Body. Why?


In Go, read The Go Programming Language Specification.

In particular, Defer statements

A "defer" statement invokes a function whose execution is deferred to the moment the surrounding function returns, either because the surrounding function executed a return statement, reached the end of its function body, or because the corresponding goroutine is panicking.

Each time a "defer" statement executes, the function value and parameters to the call are evaluated as usual and saved anew but the actual function is not invoked. Instead, deferred functions are invoked immediately before the surrounding function returns, in the reverse order they were deferred.

You write:

func storeFundingRates(minedRates <-chan []t_fundingRates) {

    for {

        db, err := sql.Open("postgres", psqlInfo)
        if nil != err {
            fmt.Println(err.Error())
            continue
        }
        defer db.Close()

    }

}

go storeFundingRates(chan_fundingRates)

You appear to be in an infinite loop and so the function does not end. The deferred functioms accumulate but are never executed.


Here's a first cut at cleaning up your getFundingRates (renamed to mineFundingRates) function.

func getFundingRates() ([]t_fundingRates, error) {
    resp, err := http.Get("https://ftx.com/api/funding_rates")
    if err != nil {
        return nil, err
    }
    defer resp.Body.Close()
    respBody, err := ioutil.ReadAll(resp.Body)
    if err != nil {
        return nil, err
    }

    var respJson t_fundingRates_apiResult
    if err := json.Unmarshal(respBody, &respJson); err != nil {
        return nil, err
    }
    if !respJson.Success {
        err := fmt.Errorf("api did not send funding rates")
        return nil, err
    }

    return respJson.Result, nil
}

func mineFundingRates(minedRates chan<- []t_fundingRates) {
    ticker := time.NewTicker(30 * time.Second)
    for {
        <-ticker.C

        rates, err := getFundingRates()
        if err != nil {
            log.Printf(err.Error())
            continue
        }
        minedRates <- rates
    }
}

go mineFundingRates(chan_fundingRates)

There is more, but I'm out of time.

\$\endgroup\$
3
  • 1
    \$\begingroup\$ The answer to most of your Whys is, that I just started learning the language and don't yet have an efficient way to find what I need in the docs. I appreciate that you went through > 200 lines of code which you find hard to read and would be interested in reading the rest of your review, should you find the time to write it. Particularly in how you would suggest to improve readability. \$\endgroup\$
    – Benj
    Oct 11 '20 at 5:50
  • 1
    \$\begingroup\$ What do you mean by Your comments largely duplicate the code.? How do you use comments? \$\endgroup\$
    – Benj
    Oct 11 '20 at 7:42
  • \$\begingroup\$ @Benj I also think your comment may be a bit too much.The variable name and method names should express the meaning of usage. Short and cohesive method often does not need comments, the method name tells what they did.So when you want to use comment, you can first think about whether you can make the code more readable by refactoring.Eg. storeFundingRates is too long and too much comments. \$\endgroup\$ Oct 22 '20 at 6:49

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Not the answer you're looking for? Browse other questions tagged or ask your own question.