I recently started to take a look at the Go programming language and decided to write a small project without practical use.
The objectives were:
- Regularly poll the latest funding rates from https://ftx.com/api/funding_rates (I chose them, because each JSON object is rather small and because new updates are available by the hour)
- Maintain a duplicate free database of all
dbRecords == jsonObjects
seen
Database setup
Dockerfile
FROM postgres:13
COPY *.sql /docker-entrypoint-initdb.d/
Database init query
-- Create Database and mark it as active so that all
-- consecutive commands are applied to this table
CREATE DATABASE crypto_mining;
\connect crypto_mining;
CREATE TABLE funding_rates(
id SERIAL PRIMARY KEY,
exchange TEXT NOT NULL,
future TEXT NOT NULL,
time TIMESTAMP NOT NULL,
rate DOUBLE PRECISION DEFAULT 0.0
);
REST Api Miner
Dockerfile
FROM golang
WORKDIR /src
RUN go get github.com/lib/pq
COPY . .
ENTRYPOINT [ "go", "run", "ftxminer.go" ]
Polling the API
package main
import "fmt"
import "os"
import "time"
import "sort"
import "net/http"
import "io/ioutil"
import "encoding/json"
import "database/sql"
import _ "github.com/lib/pq"
type t_fundingRates struct {
Future string `json:"future`
Rate float64 `json:"rate"`
Time time.Time `json:"time"`
}
type t_fundingRates_apiResult struct {
Result []t_fundingRates `json:"result"`
Success bool `json:"success"`
}
// implement logic to sort t_fundingRates by time
type rateByTime []t_fundingRates
func (r rateByTime) Len() int {
return len(r)
}
func (r rateByTime) Swap(i, j int) {
r[i], r[j] = r[j], r[i]
}
func (r rateByTime) Less(i, j int) bool {
return r[i].Time.Before(r[j].Time)
}
func getFundingRates(minedRates chan<- []t_fundingRates) {
ticker := time.NewTicker(30 * time.Second)
for {
<-ticker.C
// get response object with status code, result, etc
httpResponse, err := http.Get("https://ftx.com/api/funding_rates")
if nil != err {
fmt.Printf(err.Error())
continue
}
// extract application data
httpResponseText, err := ioutil.ReadAll(httpResponse.Body)
if nil != err {
fmt.Printf(err.Error())
continue
}
// parse application data
httpResponseJson := t_fundingRates_apiResult{}
if err := json.Unmarshal(httpResponseText, &httpResponseJson); err != nil {
fmt.Printf(err.Error())
continue
}
// if the api responds success
if httpResponseJson.Success {
// send data to consumer
minedRates <- httpResponseJson.Result
} else {
fmt.Printf("api did not send funding rates")
continue
}
}
}
func getIndex(rates []t_fundingRates, future string) int {
for i, rate := range rates {
if future == rate.Future {
return i
}
}
return -1
}
func storeFundingRates(minedRates <-chan []t_fundingRates) {
psqlInfo := os.Getenv("DATABASE_URL")
knownRates := make([]t_fundingRates, 0) // rates we have seen previously
updateCandidates := make([]t_fundingRates, 0) // rates we maybe need to compare to database and maybe update
updateRates := make([]t_fundingRates, 0) // rates we need to update in database
for {
// wait for funding rates to arrive
rates := <-minedRates
fmt.Println("Successfully retrieved", len(rates), "funding rates from ftx.com")
// evaluate which rates we need to update
updateRates = make([]t_fundingRates, 0) // clear updateRates and
updateCandidates = make([]t_fundingRates, 0) // updateCandidates first
for _, rate := range rates {
knownRatesIndex := getIndex(knownRates, rate.Future)
if -1 == knownRatesIndex {
// if we have not seen the future before, then add it to collection
updateCandidates = append(updateCandidates, rate) // mark as need to update in database
} else {
// if we have seen the future earlier, then compare dates and decide whether we
// need to update
knownRate := knownRates[knownRatesIndex]
if knownRate.Time.Before(rate.Time) {
knownRates[knownRatesIndex] = rate
updateRates = append(updateRates, rate)
}
}
}
// only pass this point if there is data to be inserted into the database
if 0 == len(updateRates) && 0 == len(updateCandidates) {
fmt.Println("No outdated or new futures found")
continue
} else {
fmt.Println("Found", len(updateRates)+len(updateCandidates), "outdated or potentially new futures")
}
// prepare the database connection
db, err := sql.Open("postgres", psqlInfo)
if nil != err {
fmt.Println(err.Error())
continue
}
defer db.Close()
// establish the connection and send a ping
err = db.Ping()
if nil != err {
fmt.Println(err.Error())
continue
}
fmt.Println("Successfully connected")
// check for update candidates and potentially mark for inserting
sort.Sort(rateByTime(updateCandidates))
for _, candidate := range updateCandidates {
candidateRecord := t_fundingRates{}
err := db.QueryRow(
"SELECT DISTINCT ON (future,time) future,time,rate FROM funding_rates WHERE exchange='ftx' AND future=$1 ORDER BY future,time DESC;",
candidate.Future).Scan(
&candidateRecord.Future,
&candidateRecord.Time,
&candidateRecord.Rate)
isCandidateRecordValid := true
// did we get the result? if not I am just going to assume that
if err != nil {
if err.Error() == "sql: no rows in result set" {
isCandidateRecordValid = false
knownRates = append(knownRates, candidate)
updateRates = append(updateRates, candidate)
} else {
fmt.Println(err.Error())
continue
}
}
// add the future to the list of known futures
if isCandidateRecordValid {
if candidateRecord.Time.Before(candidate.Time) {
updateRates = append(updateRates, candidate) // mark as definite update
knownRatesIndex := getIndex(knownRates, candidate.Future) // things get a bit messy here due to duplicates in source data
if -1 == knownRatesIndex {
knownRates = append(knownRates, candidate)
} else {
knownRate := knownRates[knownRatesIndex]
if knownRate.Time.Before(candidate.Time) {
knownRates[knownRatesIndex] = candidate
}
}
} else {
knownRates = append(knownRates, candidateRecord)
}
}
}
// insert guaranteed updates into database
for _, update := range updateRates {
_, err = db.Exec("INSERT INTO funding_rates (exchange , future , time , rate) VALUES ('ftx' , $1 , $2 , $3);", update.Future, update.Time, update.Rate)
if nil != err {
fmt.Println(err.Error())
continue
}
}
}
}
func main() {
fmt.Println("Starting FTX Miner")
chan_fundingRates := make(chan []t_fundingRates)
go storeFundingRates(chan_fundingRates)
go getFundingRates(chan_fundingRates)
fmt.Println("... Funding Rates [OK]")
sleepForever := make(chan string)
<-sleepForever
os.Exit(0)
}
Putting it all together in docker-compose.yml
version: '2.0'
services:
database:
build:
context: ./database/
dockerfile: ./Dockerfile
environment:
POSTGRES_PASSWORD: test123
ports:
- 5432:5432
volumes:
- databasevol:/var/lib/postgresql/data
networks:
- net
ftxminer:
build:
context: ./ftx/
dockerfile: ./Dockerfile
depends_on:
- database
environment:
DATABASE_URL: host=database port=5432 user=postgres password=test123 dbname=crypto_mining sslmode=disable
networks:
- net
volumes:
databasevol:
driver: local
networks:
net:
driver: bridge
Points that I found clumsy during writing the go code or that I am particularly interested in improving:
- Is it the proper way to keep all go logic in one file?
- Is it common that go code is permanently interrupted by small error checks (especially as in
storeFundingRates
)? - Is it common to distinguish error types by their error message (such as
if err.Error() == "sql: no rows in result set" {
instoreFundingRates
?