3
\$\begingroup\$

I recently created a small app in Go that generates random bson documents and insert them in a database (MongoDB). See this question for details.

After doing some profiling, I realised that garbage collection was taking a lot of time :

for inserting 1000000 JSON documents of ~220 bytes, top10 from pprof returns:

(pprof) top10
15340ms of 26740ms total (57.37%)
Dropped 230 nodes (cum <= 133.70ms)
Showing top 10 nodes out of 124 (cum >= 770ms)
      flat  flat%   sum%        cum   cum%
    4970ms 18.59% 18.59%     8630ms 32.27%  runtime.scanobject
    2570ms  9.61% 28.20%     2580ms  9.65%  runtime.greyobject
    1820ms  6.81% 35.00%     6550ms 24.50%  runtime.mallocgc
    1680ms  6.28% 41.29%     1680ms  6.28%  runtime.heapBitsForObject
     960ms  3.59% 44.88%     3860ms 14.44%  runtime.mapassign
     880ms  3.29% 48.17%      880ms  3.29%  runtime.memmove
     660ms  2.47% 50.64%     2490ms  9.31%  gopkg.in/mgo.v2/bson.(*encoder).addBytes
     620ms  2.32% 52.95%      620ms  2.32%  runtime.heapBitsSetType
     620ms  2.32% 55.27%      620ms  2.32%  runtime.memclrNoHeapPointers
     560ms  2.09% 57.37%      770ms  2.88%  runtime.mapiternext

Execution time was ~22s

So I reduced garbage collection frequency by disabling it and calling runtime.GC() manually. I got following results:

(pprof) top10
7120ms of 16320ms total (43.63%)
Dropped 128 nodes (cum <= 81.60ms)
Showing top 10 nodes out of 139 (cum >= 420ms)
      flat  flat%   sum%        cum   cum%
    1310ms  8.03%  8.03%     3120ms 19.12%  runtime.mallocgc
     900ms  5.51% 13.54%     2740ms 16.79%  runtime.mapassign
     840ms  5.15% 18.69%      840ms  5.15%  runtime.memmove
     720ms  4.41% 23.10%      720ms  4.41%  runtime.memclrNoHeapPointers
     700ms  4.29% 27.39%      700ms  4.29%  runtime.heapBitsSetType
     620ms  3.80% 31.19%     1340ms  8.21%  main.StringGenerator.Value
     610ms  3.74% 34.93%      610ms  3.74%  runtime.memhash
     530ms  3.25% 38.17%     1050ms  6.43%  runtime.scanobject
     470ms  2.88% 41.05%     7060ms 43.26%  gopkg.in/mgo.v2/bson.(*encoder).addElem
     420ms  2.57% 43.63%      420ms  2.57%  runtime.mapiternext

Execution time dropped to ~17s

But it doesn't feel right to manually call GC (maybe I'm wrong?), so I'm looking for a better way to reduce Garbage Collection and Memory allocation. Here is a simplified version of the code I'm currently using:

package main

import (
    "fmt"
    "math/rand"
    "runtime"
    "runtime/debug"
    "time"

    "gopkg.in/mgo.v2"
    "gopkg.in/mgo.v2/bson"
)

type IntGenerator struct {
    key string
    min int
    max int
}

func (g IntGenerator) Value(r rand.Rand) interface{} {
    return r.Intn(g.max-g.min) + g.min
}

type ArrayGenerator struct {
    size       int
    generators []IntGenerator
}

func (g ArrayGenerator) Value(r rand.Rand) interface{} {
    array := make([]interface{}, g.size)
    for i := 0; i < g.size; i++ {
        m := bson.M{}
        for _, gen := range g.generators {
            m[gen.key] = gen.Value(r)
        }
        array[i] = m
    }
    return array
}

func NewGeneratorsFromMap() []IntGenerator {
    gArr := make([]IntGenerator, 0)
    gArr = append(gArr, IntGenerator{key: "key1", min: 0, max: 10},
        IntGenerator{key: "key2", min: 110, max: 200},
        IntGenerator{key: "key3", min: 44, max: 11230},
        IntGenerator{key: "key4", min: 9048, max: 10000},
        IntGenerator{key: "key5", min: 0, max: 5},
        IntGenerator{key: "key5", min: 0, max: 5},
        IntGenerator{key: "key6", min: 200, max: 567},
        IntGenerator{key: "key7", min: 134, max: 12873737},
        IntGenerator{key: "key8", min: 245, max: 234211},
        IntGenerator{key: "key9", min: 12332, max: 23421},
        IntGenerator{key: "key10", min: 1, max: 20},
        IntGenerator{key: "key11", min: 17, max: 223},
        IntGenerator{key: "key12", min: 1, max: 1000},
        IntGenerator{key: "key13", min: 0, max: 22},
        IntGenerator{key: "key14", min: 489, max: 1345},
        IntGenerator{key: "key15", min: 345, max: 235988},
        IntGenerator{key: "key16", min: 1, max: 2234455})
    return gArr
}
func baseGenerator() ArrayGenerator {
    return ArrayGenerator{
        size:       1000,
        generators: NewGeneratorsFromMap(),
    }
}

func main() {
    var randSource = rand.New(rand.NewSource(time.Now().UnixNano()))

    debug.SetGCPercent(-1)
    session, err := mgo.Dial("mongodb://127.0.0.1:27017")
    if err != nil {
        fmt.Println(err.Error())
    }
    cr := session.DB("test").C("test")
    cr.DropCollection()

    generator := baseGenerator()
    // To make insertion faster, buffer the generated documents
    // and push them to a channel. The channel stores 3 x 1000 docs
    record := make(chan []interface{}, 3)
    // number of documents inserted
    count := 0
    // start []bson.M generation to feed the channel
    go func() {
        for count < 1000000 {
            record <- generator.Value(*randSource).([]interface{})
            count += generator.size
            if count%10000 == 0 {
                runtime.GC()
            }
        }
        close(record)
    }()
    // insert record in database from channel
    for r := range record {
        bulk := cr.Bulk()
        bulk.Unordered()
        bulk.Insert(r...)
        _, err := bulk.Run()
        if err != nil {
            fmt.Println(err.Error())
        }
    }
}

Edit:

I tried to implement a free list using sync.Pool to reduce memory allocations, but it didn't change anything. Am I using it correctly?

package main

import (
    "fmt"
    "math/rand"
    "runtime"
    "runtime/debug"
    "sync"
    "time"

    "gopkg.in/mgo.v2"
    "gopkg.in/mgo.v2/bson"
)

type GlobalObject struct {
    docs []interface{}
}

type IntGenerator struct {
    key string
    min int
    max int
}

func (g IntGenerator) Value(r rand.Rand) interface{} {
    return r.Intn(g.max-g.min) + g.min
}

type ArrayGenerator struct {
    size       int
    generators []IntGenerator
}

func (g ArrayGenerator) Value(r rand.Rand) interface{} {
    array := make([]interface{}, g.size)
    for i := 0; i < g.size; i++ {
        m := bson.M{}
        for _, gen := range g.generators {
            m[gen.key] = gen.Value(r)
        }
        array[i] = m
    }
    return array
}

func NewGeneratorsFromMap() []IntGenerator {
    gArr := make([]IntGenerator, 0)
    gArr = append(gArr, IntGenerator{key: "key1", min: 0, max: 10},
        IntGenerator{key: "key2", min: 110, max: 200},
        IntGenerator{key: "key3", min: 44, max: 11230},
        IntGenerator{key: "key4", min: 9048, max: 10000},
        IntGenerator{key: "key5", min: 0, max: 5},
        IntGenerator{key: "key5", min: 0, max: 5},
        IntGenerator{key: "key6", min: 200, max: 567},
        IntGenerator{key: "key7", min: 134, max: 12873737},
        IntGenerator{key: "key8", min: 245, max: 234211},
        IntGenerator{key: "key9", min: 12332, max: 23421},
        IntGenerator{key: "key10", min: 1, max: 20},
        IntGenerator{key: "key11", min: 17, max: 223},
        IntGenerator{key: "key12", min: 1, max: 1000},
        IntGenerator{key: "key13", min: 0, max: 22},
        IntGenerator{key: "key14", min: 489, max: 1345},
        IntGenerator{key: "key15", min: 345, max: 235988},
        IntGenerator{key: "key16", min: 1, max: 2234455})
    return gArr
}
func baseGenerator() ArrayGenerator {
    return ArrayGenerator{
        size:       1000,
        generators: NewGeneratorsFromMap(),
    }
}

func main() {
    var randSource = rand.New(rand.NewSource(time.Now().UnixNano()))

    debug.SetGCPercent(-1)
    session, err := mgo.Dial("mongodb://127.0.0.1:27017")
    if err != nil {
        fmt.Println(err.Error())
    }
    cr := session.DB("test").C("test")
    cr.DropCollection()
    // create the pool to avoid extra memory allocations 
    var objPool = sync.Pool{
        New: func() interface{} {
            return &GlobalObject{}
        },
    }

    generator := baseGenerator()
    // To make insertion faster, buffer the generated documents
    // and push them to a channel. The channel stores 3 x 1000 docs
    record := make(chan []interface{}, 3)
    // insert record in database from channel from a separate 
    // goroutine
    go func() {
        for r := range record {
            bulk := cr.Bulk()
            bulk.Unordered()
            bulk.Insert(r...)
            _, err := bulk.Run()
            if err != nil {
                fmt.Println(err.Error())
            }
        }
    }()
    // number of documents inserted
    count := 0
    // start []bson.M generation to feed the channel
    for count < 1000000 {
        r := objPool.Get().(*GlobalObject)
        // generate 1000 documents and send them to the channel
        r.docs = generator.Value(*randSource).([]interface{})
        record <- r.docs
        // return the object to the pool  
        objPool.Put(r)
        count += generator.size
        if count%10000 == 0 {
            runtime.GC()
        }
    }
    close(record)
}
\$\endgroup\$
  • \$\begingroup\$ Setting a big value to the percent (debug.SetGCPercent(1000)for instance ) could also help (prevents having to call runtime.GC() manually) \$\endgroup\$ – oliverpool Jul 21 '17 at 10:11

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.