Go efficiently retrieve and process large volume of AWS s3 objects

Question

Like the title says, code below tries to efficiently grab and handle a large number (1m+) of objects from an aws s3 bucket. Right now on a 10gb lambda function it can handle ~2m objects in ~10s

End goal is to use this to dump the object keys into an sqs queue for further processing, hence the list/process separation. It's faster than my first naive approach but if there's still room for improvement or clarity please let me know!

package main

import (
    "fmt"
    "os"
    "sync"

    "github.com/aws/aws-sdk-go/aws"
    "github.com/aws/aws-sdk-go/aws/credentials"
    "github.com/aws/aws-sdk-go/aws/session"
    "github.com/aws/aws-sdk-go/service/s3"
)

func makePrefixListTwoHexChars(prefix string) []string {
    charset := "0123456789abcdef"
    charsetTwoChars := make([]string, 0)

    for _, char1 := range charset {
        for _, char2 := range charset {
            charsetTwoChars = append(charsetTwoChars, prefix + string(char1) + string(char2))
        }
    }

    return charsetTwoChars
}

func handleCount(countChan, finalCountChan chan int) {
    count := 0
    
    for x := range countChan {
        count += x
        if (count+1) % 10000 == 0 {
                fmt.Println(count+1)
        }
    }
    
    finalCountChan <- count
    close(finalCountChan)

}

func handleFinalCount(finalCountChan chan int) int {
    count := 0
    for finalCount := range finalCountChan {
        count = finalCount
    }
    return count
}

func listAllPrefixes(svc *s3.S3, dataBucket string, prefixList []string, listObjectChan chan *s3.ListObjectsV2Output) *sync.WaitGroup {
    var bucketListWG = &sync.WaitGroup{}
    
    for _, prefix := range prefixList {
            bucketListWG.Add(1)    
            go listPrefix(svc, &dataBucket, prefix, listObjectChan, bucketListWG)
    }
    
    return bucketListWG
}

func listPrefix(svc *s3.S3, bucketname *string, prefix string, listObjectChan chan *s3.ListObjectsV2Output, bucketReadWG *sync.WaitGroup) { 
    defer bucketReadWG.Done()

    err := svc.ListObjectsV2Pages(&s3.ListObjectsV2Input{
        Bucket: bucketname,
        Prefix: &prefix,
    }, func(page *s3.ListObjectsV2Output, _ bool) (shouldContinue bool) {
     listObjectChan <- page
        return true
    })
    
    if err != nil {
        fmt.Println("failed to list objects: ", err)
        return
    }
}

func processAllListObjectOutputs(processListObject func(*s3.Object) error, numWorkers int, countChan chan int, listObjectChan chan *s3.ListObjectsV2Output) *sync.WaitGroup {
    var processListObjectWG = &sync.WaitGroup{}

    for i := 0; i < numWorkers; i++ {
        processListObjectWG.Add(1)
        go processListObjectOutput(processListObject, countChan, listObjectChan, processListObjectWG)
    }

    return processListObjectWG
}

func processListObjectOutput(processListObject func(*s3.Object) error, countChan chan int, listObjectChan chan *s3.ListObjectsV2Output, processListObjectWG *sync.WaitGroup) {
    defer processListObjectWG.Done()
    
    for page := range listObjectChan {
        for _, obj := range page.Contents {
            countChan <- 1
            if err := processListObject(obj); err != nil {
                fmt.Println("failed to process object: ", err)
            }
        }
    }
}

func main() {
    dataBucket := os.Getenv("DATA_BUCKET")
    region := os.Getenv("AWS_REGION")
    prefixSplit := os.Getenv("SPLIT")
    
    prefixList := makePrefixListTwoHexChars(prefixSplit)
    processListObject := func(obj *s3.Object) error {
        return nil
    }

    sess, err := session.NewSession()
    svc := s3.New(sess, &aws.Config{
        Region: aws.String(region),
        Credentials: credentials.AnonymousCredentials,
    })    
    if err != nil {
        fmt.Println("Error creating session ", err)
    }

    countChan := make(chan int)
    finalCountChan := make(chan int)
    listObjectChan := make(chan *s3.ListObjectsV2Output)

    fmt.Println("Starting...")

    go handleCount(countChan, finalCountChan)
    
    objectProcessWG := processAllListObjectOutputs(processListObject, len(prefixList), countChan, listObjectChan)
    bucketListWG := listAllPrefixes(svc, dataBucket, prefixList, listObjectChan)

    bucketListWG.Wait()
    close(listObjectChan)
    objectProcessWG.Wait()
    close(countChan)

    fmt.Println("Ending...")

    finalCount := handleFinalCount(finalCountChan)

    fmt.Println("Total objects processed: ", finalCount)
}

To run locally on 125k images of the open images dataset set the env vars:

AWS_REGION=us-east-1
DATA_BUCKET=open-images-dataset
SPLIT=test/