This is my solution (available in the Go Playground herehere — I've written code from the Fetch struct to the Crawl function, the rest was already provided by the exercise):
package main
import (
"fmt"
"sync"
"time"
"math/rand"
)
type Fetcher interface {
Fetch(url string) (body string, urls []string, err error)
}
type FetchLinks struct {
url urls string[]string
depth int
}
type CacheConcurrentStringSet struct {
visitedset map[string]bool
mux sync.Mutex
}
func (ccache *Cache*ConcurrentStringSet) onMissDoAsync(s string, f func()) {
ccache.mux.Lock()
defer ccache.mux.Unlock()
if !ccache.visited[s]set[s] {
ccache.visited[s]set[s] = true
go f()
}
}
func NewAsyncCrawler(fetcher Fetcher, timeout time.Duration) (crawl func(Fetchstring, int)) {
resultslinks := make(chan FetchLinks)
cache := CacheConcurrentStringSet{visitedset:make(map[string]bool)}
crawl = func(fetch Fetch) {
url ifstring, fetch.depth <= 0int) {
return
}
cache.onMissDoAsync(fetch.url, func() {
body, urls, err := fetcher.Fetch(fetch.url)
if err != nil {
fmt.Println(err)
} else return{
}
fmt.Printf("found: %s %q\n", fetch.url, body)
if fetch.depth > 1 {
for _, url := rangelinks urls<- Links{
urls, depth - 1}
}
if !cache.visited[url] {
}
})
results <-for Fetch{url, fetch.depth
- 1}
select {
}
case link := <-links:
}
for _, url := range link.urls }{
})
select {
go crawl(url, link.depth)
case next := <-results:
}
crawl(next)
case <-time.After(timeout):
return
}
}
}
return
}
func Crawl(url string, depth int, fetcher Fetcher) {
if (depth > 0) {
crawl := NewAsyncCrawler(fetcher, 500 * time.Millisecond)
crawl(Fetch{url, depth})
}
}
func main() {
Crawl("http://golang.org/", 4, fetcher)
}
type fakeFetcher map[string]*fakeResult
type fakeResult struct {
body string
urls []string
}
func (f fakeFetcher) Fetch(url string) (string, []string, error) {
rand.Seed(time.Now().UnixNano())
time.Sleep(time.Duration(rand.Intn(500)) * time.Millisecond)
if res, ok := f[url]; ok {
return res.body, res.urls, nil
}
return "", nil, fmt.Errorf("not found: %s", url)
}
var fetcher = fakeFetcher{
"http://golang.org/": &fakeResult{
"The Go Programming Language",
[]string{
"http://golang.org/pkg/",
"http://golang.org/cmd/",
},
},
"http://golang.org/pkg/": &fakeResult{
"Packages",
[]string{
"http://golang.org/",
"http://golang.org/cmd/",
"http://golang.org/pkg/fmt/",
"http://golang.org/pkg/os/",
},
},
"http://golang.org/pkg/fmt/": &fakeResult{
"Package fmt",
[]string{
"http://golang.org/",
"http://golang.org/pkg/",
},
},
"http://golang.org/pkg/os/": &fakeResult{
"Package os",
[]string{
"http://golang.org/",
"http://golang.org/pkg/",
},
},
}