Skip to main content
Tweeted twitter.com/StackCodeReview/status/725673179553865728
Implementation improved to actually fetch the pages asynchronously
Source Link

This is my solution (available in the Go Playground herehere — I've written code from the Fetch struct to the Crawl function, the rest was already provided by the exercise):

package main

import (
    "fmt"
    "sync"
    "time"
    "math/rand"
)

type Fetcher interface {
    Fetch(url string) (body string, urls []string, err error)
}

type FetchLinks struct {
    url urls  string[]string
    depth int
}

type CacheConcurrentStringSet struct {
    visitedset map[string]bool
    mux     sync.Mutex
}

func (ccache *Cache*ConcurrentStringSet) onMissDoAsync(s string, f func()) {
    ccache.mux.Lock()
    defer ccache.mux.Unlock()
    if !ccache.visited[s]set[s] {
        ccache.visited[s]set[s] = true
        go f()
    }
}

func NewAsyncCrawler(fetcher Fetcher, timeout time.Duration) (crawl func(Fetchstring, int)) {
    resultslinks := make(chan FetchLinks)
    cache := CacheConcurrentStringSet{visitedset:make(map[string]bool)}
    crawl = func(fetch Fetch) {
       url ifstring, fetch.depth <= 0int) {
            return
        }
        cache.onMissDoAsync(fetch.url, func() {
            body, urls, err := fetcher.Fetch(fetch.url)
            if err != nil {
                fmt.Println(err)
              } else return{
            }
            fmt.Printf("found: %s %q\n", fetch.url, body)
                if fetch.depth > 1 {
                for _, url := rangelinks urls<- Links{
 urls, depth - 1}
                }
 if !cache.visited[url] {
         }
        })
       results <-for Fetch{url, fetch.depth 
 - 1}
          select {
         }
   case link := <-links:
          }
      for _, url := range link.urls }{
        })
        select {
   go crawl(url, link.depth)
   case next := <-results:
          }
  crawl(next)
          case <-time.After(timeout):
                return
            }
        }
    }
    return
}

func Crawl(url string, depth int, fetcher Fetcher) {
    if (depth > 0) {
        crawl := NewAsyncCrawler(fetcher, 500 * time.Millisecond)
        crawl(Fetch{url, depth})
    }
}

func main() {
    Crawl("http://golang.org/", 4, fetcher)
}

type fakeFetcher map[string]*fakeResult

type fakeResult struct {
    body string
    urls []string
}

func (f fakeFetcher) Fetch(url string) (string, []string, error) {
    rand.Seed(time.Now().UnixNano())
    time.Sleep(time.Duration(rand.Intn(500)) * time.Millisecond)
    if res, ok := f[url]; ok {
        return res.body, res.urls, nil
    }
    return "", nil, fmt.Errorf("not found: %s", url)
}

var fetcher = fakeFetcher{
    "http://golang.org/": &fakeResult{
        "The Go Programming Language",
        []string{
            "http://golang.org/pkg/",
            "http://golang.org/cmd/",
        },
    },
    "http://golang.org/pkg/": &fakeResult{
        "Packages",
        []string{
            "http://golang.org/",
            "http://golang.org/cmd/",
            "http://golang.org/pkg/fmt/",
            "http://golang.org/pkg/os/",
        },
    },
    "http://golang.org/pkg/fmt/": &fakeResult{
        "Package fmt",
        []string{
            "http://golang.org/",
            "http://golang.org/pkg/",
        },
    },
    "http://golang.org/pkg/os/": &fakeResult{
        "Package os",
        []string{
            "http://golang.org/",
            "http://golang.org/pkg/",
        },
    },
}

This is my solution (available in the Go Playground here — I've written code from the Fetch struct to the Crawl function, the rest was already provided by the exercise):

package main

import (
    "fmt"
    "sync"
    "time"
    "math/rand"
)

type Fetcher interface {
    Fetch(url string) (body string, urls []string, err error)
}

type Fetch struct {
    url   string
    depth int
}

type Cache struct {
    visited map[string]bool
    mux     sync.Mutex
}

func (c *Cache) onMissDoAsync(s string, f func()) {
    c.mux.Lock()
    defer c.mux.Unlock()
    if !c.visited[s] {
        c.visited[s] = true
        go f()
    }
}

func NewAsyncCrawler(fetcher Fetcher, timeout time.Duration) (crawl func(Fetch)) {
    results := make(chan Fetch)
    cache := Cache{visited:make(map[string]bool)}
    crawl = func(fetch Fetch) {
        if fetch.depth <= 0 {
            return
        }
        cache.onMissDoAsync(fetch.url, func() {
            body, urls, err := fetcher.Fetch(fetch.url)
            if err != nil {
                fmt.Println(err)
                return
            }
            fmt.Printf("found: %s %q\n", fetch.url, body)
            if fetch.depth > 1 {
                for _, url := range urls {
                     if !cache.visited[url] {
                        results <- Fetch{url, fetch.depth - 1}
                    }
                }
            }
        })
        select {
        case next := <-results:
            crawl(next)
        case <-time.After(timeout):
            return
        }
    }
    return
}

func Crawl(url string, depth int, fetcher Fetcher) {
    crawl := NewAsyncCrawler(fetcher, 500 * time.Millisecond)
    crawl(Fetch{url, depth})
}

func main() {
    Crawl("http://golang.org/", 4, fetcher)
}

type fakeFetcher map[string]*fakeResult

type fakeResult struct {
    body string
    urls []string
}

func (f fakeFetcher) Fetch(url string) (string, []string, error) {
    rand.Seed(time.Now().UnixNano())
    time.Sleep(time.Duration(rand.Intn(500)) * time.Millisecond)
    if res, ok := f[url]; ok {
        return res.body, res.urls, nil
    }
    return "", nil, fmt.Errorf("not found: %s", url)
}

var fetcher = fakeFetcher{
    "http://golang.org/": &fakeResult{
        "The Go Programming Language",
        []string{
            "http://golang.org/pkg/",
            "http://golang.org/cmd/",
        },
    },
    "http://golang.org/pkg/": &fakeResult{
        "Packages",
        []string{
            "http://golang.org/",
            "http://golang.org/cmd/",
            "http://golang.org/pkg/fmt/",
            "http://golang.org/pkg/os/",
        },
    },
    "http://golang.org/pkg/fmt/": &fakeResult{
        "Package fmt",
        []string{
            "http://golang.org/",
            "http://golang.org/pkg/",
        },
    },
    "http://golang.org/pkg/os/": &fakeResult{
        "Package os",
        []string{
            "http://golang.org/",
            "http://golang.org/pkg/",
        },
    },
}

This is my solution (available in the Go Playground here — I've written code from the Fetch struct to the Crawl function, the rest was already provided by the exercise):

package main

import (
    "fmt"
    "sync"
    "time"
    "math/rand"
)

type Fetcher interface {
    Fetch(url string) (body string, urls []string, err error)
}

type Links struct {
    urls  []string
    depth int
}

type ConcurrentStringSet struct {
    set map[string]bool
    mux     sync.Mutex
}

func (cache *ConcurrentStringSet) onMissDoAsync(s string, f func()) {
    cache.mux.Lock()
    defer cache.mux.Unlock()
    if !cache.set[s] {
        cache.set[s] = true
        go f()
    }
}

func NewAsyncCrawler(fetcher Fetcher, timeout time.Duration) (crawl func(string, int)) {
    links := make(chan Links)
    cache := ConcurrentStringSet{set:make(map[string]bool)}
    crawl = func(url string, depth int) {
        cache.onMissDoAsync(url, func() {
            body, urls, err := fetcher.Fetch(url)
            if err != nil {
                fmt.Println(err)
            } else {
                fmt.Printf("found: %s %q\n", url, body)
                if depth > 1 {
                    links <- Links{urls, depth - 1}
                }
            }
        })
        for { 
            select {
            case link := <-links:
                for _, url := range link.urls {
                    go crawl(url, link.depth)
                }
            case <-time.After(timeout):
                return
            }
        }
    }
    return
}

func Crawl(url string, depth int, fetcher Fetcher) {
    if (depth > 0) {
        crawl := NewAsyncCrawler(fetcher, 500 * time.Millisecond)
        crawl(url, depth)
    }
}

func main() {
    Crawl("http://golang.org/", 4, fetcher)
}

type fakeFetcher map[string]*fakeResult

type fakeResult struct {
    body string
    urls []string
}

func (f fakeFetcher) Fetch(url string) (string, []string, error) {
    rand.Seed(time.Now().UnixNano())
    time.Sleep(time.Duration(rand.Intn(500)) * time.Millisecond)
    if res, ok := f[url]; ok {
        return res.body, res.urls, nil
    }
    return "", nil, fmt.Errorf("not found: %s", url)
}

var fetcher = fakeFetcher{
    "http://golang.org/": &fakeResult{
        "The Go Programming Language",
        []string{
            "http://golang.org/pkg/",
            "http://golang.org/cmd/",
        },
    },
    "http://golang.org/pkg/": &fakeResult{
        "Packages",
        []string{
            "http://golang.org/",
            "http://golang.org/cmd/",
            "http://golang.org/pkg/fmt/",
            "http://golang.org/pkg/os/",
        },
    },
    "http://golang.org/pkg/fmt/": &fakeResult{
        "Package fmt",
        []string{
            "http://golang.org/",
            "http://golang.org/pkg/",
        },
    },
    "http://golang.org/pkg/os/": &fakeResult{
        "Package os",
        []string{
            "http://golang.org/",
            "http://golang.org/pkg/",
        },
    },
}
edited tags
Link
200_success
  • 144.2k
  • 22
  • 188
  • 473
Source Link

Golang Tour Web Crawler Exercise

In the last few days I've played around with Go a little and took the language tour. The last exercise (text here) requires you to crawl a graph that simulates a set of Web pages with links, using goroutines, channels and locking primitives to do it in a concurrent fashion without ever going to the same page twice. I added a random latency to the fetch procedure to simulate the time to actually fetch a page.

My solution involves a timeout, which I'm not very fond of. In a real Web crawler I could expect some action to be taken when a timeout is hit, but in this very limited case perhaps I can use some more robust solution. Any idea? I tried using a counter to keep track of enqueued items and then closing the results channel when there were no more items to explore but the goroutine fetching the page which feeds the results channel has no awareness of other page fetching going on, so I ended up closing the channel to early (now that I'm writing this down I realize perhaps I can use some sort of "control" channel to sync the goroutines in some way).

This is my solution (available in the Go Playground here — I've written code from the Fetch struct to the Crawl function, the rest was already provided by the exercise):

package main

import (
    "fmt"
    "sync"
    "time"
    "math/rand"
)

type Fetcher interface {
    Fetch(url string) (body string, urls []string, err error)
}

type Fetch struct {
    url   string
    depth int
}

type Cache struct {
    visited map[string]bool
    mux     sync.Mutex
}

func (c *Cache) onMissDoAsync(s string, f func()) {
    c.mux.Lock()
    defer c.mux.Unlock()
    if !c.visited[s] {
        c.visited[s] = true
        go f()
    }
}

func NewAsyncCrawler(fetcher Fetcher, timeout time.Duration) (crawl func(Fetch)) {
    results := make(chan Fetch)
    cache := Cache{visited:make(map[string]bool)}
    crawl = func(fetch Fetch) {
        if fetch.depth <= 0 {
            return
        }
        cache.onMissDoAsync(fetch.url, func() {
            body, urls, err := fetcher.Fetch(fetch.url)
            if err != nil {
                fmt.Println(err)
                return
            }
            fmt.Printf("found: %s %q\n", fetch.url, body)
            if fetch.depth > 1 {
                for _, url := range urls {
                    if !cache.visited[url] {
                        results <- Fetch{url, fetch.depth - 1}
                    }
                }
            }
        })
        select {
        case next := <-results:
            crawl(next)
        case <-time.After(timeout):
            return
        }
    }
    return
}

func Crawl(url string, depth int, fetcher Fetcher) {
    crawl := NewAsyncCrawler(fetcher, 500 * time.Millisecond)
    crawl(Fetch{url, depth})
}

func main() {
    Crawl("http://golang.org/", 4, fetcher)
}

type fakeFetcher map[string]*fakeResult

type fakeResult struct {
    body string
    urls []string
}

func (f fakeFetcher) Fetch(url string) (string, []string, error) {
    rand.Seed(time.Now().UnixNano())
    time.Sleep(time.Duration(rand.Intn(500)) * time.Millisecond)
    if res, ok := f[url]; ok {
        return res.body, res.urls, nil
    }
    return "", nil, fmt.Errorf("not found: %s", url)
}

var fetcher = fakeFetcher{
    "http://golang.org/": &fakeResult{
        "The Go Programming Language",
        []string{
            "http://golang.org/pkg/",
            "http://golang.org/cmd/",
        },
    },
    "http://golang.org/pkg/": &fakeResult{
        "Packages",
        []string{
            "http://golang.org/",
            "http://golang.org/cmd/",
            "http://golang.org/pkg/fmt/",
            "http://golang.org/pkg/os/",
        },
    },
    "http://golang.org/pkg/fmt/": &fakeResult{
        "Package fmt",
        []string{
            "http://golang.org/",
            "http://golang.org/pkg/",
        },
    },
    "http://golang.org/pkg/os/": &fakeResult{
        "Package os",
        []string{
            "http://golang.org/",
            "http://golang.org/pkg/",
        },
    },
}