diff options
| author | ale <ale@incal.net> | 2014-12-20 13:10:17 +0000 |
|---|---|---|
| committer | ale <ale@incal.net> | 2014-12-20 13:10:17 +0000 |
| commit | 3af8c9a229abb85ad75bef1c3526e44435befd02 (patch) | |
| tree | ffe2794c6a65864e4bf3b1a0a5bc2dde05133a67 /cmd | |
| parent | 7c1b1f70bdae7b28dad864570d6611321a703df0 (diff) | |
| download | crawl-3af8c9a229abb85ad75bef1c3526e44435befd02.tar.gz crawl-3af8c9a229abb85ad75bef1c3526e44435befd02.zip | |
move URLInfo logic into the Crawler itself
Diffstat (limited to 'cmd')
| -rw-r--r-- | cmd/crawl/crawl.go | 2 | ||||
| -rw-r--r-- | cmd/links/links.go | 2 |
2 files changed, 2 insertions, 2 deletions
diff --git a/cmd/crawl/crawl.go b/cmd/crawl/crawl.go index 8c02089..63a5924 100644 --- a/cmd/crawl/crawl.go +++ b/cmd/crawl/crawl.go @@ -135,7 +135,7 @@ func main() { saver := NewSaveHandler(w) - crawler, err := crawl.NewCrawler("crawldb", seeds, scope, crawl.FetcherFunc(http.Get), saver) + crawler, err := crawl.NewCrawler("crawldb", seeds, scope, crawl.FetcherFunc(http.Get), crawl.NewRedirectHandler(saver)) if err != nil { log.Fatal(err) } diff --git a/cmd/links/links.go b/cmd/links/links.go index e89e22d..eb97577 100644 --- a/cmd/links/links.go +++ b/cmd/links/links.go @@ -44,7 +44,7 @@ func main() { crawl.NewSeedScope(seeds), } - crawler, err := crawl.NewCrawler("crawldb", seeds, scope, crawl.FetcherFunc(http.Get), crawl.HandlerFunc(extractLinks)) + crawler, err := crawl.NewCrawler("crawldb", seeds, scope, crawl.FetcherFunc(http.Get), crawl.NewRedirectHandler(crawl.HandlerFunc(extractLinks))) if err != nil { log.Fatal(err) } |