From 3af8c9a229abb85ad75bef1c3526e44435befd02 Mon Sep 17 00:00:00 2001 From: ale Date: Sat, 20 Dec 2014 13:10:17 +0000 Subject: move URLInfo logic into the Crawler itself --- cmd/crawl/crawl.go | 2 +- cmd/links/links.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'cmd') diff --git a/cmd/crawl/crawl.go b/cmd/crawl/crawl.go index 8c02089..63a5924 100644 --- a/cmd/crawl/crawl.go +++ b/cmd/crawl/crawl.go @@ -135,7 +135,7 @@ func main() { saver := NewSaveHandler(w) - crawler, err := crawl.NewCrawler("crawldb", seeds, scope, crawl.FetcherFunc(http.Get), saver) + crawler, err := crawl.NewCrawler("crawldb", seeds, scope, crawl.FetcherFunc(http.Get), crawl.NewRedirectHandler(saver)) if err != nil { log.Fatal(err) } diff --git a/cmd/links/links.go b/cmd/links/links.go index e89e22d..eb97577 100644 --- a/cmd/links/links.go +++ b/cmd/links/links.go @@ -44,7 +44,7 @@ func main() { crawl.NewSeedScope(seeds), } - crawler, err := crawl.NewCrawler("crawldb", seeds, scope, crawl.FetcherFunc(http.Get), crawl.HandlerFunc(extractLinks)) + crawler, err := crawl.NewCrawler("crawldb", seeds, scope, crawl.FetcherFunc(http.Get), crawl.NewRedirectHandler(crawl.HandlerFunc(extractLinks))) if err != nil { log.Fatal(err) } -- cgit v1.2.3-54-g00ecf