diff options
| author | ale <ale@incal.net> | 2017-12-19 00:16:10 +0000 |
|---|---|---|
| committer | ale <ale@incal.net> | 2017-12-19 00:16:10 +0000 |
| commit | 06f170e9ca181f145a57150c111a0982db32d023 (patch) | |
| tree | f460fedab826fc1c9dbb43425c766367e4d6e2f8 | |
| parent | 665f7ba1d075c36c010123e3e1566a0a04e89c7d (diff) | |
| download | crawl-06f170e9ca181f145a57150c111a0982db32d023.tar.gz crawl-06f170e9ca181f145a57150c111a0982db32d023.zip | |
Update cmd/links to new scope syntax
| -rw-r--r-- | cmd/links/links.go | 4 |
1 files changed, 2 insertions, 2 deletions
diff --git a/cmd/links/links.go b/cmd/links/links.go index eb97577..e8b0fcb 100644 --- a/cmd/links/links.go +++ b/cmd/links/links.go @@ -38,11 +38,11 @@ func main() { flag.Parse() seeds := crawl.MustParseURLs(flag.Args()) - scope := []crawl.Scope{ + scope := crawl.AND( crawl.NewSchemeScope(strings.Split(*validSchemes, ",")), crawl.NewDepthScope(*depth), crawl.NewSeedScope(seeds), - } + ) crawler, err := crawl.NewCrawler("crawldb", seeds, scope, crawl.FetcherFunc(http.Get), crawl.NewRedirectHandler(crawl.HandlerFunc(extractLinks))) if err != nil { |