diff options
Diffstat (limited to 'cmd')
| -rw-r--r-- | cmd/crawl/crawl.go | 6 | ||||
| -rw-r--r-- | cmd/links/links.go | 6 |
2 files changed, 10 insertions, 2 deletions
diff --git a/cmd/crawl/crawl.go b/cmd/crawl/crawl.go index 1abeca6..8c02089 100644 --- a/cmd/crawl/crawl.go +++ b/cmd/crawl/crawl.go @@ -124,7 +124,11 @@ func main() { } seeds := crawl.MustParseURLs(flag.Args()) - scope := crawl.NewSeedScope(seeds, *depth, strings.Split(*validSchemes, ",")) + scope := []crawl.Scope{ + crawl.NewSchemeScope(strings.Split(*validSchemes, ",")), + crawl.NewDepthScope(*depth), + crawl.NewSeedScope(seeds), + } w := warc.NewWriter(outf) defer w.Close() diff --git a/cmd/links/links.go b/cmd/links/links.go index 95388ce..e89e22d 100644 --- a/cmd/links/links.go +++ b/cmd/links/links.go @@ -38,7 +38,11 @@ func main() { flag.Parse() seeds := crawl.MustParseURLs(flag.Args()) - scope := crawl.NewSeedScope(seeds, *depth, strings.Split(*validSchemes, ",")) + scope := []crawl.Scope{ + crawl.NewSchemeScope(strings.Split(*validSchemes, ",")), + crawl.NewDepthScope(*depth), + crawl.NewSeedScope(seeds), + } crawler, err := crawl.NewCrawler("crawldb", seeds, scope, crawl.FetcherFunc(http.Get), crawl.HandlerFunc(extractLinks)) if err != nil { |