diff options
| author | ale <ale@incal.net> | 2014-12-20 11:41:24 +0000 |
|---|---|---|
| committer | ale <ale@incal.net> | 2014-12-20 11:41:24 +0000 |
| commit | 4c82422d2e75b9b8f4d034b1f43fda566416d6af (patch) | |
| tree | 91a66ec0d3ebd7739658794bca3e6a2ff4f6b400 /cmd/crawl/crawl.go | |
| parent | efe98903c17a9103d7830361d6ff6f98bb9e0faa (diff) | |
| download | crawl-4c82422d2e75b9b8f4d034b1f43fda566416d6af.tar.gz crawl-4c82422d2e75b9b8f4d034b1f43fda566416d6af.zip | |
make Scope checking more modular
Diffstat (limited to 'cmd/crawl/crawl.go')
| -rw-r--r-- | cmd/crawl/crawl.go | 6 |
1 files changed, 5 insertions, 1 deletions
diff --git a/cmd/crawl/crawl.go b/cmd/crawl/crawl.go index 1abeca6..8c02089 100644 --- a/cmd/crawl/crawl.go +++ b/cmd/crawl/crawl.go @@ -124,7 +124,11 @@ func main() { } seeds := crawl.MustParseURLs(flag.Args()) - scope := crawl.NewSeedScope(seeds, *depth, strings.Split(*validSchemes, ",")) + scope := []crawl.Scope{ + crawl.NewSchemeScope(strings.Split(*validSchemes, ",")), + crawl.NewDepthScope(*depth), + crawl.NewSeedScope(seeds), + } w := warc.NewWriter(outf) defer w.Close() |