From a6a6fef1c7cc7d6878e8aa36541565fb3e0c9747 Mon Sep 17 00:00:00 2001 From: Jordan Date: Mon, 14 Feb 2022 21:02:12 -0700 Subject: crawl, readme: record assembled seed URLs to seed_urls file --- cmd/crawl/crawl.go | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'cmd') diff --git a/cmd/crawl/crawl.go b/cmd/crawl/crawl.go index 8c20901..63fda1a 100644 --- a/cmd/crawl/crawl.go +++ b/cmd/crawl/crawl.go @@ -327,6 +327,16 @@ func main() { } } + // Write seed list (assembled URLs) to seed_urls file + f, err := os.Create("seed_urls") + if err != nil { + log.Fatal(err) + } + defer f.Close() + for _, v := range seeds { + fmt.Fprintln(f, v) + } + w, err := warcWriterFromFlags(host) if err != nil { log.Fatal(err) -- cgit v1.2.3-54-g00ecf