diff options
Diffstat (limited to 'cmd/crawl/crawl.go')
| -rw-r--r-- | cmd/crawl/crawl.go | 10 |
1 files changed, 10 insertions, 0 deletions
diff --git a/cmd/crawl/crawl.go b/cmd/crawl/crawl.go index 0be1572..0fc6af1 100644 --- a/cmd/crawl/crawl.go +++ b/cmd/crawl/crawl.go @@ -11,6 +11,7 @@ import ( "io" "io/ioutil" "log" + "net" "net/http" "os" "os/signal" @@ -38,6 +39,7 @@ var ( resumeDir = flag.String("resume", "", "path to directory of previous crawl to resume") warcFileSizeMB = flag.Int("output-max-size", 100, "maximum output WARC file size (in MB) when using patterns") cpuprofile = flag.String("cpuprofile", "", "create cpu profile") + bindIP = flag.String("bind", "", "IP address from which to make outbound connections") dnsMap = dnsMapFlag(make(map[string]string)) excludes []*regexp.Regexp @@ -338,6 +340,14 @@ func main() { httpClient = crawl.NewHTTPClientWithDNSOverride(dnsMap) + if *bindIP != "" { + addr, err := net.ResolveIPAddr("ip", *bindIP) + if err != nil { + log.Fatal(err) + } + httpClient = crawl.NewHTTPClientWithLocalAddrOverride(addr) + } + crawler, err := crawl.NewCrawler( *dbPath, seeds, |