diff options
author | Jordan <me@jordan.im> | 2022-02-10 14:54:24 -0700 |
---|---|---|
committer | Jordan <me@jordan.im> | 2022-02-10 14:54:24 -0700 |
commit | dc39be70c4b9dbb4cf330ddc7776340551a4764c (patch) | |
tree | 98d5ab21f92a8e1db130a3020cbc8803ff21f724 /cmd | |
parent | 2dfdfd864285c5a12cd47e446fc8bdf244fd58f8 (diff) | |
download | crawl-dc39be70c4b9dbb4cf330ddc7776340551a4764c.tar.gz crawl-dc39be70c4b9dbb4cf330ddc7776340551a4764c.zip |
client, crawl: --bind, support making outbound requests from a particular address
Diffstat (limited to 'cmd')
-rw-r--r-- | cmd/crawl/crawl.go | 10 |
1 files changed, 10 insertions, 0 deletions
diff --git a/cmd/crawl/crawl.go b/cmd/crawl/crawl.go index 0be1572..0fc6af1 100644 --- a/cmd/crawl/crawl.go +++ b/cmd/crawl/crawl.go @@ -11,6 +11,7 @@ import ( "io" "io/ioutil" "log" + "net" "net/http" "os" "os/signal" @@ -38,6 +39,7 @@ var ( resumeDir = flag.String("resume", "", "path to directory of previous crawl to resume") warcFileSizeMB = flag.Int("output-max-size", 100, "maximum output WARC file size (in MB) when using patterns") cpuprofile = flag.String("cpuprofile", "", "create cpu profile") + bindIP = flag.String("bind", "", "IP address from which to make outbound connections") dnsMap = dnsMapFlag(make(map[string]string)) excludes []*regexp.Regexp @@ -338,6 +340,14 @@ func main() { httpClient = crawl.NewHTTPClientWithDNSOverride(dnsMap) + if *bindIP != "" { + addr, err := net.ResolveIPAddr("ip", *bindIP) + if err != nil { + log.Fatal(err) + } + httpClient = crawl.NewHTTPClientWithLocalAddrOverride(addr) + } + crawler, err := crawl.NewCrawler( *dbPath, seeds, |