From dc39be70c4b9dbb4cf330ddc7776340551a4764c Mon Sep 17 00:00:00 2001
From: Jordan <me@jordan.im>
Date: Thu, 10 Feb 2022 14:54:24 -0700
Subject: client, crawl: --bind, support making outbound requests from a
 particular address

---
 cmd/crawl/crawl.go | 10 ++++++++++
 1 file changed, 10 insertions(+)

(limited to 'cmd')

diff --git a/cmd/crawl/crawl.go b/cmd/crawl/crawl.go
index 0be1572..0fc6af1 100644
--- a/cmd/crawl/crawl.go
+++ b/cmd/crawl/crawl.go
@@ -11,6 +11,7 @@ import (
 	"io"
 	"io/ioutil"
 	"log"
+	"net"
 	"net/http"
 	"os"
 	"os/signal"
@@ -38,6 +39,7 @@ var (
 	resumeDir      = flag.String("resume", "", "path to directory of previous crawl to resume")
 	warcFileSizeMB = flag.Int("output-max-size", 100, "maximum output WARC file size (in MB) when using patterns")
 	cpuprofile     = flag.String("cpuprofile", "", "create cpu profile")
+	bindIP         = flag.String("bind", "", "IP address from which to make outbound connections")
 
 	dnsMap   = dnsMapFlag(make(map[string]string))
 	excludes []*regexp.Regexp
@@ -338,6 +340,14 @@ func main() {
 
 	httpClient = crawl.NewHTTPClientWithDNSOverride(dnsMap)
 
+	if *bindIP != "" {
+		addr, err := net.ResolveIPAddr("ip", *bindIP)
+		if err != nil {
+			log.Fatal(err)
+		}
+		httpClient = crawl.NewHTTPClientWithLocalAddrOverride(addr)
+	}
+
 	crawler, err := crawl.NewCrawler(
 		*dbPath,
 		seeds,
-- 
cgit v1.2.3-54-g00ecf