diff options
Diffstat (limited to 'client.go')
| -rw-r--r-- | client.go | 43 |
1 files changed, 40 insertions, 3 deletions
@@ -1,7 +1,9 @@ package crawl import ( + "context" "crypto/tls" + "net" "net/http" "net/http/cookiejar" "time" @@ -9,14 +11,19 @@ import ( var defaultClientTimeout = 60 * time.Second -// DefaultClient returns a http.Client suitable for crawling: does not -// follow redirects, accepts invalid TLS certificates, sets a +// DefaultClient points at a shared http.Client suitable for crawling: +// does not follow redirects, accepts invalid TLS certificates, sets a // reasonable timeout for requests. var DefaultClient *http.Client func init() { + DefaultClient = NewHTTPClient() +} + +// NewHTTPClient returns an http.Client suitable for crawling. +func NewHTTPClient() *http.Client { jar, _ := cookiejar.New(nil) // nolint - DefaultClient = &http.Client{ + return &http.Client{ Timeout: defaultClientTimeout, Transport: &http.Transport{ TLSClientConfig: &tls.Config{ @@ -29,3 +36,33 @@ func init() { Jar: jar, } } + +// NewHTTPClientWithDNSOverride returns an http.Client suitable for +// crawling, with some additional DNS overrides. +func NewHTTPClientWithDNSOverride(dnsMap map[string]string) *http.Client { + jar, _ := cookiejar.New(nil) // nolint + dialer := new(net.Dialer) + transport := &http.Transport{ + DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { + host, port, err := net.SplitHostPort(addr) + if err != nil { + return nil, err + } + if override, ok := dnsMap[host]; ok { + addr = net.JoinHostPort(override, port) + } + return dialer.DialContext(ctx, network, addr) + }, + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, // nolint + }, + } + return &http.Client{ + Timeout: defaultClientTimeout, + Transport: transport, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + }, + Jar: jar, + } +} |