1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
|
package crawl
import (
"context"
"crypto/tls"
"net"
"net/http"
"net/http/cookiejar"
"time"
)
var defaultClientTimeout = 60 * time.Second
// DefaultClient points at a shared http.Client suitable for crawling:
// does not follow redirects, accepts invalid TLS certificates, sets a
// reasonable timeout for requests.
var DefaultClient *http.Client
func init() {
DefaultClient = NewHTTPClient()
}
// NewHTTPClient returns an http.Client suitable for crawling.
func NewHTTPClient() *http.Client {
jar, _ := cookiejar.New(nil) // nolint
return &http.Client{
Timeout: defaultClientTimeout,
Transport: &http.Transport{
TLSClientConfig: &tls.Config{
InsecureSkipVerify: true, // nolint
},
},
CheckRedirect: func(req *http.Request, via []*http.Request) error {
return http.ErrUseLastResponse
},
Jar: jar,
}
}
// NewHTTPClientWithDNSOverride returns an http.Client suitable for
// crawling, with some additional DNS overrides.
func NewHTTPClientWithDNSOverride(dnsMap map[string]string) *http.Client {
jar, _ := cookiejar.New(nil) // nolint
dialer := new(net.Dialer)
transport := &http.Transport{
DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
host, port, err := net.SplitHostPort(addr)
if err != nil {
return nil, err
}
if override, ok := dnsMap[host]; ok {
addr = net.JoinHostPort(override, port)
}
return dialer.DialContext(ctx, network, addr)
},
TLSClientConfig: &tls.Config{
InsecureSkipVerify: true, // nolint
},
}
return &http.Client{
Timeout: defaultClientTimeout,
Transport: transport,
CheckRedirect: func(req *http.Request, via []*http.Request) error {
return http.ErrUseLastResponse
},
Jar: jar,
}
}
// NewHTTPClientWithLocalAddrOverride returns an http.Client suitable for
// crawling, with a LocalAddr override for making outbound connections using
// an explicit interface
func NewHTTPClientWithLocalAddrOverride(addr *net.IPAddr) *http.Client {
jar, _ := cookiejar.New(nil) // nolint
localTCPAddr := net.TCPAddr{
IP: addr.IP,
}
transport := &http.Transport{
DialContext: (&net.Dialer{
LocalAddr: &localTCPAddr,
Timeout: 30 * time.Second,
KeepAlive: 30 * time.Second,
DualStack: false,
}).DialContext,
TLSClientConfig: &tls.Config{
InsecureSkipVerify: true, // nolint
},
}
return &http.Client{
Timeout: defaultClientTimeout,
Transport: transport,
CheckRedirect: func(req *http.Request, via []*http.Request) error {
return http.ErrUseLastResponse
},
Jar: jar,
}
}
|
