Use a global http.Client with sane settings

1 files changed, 31 insertions, 0 deletions

diff --git a/client.go b/client.go
new file mode 100644
index 0000000..c0c2626
--- /dev/null
+++ b/client.go
@@ -0,0 +1,31 @@
+package crawl
+
+import (
+	"crypto/tls"
+	"net/http"
+	"net/http/cookiejar"
+	"time"
+)
+
+var defaultClientTimeout = 60 * time.Second
+
+var DefaultClient *http.Client
+
+// DefaultClient returns a http.Client suitable for crawling: does not
+// follow redirects, accepts invalid TLS certificates, sets a
+// reasonable timeout for requests.
+func init() {
+	jar, _ := cookiejar.New(nil)
+	DefaultClient = &http.Client{
+		Timeout: defaultClientTimeout,
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{
+				InsecureSkipVerify: true,
+			},
+		},
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			return http.ErrUseLastResponse
+		},
+		Jar: jar,
+	}
+}