From 2dfdfd864285c5a12cd47e446fc8bdf244fd58f8 Mon Sep 17 00:00:00 2001 From: Jordan Date: Thu, 10 Feb 2022 14:31:41 -0700 Subject: crawl: set User-Agent header to appear like Firefox on Windows --- cmd/crawl/crawl.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmd/crawl/crawl.go b/cmd/crawl/crawl.go index f2c4394..0be1572 100644 --- a/cmd/crawl/crawl.go +++ b/cmd/crawl/crawl.go @@ -248,7 +248,9 @@ func (c *crawlStats) Dump() { var stats *crawlStats func fetch(urlstr string) (*http.Response, error) { - resp, err := httpClient.Get(urlstr) + req, _ := http.NewRequest("GET", urlstr, nil) + req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:77.0) Gecko/20100101 Firefox/77.0") + resp, err := httpClient.Do(req) if err == nil { stats.Update(resp) } -- cgit v1.2.3-54-g00ecf