diff options
| author | Jordan <me@jordan.im> | 2022-02-10 14:31:41 -0700 |
|---|---|---|
| committer | Jordan <me@jordan.im> | 2022-02-10 14:31:41 -0700 |
| commit | 2dfdfd864285c5a12cd47e446fc8bdf244fd58f8 (patch) | |
| tree | ac76c53062fa2b000b0af89dd0c52333875ca76e | |
| parent | 6df82fabcc0053637fdc0f02e3669cd5fa21cb46 (diff) | |
| download | crawl-2dfdfd864285c5a12cd47e446fc8bdf244fd58f8.tar.gz crawl-2dfdfd864285c5a12cd47e446fc8bdf244fd58f8.zip | |
crawl: set User-Agent header to appear like Firefox on Windows
| -rw-r--r-- | cmd/crawl/crawl.go | 4 |
1 files changed, 3 insertions, 1 deletions
diff --git a/cmd/crawl/crawl.go b/cmd/crawl/crawl.go index f2c4394..0be1572 100644 --- a/cmd/crawl/crawl.go +++ b/cmd/crawl/crawl.go @@ -248,7 +248,9 @@ func (c *crawlStats) Dump() { var stats *crawlStats func fetch(urlstr string) (*http.Response, error) { - resp, err := httpClient.Get(urlstr) + req, _ := http.NewRequest("GET", urlstr, nil) + req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:77.0) Gecko/20100101 Firefox/77.0") + resp, err := httpClient.Do(req) if err == nil { stats.Update(resp) } |