diff options
| -rw-r--r-- | cmd/crawl/crawl.go | 4 |
1 files changed, 3 insertions, 1 deletions
diff --git a/cmd/crawl/crawl.go b/cmd/crawl/crawl.go index f2c4394..0be1572 100644 --- a/cmd/crawl/crawl.go +++ b/cmd/crawl/crawl.go @@ -248,7 +248,9 @@ func (c *crawlStats) Dump() { var stats *crawlStats func fetch(urlstr string) (*http.Response, error) { - resp, err := httpClient.Get(urlstr) + req, _ := http.NewRequest("GET", urlstr, nil) + req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:77.0) Gecko/20100101 Firefox/77.0") + resp, err := httpClient.Do(req) if err == nil { stats.Update(resp) } |