diff options
Diffstat (limited to 'cmd')
| -rw-r--r-- | cmd/crawl/crawl.go | 4 |
1 files changed, 3 insertions, 1 deletions
diff --git a/cmd/crawl/crawl.go b/cmd/crawl/crawl.go index a79e0a6..93506ac 100644 --- a/cmd/crawl/crawl.go +++ b/cmd/crawl/crawl.go @@ -132,7 +132,9 @@ func (h *warcSaveHandler) Handle(p crawl.Publisher, u string, tag, depth int, re // output) and replace it with a buffer. data, derr := ioutil.ReadAll(resp.Body) if derr != nil { - return derr + // Errors at this stage are usually transport-level errors, + // and as such, retriable. + return crawl.ErrRetryRequest } resp.Body = ioutil.NopCloser(bytes.NewReader(data)) |