diff options
| author | ale <ale@incal.net> | 2020-07-30 15:10:41 +0100 |
|---|---|---|
| committer | ale <ale@incal.net> | 2020-07-30 15:10:41 +0100 |
| commit | 9d07df0980b65c95661631d3c967cf9acb8961f5 (patch) | |
| tree | a9f598770e5ba8b4459b506e9340d26cec19cf08 | |
| parent | de767cce09dd2e8aeafc79a150ef6492d80d7b2c (diff) | |
| download | crawl-9d07df0980b65c95661631d3c967cf9acb8961f5.tar.gz crawl-9d07df0980b65c95661631d3c967cf9acb8961f5.zip | |
Retry requests on transport-level errors
| -rw-r--r-- | cmd/crawl/crawl.go | 4 |
1 files changed, 3 insertions, 1 deletions
diff --git a/cmd/crawl/crawl.go b/cmd/crawl/crawl.go index a79e0a6..93506ac 100644 --- a/cmd/crawl/crawl.go +++ b/cmd/crawl/crawl.go @@ -132,7 +132,9 @@ func (h *warcSaveHandler) Handle(p crawl.Publisher, u string, tag, depth int, re // output) and replace it with a buffer. data, derr := ioutil.ReadAll(resp.Body) if derr != nil { - return derr + // Errors at this stage are usually transport-level errors, + // and as such, retriable. + return crawl.ErrRetryRequest } resp.Body = ioutil.NopCloser(bytes.NewReader(data)) |