From 9d07df0980b65c95661631d3c967cf9acb8961f5 Mon Sep 17 00:00:00 2001 From: ale Date: Thu, 30 Jul 2020 15:10:41 +0100 Subject: Retry requests on transport-level errors --- cmd/crawl/crawl.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmd/crawl/crawl.go b/cmd/crawl/crawl.go index a79e0a6..93506ac 100644 --- a/cmd/crawl/crawl.go +++ b/cmd/crawl/crawl.go @@ -132,7 +132,9 @@ func (h *warcSaveHandler) Handle(p crawl.Publisher, u string, tag, depth int, re // output) and replace it with a buffer. data, derr := ioutil.ReadAll(resp.Body) if derr != nil { - return derr + // Errors at this stage are usually transport-level errors, + // and as such, retriable. + return crawl.ErrRetryRequest } resp.Body = ioutil.NopCloser(bytes.NewReader(data)) -- cgit v1.2.3-54-g00ecf