aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--crawler.go9
1 files changed, 6 insertions, 3 deletions
diff --git a/crawler.go b/crawler.go
index b5f0822..195bab6 100644
--- a/crawler.go
+++ b/crawler.go
@@ -269,11 +269,14 @@ func (c *Crawler) urlHandler(queue <-chan queuePair) {
os.Remove(tmpFile.Name())
wb := new(leveldb.Batch)
- switch err {
- case nil:
+ switch {
+ case err == nil:
c.queue.Release(wb, p)
- case ErrRetryRequest:
+ case err == ErrRetryRequest:
Must(c.queue.Retry(wb, p, errorRetryDelay))
+ case os.IsTimeout(err):
+ log.Printf("network error in handling %s: %v", p.URL, err)
+ c.queue.Release(wb, p)
default:
log.Panicf("fatal error in handling %s: %v", p.URL, err)
}