diff options
-rw-r--r-- | crawler.go | 9 |
1 files changed, 6 insertions, 3 deletions
@@ -269,11 +269,14 @@ func (c *Crawler) urlHandler(queue <-chan queuePair) { os.Remove(tmpFile.Name()) wb := new(leveldb.Batch) - switch err { - case nil: + switch { + case err == nil: c.queue.Release(wb, p) - case ErrRetryRequest: + case err == ErrRetryRequest: Must(c.queue.Retry(wb, p, errorRetryDelay)) + case os.IsTimeout(err): + log.Printf("network error in handling %s: %v", p.URL, err) + c.queue.Release(wb, p) default: log.Panicf("fatal error in handling %s: %v", p.URL, err) } |