diff options
author | ale <ale@incal.net> | 2017-12-19 08:35:21 +0000 |
---|---|---|
committer | ale <ale@incal.net> | 2017-12-19 08:35:21 +0000 |
commit | 3ff5e7b88f3c2838e93aeb57824c82508e1e1d66 (patch) | |
tree | d3af5abc9f304ff13f5ba1a06757ed2efbb0e302 | |
parent | 06f170e9ca181f145a57150c111a0982db32d023 (diff) | |
download | crawl-3ff5e7b88f3c2838e93aeb57824c82508e1e1d66.tar.gz crawl-3ff5e7b88f3c2838e93aeb57824c82508e1e1d66.zip |
Simplify redirectHandler.Handle
-rw-r--r-- | crawler.go | 30 | ||||
-rw-r--r-- | queue.go | 2 |
2 files changed, 16 insertions, 16 deletions
@@ -293,22 +293,24 @@ type redirectHandler struct { } func (wrap *redirectHandler) Handle(c *Crawler, u string, depth int, resp *http.Response, err error) error { - if err == nil { - if resp.StatusCode == 200 { - err = wrap.h.Handle(c, u, depth, resp, err) - } else if resp.StatusCode > 300 && resp.StatusCode < 400 { - location := resp.Header.Get("Location") - if location != "" { - locationURL, err := resp.Request.URL.Parse(location) - if err != nil { - log.Printf("error parsing Location header: %v", err) - } else { - c.Enqueue(Outlink{URL: locationURL, Tag: TagPrimary}, depth+1) - } + if err != nil { + return err + } + + if resp.StatusCode == 200 { + err = wrap.h.Handle(c, u, depth, resp, err) + } else if resp.StatusCode > 300 && resp.StatusCode < 400 { + location := resp.Header.Get("Location") + if location != "" { + locationURL, err := resp.Request.URL.Parse(location) + if err != nil { + log.Printf("error parsing Location header: %v", err) + } else { + c.Enqueue(Outlink{URL: locationURL, Tag: TagPrimary}, depth+1) } - } else { - err = errors.New(resp.Status) } + } else { + err = errors.New(resp.Status) } return err } @@ -65,10 +65,8 @@ func (q *queue) Add(wb *leveldb.Batch, urlStr string, depth int, when time.Time) func (q *queue) acquire(qp queuePair) { wb := new(leveldb.Batch) - q.db.PutObjBatch(wb, activeQueueKey(qp.key), qp) wb.Delete(qp.key) - q.db.Write(wb, nil) atomic.AddInt32(&q.numActive, 1) |