aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorale <ale@incal.net>2017-12-19 08:35:21 +0000
committerale <ale@incal.net>2017-12-19 08:35:21 +0000
commit3ff5e7b88f3c2838e93aeb57824c82508e1e1d66 (patch)
treed3af5abc9f304ff13f5ba1a06757ed2efbb0e302
parent06f170e9ca181f145a57150c111a0982db32d023 (diff)
downloadcrawl-3ff5e7b88f3c2838e93aeb57824c82508e1e1d66.tar.gz
crawl-3ff5e7b88f3c2838e93aeb57824c82508e1e1d66.zip
Simplify redirectHandler.Handle
-rw-r--r--crawler.go30
-rw-r--r--queue.go2
2 files changed, 16 insertions, 16 deletions
diff --git a/crawler.go b/crawler.go
index 9fad2ef..b3c4a7b 100644
--- a/crawler.go
+++ b/crawler.go
@@ -293,22 +293,24 @@ type redirectHandler struct {
}
func (wrap *redirectHandler) Handle(c *Crawler, u string, depth int, resp *http.Response, err error) error {
- if err == nil {
- if resp.StatusCode == 200 {
- err = wrap.h.Handle(c, u, depth, resp, err)
- } else if resp.StatusCode > 300 && resp.StatusCode < 400 {
- location := resp.Header.Get("Location")
- if location != "" {
- locationURL, err := resp.Request.URL.Parse(location)
- if err != nil {
- log.Printf("error parsing Location header: %v", err)
- } else {
- c.Enqueue(Outlink{URL: locationURL, Tag: TagPrimary}, depth+1)
- }
+ if err != nil {
+ return err
+ }
+
+ if resp.StatusCode == 200 {
+ err = wrap.h.Handle(c, u, depth, resp, err)
+ } else if resp.StatusCode > 300 && resp.StatusCode < 400 {
+ location := resp.Header.Get("Location")
+ if location != "" {
+ locationURL, err := resp.Request.URL.Parse(location)
+ if err != nil {
+ log.Printf("error parsing Location header: %v", err)
+ } else {
+ c.Enqueue(Outlink{URL: locationURL, Tag: TagPrimary}, depth+1)
}
- } else {
- err = errors.New(resp.Status)
}
+ } else {
+ err = errors.New(resp.Status)
}
return err
}
diff --git a/queue.go b/queue.go
index 7621226..da4a7b8 100644
--- a/queue.go
+++ b/queue.go
@@ -65,10 +65,8 @@ func (q *queue) Add(wb *leveldb.Batch, urlStr string, depth int, when time.Time)
func (q *queue) acquire(qp queuePair) {
wb := new(leveldb.Batch)
-
q.db.PutObjBatch(wb, activeQueueKey(qp.key), qp)
wb.Delete(qp.key)
-
q.db.Write(wb, nil)
atomic.AddInt32(&q.numActive, 1)