Improve error checking

Detect write errors (both on the database and to the WARC output) and
abort with an error message.

Also fix a bunch of harmless lint warnings.

1 files changed, 8 insertions, 2 deletions

diff --git a/cmd/links/links.go b/cmd/links/links.go
index e8b0fcb..9cd741f 100644
--- a/cmd/links/links.go
+++ b/cmd/links/links.go
@@ -15,20 +15,25 @@ import (
 )
 
 var (
-	dbPath       = flag.String("state", "crawldb", "crawl state database path")
 	concurrency  = flag.Int("c", 10, "concurrent workers")
 	depth        = flag.Int("depth", 10, "maximum link depth")
 	validSchemes = flag.String("schemes", "http,https", "comma-separated list of allowed protocols")
 )
 
 func extractLinks(c *crawl.Crawler, u string, depth int, resp *http.Response, err error) error {
+	if err != nil {
+		return err
+	}
+
 	links, err := analysis.GetLinks(resp)
 	if err != nil {
 		return err
 	}
 
 	for _, link := range links {
-		c.Enqueue(link, depth+1)
+		if err := c.Enqueue(link, depth+1); err != nil {
+			return err
+		}
 	}
 
 	return nil
@@ -49,4 +54,5 @@ func main() {
 		log.Fatal(err)
 	}
 	crawler.Run(*concurrency)
+	crawler.Close()
 }