aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorale <ale@incal.net>2015-07-03 19:38:17 +0100
committerale <ale@incal.net>2015-07-03 19:38:17 +0100
commit4fc0b1d2b5f9143a3067ef31f3558774a3c6d68f (patch)
treeb7d4a4c4b8ed36ca08e28c8876e1951f2e38aa38
parentb06e5a296b16d0080afd36470b0ba59c7b8a5bc2 (diff)
downloadcrawl-4fc0b1d2b5f9143a3067ef31f3558774a3c6d68f.tar.gz
crawl-4fc0b1d2b5f9143a3067ef31f3558774a3c6d68f.zip
minor golint fixes
-rw-r--r--cmd/crawl/crawl.go4
-rw-r--r--crawler.go1
-rw-r--r--warc/warc.go2
3 files changed, 4 insertions, 3 deletions
diff --git a/cmd/crawl/crawl.go b/cmd/crawl/crawl.go
index de45494..3954682 100644
--- a/cmd/crawl/crawl.go
+++ b/cmd/crawl/crawl.go
@@ -99,7 +99,7 @@ func (h *warcSaveHandler) Handle(c *crawl.Crawler, u string, depth int, resp *ht
return extractLinks(c, u, depth, resp, err)
}
-func NewSaveHandler(w *warc.Writer) crawl.Handler {
+func newWarcSaveHandler(w *warc.Writer) crawl.Handler {
info := strings.Join([]string{
"Software: crawl/1.0\r\n",
"Format: WARC File Format 1.0\r\n",
@@ -206,7 +206,7 @@ func main() {
w := warc.NewWriter(outf)
defer w.Close()
- saver := NewSaveHandler(w)
+ saver := newWarcSaveHandler(w)
crawler, err := crawl.NewCrawler(*dbPath, seeds, scope, crawl.FetcherFunc(fetch), crawl.NewRedirectHandler(saver))
if err != nil {
diff --git a/crawler.go b/crawler.go
index d162330..f2a8968 100644
--- a/crawler.go
+++ b/crawler.go
@@ -319,6 +319,7 @@ func (c *Crawler) Run(concurrency int) {
wg.Wait()
}
+// Close the database and release resources associated with the crawler state.
func (c *Crawler) Close() {
c.db.Close()
}
diff --git a/warc/warc.go b/warc/warc.go
index 0739fb9..7f2b03b 100644
--- a/warc/warc.go
+++ b/warc/warc.go
@@ -23,7 +23,7 @@ var (
}
)
-// A WARC header. Header field names are case-sensitive.
+// Header for a WARC record. Header field names are case-sensitive.
type Header map[string]string
// Set a header to the specified value. Multiple values are not