diff options
author | ale <ale@incal.net> | 2015-07-03 19:38:17 +0100 |
---|---|---|
committer | ale <ale@incal.net> | 2015-07-03 19:38:17 +0100 |
commit | 4fc0b1d2b5f9143a3067ef31f3558774a3c6d68f (patch) | |
tree | b7d4a4c4b8ed36ca08e28c8876e1951f2e38aa38 | |
parent | b06e5a296b16d0080afd36470b0ba59c7b8a5bc2 (diff) | |
download | crawl-4fc0b1d2b5f9143a3067ef31f3558774a3c6d68f.tar.gz crawl-4fc0b1d2b5f9143a3067ef31f3558774a3c6d68f.zip |
minor golint fixes
-rw-r--r-- | cmd/crawl/crawl.go | 4 | ||||
-rw-r--r-- | crawler.go | 1 | ||||
-rw-r--r-- | warc/warc.go | 2 |
3 files changed, 4 insertions, 3 deletions
diff --git a/cmd/crawl/crawl.go b/cmd/crawl/crawl.go index de45494..3954682 100644 --- a/cmd/crawl/crawl.go +++ b/cmd/crawl/crawl.go @@ -99,7 +99,7 @@ func (h *warcSaveHandler) Handle(c *crawl.Crawler, u string, depth int, resp *ht return extractLinks(c, u, depth, resp, err) } -func NewSaveHandler(w *warc.Writer) crawl.Handler { +func newWarcSaveHandler(w *warc.Writer) crawl.Handler { info := strings.Join([]string{ "Software: crawl/1.0\r\n", "Format: WARC File Format 1.0\r\n", @@ -206,7 +206,7 @@ func main() { w := warc.NewWriter(outf) defer w.Close() - saver := NewSaveHandler(w) + saver := newWarcSaveHandler(w) crawler, err := crawl.NewCrawler(*dbPath, seeds, scope, crawl.FetcherFunc(fetch), crawl.NewRedirectHandler(saver)) if err != nil { @@ -319,6 +319,7 @@ func (c *Crawler) Run(concurrency int) { wg.Wait() } +// Close the database and release resources associated with the crawler state. func (c *Crawler) Close() { c.db.Close() } diff --git a/warc/warc.go b/warc/warc.go index 0739fb9..7f2b03b 100644 --- a/warc/warc.go +++ b/warc/warc.go @@ -23,7 +23,7 @@ var ( } ) -// A WARC header. Header field names are case-sensitive. +// Header for a WARC record. Header field names are case-sensitive. type Header map[string]string // Set a header to the specified value. Multiple values are not |