From a3019b1703b29e74cfb8fd71b9f79b0c1da11395 Mon Sep 17 00:00:00 2001 From: ale Date: Wed, 26 Aug 2020 11:48:03 +0100 Subject: Minor logging fixes --- cmd/crawl/crawl.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cmd/crawl/crawl.go b/cmd/crawl/crawl.go index 1637a62..4a98b02 100644 --- a/cmd/crawl/crawl.go +++ b/cmd/crawl/crawl.go @@ -217,6 +217,7 @@ func newWarcSaveHandler(w *warc.Writer) (crawl.Handler, error) { } type crawlStats struct { + urls int64 bytes int64 start time.Time @@ -228,6 +229,7 @@ func (c *crawlStats) Update(resp *http.Response) { c.lock.Lock() defer c.lock.Unlock() + c.urls++ c.states[resp.StatusCode]++ resp.Body = &byteCounter{resp.Body} } @@ -240,7 +242,7 @@ func (c *crawlStats) Dump() { c.lock.Lock() defer c.lock.Unlock() rate := float64(c.bytes) / time.Since(c.start).Seconds() / 1000 - fmt.Fprintf(os.Stderr, "stats: downloaded %d bytes (%.4g KB/s), status: %v\n", c.bytes, rate, c.states) // nolint + log.Printf("stats: downloaded %d urls, %d bytes (%.4g KB/s), status: %v", c.urls, c.bytes, rate, c.states) // nolint } var stats *crawlStats @@ -279,6 +281,7 @@ func warcWriterFromFlags() (w *warc.Writer, err error) { } func main() { + log.SetFlags(0) flag.Parse() if *cpuprofile != "" { -- cgit v1.2.3-54-g00ecf