diff options
author | ale <ale@incal.net> | 2020-08-26 11:48:03 +0100 |
---|---|---|
committer | ale <ale@incal.net> | 2020-08-26 11:48:03 +0100 |
commit | a3019b1703b29e74cfb8fd71b9f79b0c1da11395 (patch) | |
tree | 78fddcf5101b070cbf97f462a39d1fe8187355fb | |
parent | 9540cfe27b5e7a25da2c3b554a48ee1c8ff60069 (diff) | |
download | crawl-a3019b1703b29e74cfb8fd71b9f79b0c1da11395.tar.gz crawl-a3019b1703b29e74cfb8fd71b9f79b0c1da11395.zip |
Minor logging fixes
-rw-r--r-- | cmd/crawl/crawl.go | 5 |
1 files changed, 4 insertions, 1 deletions
diff --git a/cmd/crawl/crawl.go b/cmd/crawl/crawl.go index 1637a62..4a98b02 100644 --- a/cmd/crawl/crawl.go +++ b/cmd/crawl/crawl.go @@ -217,6 +217,7 @@ func newWarcSaveHandler(w *warc.Writer) (crawl.Handler, error) { } type crawlStats struct { + urls int64 bytes int64 start time.Time @@ -228,6 +229,7 @@ func (c *crawlStats) Update(resp *http.Response) { c.lock.Lock() defer c.lock.Unlock() + c.urls++ c.states[resp.StatusCode]++ resp.Body = &byteCounter{resp.Body} } @@ -240,7 +242,7 @@ func (c *crawlStats) Dump() { c.lock.Lock() defer c.lock.Unlock() rate := float64(c.bytes) / time.Since(c.start).Seconds() / 1000 - fmt.Fprintf(os.Stderr, "stats: downloaded %d bytes (%.4g KB/s), status: %v\n", c.bytes, rate, c.states) // nolint + log.Printf("stats: downloaded %d urls, %d bytes (%.4g KB/s), status: %v", c.urls, c.bytes, rate, c.states) // nolint } var stats *crawlStats @@ -279,6 +281,7 @@ func warcWriterFromFlags() (w *warc.Writer, err error) { } func main() { + log.SetFlags(0) flag.Parse() if *cpuprofile != "" { |