aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorale <ale@incal.net>2020-08-26 11:48:03 +0100
committerale <ale@incal.net>2020-08-26 11:48:03 +0100
commita3019b1703b29e74cfb8fd71b9f79b0c1da11395 (patch)
tree78fddcf5101b070cbf97f462a39d1fe8187355fb
parent9540cfe27b5e7a25da2c3b554a48ee1c8ff60069 (diff)
downloadcrawl-a3019b1703b29e74cfb8fd71b9f79b0c1da11395.tar.gz
crawl-a3019b1703b29e74cfb8fd71b9f79b0c1da11395.zip
Minor logging fixes
-rw-r--r--cmd/crawl/crawl.go5
1 files changed, 4 insertions, 1 deletions
diff --git a/cmd/crawl/crawl.go b/cmd/crawl/crawl.go
index 1637a62..4a98b02 100644
--- a/cmd/crawl/crawl.go
+++ b/cmd/crawl/crawl.go
@@ -217,6 +217,7 @@ func newWarcSaveHandler(w *warc.Writer) (crawl.Handler, error) {
}
type crawlStats struct {
+ urls int64
bytes int64
start time.Time
@@ -228,6 +229,7 @@ func (c *crawlStats) Update(resp *http.Response) {
c.lock.Lock()
defer c.lock.Unlock()
+ c.urls++
c.states[resp.StatusCode]++
resp.Body = &byteCounter{resp.Body}
}
@@ -240,7 +242,7 @@ func (c *crawlStats) Dump() {
c.lock.Lock()
defer c.lock.Unlock()
rate := float64(c.bytes) / time.Since(c.start).Seconds() / 1000
- fmt.Fprintf(os.Stderr, "stats: downloaded %d bytes (%.4g KB/s), status: %v\n", c.bytes, rate, c.states) // nolint
+ log.Printf("stats: downloaded %d urls, %d bytes (%.4g KB/s), status: %v", c.urls, c.bytes, rate, c.states) // nolint
}
var stats *crawlStats
@@ -279,6 +281,7 @@ func warcWriterFromFlags() (w *warc.Writer, err error) {
}
func main() {
+ log.SetFlags(0)
flag.Parse()
if *cpuprofile != "" {