aboutsummaryrefslogtreecommitdiff
path: root/crawler.go
diff options
context:
space:
mode:
Diffstat (limited to 'crawler.go')
-rw-r--r--crawler.go7
1 files changed, 4 insertions, 3 deletions
diff --git a/crawler.go b/crawler.go
index ed43b1f..709ff3c 100644
--- a/crawler.go
+++ b/crawler.go
@@ -272,8 +272,9 @@ func NewCrawler(path string, seeds []*url.URL, scope Scope, f Fetcher, h Handler
return c, nil
}
-// Run the crawl, does not exit until it is done.
-func (c *Crawler) Run() {
+// Run the crawl with the specified number of workers. This function
+// does not exit until all work is done (no URLs left in the queue).
+func (c *Crawler) Run(concurrency int) {
// Load initial seeds into the queue.
for _, u := range c.seeds {
c.Enqueue(u, 0)
@@ -282,7 +283,7 @@ func (c *Crawler) Run() {
// Start some runners and wait until they're done.
var wg sync.WaitGroup
ch := c.process()
- for i := 0; i < 3; i++ {
+ for i := 0; i < concurrency; i++ {
wg.Add(1)
go func() {
c.urlHandler(ch)