diff options
author | Jordan <me@jordan.im> | 2022-03-24 12:02:08 -0700 |
---|---|---|
committer | Jordan <me@jordan.im> | 2022-03-24 12:02:08 -0700 |
commit | 20d95915298391ea05054238c5b13337499b7af4 (patch) | |
tree | 024c080f319fc7a430343eda96ff620ed7d1febb /crawler_test.go | |
parent | 6355aa4310ff0c32b056580e812ca6f0e2a5ee2f (diff) | |
download | crawl-20d95915298391ea05054238c5b13337499b7af4.tar.gz crawl-20d95915298391ea05054238c5b13337499b7af4.zip |
misc: update handler signatures, tests, housekeeping
Diffstat (limited to 'crawler_test.go')
-rw-r--r-- | crawler_test.go | 17 |
1 files changed, 15 insertions, 2 deletions
diff --git a/crawler_test.go b/crawler_test.go index fa81c2f..d395b75 100644 --- a/crawler_test.go +++ b/crawler_test.go @@ -12,12 +12,25 @@ import ( ) func TestCrawler(t *testing.T) { + dir, err := ioutil.TempDir("", "") if err != nil { t.Fatal(err) } defer os.RemoveAll(dir) + if err := os.Chdir(dir); err != nil { + t.Fatal(err) + } + + // Create directory to (temporarily) store response bodies + if _, err := os.Stat("temp"); os.IsNotExist(err) { + err := os.Mkdir("temp", 0700) + if err != nil { + t.Fatal(err) + } + } + // Run a trivial test http server just so our test Fetcher can // return a real http.Response object. srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -33,7 +46,7 @@ func TestCrawler(t *testing.T) { ) var crawledPages int - h := HandlerFunc(func(p Publisher, u string, tag, depth int, resp *http.Response, err error) error { + h := HandlerFunc(func(p Publisher, u string, tag, depth int, resp *http.Response, body *os.File, err error) error { crawledPages++ next := fmt.Sprintf(srv.URL+"/page/%d", crawledPages) log.Printf("%s -> %s", u, next) @@ -44,7 +57,7 @@ func TestCrawler(t *testing.T) { return nil }) - crawler, err := NewCrawler(dir+"/crawl.db", seeds, scope, FetcherFunc(http.Get), HandleRetries(FilterErrors(FollowRedirects(h)))) + crawler, err := NewCrawler("crawl.db", seeds, scope, FetcherFunc(http.Get), HandleRetries(FilterErrors(FollowRedirects(h)))) if err != nil { t.Fatal("NewCrawler", err) } |