aboutsummaryrefslogtreecommitdiff
path: root/crawler_test.go
diff options
context:
space:
mode:
Diffstat (limited to 'crawler_test.go')
-rw-r--r--crawler_test.go17
1 files changed, 15 insertions, 2 deletions
diff --git a/crawler_test.go b/crawler_test.go
index fa81c2f..d395b75 100644
--- a/crawler_test.go
+++ b/crawler_test.go
@@ -12,12 +12,25 @@ import (
)
func TestCrawler(t *testing.T) {
+
dir, err := ioutil.TempDir("", "")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(dir)
+ if err := os.Chdir(dir); err != nil {
+ t.Fatal(err)
+ }
+
+ // Create directory to (temporarily) store response bodies
+ if _, err := os.Stat("temp"); os.IsNotExist(err) {
+ err := os.Mkdir("temp", 0700)
+ if err != nil {
+ t.Fatal(err)
+ }
+ }
+
// Run a trivial test http server just so our test Fetcher can
// return a real http.Response object.
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -33,7 +46,7 @@ func TestCrawler(t *testing.T) {
)
var crawledPages int
- h := HandlerFunc(func(p Publisher, u string, tag, depth int, resp *http.Response, err error) error {
+ h := HandlerFunc(func(p Publisher, u string, tag, depth int, resp *http.Response, body *os.File, err error) error {
crawledPages++
next := fmt.Sprintf(srv.URL+"/page/%d", crawledPages)
log.Printf("%s -> %s", u, next)
@@ -44,7 +57,7 @@ func TestCrawler(t *testing.T) {
return nil
})
- crawler, err := NewCrawler(dir+"/crawl.db", seeds, scope, FetcherFunc(http.Get), HandleRetries(FilterErrors(FollowRedirects(h))))
+ crawler, err := NewCrawler("crawl.db", seeds, scope, FetcherFunc(http.Get), HandleRetries(FilterErrors(FollowRedirects(h))))
if err != nil {
t.Fatal("NewCrawler", err)
}