diff options
Diffstat (limited to 'cmd/links/links.go')
-rw-r--r-- | cmd/links/links.go | 33 |
1 files changed, 3 insertions, 30 deletions
diff --git a/cmd/links/links.go b/cmd/links/links.go index 9ae2394..95388ce 100644 --- a/cmd/links/links.go +++ b/cmd/links/links.go @@ -6,14 +6,12 @@ package main import ( "flag" - "fmt" "log" "net/http" - "net/url" "strings" "git.autistici.org/ale/crawl" - "github.com/PuerkitoBio/goquery" + "git.autistici.org/ale/crawl/analysis" ) var ( @@ -23,41 +21,16 @@ var ( validSchemes = flag.String("schemes", "http,https", "comma-separated list of allowed protocols") ) -var linkMatches = []struct { - tag string - attr string -}{ - {"a", "href"}, - {"link", "href"}, - {"img", "src"}, - {"script", "src"}, -} - func extractLinks(c *crawl.Crawler, u string, depth int, resp *http.Response, err error) error { - if !strings.HasPrefix(resp.Header.Get("Content-Type"), "text/html") { - return nil - } - - doc, err := goquery.NewDocumentFromResponse(resp) + links, err := analysis.GetLinks(resp) if err != nil { return err } - links := make(map[string]*url.URL) - - for _, lm := range linkMatches { - doc.Find(fmt.Sprintf("%s[%s]", lm.tag, lm.attr)).Each(func(i int, s *goquery.Selection) { - val, _ := s.Attr(lm.attr) - if linkurl, err := resp.Request.URL.Parse(val); err == nil { - links[linkurl.String()] = linkurl - } - }) - } - for _, link := range links { - //log.Printf("%s -> %s", u, link.String()) c.Enqueue(link, depth+1) } + return nil } |