From 665f7ba1d075c36c010123e3e1566a0a04e89c7d Mon Sep 17 00:00:00 2001 From: ale Date: Tue, 19 Dec 2017 00:14:58 +0000 Subject: Skip data: URLs --- analysis/links.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/analysis/links.go b/analysis/links.go index 5d61547..3f5a795 100644 --- a/analysis/links.go +++ b/analysis/links.go @@ -71,6 +71,10 @@ func GetLinks(resp *http.Response) ([]crawl.Outlink, error) { var result []crawl.Outlink links := make(map[string]crawl.Outlink) for _, l := range outlinks { + // Skip data: URLs altogether. + if strings.HasPrefix(l.URL, "data:") { + continue + } if linkurl, err := resp.Request.URL.Parse(l.URL); err == nil { links[linkurl.String()] = crawl.Outlink{ URL: linkurl, -- cgit v1.2.3-54-g00ecf