Ignore URL decode errors

This is an internal inconsistency that should be investigated.

1 files changed, 9 insertions, 2 deletions

diff --git a/crawler.go b/crawler.go
index 49d124b..b2ad3d9 100644
--- a/crawler.go
+++ b/crawler.go
@@ -154,8 +154,8 @@ func normalizeURL(u *url.URL) *url.URL {
 			purell.FlagRemoveFragment|purell.FlagSortQuery)
 	u2, err := url.Parse(urlStr)
 	if err != nil {
-		// We *really* do not expect an error here.
-		panic(err)
+		// Ignore errors here.
+		return nil
 	}
 	return u2
 }
@@ -178,6 +178,13 @@ func (c *Crawler) Enqueue(link Outlink, depth int) error {
 	// Normalize the URL. We are going to replace link.URL in-place, to
 	// ensure that scope checks are applied to the normalized URL.
 	link.URL = normalizeURL(link.URL)
+	if link.URL == nil {
+		// We couldn't parse a URL that we have extracted
+		// ourselves from the documents. This is an internal
+		// inconsistency, but by ignoring the error we avoid
+		// failing the entire crawl.
+		return nil
+	}
 
 	// See if it's in scope.
 	if !c.scope.Check(link, depth) {