From 6355aa4310ff0c32b056580e812ca6f0e2a5ee2f Mon Sep 17 00:00:00 2001 From: Jordan Date: Thu, 24 Mar 2022 09:08:13 -0700 Subject: links, crawl: dramatically reduce memory usage to prevent excessive memory usage and OOM crashes, rather than store and pass around response bodies in memory buffers, let's store them temporarily on the filesystem wget-style and delete them when processed --- analysis/links.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) (limited to 'analysis/links.go') diff --git a/analysis/links.go b/analysis/links.go index c0663fa..f1b3e99 100644 --- a/analysis/links.go +++ b/analysis/links.go @@ -7,6 +7,7 @@ import ( "io" "io/ioutil" "net/http" + "os" "regexp" "strings" @@ -41,12 +42,12 @@ type rawOutlink struct { // GetLinks returns all the links found in a document. Currently only // parses HTML pages and CSS stylesheets. -func GetLinks(resp *http.Response) ([]crawl.Outlink, error) { +func GetLinks(resp *http.Response, body *os.File) ([]crawl.Outlink, error) { // Parse outbound links relative to the request URI, and // return unique results. var result []crawl.Outlink links := make(map[string]crawl.Outlink) - for _, l := range extractLinks(resp) { + for _, l := range extractLinks(resp, body) { // Skip data: URLs altogether. if strings.HasPrefix(l.URL, "data:") { continue @@ -64,13 +65,13 @@ func GetLinks(resp *http.Response) ([]crawl.Outlink, error) { return result, nil } -func extractLinks(resp *http.Response) []rawOutlink { +func extractLinks(resp *http.Response, body *os.File) []rawOutlink { ctype := resp.Header.Get("Content-Type") switch { case strings.HasPrefix(ctype, "text/html"): - return extractLinksFromHTML(resp.Body, nil) + return extractLinksFromHTML(body, nil) case strings.HasPrefix(ctype, "text/css"): - return extractLinksFromCSS(resp.Body, nil) + return extractLinksFromCSS(body, nil) default: return nil } -- cgit v1.2.3-54-g00ecf