From 28e4ed0e3296240ae4b86fc86cf11668e43fd452 Mon Sep 17 00:00:00 2001 From: Jordan Date: Sun, 2 Feb 2020 10:44:02 -0700 Subject: block requests for resources which lack content-length --- lib/request.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/request.py b/lib/request.py index 6cdfa57..7cf0edf 100644 --- a/lib/request.py +++ b/lib/request.py @@ -49,8 +49,12 @@ def retrieve(url, headers): request_info = dict(request.info()) headers = {k.lower(): v for k, v in request_info.items()} - # ensure size of resource falls within MAX_SIZE before downloading - if int(headers.get('content-length')) > MAX_SIZE: + # ensure size of resource falls below MAX_SIZE before downloading + # TODO: support requests for resources which lack content-length (streams) + if not headers.get('content-length'): + print('no content-length provided, blocking: %s' % (url)) + return {'url': url, 'code': 413, 'data': None, 'meta': None} + if int(headers['content-length']) > MAX_SIZE: print('exceeded MAX_SIZE of %s bytes, skipping: %s' % (MAX_SIZE, url)) return {'url': url, 'code': 413, 'data': None, 'meta': None} -- cgit v1.2.3-54-g00ecf