diff options
Diffstat (limited to 'lib/request.py')
| -rw-r--r-- | lib/request.py | 8 |
1 files changed, 6 insertions, 2 deletions
diff --git a/lib/request.py b/lib/request.py index 6cdfa57..7cf0edf 100644 --- a/lib/request.py +++ b/lib/request.py @@ -49,8 +49,12 @@ def retrieve(url, headers): request_info = dict(request.info()) headers = {k.lower(): v for k, v in request_info.items()} - # ensure size of resource falls within MAX_SIZE before downloading - if int(headers.get('content-length')) > MAX_SIZE: + # ensure size of resource falls below MAX_SIZE before downloading + # TODO: support requests for resources which lack content-length (streams) + if not headers.get('content-length'): + print('no content-length provided, blocking: %s' % (url)) + return {'url': url, 'code': 413, 'data': None, 'meta': None} + if int(headers['content-length']) > MAX_SIZE: print('exceeded MAX_SIZE of %s bytes, skipping: %s' % (MAX_SIZE, url)) return {'url': url, 'code': 413, 'data': None, 'meta': None} |