blob: 0f6bd81508661a18476de63484b3d46b645b2437 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
|
import re
import urllib.request
from html.parser import HTMLParser
class Parser(HTMLParser):
def __init__(self, links=None):
HTMLParser.__init__(self)
if links is None:
self.links = []
else:
self.links = links
self.title = []
self.current_tag = None
def handle_starttag(self, tag, attrs):
self.current_tag = tag
if tag == 'a':
self.links.append(dict(attrs).get('href'))
def handle_data(self, data):
if self.current_tag == 'title':
self.title.append(data)
def request(url, headers):
conn = urllib.request.Request(
url,
headers=headers
)
r = urllib.request.urlopen(conn)
return r
def download_file(url, headers, dest):
BLOCK = 16 * 1024
conn = urllib.request.Request(
url,
headers=headers
)
resp = urllib.request.urlopen(conn)
with open(dest, 'wb') as f:
while True:
chunk = resp.read(BLOCK)
if not chunk:
break
f.write(chunk)
def sanitize_name(name):
safe = (' ', '.', '_', '-')
name = ''.join(c for c in name if c.isalnum() or c in safe).rstrip()
return name
|