1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
|
#!/usr/bin/env python3
'''
File: generate.py (executable)
Generate complete set of relay HTML pages and copy static files to
config.CONFIG['output_root'] defined in config.py
Default output directory: ./www
'''
import os
from shutil import rmtree, copytree
import config
import countries
from jinja2 import Environment, FileSystemLoader
from relays import Relays
ABS_PATH = os.path.dirname(os.path.abspath(__file__))
ENV = Environment(loader=FileSystemLoader(os.path.join(ABS_PATH, 'templates')),
trim_blocks=True, lstrip_blocks=True)
def generate_html(relays):
'''
Render and write complete set of relay pages to disk by calling each group's
respective function
Files are written to 'www' by default (defined in config.py)
:relays: relays class object containing relay set (list of dict)
'''
if relays.json is None:
return
pages_by_key(relays, 'as')
pages_by_key(relays, 'country')
pages_by_key(relays, 'platform')
effective_family(relays)
unsorted(relays, 'index.html', is_index=True)
unsorted(relays.json['relays'], 'all.html', is_index=False)
relay_info(relays)
static_src_path = os.path.join(ABS_PATH, 'static')
static_dest_path = os.path.join(config.CONFIG['output_root'], 'static')
if not os.path.exists(static_dest_path):
copytree(static_src_path, static_dest_path)
def unsorted(relays, filename, is_index):
'''
Render and write unsorted HTML listings to disk
:relays: relays class object containing relay set (list of dict)
:filename: filename to write unsorted listing (e.g. all.html)
:is_index: whether the file is an index or not (True/False)
'''
template = ENV.get_template(filename)
template_render = template.render(relays=relays, is_index=is_index)
output = os.path.join(config.CONFIG['output_root'], filename)
with open(output, 'w', encoding='utf8') as html:
html.write(template_render)
def effective_family(relays):
'''
Render and write HTML listings to disk sorted by effective family
:relays: relays class object containing relay set (list of dict)
'''
template = ENV.get_template('effective_family.html')
output_path = os.path.join(config.CONFIG['output_root'], 'family')
if os.path.exists(output_path):
rmtree(output_path)
relay_list = relays.json['relays']
q_relays = [] # qualified relays w/ > 1 effective family member
for relay in relay_list:
if len(relay['effective_family']) > 1:
q_relays.append(relay)
for relay in q_relays:
fingerprint = relay['fingerprint']
if not fingerprint.isalnum():
continue
members = [] # list of member relays (dict)
bandwidth = 0 # total bandwidth for family subset
for p_relay in q_relays:
if fingerprint in p_relay['effective_family']:
members.append(p_relay)
bandwidth += p_relay['observed_bandwidth']
dir_path = os.path.join(output_path, fingerprint)
os.makedirs(dir_path)
f_bandwidth = round(bandwidth / 1000000, 2) # convert to MB/s
rendered = template.render(relays=members, bandwidth=f_bandwidth,
is_index=False, path_prefix='../../',
deactivate='family', family=fingerprint)
with open(os.path.join(dir_path, 'index.html'), 'w',
encoding='utf8') as html:
html.write(rendered)
def pages_by_key(relays, key):
'''
Render and write HTML listings to disk sorted by KEY
:relays: relays class object containing relay set (list of dict)
:key: onionoo JSON parameter to sort by, e.g. 'platform'
'''
template = ENV.get_template(key + '.html')
output_path = os.path.join(config.CONFIG['output_root'], key)
if os.path.exists(output_path):
rmtree(output_path)
relay_list = relays.json['relays']
values_processed = [] # record values we've already processed
for idx, relay in enumerate(relay_list):
found_relays = []
bandwidth = 0 # total bandwidth for relay subset
if not relay.get(key) or relay[key] in values_processed:
continue
if not relay[key].isalnum():
continue
values_processed.append(relay[key])
# find relays w/ matching value past outer idx
for p_relay in relay_list[idx:]:
if p_relay.get(key) and p_relay[key] == relay[key]:
found_relays.append(p_relay)
bandwidth += p_relay['observed_bandwidth']
dir_path = os.path.join(output_path, relay[key])
os.makedirs(dir_path)
f_bandwidth = round(bandwidth / 1000000, 2) # convert to MB/s
rendered = template.render(relays=found_relays,
bandwidth=f_bandwidth, is_index=False,
path_prefix='../../', deactivate=key,
special_countries=countries.THE_PREFIXED)
with open(os.path.join(dir_path, 'index.html'), 'w',
encoding='utf8') as html:
html.write(rendered)
def relay_info(relays):
'''
Render and write per-relay HTML info documents to disk
:relays: relays class object containing relay set (list of dict)
'''
template = ENV.get_template('relay-info.html')
output_path = os.path.join(config.CONFIG['output_root'], 'relay')
if os.path.exists(output_path):
rmtree(output_path)
os.makedirs(output_path)
relay_list = relays.json['relays']
for relay in relay_list:
if not relay['fingerprint'].isalnum():
continue
rendered = template.render(relay=relay, path_prefix='../')
with open(os.path.join(output_path, '%s.html' % relay['fingerprint']),
'w', encoding='utf8') as html:
html.write(rendered)
RELAY_SET = Relays()
generate_html(RELAY_SET)
|
