From 87c21d189f042876b544b248ce39e99c16f52b74 Mon Sep 17 00:00:00 2001 From: Jordan Date: Mon, 7 Sep 2020 18:27:57 -0700 Subject: add more sorting options for family and network pages --- tor-metrics/generate.py | 54 +++++++++++++++++++++++++-- tor-metrics/relays.py | 30 ++++++++++----- tor-metrics/templates/families.html | 47 ----------------------- tor-metrics/templates/index.html | 4 +- tor-metrics/templates/misc-families.html | 64 ++++++++++++++++++++++++++++++++ tor-metrics/templates/misc-networks.html | 59 +++++++++++++++++++++++++++++ tor-metrics/templates/networks.html | 46 ----------------------- 7 files changed, 196 insertions(+), 108 deletions(-) delete mode 100644 tor-metrics/templates/families.html create mode 100644 tor-metrics/templates/misc-families.html create mode 100644 tor-metrics/templates/misc-networks.html delete mode 100644 tor-metrics/templates/networks.html diff --git a/tor-metrics/generate.py b/tor-metrics/generate.py index 5be2220..26af5b2 100755 --- a/tor-metrics/generate.py +++ b/tor-metrics/generate.py @@ -22,10 +22,56 @@ if __name__ == '__main__': # generate relay HTML documents RELAY_SET.create_output_dir() - RELAY_SET.write_unsorted('index.html', is_index=True) - RELAY_SET.write_unsorted('all.html', is_index=False) - RELAY_SET.write_unsorted('families.html', is_index=False) - RELAY_SET.write_unsorted('networks.html', is_index=False) + RELAY_SET.write_misc( + template = 'index.html', + path = 'index.html', + path_prefix = './', + is_index = True, + ) + RELAY_SET.write_misc( + template = 'all.html', + path = 'misc/all.html' + ) + RELAY_SET.write_misc( + template = 'misc-families.html', + path = 'misc/families-by-bandwidth.html', + sorted_by = '1.bandwidth' + ) + RELAY_SET.write_misc( + template = 'misc-families.html', + path = 'misc/families-by-exit-count.html', + sorted_by = '1.exit_count,1.bandwidth' + ) + RELAY_SET.write_misc( + template = 'misc-families.html', + path = 'misc/families-by-middle-count.html', + sorted_by = '1.middle_count,1.bandwidth' + ) + RELAY_SET.write_misc( + template = 'misc-families.html', + path = 'misc/families-by-first-seen.html', + sorted_by = '1.first_seen,1.bandwidth' + ) + RELAY_SET.write_misc( + template = 'misc-networks.html', + path = 'misc/networks-by-bandwidth.html', + sorted_by = '1.bandwidth' + ) + RELAY_SET.write_misc( + template = 'misc-networks.html', + path = 'misc/networks-by-exit-count.html', + sorted_by = '1.exit_count,1.bandwidth' + ) + RELAY_SET.write_misc( + template = 'misc-networks.html', + path = 'misc/networks-by-middle-count.html', + sorted_by = '1.middle_count,1.bandwidth' + ) + RELAY_SET.write_misc( + template = 'misc-networks.html', + path = 'misc/networks-by-first-seen.html', + sorted_by = '1.first_seen,1.bandwidth' + ) RELAY_SET.write_pages_by_key('as') RELAY_SET.write_pages_by_key('contact') RELAY_SET.write_pages_by_key('country') diff --git a/tor-metrics/relays.py b/tor-metrics/relays.py index 01c025e..60014c0 100644 --- a/tor-metrics/relays.py +++ b/tor-metrics/relays.py @@ -141,13 +141,13 @@ class Relays: if not v in self.json['sorted'][k]: self.json['sorted'][k][v] = { 'relays': list(), - 'bw': 0, + 'bandwidth': 0, 'exit_count': 0, 'middle_count': 0 } bw = relay['observed_bandwidth'] self.json['sorted'][k][v]['relays'].append(idx) - self.json['sorted'][k][v]['bw'] += bw + self.json['sorted'][k][v]['bandwidth'] += bw if 'Exit' in relay['flags']: self.json['sorted'][k][v]['exit_count'] += 1 else: @@ -197,17 +197,29 @@ class Relays: ''' os.makedirs(config.CONFIG['output_root'],exist_ok=True) - def write_unsorted(self, filename, is_index): + def write_misc(self, template, path, path_prefix='../', sorted_by=None, + reverse=True, is_index=False): ''' Render and write unsorted HTML listings to disk - :filename: filename to write unsorted listing (e.g. all.html) - :is_index: whether the file is an index or not (True/False) + :template: jinja template name + :path_prefix: path to prefix other docs/includes + :path: path to generate HTML document + :sorted_by: key to sort by, used in family and networks pages + :reverse: passed to sort() function in family and networks pages + :is_index: whether document is main index listing, limits list to 500 ''' - template = ENV.get_template(filename) + template = ENV.get_template(template) self.json['relay_subset'] = self.json['relays'] - template_render = template.render(relays=self, is_index=is_index) - output = os.path.join(config.CONFIG['output_root'], filename) + template_render = template.render( + relays = self, + sorted_by = sorted_by, + reverse = reverse, + is_index = is_index, + path_prefix = path_prefix + ) + output = os.path.join(config.CONFIG['output_root'], path) + os.makedirs(os.path.dirname(output), exist_ok=True) with open(output, 'w', encoding='utf8') as html: html.write(template_render) @@ -232,7 +244,7 @@ class Relays: self.json['relay_subset'] = members rendered = template.render( relays = self, - bandwidth = round(i['bw'] / 1000000, 2), + bandwidth = round(i['bandwidth'] / 1000000, 2), exit_count = i['exit_count'], middle_count = i['middle_count'], is_index = False, diff --git a/tor-metrics/templates/families.html b/tor-metrics/templates/families.html deleted file mode 100644 index cbf8e8b..0000000 --- a/tor-metrics/templates/families.html +++ /dev/null @@ -1,47 +0,0 @@ -{% extends "skeleton.html" %} -{% block title %}Tor Relays :: Families{% endblock %} -{% block body %} -

Home :: Families

-

The set of all relay families with > 1 effective members. The first_seen parameter is taken from the oldest relay in the family.

- - - - - - - - - - {% set processed = dict() %} - {% for k, v in relays.json['sorted']['family'].items()|sort(attribute='1.bw', reverse=True) %} - {% if relays.json['relays'][v['relays'][0]]['fingerprint'] not in processed %} - - {% if v['bw'] > 1000000 %} - {% set obs_bandwidth = '%s %s'|format((v['bw'] / 1000000)|round(2, 'common'), 'MB/s') %} - {% else %} - {% set obs_bandwidth = '%s %s'|format((v['bw'] / 1000)|round(2, 'common'), 'KB/s') %}{% endif %} - - - - - - {% if v['contact'] %} - - {% else %} - {% endif %} - - - - - - - {% for r in v['relays'] %} - {% set _dummy = processed.update({relays.json['relay_subset'][r]['fingerprint']: None}) %} - {% endfor %} - - {% endif %} - {% endfor %} - -
FamilyBandwidthContactExit CountMiddle CountFirst Seen -
{{ k|escape }}{{ obs_bandwidth }}{{ v['contact']|truncate(50)|escape }}none{{ v['exit_count'] }}{{ v['middle_count'] }}{{ v['first_seen'].split(' ', 1)[0]|escape }}
-{% endblock %} diff --git a/tor-metrics/templates/index.html b/tor-metrics/templates/index.html index 7447be7..c3b0e96 100644 --- a/tor-metrics/templates/index.html +++ b/tor-metrics/templates/index.html @@ -1,5 +1,5 @@ {% extends "relay-list.html" %} {% block title %}Tor Relays{% endblock %} -{% block header %}Tor Metrics (v3, git) :: Networks, Families{% endblock %} -{% block description %}Relay listings are statically generated every hour. Last fetch was at {{ relays.timestamp }}. Listing top 500 relays; view full list.{% endblock %} +{% block header %}Tor Metrics (v3, git) :: Networks, Families{% endblock %} +{% block description %}Relay listings are statically generated every hour. Last fetch was at {{ relays.timestamp }}. Listing top 500 relays; view full list.{% endblock %} diff --git a/tor-metrics/templates/misc-families.html b/tor-metrics/templates/misc-families.html new file mode 100644 index 0000000..8df7d6d --- /dev/null +++ b/tor-metrics/templates/misc-families.html @@ -0,0 +1,64 @@ +{% extends "skeleton.html" %} +{% set sorted_by_label = sorted_by.split(',')[0].split('.')[1] %} +{% block title %}Tor Relays :: Families By {{ sorted_by_label|replace('_', ' ')|title }}{% endblock %} +{% block body %} +

Home :: Families By {{ sorted_by_label|replace('_', ' ')|title }}

+

The set of all relay families with > 1 effective members, sorted by {{ sorted_by_label|replace('_', ' ') }}. The first seen parameter is taken from the oldest relay in the family.

+ + + + {% if sorted_by_label == 'bandwidth' %} + + {% else %} + + {% endif %} + + {% if sorted_by_label == 'exit_count' %} + + {% else %} + + {% endif %} + {% if sorted_by_label == 'middle_count' %} + + {% else %} + + {% endif %} + {% if sorted_by_label == 'first_seen' %} + + {% else %} + + {% endif %} + + + {% set processed = dict() %} + {% for k, v in relays.json['sorted']['family'].items()|sort(attribute=sorted_by, reverse=True) %} + {% if relays.json['relays'][v['relays'][0]]['fingerprint'] not in processed %} + + {% if v['bandwidth'] > 1000000 %} + {% set obs_bandwidth = '%s %s'|format((v['bandwidth'] / 1000000)|round(2, 'common'), 'MB/s') %} + {% else %} + {% set obs_bandwidth = '%s %s'|format((v['bandwidth'] / 1000)|round(2, 'common'), 'KB/s') %}{% endif %} + + + + + + {% if v['contact'] %} + + {% else %} + {% endif %} + + + + + + + {% for r in v['relays'] %} + {% set _dummy = processed.update({relays.json['relay_subset'][r]['fingerprint']: None}) %} + {% endfor %} + + {% endif %} + {% endfor %} + +
FamilyBandwidthBandwidthContactExit CountExit CountMiddle CountMiddle CountFirst SeenFirst Seen
{{ k|escape }}{{ obs_bandwidth }}{{ v['contact']|truncate(50)|escape }}none{{ v['exit_count'] }}{{ v['middle_count'] }}{{ v['first_seen'].split(' ', 1)[0]|escape }}
+{% endblock %} diff --git a/tor-metrics/templates/misc-networks.html b/tor-metrics/templates/misc-networks.html new file mode 100644 index 0000000..97b7c28 --- /dev/null +++ b/tor-metrics/templates/misc-networks.html @@ -0,0 +1,59 @@ +{% extends "skeleton.html" %} +{% set sorted_by_label = sorted_by.split(',')[0].split('.')[1] %} +{% block title %}Tor Relays :: Networks By {{ sorted_by_label|replace('_', ' ')|title }}{% endblock %} +{% block body %} +

Home :: Networks By {{ sorted_by_label|replace('_', ' ')|title }}

+

The set of all ({{ relays.json['sorted']['as']|length }}) recognized networks running at least one relay sorted by {{ sorted_by_label|replace('_', ' ') }}. This data is onionoo-derived and might be out of date with respect to other sources.

+ + + + + + {% if sorted_by_label == 'bandwidth' %} + + {% else %} + + {% endif %} + {% if sorted_by_label == 'exit_count' %} + + {% else %} + + {% endif %} + {% if sorted_by_label == 'middle_count' %} + + {% else %} + + {% endif %} + + + {% for k, v in relays.json['sorted']['as'].items()|sort(attribute=sorted_by, reverse=True) %} + + {% if v['bandwidth'] > 1000000 %} + {% set obs_bandwidth = '%s %s'|format((v['bandwidth'] / 1000000)|round(2, 'common'), 'MB/s') %} + {% else %} + {% set obs_bandwidth = '%s %s'|format((v['bandwidth'] / 1000)|round(2, 'common'), 'KB/s') %}{% endif %} + + + + {% if v['as_name'] %} + + {% else %} + + {% endif %} + + {% if v['country'] %} + + {% else %} + + {% endif %} + + + + + + + + {% endfor %} + +
AS NumberAS NameCountryBandwidthBandwidthExit CountExit CountMiddle CountMiddle Count
{{ k|escape }}{{ v['as_name']|truncate(20)|escape }}Unknown{{ v['country_name']|escape }}X{{ obs_bandwidth }}{{ v['exit_count'] }}{{ v['middle_count'] }}
+{% endblock %} diff --git a/tor-metrics/templates/networks.html b/tor-metrics/templates/networks.html deleted file mode 100644 index 46ac549..0000000 --- a/tor-metrics/templates/networks.html +++ /dev/null @@ -1,46 +0,0 @@ -{% extends "skeleton.html" %} -{% block title %}Tor Relays :: Networks{% endblock %} -{% block body %} -

Home :: Networks

-

The set of all ({{ relays.json['sorted']['as']|length }}) recognized networks running at least one relay. This data is onionoo-derived and might be out of date with respect to other sources.

- - - - - - - - - - {% for k, v in relays.json['sorted']['as'].items()|sort(attribute='1.bw', reverse=True) %} - - {% if v['bw'] > 1000000 %} - {% set obs_bandwidth = '%s %s'|format((v['bw'] / 1000000)|round(2, 'common'), 'MB/s') %} - {% else %} - {% set obs_bandwidth = '%s %s'|format((v['bw'] / 1000)|round(2, 'common'), 'KB/s') %}{% endif %} - - - - {% if v['as_name'] %} - - {% else %} - - {% endif %} - - {% if v['country'] %} - - {% else %} - - {% endif %} - - - - - - - - {% endfor %} - -
AS NumberAS NameCountryBandwidthExit CountMiddle Count -
{{ k|escape }}{{ v['as_name']|truncate(20)|escape }}Unknown{{ v['country_name']|escape }}X{{ obs_bandwidth }}{{ v['exit_count'] }}{{ v['middle_count'] }}
-{% endblock %} -- cgit v1.2.3-54-g00ecf