From 290b78679a136ecb879d76503ea20411e3643501 Mon Sep 17 00:00:00 2001 From: Jordan Date: Mon, 7 Sep 2020 05:02:16 -0700 Subject: add network-global family and ASN pages --- tor-metrics/generate.py | 2 ++ tor-metrics/relays.py | 15 ++++++++++++ tor-metrics/templates/families.html | 47 +++++++++++++++++++++++++++++++++++++ tor-metrics/templates/index.html | 2 +- tor-metrics/templates/networks.html | 46 ++++++++++++++++++++++++++++++++++++ 5 files changed, 111 insertions(+), 1 deletion(-) create mode 100644 tor-metrics/templates/families.html create mode 100644 tor-metrics/templates/networks.html diff --git a/tor-metrics/generate.py b/tor-metrics/generate.py index b7b554b..5be2220 100755 --- a/tor-metrics/generate.py +++ b/tor-metrics/generate.py @@ -24,6 +24,8 @@ if __name__ == '__main__': RELAY_SET.create_output_dir() RELAY_SET.write_unsorted('index.html', is_index=True) RELAY_SET.write_unsorted('all.html', is_index=False) + RELAY_SET.write_unsorted('families.html', is_index=False) + RELAY_SET.write_unsorted('networks.html', is_index=False) RELAY_SET.write_pages_by_key('as') RELAY_SET.write_pages_by_key('contact') RELAY_SET.write_pages_by_key('country') diff --git a/tor-metrics/relays.py b/tor-metrics/relays.py index be3b893..01c025e 100644 --- a/tor-metrics/relays.py +++ b/tor-metrics/relays.py @@ -153,6 +153,21 @@ class Relays: else: self.json['sorted'][k][v]['middle_count'] += 1 + if k is 'as': + self.json['sorted'][k][v]['country'] = relay.get('country') + self.json['sorted'][k][v]['country_name'] = relay.get('country') + self.json['sorted'][k][v]['as_name'] = relay.get('as_name') + + if k is 'family': + self.json['sorted'][k][v]['contact'] = relay.get('contact') + + # update the first_seen parameter to always contain the oldest + # relay's first_seen date + if not self.json['sorted'][k][v].get('first_seen'): + self.json['sorted'][k][v]['first_seen'] = relay['first_seen'] + elif self.json['sorted'][k][v]['first_seen'] > relay['first_seen']: + self.json['sorted'][k][v]['first_seen'] = relay['first_seen'] + def _categorize(self): ''' Iterate over self.json['relays'] set and call self._sort() against diff --git a/tor-metrics/templates/families.html b/tor-metrics/templates/families.html new file mode 100644 index 0000000..d1ae9d7 --- /dev/null +++ b/tor-metrics/templates/families.html @@ -0,0 +1,47 @@ +{% extends "skeleton.html" %} +{% block title %}Tor Relays :: Families{% endblock %} +{% block body %} +

Home :: Families

+

The set of all ({{ relays.json['sorted']['family']|length }}) relay families with > 1 effective members. The first_seen parameter is taken from the oldest relay in the family.

+ + + + + + + + + + {% set processed = dict() %} + {% for k, v in relays.json['sorted']['family'].items()|sort(attribute='1.bw', reverse=True) %} + {% if relays.json['relays'][v['relays'][0]]['fingerprint'] not in processed %} + + {% if v['bw'] > 1000000 %} + {% set obs_bandwidth = '%s %s'|format((v['bw'] / 1000000)|round(2, 'common'), 'MB/s') %} + {% else %} + {% set obs_bandwidth = '%s %s'|format((v['bw'] / 1000)|round(2, 'common'), 'KB/s') %}{% endif %} + + + + + + {% if v['contact'] %} + + {% else %} + {% endif %} + + + + + + + {% for r in v['relays'] %} + {% set _dummy = processed.update({relays.json['relay_subset'][r]['fingerprint']: None}) %} + {% endfor %} + + {% endif %} + {% endfor %} + +
FamilyBandwidthContactExit CountMiddle CountFirst Seen +
{{ k|escape }}{{ obs_bandwidth }}{{ v['contact']|truncate(50)|escape }}none{{ v['exit_count'] }}{{ v['middle_count'] }}{{ v['first_seen'].split(' ', 1)[0]|escape }}
+{% endblock %} diff --git a/tor-metrics/templates/index.html b/tor-metrics/templates/index.html index 883fb6e..7447be7 100644 --- a/tor-metrics/templates/index.html +++ b/tor-metrics/templates/index.html @@ -1,5 +1,5 @@ {% extends "relay-list.html" %} {% block title %}Tor Relays{% endblock %} -{% block header %}Tor Metrics (v3, git){% endblock %} +{% block header %}Tor Metrics (v3, git) :: Networks, Families{% endblock %} {% block description %}Relay listings are statically generated every hour. Last fetch was at {{ relays.timestamp }}. Listing top 500 relays; view full list.{% endblock %} diff --git a/tor-metrics/templates/networks.html b/tor-metrics/templates/networks.html new file mode 100644 index 0000000..46ac549 --- /dev/null +++ b/tor-metrics/templates/networks.html @@ -0,0 +1,46 @@ +{% extends "skeleton.html" %} +{% block title %}Tor Relays :: Networks{% endblock %} +{% block body %} +

Home :: Networks

+

The set of all ({{ relays.json['sorted']['as']|length }}) recognized networks running at least one relay. This data is onionoo-derived and might be out of date with respect to other sources.

+ + + + + + + + + + {% for k, v in relays.json['sorted']['as'].items()|sort(attribute='1.bw', reverse=True) %} + + {% if v['bw'] > 1000000 %} + {% set obs_bandwidth = '%s %s'|format((v['bw'] / 1000000)|round(2, 'common'), 'MB/s') %} + {% else %} + {% set obs_bandwidth = '%s %s'|format((v['bw'] / 1000)|round(2, 'common'), 'KB/s') %}{% endif %} + + + + {% if v['as_name'] %} + + {% else %} + + {% endif %} + + {% if v['country'] %} + + {% else %} + + {% endif %} + + + + + + + + {% endfor %} + +
AS NumberAS NameCountryBandwidthExit CountMiddle Count +
{{ k|escape }}{{ v['as_name']|truncate(20)|escape }}Unknown{{ v['country_name']|escape }}X{{ obs_bandwidth }}{{ v['exit_count'] }}{{ v['middle_count'] }}
+{% endblock %} -- cgit v1.2.3-54-g00ecf