From f21a1e86291657d74c4a42b6020e2d3565ecafb4 Mon Sep 17 00:00:00 2001 From: Jordan Date: Sat, 10 Aug 2019 02:57:40 -0700 Subject: per-relay pages, template cleanup --- README.md | 9 +- tor-metrics/generate.py | 16 +++ tor-metrics/templates/all.html | 2 +- tor-metrics/templates/as.html | 2 +- tor-metrics/templates/base.html | 85 -------------- tor-metrics/templates/country.html | 2 +- tor-metrics/templates/effective_family.html | 2 +- tor-metrics/templates/index.html | 2 +- tor-metrics/templates/platform.html | 2 +- tor-metrics/templates/relay-info.html | 171 ++++++++++++++++++++++++++++ tor-metrics/templates/relay-list.html | 73 ++++++++++++ tor-metrics/templates/skeleton.html | 21 ++++ 12 files changed, 288 insertions(+), 99 deletions(-) delete mode 100644 tor-metrics/templates/base.html create mode 100644 tor-metrics/templates/relay-info.html create mode 100644 tor-metrics/templates/relay-list.html create mode 100644 tor-metrics/templates/skeleton.html diff --git a/README.md b/README.md index 2443a46..606e30a 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,6 @@ # Tor Metrics :: [yui.cat](https://yui.cat/) source -Network-oriented Tor metrics and statistics made with love (not javascript). - -* No javascript -* Static HTML/CSS generation at runtime -* Minimal library dependence (jinja2) -* Sort by ASN, country, and platform -* Respectful Onionoo queries (If-Modified-Since) +Network-oriented Tor metrics and statistics made with love (not javascript) ```bash # Requirements: python3, jinja2 (pip install jinja2) @@ -20,7 +14,6 @@ python3 generate.py ``` TODO: -* Per-relay page generation (display contact, exit policy, etc) * Top exit/guard/relay families (see https://nusenu.github.io/OrNetStats/) * Interesting statistics (ASN exit concentration, IPv6-supporting relays, etc) * Implement something similar to https://metrics.torproject.org/bubbles.html diff --git a/tor-metrics/generate.py b/tor-metrics/generate.py index 3235bd0..4bfce2b 100755 --- a/tor-metrics/generate.py +++ b/tor-metrics/generate.py @@ -18,6 +18,7 @@ def generate_html(relays): effective_family(relays) unsorted(relays, 'index.html', is_index=True) unsorted(relays.json['relays'], 'all.html', is_index=False) + relay_info(relays) static_src_path = os.path.join(abs_path, 'static') static_dest_path = os.path.join(config.CONFIG['output_root'], 'static') if not os.path.exists(static_dest_path): @@ -85,6 +86,21 @@ def pages_by_key(relays, key): encoding='utf8') as html: html.write(rendered) +def relay_info(relays): + template = env.get_template('relay-info.html') + output_path = os.path.join(config.CONFIG['output_root'], 'relay') + if os.path.exists(output_path): + rmtree(output_path) + os.makedirs(output_path) + relay_list = relays.json['relays'] + for relay in relay_list: + rendered = template.render(relay=relay, path_prefix='../') + with open(os.path.join(output_path, '%s.html' % relay['fingerprint']), + 'w', encoding='utf8') as html: + html.write(rendered) + + + relays = Relays() generate_html(relays) diff --git a/tor-metrics/templates/all.html b/tor-metrics/templates/all.html index ad1287f..d3324cf 100644 --- a/tor-metrics/templates/all.html +++ b/tor-metrics/templates/all.html @@ -1,4 +1,4 @@ -{% extends "base.html" %} +{% extends "relay-list.html" %} {% block title %}Tor Relays :: All{% endblock %} {% block header %}Home :: All Relays{% endblock %} {% block description %}All known (non-guard) relays, {{ relays|length }} in total. A smaller browser-friendly subset (top 500) is available here.{% endblock %} diff --git a/tor-metrics/templates/as.html b/tor-metrics/templates/as.html index f663080..6021dfa 100644 --- a/tor-metrics/templates/as.html +++ b/tor-metrics/templates/as.html @@ -1,4 +1,4 @@ -{% extends "base.html" %} +{% extends "relay-list.html" %} {% block title %}Tor Relays :: {{ relays[0]['as'] }}{% endblock %} {% block header %}Home :: {{ relays[0]['as'] }}{% endblock %} {% block description %}{{ relays[0]['as'] }} ({{ relays[0]['as_name']}}) is responsible for ~{{ bandwidth }} MB/s of traffic.{% endblock %} diff --git a/tor-metrics/templates/base.html b/tor-metrics/templates/base.html deleted file mode 100644 index 95ef872..0000000 --- a/tor-metrics/templates/base.html +++ /dev/null @@ -1,85 +0,0 @@ - - - {% block head %} - - {% block title %}{% endblock %} - - - - - - {% endblock %} - -
-

{% block header %}{% endblock %}

-

{% block description %}{% endblock %}

- - - - - - - - - - - - - - - {% if is_index %}{% set relay_list = relays.json['relays'][:500] %} - {% else %}{% set relay_list = relays %}{% endif %} - {% for relay in relay_list %} - - {% if relay['observed_bandwidth'] > 1000000 %} - {% set obs_bandwidth = '%s %s'|format((relay['observed_bandwidth'] / 1000000)|round(2, 'common'), 'MB/s') %} - {% else %} - {% set obs_bandwidth = '%s %s'|format((relay['observed_bandwidth'] / 1000)|round(2, 'common'), 'KB/s') %}{% endif %} - - {% if deactivate != 'family' and relay['effective_family']|length > 1 %} - - {% else %} - {% endif %} - - - - {% if relay['as'] %}{% if deactivate != 'as' %} - - {% else %} - {% endif %} - {% else %} - {% endif %} - - {% if relay['as_name'] %} - - {% else %} - {% endif %} - - {% if relay['country'] %}{% if deactivate != 'country' %} - - {% else %} - {% endif %} - {% else %} - {% endif %} - - {% if deactivate != 'platform' %} - - {% else %} - {% endif %} - - - - {% if relay['running'] %} - - {% else %} - {% endif %} - - - - {% endfor %} - -
NicknameBandwidthIP AddressAS NumberAS NameCountryPlatformFlagsRunningFirst Seen
{{ relay['nickname'] }} ({{ relay['effective_family']|length }}){{ relay['nickname'] }}{{ obs_bandwidth }}{{ relay['or_addresses'][0].split(':', 1)[0] }}{{ relay['as'] }}{{ relay['as'] }}Unknown{{ relay['as_name'] |truncate(length=20) }}Unknown{{ relay['country_name'] }}{{ relay['country_name'] }}X{{ relay['platform'] }}{{ relay['platform'] }}{% for flag in relay['flags'] %}{% if flag != 'StaleDesc' %}{{ flag }} {% endif %}{% endfor %}TrueFalse{{ relay['first_seen'].split(' ', 1)[0] }}
-

Country flags provided by GoSquared, relay flags by the Tor Project.

-
- - diff --git a/tor-metrics/templates/country.html b/tor-metrics/templates/country.html index ca98dc8..43db3c3 100644 --- a/tor-metrics/templates/country.html +++ b/tor-metrics/templates/country.html @@ -1,4 +1,4 @@ -{% extends "base.html" %} +{% extends "relay-list.html" %} {% if relays[0]['country_name'] in special_countries %} {% set country_name = 'The ' + relays[0]['country_name'] %} {% else %} diff --git a/tor-metrics/templates/effective_family.html b/tor-metrics/templates/effective_family.html index fec8609..ce85133 100644 --- a/tor-metrics/templates/effective_family.html +++ b/tor-metrics/templates/effective_family.html @@ -1,4 +1,4 @@ -{% extends "base.html" %} +{% extends "relay-list.html" %} {% block title %}Tor Relays :: Family {{ family }}{% endblock %} {% block header %}Home :: Family {{ family }}{% endblock %} {% block description %}Relays with effective family member {{ family }} are responsible for ~{{ bandwidth }} MB/s of traffic.{% endblock %} diff --git a/tor-metrics/templates/index.html b/tor-metrics/templates/index.html index c6a9fc6..883fb6e 100644 --- a/tor-metrics/templates/index.html +++ b/tor-metrics/templates/index.html @@ -1,4 +1,4 @@ -{% extends "base.html" %} +{% extends "relay-list.html" %} {% block title %}Tor Relays{% endblock %} {% block header %}Tor Metrics (v3, git){% endblock %} {% block description %}Relay listings are statically generated every hour. Last fetch was at {{ relays.timestamp }}. Listing top 500 relays; view full list.{% endblock %} diff --git a/tor-metrics/templates/platform.html b/tor-metrics/templates/platform.html index 918512c..84aace2 100644 --- a/tor-metrics/templates/platform.html +++ b/tor-metrics/templates/platform.html @@ -1,4 +1,4 @@ -{% extends "base.html" %} +{% extends "relay-list.html" %} {% block title %}Tor Relays :: {{ relays[0]['platform'] }}{% endblock %} {% block header %}Home :: {{ relays[0]['platform'] }}{% endblock %} {% block description %}{{ relays[0]['platform'] }} systems are responsible for ~{{ bandwidth }} MB/s of traffic.{% endblock %} diff --git a/tor-metrics/templates/relay-info.html b/tor-metrics/templates/relay-info.html new file mode 100644 index 0000000..79a5b24 --- /dev/null +++ b/tor-metrics/templates/relay-info.html @@ -0,0 +1,171 @@ +{% extends "skeleton.html" %} +{% block title %}Tor Relays :: {{ relay['nickname'] }}{% endblock %} +{% block body %} +
+

Home :: Relay "{{ relay['nickname'] }}"

+
+
+
+
Nickname
+
{{ relay['nickname'] }}
+ +
OR Address
+ 
{% for address in relay['or_addresses'] -%}
+                {{ address }}
+            {% endfor -%}
+
+ +
Contact
+ {% if relay['contact'] %} +
{{ relay['contact'] }}
+ {% else %} +
none
+ {% endif %} + +
Dir Address
+ {% if relay['dir_address'] %} +
{{ relay['dir_address'] }}
+ {% else %} +
none
+ {% endif %} + +
Exit Address
+ {% if relay['exit_address'] %} +
{{ relay['exit_address'] }}
+ {% else %} +
none
+ {% endif %} + +
Observed Bandwidth
+ {% if relay['observed_bandwidth'] > 1000000 %} + {% set obs_bandwidth = '%s %s'|format((relay['observed_bandwidth'] / 1000000)|round(2, 'common'), 'MB/s') %} + {% else %} + {% set obs_bandwidth = '%s %s'|format((relay['observed_bandwidth'] / 1000)|round(2, 'common'), 'KB/s') %}{% endif %} +
{{ obs_bandwidth }}
+ +
IPv4 Exit Policy Summary
+
+ {% if relay['exit_policy_summary'] -%} + {%- set v4_summary = relay['exit_policy_summary'].items() -%} + 
{% for key, value in v4_summary -%}
+                    {{ key }}: {{ value[0] }}
+                {% endfor -%}
+
+ {% else -%} + 
none
+ {% endif -%} +
+ +
IPv6 Exit Policy Summary
+
+ {% if relay['exit_policy_v6_summary'] -%} + {%- set v6_summary = relay['exit_policy_v6_summary'].items() -%} + 
{% for key, value in v6_summary -%}
+                    {{ key }}: {{ value[0] }}
+                {% endfor -%}
+
+ {% else -%} + 
none
+ {% endif -%} +
+ +
Exit Policy
+ 
{% for policy in relay['exit_policy'] -%}
+                {{ policy }}
+            {% endfor -%}
+
+ + {% if relay['effective_family']|length > 1 -%} +
Effective Family Members (view)
+ {% else %} +
Effective Family Members
+ {% endif %} + 
{% for e_relay in relay['effective_family'] -%}
+                {% if relay['effective_family']|length > 1 -%}
+                {{ e_relay }}
+                {% else -%}
+                {{ e_relay }}
+                {% endif -%}
+            {% endfor -%}
+
+ +
Alleged Family Members
+
+ {% if relay['alleged_family'] %} + 
{% for a_relay in relay['alleged_family'] -%}
+                    {% if relay['alleged_family']|length > 1 -%}
+                    {{ a_relay }}
+                    {% else -%}
+                    {{ a_relay }}
+                    {% endif -%}
+                {% endfor %}
+            {% else -%}
+                
none
+            {% endif -%}
+
+
+
+
+
+
Fingerprint
+ 
{{ relay['fingerprint'] }}
+ +
Flags
+
+ {% for flag in relay['flags'] %} + {% if flag != 'StaleDesc' %} + {{ flag }} {{ flag }} + {% endif %} + {% endfor %} +
+ +
Host Name
+
+ {% if relay['verified_host_names'] -%} + 
{{ relay['verified_host_names'][0] }}
+ {% else -%} + 
none
+ {% endif -%} + +
Country
+
+ {% if relay['country'] -%} + {{ relay['country_name'] }} {{ relay['country_name'] }} + {% else -%} + unknown + {% endif %} +
+ +
AS Number
+
+ {% if relay['as'] %} + {{ relay['as'] }} + {% else %} + unknown + {% endif %} + +
AS Name
+
+ {% if relay['as_name'] -%} + {{ relay['as_name'] }} (BGP) + {% else -%} + unknown + {% endif -%} + +
First Seen
+
{{ relay['first_seen'] }}
+ +
Last Restarted
+
{{ relay['last_restarted'] }}
+ +
Consensus Weight
+
{{ relay['consensus_weight_fraction'] }} ({{ relay['consensus_weight'] }})
+ +
Platform
+
{{ relay['platform'] }}
+ +
+
+
+
+{% endblock %} diff --git a/tor-metrics/templates/relay-list.html b/tor-metrics/templates/relay-list.html new file mode 100644 index 0000000..ec81115 --- /dev/null +++ b/tor-metrics/templates/relay-list.html @@ -0,0 +1,73 @@ +{% extends "skeleton.html" %} +{% block title %}Tor Relays{% endblock %} +{% block body %} +

{% block header %}{% endblock %}

+

{% block description %}{% endblock %}

+ + + + + + + + + + + + + + + {% if is_index %}{% set relay_list = relays.json['relays'][:500] %} + {% else %}{% set relay_list = relays %}{% endif %} + {% for relay in relay_list %} + + {% if relay['observed_bandwidth'] > 1000000 %} + {% set obs_bandwidth = '%s %s'|format((relay['observed_bandwidth'] / 1000000)|round(2, 'common'), 'MB/s') %} + {% else %} + {% set obs_bandwidth = '%s %s'|format((relay['observed_bandwidth'] / 1000)|round(2, 'common'), 'KB/s') %}{% endif %} + + {% if deactivate != 'family' and relay['effective_family']|length > 1 %} + + {% else %} + {% endif %} + + + + + {% if relay['as'] %}{% if deactivate != 'as' %} + + {% else %} + {% endif %} + {% else %} + {% endif %} + + {% if relay['as_name'] %} + + {% else %} + {% endif %} + + {% if relay['country'] %}{% if deactivate != 'country' %} + + {% else %} + {% endif %} + {% else %} + {% endif %} + + {% if deactivate != 'platform' %} + + {% else %} + {% endif %} + + + + {% if relay['running'] %} + + {% else %} + {% endif %} + + + + {% endfor %} + +
NicknameBandwidthIP AddressAS NumberAS NameCountryPlatformFlagsRunningFirst Seen
{{ relay['nickname'] }} ({{ relay['effective_family']|length }}){{ relay['nickname'] }}{{ obs_bandwidth }}{{ relay['or_addresses'][0].split(':', 1)[0] }}{{ relay['as'] }}{{ relay['as'] }}Unknown{{ relay['as_name'] |truncate(length=20) }}Unknown{{ relay['country_name'] }}{{ relay['country_name'] }}X{{ relay['platform'] }}{{ relay['platform'] }}{% for flag in relay['flags'] %}{% if flag != 'StaleDesc' %}{{ flag }} {% endif %}{% endfor %}TrueFalse{{ relay['first_seen'].split(' ', 1)[0] }}
+{% endblock %} diff --git a/tor-metrics/templates/skeleton.html b/tor-metrics/templates/skeleton.html new file mode 100644 index 0000000..17dbc8a --- /dev/null +++ b/tor-metrics/templates/skeleton.html @@ -0,0 +1,21 @@ +{% block head %} + + + + {% block title %}{% endblock %} + + + + + +{% endblock %} + +
+{% block body %} +{% endblock %} +

Country flags provided by GoSquared, relay flags by the Tor Project.

+
+ +{% block footer %} + +{% endblock %} -- cgit v1.2.3-54-g00ecf