per-relay pages, template cleanup

author: Jordan <me@jordan.im> 2019-08-10 02:57:40 -0700
committer: Jordan <me@jordan.im> 2019-08-10 02:57:40 -0700
commit: f21a1e86291657d74c4a42b6020e2d3565ecafb4 (patch)
tree: d4823d2a06b6aa838e19b76988d5a2c01bfa2a57
parent: 279721eb9df87672213ba8ae1856698efc5d3d5f (diff)
download: allium-f21a1e86291657d74c4a42b6020e2d3565ecafb4.tar.gz
allium-f21a1e86291657d74c4a42b6020e2d3565ecafb4.zip
12 files changed, 288 insertions, 99 deletions
diff --git a/README.md b/README.md
index 2443a46..606e30a 100644
--- a/README.md
+++ b/README.md
@@ -1,12 +1,6 @@
 # Tor Metrics :: [yui.cat](https://yui.cat/) source
 
-Network-oriented Tor metrics and statistics made with love (not javascript).
-
-* No javascript
-* Static HTML/CSS generation at runtime
-* Minimal library dependence (jinja2)
-* Sort by ASN, country, and platform
-* Respectful Onionoo queries (If-Modified-Since)
+Network-oriented Tor metrics and statistics made with love (not javascript)
 
 ```bash
 # Requirements: python3, jinja2 (pip install jinja2)
@@ -20,7 +14,6 @@ python3 generate.py
 
 ```
 TODO:
-* Per-relay page generation (display contact, exit policy, etc)
 * Top exit/guard/relay families (see https://nusenu.github.io/OrNetStats/)
 * Interesting statistics (ASN exit concentration, IPv6-supporting relays, etc)
 * Implement something similar to https://metrics.torproject.org/bubbles.html
diff --git a/tor-metrics/generate.py b/tor-metrics/generate.py
index 3235bd0..4bfce2b 100755
--- a/tor-metrics/generate.py
+++ b/tor-metrics/generate.py
@@ -18,6 +18,7 @@ def generate_html(relays):
         effective_family(relays)
         unsorted(relays, 'index.html', is_index=True)
         unsorted(relays.json['relays'], 'all.html', is_index=False)
+        relay_info(relays)
         static_src_path = os.path.join(abs_path, 'static')
         static_dest_path = os.path.join(config.CONFIG['output_root'], 'static')
         if not os.path.exists(static_dest_path):
@@ -85,6 +86,21 @@ def pages_by_key(relays, key):
                     encoding='utf8') as html:
                 html.write(rendered)
 
+def relay_info(relays):
+    template = env.get_template('relay-info.html')
+    output_path = os.path.join(config.CONFIG['output_root'], 'relay')
+    if os.path.exists(output_path):
+        rmtree(output_path)
+    os.makedirs(output_path)
+    relay_list = relays.json['relays']
+    for relay in relay_list:
+        rendered = template.render(relay=relay, path_prefix='../')
+        with open(os.path.join(output_path, '%s.html' % relay['fingerprint']),
+                'w', encoding='utf8') as html:
+            html.write(rendered)
+
+
+
 relays = Relays()
 generate_html(relays)
 
diff --git a/tor-metrics/templates/all.html b/tor-metrics/templates/all.html
index ad1287f..d3324cf 100644
--- a/tor-metrics/templates/all.html
+++ b/tor-metrics/templates/all.html
@@ -1,4 +1,4 @@
-{% extends "base.html" %}
+{% extends "relay-list.html" %}
 {% block title %}Tor Relays :: All{% endblock %}
 {% block header %}<a href="./">Home</a> :: All Relays{% endblock %}
 {% block description %}All known (non-guard) relays, {{ relays|length }} in total. A smaller browser-friendly subset (top 500) is <a href="./">available here</a>.{% endblock %}
diff --git a/tor-metrics/templates/as.html b/tor-metrics/templates/as.html
index f663080..6021dfa 100644
--- a/tor-metrics/templates/as.html
+++ b/tor-metrics/templates/as.html
@@ -1,4 +1,4 @@
-{% extends "base.html" %}
+{% extends "relay-list.html" %}
 {% block title %}Tor Relays :: {{ relays[0]['as'] }}{% endblock %}
 {% block header %}<a href="../../">Home</a> :: {{ relays[0]['as'] }}{% endblock %}
 {% block description %}{{ relays[0]['as'] }} ({{ relays[0]['as_name']}}) is responsible for ~{{ bandwidth }} MB/s of traffic.{% endblock %}
diff --git a/tor-metrics/templates/base.html b/tor-metrics/templates/base.html
deleted file mode 100644
index 95ef872..0000000
--- a/tor-metrics/templates/base.html
+++ /dev/null
@@ -1,85 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-    {% block head %}
-    <head>
-        <title>{% block title %}{% endblock %}</title>
-        <meta name="description" content="Javascript-free Tor metrics generated from an hourly API request.">
-        <meta charset="utf-8">
-        <meta name="viewport" content="width=device-width, initial-scale=1">
-        <link rel="stylesheet" href="{{ path_prefix }}static/css/bootstrap.min.css">
-    </head>
-    {% endblock %}
-    <body>
-        <div class="container">
-            <h2>{% block header %}{% endblock %}</h2>
-            <p>{% block description %}{% endblock %}</p>
-            <table class="table table-condensed">
-                <tr>
-                    <th>Nickname</th>
-                    <th>Bandwidth</th>
-                    <th class="visible-md visible-lg">IP Address</th>
-                    <th>AS Number</th>
-                    <th>AS Name</th>
-                    <th>Country</th>
-                    <th>Platform</th>
-                    <th>Flags</th>
-                    <th class="visible-md visible-lg">Running</th>
-                    <th class="visible-md visible-lg">First Seen</th>
-                </tr>
-                <tbody>
-                    {% if is_index %}{% set relay_list = relays.json['relays'][:500] %}
-                    {% else %}{% set relay_list = relays %}{% endif %}
-                    {% for relay in relay_list %}
-                    <tr>
-                        {% if relay['observed_bandwidth'] > 1000000 %}
-                        {% set obs_bandwidth = '%s %s'|format((relay['observed_bandwidth'] / 1000000)|round(2, 'common'), 'MB/s') %}
-                        {% else %}
-                        {% set obs_bandwidth = '%s %s'|format((relay['observed_bandwidth'] / 1000)|round(2, 'common'), 'KB/s') %}{% endif %}
-
-                        {% if deactivate != 'family' and relay['effective_family']|length > 1 %}
-                        <td>{{ relay['nickname'] }} (<a href="{{ path_prefix }}family/{{ relay['effective_family'][0] }}/">{{ relay['effective_family']|length }}</a>)</td>
-                        {% else %}
-                        <td>{{ relay['nickname'] }}</td>{% endif %}
-                        <td>{{ obs_bandwidth }}</td>
-                        <td class="visible-md visible-lg">{{ relay['or_addresses'][0].split(':', 1)[0] }}</td>
-
-                        {% if relay['as'] %}{% if deactivate != 'as' %}
-                        <td><a href="{{ path_prefix }}as/{{ relay['as'] }}/">{{ relay['as'] }}</a></td>
-                        {% else %}
-                        <td>{{ relay['as'] }}</td>{% endif %}
-                        {% else %}
-                        <td>Unknown</td>{% endif %}
-                        
-                        {% if relay['as_name'] %}
-                        <td><a href="https://bgp.he.net/{{ relay['as'] }}" title="{{ relay['as_name'] }}">{{ relay['as_name'] |truncate(length=20) }}</a></td>
-                        {% else %}
-                        <td>Unknown</td>{% endif %}
-                        
-                        {% if relay['country'] %}{% if deactivate != 'country' %}
-                        <td><a href="{{ path_prefix }}country/{{ relay['country'] }}/"><img src="{{ path_prefix }}static/images/cc/{{ relay['country'] }}.png" title="{{ relay['country_name'] }}" alt="{{ relay['country_name'] }}"></a></td>
-                        {% else %}
-                        <td><img src="{{ path_prefix }}static/images/cc/{{ relay['country'] }}.png" title="{{ relay['country_name'] }}" alt="{{ relay['country_name'] }}"></td>{% endif %}
-                        {% else %}
-                        <td>X</td>{% endif %}
-                        
-                        {% if deactivate != 'platform' %}
-                        <td><a href="{{ path_prefix }}platform/{{ relay['platform'] }}/">{{ relay['platform'] }}</a></td>
-                        {% else %}
-                        <td>{{ relay['platform'] }}</td>{% endif %}
-                        
-                        <td>{% for flag in relay['flags'] %}{% if flag != 'StaleDesc' %}<img src="{{ path_prefix}}static/images/flags/{{ flag.lower() }}.png" title="{{ flag }}" alt="{{ flag }}"> {% endif %}{% endfor %}</td>
-
-                        {% if relay['running'] %}
-                        <td class="visible-md visible-lg"><font color="green">True</font></td>
-                        {% else %}
-                        <td class="visible-md visible-lg"><font color="red">False</font></td>{% endif %}
-                        
-                        <td class="visible-md visible-lg">{{ relay['first_seen'].split(' ', 1)[0] }}</td>
-                    </tr>
-                    {% endfor %}
-                </tbody>
-            </table>
-            <p>Country flags provided by <a href="https://github.com/gosquared/flags">GoSquared</a>, relay flags by the <a href="https://gitweb.torproject.org/metrics-web.git/">Tor Project</a>.</p>
-        </div>
-    </body>
-</html>
diff --git a/tor-metrics/templates/country.html b/tor-metrics/templates/country.html
index ca98dc8..43db3c3 100644
--- a/tor-metrics/templates/country.html
+++ b/tor-metrics/templates/country.html
@@ -1,4 +1,4 @@
-{% extends "base.html" %}
+{% extends "relay-list.html" %}
 {% if relays[0]['country_name'] in special_countries %}
     {% set country_name = 'The ' + relays[0]['country_name'] %}
 {% else %}
diff --git a/tor-metrics/templates/effective_family.html b/tor-metrics/templates/effective_family.html
index fec8609..ce85133 100644
--- a/tor-metrics/templates/effective_family.html
+++ b/tor-metrics/templates/effective_family.html
@@ -1,4 +1,4 @@
-{% extends "base.html" %}
+{% extends "relay-list.html" %}
 {% block title %}Tor Relays :: Family {{ family }}{% endblock %}
 {% block header %}<a href="../../">Home</a> :: Family {{ family }}{% endblock %}
 {% block description %}Relays with effective family member {{ family }} are responsible for ~{{ bandwidth }} MB/s of traffic.{% endblock %}
diff --git a/tor-metrics/templates/index.html b/tor-metrics/templates/index.html
index c6a9fc6..883fb6e 100644
--- a/tor-metrics/templates/index.html
+++ b/tor-metrics/templates/index.html
@@ -1,4 +1,4 @@
-{% extends "base.html" %}
+{% extends "relay-list.html" %}
 {% block title %}Tor Relays{% endblock %}
 {% block header %}Tor Metrics (<a href="http://d6amkx45augz4kskvjryv3bmmwyf7vw4k3uwxsh5egnibw2igl5aozqd.onion/">v3</a>, <a href="https://github.com/tempname1024/tor-metrics">git</a>){% endblock %}
 {% block description %}Relay listings are statically generated every hour. Last fetch was at {{ relays.timestamp }}. Listing top 500 relays; <a href="all.html">view full list</a>.{% endblock %}
diff --git a/tor-metrics/templates/platform.html b/tor-metrics/templates/platform.html
index 918512c..84aace2 100644
--- a/tor-metrics/templates/platform.html
+++ b/tor-metrics/templates/platform.html
@@ -1,4 +1,4 @@
-{% extends "base.html" %}
+{% extends "relay-list.html" %}
 {% block title %}Tor Relays :: {{ relays[0]['platform'] }}{% endblock %}
 {% block header %}<a href="../../">Home</a> :: {{ relays[0]['platform'] }}{% endblock %}
 {% block description %}{{ relays[0]['platform'] }} systems are responsible for ~{{ bandwidth }} MB/s of traffic.{% endblock %}
diff --git a/tor-metrics/templates/relay-info.html b/tor-metrics/templates/relay-info.html
new file mode 100644
index 0000000..79a5b24
--- /dev/null
+++ b/tor-metrics/templates/relay-info.html
@@ -0,0 +1,171 @@
+{% extends "skeleton.html" %}
+{% block title %}Tor Relays :: {{ relay['nickname'] }}{% endblock %}
+{% block body %}
+<div id="content">
+<h2><a href="{{ path_prefix }}">Home</a> :: Relay "{{ relay['nickname'] }}"</h2>
+<div class="row">
+    <div class="col-md-6">
+        <dl>
+            <dt>Nickname</dt>
+            <dd>{{ relay['nickname'] }}</dd>
+
+            <dt>OR Address</dt>
+            <dd><pre class="pre-scrollable">{% for address in relay['or_addresses'] -%}
+                {{ address }}
+            {% endfor -%}
+            </pre></dd>
+
+            <dt>Contact</dt>
+            {% if relay['contact'] %}
+                <dd>{{ relay['contact'] }}</dd>
+            {% else %}
+                <dd>none</dd>
+            {% endif %}
+
+            <dt>Dir Address</dt>
+            {% if relay['dir_address'] %}
+                <dd>{{ relay['dir_address'] }}</dd>
+            {% else %}
+                <dd>none</dd>
+            {% endif %}
+
+            <dt>Exit Address</dt>
+            {% if relay['exit_address'] %}
+                <dd>{{ relay['exit_address'] }}</dd>
+            {% else %}
+                <dd>none</dd>
+            {% endif %}
+
+            <dt>Observed Bandwidth</dt>
+            {% if relay['observed_bandwidth'] > 1000000 %}
+            {% set obs_bandwidth = '%s %s'|format((relay['observed_bandwidth'] / 1000000)|round(2, 'common'), 'MB/s') %}
+            {% else %}
+            {% set obs_bandwidth = '%s %s'|format((relay['observed_bandwidth'] / 1000)|round(2, 'common'), 'KB/s') %}{% endif %}
+            <dd>{{ obs_bandwidth }}</dd>
+
+            <dt>IPv4 Exit Policy Summary</dt>
+            <dd>
+            {% if relay['exit_policy_summary'] -%}
+                {%- set v4_summary = relay['exit_policy_summary'].items() -%}
+                <pre class="pre-scrollable">{% for key, value in v4_summary -%}
+                    {{ key }}: {{ value[0] }}
+                {% endfor -%}
+                </pre>
+            {% else -%}
+                <pre>none</pre>
+            {% endif -%}
+            </dd>
+
+            <dt>IPv6 Exit Policy Summary</dt>
+            <dd>
+            {% if relay['exit_policy_v6_summary'] -%}
+                {%- set v6_summary = relay['exit_policy_v6_summary'].items() -%}
+                <pre class="pre-scrollable">{% for key, value in v6_summary -%}
+                    {{ key }}: {{ value[0] }}
+                {% endfor -%}
+                </pre>
+            {% else -%}
+                <pre>none</pre>
+            {% endif -%}
+            </dd>
+
+            <dt>Exit Policy</dt>
+            <dd><pre class="pre-scrollable">{% for policy in relay['exit_policy'] -%}
+                {{ policy }}
+            {% endfor -%}
+            </pre></dd>
+
+            {% if relay['effective_family']|length > 1 -%}
+                <dt>Effective Family Members (<a href="{{ path_prefix }}family/{{ relay['fingerprint'] }}">view</a>)</dt>
+            {% else %}
+                <dt>Effective Family Members</dt>
+            {% endif %}
+            <dd><pre class="pre-scrollable">{% for e_relay in relay['effective_family'] -%}
+                {% if relay['effective_family']|length > 1 -%}
+                <a href="{{ e_relay }}.html">{{ e_relay }}</a>
+                {% else -%}
+                {{ e_relay }}
+                {% endif -%}
+            {% endfor -%}
+            </pre></dd>
+
+            <dt>Alleged Family Members</dt>
+            <dd>
+            {% if relay['alleged_family'] %}
+                <pre class="pre-scrollable">{% for a_relay in relay['alleged_family'] -%}
+                    {% if relay['alleged_family']|length > 1 -%}
+                    <a href="{{ a_relay }}.html">{{ a_relay }}</a>
+                    {% else -%}
+                    {{ a_relay }}
+                    {% endif -%}
+                {% endfor %}
+            {% else -%}
+                <pre class="pre-scrollable">none
+            {% endif -%}
+            </pre></dd>
+        </dl>
+    </div>
+    <div class="col-md-6">
+        <dl>
+            <dt>Fingerprint</dt>
+            <dd><pre>{{ relay['fingerprint'] }}</pre></dd>
+
+            <dt>Flags</dt>
+            <dd>
+            {% for flag in relay['flags'] %}
+                {% if flag != 'StaleDesc' %}
+                <img src="{{ path_prefix}}static/images/flags/{{ flag.lower() }}.png" title="{{ flag }}" alt="{{ flag }}"> {{ flag }}
+                {% endif %}
+            {% endfor %}
+            </dd>
+
+            <dt>Host Name</dt>
+            <dd>
+            {% if relay['verified_host_names'] -%}
+                <pre>{{ relay['verified_host_names'][0] }}</pre>
+            {% else -%}
+                <pre>none</pre>
+            {% endif -%}
+
+            <dt>Country</dt>
+            <dd>
+            {% if relay['country'] -%}
+                <a href="{{ path_prefix }}country/{{ relay['country'] }}/"><img src="{{ path_prefix }}static/images/cc/{{ relay['country'] }}.png" title="{{ relay['country_name'] }}" alt="{{ relay['country_name'] }}"></a> {{ relay['country_name'] }}
+            {% else -%}
+                unknown
+            {% endif %}
+            </dd>
+
+            <dt>AS Number</dt>
+            <dd>
+            {% if relay['as'] %}
+                <a href='{{ path_prefix }}as/{{ relay['as'] }}'>{{ relay['as'] }}</a>
+            {% else %}
+                unknown
+            {% endif %}
+
+            <dt>AS Name</dt>
+            <dd>
+            {% if relay['as_name'] -%}
+                {{ relay['as_name'] }} (<a href='https://bgp.he.net/{{ relay['as'] }}'>BGP</a>)
+            {% else -%}
+                unknown
+            {% endif -%}
+
+            <dt>First Seen</dt>
+            <dd>{{ relay['first_seen'] }}</dd>
+
+            <dt>Last Restarted</dt>
+            <dd>{{ relay['last_restarted'] }}</dd>
+
+            <dt>Consensus Weight</dt>
+            <dd>{{ relay['consensus_weight_fraction'] }} ({{ relay['consensus_weight'] }})</dd>
+
+            <dt>Platform</dt>
+            <dd>{{ relay['platform'] }}</dd>
+
+        </dl>
+    </div>
+</div>
+</div>
+{% endblock %}
diff --git a/tor-metrics/templates/relay-list.html b/tor-metrics/templates/relay-list.html
new file mode 100644
index 0000000..ec81115
--- /dev/null
+++ b/tor-metrics/templates/relay-list.html
@@ -0,0 +1,73 @@
+{% extends "skeleton.html" %}
+{% block title %}Tor Relays{% endblock %}
+{% block body %}
+<h2>{% block header %}{% endblock %}</h2>
+<p>{% block description %}{% endblock %}</p>
+<table class="table table-condensed">
+    <tr>
+        <th>Nickname</th>
+        <th>Bandwidth</th>
+        <th class="visible-md visible-lg">IP Address</th>
+        <th>AS Number</th>
+        <th>AS Name</th>
+        <th>Country</th>
+        <th>Platform</th>
+        <th>Flags</th>
+        <th class="visible-md visible-lg">Running</th>
+        <th class="visible-md visible-lg">First Seen</th>
+    </tr>
+    <tbody>
+        {% if is_index %}{% set relay_list = relays.json['relays'][:500] %}
+        {% else %}{% set relay_list = relays %}{% endif %}
+        {% for relay in relay_list %}
+        <tr>
+            {% if relay['observed_bandwidth'] > 1000000 %}
+            {% set obs_bandwidth = '%s %s'|format((relay['observed_bandwidth'] / 1000000)|round(2, 'common'), 'MB/s') %}
+            {% else %}
+            {% set obs_bandwidth = '%s %s'|format((relay['observed_bandwidth'] / 1000)|round(2, 'common'), 'KB/s') %}{% endif %}
+
+            {% if deactivate != 'family' and relay['effective_family']|length > 1 %}
+            <td><a href="{{ path_prefix }}relay/{{ relay['fingerprint'] }}.html">{{ relay['nickname'] }}</a> (<a href="{{ path_prefix }}family/{{ relay['effective_family'][0] }}/">{{ relay['effective_family']|length }}</a>)</td>
+            {% else %}
+            <td><a href="{{ path_prefix}}relay/{{ relay['fingerprint'] }}.html">{{ relay['nickname'] }}</a></td>{% endif %}
+            <td>{{ obs_bandwidth }}</td>
+
+            <td class="visible-md visible-lg">{{ relay['or_addresses'][0].split(':', 1)[0] }}</td>
+
+            {% if relay['as'] %}{% if deactivate != 'as' %}
+            <td><a href="{{ path_prefix }}as/{{ relay['as'] }}/">{{ relay['as'] }}</a></td>
+            {% else %}
+            <td>{{ relay['as'] }}</td>{% endif %}
+            {% else %}
+            <td>Unknown</td>{% endif %}
+
+            {% if relay['as_name'] %}
+            <td><a href="https://bgp.he.net/{{ relay['as'] }}" title="{{ relay['as_name'] }}">{{ relay['as_name'] |truncate(length=20) }}</a></td>
+            {% else %}
+            <td>Unknown</td>{% endif %}
+
+            {% if relay['country'] %}{% if deactivate != 'country' %}
+            <td><a href="{{ path_prefix }}country/{{ relay['country'] }}/"><img src="{{ path_prefix }}static/images/cc/{{ relay['country'] }}.png" title="{{ relay['country_name'] }}" alt="{{ relay['country_name'] }}"></a></td>
+            {% else %}
+            <td><img src="{{ path_prefix }}static/images/cc/{{ relay['country'] }}.png" title="{{ relay['country_name'] }}" alt="{{ relay['country_name'] }}"></td>{% endif %}
+            {% else %}
+            <td>X</td>{% endif %}
+
+            {% if deactivate != 'platform' %}
+            <td><a href="{{ path_prefix }}platform/{{ relay['platform'] }}/">{{ relay['platform'] }}</a></td>
+            {% else %}
+            <td>{{ relay['platform'] }}</td>{% endif %}
+
+            <td>{% for flag in relay['flags'] %}{% if flag != 'StaleDesc' %}<img src="{{ path_prefix}}static/images/flags/{{ flag.lower() }}.png" title="{{ flag }}" alt="{{ flag }}"> {% endif %}{% endfor %}</td>
+
+            {% if relay['running'] %}
+            <td class="visible-md visible-lg"><font color="green">True</font></td>
+            {% else %}
+            <td class="visible-md visible-lg"><font color="red">False</font></td>{% endif %}
+
+            <td class="visible-md visible-lg">{{ relay['first_seen'].split(' ', 1)[0] }}</td>
+        </tr>
+        {% endfor %}
+    </tbody>
+</table>
+{% endblock %}
diff --git a/tor-metrics/templates/skeleton.html b/tor-metrics/templates/skeleton.html
new file mode 100644
index 0000000..17dbc8a
--- /dev/null
+++ b/tor-metrics/templates/skeleton.html
@@ -0,0 +1,21 @@
+{% block head %}
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <title>{% block title %}{% endblock %}</title>
+    <meta name="description" content="Javascript-free Tor metrics generated from an hourly API request.">
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <link rel="stylesheet" href="{{ path_prefix }}static/css/bootstrap.min.css">
+</head>
+{% endblock %}
+<body>
+<div class="container">
+{% block body %}
+{% endblock %}
+<p>Country flags provided by <a href="https://github.com/gosquared/flags">GoSquared</a>, relay flags by the <a href="https://gitweb.torproject.org/metrics-web.git/">Tor Project</a>.</p>
+</div>
+</body>
+{% block footer %}
+</html>
+{% endblock %}
author	Jordan <me@jordan.im>	2019-08-10 02:57:40 -0700
committer	Jordan <me@jordan.im>	2019-08-10 02:57:40 -0700
commit	f21a1e86291657d74c4a42b6020e2d3565ecafb4 (patch)
tree	d4823d2a06b6aa838e19b76988d5a2c01bfa2a57
parent	279721eb9df87672213ba8ae1856698efc5d3d5f (diff)
download	allium-f21a1e86291657d74c4a42b6020e2d3565ecafb4.tar.gz allium-f21a1e86291657d74c4a42b6020e2d3565ecafb4.zip