add network-global family and ASN pages

5 files changed, 111 insertions, 1 deletions

diff --git a/tor-metrics/generate.py b/tor-metrics/generate.py
index b7b554b..5be2220 100755
--- a/tor-metrics/generate.py
+++ b/tor-metrics/generate.py
@@ -24,6 +24,8 @@ if __name__ == '__main__':
     RELAY_SET.create_output_dir()
     RELAY_SET.write_unsorted('index.html', is_index=True)
     RELAY_SET.write_unsorted('all.html', is_index=False)
+    RELAY_SET.write_unsorted('families.html', is_index=False)
+    RELAY_SET.write_unsorted('networks.html', is_index=False)
     RELAY_SET.write_pages_by_key('as')
     RELAY_SET.write_pages_by_key('contact')
     RELAY_SET.write_pages_by_key('country')
diff --git a/tor-metrics/relays.py b/tor-metrics/relays.py
index be3b893..01c025e 100644
--- a/tor-metrics/relays.py
+++ b/tor-metrics/relays.py
@@ -153,6 +153,21 @@ class Relays:
         else:
             self.json['sorted'][k][v]['middle_count'] += 1
 
+        if k is 'as':
+            self.json['sorted'][k][v]['country'] = relay.get('country')
+            self.json['sorted'][k][v]['country_name'] = relay.get('country')
+            self.json['sorted'][k][v]['as_name'] = relay.get('as_name')
+
+        if k is 'family':
+            self.json['sorted'][k][v]['contact'] = relay.get('contact')
+
+            # update the first_seen parameter to always contain the oldest
+            # relay's first_seen date
+            if not self.json['sorted'][k][v].get('first_seen'):
+                self.json['sorted'][k][v]['first_seen'] = relay['first_seen']
+            elif self.json['sorted'][k][v]['first_seen'] > relay['first_seen']:
+                self.json['sorted'][k][v]['first_seen'] = relay['first_seen']
+
     def _categorize(self):
         '''
         Iterate over self.json['relays'] set and call self._sort() against
diff --git a/tor-metrics/templates/families.html b/tor-metrics/templates/families.html
new file mode 100644
index 0000000..d1ae9d7
--- /dev/null
+++ b/tor-metrics/templates/families.html
@@ -0,0 +1,47 @@
+{% extends "skeleton.html" %}
+{% block title %}Tor Relays :: Families{% endblock %}
+{% block body %}
+<h2><a href="./">Home</a> :: Families</h2>
+<p>The set of all ({{ relays.json['sorted']['family']|length }}) relay families with > 1 effective members. The first_seen parameter is taken from the oldest relay in the family.</p>
+<table class="table table-condensed">
+    <tr>
+        <th>Family</th>
+        <th>Bandwidth</th>
+        <th class="visible-md visible-lg">Contact</th>
+        <th>Exit Count</th>
+        <th>Middle Count</th>
+        <th>First Seen<th>
+    </tr>
+    <tbody>
+        {% set processed = dict() %}
+        {% for k, v in relays.json['sorted']['family'].items()|sort(attribute='1.bw', reverse=True) %}
+        {% if relays.json['relays'][v['relays'][0]]['fingerprint'] not in processed %}
+        <tr>
+            {% if v['bw'] > 1000000 %}
+            {% set obs_bandwidth = '%s %s'|format((v['bw'] / 1000000)|round(2, 'common'), 'MB/s') %}
+            {% else %}
+            {% set obs_bandwidth = '%s %s'|format((v['bw'] / 1000)|round(2, 'common'), 'KB/s') %}{% endif %}
+
+            <td><a href="{{ path_prefix}}family/{{ k|escape }}/">{{ k|escape }}</a></td>
+
+            <td>{{ obs_bandwidth }}</td>
+
+            {% if v['contact'] %}
+            <td class="visible-md visible-lg"><a href="{{ path_prefix }}contact/{{ v['contact']|hash }}/" title="{{ v['contact']|escape }}">{{ v['contact']|truncate(50)|escape }}<a></td>
+            {% else %}
+            <td>none</td>{% endif %}
+
+            <td>{{ v['exit_count'] }}</td>
+            <td>{{ v['middle_count'] }}</td>
+
+            <td>{{ v['first_seen'].split(' ', 1)[0]|escape }}</td>
+
+            {% for r in v['relays'] %}
+            {% set _dummy = processed.update({relays.json['relay_subset'][r]['fingerprint']: None}) %}
+            {% endfor %}
+        </tr>
+        {% endif %}
+        {% endfor %}
+    </tbody>
+</table>
+{% endblock %}
diff --git a/tor-metrics/templates/index.html b/tor-metrics/templates/index.html
index 883fb6e..7447be7 100644
--- a/tor-metrics/templates/index.html
+++ b/tor-metrics/templates/index.html
@@ -1,5 +1,5 @@
 {% extends "relay-list.html" %}
 {% block title %}Tor Relays{% endblock %}
-{% block header %}Tor Metrics (<a href="http://d6amkx45augz4kskvjryv3bmmwyf7vw4k3uwxsh5egnibw2igl5aozqd.onion/">v3</a>, <a href="https://github.com/tempname1024/tor-metrics">git</a>){% endblock %}
+{% block header %}Tor Metrics (<a href="http://d6amkx45augz4kskvjryv3bmmwyf7vw4k3uwxsh5egnibw2igl5aozqd.onion/">v3</a>, <a href="https://github.com/tempname1024/tor-metrics">git</a>) :: <a href="./networks.html">Networks</a>, <a href="./families.html">Families</a>{% endblock %}
 {% block description %}Relay listings are statically generated every hour. Last fetch was at {{ relays.timestamp }}. Listing top 500 relays; <a href="all.html">view full list</a>.{% endblock %}
 
diff --git a/tor-metrics/templates/networks.html b/tor-metrics/templates/networks.html
new file mode 100644
index 0000000..46ac549
--- /dev/null
+++ b/tor-metrics/templates/networks.html
@@ -0,0 +1,46 @@
+{% extends "skeleton.html" %}
+{% block title %}Tor Relays :: Networks{% endblock %}
+{% block body %}
+<h2><a href="./">Home</a> :: Networks</h2>
+<p>The set of all ({{ relays.json['sorted']['as']|length }}) recognized networks running at least one relay. This data is onionoo-derived and might be out of date with respect to other sources.</p>
+<table class="table table-condensed">
+    <tr>
+        <th>AS Number</th>
+        <th>AS Name</th>
+        <th>Country</th>
+        <th>Bandwidth</th>
+        <th>Exit Count</th>
+        <th>Middle Count<th>
+    </tr>
+    <tbody>
+        {% for k, v in relays.json['sorted']['as'].items()|sort(attribute='1.bw', reverse=True) %}
+        <tr>
+            {% if v['bw'] > 1000000 %}
+            {% set obs_bandwidth = '%s %s'|format((v['bw'] / 1000000)|round(2, 'common'), 'MB/s') %}
+            {% else %}
+            {% set obs_bandwidth = '%s %s'|format((v['bw'] / 1000)|round(2, 'common'), 'KB/s') %}{% endif %}
+
+            <td><a href="{{ path_prefix}}as/{{ k|escape }}/">{{ k|escape }}</a></td>
+
+            {% if v['as_name'] %}
+            <td title="{{ v['as_name']|escape }}"><a href="https://bgp.he.net/{{ k|escape }}">{{ v['as_name']|truncate(20)|escape }}</a></td>
+            {% else %}
+            <td>Unknown</td>
+            {% endif %}
+
+            {% if v['country'] %}
+            <td><a href="{{ path_prefix }}country/{{ v['country']|escape }}/"><img src="{{ path_prefix }}static/images/cc/{{ v['country']|escape }}.png" title="{{ v['country_name']|escape }}"alt="{{ v['country_name']|escape }}"></a></td>
+            {% else %}
+            <td>X</td>
+            {% endif %}
+
+            <td>{{ obs_bandwidth }}</td>
+
+            <td>{{ v['exit_count'] }}</td>
+
+            <td>{{ v['middle_count'] }}</td>
+        </tr>
+        {% endfor %}
+    </tbody>
+</table>
+{% endblock %}