add more sorting options for family and network pages

6 files changed, 158 insertions, 70 deletions

diff --git a/tor-metrics/generate.py b/tor-metrics/generate.py
index 5be2220..26af5b2 100755
--- a/tor-metrics/generate.py
+++ b/tor-metrics/generate.py
@@ -22,10 +22,56 @@ if __name__ == '__main__':
 
     # generate relay HTML documents
     RELAY_SET.create_output_dir()
-    RELAY_SET.write_unsorted('index.html', is_index=True)
-    RELAY_SET.write_unsorted('all.html', is_index=False)
-    RELAY_SET.write_unsorted('families.html', is_index=False)
-    RELAY_SET.write_unsorted('networks.html', is_index=False)
+    RELAY_SET.write_misc(
+        template    = 'index.html',
+        path        = 'index.html',
+        path_prefix = './',
+        is_index    = True,
+    )
+    RELAY_SET.write_misc(
+        template  = 'all.html',
+        path      = 'misc/all.html'
+    )
+    RELAY_SET.write_misc(
+        template  = 'misc-families.html',
+        path      = 'misc/families-by-bandwidth.html',
+        sorted_by = '1.bandwidth'
+    )
+    RELAY_SET.write_misc(
+        template  = 'misc-families.html',
+        path      = 'misc/families-by-exit-count.html',
+        sorted_by = '1.exit_count,1.bandwidth'
+    )
+    RELAY_SET.write_misc(
+        template  = 'misc-families.html',
+        path      = 'misc/families-by-middle-count.html',
+        sorted_by = '1.middle_count,1.bandwidth'
+    )
+    RELAY_SET.write_misc(
+        template  = 'misc-families.html',
+        path      = 'misc/families-by-first-seen.html',
+        sorted_by = '1.first_seen,1.bandwidth'
+    )
+    RELAY_SET.write_misc(
+        template  = 'misc-networks.html',
+        path      = 'misc/networks-by-bandwidth.html',
+        sorted_by = '1.bandwidth'
+    )
+    RELAY_SET.write_misc(
+        template  = 'misc-networks.html',
+        path      = 'misc/networks-by-exit-count.html',
+        sorted_by = '1.exit_count,1.bandwidth'
+    )
+    RELAY_SET.write_misc(
+        template  = 'misc-networks.html',
+        path      = 'misc/networks-by-middle-count.html',
+        sorted_by = '1.middle_count,1.bandwidth'
+    )
+    RELAY_SET.write_misc(
+        template  = 'misc-networks.html',
+        path      = 'misc/networks-by-first-seen.html',
+        sorted_by = '1.first_seen,1.bandwidth'
+    )
     RELAY_SET.write_pages_by_key('as')
     RELAY_SET.write_pages_by_key('contact')
     RELAY_SET.write_pages_by_key('country')
diff --git a/tor-metrics/relays.py b/tor-metrics/relays.py
index 01c025e..60014c0 100644
--- a/tor-metrics/relays.py
+++ b/tor-metrics/relays.py
@@ -141,13 +141,13 @@ class Relays:
         if not v in self.json['sorted'][k]:
             self.json['sorted'][k][v] = {
                 'relays':       list(),
-                'bw':           0,
+                'bandwidth':    0,
                 'exit_count':   0,
                 'middle_count': 0
             }
         bw = relay['observed_bandwidth']
         self.json['sorted'][k][v]['relays'].append(idx)
-        self.json['sorted'][k][v]['bw'] += bw
+        self.json['sorted'][k][v]['bandwidth'] += bw
         if 'Exit' in relay['flags']:
             self.json['sorted'][k][v]['exit_count'] += 1
         else:
@@ -197,17 +197,29 @@ class Relays:
         '''
         os.makedirs(config.CONFIG['output_root'],exist_ok=True)
 
-    def write_unsorted(self, filename, is_index):
+    def write_misc(self, template, path, path_prefix='../', sorted_by=None,
+            reverse=True, is_index=False):
         '''
         Render and write unsorted HTML listings to disk
 
-        :filename: filename to write unsorted listing (e.g. all.html)
-        :is_index: whether the file is an index or not (True/False)
+        :template:    jinja template name
+        :path_prefix: path to prefix other docs/includes
+        :path:        path to generate HTML document
+        :sorted_by:   key to sort by, used in family and networks pages
+        :reverse:     passed to sort() function in family and networks pages
+        :is_index:    whether document is main index listing, limits list to 500
         '''
-        template = ENV.get_template(filename)
+        template = ENV.get_template(template)
         self.json['relay_subset'] = self.json['relays']
-        template_render = template.render(relays=self, is_index=is_index)
-        output = os.path.join(config.CONFIG['output_root'], filename)
+        template_render = template.render(
+            relays      = self,
+            sorted_by   = sorted_by,
+            reverse     = reverse,
+            is_index    = is_index,
+            path_prefix = path_prefix
+        )
+        output = os.path.join(config.CONFIG['output_root'], path)
+        os.makedirs(os.path.dirname(output), exist_ok=True)
         with open(output, 'w', encoding='utf8') as html:
             html.write(template_render)
 
@@ -232,7 +244,7 @@ class Relays:
             self.json['relay_subset'] = members
             rendered = template.render(
                 relays       = self,
-                bandwidth    = round(i['bw'] / 1000000, 2),
+                bandwidth    = round(i['bandwidth'] / 1000000, 2),
                 exit_count   = i['exit_count'],
                 middle_count = i['middle_count'],
                 is_index     = False,
diff --git a/tor-metrics/templates/families.html b/tor-metrics/templates/families.html
deleted file mode 100644
index cbf8e8b..0000000
--- a/tor-metrics/templates/families.html
+++ /dev/null
@@ -1,47 +0,0 @@
-{% extends "skeleton.html" %}
-{% block title %}Tor Relays :: Families{% endblock %}
-{% block body %}
-<h2><a href="./">Home</a> :: Families</h2>
-<p>The set of all relay families with > 1 effective members. The first_seen parameter is taken from the oldest relay in the family.</p>
-<table class="table table-condensed">
-    <tr>
-        <th>Family</th>
-        <th>Bandwidth</th>
-        <th class="visible-md visible-lg">Contact</th>
-        <th>Exit Count</th>
-        <th>Middle Count</th>
-        <th>First Seen<th>
-    </tr>
-    <tbody>
-        {% set processed = dict() %}
-        {% for k, v in relays.json['sorted']['family'].items()|sort(attribute='1.bw', reverse=True) %}
-        {% if relays.json['relays'][v['relays'][0]]['fingerprint'] not in processed %}
-        <tr>
-            {% if v['bw'] > 1000000 %}
-            {% set obs_bandwidth = '%s %s'|format((v['bw'] / 1000000)|round(2, 'common'), 'MB/s') %}
-            {% else %}
-            {% set obs_bandwidth = '%s %s'|format((v['bw'] / 1000)|round(2, 'common'), 'KB/s') %}{% endif %}
-
-            <td><a href="{{ path_prefix}}family/{{ k|escape }}/">{{ k|escape }}</a></td>
-
-            <td>{{ obs_bandwidth }}</td>
-
-            {% if v['contact'] %}
-            <td class="visible-md visible-lg"><a href="{{ path_prefix }}contact/{{ v['contact']|hash }}/" title="{{ v['contact']|escape }}">{{ v['contact']|truncate(50)|escape }}<a></td>
-            {% else %}
-            <td>none</td>{% endif %}
-
-            <td>{{ v['exit_count'] }}</td>
-            <td>{{ v['middle_count'] }}</td>
-
-            <td>{{ v['first_seen'].split(' ', 1)[0]|escape }}</td>
-
-            {% for r in v['relays'] %}
-            {% set _dummy = processed.update({relays.json['relay_subset'][r]['fingerprint']: None}) %}
-            {% endfor %}
-        </tr>
-        {% endif %}
-        {% endfor %}
-    </tbody>
-</table>
-{% endblock %}
diff --git a/tor-metrics/templates/index.html b/tor-metrics/templates/index.html
index 7447be7..c3b0e96 100644
--- a/tor-metrics/templates/index.html
+++ b/tor-metrics/templates/index.html
@@ -1,5 +1,5 @@
 {% extends "relay-list.html" %}
 {% block title %}Tor Relays{% endblock %}
-{% block header %}Tor Metrics (<a href="http://d6amkx45augz4kskvjryv3bmmwyf7vw4k3uwxsh5egnibw2igl5aozqd.onion/">v3</a>, <a href="https://github.com/tempname1024/tor-metrics">git</a>) :: <a href="./networks.html">Networks</a>, <a href="./families.html">Families</a>{% endblock %}
-{% block description %}Relay listings are statically generated every hour. Last fetch was at {{ relays.timestamp }}. Listing top 500 relays; <a href="all.html">view full list</a>.{% endblock %}
+{% block header %}Tor Metrics (<a href="http://d6amkx45augz4kskvjryv3bmmwyf7vw4k3uwxsh5egnibw2igl5aozqd.onion/">v3</a>, <a href="https://github.com/tempname1024/tor-metrics">git</a>) :: <a href="./misc/families-by-bandwidth.html">Networks</a>, <a href="./misc/families-by-bandwidth.html">Families</a>{% endblock %}
+{% block description %}Relay listings are statically generated every hour. Last fetch was at {{ relays.timestamp }}. Listing top 500 relays; <a href="misc/all.html">view full list</a>.{% endblock %}
 
diff --git a/tor-metrics/templates/misc-families.html b/tor-metrics/templates/misc-families.html
new file mode 100644
index 0000000..8df7d6d
--- /dev/null
+++ b/tor-metrics/templates/misc-families.html
@@ -0,0 +1,64 @@
+{% extends "skeleton.html" %}
+{% set sorted_by_label = sorted_by.split(',')[0].split('.')[1] %}
+{% block title %}Tor Relays :: Families By {{ sorted_by_label|replace('_', ' ')|title }}{% endblock %}
+{% block body %}
+<h2><a href={{ path_prefix }}>Home</a> :: Families By {{ sorted_by_label|replace('_', ' ')|title }}</h2>
+<p>The set of all relay families with > 1 effective members, sorted by {{ sorted_by_label|replace('_', ' ') }}. The first seen parameter is taken from the oldest relay in the family.</p>
+<table class="table table-condensed">
+    <tr>
+        <th>Family</th>
+        {% if sorted_by_label == 'bandwidth' %}
+        <th>Bandwidth</th>
+        {% else %}
+        <th><a href="families-by-bandwidth.html">Bandwidth</a></th>
+        {% endif %}
+        <th class="visible-md visible-lg">Contact</th>
+        {% if sorted_by_label == 'exit_count' %}
+        <th>Exit Count</th>
+        {% else %}
+        <th><a href="families-by-exit-count.html">Exit Count</a></th>
+        {% endif %}
+        {% if sorted_by_label == 'middle_count' %}
+        <th>Middle Count</th>
+        {% else %}
+        <th><a href="families-by-middle-count.html">Middle Count</a></th>
+        {% endif %}
+        {% if sorted_by_label == 'first_seen' %}
+        <th>First Seen</th>
+        {% else %}
+        <th><a href="families-by-first-seen.html">First Seen</a></th>
+        {% endif %}
+    </tr>
+    <tbody>
+        {% set processed = dict() %}
+        {% for k, v in relays.json['sorted']['family'].items()|sort(attribute=sorted_by, reverse=True) %}
+        {% if relays.json['relays'][v['relays'][0]]['fingerprint'] not in processed %}
+        <tr>
+            {% if v['bandwidth'] > 1000000 %}
+            {% set obs_bandwidth = '%s %s'|format((v['bandwidth'] / 1000000)|round(2, 'common'), 'MB/s') %}
+            {% else %}
+            {% set obs_bandwidth = '%s %s'|format((v['bandwidth'] / 1000)|round(2, 'common'), 'KB/s') %}{% endif %}
+
+            <td><a href="{{ path_prefix}}family/{{ k|escape }}/">{{ k|escape }}</a></td>
+
+            <td>{{ obs_bandwidth }}</td>
+
+            {% if v['contact'] %}
+            <td class="visible-md visible-lg"><a href="{{ path_prefix }}contact/{{ v['contact']|hash }}/" title="{{ v['contact']|escape }}">{{ v['contact']|truncate(50)|escape }}<a></td>
+            {% else %}
+            <td>none</td>{% endif %}
+
+            <td>{{ v['exit_count'] }}</td>
+            <td>{{ v['middle_count'] }}</td>
+
+            <td>{{ v['first_seen'].split(' ', 1)[0]|escape }}</td>
+
+            {% for r in v['relays'] %}
+            {% set _dummy = processed.update({relays.json['relay_subset'][r]['fingerprint']: None}) %}
+            {% endfor %}
+        </tr>
+        {% endif %}
+        {% endfor %}
+    </tbody>
+</table>
+{% endblock %}
diff --git a/tor-metrics/templates/networks.html b/tor-metrics/templates/misc-networks.html
index 46ac549..97b7c28 100644
--- a/tor-metrics/templates/networks.html
+++ b/tor-metrics/templates/misc-networks.html
@@ -1,24 +1,37 @@
 {% extends "skeleton.html" %}
-{% block title %}Tor Relays :: Networks{% endblock %}
+{% set sorted_by_label = sorted_by.split(',')[0].split('.')[1] %}
+{% block title %}Tor Relays :: Networks By {{ sorted_by_label|replace('_', ' ')|title }}{% endblock %}
 {% block body %}
-<h2><a href="./">Home</a> :: Networks</h2>
-<p>The set of all ({{ relays.json['sorted']['as']|length }}) recognized networks running at least one relay. This data is onionoo-derived and might be out of date with respect to other sources.</p>
+<h2><a href={{ path_prefix }}>Home</a> :: Networks By {{ sorted_by_label|replace('_', ' ')|title }}</h2>
+<p>The set of all ({{ relays.json['sorted']['as']|length }}) recognized networks running at least one relay sorted by {{ sorted_by_label|replace('_', ' ') }}. This data is onionoo-derived and might be out of date with respect to other sources.</p>
 <table class="table table-condensed">
     <tr>
         <th>AS Number</th>
         <th>AS Name</th>
         <th>Country</th>
+        {% if sorted_by_label == 'bandwidth' %}
         <th>Bandwidth</th>
+        {% else %}
+        <th><a href="networks-by-bandwidth.html">Bandwidth</a></th>
+        {% endif %}
+        {% if sorted_by_label == 'exit_count' %}
         <th>Exit Count</th>
-        <th>Middle Count<th>
+        {% else %}
+        <th><a href="networks-by-exit-count.html">Exit Count</a></th>
+        {% endif %}
+        {% if sorted_by_label == 'middle_count' %}
+        <th>Middle Count</th>
+        {% else %}
+        <th><a href="networks-by-middle-count.html">Middle Count</a></th>
+        {% endif %}
     </tr>
     <tbody>
-        {% for k, v in relays.json['sorted']['as'].items()|sort(attribute='1.bw', reverse=True) %}
+        {% for k, v in relays.json['sorted']['as'].items()|sort(attribute=sorted_by, reverse=True) %}
         <tr>
-            {% if v['bw'] > 1000000 %}
-            {% set obs_bandwidth = '%s %s'|format((v['bw'] / 1000000)|round(2, 'common'), 'MB/s') %}
+            {% if v['bandwidth'] > 1000000 %}
+            {% set obs_bandwidth = '%s %s'|format((v['bandwidth'] / 1000000)|round(2, 'common'), 'MB/s') %}
             {% else %}
-            {% set obs_bandwidth = '%s %s'|format((v['bw'] / 1000)|round(2, 'common'), 'KB/s') %}{% endif %}
+            {% set obs_bandwidth = '%s %s'|format((v['bandwidth'] / 1000)|round(2, 'common'), 'KB/s') %}{% endif %}
 
             <td><a href="{{ path_prefix}}as/{{ k|escape }}/">{{ k|escape }}</a></td>