preliminary effective family display/sorting

4 files changed, 45 insertions, 15 deletions

diff --git a/README.md b/README.md
index 54bf7b9..2443a46 100644
--- a/README.md
+++ b/README.md
@@ -21,7 +21,6 @@ python3 generate.py
 ```
 TODO:
 * Per-relay page generation (display contact, exit policy, etc)
-* Sort by effective family, show member count globally
 * Top exit/guard/relay families (see https://nusenu.github.io/OrNetStats/)
 * Interesting statistics (ASN exit concentration, IPv6-supporting relays, etc)
 * Implement something similar to https://metrics.torproject.org/bubbles.html
diff --git a/tor-metrics/generate.py b/tor-metrics/generate.py
index 2f573a5..fdb0a25 100755
--- a/tor-metrics/generate.py
+++ b/tor-metrics/generate.py
@@ -15,6 +15,7 @@ def generate_html(relays):
         pages_by_key(relays, 'as')
         pages_by_key(relays, 'country')
         pages_by_key(relays, 'platform')
+        effective_family(relays)
         unsorted(relays, 'index.html', is_index=True)
         unsorted(relays.json['relays'], 'all.html', is_index=False)
         static_src_path = os.path.join(abs_path, 'static')
@@ -23,12 +24,31 @@ def generate_html(relays):
             copytree(static_src_path, static_dest_path)
 
 def unsorted(relays, filename, is_index):
-    template_env = env.get_template(filename)
-    template_render = template_env.render(relays=relays, is_index=is_index)
+    template = env.get_template(filename)
+    template_render = template.render(relays=relays, is_index=is_index)
     output = os.path.join(config.CONFIG['output_root'], filename)
     with open(output, 'w', encoding='utf8') as out:
         out.write(template_render)
 
+def effective_family(relays):
+    template = env.get_template('effective_family.html')
+    output_path = os.path.join(config.CONFIG['output_root'], 'family')
+    relay_list = relays.json['relays']
+    q_relays = [] # qualified relays w/ > 1 effective family member
+    for relay in relay_list:
+        if len(relay['effective_family']) > 1:
+            q_relays.append(relay)
+    for relay in q_relays:
+        fingerprint = relay['fingerprint']
+        members = []   # list of member relays (dict)
+        bandwidth = 0  # total bandwidth for family subset
+        for p_relay in q_relays:
+            if fingerprint in p_relay['effective_family']:
+                members.append(p_relay)
+                bandwidth += p_relay['observed_bandwidth']
+        write(output_path, members, bandwidth, template, fingerprint, 
+                category='family')
+
 def pages_by_key(relays, key):
     template = env.get_template(key + '.html')
     output_path = os.path.join(config.CONFIG['output_root'], key)
@@ -46,14 +66,18 @@ def pages_by_key(relays, key):
                 if p_relay.get(key) and p_relay[key] == relay[key]:
                     found_relays.append(p_relay)
                     bandwidth += p_relay['observed_bandwidth']
-            bandwidth = round(bandwidth / 1000000, 2) # convert to MB/s
-            dir_path = os.path.join(output_path, relay[key])
-            os.makedirs(dir_path)
-            rendered = template.render(relays=found_relays, bandwidth=bandwidth,
-                    deactivate=key, is_index=False, path_prefix='../../',
-                    special_countries=countries.the_prefixed)
-            with open(os.path.join(dir_path, 'index.html'), 'w', encoding='utf8') as html:
-                html.write(rendered)
+            write(output_path, found_relays, bandwidth, template, relay[key])
+
+def write(parent_dir, relays, bandwidth, template, key, category=None):
+    dir_path = os.path.join(parent_dir, key)
+    os.makedirs(dir_path)
+    f_bandwidth = round(bandwidth / 1000000, 2) # convert to MB/s
+    rendered = template.render(relays=relays, bandwidth=f_bandwidth,
+            focused=key, is_index=False, category=category, path_prefix='../../',
+            special_countries=countries.the_prefixed)
+    with open(os.path.join(dir_path, 'index.html'), 'w', encoding='utf8') as html:
+        html.write(rendered)
+
 
 relays = Relays()
 generate_html(relays)
diff --git a/tor-metrics/templates/base.html b/tor-metrics/templates/base.html
index 3e01be0..697418f 100644
--- a/tor-metrics/templates/base.html
+++ b/tor-metrics/templates/base.html
@@ -36,11 +36,14 @@
                         {% else %}
                         {% set obs_bandwidth = '%s %s'|format((relay['observed_bandwidth'] / 1000)|round(2, 'common'), 'KB/s') %}{% endif %}
 
-                        <td>{{ relay['nickname'] }}</td>
+                        {% if category != 'family' %}
+                        <td>{{ relay['nickname'] }} (<a href="{{ path_prefix }}family/{{ relay['effective_family'][0] }}/">{{ relay['effective_family']|length }}</a>)</td>
+                        {% else %}
+                        <td>{{ relay['nickname'] }}</td>{% endif %}
                         <td>{{ obs_bandwidth }}</td>
                         <td class="visible-md visible-lg">{{ relay['or_addresses'][0].split(':', 1)[0] }}</td>
 
-                        {% if relay['as'] %}{% if deactivate != 'as' %}
+                        {% if relay['as'] %}{% if focused != 'as' %}
                         <td><a href="{{ path_prefix }}as/{{ relay['as'] }}/">{{ relay['as'] }}</a></td>
                         {% else %}
                         <td>{{ relay['as'] }}</td>{% endif %}
@@ -52,14 +55,14 @@
                         {% else %}
                         <td>Unknown</td>{% endif %}
                         
-                        {% if relay['country'] %}{% if deactivate != 'country' %}
+                        {% if relay['country'] %}{% if focused != 'country' %}
                         <td><a href="{{ path_prefix }}country/{{ relay['country'] }}/"><img src="{{ path_prefix }}static/images/cc/{{ relay['country'] }}.png" title="{{ relay['country_name'] }}" alt="{{ relay['country_name'] }}"></a></td>
                         {% else %}
                         <td><img src="{{ path_prefix }}static/images/cc/{{ relay['country'] }}.png" title="{{ relay['country_name'] }}" alt="{{ relay['country_name'] }}"></td>{% endif %}
                         {% else %}
                         <td>X</td>{% endif %}
                         
-                        {% if deactivate != 'platform' %}
+                        {% if focused != 'platform' %}
                         <td><a href="{{ path_prefix }}platform/{{ relay['platform'] }}/">{{ relay['platform'] }}</a></td>
                         {% else %}
                         <td>{{ relay['platform'] }}</td>{% endif %}
diff --git a/tor-metrics/templates/effective_family.html b/tor-metrics/templates/effective_family.html
new file mode 100644
index 0000000..1acec41
--- /dev/null
+++ b/tor-metrics/templates/effective_family.html
@@ -0,0 +1,4 @@
+{% extends "base.html" %}
+{% block title %}Tor Relays :: Family {{ focused }}{% endblock %}
+{% block header %}<a href="../../">Home</a> :: Family {{ focused }}{% endblock %}
+{% block description %}Relays with effective family member {{ focused }} are responsible for ~{{ bandwidth }} MB/s of traffic.{% endblock %}