diff options
-rw-r--r-- | .gitignore | 4 | ||||
-rw-r--r-- | README | 54 | ||||
-rwxr-xr-x | allium/allium.py | 91 | ||||
-rw-r--r-- | allium/config.py | 13 | ||||
-rw-r--r-- | allium/countries.py | 26 | ||||
-rwxr-xr-x | allium/generate.py | 73 | ||||
-rw-r--r-- | allium/lib/relays.py (renamed from allium/relays.py) | 51 |
7 files changed, 160 insertions, 152 deletions
@@ -2,6 +2,6 @@ *.swo *.sh sandbox -allium/__pycache__ +__pycache__ +timestamp allium/www -allium/timestamp @@ -1,32 +1,40 @@ -allium: statically generated tor metrics and statistics - https://yui.cat/ +allium: generate static tor relay metrics and statistics - allium generates a set of HTML documents which represent the total set of tor - relays at the time of execution +usage: allium.py [-h] [--out] [--onionoo-url] - allium is heavily inspired by the official tor metrics project[0] and serves - as a javascript-free, statically-generated clean room implementation. the - primary goals of the project are to be fast (static), use few API queries - (one), and to present information in a condensed, readable format +optional arguments: + -h, --help show this help message and exit + --out directory to store rendered files (default "./www") + --onionoo-url onionoo HTTP URL (default + "https://onionoo.torproject.org/details") - INSTALL +ABOUT - $ pip install -r requirements.txt - $ cd allium - $ ./generate.py +allium generates a set of HTML documents which represent the total set of tor +relays at the time of execution - Files will be generated in the ./www directory by default, configurable by - editing config.py; the only non-standard dependency is Jinja2>=2.11.2 +allium is heavily inspired by the official tor metrics project[0] and serves +as a javascript-free, statically-generated clean room implementation. the +primary goals of the project are to be fast (static), use few API queries +(one), and to present information in a condensed, readable format - TODO +REQUIRES - - top exit/guard/relay families (see https://nusenu.github.io/OrNetStats/) - - interesting statistics (ASN exit concentration, IPv6-supporting relays) - - implement something similar to https://metrics.torproject.org/bubbles.html +* python3 +* Jinja2>=2.11.2 - note: this project includes country flags from GoSquared[1] and relay flags - from the Tor Project[2], the licenses of which are included in this project's - root directory +INSTALL - [0] https://metrics.torproject.org/ - [1] https://github.com/gosquared/flags - [2] https://www.torproject.org/ +$ pip install -r requirements.txt +$ cd allium +$ ./allium.py + +LICENSE + +this project includes country flags from GoSquared[1] and relay flags from the +Tor Project[2], the licenses of which are included in this project's root +directory; all code is published under UNLICENSE (public domain) + +[0] https://metrics.torproject.org/ +[1] https://github.com/gosquared/flags +[2] https://www.torproject.org/ diff --git a/allium/allium.py b/allium/allium.py new file mode 100755 index 0000000..9da4c00 --- /dev/null +++ b/allium/allium.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 + +''' +File: allium.py (executable) + +Generate complete set of relay HTML pages and copy static files to the +output_dir + +Default output directory: ./www +''' + +import argparse +import os +import pkg_resources +import sys +from shutil import copytree +from lib.relays import Relays + +jinja_version = pkg_resources.parse_version( + pkg_resources.get_distribution('jinja2').version) + +if jinja_version < pkg_resources.parse_version("2.11.2"): + sys.exit('Jinja2>=2.11.2 required') + +ABS_PATH = os.path.dirname(os.path.abspath(__file__)) + +if __name__ == '__main__': + desc = 'allium: generate static tor relay metrics and statistics' + parser = argparse.ArgumentParser(description=desc) + parser.add_argument('--out', dest='output_dir', action='store_true', + default="./www", + help='directory to store rendered files (default "./www")', + required=False) + parser.add_argument('--onionoo-url', dest='onionoo_url', action='store_true', + default="https://onionoo.torproject.org/details", + help='onionoo HTTP URL (default '\ + '"https://onionoo.torproject.org/details")', + required=False) + args = parser.parse_args() + + # object containing onionoo data and processing routines + RELAY_SET = Relays(args.output_dir, args.onionoo_url) + RELAY_SET.create_output_dir() + + # index and "all" HTML relay sets; index set limited to 500 relays + RELAY_SET.write_misc( + template = 'index.html', + path = 'index.html', + path_prefix = './', + is_index = True, + ) + RELAY_SET.write_misc( + template = 'all.html', + path = 'misc/all.html' + ) + + # miscellaneous page filename suffixes and sorted-by keys + misc_pages = { + 'by-bandwidth': '1.bandwidth', + 'by-exit-count': '1.exit_count,1.bandwidth', + 'by-middle-count': '1.middle_count,1.bandwidth', + 'by-first-seen': '1.first_seen,1.bandwidth' + } + + # miscellaneous-sorted (per misc_pages k/v) HTML pages + for k, v in misc_pages.items(): + RELAY_SET.write_misc( + template = 'misc-families.html', + path = 'misc/families-{}.html'.format(k), + sorted_by = v + ) + RELAY_SET.write_misc( + template = 'misc-networks.html', + path = 'misc/networks-{}.html'.format(k), + sorted_by = v + ) + + # onionoo keys used to generate pages by unique value; e.g. AS43350 + keys = ['as', 'contact', 'country', 'family', 'flag', 'platform', + 'first_seen'] + + for k in keys: + RELAY_SET.write_pages_by_key(k) + + # per-relay info pages + RELAY_SET.write_relay_info() + + # copy static directory and its contents if it doesn't exist + if not os.path.exists(os.path.join(args.output_dir, 'static')): + copytree(os.path.join(ABS_PATH, 'static'), + os.path.join(args.output_dir, 'static')) diff --git a/allium/config.py b/allium/config.py deleted file mode 100644 index 67a1934..0000000 --- a/allium/config.py +++ /dev/null @@ -1,13 +0,0 @@ -''' -File: config.py - -Configuration dict used in the generation of HTML documents - -:output_root: path to output directory (created if not exists) -:onionoo_url: url to onionoo details document -''' - -CONFIG = { - 'output_root': 'www', - 'onionoo_url': 'https://onionoo.torproject.org/details' -} diff --git a/allium/countries.py b/allium/countries.py deleted file mode 100644 index ba63358..0000000 --- a/allium/countries.py +++ /dev/null @@ -1,26 +0,0 @@ -''' -File: countries.py - -List of countries which require prefixing with "The" -''' - -THE_PREFIXED = [ - 'Dominican Republic', - 'Ivory Coast', - 'Marshall Islands', - 'Northern Marianas Islands', - 'Solomon Islands', - 'United Arab Emirates', - 'United Kingdom', - 'United States', - 'United States of America', - 'Vatican City', - 'Czech Republic', - 'Bahamas', - 'Gambia', - 'Netherlands', - 'Philippines', - 'Seychelles', - 'Sudan', - 'Ukraine' -] diff --git a/allium/generate.py b/allium/generate.py deleted file mode 100755 index 928b6e6..0000000 --- a/allium/generate.py +++ /dev/null @@ -1,73 +0,0 @@ -#!/usr/bin/env python3 - -''' -File: generate.py (executable) - -Generate complete set of relay HTML pages and copy static files to -config.CONFIG['output_root'] defined in config.py - -Default output directory: ./www -''' - -import os -import sys -from shutil import copytree -import config -from relays import Relays - -ABS_PATH = os.path.dirname(os.path.abspath(__file__)) - -if __name__ == '__main__': - # object containing onionoo data and processing routines - RELAY_SET = Relays() - - # index and "all" HTML relay sets; index set limited to 500 relays - RELAY_SET.create_output_dir() - RELAY_SET.write_misc( - template = 'index.html', - path = 'index.html', - path_prefix = './', - is_index = True, - ) - RELAY_SET.write_misc( - template = 'all.html', - path = 'misc/all.html' - ) - - # miscellaneous page filename suffixes and sorted-by keys - misc_pages = { - 'by-bandwidth': '1.bandwidth', - 'by-exit-count': '1.exit_count,1.bandwidth', - 'by-middle-count': '1.middle_count,1.bandwidth', - 'by-first-seen': '1.first_seen,1.bandwidth' - } - - # write miscellaneous-sorted (per misc_pages) HTML pages - for k, v in misc_pages.items(): - RELAY_SET.write_misc( - template = 'misc-families.html', - path = 'misc/families-{}.html'.format(k), - sorted_by = v - ) - RELAY_SET.write_misc( - template = 'misc-networks.html', - path = 'misc/networks-{}.html'.format(k), - sorted_by = v - ) - - # onionoo keys to generate pages by unique value - keys = ['as', 'contact', 'country', 'family', 'flag', 'platform', - 'first_seen'] - - for k in keys: - RELAY_SET.write_pages_by_key(k) - - # per-relay info pages - RELAY_SET.write_relay_info() - - STATIC_SRC_PATH = os.path.join(ABS_PATH, 'static') - STATIC_DEST_PATH = os.path.join(config.CONFIG['output_root'], 'static') - - # copy static directory and its contents if it doesn't exist - if not os.path.exists(STATIC_DEST_PATH): - copytree(STATIC_SRC_PATH, STATIC_DEST_PATH) diff --git a/allium/relays.py b/allium/lib/relays.py index 5c50f66..6c28d88 100644 --- a/allium/relays.py +++ b/allium/lib/relays.py @@ -12,21 +12,20 @@ import re import time import urllib.request from shutil import rmtree -import config -import countries from jinja2 import Environment, FileSystemLoader ABS_PATH = os.path.dirname(os.path.abspath(__file__)) -ENV = Environment(loader=FileSystemLoader(os.path.join(ABS_PATH, 'templates')), +ENV = Environment(loader=FileSystemLoader(os.path.join(ABS_PATH, '../templates')), trim_blocks=True, lstrip_blocks=True) -class Relays: +class Relays(): ''' Relay class consisting of processing routines and onionoo data ''' - def __init__(self): - self.url = config.CONFIG['onionoo_url'] - self.ts_file = os.path.join(ABS_PATH, "timestamp") + def __init__(self, output_dir, onionoo_url): + self.output_dir = output_dir + self.onionoo_url = onionoo_url + self.ts_file = os.path.join(os.path.dirname(ABS_PATH), "timestamp") self.json = self._fetch_onionoo_details() self.timestamp = self._write_timestamp() @@ -45,9 +44,9 @@ class Relays: with open(self.ts_file, 'r') as ts_file: prev_timestamp = ts_file.read() headers = {"If-Modified-Since": prev_timestamp} - conn = urllib.request.Request(self.url, headers=headers) + conn = urllib.request.Request(self.onionoo_url, headers=headers) else: - conn = urllib.request.Request(self.url) + conn = urllib.request.Request(self.onionoo_url) api_response = urllib.request.urlopen(conn).read() @@ -186,9 +185,9 @@ class Relays: def create_output_dir(self): ''' - Ensure config:output_root exists (required for write functions) + Ensure self.output_dir exists (required for write functions) ''' - os.makedirs(config.CONFIG['output_root'],exist_ok=True) + os.makedirs(self.output_dir,exist_ok=True) def write_misc(self, template, path, path_prefix='../', sorted_by=None, reverse=True, is_index=False): @@ -212,7 +211,7 @@ class Relays: is_index = is_index, path_prefix = path_prefix ) - output = os.path.join(config.CONFIG['output_root'], path) + output = os.path.join(self.output_dir, path) os.makedirs(os.path.dirname(output), exist_ok=True) with open(output, 'w', encoding='utf8') as html: @@ -226,7 +225,29 @@ class Relays: k: onionoo key to sort by (as, country, platform...) ''' template = ENV.get_template(k + '.html') - output_path = os.path.join(config.CONFIG['output_root'], k) + output_path = os.path.join(self.output_dir, k) + + # the "royal the" must be gramatically recognized + the_prefixed = [ + "Dominican Republic", + "Ivory Coast", + "Marshall Islands", + "Northern Marianas Islands", + "Solomon Islands", + "United Arab Emirates", + "United Kingdom", + "United States", + "United States of America", + "Vatican City", + "Czech Republic", + "Bahamas", + "Gambia", + "Netherlands", + "Philippines", + "Seychelles", + "Sudan", + "Ukraine" + ] if os.path.exists(output_path): rmtree(output_path) @@ -253,7 +274,7 @@ class Relays: path_prefix = '../../', key = k, value = v, - sp_countries = countries.THE_PREFIXED + sp_countries = the_prefixed ) with open(os.path.join(dir_path, 'index.html'), 'w', @@ -266,7 +287,7 @@ class Relays: ''' relay_list = self.json['relays'] template = ENV.get_template('relay-info.html') - output_path = os.path.join(config.CONFIG['output_root'], 'relay') + output_path = os.path.join(self.output_dir, 'relay') if os.path.exists(output_path): rmtree(output_path) |