aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJordan <me@jordan.im>2021-01-11 22:20:48 -0700
committerJordan <me@jordan.im>2021-01-11 22:20:48 -0700
commitfa85205fd0499f361b243943777590b348df290b (patch)
tree7c9d6e2f3b4d5115e7a5eae45257749cf7d2c8b5
parent7d530a90fe15fe912ffa84424d9d24e20144fb38 (diff)
downloadallium-fa85205fd0499f361b243943777590b348df290b.tar.gz
allium-fa85205fd0499f361b243943777590b348df290b.zip
replace config.py with cli args, cleanup
-rw-r--r--.gitignore4
-rw-r--r--README54
-rwxr-xr-xallium/allium.py91
-rw-r--r--allium/config.py13
-rw-r--r--allium/countries.py26
-rwxr-xr-xallium/generate.py73
-rw-r--r--allium/lib/relays.py (renamed from allium/relays.py)51
7 files changed, 160 insertions, 152 deletions
diff --git a/.gitignore b/.gitignore
index a7a359c..3437854 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,6 +2,6 @@
*.swo
*.sh
sandbox
-allium/__pycache__
+__pycache__
+timestamp
allium/www
-allium/timestamp
diff --git a/README b/README
index b4cd081..7664e47 100644
--- a/README
+++ b/README
@@ -1,32 +1,40 @@
-allium: statically generated tor metrics and statistics - https://yui.cat/
+allium: generate static tor relay metrics and statistics
- allium generates a set of HTML documents which represent the total set of tor
- relays at the time of execution
+usage: allium.py [-h] [--out] [--onionoo-url]
- allium is heavily inspired by the official tor metrics project[0] and serves
- as a javascript-free, statically-generated clean room implementation. the
- primary goals of the project are to be fast (static), use few API queries
- (one), and to present information in a condensed, readable format
+optional arguments:
+ -h, --help show this help message and exit
+ --out directory to store rendered files (default "./www")
+ --onionoo-url onionoo HTTP URL (default
+ "https://onionoo.torproject.org/details")
- INSTALL
+ABOUT
- $ pip install -r requirements.txt
- $ cd allium
- $ ./generate.py
+allium generates a set of HTML documents which represent the total set of tor
+relays at the time of execution
- Files will be generated in the ./www directory by default, configurable by
- editing config.py; the only non-standard dependency is Jinja2>=2.11.2
+allium is heavily inspired by the official tor metrics project[0] and serves
+as a javascript-free, statically-generated clean room implementation. the
+primary goals of the project are to be fast (static), use few API queries
+(one), and to present information in a condensed, readable format
- TODO
+REQUIRES
- - top exit/guard/relay families (see https://nusenu.github.io/OrNetStats/)
- - interesting statistics (ASN exit concentration, IPv6-supporting relays)
- - implement something similar to https://metrics.torproject.org/bubbles.html
+* python3
+* Jinja2>=2.11.2
- note: this project includes country flags from GoSquared[1] and relay flags
- from the Tor Project[2], the licenses of which are included in this project's
- root directory
+INSTALL
- [0] https://metrics.torproject.org/
- [1] https://github.com/gosquared/flags
- [2] https://www.torproject.org/
+$ pip install -r requirements.txt
+$ cd allium
+$ ./allium.py
+
+LICENSE
+
+this project includes country flags from GoSquared[1] and relay flags from the
+Tor Project[2], the licenses of which are included in this project's root
+directory; all code is published under UNLICENSE (public domain)
+
+[0] https://metrics.torproject.org/
+[1] https://github.com/gosquared/flags
+[2] https://www.torproject.org/
diff --git a/allium/allium.py b/allium/allium.py
new file mode 100755
index 0000000..9da4c00
--- /dev/null
+++ b/allium/allium.py
@@ -0,0 +1,91 @@
+#!/usr/bin/env python3
+
+'''
+File: allium.py (executable)
+
+Generate complete set of relay HTML pages and copy static files to the
+output_dir
+
+Default output directory: ./www
+'''
+
+import argparse
+import os
+import pkg_resources
+import sys
+from shutil import copytree
+from lib.relays import Relays
+
+jinja_version = pkg_resources.parse_version(
+ pkg_resources.get_distribution('jinja2').version)
+
+if jinja_version < pkg_resources.parse_version("2.11.2"):
+ sys.exit('Jinja2>=2.11.2 required')
+
+ABS_PATH = os.path.dirname(os.path.abspath(__file__))
+
+if __name__ == '__main__':
+ desc = 'allium: generate static tor relay metrics and statistics'
+ parser = argparse.ArgumentParser(description=desc)
+ parser.add_argument('--out', dest='output_dir', action='store_true',
+ default="./www",
+ help='directory to store rendered files (default "./www")',
+ required=False)
+ parser.add_argument('--onionoo-url', dest='onionoo_url', action='store_true',
+ default="https://onionoo.torproject.org/details",
+ help='onionoo HTTP URL (default '\
+ '"https://onionoo.torproject.org/details")',
+ required=False)
+ args = parser.parse_args()
+
+ # object containing onionoo data and processing routines
+ RELAY_SET = Relays(args.output_dir, args.onionoo_url)
+ RELAY_SET.create_output_dir()
+
+ # index and "all" HTML relay sets; index set limited to 500 relays
+ RELAY_SET.write_misc(
+ template = 'index.html',
+ path = 'index.html',
+ path_prefix = './',
+ is_index = True,
+ )
+ RELAY_SET.write_misc(
+ template = 'all.html',
+ path = 'misc/all.html'
+ )
+
+ # miscellaneous page filename suffixes and sorted-by keys
+ misc_pages = {
+ 'by-bandwidth': '1.bandwidth',
+ 'by-exit-count': '1.exit_count,1.bandwidth',
+ 'by-middle-count': '1.middle_count,1.bandwidth',
+ 'by-first-seen': '1.first_seen,1.bandwidth'
+ }
+
+ # miscellaneous-sorted (per misc_pages k/v) HTML pages
+ for k, v in misc_pages.items():
+ RELAY_SET.write_misc(
+ template = 'misc-families.html',
+ path = 'misc/families-{}.html'.format(k),
+ sorted_by = v
+ )
+ RELAY_SET.write_misc(
+ template = 'misc-networks.html',
+ path = 'misc/networks-{}.html'.format(k),
+ sorted_by = v
+ )
+
+ # onionoo keys used to generate pages by unique value; e.g. AS43350
+ keys = ['as', 'contact', 'country', 'family', 'flag', 'platform',
+ 'first_seen']
+
+ for k in keys:
+ RELAY_SET.write_pages_by_key(k)
+
+ # per-relay info pages
+ RELAY_SET.write_relay_info()
+
+ # copy static directory and its contents if it doesn't exist
+ if not os.path.exists(os.path.join(args.output_dir, 'static')):
+ copytree(os.path.join(ABS_PATH, 'static'),
+ os.path.join(args.output_dir, 'static'))
diff --git a/allium/config.py b/allium/config.py
deleted file mode 100644
index 67a1934..0000000
--- a/allium/config.py
+++ /dev/null
@@ -1,13 +0,0 @@
-'''
-File: config.py
-
-Configuration dict used in the generation of HTML documents
-
-:output_root: path to output directory (created if not exists)
-:onionoo_url: url to onionoo details document
-'''
-
-CONFIG = {
- 'output_root': 'www',
- 'onionoo_url': 'https://onionoo.torproject.org/details'
-}
diff --git a/allium/countries.py b/allium/countries.py
deleted file mode 100644
index ba63358..0000000
--- a/allium/countries.py
+++ /dev/null
@@ -1,26 +0,0 @@
-'''
-File: countries.py
-
-List of countries which require prefixing with "The"
-'''
-
-THE_PREFIXED = [
- 'Dominican Republic',
- 'Ivory Coast',
- 'Marshall Islands',
- 'Northern Marianas Islands',
- 'Solomon Islands',
- 'United Arab Emirates',
- 'United Kingdom',
- 'United States',
- 'United States of America',
- 'Vatican City',
- 'Czech Republic',
- 'Bahamas',
- 'Gambia',
- 'Netherlands',
- 'Philippines',
- 'Seychelles',
- 'Sudan',
- 'Ukraine'
-]
diff --git a/allium/generate.py b/allium/generate.py
deleted file mode 100755
index 928b6e6..0000000
--- a/allium/generate.py
+++ /dev/null
@@ -1,73 +0,0 @@
-#!/usr/bin/env python3
-
-'''
-File: generate.py (executable)
-
-Generate complete set of relay HTML pages and copy static files to
-config.CONFIG['output_root'] defined in config.py
-
-Default output directory: ./www
-'''
-
-import os
-import sys
-from shutil import copytree
-import config
-from relays import Relays
-
-ABS_PATH = os.path.dirname(os.path.abspath(__file__))
-
-if __name__ == '__main__':
- # object containing onionoo data and processing routines
- RELAY_SET = Relays()
-
- # index and "all" HTML relay sets; index set limited to 500 relays
- RELAY_SET.create_output_dir()
- RELAY_SET.write_misc(
- template = 'index.html',
- path = 'index.html',
- path_prefix = './',
- is_index = True,
- )
- RELAY_SET.write_misc(
- template = 'all.html',
- path = 'misc/all.html'
- )
-
- # miscellaneous page filename suffixes and sorted-by keys
- misc_pages = {
- 'by-bandwidth': '1.bandwidth',
- 'by-exit-count': '1.exit_count,1.bandwidth',
- 'by-middle-count': '1.middle_count,1.bandwidth',
- 'by-first-seen': '1.first_seen,1.bandwidth'
- }
-
- # write miscellaneous-sorted (per misc_pages) HTML pages
- for k, v in misc_pages.items():
- RELAY_SET.write_misc(
- template = 'misc-families.html',
- path = 'misc/families-{}.html'.format(k),
- sorted_by = v
- )
- RELAY_SET.write_misc(
- template = 'misc-networks.html',
- path = 'misc/networks-{}.html'.format(k),
- sorted_by = v
- )
-
- # onionoo keys to generate pages by unique value
- keys = ['as', 'contact', 'country', 'family', 'flag', 'platform',
- 'first_seen']
-
- for k in keys:
- RELAY_SET.write_pages_by_key(k)
-
- # per-relay info pages
- RELAY_SET.write_relay_info()
-
- STATIC_SRC_PATH = os.path.join(ABS_PATH, 'static')
- STATIC_DEST_PATH = os.path.join(config.CONFIG['output_root'], 'static')
-
- # copy static directory and its contents if it doesn't exist
- if not os.path.exists(STATIC_DEST_PATH):
- copytree(STATIC_SRC_PATH, STATIC_DEST_PATH)
diff --git a/allium/relays.py b/allium/lib/relays.py
index 5c50f66..6c28d88 100644
--- a/allium/relays.py
+++ b/allium/lib/relays.py
@@ -12,21 +12,20 @@ import re
import time
import urllib.request
from shutil import rmtree
-import config
-import countries
from jinja2 import Environment, FileSystemLoader
ABS_PATH = os.path.dirname(os.path.abspath(__file__))
-ENV = Environment(loader=FileSystemLoader(os.path.join(ABS_PATH, 'templates')),
+ENV = Environment(loader=FileSystemLoader(os.path.join(ABS_PATH, '../templates')),
trim_blocks=True, lstrip_blocks=True)
-class Relays:
+class Relays():
'''
Relay class consisting of processing routines and onionoo data
'''
- def __init__(self):
- self.url = config.CONFIG['onionoo_url']
- self.ts_file = os.path.join(ABS_PATH, "timestamp")
+ def __init__(self, output_dir, onionoo_url):
+ self.output_dir = output_dir
+ self.onionoo_url = onionoo_url
+ self.ts_file = os.path.join(os.path.dirname(ABS_PATH), "timestamp")
self.json = self._fetch_onionoo_details()
self.timestamp = self._write_timestamp()
@@ -45,9 +44,9 @@ class Relays:
with open(self.ts_file, 'r') as ts_file:
prev_timestamp = ts_file.read()
headers = {"If-Modified-Since": prev_timestamp}
- conn = urllib.request.Request(self.url, headers=headers)
+ conn = urllib.request.Request(self.onionoo_url, headers=headers)
else:
- conn = urllib.request.Request(self.url)
+ conn = urllib.request.Request(self.onionoo_url)
api_response = urllib.request.urlopen(conn).read()
@@ -186,9 +185,9 @@ class Relays:
def create_output_dir(self):
'''
- Ensure config:output_root exists (required for write functions)
+ Ensure self.output_dir exists (required for write functions)
'''
- os.makedirs(config.CONFIG['output_root'],exist_ok=True)
+ os.makedirs(self.output_dir,exist_ok=True)
def write_misc(self, template, path, path_prefix='../', sorted_by=None,
reverse=True, is_index=False):
@@ -212,7 +211,7 @@ class Relays:
is_index = is_index,
path_prefix = path_prefix
)
- output = os.path.join(config.CONFIG['output_root'], path)
+ output = os.path.join(self.output_dir, path)
os.makedirs(os.path.dirname(output), exist_ok=True)
with open(output, 'w', encoding='utf8') as html:
@@ -226,7 +225,29 @@ class Relays:
k: onionoo key to sort by (as, country, platform...)
'''
template = ENV.get_template(k + '.html')
- output_path = os.path.join(config.CONFIG['output_root'], k)
+ output_path = os.path.join(self.output_dir, k)
+
+ # the "royal the" must be gramatically recognized
+ the_prefixed = [
+ "Dominican Republic",
+ "Ivory Coast",
+ "Marshall Islands",
+ "Northern Marianas Islands",
+ "Solomon Islands",
+ "United Arab Emirates",
+ "United Kingdom",
+ "United States",
+ "United States of America",
+ "Vatican City",
+ "Czech Republic",
+ "Bahamas",
+ "Gambia",
+ "Netherlands",
+ "Philippines",
+ "Seychelles",
+ "Sudan",
+ "Ukraine"
+ ]
if os.path.exists(output_path):
rmtree(output_path)
@@ -253,7 +274,7 @@ class Relays:
path_prefix = '../../',
key = k,
value = v,
- sp_countries = countries.THE_PREFIXED
+ sp_countries = the_prefixed
)
with open(os.path.join(dir_path, 'index.html'), 'w',
@@ -266,7 +287,7 @@ class Relays:
'''
relay_list = self.json['relays']
template = ENV.get_template('relay-info.html')
- output_path = os.path.join(config.CONFIG['output_root'], 'relay')
+ output_path = os.path.join(self.output_dir, 'relay')
if os.path.exists(output_path):
rmtree(output_path)