aboutsummaryrefslogtreecommitdiff
path: root/scripts
diff options
context:
space:
mode:
authorNick Mathewson <nickm@torproject.org>2016-04-26 19:27:39 -0400
committerNick Mathewson <nickm@torproject.org>2016-04-26 19:27:39 -0400
commitfb9c9e04f002fbbd9745911a17481b49ed66c9f4 (patch)
tree2aaed320e7d13776bda66fc43b99635c9b0e607d /scripts
parentf197d189a6c310ee6725e3648ceabc1322dd24fc (diff)
parentde08f2d96bd1815a8da5c359a4f52310dc1905d7 (diff)
downloadtor-fb9c9e04f002fbbd9745911a17481b49ed66c9f4.tar.gz
tor-fb9c9e04f002fbbd9745911a17481b49ed66c9f4.zip
Merge branch 'maint-0.2.8'
Diffstat (limited to 'scripts')
-rw-r--r--scripts/maint/fallback.blacklist156
-rw-r--r--scripts/maint/fallback.whitelist678
-rwxr-xr-xscripts/maint/updateFallbackDirs.py1354
3 files changed, 1823 insertions, 365 deletions
diff --git a/scripts/maint/fallback.blacklist b/scripts/maint/fallback.blacklist
index 3c0f8fc5c6..cc0c55d4f5 100644
--- a/scripts/maint/fallback.blacklist
+++ b/scripts/maint/fallback.blacklist
@@ -39,3 +39,159 @@
# https://lists.torproject.org/pipermail/tor-relays/2016-January/008555.html
62.210.207.124:9030 orport=9001 id=58938B1A5C4029B4415D38A4F36B7724273F4755 ipv6=[2001:bc8:31eb:100::1]:9001
62.210.207.124:9130 orport=9101 id=338D0AB6DBAB7B529B9C91B2FD770658000693C4 ipv6=[2001:bc8:31eb:100::1]:9101
+
+# these fallback candidates fail the consensus download test in a way that
+# causes stem to hang (and not respond to ^C, at least on OS X)
+# (Is something sending weird responses to DirPort traffic?)
+#217.23.14.190:1194
+#151.80.164.147:80
+#148.251.255.92:80
+#78.142.19.59:80
+
+# Email sent directly to teor, verified using relay contact info
+216.17.99.183:80 orport=443 id=D52CD431CEF28E01B11F545A84347EE45524BCA7
+216.17.99.183:8080 orport=9001 id=EE21F83AB6F76E3B3FFCBA5C2496F789CB84E7C6
+65.19.167.130:80 orport=443 id=890E2EA65455FBF0FAAB4159FAC4412BDCB24295
+65.19.167.131:80 orport=443 id=0DA9BD201766EDB19F57F49F1A013A8A5432C008
+65.19.167.132:80 orport=443 id=12B80ABF019354A9D25EE8BE85EB3C0AD8F7DFC1
+65.19.167.133:80 orport=443 id=C170AE5A886C5A09D6D1CF5CF284653632EEF25D
+
+# Email sent directly to teor, verified using relay contact info
+195.154.165.227:80 orport=443 id=C08DE49658E5B3CFC6F2A952B453C4B608C9A16A
+62.210.69.34:80 orport=443 id=71AB4726D830FAE776D74AEF790CF04D8E0151B4
+163.172.13.124:80 orport=443 id=B771AA877687F88E6F1CA5354756DF6C8A7B6B24
+
+# Email sent directly to teor, verified using relay contact info
+213.136.83.225:80 orport=443 id=B411027C926A9BFFCF7DA91E3CAF1856A321EFFD
+195.154.126.78:80 orport=443 id=F6556156E2B3837248E03FDB770441CF64DBBFBE
+
+# Email sent directly to teor, verified using relay contact info
+178.63.198.113:80 orport=443 id=872B18761953254914F77C71846E8A2623C52591
+
+# Email sent directly to teor, verified using relay contact info
+63.141.226.34:80 orport=9001 id=5EF131C0C82270F40B756987FDB5D54D9C966238
+185.75.56.103:80 orport=9001 id=3763CE5C3F574670D4296573744F821C0FFFB98E
+
+# Email sent directly to teor, verified using relay contact info
+108.166.168.158:80 orport=443 id=CDAB3AE06A8C9C6BF817B3B0F1877A4B91465699
+81.7.14.227:9030 orport=9001 id=BCA197C43A44B7B9D14509637F96A45B13C233D0
+
+# Email sent directly to teor, verified using relay contact info
+84.245.32.195:9030 orport=9001 id=4CD4DFFEF3971C902A22100D911CAC639BE2EF5C
+
+# Email sent directly to teor, verified using relay contact info
+185.21.217.10:9030 orport=9001 id=41537E1D3DD3CAE86F5A3F0882F1C647FE8FC0A0
+
+# Email sent directly to teor, verified using relay contact info
+185.21.216.140:9030 orport=9001 id=921DA852C95141F8964B359F774B35502E489869
+
+# Email sent directly to teor, verified using relay contact info
+62.210.82.44:143 orport=21 id=1C90D3AEADFF3BCD079810632C8B85637924A58E ipv6=[2001:bc8:3d7c::]:21
+
+# Email sent directly to teor, verified using relay contact info
+46.101.220.161:80 orport=443 id=7DDFE5B2C306B19A79832FBE581EAA245BAE90C6 ipv6=[2a03:b0c0:3:d0::8b:3001]:443
+
+# Email sent directly to teor, verified using relay contact info
+195.154.107.23:80 orport=443 id=A1F89F26E82209169E4037B035AE7B6C94A49AEB ipv6=[2001:bc8:3829:300::1]:443
+195.154.92.70:80 orport=443 id=E7FF4ECEEFCFE3A40A6D3594898A4A3DE018BBF5 ipv6=[2001:bc8:3829:500::1]:443
+195.154.113.200:80 orport=443 id=D1A4763FA0BD71978901B1951FEE1DC29777F95A ipv6=[2001:bc8:3829:600::1]:443
+195.154.92.155:110 orport=993 id=4477D3466FE136B7FE6F7FF8EBD0D6E2FFE3288B ipv6=[2001:bc8:3829:100::1]:993
+195.154.117.182:110 orport=993 id=B1A0F1143789466AADD5FAE5948C8138548EECEC ipv6=[2001:bc8:3829:400::1]:993
+195.154.97.163:80 orport=443 id=8A2994A63B20813B7724817A8FB8C444D10BA2E2
+
+# Email sent directly to teor, verified using relay contact info
+5.135.154.206:9030 orport=9001 id=7D67B342DC1158F4CFFEE8BC530A2448848026E3
+
+# Email sent directly to teor, verified using relay contact info
+85.24.215.117:9030 orport=9001 id=5989521A85C94EE101E88B8DB2E68321673F9405 ipv6=[2001:9b0:20:2106:21a:4aff:fea5:ad05]:9001
+
+# Email sent directly to teor, verified using relay contact info
+62.210.137.230:8888 orport=8843 id=CD6B850159CFF4C068A8D0F1BA5296AE4EDCAB39 ipv6=[2001:bc8:31d3:100::1]:3443
+62.210.137.230:8080 orport=8443 id=F596E1B1EF98E1DDBBDC934DB722AF54069868F6 ipv6=[2001:bc8:31d3:100::1]:8443
+
+# Email sent directly to teor, verified using relay contact info
+195.154.99.80:80 orport=443 id=6E7CB6E783C1B67B79D0EBBE7D48BC09BD441201
+195.154.127.60:80 orport=443 id=D74ABE34845190E242EC74BA28B8C89B0A480D4B
+
+# Email sent directly to teor, verified using relay contact info
+212.51.143.20:80 orport=443 id=62DA0256BBC28992D41CBAFB549FFD7C9B846A99
+
+# Email sent directly to teor, verified using relay contact info
+195.154.90.122:80 orport=443 id=3A0D88024A30152E6F6372CFDF8F9B725F984362
+
+# Email sent directly to teor, verified using relay contact info
+188.166.118.215:9030 orport=443 id=FB5FF60F5EBA010F8A45AC6ED31A4393718A2C31 ipv6=[2a03:b0c0:2:d0::72:9001]:443
+
+# Email sent directly to teor, verified using relay contact info
+185.87.185.245:40001 orport=40000 id=2A499AEEA95FB10F07544383D562368E49BE32CA
+
+# Email sent directly to teor, verified using relay contact info
+82.161.109.71:9030 orport=9001 id=BD9CE352648B940E788A8E45393C5400CC3E87E7
+
+# Email sent directly to teor, verified using relay contact info
+212.83.40.239:9030 orport=9001 id=6DC5616BD3FC463329DCE87DD7AAAEA112C264B5
+
+# Email sent directly to teor, verified using relay contact info
+178.32.53.53:80 orport=443 id=10582C360E972EE76B0DB1C246F4E892A6BF5465
+
+# Email sent directly to teor, verified using relay contact info
+85.114.135.20:9030 orport=9001 id=ED8A9291A3139E34BBD35037B082081EC6C26C80 ipv6=[2001:4ba0:fff5:2d::8]:9001
+148.251.128.156:9030 orport=9001 id=E382042E06A0A68AFC533E5AD5FB6867A12DF9FF ipv6=[2a01:4f8:210:238a::8]:9001
+62.210.115.147:9030 orport=9001 id=7F1D94E2C36F8CC595C2AB00022A5AE38171D50B ipv6=[2001:bc8:3182:101::8]:9001
+212.47.250.24:9030 orport=9001 id=33DA0CAB7C27812EFF2E22C9705630A54D101FEB
+
+# Email sent directly to teor, verified using relay contact info
+74.208.220.222:60000 orport=59999 id=4AA22235F0E9B3795A33930343CBB3EDAC60C5B0
+
+# Email sent directly to teor, verified using relay contact info
+89.163.140.168:9030 orport=9001 id=839C1212DB15723263BE96C83DA7E1B24FA395E8
+
+# Email sent directly to teor, verified using relay contact info
+212.47.246.211:9030 orport=9001 id=AA34219475E41282095DD3C088009EE562AF14E5
+
+# Email sent directly to teor, verified using relay contact info
+85.195.235.148:9030 orport=9001 id=103336165A0D2EFCAD3605339843A0A7710B8B92
+85.195.235.148:19030 orport=19001 id=713235638AB6C64715EAFD1B4772685E38AFD52A
+
+# Email sent directly to teor, verified using relay contact info
+163.172.7.30:9030 orport=9001 id=E2EACD4752B2583202F374A34CACC844A3AECAC4
+
+# Email sent directly to teor, verified using relay contact info
+178.62.90.111:22 orport=25 id=3254D1DC1F1531D9C07C535E4991F38EE99B99E1
+
+# Email sent directly to teor, verified using relay contact info
+213.200.106.131:9030 orport=4443 id=B07CE79FD215129C381F6645B16E76DCA0845CAB
+
+# Email sent directly to teor, verified using relay contact info
+198.51.75.165:80 orport=9001 id=DABCB84A524A22FDDD3AFCB090E3090CC12D9770
+
+# Email sent directly to teor, verified using relay contact info
+204.194.29.4:80 orport=9001 id=78C7C299DB4C4BD119A22B87B57D5AF5F3741A79
+
+# Email sent directly to teor, verified using relay contact info
+104.207.132.109:9030 orport=9001 id=12D5737383C23E756A7AA1A90BB24413BA428DA7 ipv6=[2001:19f0:300:2261::1]:9001
+
+# Email sent directly to teor, verified using relay contact info
+46.252.25.249:9030 orport=443 id=80DCBB6EF4E86A7CD4FBCBDEE64979645509A610
+
+# Email sent directly to teor, verified using relay contact info
+176.10.99.200:8080 orport=443 id=2B44FD1742D26E4F28D4CACF1F0CF8A686270E45
+176.10.99.200:8000 orport=22 id=EB79F07792A065D3C534063773E83268E069F5EB
+176.10.99.201:667 orport=666 id=3EAAAB35932610411E24FA4317603CB5780B80BC
+176.10.99.201:990 orport=989 id=7C3A4CFF09C1981D41173CDE2A2ADD4A5CA109FD
+176.10.99.202:992 orport=991 id=615EBC4B48F03858FA50A3E23E5AF569D0D2308A
+176.10.99.202:994 orport=993 id=E34E25D958D46DDE5092385B14117C9B301DC0E9
+176.10.99.203:1194 orport=995 id=AD368442E9FF33C08C7407DF2DA7DB958F406CE2
+176.10.99.203:43 orport=53 id=79CF377F0ACEC5F0002D85335E4192B34202A269
+176.10.99.204:1755 orport=1723 id=69DF3CDA1CDA460C17ECAD9D6F0C117A42384FA0
+176.10.99.204:1293 orport=4321 id=3F061400B6FB1F55E7F19BB3C713884D677E55B7
+176.10.99.205:426 orport=425 id=C30B284784BF11D0D58C6A250240EE58D2084AD0
+176.10.99.205:109 orport=110 id=12D17D9F9E30FA901DE68806950A0EA278716CED
+176.10.99.206:24 orport=23 id=2C804AAB0C02F971A4386B3A1F2AC00F9E080679
+176.10.99.206:20 orport=21 id=237588726AB6BEA37FF23CA00F5BD178586CA68E
+176.10.99.207:3390 orport=3389 id=A838D5B8890B10172429ECE92EB5677DF93DC4DD
+176.10.99.207:1415 orport=1414 id=377E5E817A84FAE0F4DC3427805DB2E8A6CBBFC0
+176.10.99.208:390 orport=389 id=7C288587BA0D99CC6B8537CDC2C4639FA827B907
+176.10.99.208:3307 orport=3306 id=1F0D2A44C56F42816DED2022EFD631878C29905B
+176.10.99.209:1434 orport=1433 id=BDA7A91FF3806DE5109FDAE74CFEFB3BABB9E10F
+176.10.99.209:220 orport=219 id=B8C2030001D832066A648269CFBA94171951D34B
diff --git a/scripts/maint/fallback.whitelist b/scripts/maint/fallback.whitelist
index 6edb368345..06456c6206 100644
--- a/scripts/maint/fallback.whitelist
+++ b/scripts/maint/fallback.whitelist
@@ -20,7 +20,8 @@
84.219.173.60:9030 orport=443 id=855BC2DABE24C861CD887DB9B2E950424B49FC34
# https://lists.torproject.org/pipermail/tor-relays/2015-December/008365.html
-81.7.17.171:80 orport=443 id=00C4B4731658D3B4987132A3F77100CFCB190D97 ipv6=[2a02:180:1:1::517:11ab]:443
+# Email sent directly to teor with fingerprint update
+81.7.17.171:80 orport=443 id=CFECDDCA990E3EF7B7EC958B22441386B6B8D820 ipv6=[2a02:180:1:1::517:11ab]:443
# https://lists.torproject.org/pipermail/tor-relays/2015-December/008366.html
5.39.88.19:9030 orport=9001 id=7CB8C31432A796731EA7B6BF4025548DFEB25E0C ipv6=[2001:41d0:8:9a13::1]:9050
@@ -54,7 +55,8 @@
144.76.14.145:110 orport=143 id=14419131033443AE6E21DA82B0D307F7CAE42BDB ipv6=[2a01:4f8:190:9490::dead]:443
# https://lists.torproject.org/pipermail/tor-relays/2015-December/008379.html
-85.25.138.93:9030 orport=4029 id=6DE61A6F72C1E5418A66BFED80DFB63E4C77668F
+# changed to 91.121.84.137? Awaiting operator confirmation
+#85.25.138.93:9030 orport=4029 id=6DE61A6F72C1E5418A66BFED80DFB63E4C77668F
# https://lists.torproject.org/pipermail/tor-relays/2015-December/008380.html
5.175.233.86:80 orport=443 id=5525D0429BFE5DC4F1B0E9DE47A4CFA169661E33
@@ -95,6 +97,674 @@
# https://lists.torproject.org/pipermail/tor-relays/2016-January/008542.html
178.62.199.226:80 orport=443 id=CBEFF7BA4A4062045133C053F2D70524D8BBE5BE ipv6=[2a03:b0c0:2:d0::b7:5001]:443
-# Email sent directly to teor, verified using relay contact info
+# Emails sent directly to teor, verified using relay contact info
217.12.199.208:80 orport=443 id=DF3AED4322B1824BF5539AE54B2D1B38E080FF05
-217.12.210.214:80 orport=443 id=943C0C6841C1E914B9FCA796C6846620A5AF9BC7
+
+# Email sent directly to teor, verified using relay contact info
+94.23.204.175:9030 orport=9001 id=5665A3904C89E22E971305EE8C1997BCA4123C69
+
+# https://twitter.com/binarytenshi/status/717952514327453697
+94.126.23.174:9030 orport=9001 id=6FC6F08270D565BE89B7C819DD8E2D487397C073
+
+# Email sent directly to teor, verified using relay contact info
+171.25.193.78:80 orport=443 id=A478E421F83194C114F41E94F95999672AED51FE ipv6=[2001:67c:289c:3::78]:443
+171.25.193.77:80 orport=443 id=A10C4F666D27364036B562823E5830BC448E046A ipv6=[2001:67c:289c:3::77]:443
+171.25.193.131:80 orport=443 id=79861CF8522FC637EF046F7688F5289E49D94576
+171.25.193.20:80 orport=443 id=DD8BD7307017407FCC36F8D04A688F74A0774C02 ipv6=[2001:67c:289c::20]:443
+# OK, but same machine as 79861CF8522FC637EF046F7688F5289E49D94576
+#171.25.193.132:80 orport=443 id=01C67E0CA8F97111E652C7564CB3204361FFFAB8
+# OK, but same machine as DD8BD7307017407FCC36F8D04A688F74A0774C02
+#171.25.193.25:80 orport=443 id=185663B7C12777F052B2C2D23D7A239D8DA88A0F ipv6=[2001:67c:289c::25]:443
+
+# Email sent directly to teor, verified using relay contact info
+212.47.229.2:9030 orport=9001 id=20462CBA5DA4C2D963567D17D0B7249718114A68
+93.115.97.242:9030 orport=9001 id=B5212DB685A2A0FCFBAE425738E478D12361710D
+46.28.109.231:9030 orport=9001 id=F70B7C5CD72D74C7F9F2DC84FA9D20D51BA13610 ipv6=[2a02:2b88:2:1::4205:42]:9001
+
+# Email sent directly to teor, verified using relay contact info
+85.235.250.88:80 orport=443 id=72B2B12A3F60408BDBC98C6DF53988D3A0B3F0EE
+185.96.88.29:80 orport=443 id=86C281AD135058238D7A337D546C902BE8505DDE
+185.96.180.29:80 orport=443 id=F93D8F37E35C390BCAD9F9069E13085B745EC216
+
+# Email sent directly to teor, verified using relay contact info
+185.11.180.67:80 orport=9001 id=794D8EA8343A4E820320265D05D4FA83AB6D1778
+
+# Email sent directly to teor, verified using relay contact info
+178.16.208.62:80 orport=443 id=5CF8AFA5E4B0BB88942A44A3F3AAE08C3BDFD60B ipv6=[2a00:1c20:4089:1234:a6a4:2926:d0af:dfee]:443
+46.165.221.166:80 orport=443 id=EE5F897C752D46BCFF531641B853FC6BC78DD4A7
+178.16.208.60:80 orport=443 id=B44FBE5366AD98B46D829754FA4AC599BAE41A6A ipv6=[2a00:1c20:4089:1234:67bc:79f3:61c0:6e49]:443
+178.16.208.55:80 orport=443 id=C4AEA05CF380BAD2230F193E083B8869B4A29937 ipv6=[2a00:1c20:4089:1234:7b2c:11c5:5221:903e]:443
+178.16.208.61:80 orport=443 id=3B52392E2256C35CDCF7801FF898FC88CE6D431A ipv6=[2a00:1c20:4089:1234:2712:a3d0:666b:88a6]:443
+81.89.96.88:80 orport=443 id=55ED4BB49F6D3F36D8D9499BE43500E017A5EF82 ipv6=[2a02:180:1:1:14c5:b0b7:2d7d:5f3a]:443
+209.222.8.196:80 orport=443 id=C86D2F3DEFE287A0EEB28D4887AF14E35C172733 ipv6=[2001:19f0:1620:41c1:426c:5adf:2ed5:4e88]:443
+81.89.96.89:80 orport=443 id=28651F419F5A1CF74511BB500C58112192DD4943 ipv6=[2a02:180:1:1:2ced:24e:32ea:a03b]:443
+46.165.221.166:9030 orport=9001 id=8C7106C880FE8AA1319DD71B59623FCB8914C9F1
+178.16.208.62:80 orport=443 id=5CF8AFA5E4B0BB88942A44A3F3AAE08C3BDFD60B ipv6=[2a00:1c20:4089:1234:a6a4:2926:d0af:dfee]:443"
+46.165.221.166:80 orport=443 id=EE5F897C752D46BCFF531641B853FC6BC78DD4A7
+178.16.208.60:80 orport=443 id=B44FBE5366AD98B46D829754FA4AC599BAE41A6A ipv6=[2a00:1c20:4089:1234:67bc:79f3:61c0:6e49]:443
+178.16.208.55:80 orport=443 id=C4AEA05CF380BAD2230F193E083B8869B4A29937 ipv6=[2a00:1c20:4089:1234:7b2c:11c5:5221:903e]:443
+178.16.208.61:80 orport=443 id=3B52392E2256C35CDCF7801FF898FC88CE6D431A ipv6=[2a00:1c20:4089:1234:2712:a3d0:666b:88a6]:443
+81.89.96.88:80 orport=443 id=55ED4BB49F6D3F36D8D9499BE43500E017A5EF82 ipv6=[2a02:180:1:1:14c5:b0b7:2d7d:5f3a]:443
+209.222.8.196:80 orport=443 id=C86D2F3DEFE287A0EEB28D4887AF14E35C172733 ipv6=[2001:19f0:1620:41c1:426c:5adf:2ed5:4e88]:443
+81.89.96.89:80 orport=443 id=28651F419F5A1CF74511BB500C58112192DD4943 ipv6=[2a02:180:1:1:2ced:24e:32ea:a03b]:443
+46.165.221.166:9030 orport=9001 id=8C7106C880FE8AA1319DD71B59623FCB8914C9F1
+178.16.208.56:80 orport=443 id=2CDCFED0142B28B002E89D305CBA2E26063FADE2 ipv6=[2a00:1c20:4089:1234:cd49:b58a:9ebe:67ec]:443
+178.16.208.58:80 orport=443 id=A4C98CEA3F34E05299417E9F885A642C88EF6029 ipv6=[2a00:1c20:4089:1234:cdae:1b3e:cc38:3d45]:443
+178.16.208.57:80 orport=443 id=92CFD9565B24646CAC2D172D3DB503D69E777B8A ipv6=[2a00:1c20:4089:1234:7825:2c5d:1ecd:c66f]:443
+178.16.208.59:80 orport=443 id=136F9299A5009A4E0E96494E723BDB556FB0A26B ipv6=[2a00:1c20:4089:1234:bff6:e1bb:1ce3:8dc6]:443
+
+# Email sent directly to teor, verified using relay contact info
+195.154.8.111:80 orport=443 id=FCB6695F8F2DC240E974510A4B3A0F2B12AB5B64
+51.255.235.246:80 orport=443 id=9B99C72B02AF8E3E5BE3596964F9CACD0090D132
+5.39.76.158:80 orport=443 id=C41F60F8B00E7FEF5CCC5BC6BB514CA1B8AAB651
+
+# Email sent directly to teor, verified using relay contact info
+109.163.234.5:80 orport=443 id=5C84C35936B7100B949AC75764EEF1352550550B
+109.163.234.7:80 orport=443 id=C46524E586E1B997329703D356C07EE12B28C722
+109.163.234.9:80 orport=443 id=5714542DCBEE1DD9864824723638FD44B2122CEA
+77.247.181.162:80 orport=443 id=7BB160A8F54BD74F3DA5F2CE701E8772B841859D
+109.163.234.4:80 orport=443 id=6B1E001929AF4DDBB747D02EC28340792B7724A6
+77.247.181.164:80 orport=443 id=10E13E340651D0EF66B4DEBF610B3C0981168107
+109.163.234.8:80 orport=443 id=20B0038D7A2FD73C696922551B8344CB0893D1F8
+77.247.181.166:80 orport=443 id=06E123865C590189B3181114F23F0F13A7BC0E69
+109.163.234.2:80 orport=443 id=B4F883DB3D478C7AE569C9F6CB766FD58650DC6A
+62.102.148.67:80 orport=443 id=4A0C3E177AF684581EF780981AEAF51A98A6B5CF
+109.163.234.5:80 orport=443 id=5C84C35936B7100B949AC75764EEF1352550550B
+109.163.234.7:80 orport=443 id=C46524E586E1B997329703D356C07EE12B28C722
+109.163.234.9:80 orport=443 id=5714542DCBEE1DD9864824723638FD44B2122CEA
+77.247.181.162:80 orport=443 id=7BB160A8F54BD74F3DA5F2CE701E8772B841859D
+109.163.234.4:80 orport=443 id=6B1E001929AF4DDBB747D02EC28340792B7724A6
+77.247.181.164:80 orport=443 id=10E13E340651D0EF66B4DEBF610B3C0981168107
+109.163.234.8:80 orport=443 id=20B0038D7A2FD73C696922551B8344CB0893D1F8
+77.247.181.166:80 orport=443 id=06E123865C590189B3181114F23F0F13A7BC0E69
+109.163.234.2:80 orport=443 id=B4F883DB3D478C7AE569C9F6CB766FD58650DC6A
+62.102.148.67:80 orport=443 id=4A0C3E177AF684581EF780981AEAF51A98A6B5CF
+
+# https://twitter.com/biotimylated/status/718994247500718080
+212.47.252.149:9030 orport=9001 id=2CAC39BAA996791CEFAADC9D4754D65AF5EB77C0
+
+# Email sent directly to teor, verified using relay contact info
+46.165.230.5:80 orport=443 id=A0F06C2FADF88D3A39AA3072B406F09D7095AC9E
+
+# Email sent directly to teor, verified using relay contact info
+94.242.246.24:23 orport=8080 id=EC116BCB80565A408CE67F8EC3FE3B0B02C3A065 ipv6=[2a01:608:ffff:ff07::1:24]:9004
+176.126.252.11:443 orport=9001 id=B0279A521375F3CB2AE210BDBFC645FDD2E1973A ipv6=[2a02:59e0:0:7::11]:9003
+176.126.252.12:21 orport=8080 id=379FB450010D17078B3766C2273303C358C3A442 ipv6=[2a02:59e0:0:7::12]:81
+94.242.246.23:443 orport=9001 id=F65E0196C94DFFF48AFBF2F5F9E3E19AAE583FD0 ipv6=[2a01:608:ffff:ff07::1:23]:9003
+85.248.227.164:444 orport=9002 id=B84F248233FEA90CAD439F292556A3139F6E1B82 ipv6=[2a00:1298:8011:212::164]:9004
+85.248.227.163:443 orport=9001 id=C793AB88565DDD3C9E4C6F15CCB9D8C7EF964CE9 ipv6=[2a00:1298:8011:212::163]:9003
+
+# Email sent directly to teor, verified using relay contact info
+148.251.190.229:9030 orport=9010 id=BF0FB582E37F738CD33C3651125F2772705BB8E8 ipv6=[2a01:4f8:211:c68::2]:9010
+
+# Email sent directly to teor, verified using relay contact info
+5.79.68.161:81 orport=443 id=9030DCF419F6E2FBF84F63CBACBA0097B06F557E ipv6=[2001:1af8:4700:a012:1::1]:443
+5.79.68.161:9030 orport=9001 id=B7EC0C02D7D9F1E31B0C251A6B058880778A0CD1 ipv6=[2001:1af8:4700:a012:1::1]:9001
+
+# Email sent directly to teor, verified using relay contact info
+62.210.92.11:9030 orport=9001 id=0266B0660F3F20A7D1F3D8335931C95EF50F6C6B ipv6=[2001:bc8:338c::1]:9001
+62.210.92.11:9130 orport=9101 id=387B065A38E4DAA16D9D41C2964ECBC4B31D30FF ipv6=[2001:bc8:338c::1]:9101
+
+# Email sent directly to teor, verified using relay contact info
+188.165.194.195:9030 orport=9001 id=49E7AD01BB96F6FE3AB8C3B15BD2470B150354DF
+
+# Message sent directly to teor, verified using relay contact info
+95.215.44.110:80 orport=443 id=D56AA4A1AA71961F5279FB70A6DCF7AD7B993EB5
+95.215.44.122:80 orport=443 id=998D8FE06B867AA3F8D257A7D28FFF16964D53E2
+95.215.44.111:80 orport=443 id=A7C7FD510B20BC8BE8F2A1D911364E1A23FBD09F
+
+# Email sent directly to teor, verified using relay contact info
+86.59.119.88:80 orport=443 id=ACD889D86E02EDDAB1AFD81F598C0936238DC6D0
+
+# Email sent directly to teor, verified using relay contact info
+144.76.73.140:9030 orport=9001 id=6A640018EABF3DA9BAD9321AA37C2C87BBE1F907
+
+# Email sent directly to teor, verified using relay contact info
+193.11.164.243:9030 orport=9001 id=FFA72BD683BC2FCF988356E6BEC1E490F313FB07 ipv6=[2001:6b0:7:125::243]:9001
+109.105.109.162:52860 orport=60784 id=32EE911D968BE3E016ECA572BB1ED0A9EE43FC2F ipv6=[2001:948:7:2::163]:5001
+
+# Email sent directly to teor, verified using relay contact info
+146.0.32.144:9030 orport=9001 id=35E8B344F661F4F2E68B17648F35798B44672D7E
+
+# Email sent directly to teor, verified using relay contact info
+46.252.26.2:45212 orport=49991 id=E589316576A399C511A9781A73DA4545640B479D
+
+# Email sent directly to teor, verified using relay contact info
+89.187.142.208:80 orport=443 id=64186650FFE4469EBBE52B644AE543864D32F43C
+
+# Email sent directly to teor, verified using relay contact info
+212.51.134.123:9030 orport=9001 id=50586E25BE067FD1F739998550EDDCB1A14CA5B2 ipv6=[2a02:168:6e00:0:3a60:77ff:fe9c:8bd1]:9001
+
+# Email sent directly to teor, verified using relay contact info
+46.101.143.173:80 orport=443 id=F960DF50F0FD4075AC9B505C1D4FFC8384C490FB
+
+# Email sent directly to teor, verified using relay contact info
+217.79.190.25:9030 orport=9090 id=361D33C96D0F161275EE67E2C91EE10B276E778B
+
+# Email sent directly to teor, verified using relay contact info
+193.171.202.146:9030 orport=9001 id=01A9258A46E97FF8B2CAC7910577862C14F2C524
+
+# Email sent directly to teor, verified using relay contact info
+197.231.221.211:9030 orport=9001 id=BC630CBBB518BE7E9F4E09712AB0269E9DC7D626
+
+# Email sent directly to teor, verified using relay contact info
+185.61.138.18:8080 orport=4443 id=2541759BEC04D37811C2209A88E863320271EC9C
+
+# Email sent directly to teor, verified using relay contact info
+193.11.114.45:9031 orport=9002 id=80AAF8D5956A43C197104CEF2550CD42D165C6FB
+193.11.114.43:9030 orport=9001 id=12AD30E5D25AA67F519780E2111E611A455FDC89 ipv6=[2001:6b0:30:1000::99]:9050
+193.11.114.46:9032 orport=9003 id=B83DC1558F0D34353BB992EF93AFEAFDB226A73E
+
+# Email sent directly to teor, verified using relay contact info
+144.76.26.175:9012 orport=9011 id=2BA2C8E96B2590E1072AECE2BDB5C48921BF8510
+
+# Email sent directly to teor, verified using relay contact info
+37.221.162.226:9030 orport=9001 id=D64366987CB39F61AD21DBCF8142FA0577B92811
+
+# Email sent directly to teor, verified using relay contact info
+91.219.237.244:80 orport=443 id=92ECC9E0E2AF81BB954719B189AC362E254AD4A5
+
+# Email sent directly to teor, verified using relay contact info
+185.21.100.50:9030 orport=9001 id=58ED9C9C35E433EE58764D62892B4FFD518A3CD0 ipv6=[2a00:1158:2:cd00:0:74:6f:72]:443
+
+# Email sent directly to teor, verified using relay contact info
+193.35.52.53:9030 orport=9001 id=DAA39FC00B196B353C2A271459C305C429AF09E4
+
+# Email sent directly to teor, verified using relay contact info
+134.119.3.164:9030 orport=9001 id=D1B8AAA98C65F3DF7D8BB3AF881CAEB84A33D8EE
+
+# Email sent directly to teor, verified using relay contact info
+81.7.10.93:31336 orport=31337 id=99E246DB480B313A3012BC3363093CC26CD209C7
+
+# Email sent directly to teor, verified using relay contact info
+178.62.22.36:80 orport=443 id=A0766C0D3A667A3232C7D569DE94A28F9922FCB1 ipv6=[2a03:b0c0:1:d0::174:1]:9050
+46.101.102.71:80 orport=443 id=9504CB22EEB25D344DE63CB7A6F2C46F895C3686 ipv6=[2a03:b0c0:3:d0::2ed:7001]:9050
+188.166.23.127:80 orport=443 id=3771A8154DEA98D551607806C80A209CDAA74535 ipv6=[2a03:b0c0:2:d0::27b:7001]:9050
+198.199.64.217:80 orport=443 id=FAD306BAA59F6A02783F8606BDAA431F5FF7D1EA ipv6=[2604:a880:400:d0::1a9:b001]:9050
+159.203.32.149:80 orport=443 id=55C7554AFCEC1062DCBAC93E67B2E03C6F330EFC ipv6=[2604:a880:cad:d0::105:f001]:9050
+
+# Email sent directly to teor, verified using relay contact info
+5.196.31.80:9030 orport=9900 id=DFB2EB472643FAFCD5E73D2E37D51DB67203A695 ipv6=[2001:41d0:52:400::a65]:9900
+
+# Email sent directly to teor, verified using relay contact info
+188.138.112.60:1433 orport=1521 id=C414F28FD2BEC1553024299B31D4E726BEB8E788
+
+# Email sent directly to teor, verified using relay contact info
+213.61.66.118:9031 orport=9001 id=30648BC64CEDB3020F4A405E4AB2A6347FB8FA22
+213.61.66.117:9032 orport=9002 id=6E44A52E3D1FF7683FE5C399C3FB5E912DE1C6B4
+213.61.66.115:9034 orport=9004 id=480CCC94CEA04D2DEABC0D7373868E245D4C2AE2
+213.61.66.116:9033 orport=9003 id=A9DEB920B42B4EC1DE6249034039B06D61F38690
+
+# Email sent directly to teor, verified using relay contact info
+136.243.187.165:9030 orport=443 id=1AC65257D7BFDE7341046625470809693A8ED83E
+
+# Email sent directly to teor, verified using relay contact info
+212.47.230.49:9030 orport=9001 id=3D6D0771E54056AEFC28BB1DE816951F11826E97
+
+# Email sent directly to teor, verified using relay contact info
+176.31.180.157:143 orport=22 id=E781F4EC69671B3F1864AE2753E0890351506329 ipv6=[2001:41d0:8:eb9d::1]:22
+
+# Email sent directly to teor, verified using relay contact info
+192.99.55.69:80 orport=443 id=0682DE15222A4A4A0D67DBA72A8132161992C023
+51.254.215.121:80 orport=443 id=262B66AD25C79588AD1FC8ED0E966395B47E5C1D
+192.99.59.140:80 orport=443 id=3C9148DA49F20654730FAC83FFF693A4D49D0244
+51.254.215.13:80 orport=443 id=73C30C8ABDD6D9346C822966DE73B9F82CB6178A
+51.254.215.129:80 orport=443 id=7B4491D05144B20AE8519AE784B94F0525A8BB79
+192.99.59.139:80 orport=443 id=82EC878ADA7C205146B9F5193A7310867FAA0D7B
+51.254.215.124:80 orport=443 id=98999EBE89B5FA9AA0C58421F0B46C3D0AF51CBA
+51.254.214.208:80 orport=443 id=C3F0D1417848EAFC41277A73DEB4A9F2AEC23DDF
+192.99.59.141:80 orport=443 id=F45426551795B9DA78BEDB05CD5F2EACED8132E4
+192.99.59.14:80 orport=443 id=161A1B29A37EBF096D2F8A9B1E176D6487FE42AE
+
+# Email sent directly to teor, verified using relay contact info
+151.80.42.103:9030 orport=9001 id=9007C1D8E4F03D506A4A011B907A9E8D04E3C605 ipv6=[2001:41d0:e:f67::114]:9001
+
+# Email sent directly to teor, verified using relay contact info
+5.39.92.199:80 orport=443 id=0BEA4A88D069753218EAAAD6D22EA87B9A1319D6
+
+# Email sent directly to teor, verified using relay contact info
+176.31.159.231:80 orport=443 id=D5DBCC0B4F029F80C7B8D33F20CF7D97F0423BB1
+176.31.159.230:80 orport=443 id=631748AFB41104D77ADBB7E5CD4F8E8AE876E683
+195.154.79.128:80 orport=443 id=C697612CA5AED06B8D829FCC6065B9287212CB2F
+195.154.9.161:80 orport=443 id=B6295A9960F89BD0C743EEBC5670450EA6A34685
+46.148.18.74:8080 orport=443 id=6CACF0B5F03C779672F3C5C295F37C8D234CA3F7
+
+# Email sent directly to teor, verified using relay contact info
+178.62.36.64:9030 orport=9001 id=B87C84E38DAECFFFFDE98E5AEE5786AFDC748F2C
+
+# Email sent directly to teor, verified using relay contact info
+37.187.102.108:9090 orport=5550 id=F4263275CF54A6836EE7BD527B1328836A6F06E1
+212.47.241.21:80 orport=443 id=892F941915F6A0C6E0958E52E0A9685C190CF45C
+
+# Email sent directly to teor, verified using relay contact info
+195.191.233.221:80 orport=443 id=DE134FC8E5CC4EC8A5DE66934E70AC9D70267197
+
+# Email sent directly to teor, verified using relay contact info
+37.187.1.149:9030 orport=9001 id=08DC0F3C6E3D9C527C1FC8745D35DD1B0DE1875D ipv6=[2001:41d0:a:195::1]:9001
+
+# Email sent directly to teor, verified using relay contact info
+62.210.238.33:9030 orport=9001 id=FDF845FC159C0020E2BDDA120C30C5C5038F74B4
+
+# Email sent directly to teor, verified using relay contact info
+37.157.195.87:8030 orport=443 id=12FD624EE73CEF37137C90D38B2406A66F68FAA2
+
+# Email sent directly to teor, verified using relay contact info
+37.187.7.74:80 orport=443 id=AEA43CB1E47BE5F8051711B2BF01683DB1568E05 ipv6=[2001:41d0:a:74a::1]:443
+
+# Email sent directly to teor, verified using relay contact info
+185.66.250.141:9030 orport=9001 id=B1726B94885CE3AC3910CA8B60622B97B98E2529
+
+# Email sent directly to teor, verified using relay contact info
+185.104.120.7:9030 orport=443 id=445F1C853966624FB3CF1E12442570DC553CC2EC ipv6=[2a06:3000::120:7]:443
+185.104.120.2:9030 orport=21 id=518FF8708698E1DA09C823C36D35DF89A2CAD956
+185.104.120.4:9030 orport=9001 id=F92B3CB9BBE0CB22409843FB1AE4DBCD5EFAC835
+185.104.120.3:9030 orport=21 id=707C1B61AC72227B34487B56D04BAA3BA1179CE8 ipv6=[2a06:3000::120:3]:21
+
+# Email sent directly to teor, verified using relay contact info
+37.187.102.186:9030 orport=9001 id=489D94333DF66D57FFE34D9D59CC2D97E2CB0053 ipv6=[2001:41d0:a:26ba::1]:9001
+
+# Email sent directly to teor, verified using relay contact info
+5.35.251.247:9030 orport=9001 id=9B1F5187DFBA89DC24B37EA7BF896C12B43A27AE
+
+# Email sent directly to teor, verified using relay contact info
+198.96.155.3:8080 orport=5001 id=BCEDF6C193AA687AE471B8A22EBF6BC57C2D285E
+
+# Email sent directly to teor, verified using relay contact info
+212.83.154.33:8888 orport=443 id=3C79699D4FBC37DE1A212D5033B56DAE079AC0EF
+212.83.154.33:8080 orport=8443 id=322C6E3A973BC10FC36DE3037AD27BC89F14723B
+
+# Email sent directly to teor, verified using relay contact info
+51.255.41.65:9030 orport=9001 id=9231DF741915AA1630031A93026D88726877E93A
+
+# Email sent directly to teor, verified using relay contact info
+78.142.142.246:80 orport=443 id=5A5E03355C1908EBF424CAF1F3ED70782C0D2F74
+
+# Email sent directly to teor, verified using relay contact info
+195.154.97.91:80 orport=443 id=BD33C50D50DCA2A46AAED54CA319A1EFEBF5D714
+
+# Email sent directly to teor, verified using relay contact info
+62.210.129.246:80 orport=443 id=79E169B25E4C7CE99584F6ED06F379478F23E2B8
+
+# Email sent directly to teor, verified using relay contact info
+5.196.74.215:9030 orport=9001 id=5818055DFBAF0FA7F67E8125FD63E3E7F88E28F6
+
+# Email sent directly to teor, verified using relay contact info
+212.47.233.86:9030 orport=9001 id=B4CAFD9CBFB34EC5DAAC146920DC7DFAFE91EA20
+
+# Email sent directly to teor, verified using relay contact info
+85.214.206.219:9030 orport=9001 id=98F8D5F359949E41DE8DF3DBB1975A86E96A84A0
+
+# Email sent directly to teor, verified using relay contact info
+46.166.170.4:80 orport=443 id=19F42DB047B72C7507F939F5AEA5CD1FA4656205
+46.166.170.5:80 orport=443 id=DA705AD4591E7B4708FA2CAC3D53E81962F3E6F6
+
+# Email sent directly to teor, verified using relay contact info
+5.189.157.56:80 orport=443 id=77F6D6A6B6EAFB8F5DADDC07A918BBF378ED6725
+
+# Email sent directly to teor, verified using relay contact info
+46.28.110.244:80 orport=443 id=9F7D6E6420183C2B76D3CE99624EBC98A21A967E
+185.13.39.197:80 orport=443 id=001524DD403D729F08F7E5D77813EF12756CFA8D
+95.130.12.119:80 orport=443 id=587E0A9552E4274B251F29B5B2673D38442EE4BF
+
+# Email sent directly to teor, verified using relay contact info
+212.129.62.232:80 orport=443 id=B143D439B72D239A419F8DCE07B8A8EB1B486FA7
+
+# Email sent directly to teor, verified using relay contact info
+91.219.237.229:80 orport=443 id=1ECD73B936CB6E6B3CD647CC204F108D9DF2C9F7
+
+# Email sent directly to teor, verified using relay contact info
+# Suitable, check with operator before adding
+#212.47.240.10:82 orport=443 id=2A4C448784F5A83AFE6C78DA357D5E31F7989DEB
+212.47.240.10:81 orport=993 id=72527E3242CB15AADE28374AE0D35833FC083F60
+163.172.131.88:80 orport=443 id=AD253B49E303C6AB1E048B014392AC569E8A7DAE ipv6=[2001:bc8:4400:2100::2:1009]:443
+# Suitable, check with operator before adding
+#163.172.131.88:81 orport=993 id=D5F3FB17504744FB7ECEF46F4B1D155258A6D942 ipv6=D5F3FB17504744FB7ECEF46F4B1D155258A6D942
+
+# Email sent directly to teor, verified using relay contact info
+46.101.151.222:80 orport=443 id=1DBAED235E3957DE1ABD25B4206BE71406FB61F8
+178.62.60.37:80 orport=443 id=175921396C7C426309AB03775A9930B6F611F794
+
+# Email sent directly to teor, verified using relay contact info
+178.62.197.82:80 orport=443 id=0D3EBA17E1C78F1E9900BABDB23861D46FCAF163
+
+# Email sent directly to teor, verified using relay contact info
+82.223.21.74:9030 orport=9001 id=7A32C9519D80CA458FC8B034A28F5F6815649A98 ipv6=[2001:470:53e0::cafe]:9050
+
+# Email sent directly to teor, verified using relay contact info
+146.185.177.103:80 orport=9030 id=9EC5E097663862DF861A18C32B37C5F82284B27D
+
+# Email sent directly to teor, verified using relay contact info
+37.187.22.87:9030 orport=9001 id=36B9E7AC1E36B62A9D6F330ABEB6012BA7F0D400 ipv6=[2001:41d0:a:1657::1]:9001
+
+# Email sent directly to teor, verified using relay contact info
+37.59.46.159:9030 orport=9001 id=CBD0D1BD110EC52963082D839AC6A89D0AE243E7
+
+# Email sent directly to teor, verified using relay contact info
+212.47.250.243:9030 orport=9001 id=5B33EDBAEA92F446768B3753549F3B813836D477
+# Confirm with operator before adding these
+#163.172.133.36:9030 orport=9001 id=D8C2BD36F01FA86F4401848A0928C4CB7E5FDFF9
+#158.69.216.70:9030 orport=9001 id=0ACE25A978D4422C742D6BC6345896719BF6A7EB
+
+# Email sent directly to teor, verified using relay contact info
+5.199.142.236:9030 orport=9001 id=F4C0EDAA0BF0F7EC138746F8FEF1CE26C7860265
+
+# Email sent directly to teor, verified using relay contact info
+188.166.133.133:9030 orport=9001 id=774555642FDC1E1D4FDF2E0C31B7CA9501C5C9C7 ipv6=[2a03:b0c0:2:d0::5:f001]:9001
+
+# Email sent directly to teor, verified using relay contact info
+5.196.88.122:9030 orport=9001 id=0C2C599AFCB26F5CFC2C7592435924C1D63D9484
+
+# Email sent directly to teor, verified using relay contact info
+46.8.249.10:80 orport=443 id=31670150090A7C3513CB7914B9610E786391A95D
+
+# Email sent directly to teor, verified using relay contact info
+144.76.163.93:9030 orport=9001 id=22F08CF09764C4E8982640D77F71ED72FF26A9AC
+
+# Email sent directly to teor, verified using relay contact info
+46.4.24.161:9030 orport=9001 id=DB4C76A3AD7E234DA0F00D6F1405D8AFDF4D8DED
+46.4.24.161:9031 orport=9002 id=7460F3D12EBE861E4EE073F6233047AACFE46AB4
+46.38.51.132:9030 orport=9001 id=810DEFA7E90B6C6C383C063028EC397A71D7214A
+163.172.194.53:9030 orport=9001 id=8C00FA7369A7A308F6A137600F0FA07990D9D451
+
+# Email sent directly to teor, verified using relay contact info
+176.10.107.180:9030 orport=9001 id=3D7E274A87D9A89AF064C13D1EE4CA1F184F2600
+195.154.75.84:9030 orport=9001 id=F80FDE27EFCB3F6A7B4E2CC517133DBFFA78BA2D
+195.154.127.246:9030 orport=9001 id=4FEE77AFFD157BBCF2D896AE417FBF647860466C
+
+# Email sent directly to teor, verified using relay contact info
+46.28.207.19:80 orport=443 id=5B92FA5C8A49D46D235735504C72DBB3472BA321
+46.28.207.141:80 orport=443 id=F69BED36177ED727706512BA6A97755025EEA0FB
+46.28.205.170:80 orport=443 id=AF322D83A4D2048B22F7F1AF5F38AFF4D09D0B76
+95.183.48.12:80 orport=443 id=7187CED1A3871F837D0E60AC98F374AC541CB0DA
+
+# Email sent directly to teor, verified using relay contact info
+93.180.156.84:9030 orport=9001 id=8844D87E9B038BE3270938F05AF797E1D3C74C0F
+
+# Email sent directly to teor, verified using relay contact info
+37.187.22.172:9030 orport=9035 id=335E4117BD9A4966403C2AFA31CFDD1BC13BD46A
+
+# Email sent directly to teor, verified using relay contact info
+37.187.115.157:9030 orport=9001 id=D5039E1EBFD96D9A3F9846BF99EC9F75EDDE902A
+
+# Email sent directly to teor, verified using relay contact info
+5.34.183.205:80 orport=443 id=DDD7871C1B7FA32CB55061E08869A236E61BDDF8
+
+# Email sent directly to teor, verified using relay contact info
+51.254.246.203:9030 orport=9001 id=47B596B81C9E6277B98623A84B7629798A16E8D5
+
+# Email sent directly to teor, verified using relay contact info
+5.9.146.203:80 orport=443 id=1F45542A24A61BF9408F1C05E0DCE4E29F2CBA11
+
+# Email sent directly to teor, verified using relay contact info
+167.114.152.100:9030 orport=443 id=0EF5E5FFC5D1EABCBDA1AFF6F6D6325C5756B0B2 ipv6=[2607:5300:100:200::1608]:443
+
+# Email sent directly to teor, verified using relay contact info
+192.99.168.102:80 orport=443 id=230A8B2A8BA861210D9B4BA97745AEC217A94207
+167.114.153.21:80 orport=443 id=0B85617241252517E8ECF2CFC7F4C1A32DCD153F
+
+# Email sent directly to teor, verified using relay contact info
+204.11.50.131:9030 orport=9001 id=185F2A57B0C4620582602761097D17DB81654F70
+
+# Email sent directly to teor, verified using relay contact info
+151.236.222.217:44607 orport=9001 id=94D58704C2589C130C9C39ED148BD8EA468DBA54
+
+# Email sent directly to teor, verified using relay contact info
+194.150.168.79:11112 orport=11111 id=29F1020B94BE25E6BE1AD13E93CE19D2131B487C
+
+# Email sent directly to teor, verified using relay contact info
+185.35.202.221:9030 orport=9001 id=C13B91384CDD52A871E3ECECE4EF74A7AC7DCB08 ipv6=[2a02:ed06::221]:9001
+
+# Email sent directly to teor, verified using relay contact info
+5.9.151.241:9030 orport=4223 id=9BF04559224F0F1C3C953D641F1744AF0192543A
+
+# Email sent directly to teor, verified using relay contact info
+89.40.71.149:8081 orport=8080 id=EC639EDAA5121B47DBDF3D6B01A22E48A8CB6CC7
+
+# Email sent directly to teor, verified using relay contact info
+92.222.20.130:80 orport=443 id=0639612FF149AA19DF3BCEA147E5B8FED6F3C87C
+
+# Email sent directly to teor, verified using relay contact info
+80.112.155.100:9030 orport=9001 id=1163378F239C36CA1BDC730AC50BF4F2976141F5 ipv6=[2001:470:7b02::38]:9001
+
+# Email sent directly to teor, verified using relay contact info
+83.212.99.68:80 orport=443 id=DDBB2A38252ADDA53E4492DDF982CA6CC6E10EC0 ipv6=[2001:648:2ffc:1225:a800:bff:fe3d:67b5]:443
+
+# Email sent directly to teor, verified using relay contact info
+51.254.101.242:32991 orport=9001 id=4CC9CC9195EC38645B699A33307058624F660CCF
+
+# Email sent directly to teor, verified using relay contact info
+95.130.11.147:9030 orport=443 id=6B697F3FF04C26123466A5C0E5D1F8D91925967A
+
+# Email sent directly to teor, verified using relay contact info
+176.31.191.26:9030 orport=9001 id=7350AB9ED7568F22745198359373C04AC783C37C
+
+# Email sent directly to teor, verified using relay contact info
+128.199.55.207:9030 orport=9001 id=BCEF908195805E03E92CCFE669C48738E556B9C5 ipv6=[2a03:b0c0:2:d0::158:3001]:9001
+
+# Email sent directly to teor, verified using relay contact info
+88.190.208.4:30555 orport=30556 id=030A6EB24725C05D8E0FCE21923CBA5223E75E0E
+
+# Email sent directly to teor, verified using relay contact info
+178.32.216.146:9030 orport=9001 id=17898F9A2EBC7D69DAF87C00A1BD2FABF3C9E1D2
+
+# Email sent directly to teor, verified using relay contact info
+78.193.40.205:8080 orport=8443 id=C91450840E75AC1B654A3096744338A573A239C6
+
+# Email sent directly to teor, verified using relay contact info
+212.83.40.238:9030 orport=9001 id=F409FA7902FD89270E8DE0D7977EA23BC38E5887
+
+# Email sent directly to teor, verified using relay contact info
+204.8.156.142:80 orport=443 id=94C4B7B8C50C86A92B6A20107539EE2678CF9A28
+
+# Email sent directly to teor, verified using relay contact info
+80.240.139.111:80 orport=443 id=DD3BE7382C221F31723C7B294310EF9282B9111B
+
+# Email sent directly to teor, verified using relay contact info
+185.97.32.18:9030 orport=9001 id=3BAB316CAAEC47E71905EB6C65584636D5689A8A
+
+# Email sent directly to teor, verified using relay contact info
+149.56.45.200:9030 orport=9001 id=FE296180018833AF03A8EACD5894A614623D3F76
+
+# Email sent directly to teor, verified using relay contact info
+81.2.209.10:443 orport=80 id=B6904ADD4C0D10CDA7179E051962350A69A63243
+
+# Email sent directly to teor, verified using relay contact info
+195.154.164.243:80 orport=443 id=AC66FFA4AB35A59EBBF5BF4C70008BF24D8A7A5C ipv6=[2001:bc8:399f:f000::1]:993
+138.201.26.2:80 orport=443 id=6D3A3ED5671E4E3F58D4951438B10AE552A5FA0F
+81.7.16.182:80 orport=443 id=51E1CF613FD6F9F11FE24743C91D6F9981807D82 ipv6=[2a02:180:1:1::517:10b6]:993
+134.119.36.135:80 orport=443 id=763C9556602BD6207771A7A3D958091D44C43228 ipv6=[2a00:1158:3::2a8]:993
+46.228.199.19:80 orport=443 id=E26AFC5F718E21AC502899B20C653AEFF688B0D2 ipv6=[2001:4ba0:cafe:4a::1]:993
+37.200.98.5:80 orport=443 id=231C2B9C8C31C295C472D031E06964834B745996 ipv6=[2a00:1158:3::11a]:993
+46.23.70.195:80 orport=443 id=C9933B3725239B6FAB5227BA33B30BE7B48BB485
+185.15.244.124:80 orport=443 id=935BABE2564F82016C19AEF63C0C40B5753BA3D2 ipv6=[2001:4ba0:cafe:e35::1]:993
+195.154.116.232:80 orport=443 id=B35C5739C8C5AB72094EB2B05738FD1F8EEF6EBD ipv6=[2001:bc8:399f:200::1]:993
+195.154.121.198:80 orport=443 id=0C77421C890D16B6D201283A2244F43DF5BC89DD ipv6=[2001:bc8:399f:100::1]:993
+37.187.20.59:80 orport=443 id=91D23D8A539B83D2FB56AA67ECD4D75CC093AC55 ipv6=[2001:41d0:a:143b::1]:993
+217.12.208.117:80 orport=443 id=E6E18151300F90C235D3809F90B31330737CEB43 ipv6=[2a00:1ca8:a7::1bb]:993
+81.7.10.251:80 orport=443 id=8073670F8F852971298F8AF2C5B23AE012645901 ipv6=[2a02:180:1:1::517:afb]:993
+46.36.39.50:80 orport=443 id=ED4B0DBA79AEF5521564FA0231455DCFDDE73BB6 ipv6=[2a02:25b0:aaaa:aaaa:8d49:b692:4852:0]:995
+91.194.90.103:80 orport=443 id=75C4495F4D80522CA6F6A3FB349F1B009563F4B7 ipv6=[2a02:c200:0:10:3:0:5449:1]:993
+163.172.25.118:80 orport=22 id=0CF8F3E6590F45D50B70F2F7DA6605ECA6CD408F
+188.138.88.42:80 orport=443 id=70C55A114C0EF3DC5784A4FAEE64388434A3398F
+81.7.13.84:80 orport=443 id=0C1E7DD9ED0676C788933F68A9985ED853CA5812 ipv6=[2a02:180:1:1::5b8f:538c]:993
+213.246.56.95:80 orport=443 id=27E6E8E19C46751E7312420723C6162FF3356A4C ipv6=[2a00:c70:1:213:246:56:95:1]:993
+94.198.100.18:80 orport=443 id=BAACCB29197DB833F107E410E2BFAE5009EE7583
+217.12.203.46:80 orport=443 id=6A29FD8C00D573E6C1D47852345B0E5275BA3307
+212.117.180.107:80 orport=443 id=0B454C7EBA58657B91133A587C1BDAEDC6E23142
+217.12.199.190:80 orport=443 id=A37C47B03FF31CA6937D3D68366B157997FE7BCD ipv6=[2a02:27a8:0:2::486]:993
+216.230.230.247:80 orport=443 id=4C7BF55B1BFF47993DFF995A2926C89C81E4F04A
+69.30.215.42:80 orport=443 id=510176C07005D47B23E6796F02C93241A29AA0E9 ipv6=[2604:4300:a:2e::2]:993
+89.46.100.162:80 orport=443 id=6B7191639E179965FD694612C9B2C8FB4267B27D
+107.181.174.22:80 orport=443 id=5A551BF2E46BF26CC50A983F7435CB749C752553 ipv6=[2607:f7a0:3:4::4e]:993
+
+# Email sent directly to teor, verified using relay contact info
+212.238.208.48:9030 orport=9001 id=F406219CDD339026D160E53FCA0EF6857C70F109 ipv6=[2001:984:a8fb:1:ba27:ebff:feac:c109]:9001
+
+# Email sent directly to teor, verified using relay contact info
+176.158.132.12:9030 orport=9001 id=DC163DDEF4B6F0C6BC226F9F6656A5A30C5C5686
+
+# Email sent directly to teor, verified using relay contact info
+91.229.20.27:9030 orport=9001 id=9A0D54D3A6D2E0767596BF1515E6162A75B3293F
+
+# Email sent directly to teor, verified using relay contact info
+# Awaiting confirmation of new ORPort from relay operator
+80.127.137.19:80 orport=443 id=6EF897645B79B6CB35E853B32506375014DE3621 ipv6=[2001:981:47c1:1::6]:443
+
+# Email sent directly to teor, verified using relay contact info
+163.172.138.22:80 orport=443 id=8664DC892540F3C789DB37008236C096C871734D
+
+# Email sent directly to teor, verified using relay contact info
+97.74.237.196:9030 orport=9001 id=2F0F32AB1E5B943CA7D062C03F18960C86E70D94
+
+# Email sent directly to teor, verified using relay contact info
+192.187.124.98:9030 orport=9001 id=FD1871854BFC06D7B02F10742073069F0528B5CC
+
+# Email sent directly to teor, verified using relay contact info
+178.62.98.160:9030 orport=9001 id=8B92044763E880996A988831B15B2B0E5AD1544A
+
+# Email sent directly to teor, verified using relay contact info
+195.154.15.227:9030 orport=9001 id=6C3E3AB2F5F03CD71B637D433BAD924A1ECC5796
+
+# Email sent directly to teor, verified using relay contact info
+185.100.86.100:80 orport=443 id=0E8C0C8315B66DB5F703804B3889A1DD66C67CE0
+
+# Email sent directly to teor, verified using relay contact info
+164.132.77.175:9030 orport=9001 id=3B33F6FCA645AD4E91428A3AF7DC736AD9FB727B
+78.24.75.53:9030 orport=9001 id=DEB73705B2929AE9BE87091607388939332EF123
+
+# Email sent directly to teor, verified using relay contact info
+46.101.237.246:9030 orport=9001 id=75F1992FD3F403E9C082A5815EB5D12934CDF46C ipv6=[2a03:b0c0:3:d0::208:5001]:9050
+178.62.86.96:9030 orport=9001 id=439D0447772CB107B886F7782DBC201FA26B92D1 ipv6=[2a03:b0c0:1:d0::3cf:7001]:9050
+
+# Email sent directly to teor, verified using relay contact info
+91.233.106.121:80 orport=443 id=896364B7996F5DFBA0E15D1A2E06D0B98B555DD6
+
+# Email sent directly to teor, verified using relay contact info
+167.114.113.48:80 orport=443 id=2EC0C66EA700C44670444280AABAB1EC78B722A0
+
+# Email sent directly to teor, verified using relay contact info
+79.120.16.42:9030 orport=9001 id=BD552C165E2ED2887D3F1CCE9CFF155DDA2D86E6
+
+# Email sent directly to teor, verified using relay contact info
+95.128.43.164:80 orport=443 id=616081EC829593AF4232550DE6FFAA1D75B37A90 ipv6=[2a02:ec0:209:10::4]:443
+
+# Email sent directly to teor, verified using relay contact info
+166.82.21.200:9030 orport=9029 id=D5C33F3E203728EDF8361EA868B2939CCC43FAFB
+
+# Email sent directly to teor, verified using relay contact info
+91.121.54.8:9030 orport=9001 id=CBEE0F3303C8C50462A12107CA2AE061831931BC
+
+# Email sent directly to teor, verified using relay contact info
+178.217.184.32:9030 orport=443 id=8B7F47AE1A5D954A3E58ACDE0865D09DBA5B738D
+
+# Email sent directly to teor, verified using relay contact info
+85.10.201.47:9030 orport=9001 id=D8B7A3A6542AA54D0946B9DC0257C53B6C376679 ipv6=[2a01:4f8:a0:43eb::beef]:9001
+
+# Email sent directly to teor, verified using relay contact info
+120.29.217.46:80 orport=443 id=5E853C94AB1F655E9C908924370A0A6707508C62
+
+# Email sent directly to teor, verified using relay contact info
+37.153.1.10:9030 orport=9001 id=9772EFB535397C942C3AB8804FB35CFFAD012438
+
+# Email sent directly to teor, verified using relay contact info
+92.222.4.102:9030 orport=9001 id=1A6B8B8272632D8AD38442027F822A367128405C
+
+# Email sent directly to teor, verified using relay contact info
+31.31.78.49:80 orport=443 id=46791D156C9B6C255C2665D4D8393EC7DBAA7798
+
+# Email sent directly to teor, verified using relay contact info
+96.47.231.214:9030 orport=8080 id=F843CB5729575D76FF1FFBB2179BDCF52C0C6387
+192.99.246.48:9030 orport=9001 id=CD6B149BED1BB254EF6DFF9D75DDB11E7F8A38A4 ipv6=[2607:5300:100:200::de3]:9002
+192.160.102.164:80 orport=9001 id=823AA81E277F366505545522CEDC2F529CE4DC3F
+
+# Email sent directly to teor, verified using relay contact info
+136.243.214.137:80 orport=443 id=B291D30517D23299AD7CEE3E60DFE60D0E3A4664
+
+# Email sent directly to teor, verified using relay contact info
+192.87.28.28:9030 orport=9001 id=ED2338CAC2711B3E331392E1ED2831219B794024
+192.87.28.82:9030 orport=9001 id=844AE9CAD04325E955E2BE1521563B79FE7094B7
+
+# Email sent directly to teor, verified using relay contact info
+192.87.28.28:9030 orport=9001 id=ED2338CAC2711B3E331392E1ED2831219B794024
+# OK, but same machine as ED2338CAC2711B3E331392E1ED2831219B794024
+#192.87.28.82:9030 orport=9001 id=844AE9CAD04325E955E2BE1521563B79FE7094B7
+
+# https://twitter.com/kosjoli/status/719507270904758272
+85.10.202.87:9030 orport=9001 id=971AFB23C168DCD8EDA17473C1C452B359DE3A5A
+176.9.5.116:9030 orport=9001 id=A1EB8D8F1EE28DB98BBB1EAA3B4BEDD303BAB911
+46.4.111.124:9030 orport=9001 id=D9065F9E57899B3D272AA212317AF61A9B14D204
+
+# Email sent directly to teor, verified using relay contact info
+78.46.164.129:9030 orport=9001 id=52AEA31188331F421B2EDB494DB65CD181E5B257
+
+# Email sent directly to teor, verified using relay contact info
+185.100.85.61:80 orport=443 id=025B66CEBC070FCB0519D206CF0CF4965C20C96E
+
+# Email sent directly to teor, verified using relay contact info
+108.166.168.158:80 orport=443 id=CDAB3AE06A8C9C6BF817B3B0F1877A4B91465699
+
+# Email sent directly to teor, verified using relay contact info
+91.219.236.222:80 orport=443 id=EC413181CEB1C8EDC17608BBB177CD5FD8535E99
+
+# Email sent directly to teor, verified using relay contact info
+185.14.185.240:9030 orport=443 id=D62FB817B0288085FAC38A6DC8B36DCD85B70260
+192.34.63.137:9030 orport=443 id=ABCB4965F1FEE193602B50A365425105C889D3F8
+
+# Email sent directly to teor, verified using relay contact info
+185.13.38.75:9030 orport=9001 id=D2A1703758A0FBBA026988B92C2F88BAB59F9361
+
+# Email sent directly to teor, verified using relay contact info
+128.204.39.106:9030 orport=9001 id=6F0F3C09AF9580F7606B34A7678238B3AF7A57B7
+
+# Email sent directly to teor, verified using relay contact info
+198.50.191.95:80 orport=443 id=39F096961ED2576975C866D450373A9913AFDC92
+
+# Email sent directly to teor, verified using relay contact info
+185.100.85.138:80 orport=46356 id=5C4DF16A0029CC4F67D3E127356E68F219269859
+
+# Email sent directly to teor, verified using relay contact info
+167.114.66.61:9696 orport=443 id=DE6CD5F09DF26076F26321B0BDFBE78ACD935C65 ipv6=[2607:5300:100::78d]:443
+
+# Email sent directly to teor, verified using relay contact info
+66.111.2.20:9030 orport=9001 id=9A68B85A02318F4E7E87F2828039FBD5D75B0142
+66.111.2.16:9030 orport=9001 id=3F092986E9B87D3FDA09B71FA3A602378285C77A
+
+# Email sent directly to teor, verified using relay contact info
+92.222.38.67:80 orport=443 id=DED6892FF89DBD737BA689698A171B2392EB3E82
+
+# Email sent directly to teor, verified using relay contact info
+212.47.228.115:9030 orport=443 id=BCA017ACDA48330D02BB70716639ED565493E36E
+
+# Email sent directly to teor, verified using relay contact info
+185.100.84.175:80 orport=443 id=39B59AF4FE54FAD8C5085FA9C15FDF23087250DB
+
+# Email sent directly to teor, verified using relay contact info
+166.70.207.2:9030 orport=9001 id=E3DB2E354B883B59E8DC56B3E7A353DDFD457812
+
+# Emails sent directly to teor, verified using relay contact info
+#69.162.139.9:9030 orport=9001 id=4791FC0692EAB60DF2BCCAFF940B95B74E7654F6 ipv6=[2607:f128:40:1212::45a2:8b09]:9001
+
+# Email sent directly to teor, verified using relay contact info
+213.239.217.18:1338 orport=1337 id=C37BC191AC389179674578C3E6944E925FE186C2 ipv6=[2a01:4f8:a0:746a:101:1:1:1]:1337
+
+# Email sent directly to teor, verified using relay contact info
+188.40.128.246:9030 orport=9001 id=AD19490C7DBB26D3A68EFC824F67E69B0A96E601
+
+# Email sent directly to teor, verified using relay contact info
+88.198.253.13:9030 orport=9001 id=DF924196D69AAE3C00C115A9CCDF7BB62A175310 ipv6=[2a01:4f8:11a:b1f::2]:9001
+
+# Email sent directly to teor, verified using relay contact info
+185.100.86.128:9030 orport=9001 id=9B31F1F1C1554F9FFB3455911F82E818EF7C7883
+46.36.36.127:9030 orport=9001 id=C80DF89B21FF932DEC0D7821F679B6C79E1449C3
+
+# Email sent directly to teor, verified using relay contact info
+176.10.104.240:80 orport=443 id=0111BA9B604669E636FFD5B503F382A4B7AD6E80
+176.10.104.240:8080 orport=8443 id=AD86CD1A49573D52A7B6F4A35750F161AAD89C88
+176.10.104.243:80 orport=443 id=88487BDD980BF6E72092EE690E8C51C0AA4A538C
+176.10.104.243:8080 orport=8443 id=95DA61AEF23A6C851028C1AA88AD8593F659E60F
+
+# Email sent directly to teor, verified using relay contact info
+107.170.101.39:9030 orport=443 id=30973217E70AF00EBE51797FF6D9AA720A902EAA
+
+# Email sent directly to teor, verified using relay contact info
+192.99.212.139:80 orport=443 id=F10BDE279AE71515DDCCCC61DC19AC8765F8A3CC
diff --git a/scripts/maint/updateFallbackDirs.py b/scripts/maint/updateFallbackDirs.py
index d110335de4..d27c1449ee 100755
--- a/scripts/maint/updateFallbackDirs.py
+++ b/scripts/maint/updateFallbackDirs.py
@@ -1,9 +1,13 @@
#!/usr/bin/python
# Usage: scripts/maint/updateFallbackDirs.py > src/or/fallback_dirs.inc
+# Needs stem available in your PYTHONPATH, or just ln -s ../stem/stem .
+# Optionally uses ipaddress (python 3 builtin) or py2-ipaddress (package)
+# for netblock analysis, in PYTHONPATH, or just
+# ln -s ../py2-ipaddress-3.4.1/ipaddress.py .
#
-# Then read the generated list to ensure no-one slipped anything funny into
-# their name or contactinfo
+# Then read the logs to make sure the fallbacks aren't dominated by a single
+# netblock or port
# Script by weasel, April 2015
# Portions by gsathya & karsten, 2013
@@ -25,20 +29,55 @@ import hashlib
import dateutil.parser
# bson_lazy provides bson
#from bson import json_util
+import copy
from stem.descriptor.remote import DescriptorDownloader
import logging
-logging.basicConfig(level=logging.DEBUG)
+# INFO tells you why each relay was included or excluded
+# WARN tells you about potential misconfigurations
+logging.basicConfig(level=logging.WARNING)
+logging.root.name = ''
+# INFO tells you about each consensus download attempt
+logging.getLogger('stem').setLevel(logging.WARNING)
+
+HAVE_IPADDRESS = False
+try:
+ # python 3 builtin, or install package py2-ipaddress
+ # there are several ipaddress implementations for python 2
+ # with slightly different semantics with str typed text
+ # fortunately, all our IP addresses are in unicode
+ import ipaddress
+ HAVE_IPADDRESS = True
+except ImportError:
+ # if this happens, we avoid doing netblock analysis
+ logging.warning('Unable to import ipaddress, please install py2-ipaddress')
## Top-Level Configuration
+# Output all candidate fallbacks, or only output selected fallbacks?
+OUTPUT_CANDIDATES = False
+
+# Perform DirPort checks over IPv4?
+# Change this to False if IPv4 doesn't work for you, or if you don't want to
+# download a consensus for each fallback
+# Don't check ~1000 candidates when OUTPUT_CANDIDATES is True
+PERFORM_IPV4_DIRPORT_CHECKS = False if OUTPUT_CANDIDATES else True
+
# Perform DirPort checks over IPv6?
# If you know IPv6 works for you, set this to True
-PERFORM_IPV6_DIRPORT_CHECKS = False
+# This will exclude IPv6 relays without an IPv6 DirPort configured
+# So it's best left at False until #18394 is implemented
+# Don't check ~1000 candidates when OUTPUT_CANDIDATES is True
+PERFORM_IPV6_DIRPORT_CHECKS = False if OUTPUT_CANDIDATES else False
-# Output all candidate fallbacks, or only output selected fallbacks?
-OUTPUT_CANDIDATES = False
+# Output fallback name, flags, and ContactInfo in a C comment?
+OUTPUT_COMMENTS = True if OUTPUT_CANDIDATES else False
+
+# Output matching ContactInfo in fallbacks list or the blacklist?
+# Useful if you're trying to contact operators
+CONTACT_COUNT = True if OUTPUT_CANDIDATES else False
+CONTACT_BLACKLIST_COUNT = True if OUTPUT_CANDIDATES else False
## OnionOO Settings
@@ -81,7 +120,7 @@ MAX_LIST_FILE_SIZE = 1024 * 1024
# Reduced due to a bug in tor where a relay submits a 0 DirPort when restarted
# This causes OnionOO to (correctly) reset its stability timer
-# This issue is fixed in 0.2.7.7 and master.
+# This issue will be fixed in 0.2.7.7 and 0.2.8.2
# Until then, the CUTOFFs below ensure a decent level of stability.
ADDRESS_AND_PORT_STABLE_DAYS = 7
# What time-weighted-fraction of these flags must FallbackDirs
@@ -94,6 +133,42 @@ CUTOFF_GUARD = .95
# .00 means no bad exits
PERMITTED_BADEXIT = .00
+# older entries' weights are adjusted with ALPHA^(age in days)
+AGE_ALPHA = 0.99
+
+# this factor is used to scale OnionOO entries to [0,1]
+ONIONOO_SCALE_ONE = 999.
+
+## Fallback Count Limits
+
+# The target for these parameters is 20% of the guards in the network
+# This is around 200 as of October 2015
+_FB_POG = 0.2
+FALLBACK_PROPORTION_OF_GUARDS = None if OUTPUT_CANDIDATES else _FB_POG
+
+# We want exactly 100 fallbacks for the initial release
+# This gives us scope to add extra fallbacks to the list as needed
+# Limit the number of fallbacks (eliminating lowest by advertised bandwidth)
+MAX_FALLBACK_COUNT = None if OUTPUT_CANDIDATES else 100
+# Emit a C #error if the number of fallbacks is below
+MIN_FALLBACK_COUNT = 100
+
+## Fallback Bandwidth Requirements
+
+# Any fallback with the Exit flag has its bandwidth multipled by this fraction
+# to make sure we aren't further overloading exits
+# (Set to 1.0, because we asked that only lightly loaded exits opt-in,
+# and the extra load really isn't that much for large relays.)
+EXIT_BANDWIDTH_FRACTION = 1.0
+
+# If a single fallback's bandwidth is too low, it's pointless adding it
+# We expect fallbacks to handle an extra 30 kilobytes per second of traffic
+# Make sure they can support a hundred times the expected extra load
+# (Use 102.4 to make it come out nicely in MB/s)
+# We convert this to a consensus weight before applying the filter,
+# because all the bandwidth amounts are specified by the relay
+MIN_BANDWIDTH = 102.4 * 30.0 * 1024.0
+
# Clients will time out after 30 seconds trying to download a consensus
# So allow fallback directories half that to deliver a consensus
# The exact download times might change based on the network connection
@@ -104,51 +179,20 @@ CONSENSUS_DOWNLOAD_SPEED_MAX = 15.0
# This avoids delisting a relay due to transient network conditions
CONSENSUS_DOWNLOAD_RETRY = True
-## List Length Limits
-
-# The target for these parameters is 20% of the guards in the network
-# This is around 200 as of October 2015
-FALLBACK_PROPORTION_OF_GUARDS = None if OUTPUT_CANDIDATES else 0.2
-
-# Limit the number of fallbacks (eliminating lowest by weight)
-MAX_FALLBACK_COUNT = None if OUTPUT_CANDIDATES else 500
-# Emit a C #error if the number of fallbacks is below
-MIN_FALLBACK_COUNT = 50
-
-## Fallback Weight Settings
-
-# Any fallback with the Exit flag has its weight multipled by this fraction
-EXIT_WEIGHT_FRACTION = 1.0
-
-# If True, emit a C #error if we can't satisfy various constraints
-# If False, emit a C comment instead
-STRICT_FALLBACK_WEIGHTS = False
-
-# Limit the proportional weight
-# If a single fallback's weight is too high, it will see too many clients
-# We reweight using a lower threshold to provide some leeway for:
-# * elimination of low weight relays
-# * consensus weight changes
-# * fallback directory losses over time
-# A relay weighted at 1 in 10 fallbacks will see about 10% of clients that
-# use the fallback directories. (The 9 directory authorities see a similar
-# proportion of clients.)
-TARGET_MAX_WEIGHT_FRACTION = 1/10.0
-REWEIGHTING_FUDGE_FACTOR = 0.8
-MAX_WEIGHT_FRACTION = TARGET_MAX_WEIGHT_FRACTION * REWEIGHTING_FUDGE_FACTOR
-# If a single fallback's weight is too low, it's pointless adding it.
-# (Final weights may be slightly higher than this, due to low weight relays
-# being excluded.)
-# A relay weighted at 1 in 1000 fallbacks will see about 0.1% of clients.
-MIN_WEIGHT_FRACTION = 0.0 if OUTPUT_CANDIDATES else 1/1000.0
-
-## Other Configuration Parameters
-
-# older entries' weights are adjusted with ALPHA^(age in days)
-AGE_ALPHA = 0.99
-
-# this factor is used to scale OnionOO entries to [0,1]
-ONIONOO_SCALE_ONE = 999.
+## Fallback Weights for Client Selection
+
+# All fallback weights are equal, and set to the value below
+# Authorities are weighted 1.0 by default
+# Clients use these weights to select fallbacks and authorities at random
+# If there are 100 fallbacks and 9 authorities:
+# - each fallback is chosen with probability 10.0/(10.0*100 + 1.0*9) ~= 0.99%
+# - each authority is chosen with probability 1.0/(10.0*100 + 1.0*9) ~= 0.09%
+# A client choosing a bootstrap directory server will choose a fallback for
+# 10.0/(10.0*100 + 1.0*9) * 100 = 99.1% of attempts, and an authority for
+# 1.0/(10.0*100 + 1.0*9) * 9 = 0.9% of attempts.
+# (This disregards the bootstrap schedules, where clients start by choosing
+# from fallbacks & authoritites, then later choose from only authorities.)
+FALLBACK_OUTPUT_WEIGHT = 10.0
## Parsing Functions
@@ -157,36 +201,52 @@ def parse_ts(t):
def remove_bad_chars(raw_string, bad_char_list):
# Remove each character in the bad_char_list
- escaped_string = raw_string
+ cleansed_string = raw_string
for c in bad_char_list:
- escaped_string = escaped_string.replace(c, '')
- return escaped_string
+ cleansed_string = cleansed_string.replace(c, '')
+ return cleansed_string
+
+def cleanse_unprintable(raw_string):
+ # Remove all unprintable characters
+ cleansed_string = ''
+ for c in raw_string:
+ if (c in string.ascii_letters or c in string.digits
+ or c in string.punctuation or c in string.whitespace):
+ cleansed_string += c
+ return cleansed_string
def cleanse_whitespace(raw_string):
# Replace all whitespace characters with a space
- escaped_string = raw_string
+ cleansed_string = raw_string
for c in string.whitespace:
- escaped_string = escaped_string.replace(c, ' ')
- return escaped_string
+ cleansed_string = cleansed_string.replace(c, ' ')
+ return cleansed_string
def cleanse_c_multiline_comment(raw_string):
+ cleansed_string = raw_string
+ # Embedded newlines should be removed by tor/onionoo, but let's be paranoid
+ cleansed_string = cleanse_whitespace(cleansed_string)
+ # ContactInfo and Version can be arbitrary binary data
+ cleansed_string = cleanse_unprintable(cleansed_string)
# Prevent a malicious / unanticipated string from breaking out
# of a C-style multiline comment
- # This removes '/*' and '*/'
- # To deal with '//', the end comment must be on its own line
- bad_char_list = '*'
+ # This removes '/*' and '*/' and '//'
+ bad_char_list = '*/'
# Prevent a malicious string from using C nulls
bad_char_list += '\0'
# Be safer by removing bad characters entirely
- escaped_string = remove_bad_chars(raw_string, bad_char_list)
- # Embedded newlines should be removed by tor/onionoo, but let's be paranoid
- escaped_string = cleanse_whitespace(escaped_string)
+ cleansed_string = remove_bad_chars(cleansed_string, bad_char_list)
# Some compilers may further process the content of comments
# There isn't much we can do to cover every possible case
# But comment-based directives are typically only advisory
- return escaped_string
+ return cleansed_string
def cleanse_c_string(raw_string):
+ cleansed_string = raw_string
+ # Embedded newlines should be removed by tor/onionoo, but let's be paranoid
+ cleansed_string = cleanse_whitespace(cleansed_string)
+ # ContactInfo and Version can be arbitrary binary data
+ cleansed_string = cleanse_unprintable(cleansed_string)
# Prevent a malicious address/fingerprint string from breaking out
# of a C-style string
bad_char_list = '"'
@@ -195,13 +255,11 @@ def cleanse_c_string(raw_string):
# Prevent a malicious string from using C nulls
bad_char_list += '\0'
# Be safer by removing bad characters entirely
- escaped_string = remove_bad_chars(raw_string, bad_char_list)
- # Embedded newlines should be removed by tor/onionoo, but let's be paranoid
- escaped_string = cleanse_whitespace(escaped_string)
+ cleansed_string = remove_bad_chars(cleansed_string, bad_char_list)
# Some compilers may further process the content of strings
# There isn't much we can do to cover every possible case
# But this typically only results in changes to the string data
- return escaped_string
+ return cleansed_string
## OnionOO Source Functions
@@ -244,11 +302,11 @@ def write_to_file(str, file_name, max_len):
with open(file_name, 'w') as f:
f.write(str[0:max_len])
except EnvironmentError, error:
- logging.debug('Writing file %s failed: %d: %s'%
- (file_name,
- error.errno,
- error.strerror)
- )
+ logging.warning('Writing file %s failed: %d: %s'%
+ (file_name,
+ error.errno,
+ error.strerror)
+ )
def read_from_file(file_name, max_len):
try:
@@ -256,11 +314,11 @@ def read_from_file(file_name, max_len):
with open(file_name, 'r') as f:
return f.read(max_len)
except EnvironmentError, error:
- logging.debug('Loading file %s failed: %d: %s'%
- (file_name,
- error.errno,
- error.strerror)
- )
+ logging.info('Loading file %s failed: %d: %s'%
+ (file_name,
+ error.errno,
+ error.strerror)
+ )
return None
def load_possibly_compressed_response_json(response):
@@ -423,6 +481,14 @@ class Candidate(object):
details['contact'] = None
if not 'flags' in details or details['flags'] is None:
details['flags'] = []
+ if (not 'advertised_bandwidth' in details
+ or details['advertised_bandwidth'] is None):
+ # relays without advertised bandwdith have it calculated from their
+ # consensus weight
+ details['advertised_bandwidth'] = 0
+ if (not 'effective_family' in details
+ or details['effective_family'] is None):
+ details['effective_family'] = []
details['last_changed_address_or_port'] = parse_ts(
details['last_changed_address_or_port'])
self._data = details
@@ -435,14 +501,8 @@ class Candidate(object):
if self.orport is None:
raise Exception("Failed to get an orport for %s."%(self._fpr,))
self._compute_ipv6addr()
- if self.ipv6addr is None:
+ if not self.has_ipv6():
logging.debug("Failed to get an ipv6 address for %s."%(self._fpr,))
- # Reduce the weight of exits to EXIT_WEIGHT_FRACTION * consensus_weight
- if self.is_exit():
- current_weight = self._data['consensus_weight']
- exit_weight = current_weight * EXIT_WEIGHT_FRACTION
- self._data['original_consensus_weight'] = current_weight
- self._data['consensus_weight'] = exit_weight
def _stable_sort_or_addresses(self):
# replace self._data['or_addresses'] with a stable ordering,
@@ -545,14 +605,14 @@ class Candidate(object):
(ipaddr, port) = i.rsplit(':', 1)
if (port == self.orport) and Candidate.is_valid_ipv6_address(ipaddr):
self.ipv6addr = ipaddr
- self.ipv6orport = port
+ self.ipv6orport = int(port)
return
# Choose the first IPv6 address in the list
for i in self._data['or_addresses']:
(ipaddr, port) = i.rsplit(':', 1)
if Candidate.is_valid_ipv6_address(ipaddr):
self.ipv6addr = ipaddr
- self.ipv6orport = port
+ self.ipv6orport = int(port)
return
@staticmethod
@@ -699,30 +759,41 @@ class Candidate(object):
self._badexit = self._avg_generic_history(badexit) / ONIONOO_SCALE_ONE
def is_candidate(self):
+ must_be_running_now = (PERFORM_IPV4_DIRPORT_CHECKS
+ or PERFORM_IPV6_DIRPORT_CHECKS)
+ if (must_be_running_now and not self.is_running()):
+ logging.info('%s not a candidate: not running now, unable to check ' +
+ 'DirPort consensus download', self._fpr)
+ return False
if (self._data['last_changed_address_or_port'] >
self.CUTOFF_ADDRESS_AND_PORT_STABLE):
- logging.debug('%s not a candidate: changed address/port recently (%s)',
- self._fpr, self._data['last_changed_address_or_port'])
+ logging.info('%s not a candidate: changed address/port recently (%s)',
+ self._fpr, self._data['last_changed_address_or_port'])
return False
if self._running < CUTOFF_RUNNING:
- logging.debug('%s not a candidate: running avg too low (%lf)',
- self._fpr, self._running)
+ logging.info('%s not a candidate: running avg too low (%lf)',
+ self._fpr, self._running)
return False
if self._v2dir < CUTOFF_V2DIR:
- logging.debug('%s not a candidate: v2dir avg too low (%lf)',
- self._fpr, self._v2dir)
+ logging.info('%s not a candidate: v2dir avg too low (%lf)',
+ self._fpr, self._v2dir)
return False
if self._badexit is not None and self._badexit > PERMITTED_BADEXIT:
- logging.debug('%s not a candidate: badexit avg too high (%lf)',
- self._fpr, self._badexit)
+ logging.info('%s not a candidate: badexit avg too high (%lf)',
+ self._fpr, self._badexit)
return False
# if the relay doesn't report a version, also exclude the relay
if (not self._data.has_key('recommended_version')
or not self._data['recommended_version']):
+ logging.info('%s not a candidate: version not recommended', self._fpr)
return False
if self._guard < CUTOFF_GUARD:
- logging.debug('%s not a candidate: guard avg too low (%lf)',
- self._fpr, self._guard)
+ logging.info('%s not a candidate: guard avg too low (%lf)',
+ self._fpr, self._guard)
+ return False
+ if (not self._data.has_key('consensus_weight')
+ or self._data['consensus_weight'] < 1):
+ logging.info('%s not a candidate: consensus weight invalid', self._fpr)
return False
return True
@@ -735,25 +806,50 @@ class Candidate(object):
ipv6 address and port (if present)
If the fallback has an ipv6 key, the whitelist line must also have
it, and vice versa, otherwise they don't match. """
+ ipv6 = None
+ if self.has_ipv6():
+ ipv6 = '%s:%d'%(self.ipv6addr, self.ipv6orport)
for entry in relaylist:
+ if entry['id'] != self._fpr:
+ # can't log here unless we match an IP and port, because every relay's
+ # fingerprint is compared to every entry's fingerprint
+ if entry['ipv4'] == self.dirip and int(entry['orport']) == self.orport:
+ logging.warning('%s excluded: has OR %s:%d changed fingerprint to ' +
+ '%s?', entry['id'], self.dirip, self.orport,
+ self._fpr)
+ if self.has_ipv6() and entry.has_key('ipv6') and entry['ipv6'] == ipv6:
+ logging.warning('%s excluded: has OR %s changed fingerprint to ' +
+ '%s?', entry['id'], ipv6, self._fpr)
+ continue
if entry['ipv4'] != self.dirip:
+ logging.warning('%s excluded: has it changed IPv4 from %s to %s?',
+ self._fpr, entry['ipv4'], self.dirip)
continue
if int(entry['dirport']) != self.dirport:
+ logging.warning('%s excluded: has it changed DirPort from %s:%d to ' +
+ '%s:%d?', self._fpr, self.dirip, int(entry['dirport']),
+ self.dirip, self.dirport)
continue
if int(entry['orport']) != self.orport:
+ logging.warning('%s excluded: has it changed ORPort from %s:%d to ' +
+ '%s:%d?', self._fpr, self.dirip, int(entry['orport']),
+ self.dirip, self.orport)
continue
- if entry['id'] != self._fpr:
- continue
- if (entry.has_key('ipv6')
- and self.ipv6addr is not None and self.ipv6orport is not None):
+ if entry.has_key('ipv6') and self.has_ipv6():
# if both entry and fallback have an ipv6 address, compare them
- if entry['ipv6'] != self.ipv6addr + ':' + self.ipv6orport:
+ if entry['ipv6'] != ipv6:
+ logging.warning('%s excluded: has it changed IPv6 ORPort from %s ' +
+ 'to %s?', self._fpr, entry['ipv6'], ipv6)
continue
# if the fallback has an IPv6 address but the whitelist entry
# doesn't, or vice versa, the whitelist entry doesn't match
- elif entry.has_key('ipv6') and self.ipv6addr is None:
+ elif entry.has_key('ipv6') and not self.has_ipv6():
+ logging.warning('%s excluded: has it lost its former IPv6 address %s?',
+ self._fpr, entry['ipv6'])
continue
- elif not entry.has_key('ipv6') and self.ipv6addr is not None:
+ elif not entry.has_key('ipv6') and self.has_ipv6():
+ logging.warning('%s excluded: has it gained an IPv6 address %s?',
+ self._fpr, ipv6)
continue
return True
return False
@@ -773,118 +869,283 @@ class Candidate(object):
for entry in relaylist:
for key in entry:
value = entry[key]
+ if key == 'id' and value == self._fpr:
+ logging.info('%s is in the blacklist: fingerprint matches',
+ self._fpr)
+ return True
if key == 'ipv4' and value == self.dirip:
# if the dirport is present, check it too
if entry.has_key('dirport'):
if int(entry['dirport']) == self.dirport:
+ logging.info('%s is in the blacklist: IPv4 (%s) and ' +
+ 'DirPort (%d) match', self._fpr, self.dirip,
+ self.dirport)
return True
# if the orport is present, check it too
elif entry.has_key('orport'):
if int(entry['orport']) == self.orport:
+ logging.info('%s is in the blacklist: IPv4 (%s) and ' +
+ 'ORPort (%d) match', self._fpr, self.dirip,
+ self.orport)
return True
else:
+ logging.info('%s is in the blacklist: IPv4 (%s) matches, and ' +
+ 'entry has no DirPort or ORPort', self._fpr,
+ self.dirip)
return True
- if key == 'id' and value == self._fpr:
- return True
- if (key == 'ipv6'
- and self.ipv6addr is not None and self.ipv6orport is not None):
+ ipv6 = None
+ if self.has_ipv6():
+ ipv6 = '%s:%d'%(self.ipv6addr, self.ipv6orport)
+ if (key == 'ipv6' and self.has_ipv6()):
# if both entry and fallback have an ipv6 address, compare them,
# otherwise, disregard ipv6 addresses
- if value == self.ipv6addr + ':' + self.ipv6orport:
+ if value == ipv6:
# if the dirport is present, check it too
if entry.has_key('dirport'):
if int(entry['dirport']) == self.dirport:
+ logging.info('%s is in the blacklist: IPv6 (%s) and ' +
+ 'DirPort (%d) match', self._fpr, ipv6,
+ self.dirport)
return True
- # if the orport is present, check it too
- elif entry.has_key('orport'):
- if int(entry['orport']) == self.orport:
- return True
+ # we've already checked the ORPort, it's part of entry['ipv6']
else:
+ logging.info('%s is in the blacklist: IPv6 (%s) matches, and' +
+ 'entry has no DirPort', self._fpr, ipv6)
return True
+ elif (key == 'ipv6' or self.has_ipv6()):
+ # only log if the fingerprint matches but the IPv6 doesn't
+ if entry.has_key('id') and entry['id'] == self._fpr:
+ logging.info('%s skipping IPv6 blacklist comparison: relay ' +
+ 'has%s IPv6%s, but entry has%s IPv6%s', self._fpr,
+ '' if self.has_ipv6() else ' no',
+ (' (' + ipv6 + ')') if self.has_ipv6() else '',
+ '' if key == 'ipv6' else ' no',
+ (' (' + value + ')') if key == 'ipv6' else '')
+ logging.warning('Has %s %s IPv6 address %s?', self._fpr,
+ 'gained an' if self.has_ipv6() else 'lost its former',
+ ipv6 if self.has_ipv6() else value)
return False
+ def cw_to_bw_factor(self):
+ # any relays with a missing or zero consensus weight are not candidates
+ # any relays with a missing advertised bandwidth have it set to zero
+ return self._data['advertised_bandwidth'] / self._data['consensus_weight']
+
+ # since advertised_bandwidth is reported by the relay, it can be gamed
+ # to avoid this, use the median consensus weight to bandwidth factor to
+ # estimate this relay's measured bandwidth, and make that the upper limit
+ def measured_bandwidth(self, median_cw_to_bw_factor):
+ cw_to_bw= median_cw_to_bw_factor
+ # Reduce exit bandwidth to make sure we're not overloading them
+ if self.is_exit():
+ cw_to_bw *= EXIT_BANDWIDTH_FRACTION
+ measured_bandwidth = self._data['consensus_weight'] * cw_to_bw
+ if self._data['advertised_bandwidth'] != 0:
+ # limit advertised bandwidth (if available) to measured bandwidth
+ return min(measured_bandwidth, self._data['advertised_bandwidth'])
+ else:
+ return measured_bandwidth
+
+ def set_measured_bandwidth(self, median_cw_to_bw_factor):
+ self._data['measured_bandwidth'] = self.measured_bandwidth(
+ median_cw_to_bw_factor)
+
def is_exit(self):
return 'Exit' in self._data['flags']
def is_guard(self):
return 'Guard' in self._data['flags']
- def fallback_weight_fraction(self, total_weight):
- return float(self._data['consensus_weight']) / total_weight
+ def is_running(self):
+ return 'Running' in self._data['flags']
- # return the original consensus weight, if it exists,
- # or, if not, return the consensus weight
- def original_consensus_weight(self):
- if self._data.has_key('original_consensus_weight'):
- return self._data['original_consensus_weight']
- else:
- return self._data['consensus_weight']
+ # does this fallback have an IPv6 address and orport?
+ def has_ipv6(self):
+ return self.ipv6addr is not None and self.ipv6orport is not None
- def original_fallback_weight_fraction(self, total_weight):
- return float(self.original_consensus_weight()) / total_weight
+ # strip leading and trailing brackets from an IPv6 address
+ # safe to use on non-bracketed IPv6 and on IPv4 addresses
+ # also convert to unicode, and make None appear as ''
+ @staticmethod
+ def strip_ipv6_brackets(ip):
+ if ip is None:
+ return unicode('')
+ if len(ip) < 2:
+ return unicode(ip)
+ if ip[0] == '[' and ip[-1] == ']':
+ return unicode(ip[1:-1])
+ return unicode(ip)
+
+ # are ip_a and ip_b in the same netblock?
+ # mask_bits is the size of the netblock
+ # takes both IPv4 and IPv6 addresses
+ # the versions of ip_a and ip_b must be the same
+ # the mask must be valid for the IP version
+ @staticmethod
+ def netblocks_equal(ip_a, ip_b, mask_bits):
+ if ip_a is None or ip_b is None:
+ return False
+ ip_a = Candidate.strip_ipv6_brackets(ip_a)
+ ip_b = Candidate.strip_ipv6_brackets(ip_b)
+ a = ipaddress.ip_address(ip_a)
+ b = ipaddress.ip_address(ip_b)
+ if a.version != b.version:
+ raise Exception('Mismatching IP versions in %s and %s'%(ip_a, ip_b))
+ if mask_bits > a.max_prefixlen:
+ logging.warning('Bad IP mask %d for %s and %s'%(mask_bits, ip_a, ip_b))
+ mask_bits = a.max_prefixlen
+ if mask_bits < 0:
+ logging.warning('Bad IP mask %d for %s and %s'%(mask_bits, ip_a, ip_b))
+ mask_bits = 0
+ a_net = ipaddress.ip_network('%s/%d'%(ip_a, mask_bits), strict=False)
+ return b in a_net
+
+ # is this fallback's IPv4 address (dirip) in the same netblock as other's
+ # IPv4 address?
+ # mask_bits is the size of the netblock
+ def ipv4_netblocks_equal(self, other, mask_bits):
+ return Candidate.netblocks_equal(self.dirip, other.dirip, mask_bits)
+
+ # is this fallback's IPv6 address (ipv6addr) in the same netblock as
+ # other's IPv6 address?
+ # Returns False if either fallback has no IPv6 address
+ # mask_bits is the size of the netblock
+ def ipv6_netblocks_equal(self, other, mask_bits):
+ if not self.has_ipv6() or not other.has_ipv6():
+ return False
+ return Candidate.netblocks_equal(self.ipv6addr, other.ipv6addr, mask_bits)
+
+ # is this fallback's IPv4 DirPort the same as other's IPv4 DirPort?
+ def dirport_equal(self, other):
+ return self.dirport == other.dirport
+ # is this fallback's IPv4 ORPort the same as other's IPv4 ORPort?
+ def ipv4_orport_equal(self, other):
+ return self.orport == other.orport
+
+ # is this fallback's IPv6 ORPort the same as other's IPv6 ORPort?
+ # Returns False if either fallback has no IPv6 address
+ def ipv6_orport_equal(self, other):
+ if not self.has_ipv6() or not other.has_ipv6():
+ return False
+ return self.ipv6orport == other.ipv6orport
+
+ # does this fallback have the same DirPort, IPv4 ORPort, or
+ # IPv6 ORPort as other?
+ # Ignores IPv6 ORPort if either fallback has no IPv6 address
+ def port_equal(self, other):
+ return (self.dirport_equal(other) or self.ipv4_orport_equal(other)
+ or self.ipv6_orport_equal(other))
+
+ # return a list containing IPv4 ORPort, DirPort, and IPv6 ORPort (if present)
+ def port_list(self):
+ ports = [self.dirport, self.orport]
+ if self.has_ipv6() and not self.ipv6orport in ports:
+ ports.append(self.ipv6orport)
+ return ports
+
+ # does this fallback share a port with other, regardless of whether the
+ # port types match?
+ # For example, if self's IPv4 ORPort is 80 and other's DirPort is 80,
+ # return True
+ def port_shared(self, other):
+ for p in self.port_list():
+ if p in other.port_list():
+ return True
+ return False
+
+ # report how long it takes to download a consensus from dirip:dirport
@staticmethod
- def fallback_consensus_dl_speed(dirip, dirport, nickname, max_time):
+ def fallback_consensus_download_speed(dirip, dirport, nickname, max_time):
+ download_failed = False
downloader = DescriptorDownloader()
start = datetime.datetime.utcnow()
+ # some directory mirrors respond to requests in ways that hang python
+ # sockets, which is why we long this line here
+ logging.info('Initiating consensus download from %s (%s:%d).', nickname,
+ dirip, dirport)
# there appears to be about 1 second of overhead when comparing stem's
# internal trace time and the elapsed time calculated here
- downloader.get_consensus(endpoints = [(dirip, dirport)]).run()
+ TIMEOUT_SLOP = 1.0
+ try:
+ downloader.get_consensus(endpoints = [(dirip, dirport)],
+ timeout = (max_time + TIMEOUT_SLOP),
+ validate = True,
+ retries = 0,
+ fall_back_to_authority = False).run()
+ except Exception, stem_error:
+ logging.debug('Unable to retrieve a consensus from %s: %s', nickname,
+ stem_error)
+ status = 'error: "%s"' % (stem_error)
+ level = logging.WARNING
+ download_failed = True
elapsed = (datetime.datetime.utcnow() - start).total_seconds()
if elapsed > max_time:
status = 'too slow'
+ level = logging.WARNING
+ download_failed = True
else:
status = 'ok'
- logging.debug(('Consensus download: %0.2fs %s from %s (%s:%d), '
- + 'max download time %0.2fs.') % (elapsed, status,
- nickname, dirip, dirport,
- max_time))
- return elapsed
-
- def fallback_consensus_dl_check(self):
- ipv4_speed = Candidate.fallback_consensus_dl_speed(self.dirip,
+ level = logging.DEBUG
+ logging.log(level, 'Consensus download: %0.1fs %s from %s (%s:%d), ' +
+ 'max download time %0.1fs.', elapsed, status, nickname,
+ dirip, dirport, max_time)
+ return download_failed
+
+ # does this fallback download the consensus fast enough?
+ def check_fallback_download_consensus(self):
+ # include the relay if we're not doing a check, or we can't check (IPv6)
+ ipv4_failed = False
+ ipv6_failed = False
+ if PERFORM_IPV4_DIRPORT_CHECKS:
+ ipv4_failed = Candidate.fallback_consensus_download_speed(self.dirip,
self.dirport,
self._data['nickname'],
CONSENSUS_DOWNLOAD_SPEED_MAX)
- if self.ipv6addr is not None and PERFORM_IPV6_DIRPORT_CHECKS:
+ if self.has_ipv6() and PERFORM_IPV6_DIRPORT_CHECKS:
# Clients assume the IPv6 DirPort is the same as the IPv4 DirPort
- ipv6_speed = Candidate.fallback_consensus_dl_speed(self.ipv6addr,
- self.dirport,
- self._data['nickname'],
- CONSENSUS_DOWNLOAD_SPEED_MAX)
- else:
- ipv6_speed = None
- # Now retry the relay if it took too long the first time
- if (ipv4_speed > CONSENSUS_DOWNLOAD_SPEED_MAX
- and CONSENSUS_DOWNLOAD_RETRY):
- ipv4_speed = Candidate.fallback_consensus_dl_speed(self.dirip,
+ ipv6_failed = Candidate.fallback_consensus_download_speed(self.ipv6addr,
self.dirport,
self._data['nickname'],
CONSENSUS_DOWNLOAD_SPEED_MAX)
- if (self.ipv6addr is not None and PERFORM_IPV6_DIRPORT_CHECKS
- and ipv6_speed > CONSENSUS_DOWNLOAD_SPEED_MAX
- and CONSENSUS_DOWNLOAD_RETRY):
- ipv6_speed = Candidate.fallback_consensus_dl_speed(self.ipv6addr,
- self.dirport,
- self._data['nickname'],
- CONSENSUS_DOWNLOAD_SPEED_MAX)
-
- return (ipv4_speed <= CONSENSUS_DOWNLOAD_SPEED_MAX
- and (not PERFORM_IPV6_DIRPORT_CHECKS
- or ipv6_speed <= CONSENSUS_DOWNLOAD_SPEED_MAX))
+ return ((not ipv4_failed) and (not ipv6_failed))
+
+ # if this fallback has not passed a download check, try it again,
+ # and record the result, available in get_fallback_download_consensus
+ def try_fallback_download_consensus(self):
+ if not self.get_fallback_download_consensus():
+ self._data['download_check'] = self.check_fallback_download_consensus()
+
+ # did this fallback pass the download check?
+ def get_fallback_download_consensus(self):
+ # if we're not performing checks, return True
+ if not PERFORM_IPV4_DIRPORT_CHECKS and not PERFORM_IPV6_DIRPORT_CHECKS:
+ return True
+ # if we are performing checks, but haven't done one, return False
+ if not self._data.has_key('download_check'):
+ return False
+ return self._data['download_check']
+
+ # output an optional header comment and info for this fallback
+ # try_fallback_download_consensus before calling this
+ def fallbackdir_line(self, fallbacks, prefilter_fallbacks):
+ s = ''
+ if OUTPUT_COMMENTS:
+ s += self.fallbackdir_comment(fallbacks, prefilter_fallbacks)
+ # if the download speed is ok, output a C string
+ # if it's not, but we OUTPUT_COMMENTS, output a commented-out C string
+ if self.get_fallback_download_consensus() or OUTPUT_COMMENTS:
+ s += self.fallbackdir_info(self.get_fallback_download_consensus())
+ return s
- def fallbackdir_line(self, total_weight, original_total_weight, dl_speed_ok):
+ # output a header comment for this fallback
+ def fallbackdir_comment(self, fallbacks, prefilter_fallbacks):
# /*
# nickname
# flags
- # weight / total (percentage)
- # [original weight / original total (original percentage)]
# [contact]
+ # [identical contact counts]
# */
- # "address:dirport orport=port id=fingerprint"
- # "[ipv6=addr:orport]"
- # "weight=num",
- #
# Multiline C comment
s = '/*'
s += '\n'
@@ -893,27 +1154,47 @@ class Candidate(object):
s += 'Flags: '
s += cleanse_c_multiline_comment(' '.join(sorted(self._data['flags'])))
s += '\n'
- weight = self._data['consensus_weight']
- percent_weight = self.fallback_weight_fraction(total_weight)*100
- s += 'Fallback Weight: %d / %d (%.3f%%)'%(weight, total_weight,
- percent_weight)
- s += '\n'
- o_weight = self.original_consensus_weight()
- if o_weight != weight:
- o_percent_weight = self.original_fallback_weight_fraction(
- original_total_weight)*100
- s += 'Consensus Weight: %d / %d (%.3f%%)'%(o_weight,
- original_total_weight,
- o_percent_weight)
- s += '\n'
if self._data['contact'] is not None:
s += cleanse_c_multiline_comment(self._data['contact'])
+ if CONTACT_COUNT or CONTACT_BLACKLIST_COUNT:
+ fallback_count = len([f for f in fallbacks
+ if f._data['contact'] == self._data['contact']])
+ if fallback_count > 1:
+ s += '\n'
+ s += '%d identical contacts listed' % (fallback_count)
+ if CONTACT_BLACKLIST_COUNT:
+ prefilter_count = len([f for f in prefilter_fallbacks
+ if f._data['contact'] == self._data['contact']])
+ filter_count = prefilter_count - fallback_count
+ if filter_count > 0:
+ if fallback_count > 1:
+ s += ' '
+ else:
+ s += '\n'
+ s += '%d blacklisted' % (filter_count)
s += '\n'
s += '*/'
s += '\n'
+
+ # output the fallback info C string for this fallback
+ # this is the text that would go after FallbackDir in a torrc
+ # if this relay failed the download test and we OUTPUT_COMMENTS,
+ # comment-out the returned string
+ def fallbackdir_info(self, dl_speed_ok):
+ # "address:dirport orport=port id=fingerprint"
+ # "[ipv6=addr:orport]"
+ # "weight=FALLBACK_OUTPUT_WEIGHT",
+ #
+ # Do we want a C string, or a commented-out string?
+ c_string = dl_speed_ok
+ comment_string = not dl_speed_ok and OUTPUT_COMMENTS
+ # If we don't want either kind of string, bail
+ if not c_string and not comment_string:
+ return ''
+ s = ''
# Comment out the fallback directory entry if it's too slow
# See the debug output for which address and port is failing
- if not dl_speed_ok:
+ if comment_string:
s += '/* Consensus download failed or was too slow:\n'
# Multi-Line C string with trailing comma (part of a string list)
# This makes it easier to diff the file, and remove IPv6 lines using grep
@@ -923,12 +1204,11 @@ class Candidate(object):
self.orport,
cleanse_c_string(self._fpr))
s += '\n'
- if self.ipv6addr is not None:
- s += '" ipv6=%s:%s"'%(
- cleanse_c_string(self.ipv6addr), cleanse_c_string(self.ipv6orport))
+ if self.has_ipv6():
+ s += '" ipv6=%s:%d"'%(cleanse_c_string(self.ipv6addr), self.ipv6orport)
s += '\n'
- s += '" weight=%d",'%(weight)
- if not dl_speed_ok:
+ s += '" weight=%d",'%(FALLBACK_OUTPUT_WEIGHT)
+ if comment_string:
s += '\n'
s += '*/'
return s
@@ -962,8 +1242,8 @@ class CandidateList(dict):
logging.debug('Loading details document.')
d = fetch('details',
fields=('fingerprint,nickname,contact,last_changed_address_or_port,' +
- 'consensus_weight,or_addresses,dir_address,' +
- 'recommended_version,flags'))
+ 'consensus_weight,advertised_bandwidth,or_addresses,' +
+ 'dir_address,recommended_version,flags,effective_family'))
logging.debug('Loading details document done.')
if not 'relays' in d: raise Exception("No relays found in document.")
@@ -989,15 +1269,30 @@ class CandidateList(dict):
guard_count += 1
return guard_count
- # Find fallbacks that fit the uptime, stability, and flags criteria
+ # Find fallbacks that fit the uptime, stability, and flags criteria,
+ # and make an array of them in self.fallbacks
def compute_fallbacks(self):
self.fallbacks = map(lambda x: self[x],
- sorted(
- filter(lambda x: self[x].is_candidate(),
- self.keys()),
- key=lambda x: self[x]._data['consensus_weight'],
+ filter(lambda x: self[x].is_candidate(),
+ self.keys()))
+
+ # sort fallbacks by their consensus weight to advertised bandwidth factor,
+ # lowest to highest
+ # used to find the median cw_to_bw_factor()
+ def sort_fallbacks_by_cw_to_bw_factor(self):
+ self.fallbacks.sort(key=lambda f: f.cw_to_bw_factor())
+
+ # sort fallbacks by their measured bandwidth, highest to lowest
+ # calculate_measured_bandwidth before calling this
+ # this is useful for reviewing candidates in priority order
+ def sort_fallbacks_by_measured_bandwidth(self):
+ self.fallbacks.sort(key=lambda f: f._data['measured_bandwidth'],
reverse=True)
- )
+
+ # sort fallbacks by their fingerprint, lowest to highest
+ # this is useful for stable diffs of fallback lists
+ def sort_fallbacks_by_fingerprint(self):
+ self.fallbacks.sort(key=lambda f: f._fpr)
@staticmethod
def load_relaylist(file_name):
@@ -1071,8 +1366,8 @@ class CandidateList(dict):
if BLACKLIST_EXCLUDES_WHITELIST_ENTRIES:
# exclude
excluded_count += 1
- logging.debug('Excluding %s: in both blacklist and whitelist.' %
- f._fpr)
+ logging.warning('Excluding %s: in both blacklist and whitelist.',
+ f._fpr)
else:
# include
filtered_fallbacks.append(f)
@@ -1082,8 +1377,7 @@ class CandidateList(dict):
elif in_blacklist:
# exclude
excluded_count += 1
- logging.debug('Excluding %s: in blacklist.' %
- f._fpr)
+ logging.debug('Excluding %s: in blacklist.', f._fpr)
else:
if INCLUDE_UNLISTED_ENTRIES:
# include
@@ -1091,8 +1385,8 @@ class CandidateList(dict):
else:
# exclude
excluded_count += 1
- logging.debug('Excluding %s: in neither blacklist nor whitelist.' %
- f._fpr)
+ logging.info('Excluding %s: in neither blacklist nor whitelist.',
+ f._fpr)
self.fallbacks = filtered_fallbacks
return excluded_count
@@ -1101,147 +1395,480 @@ class CandidateList(dict):
return '/* Whitelist & blacklist excluded %d of %d candidates. */'%(
excluded_count, initial_count)
- # Remove any fallbacks in excess of MAX_FALLBACK_COUNT,
- # starting with the lowest-weighted fallbacks
- # total_weight should be recalculated after calling this
- def exclude_excess_fallbacks(self):
- if MAX_FALLBACK_COUNT is not None:
- self.fallbacks = self.fallbacks[:MAX_FALLBACK_COUNT]
-
- # Clamp the weight of all fallbacks to MAX_WEIGHT_FRACTION * total_weight
- # fallbacks are kept sorted, but since excessive weights are reduced to
- # the maximum acceptable weight, these relays end up with equal weights
- def clamp_high_weight_fallbacks(self, total_weight):
- if MAX_WEIGHT_FRACTION * len(self.fallbacks) < 1.0:
- error_str = 'Max Fallback Weight %.3f%% is unachievable'%(
- MAX_WEIGHT_FRACTION)
- error_str += ' with Current Fallback Count %d.'%(len(self.fallbacks))
- if STRICT_FALLBACK_WEIGHTS:
- print '#error ' + error_str
- else:
- print '/* ' + error_str + ' */'
- relays_clamped = 0
- max_acceptable_weight = total_weight * MAX_WEIGHT_FRACTION
+ # calculate each fallback's measured bandwidth based on the median
+ # consensus weight to advertised bandwdith ratio
+ def calculate_measured_bandwidth(self):
+ self.sort_fallbacks_by_cw_to_bw_factor()
+ median_fallback = self.fallback_median(True)
+ median_cw_to_bw_factor = median_fallback.cw_to_bw_factor()
for f in self.fallbacks:
- frac_weight = f.fallback_weight_fraction(total_weight)
- if frac_weight > MAX_WEIGHT_FRACTION:
- relays_clamped += 1
- current_weight = f._data['consensus_weight']
- # if we already have an original weight, keep it
- if (not f._data.has_key('original_consensus_weight')
- or f._data['original_consensus_weight'] == current_weight):
- f._data['original_consensus_weight'] = current_weight
- f._data['consensus_weight'] = max_acceptable_weight
- return relays_clamped
-
- # Remove any fallbacks with weights lower than MIN_WEIGHT_FRACTION
- # total_weight should be recalculated after calling this
- def exclude_low_weight_fallbacks(self, total_weight):
- self.fallbacks = filter(
- lambda x:
- x.fallback_weight_fraction(total_weight) >= MIN_WEIGHT_FRACTION,
- self.fallbacks)
-
- def fallback_weight_total(self):
- return sum(f._data['consensus_weight'] for f in self.fallbacks)
-
- def fallback_min_weight(self):
+ f.set_measured_bandwidth(median_cw_to_bw_factor)
+
+ # remove relays with low measured bandwidth from the fallback list
+ # calculate_measured_bandwidth for each relay before calling this
+ def remove_low_bandwidth_relays(self):
+ if MIN_BANDWIDTH is None:
+ return
+ above_min_bw_fallbacks = []
+ for f in self.fallbacks:
+ if f._data['measured_bandwidth'] >= MIN_BANDWIDTH:
+ above_min_bw_fallbacks.append(f)
+ else:
+ # the bandwidth we log here is limited by the relay's consensus weight
+ # as well as its adverttised bandwidth. See set_measured_bandwidth
+ # for details
+ logging.info('%s not a candidate: bandwidth %.1fMB/s too low, must ' +
+ 'be at least %.1fMB/s', f._fpr,
+ f._data['measured_bandwidth']/(1024.0*1024.0),
+ MIN_BANDWIDTH/(1024.0*1024.0))
+ self.fallbacks = above_min_bw_fallbacks
+
+ # the minimum fallback in the list
+ # call one of the sort_fallbacks_* functions before calling this
+ def fallback_min(self):
if len(self.fallbacks) > 0:
return self.fallbacks[-1]
else:
return None
- def fallback_max_weight(self):
+ # the median fallback in the list
+ # call one of the sort_fallbacks_* functions before calling this
+ def fallback_median(self, require_advertised_bandwidth):
+ # use the low-median when there are an evan number of fallbacks,
+ # for consistency with the bandwidth authorities
+ if len(self.fallbacks) > 0:
+ median_position = (len(self.fallbacks) - 1) / 2
+ if not require_advertised_bandwidth:
+ return self.fallbacks[median_position]
+ # if we need advertised_bandwidth but this relay doesn't have it,
+ # move to a fallback with greater consensus weight until we find one
+ while not self.fallbacks[median_position]._data['advertised_bandwidth']:
+ median_position += 1
+ if median_position >= len(self.fallbacks):
+ return None
+ return self.fallbacks[median_position]
+ else:
+ return None
+
+ # the maximum fallback in the list
+ # call one of the sort_fallbacks_* functions before calling this
+ def fallback_max(self):
if len(self.fallbacks) > 0:
return self.fallbacks[0]
else:
return None
- def summarise_fallbacks(self, eligible_count, eligible_weight,
- relays_clamped, clamped_weight,
- guard_count, target_count, max_count):
+ # does exclusion_list contain attribute?
+ # if so, return False
+ # if not, return True
+ # if attribute is None or the empty string, always return True
+ @staticmethod
+ def allow(attribute, exclusion_list):
+ if attribute is None or attribute == '':
+ return True
+ elif attribute in exclusion_list:
+ return False
+ else:
+ return True
+
+ # make sure there is only one fallback per IPv4 address, and per IPv6 address
+ # there is only one IPv4 address on each fallback: the IPv4 DirPort address
+ # (we choose the IPv4 ORPort which is on the same IPv4 as the DirPort)
+ # there is at most one IPv6 address on each fallback: the IPv6 ORPort address
+ # we try to match the IPv4 ORPort, but will use any IPv6 address if needed
+ # (clients assume the IPv6 DirPort is the same as the IPv4 DirPort, but
+ # typically only use the IPv6 ORPort)
+ # if there is no IPv6 address, only the IPv4 address is checked
+ # return the number of candidates we excluded
+ def limit_fallbacks_same_ip(self):
+ ip_limit_fallbacks = []
+ ip_list = []
+ for f in self.fallbacks:
+ if (CandidateList.allow(f.dirip, ip_list)
+ and CandidateList.allow(f.ipv6addr, ip_list)):
+ ip_limit_fallbacks.append(f)
+ ip_list.append(f.dirip)
+ if f.has_ipv6():
+ ip_list.append(f.ipv6addr)
+ elif not CandidateList.allow(f.dirip, ip_list):
+ logging.debug('Eliminated %s: already have fallback on IPv4 %s'%(
+ f._fpr, f.dirip))
+ elif f.has_ipv6() and not CandidateList.allow(f.ipv6addr, ip_list):
+ logging.debug('Eliminated %s: already have fallback on IPv6 %s'%(
+ f._fpr, f.ipv6addr))
+ original_count = len(self.fallbacks)
+ self.fallbacks = ip_limit_fallbacks
+ return original_count - len(self.fallbacks)
+
+ # make sure there is only one fallback per ContactInfo
+ # if there is no ContactInfo, allow the fallback
+ # this check can be gamed by providing no ContactInfo, or by setting the
+ # ContactInfo to match another fallback
+ # However, given the likelihood that relays with the same ContactInfo will
+ # go down at similar times, its usefulness outweighs the risk
+ def limit_fallbacks_same_contact(self):
+ contact_limit_fallbacks = []
+ contact_list = []
+ for f in self.fallbacks:
+ if CandidateList.allow(f._data['contact'], contact_list):
+ contact_limit_fallbacks.append(f)
+ contact_list.append(f._data['contact'])
+ else:
+ logging.debug(('Eliminated %s: already have fallback on ' +
+ 'ContactInfo %s')%(f._fpr, f._data['contact']))
+ original_count = len(self.fallbacks)
+ self.fallbacks = contact_limit_fallbacks
+ return original_count - len(self.fallbacks)
+
+ # make sure there is only one fallback per effective family
+ # if there is no family, allow the fallback
+ # this check can't be gamed, because we use effective family, which ensures
+ # mutual family declarations
+ # if any indirect families exist, the result depends on the order in which
+ # fallbacks are sorted in the list
+ def limit_fallbacks_same_family(self):
+ family_limit_fallbacks = []
+ fingerprint_list = []
+ for f in self.fallbacks:
+ if CandidateList.allow(f._fpr, fingerprint_list):
+ family_limit_fallbacks.append(f)
+ fingerprint_list.append(f._fpr)
+ fingerprint_list.extend(f._data['effective_family'])
+ else:
+ # technically, we already have a fallback with this fallback in its
+ # effective family
+ logging.debug('Eliminated %s: already have fallback in effective ' +
+ 'family'%(f._fpr))
+ original_count = len(self.fallbacks)
+ self.fallbacks = family_limit_fallbacks
+ return original_count - len(self.fallbacks)
+
+ # try a download check on each fallback candidate in order
+ # stop after max_count successful downloads
+ # but don't remove any candidates from the array
+ def try_download_consensus_checks(self, max_count):
+ dl_ok_count = 0
+ for f in self.fallbacks:
+ f.try_fallback_download_consensus()
+ if f.get_fallback_download_consensus():
+ # this fallback downloaded a consensus ok
+ dl_ok_count += 1
+ if dl_ok_count >= max_count:
+ # we have enough fallbacks
+ return
+
+ # put max_count successful candidates in the fallbacks array:
+ # - perform download checks on each fallback candidate
+ # - retry failed candidates if CONSENSUS_DOWNLOAD_RETRY is set
+ # - eliminate failed candidates
+ # - if there are more than max_count candidates, eliminate lowest bandwidth
+ # - if there are fewer than max_count candidates, leave only successful
+ # Return the number of fallbacks that failed the consensus check
+ def perform_download_consensus_checks(self, max_count):
+ self.sort_fallbacks_by_measured_bandwidth()
+ self.try_download_consensus_checks(max_count)
+ if CONSENSUS_DOWNLOAD_RETRY:
+ # try unsuccessful candidates again
+ # we could end up with more than max_count successful candidates here
+ self.try_download_consensus_checks(max_count)
+ # now we have at least max_count successful candidates,
+ # or we've tried them all
+ original_count = len(self.fallbacks)
+ self.fallbacks = filter(lambda x: x.get_fallback_download_consensus(),
+ self.fallbacks)
+ # some of these failed the check, others skipped the check,
+ # if we already had enough successful downloads
+ failed_count = original_count - len(self.fallbacks)
+ self.fallbacks = self.fallbacks[:max_count]
+ return failed_count
+
+ # return a string that describes a/b as a percentage
+ @staticmethod
+ def describe_percentage(a, b):
+ return '%d/%d = %.0f%%'%(a, b, (a*100.0)/b)
+
+ # return a dictionary of lists of fallbacks by IPv4 netblock
+ # the dictionary is keyed by the fingerprint of an arbitrary fallback
+ # in each netblock
+ # mask_bits is the size of the netblock
+ def fallbacks_by_ipv4_netblock(self, mask_bits):
+ netblocks = {}
+ for f in self.fallbacks:
+ found_netblock = False
+ for b in netblocks.keys():
+ # we found an existing netblock containing this fallback
+ if f.ipv4_netblocks_equal(self[b], mask_bits):
+ # add it to the list
+ netblocks[b].append(f)
+ found_netblock = True
+ break
+ # make a new netblock based on this fallback's fingerprint
+ if not found_netblock:
+ netblocks[f._fpr] = [f]
+ return netblocks
+
+ # return a dictionary of lists of fallbacks by IPv6 netblock
+ # where mask_bits is the size of the netblock
+ def fallbacks_by_ipv6_netblock(self, mask_bits):
+ netblocks = {}
+ for f in self.fallbacks:
+ # skip fallbacks without IPv6 addresses
+ if not f.has_ipv6():
+ continue
+ found_netblock = False
+ for b in netblocks.keys():
+ # we found an existing netblock containing this fallback
+ if f.ipv6_netblocks_equal(self[b], mask_bits):
+ # add it to the list
+ netblocks[b].append(f)
+ found_netblock = True
+ break
+ # make a new netblock based on this fallback's fingerprint
+ if not found_netblock:
+ netblocks[f._fpr] = [f]
+ return netblocks
+
+ # log a message about the proportion of fallbacks in each IPv4 netblock,
+ # where mask_bits is the size of the netblock
+ def describe_fallback_ipv4_netblock_mask(self, mask_bits):
+ fallback_count = len(self.fallbacks)
+ shared_netblock_fallback_count = 0
+ most_frequent_netblock = None
+ netblocks = self.fallbacks_by_ipv4_netblock(mask_bits)
+ for b in netblocks.keys():
+ if len(netblocks[b]) > 1:
+ # how many fallbacks are in a netblock with other fallbacks?
+ shared_netblock_fallback_count += len(netblocks[b])
+ # what's the netblock with the most fallbacks?
+ if (most_frequent_netblock is None
+ or len(netblocks[b]) > len(netblocks[most_frequent_netblock])):
+ most_frequent_netblock = b
+ logging.debug('Fallback IPv4 addresses in the same /%d:'%(mask_bits))
+ for f in netblocks[b]:
+ logging.debug('%s - %s', f.dirip, f._fpr)
+ if most_frequent_netblock is not None:
+ logging.warning('There are %s fallbacks in the IPv4 /%d containing %s'%(
+ CandidateList.describe_percentage(
+ len(netblocks[most_frequent_netblock]),
+ fallback_count),
+ mask_bits,
+ self[most_frequent_netblock].dirip))
+ if shared_netblock_fallback_count > 0:
+ logging.warning(('%s of fallbacks are in an IPv4 /%d with other ' +
+ 'fallbacks')%(CandidateList.describe_percentage(
+ shared_netblock_fallback_count,
+ fallback_count),
+ mask_bits))
+
+ # log a message about the proportion of fallbacks in each IPv6 netblock,
+ # where mask_bits is the size of the netblock
+ def describe_fallback_ipv6_netblock_mask(self, mask_bits):
+ fallback_count = len(self.fallbacks_with_ipv6())
+ shared_netblock_fallback_count = 0
+ most_frequent_netblock = None
+ netblocks = self.fallbacks_by_ipv6_netblock(mask_bits)
+ for b in netblocks.keys():
+ if len(netblocks[b]) > 1:
+ # how many fallbacks are in a netblock with other fallbacks?
+ shared_netblock_fallback_count += len(netblocks[b])
+ # what's the netblock with the most fallbacks?
+ if (most_frequent_netblock is None
+ or len(netblocks[b]) > len(netblocks[most_frequent_netblock])):
+ most_frequent_netblock = b
+ logging.debug('Fallback IPv6 addresses in the same /%d:'%(mask_bits))
+ for f in netblocks[b]:
+ logging.debug('%s - %s', f.ipv6addr, f._fpr)
+ if most_frequent_netblock is not None:
+ logging.warning('There are %s fallbacks in the IPv6 /%d containing %s'%(
+ CandidateList.describe_percentage(
+ len(netblocks[most_frequent_netblock]),
+ fallback_count),
+ mask_bits,
+ self[most_frequent_netblock].ipv6addr))
+ if shared_netblock_fallback_count > 0:
+ logging.warning(('%s of fallbacks are in an IPv6 /%d with other ' +
+ 'fallbacks')%(CandidateList.describe_percentage(
+ shared_netblock_fallback_count,
+ fallback_count),
+ mask_bits))
+
+ # log a message about the proportion of fallbacks in each IPv4 /8, /16,
+ # and /24
+ def describe_fallback_ipv4_netblocks(self):
+ # this doesn't actually tell us anything useful
+ #self.describe_fallback_ipv4_netblock_mask(8)
+ self.describe_fallback_ipv4_netblock_mask(16)
+ self.describe_fallback_ipv4_netblock_mask(24)
+
+ # log a message about the proportion of fallbacks in each IPv6 /12 (RIR),
+ # /23 (smaller RIR blocks), /32 (LIR), /48 (Customer), and /64 (Host)
+ # https://www.iana.org/assignments/ipv6-unicast-address-assignments/
+ def describe_fallback_ipv6_netblocks(self):
+ # these don't actually tell us anything useful
+ #self.describe_fallback_ipv6_netblock_mask(12)
+ #self.describe_fallback_ipv6_netblock_mask(23)
+ self.describe_fallback_ipv6_netblock_mask(32)
+ self.describe_fallback_ipv6_netblock_mask(48)
+ self.describe_fallback_ipv6_netblock_mask(64)
+
+ # log a message about the proportion of fallbacks in each IPv4 and IPv6
+ # netblock
+ def describe_fallback_netblocks(self):
+ self.describe_fallback_ipv4_netblocks()
+ self.describe_fallback_ipv6_netblocks()
+
+ # return a list of fallbacks which are on the IPv4 ORPort port
+ def fallbacks_on_ipv4_orport(self, port):
+ return filter(lambda x: x.orport == port, self.fallbacks)
+
+ # return a list of fallbacks which are on the IPv6 ORPort port
+ def fallbacks_on_ipv6_orport(self, port):
+ return filter(lambda x: x.ipv6orport == port, self.fallbacks_with_ipv6())
+
+ # return a list of fallbacks which are on the DirPort port
+ def fallbacks_on_dirport(self, port):
+ return filter(lambda x: x.dirport == port, self.fallbacks)
+
+ # log a message about the proportion of fallbacks on IPv4 ORPort port
+ # and return that count
+ def describe_fallback_ipv4_orport(self, port):
+ port_count = len(self.fallbacks_on_ipv4_orport(port))
+ fallback_count = len(self.fallbacks)
+ logging.warning('%s of fallbacks are on IPv4 ORPort %d'%(
+ CandidateList.describe_percentage(port_count,
+ fallback_count),
+ port))
+ return port_count
+
+ # log a message about the proportion of IPv6 fallbacks on IPv6 ORPort port
+ # and return that count
+ def describe_fallback_ipv6_orport(self, port):
+ port_count = len(self.fallbacks_on_ipv6_orport(port))
+ fallback_count = len(self.fallbacks_with_ipv6())
+ logging.warning('%s of IPv6 fallbacks are on IPv6 ORPort %d'%(
+ CandidateList.describe_percentage(port_count,
+ fallback_count),
+ port))
+ return port_count
+
+ # log a message about the proportion of fallbacks on DirPort port
+ # and return that count
+ def describe_fallback_dirport(self, port):
+ port_count = len(self.fallbacks_on_dirport(port))
+ fallback_count = len(self.fallbacks)
+ logging.warning('%s of fallbacks are on DirPort %d'%(
+ CandidateList.describe_percentage(port_count,
+ fallback_count),
+ port))
+ return port_count
+
+ # log a message about the proportion of fallbacks on each dirport,
+ # each IPv4 orport, and each IPv6 orport
+ def describe_fallback_ports(self):
+ fallback_count = len(self.fallbacks)
+ ipv4_or_count = fallback_count
+ ipv4_or_count -= self.describe_fallback_ipv4_orport(443)
+ ipv4_or_count -= self.describe_fallback_ipv4_orport(9001)
+ logging.warning('%s of fallbacks are on other IPv4 ORPorts'%(
+ CandidateList.describe_percentage(ipv4_or_count,
+ fallback_count)))
+ ipv6_fallback_count = len(self.fallbacks_with_ipv6())
+ ipv6_or_count = ipv6_fallback_count
+ ipv6_or_count -= self.describe_fallback_ipv6_orport(443)
+ ipv6_or_count -= self.describe_fallback_ipv6_orport(9001)
+ logging.warning('%s of IPv6 fallbacks are on other IPv6 ORPorts'%(
+ CandidateList.describe_percentage(ipv6_or_count,
+ ipv6_fallback_count)))
+ dir_count = fallback_count
+ dir_count -= self.describe_fallback_dirport(80)
+ dir_count -= self.describe_fallback_dirport(9030)
+ logging.warning('%s of fallbacks are on other DirPorts'%(
+ CandidateList.describe_percentage(dir_count,
+ fallback_count)))
+
+ # return a list of fallbacks which have the Exit flag
+ def fallbacks_with_exit(self):
+ return filter(lambda x: x.is_exit(), self.fallbacks)
+
+ # log a message about the proportion of fallbacks with an Exit flag
+ def describe_fallback_exit_flag(self):
+ exit_falback_count = len(self.fallbacks_with_exit())
+ fallback_count = len(self.fallbacks)
+ logging.warning('%s of fallbacks have the Exit flag'%(
+ CandidateList.describe_percentage(exit_falback_count,
+ fallback_count)))
+
+ # return a list of fallbacks which have an IPv6 address
+ def fallbacks_with_ipv6(self):
+ return filter(lambda x: x.has_ipv6(), self.fallbacks)
+
+ # log a message about the proportion of fallbacks on IPv6
+ def describe_fallback_ip_family(self):
+ ipv6_falback_count = len(self.fallbacks_with_ipv6())
+ fallback_count = len(self.fallbacks)
+ logging.warning('%s of fallbacks are on IPv6'%(
+ CandidateList.describe_percentage(ipv6_falback_count,
+ fallback_count)))
+
+ def summarise_fallbacks(self, eligible_count, operator_count, failed_count,
+ guard_count, target_count):
# Report:
- # the number of fallback directories (with min & max limits);
- # #error if below minimum count
- # the total weight, min & max fallback proportions
- # #error if outside max weight proportion
- # Multiline C comment with #error if things go bad
- s = '/*'
+ # whether we checked consensus download times
+ # the number of fallback directories (and limits/exclusions, if relevant)
+ # min & max fallback bandwidths
+ # #error if below minimum count
+ if PERFORM_IPV4_DIRPORT_CHECKS or PERFORM_IPV6_DIRPORT_CHECKS:
+ s = '/* Checked %s%s%s DirPorts served a consensus within %.1fs. */'%(
+ 'IPv4' if PERFORM_IPV4_DIRPORT_CHECKS else '',
+ ' and ' if (PERFORM_IPV4_DIRPORT_CHECKS
+ and PERFORM_IPV6_DIRPORT_CHECKS) else '',
+ 'IPv6' if PERFORM_IPV6_DIRPORT_CHECKS else '',
+ CONSENSUS_DOWNLOAD_SPEED_MAX)
+ else:
+ s = '/* Did not check IPv4 or IPv6 DirPort consensus downloads. */'
s += '\n'
- s += 'Fallback Directory Summary'
+ # Multiline C comment with #error if things go bad
+ s += '/*'
s += '\n'
# Integers don't need escaping in C comments
fallback_count = len(self.fallbacks)
if FALLBACK_PROPORTION_OF_GUARDS is None:
fallback_proportion = ''
else:
- fallback_proportion = ' (%d * %f)'%(guard_count,
- FALLBACK_PROPORTION_OF_GUARDS)
- s += 'Final Count: %d (Eligible %d, Usable %d, Target %d%s'%(
- min(max_count, fallback_count),
- eligible_count,
- fallback_count,
- target_count,
- fallback_proportion)
+ fallback_proportion = ', Target %d (%d * %.2f)'%(target_count,
+ guard_count,
+ FALLBACK_PROPORTION_OF_GUARDS)
+ s += 'Final Count: %d (Eligible %d%s'%(fallback_count, eligible_count,
+ fallback_proportion)
if MAX_FALLBACK_COUNT is not None:
- s += ', Clamped to %d'%(MAX_FALLBACK_COUNT)
+ s += ', Max %d'%(MAX_FALLBACK_COUNT)
s += ')\n'
- if fallback_count < MIN_FALLBACK_COUNT:
- s += '*/'
+ if eligible_count != fallback_count:
+ removed_count = eligible_count - fallback_count
+ excess_to_target_or_max = (eligible_count - operator_count - failed_count
+ - fallback_count)
+ # some 'Failed' failed the check, others 'Skipped' the check,
+ # if we already had enough successful downloads
+ s += ('Excluded: %d (Same Operator %d, Failed/Skipped Download %d, ' +
+ 'Excess %d)')%(removed_count, operator_count, failed_count,
+ excess_to_target_or_max)
s += '\n'
+ min_fb = self.fallback_min()
+ min_bw = min_fb._data['measured_bandwidth']
+ max_fb = self.fallback_max()
+ max_bw = max_fb._data['measured_bandwidth']
+ s += 'Bandwidth Range: %.1f - %.1f MB/s'%(min_bw/(1024.0*1024.0),
+ max_bw/(1024.0*1024.0))
+ s += '\n'
+ s += '*/'
+ if fallback_count < MIN_FALLBACK_COUNT:
# We must have a minimum number of fallbacks so they are always
# reachable, and are in diverse locations
+ s += '\n'
s += '#error Fallback Count %d is too low. '%(fallback_count)
s += 'Must be at least %d for diversity. '%(MIN_FALLBACK_COUNT)
s += 'Try adding entries to the whitelist, '
s += 'or setting INCLUDE_UNLISTED_ENTRIES = True.'
- s += '\n'
- s += '/*'
- s += '\n'
- total_weight = self.fallback_weight_total()
- min_fb = self.fallback_min_weight()
- min_weight = min_fb._data['consensus_weight']
- min_percent = min_fb.fallback_weight_fraction(total_weight)*100.0
- max_fb = self.fallback_max_weight()
- max_weight = max_fb._data['consensus_weight']
- max_frac = max_fb.fallback_weight_fraction(total_weight)
- max_percent = max_frac*100.0
- s += 'Final Weight: %d (Eligible %d)'%(total_weight, eligible_weight)
- s += '\n'
- s += 'Max Weight: %d (%.3f%%) (Clamped to %.3f%%)'%(
- max_weight,
- max_percent,
- TARGET_MAX_WEIGHT_FRACTION*100)
- s += '\n'
- s += 'Min Weight: %d (%.3f%%) (Clamped to %.3f%%)'%(
- min_weight,
- min_percent,
- MIN_WEIGHT_FRACTION*100)
- s += '\n'
- if eligible_count != fallback_count:
- s += 'Excluded: %d (Clamped, Below Target, or Low Weight)'%(
- eligible_count - fallback_count)
- s += '\n'
- if relays_clamped > 0:
- s += 'Clamped: %d (%.3f%%) Excess Weight, '%(
- clamped_weight,
- (100.0 * clamped_weight) / total_weight)
- s += '%d High Weight Fallbacks (%.1f%%)'%(
- relays_clamped,
- (100.0 * relays_clamped) / fallback_count)
- s += '\n'
- s += '*/'
- if max_frac > TARGET_MAX_WEIGHT_FRACTION:
- s += '\n'
- # We must restrict the maximum fallback weight, so an adversary
- # at or near the fallback doesn't see too many clients
- error_str = 'Max Fallback Weight %.3f%% is too high. '%(max_frac*100)
- error_str += 'Must be at most %.3f%% for client anonymity.'%(
- TARGET_MAX_WEIGHT_FRACTION*100)
- if STRICT_FALLBACK_WEIGHTS:
- s += '#error ' + error_str
- else:
- s += '/* ' + error_str + ' */'
return s
## Main Function
@@ -1250,9 +1877,11 @@ def list_fallbacks():
""" Fetches required onionoo documents and evaluates the
fallback directory criteria for each of the relays """
+ # find relays that could be fallbacks
candidates = CandidateList()
candidates.add_relays()
+ # work out how many fallbacks we want
guard_count = candidates.count_guards()
if FALLBACK_PROPORTION_OF_GUARDS is None:
target_count = guard_count
@@ -1262,76 +1891,79 @@ def list_fallbacks():
# - the target fallback count (FALLBACK_PROPORTION_OF_GUARDS * guard count)
# - the maximum fallback count (MAX_FALLBACK_COUNT)
if MAX_FALLBACK_COUNT is None:
- max_count = guard_count
+ max_count = target_count
else:
max_count = min(target_count, MAX_FALLBACK_COUNT)
candidates.compute_fallbacks()
+ prefilter_fallbacks = copy.copy(candidates.fallbacks)
+ # filter with the whitelist and blacklist
initial_count = len(candidates.fallbacks)
excluded_count = candidates.apply_filter_lists()
print candidates.summarise_filters(initial_count, excluded_count)
-
eligible_count = len(candidates.fallbacks)
- eligible_weight = candidates.fallback_weight_total()
+
+ # calculate the measured bandwidth of each relay,
+ # then remove low-bandwidth relays
+ candidates.calculate_measured_bandwidth()
+ candidates.remove_low_bandwidth_relays()
# print the raw fallback list
- #total_weight = candidates.fallback_weight_total()
#for x in candidates.fallbacks:
- # print x.fallbackdir_line(total_weight, total_weight)
-
- # When candidates are excluded, total_weight decreases, and
- # the proportional weight of other candidates increases.
- candidates.exclude_excess_fallbacks()
- total_weight = candidates.fallback_weight_total()
-
- # When candidates are reweighted, total_weight decreases, and
- # the proportional weight of other candidates increases.
- # Previously low-weight candidates might obtain sufficient proportional
- # weights to be included.
- # Save the weight at which we reweighted fallbacks for the summary.
- pre_clamp_total_weight = total_weight
- relays_clamped = candidates.clamp_high_weight_fallbacks(total_weight)
-
- # When candidates are excluded, total_weight decreases, and
- # the proportional weight of other candidates increases.
- # No new low weight candidates will be created during exclusions.
- # However, high weight candidates may increase over the maximum proportion.
- # This should not be an issue, except in pathological cases.
- candidates.exclude_low_weight_fallbacks(total_weight)
- total_weight = candidates.fallback_weight_total()
-
- # check we haven't exceeded TARGET_MAX_WEIGHT_FRACTION
- # since reweighting preserves the orginal sort order,
- # the maximum weights will be at the head of the list
+ # print x.fallbackdir_line(True)
+ # print json.dumps(candidates[x]._data, sort_keys=True, indent=4,
+ # separators=(',', ': '), default=json_util.default)
+
+ # impose mandatory conditions here, like one per contact, family, IP
+ # in measured bandwidth order
+ candidates.sort_fallbacks_by_measured_bandwidth()
+ operator_count = 0
+ # only impose these limits on the final list - operators can nominate
+ # multiple candidate fallbacks, and then we choose the best set
+ if not OUTPUT_CANDIDATES:
+ operator_count += candidates.limit_fallbacks_same_ip()
+ operator_count += candidates.limit_fallbacks_same_contact()
+ operator_count += candidates.limit_fallbacks_same_family()
+
+ # check if each candidate can serve a consensus
+ # there's a small risk we've eliminated relays from the same operator that
+ # can serve a consensus, in favour of one that can't
+ # but given it takes up to 15 seconds to check each consensus download,
+ # the risk is worth it
+ failed_count = candidates.perform_download_consensus_checks(max_count)
+
+ # analyse and log interesting diversity metrics
+ # like netblock, ports, exit, IPv4-only
+ # (we can't easily analyse AS, and it's hard to accurately analyse country)
+ candidates.describe_fallback_ip_family()
+ # if we can't import the ipaddress module, we can't do netblock analysis
+ if HAVE_IPADDRESS:
+ candidates.describe_fallback_netblocks()
+ candidates.describe_fallback_ports()
+ candidates.describe_fallback_exit_flag()
+
+ # output C comments summarising the fallback selection process
if len(candidates.fallbacks) > 0:
- max_weight_fb = candidates.fallback_max_weight()
- max_weight = max_weight_fb.fallback_weight_fraction(total_weight)
- if max_weight > TARGET_MAX_WEIGHT_FRACTION:
- error_str = 'Maximum fallback weight: %.3f%% exceeds target %.3f%%. '%(
- max_weight*100.0,
- TARGET_MAX_WEIGHT_FRACTION*100.0)
- error_str += 'Try decreasing REWEIGHTING_FUDGE_FACTOR.'
- if STRICT_FALLBACK_WEIGHTS:
- print '#error ' + error_str
- else:
- print '/* ' + error_str + ' */'
-
- print candidates.summarise_fallbacks(eligible_count, eligible_weight,
- relays_clamped,
- pre_clamp_total_weight - total_weight,
- guard_count, target_count, max_count)
+ print candidates.summarise_fallbacks(eligible_count, operator_count,
+ failed_count, guard_count,
+ target_count)
else:
print '/* No Fallbacks met criteria */'
+ # output C comments specifying the OnionOO data used to create the list
for s in fetch_source_list():
print describe_fetch_source(s)
- for x in candidates.fallbacks[:max_count]:
- dl_speed_ok = x.fallback_consensus_dl_check()
- print x.fallbackdir_line(total_weight, pre_clamp_total_weight, dl_speed_ok)
- #print json.dumps(candidates[x]._data, sort_keys=True, indent=4,
- # separators=(',', ': '), default=json_util.default)
+ # if we're outputting the final fallback list, sort by fingerprint
+ # this makes diffs much more stable
+ # otherwise, leave sorted by bandwidth, which allows operators to be
+ # contacted in priority order
+ if not OUTPUT_CANDIDATES:
+ candidates.sort_fallbacks_by_fingerprint()
+
+ for x in candidates.fallbacks:
+ print x.fallbackdir_line(candidates.fallbacks, prefilter_fallbacks)
if __name__ == "__main__":
list_fallbacks()
7apu2bq3rhoylwmucxizk54afw5jyda7pho4j5zdcqpwkufke7zdzwad.onion