Merge remote-tracking branch 'teor/fallback-inputs'

5 files changed, 56 insertions, 21 deletions

diff --git a/changes/bug20913 b/changes/bug20913
new file mode 100644
index 0000000000..5086edcdba
--- /dev/null
+++ b/changes/bug20913
@@ -0,0 +1,9 @@
+  o Minor bugfixes (fallbacks):
+    - Make sure fallback directory mirrors have the same address, port, and
+      relay identity key for at least 30 days before they are selected.
+      Fixes 20913, bugfix on 0.2.8.1-alpha.
+    - Decrease the guard flag average required to be a fallback. This allows
+      us to keep relays that have their guard flag removed when they restart.
+      Fixes 20913, bugfix on 0.2.8.1-alpha.
+    - Decrease the minimum number of fallbacks to 100.
+      Fixes 20913, bugfix on 0.2.8.1-alpha.
diff --git a/changes/bug21121 b/changes/bug21121
new file mode 100644
index 0000000000..b74332c44c
--- /dev/null
+++ b/changes/bug21121
@@ -0,0 +1,3 @@
+  o Minor enhancements (fallbacks):
+    - Update the fallback directory mirror whitelist and blacklist based on
+      operator emails. Closes task 21121.
diff --git a/scripts/maint/fallback.blacklist b/scripts/maint/fallback.blacklist
index 974b304729..1417a13a98 100644
--- a/scripts/maint/fallback.blacklist
+++ b/scripts/maint/fallback.blacklist
@@ -252,3 +252,23 @@ id=9C8A123081EFBE022EF795630F447839DDFDDDEC
 
 # Email sent directly to teor, verified using relay contact info
 163.172.35.245:80 orport=443 id=B771AA877687F88E6F1CA5354756DF6C8A7B6B24
+
+# Email sent directly to teor, verified using relay contact info
+104.243.35.196:9030 orport=9001 id=FA3415659444AE006E7E9E5375E82F29700CFDFD
+
+# Relay changed IPv4 address, operator uncontactable
+138.201.130.32:9030 orport=9001 id=52AEA31188331F421B2EDB494DB65CD181E5B257
+
+# Emails sent directly to teor, verified using relay contact info
+217.12.199.208:80 orport=443 id=DF3AED4322B1824BF5539AE54B2D1B38E080FF05 ipv6=[2a02:27a8:0:2::7e]:443
+
+# Emails sent directly to teor, verified using relay contact info
+195.154.75.84:9030 orport=9001 id=F80FDE27EFCB3F6A7B4E2CC517133DBFFA78BA2D
+195.154.127.246:9030 orport=9001 id=4FEE77AFFD157BBCF2D896AE417FBF647860466C
+
+# Email sent directly to teor, verified using relay contact info
+5.35.251.247:9030 orport=9001 id=9B1F5187DFBA89DC24B37EA7BF896C12B43A27AE
+
+#​https://lists.torproject.org/pipermail/tor-relays/2017-May/012281.html
+62.210.124.124:9030 orport=9001 id=86E78DD3720C78DA8673182EF96C54B162CD660C ipv6=[2001:bc8:3f23:100::1]:9001
+62.210.124.124:9130 orport=9101 id=2EBD117806EE43C3CC885A8F1E4DC60F207E7D3E ipv6=[2001:bc8:3f23:100::1]:9101
diff --git a/scripts/maint/fallback.whitelist b/scripts/maint/fallback.whitelist
index c993be97d3..0620d6b5fe 100644
--- a/scripts/maint/fallback.whitelist
+++ b/scripts/maint/fallback.whitelist
@@ -37,8 +37,6 @@
 # https://lists.torproject.org/pipermail/tor-relays/2015-December/008370.html
 # https://lists.torproject.org/pipermail/tor-relays/2016-January/008517.html
 # https://lists.torproject.org/pipermail/tor-relays/2016-January/008555.html
-62.210.124.124:9030 orport=9001 id=86E78DD3720C78DA8673182EF96C54B162CD660C ipv6=[2001:bc8:3f23:100::1]:9001
-62.210.124.124:9130 orport=9101 id=2EBD117806EE43C3CC885A8F1E4DC60F207E7D3E ipv6=[2001:bc8:3f23:100::1]:9101
 212.47.237.95:9030 orport=9001 id=3F5D8A879C58961BB45A3D26AC41B543B40236D6
 212.47.237.95:9130 orport=9101 id=6FB38EB22E57EF7ED5EF00238F6A48E553735D88
 
@@ -49,15 +47,13 @@
 # https://lists.torproject.org/pipermail/tor-relays/2015-December/008373.html
 167.114.35.28:9030 orport=9001 id=E65D300F11E1DB12C534B0146BDAB6972F1A8A48
 
-# https://lists.torproject.org/pipermail/tor-relays/2015-December/008374.html
-104.243.35.196:9030 orport=9001 id=FA3415659444AE006E7E9E5375E82F29700CFDFD
-
 # https://lists.torproject.org/pipermail/tor-relays/2015-December/008378.html
 144.76.14.145:110 orport=143 id=14419131033443AE6E21DA82B0D307F7CAE42BDB ipv6=[2a01:4f8:190:9490::dead]:443
 
 # https://lists.torproject.org/pipermail/tor-relays/2015-December/008379.html
 # Email sent directly to teor, verified using relay contact info
 91.121.84.137:4951 orport=4051 id=6DE61A6F72C1E5418A66BFED80DFB63E4C77668F ipv6=[2001:41d0:1:8989::1]:4051
+91.121.84.137:4952 orport=4052 id=9FBEB75E8BC142565F12CBBE078D63310236A334 ipv6=[2001:41d0:1:8989::1]:4052
 
 # https://lists.torproject.org/pipermail/tor-relays/2015-December/008381.html
 # Sent additional email to teor with more relays
@@ -95,9 +91,6 @@
 # https://lists.torproject.org/pipermail/tor-relays/2016-January/008542.html
 178.62.199.226:80 orport=443 id=CBEFF7BA4A4062045133C053F2D70524D8BBE5BE ipv6=[2a03:b0c0:2:d0::b7:5001]:443
 
-# Emails sent directly to teor, verified using relay contact info
-217.12.199.208:80 orport=443 id=DF3AED4322B1824BF5539AE54B2D1B38E080FF05 ipv6=[2a02:27a8:0:2::7e]:443
-
 # Email sent directly to teor, verified using relay contact info
 94.23.204.175:9030 orport=9001 id=5665A3904C89E22E971305EE8C1997BCA4123C69
 
@@ -330,9 +323,6 @@
 37.187.102.186:9030 orport=9001 id=489D94333DF66D57FFE34D9D59CC2D97E2CB0053 ipv6=[2001:41d0:a:26ba::1]:9001
 
 # Email sent directly to teor, verified using relay contact info
-5.35.251.247:9030 orport=9001 id=9B1F5187DFBA89DC24B37EA7BF896C12B43A27AE
-
-# Email sent directly to teor, verified using relay contact info
 198.96.155.3:8080 orport=5001 id=BCEDF6C193AA687AE471B8A22EBF6BC57C2D285E
 
 # Email sent directly to teor, verified using relay contact info
@@ -427,12 +417,10 @@
 46.4.24.161:9030 orport=9001 id=DB4C76A3AD7E234DA0F00D6F1405D8AFDF4D8DED
 46.4.24.161:9031 orport=9002 id=7460F3D12EBE861E4EE073F6233047AACFE46AB4
 46.38.51.132:9030 orport=9001 id=810DEFA7E90B6C6C383C063028EC397A71D7214A
-163.172.194.53:9030 orport=9001 id=8C00FA7369A7A308F6A137600F0FA07990D9D451
+163.172.194.53:9030 orport=9001 id=8C00FA7369A7A308F6A137600F0FA07990D9D451 ipv6=[2001:bc8:225f:142:6c69:7461:7669:73]:9001
 
 # Email sent directly to teor, verified using relay contact info
 176.10.107.180:9030 orport=9001 id=3D7E274A87D9A89AF064C13D1EE4CA1F184F2600
-195.154.75.84:9030 orport=9001 id=F80FDE27EFCB3F6A7B4E2CC517133DBFFA78BA2D
-195.154.127.246:9030 orport=9001 id=4FEE77AFFD157BBCF2D896AE417FBF647860466C
 
 # Email sent directly to teor, verified using relay contact info
 46.28.207.19:80 orport=443 id=5B92FA5C8A49D46D235735504C72DBB3472BA321
@@ -471,7 +459,7 @@
 185.35.202.221:9030 orport=9001 id=C13B91384CDD52A871E3ECECE4EF74A7AC7DCB08 ipv6=[2a02:ed06::221]:9001
 
 # Email sent directly to teor, verified using relay contact info
-5.9.151.241:9030 orport=4223 id=9BF04559224F0F1C3C953D641F1744AF0192543A
+5.9.151.241:9030 orport=4223 id=9BF04559224F0F1C3C953D641F1744AF0192543A ipv6=[2a01:4f8:190:34f0::2]:4223
 
 # Email sent directly to teor, verified using relay contact info
 89.40.71.149:8081 orport=8080 id=EC639EDAA5121B47DBDF3D6B01A22E48A8CB6CC7
@@ -574,6 +562,7 @@
 
 # Email sent directly to teor, verified using relay contact info
 185.100.86.100:80 orport=443 id=0E8C0C8315B66DB5F703804B3889A1DD66C67CE0
+185.100.84.82:80 orport=443 id=7D05A38E39FC5D29AFE6BE487B9B4DC9E635D09E
 
 # Email sent directly to teor, verified using relay contact info
 164.132.77.175:9030 orport=9001 id=3B33F6FCA645AD4E91428A3AF7DC736AD9FB727B
@@ -643,9 +632,6 @@
 46.4.111.124:9030 orport=9001 id=D9065F9E57899B3D272AA212317AF61A9B14D204
 
 # Email sent directly to teor, verified using relay contact info
-138.201.130.32:9030 orport=9001 id=52AEA31188331F421B2EDB494DB65CD181E5B257
-
-# Email sent directly to teor, verified using relay contact info
 185.100.85.61:80 orport=443 id=025B66CEBC070FCB0519D206CF0CF4965C20C96E
 
 # Email sent directly to teor, verified using relay contact info
@@ -828,3 +814,16 @@
 
 # Email sent directly to teor, verified using relay contact info
 95.85.8.226:80 orport=443 id=1211AC1BBB8A1AF7CBA86BCE8689AA3146B86423
+
+# Email sent directly to teor, verified using relay contact info
+85.214.151.72:9030 orport=9001 id=722D365140C8C52DBB3C9FF6986E3CEFFE2BA812
+
+# Email sent directly to teor, verified using relay contact info
+72.52.75.27:9030 orport=9001 id=1220F0F20E80D348244C5F3B6D126DAA0A446DFD
+
+# Email sent directly to teor, verified using relay contact info
+5.9.146.203:80 orport=443 id=1F45542A24A61BF9408F1C05E0DCE4E29F2CBA11
+5.9.159.14:9030 orport=9001 id=0F100F60C7A63BED90216052324D29B08CFCF797
+
+# Email sent directly to teor, verified using relay contact info
+5.9.147.226:9030 orport=9001 id=B0553175AADB0501E5A61FC61CEA3970BE130FF2
diff --git a/scripts/maint/updateFallbackDirs.py b/scripts/maint/updateFallbackDirs.py
index 50a2bd620e..82a60420b4 100755
--- a/scripts/maint/updateFallbackDirs.py
+++ b/scripts/maint/updateFallbackDirs.py
@@ -159,20 +159,24 @@ MAX_LIST_FILE_SIZE = 1024 * 1024
 ## Eligibility Settings
 
 # Require fallbacks to have the same address and port for a set amount of time
+# We used to have this at 1 week, but that caused many fallback failures, which
+# meant that we had to rebuild the list more often.
 #
 # There was a bug in Tor 0.2.8.1-alpha and earlier where a relay temporarily
 # submits a 0 DirPort when restarted.
 # This causes OnionOO to (correctly) reset its stability timer.
 # Affected relays should upgrade to Tor 0.2.8.7 or later, which has a fix
 # for this issue.
-ADDRESS_AND_PORT_STABLE_DAYS = 7
+ADDRESS_AND_PORT_STABLE_DAYS = 30
 # We ignore relays that have been down for more than this period
 MAX_DOWNTIME_DAYS = 0 if MUST_BE_RUNNING_NOW else 7
 # What time-weighted-fraction of these flags must FallbackDirs
 # Equal or Exceed?
 CUTOFF_RUNNING = .90
 CUTOFF_V2DIR = .90
-CUTOFF_GUARD = .90
+# Tolerate lower guard flag averages, as guard flags are removed for some time
+# after a relay restarts
+CUTOFF_GUARD = .80
 # What time-weighted-fraction of these flags must FallbackDirs
 # Equal or Fall Under?
 # .00 means no bad exits
@@ -194,7 +198,7 @@ FALLBACK_PROPORTION_OF_GUARDS = None if OUTPUT_CANDIDATES else _FB_POG
 # Limit the number of fallbacks (eliminating lowest by advertised bandwidth)
 MAX_FALLBACK_COUNT = None if OUTPUT_CANDIDATES else 200
 # Emit a C #error if the number of fallbacks is less than expected
-MIN_FALLBACK_COUNT = 0 if OUTPUT_CANDIDATES else MAX_FALLBACK_COUNT*0.75
+MIN_FALLBACK_COUNT = 0 if OUTPUT_CANDIDATES else MAX_FALLBACK_COUNT*0.5
 
 # The maximum number of fallbacks on the same address, contact, or family
 # With 200 fallbacks, this means each operator can see 1% of client bootstraps