diff options
38 files changed, 1349 insertions, 462 deletions
@@ -1,6 +1,39 @@ Changes in version 0.3.1.1-alpha - 2017-??-?? +Changes in version 0.3.0.7 - 2017-05-15 + Tor 0.3.0.7 fixes a medium-severity security bug in earlier versions + of Tor 0.3.0.x, where an attacker could cause a Tor relay process + to exit. Relays running earlier versions of Tor 0.3.0.x should upgrade; + clients are not affected. + + o Major bugfixes (hidden service directory, security): + - Fix an assertion failure in the hidden service directory code, which + could be used by an attacker to remotely cause a Tor relay process to + exit. Relays running earlier versions of Tor 0.3.0.x should upgrade. + should upgrade. This security issue is tracked as TROVE-2017-002. + Fixes bug 22246; bugfix on 0.3.0.1-alpha. + + o Minor features: + - Update geoip and geoip6 to the May 2 2017 Maxmind GeoLite2 + Country database. + + o Minor features (future-proofing): + - Tor no longer refuses to download microdescriptors or descriptors + if they are listed as "published in the future". This change will + eventually allow us to stop listing meaningful "published" dates + in microdescriptor consensuses, and thereby allow us to reduce the + resources required to download consensus diffs by over 50%. + Implements part of ticket 21642; implements part of proposal 275. + + o Minor bugfixes (Linux seccomp2 sandbox): + - The getpid() system call is now permitted under the Linux seccomp2 + sandbox, to avoid crashing with versions of OpenSSL (and other + libraries) that attempt to learn the process's PID by using the + syscall rather than the VDSO code. Fixes bug 21943; bugfix + on 0.2.5.1-alpha. + + Changes in version 0.3.0.6 - 2017-04-26 Tor 0.3.0.6 is the first stable release of the Tor 0.3.0 series. diff --git a/ReleaseNotes b/ReleaseNotes index ca5a30dd43..e88dc2f81c 100644 --- a/ReleaseNotes +++ b/ReleaseNotes @@ -3,6 +3,39 @@ of Tor. If you want to see more detailed descriptions of the changes in each development snapshot, see the ChangeLog file. +Changes in version 0.3.0.7 - 2017-05-15 + Tor 0.3.0.7 fixes a medium-severity security bug in earlier versions + of Tor 0.3.0.x, where an attacker could cause a Tor relay process + to exit. Relays running earlier versions of Tor 0.3.0.x should upgrade; + clients are not affected. + + o Major bugfixes (hidden service directory, security): + - Fix an assertion failure in the hidden service directory code, which + could be used by an attacker to remotely cause a Tor relay process to + exit. Relays running earlier versions of Tor 0.3.0.x should upgrade. + should upgrade. This security issue is tracked as TROVE-2017-002. + Fixes bug 22246; bugfix on 0.3.0.1-alpha. + + o Minor features: + - Update geoip and geoip6 to the May 2 2017 Maxmind GeoLite2 + Country database. + + o Minor features (future-proofing): + - Tor no longer refuses to download microdescriptors or descriptors + if they are listed as "published in the future". This change will + eventually allow us to stop listing meaningful "published" dates + in microdescriptor consensuses, and thereby allow us to reduce the + resources required to download consensus diffs by over 50%. + Implements part of ticket 21642; implements part of proposal 275. + + o Minor bugfixes (Linux seccomp2 sandbox): + - The getpid() system call is now permitted under the Linux seccomp2 + sandbox, to avoid crashing with versions of OpenSSL (and other + libraries) that attempt to learn the process's PID by using the + syscall rather than the VDSO code. Fixes bug 21943; bugfix + on 0.2.5.1-alpha. + + Changes in version 0.3.0.6 - 2017-04-26 Tor 0.3.0.6 is the first stable release of the Tor 0.3.0 series. diff --git a/changes/bug17592 b/changes/bug17592 index 856e75b622..131791b31f 100644 --- a/changes/bug17592 +++ b/changes/bug17592 @@ -3,7 +3,7 @@ CircuitIdleTimeout and PredictedPortsRelevanceTime into a single option called CircuitsAvailableTimeout. Also, allow the consensus to control the default values for both this preference, as well as the lifespan - of relay-to-relay connections. Fixes bug 17592. + of relay-to-relay connections. Fixes bug 17592; bugfix on 0.2.5.5-alpha. - Increase the intial circuit build timeout testing frequency, to help ensure that ReducedConnectionPadding clients finish learning a timeout before their orconn would expire. The initial testing rate was set back diff --git a/changes/bug17604 b/changes/bug17604 index 0c7b983a78..6cd9e2e50a 100644 --- a/changes/bug17604 +++ b/changes/bug17604 @@ -6,7 +6,8 @@ status early. Also, additionally alter the connection selection logic to prefer the oldest valid connection for extend cells. These two changes should reduce the number of long-term connections - that are kept open between relays. Fixes bug 17604. + that are kept open between relays. Fixes bug 17604; bugfix on + 0.2.5.5-alpha. - Relays will now log hourly statistics on the total number of connections to other relays. If the number of connections per relay unexpectedly large, this log message is at notice level. Otherwise diff --git a/changes/bug19699 b/changes/bug19699 index 8bfe66210e..10ba57f73e 100644 --- a/changes/bug19699 +++ b/changes/bug19699 @@ -2,4 +2,4 @@ - Make test-network.sh always call chutney's test-network.sh. Previously, this only worked on systems which had bash installed, due to some bash-specific code in the script. - Fixes bug 19699; follow-up to 21581. + Fixes bug 19699; bugfix on 0.3.0.4-rc. Follow-up to ticket 21581. diff --git a/changes/bug20341 b/changes/bug20341 index 399ee509e6..419240c3f0 100644 --- a/changes/bug20341 +++ b/changes/bug20341 @@ -1,3 +1,3 @@ o Documentation: - Document key=value pluggable transport arguments for Bridge lines in - torrc. Fixes bug 20341; feature first appeared in 0.2.5.1-alpha. + torrc. Fixes bug 20341; bugfix on 0.2.5.1-alpha. diff --git a/changes/bug20913 b/changes/bug20913 new file mode 100644 index 0000000000..df7f106759 --- /dev/null +++ b/changes/bug20913 @@ -0,0 +1,9 @@ + o Minor bugfixes (fallbacks): + - Make sure fallback directory mirrors have the same address, port, and + relay identity key for at least 30 days before they are selected. + Fixes bug 20913; bugfix on 0.2.8.1-alpha. + - Decrease the guard flag average required to be a fallback. This allows + us to keep relays that have their guard flag removed when they restart. + Fixes bug 20913; bugfix on 0.2.8.1-alpha. + - Decrease the minimum number of fallbacks to 100. + Fixes bug 20913; bugfix on 0.2.8.1-alpha. diff --git a/changes/bug21121 b/changes/bug21121 new file mode 100644 index 0000000000..96b973028e --- /dev/null +++ b/changes/bug21121 @@ -0,0 +1,3 @@ + o Minor features (fallback directories): + - Update the fallback directory mirror whitelist and blacklist based on + operator emails. Closes task 21121. diff --git a/changes/bug21155 b/changes/bug21155 index 33c0cd10d2..67e4a64d14 100644 --- a/changes/bug21155 +++ b/changes/bug21155 @@ -1,5 +1,5 @@ - o Minor bugfixes (hidden service, logging): + o Minor features (hidden service, logging): - Warn user if multiple entries in EntryNodes and at least one HiddenService are used together. Pinning EntryNodes along with an hidden service can be possibly harmful for instance see ticket 14917 or 21155. - Fixes bug 21155. + Closes ticket 21155. diff --git a/changes/bug21654 b/changes/bug21654 index 6b225877b2..c3badd2bbf 100644 --- a/changes/bug21654 +++ b/changes/bug21654 @@ -1,4 +1,4 @@ o Minor bugfixes (testing): - Use unbuffered I/O for utility functions around the process_handle_t type. This fixes unit test failures reported on OpenBSD and FreeBSD. - Fixes bug 21654. + Fixes bug 21654; bugfix on 0.2.3.1-alpha. diff --git a/changes/bug21715 b/changes/bug21715 index f8b5ae23a7..54ad1ad246 100644 --- a/changes/bug21715 +++ b/changes/bug21715 @@ -2,4 +2,4 @@ - Default of NumEntryGuards is 1 if the consensus parameter guard-n-primary-guards-to-use isn't set. Default of NumDirectoryGuards is 3 if the consensus parameter guard-n-primary-dir-guards-to-use isn't - set. Fixes bug 21715. + set. Fixes bug 21715; bugfix on 0.3.0.1-alpha. diff --git a/changes/bug22042 b/changes/bug22042 index c1188df478..dccf83da8e 100644 --- a/changes/bug22042 +++ b/changes/bug22042 @@ -2,4 +2,6 @@ - Trigger HS descriptor events on the control port when the client is unable to pick a suitable hidden service directory. This can happen if they are all in the ExcludeNodes list or they all have been queried - inside the allowed 15 minutes. Fixes bug 22042. + inside the allowed 15 minutes. Fixes bug 22042; bugfix on + 0.2.5.2-alpha. + diff --git a/changes/bug22246 b/changes/bug22246 new file mode 100644 index 0000000000..dbdf31a433 --- /dev/null +++ b/changes/bug22246 @@ -0,0 +1,6 @@ + o Major bugfixes (hidden service directory, security): + - Fix an assertion failure in the hidden service directory code, which + could be used by an attacker to remotely cause a Tor relay process to + exit. Relays running earlier versions of Tor 0.3.0.x should upgrade. + This security issue is tracked as tracked as + TROVE-2017-002. Fixes bug 22246; bugfix on 0.3.0.1-alpha. diff --git a/changes/bug22252 b/changes/bug22252 new file mode 100644 index 0000000000..42b9d8e095 --- /dev/null +++ b/changes/bug22252 @@ -0,0 +1,3 @@ + o Minor bugfixes (configuration): + - Do not crash when starting with LearnCircuitBuildTimeout 0. + Fixes bug 22252; bugfix on 0.2.9.3-alpha. diff --git a/changes/bug22270 b/changes/bug22270 new file mode 100644 index 0000000000..6b58446402 --- /dev/null +++ b/changes/bug22270 @@ -0,0 +1,4 @@ + o Minor bugfixes (fallback directory mirrors): + - Make the usage example in updateFallbackDirs.py actually work. + (And explain what it does.) + Fixes bug 22270; bugfix on 0.3.0.3-alpha. diff --git a/changes/ticket21564 b/changes/ticket21564 new file mode 100644 index 0000000000..7e01f41f8f --- /dev/null +++ b/changes/ticket21564 @@ -0,0 +1,6 @@ + o Minor features (fallback directory list): + - Replace the 177 fallbacks originally introduced in Tor 0.2.9.8 in + December 2016 (of which ~126 were still functional), with a list of + 151 fallbacks (32 new, 119 existing, 58 removed) generated in + May 2017. + Resolves ticket 21564. diff --git a/scripts/maint/fallback.blacklist b/scripts/maint/fallback.blacklist index 974b304729..1417a13a98 100644 --- a/scripts/maint/fallback.blacklist +++ b/scripts/maint/fallback.blacklist @@ -252,3 +252,23 @@ id=9C8A123081EFBE022EF795630F447839DDFDDDEC # Email sent directly to teor, verified using relay contact info 163.172.35.245:80 orport=443 id=B771AA877687F88E6F1CA5354756DF6C8A7B6B24 + +# Email sent directly to teor, verified using relay contact info +104.243.35.196:9030 orport=9001 id=FA3415659444AE006E7E9E5375E82F29700CFDFD + +# Relay changed IPv4 address, operator uncontactable +138.201.130.32:9030 orport=9001 id=52AEA31188331F421B2EDB494DB65CD181E5B257 + +# Emails sent directly to teor, verified using relay contact info +217.12.199.208:80 orport=443 id=DF3AED4322B1824BF5539AE54B2D1B38E080FF05 ipv6=[2a02:27a8:0:2::7e]:443 + +# Emails sent directly to teor, verified using relay contact info +195.154.75.84:9030 orport=9001 id=F80FDE27EFCB3F6A7B4E2CC517133DBFFA78BA2D +195.154.127.246:9030 orport=9001 id=4FEE77AFFD157BBCF2D896AE417FBF647860466C + +# Email sent directly to teor, verified using relay contact info +5.35.251.247:9030 orport=9001 id=9B1F5187DFBA89DC24B37EA7BF896C12B43A27AE + +#​https://lists.torproject.org/pipermail/tor-relays/2017-May/012281.html +62.210.124.124:9030 orport=9001 id=86E78DD3720C78DA8673182EF96C54B162CD660C ipv6=[2001:bc8:3f23:100::1]:9001 +62.210.124.124:9130 orport=9101 id=2EBD117806EE43C3CC885A8F1E4DC60F207E7D3E ipv6=[2001:bc8:3f23:100::1]:9101 diff --git a/scripts/maint/fallback.whitelist b/scripts/maint/fallback.whitelist index c993be97d3..0620d6b5fe 100644 --- a/scripts/maint/fallback.whitelist +++ b/scripts/maint/fallback.whitelist @@ -37,8 +37,6 @@ # https://lists.torproject.org/pipermail/tor-relays/2015-December/008370.html # https://lists.torproject.org/pipermail/tor-relays/2016-January/008517.html # https://lists.torproject.org/pipermail/tor-relays/2016-January/008555.html -62.210.124.124:9030 orport=9001 id=86E78DD3720C78DA8673182EF96C54B162CD660C ipv6=[2001:bc8:3f23:100::1]:9001 -62.210.124.124:9130 orport=9101 id=2EBD117806EE43C3CC885A8F1E4DC60F207E7D3E ipv6=[2001:bc8:3f23:100::1]:9101 212.47.237.95:9030 orport=9001 id=3F5D8A879C58961BB45A3D26AC41B543B40236D6 212.47.237.95:9130 orport=9101 id=6FB38EB22E57EF7ED5EF00238F6A48E553735D88 @@ -49,15 +47,13 @@ # https://lists.torproject.org/pipermail/tor-relays/2015-December/008373.html 167.114.35.28:9030 orport=9001 id=E65D300F11E1DB12C534B0146BDAB6972F1A8A48 -# https://lists.torproject.org/pipermail/tor-relays/2015-December/008374.html -104.243.35.196:9030 orport=9001 id=FA3415659444AE006E7E9E5375E82F29700CFDFD - # https://lists.torproject.org/pipermail/tor-relays/2015-December/008378.html 144.76.14.145:110 orport=143 id=14419131033443AE6E21DA82B0D307F7CAE42BDB ipv6=[2a01:4f8:190:9490::dead]:443 # https://lists.torproject.org/pipermail/tor-relays/2015-December/008379.html # Email sent directly to teor, verified using relay contact info 91.121.84.137:4951 orport=4051 id=6DE61A6F72C1E5418A66BFED80DFB63E4C77668F ipv6=[2001:41d0:1:8989::1]:4051 +91.121.84.137:4952 orport=4052 id=9FBEB75E8BC142565F12CBBE078D63310236A334 ipv6=[2001:41d0:1:8989::1]:4052 # https://lists.torproject.org/pipermail/tor-relays/2015-December/008381.html # Sent additional email to teor with more relays @@ -95,9 +91,6 @@ # https://lists.torproject.org/pipermail/tor-relays/2016-January/008542.html 178.62.199.226:80 orport=443 id=CBEFF7BA4A4062045133C053F2D70524D8BBE5BE ipv6=[2a03:b0c0:2:d0::b7:5001]:443 -# Emails sent directly to teor, verified using relay contact info -217.12.199.208:80 orport=443 id=DF3AED4322B1824BF5539AE54B2D1B38E080FF05 ipv6=[2a02:27a8:0:2::7e]:443 - # Email sent directly to teor, verified using relay contact info 94.23.204.175:9030 orport=9001 id=5665A3904C89E22E971305EE8C1997BCA4123C69 @@ -330,9 +323,6 @@ 37.187.102.186:9030 orport=9001 id=489D94333DF66D57FFE34D9D59CC2D97E2CB0053 ipv6=[2001:41d0:a:26ba::1]:9001 # Email sent directly to teor, verified using relay contact info -5.35.251.247:9030 orport=9001 id=9B1F5187DFBA89DC24B37EA7BF896C12B43A27AE - -# Email sent directly to teor, verified using relay contact info 198.96.155.3:8080 orport=5001 id=BCEDF6C193AA687AE471B8A22EBF6BC57C2D285E # Email sent directly to teor, verified using relay contact info @@ -427,12 +417,10 @@ 46.4.24.161:9030 orport=9001 id=DB4C76A3AD7E234DA0F00D6F1405D8AFDF4D8DED 46.4.24.161:9031 orport=9002 id=7460F3D12EBE861E4EE073F6233047AACFE46AB4 46.38.51.132:9030 orport=9001 id=810DEFA7E90B6C6C383C063028EC397A71D7214A -163.172.194.53:9030 orport=9001 id=8C00FA7369A7A308F6A137600F0FA07990D9D451 +163.172.194.53:9030 orport=9001 id=8C00FA7369A7A308F6A137600F0FA07990D9D451 ipv6=[2001:bc8:225f:142:6c69:7461:7669:73]:9001 # Email sent directly to teor, verified using relay contact info 176.10.107.180:9030 orport=9001 id=3D7E274A87D9A89AF064C13D1EE4CA1F184F2600 -195.154.75.84:9030 orport=9001 id=F80FDE27EFCB3F6A7B4E2CC517133DBFFA78BA2D -195.154.127.246:9030 orport=9001 id=4FEE77AFFD157BBCF2D896AE417FBF647860466C # Email sent directly to teor, verified using relay contact info 46.28.207.19:80 orport=443 id=5B92FA5C8A49D46D235735504C72DBB3472BA321 @@ -471,7 +459,7 @@ 185.35.202.221:9030 orport=9001 id=C13B91384CDD52A871E3ECECE4EF74A7AC7DCB08 ipv6=[2a02:ed06::221]:9001 # Email sent directly to teor, verified using relay contact info -5.9.151.241:9030 orport=4223 id=9BF04559224F0F1C3C953D641F1744AF0192543A +5.9.151.241:9030 orport=4223 id=9BF04559224F0F1C3C953D641F1744AF0192543A ipv6=[2a01:4f8:190:34f0::2]:4223 # Email sent directly to teor, verified using relay contact info 89.40.71.149:8081 orport=8080 id=EC639EDAA5121B47DBDF3D6B01A22E48A8CB6CC7 @@ -574,6 +562,7 @@ # Email sent directly to teor, verified using relay contact info 185.100.86.100:80 orport=443 id=0E8C0C8315B66DB5F703804B3889A1DD66C67CE0 +185.100.84.82:80 orport=443 id=7D05A38E39FC5D29AFE6BE487B9B4DC9E635D09E # Email sent directly to teor, verified using relay contact info 164.132.77.175:9030 orport=9001 id=3B33F6FCA645AD4E91428A3AF7DC736AD9FB727B @@ -643,9 +632,6 @@ 46.4.111.124:9030 orport=9001 id=D9065F9E57899B3D272AA212317AF61A9B14D204 # Email sent directly to teor, verified using relay contact info -138.201.130.32:9030 orport=9001 id=52AEA31188331F421B2EDB494DB65CD181E5B257 - -# Email sent directly to teor, verified using relay contact info 185.100.85.61:80 orport=443 id=025B66CEBC070FCB0519D206CF0CF4965C20C96E # Email sent directly to teor, verified using relay contact info @@ -828,3 +814,16 @@ # Email sent directly to teor, verified using relay contact info 95.85.8.226:80 orport=443 id=1211AC1BBB8A1AF7CBA86BCE8689AA3146B86423 + +# Email sent directly to teor, verified using relay contact info +85.214.151.72:9030 orport=9001 id=722D365140C8C52DBB3C9FF6986E3CEFFE2BA812 + +# Email sent directly to teor, verified using relay contact info +72.52.75.27:9030 orport=9001 id=1220F0F20E80D348244C5F3B6D126DAA0A446DFD + +# Email sent directly to teor, verified using relay contact info +5.9.146.203:80 orport=443 id=1F45542A24A61BF9408F1C05E0DCE4E29F2CBA11 +5.9.159.14:9030 orport=9001 id=0F100F60C7A63BED90216052324D29B08CFCF797 + +# Email sent directly to teor, verified using relay contact info +5.9.147.226:9030 orport=9001 id=B0553175AADB0501E5A61FC61CEA3970BE130FF2 diff --git a/scripts/maint/updateFallbackDirs.py b/scripts/maint/updateFallbackDirs.py index 117ac5cccb..82a60420b4 100755 --- a/scripts/maint/updateFallbackDirs.py +++ b/scripts/maint/updateFallbackDirs.py @@ -1,8 +1,13 @@ #!/usr/bin/python # Usage: +# +# Regenerate the list: # scripts/maint/updateFallbackDirs.py > src/or/fallback_dirs.inc -# scripts/maint/updateFallbackDirs.py check_existing > src/or/fallback_dirs.inc +# +# Check the existing list: +# scripts/maint/updateFallbackDirs.py check_existing > fallback_dirs.inc.ok +# mv fallback_dirs.inc.ok src/or/fallback_dirs.inc # # This script should be run from a stable, reliable network connection, # with no other network activity (and not over tor). @@ -154,20 +159,24 @@ MAX_LIST_FILE_SIZE = 1024 * 1024 ## Eligibility Settings # Require fallbacks to have the same address and port for a set amount of time +# We used to have this at 1 week, but that caused many fallback failures, which +# meant that we had to rebuild the list more often. # # There was a bug in Tor 0.2.8.1-alpha and earlier where a relay temporarily # submits a 0 DirPort when restarted. # This causes OnionOO to (correctly) reset its stability timer. # Affected relays should upgrade to Tor 0.2.8.7 or later, which has a fix # for this issue. -ADDRESS_AND_PORT_STABLE_DAYS = 7 +ADDRESS_AND_PORT_STABLE_DAYS = 30 # We ignore relays that have been down for more than this period MAX_DOWNTIME_DAYS = 0 if MUST_BE_RUNNING_NOW else 7 # What time-weighted-fraction of these flags must FallbackDirs # Equal or Exceed? CUTOFF_RUNNING = .90 CUTOFF_V2DIR = .90 -CUTOFF_GUARD = .90 +# Tolerate lower guard flag averages, as guard flags are removed for some time +# after a relay restarts +CUTOFF_GUARD = .80 # What time-weighted-fraction of these flags must FallbackDirs # Equal or Fall Under? # .00 means no bad exits @@ -189,7 +198,7 @@ FALLBACK_PROPORTION_OF_GUARDS = None if OUTPUT_CANDIDATES else _FB_POG # Limit the number of fallbacks (eliminating lowest by advertised bandwidth) MAX_FALLBACK_COUNT = None if OUTPUT_CANDIDATES else 200 # Emit a C #error if the number of fallbacks is less than expected -MIN_FALLBACK_COUNT = 0 if OUTPUT_CANDIDATES else MAX_FALLBACK_COUNT*0.75 +MIN_FALLBACK_COUNT = 0 if OUTPUT_CANDIDATES else MAX_FALLBACK_COUNT*0.5 # The maximum number of fallbacks on the same address, contact, or family # With 200 fallbacks, this means each operator can see 1% of client bootstraps diff --git a/src/common/compress.c b/src/common/compress.c index 6fe4569868..6513029f9c 100644 --- a/src/common/compress.c +++ b/src/common/compress.c @@ -343,6 +343,32 @@ compression_method_get_name(compress_method_t method) return NULL; } +/** Table of compression human readable method names. */ +static const struct { + compress_method_t method; + const char *name; +} compression_method_human_names[] = { + { NO_METHOD, "uncompressed" }, + { GZIP_METHOD, "gzipped" }, + { ZLIB_METHOD, "deflated" }, + { LZMA_METHOD, "LZMA compressed" }, + { ZSTD_METHOD, "Zstandard compressed" }, + { UNKNOWN_METHOD, "unknown encoding" }, +}; + +/** Return a human readable string representation of the compression method + * <b>method</b>, or NULL if the method isn't recognized. */ +const char * +compression_method_get_human_name(compress_method_t method) +{ + unsigned i; + for (i = 0; i < ARRAY_LENGTH(compression_method_human_names); ++i) { + if (method == compression_method_human_names[i].method) + return compression_method_human_names[i].name; + } + return NULL; +} + /** Return the compression method represented by the string <b>name</b>, or * UNKNOWN_METHOD if the string isn't recognized. */ compress_method_t diff --git a/src/common/compress.h b/src/common/compress.h index 5b47c5d458..7c0dc14061 100644 --- a/src/common/compress.h +++ b/src/common/compress.h @@ -50,6 +50,7 @@ int tor_compress_is_compression_bomb(size_t size_in, size_t size_out); int tor_compress_supports_method(compress_method_t method); unsigned tor_compress_get_supported_method_bitmask(void); const char * compression_method_get_name(compress_method_t method); +const char *compression_method_get_human_name(compress_method_t method); compress_method_t compression_method_get_by_name(const char *name); const char *tor_compress_version_str(compress_method_t method); diff --git a/src/common/util_format.h b/src/common/util_format.h index adf48c0077..4af8832bbe 100644 --- a/src/common/util_format.h +++ b/src/common/util_format.h @@ -23,11 +23,11 @@ #define BASE32_BUFSIZE(n) (BASE32_LEN(n) + 1) #define BASE16_BUFSIZE(n) (BASE16_LEN(n) + 1) -#define BASE64_NOPAD_LEN(n) (CEIL_DIV((n) * 4, 3) -#define BASE32_NOPAD_LEN(n) (CEIL_DIV((n) * 8, 5) +#define BASE64_NOPAD_LEN(n) (CEIL_DIV((n) * 4, 3)) +#define BASE32_NOPAD_LEN(n) (CEIL_DIV((n) * 8, 5)) -#define BASE64_NOPAD_BUFSIZE(n) (BASE64_NOPAD_LEN(n) + 1)) -#define BASE32_NOPAD_BUFSIZE(n) (BASE32_NOPAD_LEN(n) + 1)) +#define BASE64_NOPAD_BUFSIZE(n) (BASE64_NOPAD_LEN(n) + 1) +#define BASE32_NOPAD_BUFSIZE(n) (BASE32_NOPAD_LEN(n) + 1) /** @} */ #define BASE64_ENCODE_MULTILINE 1 diff --git a/src/or/circuitstats.c b/src/or/circuitstats.c index fe6f723bd5..51d580a1a4 100644 --- a/src/or/circuitstats.c +++ b/src/or/circuitstats.c @@ -107,10 +107,19 @@ get_circuit_build_timeout_ms(void) int circuit_build_times_disabled(const or_options_t *options) { + return circuit_build_times_disabled_(options, 0); +} + +/** As circuit_build_times_disabled, but take options as an argument. */ +int +circuit_build_times_disabled_(const or_options_t *options, + int ignore_consensus) +{ if (unit_tests) { return 0; } else { - int consensus_disabled = networkstatus_get_param(NULL, "cbtdisabled", + int consensus_disabled = + ignore_consensus ? 0 : networkstatus_get_param(NULL, "cbtdisabled", 0, 0, 1); int config_disabled = !options->LearnCircuitBuildTimeout; int dirauth_disabled = options->AuthoritativeDir; diff --git a/src/or/circuitstats.h b/src/or/circuitstats.h index 4977b26c0d..8a1dec4bfd 100644 --- a/src/or/circuitstats.h +++ b/src/or/circuitstats.h @@ -18,6 +18,9 @@ double get_circuit_build_close_time_ms(void); double get_circuit_build_timeout_ms(void); int circuit_build_times_disabled(const or_options_t *options); +int circuit_build_times_disabled_(const or_options_t *options, + int ignore_consensus); + int circuit_build_times_enough_to_compute(const circuit_build_times_t *cbt); void circuit_build_times_update_state(const circuit_build_times_t *cbt, or_state_t *state); diff --git a/src/or/config.c b/src/or/config.c index a1212cfd3c..461cf0aa49 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -2775,7 +2775,7 @@ compute_publishserverdescriptor(or_options_t *options) #define MIN_REND_POST_PERIOD (10*60) #define MIN_REND_POST_PERIOD_TESTING (5) -/** Higest allowable value for CircuitsAvailableTimeout. +/** Highest allowable value for CircuitsAvailableTimeout. * If this is too large, client connections will stay open for too long, * incurring extra padding overhead. */ #define MAX_CIRCS_AVAILABLE_TIME (24*60*60) @@ -3490,7 +3490,7 @@ options_validate(or_options_t *old_options, or_options_t *options, int severity = LOG_NOTICE; /* Be a little quieter if we've deliberately disabled * LearnCircuitBuildTimeout. */ - if (circuit_build_times_disabled(options)) { + if (circuit_build_times_disabled_(options, 1)) { severity = LOG_INFO; } log_fn(severity, LD_CONFIG, "You disabled LearnCircuitBuildTimeout, but " diff --git a/src/or/consdiffmgr.c b/src/or/consdiffmgr.c index d478101c6b..2af104733b 100644 --- a/src/or/consdiffmgr.c +++ b/src/or/consdiffmgr.c @@ -32,6 +32,15 @@ /* The valid-after time for a consensus (or for the target consensus of a * diff), encoded as ISO UTC. */ #define LABEL_VALID_AFTER "consensus-valid-after" +/* The fresh-until time for a consensus (or for the target consensus of a + * diff), encoded as ISO UTC. */ +#define LABEL_FRESH_UNTIL "consensus-fresh-until" +/* The valid-until time for a consensus (or for the target consensus of a + * diff), encoded as ISO UTC. */ +#define LABEL_VALID_UNTIL "consensus-valid-until" +/* Comma-separated list of hex-encoded identity digests for the voting + * authorities. */ +#define LABEL_SIGNATORIES "consensus-signatories" /* A hex encoded SHA3 digest of the object, as compressed (if any) */ #define LABEL_SHA3_DIGEST "sha3-digest" /* A hex encoded SHA3 digest of the object before compression. */ @@ -96,6 +105,36 @@ n_diff_compression_methods(void) return ARRAY_LENGTH(compress_diffs_with); } +/** Which methods do we use for precompressing consensuses? */ +static const compress_method_t compress_consensus_with[] = { + ZLIB_METHOD, +#ifdef HAVE_LZMA + LZMA_METHOD, +#endif +#ifdef HAVE_ZSTD + ZSTD_METHOD, +#endif +}; + +/** How many different methods will we try to use for diff compression? */ +STATIC unsigned +n_consensus_compression_methods(void) +{ + return ARRAY_LENGTH(compress_consensus_with); +} + +/** For which compression method do we retain old consensuses? There's no + * need to keep all of them, since we won't be serving them. We'll + * go with ZLIB_METHOD because it's pretty fast and everyone has it. + */ +#define RETAIN_CONSENSUS_COMPRESSED_WITH_METHOD ZLIB_METHOD + +/** Handles pointing to the latest consensus entries as compressed and + * stored. */ +static consensus_cache_entry_handle_t * + latest_consensus[N_CONSENSUS_FLAVORS] + [ARRAY_LENGTH(compress_consensus_with)]; + /** Hashtable node used to remember the current status of the diff * from a given sha3 digest to the current consensus. */ typedef struct cdm_diff_t { @@ -127,7 +166,6 @@ static HT_HEAD(cdm_diff_ht, cdm_diff_t) cdm_diff_ht = HT_INITIALIZER(); * Configuration for this module */ static consdiff_cfg_t consdiff_cfg = { - /* .cache_max_age_hours = */ 24 * 90, // XXXX I'd like to make this number bigger, but it interferes with the // XXXX seccomp2 syscall filter, which tops out at BPF_MAXINS (4096) // XXXX rules. @@ -135,13 +173,12 @@ static consdiff_cfg_t consdiff_cfg = { }; static int consdiffmgr_ensure_space_for_files(int n); +static int consensus_queue_compression_work(const char *consensus, + const networkstatus_t *as_parsed); static int consensus_diff_queue_diff_work(consensus_cache_entry_t *diff_from, consensus_cache_entry_t *diff_to); static void consdiffmgr_set_cache_flags(void); -/* Just gzip consensuses for now. */ -#define COMPRESS_CONSENSUS_WITH GZIP_METHOD - /* ===== * Hashtable setup * ===== */ @@ -410,11 +447,6 @@ cdm_cache_lookup_consensus(consensus_flavor_t flavor, time_t valid_after) consensus_cache_filter_list(matches, LABEL_DOCTYPE, DOCTYPE_CONSENSUS); consensus_cache_entry_t *result = NULL; - if (smartlist_len(matches) > 1) { - log_warn(LD_BUG, "How odd; there appear to be two matching consensuses " - "with flavor %s published at %s.", - flavname, formatted_time); - } if (smartlist_len(matches)) { result = smartlist_get(matches, 0); } @@ -423,6 +455,25 @@ cdm_cache_lookup_consensus(consensus_flavor_t flavor, time_t valid_after) return result; } +/** Return the maximum age (in seconds) of consensuses that we should consider + * storing. The available space in the directory may impose additional limits + * on how much we store. */ +static int32_t +get_max_age_to_cache(void) +{ + /* The parameter is in hours. */ + const int32_t DEFAULT_MAX_AGE_TO_CACHE = 8192; + const int32_t MIN_MAX_AGE_TO_CACHE = 0; + const int32_t MAX_MAX_AGE_TO_CACHE = 8192; + const char MAX_AGE_TO_CACHE_NAME[] = "max-consensus-age-to-cache-for-diff"; + + return 3600 * networkstatus_get_param(NULL, + MAX_AGE_TO_CACHE_NAME, + DEFAULT_MAX_AGE_TO_CACHE, + MIN_MAX_AGE_TO_CACHE, + MAX_MAX_AGE_TO_CACHE); +} + /** * Given a string containing a networkstatus consensus, and the results of * having parsed that consensus, add that consensus to the cache if it is not @@ -443,7 +494,7 @@ consdiffmgr_add_consensus(const char *consensus, const consensus_flavor_t flavor = as_parsed->flavor; const time_t valid_after = as_parsed->valid_after; - if (valid_after < approx_time() - 3600 * consdiff_cfg.cache_max_age_hours) { + if (valid_after < approx_time() - get_max_age_to_cache()) { log_info(LD_DIRSERV, "We don't care about this consensus document; it's " "too old."); return -1; @@ -458,59 +509,7 @@ consdiffmgr_add_consensus(const char *consensus, } /* We don't have it. Add it to the cache. */ - consdiffmgr_ensure_space_for_files(1); - - { - size_t bodylen = strlen(consensus); - config_line_t *labels = NULL; - char formatted_time[ISO_TIME_LEN+1]; - format_iso_time_nospace(formatted_time, valid_after); - const char *flavname = networkstatus_get_flavor_name(flavor); - - cdm_labels_prepend_sha3(&labels, LABEL_SHA3_DIGEST_UNCOMPRESSED, - (const uint8_t *)consensus, bodylen); - { - const char *start, *end; - if (router_get_networkstatus_v3_signed_boundaries(consensus, - &start, &end) < 0) { - start = consensus; - end = consensus+bodylen; - } - cdm_labels_prepend_sha3(&labels, LABEL_SHA3_DIGEST_AS_SIGNED, - (const uint8_t *)start, - end - start); - } - - char *body_compressed = NULL; - size_t size_compressed = 0; - if (tor_compress(&body_compressed, &size_compressed, - consensus, bodylen, COMPRESS_CONSENSUS_WITH) < 0) { - config_free_lines(labels); - return -1; - } - cdm_labels_prepend_sha3(&labels, LABEL_SHA3_DIGEST, - (const uint8_t *)body_compressed, size_compressed); - config_line_prepend(&labels, LABEL_COMPRESSION_TYPE, - compression_method_get_name(COMPRESS_CONSENSUS_WITH)); - config_line_prepend(&labels, LABEL_FLAVOR, flavname); - config_line_prepend(&labels, LABEL_VALID_AFTER, formatted_time); - config_line_prepend(&labels, LABEL_DOCTYPE, DOCTYPE_CONSENSUS); - - entry = consensus_cache_add(cdm_cache_get(), - labels, - (const uint8_t *)body_compressed, - size_compressed); - tor_free(body_compressed); - config_free_lines(labels); - } - - if (entry) { - consensus_cache_entry_mark_for_aggressive_release(entry); - consensus_cache_entry_decref(entry); - } - - cdm_cache_dirty = 1; - return entry ? 0 : -1; + return consensus_queue_compression_work(consensus, as_parsed); } /** @@ -543,6 +542,48 @@ sort_and_find_most_recent(smartlist_t *lst) } } +/** Return i such that compress_consensus_with[i] == method. Return + * -1 if no such i exists. */ +static int +consensus_compression_method_pos(compress_method_t method) +{ + unsigned i; + for (i = 0; i < n_consensus_compression_methods(); ++i) { + if (compress_consensus_with[i] == method) { + return i; + } + } + return -1; +} + +/** + * If we know a consensus with the flavor <b>flavor</b> compressed with + * <b>method</b>, set *<b>entry_out</b> to that value. Return values are as + * for consdiffmgr_find_diff_from(). + */ +consdiff_status_t +consdiffmgr_find_consensus(struct consensus_cache_entry_t **entry_out, + consensus_flavor_t flavor, + compress_method_t method) +{ + tor_assert(entry_out); + tor_assert((int)flavor < N_CONSENSUS_FLAVORS); + + int pos = consensus_compression_method_pos(method); + if (pos < 0) { + // We don't compress consensuses with this method. + return CONSDIFF_NOT_FOUND; + } + consensus_cache_entry_handle_t *handle = latest_consensus[flavor][pos]; + if (!handle) + return CONSDIFF_NOT_FOUND; + *entry_out = consensus_cache_entry_handle_get(handle); + if (*entry_out) + return CONSDIFF_AVAILABLE; + else + return CONSDIFF_NOT_FOUND; +} + /** * Look up consensus_cache_entry_t for the consensus of type <b>flavor</b>, * from the source consensus with the specified digest (which must be SHA3). @@ -621,8 +662,7 @@ consdiffmgr_cleanup(void) log_debug(LD_DIRSERV, "Looking for consdiffmgr entries to remove"); // 1. Delete any consensus or diff or anything whose valid_after is too old. - const time_t valid_after_cutoff = - approx_time() - 3600 * consdiff_cfg.cache_max_age_hours; + const time_t valid_after_cutoff = approx_time() - get_max_age_to_cache(); consensus_cache_find_all(objects, cdm_cache_get(), NULL, NULL); @@ -684,6 +724,42 @@ consdiffmgr_cleanup(void) smartlist_clear(diffs); } + // 3. Delete all consensuses except the most recent that are compressed with + // an un-preferred method. + for (int flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) { + const char *flavname = networkstatus_get_flavor_name(flav); + /* Determine the most recent consensus of this flavor */ + consensus_cache_find_all(consensuses, cdm_cache_get(), + LABEL_DOCTYPE, DOCTYPE_CONSENSUS); + consensus_cache_filter_list(consensuses, LABEL_FLAVOR, flavname); + consensus_cache_entry_t *most_recent = + sort_and_find_most_recent(consensuses); + if (most_recent == NULL) + continue; + const char *most_recent_sha3_uncompressed = + consensus_cache_entry_get_value(most_recent, + LABEL_SHA3_DIGEST_UNCOMPRESSED); + const char *retain_methodname = compression_method_get_name( + RETAIN_CONSENSUS_COMPRESSED_WITH_METHOD); + + if (BUG(most_recent_sha3_uncompressed == NULL)) + continue; + SMARTLIST_FOREACH_BEGIN(consensuses, consensus_cache_entry_t *, ent) { + const char *lv_sha3_uncompressed = + consensus_cache_entry_get_value(ent, LABEL_SHA3_DIGEST_UNCOMPRESSED); + if (BUG(! lv_sha3_uncompressed)) + continue; + if (!strcmp(lv_sha3_uncompressed, most_recent_sha3_uncompressed)) + continue; // This _is_ the most recent. + const char *lv_methodname = + consensus_cache_entry_get_value(ent, LABEL_COMPRESSION_TYPE); + if (! lv_methodname || strcmp(lv_methodname, retain_methodname)) { + consensus_cache_entry_mark_for_removal(ent); + ++n_to_delete; + } + } SMARTLIST_FOREACH_END(ent); + } + smartlist_free(objects); smartlist_free(consensuses); smartlist_free(diffs); @@ -787,10 +863,14 @@ consdiffmgr_rescan_flavor_(consensus_flavor_t flavor) // 1. find the most recent consensus, and the ones that we might want // to diff to it. + const char *methodname = compression_method_get_name( + RETAIN_CONSENSUS_COMPRESSED_WITH_METHOD); + matches = smartlist_new(); consensus_cache_find_all(matches, cdm_cache_get(), LABEL_FLAVOR, flavname); consensus_cache_filter_list(matches, LABEL_DOCTYPE, DOCTYPE_CONSENSUS); + consensus_cache_filter_list(matches, LABEL_COMPRESSION_TYPE, methodname); consensus_cache_entry_t *most_recent = sort_and_find_most_recent(matches); if (!most_recent) { log_info(LD_DIRSERV, "No 'most recent' %s consensus found; " @@ -835,6 +915,10 @@ consdiffmgr_rescan_flavor_(consensus_flavor_t flavor) if (strmap_get(have_diff_from, va) != NULL) continue; /* we already have this one. */ smartlist_add(compute_diffs_from, ent); + /* Since we are not going to serve this as the most recent consensus + * any more, we should stop keeping it mmap'd when it's not in use. + */ + consensus_cache_entry_mark_for_aggressive_release(ent); } SMARTLIST_FOREACH_END(ent); log_info(LD_DIRSERV, @@ -880,6 +964,48 @@ consdiffmgr_rescan_flavor_(consensus_flavor_t flavor) } /** + * Scan the cache for the latest consensuses and add their handles to + * latest_consensus + */ +static void +consdiffmgr_consensus_load(void) +{ + smartlist_t *matches = smartlist_new(); + for (int flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) { + const char *flavname = networkstatus_get_flavor_name(flav); + smartlist_clear(matches); + consensus_cache_find_all(matches, cdm_cache_get(), + LABEL_FLAVOR, flavname); + consensus_cache_filter_list(matches, LABEL_DOCTYPE, DOCTYPE_CONSENSUS); + consensus_cache_entry_t *most_recent = sort_and_find_most_recent(matches); + if (! most_recent) + continue; // no consensuses. + const char *most_recent_sha3 = + consensus_cache_entry_get_value(most_recent, + LABEL_SHA3_DIGEST_UNCOMPRESSED); + if (BUG(most_recent_sha3 == NULL)) + continue; // LCOV_EXCL_LINE + consensus_cache_filter_list(matches, LABEL_SHA3_DIGEST_UNCOMPRESSED, + most_recent_sha3); + + // Everything that remains matches the most recent consensus of this + // flavor. + SMARTLIST_FOREACH_BEGIN(matches, consensus_cache_entry_t *, ent) { + const char *lv_compression = + consensus_cache_entry_get_value(ent, LABEL_COMPRESSION_TYPE); + compress_method_t method = + compression_method_get_by_name(lv_compression); + int pos = consensus_compression_method_pos(method); + if (pos < 0) + continue; + consensus_cache_entry_handle_free(latest_consensus[flav][pos]); + latest_consensus[flav][pos] = consensus_cache_entry_handle_new(ent); + } SMARTLIST_FOREACH_END(ent); + } + smartlist_free(matches); +} + +/** * Scan the cache for diffs, and add them to the hashtable. */ static void @@ -936,6 +1062,7 @@ consdiffmgr_rescan(void) if (cdm_cache_loaded == 0) { consdiffmgr_diffs_load(); + consdiffmgr_consensus_load(); cdm_cache_loaded = 1; } @@ -1051,6 +1178,14 @@ consdiffmgr_free_all(void) next = HT_NEXT_RMV(cdm_diff_ht, &cdm_diff_ht, diff); cdm_diff_free(this); } + int i; + unsigned j; + for (i = 0; i < N_CONSENSUS_FLAVORS; ++i) { + for (j = 0; j < n_consensus_compression_methods(); ++j) { + consensus_cache_entry_handle_free(latest_consensus[i][j]); + } + } + memset(latest_consensus, 0, sizeof(latest_consensus)); consensus_cache_free(cons_diff_cache); cons_diff_cache = NULL; } @@ -1072,6 +1207,93 @@ typedef struct compressed_result_t { } compressed_result_t; /** + * Compress the bytestring <b>input</b> of length <b>len</b> using the + * <n>n_methods</b> compression methods listed in the array <b>methods</b>. + * + * For each successful compression, set the fields in the <b>results_out</b> + * array in the position corresponding to the compression method. Use + * <b>labels_in</b> as a basis for the labels of the result. + * + * Return 0 if all compression succeeded; -1 if any failed. + */ +static int +compress_multiple(compressed_result_t *results_out, int n_methods, + const compress_method_t *methods, + const uint8_t *input, size_t len, + const config_line_t *labels_in) +{ + int rv = 0; + int i; + for (i = 0; i < n_methods; ++i) { + compress_method_t method = methods[i]; + const char *methodname = compression_method_get_name(method); + char *result; + size_t sz; + if (0 == tor_compress(&result, &sz, (const char*)input, len, method)) { + results_out[i].body = (uint8_t*)result; + results_out[i].bodylen = sz; + results_out[i].labels = config_lines_dup(labels_in); + cdm_labels_prepend_sha3(&results_out[i].labels, LABEL_SHA3_DIGEST, + results_out[i].body, + results_out[i].bodylen); + config_line_prepend(&results_out[i].labels, + LABEL_COMPRESSION_TYPE, + methodname); + } else { + rv = -1; + } + } + return rv; +} + +/** + * Given an array of <b>n</b> compressed_result_t in <b>results</b>, + * as produced by compress_multiple, store them all into the + * consdiffmgr, and store handles to them in the <b>handles_out</b> + * array. + * + * Return CDM_DIFF_PRESENT if any was stored, and CDM_DIFF_ERROR if none + * was stored. + */ +static cdm_diff_status_t +store_multiple(consensus_cache_entry_handle_t **handles_out, + int n, + const compress_method_t *methods, + const compressed_result_t *results, + const char *description) +{ + cdm_diff_status_t status = CDM_DIFF_ERROR; + consdiffmgr_ensure_space_for_files(n); + + int i; + for (i = 0; i < n; ++i) { + compress_method_t method = methods[i]; + uint8_t *body_out = results[i].body; + size_t bodylen_out = results[i].bodylen; + config_line_t *labels = results[i].labels; + const char *methodname = compression_method_get_name(method); + if (body_out && bodylen_out && labels) { + /* Success! Store the results */ + log_info(LD_DIRSERV, "Adding %s, compressed with %s", + description, methodname); + + consensus_cache_entry_t *ent = + consensus_cache_add(cdm_cache_get(), + labels, + body_out, + bodylen_out); + if (BUG(ent == NULL)) + continue; + + status = CDM_DIFF_PRESENT; + handles_out[i] = consensus_cache_entry_handle_new(ent); + consensus_cache_entry_decref(ent); + } + } + return status; +} + +/** * An object passed to a worker thread that will try to produce a consensus * diff. */ @@ -1144,6 +1366,12 @@ consensus_diff_worker_threadfn(void *state_, void *work_) const char *lv_to_valid_after = consensus_cache_entry_get_value(job->diff_to, LABEL_VALID_AFTER); + const char *lv_to_fresh_until = + consensus_cache_entry_get_value(job->diff_to, LABEL_FRESH_UNTIL); + const char *lv_to_valid_until = + consensus_cache_entry_get_value(job->diff_to, LABEL_VALID_UNTIL); + const char *lv_to_signatories = + consensus_cache_entry_get_value(job->diff_to, LABEL_SIGNATORIES); const char *lv_from_valid_after = consensus_cache_entry_get_value(job->diff_from, LABEL_VALID_AFTER); const char *lv_from_digest = @@ -1213,6 +1441,12 @@ consensus_diff_worker_threadfn(void *state_, void *work_) job->out[0].bodylen = difflen; config_line_t *common_labels = NULL; + if (lv_to_valid_until) + config_line_prepend(&common_labels, LABEL_VALID_UNTIL, lv_to_valid_until); + if (lv_to_fresh_until) + config_line_prepend(&common_labels, LABEL_FRESH_UNTIL, lv_to_fresh_until); + if (lv_to_signatories) + config_line_prepend(&common_labels, LABEL_SIGNATORIES, lv_to_signatories); cdm_labels_prepend_sha3(&common_labels, LABEL_SHA3_DIGEST_UNCOMPRESSED, job->out[0].body, @@ -1235,24 +1469,10 @@ consensus_diff_worker_threadfn(void *state_, void *work_) job->out[0].body, job->out[0].bodylen); - unsigned u; - for (u = 1; u < n_diff_compression_methods(); ++u) { - compress_method_t method = compress_diffs_with[u]; - const char *methodname = compression_method_get_name(method); - char *result; - size_t sz; - if (0 == tor_compress(&result, &sz, consensus_diff, difflen, method)) { - job->out[u].body = (uint8_t*)result; - job->out[u].bodylen = sz; - job->out[u].labels = config_lines_dup(common_labels); - cdm_labels_prepend_sha3(&job->out[u].labels, LABEL_SHA3_DIGEST, - job->out[u].body, - job->out[u].bodylen); - config_line_prepend(&job->out[u].labels, - LABEL_COMPRESSION_TYPE, - methodname); - } - } + compress_multiple(job->out+1, + n_diff_compression_methods()-1, + compress_diffs_with+1, + (const uint8_t*)consensus_diff, difflen, common_labels); config_free_lines(common_labels); return WQ_RPL_REPLY; @@ -1318,36 +1538,20 @@ consensus_diff_worker_replyfn(void *work_) cache = 0; } - int status = CDM_DIFF_ERROR; consensus_cache_entry_handle_t *handles[ARRAY_LENGTH(compress_diffs_with)]; memset(handles, 0, sizeof(handles)); - consdiffmgr_ensure_space_for_files(n_diff_compression_methods()); + char description[128]; + tor_snprintf(description, sizeof(description), + "consensus diff from %s to %s", + lv_from_digest, lv_to_digest); - unsigned u; - for (u = 0; u < n_diff_compression_methods(); ++u) { - compress_method_t method = compress_diffs_with[u]; - uint8_t *body_out = job->out[u].body; - size_t bodylen_out = job->out[u].bodylen; - config_line_t *labels = job->out[u].labels; - const char *methodname = compression_method_get_name(method); - if (body_out && bodylen_out && labels) { - /* Success! Store the results */ - log_info(LD_DIRSERV, "Adding consensus diff from %s to %s, " - "compressed with %s", - lv_from_digest, lv_to_digest, methodname); - - consensus_cache_entry_t *ent = - consensus_cache_add(cdm_cache_get(), - labels, - body_out, - bodylen_out); + int status = store_multiple(handles, + n_diff_compression_methods(), + compress_diffs_with, + job->out, + description); - status = CDM_DIFF_PRESENT; - handles[u] = consensus_cache_entry_handle_new(ent); - consensus_cache_entry_decref(ent); - } - } if (status != CDM_DIFF_PRESENT) { /* Failure! Nothing to do but complain */ log_warn(LD_DIRSERV, @@ -1357,6 +1561,7 @@ consensus_diff_worker_replyfn(void *work_) status = CDM_DIFF_ERROR; } + unsigned u; for (u = 0; u < ARRAY_LENGTH(handles); ++u) { compress_method_t method = compress_diffs_with[u]; if (cache) { @@ -1408,3 +1613,230 @@ consensus_diff_queue_diff_work(consensus_cache_entry_t *diff_from, return -1; } +/** + * Holds requests and replies for consensus_compress_workers. + */ +typedef struct consensus_compress_worker_job_t { + char *consensus; + size_t consensus_len; + consensus_flavor_t flavor; + config_line_t *labels_in; + compressed_result_t out[ARRAY_LENGTH(compress_consensus_with)]; +} consensus_compress_worker_job_t; + +/** + * Free all resources held in <b>job</b> + */ +static void +consensus_compress_worker_job_free(consensus_compress_worker_job_t *job) +{ + if (!job) + return; + tor_free(job->consensus); + config_free_lines(job->labels_in); + unsigned u; + for (u = 0; u < n_consensus_compression_methods(); ++u) { + config_free_lines(job->out[u].labels); + tor_free(job->out[u].body); + } + tor_free(job); +} +/** + * Worker function. This function runs inside a worker thread and receives + * a consensus_compress_worker_job_t as its input. + */ +static workqueue_reply_t +consensus_compress_worker_threadfn(void *state_, void *work_) +{ + (void)state_; + consensus_compress_worker_job_t *job = work_; + consensus_flavor_t flavor = job->flavor; + const char *consensus = job->consensus; + size_t bodylen = job->consensus_len; + + config_line_t *labels = config_lines_dup(job->labels_in); + const char *flavname = networkstatus_get_flavor_name(flavor); + + cdm_labels_prepend_sha3(&labels, LABEL_SHA3_DIGEST_UNCOMPRESSED, + (const uint8_t *)consensus, bodylen); + { + const char *start, *end; + if (router_get_networkstatus_v3_signed_boundaries(consensus, + &start, &end) < 0) { + start = consensus; + end = consensus+bodylen; + } + cdm_labels_prepend_sha3(&labels, LABEL_SHA3_DIGEST_AS_SIGNED, + (const uint8_t *)start, + end - start); + } + config_line_prepend(&labels, LABEL_FLAVOR, flavname); + config_line_prepend(&labels, LABEL_DOCTYPE, DOCTYPE_CONSENSUS); + + compress_multiple(job->out, + n_consensus_compression_methods(), + compress_consensus_with, + (const uint8_t*)consensus, bodylen, labels); + config_free_lines(labels); + return WQ_RPL_REPLY; +} + +/** + * Worker function: This function runs in the main thread, and receives + * a consensus_diff_compress_job_t that the worker thread has already + * processed. + */ +static void +consensus_compress_worker_replyfn(void *work_) +{ + consensus_compress_worker_job_t *job = work_; + + consensus_cache_entry_handle_t *handles[ + ARRAY_LENGTH(compress_consensus_with)]; + memset(handles, 0, sizeof(handles)); + + store_multiple(handles, + n_consensus_compression_methods(), + compress_consensus_with, + job->out, + "consensus"); + cdm_cache_dirty = 1; + + unsigned u; + consensus_flavor_t f = job->flavor; + tor_assert((int)f < N_CONSENSUS_FLAVORS); + for (u = 0; u < ARRAY_LENGTH(handles); ++u) { + if (handles[u] == NULL) + continue; + consensus_cache_entry_handle_free(latest_consensus[f][u]); + latest_consensus[f][u] = handles[u]; + } + + consensus_compress_worker_job_free(job); +} + +/** + * If true, we compress in worker threads. + */ +static int background_compression = 0; + +/** + * Queue a job to compress <b>consensus</b> and store its compressed + * text in the cache. + */ +static int +consensus_queue_compression_work(const char *consensus, + const networkstatus_t *as_parsed) +{ + tor_assert(consensus); + tor_assert(as_parsed); + + consensus_compress_worker_job_t *job = tor_malloc_zero(sizeof(*job)); + job->consensus = tor_strdup(consensus); + job->consensus_len = strlen(consensus); + job->flavor = as_parsed->flavor; + + char va_str[ISO_TIME_LEN+1]; + char vu_str[ISO_TIME_LEN+1]; + char fu_str[ISO_TIME_LEN+1]; + format_iso_time_nospace(va_str, as_parsed->valid_after); + format_iso_time_nospace(fu_str, as_parsed->fresh_until); + format_iso_time_nospace(vu_str, as_parsed->valid_until); + config_line_append(&job->labels_in, LABEL_VALID_AFTER, va_str); + config_line_append(&job->labels_in, LABEL_FRESH_UNTIL, fu_str); + config_line_append(&job->labels_in, LABEL_VALID_UNTIL, vu_str); + if (as_parsed->voters) { + smartlist_t *hexvoters = smartlist_new(); + SMARTLIST_FOREACH_BEGIN(as_parsed->voters, + networkstatus_voter_info_t *, vi) { + if (smartlist_len(vi->sigs) == 0) + continue; // didn't sign. + char d[HEX_DIGEST_LEN+1]; + base16_encode(d, sizeof(d), vi->identity_digest, DIGEST_LEN); + smartlist_add_strdup(hexvoters, d); + } SMARTLIST_FOREACH_END(vi); + char *signers = smartlist_join_strings(hexvoters, ",", 0, NULL); + config_line_prepend(&job->labels_in, LABEL_SIGNATORIES, signers); + tor_free(signers); + SMARTLIST_FOREACH(hexvoters, char *, cp, tor_free(cp)); + smartlist_free(hexvoters); + } + + if (background_compression) { + workqueue_entry_t *work; + work = cpuworker_queue_work(consensus_compress_worker_threadfn, + consensus_compress_worker_replyfn, + job); + if (!work) { + consensus_compress_worker_job_free(job); + return -1; + } + + return 0; + } else { + consensus_compress_worker_threadfn(NULL, job); + consensus_compress_worker_replyfn(job); + return 0; + } +} + +/** + * Tell the consdiffmgr backend to compress consensuses in worker threads. + */ +void +consdiffmgr_enable_background_compression(void) +{ + // This isn't the default behavior because it would break unit tests. + background_compression = 1; +} + +/** Read the set of voters from the cached object <b>ent</b> into + * <b>out</b>, as a list of hex-encoded digests. Return 0 on success, + * -1 if no signatories were recorded. */ +int +consensus_cache_entry_get_voter_id_digests(const consensus_cache_entry_t *ent, + smartlist_t *out) +{ + tor_assert(ent); + tor_assert(out); + const char *s; + s = consensus_cache_entry_get_value(ent, LABEL_SIGNATORIES); + if (s == NULL) + return -1; + smartlist_split_string(out, s, ",", SPLIT_SKIP_SPACE|SPLIT_STRIP_SPACE, 0); + return 0; +} + +/** Read the fresh-until time of cached object <b>ent</b> into *<b>out</b> + * and return 0, or return -1 if no such time was recorded. */ +int +consensus_cache_entry_get_fresh_until(const consensus_cache_entry_t *ent, + time_t *out) +{ + tor_assert(ent); + tor_assert(out); + const char *s; + s = consensus_cache_entry_get_value(ent, LABEL_FRESH_UNTIL); + if (s == NULL || parse_iso_time_nospace(s, out) < 0) + return -1; + else + return 0; +} + +/** Read the valid until timestamp from the cached object <b>ent</b> into + * *<b>out</b> and return 0, or return -1 if no such time was recorded. */ +int +consensus_cache_entry_get_valid_until(const consensus_cache_entry_t *ent, + time_t *out) +{ + tor_assert(ent); + tor_assert(out); + + const char *s; + s = consensus_cache_entry_get_value(ent, LABEL_VALID_UNTIL); + if (s == NULL || parse_iso_time_nospace(s, out) < 0) + return -1; + else + return 0; +} + diff --git a/src/or/consdiffmgr.h b/src/or/consdiffmgr.h index 048dae432c..fe4f9ee239 100644 --- a/src/or/consdiffmgr.h +++ b/src/or/consdiffmgr.h @@ -14,7 +14,6 @@ typedef enum consdiff_status_t { } consdiff_status_t; typedef struct consdiff_cfg_t { - int32_t cache_max_age_hours; int32_t cache_max_num; } consdiff_cfg_t; @@ -23,6 +22,11 @@ struct consensus_cache_entry_t; // from conscache.h int consdiffmgr_add_consensus(const char *consensus, const networkstatus_t *as_parsed); +consdiff_status_t consdiffmgr_find_consensus( + struct consensus_cache_entry_t **entry_out, + consensus_flavor_t flavor, + compress_method_t method); + consdiff_status_t consdiffmgr_find_diff_from( struct consensus_cache_entry_t **entry_out, consensus_flavor_t flavor, @@ -30,8 +34,20 @@ consdiff_status_t consdiffmgr_find_diff_from( const uint8_t *digest, size_t digestlen, compress_method_t method); + +int consensus_cache_entry_get_voter_id_digests( + const struct consensus_cache_entry_t *ent, + smartlist_t *out); +int consensus_cache_entry_get_fresh_until( + const struct consensus_cache_entry_t *ent, + time_t *out); +int consensus_cache_entry_get_valid_until( + const struct consensus_cache_entry_t *ent, + time_t *out); + void consdiffmgr_rescan(void); int consdiffmgr_cleanup(void); +void consdiffmgr_enable_background_compression(void); void consdiffmgr_configure(const consdiff_cfg_t *cfg); struct sandbox_cfg_elem; int consdiffmgr_register_with_sandbox(struct sandbox_cfg_elem **cfg); @@ -40,6 +56,7 @@ int consdiffmgr_validate(void); #ifdef CONSDIFFMGR_PRIVATE STATIC unsigned n_diff_compression_methods(void); +STATIC unsigned n_consensus_compression_methods(void); STATIC consensus_cache_t *cdm_cache_get(void); STATIC consensus_cache_entry_t *cdm_cache_lookup_consensus( consensus_flavor_t flavor, time_t valid_after); diff --git a/src/or/directory.c b/src/or/directory.c index da92e0f897..ef74c0fed4 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -13,6 +13,7 @@ #include "config.h" #include "connection.h" #include "connection_edge.h" +#include "conscache.h" #include "consdiff.h" #include "consdiffmgr.h" #include "control.h" @@ -117,7 +118,8 @@ static void dir_routerdesc_download_failed(smartlist_t *failed, int was_descriptor_digests); static void dir_microdesc_download_failed(smartlist_t *failed, int status_code); -static int client_likes_consensus(networkstatus_t *v, const char *want_url); +static int client_likes_consensus(const struct consensus_cache_entry_t *ent, + const char *want_url); static void connection_dir_close_consensus_fetches( dir_connection_t *except_this_one, const char *resource); @@ -497,11 +499,23 @@ dir_consensus_request_set_additional_headers(directory_request_t *req, * period of 1 hour. */ const int DEFAULT_IF_MODIFIED_SINCE_DELAY = 180; + const int32_t DEFAULT_TRY_DIFF_FOR_CONSENSUS_NEWER = 72; + const int32_t MIN_TRY_DIFF_FOR_CONSENSUS_NEWER = 0; + const int32_t MAX_TRY_DIFF_FOR_CONSENSUS_NEWER = 8192; + const char TRY_DIFF_FOR_CONSENSUS_NEWER_NAME[] = + "try-diff-for-consensus-newer-than"; int flav = FLAV_NS; if (resource) flav = networkstatus_parse_flavor_name(resource); + int32_t max_age_for_diff = 3600 * + networkstatus_get_param(NULL, + TRY_DIFF_FOR_CONSENSUS_NEWER_NAME, + DEFAULT_TRY_DIFF_FOR_CONSENSUS_NEWER, + MIN_TRY_DIFF_FOR_CONSENSUS_NEWER, + MAX_TRY_DIFF_FOR_CONSENSUS_NEWER); + if (flav != -1) { /* IF we have a parsed consensus of this type, we can do an * if-modified-time based on it. */ @@ -517,8 +531,10 @@ dir_consensus_request_set_additional_headers(directory_request_t *req, ims_delay = (v->fresh_until - v->valid_after)/2; } if_modified_since = v->valid_after + ims_delay; - memcpy(or_diff_from, v->digest_sha3_as_signed, DIGEST256_LEN); - or_diff_from_is_set = 1; + if (v->valid_after >= approx_time() - max_age_for_diff) { + memcpy(or_diff_from, v->digest_sha3_as_signed, DIGEST256_LEN); + or_diff_from_is_set = 1; + } } } else { /* Otherwise it might be a consensus we don't parse, but which we @@ -528,8 +544,10 @@ dir_consensus_request_set_additional_headers(directory_request_t *req, * unparsed consensus, so we use the default. */ if (cd) { if_modified_since = cd->published + DEFAULT_IF_MODIFIED_SINCE_DELAY; - memcpy(or_diff_from, cd->digest_sha3_as_signed, DIGEST256_LEN); - or_diff_from_is_set = 1; + if (cd->published >= approx_time() - max_age_for_diff) { + memcpy(or_diff_from, cd->digest_sha3_as_signed, DIGEST256_LEN); + or_diff_from_is_set = 1; + } } } @@ -1667,6 +1685,7 @@ directory_send_command(dir_connection_t *conn, char decorated_address[128]; smartlist_t *headers = smartlist_new(); char *url; + char *accept_encoding; size_t url_len; char request[8192]; size_t request_len, total_request_len = 0; @@ -1723,6 +1742,12 @@ directory_send_command(dir_connection_t *conn, proxystring[0] = 0; } + /* Add Accept-Encoding. */ + accept_encoding = accept_encoding_header(); + smartlist_add_asprintf(headers, "Accept-Encoding: %s\r\n", + accept_encoding); + tor_free(accept_encoding); + /* Add additional headers, if any */ { config_line_t *h; @@ -2050,16 +2075,15 @@ parse_http_response(const char *headers, int *code, time_t *date, if (!strcmpstart(s, "Content-Encoding: ")) { enc = s+18; break; }); - if (!enc || !strcmp(enc, "identity")) { + + if (enc == NULL) *compression = NO_METHOD; - } else if (!strcmp(enc, "deflate") || !strcmp(enc, "x-deflate")) { - *compression = ZLIB_METHOD; - } else if (!strcmp(enc, "gzip") || !strcmp(enc, "x-gzip")) { - *compression = GZIP_METHOD; - } else { - log_info(LD_HTTP, "Unrecognized content encoding: %s. Trying to deal.", - escaped(enc)); - *compression = UNKNOWN_METHOD; + else { + *compression = compression_method_get_by_name(enc); + + if (*compression == UNKNOWN_METHOD) + log_info(LD_HTTP, "Unrecognized content encoding: %s. Trying to deal.", + escaped(enc)); } } SMARTLIST_FOREACH(parsed_headers, char *, s, tor_free(s)); @@ -2301,37 +2325,31 @@ connection_dir_client_reached_eof(dir_connection_t *conn) if (compression == UNKNOWN_METHOD || guessed != compression) { /* Tell the user if we don't believe what we're told about compression.*/ const char *description1, *description2; - if (compression == ZLIB_METHOD) - description1 = "as deflated"; - else if (compression == GZIP_METHOD) - description1 = "as gzipped"; - else if (compression == NO_METHOD) - description1 = "as uncompressed"; - else - description1 = "with an unknown Content-Encoding"; - if (guessed == ZLIB_METHOD) - description2 = "deflated"; - else if (guessed == GZIP_METHOD) - description2 = "gzipped"; - else if (!plausible) + + description1 = compression_method_get_human_name(compression); + + if (BUG(description1 == NULL)) + description1 = compression_method_get_human_name(UNKNOWN_METHOD); + + if (guessed == UNKNOWN_METHOD && !plausible) description2 = "confusing binary junk"; else - description2 = "uncompressed"; + description2 = compression_method_get_human_name(guessed); - log_info(LD_HTTP, "HTTP body from server '%s:%d' was labeled %s, " + log_info(LD_HTTP, "HTTP body from server '%s:%d' was labeled as %s, " "but it seems to be %s.%s", conn->base_.address, conn->base_.port, description1, description2, (compression>0 && guessed>0)?" Trying both.":""); } - /* Try declared compression first if we can. */ - if (compression == GZIP_METHOD || compression == ZLIB_METHOD) + /* Try declared compression first if we can. + * tor_compress_supports_method() also returns true for NO_METHOD. */ + if (tor_compress_supports_method(compression)) tor_uncompress(&new_body, &new_len, body, body_len, compression, !allow_partial, LOG_PROTOCOL_WARN); /* Okay, if that didn't work, and we think that it was compressed * differently, try that. */ - if (!new_body && - (guessed == GZIP_METHOD || guessed == ZLIB_METHOD) && + if (!new_body && tor_compress_supports_method(guessed) && compression != guessed) tor_uncompress(&new_body, &new_len, body, body_len, guessed, !allow_partial, LOG_PROTOCOL_WARN); @@ -3270,8 +3288,9 @@ write_http_response_header_impl(dir_connection_t *conn, ssize_t length, /** As write_http_response_header_impl, but sets encoding and content-typed * based on whether the response will be <b>compressed</b> or not. */ static void -write_http_response_header(dir_connection_t *conn, ssize_t length, - compress_method_t method, long cache_lifetime) +write_http_response_headers(dir_connection_t *conn, ssize_t length, + compress_method_t method, + const char *extra_headers, long cache_lifetime) { const char *methodname = compression_method_get_name(method); const char *doctype; @@ -3282,10 +3301,19 @@ write_http_response_header(dir_connection_t *conn, ssize_t length, write_http_response_header_impl(conn, length, doctype, methodname, - NULL, + extra_headers, cache_lifetime); } +/** As write_http_response_headers, but assumes extra_headers is NULL */ +static void +write_http_response_header(dir_connection_t *conn, ssize_t length, + compress_method_t method, + long cache_lifetime) +{ + write_http_response_headers(conn, length, method, NULL, cache_lifetime); +} + /** Array of compression methods to use (if supported) for serving * precompressed data, ordered from best to worst. */ static compress_method_t srv_meth_pref_precompressed[] = { @@ -3296,6 +3324,15 @@ static compress_method_t srv_meth_pref_precompressed[] = { NO_METHOD }; +/** Array of compression methods to use (if supported) for serving + * streamed data, ordered from best to worst. */ +static compress_method_t srv_meth_pref_streaming_compression[] = { + ZSTD_METHOD, + ZLIB_METHOD, + GZIP_METHOD, + NO_METHOD +}; + /** Parse the compression methods listed in an Accept-Encoding header <b>h</b>, * and convert them to a bitfield where compression method x is supported if * and only if 1 << x is set in the bitfield. */ @@ -3321,6 +3358,38 @@ parse_accept_encoding_header(const char *h) return result; } +/** Array of compression methods to use (if supported) for requesting + * compressed data, ordered from best to worst. */ +static compress_method_t client_meth_pref[] = { + LZMA_METHOD, + ZSTD_METHOD, + ZLIB_METHOD, + GZIP_METHOD, + NO_METHOD +}; + +/** Return a newly allocated string containing a comma separated list of + * supported encodings. */ +STATIC char * +accept_encoding_header(void) +{ + smartlist_t *methods = smartlist_new(); + char *header = NULL; + compress_method_t method; + unsigned i; + + for (i = 0; i < ARRAY_LENGTH(client_meth_pref); ++i) { + method = client_meth_pref[i]; + if (tor_compress_supports_method(method)) + smartlist_add(methods, (char *)compression_method_get_name(method)); + } + + header = smartlist_join_strings(methods, ", ", 0, NULL); + smartlist_free(methods); + + return header; +} + /** Decide whether a client would accept the consensus we have. * * Clients can say they only want a consensus if it's signed by more @@ -3336,42 +3405,39 @@ parse_accept_encoding_header(const char *h) * consensus, 0 otherwise. */ int -client_likes_consensus(networkstatus_t *v, const char *want_url) +client_likes_consensus(const struct consensus_cache_entry_t *ent, + const char *want_url) { - smartlist_t *want_authorities = smartlist_new(); + smartlist_t *voters = smartlist_new(); int need_at_least; int have = 0; + if (consensus_cache_entry_get_voter_id_digests(ent, voters) != 0) { + return 1; // We don't know the voters; assume the client won't mind. */ + } + + smartlist_t *want_authorities = smartlist_new(); dir_split_resource_into_fingerprints(want_url, want_authorities, NULL, 0); need_at_least = smartlist_len(want_authorities)/2+1; - SMARTLIST_FOREACH_BEGIN(want_authorities, const char *, d) { - char want_digest[DIGEST_LEN]; - size_t want_len = strlen(d)/2; - if (want_len > DIGEST_LEN) - want_len = DIGEST_LEN; - - if (base16_decode(want_digest, DIGEST_LEN, d, want_len*2) - != (int) want_len) { - log_fn(LOG_PROTOCOL_WARN, LD_DIR, - "Failed to decode requested authority digest %s.", escaped(d)); - continue; - }; - SMARTLIST_FOREACH_BEGIN(v->voters, networkstatus_voter_info_t *, vi) { - if (smartlist_len(vi->sigs) && - tor_memeq(vi->identity_digest, want_digest, want_len)) { + SMARTLIST_FOREACH_BEGIN(want_authorities, const char *, want_digest) { + + SMARTLIST_FOREACH_BEGIN(voters, const char *, digest) { + if (!strcasecmpstart(digest, want_digest)) { have++; break; }; - } SMARTLIST_FOREACH_END(vi); + } SMARTLIST_FOREACH_END(digest); /* early exit, if we already have enough */ if (have >= need_at_least) break; - } SMARTLIST_FOREACH_END(d); + } SMARTLIST_FOREACH_END(want_digest); SMARTLIST_FOREACH(want_authorities, char *, d, tor_free(d)); smartlist_free(want_authorities); + SMARTLIST_FOREACH(voters, char *, cp, tor_free(cp)); + smartlist_free(voters); return (have >= need_at_least); } @@ -3578,20 +3644,25 @@ handle_get_frontpage(dir_connection_t *conn, const get_handler_args_t *args) return 0; } -/** Warn that the consensus <b>v</b> of type <b>flavor</b> is too old and will - * not be served to clients. Rate-limit the warning to avoid logging an entry - * on every request. +/** Warn that the cached consensus <b>consensus</b> of type + * <b>flavor</b> is too old and will not be served to clients. Rate-limit the + * warning to avoid logging an entry on every request. */ static void -warn_consensus_is_too_old(networkstatus_t *v, const char *flavor, time_t now) +warn_consensus_is_too_old(const struct consensus_cache_entry_t *consensus, + const char *flavor, time_t now) { #define TOO_OLD_WARNING_INTERVAL (60*60) static ratelim_t warned = RATELIM_INIT(TOO_OLD_WARNING_INTERVAL); char timestamp[ISO_TIME_LEN+1]; + time_t valid_until; char *dupes; + if (consensus_cache_entry_get_valid_until(consensus, &valid_until)) + return; + if ((dupes = rate_limit_log(&warned, now))) { - format_local_iso_time(timestamp, v->valid_until); + format_local_iso_time(timestamp, valid_until); log_warn(LD_DIRSERV, "Our %s%sconsensus is too old, so we will not " "serve it to clients. It was valid until %s local time and we " "continued to serve it for up to 24 hours after it expired.%s", @@ -3600,6 +3671,27 @@ warn_consensus_is_too_old(networkstatus_t *v, const char *flavor, time_t now) } } +/** + * Parse a single hex-encoded sha3-256 digest from <b>hex</b> into + * <b>digest</b>. Return 0 on success. On failure, report that the hash came + * from <b>location</b>, report that we are taking <b>action</b> with it, and + * return -1. + */ +static int +parse_one_diff_hash(uint8_t *digest, const char *hex, const char *location, + const char *action) +{ + if (base16_decode((char*)digest, DIGEST256_LEN, hex, strlen(hex)) == + DIGEST256_LEN) { + return 0; + } else { + log_fn(LOG_PROTOCOL_WARN, LD_DIR, + "%s contained bogus digest %s; %s.", + location, escaped(hex), action); + return -1; + } +} + /** If there is an X-Or-Diff-From-Consensus header included in <b>headers</b>, * set <b>digest_out<b> to a new smartlist containing every 256-bit * hex-encoded digest listed in that header and return 0. Otherwise return @@ -3617,13 +3709,9 @@ parse_or_diff_from_header(smartlist_t **digests_out, const char *headers) SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, -1); SMARTLIST_FOREACH_BEGIN(hex_digests, const char *, hex) { uint8_t digest[DIGEST256_LEN]; - if (base16_decode((char*)digest, sizeof(digest), hex, strlen(hex)) == - DIGEST256_LEN) { + if (!parse_one_diff_hash(digest, hex, "X-Or-Diff-From-Consensus header", + "ignoring")) { smartlist_add(*digests_out, tor_memdup(digest, sizeof(digest))); - } else { - log_fn(LOG_PROTOCOL_WARN, LD_DIR, - "X-Or-Diff-From-Consensus header contained bogus digest %s; " - "ignoring.", escaped(hex)); } } SMARTLIST_FOREACH_END(hex); SMARTLIST_FOREACH(hex_digests, char *, cp, tor_free(cp)); @@ -3632,6 +3720,13 @@ parse_or_diff_from_header(smartlist_t **digests_out, const char *headers) return 0; } +/** Fallback compression method. The fallback compression method is used in + * case a client requests a non-compressed document. We only store compressed + * documents, so we use this compression method to fetch the document and let + * the spooling system do the streaming decompression. + */ +#define FALLBACK_COMPRESS_METHOD ZLIB_METHOD + /** * Try to find the best consensus diff possible in order to serve a client * request for a diff from one of the consensuses in <b>digests</b> to the @@ -3661,110 +3756,280 @@ find_best_diff(const smartlist_t *digests, int flav, } } } SMARTLIST_FOREACH_END(diff_from); + + SMARTLIST_FOREACH_BEGIN(digests, const uint8_t *, diff_from) { + if (consdiffmgr_find_diff_from(&result, flav, DIGEST_SHA3_256, diff_from, + DIGEST256_LEN, FALLBACK_COMPRESS_METHOD) == CONSDIFF_AVAILABLE) { + tor_assert_nonfatal(result); + *compression_used_out = FALLBACK_COMPRESS_METHOD; + return result; + } + } SMARTLIST_FOREACH_END(diff_from); + return NULL; } -/** Helper function for GET /tor/status-vote/current/consensus +/** Lookup the cached consensus document by the flavor found in <b>flav</b>. + * The prefered set of compression methods should be listed in the + * <b>compression_methods</b> bitfield. The compression method chosen (if any) + * is stored in <b>compression_used_out</b>. */ +static struct consensus_cache_entry_t * +find_best_consensus(int flav, + unsigned compression_methods, + compress_method_t *compression_used_out) +{ + struct consensus_cache_entry_t *result = NULL; + unsigned u; + + for (u = 0; u < ARRAY_LENGTH(srv_meth_pref_precompressed); ++u) { + compress_method_t method = srv_meth_pref_precompressed[u]; + + if (0 == (compression_methods & (1u<<method))) + continue; + + if (consdiffmgr_find_consensus(&result, flav, + method) == CONSDIFF_AVAILABLE) { + tor_assert_nonfatal(result); + *compression_used_out = method; + return result; + } + } + + if (consdiffmgr_find_consensus(&result, flav, + FALLBACK_COMPRESS_METHOD) == CONSDIFF_AVAILABLE) { + tor_assert_nonfatal(result); + *compression_used_out = FALLBACK_COMPRESS_METHOD; + return result; + } + + return NULL; +} + +/** Try to find the best supported compression method possible from a given + * <b>compression_methods</b>. Return NO_METHOD if no mutually supported + * compression method could be found. */ +static compress_method_t +find_best_compression_method(unsigned compression_methods, int stream) +{ + unsigned u; + compress_method_t *methods; + size_t length; + + if (stream) { + methods = srv_meth_pref_streaming_compression; + length = ARRAY_LENGTH(srv_meth_pref_streaming_compression); + } else { + methods = srv_meth_pref_precompressed; + length = ARRAY_LENGTH(srv_meth_pref_precompressed); + } + + for (u = 0; u < length; ++u) { + compress_method_t method = methods[u]; + if (compression_methods & (1u<<method)) + return method; + } + + return NO_METHOD; +} + +/** Encodes the results of parsing a consensus request to figure out what + * consensus, and possibly what diffs, the user asked for. */ +typedef struct { + /** name of the flavor to retrieve. */ + char *flavor; + /** flavor to retrive, as enum. */ + consensus_flavor_t flav; + /** plus-separated list of authority fingerprints; see + * client_likes_consensus(). Aliases the URL in the request passed to + * parse_consensus_request(). */ + const char *want_fps; + /** Optionally, a smartlist of sha3 digests-as-signed of the consensuses + * to return a diff from. */ + smartlist_t *diff_from_digests; + /** If true, never send a full consensus. If there is no diff, send + * a 404 instead. */ + int diff_only; +} parsed_consensus_request_t; + +/** Remove all data held in <b>req</b>. Do not free <b>req</b> itself, since + * it is stack-allocated. */ +static void +parsed_consensus_request_clear(parsed_consensus_request_t *req) +{ + if (!req) + return; + tor_free(req->flavor); + if (req->diff_from_digests) { + SMARTLIST_FOREACH(req->diff_from_digests, uint8_t *, d, tor_free(d)); + smartlist_free(req->diff_from_digests); + } + memset(req, 0, sizeof(parsed_consensus_request_t)); +} + +/** + * Parse the URL and relevant headers of <b>args</b> for a current-consensus + * request to learn what flavor of consensus we want, what keys it must be + * signed with, and what diffs we would accept (or demand) instead. Return 0 + * on success and -1 on failure. */ static int -handle_get_current_consensus(dir_connection_t *conn, - const get_handler_args_t *args) +parse_consensus_request(parsed_consensus_request_t *out, + const get_handler_args_t *args) { const char *url = args->url; - const int compressed = args->compression_supported & (1u << ZLIB_METHOD); - const time_t if_modified_since = args->if_modified_since; - int clear_spool = 0; + memset(out, 0, sizeof(parsed_consensus_request_t)); + out->flav = FLAV_NS; - /* v3 network status fetch. */ - long lifetime = NETWORKSTATUS_CACHE_LIFETIME; + const char CONSENSUS_URL_PREFIX[] = "/tor/status-vote/current/consensus/"; + const char CONSENSUS_FLAVORED_PREFIX[] = + "/tor/status-vote/current/consensus-"; - networkstatus_t *v; - time_t now = time(NULL); - const char *want_fps = NULL; - char *flavor = NULL; - int flav = FLAV_NS; -#define CONSENSUS_URL_PREFIX "/tor/status-vote/current/consensus/" -#define CONSENSUS_FLAVORED_PREFIX "/tor/status-vote/current/consensus-" /* figure out the flavor if any, and who we wanted to sign the thing */ + const char *after_flavor = NULL; + if (!strcmpstart(url, CONSENSUS_FLAVORED_PREFIX)) { const char *f, *cp; f = url + strlen(CONSENSUS_FLAVORED_PREFIX); cp = strchr(f, '/'); if (cp) { - want_fps = cp+1; - flavor = tor_strndup(f, cp-f); + after_flavor = cp+1; + out->flavor = tor_strndup(f, cp-f); } else { - flavor = tor_strdup(f); + out->flavor = tor_strdup(f); } - flav = networkstatus_parse_flavor_name(flavor); + int flav = networkstatus_parse_flavor_name(out->flavor); if (flav < 0) flav = FLAV_NS; + out->flav = flav; } else { if (!strcmpstart(url, CONSENSUS_URL_PREFIX)) - want_fps = url+strlen(CONSENSUS_URL_PREFIX); + after_flavor = url+strlen(CONSENSUS_URL_PREFIX); + } + + /* see whether we've been asked explicitly for a diff from an older + * consensus. (The user might also have said that a diff would be okay, + * via X-Or-Diff-From-Consensus */ + const char DIFF_COMPONENT[] = "diff/"; + char *diff_hash_in_url = NULL; + if (after_flavor && !strcmpstart(after_flavor, DIFF_COMPONENT)) { + after_flavor += strlen(DIFF_COMPONENT); + const char *cp = strchr(after_flavor, '/'); + if (cp) { + diff_hash_in_url = tor_strndup(after_flavor, cp-after_flavor); + out->want_fps = cp+1; + } else { + diff_hash_in_url = tor_strdup(after_flavor); + out->want_fps = NULL; + } + } else { + out->want_fps = after_flavor; + } + + if (diff_hash_in_url) { + uint8_t diff_from[DIGEST256_LEN]; + out->diff_from_digests = smartlist_new(); + out->diff_only = 1; + int ok = !parse_one_diff_hash(diff_from, diff_hash_in_url, "URL", + "rejecting"); + tor_free(diff_hash_in_url); + if (ok) { + smartlist_add(out->diff_from_digests, + tor_memdup(diff_from, DIGEST256_LEN)); + } else { + return -1; + } + } else { + parse_or_diff_from_header(&out->diff_from_digests, args->headers); } - v = networkstatus_get_latest_consensus_by_flavor(flav); + return 0; +} + +/** Helper function for GET /tor/status-vote/current/consensus + */ +static int +handle_get_current_consensus(dir_connection_t *conn, + const get_handler_args_t *args) +{ + const compress_method_t compress_method = + find_best_compression_method(args->compression_supported, 0); + const time_t if_modified_since = args->if_modified_since; + int clear_spool = 0; + + /* v3 network status fetch. */ + long lifetime = NETWORKSTATUS_CACHE_LIFETIME; + + time_t now = time(NULL); + parsed_consensus_request_t req; + + if (parse_consensus_request(&req, args) < 0) { + write_http_status_line(conn, 404, "Couldn't parse request"); + goto done; + } + + struct consensus_cache_entry_t *cached_consensus = NULL; - if (v && !networkstatus_consensus_reasonably_live(v, now)) { + compress_method_t compression_used = NO_METHOD; + if (req.diff_from_digests) { + cached_consensus = find_best_diff(req.diff_from_digests, req.flav, + args->compression_supported, + &compression_used); + } + + if (req.diff_only && !cached_consensus) { + write_http_status_line(conn, 404, "No such diff available"); + // XXXX warn_consensus_is_too_old(v, req.flavor, now); + geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND); + goto done; + } + + if (! cached_consensus) { + cached_consensus = find_best_consensus(req.flav, + args->compression_supported, + &compression_used); + } + + time_t fresh_until, valid_until; + int have_fresh_until = 0, have_valid_until = 0; + if (cached_consensus) { + have_fresh_until = + !consensus_cache_entry_get_fresh_until(cached_consensus, &fresh_until); + have_valid_until = + !consensus_cache_entry_get_valid_until(cached_consensus, &valid_until); + } + + if (cached_consensus && have_valid_until && + !networkstatus_valid_until_is_reasonably_live(valid_until, now)) { write_http_status_line(conn, 404, "Consensus is too old"); - warn_consensus_is_too_old(v, flavor, now); + warn_consensus_is_too_old(cached_consensus, req.flavor, now); geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND); - tor_free(flavor); goto done; } - if (v && want_fps && - !client_likes_consensus(v, want_fps)) { + if (cached_consensus && req.want_fps && + !client_likes_consensus(cached_consensus, req.want_fps)) { write_http_status_line(conn, 404, "Consensus not signed by sufficient " "number of requested authorities"); geoip_note_ns_response(GEOIP_REJECT_NOT_ENOUGH_SIGS); - tor_free(flavor); goto done; } - struct consensus_cache_entry_t *cached_diff = NULL; - smartlist_t *diff_from_digests = NULL; - compress_method_t compression_used = NO_METHOD; - if (!parse_or_diff_from_header(&diff_from_digests, args->headers)) { - tor_assert(diff_from_digests); - cached_diff = find_best_diff(diff_from_digests, flav, - args->compression_supported, - &compression_used); - SMARTLIST_FOREACH(diff_from_digests, uint8_t *, d, tor_free(d)); - smartlist_free(diff_from_digests); - } - conn->spool = smartlist_new(); clear_spool = 1; { spooled_resource_t *spooled; - if (cached_diff) { - spooled = spooled_resource_new_from_cache_entry(cached_diff); - } else if (flavor) { - spooled = spooled_resource_new(DIR_SPOOL_NETWORKSTATUS, - (uint8_t*)flavor, strlen(flavor)); - compression_used = compressed ? ZLIB_METHOD : NO_METHOD; - } else { - spooled = spooled_resource_new(DIR_SPOOL_NETWORKSTATUS, - NULL, 0); - compression_used = compressed ? ZLIB_METHOD : NO_METHOD; + if (cached_consensus) { + spooled = spooled_resource_new_from_cache_entry(cached_consensus); + smartlist_add(conn->spool, spooled); } - tor_free(flavor); - smartlist_add(conn->spool, spooled); } - lifetime = (v && v->fresh_until > now) ? v->fresh_until - now : 0; - if (!smartlist_len(conn->spool)) { /* we failed to create/cache cp */ - write_http_status_line(conn, 503, "Network status object unavailable"); - geoip_note_ns_response(GEOIP_REJECT_UNAVAILABLE); - goto done; - } + lifetime = (have_fresh_until && fresh_until > now) ? fresh_until - now : 0; size_t size_guess = 0; int n_expired = 0; dirserv_spool_remove_missing_and_guess_size(conn, if_modified_since, - compressed, + compress_method != NO_METHOD, &size_guess, &n_expired); @@ -3802,12 +4067,23 @@ handle_get_current_consensus(dir_connection_t *conn, DIRREQ_DIRECT); } + /* Use this header to tell caches that the response depends on the + * X-Or-Diff-From-Consensus header (or lack thereof). */ + const char vary_header[] = "Vary: X-Or-Diff-From-Consensus\r\n"; + clear_spool = 0; - write_http_response_header(conn, -1, - compression_used, + + // The compress_method might have been NO_METHOD, but we store the data + // compressed. Decompress them using `compression_used`. See fallback code in + // find_best_consensus() and find_best_diff(). + write_http_response_headers(conn, -1, + compress_method == NO_METHOD ? + NO_METHOD : compression_used, + vary_header, smartlist_len(conn->spool) == 1 ? lifetime : 0); - if (! compressed) - conn->compress_state = tor_compress_new(0, ZLIB_METHOD, + + if (compress_method == NO_METHOD && smartlist_len(conn->spool)) + conn->compress_state = tor_compress_new(0, compression_used, HIGH_COMPRESSION); /* Prime the connection with some data. */ @@ -3816,6 +4092,7 @@ handle_get_current_consensus(dir_connection_t *conn, goto done; done: + parsed_consensus_request_clear(&req); if (clear_spool) { dir_conn_clear_spool(conn); } @@ -3828,7 +4105,8 @@ static int handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args) { const char *url = args->url; - const int compressed = args->compression_supported & (1u << ZLIB_METHOD); + const compress_method_t compress_method = + find_best_compression_method(args->compression_supported, 1); { int current; ssize_t body_len = 0; @@ -3885,11 +4163,12 @@ handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args) goto vote_done; } SMARTLIST_FOREACH(dir_items, cached_dir_t *, d, - body_len += compressed ? d->dir_z_len : d->dir_len); + body_len += compress_method != NO_METHOD ? + d->dir_compressed_len : d->dir_len); estimated_len += body_len; SMARTLIST_FOREACH(items, const char *, item, { size_t ln = strlen(item); - if (compressed) { + if (compress_method != NO_METHOD) { estimated_len += ln/2; } else { body_len += ln; estimated_len += ln; @@ -3901,12 +4180,12 @@ handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args) goto vote_done; } write_http_response_header(conn, body_len ? body_len : -1, - compressed ? ZLIB_METHOD : NO_METHOD, + compress_method, lifetime); if (smartlist_len(items)) { - if (compressed) { - conn->compress_state = tor_compress_new(1, ZLIB_METHOD, + if (compress_method != NO_METHOD) { + conn->compress_state = tor_compress_new(1, compress_method, choose_compression_level(estimated_len)); SMARTLIST_FOREACH(items, const char *, c, connection_write_to_buf_compress(c, strlen(c), conn, 0)); @@ -3917,8 +4196,10 @@ handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args) } } else { SMARTLIST_FOREACH(dir_items, cached_dir_t *, d, - connection_write_to_buf(compressed ? d->dir_z : d->dir, - compressed ? d->dir_z_len : d->dir_len, + connection_write_to_buf(compress_method != NO_METHOD ? + d->dir_compressed : d->dir, + compress_method != NO_METHOD ? + d->dir_compressed_len : d->dir_len, TO_CONN(conn))); } vote_done: @@ -3936,7 +4217,8 @@ static int handle_get_microdesc(dir_connection_t *conn, const get_handler_args_t *args) { const char *url = args->url; - const int compressed = args->compression_supported & (1u << ZLIB_METHOD); + const compress_method_t compress_method = + find_best_compression_method(args->compression_supported, 1); int clear_spool = 1; { conn->spool = smartlist_new(); @@ -3947,7 +4229,8 @@ handle_get_microdesc(dir_connection_t *conn, const get_handler_args_t *args) DSR_DIGEST256|DSR_BASE64|DSR_SORT_UNIQ); size_t size_guess = 0; - dirserv_spool_remove_missing_and_guess_size(conn, 0, compressed, + dirserv_spool_remove_missing_and_guess_size(conn, 0, + compress_method != NO_METHOD, &size_guess, NULL); if (smartlist_len(conn->spool) == 0) { write_http_status_line(conn, 404, "Not found"); @@ -3963,11 +4246,11 @@ handle_get_microdesc(dir_connection_t *conn, const get_handler_args_t *args) clear_spool = 0; write_http_response_header(conn, -1, - compressed ? ZLIB_METHOD : NO_METHOD, + compress_method, MICRODESC_CACHE_LIFETIME); - if (compressed) - conn->compress_state = tor_compress_new(1, ZLIB_METHOD, + if (compress_method != NO_METHOD) + conn->compress_state = tor_compress_new(1, compress_method, choose_compression_level(size_guess)); const int initial_flush_result = connection_dirserv_flushed_some(conn); @@ -3988,7 +4271,8 @@ static int handle_get_descriptor(dir_connection_t *conn, const get_handler_args_t *args) { const char *url = args->url; - const int compressed = args->compression_supported & (1u << ZLIB_METHOD); + const compress_method_t compress_method = + find_best_compression_method(args->compression_supported, 1); const or_options_t *options = get_options(); int clear_spool = 1; if (!strcmpstart(url,"/tor/server/") || @@ -4027,8 +4311,8 @@ handle_get_descriptor(dir_connection_t *conn, const get_handler_args_t *args) size_t size_guess = 0; int n_expired = 0; dirserv_spool_remove_missing_and_guess_size(conn, publish_cutoff, - compressed, &size_guess, - &n_expired); + compress_method != NO_METHOD, + &size_guess, &n_expired); /* If we are the bridge authority and the descriptor is a bridge * descriptor, remember that we served this descriptor for desc stats. */ @@ -4058,11 +4342,9 @@ handle_get_descriptor(dir_connection_t *conn, const get_handler_args_t *args) dir_conn_clear_spool(conn); goto done; } - write_http_response_header(conn, -1, - compressed ? ZLIB_METHOD : NO_METHOD, - cache_lifetime); - if (compressed) - conn->compress_state = tor_compress_new(1, ZLIB_METHOD, + write_http_response_header(conn, -1, compress_method, cache_lifetime); + if (compress_method != NO_METHOD) + conn->compress_state = tor_compress_new(1, compress_method, choose_compression_level(size_guess)); clear_spool = 0; /* Prime the connection with some data. */ @@ -4083,7 +4365,8 @@ static int handle_get_keys(dir_connection_t *conn, const get_handler_args_t *args) { const char *url = args->url; - const int compressed = args->compression_supported & (1u << ZLIB_METHOD); + const compress_method_t compress_method = + find_best_compression_method(args->compression_supported, 1); const time_t if_modified_since = args->if_modified_since; { smartlist_t *certs = smartlist_new(); @@ -4146,16 +4429,19 @@ handle_get_keys(dir_connection_t *conn, const get_handler_args_t *args) SMARTLIST_FOREACH(certs, authority_cert_t *, c, len += c->cache_info.signed_descriptor_len); - if (global_write_bucket_low(TO_CONN(conn), compressed?len/2:len, 2)) { + if (global_write_bucket_low(TO_CONN(conn), + compress_method != NO_METHOD ? len/2 : len, + 2)) { write_http_status_line(conn, 503, "Directory busy, try again later"); goto keys_done; } - write_http_response_header(conn, compressed?-1:len, - compressed ? ZLIB_METHOD : NO_METHOD, + write_http_response_header(conn, + compress_method != NO_METHOD ? -1 : len, + compress_method, 60*60); - if (compressed) { - conn->compress_state = tor_compress_new(1, ZLIB_METHOD, + if (compress_method != NO_METHOD) { + conn->compress_state = tor_compress_new(1, compress_method, choose_compression_level(len)); SMARTLIST_FOREACH(certs, authority_cert_t *, c, connection_write_to_buf_compress( diff --git a/src/or/directory.h b/src/or/directory.h index 125333da37..a015c7045d 100644 --- a/src/or/directory.h +++ b/src/or/directory.h @@ -160,6 +160,7 @@ struct get_handler_args_t; STATIC int handle_get_hs_descriptor_v3(dir_connection_t *conn, const struct get_handler_args_t *args); STATIC int directory_handle_command(dir_connection_t *conn); +STATIC char *accept_encoding_header(void); #endif diff --git a/src/or/dirserv.c b/src/or/dirserv.c index 77e99fbc17..2b10a09ead 100644 --- a/src/or/dirserv.c +++ b/src/or/dirserv.c @@ -1184,8 +1184,8 @@ new_cached_dir(char *s, time_t published) d->dir = s; d->dir_len = strlen(s); d->published = published; - if (tor_compress(&(d->dir_z), &(d->dir_z_len), d->dir, d->dir_len, - ZLIB_METHOD)) { + if (tor_compress(&(d->dir_compressed), &(d->dir_compressed_len), + d->dir, d->dir_len, ZLIB_METHOD)) { log_warn(LD_BUG, "Error compressing directory"); } return d; @@ -1196,7 +1196,7 @@ static void clear_cached_dir(cached_dir_t *d) { tor_free(d->dir); - tor_free(d->dir_z); + tor_free(d->dir_compressed); memset(d, 0, sizeof(cached_dir_t)); } @@ -3513,7 +3513,7 @@ spooled_resource_estimate_size(const spooled_resource_t *spooled, if (cached == NULL) { return 0; } - size_t result = compressed ? cached->dir_z_len : cached->dir_len; + size_t result = compressed ? cached->dir_compressed_len : cached->dir_len; return result; } } @@ -3572,8 +3572,8 @@ spooled_resource_flush_some(spooled_resource_t *spooled, int64_t total_len; const char *ptr; if (cached) { - total_len = cached->dir_z_len; - ptr = cached->dir_z; + total_len = cached->dir_compressed_len; + ptr = cached->dir_compressed; } else { total_len = spooled->cce_len; ptr = (const char *)spooled->cce_body; diff --git a/src/or/fallback_dirs.inc b/src/or/fallback_dirs.inc index be94ff50fc..cc37e5f9af 100644 --- a/src/or/fallback_dirs.inc +++ b/src/or/fallback_dirs.inc @@ -1,54 +1,58 @@ -/* Whitelist & blacklist excluded 1177 of 1389 candidates. */ +/* Whitelist & blacklist excluded 1326 of 1513 candidates. */ /* To comment-out entries in this file, use C comments, and add * to the start of each line. (stem finds fallback entries using " at the start of a line.) */ /* Checked IPv4 DirPorts served a consensus within 15.0s. */ /* -Final Count: 177 (Eligible 212, Target 392 (1963 * 0.20), Max 200) -Excluded: 35 (Same Operator 35, Failed/Skipped Download 0, Excess 0) -Bandwidth Range: 1.2 - 107.3 MByte/s +Final Count: 151 (Eligible 187, Target 392 (1963 * 0.20), Max 200) +Excluded: 36 (Same Operator 27, Failed/Skipped Download 9, Excess 0) +Bandwidth Range: 1.3 - 40.0 MByte/s */ /* -Onionoo Source: details Date: 2016-12-19 03:00:00 Version: 3.1 -URL: https:onionoo.torproject.orgdetails?fields=fingerprint%2Cnickname%2Ccontact%2Clast_changed_address_or_port%2Cconsensus_weight%2Cadvertised_bandwidth%2Cor_addresses%2Cdir_address%2Crecommended_version%2Cflags%2Ceffective_family%2Cplatform&flag=V2Dir&type=relay&last_seen_days=-0&first_seen_days=7- +Onionoo Source: details Date: 2017-05-16 07:00:00 Version: 4.0 +URL: https:onionoo.torproject.orgdetails?fields=fingerprint%2Cnickname%2Ccontact%2Clast_changed_address_or_port%2Cconsensus_weight%2Cadvertised_bandwidth%2Cor_addresses%2Cdir_address%2Crecommended_version%2Cflags%2Ceffective_family%2Cplatform&flag=V2Dir&type=relay&last_seen_days=-0&first_seen_days=30- */ /* -Onionoo Source: uptime Date: 2016-12-19 03:00:00 Version: 3.1 -URL: https:onionoo.torproject.orguptime?first_seen_days=7-&flag=V2Dir&type=relay&last_seen_days=-0 +Onionoo Source: uptime Date: 2017-05-16 07:00:00 Version: 4.0 +URL: https:onionoo.torproject.orguptime?first_seen_days=30-&flag=V2Dir&type=relay&last_seen_days=-0 */ -"185.13.39.197:80 orport=443 id=001524DD403D729F08F7E5D77813EF12756CFA8D" +"176.10.104.240:80 orport=443 id=0111BA9B604669E636FFD5B503F382A4B7AD6E80" " weight=10", -"185.100.85.61:80 orport=443 id=025B66CEBC070FCB0519D206CF0CF4965C20C96E" +"193.171.202.146:9030 orport=9001 id=01A9258A46E97FF8B2CAC7910577862C14F2C524" " weight=10", -"62.210.92.11:9030 orport=9001 id=0266B0660F3F20A7D1F3D8335931C95EF50F6C6B" -" ipv6=[2001:bc8:338c::1]:9001" +"185.100.85.61:80 orport=443 id=025B66CEBC070FCB0519D206CF0CF4965C20C96E" " weight=10", "185.97.32.18:9030 orport=9001 id=04250C3835019B26AA6764E85D836088BE441088" " weight=10", -"92.222.20.130:80 orport=443 id=0639612FF149AA19DF3BCEA147E5B8FED6F3C87C" +"5.9.110.236:9030 orport=9001 id=0756B7CD4DFC8182BE23143FAC0642F515182CEB" +" ipv6=[2a01:4f8:162:51e2::2]:9001" +" weight=10", +"109.163.234.8:80 orport=443 id=0818DAE0E2DDF795AEDEAC60B15E71901084F281" " weight=10", "163.172.149.155:80 orport=443 id=0B85617241252517E8ECF2CFC7F4C1A32DCD153F" " weight=10", "5.39.92.199:80 orport=443 id=0BEA4A88D069753218EAAAD6D22EA87B9A1319D6" " ipv6=[2001:41d0:8:b1c7::1]:443" " weight=10", -"163.172.25.118:80 orport=22 id=0CF8F3E6590F45D50B70F2F7DA6605ECA6CD408F" -" weight=10", "178.62.197.82:80 orport=443 id=0D3EBA17E1C78F1E9900BABDB23861D46FCAF163" " weight=10", "185.100.86.100:80 orport=443 id=0E8C0C8315B66DB5F703804B3889A1DD66C67CE0" " weight=10", -"5.9.159.14:9030 orport=9001 id=0F100F60C7A63BED90216052324D29B08CFCF797" +"95.85.8.226:80 orport=443 id=1211AC1BBB8A1AF7CBA86BCE8689AA3146B86423" " weight=10", "193.11.114.43:9030 orport=9001 id=12AD30E5D25AA67F519780E2111E611A455FDC89" " ipv6=[2001:6b0:30:1000::99]:9050" " weight=10", "37.157.195.87:8030 orport=443 id=12FD624EE73CEF37137C90D38B2406A66F68FAA2" " weight=10", +"178.16.208.59:80 orport=443 id=136F9299A5009A4E0E96494E723BDB556FB0A26B" +" ipv6=[2a00:1c20:4089:1234:bff6:e1bb:1ce3:8dc6]:443" +" weight=10", +"144.76.14.145:110 orport=143 id=14419131033443AE6E21DA82B0D307F7CAE42BDB" +" ipv6=[2a01:4f8:190:9490::dead]:443" +" weight=10", "178.62.60.37:80 orport=443 id=175921396C7C426309AB03775A9930B6F611F794" " weight=10", "204.11.50.131:9030 orport=9001 id=185F2A57B0C4620582602761097D17DB81654F70" " weight=10", -"92.222.4.102:9030 orport=9001 id=1A6B8B8272632D8AD38442027F822A367128405C" -" weight=10", "5.9.158.75:80 orport=443 id=1AF72E8906E6C49481A791A6F8F84F8DFEBBB2BA" " ipv6=[2a01:4f8:190:514a::2]:443" " weight=10", @@ -56,31 +60,22 @@ URL: https:onionoo.torproject.orguptime?first_seen_days=7-&flag=V2Dir&type=relay " weight=10", "91.219.237.229:80 orport=443 id=1ECD73B936CB6E6B3CD647CC204F108D9DF2C9F7" " weight=10", -"5.9.146.203:80 orport=443 id=1F45542A24A61BF9408F1C05E0DCE4E29F2CBA11" -" weight=10", "212.47.229.2:9030 orport=9001 id=20462CBA5DA4C2D963567D17D0B7249718114A68" " ipv6=[2001:bc8:4400:2100::f03]:9001" " weight=10", -"91.219.236.222:80 orport=443 id=20704E7DD51501DC303FA51B738D7B7E61397CF6" -" weight=10", "144.76.163.93:9030 orport=9001 id=22F08CF09764C4E8982640D77F71ED72FF26A9AC" " weight=10", "163.172.176.167:80 orport=443 id=230A8B2A8BA861210D9B4BA97745AEC217A94207" " weight=10", +"37.200.98.5:80 orport=443 id=231C2B9C8C31C295C472D031E06964834B745996" +" ipv6=[2a00:1158:3::11a]:993" +" weight=10", "212.47.240.10:82 orport=443 id=2A4C448784F5A83AFE6C78DA357D5E31F7989DEB" " weight=10", "144.76.26.175:9012 orport=9011 id=2BA2C8E96B2590E1072AECE2BDB5C48921BF8510" " weight=10", -"178.16.208.56:80 orport=443 id=2CDCFED0142B28B002E89D305CBA2E26063FADE2" -" ipv6=[2a00:1c20:4089:1234:cd49:b58a:9ebe:67ec]:443" -" weight=10", -"62.210.124.124:9130 orport=9101 id=2EBD117806EE43C3CC885A8F1E4DC60F207E7D3E" -" ipv6=[2001:bc8:3f23:100::1]:9101" -" weight=10", "97.74.237.196:9030 orport=9001 id=2F0F32AB1E5B943CA7D062C03F18960C86E70D94" " weight=10", -"213.61.66.118:9031 orport=9001 id=30648BC64CEDB3020F4A405E4AB2A6347FB8FA22" -" weight=10", "107.170.101.39:9030 orport=443 id=30973217E70AF00EBE51797FF6D9AA720A902EAA" " weight=10", "64.113.32.29:9030 orport=9001 id=30C19B81981F450C402306E2E7CFB6C3F79CB6B2" @@ -90,21 +85,16 @@ URL: https:onionoo.torproject.orguptime?first_seen_days=7-&flag=V2Dir&type=relay "109.105.109.162:52860 orport=60784 id=32EE911D968BE3E016ECA572BB1ED0A9EE43FC2F" " ipv6=[2001:948:7:2::163]:5001" " weight=10", -"185.100.84.212:80 orport=443 id=330CD3DB6AD266DC70CDB512B036957D03D9BC59" -" ipv6=[2a06:1700:0:7::1]:443" -" weight=10", "163.172.13.165:9030 orport=9001 id=33DA0CAB7C27812EFF2E22C9705630A54D101FEB" " ipv6=[2001:bc8:38cb:201::8]:9001" " weight=10", -"45.62.255.25:80 orport=443 id=3473ED788D9E63361D1572B7E82EC54338953D2A" -" weight=10", "217.79.190.25:9030 orport=9090 id=361D33C96D0F161275EE67E2C91EE10B276E778B" " weight=10", "37.187.22.87:9030 orport=9001 id=36B9E7AC1E36B62A9D6F330ABEB6012BA7F0D400" " ipv6=[2001:41d0:a:1657::1]:9001" " weight=10", -"176.126.252.12:21 orport=8080 id=379FB450010D17078B3766C2273303C358C3A442" -" ipv6=[2a02:59e0:0:7::12]:81" +"62.210.92.11:9130 orport=9101 id=387B065A38E4DAA16D9D41C2964ECBC4B31D30FF" +" ipv6=[2001:bc8:338c::1]:9101" " weight=10", "198.50.191.95:80 orport=443 id=39F096961ED2576975C866D450373A9913AFDC92" " weight=10", @@ -112,13 +102,11 @@ URL: https:onionoo.torproject.orguptime?first_seen_days=7-&flag=V2Dir&type=relay " weight=10", "212.47.230.49:9030 orport=9001 id=3D6D0771E54056AEFC28BB1DE816951F11826E97" " weight=10", +"176.10.107.180:9030 orport=9001 id=3D7E274A87D9A89AF064C13D1EE4CA1F184F2600" +" weight=10", "217.79.179.177:9030 orport=9001 id=3E53D3979DB07EFD736661C934A1DED14127B684" " ipv6=[2001:4ba0:fff9:131:6c4f::90d3]:9001" " weight=10", -"212.47.237.95:9030 orport=9001 id=3F5D8A879C58961BB45A3D26AC41B543B40236D6" -" weight=10", -"185.100.85.101:9030 orport=9001 id=4061C553CA88021B8302F0814365070AAE617270" -" weight=10", "178.62.86.96:9030 orport=9001 id=439D0447772CB107B886F7782DBC201FA26B92D1" " ipv6=[2a03:b0c0:1:d0::3cf:7001]:9050" " weight=10", @@ -143,8 +131,6 @@ URL: https:onionoo.torproject.orguptime?first_seen_days=7-&flag=V2Dir&type=relay "81.7.16.182:80 orport=443 id=51E1CF613FD6F9F11FE24743C91D6F9981807D82" " ipv6=[2a02:180:1:1::517:10b6]:993" " weight=10", -"138.201.130.32:9030 orport=9001 id=52AEA31188331F421B2EDB494DB65CD181E5B257" -" weight=10", "94.23.204.175:9030 orport=9001 id=5665A3904C89E22E971305EE8C1997BCA4123C69" " weight=10", "95.130.12.119:80 orport=443 id=587E0A9552E4274B251F29B5B2673D38442EE4BF" @@ -154,37 +140,30 @@ URL: https:onionoo.torproject.orguptime?first_seen_days=7-&flag=V2Dir&type=relay " weight=10", "78.142.142.246:80 orport=443 id=5A5E03355C1908EBF424CAF1F3ED70782C0D2F74" " weight=10", -"46.28.207.19:80 orport=443 id=5B92FA5C8A49D46D235735504C72DBB3472BA321" -" weight=10", "120.29.217.46:80 orport=443 id=5E853C94AB1F655E9C908924370A0A6707508C62" " weight=10", +"109.163.234.5:80 orport=443 id=5EB8D862E70981B8690DEDEF546789E26AB2BD24" +" weight=10", "95.128.43.164:80 orport=443 id=616081EC829593AF4232550DE6FFAA1D75B37A90" " ipv6=[2a02:ec0:209:10::4]:443" " weight=10", -"195.154.122.54:80 orport=443 id=64E99CB34C595A02A3165484BD1215E7389322C6" -" weight=10", "163.172.139.104:8080 orport=443 id=68F175CCABE727AA2D2309BCD8789499CEE36ED7" " weight=10", "85.214.62.48:80 orport=443 id=6A7551EEE18F78A9813096E82BF84F740D32B911" " weight=10", -"95.130.11.147:9030 orport=443 id=6B697F3FF04C26123466A5C0E5D1F8D91925967A" -" weight=10", -"91.121.84.137:4951 orport=4051 id=6DE61A6F72C1E5418A66BFED80DFB63E4C77668F" -" ipv6=[2001:41d0:1:8989::1]:4051" -" weight=10", -"213.61.66.117:9032 orport=9002 id=6E44A52E3D1FF7683FE5C399C3FB5E912DE1C6B4" -" weight=10", "80.127.137.19:80 orport=443 id=6EF897645B79B6CB35E853B32506375014DE3621" " ipv6=[2001:981:47c1:1::6]:443" " weight=10", "95.183.48.12:80 orport=443 id=7187CED1A3871F837D0E60AC98F374AC541CB0DA" " weight=10", -"163.172.35.247:80 orport=443 id=71AB4726D830FAE776D74AEF790CF04D8E0151B4" +"85.214.151.72:9030 orport=9001 id=722D365140C8C52DBB3C9FF6986E3CEFFE2BA812" " weight=10", "85.235.250.88:80 orport=443 id=72B2B12A3F60408BDBC98C6DF53988D3A0B3F0EE" " weight=10", -"46.101.237.246:9030 orport=9001 id=75F1992FD3F403E9C082A5815EB5D12934CDF46C" -" ipv6=[2a03:b0c0:3:d0::208:5001]:9050" +"176.31.191.26:80 orport=443 id=7350AB9ED7568F22745198359373C04AC783C37C" +" weight=10", +"134.119.36.135:80 orport=443 id=763C9556602BD6207771A7A3D958091D44C43228" +" ipv6=[2a00:1158:3::2a8]:993" " weight=10", "188.166.133.133:9030 orport=9001 id=774555642FDC1E1D4FDF2E0C31B7CA9501C5C9C7" " ipv6=[2a03:b0c0:2:d0::5:f001]:9001" @@ -192,12 +171,8 @@ URL: https:onionoo.torproject.orguptime?first_seen_days=7-&flag=V2Dir&type=relay "81.30.158.213:9030 orport=9001 id=789EA6C9AE9ADDD8760903171CFA9AC5741B0C70" " ipv6=[2001:4ba0:cafe:e84::1]:9001" " weight=10", -"185.11.180.67:80 orport=9001 id=794D8EA8343A4E820320265D05D4FA83AB6D1778" -" weight=10", "171.25.193.131:80 orport=443 id=79861CF8522FC637EF046F7688F5289E49D94576" " weight=10", -"62.210.129.246:80 orport=443 id=79E169B25E4C7CE99584F6ED06F379478F23E2B8" -" weight=10", "82.223.21.74:9030 orport=9001 id=7A32C9519D80CA458FC8B034A28F5F6815649A98" " ipv6=[2001:470:53e0::cafe]:9050" " weight=10", @@ -210,9 +185,6 @@ URL: https:onionoo.torproject.orguptime?first_seen_days=7-&flag=V2Dir&type=relay " weight=10", "192.87.28.82:9030 orport=9001 id=844AE9CAD04325E955E2BE1521563B79FE7094B7" " weight=10", -"163.172.138.22:80 orport=443 id=8664DC892540F3C789DB37008236C096C871734D" -" ipv6=[2001:bc8:4400:2100::1:3]:443" -" weight=10", "188.166.23.127:80 orport=443 id=8672E8A01B4D3FA4C0BBE21C740D4506302EA487" " ipv6=[2a03:b0c0:2:d0::27b:7001]:9050" " weight=10", @@ -221,6 +193,7 @@ URL: https:onionoo.torproject.orguptime?first_seen_days=7-&flag=V2Dir&type=relay "212.47.241.21:80 orport=443 id=892F941915F6A0C6E0958E52E0A9685C190CF45C" " weight=10", "163.172.194.53:9030 orport=9001 id=8C00FA7369A7A308F6A137600F0FA07990D9D451" +" ipv6=[2001:bc8:225f:142:6c69:7461:7669:73]:9001" " weight=10", "178.254.44.135:9030 orport=9001 id=8FA37B93397015B2BC5A525C908485260BE9F422" " weight=10", @@ -238,12 +211,6 @@ URL: https:onionoo.torproject.orguptime?first_seen_days=7-&flag=V2Dir&type=relay " weight=10", "204.8.156.142:80 orport=443 id=94C4B7B8C50C86A92B6A20107539EE2678CF9A28" " weight=10", -"176.10.104.243:8080 orport=8443 id=95DA61AEF23A6C851028C1AA88AD8593F659E60F" -" weight=10", -"85.10.202.87:9030 orport=9001 id=971AFB23C168DCD8EDA17473C1C452B359DE3A5A" -" weight=10", -"85.214.206.219:9030 orport=9001 id=98F8D5F359949E41DE8DF3DBB1975A86E96A84A0" -" weight=10", "163.172.223.200:80 orport=443 id=998BF3ED7F70E33D1C307247B9626D9E7573C438" " weight=10", "81.7.10.93:31336 orport=31337 id=99E246DB480B313A3012BC3363093CC26CD209C7" @@ -252,18 +219,20 @@ URL: https:onionoo.torproject.orguptime?first_seen_days=7-&flag=V2Dir&type=relay " weight=10", "66.111.2.20:9030 orport=9001 id=9A68B85A02318F4E7E87F2828039FBD5D75B0142" " weight=10", -"5.35.251.247:9030 orport=9001 id=9B1F5187DFBA89DC24B37EA7BF896C12B43A27AE" +"185.100.86.128:9030 orport=9001 id=9B31F1F1C1554F9FFB3455911F82E818EF7C7883" " weight=10", "5.9.151.241:9030 orport=4223 id=9BF04559224F0F1C3C953D641F1744AF0192543A" +" ipv6=[2a01:4f8:190:34f0::2]:4223" " weight=10", "86.105.212.130:9030 orport=443 id=9C900A7F6F5DD034CFFD192DAEC9CCAA813DB022" " weight=10", -"146.185.177.103:80 orport=9030 id=9EC5E097663862DF861A18C32B37C5F82284B27D" -" weight=10", "178.254.20.134:80 orport=443 id=9F5068310818ED7C70B0BC4087AB55CB12CB4377" " weight=10", "46.28.110.244:80 orport=443 id=9F7D6E6420183C2B76D3CE99624EBC98A21A967E" " weight=10", +"91.121.84.137:4952 orport=4052 id=9FBEB75E8BC142565F12CBBE078D63310236A334" +" ipv6=[2001:41d0:1:8989::1]:4052" +" weight=10", "178.62.22.36:80 orport=443 id=A0766C0D3A667A3232C7D569DE94A28F9922FCB1" " ipv6=[2a03:b0c0:1:d0::174:1]:9050" " weight=10", @@ -273,47 +242,38 @@ URL: https:onionoo.torproject.orguptime?first_seen_days=7-&flag=V2Dir&type=relay "171.25.193.78:80 orport=443 id=A478E421F83194C114F41E94F95999672AED51FE" " ipv6=[2001:67c:289c:3::78]:443" " weight=10", -"178.16.208.58:80 orport=443 id=A4C98CEA3F34E05299417E9F885A642C88EF6029" -" ipv6=[2a00:1c20:4089:1234:cdae:1b3e:cc38:3d45]:443" -" weight=10", "163.172.149.122:80 orport=443 id=A9406A006D6E7B5DA30F2C6D4E42A338B5E340B2" " weight=10", -"213.61.66.116:9033 orport=9003 id=A9DEB920B42B4EC1DE6249034039B06D61F38690" -" weight=10", "192.34.63.137:9030 orport=443 id=ABCB4965F1FEE193602B50A365425105C889D3F8" " weight=10", -"195.154.164.243:80 orport=443 id=AC66FFA4AB35A59EBBF5BF4C70008BF24D8A7A5C" +"109.163.234.9:80 orport=443 id=ABF7FBF389C9A747938B639B20E80620B460B2A9" " weight=10", "86.59.119.88:80 orport=443 id=ACD889D86E02EDDAB1AFD81F598C0936238DC6D0" " weight=10", "185.129.62.62:9030 orport=9001 id=ACDD9E85A05B127BA010466C13C8C47212E8A38F" " ipv6=[2a06:d380:0:3700::62]:9001" " weight=10", -"188.40.128.246:9030 orport=9001 id=AD19490C7DBB26D3A68EFC824F67E69B0A96E601" -" weight=10", "163.172.131.88:80 orport=443 id=AD253B49E303C6AB1E048B014392AC569E8A7DAE" " ipv6=[2001:bc8:4400:2100::2:1009]:443" " weight=10", -"176.10.104.240:8080 orport=8443 id=AD86CD1A49573D52A7B6F4A35750F161AAD89C88" -" weight=10", "31.185.104.20:80 orport=443 id=ADB2C26629643DBB9F8FE0096E7D16F9414B4F8D" " weight=10", "37.187.7.74:80 orport=443 id=AEA43CB1E47BE5F8051711B2BF01683DB1568E05" " ipv6=[2001:41d0:a:74a::1]:443" " weight=10", -"176.126.252.11:443 orport=9001 id=B0279A521375F3CB2AE210BDBFC645FDD2E1973A" -" ipv6=[2a02:59e0:0:7::11]:9003" +"46.28.205.170:80 orport=443 id=AF322D83A4D2048B22F7F1AF5F38AFF4D09D0B76" " weight=10", -"212.129.62.232:80 orport=443 id=B143D439B72D239A419F8DCE07B8A8EB1B486FA7" +"5.9.147.226:9030 orport=9001 id=B0553175AADB0501E5A61FC61CEA3970BE130FF2" " weight=10", -"185.66.250.141:9030 orport=9001 id=B1726B94885CE3AC3910CA8B60622B97B98E2529" +"212.129.62.232:80 orport=443 id=B143D439B72D239A419F8DCE07B8A8EB1B486FA7" " weight=10", "198.199.64.217:80 orport=443 id=B1D81825CFD7209BD1B4520B040EF5653C204A23" " ipv6=[2604:a880:400:d0::1a9:b001]:9050" " weight=10", "136.243.214.137:80 orport=443 id=B291D30517D23299AD7CEE3E60DFE60D0E3A4664" " weight=10", -"212.47.233.86:9030 orport=9001 id=B4CAFD9CBFB34EC5DAAC146920DC7DFAFE91EA20" +"178.16.208.60:80 orport=443 id=B44FBE5366AD98B46D829754FA4AC599BAE41A6A" +" ipv6=[2a00:1c20:4089:1234:67bc:79f3:61c0:6e49]:443" " weight=10", "93.115.97.242:9030 orport=9001 id=B5212DB685A2A0FCFBAE425738E478D12361710D" " weight=10", @@ -325,20 +285,14 @@ URL: https:onionoo.torproject.orguptime?first_seen_days=7-&flag=V2Dir&type=relay "85.248.227.164:444 orport=9002 id=B84F248233FEA90CAD439F292556A3139F6E1B82" " ipv6=[2a00:1298:8011:212::164]:9004" " weight=10", -"197.231.221.211:9030 orport=9001 id=BC630CBBB518BE7E9F4E09712AB0269E9DC7D626" -" weight=10", "89.163.247.43:9030 orport=9001 id=BC7ACFAC04854C77167C7D66B7E471314ED8C410" +" ipv6=[2001:4ba0:fff7:25::5]:9001" " weight=10", "198.96.155.3:8080 orport=5001 id=BCEDF6C193AA687AE471B8A22EBF6BC57C2D285E" " weight=10", "128.199.55.207:9030 orport=9001 id=BCEF908195805E03E92CCFE669C48738E556B9C5" " ipv6=[2a03:b0c0:2:d0::158:3001]:9001" " weight=10", -"148.251.190.229:9030 orport=9010 id=BF0FB582E37F738CD33C3651125F2772705BB8E8" -" ipv6=[2a01:4f8:211:c68::2]:9010" -" weight=10", -"163.172.35.249:80 orport=443 id=C08DE49658E5B3CFC6F2A952B453C4B608C9A16A" -" weight=10", "185.35.202.221:9030 orport=9001 id=C13B91384CDD52A871E3ECECE4EF74A7AC7DCB08" " ipv6=[2a02:ed06::221]:9001" " weight=10", @@ -347,16 +301,21 @@ URL: https:onionoo.torproject.orguptime?first_seen_days=7-&flag=V2Dir&type=relay " weight=10", "188.138.112.60:1433 orport=1521 id=C414F28FD2BEC1553024299B31D4E726BEB8E788" " weight=10", -"37.59.46.159:9030 orport=9001 id=CBD0D1BD110EC52963082D839AC6A89D0AE243E7" +"85.248.227.163:443 orport=9001 id=C793AB88565DDD3C9E4C6F15CCB9D8C7EF964CE9" +" ipv6=[2a00:1298:8011:212::163]:9003" " weight=10", "178.62.199.226:80 orport=443 id=CBEFF7BA4A4062045133C053F2D70524D8BBE5BE" " ipv6=[2a03:b0c0:2:d0::b7:5001]:443" " weight=10", "134.119.3.164:9030 orport=9001 id=D1B8AAA98C65F3DF7D8BB3AF881CAEB84A33D8EE" " weight=10", -"185.13.38.75:9030 orport=9001 id=D2A1703758A0FBBA026988B92C2F88BAB59F9361" +"31.171.155.108:9030 orport=9001 id=D3E5EDDBE5159388704D6785BE51930AAFACEC6F" " weight=10", -"37.221.162.226:9030 orport=9001 id=D64366987CB39F61AD21DBCF8142FA0577B92811" +"37.187.115.157:9030 orport=9001 id=D5039E1EBFD96D9A3F9846BF99EC9F75EDDE902A" +" weight=10", +"166.82.21.200:9030 orport=9029 id=D5C33F3E203728EDF8361EA868B2939CCC43FAFB" +" weight=10", +"185.14.185.240:9030 orport=443 id=D62FB817B0288085FAC38A6DC8B36DCD85B70260" " weight=10", "46.101.169.151:9030 orport=9001 id=D760C5B436E42F93D77EF2D969157EEA14F9B39C" " ipv6=[2a03:b0c0:3:d0::74f:a001]:9001" @@ -371,34 +330,26 @@ URL: https:onionoo.torproject.orguptime?first_seen_days=7-&flag=V2Dir&type=relay "178.62.173.203:9030 orport=9001 id=DD85503F2D1F52EF9EAD621E942298F46CD2FC10" " ipv6=[2a03:b0c0:0:1010::a4:b001]:9001" " weight=10", -"83.212.99.68:80 orport=443 id=DDBB2A38252ADDA53E4492DDF982CA6CC6E10EC0" -" ipv6=[2001:648:2ffc:1225:a800:bff:fe3d:67b5]:443" -" weight=10", "5.34.183.205:80 orport=443 id=DDD7871C1B7FA32CB55061E08869A236E61BDDF8" " weight=10", -"167.114.66.61:9696 orport=443 id=DE6CD5F09DF26076F26321B0BDFBE78ACD935C65" -" ipv6=[2607:5300:100::78d]:443" -" weight=10", "78.24.75.53:9030 orport=9001 id=DEB73705B2929AE9BE87091607388939332EF123" " weight=10", "92.222.38.67:80 orport=443 id=DED6892FF89DBD737BA689698A171B2392EB3E82" " weight=10", -"217.12.199.208:80 orport=443 id=DF3AED4322B1824BF5539AE54B2D1B38E080FF05" -" ipv6=[2a02:27a8:0:2::7e]:443" +"166.70.207.2:9030 orport=9001 id=E3DB2E354B883B59E8DC56B3E7A353DDFD457812" " weight=10", -"167.114.35.28:9030 orport=9001 id=E65D300F11E1DB12C534B0146BDAB6972F1A8A48" +"46.252.26.2:45212 orport=49991 id=E589316576A399C511A9781A73DA4545640B479D" " weight=10", -"212.47.244.38:8080 orport=443 id=E81EF60A73B3809F8964F73766B01BAA0A171E20" +"167.114.35.28:9030 orport=9001 id=E65D300F11E1DB12C534B0146BDAB6972F1A8A48" " weight=10", "131.188.40.188:443 orport=80 id=EBE718E1A49EE229071702964F8DB1F318075FF8" " weight=10", -"89.40.71.149:8081 orport=8080 id=EC639EDAA5121B47DBDF3D6B01A22E48A8CB6CC7" -" weight=10", "192.87.28.28:9030 orport=9001 id=ED2338CAC2711B3E331392E1ED2831219B794024" " weight=10", -"212.83.40.238:9030 orport=9001 id=F409FA7902FD89270E8DE0D7977EA23BC38E5887" +"192.99.212.139:80 orport=443 id=F10BDE279AE71515DDCCCC61DC19AC8765F8A3CC" " weight=10", -"5.199.142.236:9030 orport=9001 id=F4C0EDAA0BF0F7EC138746F8FEF1CE26C7860265" +"212.238.208.48:9030 orport=9001 id=F406219CDD339026D160E53FCA0EF6857C70F109" +" ipv6=[2001:984:a8fb:1:ba27:ebff:feac:c109]:9001" " weight=10", "46.28.207.141:80 orport=443 id=F69BED36177ED727706512BA6A97755025EEA0FB" " weight=10", @@ -408,14 +359,10 @@ URL: https:onionoo.torproject.orguptime?first_seen_days=7-&flag=V2Dir&type=relay " weight=10", "185.96.180.29:80 orport=443 id=F93D8F37E35C390BCAD9F9069E13085B745EC216" " weight=10", -"104.243.35.196:9030 orport=9001 id=FA3415659444AE006E7E9E5375E82F29700CFDFD" -" weight=10", "86.59.119.83:80 orport=443 id=FC9AC8EA0160D88BCCFDE066940D7DD9FA45495B" " weight=10", "192.187.124.98:9030 orport=9001 id=FD1871854BFC06D7B02F10742073069F0528B5CC" " weight=10", -"212.129.38.254:9030 orport=9001 id=FDF845FC159C0020E2BDDA120C30C5C5038F74B4" -" weight=10", "149.56.45.200:9030 orport=9001 id=FE296180018833AF03A8EACD5894A614623D3F76" " weight=10", "193.11.164.243:9030 orport=9001 id=FFA72BD683BC2FCF988356E6BEC1E490F313FB07" diff --git a/src/or/main.c b/src/or/main.c index f1a8cfb96e..398c4f63d2 100644 --- a/src/or/main.c +++ b/src/or/main.c @@ -2482,6 +2482,7 @@ do_main_loop(void) /* launch cpuworkers. Need to do this *after* we've read the onion key. */ cpu_init(); } + consdiffmgr_enable_background_compression(); /* Setup shared random protocol subsystem. */ if (authdir_mode_v3(get_options())) { @@ -3005,6 +3006,7 @@ tor_init(int argc, char *argv[]) rep_hist_init(); /* Initialize the service cache. */ rend_cache_init(); + hs_cache_init(); addressmap_init(); /* Init the client dns cache. Do it always, since it's * cheap. */ @@ -3665,6 +3667,9 @@ tor_main(int argc, char *argv[]) int result = 0; #ifdef _WIN32 +#ifndef HeapEnableTerminationOnCorruption +#define HeapEnableTerminationOnCorruption 1 +#endif /* On heap corruption, just give up; don't try to play along. */ HeapSetInformation(NULL, HeapEnableTerminationOnCorruption, NULL, 0); /* Call SetProcessDEPPolicy to permanently enable DEP. diff --git a/src/or/networkstatus.c b/src/or/networkstatus.c index bd106fd9a6..fffd1078be 100644 --- a/src/or/networkstatus.c +++ b/src/or/networkstatus.c @@ -1407,16 +1407,24 @@ networkstatus_get_live_consensus,(time_t now)) * Return 1 if the consensus is reasonably live, or 0 if it is too old. */ int -networkstatus_consensus_reasonably_live(networkstatus_t *consensus, time_t now) +networkstatus_consensus_reasonably_live(const networkstatus_t *consensus, + time_t now) { -#define REASONABLY_LIVE_TIME (24*60*60) if (BUG(!consensus)) return 0; - if (now <= consensus->valid_until + REASONABLY_LIVE_TIME) - return 1; + return networkstatus_valid_until_is_reasonably_live(consensus->valid_until, + now); +} - return 0; +/** As networkstatus_consensus_reasonably_live, but takes a valid_until + * time rather than an entire consensus. */ +int +networkstatus_valid_until_is_reasonably_live(time_t valid_until, + time_t now) +{ +#define REASONABLY_LIVE_TIME (24*60*60) + return (now <= valid_until + REASONABLY_LIVE_TIME); } /* XXXX remove this in favor of get_live_consensus. But actually, diff --git a/src/or/networkstatus.h b/src/or/networkstatus.h index 37a5a3b7a0..e774c4d266 100644 --- a/src/or/networkstatus.h +++ b/src/or/networkstatus.h @@ -81,8 +81,10 @@ MOCK_DECL(networkstatus_t *,networkstatus_get_latest_consensus,(void)); MOCK_DECL(networkstatus_t *,networkstatus_get_latest_consensus_by_flavor, (consensus_flavor_t f)); MOCK_DECL(networkstatus_t *, networkstatus_get_live_consensus,(time_t now)); -int networkstatus_consensus_reasonably_live(networkstatus_t *consensus, +int networkstatus_consensus_reasonably_live(const networkstatus_t *consensus, time_t now); +int networkstatus_valid_until_is_reasonably_live(time_t valid_until, + time_t now); networkstatus_t *networkstatus_get_reasonably_live_consensus(time_t now, int flavor); MOCK_DECL(int, networkstatus_consensus_is_bootstrapping,(time_t now)); diff --git a/src/or/or.h b/src/or/or.h index 41d9e8445f..0d29e19d75 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -1934,9 +1934,9 @@ typedef struct addr_policy_t { * compressed form. */ typedef struct cached_dir_t { char *dir; /**< Contents of this object, NUL-terminated. */ - char *dir_z; /**< Compressed contents of this object. */ + char *dir_compressed; /**< Compressed contents of this object. */ size_t dir_len; /**< Length of <b>dir</b> (not counting its NUL). */ - size_t dir_z_len; /**< Length of <b>dir_z</b>. */ + size_t dir_compressed_len; /**< Length of <b>dir_compressed</b>. */ time_t published; /**< When was this object published. */ common_digests_t digests; /**< Digests of this object (networkstatus only) */ /** Sha3 digest (also ns only) */ diff --git a/src/or/protover.c b/src/or/protover.c index ee58927fc5..1a3e69be10 100644 --- a/src/or/protover.c +++ b/src/or/protover.c @@ -288,7 +288,7 @@ protover_get_supported_protocols(void) return "Cons=1-2 " "Desc=1-2 " - "DirCache=1 " + "DirCache=1-2 " "HSDir=1-2 " "HSIntro=3-4 " "HSRend=1-2 " diff --git a/src/test/test_consdiffmgr.c b/src/test/test_consdiffmgr.c index 9d0c5b5b71..746d17a038 100644 --- a/src/test/test_consdiffmgr.c +++ b/src/test/test_consdiffmgr.c @@ -28,7 +28,7 @@ consdiffmgr_test_setup(const struct testcase_t *arg) get_options_mutable()->DataDirectory = ddir_fname; // now owns the pointer. check_private_dir(ddir_fname, CPD_CREATE, NULL); - consdiff_cfg_t consdiff_cfg = { 7200, 300 }; + consdiff_cfg_t consdiff_cfg = { 300 }; consdiffmgr_configure(&consdiff_cfg); return (void *)1; // must return something non-null. } @@ -759,9 +759,11 @@ test_consdiffmgr_cleanup_old_diffs(void *arg) consensus_cache_entry_incref(hold_ent); // incref, so it is preserved. /* Now add an even-more-recent consensus; this should make all previous - * diffs deletable */ + * diffs deletable, and make delete */ tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[3], md_ns[3])); - tt_int_op(2 * n_diff_compression_methods(), OP_EQ, consdiffmgr_cleanup()); + tt_int_op(2 * n_diff_compression_methods() + + (n_consensus_compression_methods() - 1) , OP_EQ, + consdiffmgr_cleanup()); tt_int_op(CONSDIFF_NOT_FOUND, OP_EQ, lookup_diff_from(&ent, FLAV_MICRODESC, md_body[0])); diff --git a/src/test/test_dir_handle_get.c b/src/test/test_dir_handle_get.c index c98938b2db..75fe6249ad 100644 --- a/src/test/test_dir_handle_get.c +++ b/src/test/test_dir_handle_get.c @@ -12,6 +12,7 @@ #include "or.h" #include "config.h" #include "connection.h" +#include "consdiffmgr.h" #include "directory.h" #include "test.h" #include "compress.h" @@ -63,6 +64,7 @@ new_dir_conn(void) { dir_connection_t *conn = dir_connection_new(AF_INET); tor_addr_from_ipv4h(&conn->base_.addr, 0x7f000001); + TO_CONN(conn)->address = tor_strdup("127.0.0.1"); return conn; } @@ -465,6 +467,8 @@ init_mock_options(void) mock_options = tor_malloc(sizeof(or_options_t)); memset(mock_options, 0, sizeof(or_options_t)); mock_options->TestingTorNetwork = 1; + mock_options->DataDirectory = tor_strdup(get_fname_rnd("datadir_tmp")); + check_private_dir(mock_options->DataDirectory, CPD_CREATE, NULL); } static const or_options_t * @@ -501,14 +505,6 @@ test_dir_handle_get_micro_d(void *data) /* SETUP */ init_mock_options(); - const char *fn = get_fname("dir_handle_datadir_test1"); - mock_options->DataDirectory = tor_strdup(fn); - -#ifdef _WIN32 - tt_int_op(0, OP_EQ, mkdir(mock_options->DataDirectory)); -#else - tt_int_op(0, OP_EQ, mkdir(mock_options->DataDirectory, 0700)); -#endif /* Add microdesc to cache */ crypto_digest256(digest, microdesc, strlen(microdesc), DIGEST_SHA256); @@ -568,14 +564,6 @@ test_dir_handle_get_micro_d_server_busy(void *data) /* SETUP */ init_mock_options(); - const char *fn = get_fname("dir_handle_datadir_test2"); - mock_options->DataDirectory = tor_strdup(fn); - -#ifdef _WIN32 - tt_int_op(0, OP_EQ, mkdir(mock_options->DataDirectory)); -#else - tt_int_op(0, OP_EQ, mkdir(mock_options->DataDirectory, 0700)); -#endif /* Add microdesc to cache */ crypto_digest256(digest, microdesc, strlen(microdesc), DIGEST_SHA256); @@ -1621,8 +1609,13 @@ test_dir_handle_get_status_vote_current_consensus_ns_not_enough_sigs(void* d) /* init mock */ mock_ns_val = tor_malloc_zero(sizeof(networkstatus_t)); mock_ns_val->flavor = FLAV_NS; + mock_ns_val->type = NS_TYPE_CONSENSUS; mock_ns_val->voters = smartlist_new(); - mock_ns_val->valid_until = time(NULL); + mock_ns_val->valid_after = time(NULL) - 1800; + mock_ns_val->valid_until = time(NULL) - 60; + + #define NETWORK_STATUS "some network status string" + consdiffmgr_add_consensus(NETWORK_STATUS, mock_ns_val); /* init mock */ init_mock_options(); @@ -1710,10 +1703,17 @@ test_dir_handle_get_status_vote_current_consensus_too_old(void *data) (void)data; mock_ns_val = tor_malloc_zero(sizeof(networkstatus_t)); + mock_ns_val->type = NS_TYPE_CONSENSUS; mock_ns_val->flavor = FLAV_MICRODESC; - mock_ns_val->valid_until = time(NULL) - (60 * 60 * 24) - 1; + mock_ns_val->valid_after = time(NULL) - (24 * 60 * 60 + 1800); + mock_ns_val->fresh_until = time(NULL) - (24 * 60 * 60 + 900); + mock_ns_val->valid_until = time(NULL) - (24 * 60 * 60 + 20); + + #define NETWORK_STATUS "some network status string" + consdiffmgr_add_consensus(NETWORK_STATUS, mock_ns_val); init_mock_options(); + MOCK(get_options, mock_get_options); MOCK(connection_write_to_buf_impl_, connection_write_to_buf_mock); MOCK(networkstatus_get_latest_consensus_by_flavor, mock_ns_get_by_flavor); @@ -1734,6 +1734,17 @@ test_dir_handle_get_status_vote_current_consensus_too_old(void *data) tor_free(header); teardown_capture_of_logs(); + tor_free(mock_ns_val); + + mock_ns_val = tor_malloc_zero(sizeof(networkstatus_t)); + mock_ns_val->type = NS_TYPE_CONSENSUS; + mock_ns_val->flavor = FLAV_NS; + mock_ns_val->valid_after = time(NULL) - (24 * 60 * 60 + 1800); + mock_ns_val->fresh_until = time(NULL) - (24 * 60 * 60 + 900); + mock_ns_val->valid_until = time(NULL) - (24 * 60 * 60 + 20); + + #define NETWORK_STATUS "some network status string" + consdiffmgr_add_consensus(NETWORK_STATUS, mock_ns_val); setup_capture_of_logs(LOG_WARN); @@ -1772,16 +1783,26 @@ static void status_vote_current_consensus_ns_test(char **header, char **body, size_t *body_len) { - common_digests_t digests; - uint8_t sha3[DIGEST256_LEN]; dir_connection_t *conn = NULL; #define NETWORK_STATUS "some network status string" +#if 0 + common_digests_t digests; + uint8_t sha3[DIGEST256_LEN]; memset(&digests, 0x60, sizeof(digests)); memset(sha3, 0x06, sizeof(sha3)); dirserv_set_cached_consensus_networkstatus(NETWORK_STATUS, "ns", &digests, sha3, time(NULL)); +#endif + networkstatus_t *ns = tor_malloc_zero(sizeof(networkstatus_t)); + ns->type = NS_TYPE_CONSENSUS; + ns->flavor = FLAV_NS; + ns->valid_after = time(NULL) - 1800; + ns->fresh_until = time(NULL) - 900; + ns->valid_until = time(NULL) - 60; + consdiffmgr_add_consensus(NETWORK_STATUS, ns); + networkstatus_vote_free(ns); MOCK(connection_write_to_buf_impl_, connection_write_to_buf_mock); @@ -1794,7 +1815,6 @@ status_vote_current_consensus_ns_test(char **header, char **body, tt_str_op("ab", OP_EQ, geoip_get_country_name(1)); conn = new_dir_conn(); - TO_CONN(conn)->address = tor_strdup("127.0.0.1"); tt_int_op(0, OP_EQ, directory_handle_command_get(conn, GET("/tor/status-vote/current/consensus-ns"), NULL, 0)); @@ -2592,11 +2612,11 @@ struct testcase_t dir_handle_get_tests[] = { DIR_HANDLE_CMD(status_vote_current_authority, 0), DIR_HANDLE_CMD(status_vote_next_authority_not_found, 0), DIR_HANDLE_CMD(status_vote_next_authority, 0), - DIR_HANDLE_CMD(status_vote_current_consensus_ns_not_enough_sigs, 0), - DIR_HANDLE_CMD(status_vote_current_consensus_ns_not_found, 0), - DIR_HANDLE_CMD(status_vote_current_consensus_too_old, 0), - DIR_HANDLE_CMD(status_vote_current_consensus_ns_busy, 0), - DIR_HANDLE_CMD(status_vote_current_consensus_ns, 0), + DIR_HANDLE_CMD(status_vote_current_consensus_ns_not_enough_sigs, TT_FORK), + DIR_HANDLE_CMD(status_vote_current_consensus_ns_not_found, TT_FORK), + DIR_HANDLE_CMD(status_vote_current_consensus_too_old, TT_FORK), + DIR_HANDLE_CMD(status_vote_current_consensus_ns_busy, TT_FORK), + DIR_HANDLE_CMD(status_vote_current_consensus_ns, TT_FORK), DIR_HANDLE_CMD(status_vote_current_d_not_found, 0), DIR_HANDLE_CMD(status_vote_next_d_not_found, 0), DIR_HANDLE_CMD(status_vote_d, 0), |