Add Fallback Directory Candidate Selection Script

"Tor has included a feature to fetch the initial consensus from nodes
 other than the authorities for a while now. We just haven't shipped a
 list of alternate locations for clients to go to yet.

 Reasons why we might want to ship tor with a list of additional places
 where clients can find the consensus is that it makes authority
 reachability and BW less important.

 We want them to have been around and using their current key, address,
 and port for a while now (120 days), and have been running, a guard,
 and a v2 directory mirror for most of that time."

Features:
* whitelist and blacklist for an opt-in/opt-out trial.
* excludes BadExits, tor versions that aren't recommended, and low
  consensus weight directory mirrors.
* reduces the weighting of Exits to avoid overloading them.
* places limits on the weight of any one fallback.
* includes an IPv6 address and orport for each FallbackDir, as
  implemented in #17327. (Tor won't bootstrap using IPv6 fallbacks
  until #17840 is merged.)
* generated output includes timestamps & Onionoo URL for traceability.
* unit test ensures that we successfully load all included default
  fallback directories.

Closes ticket #15775. Patch by "teor".
OnionOO script by "weasel", "teor", "gsathya", and "karsten".

5 files changed, 40 insertions, 22 deletions

diff --git a/src/or/config.c b/src/or/config.c
index 7b42c9fdb3..148fa9fe04 100644
--- a/src/or/config.c
+++ b/src/or/config.c
@@ -916,6 +916,7 @@ add_default_fallback_dir_servers,(void))
 {
   int i;
   const char *fallback[] = {
+#include "fallback_dirs.inc"
     NULL
   };
   for (i=0; fallback[i]; i++) {
diff --git a/src/or/fallback_dirs.inc b/src/or/fallback_dirs.inc
new file mode 100644
index 0000000000..d9214cb884
--- /dev/null
+++ b/src/or/fallback_dirs.inc
@@ -0,0 +1 @@
+/* This list will be empty until opt-ins are finalised. */
diff --git a/src/or/include.am b/src/or/include.am
index 264c4ae802..7371e55324 100644
--- a/src/or/include.am
+++ b/src/or/include.am
@@ -158,6 +158,7 @@ ORHEADERS = \
 	src/or/dnsserv.h				\
 	src/or/eventdns_tor.h				\
 	src/or/ext_orport.h				\
+	src/or/fallback_dirs.inc			\
 	src/or/fp_pair.h				\
 	src/or/geoip.h					\
 	src/or/entrynodes.h				\
diff --git a/src/or/routerlist.c b/src/or/routerlist.c
index 5e7906475f..638c6c551d 100644
--- a/src/or/routerlist.c
+++ b/src/or/routerlist.c
@@ -1295,8 +1295,8 @@ router_get_fallback_dir_servers(void)
 /** Try to find a running dirserver that supports operations of <b>type</b>.
  *
  * If there are no running dirservers in our routerlist and the
- * <b>PDS_RETRY_IF_NO_SERVERS</b> flag is set, set all the authoritative ones
- * as running again, and pick one.
+ * <b>PDS_RETRY_IF_NO_SERVERS</b> flag is set, set all the fallback ones
+ * (including authorities) as running again, and pick one.
  *
  * If the <b>PDS_IGNORE_FASCISTFIREWALL</b> flag is set, then include
  * dirservers that we can't reach.
@@ -1304,8 +1304,9 @@ router_get_fallback_dir_servers(void)
  * If the <b>PDS_ALLOW_SELF</b> flag is not set, then don't include ourself
  * (if we're a dirserver).
  *
- * Don't pick an authority if any non-authority is viable; try to avoid using
- * servers that have returned 503 recently.
+ * Don't pick a fallback directory mirror if any non-fallback is viable;
+ * (the fallback directory mirrors include the authorities)
+ * try to avoid using servers that have returned 503 recently.
  */
 const routerstatus_t *
 router_pick_directory_server(dirinfo_type_t type, int flags)
@@ -1332,7 +1333,7 @@ router_pick_directory_server(dirinfo_type_t type, int flags)
   log_info(LD_DIR,
            "No reachable router entries for dirservers. "
            "Trying them all again.");
-  /* mark all authdirservers as up again */
+  /* mark all fallback directory mirrors as up again */
   mark_all_dirservers_up(fallback_dir_servers);
   /* try again */
   choice = router_pick_directory_server_impl(type, flags, NULL);
diff --git a/src/test/test_config.c b/src/test/test_config.c
index 28e9fa0f32..00489d1d1a 100644
--- a/src/test/test_config.c
+++ b/src/test/test_config.c
@@ -1521,9 +1521,9 @@ test_config_adding_dir_servers(void *arg)
     );
 
   /* We need to know if add_default_fallback_dir_servers is called,
+   * whatever the size of the list in fallback_dirs.inc,
    * so we use a version of add_default_fallback_dir_servers that adds
-   * one known default fallback directory.
-   * There doesn't appear to be any need to test it unmocked. */
+   * one known default fallback directory. */
   MOCK(add_default_fallback_dir_servers,
        add_default_fallback_dir_servers_known_default);
 
@@ -1581,20 +1581,6 @@ test_config_adding_dir_servers(void *arg)
    *  Cases expected to yield this outcome:
    *    0 (DirAuthorities, AlternateBridgeAuthority, AlternateDirAuthority
    *       and FallbackDir are all NULL)
-   *
-   * Before #15642 but after #13163 - Stop using default authorities when both
-   * Alternate Dir and Bridge Authority are set
-   * (#13163 was committed in 0.2.6 as c1dd43d823c7)
-   *
-   * The behaviour is different in the following cases
-   * where FallbackDir is NULL:
-   *  2, 6, 8
-   *
-   * In these cases, the Default Fallback Directories are applied, even when
-   * DirAuthorities or AlternateDirAuthority are set.
-   *
-   * However, as the list of default fallback directories is currently empty,
-   * this change doesn't modify any user-visible behaviour.
    */
 
   /*
@@ -3209,11 +3195,39 @@ test_config_adding_dir_servers(void *arg)
   UNMOCK(add_default_fallback_dir_servers);
 }
 
+static void
+test_config_default_fallback_dirs(void *arg)
+{
+  const char *fallback[] = {
+#include "../or/fallback_dirs.inc"
+    NULL
+  };
+
+  int n_included_fallback_dirs = 0;
+  int n_added_fallback_dirs = 0;
+
+  (void)arg;
+  clear_dir_servers();
+
+  while (fallback[n_included_fallback_dirs])
+    n_included_fallback_dirs++;
+
+  add_default_fallback_dir_servers();
+
+  n_added_fallback_dirs = smartlist_len(router_get_fallback_dir_servers());
+
+  tt_assert(n_included_fallback_dirs == n_added_fallback_dirs);
+
+  done:
+  clear_dir_servers();
+}
+
 #define CONFIG_TEST(name, flags)                          \
   { #name, test_config_ ## name, flags, NULL, NULL }
 
 struct testcase_t config_tests[] = {
-  CONFIG_TEST(adding_dir_servers, TT_FORK),
+  CONFIG_TEST(adding_dir_servers, 0),
+  CONFIG_TEST(default_fallback_dirs, 0),
   CONFIG_TEST(resolve_my_address, TT_FORK),
   CONFIG_TEST(addressmap, 0),
   CONFIG_TEST(parse_bridge_line, 0),