Merge branch 'feature17840-v11-squashed' into feature17840-v11-merged

Conflicts:
	src/or/directory.c
	src/test/test_routerlist.c

Fix minor conflicts.

1 files changed, 196 insertions, 55 deletions

diff --git a/src/or/directory.c b/src/or/directory.c
index c1fa37f242..9419f2a590 100644
--- a/src/or/directory.c
+++ b/src/or/directory.c
@@ -4,6 +4,7 @@
 /* See LICENSE for licensing information */
 
 #include "or.h"
+#include "backtrace.h"
 #include "buffers.h"
 #include "circuitbuild.h"
 #include "config.h"
@@ -82,18 +83,18 @@ static void dir_microdesc_download_failed(smartlist_t *failed,
 static void note_client_request(int purpose, int compressed, size_t bytes);
 static int client_likes_consensus(networkstatus_t *v, const char *want_url);
 
-static void directory_initiate_command_rend(const tor_addr_t *addr,
-                                            uint16_t or_port,
-                                            uint16_t dir_port,
-                                            const char *digest,
-                                            uint8_t dir_purpose,
-                                            uint8_t router_purpose,
-                                            dir_indirection_t indirection,
-                                            const char *resource,
-                                            const char *payload,
-                                            size_t payload_len,
-                                            time_t if_modified_since,
-                                            const rend_data_t *rend_query);
+static void directory_initiate_command_rend(
+                                          const tor_addr_port_t *or_addr_port,
+                                          const tor_addr_port_t *dir_addr_port,
+                                          const char *digest,
+                                          uint8_t dir_purpose,
+                                          uint8_t router_purpose,
+                                          dir_indirection_t indirection,
+                                          const char *resource,
+                                          const char *payload,
+                                          size_t payload_len,
+                                          time_t if_modified_since,
+                                          const rend_data_t *rend_query);
 
 /********* START VARIABLES **********/
 
@@ -313,7 +314,6 @@ directory_post_to_dirservers(uint8_t dir_purpose, uint8_t router_purpose,
   SMARTLIST_FOREACH_BEGIN(dirservers, dir_server_t *, ds) {
       routerstatus_t *rs = &(ds->fake_status);
       size_t upload_len = payload_len;
-      tor_addr_t ds_addr;
 
       if ((type & ds->type) == 0)
         continue;
@@ -344,11 +344,12 @@ directory_post_to_dirservers(uint8_t dir_purpose, uint8_t router_purpose,
         log_info(LD_DIR, "Uploading an extrainfo too (length %d)",
                  (int) extrainfo_len);
       }
-      tor_addr_from_ipv4h(&ds_addr, ds->addr);
       if (purpose_needs_anonymity(dir_purpose, router_purpose)) {
         indirection = DIRIND_ANONYMOUS;
-      } else if (!fascist_firewall_allows_address_dir(&ds_addr,ds->dir_port)) {
-        if (fascist_firewall_allows_address_or(&ds_addr,ds->or_port))
+      } else if (!fascist_firewall_allows_dir_server(ds,
+                                                     FIREWALL_DIR_CONNECTION,
+                                                     0)) {
+        if (fascist_firewall_allows_dir_server(ds, FIREWALL_OR_CONNECTION, 0))
           indirection = DIRIND_ONEHOP;
         else
           indirection = DIRIND_ANONYMOUS;
@@ -496,11 +497,14 @@ MOCK_IMPL(void, directory_get_from_dirserver, (
       const node_t *node = choose_random_dirguard(type);
       if (node && node->ri) {
         /* every bridge has a routerinfo. */
-        tor_addr_t addr;
         routerinfo_t *ri = node->ri;
-        node_get_addr(node, &addr);
-        directory_initiate_command(&addr,
-                                   ri->or_port, 0/*no dirport*/,
+        /* clients always make OR connections to bridges */
+        tor_addr_port_t or_ap;
+        /* we are willing to use a non-preferred address if we need to */
+        fascist_firewall_choose_address_node(node, FIREWALL_OR_CONNECTION, 0,
+                                             &or_ap);
+        directory_initiate_command(&or_ap.addr, or_ap.port,
+                                   NULL, 0, /*no dirport*/
                                    ri->cache_info.identity_digest,
                                    dir_purpose,
                                    router_purpose,
@@ -609,6 +613,80 @@ dirind_is_anon(dir_indirection_t ind)
   return ind == DIRIND_ANON_DIRPORT || ind == DIRIND_ANONYMOUS;
 }
 
+/* Choose reachable OR and Dir addresses and ports from status, copying them
+ * into use_or_ap and use_dir_ap. If indirection is anonymous, then we're
+ * connecting via another relay, so choose the primary IPv4 address and ports.
+ *
+ * status should have at least one reachable address, if we can't choose a
+ * reachable address, warn and return -1. Otherwise, return 0.
+ */
+static int
+directory_choose_address_routerstatus(const routerstatus_t *status,
+                                      dir_indirection_t indirection,
+                                      tor_addr_port_t *use_or_ap,
+                                      tor_addr_port_t *use_dir_ap)
+{
+  tor_assert(status != NULL);
+  tor_assert(use_or_ap != NULL);
+  tor_assert(use_dir_ap != NULL);
+
+  const int anonymized_connection = dirind_is_anon(indirection);
+  int have_or = 0, have_dir = 0;
+
+  /* We expect status to have at least one reachable address if we're
+   * connecting to it directly.
+   *
+   * Therefore, we can simply use the other address if the one we want isn't
+   * allowed by the firewall.
+   *
+   * (When Tor uploads and downloads a hidden service descriptor, it uses
+   * DIRIND_ANONYMOUS, except for Tor2Web, which uses DIRIND_ONEHOP.
+   * So this code will only modify the address for Tor2Web's HS descriptor
+   * fetches. Even Single Onion Servers (NYI) use DIRIND_ANONYMOUS, to avoid
+   * HSDirs denying service by rejecting descriptors.)
+   */
+
+  /* Initialise the OR / Dir addresses */
+  tor_addr_make_null(&use_or_ap->addr, AF_UNSPEC);
+  use_or_ap->port = 0;
+  tor_addr_make_null(&use_dir_ap->addr, AF_UNSPEC);
+  use_dir_ap->port = 0;
+
+  if (anonymized_connection) {
+    /* Use the primary (IPv4) OR address if we're making an indirect
+     * connection. */
+    tor_addr_from_ipv4h(&use_or_ap->addr, status->addr);
+    use_or_ap->port = status->or_port;
+    have_or = 1;
+  } else {
+    /* We use an IPv6 address if we have one and we prefer it.
+     * Use the preferred address and port if they are reachable, otherwise,
+     * use the alternate address and port (if any).
+     */
+    have_or = fascist_firewall_choose_address_rs(status,
+                                                 FIREWALL_OR_CONNECTION, 0,
+                                                 use_or_ap);
+  }
+
+  have_dir = fascist_firewall_choose_address_rs(status,
+                                                FIREWALL_DIR_CONNECTION, 0,
+                                                use_dir_ap);
+
+  /* We rejected both addresses. This isn't great. */
+  if (!have_or && !have_dir) {
+    log_warn(LD_BUG, "Rejected all OR and Dir addresses from %s when "
+             "launching a directory connection to: IPv4 %s OR %d Dir %d "
+             "IPv6 %s OR %d Dir %d", routerstatus_describe(status),
+             fmt_addr32(status->addr), status->or_port,
+             status->dir_port, fmt_addr(&status->ipv6_addr),
+             status->ipv6_orport, status->dir_port);
+    log_backtrace(LOG_WARN, LD_BUG, "Addresses came from");
+    return -1;
+  }
+
+  return 0;
+}
+
 /** Same as directory_initiate_command_routerstatus(), but accepts
  * rendezvous data to fetch a hidden service descriptor. */
 void
@@ -624,8 +702,11 @@ directory_initiate_command_routerstatus_rend(const routerstatus_t *status,
 {
   const or_options_t *options = get_options();
   const node_t *node;
-  tor_addr_t addr;
+  tor_addr_port_t use_or_ap, use_dir_ap;
   const int anonymized_connection = dirind_is_anon(indirection);
+
+  tor_assert(status != NULL);
+
   node = node_get_by_id(status->identity_digest);
 
   if (!node && anonymized_connection) {
@@ -634,7 +715,6 @@ directory_initiate_command_routerstatus_rend(const routerstatus_t *status,
              routerstatus_describe(status));
     return;
   }
-  tor_addr_from_ipv4h(&addr, status->addr);
 
   if (options->ExcludeNodes && options->StrictNodes &&
       routerset_contains_routerstatus(options->ExcludeNodes, status, -1)) {
@@ -646,13 +726,30 @@ directory_initiate_command_routerstatus_rend(const routerstatus_t *status,
     return;
   }
 
-  directory_initiate_command_rend(&addr,
-                             status->or_port, status->dir_port,
-                             status->identity_digest,
-                             dir_purpose, router_purpose,
-                             indirection, resource,
-                             payload, payload_len, if_modified_since,
-                             rend_query);
+  /* At this point, if we are a clients making a direct connection to a
+   * directory server, we have selected a server that has at least one address
+   * allowed by ClientUseIPv4/6 and Reachable{"",OR,Dir}Addresses. This
+   * selection uses the preference in ClientPreferIPv6{OR,Dir}Port, if
+   * possible. (If UseBridges is set, clients always use IPv6, and prefer it
+   * by default.)
+   *
+   * Now choose an address that we can use to connect to the directory server.
+   */
+  if (directory_choose_address_routerstatus(status, indirection, &use_or_ap,
+                                            &use_dir_ap) < 0) {
+    return;
+  }
+
+  /* We don't retry the alternate OR/Dir address for the same directory if
+   * the address we choose fails (#6772).
+   * Instead, we'll retry another directory on failure. */
+
+  directory_initiate_command_rend(&use_or_ap, &use_dir_ap,
+                                  status->identity_digest,
+                                  dir_purpose, router_purpose,
+                                  indirection, resource,
+                                  payload, payload_len, if_modified_since,
+                                  rend_query);
 }
 
 /** Launch a new connection to the directory server <b>status</b> to
@@ -874,27 +971,52 @@ directory_command_should_use_begindir(const or_options_t *options,
   if (indirection == DIRIND_DIRECT_CONN || indirection == DIRIND_ANON_DIRPORT)
     return 0;
   if (indirection == DIRIND_ONEHOP)
-    if (!fascist_firewall_allows_address_or(addr, or_port) ||
+    if (!fascist_firewall_allows_address_addr(addr, or_port,
+                                              FIREWALL_OR_CONNECTION, 0) ||
         directory_fetches_from_authorities(options))
       return 0; /* We're firewalled or are acting like a relay -- also no. */
   return 1;
 }
 
-/** Helper for directory_initiate_command_routerstatus: send the
- * command to a server whose address is <b>address</b>, whose IP is
- * <b>addr</b>, whose directory port is <b>dir_port</b>, whose tor version
- * <b>supports_begindir</b>, and whose identity key digest is
- * <b>digest</b>. */
+/** Helper for directory_initiate_command_rend: send the
+ * command to a server whose OR address/port is <b>or_addr</b>/<b>or_port</b>,
+ * whose directory address/port is <b>dir_addr</b>/<b>dir_port</b>, whose
+ * identity key digest is <b>digest</b>, with purposes <b>dir_purpose</b> and
+ * <b>router_purpose</b>, making an (in)direct connection as specified in
+ * <b>indirection</b>, with command <b>resource</b>, <b>payload</b> of
+ * <b>payload_len</b>, and asking for a result only <b>if_modified_since</b>.
+ */
 void
-directory_initiate_command(const tor_addr_t *_addr,
-                           uint16_t or_port, uint16_t dir_port,
+directory_initiate_command(const tor_addr_t *or_addr, uint16_t or_port,
+                           const tor_addr_t *dir_addr, uint16_t dir_port,
                            const char *digest,
                            uint8_t dir_purpose, uint8_t router_purpose,
                            dir_indirection_t indirection, const char *resource,
                            const char *payload, size_t payload_len,
                            time_t if_modified_since)
 {
-  directory_initiate_command_rend(_addr, or_port, dir_port,
+  tor_addr_port_t or_ap, dir_ap;
+
+  /* Use the null tor_addr and 0 port if the address or port isn't valid. */
+  if (tor_addr_port_is_valid(or_addr, or_port, 0)) {
+    tor_addr_copy(&or_ap.addr, or_addr);
+    or_ap.port = or_port;
+  } else {
+    /* the family doesn't matter here, so make it IPv4 */
+    tor_addr_make_null(&or_ap.addr, AF_INET);
+    or_port = 0;
+  }
+
+  if (tor_addr_port_is_valid(dir_addr, dir_port, 0)) {
+    tor_addr_copy(&dir_ap.addr, dir_addr);
+    dir_ap.port = dir_port;
+  } else {
+    /* the family doesn't matter here, so make it IPv4 */
+    tor_addr_make_null(&dir_ap.addr, AF_INET);
+    dir_port = 0;
+  }
+
+  directory_initiate_command_rend(&or_ap, &dir_ap,
                              digest, dir_purpose,
                              router_purpose, indirection,
                              resource, payload, payload_len,
@@ -914,10 +1036,11 @@ is_sensitive_dir_purpose(uint8_t dir_purpose)
 }
 
 /** Same as directory_initiate_command(), but accepts rendezvous data to
- * fetch a hidden service descriptor. */
+ * fetch a hidden service descriptor, and takes its address & port arguments
+ * as tor_addr_port_t. */
 static void
-directory_initiate_command_rend(const tor_addr_t *_addr,
-                                uint16_t or_port, uint16_t dir_port,
+directory_initiate_command_rend(const tor_addr_port_t *or_addr_port,
+                                const tor_addr_port_t *dir_addr_port,
                                 const char *digest,
                                 uint8_t dir_purpose, uint8_t router_purpose,
                                 dir_indirection_t indirection,
@@ -926,29 +1049,34 @@ directory_initiate_command_rend(const tor_addr_t *_addr,
                                 time_t if_modified_since,
                                 const rend_data_t *rend_query)
 {
+  tor_assert(or_addr_port);
+  tor_assert(dir_addr_port);
+  tor_assert(or_addr_port->port || dir_addr_port->port);
+  tor_assert(digest);
+
   dir_connection_t *conn;
   const or_options_t *options = get_options();
   int socket_error = 0;
-  int use_begindir = directory_command_should_use_begindir(options, _addr,
-                                     or_port, router_purpose, indirection);
+  const int use_begindir = directory_command_should_use_begindir(options,
+                                     &or_addr_port->addr, or_addr_port->port,
+                                     router_purpose, indirection);
   const int anonymized_connection = dirind_is_anon(indirection);
-  tor_addr_t addr;
-
-  tor_assert(_addr);
-  tor_assert(or_port || dir_port);
-  tor_assert(digest);
+  const int or_connection = use_begindir || anonymized_connection;
 
-  tor_addr_copy(&addr, _addr);
+  tor_addr_t addr;
+  tor_addr_copy(&addr, &(or_connection ? or_addr_port : dir_addr_port)->addr);
+  uint16_t port = (or_connection ? or_addr_port : dir_addr_port)->port;
+  uint16_t dir_port = dir_addr_port->port;
 
   log_debug(LD_DIR, "anonymized %d, use_begindir %d.",
             anonymized_connection, use_begindir);
 
   if (!dir_port && !use_begindir) {
     char ipaddr[TOR_ADDR_BUF_LEN];
-    tor_addr_to_str(ipaddr, _addr, TOR_ADDR_BUF_LEN, 0);
+    tor_addr_to_str(ipaddr, &addr, TOR_ADDR_BUF_LEN, 0);
     log_warn(LD_BUG, "Cannot use directory server without dirport or "
-                     "begindir! Address: %s, ORPort: %d, DirPort: %d",
-                     escaped_safe_str_client(ipaddr), or_port, dir_port);
+                     "begindir! Address: %s, DirPort: %d, Connection Port: %d",
+                     escaped_safe_str_client(ipaddr), dir_port, port);
     return;
   }
 
@@ -963,13 +1091,26 @@ directory_initiate_command_rend(const tor_addr_t *_addr,
 
   /* ensure that we don't make direct connections when a SOCKS server is
    * configured. */
-  if (!anonymized_connection && !use_begindir && !options->HTTPProxy &&
+  if (!or_connection && !options->HTTPProxy &&
       (options->Socks4Proxy || options->Socks5Proxy)) {
     log_warn(LD_DIR, "Cannot connect to a directory server through a "
                      "SOCKS proxy!");
     return;
   }
 
+  if (or_connection && (!or_addr_port->port
+                        || tor_addr_is_null(&or_addr_port->addr))) {
+    log_warn(LD_DIR, "Cannot make an OR connection without an OR port.");
+    log_backtrace(LOG_WARN, LD_BUG, "Address came from");
+    return;
+  } else if (!or_connection && (!dir_addr_port->port
+                                || tor_addr_is_null(&dir_addr_port->addr))) {
+    log_warn(LD_DIR, "Cannot make a Dir connection without a Dir port.");
+    log_backtrace(LOG_WARN, LD_BUG, "Address came from");
+
+    return;
+  }
+
   /* ensure we don't make excess connections when we're already downloading
    * a consensus during bootstrap */
   if (connection_dir_avoid_extra_connection_for_purpose(dir_purpose)) {
@@ -980,7 +1121,7 @@ directory_initiate_command_rend(const tor_addr_t *_addr,
 
   /* set up conn so it's got all the data we need to remember */
   tor_addr_copy(&conn->base_.addr, &addr);
-  conn->base_.port = use_begindir ? or_port : dir_port;
+  conn->base_.port = port;
   conn->base_.address = tor_dup_addr(&addr);
   memcpy(conn->identity_digest, digest, DIGEST_LEN);
 
@@ -998,7 +1139,7 @@ directory_initiate_command_rend(const tor_addr_t *_addr,
   if (rend_query)
     conn->rend_data = rend_data_dup(rend_query);
 
-  if (!anonymized_connection && !use_begindir) {
+  if (!or_connection) {
     /* then we want to connect to dirport directly */
 
     if (options->HTTPProxy) {