Merge branch 'ticket33029_042_01' into ticket33029_043_03

Conflicts:
	doc/tor.1.txt
	src/app/config/config.c
	src/app/config/or_options_st.h
	src/core/mainloop/connection.h

Between 042 and 043, the dirauth options were modularized so this merge commit
address this by moving the AuthDirRejectUncompressedRequests to the module
along with a series of accessors.

Signed-off-by: David Goulet <dgoulet@torproject.org>

8 files changed, 96 insertions, 15 deletions

diff --git a/src/feature/dirauth/dirauth_config.c b/src/feature/dirauth/dirauth_config.c
index 3aeeab3b31..ca16dc8424 100644
--- a/src/feature/dirauth/dirauth_config.c
+++ b/src/feature/dirauth/dirauth_config.c
@@ -27,6 +27,7 @@
 #include "feature/dirauth/authmode.h"
 #include "feature/dirauth/bwauth.h"
 #include "feature/dirauth/dirauth_periodic.h"
+#include "feature/dirauth/dirauth_sys.h"
 #include "feature/dirauth/dirvote.h"
 #include "feature/dirauth/guardfraction.h"
 #include "feature/dirauth/dirauth_options_st.h"
@@ -45,6 +46,14 @@
 #define YES_IF_CHANGED_INT(opt) \
   if (!CFG_EQ_INT(old_options, new_options, opt)) return 1;
 
+/** Return true iff we are configured to reject request under load for non
+ * relay connections. */
+bool
+dirauth_should_reject_requests_under_load(void)
+{
+  return !!dirauth_get_options()->AuthDirRejectRequestsUnderLoad;
+}
+
 /**
  * Legacy validation/normalization function for the dirauth mode options in
  * options. Uses old_options as the previous options.
diff --git a/src/feature/dirauth/dirauth_config.h b/src/feature/dirauth/dirauth_config.h
index 2ebafd917e..1ec599717c 100644
--- a/src/feature/dirauth/dirauth_config.h
+++ b/src/feature/dirauth/dirauth_config.h
@@ -35,6 +35,8 @@ int options_act_dirauth_mtbf(const struct or_options_t *old_options);
 int options_act_dirauth_stats(const struct or_options_t *old_options,
                               bool *print_notice_out);
 
+bool dirauth_should_reject_requests_under_load(void);
+
 extern const struct config_format_t dirauth_options_fmt;
 
 #else /* !defined(HAVE_MODULE_DIRAUTH) */
@@ -78,6 +80,8 @@ options_validate_dirauth_mode(const struct or_options_t *old_options,
 #define options_act_dirauth_stats(old_options, print_notice_out) \
   (((void)(old_options)),((void)(print_notice_out)),0)
 
+#define dirauth_should_reject_requests_under_load() (false)
+
 #endif /* defined(HAVE_MODULE_DIRAUTH) */
 
 #endif /* !defined(TOR_FEATURE_DIRAUTH_DIRAUTH_CONFIG_H) */
diff --git a/src/feature/dirauth/dirauth_options.inc b/src/feature/dirauth/dirauth_options.inc
index 5939010fe7..21f4996c39 100644
--- a/src/feature/dirauth/dirauth_options.inc
+++ b/src/feature/dirauth/dirauth_options.inc
@@ -95,4 +95,11 @@ CONF_VAR(TestingMinFastFlagThreshold, MEMUNIT, 0, "0")
  * versions? */
 CONF_VAR(VersioningAuthoritativeDirectory, BOOL, 0, "0")
 
+/** Boolean: Under bandwidth pressure, if set to 1, the authority will always
+ * answer directory requests from relays but will start sending 503 error code
+ * for the other connections. If set to 0, all connections are considered the
+ * same and the authority will try to answer them all regardless of bandwidth
+ * pressure or not. */
+CONF_VAR(AuthDirRejectRequestsUnderLoad, BOOL, 0, "1")
+
 END_CONF_STRUCT(dirauth_options_t)
diff --git a/src/feature/dircache/dircache.c b/src/feature/dircache/dircache.c
index ef7054001e..3b8775968a 100644
--- a/src/feature/dircache/dircache.c
+++ b/src/feature/dircache/dircache.c
@@ -957,7 +957,7 @@ handle_get_current_consensus(dir_connection_t *conn,
     goto done;
   }
 
-  if (global_write_bucket_low(TO_CONN(conn), size_guess, 2)) {
+  if (connection_dir_is_global_write_low(TO_CONN(conn), size_guess)) {
     log_debug(LD_DIRSERV,
               "Client asked for network status lists, but we've been "
               "writing too many bytes lately. Sending 503 Dir busy.");
@@ -1066,7 +1066,7 @@ handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args)
         }
       });
 
-    if (global_write_bucket_low(TO_CONN(conn), estimated_len, 2)) {
+    if (connection_dir_is_global_write_low(TO_CONN(conn), estimated_len)) {
       write_short_http_response(conn, 503, "Directory busy, try again later");
       goto vote_done;
     }
@@ -1125,7 +1125,7 @@ handle_get_microdesc(dir_connection_t *conn, const get_handler_args_t *args)
       write_short_http_response(conn, 404, "Not found");
       goto done;
     }
-    if (global_write_bucket_low(TO_CONN(conn), size_guess, 2)) {
+    if (connection_dir_is_global_write_low(TO_CONN(conn), size_guess)) {
       log_info(LD_DIRSERV,
                "Client asked for server descriptors, but we've been "
                "writing too many bytes lately. Sending 503 Dir busy.");
@@ -1223,7 +1223,7 @@ handle_get_descriptor(dir_connection_t *conn, const get_handler_args_t *args)
         msg = "Not found";
       write_short_http_response(conn, 404, msg);
     } else {
-      if (global_write_bucket_low(TO_CONN(conn), size_guess, 2)) {
+      if (connection_dir_is_global_write_low(TO_CONN(conn), size_guess)) {
         log_info(LD_DIRSERV,
                  "Client asked for server descriptors, but we've been "
                  "writing too many bytes lately. Sending 503 Dir busy.");
@@ -1319,9 +1319,8 @@ handle_get_keys(dir_connection_t *conn, const get_handler_args_t *args)
     SMARTLIST_FOREACH(certs, authority_cert_t *, c,
                       len += c->cache_info.signed_descriptor_len);
 
-    if (global_write_bucket_low(TO_CONN(conn),
-                                compress_method != NO_METHOD ? len/2 : len,
-                                2)) {
+    if (connection_dir_is_global_write_low(TO_CONN(conn),
+                                compress_method != NO_METHOD ? len/2 : len)) {
       write_short_http_response(conn, 503, "Directory busy, try again later");
       goto keys_done;
     }
diff --git a/src/feature/nodelist/dirlist.c b/src/feature/nodelist/dirlist.c
index 06f4f5482b..ad3af0a143 100644
--- a/src/feature/nodelist/dirlist.c
+++ b/src/feature/nodelist/dirlist.c
@@ -49,6 +49,37 @@ static smartlist_t *trusted_dir_servers = NULL;
  * and all fallback directory servers. */
 static smartlist_t *fallback_dir_servers = NULL;
 
+/** Helper: From a given trusted directory entry, add the v4 or/and v6 address
+ * to the nodelist address set. */
+static void
+add_trusted_dir_to_nodelist_addr_set(const dir_server_t *dir)
+{
+  tor_assert(dir);
+  tor_assert(dir->is_authority);
+
+  /* Add IPv4 and then IPv6 if applicable. */
+  nodelist_add_addr4_to_address_set(dir->addr);
+  if (!tor_addr_is_null(&dir->ipv6_addr)) {
+    nodelist_add_addr6_to_address_set(&dir->ipv6_addr);
+  }
+}
+
+/** Go over the trusted directory server list and add their address(es) to the
+ * nodelist address set. This is called everytime a new consensus is set. */
+MOCK_IMPL(void,
+dirlist_add_trusted_dir_addresses, (void))
+{
+  if (!trusted_dir_servers) {
+    return;
+  }
+
+  SMARTLIST_FOREACH_BEGIN(trusted_dir_servers, const dir_server_t *, ent) {
+    if (ent->is_authority) {
+      add_trusted_dir_to_nodelist_addr_set(ent);
+    }
+  } SMARTLIST_FOREACH_END(ent);
+}
+
 /** Return the number of directory authorities whose type matches some bit set
  * in <b>type</b>  */
 int
diff --git a/src/feature/nodelist/dirlist.h b/src/feature/nodelist/dirlist.h
index 6bfc2e1b01..9201e76a9c 100644
--- a/src/feature/nodelist/dirlist.h
+++ b/src/feature/nodelist/dirlist.h
@@ -44,4 +44,6 @@ void dir_server_add(dir_server_t *ent);
 void clear_dir_servers(void);
 void dirlist_free_all(void);
 
+MOCK_DECL(void, dirlist_add_trusted_dir_addresses, (void));
+
 #endif /* !defined(TOR_DIRLIST_H) */
diff --git a/src/feature/nodelist/nodelist.c b/src/feature/nodelist/nodelist.c
index 94ff08826f..b7c7552561 100644
--- a/src/feature/nodelist/nodelist.c
+++ b/src/feature/nodelist/nodelist.c
@@ -455,22 +455,43 @@ node_add_to_address_set(const node_t *node)
 
   if (node->rs) {
     if (node->rs->addr)
-      address_set_add_ipv4h(the_nodelist->node_addrs, node->rs->addr);
+      nodelist_add_addr4_to_address_set(node->rs->addr);
     if (!tor_addr_is_null(&node->rs->ipv6_addr))
-      address_set_add(the_nodelist->node_addrs, &node->rs->ipv6_addr);
+      nodelist_add_addr6_to_address_set(&node->rs->ipv6_addr);
   }
   if (node->ri) {
     if (node->ri->addr)
-      address_set_add_ipv4h(the_nodelist->node_addrs, node->ri->addr);
+      nodelist_add_addr4_to_address_set(node->ri->addr);
     if (!tor_addr_is_null(&node->ri->ipv6_addr))
-      address_set_add(the_nodelist->node_addrs, &node->ri->ipv6_addr);
+      nodelist_add_addr6_to_address_set(&node->ri->ipv6_addr);
   }
   if (node->md) {
     if (!tor_addr_is_null(&node->md->ipv6_addr))
-      address_set_add(the_nodelist->node_addrs, &node->md->ipv6_addr);
+      nodelist_add_addr6_to_address_set(&node->md->ipv6_addr);
   }
 }
 
+/** Add the given v4 address into the nodelist address set. */
+void
+nodelist_add_addr4_to_address_set(const uint32_t addr)
+{
+  if (!the_nodelist || !the_nodelist->node_addrs || addr == 0) {
+    return;
+  }
+  address_set_add_ipv4h(the_nodelist->node_addrs, addr);
+}
+
+/** Add the given v6 address into the nodelist address set. */
+void
+nodelist_add_addr6_to_address_set(const tor_addr_t *addr)
+{
+  if (BUG(!addr) || tor_addr_is_null(addr) || tor_addr_is_v4(addr) ||
+      !the_nodelist || !the_nodelist->node_addrs) {
+    return;
+  }
+  address_set_add(the_nodelist->node_addrs, addr);
+}
+
 /** Return true if <b>addr</b> is the address of some node in the nodelist.
  * If not, probably return false. */
 int
@@ -612,9 +633,12 @@ nodelist_set_consensus(networkstatus_t *ns)
   SMARTLIST_FOREACH(the_nodelist->nodes, node_t *, node,
                     node->rs = NULL);
 
-  /* Conservatively estimate that every node will have 2 addresses. */
-  const int estimated_addresses = smartlist_len(ns->routerstatus_list) *
-                                  get_estimated_address_per_node();
+  /* Conservatively estimate that every node will have 2 addresses (v4 and
+   * v6). Then we add the number of configured trusted authorities we have. */
+  int estimated_addresses = smartlist_len(ns->routerstatus_list) *
+                            get_estimated_address_per_node();
+  estimated_addresses += (get_n_authorities(V3_DIRINFO & BRIDGE_DIRINFO) *
+                          get_estimated_address_per_node());
   address_set_free(the_nodelist->node_addrs);
   the_nodelist->node_addrs = address_set_new(estimated_addresses);
 
@@ -665,6 +689,9 @@ nodelist_set_consensus(networkstatus_t *ns)
   SMARTLIST_FOREACH_BEGIN(the_nodelist->nodes, node_t *, node) {
     node_add_to_address_set(node);
   } SMARTLIST_FOREACH_END(node);
+  /* Then, add all trusted configured directories. Some might not be in the
+   * consensus so make sure we know them. */
+  dirlist_add_trusted_dir_addresses();
 
   if (! authdir) {
     SMARTLIST_FOREACH_BEGIN(the_nodelist->nodes, node_t *, node) {
diff --git a/src/feature/nodelist/nodelist.h b/src/feature/nodelist/nodelist.h
index 9742e3dff2..87020b81eb 100644
--- a/src/feature/nodelist/nodelist.h
+++ b/src/feature/nodelist/nodelist.h
@@ -35,6 +35,8 @@ node_t *nodelist_add_microdesc(microdesc_t *md);
 void nodelist_set_consensus(networkstatus_t *ns);
 void nodelist_ensure_freshness(networkstatus_t *ns);
 int nodelist_probably_contains_address(const tor_addr_t *addr);
+void nodelist_add_addr4_to_address_set(const uint32_t addr);
+void nodelist_add_addr6_to_address_set(const tor_addr_t *addr);
 
 void nodelist_remove_microdesc(const char *identity_digest, microdesc_t *md);
 void nodelist_remove_routerinfo(routerinfo_t *ri);