Merge remote-tracking branch 'teor/bug13718-consensus-interval'

12 files changed, 216 insertions, 30 deletions

diff --git a/changes/bug13823-decrease-consensus-interval b/changes/bug13823-decrease-consensus-interval
new file mode 100644
index 0000000000..1d99bd73cb
--- /dev/null
+++ b/changes/bug13823-decrease-consensus-interval
@@ -0,0 +1,8 @@
+  o Minor bugfixes:
+    - Decrease minimum consensus interval to 10 seconds
+      when TestingTorNetwork is set. (Or 5 seconds for
+      the first consensus.)
+      Fix code that assumes larger interval values.
+      This assists in quickly bootstrapping a testing
+      Tor network.
+      Fixes bugs 13718 & 13823.
diff --git a/changes/bug13839-fix-TestingMinExitFlagThreshold b/changes/bug13839-fix-TestingMinExitFlagThreshold
new file mode 100644
index 0000000000..947614f550
--- /dev/null
+++ b/changes/bug13839-fix-TestingMinExitFlagThreshold
@@ -0,0 +1,7 @@
+  o Minor bugfixes:
+    - Stop requiring exits to have non-zero bandwithcapacity in a
+      TestingTorNetwork. Instead, when TestingMinExitFlagThreshold is 0,
+      ignore exit bandwidthcapacity.
+      This assists in bootstrapping a testing Tor network.
+      Fixes bugs 13718 & 13839.
+      Makes bug 13161's TestingDirAuthVoteExit non-essential.
diff --git a/changes/bug13963-decrease-if-modified-since-delay b/changes/bug13963-decrease-if-modified-since-delay
new file mode 100644
index 0000000000..62371444c4
--- /dev/null
+++ b/changes/bug13963-decrease-if-modified-since-delay
@@ -0,0 +1,7 @@
+  o Minor bugfixes:
+    - When V3AuthVotingInterval is low, decrease the delay on the
+      If-Modified-Since header passed to directory servers.
+      This allows us to obtain consensuses promptly when the consensus
+      interval is very short.
+      This assists in bootstrapping a testing Tor network.
+      Fixes bugs 13718 & 13963.
diff --git a/src/or/config.c b/src/or/config.c
index 701d8977ac..e7891a5bd3 100644
--- a/src/or/config.c
+++ b/src/or/config.c
@@ -469,7 +469,7 @@ static const config_var_t testing_tor_network_defaults[] = {
   V(V3AuthVotingInterval,        INTERVAL, "5 minutes"),
   V(V3AuthVoteDelay,             INTERVAL, "20 seconds"),
   V(V3AuthDistDelay,             INTERVAL, "20 seconds"),
-  V(TestingV3AuthInitialVotingInterval, INTERVAL, "5 minutes"),
+  V(TestingV3AuthInitialVotingInterval, INTERVAL, "150 seconds"),
   V(TestingV3AuthInitialVoteDelay, INTERVAL, "20 seconds"),
   V(TestingV3AuthInitialDistDelay, INTERVAL, "20 seconds"),
   V(TestingV3AuthVotingStartOffset, INTERVAL, "0"),
@@ -3414,19 +3414,68 @@ options_validate(or_options_t *old_options, or_options_t *options,
 
   if (options->V3AuthVoteDelay + options->V3AuthDistDelay >=
       options->V3AuthVotingInterval/2) {
-    REJECT("V3AuthVoteDelay plus V3AuthDistDelay must be less than half "
-           "V3AuthVotingInterval");
+    /*
+    This doesn't work, but it seems like it should:
+     what code is preventing the interval being less than twice the lead-up?
+    if (options->TestingTorNetwork) {
+      if (options->V3AuthVoteDelay + options->V3AuthDistDelay >=
+          options->V3AuthVotingInterval) {
+        REJECT("V3AuthVoteDelay plus V3AuthDistDelay must be less than "
+               "V3AuthVotingInterval");
+      } else {
+        COMPLAIN("V3AuthVoteDelay plus V3AuthDistDelay is more than half "
+                 "V3AuthVotingInterval. This may lead to "
+                 "consensus instability, particularly if clocks drift.");
+      }
+    } else {
+     */
+      REJECT("V3AuthVoteDelay plus V3AuthDistDelay must be less than half "
+             "V3AuthVotingInterval");
+    /*
+    }
+     */
+  }
+
+  if (options->V3AuthVoteDelay < MIN_VOTE_SECONDS) {
+    if (options->TestingTorNetwork) {
+      if (options->V3AuthVoteDelay < MIN_VOTE_SECONDS_TESTING) {
+        REJECT("V3AuthVoteDelay is way too low.");
+      } else {
+        COMPLAIN("V3AuthVoteDelay is very low. "
+                 "This may lead to failure to vote for a consensus.");
+      }
+    } else {
+      REJECT("V3AuthVoteDelay is way too low.");
+    }
+  }
+
+  if (options->V3AuthDistDelay < MIN_DIST_SECONDS) {
+    if (options->TestingTorNetwork) {
+      if (options->V3AuthDistDelay < MIN_DIST_SECONDS_TESTING) {
+        REJECT("V3AuthDistDelay is way too low.");
+      } else {
+        COMPLAIN("V3AuthDistDelay is very low. "
+                 "This may lead to missing votes in a consensus.");
+      }
+    } else {
+      REJECT("V3AuthDistDelay is way too low.");
+    }
   }
-  if (options->V3AuthVoteDelay < MIN_VOTE_SECONDS)
-    REJECT("V3AuthVoteDelay is way too low.");
-  if (options->V3AuthDistDelay < MIN_DIST_SECONDS)
-    REJECT("V3AuthDistDelay is way too low.");
 
   if (options->V3AuthNIntervalsValid < 2)
     REJECT("V3AuthNIntervalsValid must be at least 2.");
 
   if (options->V3AuthVotingInterval < MIN_VOTE_INTERVAL) {
-    REJECT("V3AuthVotingInterval is insanely low.");
+    if (options->TestingTorNetwork) {
+      if (options->V3AuthVotingInterval < MIN_VOTE_INTERVAL_TESTING) {
+        REJECT("V3AuthVotingInterval is insanely low.");
+      } else {
+        COMPLAIN("V3AuthVotingInterval is very low. "
+                 "This may lead to failure to synchronise for a consensus.");
+      }
+    } else {
+      REJECT("V3AuthVotingInterval is insanely low.");
+    }
   } else if (options->V3AuthVotingInterval > 24*60*60) {
     REJECT("V3AuthVotingInterval is insanely high.");
   } else if (((24*60*60) % options->V3AuthVotingInterval) != 0) {
@@ -3501,26 +3550,27 @@ options_validate(or_options_t *old_options, or_options_t *options,
   CHECK_DEFAULT(TestingCertMaxDownloadTries);
 #undef CHECK_DEFAULT
 
-  if (options->TestingV3AuthInitialVotingInterval < MIN_VOTE_INTERVAL) {
+  if (options->TestingV3AuthInitialVotingInterval
+      < MIN_VOTE_INTERVAL_TESTING_INITIAL) {
     REJECT("TestingV3AuthInitialVotingInterval is insanely low.");
   } else if (((30*60) % options->TestingV3AuthInitialVotingInterval) != 0) {
     REJECT("TestingV3AuthInitialVotingInterval does not divide evenly into "
            "30 minutes.");
   }
 
-  if (options->TestingV3AuthInitialVoteDelay < MIN_VOTE_SECONDS) {
+  if (options->TestingV3AuthInitialVoteDelay < MIN_VOTE_SECONDS_TESTING) {
     REJECT("TestingV3AuthInitialVoteDelay is way too low.");
   }
 
-  if (options->TestingV3AuthInitialDistDelay < MIN_DIST_SECONDS) {
+  if (options->TestingV3AuthInitialDistDelay < MIN_DIST_SECONDS_TESTING) {
     REJECT("TestingV3AuthInitialDistDelay is way too low.");
   }
 
   if (options->TestingV3AuthInitialVoteDelay +
       options->TestingV3AuthInitialDistDelay >=
-      options->TestingV3AuthInitialVotingInterval/2) {
+      options->TestingV3AuthInitialVotingInterval) {
     REJECT("TestingV3AuthInitialVoteDelay plus TestingV3AuthInitialDistDelay "
-           "must be less than half TestingV3AuthInitialVotingInterval");
+           "must be less than TestingV3AuthInitialVotingInterval");
   }
 
   if (options->TestingV3AuthVotingStartOffset >
@@ -3528,6 +3578,8 @@ options_validate(or_options_t *old_options, or_options_t *options,
           options->V3AuthVotingInterval)) {
     REJECT("TestingV3AuthVotingStartOffset is higher than the voting "
            "interval.");
+  } else if (options->TestingV3AuthVotingStartOffset < 0) {
+    REJECT("TestingV3AuthVotingStartOffset must be non-negative.");
   }
 
   if (options->TestingAuthDirTimeToLearnReachability < 0) {
diff --git a/src/or/directory.c b/src/or/directory.c
index cca4b54e24..b88c9d9f10 100644
--- a/src/or/directory.c
+++ b/src/or/directory.c
@@ -433,18 +433,33 @@ directory_get_from_dirserver(uint8_t dir_purpose, uint8_t router_purpose,
     if (resource)
       flav = networkstatus_parse_flavor_name(resource);
 
+    /* DEFAULT_IF_MODIFIED_SINCE_DELAY is 1/20 of the default consensus
+     * period of 1 hour.
+     */
+#define DEFAULT_IF_MODIFIED_SINCE_DELAY (180)
     if (flav != -1) {
       /* IF we have a parsed consensus of this type, we can do an
        * if-modified-time based on it. */
       v = networkstatus_get_latest_consensus_by_flavor(flav);
-      if (v)
-        if_modified_since = v->valid_after + 180;
+      if (v) {
+        /* In networks with particularly short V3AuthVotingIntervals,
+         * ask for the consensus if it's been modified since half the
+         * V3AuthVotingInterval of the most recent consensus. */
+        time_t ims_delay = DEFAULT_IF_MODIFIED_SINCE_DELAY;
+        if (v->fresh_until > v->valid_after
+            && ims_delay > (v->fresh_until - v->valid_after)/2) {
+          ims_delay = (v->fresh_until - v->valid_after)/2;
+        }
+        if_modified_since = v->valid_after + ims_delay;
+      }
     } else {
       /* Otherwise it might be a consensus we don't parse, but which we
        * do cache.  Look at the cached copy, perhaps. */
       cached_dir_t *cd = dirserv_get_consensus(resource);
+      /* We have no method of determining the voting interval from an
+       * unparsed consensus, so we use the default. */
       if (cd)
-        if_modified_since = cd->published + 180;
+        if_modified_since = cd->published + DEFAULT_IF_MODIFIED_SINCE_DELAY;
     }
   }
 
@@ -2258,6 +2273,7 @@ write_http_status_line(dir_connection_t *conn, int status,
     log_warn(LD_BUG,"status line too long.");
     return;
   }
+  log_debug(LD_DIRSERV,"Wrote status 'HTTP/1.0 %d %s'", status, reason_phrase);
   connection_write_to_buf(buf, strlen(buf), TO_CONN(conn));
 }
 
@@ -2554,8 +2570,11 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers,
   if ((header = http_get_header(headers, "If-Modified-Since: "))) {
     struct tm tm;
     if (parse_http_time(header, &tm) == 0) {
-      if (tor_timegm(&tm, &if_modified_since)<0)
+      if (tor_timegm(&tm, &if_modified_since)<0) {
         if_modified_since = 0;
+      } else {
+        log_debug(LD_DIRSERV, "If-Modified-Since is '%s'.", escaped(header));
+      }
     }
     /* The correct behavior on a malformed If-Modified-Since header is to
      * act as if no If-Modified-Since header had been given. */
diff --git a/src/or/dirserv.c b/src/or/dirserv.c
index d31bb72361..a1d22b041f 100644
--- a/src/or/dirserv.c
+++ b/src/or/dirserv.c
@@ -887,12 +887,26 @@ static int
 router_is_active(const routerinfo_t *ri, const node_t *node, time_t now)
 {
   time_t cutoff = now - ROUTER_MAX_AGE_TO_PUBLISH;
-  if (ri->cache_info.published_on < cutoff)
+  if (ri->cache_info.published_on < cutoff) {
     return 0;
-  if (!node->is_running || !node->is_valid || ri->is_hibernating)
+  }
+  if (!node->is_running || !node->is_valid || ri->is_hibernating) {
     return 0;
-  if (!ri->bandwidthcapacity)
+  }
+  /* Only require bandwith capacity in non-test networks, or
+   * if TestingTorNetwork, and TestingMinExitFlagThreshold is non-zero */
+  if (!ri->bandwidthcapacity) {
+    if (get_options()->TestingTorNetwork) {
+      if (get_options()->TestingMinExitFlagThreshold > 0) {
+        /* If we're in a TestingTorNetwork, and TestingMinExitFlagThreshold is,
+         * then require bandwidthcapacity */
+        return 0;
+      }
+    } else {
+      /* If we're not in a TestingTorNetwork, then require bandwidthcapacity */
       return 0;
+    }
+  }
   return 1;
 }
 
@@ -1037,7 +1051,7 @@ directory_fetches_dir_info_later(const or_options_t *options)
 }
 
 /** Return true iff we want to fetch and keep certificates for authorities
- * that we don't acknowledge as aurthorities ourself.
+ * that we don't acknowledge as authorities ourself.
  */
 int
 directory_caches_unknown_auth_certs(const or_options_t *options)
@@ -1498,7 +1512,7 @@ dirserv_compute_performance_thresholds(routerlist_t *rl,
       (unsigned long)guard_tk,
       (unsigned long)guard_bandwidth_including_exits_kb,
       (unsigned long)guard_bandwidth_excluding_exits_kb,
-      enough_mtbf_info ? "" : " don't ");
+      enough_mtbf_info ? "" : " don't");
 
   tor_free(uptimes);
   tor_free(mtbfs);
diff --git a/src/or/dirvote.c b/src/or/dirvote.c
index 39505a4f9e..c6068228f9 100644
--- a/src/or/dirvote.c
+++ b/src/or/dirvote.c
@@ -1107,8 +1107,12 @@ networkstatus_compute_consensus(smartlist_t *votes,
     vote_seconds = median_int(votesec_list, n_votes);
     dist_seconds = median_int(distsec_list, n_votes);
 
-    tor_assert(valid_after+MIN_VOTE_INTERVAL <= fresh_until);
-    tor_assert(fresh_until+MIN_VOTE_INTERVAL <= valid_until);
+    tor_assert(valid_after +
+               (get_options()->TestingTorNetwork ?
+                MIN_VOTE_INTERVAL_TESTING : MIN_VOTE_INTERVAL) <= fresh_until);
+    tor_assert(fresh_until +
+               (get_options()->TestingTorNetwork ?
+                MIN_VOTE_INTERVAL_TESTING : MIN_VOTE_INTERVAL) <= valid_until);
     tor_assert(vote_seconds >= MIN_VOTE_SECONDS);
     tor_assert(dist_seconds >= MIN_DIST_SECONDS);
 
diff --git a/src/or/dirvote.h b/src/or/dirvote.h
index 5d44ba4320..b570e9d251 100644
--- a/src/or/dirvote.h
+++ b/src/or/dirvote.h
@@ -14,12 +14,42 @@
 
 #include "testsupport.h"
 
+/*
+ * Ideally, assuming synced clocks, we should only need 1 second for each of:
+ *  - Vote
+ *  - Distribute
+ *  - Consensus Publication
+ * As we can gather descriptors continuously.
+ * (Could we even go as far as publishing the previous consensus,
+ *  in the same second that we vote for the next one?)
+ * But we're not there yet: these are the lowest working values at this time.
+ */
+
 /** Lowest allowable value for VoteSeconds. */
 #define MIN_VOTE_SECONDS 2
+/** Lowest allowable value for VoteSeconds when TestingTorNetwork is 1 */
+#define MIN_VOTE_SECONDS_TESTING 2
+
 /** Lowest allowable value for DistSeconds. */
 #define MIN_DIST_SECONDS 2
-/** Smallest allowable voting interval. */
+/** Lowest allowable value for DistSeconds when TestingTorNetwork is 1 */
+#define MIN_DIST_SECONDS_TESTING 2
+
+/** Lowest allowable voting interval. */
 #define MIN_VOTE_INTERVAL 300
+/** Lowest allowable voting interval when TestingTorNetwork is 1:
+ * Voting Interval can be:
+ *   10, 12, 15, 18, 20, 24, 25, 30, 36, 40, 45, 50, 60, ...
+ * Testing Initial Voting Interval can be:
+ *    5,  6,  8,  9, or any of the possible values for Voting Interval,
+ * as they both need to evenly divide 30 minutes.
+ * If clock desynchronisation is an issue, use an interval of at least:
+ *   18 * drift in seconds, to allow for a clock slop factor */
+#define MIN_VOTE_INTERVAL_TESTING \
+                (((MIN_VOTE_SECONDS_TESTING)+(MIN_DIST_SECONDS_TESTING)+1)*2)
+
+#define MIN_VOTE_INTERVAL_TESTING_INITIAL \
+                ((MIN_VOTE_SECONDS_TESTING)+(MIN_DIST_SECONDS_TESTING)+1)
 
 /** The lowest consensus method that we currently support. */
 #define MIN_SUPPORTED_CONSENSUS_METHOD 13
diff --git a/src/or/networkstatus.c b/src/or/networkstatus.c
index 21efdd129d..9b24405951 100644
--- a/src/or/networkstatus.c
+++ b/src/or/networkstatus.c
@@ -832,6 +832,10 @@ update_consensus_networkstatus_fetch_time_impl(time_t now, int flav)
          a crazy-fast voting interval, though, 2 minutes may be too
          much. */
       min_sec_before_caching = interval/16;
+      /* make sure we always delay by at least a second before caching */
+      if (min_sec_before_caching == 0) {
+        min_sec_before_caching = 1;
+      }
     }
 
     if (directory_fetches_dir_info_early(options)) {
@@ -863,8 +867,16 @@ update_consensus_networkstatus_fetch_time_impl(time_t now, int flav)
         dl_interval = (c->valid_until - start) - min_sec_before_caching;
       }
     }
+    /* catch low dl_interval in crazy-fast networks */
     if (dl_interval < 1)
       dl_interval = 1;
+    /* catch late start in crazy-fast networks */
+    if (start+dl_interval >= c->valid_until)
+      start = c->valid_until - dl_interval - 1;
+    log_debug(LD_DIR,
+              "fresh_until: %ld start: %ld "
+              "dl_interval: %ld valid_until: %ld ",
+              c->fresh_until, start, dl_interval, c->valid_until);
     /* We must not try to replace c while it's still fresh: */
     tor_assert(c->fresh_until < start);
     /* We must download the next one before c is invalid: */
diff --git a/src/or/router.c b/src/or/router.c
index 7119a29d68..f6b2250a47 100644
--- a/src/or/router.c
+++ b/src/or/router.c
@@ -1223,6 +1223,11 @@ router_orport_found_reachable(void)
                  " Publishing server descriptor." : "");
     can_reach_or_port = 1;
     mark_my_descriptor_dirty("ORPort found reachable");
+    /* This is a significant enough change to upload immediately,
+     * at least in a test network */
+    if (get_options()->TestingTorNetwork == 1) {
+      reschedule_descriptor_update_check();
+    }
     control_event_server_status(LOG_NOTICE,
                                 "REACHABILITY_SUCCEEDED ORADDRESS=%s:%d",
                                 address, me->or_port);
@@ -1240,8 +1245,14 @@ router_dirport_found_reachable(void)
     log_notice(LD_DIRSERV,"Self-testing indicates your DirPort is reachable "
                "from the outside. Excellent.");
     can_reach_dir_port = 1;
-    if (decide_to_advertise_dirport(get_options(), me->dir_port))
+    if (decide_to_advertise_dirport(get_options(), me->dir_port)) {
       mark_my_descriptor_dirty("DirPort found reachable");
+      /* This is a significant enough change to upload immediately,
+       * at least in a test network */
+      if (get_options()->TestingTorNetwork == 1) {
+        reschedule_descriptor_update_check();
+      }
+    }
     control_event_server_status(LOG_NOTICE,
                                 "REACHABILITY_SUCCEEDED DIRADDRESS=%s:%d",
                                 address, me->dir_port);
diff --git a/src/or/routerlist.c b/src/or/routerlist.c
index 8379bc80b3..60d8e71a28 100644
--- a/src/or/routerlist.c
+++ b/src/or/routerlist.c
@@ -2210,11 +2210,29 @@ router_choose_random_node(smartlist_t *excludedsmartlist,
   router_add_running_nodes_to_smartlist(sl, allow_invalid,
                                         need_uptime, need_capacity,
                                         need_guard, need_desc);
+  log_debug(LD_CIRC,
+           "We found %d running nodes.",
+            smartlist_len(sl));
+
   smartlist_subtract(sl,excludednodes);
-  if (excludedsmartlist)
+  log_debug(LD_CIRC,
+            "We removed %d excludednodes, leaving %d nodes.",
+            smartlist_len(excludednodes),
+            smartlist_len(sl));
+
+  if (excludedsmartlist) {
     smartlist_subtract(sl,excludedsmartlist);
-  if (excludedset)
+    log_debug(LD_CIRC,
+              "We removed %d excludedsmartlist, leaving %d nodes.",
+              smartlist_len(excludedsmartlist),
+              smartlist_len(sl));
+  }
+  if (excludedset) {
     routerset_subtract_nodes(sl,excludedset);
+    log_debug(LD_CIRC,
+              "We removed excludedset, leaving %d nodes.",
+              smartlist_len(sl));
+  }
 
   // Always weight by bandwidth
   choice = node_sl_choose_by_bandwidth(sl, rule);
diff --git a/src/or/routerparse.c b/src/or/routerparse.c
index bc3b00226a..8176d47262 100644
--- a/src/or/routerparse.c
+++ b/src/or/routerparse.c
@@ -2598,11 +2598,15 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out,
     (int) tor_parse_long(tok->args[1], 10, 0, INT_MAX, &ok, NULL);
   if (!ok)
     goto err;
-  if (ns->valid_after + MIN_VOTE_INTERVAL > ns->fresh_until) {
+  if (ns->valid_after +
+      (get_options()->TestingTorNetwork ?
+       MIN_VOTE_INTERVAL_TESTING : MIN_VOTE_INTERVAL) > ns->fresh_until) {
     log_warn(LD_DIR, "Vote/consensus freshness interval is too short");
     goto err;
   }
-  if (ns->valid_after + MIN_VOTE_INTERVAL*2 > ns->valid_until) {
+  if (ns->valid_after +
+      (get_options()->TestingTorNetwork ?
+       MIN_VOTE_INTERVAL_TESTING : MIN_VOTE_INTERVAL)*2 > ns->valid_until) {
     log_warn(LD_DIR, "Vote/consensus liveness interval is too short");
     goto err;
   }