r13989@catbus: nickm | 2007-07-29 19:11:07 -0400

 More directory voting code.  Now, if everything works, and I haven't forgotten anything, it is possible to set up some v3 authorities and start voting. Of course, I have probably forgotten something, and there are probably bugs in there somewhere too.


svn:r10976

8 files changed, 176 insertions, 24 deletions

diff --git a/ChangeLog b/ChangeLog
index 104a07b060..f989efb033 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -10,6 +10,12 @@ Changes in version 0.2.0.3-alpha - 2007-07-29
     - Be even more aggressive about separating local traffic from relayed
       traffic when RelayBandwidthRate is set. (Refines proposal 111.)
 
+  o Major features (experimental):
+    - First cut of code for directory authorities to vote on a common
+      network status document rather than each publishing their own
+      opinion.  This code needs more testing and more corner-case handling
+      before it's ready for use.
+
   o Security fixes:
     - Directory authorities now call routers Fast if their bandwidth is
       at least 100KB/s, and consider their bandwidth adequate to be a
diff --git a/doc/TODO b/doc/TODO
index ceff559f1d..c6bdd56319 100644
--- a/doc/TODO
+++ b/doc/TODO
@@ -95,29 +95,30 @@ Things we'd like to do in 0.2.0.x:
           - Download as needed.
           o Actually invoke trusted_dirs_flush_certs_to_disk()
           - Serve list as needed.
-          * Detect whether votes are really all for the same period.
           o Avoid double-checking signatures every time we get a vote.
           - Warn about expired stuff.
           - Fix all XXXX020s in vote code
-          * Unit tests for detached signatures and signature manipulation.
         o Code to generate votes
         o Code to generate consensus from a list of votes
+          * Detect whether votes are really all for the same period.
         o Add a signature to a consensus.
+          * Unit tests for detached signatures and signature manipulation.
         o Code to check signatures on a consensus
         - Push/pull documents as appropriate.
-          . Push vote on voting
+          o Push vote on voting
             o Push vote
             o Process vote when received
               o Even if we get it before we start voting ourself.
-          * Push signature on forming consensus.
+          o Push signature on forming consensus.
             o Push signature
             o Add signatures when received
             o Queue received signatures before consensus is ready
-            * When consensus is ready, use queued signatures.
+            o When consensus is ready, use queued signatures.
           - Pull votes and signatures if we don't get them.
-        * Serve and store consensuses.
+        o Serve consensuses.
+        - Store consensuses
         - Cache votes and signatures on disk.
-        * Discard votes in advance of next voting period.
+        o Discard votes in advance of next voting period.
         o Have clients know which authorities are v3 authorities, and what
           their keys are.
           - While we're at it, let v3 authorities have fqdns lines.
@@ -140,6 +141,9 @@ Things we'd like to do in 0.2.0.x:
       - Drop bandwidth history from router-descriptors
     - 105: Version negotiation for the Tor protocol
     - 108: Base "Stable" Flag on Mean Time Between Failures
+      - Track mtbf in rephist.c
+      - Record mtbf between invocations
+      - Base stable on mtbf.
     o 109: No more than one server per IP address
     o 103: Splitting identity key from regularly used signing key
       o Merge with 101 into a new dir-spec.txt
diff --git a/src/or/directory.c b/src/or/directory.c
index ca6aae483d..275271c396 100644
--- a/src/or/directory.c
+++ b/src/or/directory.c
@@ -1817,6 +1817,7 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers,
     return 0;
   }
 
+
   if (!strcmp(url,"/tor/running-routers") ||
       !strcmp(url,"/tor/running-routers.z")) { /* running-routers fetch */
     int deflated = !strcmp(url,"/tor/running-routers.z");
@@ -1856,25 +1857,35 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers,
     return 0;
   }
 
-  if (!strcmpstart(url,"/tor/status/")) {
-    /* v2 network status fetch. */
+  if (!strcmpstart(url,"/tor/status/")
+      || !strcmp(url, "/tor/status-vote/current/consensus")
+      || !strcmp(url, "/tor/status-vote/current/consensus.z")) {
+    /* v2 or v3 network status fetch. */
     size_t url_len = strlen(url);
     int deflated = !strcmp(url+url_len-2, ".z");
     smartlist_t *dir_fps = smartlist_create();
+    int is_v3 = !strcmpstart(url, "/tor/status-vote");
     const char *request_type = NULL;
     const char *key = url + strlen("/tor/status/");
     if (deflated)
       url[url_len-2] = '\0';
-    dirserv_get_networkstatus_v2_fingerprints(dir_fps, key);
-    if (!strcmpstart(key, "fp/"))
-      request_type = deflated?"/tor/status/fp.z":"/tor/status/fp";
-    else if (!strcmpstart(key, "authority"))
-      request_type = deflated?"/tor/status/authority.z":
-        "/tor/status/authority";
-    else if (!strcmpstart(key, "all"))
-      request_type = deflated?"/tor/status/all.z":"/tor/status/all";
-    else
-      request_type = "/tor/status/?";
+    if (!is_v3) {
+      dirserv_get_networkstatus_v2_fingerprints(dir_fps, key);
+      if (!strcmpstart(key, "fp/"))
+        request_type = deflated?"/tor/status/fp.z":"/tor/status/fp";
+      else if (!strcmpstart(key, "authority"))
+        request_type = deflated?"/tor/status/authority.z":
+          "/tor/status/authority";
+      else if (!strcmpstart(key, "all"))
+        request_type = deflated?"/tor/status/all.z":"/tor/status/all";
+      else
+        request_type = "/tor/status/?";
+    } else {
+      smartlist_add(dir_fps, tor_memdup("\0\0\0\0\0\0\0\0\0\0"
+                                        "\0\0\0\0\0\0\0\0\0\0", 20));
+      request_type = deflated?"v3.z":"v3";
+    }
+
     tor_free(url);
     if (!smartlist_len(dir_fps)) { /* we failed to create/cache cp */
       write_http_status_line(conn, 503, "Network status object unavailable");
diff --git a/src/or/dirserv.c b/src/or/dirserv.c
index 72e1694db6..b1ea8cf84b 100644
--- a/src/or/dirserv.c
+++ b/src/or/dirserv.c
@@ -1079,6 +1079,9 @@ static cached_dir_t cached_runningrouters = { NULL, NULL, 0, 0, 0, -1 };
  * cached_dir_t. */
 static digestmap_t *cached_v2_networkstatus = NULL;
 
+/** DOCDOC */
+static cached_dir_t *cached_v3_networkstatus = NULL;
+
 /** Possibly replace the contents of <b>d</b> with the value of
  * <b>directory</b> published on <b>when</b>, unless <b>when</b> is older than
  * the last value, or too far in the future.
@@ -1245,6 +1248,17 @@ dirserv_set_cached_networkstatus_v2(const char *networkstatus,
   }
 }
 
+/* DOCDOC */
+void
+dirserv_set_cached_networkstatus_v3(const char *networkstatus,
+                                    time_t published)
+{
+  if (cached_v3_networkstatus)
+    cached_dir_decref(cached_v3_networkstatus);
+  cached_v3_networkstatus = new_cached_dir(
+                                  tor_strdup(networkstatus), published);
+}
+
 /** Remove any v2 networkstatus from the directory cache that was published
  * before <b>cutoff</b>. */
 void
@@ -1446,6 +1460,12 @@ dirserv_get_runningrouters(void)
                          "v1 network status list", V1_AUTHORITY);
 }
 
+cached_dir_t *
+dirserv_get_consensus(void)
+{
+  return cached_v3_networkstatus;
+}
+
 /** For authoritative directories: the current (v2) network status. */
 static cached_dir_t *the_v2_networkstatus = NULL;
 
@@ -2909,7 +2929,9 @@ connection_dirserv_add_networkstatus_bytes_to_outbuf(dir_connection_t *conn)
       /* Add another networkstatus; start serving it. */
       char *fp = smartlist_pop_last(conn->fingerprint_stack);
       cached_dir_t *d;
-      if (router_digest_is_me(fp))
+      if (tor_digest_is_zero(fp)) /* XXXX020 document this "feature". */
+        d = cached_v3_networkstatus;
+      else if (router_digest_is_me(fp))
         d = the_v2_networkstatus;
       else
         d = digestmap_get(cached_v2_networkstatus, fp);
diff --git a/src/or/dirvote.c b/src/or/dirvote.c
index 23ef147a99..321c78a696 100644
--- a/src/or/dirvote.c
+++ b/src/or/dirvote.c
@@ -13,6 +13,10 @@ const char dirvote_c_id[] =
  * \brief Functions to compute directory consensus, and schedule voting.
  **/
 
+static int dirvote_add_signatures_to_pending_consensus(
+                       const char *detached_signatures_body,
+                       const char **msg_out);
+
 /* =====
  * Voting and consensus generation
  * ===== */
@@ -721,6 +725,7 @@ networkstatus_check_consensus_signature(networkstatus_vote_t *consensus)
   int n_bad = 0;
   int n_unknown = 0;
   int n_no_signature = 0;
+  int n_required = 1; /* XXXX020 This is completely wrong. */
 
   tor_assert(! consensus->is_vote);
 
@@ -748,9 +753,10 @@ networkstatus_check_consensus_signature(networkstatus_vote_t *consensus)
       ++n_no_signature;
   });
 
-  /* XXXX020 actually use the result. */
-
-  return 0;
+  if (n_good > n_required)
+    return 0;
+  else
+    return -1;
 }
 
 /** DOCDOC */
@@ -1027,7 +1033,13 @@ static struct {
   time_t voting_starts;
   time_t voting_ends;
   time_t interval_starts;
-} voting_schedule;
+
+  time_t discard_old_votes;
+
+  int have_voted;
+  int have_built_consensus;
+  int have_published_consensus;
+} voting_schedule = {0,0,0,0,0,0,0};
 
 /** DOCDOC */
 void
@@ -1037,6 +1049,8 @@ dirvote_recalculate_timing(time_t now)
   time_t start;
   networkstatus_vote_t *consensus = networkstatus_get_latest_consensus();
 
+  memset(&voting_schedule, 0, sizeof(voting_schedule));
+
   if (consensus) {
     /* XXXX020 sanity-check these somewhere! */
     interval = consensus->fresh_until - consensus->valid_after;
@@ -1052,6 +1066,39 @@ dirvote_recalculate_timing(time_t now)
     dirvote_get_start_of_next_interval(now,interval);
   voting_schedule.voting_ends = start - vote_delay;
   voting_schedule.voting_starts = start - vote_delay - dist_delay;
+
+  voting_schedule.discard_old_votes = start + 600; /* XXXX020 */
+}
+
+/** DOCDOC */
+void
+dirvote_act(time_t now)
+{
+  if (!voting_schedule.voting_starts)
+    dirvote_recalculate_timing(now);
+  if (voting_schedule.voting_starts < now && !voting_schedule.have_voted) {
+    dirvote_perform_vote();
+    voting_schedule.have_voted = 1;
+  }
+  /* XXXX020 after a couple minutes here, start trying to fetch votes. */
+  if (voting_schedule.voting_ends < now &&
+      !voting_schedule.have_built_consensus) {
+    dirvote_compute_consensus();
+    /* XXXX020 we will want to try again later if we haven't got enough
+     * votes yet. */
+    voting_schedule.have_built_consensus = 1;
+  }
+  if (voting_schedule.interval_starts < now &&
+      !voting_schedule.have_published_consensus) {
+    dirvote_publish_consensus();
+    /* XXXX020 we will want to try again later if we haven't got enough
+     * signatures yet. */
+    voting_schedule.have_published_consensus = 1;
+  }
+  if (voting_schedule.discard_old_votes < now) {
+    dirvote_clear_pending_votes();
+    dirvote_recalculate_timing(now);
+  }
 }
 
 /** DOCDOC */
@@ -1240,6 +1287,19 @@ dirvote_compute_consensus(void)
     networkstatus_vote_free(pending_consensus);
   pending_consensus = consensus;
 
+  if (pending_consensus_signature_list) {
+    /* we may have gotten signatures for this consensus before we built
+     * it ourself.  Add them now. */
+    SMARTLIST_FOREACH(pending_consensus_signature_list, char *, sig,
+      {
+        const char *msg = NULL;
+        dirvote_add_signatures_to_pending_consensus(sig, &msg);
+        /* XXXX020 log result. */
+        tor_free(sig);
+      });
+    smartlist_clear(pending_consensus_signature_list);
+  }
+
   return 0;
  err:
   if (votes)
@@ -1325,6 +1385,19 @@ dirvote_add_signatures(const char *detached_signatures_body)
   }
 }
 
+/** DOCDOC */
+int
+dirvote_publish_consensus(void)
+{
+  /* Can we actually publish it yet? */
+  if (!pending_consensus ||
+      networkstatus_check_consensus_signature(pending_consensus)<0)
+    return -1;
+
+  networkstatus_set_current_consensus(pending_consensus_body);
+  return 0;
+}
+
 /** Release all static storage held in dirvote.c */
 void
 dirvote_free_all(void)
diff --git a/src/or/main.c b/src/or/main.c
index 83954789e8..7ca7b16aea 100644
--- a/src/or/main.c
+++ b/src/or/main.c
@@ -1006,6 +1006,10 @@ run_scheduled_events(time_t now)
     update_networkstatus_downloads(now);
   }
 
+  /** 2c. Let directory voting happen. */
+  if (authdir_mode_v3(options))
+    dirvote_act(now);
+
   /** 3a. Every second, we examine pending circuits and prune the
    *    ones which have been pending for more than a few seconds.
    *    We do this before step 4, so it can try building more if
diff --git a/src/or/or.h b/src/or/or.h
index a189d7247d..e60f9887be 100644
--- a/src/or/or.h
+++ b/src/or/or.h
@@ -2759,11 +2759,14 @@ int dirserv_dump_directory_to_string(char **dir_out,
 void directory_set_dirty(void);
 cached_dir_t *dirserv_get_directory(void);
 cached_dir_t *dirserv_get_runningrouters(void);
+cached_dir_t *dirserv_get_consensus(void);
 void dirserv_set_cached_directory(const char *directory, time_t when,
                                   int is_running_routers);
 void dirserv_set_cached_networkstatus_v2(const char *directory,
                                          const char *identity,
                                          time_t published);
+void dirserv_set_cached_networkstatus_v3(const char *consensus,
+                                         time_t published);
 void dirserv_clear_old_networkstatuses(time_t cutoff);
 void dirserv_clear_old_v1_info(time_t now);
 void dirserv_get_networkstatus_v2(smartlist_t *result, const char *key);
@@ -2837,6 +2840,7 @@ typedef struct vote_timing_t {
 void dirvote_get_preferred_voting_intervals(vote_timing_t *timing_out);
 time_t dirvote_get_start_of_next_interval(time_t now, int interval);
 void dirvote_recalculate_timing(time_t now);
+void dirvote_act(time_t now);
 
 /* invoked on timers and by outside triggers. */
 void dirvote_perform_vote(void);
@@ -2845,6 +2849,7 @@ struct pending_vote_t * dirvote_add_vote(const char *vote_body,
                                          const char **msg_out);
 int dirvote_compute_consensus(void);
 int dirvote_add_signatures(const char *detached_signatures_body);
+int dirvote_publish_consensus(void);
 
 #ifdef DIRVOTE_PRIVATE
 int networkstatus_check_voter_signature(networkstatus_vote_t *consensus,
@@ -3419,6 +3424,7 @@ local_routerstatus_t *router_get_combined_status_by_descriptor_digest(
 /* for consensuses. */
 networkstatus_vote_t *networkstatus_get_latest_consensus(void);
 networkstatus_vote_t *networkstatus_get_live_consensus(time_t now);
+int networkstatus_set_current_consensus(const char *consensus);
 
 //routerstatus_t *routerstatus_get_by_hexdigest(const char *hexdigest);
 int should_delay_dir_fetches(or_options_t *options);
diff --git a/src/or/routerlist.c b/src/or/routerlist.c
index 24c01a0b3a..88964611ba 100644
--- a/src/or/routerlist.c
+++ b/src/or/routerlist.c
@@ -3832,6 +3832,32 @@ networkstatus_get_live_consensus(time_t now)
   return current_consensus;
 }
 
+int
+networkstatus_set_current_consensus(const char *consensus)
+{
+  networkstatus_vote_t *c;
+  /* Make sure it's parseable. */
+  c = networkstatus_parse_vote_from_string(consensus, 0);
+  if (!c)
+    return -1;
+
+  /* Make sure it's signed enough. */
+  if (networkstatus_check_consensus_signature(c)<0) {
+    networkstatus_vote_free(c);
+    return -1;
+  }
+
+  if (current_consensus)
+    networkstatus_vote_free(current_consensus);
+
+  current_consensus = c;
+
+  if (get_options()->DirPort)
+    dirserv_set_cached_networkstatus_v3(consensus, c->valid_after);
+
+  return 0;
+}
+
 /** We believe networkstatuses more recent than this when they tell us that
  * our server is broken, invalid, obsolete, etc. */
 #define SELF_OPINION_INTERVAL (90*60)