r14769@Kushana: nickm | 2007-10-07 14:00:17 -0400

 Fetch networkstatus consensus documents as needed.  Disabled for clients until more caches support it.


svn:r11788

6 files changed, 116 insertions, 7 deletions

diff --git a/ChangeLog b/ChangeLog
index 34b03d4e96..86721841ef 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -15,6 +15,7 @@ Changes in version 0.2.0.8-alpha - 2007-??-??
   o Minor features (v3 authority system):
     - Add more ways for tools to download the votes that lead to the current
       consensus.
+    - Caches now download v3 network status documents as needed.
 
   o Major bugfixes (performance):
     - Fix really bad O(n^2) performance when parsing a long list of routers:
diff --git a/doc/TODO b/doc/TODO
index 20a4464d90..8d21f8b3bc 100644
--- a/doc/TODO
+++ b/doc/TODO
@@ -78,8 +78,18 @@ Things we'd like to do in 0.2.0.x:
         - When checking a consensus, make sure that its times are plausible.
         o Add a function that will eventually tell us about our clock skew.
           For now, just require that authorities not be skewed.
-      - Start caching consensus documents once authorities make them
-      - Start downloading and using consensus documents once caches serve them
+      . Start caching consensus documents once authorities make them;
+        start downloading consensus documents once caches serve
+        them
+        o Download code
+        o Code to schedule downloads
+        - Code to retry fail downloads
+        - Code to delay next download while fetching certificates
+        - Enable for non-caches
+      - Code to use v3 networkstatus documents once clients are
+        fetching them
+        - Implement
+        - Enable
       - Controller support
         - GETINFO to get consensus
         - Event when new consensus arrives
diff --git a/src/or/directory.c b/src/or/directory.c
index c9fe224bcc..43792b2853 100644
--- a/src/or/directory.c
+++ b/src/or/directory.c
@@ -497,8 +497,12 @@ connection_dir_request_failed(dir_connection_t *conn)
     log_info(LD_DIR, "Giving up on directory server at '%s'; retrying",
              conn->_base.address);
     connection_dir_download_routerdesc_failed(conn);
+  } else if (conn->_base.purpose == DIR_PURPOSE_FETCH_CONSENSUS) {
+    /* XXXX020 NMNM */
+  } else if (conn->_base.purpose == DIR_PURPOSE_FETCH_CERTIFICATE) {
+    /* XXXX020 NMNM */
   } else {
-    /* XXXX020 handle failing: votes. signatures. certificates. consensus. */
+    /* XXXX020 handle failing: votes. signatures. */
   }
 }
 
@@ -1374,11 +1378,29 @@ connection_dir_client_reached_eof(dir_connection_t *conn)
   }
 
   if (conn->_base.purpose == DIR_PURPOSE_FETCH_CONSENSUS) {
-    /*XXXX020*/;
+    /* XXXX020 NMNM */;
+    if (status_code != 200) {
+      log_fn(status_code == 403 ? LOG_INFO : LOG_WARN, LD_DIR,
+          "Received http status code %d (%s) from server "
+          "'%s:%d' while fetching consensus directory.",
+           status_code, escaped(reason), conn->_base.address,
+           conn->_base.port);
+      tor_free(body); tor_free(headers); tor_free(reason);
+      /* XXXX020NMNM retry. */
+      return -1;
+    }
+    log_info(LD_DIR,"Received consensus directory (size %d) from server "
+             "'%s:%d'",(int) body_len, conn->_base.address, conn->_base.port);
+    if (networkstatus_set_current_consensus(body, 0, 0)<0) {
+      log_warn(LD_DIR, "Unable to load consensus directory dowloaded from "
+               "server '%s:%d'", conn->_base.address, conn->_base.port);
+      tor_free(body); tor_free(headers); tor_free(reason);
+      /* XXXX020 NMNM retry. */
+      return -1;
+    }
+    log_info(LD_DIR, "Successfully loaded consensus.");
   }
   if (conn->_base.purpose == DIR_PURPOSE_FETCH_CERTIFICATE) {
-    log_info(LD_DIR,"Received authority certificatess (size %d) from server "
-             "'%s:%d'",(int) body_len, conn->_base.address, conn->_base.port);
     if (status_code != 200) {
       log_fn(status_code == 403 ? LOG_INFO : LOG_WARN, LD_DIR,
           "Received http status code %d (%s) from server "
@@ -1386,8 +1408,11 @@ connection_dir_client_reached_eof(dir_connection_t *conn)
            status_code, escaped(reason), conn->_base.address,
            conn->_base.port, conn->requested_resource);
       tor_free(body); tor_free(headers); tor_free(reason);
+      /* XXXX020NMNM retry. */
       return -1;
     }
+    log_info(LD_DIR,"Received authority certificatess (size %d) from server "
+             "'%s:%d'",(int) body_len, conn->_base.address, conn->_base.port);
     if (trusted_dirs_load_certs_from_string(body, 0)<0) {
       log_warn(LD_DIR, "Unable to parse fetched certificates");
     } else {
diff --git a/src/or/or.h b/src/or/or.h
index ec5fbec182..e049ab4445 100644
--- a/src/or/or.h
+++ b/src/or/or.h
@@ -1247,6 +1247,9 @@ typedef struct routerstatus_t {
   unsigned int version_supports_begindir:1;
   /** True iff this router is a version that we can post extrainfo docs to. */
   unsigned int version_supports_extrainfo_upload:1;
+  /** True iff this router is a version that, if it caches directory info,
+   * we can get v3 downloads from. */
+  unsigned int version_supports_v3_dir:1;
 
   /** True if we, as a directory mirror, want to download the corresponding
    * routerinfo from the authority who gave us this routerstatus.  (That is,
diff --git a/src/or/routerlist.c b/src/or/routerlist.c
index 4da516e0b9..8082bc5f5c 100644
--- a/src/or/routerlist.c
+++ b/src/or/routerlist.c
@@ -34,6 +34,8 @@ static void local_routerstatus_free(local_routerstatus_t *rs);
 static void trusted_dir_server_free(trusted_dir_server_t *ds);
 static void update_networkstatus_cache_downloads(time_t now);
 static void update_networkstatus_client_downloads(time_t now);
+static void update_consensus_networkstatus_fetch_time(time_t now);
+static void update_consensus_networkstatus_downloads(time_t now);
 static int signed_desc_digest_is_recognized(signed_descriptor_t *desc);
 static int have_tried_downloading_all_statuses(int n_failures);
 static routerstatus_t *networkstatus_find_entry(networkstatus_t *ns,
@@ -111,6 +113,13 @@ static time_t last_routerdesc_download_attempted = 0;
  * mirrors).  Clients don't use this now. */
 static time_t last_networkstatus_download_attempted = 0;
 
+/** The last time we tried to download a networkstatus, or 0 for "never".  We
+ * use this to rate-limit download attempts for directory caches (including
+ * mirrors).  Clients don't use this now. */
+static time_t last_consensus_networkstatus_download_attempted = 0;
+/**DOCDOC*/
+static time_t time_to_download_next_consensus = 0;
+
 /** True iff we have logged a warning about this OR not being valid or
  * not being named. */
 static int have_warned_about_invalid_status = 0;
@@ -982,6 +991,9 @@ router_pick_directory_server_impl(int requireother, int fascistfirewall,
     if (requireother && router_digest_is_me(status->identity_digest))
       continue;
     is_trusted = router_digest_is_trusted_dir(status->identity_digest);
+    if ((type & V3_AUTHORITY &&
+         !(status->version_supports_v3_dir || is_trusted)))
+      continue; /* is_trusted is not quite right XXXX020. */
     if ((type & V2_AUTHORITY) && !(status->is_v2_dir || is_trusted))
       continue;
     if ((type & EXTRAINFO_CACHE) &&
@@ -3921,6 +3933,56 @@ update_networkstatus_client_downloads(time_t now)
   smartlist_free(missing);
 }
 
+/** DOCDOC */
+static void
+update_consensus_networkstatus_downloads(time_t now)
+{
+  or_options_t *options = get_options();
+  if (!options->DirPort) /*XXXX020 remove this. */
+    return;
+  if (time_to_download_next_consensus > now)
+    return;
+  if (authdir_mode_v3(options))
+    return;
+  if (connection_get_by_type_purpose(CONN_TYPE_DIR,
+                                     DIR_PURPOSE_FETCH_CONSENSUS))
+    return;
+  /* XXXX020 on failure, delay until next retry. */
+
+  last_consensus_networkstatus_download_attempted = now;/*XXXX020 use this*/
+  directory_get_from_dirserver(DIR_PURPOSE_FETCH_CONSENSUS,
+                               ROUTER_PURPOSE_GENERAL, NULL, 1);
+  // XXXX020 time_to_download_next_consensus = put it off for a while?
+}
+
+/** DOCDOC */
+static void
+update_consensus_networkstatus_fetch_time(time_t now)
+{
+  or_options_t *options = get_options();
+  /* XXXX020 call this when DirPort switches on or off. NMNM */
+  if (current_consensus) {
+    const networkstatus_vote_t *c = current_consensus;
+    time_t start;
+    long interval;
+    if (options->DirPort) {
+      start = c->valid_after + 120; /*XXXX020 make this a macro. */
+      /* XXXX020 too much magic. */
+      interval = (c->fresh_until - c->valid_after) / 2;
+    } else {
+      start = c->fresh_until;
+      /* XXXX020 too much magic. */
+      interval = (c->valid_until - c->fresh_until) * 7 / 8;
+    }
+    if (interval < 1)
+      interval = 1;
+    tor_assert(start+interval < c->valid_until);
+    time_to_download_next_consensus = start + crypto_rand_int(interval);
+  } else {
+    time_to_download_next_consensus = now;
+  }
+}
+
 /** Return 1 if there's a reason we shouldn't try any directory
  * fetches yet (e.g. we demand bridges and none are yet known).
  * Else return 0. */
@@ -3945,6 +4007,7 @@ update_networkstatus_downloads(time_t now)
     update_networkstatus_cache_downloads(now);
   else
     update_networkstatus_client_downloads(now);
+  update_consensus_networkstatus_downloads(now);
 }
 
 /** Clear all our timeouts for fetching v2 directory stuff, and then
@@ -4143,6 +4206,8 @@ networkstatus_set_current_consensus(const char *consensus, int from_cache,
 {
   networkstatus_vote_t *c;
   int r;
+  time_t now = time(NULL);
+
   /* Make sure it's parseable. */
   c = networkstatus_parse_vote_from_string(consensus, NULL, 0);
   if (!c) {
@@ -4164,6 +4229,7 @@ networkstatus_set_current_consensus(const char *consensus, int from_cache,
         tor_free(consensus_waiting_for_certs_body);
         consensus_waiting_for_certs = c;
         consensus_waiting_for_certs_body = tor_strdup(consensus);
+        /*XXXX020 delay next update. NMNM */
         if (!from_cache) {
           or_options_t *options = get_options();
           char filename[512];
@@ -4177,7 +4243,7 @@ networkstatus_set_current_consensus(const char *consensus, int from_cache,
                                             DIR_PURPOSE_FETCH_CERTIFICATE))
           authority_certs_fetch_missing(c);
       }
-      return -1; /* XXXX020 shoul*/
+      return 0;
     } else {
       if (!was_waiting_for_certs)
         log_warn(LD_DIR, "Not enough good signatures on networkstatus "
@@ -4208,6 +4274,8 @@ networkstatus_set_current_consensus(const char *consensus, int from_cache,
 
   current_consensus = c;
 
+  update_consensus_networkstatus_fetch_time(now);
+
   if (!from_cache) {
     or_options_t *options = get_options();
     char filename[512];
diff --git a/src/or/routerparse.c b/src/or/routerparse.c
index 1ab5e73e9f..0198233955 100644
--- a/src/or/routerparse.c
+++ b/src/or/routerparse.c
@@ -1669,6 +1669,8 @@ routerstatus_parse_entry_from_string(const char **s, smartlist_t *tokens,
         tor_version_as_new_as(tok->args[0], "0.1.2.2-alpha");
       rs->version_supports_extrainfo_upload =
         tor_version_as_new_as(tok->args[0], "0.2.0.0-alpha-dev (r10070)");
+      rs->version_supports_v3_dir =
+        tor_version_as_new_as(tok->args[0], "0.2.0.8-alpha");
     }
     if (vote_rs) {
       vote_rs->version = tok->args[0];