Start spooling v2 networkstatus docs as well.

svn:r6664

3 files changed, 145 insertions, 67 deletions

diff --git a/trunk/src/or/directory.c b/trunk/src/or/directory.c
index 16ec510819..84b2db7d74 100644
--- a/trunk/src/or/directory.c
+++ b/trunk/src/or/directory.c
@@ -1418,7 +1418,7 @@ directory_handle_command_get(connection_t *conn, char *headers,
     ++d->refcnt;
 
     /* Prime the connection with some data. */
-    conn->dir_refresh_src = DIR_REFRESH_CACHED_DIR;
+    conn->dir_spool_src = DIR_SPOOL_CACHED_DIR;
     connection_dirserv_flushed_some(conn);
     return 0;
   }
@@ -1454,12 +1454,12 @@ directory_handle_command_get(connection_t *conn, char *headers,
     /* v2 network status fetch. */
     size_t url_len = strlen(url);
     int deflated = !strcmp(url+url_len-2, ".z");
-    smartlist_t *dir_objs = smartlist_create();
+    smartlist_t *dir_fps = smartlist_create();
     const char *request_type = NULL;
     const char *key = url + strlen("/tor/status/");
     if (deflated)
       url[url_len-2] = '\0';
-    dirserv_get_networkstatus_v2(dir_objs, key);
+    dirserv_get_networkstatus_v2_fingerprints(dir_fps, key);
     if (!strcmpstart(key, "fp/"))
       request_type = deflated?"/tor/status/fp.z":"/tor/status/fp";
     else if (!strcmpstart(key, "authority"))
@@ -1470,32 +1470,29 @@ directory_handle_command_get(connection_t *conn, char *headers,
     else
       request_type = "/tor/status/?";
     tor_free(url);
-    if (!smartlist_len(dir_objs)) { /* we failed to create/cache cp */
+    if (!smartlist_len(dir_fps)) { /* we failed to create/cache cp */
       write_http_status_line(conn, 503, "Network status object unavailable");
-      smartlist_free(dir_objs);
+      smartlist_free(dir_fps);
       return 0;
     }
-    dlen = 0;
-    SMARTLIST_FOREACH(dir_objs, cached_dir_t *, d,
-                      dlen += deflated?d->dir_z_len:d->dir_len);
-    note_request(request_type,dlen);
+    // note_request(request_type,dlen);
     format_rfc1123_time(date, time(NULL));
     tor_snprintf(tmp, sizeof(tmp),
-                 "HTTP/1.0 200 OK\r\nDate: %s\r\nContent-Length: %d\r\n"
+                 "HTTP/1.0 200 OK\r\nDate: %s\r\n"
                  "Content-Type: %s\r\nContent-Encoding: %s\r\n\r\n",
                  date,
-                 (int)dlen,
                  deflated?"application/octet-stream":"text/plain",
                  deflated?"deflate":"identity");
     connection_write_to_buf(tmp, strlen(tmp), conn);
-    SMARTLIST_FOREACH(dir_objs, cached_dir_t *, d,
-       {
-         if (deflated)
-           connection_write_to_buf(d->dir_z, d->dir_z_len, conn);
-         else
-           connection_write_to_buf(d->dir, d->dir_len, conn);
-       });
-    smartlist_free(dir_objs);
+
+    conn->fingerprint_stack = dir_fps;
+    if (! deflated)
+      conn->zlib_state = tor_zlib_new(0, ZLIB_METHOD);
+
+    /* Prime the connection with some data. */
+    conn->dir_spool_src = DIR_SPOOL_NETWORKSTATUS;
+    connection_dirserv_flushed_some(conn);
+
     return 0;
   }
 
@@ -1523,9 +1520,9 @@ directory_handle_command_get(connection_t *conn, char *headers,
     else
       request_type = "/tor/server/?";
     if (!strcmpstart(url, "/tor/server/d/"))
-      conn->dir_refresh_src = DIR_REFRESH_SERVER_BY_DIGEST;
+      conn->dir_spool_src = DIR_SPOOL_SERVER_BY_DIGEST;
     else
-      conn->dir_refresh_src = DIR_REFRESH_SERVER_BY_FP;
+      conn->dir_spool_src = DIR_SPOOL_SERVER_BY_FP;
     tor_free(url);
     if (res < 0)
       write_http_status_line(conn, 404, msg);
diff --git a/trunk/src/or/dirserv.c b/trunk/src/or/dirserv.c
index d4a1e47f6e..3ddc78cd3f 100644
--- a/trunk/src/or/dirserv.c
+++ b/trunk/src/or/dirserv.c
@@ -945,11 +945,10 @@ clear_cached_dir(cached_dir_t *d)
 
 /** Free all storage held by the cached_dir_t in <b>d</b>. */
 static void
-free_cached_dir(void *_d)
+_free_cached_dir(void *_d)
 {
   cached_dir_t *d = (cached_dir_t *)_d;
-  clear_cached_dir(d);
-  tor_free(d);
+  cached_dir_decref(d);
 }
 
 /** If we have no cached directory, or it is older than <b>when</b>, then
@@ -978,28 +977,30 @@ dirserv_set_cached_networkstatus_v2(const char *networkstatus,
                                     const char *identity,
                                     time_t published)
 {
-  cached_dir_t *d;
+  cached_dir_t *d, *old_d;
   smartlist_t *trusted_dirs;
   if (!cached_v2_networkstatus)
     cached_v2_networkstatus = digestmap_new();
 
-  if (!(d = digestmap_get(cached_v2_networkstatus, identity))) {
-    if (!networkstatus)
-      return;
-    d = tor_malloc_zero(sizeof(cached_dir_t));
-    digestmap_set(cached_v2_networkstatus, identity, d);
-  }
+  old_d = digestmap_get(cached_v2_networkstatus, identity);
+  if (!old_d && !networkstatus)
+    return;
 
-  tor_assert(d);
   if (networkstatus) {
-    if (published > d->published) {
-      set_cached_dir(d, tor_strdup(networkstatus), published);
+    if (!old_d || published > old_d->published) {
+      d = new_cached_dir(tor_strdup(networkstatus), published);
+      digestmap_set(cached_v2_networkstatus, identity, d);
+      if (old_d)
+        cached_dir_decref(old_d);
     }
   } else {
-    free_cached_dir(d);
-    digestmap_remove(cached_v2_networkstatus, identity);
+    if (old_d) {
+      digestmap_remove(cached_v2_networkstatus, identity);
+      cached_dir_decref(old_d);
+    }
   }
 
+  /* Now purge old entries. */
   trusted_dirs = router_get_trusted_dir_servers();
   if (digestmap_size(cached_v2_networkstatus) >
       smartlist_len(trusted_dirs) + MAX_UNTRUSTED_NETWORKSTATUSES) {
@@ -1024,7 +1025,7 @@ dirserv_set_cached_networkstatus_v2(const char *networkstatus,
     tor_assert(oldest);
     d = digestmap_remove(cached_v2_networkstatus, oldest);
     if (d)
-      free_cached_dir(d);
+      cached_dir_decref(d);
   }
 }
 
@@ -1209,7 +1210,7 @@ dirserv_get_runningrouters(const char **rr, int compress)
 }
 
 /** For authoritative directories: the current (v2) network status */
-static cached_dir_t the_v2_networkstatus = { NULL, NULL, 0, 0, 0, -1 };
+static cached_dir_t *the_v2_networkstatus = NULL;
 
 static int
 should_generate_v2_networkstatus(void)
@@ -1485,13 +1486,14 @@ generate_v2_networkstatus(void)
     goto done;
   }
 
-  set_cached_dir(&the_v2_networkstatus, status, time(NULL));
+  if (the_v2_networkstatus)
+    cached_dir_decref(the_v2_networkstatus);
+  the_v2_networkstatus = new_cached_dir(status, time(NULL));
   status = NULL; /* So it doesn't get double-freed. */
   the_v2_networkstatus_is_dirty = 0;
-  router_set_networkstatus(the_v2_networkstatus.dir, time(NULL), NS_GENERATED,
-                           NULL);
-
-  r = &the_v2_networkstatus;
+  router_set_networkstatus(the_v2_networkstatus->dir,
+                           time(NULL), NS_GENERATED, NULL);
+  r = the_v2_networkstatus;
  done:
   tor_free(client_versions);
   tor_free(server_versions);
@@ -1501,6 +1503,44 @@ generate_v2_networkstatus(void)
   return r;
 }
 
+/* DOCDOC */
+void
+dirserv_get_networkstatus_v2_fingerprints(smartlist_t *result,
+                                          const char *key)
+{
+  tor_assert(result);
+
+  if (!cached_v2_networkstatus)
+    cached_v2_networkstatus = digestmap_new();
+
+  if (should_generate_v2_networkstatus())
+    generate_v2_networkstatus();
+
+  if (!strcmp(key,"authority")) {
+    if (get_options()->AuthoritativeDir) {
+      routerinfo_t *me = router_get_my_routerinfo();
+      if (me)
+        smartlist_add(result,
+                      tor_memdup(me->cache_info.identity_digest, DIGEST_LEN));
+    }
+  } else if (!strcmp(key, "all")) {
+    digestmap_iter_t *iter;
+    iter = digestmap_iter_init(cached_v2_networkstatus);
+    while (!digestmap_iter_done(iter)) {
+      const char *ident;
+      void *val;
+      digestmap_iter_get(iter, &ident, &val);
+      smartlist_add(result, tor_memdup(ident, DIGEST_LEN));
+      iter = digestmap_iter_next(cached_v2_networkstatus, iter);
+    }
+    if (smartlist_len(result) == 0)
+      log_warn(LD_DIRSERV,
+               "Client requested 'all' network status objects; we have none.");
+  } else if (!strcmpstart(key, "fp/")) {
+    dir_split_resource_into_fingerprints(key+3, result, NULL, 1);
+  }
+}
+
 /** Look for a network status object as specified by <b>key</b>, which should
  * be either "authority" (to find a network status generated by us), a hex
  * identity digest (to find a network status generated by given directory), or
@@ -1519,7 +1559,7 @@ dirserv_get_networkstatus_v2(smartlist_t *result,
     if (get_options()->AuthoritativeDir) {
       cached_dir_t *d =
         dirserv_pick_cached_dir_obj(NULL,
-                                    &the_v2_networkstatus,
+                                    the_v2_networkstatus,
                                     the_v2_networkstatus_is_dirty,
                                     generate_v2_networkstatus,
                                     "network status list", 0);
@@ -1747,11 +1787,23 @@ dirserv_orconn_tls_done(const char *address,
  * below this threshold. */
 #define DIRSERV_BUFFER_MIN 16384
 
+static int
+connection_dirserv_finish_spooling(connection_t *conn)
+{
+  if (conn->zlib_state) {
+    connection_write_to_buf_zlib(conn, conn->zlib_state, "", 0, 1);
+    tor_zlib_free(conn->zlib_state);
+    conn->zlib_state = NULL;
+  }
+  conn->dir_spool_src = DIR_SPOOL_NONE;
+  return 0;
+}
+
 /** DOCDOC */
 static int
 connection_dirserv_add_servers_to_outbuf(connection_t *conn)
 {
-  int by_fp = conn->dir_refresh_src == DIR_REFRESH_SERVER_BY_FP;
+  int by_fp = conn->dir_spool_src == DIR_SPOOL_SERVER_BY_FP;
 
   while (smartlist_len(conn->fingerprint_stack) &&
          buf_datalen(conn->outbuf) < DIRSERV_BUFFER_MIN) {
@@ -1785,14 +1837,9 @@ connection_dirserv_add_servers_to_outbuf(connection_t *conn)
 
   if (!smartlist_len(conn->fingerprint_stack)) {
     /* We just wrote the last one; finish up. */
-    if (conn->zlib_state) {
-      connection_write_to_buf_zlib(conn, conn->zlib_state, "", 0, 1);
-      tor_zlib_free(conn->zlib_state);
-      conn->zlib_state = NULL;
-    }
+    connection_dirserv_finish_spooling(conn);
     smartlist_free(conn->fingerprint_stack);
     conn->fingerprint_stack = NULL;
-    conn->dir_refresh_src = DIR_REFRESH_NONE;
   }
   return 0;
 }
@@ -1823,14 +1870,44 @@ connection_dirserv_add_dir_bytes_to_outbuf(connection_t *conn)
   conn->cached_dir_offset += bytes;
   if (conn->cached_dir_offset == (int)conn->cached_dir->dir_z_len) {
     /* We just wrote the last one; finish up. */
-    if (conn->zlib_state) {
-      connection_write_to_buf_zlib(conn, conn->zlib_state, "", 0, 1);
-      tor_zlib_free(conn->zlib_state);
-      conn->zlib_state = NULL;
-    }
+    if (conn->dir_spool_src == DIR_SPOOL_CACHED_DIR)
+      connection_dirserv_finish_spooling(conn);
     cached_dir_decref(conn->cached_dir);
     conn->cached_dir = NULL;
-    conn->dir_refresh_src = DIR_REFRESH_NONE;
+  }
+  return 0;
+}
+
+static int
+connection_dirserv_add_networkstatus_bytes_to_outbuf(connection_t *conn)
+{
+  int r;
+  while (buf_datalen(conn->outbuf) < DIRSERV_BUFFER_MIN) {
+    if (conn->cached_dir) {
+      if ((r = connection_dirserv_add_dir_bytes_to_outbuf(conn)))
+        return r;
+    } else if (conn->fingerprint_stack &&
+               smartlist_len(conn->fingerprint_stack)) {
+      /* Add another networkstatus; start serving it. */
+      char *fp = smartlist_pop_last(conn->fingerprint_stack);
+      cached_dir_t *d;
+      if (router_digest_is_me(fp))
+        d = the_v2_networkstatus;
+      else
+        d = digestmap_get(cached_v2_networkstatus, fp);
+      tor_free(fp);
+      if (d) {
+        ++d->refcnt;
+        conn->cached_dir = d;
+        conn->cached_dir_offset = 0;
+      }
+    } else {
+      connection_dirserv_finish_spooling(conn);
+      if (conn->fingerprint_stack)
+        smartlist_free(conn->fingerprint_stack);
+      conn->fingerprint_stack = NULL;
+      return 0;
+    }
   }
   return 0;
 }
@@ -1843,16 +1920,18 @@ connection_dirserv_flushed_some(connection_t *conn)
   tor_assert(conn->type == CONN_TYPE_DIR);
   tor_assert(conn->state == DIR_CONN_STATE_SERVER_WRITING);
 
-  if (conn->dir_refresh_src == DIR_REFRESH_NONE
+  if (conn->dir_spool_src == DIR_SPOOL_NONE
       || buf_datalen(conn->outbuf) >= DIRSERV_BUFFER_MIN)
     return 0;
 
-  switch (conn->dir_refresh_src) {
-    case DIR_REFRESH_SERVER_BY_DIGEST:
-    case DIR_REFRESH_SERVER_BY_FP:
+  switch (conn->dir_spool_src) {
+    case DIR_SPOOL_SERVER_BY_DIGEST:
+    case DIR_SPOOL_SERVER_BY_FP:
       return connection_dirserv_add_servers_to_outbuf(conn);
-    case DIR_REFRESH_CACHED_DIR:
+    case DIR_SPOOL_CACHED_DIR:
       return connection_dirserv_add_dir_bytes_to_outbuf(conn);
+    case DIR_SPOOL_NETWORKSTATUS:
+      return connection_dirserv_add_networkstatus_bytes_to_outbuf(conn);
     default:
       return 0;
   }
@@ -1872,11 +1951,11 @@ dirserv_free_all(void)
   }
   cached_dir_decref(the_directory);
   clear_cached_dir(&the_runningrouters);
-  clear_cached_dir(&the_v2_networkstatus);
+  cached_dir_decref(the_v2_networkstatus);
   cached_dir_decref(cached_directory);
   clear_cached_dir(&cached_runningrouters);
   if (cached_v2_networkstatus) {
-    digestmap_free(cached_v2_networkstatus, free_cached_dir);
+    digestmap_free(cached_v2_networkstatus, _free_cached_dir);
     cached_v2_networkstatus = NULL;
   }
 }
diff --git a/trunk/src/or/or.h b/trunk/src/or/or.h
index 08527e066b..7346e225ec 100644
--- a/trunk/src/or/or.h
+++ b/trunk/src/or/or.h
@@ -699,9 +699,9 @@ struct connection_t {
                              * for?*/
 /* Used only for server sides of some dir connections. */
   enum {
-    DIR_REFRESH_NONE=0, DIR_REFRESH_SERVER_BY_DIGEST, DIR_REFRESH_SERVER_BY_FP,
-    DIR_REFRESH_CACHED_DIR
-  } dir_refresh_src;
+    DIR_SPOOL_NONE=0, DIR_SPOOL_SERVER_BY_DIGEST, DIR_SPOOL_SERVER_BY_FP,
+    DIR_SPOOL_CACHED_DIR, DIR_SPOOL_NETWORKSTATUS
+  } dir_spool_src;
   smartlist_t *fingerprint_stack;
   struct cached_dir_t *cached_dir;
   off_t cached_dir_offset;
@@ -1938,6 +1938,8 @@ void dirserv_set_cached_networkstatus_v2(const char *directory,
                                          const char *identity,
                                          time_t published);
 void dirserv_get_networkstatus_v2(smartlist_t *result, const char *key);
+void dirserv_get_networkstatus_v2_fingerprints(smartlist_t *result,
+                                               const char *key);
 int dirserv_get_routerdesc_fingerprints(smartlist_t *fps_out, const char *key,
                                         const char **msg);
 int dirserv_get_routerdescs(smartlist_t *descs_out, const char *key,