Include contents of *-stats files in descriptor.

6 files changed, 185 insertions, 32 deletions

diff --git a/src/or/config.c b/src/or/config.c
index 89c94b1997..7b3f0ad07f 100644
--- a/src/or/config.c
+++ b/src/or/config.c
@@ -208,6 +208,7 @@ static config_var_t _option_vars[] = {
   V(ExitPolicy,                  LINELIST, NULL),
   V(ExitPolicyRejectPrivate,     BOOL,     "1"),
   V(ExitPortStatistics,          BOOL,     "0"),
+  V(ExtraInfoStatistics,         BOOL,     "0"),
   V(FallbackNetworkstatusFile,   FILENAME,
     SHARE_DATADIR PATH_SEPARATOR "tor" PATH_SEPARATOR "fallback-consensus"),
   V(FascistFirewall,             BOOL,     "0"),
diff --git a/src/or/geoip.c b/src/or/geoip.c
index a07e4b585f..824428b7d9 100644
--- a/src/or/geoip.c
+++ b/src/or/geoip.c
@@ -948,7 +948,7 @@ dump_geoip_stats(void)
   time_t request_start;
   char *filename = get_datadir_fname("dirreq-stats");
   char *data_v2 = NULL, *data_v3 = NULL;
-  char since[ISO_TIME_LEN+1], written[ISO_TIME_LEN+1];
+  char written[ISO_TIME_LEN+1];
   open_file_t *open_file = NULL;
   double v2_share = 0.0, v3_share = 0.0;
   FILE *out;
@@ -961,14 +961,13 @@ dump_geoip_stats(void)
                 GEOIP_CLIENT_NETWORKSTATUS_V2);
   data_v3 = geoip_get_client_history_dirreq(now,
                 GEOIP_CLIENT_NETWORKSTATUS);
-  format_iso_time(since, geoip_get_history_start());
-  format_iso_time(written, now);
+  format_iso_time(written, geoip_get_history_start() + REQUEST_HIST_PERIOD);
   out = start_writing_to_stdio_file(filename, OPEN_FLAGS_APPEND,
                                     0600, &open_file);
   if (!out)
     goto done;
-  if (fprintf(out, "written %s\nstarted-at %s\nns-ips %s\nns-v2-ips %s\n",
-              written, since,
+  if (fprintf(out, "dirreq-stats-end %s (%d s)\ndirreq-v3-ips %s\n"
+              "dirreq-v2-ips %s\n", written, REQUEST_HIST_PERIOD,
               data_v3 ? data_v3 : "", data_v2 ? data_v2 : "") < 0)
     goto done;
   tor_free(data_v2);
@@ -976,11 +975,9 @@ dump_geoip_stats(void)
 
   request_start = current_request_period_starts -
     (n_old_request_periods * REQUEST_HIST_PERIOD);
-  format_iso_time(since, request_start);
   data_v2 = geoip_get_request_history(now, GEOIP_CLIENT_NETWORKSTATUS_V2);
   data_v3 = geoip_get_request_history(now, GEOIP_CLIENT_NETWORKSTATUS);
-  if (fprintf(out, "requests-start %s\nn-ns-reqs %s\nn-v2-ns-reqs %s\n",
-              since,
+  if (fprintf(out, "dirreq-v3-reqs %s\ndirreq-v2-reqs %s\n",
               data_v3 ? data_v3 : "", data_v2 ? data_v2 : "") < 0)
     goto done;
 #define RESPONSE_GRANULARITY 8
@@ -991,7 +988,7 @@ dump_geoip_stats(void)
                                ns_v3_responses[i], RESPONSE_GRANULARITY);
   }
 #undef RESPONSE_GRANULARITY
-  if (fprintf(out, "n-ns-resp ok=%u,not-enough-sigs=%u,unavailable=%u,"
+  if (fprintf(out, "dirreq-v3-resp ok=%u,not-enough-sigs=%u,unavailable=%u,"
                    "not-found=%u,not-modified=%u,busy=%u\n",
                    ns_v3_responses[GEOIP_SUCCESS],
                    ns_v3_responses[GEOIP_REJECT_NOT_ENOUGH_SIGS],
@@ -1000,7 +997,7 @@ dump_geoip_stats(void)
                    ns_v3_responses[GEOIP_REJECT_NOT_MODIFIED],
                    ns_v3_responses[GEOIP_REJECT_BUSY]) < 0)
     goto done;
-  if (fprintf(out, "n-v2-ns-resp ok=%u,unavailable=%u,"
+  if (fprintf(out, "dirreq-v2-resp ok=%u,unavailable=%u,"
                    "not-found=%u,not-modified=%u,busy=%u\n",
                    ns_v2_responses[GEOIP_SUCCESS],
                    ns_v2_responses[GEOIP_REJECT_UNAVAILABLE],
@@ -1011,9 +1008,9 @@ dump_geoip_stats(void)
   memset(ns_v2_responses, 0, sizeof(ns_v2_responses));
   memset(ns_v3_responses, 0, sizeof(ns_v3_responses));
   if (!geoip_get_mean_shares(now, &v2_share, &v3_share)) {
-    if (fprintf(out, "v2-ns-share %0.2lf%%\n", v2_share*100) < 0)
+    if (fprintf(out, "dirreq-v2-share %0.2lf%%\n", v2_share*100) < 0)
       goto done;
-    if (fprintf(out, "v3-ns-share %0.2lf%%\n", v3_share*100) < 0)
+    if (fprintf(out, "dirreq-v3-share %0.2lf%%\n", v3_share*100) < 0)
       goto done;
   }
 
@@ -1021,7 +1018,7 @@ dump_geoip_stats(void)
                                        DIRREQ_DIRECT);
   data_v3 = geoip_get_dirreq_history(GEOIP_CLIENT_NETWORKSTATUS,
                                        DIRREQ_DIRECT);
-  if (fprintf(out, "ns-direct-dl %s\nns-v2-direct-dl %s\n",
+  if (fprintf(out, "dirreq-v3-direct-dl %s\ndirreq-v2-direct-dl %s\n",
               data_v3 ? data_v3 : "", data_v2 ? data_v2 : "") < 0)
     goto done;
   tor_free(data_v2);
@@ -1030,7 +1027,7 @@ dump_geoip_stats(void)
                                        DIRREQ_TUNNELED);
   data_v3 = geoip_get_dirreq_history(GEOIP_CLIENT_NETWORKSTATUS,
                                        DIRREQ_TUNNELED);
-  if (fprintf(out, "ns-tunneled-dl %s\nns-v2-tunneled-dl %s\n",
+  if (fprintf(out, "dirreq-v3-tunneled-dl %s\ndirreq-v2-tunneled-dl %s\n",
               data_v3 ? data_v3 : "", data_v2 ? data_v2 : "") < 0)
     goto done;
 
@@ -1053,19 +1050,18 @@ dump_entry_stats(void)
   time_t now = time(NULL);
   char *filename = get_datadir_fname("entry-stats");
   char *data = NULL;
-  char since[ISO_TIME_LEN+1], written[ISO_TIME_LEN+1];
+  char written[ISO_TIME_LEN+1];
   open_file_t *open_file = NULL;
   FILE *out;
 
   data = geoip_get_client_history(now, GEOIP_CLIENT_CONNECT);
-  format_iso_time(since, geoip_get_history_start());
-  format_iso_time(written, now);
+  format_iso_time(written, geoip_get_history_start() + REQUEST_HIST_PERIOD);
   out = start_writing_to_stdio_file(filename, OPEN_FLAGS_APPEND,
                                     0600, &open_file);
   if (!out)
     goto done;
-  if (fprintf(out, "written %s\nstarted-at %s\nips %s\n",
-              written, since, data ? data : "") < 0)
+  if (fprintf(out, "entry-stats-end %s (%d s)\nentry-ips %s\n",
+              written, REQUEST_HIST_PERIOD, data ? data : "") < 0)
     goto done;
 
   finish_writing_to_file(open_file);
diff --git a/src/or/or.h b/src/or/or.h
index ff2e65c036..ce4fbd074e 100644
--- a/src/or/or.h
+++ b/src/or/or.h
@@ -2516,6 +2516,9 @@ typedef struct {
   /** If true, the user wants us to collect statistics as entry node. */
   int EntryStatistics;
 
+  /** If true, include statistics file contents in extra-info documents. */
+  int ExtraInfoStatistics;
+
   /** If true, do not believe anybody who tells us that a domain resolves
    * to an internal address, or that an internal address has a PTR mapping.
    * Helps avoid some cross-site attacks. */
diff --git a/src/or/rephist.c b/src/or/rephist.c
index abfd47742c..a9bf95da1b 100644
--- a/src/or/rephist.c
+++ b/src/or/rephist.c
@@ -1390,7 +1390,8 @@ write_exit_stats(time_t when)
     }
 
     /* written yyyy-mm-dd HH:MM:SS (n s) */
-    if (fprintf(out, "written %s (%d s)\n", t, EXIT_STATS_INTERVAL_SEC) < 0)
+    if (fprintf(out, "exit-stats-end %s (%d s)\n", t,
+                EXIT_STATS_INTERVAL_SEC) < 0)
       goto done;
 
     /* Count the total number of bytes, so that we can attribute all
@@ -1408,7 +1409,8 @@ write_exit_stats(time_t when)
       b = r ? exit_bytes_read : exit_bytes_written;
       tor_assert(b);
       if (fprintf(out, "%s ",
-                  r ? "kibibytes-read" : "kibibytes-written")<0)
+                  r ? "exit-kibibytes-read"
+                    : "exit-kibibytes-written") < 0)
         goto done;
 
       comma = 0;
@@ -1435,7 +1437,7 @@ write_exit_stats(time_t when)
         goto done;
     }
     /* streams-opened port=num,.. */
-    if (fprintf(out, "streams-opened ")<0)
+    if (fprintf(out, "exit-streams-opened ") < 0)
       goto done;
     comma = 0;
     other_streams = 0;
@@ -2733,7 +2735,7 @@ dump_buffer_stats(void)
   if (!out)
     goto done;
   format_iso_time(written, now);
-  if (fprintf(out, "written %s (%d s)\n", written,
+  if (fprintf(out, "cell-stats-end %s (%d s)\n", written,
               DUMP_BUFFER_STATS_INTERVAL) < 0)
     goto done;
   for (i = 0; i < SHARES; i++) {
@@ -2742,7 +2744,7 @@ dump_buffer_stats(void)
     smartlist_add(str_build, tor_strdup(buf));
   }
   str = smartlist_join_strings(str_build, ",", 0, NULL);
-  if (fprintf(out, "processed-cells %s\n", str) < 0)
+  if (fprintf(out, "cell-processed-cells %s\n", str) < 0)
     goto done;
   tor_free(str);
   SMARTLIST_FOREACH(str_build, char *, c, tor_free(c));
@@ -2753,7 +2755,7 @@ dump_buffer_stats(void)
     smartlist_add(str_build, tor_strdup(buf));
   }
   str = smartlist_join_strings(str_build, ",", 0, NULL);
-  if (fprintf(out, "queued-cells %s\n", str) < 0)
+  if (fprintf(out, "cell-queued-cells %s\n", str) < 0)
     goto done;
   tor_free(str);
   SMARTLIST_FOREACH(str_build, char *, c, tor_free(c));
@@ -2764,13 +2766,13 @@ dump_buffer_stats(void)
     smartlist_add(str_build, tor_strdup(buf));
   }
   str = smartlist_join_strings(str_build, ",", 0, NULL);
-  if (fprintf(out, "time-in-queue %s\n", str) < 0)
+  if (fprintf(out, "cell-time-in-queue %s\n", str) < 0)
     goto done;
   tor_free(str);
   SMARTLIST_FOREACH(str_build, char *, c, tor_free(c));
   smartlist_free(str_build);
   str_build = NULL;
-  if (fprintf(out, "number-of-circuits-per-share %d\n",
+  if (fprintf(out, "cell-circuits-per-decile %d\n",
               (number_of_circuits + SHARES - 1) / SHARES) < 0)
     goto done;
   finish_writing_to_file(open_file);
diff --git a/src/or/router.c b/src/or/router.c
index c66b66f0ea..e1181971c4 100644
--- a/src/or/router.c
+++ b/src/or/router.c
@@ -1382,8 +1382,10 @@ router_rebuild_descriptor(int force)
   ei->cache_info.published_on = ri->cache_info.published_on;
   memcpy(ei->cache_info.identity_digest, ri->cache_info.identity_digest,
          DIGEST_LEN);
-  ei->cache_info.signed_descriptor_body = tor_malloc(8192);
-  if (extrainfo_dump_to_string(ei->cache_info.signed_descriptor_body, 8192,
+  ei->cache_info.signed_descriptor_body =
+      tor_malloc(MAX_EXTRAINFO_UPLOAD_SIZE);
+  if (extrainfo_dump_to_string(ei->cache_info.signed_descriptor_body,
+                               MAX_EXTRAINFO_UPLOAD_SIZE,
                                ei, get_identity_key()) < 0) {
     log_warn(LD_BUG, "Couldn't generate extra-info descriptor.");
     extrainfo_free(ei);
@@ -1824,6 +1826,31 @@ router_dump_router_to_string(char *s, size_t maxlen, routerinfo_t *router,
   return (int)written+1;
 }
 
+/** Load the contents of <b>filename</b> and write its contents to
+ * **<b>contents</b>. Return 1 for success, 0 if the file does not exit,
+ * or -1 for failure. */
+static int
+load_stats_file(const char *filename, char **contents)
+{
+  int r = -1;
+  char *fname = get_datadir_fname(filename);
+  switch (file_status(fname)) {
+    case FN_FILE:
+      if ((*contents = read_file_to_str(fname, 0, NULL)))
+        r = 1;
+      break;
+    case FN_NOENT:
+      r = 0;
+      break;
+    case FN_ERROR:
+    case FN_DIR:
+    default:
+      break;
+  }
+  tor_free(fname);
+  return r;
+}
+
 /** Write the contents of <b>extrainfo</b> to the <b>maxlen</b>-byte string
  * <b>s</b>, signing them with <b>ident_key</b>.  Return 0 on success,
  * negative on failure. */
@@ -1838,6 +1865,7 @@ extrainfo_dump_to_string(char *s, size_t maxlen, extrainfo_t *extrainfo,
   char *bandwidth_usage;
   int result;
   size_t len;
+  static int write_stats_to_extrainfo = 1;
 
   base16_encode(identity, sizeof(identity),
                 extrainfo->cache_info.identity_digest, DIGEST_LEN);
@@ -1849,6 +1877,56 @@ extrainfo_dump_to_string(char *s, size_t maxlen, extrainfo_t *extrainfo,
                         "published %s\n%s",
                         extrainfo->nickname, identity,
                         published, bandwidth_usage);
+
+  if (options->ExtraInfoStatistics && write_stats_to_extrainfo) {
+    char *contents = NULL;
+    log_info(LD_GENERAL, "Adding stats to extra-info descriptor.");
+    if (options->DirReqStatistics &&
+        load_stats_file("dirreq-stats", &contents) > 0) {
+      int pos = strlen(s);
+      if (strlcpy(s + pos, contents, maxlen - strlen(s)) !=
+          strlen(contents)) {
+        log_warn(LD_DIR, "Could not write dirreq-stats to extra-info "
+                 "descriptor.");
+        s[pos] = '\0';
+      }
+      tor_free(contents);
+    }
+    if (options->EntryStatistics &&
+        load_stats_file("entry-stats", &contents) > 0) {
+      int pos = strlen(s);
+      if (strlcpy(s + pos, contents, maxlen - strlen(s)) !=
+          strlen(contents)) {
+        log_warn(LD_DIR, "Could not write entry-stats to extra-info "
+                 "descriptor.");
+        s[pos] = '\0';
+      }
+      tor_free(contents);
+    }
+    if (options->CellStatistics &&
+        load_stats_file("buffer-stats", &contents) > 0) {
+      int pos = strlen(s);
+      if (strlcpy(s + pos, contents, maxlen - strlen(s)) !=
+          strlen(contents)) {
+        log_warn(LD_DIR, "Could not write buffer-stats to extra-info "
+                 "descriptor.");
+        s[pos] = '\0';
+      }
+      tor_free(contents);
+    }
+    if (options->ExitPortStatistics &&
+        load_stats_file("exit-stats", &contents) > 0) {
+      int pos = strlen(s);
+      if (strlcpy(s + pos, contents, maxlen - strlen(s)) !=
+          strlen(contents)) {
+        log_warn(LD_DIR, "Could not write exit-stats to extra-info "
+                 "descriptor.");
+        s[pos] = '\0';
+      }
+      tor_free(contents);
+    }
+  }
+
   tor_free(bandwidth_usage);
   if (result<0)
     return -1;
@@ -1877,22 +1955,45 @@ extrainfo_dump_to_string(char *s, size_t maxlen, extrainfo_t *extrainfo,
   if (router_append_dirobj_signature(s+len, maxlen-len, digest, ident_key)<0)
     return -1;
 
-#ifdef DEBUG_ROUTER_DUMP_ROUTER_TO_STRING
   {
     char *cp, *s_dup;
     extrainfo_t *ei_tmp;
     cp = s_dup = tor_strdup(s);
     ei_tmp = extrainfo_parse_entry_from_string(cp, NULL, 1, NULL);
-    if (!ei_tmp) {
+    if (ei_tmp) {
       log_err(LD_BUG,
               "We just generated an extrainfo descriptor we can't parse.");
       log_err(LD_BUG, "Descriptor was: <<%s>>", s);
       return -1;
+    } else {
+      log_debug(LD_GENERAL, "We generated an extra-info descriptor:\n%s",
+                s);
     }
     tor_free(s_dup);
     extrainfo_free(ei_tmp);
   }
-#endif
+
+  if (options->ExtraInfoStatistics && write_stats_to_extrainfo) {
+    char *cp, *s_dup;
+    extrainfo_t *ei_tmp;
+    cp = s_dup = tor_strdup(s);
+    ei_tmp = extrainfo_parse_entry_from_string(cp, NULL, 1, NULL);
+    if (!ei_tmp) {
+      log_warn(LD_GENERAL,
+               "We just generated an extra-info descriptor with "
+               "statistics that we can't parse. Not adding statistics to "
+               "this or any future extra-info descriptors. Descriptor "
+               "was:\n%s", s);
+      write_stats_to_extrainfo = 0;
+      tor_free(s);
+      s = NULL;
+      extrainfo_dump_to_string(s, maxlen, extrainfo, ident_key);
+    }
+    tor_free(s_dup);
+    extrainfo_free(ei_tmp);
+  }
+
+  log_info(LD_GENERAL, "Done with dumping our extra-info descriptor.");
 
   return (int)strlen(s)+1;
 }
diff --git a/src/or/routerparse.c b/src/or/routerparse.c
index 8021158e31..1e505d67c7 100644
--- a/src/or/routerparse.c
+++ b/src/or/routerparse.c
@@ -62,6 +62,31 @@ typedef enum {
   K_HIDDEN_SERVICE_DIR,
   K_ALLOW_SINGLE_HOP_EXITS,
 
+  K_DIRREQ_END,
+  K_DIRREQ_V2_IPS,
+  K_DIRREQ_V3_IPS,
+  K_DIRREQ_V2_REQS,
+  K_DIRREQ_V3_REQS,
+  K_DIRREQ_V2_SHARE,
+  K_DIRREQ_V3_SHARE,
+  K_DIRREQ_V2_RESP,
+  K_DIRREQ_V3_RESP,
+  K_DIRREQ_V2_DIR,
+  K_DIRREQ_V3_DIR,
+  K_DIRREQ_V2_TUN,
+  K_DIRREQ_V3_TUN,
+  K_ENTRY_END,
+  K_ENTRY_IPS,
+  K_CELL_END,
+  K_CELL_PROCESSED,
+  K_CELL_QUEUED,
+  K_CELL_TIME,
+  K_CELL_CIRCS,
+  K_EXIT_END,
+  K_EXIT_WRITTEN,
+  K_EXIT_READ,
+  K_EXIT_OPENED,
+
   K_DIR_KEY_CERTIFICATE_VERSION,
   K_DIR_IDENTITY_KEY,
   K_DIR_KEY_PUBLISHED,
@@ -257,6 +282,31 @@ static token_rule_t extrainfo_token_table[] = {
   T0N("opt",                 K_OPT,             CONCAT_ARGS, OBJ_OK ),
   T01("read-history",        K_READ_HISTORY,        ARGS,    NO_OBJ ),
   T01("write-history",       K_WRITE_HISTORY,       ARGS,    NO_OBJ ),
+  T01("dirreq-stats-end",    K_DIRREQ_END,          ARGS,    NO_OBJ ),
+  T01("dirreq-v2-ips",       K_DIRREQ_V2_IPS,       ARGS,    NO_OBJ ),
+  T01("dirreq-v3-ips",       K_DIRREQ_V3_IPS,       ARGS,    NO_OBJ ),
+  T01("dirreq-v2-reqs",      K_DIRREQ_V2_REQS,      ARGS,    NO_OBJ ),
+  T01("dirreq-v3-reqs",      K_DIRREQ_V3_REQS,      ARGS,    NO_OBJ ),
+  T01("dirreq-v2-share",     K_DIRREQ_V2_SHARE,     ARGS,    NO_OBJ ),
+  T01("dirreq-v3-share",     K_DIRREQ_V3_SHARE,     ARGS,    NO_OBJ ),
+  T01("dirreq-v2-resp",      K_DIRREQ_V2_RESP,      ARGS,    NO_OBJ ),
+  T01("dirreq-v3-resp",      K_DIRREQ_V3_RESP,      ARGS,    NO_OBJ ),
+  T01("dirreq-v2-direct-dl", K_DIRREQ_V2_DIR,       ARGS,    NO_OBJ ),
+  T01("dirreq-v3-direct-dl", K_DIRREQ_V3_DIR,       ARGS,    NO_OBJ ),
+  T01("dirreq-v2-tunneled-dl", K_DIRREQ_V2_TUN,     ARGS,    NO_OBJ ),
+  T01("dirreq-v3-tunneled-dl", K_DIRREQ_V3_TUN,     ARGS,    NO_OBJ ),
+  T01("entry-stats-end",     K_ENTRY_END,           ARGS,    NO_OBJ ),
+  T01("entry-ips",           K_ENTRY_IPS,           ARGS,    NO_OBJ ),
+  T01("cell-stats-end",      K_CELL_END,            ARGS,    NO_OBJ ),
+  T01("cell-processed-cells", K_CELL_PROCESSED,     ARGS,    NO_OBJ ),
+  T01("cell-queued-cells",   K_CELL_QUEUED,         ARGS,    NO_OBJ ),
+  T01("cell-time-in-queue",  K_CELL_TIME,           ARGS,    NO_OBJ ),
+  T01("cell-circuits-per-decile", K_CELL_CIRCS,     ARGS,    NO_OBJ ),
+  T01("exit-stats-end",      K_EXIT_END,            ARGS,    NO_OBJ ),
+  T01("exit-kibibytes-written", K_EXIT_WRITTEN,     ARGS,    NO_OBJ ),
+  T01("exit-kibibytes-read", K_EXIT_READ,           ARGS,    NO_OBJ ),
+  T01("exit-streams-opened", K_EXIT_OPENED,         ARGS,    NO_OBJ ),
+
   T1_START( "extra-info",          K_EXTRA_INFO,          GE(2),   NO_OBJ ),
 
   END_OF_TABLE