Merge commit 'karsten/proposal-166-impl-master'

1 files changed, 145 insertions, 108 deletions

diff --git a/src/or/geoip.c b/src/or/geoip.c
index c61ac3c5fd..befc3d9e06 100644
--- a/src/or/geoip.c
+++ b/src/or/geoip.c
@@ -12,8 +12,6 @@
 #include "ht.h"
 
 static void clear_geoip_db(void);
-static void dump_geoip_stats(void);
-static void dump_entry_stats(void);
 
 /** An entry from the GeoIP file: maps an IP range to a country. */
 typedef struct geoip_entry_t {
@@ -347,7 +345,6 @@ geoip_determine_shares(time_t now)
   last_time_determined_shares = now;
 }
 
-#ifdef ENABLE_DIRREQ_STATS
 /** Calculate which fraction of v2 and v3 directory requests aimed at caches
  * have been sent to us since the last call of this function up to time
  * <b>now</b>. Set *<b>v2_share_out</b> and *<b>v3_share_out</b> to the
@@ -367,7 +364,23 @@ geoip_get_mean_shares(time_t now, double *v2_share_out,
   share_seconds = 0;
   return 0;
 }
-#endif
+
+/* Rotate period of v2 and v3 network status requests. */
+static void
+rotate_request_period(void)
+{
+  SMARTLIST_FOREACH(geoip_countries, geoip_country_t *, c, {
+      memmove(&c->n_v2_ns_requests[0], &c->n_v2_ns_requests[1],
+              sizeof(uint32_t)*(REQUEST_HIST_LEN-1));
+      memmove(&c->n_v3_ns_requests[0], &c->n_v3_ns_requests[1],
+              sizeof(uint32_t)*(REQUEST_HIST_LEN-1));
+      c->n_v2_ns_requests[REQUEST_HIST_LEN-1] = 0;
+      c->n_v3_ns_requests[REQUEST_HIST_LEN-1] = 0;
+    });
+  current_request_period_starts += REQUEST_HIST_PERIOD;
+  if (n_old_request_periods < REQUEST_HIST_LEN-1)
+    ++n_old_request_periods;
+}
 
 /** Note that we've seen a client connect from the IP <b>addr</b> (host order)
  * at time <b>now</b>. Ignored by all but bridges and directories if
@@ -379,55 +392,37 @@ geoip_note_client_seen(geoip_client_action_t action,
   or_options_t *options = get_options();
   clientmap_entry_t lookup, *ent;
   if (action == GEOIP_CLIENT_CONNECT) {
-#ifdef ENABLE_ENTRY_STATS
-    if (!options->EntryStatistics)
+    /* Only remember statistics as entry guard or as bridge. */
+    if (!options->EntryStatistics ||
+        (!(options->BridgeRelay && options->BridgeRecordUsageByCountry)))
       return;
-#else
-    if (!(options->BridgeRelay && options->BridgeRecordUsageByCountry))
-      return;
-#endif
     /* Did we recently switch from bridge to relay or back? */
     if (client_history_starts > now)
       return;
   } else {
-#ifndef ENABLE_DIRREQ_STATS
-    return;
-#else
     if (options->BridgeRelay || options->BridgeAuthoritativeDir ||
         !options->DirReqStatistics)
       return;
-#endif
   }
 
-  /* Rotate the current request period. */
-  while (current_request_period_starts + REQUEST_HIST_PERIOD < now) {
-    if (!geoip_countries)
-      geoip_countries = smartlist_create();
-    if (!current_request_period_starts) {
-      current_request_period_starts = now;
-      break;
+  /* As a bridge that doesn't rotate request periods every 24 hours,
+   * possibly rotate now. */
+  if (options->BridgeRelay) {
+    while (current_request_period_starts + REQUEST_HIST_PERIOD < now) {
+      if (!geoip_countries)
+        geoip_countries = smartlist_create();
+      if (!current_request_period_starts) {
+        current_request_period_starts = now;
+        break;
+      }
+      /* Also discard all items in the client history that are too old.
+       * (This only works here because bridge and directory stats are
+       * independent. Otherwise, we'd only want to discard those items
+       * with action GEOIP_CLIENT_NETWORKSTATUS{_V2}.) */
+      geoip_remove_old_clients(current_request_period_starts);
+      /* Now rotate request period */
+      rotate_request_period();
     }
-    /* Also discard all items in the client history that are too old.
-     * (This only works here because bridge and directory stats are
-     * independent. Otherwise, we'd only want to discard those items
-     * with action GEOIP_CLIENT_NETWORKSTATUS{_V2}.) */
-    geoip_remove_old_clients(current_request_period_starts);
-    /* Before rotating, write the current stats to disk. */
-    dump_geoip_stats();
-    if (get_options()->EntryStatistics)
-      dump_entry_stats();
-    /* Now rotate request period */
-    SMARTLIST_FOREACH(geoip_countries, geoip_country_t *, c, {
-        memmove(&c->n_v2_ns_requests[0], &c->n_v2_ns_requests[1],
-                sizeof(uint32_t)*(REQUEST_HIST_LEN-1));
-        memmove(&c->n_v3_ns_requests[0], &c->n_v3_ns_requests[1],
-                sizeof(uint32_t)*(REQUEST_HIST_LEN-1));
-        c->n_v2_ns_requests[REQUEST_HIST_LEN-1] = 0;
-        c->n_v3_ns_requests[REQUEST_HIST_LEN-1] = 0;
-      });
-    current_request_period_starts += REQUEST_HIST_PERIOD;
-    if (n_old_request_periods < REQUEST_HIST_LEN-1)
-      ++n_old_request_periods;
   }
 
   lookup.ipaddr = addr;
@@ -495,7 +490,6 @@ geoip_remove_old_clients(time_t cutoff)
     client_history_starts = cutoff;
 }
 
-#ifdef ENABLE_DIRREQ_STATS
 /** How many responses are we giving to clients requesting v2 network
  * statuses? */
 static uint32_t ns_v2_responses[GEOIP_NS_RESPONSE_NUM];
@@ -503,7 +497,6 @@ static uint32_t ns_v2_responses[GEOIP_NS_RESPONSE_NUM];
 /** How many responses are we giving to clients requesting v3 network
  * statuses? */
 static uint32_t ns_v3_responses[GEOIP_NS_RESPONSE_NUM];
-#endif
 
 /** Note that we've rejected a client's request for a v2 or v3 network
  * status, encoded in <b>action</b> for reason <b>reason</b> at time
@@ -512,7 +505,6 @@ void
 geoip_note_ns_response(geoip_client_action_t action,
                        geoip_ns_response_t response)
 {
-#ifdef ENABLE_DIRREQ_STATS
   static int arrays_initialized = 0;
   if (!get_options()->DirReqStatistics)
     return;
@@ -528,10 +520,6 @@ geoip_note_ns_response(geoip_client_action_t action,
     ns_v3_responses[response]++;
   else
     ns_v2_responses[response]++;
-#else
-  (void) action;
-  (void) response;
-#endif
 }
 
 /** Do not mention any country from which fewer than this number of IPs have
@@ -709,7 +697,6 @@ geoip_change_dirreq_state(uint64_t dirreq_id, dirreq_type_t type,
   }
 }
 
-#ifdef ENABLE_DIRREQ_STATS
 /** Return a newly allocated comma-separated string containing statistics
  * on network status downloads. The string contains the number of completed
  * requests, timeouts, and still running requests as well as the download
@@ -811,25 +798,18 @@ geoip_get_dirreq_history(geoip_client_action_t action,
   smartlist_free(dirreq_completed);
   return result;
 }
-#endif
 
 /** How long do we have to have observed per-country request history before we
  * are willing to talk about it? */
 #define GEOIP_MIN_OBSERVATION_TIME (12*60*60)
 
-/** Return a newly allocated comma-separated string containing entries for all
- * the countries from which we've seen enough clients connect. The entry
- * format is cc=num where num is the number of IPs we've seen connecting from
- * that country, and cc is a lowercased country code. Returns NULL if we don't
- * want to export geoip data yet. */
-char *
-geoip_get_client_history(time_t now, geoip_client_action_t action)
+/** Helper for geoip_get_client_history_dirreq() and
+ * geoip_get_client_history_bridge(). */
+static char *
+geoip_get_client_history(time_t now, geoip_client_action_t action,
+                         int min_observation_time, unsigned granularity)
 {
   char *result = NULL;
-  int min_observation_time = GEOIP_MIN_OBSERVATION_TIME;
-#ifdef ENABLE_DIRREQ_STATS
-  min_observation_time = DIR_RECORD_USAGE_MIN_OBSERVATION_TIME;
-#endif
   if (!geoip_is_loaded())
     return NULL;
   if (client_history_starts < (now - min_observation_time)) {
@@ -841,10 +821,6 @@ geoip_get_client_history(time_t now, geoip_client_action_t action)
     clientmap_entry_t **ent;
     unsigned *counts = tor_malloc_zero(sizeof(unsigned)*n_countries);
     unsigned total = 0;
-    unsigned granularity = IP_GRANULARITY;
-#ifdef ENABLE_DIRREQ_STATS
-    granularity = DIR_RECORD_USAGE_GRANULARITY;
-#endif
     HT_FOREACH(ent, clientmap, &client_history) {
       int country;
       if ((*ent)->action != (int)action)
@@ -900,6 +876,34 @@ geoip_get_client_history(time_t now, geoip_client_action_t action)
   return result;
 }
 
+/** Return a newly allocated comma-separated string containing entries for
+ * all the countries from which we've seen enough clients connect as a
+ * directory. The entry format is cc=num where num is the number of IPs
+ * we've seen connecting from that country, and cc is a lowercased country
+ * code. Returns NULL if we don't want to export geoip data yet. */
+char *
+geoip_get_client_history_dirreq(time_t now,
+                                geoip_client_action_t action)
+{
+  return geoip_get_client_history(now, action,
+                                  DIR_RECORD_USAGE_MIN_OBSERVATION_TIME,
+                                  DIR_RECORD_USAGE_GRANULARITY);
+}
+
+/** Return a newly allocated comma-separated string containing entries for
+ * all the countries from which we've seen enough clients connect as a
+ * bridge. The entry format is cc=num where num is the number of IPs
+ * we've seen connecting from that country, and cc is a lowercased country
+ * code. Returns NULL if we don't want to export geoip data yet. */
+char *
+geoip_get_client_history_bridge(time_t now,
+                                geoip_client_action_t action)
+{
+  return geoip_get_client_history(now, action,
+                                  GEOIP_MIN_OBSERVATION_TIME,
+                                  IP_GRANULARITY);
+}
+
 /** Return a newly allocated string holding the per-country request history
  * for <b>action</b> in a format suitable for an extra-info document, or NULL
  * on failure. */
@@ -910,10 +914,6 @@ geoip_get_request_history(time_t now, geoip_client_action_t action)
   char *result;
   unsigned granularity = IP_GRANULARITY;
   int min_observation_time = GEOIP_MIN_OBSERVATION_TIME;
-#ifdef ENABLE_DIRREQ_STATS
-  granularity = DIR_RECORD_USAGE_GRANULARITY;
-  min_observation_time = DIR_RECORD_USAGE_MIN_OBSERVATION_TIME;
-#endif
 
   if (client_history_starts >= (now - min_observation_time))
     return NULL;
@@ -955,16 +955,23 @@ geoip_get_request_history(time_t now, geoip_client_action_t action)
   return result;
 }
 
-/** Store all our geoip statistics into $DATADIR/dirreq-stats. */
-static void
-dump_geoip_stats(void)
+/** Start time of directory request stats. */
+static time_t start_of_dirreq_stats_interval;
+
+/** Initialize directory request stats. */
+void
+geoip_dirreq_stats_init(time_t now)
+{
+  start_of_dirreq_stats_interval = now;
+}
+
+/** Write dirreq statistics to $DATADIR/stats/dirreq-stats. */
+void
+geoip_dirreq_stats_write(time_t now)
 {
-#ifdef ENABLE_DIRREQ_STATS
-  time_t now = time(NULL);
-  time_t request_start;
-  char *filename = get_datadir_fname("dirreq-stats");
+  char *statsdir = NULL, *filename = NULL;
   char *data_v2 = NULL, *data_v3 = NULL;
-  char since[ISO_TIME_LEN+1], written[ISO_TIME_LEN+1];
+  char written[ISO_TIME_LEN+1];
   open_file_t *open_file = NULL;
   double v2_share = 0.0, v3_share = 0.0;
   FILE *out;
@@ -973,28 +980,33 @@ dump_geoip_stats(void)
   if (!get_options()->DirReqStatistics)
     goto done;
 
-  data_v2 = geoip_get_client_history(now, GEOIP_CLIENT_NETWORKSTATUS_V2);
-  data_v3 = geoip_get_client_history(now, GEOIP_CLIENT_NETWORKSTATUS);
-  format_iso_time(since, geoip_get_history_start());
+  /* Discard all items in the client history that are too old. */
+  geoip_remove_old_clients(start_of_dirreq_stats_interval);
+
+  statsdir = get_datadir_fname("stats");
+  if (check_private_dir(statsdir, CPD_CREATE) < 0)
+    goto done;
+  filename = get_datadir_fname("stats"PATH_SEPARATOR"dirreq-stats");
+  data_v2 = geoip_get_client_history_dirreq(now,
+                GEOIP_CLIENT_NETWORKSTATUS_V2);
+  data_v3 = geoip_get_client_history_dirreq(now,
+                GEOIP_CLIENT_NETWORKSTATUS);
   format_iso_time(written, now);
   out = start_writing_to_stdio_file(filename, OPEN_FLAGS_APPEND,
                                     0600, &open_file);
   if (!out)
     goto done;
-  if (fprintf(out, "written %s\nstarted-at %s\nns-ips %s\nns-v2-ips %s\n",
-              written, since,
+  if (fprintf(out, "dirreq-stats-end %s (%d s)\ndirreq-v3-ips %s\n"
+              "dirreq-v2-ips %s\n", written,
+              (unsigned) (now - start_of_dirreq_stats_interval),
               data_v3 ? data_v3 : "", data_v2 ? data_v2 : "") < 0)
     goto done;
   tor_free(data_v2);
   tor_free(data_v3);
 
-  request_start = current_request_period_starts -
-    (n_old_request_periods * REQUEST_HIST_PERIOD);
-  format_iso_time(since, request_start);
   data_v2 = geoip_get_request_history(now, GEOIP_CLIENT_NETWORKSTATUS_V2);
   data_v3 = geoip_get_request_history(now, GEOIP_CLIENT_NETWORKSTATUS);
-  if (fprintf(out, "requests-start %s\nn-ns-reqs %s\nn-v2-ns-reqs %s\n",
-              since,
+  if (fprintf(out, "dirreq-v3-reqs %s\ndirreq-v2-reqs %s\n",
               data_v3 ? data_v3 : "", data_v2 ? data_v2 : "") < 0)
     goto done;
 #define RESPONSE_GRANULARITY 8
@@ -1005,7 +1017,7 @@ dump_geoip_stats(void)
                                ns_v3_responses[i], RESPONSE_GRANULARITY);
   }
 #undef RESPONSE_GRANULARITY
-  if (fprintf(out, "n-ns-resp ok=%u,not-enough-sigs=%u,unavailable=%u,"
+  if (fprintf(out, "dirreq-v3-resp ok=%u,not-enough-sigs=%u,unavailable=%u,"
                    "not-found=%u,not-modified=%u,busy=%u\n",
                    ns_v3_responses[GEOIP_SUCCESS],
                    ns_v3_responses[GEOIP_REJECT_NOT_ENOUGH_SIGS],
@@ -1014,7 +1026,7 @@ dump_geoip_stats(void)
                    ns_v3_responses[GEOIP_REJECT_NOT_MODIFIED],
                    ns_v3_responses[GEOIP_REJECT_BUSY]) < 0)
     goto done;
-  if (fprintf(out, "n-v2-ns-resp ok=%u,unavailable=%u,"
+  if (fprintf(out, "dirreq-v2-resp ok=%u,unavailable=%u,"
                    "not-found=%u,not-modified=%u,busy=%u\n",
                    ns_v2_responses[GEOIP_SUCCESS],
                    ns_v2_responses[GEOIP_REJECT_UNAVAILABLE],
@@ -1025,9 +1037,9 @@ dump_geoip_stats(void)
   memset(ns_v2_responses, 0, sizeof(ns_v2_responses));
   memset(ns_v3_responses, 0, sizeof(ns_v3_responses));
   if (!geoip_get_mean_shares(now, &v2_share, &v3_share)) {
-    if (fprintf(out, "v2-ns-share %0.2lf%%\n", v2_share*100) < 0)
+    if (fprintf(out, "dirreq-v2-share %0.2lf%%\n", v2_share*100) < 0)
       goto done;
-    if (fprintf(out, "v3-ns-share %0.2lf%%\n", v3_share*100) < 0)
+    if (fprintf(out, "dirreq-v3-share %0.2lf%%\n", v3_share*100) < 0)
       goto done;
   }
 
@@ -1035,7 +1047,7 @@ dump_geoip_stats(void)
                                        DIRREQ_DIRECT);
   data_v3 = geoip_get_dirreq_history(GEOIP_CLIENT_NETWORKSTATUS,
                                        DIRREQ_DIRECT);
-  if (fprintf(out, "ns-direct-dl %s\nns-v2-direct-dl %s\n",
+  if (fprintf(out, "dirreq-v3-direct-dl %s\ndirreq-v2-direct-dl %s\n",
               data_v3 ? data_v3 : "", data_v2 ? data_v2 : "") < 0)
     goto done;
   tor_free(data_v2);
@@ -1044,53 +1056,78 @@ dump_geoip_stats(void)
                                        DIRREQ_TUNNELED);
   data_v3 = geoip_get_dirreq_history(GEOIP_CLIENT_NETWORKSTATUS,
                                        DIRREQ_TUNNELED);
-  if (fprintf(out, "ns-tunneled-dl %s\nns-v2-tunneled-dl %s\n",
+  if (fprintf(out, "dirreq-v3-tunneled-dl %s\ndirreq-v2-tunneled-dl %s\n",
               data_v3 ? data_v3 : "", data_v2 ? data_v2 : "") < 0)
     goto done;
 
   finish_writing_to_file(open_file);
   open_file = NULL;
+
+  /* Rotate request period */
+  rotate_request_period();
+
+  start_of_dirreq_stats_interval = now;
+
  done:
   if (open_file)
     abort_writing_to_file(open_file);
   tor_free(filename);
+  tor_free(statsdir);
   tor_free(data_v2);
   tor_free(data_v3);
-#endif
 }
 
-/** Store all our geoip statistics as entry guards into
- * $DATADIR/entry-stats. */
-static void
-dump_entry_stats(void)
+/** Start time of entry stats. */
+static time_t start_of_entry_stats_interval;
+
+/** Initialize entry stats. */
+void
+geoip_entry_stats_init(time_t now)
+{
+  start_of_entry_stats_interval = now;
+}
+
+/** Write entry statistics to $DATADIR/stats/entry-stats. */
+void
+geoip_entry_stats_write(time_t now)
 {
-#ifdef ENABLE_ENTRY_STATS
-  time_t now = time(NULL);
-  char *filename = get_datadir_fname("entry-stats");
+  char *statsdir = NULL, *filename = NULL;
   char *data = NULL;
-  char since[ISO_TIME_LEN+1], written[ISO_TIME_LEN+1];
+  char written[ISO_TIME_LEN+1];
   open_file_t *open_file = NULL;
   FILE *out;
 
-  data = geoip_get_client_history(now, GEOIP_CLIENT_CONNECT);
-  format_iso_time(since, geoip_get_history_start());
+  if (!get_options()->EntryStatistics)
+    goto done;
+
+  /* Discard all items in the client history that are too old. */
+  geoip_remove_old_clients(start_of_entry_stats_interval);
+
+  statsdir = get_datadir_fname("stats");
+  if (check_private_dir(statsdir, CPD_CREATE) < 0)
+    goto done;
+  filename = get_datadir_fname("stats"PATH_SEPARATOR"entry-stats");
+  data = geoip_get_client_history_dirreq(now, GEOIP_CLIENT_CONNECT);
   format_iso_time(written, now);
   out = start_writing_to_stdio_file(filename, OPEN_FLAGS_APPEND,
                                     0600, &open_file);
   if (!out)
     goto done;
-  if (fprintf(out, "written %s\nstarted-at %s\nips %s\n",
-              written, since, data ? data : "") < 0)
+  if (fprintf(out, "entry-stats-end %s (%u s)\nentry-ips %s\n",
+              written, (unsigned) (now - start_of_entry_stats_interval),
+              data ? data : "") < 0)
     goto done;
 
+  start_of_entry_stats_interval = now;
+
   finish_writing_to_file(open_file);
   open_file = NULL;
  done:
   if (open_file)
     abort_writing_to_file(open_file);
   tor_free(filename);
+  tor_free(statsdir);
   tor_free(data);
-#endif
 }
 
 /** Helper used to implement GETINFO ip-to-country/... controller command. */