Introduce v3_stats_t structure and some of its methods.

6 files changed, 117 insertions, 1 deletions

diff --git a/src/app/main/main.c b/src/app/main/main.c
index ff530c0ad0..c2ced30851 100644
--- a/src/app/main/main.c
+++ b/src/app/main/main.c
@@ -1070,6 +1070,7 @@ sandbox_init_filter(void)
     OPEN_DATADIR2_SUFFIX("stats", "buffer-stats", ".tmp");
     OPEN_DATADIR2_SUFFIX("stats", "conn-stats", ".tmp");
     OPEN_DATADIR2_SUFFIX("stats", "hidserv-stats", ".tmp");
+    OPEN_DATADIR2_SUFFIX("stats", "hidserv-v3-stats", ".tmp");
 
     OPEN_DATADIR("approved-routers");
     OPEN_DATADIR_SUFFIX("fingerprint", ".tmp");
@@ -1095,6 +1096,7 @@ sandbox_init_filter(void)
     RENAME_SUFFIX2("stats", "buffer-stats", ".tmp");
     RENAME_SUFFIX2("stats", "conn-stats", ".tmp");
     RENAME_SUFFIX2("stats", "hidserv-stats", ".tmp");
+    RENAME_SUFFIX2("stats", "hidserv-v3-stats", ".tmp");
     RENAME_SUFFIX("hashed-fingerprint", ".tmp");
     RENAME_SUFFIX("router-stability", ".tmp");
 
diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c
index 4b4ac5c4a7..25555a3f22 100644
--- a/src/core/mainloop/mainloop.c
+++ b/src/core/mainloop/mainloop.c
@@ -1940,6 +1940,10 @@ write_stats_file_callback(time_t now, const or_options_t *options)
     time_t next_write = rep_hist_hs_v2_stats_write(now);
     if (next_write && next_write < next_time_to_write_stats_files)
       next_time_to_write_stats_files = next_write;
+
+    next_write = rep_hist_hs_v3_stats_write(now);
+    if (next_write && next_write < next_time_to_write_stats_files)
+      next_time_to_write_stats_files = next_write;
   }
   if (options->ExitPortStatistics) {
     time_t next_write = rep_hist_exit_stats_write(now);
diff --git a/src/feature/hs/hs_cache.c b/src/feature/hs/hs_cache.c
index 44cd2505fd..0688d7765d 100644
--- a/src/feature/hs/hs_cache.c
+++ b/src/feature/hs/hs_cache.c
@@ -19,6 +19,7 @@
 #include "feature/hs/hs_descriptor.h"
 #include "feature/nodelist/networkstatus.h"
 #include "feature/rend/rendcache.h"
+#include "feature/stats/rephist.h"
 
 #include "feature/hs/hs_cache.h"
 
diff --git a/src/feature/relay/router.c b/src/feature/relay/router.c
index 5ca21964b6..ea631e18fe 100644
--- a/src/feature/relay/router.c
+++ b/src/feature/relay/router.c
@@ -3288,6 +3288,11 @@ extrainfo_dump_to_string_stats_helper(smartlist_t *chunks,
                         "hidserv-stats-end", now, &contents) > 0) {
       smartlist_add(chunks, contents);
     }
+    if (options->HiddenServiceStatistics &&
+        load_stats_file("stats"PATH_SEPARATOR"hidserv-v3-stats",
+                        "hidserv-v3-stats-end", now, &contents) > 0) {
+      smartlist_add(chunks, contents);
+    }
     if (options->EntryStatistics &&
         load_stats_file("stats"PATH_SEPARATOR"entry-stats",
                         "entry-stats-end", now, &contents) > 0) {
diff --git a/src/feature/stats/rephist.c b/src/feature/stats/rephist.c
index bde65ea9d9..daf9db074c 100644
--- a/src/feature/stats/rephist.c
+++ b/src/feature/stats/rephist.c
@@ -1818,6 +1818,96 @@ rep_hist_hsdir_stored_maybe_new_v2_onion(const crypto_pk_t *pubkey)
   }
 }
 
+/*** HSv3 stats ******/
+
+/** Start of the current hidden service stats interval or 0 if we're not
+ *  collecting hidden service statistics.
+ *
+ *  This is particularly important for v3 statistics since this variable
+ *  controls the start time of initial v3 stats collection. It's initialized by
+ *  rep_hist_hs_stats_init() to the next time period start (i.e. 12:00UTC), and
+ *  should_collect_v3_stats() ensures that functions that collect v3 stats do
+ *  not do so sooner than that.
+ *
+ *  Collecting stats from 12:00UTC to 12:00UTC is extremely important for v3
+ *  stats because rep_hist_hsdir_stored_maybe_new_v3_onion() uses the blinded
+ *  key of each onion service as its double-counting index. Onion services
+ *  rotate their descriptor at around 00:00UTC which means that their blinded
+ *  key also changes around that time. However the precise time that onion
+ *  services rotate their descriptors is actually when they fetch a new
+ *  00:00UTC consensus and that happens at a random time (e.g. it can even
+ *  happen at 02:00UTC). This means that if we started keeping v3 stats at
+ *  around 00:00UTC we wouldn't be able to tell when onion services change
+ *  their blinded key and hence we would double count an unpredictable amount
+ *  of them (for example, if an onion service fetches the 00:00UTC consensus at
+ *  01:00UTC it would upload to its old HSDir at 00:45UTC, and then to a
+ *  different HSDir at 01:50UTC).
+ *
+ *  For this reason, we start collecting statistics at 12:00UTC. This way we
+ *  know that by the time we stop collecting statistics for that time period 24
+ *  hours later, all the onion services have switched to their new blinded
+ *  key. This way we can predict much better how much double counting has been
+ *  performed.
+ */
+static time_t start_of_hs_v3_stats_interval;
+
+/** Our v3 statistics structure singleton. */
+static hs_v3_stats_t *hs_v3_stats = NULL;
+
+/** Allocate, initialize and return an hs_v3_stats_t structure. */
+static hs_v3_stats_t *
+hs_v3_stats_new(void)
+{
+  hs_v3_stats_t *new_hs_v3_stats = tor_malloc_zero(sizeof(hs_v3_stats_t));
+  new_hs_v3_stats->v3_onions_seen_this_period = digestmap_new();
+
+  return new_hs_v3_stats;
+}
+
+#define hs_v3_stats_free(val) \
+  FREE_AND_NULL(hs_v3_stats_t, hs_v3_stats_free_, (val))
+
+/** Free an hs_v3_stats_t structure. */
+static void
+hs_v3_stats_free_(hs_v3_stats_t *victim_hs_v3_stats)
+{
+  if (!victim_hs_v3_stats) {
+    return;
+  }
+
+  digestmap_free(victim_hs_v3_stats->v3_onions_seen_this_period, NULL);
+  tor_free(victim_hs_v3_stats);
+}
+
+/** Clear history of hidden service statistics and set the measurement
+ * interval start to <b>now</b>. */
+static void
+rep_hist_reset_hs_v3_stats(time_t now)
+{
+  if (!hs_v3_stats) {
+    hs_v3_stats = hs_v3_stats_new();
+  }
+
+  digestmap_free(hs_v3_stats->v3_onions_seen_this_period, NULL);
+  hs_v3_stats->v3_onions_seen_this_period = digestmap_new();
+
+  hs_v3_stats->rp_v3_relay_cells_seen = 0;
+
+  start_of_hs_v3_stats_interval = now;
+}
+
+/** Return true if it's a good time to collect v3 stats.
+ *
+ *  v3 stats have a strict stats collection period (from 12:00UTC to 12:00UTC
+ *  on the real network) and hence we don't want to collect statistics if it's
+ *  not yet the time to do so.
+ */
+static bool
+should_collect_v3_stats(void)
+{
+  return start_of_hs_v3_stats_interval <= approx_time();
+}
+
 /* The number of cells that are supposed to be hidden from the adversary
  * by adding noise from the Laplace distribution.  This value, divided by
  * EPSILON, is Laplace parameter b. It must be greather than 0. */
@@ -2125,6 +2215,7 @@ void
 rep_hist_free_all(void)
 {
   hs_v2_stats_free(hs_v2_stats);
+  hs_v3_stats_free(hs_v3_stats);
   digestmap_free(history_map, free_or_history);
 
   tor_free(exit_bytes_read);
@@ -2154,3 +2245,10 @@ rep_hist_get_hs_v2_stats(void)
   return hs_v2_stats;
 }
 
+/* only exists for unit tests: get HSv2 stats object */
+const hs_v3_stats_t *
+rep_hist_get_hs_v3_stats(void)
+{
+  return hs_v3_stats;
+}
+#endif /* defined(TOR_UNIT_TESTS) */
diff --git a/src/feature/stats/rephist.h b/src/feature/stats/rephist.h
index d5ad21e228..e6c1509498 100644
--- a/src/feature/stats/rephist.h
+++ b/src/feature/stats/rephist.h
@@ -69,6 +69,9 @@ char *rep_hist_get_hs_v2_stats_string(void);
 void rep_hist_seen_new_rp_cell(void);
 void rep_hist_hsdir_stored_maybe_new_v2_onion(const crypto_pk_t *pubkey);
 
+time_t rep_hist_hs_v3_stats_write(time_t now);
+char *rep_hist_get_hs_v3_stats_string(void);
+
 void rep_hist_free_all(void);
 
 void rep_hist_note_negotiated_link_proto(unsigned link_proto,
@@ -111,7 +114,8 @@ typedef struct hs_v3_stats_t {
   digestmap_t *v3_onions_seen_this_period;
 } hs_v3_stats_t;
 
-STATIC char *rep_hist_format_hs_v2_stats(time_t now, bool is_v3);
+STATIC char *rep_hist_format_hs_v2_stats(time_t now);
+STATIC char *rep_hist_format_hs_v3_stats(time_t now);
 #endif /* defined(REPHIST_PRIVATE) */
 
 /**
@@ -142,6 +146,8 @@ void rep_hist_padding_count_timers(uint64_t num_timers);
 #ifdef TOR_UNIT_TESTS
 typedef struct hs_v2_stats_t hs_v2_stats_t;
 const hs_v2_stats_t *rep_hist_get_hs_v2_stats(void);
+typedef struct hs_v3_stats_t hs_v3_stats_t;
+const hs_v3_stats_t *rep_hist_get_hs_v3_stats(void);
 #endif
 
 #endif /* !defined(TOR_REPHIST_H) */