1 files changed, 354 insertions, 20 deletions

diff --git a/src/test/test_stats.c b/src/test/test_stats.c
index 291473ebc9..d45afc7b15 100644
--- a/src/test/test_stats.c
+++ b/src/test/test_stats.c
@@ -30,6 +30,8 @@
 #define CIRCUITLIST_PRIVATE
 #define MAINLOOP_PRIVATE
 #define STATEFILE_PRIVATE
+#define BWHIST_PRIVATE
+#define ROUTER_PRIVATE
 
 #include "core/or/or.h"
 #include "lib/err/backtrace.h"
@@ -39,8 +41,12 @@
 #include "test/test.h"
 #include "core/mainloop/mainloop.h"
 #include "lib/memarea/memarea.h"
+#include "feature/stats/connstats.h"
 #include "feature/stats/rephist.h"
 #include "app/config/statefile.h"
+#include "feature/stats/bwhist.h"
+#include "feature/stats/bw_array_st.h"
+#include "feature/relay/router.h"
 
 /** Run unit tests for some stats code. */
 static void
@@ -111,37 +117,41 @@ test_stats(void *arg)
 
   /* Continue with testing connection statistics; we shouldn't collect
    * conn stats without initializing them. */
-  rep_hist_note_or_conn_bytes(1, 20, 400, now);
-  s = rep_hist_format_conn_stats(now + 86400);
+  conn_stats_note_or_conn_bytes(1, 20, 400, now, false);
+  s = conn_stats_format(now + 86400);
   tt_ptr_op(s, OP_EQ, NULL);
 
   /* Initialize stats, note bytes, and generate history string. */
-  rep_hist_conn_stats_init(now);
-  rep_hist_note_or_conn_bytes(1, 30000, 400000, now);
-  rep_hist_note_or_conn_bytes(1, 30000, 400000, now + 5);
-  rep_hist_note_or_conn_bytes(2, 400000, 30000, now + 10);
-  rep_hist_note_or_conn_bytes(2, 400000, 30000, now + 15);
-  s = rep_hist_format_conn_stats(now + 86400);
-  tt_str_op("conn-bi-direct 2010-08-12 13:27:30 (86400 s) 0,0,1,0\n",OP_EQ, s);
+  conn_stats_init(now);
+  conn_stats_note_or_conn_bytes(1, 30000, 400000, now, false);
+  conn_stats_note_or_conn_bytes(1, 30000, 400000, now + 5, false);
+  conn_stats_note_or_conn_bytes(2, 400000, 30000, now + 10, true);
+  conn_stats_note_or_conn_bytes(2, 400000, 30000, now + 15, true);
+  s = conn_stats_format(now + 86400);
+  tt_str_op("conn-bi-direct 2010-08-12 13:27:30 (86400 s) 0,0,1,0\n"
+            "ipv6-conn-bi-direct 2010-08-12 13:27:30 (86400 s) 0,0,0,0\n",
+            OP_EQ, s);
   tor_free(s);
 
   /* Stop collecting stats, add some bytes, and ensure we don't generate
    * a history string. */
-  rep_hist_conn_stats_term();
-  rep_hist_note_or_conn_bytes(2, 400000, 30000, now + 15);
-  s = rep_hist_format_conn_stats(now + 86400);
+  conn_stats_terminate();
+  conn_stats_note_or_conn_bytes(2, 400000, 30000, now + 15, true);
+  s = conn_stats_format(now + 86400);
   tt_ptr_op(s, OP_EQ, NULL);
 
   /* Re-start stats, add some bytes, reset stats, and see what history we
    * get when observing no bytes at all. */
-  rep_hist_conn_stats_init(now);
-  rep_hist_note_or_conn_bytes(1, 30000, 400000, now);
-  rep_hist_note_or_conn_bytes(1, 30000, 400000, now + 5);
-  rep_hist_note_or_conn_bytes(2, 400000, 30000, now + 10);
-  rep_hist_note_or_conn_bytes(2, 400000, 30000, now + 15);
-  rep_hist_reset_conn_stats(now);
-  s = rep_hist_format_conn_stats(now + 86400);
-  tt_str_op("conn-bi-direct 2010-08-12 13:27:30 (86400 s) 0,0,0,0\n",OP_EQ, s);
+  conn_stats_init(now);
+  conn_stats_note_or_conn_bytes(1, 30000, 400000, now, false);
+  conn_stats_note_or_conn_bytes(1, 30000, 400000, now + 5, false);
+  conn_stats_note_or_conn_bytes(2, 400000, 30000, now + 10, true);
+  conn_stats_note_or_conn_bytes(2, 400000, 30000, now + 15, true);
+  conn_stats_reset(now);
+  s = conn_stats_format(now + 86400);
+  tt_str_op("conn-bi-direct 2010-08-12 13:27:30 (86400 s) 0,0,0,0\n"
+            "ipv6-conn-bi-direct 2010-08-12 13:27:30 (86400 s) 0,0,0,0\n",
+            OP_EQ, s);
   tor_free(s);
 
   /* Continue with testing buffer statistics; we shouldn't collect buffer
@@ -245,6 +255,324 @@ test_rephist_mtbf(void *arg)
   tor_free(ddir_fname);
 }
 
+static void
+test_commit_max(void *arg)
+{
+  (void) arg;
+  bw_array_t *b = bw_array_new();
+  time_t now = b->cur_obs_time;
+
+  commit_max(b);
+  tt_int_op(b->next_period, OP_EQ, now + 2*86400);
+
+  b->total_in_period = 100;
+  b->max_total = 10;
+  commit_max(b);
+  tor_assert(b->total_in_period == 0);
+  tor_assert(b->max_total == 0);
+  tt_int_op(b->totals[1], OP_EQ, 100);
+  tt_int_op(b->maxima[1], OP_EQ, 10);
+  tt_int_op(b->next_period, OP_EQ, now + 3*86400);
+
+  commit_max(b);
+  tt_int_op(b->next_period, OP_EQ, now + 4*86400);
+
+  commit_max(b);
+  tt_int_op(b->next_period, OP_EQ, now + 5*86400);
+
+  b->total_in_period = 100;
+  b->max_total = 10;
+  commit_max(b);
+  tor_assert(!b->next_max_idx);
+  tt_int_op(b->cur_obs_idx, OP_EQ, 0);
+  tt_int_op(b->totals[4], OP_EQ, 100);
+  tt_int_op(b->maxima[4], OP_EQ, 10);
+  tt_int_op(b->next_period, OP_EQ, now + 6*86400);
+ done:
+  bw_array_free(b);
+}
+
+#define test_obs(b, idx, time, tot, max) STMT_BEGIN \
+    tt_int_op(b->cur_obs_idx, OP_EQ, idx); \
+    tt_int_op(b->cur_obs_time, OP_EQ, time); \
+    tt_int_op(b->total_obs, OP_EQ, tot); \
+    tt_int_op(b->max_total, OP_EQ, max); \
+  STMT_END;
+
+static void
+test_advance_obs(void *arg)
+{
+  (void) arg;
+  int iter, tot = 0;
+  bw_array_t *b = bw_array_new();
+  time_t now = b->cur_obs_time;
+
+  for (iter = 0; iter < 10; ++iter) {
+    b->obs[b->cur_obs_idx] += 10;
+    tot += 10;
+    advance_obs(b);
+    if (iter == 9) {
+      /* The current value under cur_obs_idx was zeroed in last iterN. */
+      test_obs(b, 0, now+iter+1, tot - 10, tot);
+      break;
+    }
+    test_obs(b, iter+1, now+iter+1, tot, tot);
+  }
+
+  b->total_in_period = 100;
+  b->cur_obs_time = now + NUM_SECS_BW_SUM_INTERVAL - 1;
+  advance_obs(b);
+  test_obs(b, 1, now+NUM_SECS_BW_SUM_INTERVAL, 80, 0);
+  tt_int_op(b->maxima[0], OP_EQ, 100);
+  tt_int_op(b->totals[0], OP_EQ, 100);
+  tt_int_op(b->num_maxes_set, OP_EQ, 1);
+ done:
+  bw_array_free(b);
+}
+
+#define test_add_obs_(b, now, checknow, bw, tot) STMT_BEGIN \
+    tot += bw; \
+    add_obs(b, now, bw); \
+    tt_int_op(b->cur_obs_time, OP_EQ, checknow); \
+    tt_int_op(b->obs[b->cur_obs_idx], OP_EQ, bw); \
+    tt_int_op(b->total_in_period, OP_EQ, tot); \
+  STMT_END;
+
+static void
+test_add_obs(void *arg)
+{
+  (void) arg;
+  bw_array_t *b = bw_array_new();
+  time_t now = b->cur_obs_time;
+  uint64_t bw = 0, tot = 0;
+  /* Requests for the past should not be entertained. */
+  test_add_obs_(b, now-1, now, bw, tot);
+  /* Test the expected functionalities for random values. */
+  now += 53;
+  bw = 97;
+  test_add_obs_(b, now, now, bw, tot);
+
+  now += 60*60;
+  bw = 90;
+  test_add_obs_(b, now, now, bw, tot);
+
+  now += 24*60*60;
+  bw = 100;
+  tot = 0;
+  test_add_obs_(b, now, now, bw, tot);
+ done:
+  bw_array_free(b);
+}
+
+static or_options_t mock_options;
+
+static const or_options_t *
+mock_get_options(void)
+{
+  return &mock_options;
+}
+
+#define MAX_HIST_VALUE_LEN 21*NUM_TOTALS
+
+#define set_test_case(b, max, idx, a1, a2, a3, a4, a5) STMT_BEGIN \
+    b->num_maxes_set = max; \
+    b->next_max_idx = idx; \
+    b->totals[0] = a1; \
+    b->totals[1] = a2; \
+    b->totals[2] = a3; \
+    b->totals[3] = a4; \
+    b->totals[4] = a5; \
+  STMT_END;
+
+#define test_fill_bw(b, buf, rv, str, checkrv) STMT_BEGIN \
+    buf = tor_malloc_zero(MAX_HIST_VALUE_LEN); \
+    rv = bwhist_fill_bandwidth_history(buf, MAX_HIST_VALUE_LEN, b); \
+    tt_str_op(buf, OP_EQ, str); \
+    tt_int_op(rv, OP_EQ, checkrv); \
+    tor_free(buf); \
+  STMT_END;
+
+static void
+test_fill_bandwidth_history(void *arg)
+{
+  (void) arg;
+  bw_array_t *b = bw_array_new();
+  char *buf;
+  size_t rv;
+  /* Remember bandwidth is rounded down to the nearest 1K. */
+  /* Day 1. */
+  set_test_case(b, 0, 0, 0, 0, 0, 0, 0);
+  buf = tor_malloc_zero(MAX_HIST_VALUE_LEN);
+  rv = bwhist_fill_bandwidth_history(buf, MAX_HIST_VALUE_LEN, b);
+  tt_int_op(rv, OP_EQ, 0);
+  tor_free(buf);
+  /* Day 2. */
+  set_test_case(b, 1, 1, 1000, 0, 0, 0, 0);
+  test_fill_bw(b, buf, rv, "0", 1);
+  /* Day 3. */
+  set_test_case(b, 2, 2, 1000, 1500, 0, 0, 0);
+  test_fill_bw(b, buf, rv, "0,1024", 6);
+  /* Day 4. */
+  set_test_case(b, 3, 3, 1000, 1500, 3500, 0, 0);
+  test_fill_bw(b, buf, rv, "0,1024,3072", 11);
+  /* Day 5. */
+  set_test_case(b, 4, 4, 1000, 1500, 3500, 8000, 0);
+  test_fill_bw(b, buf, rv, "0,1024,3072,7168", 16);
+  /* Day 6. */
+  set_test_case(b, 5, 0, 1000, 1500, 3500, 8000, 6000);
+  test_fill_bw(b, buf, rv, "0,1024,3072,7168,5120", 21);
+  /* Day 7. */
+  /* Remember oldest entry first. */
+  set_test_case(b, 5, 1, 10000, 1500, 3500, 8000, 6000);
+  test_fill_bw(b, buf, rv, "1024,3072,7168,5120,9216", 24);
+  /* Mocking get_options to manipulate RelayBandwidthRate. */
+  MOCK(get_options, mock_get_options);
+  /* Limits bandwidth to 1 KBps. */
+  /* Cutoff is set to 88473600. */
+  mock_options.RelayBandwidthRate = 1024;
+  set_test_case(b, 5, 2, 88573600, 88473600, 10000, 8000, 6000);
+  test_fill_bw(b, buf, rv, "9216,7168,5120,88473600,88473600", 32);
+ done:
+  UNMOCK(get_options);
+  bw_array_free(b);
+}
+
+#define set_test_bw_lines(r, w, dr, dw, when) STMT_BEGIN \
+    bwhist_note_bytes_read(r, when, false); \
+    bwhist_note_bytes_written(w, when, false); \
+    bwhist_note_dir_bytes_read(dr, when); \
+    bwhist_note_dir_bytes_written(dw, when); \
+  STMT_END;
+
+#define test_get_bw_lines(str, checkstr) STMT_BEGIN \
+    str = bwhist_get_bandwidth_lines(); \
+    tt_str_op(str, OP_EQ, checkstr); \
+    tor_free(str); \
+  STMT_END;
+
+static void
+test_get_bandwidth_lines(void *arg)
+{
+  (void) arg;
+  char *str = NULL, *checkstr = NULL;
+  char t[ISO_TIME_LEN+1];
+  int len = (67+MAX_HIST_VALUE_LEN)*4;
+  checkstr = tor_malloc_zero(len);
+  time_t now = time(NULL);
+  bwhist_init();
+
+  /* Day 1. */
+  now += 86400;
+  set_test_bw_lines(5000, 5500, 3000, 3500, now - 6*60*60);
+  /* Day 2. */
+  now += 86400;
+  set_test_bw_lines(50000, 55000, 30000, 35000, now - 6*60*60);
+  /* Day 3. */
+  now += 86400;
+  set_test_bw_lines(25000, 27500, 15000, 17500, now - 6*60*60);
+  /* Day 4. */
+  now += 86400;
+  set_test_bw_lines(90000, 76000, 60000, 45000, now - 6*60*60);
+  /* Day 5. */
+  now += 86400;
+  set_test_bw_lines(500, 55000, 30000, 35000, now - 6*60*60);
+  set_test_bw_lines(0, 0, 0, 0, now);
+  format_iso_time(t, now);
+  tor_snprintf(checkstr, len, "write-history %s (86400 s) "
+                    "5120,54272,26624,75776,54272\n"
+                    "read-history %s (86400 s) "
+                    "4096,49152,24576,89088,0\n"
+                    "dirreq-write-history %s (86400 s) "
+                    "3072,34816,17408,44032,34816\n"
+                    "dirreq-read-history %s (86400 s) "
+                    "2048,29696,14336,59392,29696\n",
+                    t, t, t, t);
+  test_get_bw_lines(str, checkstr);
+
+ done:
+  tor_free(str);
+  tor_free(checkstr);
+  bwhist_free_all();
+}
+
+static void
+test_load_stats_file(void *arg)
+{
+  int ret;
+  char *content = NULL, *read_file_content = NULL, *fname = NULL;
+
+  (void) arg;
+
+  /* Load conn-stats. */
+  fname = get_datadir_fname("conn-stats");
+  tt_assert(fname);
+  read_file_content = tor_strdup(
+    "conn-bi-direct 2020-12-13 15:48:53 (86400 s) 12,34,56,78\n"
+    "ipv6-conn-bi-direct 2020-12-14 15:48:53 (86400 s) 21,43,65,87\n");
+  write_str_to_file(fname, read_file_content, 0);
+  ret = load_stats_file("conn-stats", "conn-bi-direct", 1607874000, &content);
+  tt_int_op(ret, OP_EQ, 1);
+  tt_str_op(read_file_content, OP_EQ, content);
+
+  /* Load hidserv-stats. */
+  tor_free(fname);
+  fname = get_datadir_fname("hidserv-stats");
+  tt_assert(fname);
+  tor_free(read_file_content);
+  read_file_content = tor_strdup(
+    "hidserv-stats-end 2020-12-13 15:48:53 (86400 s)\n"
+    "hidserv-rend-relayed-cells 48754891 delta_f=2048 epsilon=0.30 "
+      "bin_size=1024\n"
+    "hidserv-dir-onions-seen 53 delta_f=8 epsilon=0.30 bin_size=8\n");
+  write_str_to_file(fname, read_file_content, 0);
+  tor_free(content);
+  ret = load_stats_file("hidserv-stats", "hidserv-stats-end", 1607874000,
+                        &content);
+  tt_int_op(ret, OP_EQ, 1);
+  tt_str_op(read_file_content, OP_EQ, content);
+
+  /* Load dirreq-stats. */
+  tor_free(fname);
+  fname = get_datadir_fname("dirreq-stats");
+  tt_assert(fname);
+  tor_free(read_file_content);
+  read_file_content = tor_strdup(
+    "dirreq-stats-end 2020-12-13 15:48:53 (86400 s)\n"
+    "dirreq-v3-ips ru=1728,us=1144,de=696,ir=432,gb=328,fr=304,in=296,ua=232\n"
+    "dirreq-v3-reqs ru=3616,us=3576,de=1896,fr=800,gb=632,ir=616\n"
+    "dirreq-v3-resp ok=18472,not-enough-sigs=0,unavailable=0,not-found=0,"
+      "not-modified=3136,busy=0\n"
+    "dirreq-v3-direct-dl complete=0,timeout=0,running=0\n"
+    "dirreq-v3-tunneled-dl complete=18124,timeout=348,running=4,min=257,"
+      "d1=133653,d2=221050,q1=261242,d3=300622,d4=399758,md=539051,d6=721322,"
+      "d7=959866,q3=1103363,d8=1302035,d9=2046125,max=113404000\n");
+  write_str_to_file(fname, read_file_content, 0);
+  tor_free(content);
+  ret = load_stats_file("dirreq-stats", "dirreq-stats-end", 1607874000,
+                        &content);
+  tt_int_op(ret, OP_EQ, 1);
+  tt_str_op(read_file_content, OP_EQ, content);
+
+  /* Attempt to load future-stats file not starting with timestamp tag. */
+  tor_free(fname);
+  fname = get_datadir_fname("future-stats");
+  tt_assert(fname);
+  tor_free(read_file_content);
+  read_file_content = tor_strdup(
+    "future-stuff-at-file-start\n"
+    "future-stats 2020-12-13 15:48:53 (86400 s)\n");
+  write_str_to_file(fname, read_file_content, 0);
+  tor_free(content);
+  ret = load_stats_file("future-stats", "future-stats", 1607874000, &content);
+  tt_int_op(ret, OP_EQ, 1);
+  tt_str_op(read_file_content, OP_EQ, content);
+
+ done:
+  tor_free(fname);
+  tor_free(read_file_content);
+  tor_free(content);
+}
+
 #define ENT(name)                                                       \
   { #name, test_ ## name , 0, NULL, NULL }
 #define FORK(name)                                                      \
@@ -253,6 +581,12 @@ test_rephist_mtbf(void *arg)
 struct testcase_t stats_tests[] = {
   FORK(stats),
   ENT(rephist_mtbf),
+  FORK(commit_max),
+  FORK(advance_obs),
+  FORK(add_obs),
+  FORK(fill_bandwidth_history),
+  FORK(get_bandwidth_lines),
+  FORK(load_stats_file),
 
   END_OF_TESTCASES
 };