Record the states of failing OR connections

This code lets us record the state of any outgoing OR connection
that fails before it becomes open, so we can notice if they're all
dying in the same SSL state or the same OR handshake state.

More work is still needed:
  - We need documentation
  - We need to actually call the code that reports the failure when
    we realize that we're having a hard time connecting out or
    making circuits.
  - We need to periodically clear out all this data -- perhaps,
    whenever we build a circuit successfully?
  - We'll eventually want to expose it to controllers, perhaps.

Partial implementation of feature 3116.

5 files changed, 144 insertions, 0 deletions

diff --git a/src/common/tortls.c b/src/common/tortls.c
index 94ca81ba49..8db47a6f21 100644
--- a/src/common/tortls.c
+++ b/src/common/tortls.c
@@ -222,6 +222,37 @@ ssl_state_to_string(int ssl_state)
   return buf;
 }
 
+/** DOCDOC 3116 */
+void
+tor_tls_get_state_description(tor_tls_t *tls, char *buf, size_t sz)
+{
+  const char *ssl_state;
+  const char *tortls_state;
+
+  if (PREDICT_UNLIKELY(!tls || !tls->ssl)) {
+    strlcpy(buf, "(No SSL object)", sz);
+    return;
+  }
+
+  ssl_state = ssl_state_to_string(tls->ssl->state);
+  switch (tls->state) {
+#define CASE(st) case TOR_TLS_ST_##st: tortls_state = #st ; break
+    CASE(HANDSHAKE);
+    CASE(OPEN);
+    CASE(GOTCLOSE);
+    CASE(SENTCLOSE);
+    CASE(CLOSED);
+    CASE(RENEGOTIATE);
+    CASE(BUFFEREVENT);
+#undef CASE
+  default:
+    tortls_state = "unknown";
+    break;
+  }
+
+  tor_snprintf(buf, sz, "%s in %s", ssl_state, tortls_state);
+}
+
 void
 tor_tls_log_one_error(tor_tls_t *tls, unsigned long err,
                   int severity, int domain, const char *doing)
diff --git a/src/common/tortls.h b/src/common/tortls.h
index ecb5bd2fbe..9b8108b42b 100644
--- a/src/common/tortls.h
+++ b/src/common/tortls.h
@@ -48,6 +48,7 @@ typedef struct tor_tls_t tor_tls_t;
 
 #define TOR_TLS_IS_ERROR(rv) ((rv) < TOR_TLS_CLOSE)
 const char *tor_tls_err_to_string(int err);
+void tor_tls_get_state_description(tor_tls_t *tls, char *buf, size_t sz);
 
 void tor_tls_free_all(void);
 int tor_tls_context_init(int is_public_server,
diff --git a/src/or/connection.c b/src/or/connection.c
index e8a17e7641..b0e0f84371 100644
--- a/src/or/connection.c
+++ b/src/or/connection.c
@@ -554,6 +554,9 @@ connection_free_all(void)
   /* Unlink everything from the identity map. */
   connection_or_clear_identity_map();
 
+  /* Clear out our list of broken connections */
+  clear_broken_connection_map();
+
   SMARTLIST_FOREACH(conns, connection_t *, conn, _connection_free(conn));
 
   if (outgoing_addrs) {
diff --git a/src/or/connection_or.c b/src/or/connection_or.c
index fa8624135b..85df78bd6c 100644
--- a/src/or/connection_or.c
+++ b/src/or/connection_or.c
@@ -150,6 +150,111 @@ connection_or_set_identity_digest(or_connection_t *conn, const char *digest)
 #endif
 }
 
+/**************************************************************/
+
+/** DOCDOC */
+static strmap_t *broken_connection_counts;
+
+/** DOCDOC */
+static void
+note_broken_connection(const char *state)
+{
+  void *ptr;
+  intptr_t val;
+  if (!broken_connection_counts)
+    broken_connection_counts = strmap_new();
+
+  ptr = strmap_get(broken_connection_counts, state);
+  val = (intptr_t)ptr;
+  val++;
+  ptr = (void*)val;
+  strmap_set(broken_connection_counts, state, ptr);
+}
+
+/** DOCDOC */
+void
+clear_broken_connection_map(void)
+{
+  if (broken_connection_counts)
+    strmap_free(broken_connection_counts, NULL);
+  broken_connection_counts = NULL;
+}
+
+/** DOCDOC */
+static void
+connection_or_get_state_description(or_connection_t *orconn,
+                                    char *buf, size_t buflen)
+{
+  connection_t *conn = TO_CONN(orconn);
+  const char *conn_state;
+  char tls_state[256];
+
+  tor_assert(conn->type == CONN_TYPE_OR);
+
+  conn_state = conn_state_to_string(conn->type, conn->state);
+  tor_tls_get_state_description(orconn->tls, tls_state, sizeof(tls_state));
+
+  tor_snprintf(buf, buflen, "%s with SSL state %s", conn_state, tls_state);
+}
+
+/** DOCDOC */
+static void
+connection_or_note_state_when_broken(or_connection_t *orconn)
+{
+  char buf[256];
+  connection_or_get_state_description(orconn, buf, sizeof(buf));
+  log_info(LD_HANDSHAKE,"Connection died in state '%s'", buf);
+  note_broken_connection(buf);
+}
+
+/** DOCDOC */
+typedef struct broken_state_count_t {
+  intptr_t count;
+  const char *state;
+} broken_state_count_t;
+
+/** DOCDOC */
+static int
+broken_state_count_compare(const void **a_ptr, const void **b_ptr)
+{
+  const broken_state_count_t *a = *a_ptr, *b = *b_ptr;
+  return a->count - b->count;
+}
+
+/** DOCDOC */
+void
+connection_or_report_broken_states(int severity, int domain)
+{
+  int total = 0;
+  smartlist_t *items;
+
+  if (!broken_connection_counts) {
+    log(severity, domain, "No broken connections reported");
+    return;
+  }
+  items = smartlist_create();
+  STRMAP_FOREACH(broken_connection_counts, state, void *, countptr) {
+    broken_state_count_t *c = tor_malloc(sizeof(broken_state_count_t));
+    total += c->count = (intptr_t)countptr;
+    c->state = state;
+    smartlist_add(items, c);
+  } STRMAP_FOREACH_END;
+
+  smartlist_sort(items, broken_state_count_compare);
+
+  log(severity, domain, "%d connections have failed:", total);
+
+  SMARTLIST_FOREACH_BEGIN(items, const broken_state_count_t *, c) {
+    log(severity, domain,
+        " %d connections died in state %s", (int)c->count, c->state);
+  } SMARTLIST_FOREACH_END(c);
+
+  SMARTLIST_FOREACH(items, broken_state_count_t *, c, tor_free(c));
+  smartlist_free(items);
+}
+
+/**************************************************************/
+
 /** Pack the cell_t host-order structure <b>src</b> into network-order
  * in the buffer <b>dest</b>. See tor-spec.txt for details about the
  * wire format.
@@ -364,6 +469,7 @@ connection_or_about_to_close(or_connection_t *or_conn)
     /* now mark things down as needed */
     if (connection_or_nonopen_was_started_here(or_conn)) {
       const or_options_t *options = get_options();
+      connection_or_note_state_when_broken(or_conn);
       rep_hist_note_connect_failed(or_conn->identity_digest, now);
       entry_guard_register_connect_status(or_conn->identity_digest,0,
                                           !options->HTTPSProxy, now);
diff --git a/src/or/connection_or.h b/src/or/connection_or.h
index f1acfd187b..abcfe83b99 100644
--- a/src/or/connection_or.h
+++ b/src/or/connection_or.h
@@ -14,6 +14,7 @@
 
 void connection_or_remove_from_identity_map(or_connection_t *conn);
 void connection_or_clear_identity_map(void);
+void clear_broken_connection_map(void);
 or_connection_t *connection_or_get_for_extend(const char *digest,
                                               const tor_addr_t *target_addr,
                                               const char **msg_out,
@@ -35,6 +36,8 @@ void connection_or_connect_failed(or_connection_t *conn,
 or_connection_t *connection_or_connect(const tor_addr_t *addr, uint16_t port,
                                        const char *id_digest);
 
+void connection_or_report_broken_states(int severity, int domain);
+
 int connection_tls_start_handshake(or_connection_t *conn, int receiving);
 int connection_tls_continue_handshake(or_connection_t *conn);