Prop#329 sendme: Adjust sendme sending and tracking for conflux

Because circuit-level sendmes are sent before relay data cells are processed,
we can safely move this to before the conflux decision. In this way,
regardless of conflux being negotiated, we still send sendmes as soon as data
cells are recieved. This avoids introducing conflux queue delay into RTT
measurement, which is important for measuring actual circuit capacity.

The circuit-level tracking must happen inside the call to send a data cell,
since that call now chooses a circuit to send on. Turns out, we were already
doing this kind of here, but only for the digest. Now we do both things here.

1 files changed, 30 insertions, 22 deletions

diff --git a/src/core/or/relay.c b/src/core/or/relay.c
index 0f8fd12401..38fb560e34 100644
--- a/src/core/or/relay.c
+++ b/src/core/or/relay.c
@@ -747,6 +747,15 @@ relay_send_command_from_edge_,(streamid_t stream_id, circuit_t *orig_circ,
    * because the cell digest is set within that function. */
   if (relay_command == RELAY_COMMAND_DATA) {
     sendme_record_cell_digest_on_circ(circ, cpath_layer);
+
+    /* Handle the circuit-level SENDME package window. */
+    if (sendme_note_circuit_data_packaged(circ, cpath_layer) < 0) {
+      /* Package window has gone under 0. Protocol issue. */
+      log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+             "Circuit package window is below 0. Closing circuit.");
+      circuit_mark_for_close(circ, END_CIRC_REASON_TORPROTOCOL);
+      return -1;
+    }
   }
 
   return 0;
@@ -1696,19 +1705,6 @@ handle_relay_cell_command(cell_t *cell, circuit_t *circ,
     case RELAY_COMMAND_DATA:
       ++stats_n_data_cells_received;
 
-      /* Update our circuit-level deliver window that we received a DATA cell.
-       * If the deliver window goes below 0, we end the circuit and stream due
-       * to a protocol failure. */
-      if (sendme_circuit_data_received(circ, layer_hint) < 0) {
-        log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
-               "(relay data) circ deliver_window below 0. Killing.");
-        connection_edge_end_close(conn, END_STREAM_REASON_TORPROTOCOL);
-        return -END_CIRC_REASON_TORPROTOCOL;
-      }
-
-      /* Consider sending a circuit-level SENDME cell. */
-      sendme_circuit_consider_sending(circ, layer_hint);
-
       if (rh->stream_id == 0) {
         log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, "Relay data cell with zero "
                "stream_id. Dropping.");
@@ -2113,6 +2109,26 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
     }
   }
 
+  /* Regardless of conflux or not, we always decide to send a SENDME
+   * for RELAY_DATA immediately
+   */
+  if (rh.command == RELAY_COMMAND_DATA) {
+    /* Update our circuit-level deliver window that we received a DATA cell.
+     * If the deliver window goes below 0, we end the circuit and stream due
+     * to a protocol failure. */
+    if (sendme_circuit_data_received(circ, layer_hint) < 0) {
+      log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+             "(relay data) circ deliver_window below 0. Killing.");
+      connection_edge_end_close(conn, END_STREAM_REASON_TORPROTOCOL);
+      return -END_CIRC_REASON_TORPROTOCOL;
+    }
+
+    /* Consider sending a circuit-level SENDME cell. */
+    sendme_circuit_consider_sending(circ, layer_hint);
+
+    /* Continue on to process the data cell via conflux or not */
+  }
+
   /* Conflux handling: If conflux is disabled, or the relay command is not
    * multiplexed across circuits, then process it immediately.
    *
@@ -2398,21 +2414,13 @@ connection_edge_package_raw_inbuf(edge_connection_t *conn, int package_partial,
     buf_add(entry_conn->pending_optimistic_data, payload, length);
   }
 
+  /* Send a data cell. This handles the circuit package window. */
   if (connection_edge_send_command(conn, RELAY_COMMAND_DATA,
                                    payload, length) < 0 ) {
     /* circuit got marked for close, don't continue, don't need to mark conn */
     return 0;
   }
 
-  /* Handle the circuit-level SENDME package window. */
-  if (sendme_note_circuit_data_packaged(circ, cpath_layer) < 0) {
-    /* Package window has gone under 0. Protocol issue. */
-    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
-           "Circuit package window is below 0. Closing circuit.");
-    conn->end_reason = END_STREAM_REASON_TORPROTOCOL;
-    return -1;
-  }
-
   /* Handle the stream-level SENDME package window. */
   if (sendme_note_stream_data_packaged(conn, length) < 0) {
     connection_stop_reading(TO_CONN(conn));