Merge remote-tracking branch 'public/prop227_v2'

Conflicts:
	src/test/test_dir.c

9 files changed, 250 insertions, 2 deletions

diff --git a/src/or/config.c b/src/or/config.c
index f24b19a388..a0d1346afb 100644
--- a/src/or/config.c
+++ b/src/or/config.c
@@ -86,6 +86,7 @@ static config_abbrev_t option_abbrevs_[] = {
   PLURAL(HiddenServiceExcludeNode),
   PLURAL(NumCPU),
   PLURAL(RendNode),
+  PLURAL(RecommendedPackage),
   PLURAL(RendExcludeNode),
   PLURAL(StrictEntryNode),
   PLURAL(StrictExitNode),
@@ -367,6 +368,7 @@ static config_var_t option_vars_[] = {
   V(RecommendedVersions,         LINELIST, NULL),
   V(RecommendedClientVersions,   LINELIST, NULL),
   V(RecommendedServerVersions,   LINELIST, NULL),
+  V(RecommendedPackages,         LINELIST, NULL),
   V(RefuseUnknownExits,          AUTOBOOL, "auto"),
   V(RejectPlaintextPorts,        CSV,      ""),
   V(RelayBandwidthBurst,         MEMUNIT,  "0"),
@@ -2743,6 +2745,13 @@ options_validate(or_options_t *old_options, or_options_t *options,
              "features to be broken in unpredictable ways.");
   }
 
+  for (cl = options->RecommendedPackages; cl; cl = cl->next) {
+    if (! validate_recommended_package_line(cl->value)) {
+      log_warn(LD_CONFIG, "Invalid RecommendedPackage line %s will be ignored",
+               escaped(cl->value));
+    }
+  }
+
   if (options->AuthoritativeDir) {
     if (!options->ContactInfo && !options->TestingTorNetwork)
       REJECT("Authoritative directory servers must set ContactInfo");
diff --git a/src/or/control.c b/src/or/control.c
index e963aeab7f..24e1479b50 100644
--- a/src/or/control.c
+++ b/src/or/control.c
@@ -2183,6 +2183,8 @@ static const getinfo_item_t getinfo_items[] = {
          "Brief summary of router status by nickname (v2 directory format)."),
   PREFIX("ns/purpose/", networkstatus,
          "Brief summary of router status by purpose (v2 directory format)."),
+  PREFIX("consensus/", networkstatus,
+         "Information about and from the ns consensus."),
   ITEM("network-status", dir,
        "Brief summary of router status (v1 directory format)"),
   ITEM("circuit-status", events, "List of current circuits originating here."),
diff --git a/src/or/dirserv.c b/src/or/dirserv.c
index b694f8af77..0be2a2071e 100644
--- a/src/or/dirserv.c
+++ b/src/or/dirserv.c
@@ -2511,6 +2511,15 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key,
 
   v3_out->client_versions = client_versions;
   v3_out->server_versions = server_versions;
+  v3_out->package_lines = smartlist_new();
+  {
+    config_line_t *cl;
+    for (cl = get_options()->RecommendedPackages; cl; cl = cl->next) {
+      if (validate_recommended_package_line(cl->value))
+        smartlist_add(v3_out->package_lines, tor_strdup(cl->value));
+    }
+  }
+
   v3_out->known_flags = smartlist_new();
   smartlist_split_string(v3_out->known_flags,
                 "Authority Exit Fast Guard Stable V2Dir Valid",
@@ -3256,6 +3265,83 @@ connection_dirserv_flushed_some(dir_connection_t *conn)
   }
 }
 
+/** Return true iff <b>line</b> is a valid RecommendedPackages line.
+ */
+/*
+  The grammar is:
+
+    "package" SP PACKAGENAME SP VERSION SP URL SP DIGESTS NL
+
+      PACKAGENAME = NONSPACE
+      VERSION = NONSPACE
+      URL = NONSPACE
+      DIGESTS = DIGEST | DIGESTS SP DIGEST
+      DIGEST = DIGESTTYPE "=" DIGESTVAL
+
+      NONSPACE = one or more non-space printing characters
+
+      DIGESTVAL = DIGESTTYPE = one or more non-=, non-" " characters.
+
+      SP = " "
+      NL = a newline
+
+ */
+int
+validate_recommended_package_line(const char *line)
+{
+  const char *cp = line;
+
+#define WORD()                                  \
+  do {                                          \
+    if (*cp == ' ')                             \
+      return 0;                                 \
+    cp = strchr(cp, ' ');                       \
+    if (!cp)                                    \
+      return 0;                                 \
+  } while (0)
+
+  WORD(); /* skip packagename */
+  ++cp;
+  WORD(); /* skip version */
+  ++cp;
+  WORD(); /* Skip URL */
+  ++cp;
+
+  /* Skip digesttype=digestval + */
+  int n_entries = 0;
+  while (1) {
+    const char *start_of_word = cp;
+    const char *end_of_word = strchr(cp, ' ');
+    if (! end_of_word)
+      end_of_word = cp + strlen(cp);
+
+    if (start_of_word == end_of_word)
+      return 0;
+
+    const char *eq = memchr(start_of_word, '=', end_of_word - start_of_word);
+
+    if (!eq)
+      return 0;
+    if (eq == start_of_word)
+      return 0;
+    if (eq == end_of_word - 1)
+      return 0;
+    if (memchr(eq+1, '=', end_of_word - (eq+1)))
+      return 0;
+
+    ++n_entries;
+    if (0 == *end_of_word)
+      break;
+
+    cp = end_of_word + 1;
+  }
+
+  if (n_entries == 0)
+    return 0;
+
+  return 1;
+}
+
 /** Release all storage used by the directory server. */
 void
 dirserv_free_all(void)
diff --git a/src/or/dirserv.h b/src/or/dirserv.h
index d4ce54260c..514ec444e6 100644
--- a/src/or/dirserv.h
+++ b/src/or/dirserv.h
@@ -104,6 +104,8 @@ void dirserv_free_all(void);
 void cached_dir_decref(cached_dir_t *d);
 cached_dir_t *new_cached_dir(char *s, time_t published);
 
+int validate_recommended_package_line(const char *line);
+
 #ifdef DIRSERV_PRIVATE
 
 /* Put the MAX_MEASUREMENT_AGE #define here so unit tests can see it */
diff --git a/src/or/dirvote.c b/src/or/dirvote.c
index f0dcc88070..7739c52d15 100644
--- a/src/or/dirvote.c
+++ b/src/or/dirvote.c
@@ -66,6 +66,7 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key,
 {
   smartlist_t *chunks = smartlist_new();
   const char *client_versions = NULL, *server_versions = NULL;
+  char *packages = NULL;
   char fingerprint[FINGERPRINT_LEN+1];
   char digest[DIGEST_LEN];
   uint32_t addr;
@@ -98,6 +99,18 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key,
     server_versions_line = tor_strdup("");
   }
 
+  if (v3_ns->package_lines) {
+    smartlist_t *tmp = smartlist_new();
+    SMARTLIST_FOREACH(v3_ns->package_lines, const char *, p,
+                      if (validate_recommended_package_line(p))
+                        smartlist_add_asprintf(tmp, "package %s\n", p));
+    packages = smartlist_join_strings(tmp, "", 0, NULL);
+    SMARTLIST_FOREACH(tmp, char *, cp, tor_free(cp));
+    smartlist_free(tmp);
+  } else {
+    packages = tor_strdup("");
+  }
+
   {
     char published[ISO_TIME_LEN+1];
     char va[ISO_TIME_LEN+1];
@@ -132,6 +145,7 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key,
                  "valid-until %s\n"
                  "voting-delay %d %d\n"
                  "%s%s" /* versions */
+                 "%s" /* packages */
                  "known-flags %s\n"
                  "flag-thresholds %s\n"
                  "params %s\n"
@@ -143,6 +157,7 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key,
                  v3_ns->vote_seconds, v3_ns->dist_seconds,
                  client_versions_line,
                  server_versions_line,
+                 packages,
                  flags,
                  flag_thresholds,
                  params,
@@ -230,6 +245,7 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key,
  done:
   tor_free(client_versions_line);
   tor_free(server_versions_line);
+  tor_free(packages);
 
   SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp));
   smartlist_free(chunks);
@@ -1037,6 +1053,7 @@ networkstatus_compute_consensus(smartlist_t *votes,
   const routerstatus_format_type_t rs_format =
     flavor == FLAV_NS ? NS_V3_CONSENSUS : NS_V3_CONSENSUS_MICRODESC;
   char *params = NULL;
+  char *packages = NULL;
   int added_weights = 0;
   tor_assert(flavor == FLAV_NS || flavor == FLAV_MICRODESC);
   tor_assert(total_authorities >= smartlist_len(votes));
@@ -1120,6 +1137,11 @@ networkstatus_compute_consensus(smartlist_t *votes,
                                                       n_versioning_servers);
     client_versions = compute_consensus_versions_list(combined_client_versions,
                                                       n_versioning_clients);
+    if (consensus_method >= MIN_METHOD_FOR_PACKAGE_LINES) {
+      packages = compute_consensus_package_lines(votes);
+    } else {
+      packages = tor_strdup("");
+    }
 
     SMARTLIST_FOREACH(combined_server_versions, char *, cp, tor_free(cp));
     SMARTLIST_FOREACH(combined_client_versions, char *, cp, tor_free(cp));
@@ -1162,10 +1184,13 @@ networkstatus_compute_consensus(smartlist_t *votes,
                  "voting-delay %d %d\n"
                  "client-versions %s\n"
                  "server-versions %s\n"
+                 "%s" /* packages */
                  "known-flags %s\n",
                  va_buf, fu_buf, vu_buf,
                  vote_seconds, dist_seconds,
-                 client_versions, server_versions, flaglist);
+                 client_versions, server_versions,
+                 packages,
+                 flaglist);
 
     tor_free(flaglist);
   }
@@ -1852,6 +1877,7 @@ networkstatus_compute_consensus(smartlist_t *votes,
 
   tor_free(client_versions);
   tor_free(server_versions);
+  tor_free(packages);
   SMARTLIST_FOREACH(flags, char *, cp, tor_free(cp));
   smartlist_free(flags);
   SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp));
@@ -1860,6 +1886,78 @@ networkstatus_compute_consensus(smartlist_t *votes,
   return result;
 }
 
+/** Given a list of networkstatus_t for each vote, return a newly allocated
+ * string containing the "package" lines for the vote. */
+STATIC char *
+compute_consensus_package_lines(smartlist_t *votes)
+{
+  const int n_votes = smartlist_len(votes);
+
+  /* This will be a map from "packagename version" strings to arrays
+   * of const char *, with the i'th member of the array corresponding to the
+   * package line from the i'th vote.
+   */
+  strmap_t *package_status = strmap_new();
+
+  SMARTLIST_FOREACH_BEGIN(votes, networkstatus_t *, v) {
+    if (! v->package_lines)
+      continue;
+    SMARTLIST_FOREACH_BEGIN(v->package_lines, const char *, line) {
+      if (! validate_recommended_package_line(line))
+        continue;
+
+      /* Skip 'cp' to the second space in the line. */
+      const char *cp = strchr(line, ' ');
+      if (!cp) continue;
+      ++cp;
+      cp = strchr(cp, ' ');
+      if (!cp) continue;
+
+      char *key = tor_strndup(line, cp - line);
+
+      const char **status = strmap_get(package_status, key);
+      if (!status) {
+        status = tor_calloc(n_votes, sizeof(const char *));
+        strmap_set(package_status, key, status);
+      }
+      status[v_sl_idx] = line; /* overwrite old value */
+      tor_free(key);
+    } SMARTLIST_FOREACH_END(line);
+  } SMARTLIST_FOREACH_END(v);
+
+  smartlist_t *entries = smartlist_new(); /* temporary */
+  smartlist_t *result_list = smartlist_new(); /* output */
+  STRMAP_FOREACH(package_status, key, const char **, values) {
+    int i, count=-1;
+    for (i = 0; i < n_votes; ++i) {
+      if (values[i])
+        smartlist_add(entries, (void*) values[i]);
+    }
+    smartlist_sort_strings(entries);
+    int n_voting_for_entry = smartlist_len(entries);
+    const char *most_frequent =
+      smartlist_get_most_frequent_string_(entries, &count);
+
+    if (n_voting_for_entry >= 3 && count > n_voting_for_entry / 2) {
+      smartlist_add_asprintf(result_list, "package %s\n", most_frequent);
+    }
+
+    smartlist_clear(entries);
+
+  } STRMAP_FOREACH_END;
+
+  smartlist_sort_strings(result_list);
+
+  char *result = smartlist_join_strings(result_list, "", 0, NULL);
+
+  SMARTLIST_FOREACH(result_list, char *, cp, tor_free(cp));
+  smartlist_free(result_list);
+  smartlist_free(entries);
+  strmap_free(package_status, tor_free_);
+
+  return result;
+}
+
 /** Given a consensus vote <b>target</b> and a set of detached signatures in
  * <b>sigs</b> that correspond to the same consensus, check whether there are
  * any new signatures in <b>src_voter_list</b> that should be added to
diff --git a/src/or/dirvote.h b/src/or/dirvote.h
index 8908336fa1..20dcbcd5b6 100644
--- a/src/or/dirvote.h
+++ b/src/or/dirvote.h
@@ -55,7 +55,7 @@
 #define MIN_SUPPORTED_CONSENSUS_METHOD 13
 
 /** The highest consensus method that we currently support. */
-#define MAX_SUPPORTED_CONSENSUS_METHOD 18
+#define MAX_SUPPORTED_CONSENSUS_METHOD 19
 
 /** Lowest consensus method where microdesc consensuses omit any entry
  * with no microdesc. */
@@ -79,6 +79,9 @@
  * microdescriptors. */
 #define MIN_METHOD_FOR_ID_HASH_IN_MD 18
 
+/** Lowest consensus method where we include "package" lines*/
+#define MIN_METHOD_FOR_PACKAGE_LINES 19
+
 /** Default bandwidth to clip unmeasured bandwidths to using method >=
  * MIN_METHOD_TO_CLIP_UNMEASURED_BW */
 #define DEFAULT_MAX_UNMEASURED_BW_KB 20
@@ -160,6 +163,7 @@ STATIC char *format_networkstatus_vote(crypto_pk_t *private_key,
                                  networkstatus_t *v3_ns);
 STATIC char *dirvote_compute_params(smartlist_t *votes, int method,
                              int total_authorities);
+STATIC char *compute_consensus_package_lines(smartlist_t *votes);
 #endif
 
 #endif
diff --git a/src/or/networkstatus.c b/src/or/networkstatus.c
index 59ba1e6cb7..da110fdff6 100644
--- a/src/or/networkstatus.c
+++ b/src/or/networkstatus.c
@@ -257,6 +257,10 @@ networkstatus_vote_free(networkstatus_t *ns)
     SMARTLIST_FOREACH(ns->supported_methods, char *, c, tor_free(c));
     smartlist_free(ns->supported_methods);
   }
+  if (ns->package_lines) {
+    SMARTLIST_FOREACH(ns->package_lines, char *, c, tor_free(c));
+    smartlist_free(ns->package_lines);
+  }
   if (ns->voters) {
     SMARTLIST_FOREACH_BEGIN(ns->voters, networkstatus_voter_info_t *, voter) {
       tor_free(voter->nickname);
@@ -1909,6 +1913,33 @@ getinfo_helper_networkstatus(control_connection_t *conn,
   } else if (!strcmpstart(question, "ns/purpose/")) {
     *answer = networkstatus_getinfo_by_purpose(question+11, time(NULL));
     return *answer ? 0 : -1;
+  } else if (!strcmp(question, "consensus/packages")) {
+    const networkstatus_t *ns = networkstatus_get_latest_consensus();
+    if (ns && ns->package_lines)
+      *answer = smartlist_join_strings(ns->package_lines, "\n", 0, NULL);
+    else
+      *errmsg = "No consensus available";
+    return *answer ? 0 : -1;
+  } else if (!strcmp(question, "consensus/valid-after") ||
+             !strcmp(question, "consensus/fresh-until") ||
+             !strcmp(question, "consensus/valid-until")) {
+    const networkstatus_t *ns = networkstatus_get_latest_consensus();
+    if (ns) {
+      time_t t;
+      if (!strcmp(question, "consensus/valid-after"))
+        t = ns->valid_after;
+      else if (!strcmp(question, "consensus/fresh-until"))
+        t = ns->fresh_until;
+      else
+        t = ns->valid_until;
+
+      char tbuf[ISO_TIME_LEN+1];
+      format_iso_time(tbuf, t);
+      *answer = tor_strdup(tbuf);
+    } else {
+      *errmsg = "No consensus available";
+    }
+    return *answer ? 0 : -1;
   } else {
     return 0;
   }
diff --git a/src/or/or.h b/src/or/or.h
index 4207f3d7b3..520b7dba89 100644
--- a/src/or/or.h
+++ b/src/or/or.h
@@ -2417,6 +2417,9 @@ typedef struct networkstatus_t {
   /** Vote only: what methods is this voter willing to use? */
   smartlist_t *supported_methods;
 
+  /** List of 'package' lines describing hashes of downloadable packages */
+  smartlist_t *package_lines;
+
   /** How long does this vote/consensus claim that authorities take to
    * distribute their votes to one another? */
   int vote_seconds;
@@ -3433,6 +3436,7 @@ typedef struct {
   config_line_t *RecommendedVersions;
   config_line_t *RecommendedClientVersions;
   config_line_t *RecommendedServerVersions;
+  config_line_t *RecommendedPackages;
   /** Whether dirservers allow router descriptors with private IPs. */
   int DirAllowPrivateAddresses;
   /** Whether routers accept EXTEND cells to routers with private IPs. */
diff --git a/src/or/routerparse.c b/src/or/routerparse.c
index a2bc8fbb93..f7687e0e40 100644
--- a/src/or/routerparse.c
+++ b/src/or/routerparse.c
@@ -131,6 +131,7 @@ typedef enum {
   K_CONSENSUS_METHOD,
   K_LEGACY_DIR_KEY,
   K_DIRECTORY_FOOTER,
+  K_PACKAGE,
 
   A_PURPOSE,
   A_LAST_LISTED,
@@ -420,6 +421,7 @@ static token_rule_t networkstatus_token_table[] = {
   T1("known-flags",            K_KNOWN_FLAGS,      ARGS,        NO_OBJ ),
   T01("params",                K_PARAMS,           ARGS,        NO_OBJ ),
   T( "fingerprint",            K_FINGERPRINT,      CONCAT_ARGS, NO_OBJ ),
+  T0N("package",               K_PACKAGE,          CONCAT_ARGS, NO_OBJ ),
 
   CERTIFICATE_MEMBERS
 
@@ -2626,6 +2628,16 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out,
     ns->server_versions = tor_strdup(tok->args[0]);
   }
 
+  {
+    smartlist_t *package_lst = find_all_by_keyword(tokens, K_PACKAGE);
+    ns->package_lines = smartlist_new();
+    if (package_lst) {
+      SMARTLIST_FOREACH(package_lst, directory_token_t *, t,
+                    smartlist_add(ns->package_lines, tor_strdup(t->args[0])));
+    }
+    smartlist_free(package_lst);
+  }
+
   tok = find_by_keyword(tokens, K_KNOWN_FLAGS);
   ns->known_flags = smartlist_new();
   inorder = 1;