diff options
449 files changed, 30910 insertions, 925 deletions
@@ -11,3 +11,12 @@ # OS X folder cruft .DS_Store + +# mdbook build dir +/build + +# generated html (reorganized from the build dir) +/html + +# bin/check_links output +/html.link-check diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6f56955..ddd399d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,52 +1,71 @@ - +--- variables: - HEADER: | - <!DOCTYPE html> - <html lang="en-US"> - <head> - <meta charset="utf-8"> - <meta http-equiv="X-UA-Compatible" content="IE=edge"> - <meta name="viewport" content="width\=device-width, initial-scale=1"> - <meta name="author" content="The Tor Project, Inc."> - <title>torspec</title> - <link href="https://2019.www.torproject.org/css/master.min.css" rel="stylesheet"> - </head> - <body> - <div id="wrap"><div id="content"><div id="maincol"> - FOOTER: "</div></div></div></body></html>" + SITE_URL: spec.torproject.org -pages: - image: debian:buster +build: + image: rust:bookworm + rules: + # We install mermaid-cli only when we are building for the + # actual website, since it is pretty expensive. If we switch to + # faster install tool, we should revisit that. + - if: '$CI_PROJECT_ROOT_NAMESPACE == "tpo" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' + variables: + MMDC: mmdc -p bin/ci-puppeteer-config.json + # ^ See note + # But we _do_ want to build unconditionally. + - when: always + script: + - env + - apt-get update && apt-get install -y git python3 python3-yaml + - ./bin/via-cargo-install-in-ci mdbook mdbook-linkcheck + - | + if test "$MMDC" != "" ; then + ./bin/mermaid-cli-in-ci + else + ./bin/via-cargo-install-in-ci mdbook-mermaid + fi + - ./bin/check_index + - ./bin/build_html + - mv html public + artifacts: + paths: + - public + cache: + paths: + - cache + +# Note re MMDC above: +# https://github.com/mermaid-js/mermaid-cli/blob/HEAD/docs/linux-sandbox-issue.md +# +# bin/ is the wrong place for that file +# but I don't want to rename all of bin/ to maint/ right now. + +link-check: + image: rust:bookworm + rules: + - when: always script: - - apt-get update - - apt-get -qy install --no-install-recommends pandoc - - test -d public || mkdir public - - printf "${HEADER}<h1>%s</h1><ul>" $CI_PROJECT_PATH > public/index.html - - for f in *.txt; do - set -x; - name=`echo $f | sed s,\.txt$,,`; - md=${name}.md; - cat $f | sed --regexp-extended - -e '0,/^ +/{s/^ +/# /}' - -e 's/^ {1,3}([^ ])/\1/' - -e '/^[0-9]+\. +http/! s/^([0-9]+\. )/## \1/' - -e 's/^([0-9]+\.[0-9]+\. )/### \1/' - -e 's/^([0-9]+\.[0-9]+\.[0-9]+\. )/#### \1/' - -e 's/^([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+\. )/##### \1/' - > $md; - printf "\n---\n\noriginal source\x3a [$f](https://gitweb.torproject.org/torspec.git/tree/$f)\n" >> $md; - title=`sed -En '0,/^# /s/^# (.*)/\1/p' $md`; - printf "<li><a href=\"${name}.html\"><tt>$name</tt>: $title</a></li>" >> public/index.html; - pandoc --from=markdown $md --output=${name}.html; - printf "$HEADER" > public/${name}.html; - cat ${name}.html >> public/${name}.html; - printf "$FOOTER" >> public/${name}.html; - mkdir public/$name; - cp public/${name}.html public/$name/index.html; - done - - printf "</ul>$FOOTER" >> public/index.html + - apt-get update && apt-get install -y git python3 python3-yaml linklint + - ./bin/via-cargo-install-in-ci mdbook mdbook-linkcheck + - ./bin/build_html + - ./bin/check_links artifacts: paths: - - public - only: - - main + - html.link-check + cache: + paths: + - cache + +include: + project: tpo/tpa/ci-templates + file: + - static-shim-deploy.yml + - pages-deploy.yml + +# use pages for merge requests, not main (which is the default in the +# above template). the logic here is that we want people that do not +# have acess to the repository to still have a preview of their merge +# request. +pages: + rules: + - if: '$CI_PROJECT_ROOT_NAMESPACE != "tpo" && $CI_PIPELINE_SOURCE == "merge_request_event"' @@ -1,69 +1,12 @@ -# Tor specifications +# Tor specification repository -This repository holds the specifications that describe how Tor works. -They try to present Tor's protocols in sufficient detail to allow -the reader to implement a compatible implementation of Tor without ever -having to read the Tor source code. +This is the source code of the Tor specification documents. -The [proposals](/proposals) directory holds our design proposals. These -include historical documents that have now been merged into the main -specifications. For more information on the proposal process, including an -explanation of how to make new proposals, see -[001-process.txt](/proposals/001-process.txt). +**To read the Tor Specifications online**, +please go to the canonical site <https://spec.torproject.org/>. -## What you can find here +Overview of the source code layout, and build instructions, +are in `spec/back-matter.md`. -Tor's specification is pretty big, and we've broken it into a bunch of -files. - -* General interest - * [tor-spec.txt](tor-spec.txt) - contains the specification for the core Tor protocol - itself; this is a good place to start reading. - * [cert-spec.txt](cert-spec.txt) describes a certificate format used - in the other parts of the protocol. - * [dir-spec.txt](dir-spec.txt) specifies the operations and formats used to - maintain a view of the network directory. - * [padding-spec.txt](padding-spec.txt) describes a set of padding mechanisms - used to impede traffic analysis. - * [version-spec.txt](version-spec.txt) explains how to parse Tor - version numbers. - * [glossary.txt](glossary.txt) is a glossary of terms used - in the other specifications. -* Client operations - * [address-spec.txt](address-spec.txt) lists a set of special - addresses that Tor handles differently from the regular DNS system. - * [guard-spec.txt](guard-spec.txt) explains the "guard node" algorithm - that Tor clients use to avoid sampling attacks. - * [path-spec.txt](path-spec.txt) explains how clients choose their paths - through the Tor network. - * [socks-extensions](socks-extensions.txt) specifies Tor-specific - extensions to the SOCKS protocol. -* Onion services - * [rend-spec-v2.txt](rend-spec-v2.txt) is the old, deprecated version - of the onion service protocol. - * [rend-spec-v3.txt](rend-spec-v3.txt) is the current version of the - onion service protocol. -* Censorship resistance - * [bridgedb-spec.txt](bridgedb-spec.txt) explains how the `bridgedb` - server gives out bridges to censored clients. - * [gettor-spec.txt](gettor-spec.txt) describes the `gettor` tool, - which is used to download Tor in censored areas. - * [pt-spec.txt](pt-spec.txt) describes the protocol that Tor clients - and relays use to communicate with pluggable transports used for - traffic obfuscation. -* Directory authorities - * [bandwidth-file-spec.txt](bandwidth-file-spec.txt) specifies the - file format used by bandwidth-measuring tools to report their - observations to directory authorities. - * [srv-spec.txt](srv-spec.txt) specifies the protocol that - directory authorities use to securely compute shared random values - for the network. -* Controller protocol - * [control-spec.txt](control-spec.txt) explains the protocol used by - controllers to communicate with a running Tor process. -* Miscellaneous - * [dir-list-spec.txt](dir-list-spec.txt) explains the format used by - tools like the fallback directory scripts to output a list of - Tor directories for inclusion in the Tor source code. - * The [attic](attic) directory has obsolete or historical documents. +The canonical source repository is on the Tor Project's gitab, at +<https://gitlab.torproject.org/tpo/core/torspec/> diff --git a/attic/text_formats/README.md b/attic/text_formats/README.md new file mode 100644 index 0000000..b68d031 --- /dev/null +++ b/attic/text_formats/README.md @@ -0,0 +1,2 @@ +This directory contains our specifications before our conversion +to mdbook on Thu Oct 12 12:27:58 PM EDT 2023. diff --git a/address-spec.txt b/attic/text_formats/address-spec.txt index 1e90e6e..1e90e6e 100644 --- a/address-spec.txt +++ b/attic/text_formats/address-spec.txt diff --git a/bandwidth-file-spec.txt b/attic/text_formats/bandwidth-file-spec.txt index bad13f6..bad13f6 100644 --- a/bandwidth-file-spec.txt +++ b/attic/text_formats/bandwidth-file-spec.txt diff --git a/bridgedb-spec.txt b/attic/text_formats/bridgedb-spec.txt index 51f6e5d..51f6e5d 100644 --- a/bridgedb-spec.txt +++ b/attic/text_formats/bridgedb-spec.txt diff --git a/cert-spec.txt b/attic/text_formats/cert-spec.txt index a70e100..a70e100 100644 --- a/cert-spec.txt +++ b/attic/text_formats/cert-spec.txt diff --git a/control-spec.txt b/attic/text_formats/control-spec.txt index 52e11a0..52e11a0 100644 --- a/control-spec.txt +++ b/attic/text_formats/control-spec.txt diff --git a/dir-list-spec.txt b/attic/text_formats/dir-list-spec.txt index 65af536..65af536 100644 --- a/dir-list-spec.txt +++ b/attic/text_formats/dir-list-spec.txt diff --git a/dir-spec.txt b/attic/text_formats/dir-spec.txt index f133c39..f133c39 100644 --- a/dir-spec.txt +++ b/attic/text_formats/dir-spec.txt diff --git a/ext-orport-spec.txt b/attic/text_formats/ext-orport-spec.txt index 6b8f8e1..6b8f8e1 100644 --- a/ext-orport-spec.txt +++ b/attic/text_formats/ext-orport-spec.txt diff --git a/gettor-spec.txt b/attic/text_formats/gettor-spec.txt index a4959b4..a4959b4 100644 --- a/gettor-spec.txt +++ b/attic/text_formats/gettor-spec.txt diff --git a/glossary.txt b/attic/text_formats/glossary.txt index 68de376..68de376 100644 --- a/glossary.txt +++ b/attic/text_formats/glossary.txt diff --git a/guard-spec.txt b/attic/text_formats/guard-spec.txt index 154edae..154edae 100644 --- a/guard-spec.txt +++ b/attic/text_formats/guard-spec.txt diff --git a/padding-spec.txt b/attic/text_formats/padding-spec.txt index 206a7f1..206a7f1 100644 --- a/padding-spec.txt +++ b/attic/text_formats/padding-spec.txt diff --git a/param-spec.txt b/attic/text_formats/param-spec.txt index 123cedc..af2da8c 100644 --- a/param-spec.txt +++ b/attic/text_formats/param-spec.txt @@ -1,4 +1,3 @@ - Tor network parameters This file lists the recognized parameters that can appear on the "params" @@ -126,6 +125,15 @@ Table of Contents "maxunmeasuredbw" -- Used by authorities during voting with method 17 or later. The maximum value to give for any Bandwidth= entry for a router that isn't based on at least three measurements. + + (Note: starting in version 0.4.6.1-alpha + there was a bug where Tor authorities would instead look at + a parameter called "maxunmeasurdbw", without the "e". + This bug was fixed in 0.4.9.1-alpha and in 0.4.8.8. + Until all relays are running a fixed version, then either this parameter + must not be set, or it must be set to the same value for both + spellings.) + First-appeared: 0.2.4.11-alpha "FastFlagMinThreshold", "FastFlagMaxThreshold" -- lowest and highest @@ -359,27 +367,75 @@ Table of Contents Min: 0. Max: 50000. Default 1000. First appeared: 0.4.0.3-alpha. - "circpad_global_allowed_cells" -- DOCDOC - - "circpad_global_max_padding_pct" -- DOCDOC - - "circpad_padding_disabled" -- DOCDOC - - "circpad_padding_reduced" -- DOCDOC - - "nf_conntimeout_clients" -- DOCDOC - - "nf_conntimeout_relays" -- DOCDOC - - "nf_ito_high_reduced" -- DOCDOC - - "nf_ito_low" -- DOCDOC - - "nf_ito_low_reduced" -- DOCDOC + "circpad_global_allowed_cells" -- This is the number of padding cells + that must be sent before the 'circpad_global_max_padding_percent' + parameter is applied. + Min: 0. Max: 65535. Default: 0 + + "circpad_global_max_padding_pct" -- This is the maximum ratio of + padding cells to total cells, specified as a percent. If the global + ratio of padding cells to total cells across all circuits exceeds + this percent value, no more padding is sent until the ratio becomes + lower. 0 means no limit. + Min: 0. Max: 100. Default: 0 + + "circpad_padding_disabled" -- If set to 1, no circuit padding machines + will negotiate, and all current padding machines will cease padding + immediately. + Min: 0. Max: 1. Default: 0 - "nf_pad_before_usage" -- DOCDOC + "circpad_padding_reduced" -- If set to 1, only circuit padding + machines marked as "reduced"/"low overhead" will be used. + (Currently no such machines are marked as "reduced overhead"). + Min: 0. Max: 1. Default: 0 - "nf_pad_relays" -- DOCDOC + "nf_conntimeout_clients" + - The number of seconds to keep never-used circuits opened and + available for clients to use. Note that the actual client timeout is + randomized uniformly from this value to twice this value. + - The number of seconds to keep idle (not currently used) canonical + channels are open and available. (We do this to ensure a sufficient + time duration of padding, which is the ultimate goal.) + - This value is also used to determine how long, after a port has been + used, we should attempt to keep building predicted circuits for that + port. (See path-spec.txt section 2.1.1.) This behavior was + originally added to work around implementation limitations, but it + serves as a reasonable default regardless of implementation. + - For all use cases, reduced padding clients use half the consensus + value. + - Implementations MAY mark circuits held open past the reduced padding + quantity (half the consensus value) as "not to be used for streams", + to prevent their use from becoming a distinguisher. + Min: 60. Max: 86400. Default: 1800 + + "nf_conntimeout_relays" -- The number of seconds that idle + relay-to-relay connections are kept open. + Min: 60. Max: 604800. Default: 3600 + + "nf_ito_low" -- The low end of the range to send padding when + inactive, in ms. + Min: 0. Max: 60000. Default: 1500 + + "nf_ito_high" -- The high end of the range to send padding, in ms. + If nf_ito_low == nf_ito_high == 0, padding will be disabled. + Min: nf_ito_low. Max: 60000. Default: 9500 + + "nf_ito_low_reduced" -- For reduced padding clients: the low + end of the range to send padding when inactive, in ms. + Min: 0. Max: 60000. Default: 9000 + + "nf_ito_high_reduced" -- For reduced padding clients: the high + end of the range to send padding, in ms. + Min: nf_ito_low_reduced. Max: 60000. Default: 14000 + + "nf_pad_before_usage" -- If set to 1, OR connections are padded + before the client uses them for any application traffic. If 0, + OR connections are not padded until application data begins. + Min: 0. Max: 1. Default: 1 + + "nf_pad_relays" -- If set to 1, we also pad inactive + relay-to-relay connections. + Min: 0. Max: 1. Default: 0 "nf_pad_single_onion" -- DOCDOC diff --git a/path-spec.txt b/attic/text_formats/path-spec.txt index 33d50e5..33d50e5 100644 --- a/path-spec.txt +++ b/attic/text_formats/path-spec.txt diff --git a/pt-spec.txt b/attic/text_formats/pt-spec.txt index 45b4c31..45b4c31 100644 --- a/pt-spec.txt +++ b/attic/text_formats/pt-spec.txt diff --git a/rend-spec-v3.txt b/attic/text_formats/rend-spec-v3.txt index d836d23..d836d23 100644 --- a/rend-spec-v3.txt +++ b/attic/text_formats/rend-spec-v3.txt diff --git a/socks-extensions.txt b/attic/text_formats/socks-extensions.txt index c35069d..c35069d 100644 --- a/socks-extensions.txt +++ b/attic/text_formats/socks-extensions.txt diff --git a/srv-spec.txt b/attic/text_formats/srv-spec.txt index f768b73..f768b73 100644 --- a/srv-spec.txt +++ b/attic/text_formats/srv-spec.txt diff --git a/tor-spec.txt b/attic/text_formats/tor-spec.txt index 4d21c9a..4d21c9a 100644 --- a/tor-spec.txt +++ b/attic/text_formats/tor-spec.txt diff --git a/version-spec.txt b/attic/text_formats/version-spec.txt index 615f6f2..615f6f2 100644 --- a/version-spec.txt +++ b/attic/text_formats/version-spec.txt diff --git a/bin/build_html b/bin/build_html new file mode 100755 index 0000000..b6ee26d --- /dev/null +++ b/bin/build_html @@ -0,0 +1,90 @@ +#!/usr/bin/env bash + +set -e -u -o pipefail -x + +: "${MDBOOK:=mdbook}" + +TOPLEVEL=$(realpath "$(dirname "$0")")/.. +cd "${TOPLEVEL}" +./bin/reindex + +./bin/make_redirects + +# Now we deal with mermaid diagrams. +# +# This may require changes to the md files in {spec,proposals} +# or to the mdbook.toml files in mdbook/*/. +# We will make a copy of whatever we need to change, +# and then make changes to that copy. +# +# When we are done with these changes, we will set some variables: +# - MDBOOK_OUTPUT_DIR is "", or the location where we have put our raw mdbook output. +# - MDBOOK_DIR is the parent directory of the possibly modified copies +# of mdbook/{spec,proposals}. + +if test -n "${MMDC:-}" || command -v "mmdc" >&/dev/null; then + # CASE 1: mermaid-cli is installed. + # + # We will convert mermaid diagrams to svg. The mermaid_cvt_svg + # script does this with a temporary copy of our markdown directories, + # so as not to alter the original. + # + # (The conversion involves npm and a headless chrome browser, + # to it is understandable that not everybody would want to do it + # this way.) + echo "Using mermaid-cli to pre-render mermaid diagrams" + + TMPDIR=$(mktemp -d "${TOPLEVEL}/tmp_mmdc.XXXXXXXX") + trap 'rm -rf "$TMPDIR"' 0 + ./bin/mermaid_cvt_svg "$TMPDIR" + MDBOOK_OUTPUT_DIR="$TMPDIR/build" + MDBOOK_DIR="$TMPDIR/mdbook" +elif test -n "${MDBOOK_MERMAID:-}" || command -v "mdbook-mermaid" >&/dev/null; then + # CASE 2: mdbook_mermaid is installed. + # + # We will make a temporary copy of the mdbook configuration directory + # only, and use mdbook-mermaid to alter that. + # + # This is much easier to run locally, but it requires that your + # browser has enough client-side javascript in order to run + # mermaid. It doesn't touch npm. + echo "Using mdbook-mermaid to set up dynamic rendering of mermaid diagrams" + + MDBOOK_MERMAID=${MDBOOK_MERMAID:=mdbook-mermaid} + TMPDIR=$(mktemp -d "${TOPLEVEL}/tmp_mdbook_mermaid.XXXXXXXX") + trap 'rm -rf "$TMPDIR"' 0 + cp -r ./mdbook/proposals ./mdbook/spec ./mdbook/theme "$TMPDIR" + mdbook-mermaid install "$TMPDIR/spec" + mdbook-mermaid install "$TMPDIR/proposals" + MDBOOK_OUTPUT_DIR="" + MDBOOK_DIR="$TMPDIR" +else + # CASE 3: No mermaid support. + # + # In this case we run mdbook on our inputs unchanged. + # The mermaid blocks will render as code. + echo "No mermaid support found; mermaid diagrams will be unrendered" + + MDBOOK_OUTPUT_DIR="" + MDBOOK_DIR="$TOPLEVEL/mdbook" +fi + +# mdbook-linkcheck is a non-obvious dependency, and the mdbook output when it's +# not found doesn't spell out how to install it. +if ! command -v mdbook-linkcheck; then + echo 'ERROR: mdbook-linkcheck not found. You should probably install it with "cargo install mdbook-linkcheck"' + exit 1 +fi + +$MDBOOK build "${MDBOOK_DIR}/spec" +$MDBOOK build "${MDBOOK_DIR}/proposals" + +if test -n "${MDBOOK_OUTPUT_DIR}"; then + rm -rf "${TOPLEVEL}/build" + mv "${MDBOOK_OUTPUT_DIR}" "${TOPLEVEL}/build" +fi + +rm -rf "${TOPLEVEL}/html/" +mv "${TOPLEVEL}/build/spec/html" "${TOPLEVEL}/html" +mv "${TOPLEVEL}/build/proposals/html" "${TOPLEVEL}/html/proposals" + diff --git a/bin/check_index b/bin/check_index new file mode 100755 index 0000000..a2e8c4f --- /dev/null +++ b/bin/check_index @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# +# Give an error if somebody forgot to run ./reindex.py. +# +# Only works on a clean git checkout. + +set -e -u -o pipefail -x + +TOPLEVEL=$(dirname "$0")/.. + +if ! git diff --quiet ; then + echo "Git repository is not clean. Cannot procede." + exit 2 +fi + +cd "$TOPLEVEL/" +./bin/reindex + +if ! git diff --quiet ; then + echo "Proposal index is not up-to-date. Run ./reindex.py to regenerate it." >&2 + exit 1 +fi diff --git a/bin/check_links b/bin/check_links new file mode 100755 index 0000000..a079976 --- /dev/null +++ b/bin/check_links @@ -0,0 +1,41 @@ +#!/bin/bash +# +# Check rustdoc HTML links and anchors +# bin/build_html must have been run first. +# +# Adapted from +# https://gitlab.torproject.org/Diziet/rust-derive-adhoc/-/blob/main/maint/check-doc-links?ref_type=heads + +set -e +set -o pipefail + +# nailing-cargo --no-nail --- bin/build_html +# nailing-cargo --preclean=no --- bin/check_links + +chk_dir=html.link-check + +rm -rf html.link-check +cp -al html $chk_dir + +# Fix up https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=425632 + +# shellcheck disable=SC2016 +find $chk_dir -name \*.html -print0 | +xargs -0r -- perl -i~ -pe ' + s{\bid=("[^"]+")[^<>]*\>}{$&<a name=$1>}g; +' + +linklint -xref -out linklint.errors -error -root $chk_dir /@ + +cat linklint.errors + +set +e +grep ERROR linklint.errors +rc=$? +set -e + +case $rc in +0) echo >&2 '** found linkcheck errors **'; exit 1;; +1) ;; +*) echo >&2 "linkcheck failed $rc"; exit 1;; +esac diff --git a/bin/ci-puppeteer-config.json b/bin/ci-puppeteer-config.json new file mode 100644 index 0000000..3201af7 --- /dev/null +++ b/bin/ci-puppeteer-config.json @@ -0,0 +1,3 @@ +{ + "args": ["--no-sandbox"] +} diff --git a/bin/make_redirects b/bin/make_redirects new file mode 100755 index 0000000..a26b871 --- /dev/null +++ b/bin/make_redirects @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 + +import yaml, os, re, sys + +def update_file(fname, start_marker, end_marker, replacement): + content = list(open(fname, 'r').readlines()) + startline = content.index(start_marker) + endline = content.index(end_marker) + assert endline > startline + content[startline+1:endline] = replacement + + with open(fname+".tmp", 'w') as f: + f.write("".join(content)) + os.rename(fname+".tmp", fname) + +BOOK_START = "# BEGIN AUTO-GENERATED REDIRECTS\n" +BOOK_END = "# END AUTO-GENERATED REDIRECTS\n" + +HTML_MARKER_START = "<!-- BEGIN AUTO-GENERATED REDIRECTS -->\n" +HTML_MARKER_END = "<!-- END AUTO-GENERATED REDIRECTS -->\n" + +def book_redirects(rs, spec_dir): + lines = [] + for kwd, info in rs.items(): + if os.path.isdir(os.path.join(spec_dir, kwd)): + assert os.path.isfile(os.path.join(spec_dir, kwd, "index.md")) + assert info.get('implicit') + continue + elif any((os.path.isfile(os.path.join(spec_dir, kwd) + ext)) for ext in [".txt", ".md"]): + assert info.get('implicit') + continue + else: + assert not info.get('implicit') + source = kwd + ".html" + target = info['target'] + lines.append( + f'"/{source}" = "{target}"\n' + ) + return "".join(lines) + +def permalinks_redirects(rs): + lines = [ "<dl>\n" ] + for kwd, info in rs.items(): + target = info['target'] + desc = info['description'] + lines.append(f'<dt><code>/{kwd}</code></dt>\n') + lines.append(f'<dd><a href="{target}"><code>{target}</code> ({desc})</a></dt>\n') + + lines.append("</dl>\n") + return "".join(lines) + +def proposal_redirects(proposals_dir): + lines = [] + for fname in os.listdir(proposals_dir): + m = re.match(r'^(\d+)-.*\.(?:md|txt)$', fname) + if m: + source = m.group(1) + ".html" + target, targetext = os.path.splitext(fname) + if targetext == '.md': + targetext = ".html" + lines.append(f'"/{source}" = "./{target}{targetext}"\n') + lines.sort() + return "".join(lines) + +if __name__ == '__main__': + toplevel = os.path.join(os.path.dirname(sys.argv[0]), "..") + spec_book_fname = os.path.join(toplevel, "mdbook", "spec", "book.toml") + spec_dir = os.path.join(toplevel, "spec") + permalinks_fname = os.path.join(toplevel, "spec", "permalinks.md") + prop_dir = os.path.join(toplevel, "proposals") + prop_book_fname = os.path.join(toplevel, "mdbook", "proposals", "book.toml") + yaml_fname = os.path.join(toplevel, "mdbook", "spec", "spec-redirects.yaml") + + rs = yaml.load(open(yaml_fname), yaml.Loader)['redirects'] + + update_file(spec_book_fname, BOOK_START, BOOK_END, book_redirects(rs, spec_dir)) + update_file(permalinks_fname, HTML_MARKER_START, HTML_MARKER_END, permalinks_redirects(rs)) + update_file(prop_book_fname, BOOK_START, BOOK_END, proposal_redirects(prop_dir)) diff --git a/bin/mermaid-cli-in-ci b/bin/mermaid-cli-in-ci new file mode 100755 index 0000000..07700fa --- /dev/null +++ b/bin/mermaid-cli-in-ci @@ -0,0 +1,52 @@ +#!/usr/bin/env bash + +set -euo pipefail + +set -x + +# Bump this to "clear" the cache. +# Actually, it just causes us to ignore previous cached results. +cache_clear_token=2023-11-09c + +cache_dir="cache/$CI_JOB_IMAGE,$cache_clear_token,puppeteer" +local_dir="$HOME/.cache/puppeteer" + +if test -d "$cache_dir"; then + mkdir -p "$local_dir"/. + cp -a "$cache_dir"/. "$local_dir"/. +fi + +./bin/via-yarn-install-in-ci mmdc https://github.com/mermaid-js/mermaid-cli \ + bba0240ad87f6fbf44d8a24941e37f4bc2c8bf30 + +( + cd "$local_dir" + find . -printf '%y %p %l\n' | sort + find . -type f | sort | xargs sha256sum -- +) >puppeteer-cache-got-listings + +# Empirically, the mermaid-cli locked install produced a very old +# chromium today. So I suspect it's not fetching "latest", but a controlled +# version. Nevertheless, we should check that what we got is actually +# the same and hasn't been changed by Google (or something on the way). + +# Doing this now isn't ideal, because I don't actually know if +# the mermaid-cli *install* process runs anything from here. +# But I don't think it runs the main chrome binary, since at one point +# in our tests we got as far as this and then the chrome binary +# failed with due to a missing OS shared library. + +# This expected output listing shouldn't be in bin/ but the best way +# to fix that would be to rename the whole bin directory to maint. +diff -u bin/puppeteer-cache-expect-listings puppeteer-cache-got-listings + +if ! test -d "$cache_dir"; then + mkdir -p "$cache_dir" + cp -a "$local_dir"/. "$cache_dir"/. +fi + +# This is the easiest way to get the shared libraries that chromium +# depends on. Obviously, using the Debian package's dependencies +# is totally wrong, but it works in practice, and we don't have a proper +# dependency list from the binaries from the ad-hoc downloads. +apt-get install -y chromium diff --git a/bin/mermaid_cvt_svg b/bin/mermaid_cvt_svg new file mode 100755 index 0000000..a48bbee --- /dev/null +++ b/bin/mermaid_cvt_svg @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +set -e -u -o pipefail -x + +TOPLEVEL=$(realpath "$(dirname "$0")")/.. +TMPDIR="$1" +: "${MMDC:=mmdc}" + +cd "$TOPLEVEL" + +# We make a mirror of the specs, since we will need to make changes to them. +cp -rl spec proposals mdbook "$TMPDIR" + +# We use mermaid-cli to extract the mermaid from any file containing it, +# and generate a new version that uses svg instead. +shopt -s globstar nullglob +for fname in "$TMPDIR"/**/*.md; do + if grep '^```mermaid' "$fname"; then + ORIG="${fname%.md}.__orig.md" + mv "$fname" "$ORIG" + $MMDC -i "$ORIG" -o "$fname" + fi +done + + + diff --git a/bin/puppeteer-cache-expect-listings b/bin/puppeteer-cache-expect-listings new file mode 100644 index 0000000..a5d5783 --- /dev/null +++ b/bin/puppeteer-cache-expect-listings @@ -0,0 +1,289 @@ +d . +d ./chrome +d ./chrome/linux-1108766 +d ./chrome/linux-1108766/chrome-linux +d ./chrome/linux-1108766/chrome-linux/ClearKeyCdm +d ./chrome/linux-1108766/chrome-linux/ClearKeyCdm/_platform_specific +d ./chrome/linux-1108766/chrome-linux/ClearKeyCdm/_platform_specific/linux_x64 +d ./chrome/linux-1108766/chrome-linux/MEIPreload +d ./chrome/linux-1108766/chrome-linux/locales +d ./chrome/linux-1108766/chrome-linux/resources +d ./chrome/linux-1108766/chrome-linux/resources/inspector_overlay +f ./chrome/linux-1108766/chrome-linux/ClearKeyCdm/_platform_specific/linux_x64/libclearkeycdm.so +f ./chrome/linux-1108766/chrome-linux/MEIPreload/manifest.json +f ./chrome/linux-1108766/chrome-linux/MEIPreload/preloaded_data.pb +f ./chrome/linux-1108766/chrome-linux/chrome +f ./chrome/linux-1108766/chrome-linux/chrome-wrapper +f ./chrome/linux-1108766/chrome-linux/chrome_100_percent.pak +f ./chrome/linux-1108766/chrome-linux/chrome_200_percent.pak +f ./chrome/linux-1108766/chrome-linux/chrome_crashpad_handler +f ./chrome/linux-1108766/chrome-linux/chrome_sandbox +f ./chrome/linux-1108766/chrome-linux/icudtl.dat +f ./chrome/linux-1108766/chrome-linux/libEGL.so +f ./chrome/linux-1108766/chrome-linux/libGLESv2.so +f ./chrome/linux-1108766/chrome-linux/libvk_swiftshader.so +f ./chrome/linux-1108766/chrome-linux/libvulkan.so.1 +f ./chrome/linux-1108766/chrome-linux/locales/af.pak +f ./chrome/linux-1108766/chrome-linux/locales/af.pak.info +f ./chrome/linux-1108766/chrome-linux/locales/am.pak +f ./chrome/linux-1108766/chrome-linux/locales/am.pak.info +f ./chrome/linux-1108766/chrome-linux/locales/ar-XB.pak +f ./chrome/linux-1108766/chrome-linux/locales/ar-XB.pak.info +f ./chrome/linux-1108766/chrome-linux/locales/ar.pak +f ./chrome/linux-1108766/chrome-linux/locales/ar.pak.info +f ./chrome/linux-1108766/chrome-linux/locales/bg.pak +f ./chrome/linux-1108766/chrome-linux/locales/bg.pak.info +f ./chrome/linux-1108766/chrome-linux/locales/bn.pak +f ./chrome/linux-1108766/chrome-linux/locales/bn.pak.info +f ./chrome/linux-1108766/chrome-linux/locales/ca.pak +f ./chrome/linux-1108766/chrome-linux/locales/ca.pak.info +f ./chrome/linux-1108766/chrome-linux/locales/cs.pak +f ./chrome/linux-1108766/chrome-linux/locales/cs.pak.info +f ./chrome/linux-1108766/chrome-linux/locales/da.pak +f ./chrome/linux-1108766/chrome-linux/locales/da.pak.info +f ./chrome/linux-1108766/chrome-linux/locales/de.pak +f ./chrome/linux-1108766/chrome-linux/locales/de.pak.info +f ./chrome/linux-1108766/chrome-linux/locales/el.pak +f ./chrome/linux-1108766/chrome-linux/locales/el.pak.info +f ./chrome/linux-1108766/chrome-linux/locales/en-GB.pak +f ./chrome/linux-1108766/chrome-linux/locales/en-GB.pak.info +f ./chrome/linux-1108766/chrome-linux/locales/en-US.pak +f ./chrome/linux-1108766/chrome-linux/locales/en-US.pak.info +f ./chrome/linux-1108766/chrome-linux/locales/en-XA.pak +f ./chrome/linux-1108766/chrome-linux/locales/en-XA.pak.info +f ./chrome/linux-1108766/chrome-linux/locales/es-419.pak +f ./chrome/linux-1108766/chrome-linux/locales/es-419.pak.info +f ./chrome/linux-1108766/chrome-linux/locales/es.pak +f ./chrome/linux-1108766/chrome-linux/locales/es.pak.info +f ./chrome/linux-1108766/chrome-linux/locales/et.pak +f ./chrome/linux-1108766/chrome-linux/locales/et.pak.info +f ./chrome/linux-1108766/chrome-linux/locales/fa.pak +f ./chrome/linux-1108766/chrome-linux/locales/fa.pak.info +f ./chrome/linux-1108766/chrome-linux/locales/fi.pak +f ./chrome/linux-1108766/chrome-linux/locales/fi.pak.info +f ./chrome/linux-1108766/chrome-linux/locales/fil.pak +f ./chrome/linux-1108766/chrome-linux/locales/fil.pak.info +f ./chrome/linux-1108766/chrome-linux/locales/fr.pak +f ./chrome/linux-1108766/chrome-linux/locales/fr.pak.info +f ./chrome/linux-1108766/chrome-linux/locales/gu.pak +f ./chrome/linux-1108766/chrome-linux/locales/gu.pak.info +f ./chrome/linux-1108766/chrome-linux/locales/he.pak +f ./chrome/linux-1108766/chrome-linux/locales/he.pak.info +f ./chrome/linux-1108766/chrome-linux/locales/hi.pak +f ./chrome/linux-1108766/chrome-linux/locales/hi.pak.info +f ./chrome/linux-1108766/chrome-linux/locales/hr.pak +f ./chrome/linux-1108766/chrome-linux/locales/hr.pak.info +f ./chrome/linux-1108766/chrome-linux/locales/hu.pak +f ./chrome/linux-1108766/chrome-linux/locales/hu.pak.info +f ./chrome/linux-1108766/chrome-linux/locales/id.pak +f ./chrome/linux-1108766/chrome-linux/locales/id.pak.info +f ./chrome/linux-1108766/chrome-linux/locales/it.pak +f ./chrome/linux-1108766/chrome-linux/locales/it.pak.info +f ./chrome/linux-1108766/chrome-linux/locales/ja.pak +f ./chrome/linux-1108766/chrome-linux/locales/ja.pak.info +f ./chrome/linux-1108766/chrome-linux/locales/kn.pak +f ./chrome/linux-1108766/chrome-linux/locales/kn.pak.info +f ./chrome/linux-1108766/chrome-linux/locales/ko.pak +f ./chrome/linux-1108766/chrome-linux/locales/ko.pak.info +f ./chrome/linux-1108766/chrome-linux/locales/lt.pak +f ./chrome/linux-1108766/chrome-linux/locales/lt.pak.info +f ./chrome/linux-1108766/chrome-linux/locales/lv.pak +f ./chrome/linux-1108766/chrome-linux/locales/lv.pak.info +f ./chrome/linux-1108766/chrome-linux/locales/ml.pak +f ./chrome/linux-1108766/chrome-linux/locales/ml.pak.info +f ./chrome/linux-1108766/chrome-linux/locales/mr.pak +f ./chrome/linux-1108766/chrome-linux/locales/mr.pak.info +f ./chrome/linux-1108766/chrome-linux/locales/ms.pak +f ./chrome/linux-1108766/chrome-linux/locales/ms.pak.info +f ./chrome/linux-1108766/chrome-linux/locales/nb.pak +f ./chrome/linux-1108766/chrome-linux/locales/nb.pak.info +f ./chrome/linux-1108766/chrome-linux/locales/nl.pak +f ./chrome/linux-1108766/chrome-linux/locales/nl.pak.info +f ./chrome/linux-1108766/chrome-linux/locales/pl.pak +f ./chrome/linux-1108766/chrome-linux/locales/pl.pak.info +f ./chrome/linux-1108766/chrome-linux/locales/pt-BR.pak +f ./chrome/linux-1108766/chrome-linux/locales/pt-BR.pak.info +f ./chrome/linux-1108766/chrome-linux/locales/pt-PT.pak +f ./chrome/linux-1108766/chrome-linux/locales/pt-PT.pak.info +f ./chrome/linux-1108766/chrome-linux/locales/ro.pak +f ./chrome/linux-1108766/chrome-linux/locales/ro.pak.info +f ./chrome/linux-1108766/chrome-linux/locales/ru.pak +f ./chrome/linux-1108766/chrome-linux/locales/ru.pak.info +f ./chrome/linux-1108766/chrome-linux/locales/sk.pak +f ./chrome/linux-1108766/chrome-linux/locales/sk.pak.info +f ./chrome/linux-1108766/chrome-linux/locales/sl.pak +f ./chrome/linux-1108766/chrome-linux/locales/sl.pak.info +f ./chrome/linux-1108766/chrome-linux/locales/sr.pak +f ./chrome/linux-1108766/chrome-linux/locales/sr.pak.info +f ./chrome/linux-1108766/chrome-linux/locales/sv.pak +f ./chrome/linux-1108766/chrome-linux/locales/sv.pak.info +f ./chrome/linux-1108766/chrome-linux/locales/sw.pak +f ./chrome/linux-1108766/chrome-linux/locales/sw.pak.info +f ./chrome/linux-1108766/chrome-linux/locales/ta.pak +f ./chrome/linux-1108766/chrome-linux/locales/ta.pak.info +f ./chrome/linux-1108766/chrome-linux/locales/te.pak +f ./chrome/linux-1108766/chrome-linux/locales/te.pak.info +f ./chrome/linux-1108766/chrome-linux/locales/th.pak +f ./chrome/linux-1108766/chrome-linux/locales/th.pak.info +f ./chrome/linux-1108766/chrome-linux/locales/tr.pak +f ./chrome/linux-1108766/chrome-linux/locales/tr.pak.info +f ./chrome/linux-1108766/chrome-linux/locales/uk.pak +f ./chrome/linux-1108766/chrome-linux/locales/uk.pak.info +f ./chrome/linux-1108766/chrome-linux/locales/ur.pak +f ./chrome/linux-1108766/chrome-linux/locales/ur.pak.info +f ./chrome/linux-1108766/chrome-linux/locales/vi.pak +f ./chrome/linux-1108766/chrome-linux/locales/vi.pak.info +f ./chrome/linux-1108766/chrome-linux/locales/zh-CN.pak +f ./chrome/linux-1108766/chrome-linux/locales/zh-CN.pak.info +f ./chrome/linux-1108766/chrome-linux/locales/zh-TW.pak +f ./chrome/linux-1108766/chrome-linux/locales/zh-TW.pak.info +f ./chrome/linux-1108766/chrome-linux/nacl_helper +f ./chrome/linux-1108766/chrome-linux/nacl_helper_bootstrap +f ./chrome/linux-1108766/chrome-linux/nacl_irt_x86_64.nexe +f ./chrome/linux-1108766/chrome-linux/product_logo_48.png +f ./chrome/linux-1108766/chrome-linux/resources.pak +f ./chrome/linux-1108766/chrome-linux/resources/inspector_overlay/inspector_overlay_resources.grd +f ./chrome/linux-1108766/chrome-linux/resources/inspector_overlay/main.js +f ./chrome/linux-1108766/chrome-linux/v8_context_snapshot.bin +f ./chrome/linux-1108766/chrome-linux/vk_swiftshader_icd.json +f ./chrome/linux-1108766/chrome-linux/xdg-mime +f ./chrome/linux-1108766/chrome-linux/xdg-settings +c0b731f77524b864f90bd6ba538880169a1ae2a8ece21a849a752e1c8809ef1d ./chrome/linux-1108766/chrome-linux/ClearKeyCdm/_platform_specific/linux_x64/libclearkeycdm.so +44844cf3dde6e80087ae0e6bf0d9326d7ef7d23326d24ac83af0850be26923d2 ./chrome/linux-1108766/chrome-linux/MEIPreload/manifest.json +c6070a157b4e28d16fbccbd233e93846ddb070c85e1a1bc64469b7a5f1424fad ./chrome/linux-1108766/chrome-linux/MEIPreload/preloaded_data.pb +9fb6757b5c61b28cbdd23a6c17a9acc61da856f812c02dae0b5156c0ee168a48 ./chrome/linux-1108766/chrome-linux/chrome +17b0432a26e628287789f1efa53056384a46b6265939e8bf819b1b6cf8472ada ./chrome/linux-1108766/chrome-linux/chrome-wrapper +7ab09e1609f2dd97cfbfa8988c71575d6abbe329d9c5a0d555b75504d5c4b980 ./chrome/linux-1108766/chrome-linux/chrome_100_percent.pak +2e3fe3b1da0944d3704d1593c73e6ce3e286f8733829138d60a49766b69c03ff ./chrome/linux-1108766/chrome-linux/chrome_200_percent.pak +f429e74163d634d37253d07730adc7651ce68fa16ebf794c1640193f94156392 ./chrome/linux-1108766/chrome-linux/chrome_crashpad_handler +c97779a42a6edebdb6e94ff4836b87444380e4fcb6df007f715cbc3b362231bb ./chrome/linux-1108766/chrome-linux/chrome_sandbox +e185ba581543fe286f1640a579d6f2219c181d95fa76e64f66c5a706f308860a ./chrome/linux-1108766/chrome-linux/icudtl.dat +1647b2a56c450d02dc735f334ee12a6e442e6b2f6dae5ff0f6e4b05efedc9465 ./chrome/linux-1108766/chrome-linux/libEGL.so +2af932a8f93999a0a32b87be6002fe83ca0e4579fc8ddba72a1a150323c38b58 ./chrome/linux-1108766/chrome-linux/libGLESv2.so +fd7894d32ccbaed94672b63e9ff76f970e995f193716271005c581a4a2f3c947 ./chrome/linux-1108766/chrome-linux/libvk_swiftshader.so +a94387cd9baf120b97fce4dcf8569c1a83e99ce85dd41978291b7fff25541aad ./chrome/linux-1108766/chrome-linux/libvulkan.so.1 +323c7a7ca326e7ef582b7034d10f4e81deaf630722cf43d5466b1deb419ec801 ./chrome/linux-1108766/chrome-linux/locales/af.pak +cb87a868d66f88bb0479c20536bf22ec93dd0bd775a88c57777d27dd9a128add ./chrome/linux-1108766/chrome-linux/locales/af.pak.info +f087f1f2bb83c48016b647799190f0f7f172df2fb5577ae9597ec17071caab90 ./chrome/linux-1108766/chrome-linux/locales/am.pak +cb87a868d66f88bb0479c20536bf22ec93dd0bd775a88c57777d27dd9a128add ./chrome/linux-1108766/chrome-linux/locales/am.pak.info +5dae1653f48a41abe2869245fd4fbc8a8e415dbfa125942706a6ff039e9a7f0c ./chrome/linux-1108766/chrome-linux/locales/ar-XB.pak +cb87a868d66f88bb0479c20536bf22ec93dd0bd775a88c57777d27dd9a128add ./chrome/linux-1108766/chrome-linux/locales/ar-XB.pak.info +5e22492cd14f8d739d2b14c7c5b0c10ed4f8585b5dddb402f36652d2c990b170 ./chrome/linux-1108766/chrome-linux/locales/ar.pak +cb87a868d66f88bb0479c20536bf22ec93dd0bd775a88c57777d27dd9a128add ./chrome/linux-1108766/chrome-linux/locales/ar.pak.info +dde3b5d59f81d593045518a40be3d9236d3f66a678f5823e342f3f379056858f ./chrome/linux-1108766/chrome-linux/locales/bg.pak +cb87a868d66f88bb0479c20536bf22ec93dd0bd775a88c57777d27dd9a128add ./chrome/linux-1108766/chrome-linux/locales/bg.pak.info +afb37a07a9fae9cf5d278f0426c047334f24a08c5f52aac76f75f3bfc1957bf6 ./chrome/linux-1108766/chrome-linux/locales/bn.pak +cb87a868d66f88bb0479c20536bf22ec93dd0bd775a88c57777d27dd9a128add ./chrome/linux-1108766/chrome-linux/locales/bn.pak.info +628c2f24f00ef37ba42af1ecf74136b5620a4f304a1bba9358129ba52f1124b2 ./chrome/linux-1108766/chrome-linux/locales/ca.pak +cb87a868d66f88bb0479c20536bf22ec93dd0bd775a88c57777d27dd9a128add ./chrome/linux-1108766/chrome-linux/locales/ca.pak.info +b5e95d9d94bfdef932f6fcaadf828b000e2f5a3b6d4a7c3a1b8ff1e55f2809aa ./chrome/linux-1108766/chrome-linux/locales/cs.pak +cb87a868d66f88bb0479c20536bf22ec93dd0bd775a88c57777d27dd9a128add ./chrome/linux-1108766/chrome-linux/locales/cs.pak.info +e5ea8dc83b39971db4bc62616d90c574311ad111ef38269befeff04c76b5a3bb ./chrome/linux-1108766/chrome-linux/locales/da.pak +cb87a868d66f88bb0479c20536bf22ec93dd0bd775a88c57777d27dd9a128add ./chrome/linux-1108766/chrome-linux/locales/da.pak.info +6983d893c1005c5a91718870b24c9444164427b901dd788435fcc78357a140b2 ./chrome/linux-1108766/chrome-linux/locales/de.pak +cb87a868d66f88bb0479c20536bf22ec93dd0bd775a88c57777d27dd9a128add ./chrome/linux-1108766/chrome-linux/locales/de.pak.info +e20982ee140a95bd8d440b645b593faab16ddc5a3c41fe40ca3b77243bfbacf8 ./chrome/linux-1108766/chrome-linux/locales/el.pak +cb87a868d66f88bb0479c20536bf22ec93dd0bd775a88c57777d27dd9a128add ./chrome/linux-1108766/chrome-linux/locales/el.pak.info +0918c68b0c5cd09f0ef1e9cc984fecb8ce79c0d0a509cdac2b77f9c0c29ce143 ./chrome/linux-1108766/chrome-linux/locales/en-GB.pak +cb87a868d66f88bb0479c20536bf22ec93dd0bd775a88c57777d27dd9a128add ./chrome/linux-1108766/chrome-linux/locales/en-GB.pak.info +b090180d5cd6ebfc25cb245c44156a4b945b52f581725b4184b0e0e5d78ff381 ./chrome/linux-1108766/chrome-linux/locales/en-US.pak +cb87a868d66f88bb0479c20536bf22ec93dd0bd775a88c57777d27dd9a128add ./chrome/linux-1108766/chrome-linux/locales/en-US.pak.info +7376d5f7cdbb0763efff84f2d8fa546be589577af5291cc629387425ca4e207e ./chrome/linux-1108766/chrome-linux/locales/en-XA.pak +cb87a868d66f88bb0479c20536bf22ec93dd0bd775a88c57777d27dd9a128add ./chrome/linux-1108766/chrome-linux/locales/en-XA.pak.info +290017770c41c26844032b8a2a1b97f7674653083411e23c12b72cc9ed8f49ea ./chrome/linux-1108766/chrome-linux/locales/es-419.pak +cb87a868d66f88bb0479c20536bf22ec93dd0bd775a88c57777d27dd9a128add ./chrome/linux-1108766/chrome-linux/locales/es-419.pak.info +a6d11beb2cf80c222d1a62133a6243776d4c379c18392794027ec0e64489af1b ./chrome/linux-1108766/chrome-linux/locales/es.pak +cb87a868d66f88bb0479c20536bf22ec93dd0bd775a88c57777d27dd9a128add ./chrome/linux-1108766/chrome-linux/locales/es.pak.info +42a74c9af1d7eee8c912c0b06b0439b0082ac453bf247ab044a2dccf8abbe7a5 ./chrome/linux-1108766/chrome-linux/locales/et.pak +cb87a868d66f88bb0479c20536bf22ec93dd0bd775a88c57777d27dd9a128add ./chrome/linux-1108766/chrome-linux/locales/et.pak.info +5ac624214ee7eded46e70368653f42b15ede5348621d0a57e7e615a56d8c9c05 ./chrome/linux-1108766/chrome-linux/locales/fa.pak +cb87a868d66f88bb0479c20536bf22ec93dd0bd775a88c57777d27dd9a128add ./chrome/linux-1108766/chrome-linux/locales/fa.pak.info +13c852eb058b37b736df5d87a81ed5370404cdb0c9a5f35d64d6d6ca03eaf013 ./chrome/linux-1108766/chrome-linux/locales/fi.pak +cb87a868d66f88bb0479c20536bf22ec93dd0bd775a88c57777d27dd9a128add ./chrome/linux-1108766/chrome-linux/locales/fi.pak.info +34572afd7489460793ea95b65e4bd68cb2f70a4873f74f2230fc71865bd514fa ./chrome/linux-1108766/chrome-linux/locales/fil.pak +cb87a868d66f88bb0479c20536bf22ec93dd0bd775a88c57777d27dd9a128add ./chrome/linux-1108766/chrome-linux/locales/fil.pak.info +4f3760d38ae95943ff8f2b79aa7c7d41b4143cad9ac81fe34080fc0aff66838b ./chrome/linux-1108766/chrome-linux/locales/fr.pak +cb87a868d66f88bb0479c20536bf22ec93dd0bd775a88c57777d27dd9a128add ./chrome/linux-1108766/chrome-linux/locales/fr.pak.info +01732e407c1ef2ffb678f2e1d8ba47e64b4d7583ba6fc2e3cfa4d40e89b325d9 ./chrome/linux-1108766/chrome-linux/locales/gu.pak +cb87a868d66f88bb0479c20536bf22ec93dd0bd775a88c57777d27dd9a128add ./chrome/linux-1108766/chrome-linux/locales/gu.pak.info +c191ec5dfc7eee295fe784e529bdf87878dec24d9705ae87a23a0a7be2daf742 ./chrome/linux-1108766/chrome-linux/locales/he.pak +cb87a868d66f88bb0479c20536bf22ec93dd0bd775a88c57777d27dd9a128add ./chrome/linux-1108766/chrome-linux/locales/he.pak.info +518729e2d63e92b36b146a6017eda1f36d5a1b93c4733509a8d8ed30ad8a2abe ./chrome/linux-1108766/chrome-linux/locales/hi.pak +cb87a868d66f88bb0479c20536bf22ec93dd0bd775a88c57777d27dd9a128add ./chrome/linux-1108766/chrome-linux/locales/hi.pak.info +7faa324f5deb95621f19401bab3b616bc8f4fc99b1864d8dd02b53f64ff13972 ./chrome/linux-1108766/chrome-linux/locales/hr.pak +cb87a868d66f88bb0479c20536bf22ec93dd0bd775a88c57777d27dd9a128add ./chrome/linux-1108766/chrome-linux/locales/hr.pak.info +eb99d8d7345da980371850ee26d74b1f98bebf5227a3e2b42fc55621c4ecb826 ./chrome/linux-1108766/chrome-linux/locales/hu.pak +cb87a868d66f88bb0479c20536bf22ec93dd0bd775a88c57777d27dd9a128add ./chrome/linux-1108766/chrome-linux/locales/hu.pak.info +cd5f6af3d83e9b49b66057b5de4defef8c925c2eea29171d336846c4f1b8c0e2 ./chrome/linux-1108766/chrome-linux/locales/id.pak +cb87a868d66f88bb0479c20536bf22ec93dd0bd775a88c57777d27dd9a128add ./chrome/linux-1108766/chrome-linux/locales/id.pak.info +aba599aef83877ec43647f17381b5ea677b115573f829fee19b5e95a106d82d2 ./chrome/linux-1108766/chrome-linux/locales/it.pak +cb87a868d66f88bb0479c20536bf22ec93dd0bd775a88c57777d27dd9a128add ./chrome/linux-1108766/chrome-linux/locales/it.pak.info +77a6f638d6e20c57335b1a419abbd5419a45b996f20a8b247f8323f95d8b8a7d ./chrome/linux-1108766/chrome-linux/locales/ja.pak +cb87a868d66f88bb0479c20536bf22ec93dd0bd775a88c57777d27dd9a128add ./chrome/linux-1108766/chrome-linux/locales/ja.pak.info +4251e497322a09eb9d11fcdeb79eacbd1d5ca738824b6efa41c20742bf05e041 ./chrome/linux-1108766/chrome-linux/locales/kn.pak +cb87a868d66f88bb0479c20536bf22ec93dd0bd775a88c57777d27dd9a128add ./chrome/linux-1108766/chrome-linux/locales/kn.pak.info +9cbf60872e4853badc62264af46de01f3700049f72d40412b805c1f2f0fc1c1d ./chrome/linux-1108766/chrome-linux/locales/ko.pak +cb87a868d66f88bb0479c20536bf22ec93dd0bd775a88c57777d27dd9a128add ./chrome/linux-1108766/chrome-linux/locales/ko.pak.info +a54f4825d04fef04066c88e3a1cfe2e014551cbfb19d4de3df71e38b118aec54 ./chrome/linux-1108766/chrome-linux/locales/lt.pak +cb87a868d66f88bb0479c20536bf22ec93dd0bd775a88c57777d27dd9a128add ./chrome/linux-1108766/chrome-linux/locales/lt.pak.info +f23f05077da21b8c811b60181389c15fddffd41327c36fe0c6e7a691fa9433d2 ./chrome/linux-1108766/chrome-linux/locales/lv.pak +cb87a868d66f88bb0479c20536bf22ec93dd0bd775a88c57777d27dd9a128add ./chrome/linux-1108766/chrome-linux/locales/lv.pak.info +6b1f3aad63a2152ae7343c6c20fe4f77f1eeb973ba957893feece942ba88e54a ./chrome/linux-1108766/chrome-linux/locales/ml.pak +cb87a868d66f88bb0479c20536bf22ec93dd0bd775a88c57777d27dd9a128add ./chrome/linux-1108766/chrome-linux/locales/ml.pak.info +c55c1e54776346c536abd431e289c467a01ba963a93ee2ff1845a150b207c744 ./chrome/linux-1108766/chrome-linux/locales/mr.pak +cb87a868d66f88bb0479c20536bf22ec93dd0bd775a88c57777d27dd9a128add ./chrome/linux-1108766/chrome-linux/locales/mr.pak.info +2a7f4e67717d25e9ccc74d16cfb1bab5dd18eaf16d8771d80fe38b226ec2ad5a ./chrome/linux-1108766/chrome-linux/locales/ms.pak +cb87a868d66f88bb0479c20536bf22ec93dd0bd775a88c57777d27dd9a128add ./chrome/linux-1108766/chrome-linux/locales/ms.pak.info +2971e09f96b9249b74ad8be35d777516fe1e49ae8ee8099935a40280105ff759 ./chrome/linux-1108766/chrome-linux/locales/nb.pak +cb87a868d66f88bb0479c20536bf22ec93dd0bd775a88c57777d27dd9a128add ./chrome/linux-1108766/chrome-linux/locales/nb.pak.info +c6df365597b3dd966ba84e6cd0d7fc4027a520ff85b36a47da9ed61e2b3a994a ./chrome/linux-1108766/chrome-linux/locales/nl.pak +cb87a868d66f88bb0479c20536bf22ec93dd0bd775a88c57777d27dd9a128add ./chrome/linux-1108766/chrome-linux/locales/nl.pak.info +ce50ebb59577f58a81444d074bb3602badf7df026c8a778333870647e9b54b52 ./chrome/linux-1108766/chrome-linux/locales/pl.pak +cb87a868d66f88bb0479c20536bf22ec93dd0bd775a88c57777d27dd9a128add ./chrome/linux-1108766/chrome-linux/locales/pl.pak.info +30d3eae85ccb68a513f49981c5dd8b67d4a42ea4c330016cbe30e37ee38e0a1d ./chrome/linux-1108766/chrome-linux/locales/pt-BR.pak +cb87a868d66f88bb0479c20536bf22ec93dd0bd775a88c57777d27dd9a128add ./chrome/linux-1108766/chrome-linux/locales/pt-BR.pak.info +bdcfdb657a204904faef4c2f627954dab097702e1dfd2e3e15971bb4d959c843 ./chrome/linux-1108766/chrome-linux/locales/pt-PT.pak +cb87a868d66f88bb0479c20536bf22ec93dd0bd775a88c57777d27dd9a128add ./chrome/linux-1108766/chrome-linux/locales/pt-PT.pak.info +d67a006273ce46f0c5ba724efe2c1bdf7943e88a9d88c7563421c31d14ba42a9 ./chrome/linux-1108766/chrome-linux/locales/ro.pak +cb87a868d66f88bb0479c20536bf22ec93dd0bd775a88c57777d27dd9a128add ./chrome/linux-1108766/chrome-linux/locales/ro.pak.info +723a919b66af7e2c428e4724376237fce747a47a9a0128810132f9f0b07d64ba ./chrome/linux-1108766/chrome-linux/locales/ru.pak +cb87a868d66f88bb0479c20536bf22ec93dd0bd775a88c57777d27dd9a128add ./chrome/linux-1108766/chrome-linux/locales/ru.pak.info +de81e6beb671141293605e396ba0992cc040ad8d2b4a9a019f60a2e40c954562 ./chrome/linux-1108766/chrome-linux/locales/sk.pak +cb87a868d66f88bb0479c20536bf22ec93dd0bd775a88c57777d27dd9a128add ./chrome/linux-1108766/chrome-linux/locales/sk.pak.info +16eee255672f7803ae9b282ce9d3aab6a29797a8ce1c787f949ab208a1f35352 ./chrome/linux-1108766/chrome-linux/locales/sl.pak +cb87a868d66f88bb0479c20536bf22ec93dd0bd775a88c57777d27dd9a128add ./chrome/linux-1108766/chrome-linux/locales/sl.pak.info +01b9a0a613d1d7504e90f9a523a90102d245f999ee8ff93d284c4186e6c2f463 ./chrome/linux-1108766/chrome-linux/locales/sr.pak +cb87a868d66f88bb0479c20536bf22ec93dd0bd775a88c57777d27dd9a128add ./chrome/linux-1108766/chrome-linux/locales/sr.pak.info +b8f738b3b5f334230f7ac8849a4eef3d77862e54202728e0c5608426b4add140 ./chrome/linux-1108766/chrome-linux/locales/sv.pak +cb87a868d66f88bb0479c20536bf22ec93dd0bd775a88c57777d27dd9a128add ./chrome/linux-1108766/chrome-linux/locales/sv.pak.info +4d7a3154fa0dbafefb96ef1fff1f60563a16de55deccfd11f2c5d3f733e29d0a ./chrome/linux-1108766/chrome-linux/locales/sw.pak +cb87a868d66f88bb0479c20536bf22ec93dd0bd775a88c57777d27dd9a128add ./chrome/linux-1108766/chrome-linux/locales/sw.pak.info +2dde4957d13101ade6f393a88332f8fc774f0565c73ece361f2f1ad12be235a2 ./chrome/linux-1108766/chrome-linux/locales/ta.pak +cb87a868d66f88bb0479c20536bf22ec93dd0bd775a88c57777d27dd9a128add ./chrome/linux-1108766/chrome-linux/locales/ta.pak.info +331489c0918c2b105b1112cdd620eba4af78bc80284730ed0c5691685be622fe ./chrome/linux-1108766/chrome-linux/locales/te.pak +cb87a868d66f88bb0479c20536bf22ec93dd0bd775a88c57777d27dd9a128add ./chrome/linux-1108766/chrome-linux/locales/te.pak.info +e04402b90dd5bbae71b68f85c834247c46eead0bcf0ac1996f4cc191f204d0c0 ./chrome/linux-1108766/chrome-linux/locales/th.pak +cb87a868d66f88bb0479c20536bf22ec93dd0bd775a88c57777d27dd9a128add ./chrome/linux-1108766/chrome-linux/locales/th.pak.info +fef2df11ed39c4f0f1687d49ff60b97d07513e03f9ff56ebe09ef088d7918473 ./chrome/linux-1108766/chrome-linux/locales/tr.pak +cb87a868d66f88bb0479c20536bf22ec93dd0bd775a88c57777d27dd9a128add ./chrome/linux-1108766/chrome-linux/locales/tr.pak.info +7a90a56c50fdc2a9add8c2399c18cc97f4af00d6ad067990f5758158987af6f2 ./chrome/linux-1108766/chrome-linux/locales/uk.pak +cb87a868d66f88bb0479c20536bf22ec93dd0bd775a88c57777d27dd9a128add ./chrome/linux-1108766/chrome-linux/locales/uk.pak.info +599d28b9b15b4a7e49a6747899310b919a5a05c428e7fb8e62fd67498ce46cdb ./chrome/linux-1108766/chrome-linux/locales/ur.pak +cb87a868d66f88bb0479c20536bf22ec93dd0bd775a88c57777d27dd9a128add ./chrome/linux-1108766/chrome-linux/locales/ur.pak.info +858a3ace87bff36cd4dc1532ad95626b061fa2af07dff40d896f266facc46ca7 ./chrome/linux-1108766/chrome-linux/locales/vi.pak +cb87a868d66f88bb0479c20536bf22ec93dd0bd775a88c57777d27dd9a128add ./chrome/linux-1108766/chrome-linux/locales/vi.pak.info +27d1f81e8302bd24bdaaf39ec9734e732f812644cf24cf282c86851bd1a4ee1a ./chrome/linux-1108766/chrome-linux/locales/zh-CN.pak +cb87a868d66f88bb0479c20536bf22ec93dd0bd775a88c57777d27dd9a128add ./chrome/linux-1108766/chrome-linux/locales/zh-CN.pak.info +72fa334fe28ab855115c32d9005c34620115960c0fc1e3709c2fb6e9e53d6976 ./chrome/linux-1108766/chrome-linux/locales/zh-TW.pak +cb87a868d66f88bb0479c20536bf22ec93dd0bd775a88c57777d27dd9a128add ./chrome/linux-1108766/chrome-linux/locales/zh-TW.pak.info +a20f2ff7a775859e2493c9fc4118080b5d31c2b28c4e79c08255f7d0e411a12b ./chrome/linux-1108766/chrome-linux/nacl_helper +2bb2fa72fab83a34dfa6690973177d3537a3a946300d66ede276c8b6de2cbec7 ./chrome/linux-1108766/chrome-linux/nacl_helper_bootstrap +17dcbaeb7337547b75884c44bd38377dccf36b444c6cab9a740cf0aef1b7abf5 ./chrome/linux-1108766/chrome-linux/nacl_irt_x86_64.nexe +21b2927bb47d305bbda718b9c08c6b51fbc0597454587978ea0046bc595393fb ./chrome/linux-1108766/chrome-linux/product_logo_48.png +64b6feb043f51c13b25b5b72290bf30ec48c4181ed9716b1b4bee76a7c0f55ba ./chrome/linux-1108766/chrome-linux/resources.pak +68760fc7002c3999d3b2ac6eb80febe28182134a0dd1d53829f660b3d96bc9ef ./chrome/linux-1108766/chrome-linux/resources/inspector_overlay/inspector_overlay_resources.grd +cd1db1a3000b7397c4232bcc2c05b9fa436c25f0c00aaab683d448b325588256 ./chrome/linux-1108766/chrome-linux/resources/inspector_overlay/main.js +46639e627710cf843878115db26b97da94678782a8a0d109a5c29ac04b6b0d9e ./chrome/linux-1108766/chrome-linux/v8_context_snapshot.bin +c9eef28b6b984fec220ef0abfecc40b502d46946706e47bfe97707027cb818bd ./chrome/linux-1108766/chrome-linux/vk_swiftshader_icd.json +ad2d5802bb3604daa849ce5ac86a81acf7548c8ffd10ab4ecae1d56a5f10d63b ./chrome/linux-1108766/chrome-linux/xdg-mime +442777b1f7d206c5ff661fea00dc1a8319df6256feccc54f186f6edacf235801 ./chrome/linux-1108766/chrome-linux/xdg-settings diff --git a/proposals/reindex.py b/bin/reindex index 72b1234..24e76eb 100755 --- a/proposals/reindex.py +++ b/bin/reindex @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 import sys if sys.version_info[0] < 3: @@ -98,6 +98,8 @@ def checkProposal(fn, fields): for f in [ 'Implemented-In', 'Target' ]: if f in fields: del fields[f] + fields['FilenameTruncated'] = os.path.splitext(fields['Filename'])[0] + def readProposals(): res = [] for fn in os.listdir(DIR): @@ -142,16 +144,21 @@ def writeTextIndexFile(proposals): if "Implemented-In" in prop: out.write(" [in %(Implemented-In)s]"%prop) out.write("\n") + + out.write("```\n") out.close() os.rename(TMPFILE_TXT, OUTFILE_TXT) def formatMarkdownEntry(prop, withStatus=False): if withStatus: - fmt = "* [`{Filename}`](/proposals/{Filename}): {Title} [{Status}]\n" + fmt = "* [`{Filename}`](/proposals/{Filename}): {Title} \\[{Status}\\]\n" else: fmt = "* [`{Filename}`](/proposals/{Filename}): {Title}\n" return fmt.format(**prop) +def formatSummaryEntry(prop): + return " - [`{FilenameTruncated}`](./{Filename}): {Title} ({Status})\n".format(**prop) + def writeMarkdownFile(prefix, format_inputs): template = prefix+"_template.md" output = prefix+".md" @@ -161,7 +168,7 @@ def writeMarkdownFile(prefix, format_inputs): f.write(content) def writeMarkdownIndexFiles(proposals): - markdown_files = [ "README", "BY_INDEX" ] + markdown_files = [ "BY_INDEX", "BY_STATUS", "SUMMARY" ] format_inputs = {} format_inputs['warning'] = "<!-- DO NOT EDIT THIS FILE -->" @@ -171,6 +178,11 @@ def writeMarkdownIndexFiles(proposals): entries.append(formatMarkdownEntry(prop, withStatus=True)) format_inputs["BY_INDEX"] = "".join(entries) + entries = [] + for prop in proposals: + entries.append(formatSummaryEntry(prop)) + format_inputs["SUMMARY_TABLE"] = "".join(entries) + for s in STATUSES: entries = [] for prop in proposals: @@ -191,6 +203,9 @@ def writeMarkdownIndexFiles(proposals): writeMarkdownFile(prefix, format_inputs) if __name__ == '__main__': + proposal_dir = os.path.join(os.path.dirname(sys.argv[0]), "..", "proposals") + os.chdir(proposal_dir) + proposals = readProposals() try: writeTextIndexFile(proposals) diff --git a/bin/via-cargo-install-in-ci b/bin/via-cargo-install-in-ci new file mode 100755 index 0000000..c104ff9 --- /dev/null +++ b/bin/via-cargo-install-in-ci @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# Bump this to "clear" the cache. +# Actually, it just causes us to ignore previous cached results. +cache_clear_token=2023-09-11 + +cache_dir=cache/"$CI_JOB_IMAGE","$cache_clear_token" + +mkdir -p "$cache_dir" + +for cmd in "$@"; do + cache="$cache_dir/$cmd" + + if cp "$cache" "$CARGO_HOME"/bin/; then continue; fi + + cargo install --locked "$cmd" + cp -v "$CARGO_HOME/bin/$cmd" "$cache" +done diff --git a/bin/via-yarn-install-in-ci b/bin/via-yarn-install-in-ci new file mode 100755 index 0000000..5aa9fcb --- /dev/null +++ b/bin/via-yarn-install-in-ci @@ -0,0 +1,59 @@ +#!/usr/bin/env bash + +set -euo pipefail + +set -x + +# usage: +# bin/via-yarn-install-in-ci COMMAND GIT-URL-FOR-PACKAGE COMMITID +# +# We demand a commitid to mitigate the security risks of npm + +command="$1" +git_url="$2" +commitid="$3" + +# Bump this to "clear" the cache. +# Actually, it just causes us to ignore previous cached results. +cache_clear_token=2023-11-09c + +cache_dir=cache/"$CI_JOB_IMAGE","$cache_clear_token","$command","$commitid" +local_dir="yarn-build,$command" + +mkdir -p "$local_dir" + +if test -d "$cache_dir"/; then + cp -al "$cache_dir"/. "$local_dir"/. + + type -p node || apt-get install -y nodejs +else + + if [ -z "${YARN-}" ]; then + type -p yarnpkg || apt-get install -y yarnpkg + YARN=yarnpkg + fi + + cd "$local_dir" + git clone "$git_url" "$command".git + cd "$command".git + git config advice.detachedHead false + git checkout "$commitid" + + : ----- invoke yarn to build "$command": running ----- + $YARN install --frozen-lockfile --non-interactive + : ----- invoke yarn to build "$command": complete ----- + + cd ../.. + ls -al "$PWD"/"$local_dir"/"$command".git/src/cli.js + + mkdir -p "$cache_dir" + cp -al "$local_dir"/. "$cache_dir"/. +fi + +# We abuse $CARGO_HOME/bin. +# The rust: CI images we're using have that on PATH. +# If one were to run this outside CI, putting a symlink +# to some NPM thing in ~/.cargo/bin doesn't seem terrible. + +ln -s "$PWD"/"$local_dir"/"$command".git/src/cli.js \ + "$CARGO_HOME"/bin/"$command" diff --git a/mdbook/proposals/book.toml b/mdbook/proposals/book.toml new file mode 100644 index 0000000..e466b1b --- /dev/null +++ b/mdbook/proposals/book.toml @@ -0,0 +1,290 @@ +[book] +authors = ["The Tor Project"] +language = "en" +multilingual = false +src = "../../proposals" +title = "Tor design proposals" +use-default-preprocessors = false + +[build] +build-dir = "../../build/proposals" + +[preprocessor.links] + +# [preprocessor.pagetoc] + +[output.html] +theme = "../theme" +# additional-css = ["theme/pagetoc.css"] +# additional-js = ["theme/pagetoc.js"] +no-section-label = true + +[output.linkcheck] +warning-policy = "error" +# We effectively disable checking link destinations by excluding +# all links (`.*`). +# +# We're only using linkcheck to validate that reference-style links have *some* +# definition in the markdown, ensuring that *some* link will be generated. We +# validate that those links are correct using a separate tool (`bin/check_links`) +exclude = [ '.*' ] + +[output.html.search] +enable = false + +[output.html.redirect] + +# BEGIN AUTO-GENERATED REDIRECTS +"/000.html" = "./000-index.txt" +"/001.html" = "./001-process.txt" +"/098.html" = "./098-todo.txt" +"/099.html" = "./099-misc.txt" +"/100.html" = "./100-tor-spec-udp.txt" +"/101.html" = "./101-dir-voting.txt" +"/102.html" = "./102-drop-opt.txt" +"/103.html" = "./103-multilevel-keys.txt" +"/104.html" = "./104-short-descriptors.txt" +"/105.html" = "./105-handshake-revision.txt" +"/106.html" = "./106-less-tls-constraint.txt" +"/107.html" = "./107-uptime-sanity-checking.txt" +"/108.html" = "./108-mtbf-based-stability.txt" +"/109.html" = "./109-no-sharing-ips.txt" +"/110.html" = "./110-avoid-infinite-circuits.txt" +"/111.html" = "./111-local-traffic-priority.txt" +"/112.html" = "./112-bring-back-pathlencoinweight.txt" +"/113.html" = "./113-fast-authority-interface.txt" +"/114.html" = "./114-distributed-storage.txt" +"/115.html" = "./115-two-hop-paths.txt" +"/116.html" = "./116-two-hop-paths-from-guard.txt" +"/117.html" = "./117-ipv6-exits.txt" +"/118.html" = "./118-multiple-orports.txt" +"/119.html" = "./119-controlport-auth.txt" +"/120.html" = "./120-shutdown-descriptors.txt" +"/121.html" = "./121-hidden-service-authentication.txt" +"/122.html" = "./122-unnamed-flag.txt" +"/123.html" = "./123-autonaming.txt" +"/124.html" = "./124-tls-certificates.txt" +"/125.html" = "./125-bridges.txt" +"/126.html" = "./126-geoip-reporting.txt" +"/127.html" = "./127-dirport-mirrors-downloads.txt" +"/128.html" = "./128-bridge-families.txt" +"/129.html" = "./129-reject-plaintext-ports.txt" +"/130.html" = "./130-v2-conn-protocol.txt" +"/131.html" = "./131-verify-tor-usage.txt" +"/132.html" = "./132-browser-check-tor-service.txt" +"/133.html" = "./133-unreachable-ors.txt" +"/134.html" = "./134-robust-voting.txt" +"/135.html" = "./135-private-tor-networks.txt" +"/136.html" = "./136-legacy-keys.txt" +"/137.html" = "./137-bootstrap-phases.txt" +"/138.html" = "./138-remove-down-routers-from-consensus.txt" +"/139.html" = "./139-conditional-consensus-download.txt" +"/140.html" = "./140-consensus-diffs.txt" +"/141.html" = "./141-jit-sd-downloads.txt" +"/142.html" = "./142-combine-intro-and-rend-points.txt" +"/143.html" = "./143-distributed-storage-improvements.txt" +"/144.html" = "./144-enforce-distinct-providers.txt" +"/145.html" = "./145-newguard-flag.txt" +"/146.html" = "./146-long-term-stability.txt" +"/147.html" = "./147-prevoting-opinions.txt" +"/148.html" = "./148-uniform-client-end-reason.txt" +"/149.html" = "./149-using-netinfo-data.txt" +"/150.html" = "./150-exclude-exit-nodes.txt" +"/151.html" = "./151-path-selection-improvements.txt" +"/152.html" = "./152-single-hop-circuits.txt" +"/153.html" = "./153-automatic-software-update-protocol.txt" +"/154.html" = "./154-automatic-updates.txt" +"/155.html" = "./155-four-hidden-service-improvements.txt" +"/156.html" = "./156-tracking-blocked-ports.txt" +"/157.html" = "./157-specific-cert-download.txt" +"/158.html" = "./158-microdescriptors.txt" +"/159.html" = "./159-exit-scanning.txt" +"/160.html" = "./160-bandwidth-offset.txt" +"/161.html" = "./161-computing-bandwidth-adjustments.txt" +"/162.html" = "./162-consensus-flavors.txt" +"/163.html" = "./163-detecting-clients.txt" +"/164.html" = "./164-reporting-server-status.txt" +"/165.html" = "./165-simple-robust-voting.txt" +"/166.html" = "./166-statistics-extra-info-docs.txt" +"/167.html" = "./167-params-in-consensus.txt" +"/168.html" = "./168-reduce-circwindow.txt" +"/169.html" = "./169-eliminating-renegotiation.txt" +"/170.html" = "./170-user-path-config.txt" +"/171.html" = "./171-separate-streams.txt" +"/172.html" = "./172-circ-getinfo-option.txt" +"/173.html" = "./173-getinfo-option-expansion.txt" +"/174.html" = "./174-optimistic-data-server.txt" +"/175.html" = "./175-automatic-node-promotion.txt" +"/176.html" = "./176-revising-handshake.txt" +"/177.html" = "./177-flag-abstention.txt" +"/178.html" = "./178-param-voting.txt" +"/179.html" = "./179-TLS-cert-and-parameter-normalization.txt" +"/180.html" = "./180-pluggable-transport.txt" +"/181.html" = "./181-optimistic-data-client.txt" +"/182.html" = "./182-creditbucket.txt" +"/183.html" = "./183-refillintervals.txt" +"/184.html" = "./184-v3-link-protocol.txt" +"/185.html" = "./185-dir-without-dirport.txt" +"/186.html" = "./186-multiple-orports.txt" +"/187.html" = "./187-allow-client-auth.txt" +"/188.html" = "./188-bridge-guards.txt" +"/189.html" = "./189-authorize-cell.txt" +"/190.html" = "./190-shared-secret-bridge-authorization.txt" +"/191.html" = "./191-mitm-bridge-detection-resistance.txt" +"/192.html" = "./192-store-bridge-information.txt" +"/193.html" = "./193-safe-cookie-authentication.txt" +"/194.html" = "./194-mnemonic-urls.txt" +"/195.html" = "./195-TLS-normalization-for-024.txt" +"/196.html" = "./196-transport-control-ports.txt" +"/197.html" = "./197-postmessage-ipc.txt" +"/198.html" = "./198-restore-clienthello-semantics.txt" +"/199.html" = "./199-bridgefinder-integration.txt" +"/200.html" = "./200-new-create-and-extend-cells.txt" +"/201.html" = "./201-bridge-v3-reqs-stats.txt" +"/202.html" = "./202-improved-relay-crypto.txt" +"/203.html" = "./203-https-frontend.txt" +"/204.html" = "./204-hidserv-subdomains.txt" +"/205.html" = "./205-local-dnscache.txt" +"/206.html" = "./206-directory-sources.txt" +"/207.html" = "./207-directory-guards.txt" +"/208.html" = "./208-ipv6-exits-redux.txt" +"/209.html" = "./209-path-bias-tuning.txt" +"/210.html" = "./210-faster-headless-consensus-bootstrap.txt" +"/211.html" = "./211-mapaddress-tor-status.txt" +"/212.html" = "./212-using-old-consensus.txt" +"/213.html" = "./213-remove-stream-sendmes.txt" +"/214.html" = "./214-longer-circids.txt" +"/215.html" = "./215-update-min-consensus-ver.txt" +"/216.html" = "./216-ntor-handshake.txt" +"/217.html" = "./217-ext-orport-auth.txt" +"/218.html" = "./218-usage-controller-events.txt" +"/219.html" = "./219-expanded-dns.txt" +"/220.html" = "./220-ecc-id-keys.txt" +"/221.html" = "./221-stop-using-create-fast.txt" +"/222.html" = "./222-remove-client-timestamps.txt" +"/223.html" = "./223-ace-handshake.txt" +"/224.html" = "./224-rend-spec-ng.txt" +"/225.html" = "./225-strawman-shared-rand.txt" +"/226.html" = "./226-bridgedb-database-improvements.txt" +"/227.html" = "./227-vote-on-package-fingerprints.txt" +"/228.html" = "./228-cross-certification-onionkeys.txt" +"/229.html" = "./229-further-socks5-extensions.txt" +"/230.html" = "./230-rsa1024-relay-id-migration.txt" +"/231.html" = "./231-migrate-authority-rsa1024-ids.txt" +"/232.html" = "./232-pluggable-transports-through-proxy.txt" +"/233.html" = "./233-quicken-tor2web-mode.txt" +"/234.html" = "./234-remittance-addresses.txt" +"/235.html" = "./235-kill-named-flag.txt" +"/236.html" = "./236-single-guard-node.txt" +"/237.html" = "./237-directory-servers-for-all.txt" +"/238.html" = "./238-hs-relay-stats.txt" +"/239.html" = "./239-consensus-hash-chaining.txt" +"/240.html" = "./240-auth-cert-revocation.txt" +"/241.html" = "./241-suspicious-guard-turnover.txt" +"/242.html" = "./242-better-families.txt" +"/243.html" = "./243-hsdir-flag-need-stable.txt" +"/244.html" = "./244-use-rfc5705-for-tls-binding.txt" +"/245.html" = "./245-tap-out.txt" +"/246.html" = "./246-merge-hsdir-and-intro.txt" +"/247.html" = "./247-hs-guard-discovery.txt" +"/248.html" = "./248-removing-rsa-identities.txt" +"/249.html" = "./249-large-create-cells.txt" +"/250.html" = "./250-commit-reveal-consensus.txt" +"/251.html" = "./251-netflow-padding.txt" +"/252.html" = "./252-single-onion.txt" +"/253.html" = "./253-oob-hmac.txt" +"/254.html" = "./254-padding-negotiation.txt" +"/255.html" = "./255-hs-load-balancing.txt" +"/256.html" = "./256-key-revocation.txt" +"/257.html" = "./257-hiding-authorities.txt" +"/258.html" = "./258-dirauth-dos.txt" +"/259.html" = "./259-guard-selection.txt" +"/260.html" = "./260-rend-single-onion.txt" +"/261.html" = "./261-aez-crypto.txt" +"/262.html" = "./262-rekey-circuits.txt" +"/263.html" = "./263-ntru-for-pq-handshake.txt" +"/264.html" = "./264-subprotocol-versions.txt" +"/265.html" = "./265-load-balancing-with-overhead.txt" +"/266.html" = "./266-removing-current-obsolete-clients.txt" +"/267.html" = "./267-tor-consensus-transparency.txt" +"/268.html" = "./268-guard-selection.txt" +"/269.html" = "./269-hybrid-handshake.txt" +"/270.html" = "./270-newhope-hybrid-handshake.txt" +"/271.html" = "./271-another-guard-selection.txt" +"/272.html" = "./272-valid-and-running-by-default.txt" +"/273.html" = "./273-exit-relay-pinning.txt" +"/274.html" = "./274-rotate-onion-keys-less.txt" +"/275.html" = "./275-md-published-time-is-silly.txt" +"/276.html" = "./276-lower-bw-granularity.txt" +"/277.html" = "./277-detect-id-sharing.txt" +"/278.html" = "./278-directory-compression-scheme-negotiation.txt" +"/279.html" = "./279-naming-layer-api.txt" +"/280.html" = "./280-privcount-in-tor.txt" +"/281.html" = "./281-bulk-md-download.txt" +"/282.html" = "./282-remove-named-from-consensus.txt" +"/283.html" = "./283-ipv6-in-micro-consensus.txt" +"/284.html" = "./284-hsv3-control-port.txt" +"/285.html" = "./285-utf-8.txt" +"/286.html" = "./286-hibernation-api.txt" +"/287.html" = "./287-reduce-lifetime.txt" +"/288.html" = "./288-privcount-with-shamir.txt" +"/289.html" = "./289-authenticated-sendmes.txt" +"/290.html" = "./290-deprecate-consensus-methods.txt" +"/291.html" = "./291-two-guard-nodes.txt" +"/292.html" = "./292-mesh-vanguards.txt" +"/293.html" = "./293-know-when-to-publish.txt" +"/294.html" = "./294-tls-1.3.txt" +"/295.html" = "./295-relay-crypto-with-adl.txt" +"/296.html" = "./296-expose-bandwidth-files.txt" +"/297.html" = "./297-safer-protover-shutdowns.txt" +"/298.html" = "./298-canonical-families.txt" +"/299.html" = "./299-ip-failure-count.txt" +"/300.html" = "./300-walking-onions.txt" +"/301.html" = "./301-dont-vote-on-package-fingerprints.txt" +"/302.html" = "./302-padding-machines-for-onion-clients.txt" +"/303.html" = "./303-protover-removal-policy.txt" +"/304.html" = "./304-socks5-extending-hs-error-codes.txt" +"/305.html" = "./305-establish-intro-dos-defense-extention.txt" +"/306.html" = "./306-ipv6-happy-eyeballs.txt" +"/307.html" = "./307-onionbalance-v3.txt" +"/308.html" = "./308-counter-galois-onion.txt" +"/309.html" = "./309-optimistic-socks-in-tor.txt" +"/310.html" = "./310-bandaid-on-guard-selection.txt" +"/311.html" = "./311-relay-ipv6-reachability.txt" +"/312.html" = "./312-relay-auto-ipv6-addr.txt" +"/313.html" = "./313-relay-ipv6-stats.txt" +"/314.html" = "./314-allow-markdown-proposals.html" +"/315.html" = "./315-update-dir-required-fields.txt" +"/316.html" = "./316-flashflow.html" +"/317.html" = "./317-secure-dns-name-resolution.txt" +"/318.html" = "./318-limit-protovers.html" +"/319.html" = "./319-wide-everything.html" +"/320.html" = "./320-tap-out-again.html" +"/321.html" = "./321-happy-families.html" +"/322.html" = "./322-dirport-linkspec.html" +"/323.html" = "./323-walking-onions-full.html" +"/324.html" = "./324-rtt-congestion-control.txt" +"/325.html" = "./325-packed-relay-cells.html" +"/326.html" = "./326-tor-relay-well-known-uri-rfc8615.html" +"/327.html" = "./327-pow-over-intro.txt" +"/328.html" = "./328-relay-overload-report.html" +"/329.html" = "./329-traffic-splitting.txt" +"/330.html" = "./330-authority-contact.html" +"/331.html" = "./331-res-tokens-for-anti-dos.html" +"/332.html" = "./332-ntor-v3-with-extra-data.html" +"/333.html" = "./333-vanguards-lite.html" +"/334.html" = "./334-middle-only-flag.txt" +"/335.html" = "./335-middle-only-redux.html" +"/336.html" = "./336-randomize-guard-retries.html" +"/337.html" = "./337-simpler-guard-usability.html" +"/338.html" = "./338-netinfo-y2038.html" +"/339.html" = "./339-udp-over-tor.html" +"/340.html" = "./340-packed-and-fragmented.html" +"/341.html" = "./341-better-oos.html" +"/342.html" = "./342-decouple-hs-interval.html" +"/343.html" = "./343-rend-caa.txt" +"/344.html" = "./344-protocol-info-leaks.txt" +"/345.html" = "./345-specs-in-mdbook.html" +"/346.html" = "./346-protovers-again.html" +"/347.html" = "./347-domain-separation.html" +# END AUTO-GENERATED REDIRECTS diff --git a/mdbook/proposals/mermaid-init.js b/mdbook/proposals/mermaid-init.js new file mode 100644 index 0000000..313a6e8 --- /dev/null +++ b/mdbook/proposals/mermaid-init.js @@ -0,0 +1 @@ +mermaid.initialize({startOnLoad:true}); diff --git a/mdbook/spec/book.toml b/mdbook/spec/book.toml new file mode 100644 index 0000000..16668c9 --- /dev/null +++ b/mdbook/spec/book.toml @@ -0,0 +1,37 @@ +[book] +authors = ["The Tor Project"] +language = "en" +multilingual = false +src = "../../spec" +title = "Tor Specifications" + +[build] +build-dir = "../../build/spec" + +[preprocessor] + +[output.html] +theme = "../theme" + +[output.html.redirect] + +# BEGIN AUTO-GENERATED REDIRECTS +"/collector-protocol.html" = "https://gitlab.torproject.org/tpo/network-health/metrics/collector/-/blob/master/src/main/resources/docs/PROTOCOL?ref_type=heads" +"/gettor-spec.html" = "https://gitlab.torproject.org/tpo/core/torspec/-/raw/main/attic/text_formats/gettor-spec.txt?ref_type=heads" +"/rend-spec-v2.html" = "https://gitlab.torproject.org/tpo/core/torspec/-/blob/main/attic/rend-spec-v2.txt?ref_type=heads" +"/rend-spec-v3.html" = "https://spec.torproject.org/rend-spec" +"/tor-fw-helper-spec.html" = "https://gitlab.torproject.org/tpo/core/torspec/-/blob/main/attic/tor-fw-helper-spec.txt?ref_type=heads" +"/torbrowser-design.html" = "https://2019.www.torproject.org/projects/torbrowser/design/" +"/tor-design.html" = "https://svn.torproject.org/svn/projects/design-paper/tor-design.pdf" +"/walking-onions.html" = "https://spec.torproject.org/proposals/323-walking-onions-full.html" +# END AUTO-GENERATED REDIRECTS + +[output.linkcheck] +warning-policy = "error" +# We effectively disable checking link destinations by excluding +# all links (`.*`). +# +# We're only using linkcheck to validate that reference-style links have *some* +# definition in the markdown, ensuring that *some* link will be generated. We +# validate that those links are correct using a separate tool (`bin/check_links`) +exclude = [ '.*' ] diff --git a/mdbook/spec/mermaid-init.js b/mdbook/spec/mermaid-init.js new file mode 100644 index 0000000..313a6e8 --- /dev/null +++ b/mdbook/spec/mermaid-init.js @@ -0,0 +1 @@ +mermaid.initialize({startOnLoad:true}); diff --git a/mdbook/spec/spec-redirects.yaml b/mdbook/spec/spec-redirects.yaml new file mode 100644 index 0000000..456d464 --- /dev/null +++ b/mdbook/spec/spec-redirects.yaml @@ -0,0 +1,100 @@ +# yamllint disable rule:line-length +--- +# NOTE: that this file is for permalink redirects only. If you want a +# temporary redirect, or a redirect that is not a documentated +# permanent link, do not add it here! + +# A permalink is marked as "implicit" below if it already has the +# right URL, and no further redirect HTML needs to be created in the +# mdbook configuration. + +redirects: + 'address-spec': + target: https://spec.torproject.org/address-spec + description: Special Hostnames in Tor + implicit: yes + 'bandwidth-file-spec': + target: https://spec.torproject.org/bandwidth-file-spec + description: Directory Authority Bandwidth File spec + implicit: yes + 'bridgedb-spec': + target: https://spec.torproject.org/bridgedb-spec + description: BridgeDB specification + implicit: yes + 'cert-spec': + target: https://spec.torproject.org/cert-spec + description: Ed25519 certificates in Tor + implicit: yes + 'collector-protocol': + target: https://gitlab.torproject.org/tpo/network-health/metrics/collector/-/blob/master/src/main/resources/docs/PROTOCOL?ref_type=heads + description: Protocol of CollecTor's File Structure + 'control-spec': + target: https://spec.torproject.org/control-spec + description: Tor control protocol, version 1 + implicit: yes + 'dir-spec': + target: https://spec.torproject.org/dir-spec + description: Tor directory protocol, version 3 + implicit: yes + 'dir-list-spec': + target: https://spec.torproject.org/dir-list-spec + description: Tor Directory List file format + implicit: yes + 'ext-orport-spec': + target: https://spec.torproject.org/ext-orport-spec + description: Extended ORPort for pluggable transports + implicit: yes + 'gettor-spec': + target: https://gitlab.torproject.org/tpo/core/torspec/-/raw/main/attic/text_formats/gettor-spec.txt?ref_type=heads + description: GetTor specification + 'padding-spec': + target: https://spec.torproject.org/padding-spec + description: Tor Padding Specification + implicit: yes + 'path-spec': + target: https://spec.torproject.org/path-spec + description: Tor Path Specification + implicit: yes + 'pt-spec': + target: https://spec.torproject.org/pt-spec + description: Tor Pluggable Transport Specification, version 1 + implicit: yes + 'rend-spec': + target: https://spec.torproject.org/rend-spec + description: Tor Onion Service Rendezvous Specification, latest version + implicit: yes + 'rend-spec-v2': + target: https://gitlab.torproject.org/tpo/core/torspec/-/blob/main/attic/rend-spec-v2.txt?ref_type=heads + description: Tor Onion Service Rendezvous Specification, Version 2 (Obsolete) + 'rend-spec-v3': + target: https://spec.torproject.org/rend-spec + description: Tor Onion Service Rendezvous Specification, Version 3 (Latest) + implicit: no + 'socks-extensions': + target: https://spec.torproject.org/socks-extensions + description: "Tor's extensions to the SOCKS protocol" + implicit: yes + 'srv-spec': + target: https://spec.torproject.org/srv-spec + description: Tor Shared Random Subsystem Specification + implicit: yes + 'tor-fw-helper-spec': + target: https://gitlab.torproject.org/tpo/core/torspec/-/blob/main/attic/tor-fw-helper-spec.txt?ref_type=heads + description: "Tor's (little) Firewall Helper specification" + 'tor-spec': + target: https://spec.torproject.org/tor-spec + description: Tor Protocol Specification + implicit: yes + 'torbrowser-design': + target: https://2019.www.torproject.org/projects/torbrowser/design/ + description: The Design and Implementation of the Tor Browser + 'version-spec': + target: https://spec.torproject.org/version-spec + description: How Tor Version Numbers Work + implicit: yes + 'tor-design': + target: https://svn.torproject.org/svn/projects/design-paper/tor-design.pdf + description: "Tor: The Second-Generation Onion Router" + 'walking-onions': + target: https://spec.torproject.org/proposals/323-walking-onions-full.html + description: Walking Onions specifications diff --git a/mdbook/theme/favicon.png b/mdbook/theme/favicon.png Binary files differnew file mode 100644 index 0000000..514eb90 --- /dev/null +++ b/mdbook/theme/favicon.png diff --git a/proposals/000-index.txt b/proposals/000-index.txt index 48765d0..8c9668f 100644 --- a/proposals/000-index.txt +++ b/proposals/000-index.txt @@ -1,3 +1,4 @@ +``` Filename: 000-index.txt Title: Index of Tor Proposals Author: Nick Mathewson @@ -212,7 +213,7 @@ Proposals by number: 289 Authenticating sendme cells to mitigate bandwidth attacks [CLOSED] 290 Continuously update consensus methods [META] 291 The move to two guard nodes [FINISHED] -292 Mesh-based vanguards [ACCEPTED] +292 Mesh-based vanguards [CLOSED] 293 Other ways for relays to know when to publish [CLOSED] 294 TLS 1.3 Migration [DRAFT] 295 Using ADL for relay cryptography (solving the crypto-tagging attack) [OPEN] @@ -247,13 +248,13 @@ Proposals by number: 324 RTT-based Congestion Control for Tor [FINISHED] 325 Packed relay cells: saving space on small commands [OBSOLETE] 326 The "tor-relay" Well-Known Resource Identifier [OPEN] -327 A First Take at PoW Over Introduction Circuits [FINISHED] +327 A First Take at PoW Over Introduction Circuits [CLOSED] 328 Make Relays Report When They Are Overloaded [CLOSED] 329 Overcoming Tor's Bottlenecks with Traffic Splitting [FINISHED] 330 Modernizing authority contact entries [OPEN] 331 Res tokens: Anonymous Credentials for Onion Service DoS Resilience [DRAFT] 332 Ntor protocol with extra data, version 3 [CLOSED] -333 Vanguards lite [FINISHED] +333 Vanguards lite [CLOSED] 334 A Directory Authority Flag To Mark Relays As Middle-only [SUPERSEDED] 335 An authority-only design for MiddleOnly [CLOSED] 336 Randomized schedule for guard retries [CLOSED] @@ -265,7 +266,10 @@ Proposals by number: 342 Decoupling hs_interval and SRV lifetime [DRAFT] 343 CAA Extensions for the Tor Rendezvous Specification [OPEN] 344 Prioritizing Protocol Information Leaks in Tor [OPEN] -345 Migrating the tor specifications to mdbook [OPEN] +345 Migrating the tor specifications to mdbook [CLOSED] +346 Clarifying and extending the use of protocol versioning [OPEN] +347 Domain separation for certificate signing keys [OPEN] +348 UDP Application Support in Tor [OPEN] Proposals by status: @@ -302,11 +306,12 @@ Proposals by status: 341 A better algorithm for out-of-sockets eviction 343 CAA Extensions for the Tor Rendezvous Specification 344 Prioritizing Protocol Information Leaks in Tor - 345 Migrating the tor specifications to mdbook + 346 Clarifying and extending the use of protocol versioning + 347 Domain separation for certificate signing keys + 348 UDP Application Support in Tor ACCEPTED: 282 Remove "Named" and "Unnamed" handling from consensus voting [for arti-dirauth] 285 Directory documents should be standardized as UTF-8 [for arti-dirauth] - 292 Mesh-based vanguards 311 Tor Relay IPv6 Reachability 312 Tor Relay Automatic IPv6 Address Discovery 313 Tor Relay IPv6 Statistics @@ -323,9 +328,7 @@ Proposals by status: 260 Rendezvous Single Onion Services [in 0.2.9.3-alpha] 291 The move to two guard nodes 324 RTT-based Congestion Control for Tor - 327 A First Take at PoW Over Introduction Circuits 329 Overcoming Tor's Bottlenecks with Traffic Splitting - 333 Vanguards lite [in 0.4.7.1-alpha] CLOSED: 101 Voting on the Tor Directory System [in 0.2.0.x] 102 Dropping "opt" from the directory format [in 0.2.0.x] @@ -416,6 +419,7 @@ Proposals by status: 283 Move IPv6 ORPorts from microdescriptors to the microdesc consensus [for 0.3.3.x] [in 0.3.3.1-alpha] 284 Hidden Service v3 Control Port 289 Authenticating sendme cells to mitigate bandwidth attacks [in 0.4.1.1-alpha] + 292 Mesh-based vanguards 293 Other ways for relays to know when to publish [for 0.3.5] [in 0.4.0.1-alpha] 296 Have Directory Authorities expose raw bandwidth list files [in 0.4.0.1-alpha] 297 Relaxing the protover-based shutdown rules [for 0.3.5.x] [in 0.4.0.x] @@ -428,11 +432,14 @@ Proposals by status: 314 Allow Markdown for proposal format 315 Updating the list of fields required in directory documents [in 0.4.5.1-alpha] 318 Limit protover values to 0-63 [in 0.4.5.1-alpha] + 327 A First Take at PoW Over Introduction Circuits 328 Make Relays Report When They Are Overloaded 332 Ntor protocol with extra data, version 3 + 333 Vanguards lite [in 0.4.7.1-alpha] 335 An authority-only design for MiddleOnly [in 0.4.7.2-alpha] 336 Randomized schedule for guard retries 337 A simpler way to decide, "Is this guard usable?" + 345 Migrating the tor specifications to mdbook SUPERSEDED: 112 Bring Back Pathlen Coin Weight 113 Simplifying directory authority administration @@ -533,3 +540,4 @@ Proposals by status: INFORMATIONAL: 159 Exit Scanning 300 Walking Onions: Scaling and Saving Bandwidth +``` diff --git a/proposals/001-process.txt b/proposals/001-process.txt index 7211c3b..51526bb 100644 --- a/proposals/001-process.txt +++ b/proposals/001-process.txt @@ -1,3 +1,4 @@ +``` Filename: 001-process.txt Title: The Tor Proposal Process Author: Nick Mathewson @@ -207,3 +208,4 @@ Proposal numbering: Numbers 000-099 are reserved for special and meta-proposals. 100 and up are used for actual proposals. Numbers aren't recycled. +``` diff --git a/proposals/098-todo.txt b/proposals/098-todo.txt index 0acb464..91202c9 100644 --- a/proposals/098-todo.txt +++ b/proposals/098-todo.txt @@ -1,3 +1,4 @@ +``` Filename: 098-todo.txt Title: Proposals that should be written Author: Nick Mathewson, Roger Dingledine @@ -107,3 +108,4 @@ B.2. ... and that we have no idea how to do. Let onion keys be not just RSA but maybe DH too, for Paul's reply onion design. +``` diff --git a/proposals/099-misc.txt b/proposals/099-misc.txt index ef34c80..bc44540 100644 --- a/proposals/099-misc.txt +++ b/proposals/099-misc.txt @@ -1,3 +1,4 @@ +``` Filename: 099-misc.txt Title: Miscellaneous proposals Author: Various @@ -29,3 +30,4 @@ Proposals -- Nick Mathewson +``` diff --git a/proposals/100-tor-spec-udp.txt b/proposals/100-tor-spec-udp.txt index 7f06222..e9f1eae 100644 --- a/proposals/100-tor-spec-udp.txt +++ b/proposals/100-tor-spec-udp.txt @@ -1,3 +1,4 @@ +``` Filename: 100-tor-spec-udp.txt Title: Tor Unreliable Datagram Extension Proposal Author: Marc Liberatore @@ -420,3 +421,4 @@ traffic "for free". example, what happens when a UDP packet doesn't fit in a single cell?) -NM +``` diff --git a/proposals/101-dir-voting.txt b/proposals/101-dir-voting.txt index 634d3f1..39fa6c6 100644 --- a/proposals/101-dir-voting.txt +++ b/proposals/101-dir-voting.txt @@ -1,3 +1,4 @@ +``` Filename: 101-dir-voting.txt Title: Voting on the Tor Directory System Author: Nick Mathewson @@ -281,3 +282,4 @@ by the authorities. -RD] off very quickly from downloading consensus docs until they're actually implemented. +``` diff --git a/proposals/102-drop-opt.txt b/proposals/102-drop-opt.txt index 490376b..a7d0cbe 100644 --- a/proposals/102-drop-opt.txt +++ b/proposals/102-drop-opt.txt @@ -1,3 +1,4 @@ +``` Filename: 102-drop-opt.txt Title: Dropping "opt" from the directory format Author: Nick Mathewson @@ -36,3 +37,4 @@ Status: once earlier formats are obsolete. +``` diff --git a/proposals/103-multilevel-keys.txt b/proposals/103-multilevel-keys.txt index 7ec254a..3bb3c28 100644 --- a/proposals/103-multilevel-keys.txt +++ b/proposals/103-multilevel-keys.txt @@ -1,3 +1,4 @@ +``` Filename: 103-multilevel-keys.txt Title: Splitting identity key from regularly used signing key Author: Nick Mathewson @@ -202,3 +203,4 @@ Extensions to Proposal 101. unknown signing keys might be enough to make the consensus correctly signed, do not use the consensus, but do not discard it until we have a new keys document. +``` diff --git a/proposals/104-short-descriptors.txt b/proposals/104-short-descriptors.txt index 90e0764..c543bf1 100644 --- a/proposals/104-short-descriptors.txt +++ b/proposals/104-short-descriptors.txt @@ -1,3 +1,4 @@ +``` Filename: 104-short-descriptors.txt Title: Long and Short Router Descriptors Author: Nick Mathewson @@ -179,3 +180,4 @@ Migration: * Once tools that want bandwidth info support fetching extra info: * Have routers stop including bandwidth info in their router descriptors. +``` diff --git a/proposals/105-handshake-revision.txt b/proposals/105-handshake-revision.txt index 791a016..45a937c 100644 --- a/proposals/105-handshake-revision.txt +++ b/proposals/105-handshake-revision.txt @@ -1,3 +1,4 @@ +``` Filename: 105-handshake-revision.txt Title: Version negotiation for the Tor protocol. Author: Nick Mathewson, Roger Dingledine @@ -321,3 +322,4 @@ Security issues: authorities' RecommmendedVersions mechanism, even if it is still technically possible to use them. +``` diff --git a/proposals/106-less-tls-constraint.txt b/proposals/106-less-tls-constraint.txt index 7e7621d..1655525 100644 --- a/proposals/106-less-tls-constraint.txt +++ b/proposals/106-less-tls-constraint.txt @@ -1,3 +1,4 @@ +``` Filename: 106-less-tls-constraint.txt Title: Checking fewer things during TLS handshakes Author: Nick Mathewson @@ -109,3 +110,4 @@ OTHER SPEC CHANGES: When a client doesn't give us an identity, we should never extend any circuits to it (duh), and we should allow it to set circuit ID however it wants. +``` diff --git a/proposals/107-uptime-sanity-checking.txt b/proposals/107-uptime-sanity-checking.txt index 922129b..7764889 100644 --- a/proposals/107-uptime-sanity-checking.txt +++ b/proposals/107-uptime-sanity-checking.txt @@ -1,3 +1,4 @@ +``` Filename: 107-uptime-sanity-checking.txt Title: Uptime Sanity Checking Author: Kevin Bauer & Damon McCoy @@ -52,3 +53,4 @@ Discussion: This proposal also led to 108-mtbf-based-stability.txt +``` diff --git a/proposals/108-mtbf-based-stability.txt b/proposals/108-mtbf-based-stability.txt index 2941037..962d273 100644 --- a/proposals/108-mtbf-based-stability.txt +++ b/proposals/108-mtbf-based-stability.txt @@ -1,3 +1,4 @@ +``` Filename: 108-mtbf-based-stability.txt Title: Base "Stable" Flag on Mean Time Between Failures Author: Nick Mathewson @@ -86,3 +87,4 @@ Implementation: Servers should probe at random intervals to test whether servers are running. +``` diff --git a/proposals/109-no-sharing-ips.txt b/proposals/109-no-sharing-ips.txt index a0f1fff..98927e9 100644 --- a/proposals/109-no-sharing-ips.txt +++ b/proposals/109-no-sharing-ips.txt @@ -1,3 +1,4 @@ +``` Filename: 109-no-sharing-ips.txt Title: No more than one server per IP address Author: Kevin Bauer & Damon McCoy @@ -88,3 +89,4 @@ Compatibility: [Note Roger's tweak to this behavior, in http://archives.seul.org/or/cvs/Oct-2007/msg00118.html] +``` diff --git a/proposals/110-avoid-infinite-circuits.txt b/proposals/110-avoid-infinite-circuits.txt index 7d6772d..07f5461 100644 --- a/proposals/110-avoid-infinite-circuits.txt +++ b/proposals/110-avoid-infinite-circuits.txt @@ -1,3 +1,4 @@ +``` Filename: 110-avoid-infinite-circuits.txt Title: Avoiding infinite length circuits Author: Roger Dingledine @@ -118,3 +119,4 @@ Acknowledgements: is working on implementing a fix based on this design in the summer 2007 timeframe.) +``` diff --git a/proposals/111-local-traffic-priority.txt b/proposals/111-local-traffic-priority.txt index 9411463..762634e 100644 --- a/proposals/111-local-traffic-priority.txt +++ b/proposals/111-local-traffic-priority.txt @@ -1,3 +1,4 @@ +``` Filename: 111-local-traffic-priority.txt Title: Prioritizing local traffic over relayed traffic Author: Roger Dingledine @@ -149,3 +150,4 @@ Prognosis: We could probably remove the third case and it would still work, but hey. +``` diff --git a/proposals/112-bring-back-pathlencoinweight.txt b/proposals/112-bring-back-pathlencoinweight.txt index 3f6c337..5109328 100644 --- a/proposals/112-bring-back-pathlencoinweight.txt +++ b/proposals/112-bring-back-pathlencoinweight.txt @@ -1,3 +1,4 @@ +``` Filename: 112-bring-back-pathlencoinweight.txt Title: Bring Back Pathlen Coin Weight Author: Mike Perry @@ -161,3 +162,4 @@ Migration: [1] http://www.cs.umass.edu/~mwright/papers/levine-timing.pdf +``` diff --git a/proposals/113-fast-authority-interface.txt b/proposals/113-fast-authority-interface.txt index 8912b53..55e925f 100644 --- a/proposals/113-fast-authority-interface.txt +++ b/proposals/113-fast-authority-interface.txt @@ -1,3 +1,4 @@ +``` Filename: 113-fast-authority-interface.txt Title: Simplifying directory authority administration Author: Nick Mathewson @@ -83,3 +84,4 @@ Resolution: automatically create bindings", and that approach is implemented. There are remaining issues in the problem statement above that need their own solutions. +``` diff --git a/proposals/114-distributed-storage.txt b/proposals/114-distributed-storage.txt index 91a787d..1c0f40f 100644 --- a/proposals/114-distributed-storage.txt +++ b/proposals/114-distributed-storage.txt @@ -1,3 +1,4 @@ +``` Filename: 114-distributed-storage.txt Title: Distributed Storage for Tor Hidden Service Descriptors Author: Karsten Loesing @@ -437,3 +438,4 @@ Compatibility: After the second transition phase, the authoritative directories should stop serving hidden service descriptors. +``` diff --git a/proposals/115-two-hop-paths.txt b/proposals/115-two-hop-paths.txt index 9854c9a..7ef07b4 100644 --- a/proposals/115-two-hop-paths.txt +++ b/proposals/115-two-hop-paths.txt @@ -1,3 +1,4 @@ +``` Filename: 115-two-hop-paths.txt Title: Two Hop Paths Author: Mike Perry @@ -383,3 +384,4 @@ Migration: [1] http://p2pnet.net/story/11279 [2] http://www.cs.umass.edu/~mwright/papers/levine-timing.pdf [3] Proof available upon request ;) +``` diff --git a/proposals/116-two-hop-paths-from-guard.txt b/proposals/116-two-hop-paths-from-guard.txt index f456253..8228c7a 100644 --- a/proposals/116-two-hop-paths-from-guard.txt +++ b/proposals/116-two-hop-paths-from-guard.txt @@ -1,3 +1,4 @@ +``` Filename: 116-two-hop-paths-from-guard.txt Title: Two hop paths from entry guards Author: Michael Lieberman @@ -116,3 +117,4 @@ MIT mentioned above. 2) It would give governments an incentive to confiscate exit nodes to see if they are pulling this trick. +``` diff --git a/proposals/117-ipv6-exits.txt b/proposals/117-ipv6-exits.txt index f343ff7..e5231db 100644 --- a/proposals/117-ipv6-exits.txt +++ b/proposals/117-ipv6-exits.txt @@ -1,3 +1,4 @@ +``` Filename: 117-ipv6-exits.txt Title: IPv6 exits Author: coderman @@ -409,3 +410,4 @@ Contents 'Network Address Translation - Protocol Translation (NAT-PT)' http://www.ietf.org/rfc/rfc2766.txt +``` diff --git a/proposals/118-multiple-orports.txt b/proposals/118-multiple-orports.txt index 69ab382..0378589 100644 --- a/proposals/118-multiple-orports.txt +++ b/proposals/118-multiple-orports.txt @@ -1,3 +1,4 @@ +``` Filename: 118-multiple-orports.txt Title: Advertising multiple ORPorts at once Author: Nick Mathewson @@ -85,3 +86,4 @@ Notes 3 July: - Write up the simple version of this. No ranges needed yet. No networkstatus chagnes yet. +``` diff --git a/proposals/119-controlport-auth.txt b/proposals/119-controlport-auth.txt index 9ed1cc1..32c70b6 100644 --- a/proposals/119-controlport-auth.txt +++ b/proposals/119-controlport-auth.txt @@ -1,3 +1,4 @@ +``` Filename: 119-controlport-auth.txt Title: New PROTOCOLINFO command for controllers Author: Roger Dingledine @@ -138,3 +139,4 @@ Unsolved problems: to think about how to demand password-based authentication without bothering the user about it. +``` diff --git a/proposals/120-shutdown-descriptors.txt b/proposals/120-shutdown-descriptors.txt index 5cfe2b5..465ef86 100644 --- a/proposals/120-shutdown-descriptors.txt +++ b/proposals/120-shutdown-descriptors.txt @@ -1,3 +1,4 @@ +``` Filename: 120-shutdown-descriptors.txt Title: Shutdown descriptors when Tor servers stop Author: Roger Dingledine @@ -81,3 +82,4 @@ Comments: 2) Maybe add a rule "Don't do this for hibernation if we expect to wake up before the next consensus is published"? - NM 9 Oct 2007 +``` diff --git a/proposals/121-hidden-service-authentication.txt b/proposals/121-hidden-service-authentication.txt index 404ca57..678d4d4 100644 --- a/proposals/121-hidden-service-authentication.txt +++ b/proposals/121-hidden-service-authentication.txt @@ -1,3 +1,4 @@ +``` Filename: 121-hidden-service-authentication.txt Title: Hidden Service Authentication Author: Tobias Kamm, Thomas Lauterbach, Karsten Loesing, Ferdinand Rieger, @@ -774,3 +775,4 @@ Implementation: checking access counters. - Log access for accountability. +``` diff --git a/proposals/122-unnamed-flag.txt b/proposals/122-unnamed-flag.txt index 2ce7bb2..d73f2e4 100644 --- a/proposals/122-unnamed-flag.txt +++ b/proposals/122-unnamed-flag.txt @@ -1,3 +1,4 @@ +``` Filename: 122-unnamed-flag.txt Title: Network status entries need a new Unnamed flag Author: Roger Dingledine @@ -134,3 +135,4 @@ Implemented-In: 0.2.0.x blacklisted from the network, the really bad part is that this logic is really unintuitive to prospective new server operators. +``` diff --git a/proposals/123-autonaming.txt b/proposals/123-autonaming.txt index 74c4869..bc9cdfa 100644 --- a/proposals/123-autonaming.txt +++ b/proposals/123-autonaming.txt @@ -1,3 +1,4 @@ +``` Filename: 123-autonaming.txt Title: Naming authorities automatically create bindings Author: Peter Palfrader @@ -52,3 +53,4 @@ Proposal: This automaton does not necessarily need to live in the Tor code, it can do its job just as well when it's an external tool. +``` diff --git a/proposals/124-tls-certificates.txt b/proposals/124-tls-certificates.txt index 9472d14..0149da7 100644 --- a/proposals/124-tls-certificates.txt +++ b/proposals/124-tls-certificates.txt @@ -1,3 +1,4 @@ +``` Filename: 124-tls-certificates.txt Title: Blocking resistant TLS certificate usage Author: Steven J. Murdoch @@ -311,3 +312,4 @@ References: [5] Elliptic Curve Cryptography (ECC) Cipher Suites for Transport Layer Security (TLS) % <!-- Local IspellDict: american --> +``` diff --git a/proposals/125-bridges.txt b/proposals/125-bridges.txt index 9d95729..2b4d0e6 100644 --- a/proposals/125-bridges.txt +++ b/proposals/125-bridges.txt @@ -1,3 +1,4 @@ +``` Filename: 125-bridges.txt Title: Behavior for bridge users, bridge relays, and bridge authorities Author: Roger Dingledine @@ -289,3 +290,4 @@ Implemented-In: 0.2.0.x We should figure this out in the 0.2.1.x timeframe. +``` diff --git a/proposals/126-geoip-reporting.txt b/proposals/126-geoip-reporting.txt index 9f3b21c..131b25f 100644 --- a/proposals/126-geoip-reporting.txt +++ b/proposals/126-geoip-reporting.txt @@ -1,3 +1,4 @@ +``` Filename: 126-geoip-reporting.txt Title: Getting GeoIP data and publishing usage summaries Author: Roger Dingledine @@ -408,3 +409,4 @@ Implemented-In: 0.2.0.x Other ideas? +``` diff --git a/proposals/127-dirport-mirrors-downloads.txt b/proposals/127-dirport-mirrors-downloads.txt index 377492c..fcdec60 100644 --- a/proposals/127-dirport-mirrors-downloads.txt +++ b/proposals/127-dirport-mirrors-downloads.txt @@ -1,3 +1,4 @@ +``` Filename: 127-dirport-mirrors-downloads.txt Title: Relaying dirport requests to Tor download site / website Author: Roger Dingledine @@ -153,3 +154,4 @@ Status: Obsolete you about a gmail autoresponder address just as easily, and then you'd get better authentication of the Tor program to boot. +``` diff --git a/proposals/128-bridge-families.txt b/proposals/128-bridge-families.txt index e5bdcf9..67f208e 100644 --- a/proposals/128-bridge-families.txt +++ b/proposals/128-bridge-families.txt @@ -1,3 +1,4 @@ +``` Filename: 128-bridge-families.txt Title: Families of private bridges Author: Roger Dingledine @@ -62,3 +63,4 @@ Status: Dead ideas/ directory. Somebody should pick it up sometime and finish the design and implementation. +``` diff --git a/proposals/129-reject-plaintext-ports.txt b/proposals/129-reject-plaintext-ports.txt index 8080ff5..91b9bd4 100644 --- a/proposals/129-reject-plaintext-ports.txt +++ b/proposals/129-reject-plaintext-ports.txt @@ -1,3 +1,4 @@ +``` Filename: 129-reject-plaintext-ports.txt Title: Block Insecure Protocols by Default Author: Kevin Bauer & Damon McCoy @@ -112,3 +113,4 @@ Next steps: b) We should decide which ports to reject by default. The current consensus is 23,109,110,143 -- the same set that we warn for now. +``` diff --git a/proposals/130-v2-conn-protocol.txt b/proposals/130-v2-conn-protocol.txt index f5c31cc..fddbb75 100644 --- a/proposals/130-v2-conn-protocol.txt +++ b/proposals/130-v2-conn-protocol.txt @@ -1,3 +1,4 @@ +``` Filename: 130-v2-conn-protocol.txt Title: Version 2 Tor connection protocol Author: Nick Mathewson @@ -182,3 +183,4 @@ Proposal: Until NETINFO cells have been exchanged, the connection is not open. +``` diff --git a/proposals/131-verify-tor-usage.txt b/proposals/131-verify-tor-usage.txt index 8af1151..19a5737 100644 --- a/proposals/131-verify-tor-usage.txt +++ b/proposals/131-verify-tor-usage.txt @@ -1,3 +1,4 @@ +``` Filename: 131-verify-tor-usage.txt Title: Help users to verify they are using Tor Author: Steven J. Murdoch @@ -146,3 +147,4 @@ Related work: Tor should be able to respond with a webpage to notify the user of how to fix the problem. This also implies that Tor must be told of the special IP address, and so must be effectively permanent. +``` diff --git a/proposals/132-browser-check-tor-service.txt b/proposals/132-browser-check-tor-service.txt index 7aecaaf..d652d6f 100644 --- a/proposals/132-browser-check-tor-service.txt +++ b/proposals/132-browser-check-tor-service.txt @@ -1,3 +1,4 @@ +``` Filename: 132-browser-check-tor-service.txt Title: A Tor Web Service For Verifying Correct Browser Configuration Author: Robert Hogan @@ -143,3 +144,4 @@ Open Issues: future. Or maybe it's not such a bad thing, Tor is merely attempting to make normally invalid resource requests valid for a given purpose. +``` diff --git a/proposals/133-unreachable-ors.txt b/proposals/133-unreachable-ors.txt index 5e2916b..ed80d7a 100644 --- a/proposals/133-unreachable-ors.txt +++ b/proposals/133-unreachable-ors.txt @@ -1,3 +1,4 @@ +``` Filename: 133-unreachable-ors.txt Title: Incorporate Unreachable ORs into the Tor Network Author: Robert Hogan @@ -126,3 +127,4 @@ Implementation Issues: published OR should be willing to accept is an open question. Is one hundred, mostly idle, such orconns too onerous? +``` diff --git a/proposals/134-robust-voting.txt b/proposals/134-robust-voting.txt index c5dfb3b..66dfef5 100644 --- a/proposals/134-robust-voting.txt +++ b/proposals/134-robust-voting.txt @@ -1,3 +1,4 @@ +``` Filename: 134-robust-voting.txt Title: More robust consensus voting with diverse authority sets Author: Peter Palfrader @@ -121,3 +122,4 @@ requirement that an adversary must compromise a majority of authorities in order to control the consensus. -- Nick +``` diff --git a/proposals/135-private-tor-networks.txt b/proposals/135-private-tor-networks.txt index 19ef68b..7ce0d4c 100644 --- a/proposals/135-private-tor-networks.txt +++ b/proposals/135-private-tor-networks.txt @@ -1,3 +1,4 @@ +``` Filename: 135-private-tor-networks.txt Title: Simplify Configuration of Private Tor Networks Author: Karsten Loesing @@ -279,3 +280,4 @@ Test: [warn] Failed to parse/validate config: TestingTorNetwork may only be configured in combination with a non-default set of DirServers. +``` diff --git a/proposals/136-legacy-keys.txt b/proposals/136-legacy-keys.txt index f2b1b5c..ab8c9de 100644 --- a/proposals/136-legacy-keys.txt +++ b/proposals/136-legacy-keys.txt @@ -1,3 +1,4 @@ +``` Filename: 136-legacy-keys.txt Title: Mass authority migration with legacy keys Author: Nick Mathewson @@ -98,3 +99,4 @@ Notes: I don't think this achieves us anything either, though. +``` diff --git a/proposals/137-bootstrap-phases.txt b/proposals/137-bootstrap-phases.txt index ebe044c..fbb5081 100644 --- a/proposals/137-bootstrap-phases.txt +++ b/proposals/137-bootstrap-phases.txt @@ -1,3 +1,4 @@ +``` Filename: 137-bootstrap-phases.txt Title: Keep controllers informed as Tor bootstraps Author: Roger Dingledine @@ -233,3 +234,4 @@ Implemented-In: 0.2.1.x Controllers should use this getinfo when they connect or attach to Tor to learn its current state. +``` diff --git a/proposals/138-remove-down-routers-from-consensus.txt b/proposals/138-remove-down-routers-from-consensus.txt index 776911b..037edbb 100644 --- a/proposals/138-remove-down-routers-from-consensus.txt +++ b/proposals/138-remove-down-routers-from-consensus.txt @@ -1,3 +1,4 @@ +``` Filename: 138-remove-down-routers-from-consensus.txt Title: Remove routers that are not Running from consensus documents Author: Peter Palfrader @@ -47,3 +48,4 @@ Implemented-In: 0.2.1.2-alpha consensus document is formed like before but a new last step removes all routers from the listing that are not marked as Running. +``` diff --git a/proposals/139-conditional-consensus-download.txt b/proposals/139-conditional-consensus-download.txt index 941f5ad..ea00acc 100644 --- a/proposals/139-conditional-consensus-download.txt +++ b/proposals/139-conditional-consensus-download.txt @@ -1,3 +1,4 @@ +``` Filename: 139-conditional-consensus-download.txt Title: Download consensus documents only when it will be trusted Author: Peter Palfrader @@ -92,3 +93,4 @@ Footnotes: 1. For the purpose of this proposal a client can be any Tor instance that downloads a consensus document. This includes relays, directory caches as well as end users. +``` diff --git a/proposals/140-consensus-diffs.txt b/proposals/140-consensus-diffs.txt index c50aba0..d1a5a1d 100644 --- a/proposals/140-consensus-diffs.txt +++ b/proposals/140-consensus-diffs.txt @@ -1,3 +1,4 @@ +``` Filename: 140-consensus-diffs.txt Title: Provide diffs between consensuses Author: Peter Palfrader @@ -231,3 +232,4 @@ Ticket: https://bugs.torproject.org/13339 This parameter determines how old a consensus can be (in hours) before a client should no longer try to find a diff for it. +``` diff --git a/proposals/141-jit-sd-downloads.txt b/proposals/141-jit-sd-downloads.txt index bba6493..1216ded 100644 --- a/proposals/141-jit-sd-downloads.txt +++ b/proposals/141-jit-sd-downloads.txt @@ -1,3 +1,4 @@ +``` Filename: 141-jit-sd-downloads.txt Title: Download server descriptors on demand Author: Peter Palfrader @@ -321,3 +322,4 @@ Status: Obsolete B needs to contact C so it can ask about the descriptor before passing it back to the client. +``` diff --git a/proposals/142-combine-intro-and-rend-points.txt b/proposals/142-combine-intro-and-rend-points.txt index 3abd5c8..6ad34eb 100644 --- a/proposals/142-combine-intro-and-rend-points.txt +++ b/proposals/142-combine-intro-and-rend-points.txt @@ -1,3 +1,4 @@ +``` Filename: 142-combine-intro-and-rend-points.txt Title: Combine Introduction and Rendezvous Points Author: Karsten Loesing, Christian Wilms @@ -275,3 +276,4 @@ References: [5] Lasse Øverlier and Paul Syverson, Locating Hidden Servers. In the Proceedings of the 2006 IEEE Symposium on Security and Privacy, May 2006. +``` diff --git a/proposals/143-distributed-storage-improvements.txt b/proposals/143-distributed-storage-improvements.txt index ce3f781..c89c309 100644 --- a/proposals/143-distributed-storage-improvements.txt +++ b/proposals/143-distributed-storage-improvements.txt @@ -1,3 +1,4 @@ +``` Filename: 143-distributed-storage-improvements.txt Title: Improvements of Distributed Storage for Tor Hidden Service Descriptors Author: Karsten Loesing @@ -191,3 +192,4 @@ Compatibility: All proposed improvements are compatible to the currently implemented design as described in proposal 114. +``` diff --git a/proposals/144-enforce-distinct-providers.txt b/proposals/144-enforce-distinct-providers.txt index 1a682f3..da3845e 100644 --- a/proposals/144-enforce-distinct-providers.txt +++ b/proposals/144-enforce-distinct-providers.txt @@ -1,3 +1,4 @@ +``` Filename: 144-enforce-distinct-providers.txt Title: Increase the diversity of circuits by detecting nodes belonging the same provider @@ -163,3 +164,4 @@ In the Proceedings of the Seventh Workshop on Privacy Enhancing Technologies (PET 2007), Ottawa, Canada, June 2007. http://freehaven.net/anonbib/#murdoch-pet2007 [5] http://bugs.noreply.org/flyspray/index.php?do=details&id=690 +``` diff --git a/proposals/145-newguard-flag.txt b/proposals/145-newguard-flag.txt index 98c6508..7646727 100644 --- a/proposals/145-newguard-flag.txt +++ b/proposals/145-newguard-flag.txt @@ -1,3 +1,4 @@ +``` Filename: 145-newguard-flag.txt Title: Separate "suitable as a guard" from "suitable as a new guard" Author: Nick Mathewson @@ -38,3 +39,4 @@ Alternatives We might instead have authorities list weights with which nodes should be picked as guards. +``` diff --git a/proposals/146-long-term-stability.txt b/proposals/146-long-term-stability.txt index 650be33..110231f 100644 --- a/proposals/146-long-term-stability.txt +++ b/proposals/146-long-term-stability.txt @@ -1,3 +1,4 @@ +``` Filename: 146-long-term-stability.txt Title: Add new flag to reflect long-term stability Author: Nick Mathewson @@ -91,3 +92,4 @@ Issues and future possibilities: Longterm is a really awkward name. +``` diff --git a/proposals/147-prevoting-opinions.txt b/proposals/147-prevoting-opinions.txt index 22f5c0f..42787a5 100644 --- a/proposals/147-prevoting-opinions.txt +++ b/proposals/147-prevoting-opinions.txt @@ -1,3 +1,4 @@ +``` Filename: 147-prevoting-opinions.txt Title: Eliminate the need for v2 directories in generating v3 directories Author: Nick Mathewson @@ -118,3 +119,4 @@ Reasons for rejection: -c | sort | grep " [1-4] " | wc -l`, result is 63, divide by 7*24 published consensuses, obtain 0.375 as end result.) +``` diff --git a/proposals/148-uniform-client-end-reason.txt b/proposals/148-uniform-client-end-reason.txt index 1db3b3e..0ee8145 100644 --- a/proposals/148-uniform-client-end-reason.txt +++ b/proposals/148-uniform-client-end-reason.txt @@ -1,3 +1,4 @@ +``` Filename: 148-uniform-client-end-reason.txt Title: Stream end reasons from the client side should be uniform Author: Roger Dingledine @@ -55,3 +56,4 @@ Design: like most of the Windows clients currently do and like the non-Windows clients already do sometimes. +``` diff --git a/proposals/149-using-netinfo-data.txt b/proposals/149-using-netinfo-data.txt index 0ad663c..d8d052c 100644 --- a/proposals/149-using-netinfo-data.txt +++ b/proposals/149-using-netinfo-data.txt @@ -1,3 +1,4 @@ +``` Filename: 149-using-netinfo-data.txt Title: Using data from NETINFO cells Author: Nick Mathewson @@ -43,3 +44,4 @@ Avoiding MITM attacks connections without first trying to build a canonical one. +``` diff --git a/proposals/150-exclude-exit-nodes.txt b/proposals/150-exclude-exit-nodes.txt index b497ae6..460058c 100644 --- a/proposals/150-exclude-exit-nodes.txt +++ b/proposals/150-exclude-exit-nodes.txt @@ -1,3 +1,4 @@ +``` Filename: 150-exclude-exit-nodes.txt Title: Exclude Exit Nodes from a circuit Author: Mfr @@ -45,3 +46,4 @@ Risks: about broken exits because of this option. This issue can probably be addressed sufficiently with documentation. +``` diff --git a/proposals/151-path-selection-improvements.txt b/proposals/151-path-selection-improvements.txt index e92b574..5eacfa7 100644 --- a/proposals/151-path-selection-improvements.txt +++ b/proposals/151-path-selection-improvements.txt @@ -1,3 +1,4 @@ +``` Filename: 151-path-selection-improvements.txt Title: Improving Tor Path Selection Author: Fallon Chen, Mike Perry @@ -147,3 +148,4 @@ Issues timeout can be achieved without cutting off a great number of the total paths. This will eliminate a great deal of the performance variation of Tor usage. +``` diff --git a/proposals/152-single-hop-circuits.txt b/proposals/152-single-hop-circuits.txt index d0b28b1..1e271af 100644 --- a/proposals/152-single-hop-circuits.txt +++ b/proposals/152-single-hop-circuits.txt @@ -1,3 +1,4 @@ +``` Filename: 152-single-hop-circuits.txt Title: Optionally allow exit from single-hop circuits Author: Geoff Goodell @@ -60,3 +61,4 @@ Security Considerations exit-from-single-hop-circuits" when choosing random paths through the Tor network. +``` diff --git a/proposals/153-automatic-software-update-protocol.txt b/proposals/153-automatic-software-update-protocol.txt index c2979bb..7cf0c0a 100644 --- a/proposals/153-automatic-software-update-protocol.txt +++ b/proposals/153-automatic-software-update-protocol.txt @@ -1,3 +1,4 @@ +``` Filename: 153-automatic-software-update-protocol.txt Title: Automatic software update protocol Author: Jacob Appelbaum @@ -173,3 +174,4 @@ question. x.x Other Things Other things should be added to this proposal. What are they? +``` diff --git a/proposals/154-automatic-updates.txt b/proposals/154-automatic-updates.txt index 4c2c6d3..e893344 100644 --- a/proposals/154-automatic-updates.txt +++ b/proposals/154-automatic-updates.txt @@ -1,3 +1,4 @@ +``` Filename: 154-automatic-updates.txt Title: Automatic Software Update Protocol Author: Matt Edman @@ -375,3 +376,4 @@ References [3] Tor version specification, https://tor-svn.freehaven.net/svn/tor/trunk/doc/spec/version-spec.txt +``` diff --git a/proposals/155-four-hidden-service-improvements.txt b/proposals/155-four-hidden-service-improvements.txt index 423c9e8..3406481 100644 --- a/proposals/155-four-hidden-service-improvements.txt +++ b/proposals/155-four-hidden-service-improvements.txt @@ -1,3 +1,4 @@ +``` Filename: 155-four-hidden-service-improvements.txt Title: Four Improvements of Hidden Service Performance Author: Karsten Loesing, Christian Wilms @@ -118,3 +119,4 @@ References: [3] http://freehaven.net/~karsten/hidserv/design-2008-08-15.pdf +``` diff --git a/proposals/156-tracking-blocked-ports.txt b/proposals/156-tracking-blocked-ports.txt index acc0625..578621f 100644 --- a/proposals/156-tracking-blocked-ports.txt +++ b/proposals/156-tracking-blocked-ports.txt @@ -1,3 +1,4 @@ +``` Filename: 156-tracking-blocked-ports.txt Title: Tracking blocked ports on the client side Author: Robert Hogan @@ -527,3 +528,4 @@ Index: src/or/circuitbuild.c /** If <b>question</b> is the string "entry-guards", then dump +``` diff --git a/proposals/157-specific-cert-download.txt b/proposals/157-specific-cert-download.txt index e0ef0e8..8a69611 100644 --- a/proposals/157-specific-cert-download.txt +++ b/proposals/157-specific-cert-download.txt @@ -1,3 +1,4 @@ +``` Filename: 157-specific-cert-download.txt Title: Make certificate downloads specific Author: Nick Mathewson @@ -100,3 +101,4 @@ Why cross-certify too? References: This is related to part 2 of bug 854. +``` diff --git a/proposals/158-microdescriptors.txt b/proposals/158-microdescriptors.txt index 442d705..d267c0b 100644 --- a/proposals/158-microdescriptors.txt +++ b/proposals/158-microdescriptors.txt @@ -1,3 +1,4 @@ +``` Filename: 158-microdescriptors.txt Title: Clients download consensus + microdescriptors Author: Roger Dingledine @@ -197,3 +198,4 @@ Implemented-In: 0.2.3.1-alpha Phase three, clients should start fetching and caching them instead of normal descriptors. +``` diff --git a/proposals/159-exit-scanning.txt b/proposals/159-exit-scanning.txt index d2a7f3b..0f4d329 100644 --- a/proposals/159-exit-scanning.txt +++ b/proposals/159-exit-scanning.txt @@ -1,3 +1,4 @@ +``` Filename: 159-exit-scanning.txt Title: Exit Scanning Author: Mike Perry @@ -140,3 +141,4 @@ Having a separate file that is under the exclusive control of the scanner allows us to heavily isolate the scanner from the Tor authority, potentially even running them on separate machines. +``` diff --git a/proposals/160-bandwidth-offset.txt b/proposals/160-bandwidth-offset.txt index 8ad4d18..33abd45 100644 --- a/proposals/160-bandwidth-offset.txt +++ b/proposals/160-bandwidth-offset.txt @@ -1,3 +1,4 @@ +``` Filename: 160-bandwidth-offset.txt Title: Authorities vote for bandwidth offsets in consensus Author: Roger Dingledine @@ -103,3 +104,4 @@ Target: 0.2.1.x +``` diff --git a/proposals/161-computing-bandwidth-adjustments.txt b/proposals/161-computing-bandwidth-adjustments.txt index 6968724..38a1f9e 100644 --- a/proposals/161-computing-bandwidth-adjustments.txt +++ b/proposals/161-computing-bandwidth-adjustments.txt @@ -1,3 +1,4 @@ +``` Title: Computing Bandwidth Adjustments Filename: 161-computing-bandwidth-adjustments.txt Author: Mike Perry @@ -172,3 +173,4 @@ Status: Closed This file can be either copied or rsynced into a directory readable by the directory authority. +``` diff --git a/proposals/162-consensus-flavors.txt b/proposals/162-consensus-flavors.txt index 2a227a0..85e6421 100644 --- a/proposals/162-consensus-flavors.txt +++ b/proposals/162-consensus-flavors.txt @@ -1,3 +1,4 @@ +``` Filename: 162-consensus-flavors.txt Title: Publish the consensus in multiple flavors Author: Nick Mathewson @@ -188,3 +189,4 @@ Acknowledgements: Aspects of this design and its applications to hash migration were heavily influenced by IRC conversations with Marian. +``` diff --git a/proposals/163-detecting-clients.txt b/proposals/163-detecting-clients.txt index f2a17a5..5f77703 100644 --- a/proposals/163-detecting-clients.txt +++ b/proposals/163-detecting-clients.txt @@ -1,3 +1,4 @@ +``` Filename: 163-detecting-clients.txt Title: Detecting whether a connection comes from a client Author: Nick Mathewson @@ -116,3 +117,4 @@ Open questions: [Suggested answer: for now, just try making circuits through the node. Extend this to extending circuits as needed.] +``` diff --git a/proposals/164-reporting-server-status.txt b/proposals/164-reporting-server-status.txt index 24fcc8f..418b3f8 100644 --- a/proposals/164-reporting-server-status.txt +++ b/proposals/164-reporting-server-status.txt @@ -1,3 +1,4 @@ +``` Filename: 164-reporting-server-status.txt Title: Reporting the status of server votes Author: Nick Mathewson @@ -90,3 +91,4 @@ Risks: +``` diff --git a/proposals/165-simple-robust-voting.txt b/proposals/165-simple-robust-voting.txt index 28ea6d9..c759ee8 100644 --- a/proposals/165-simple-robust-voting.txt +++ b/proposals/165-simple-robust-voting.txt @@ -1,3 +1,4 @@ +``` Filename: 165-simple-robust-voting.txt Title: Easy migration for voting authority sets Author: Nick Mathewson @@ -134,3 +135,4 @@ Acknowledgments: The design came out of an IRC conversation with Peter Palfrader. He had the basic idea first. +``` diff --git a/proposals/166-statistics-extra-info-docs.txt b/proposals/166-statistics-extra-info-docs.txt index 8b0c6a1..0075ade 100644 --- a/proposals/166-statistics-extra-info-docs.txt +++ b/proposals/166-statistics-extra-info-docs.txt @@ -1,3 +1,4 @@ +``` Filename: 166-statistics-extra-info-docs.txt Title: Including Network Statistics in Extra-Info Documents Author: Karsten Loesing @@ -389,3 +390,4 @@ Implementation notes: gathering statistics. However, in the near future, statistics shall be enabled gathered by all relays by default, where requiring a ./configure option would be a barrier for many relay operators. +``` diff --git a/proposals/167-params-in-consensus.txt b/proposals/167-params-in-consensus.txt index d23bc9c..f616c35 100644 --- a/proposals/167-params-in-consensus.txt +++ b/proposals/167-params-in-consensus.txt @@ -1,3 +1,4 @@ +``` Filename: 167-params-in-consensus.txt Title: Vote on network parameters in consensus Author: Roger Dingledine @@ -45,3 +46,4 @@ Implemented-In: 0.2.2 I don't have any non-integer values in mind yet. So I say we cross that bridge when we get to it. +``` diff --git a/proposals/168-reduce-circwindow.txt b/proposals/168-reduce-circwindow.txt index 9932ff6..48fc146 100644 --- a/proposals/168-reduce-circwindow.txt +++ b/proposals/168-reduce-circwindow.txt @@ -1,3 +1,4 @@ +``` Filename: 168-reduce-circwindow.txt Title: Reduce default circuit window Author: Roger Dingledine @@ -132,3 +133,4 @@ Status: Rejected isn't very complex. Let's shoot for that, and fall back to (b) if the patch turns out to be so big that we reconsider. +``` diff --git a/proposals/169-eliminating-renegotiation.txt b/proposals/169-eliminating-renegotiation.txt index fe4cd58..36f8acd 100644 --- a/proposals/169-eliminating-renegotiation.txt +++ b/proposals/169-eliminating-renegotiation.txt @@ -1,3 +1,4 @@ +``` Filename: 169-eliminating-renegotiation.txt Title: Eliminate TLS renegotiation for the Tor connection handshake Author: Nick Mathewson @@ -404,3 +405,4 @@ Superseded-By: 176 now, I think they should just get it. Eventually we can deprecate the V2 handshake as we did with the V1 handshake. +``` diff --git a/proposals/170-user-path-config.txt b/proposals/170-user-path-config.txt index 42163f1..da85700 100644 --- a/proposals/170-user-path-config.txt +++ b/proposals/170-user-path-config.txt @@ -1,3 +1,4 @@ +``` Title: Configuration options regarding circuit building Filename: 170-user-path-config.txt Author: Sebastian Hahn @@ -93,3 +94,4 @@ Compatibility: The old Strict*Nodes options are deprecated, and the StrictNodes option is new. Tor users may need to update their configuration file. +``` diff --git a/proposals/171-separate-streams.txt b/proposals/171-separate-streams.txt index 01668a4..f482b09 100644 --- a/proposals/171-separate-streams.txt +++ b/proposals/171-separate-streams.txt @@ -1,3 +1,4 @@ +``` Filename: 171-separate-streams.txt Title: Separate streams across circuits by connection metadata Author: Robert Hogan, Jacob Appelbaum, Damon McCoy, Nick Mathewson @@ -356,3 +357,4 @@ Lingering questions: I suspect there are issues remaining with DNS and TransPort users, and that my "just use AutomapHostsOnResolve" suggestion may be insufficient. +``` diff --git a/proposals/172-circ-getinfo-option.txt b/proposals/172-circ-getinfo-option.txt index 38b0da7..9c9cea7 100644 --- a/proposals/172-circ-getinfo-option.txt +++ b/proposals/172-circ-getinfo-option.txt @@ -1,3 +1,4 @@ +``` Filename: 172-circ-getinfo-option.txt Title: GETINFO controller option for circuit information Author: Damian Johnson @@ -136,3 +137,4 @@ Specification: provides the same fields that would be given by calling "GETINFO rcirc/id/" with the CircID. +``` diff --git a/proposals/173-getinfo-option-expansion.txt b/proposals/173-getinfo-option-expansion.txt index 035471d..9c6cc12 100644 --- a/proposals/173-getinfo-option-expansion.txt +++ b/proposals/173-getinfo-option-expansion.txt @@ -1,3 +1,4 @@ +``` Filename: 173-getinfo-option-expansion.txt Title: GETINFO Option Expansion Author: Damian Johnson @@ -103,3 +104,4 @@ Specification: 650-EXITS_SEEN TimeStarted="2008-12-25 23:50:43" PortSummary=80=16,443=8 +``` diff --git a/proposals/174-optimistic-data-server.txt b/proposals/174-optimistic-data-server.txt index 193565b..ef34526 100644 --- a/proposals/174-optimistic-data-server.txt +++ b/proposals/174-optimistic-data-server.txt @@ -1,3 +1,4 @@ +``` Filename: 174-optimistic-data-server.txt Title: Optimistic Data for Tor: Server Side Author: Ian Goldberg @@ -241,3 +242,4 @@ Performance and scalability notes: There may be more RAM used at Exit nodes, as mentioned above, but it is transient. +``` diff --git a/proposals/175-automatic-node-promotion.txt b/proposals/175-automatic-node-promotion.txt index d6d86a6..2713a2a 100644 --- a/proposals/175-automatic-node-promotion.txt +++ b/proposals/175-automatic-node-promotion.txt @@ -1,3 +1,4 @@ +``` Filename: 175-automatic-node-promotion.txt Title: Automatically promoting Tor clients to nodes Author: Steven Murdoch @@ -238,3 +239,4 @@ Status: Rejected http://www.itl.nist.gov/div898/handbook/pmc/section3/pmc331.htm (e-Handbook of Statistical Methods) % vim: spell ai et: +``` diff --git a/proposals/176-revising-handshake.txt b/proposals/176-revising-handshake.txt index 20a3ef2..f53190a 100644 --- a/proposals/176-revising-handshake.txt +++ b/proposals/176-revising-handshake.txt @@ -1,3 +1,4 @@ +``` Filename: 176-revising-handshake.txt Title: Proposed version-3 link handshake for Tor Author: Nick Mathewson @@ -622,3 +623,4 @@ Supersedes: 169 now, I think they should just stick with V1. Eventually we can deprecate the V2 handshake as we did with the V1 handshake. When that happens, servers can be V3-only. +``` diff --git a/proposals/177-flag-abstention.txt b/proposals/177-flag-abstention.txt index cbc0819..02a4657 100644 --- a/proposals/177-flag-abstention.txt +++ b/proposals/177-flag-abstention.txt @@ -1,3 +1,4 @@ +``` Filename: 177-flag-abstention.txt Title: Abstaining from votes on individual flags Author: Nick Mathewson @@ -103,3 +104,4 @@ Examples: it is not run by wombats. (As now, "no" votes win ties.) +``` diff --git a/proposals/178-param-voting.txt b/proposals/178-param-voting.txt index 58d2a62..fc7fe25 100644 --- a/proposals/178-param-voting.txt +++ b/proposals/178-param-voting.txt @@ -1,3 +1,4 @@ +``` Filename: 178-param-voting.txt Title: Require majority of authorities to vote for consensus parameters Author: Sebastian Hahn @@ -99,3 +100,4 @@ Implementation: An example implementation of this feature can be found in https://gitweb.torproject.org/sebastian/tor.git, branch safer_params. +``` diff --git a/proposals/179-TLS-cert-and-parameter-normalization.txt b/proposals/179-TLS-cert-and-parameter-normalization.txt index 4272d4f..cf455a8 100644 --- a/proposals/179-TLS-cert-and-parameter-normalization.txt +++ b/proposals/179-TLS-cert-and-parameter-normalization.txt @@ -1,3 +1,4 @@ +``` Filename: 179-TLS-cert-and-parameter-normalization.txt Title: TLS certificate and parameter normalization Author: Jacob Appelbaum, Gladys Shufflebottom @@ -384,3 +385,4 @@ present in the Tor TLS setup process? [4] httpd-2.2.17/modules/ss/ssl_engine_dh.c [5] http://tools.ietf.org/html/rfc4419 [6] http://archives.seul.org/or/dev/Jan-2011/msg00037.html +``` diff --git a/proposals/180-pluggable-transport.txt b/proposals/180-pluggable-transport.txt index c23e5fa..8626375 100644 --- a/proposals/180-pluggable-transport.txt +++ b/proposals/180-pluggable-transport.txt @@ -1,3 +1,4 @@ +``` Filename: 180-pluggable-transport.txt Title: Pluggable transports for circumvention Author: Jacob Appelbaum, Nick Mathewson @@ -548,3 +549,4 @@ Appendix: recommendations for transports protocols -- and in many cases, most possible variants of a given protocol won't actually exist in the wild. +``` diff --git a/proposals/181-optimistic-data-client.txt b/proposals/181-optimistic-data-client.txt index 6d0a929..e1544df 100644 --- a/proposals/181-optimistic-data-client.txt +++ b/proposals/181-optimistic-data-client.txt @@ -1,3 +1,4 @@ +``` Filename: 181-optimistic-data-client.txt Title: Optimistic Data for Tor: Client Side Author: Ian Goldberg @@ -126,3 +127,4 @@ Performance and scalability notes: OPs may queue a little more data, if the SOCKS client pushes it faster than the OP can write it out. But that's also true today after the SOCKS CONNECT returns success, right? +``` diff --git a/proposals/182-creditbucket.txt b/proposals/182-creditbucket.txt index fc3cb17..b462a61 100644 --- a/proposals/182-creditbucket.txt +++ b/proposals/182-creditbucket.txt @@ -1,3 +1,4 @@ +``` Filename: 182-creditbucket.txt Title: Credit Bucket Author: Florian Tschorsch and Björn Scheuermann @@ -181,3 +182,4 @@ References: [1] Karsten Loesing. Analysis of Circuit Queues in Tor. August 25, 2009. [2] https://trac.torproject.org/projects/tor/wiki/sponsors/SponsorD/June2011 +``` diff --git a/proposals/183-refillintervals.txt b/proposals/183-refillintervals.txt index 81ff5b0..b2425fd 100644 --- a/proposals/183-refillintervals.txt +++ b/proposals/183-refillintervals.txt @@ -1,3 +1,4 @@ +``` Filename: 183-refillintervals.txt Title: Refill Intervals Author: Florian Tschorsch and Björn Scheuermann @@ -88,3 +89,4 @@ References: [1] Karsten Loesing. Analysis of Circuit Queues in Tor. August 25, 2009. [2] https://trac.torproject.org/projects/tor/wiki/sponsors/SponsorD/June2011 +``` diff --git a/proposals/184-v3-link-protocol.txt b/proposals/184-v3-link-protocol.txt index 820e779..5de9294 100644 --- a/proposals/184-v3-link-protocol.txt +++ b/proposals/184-v3-link-protocol.txt @@ -1,3 +1,4 @@ +``` Filename: 184-v3-link-protocol.txt Title: Miscellaneous changes for a v3 Tor link protocol Author: Nick Mathewson @@ -96,3 +97,4 @@ Notes on future-proofing: tweaking the link protocol to have a variable-length cell type encoding. +``` diff --git a/proposals/185-dir-without-dirport.txt b/proposals/185-dir-without-dirport.txt index 65cfd43..21c5b8c 100644 --- a/proposals/185-dir-without-dirport.txt +++ b/proposals/185-dir-without-dirport.txt @@ -1,3 +1,4 @@ +``` Filename: 185-dir-without-dirport.txt Title: Directory caches without DirPort Author: Nick Mathewson @@ -47,3 +48,4 @@ Consensus: as a directory cache. This does not require a new version of the consensus algorithm. +``` diff --git a/proposals/186-multiple-orports.txt b/proposals/186-multiple-orports.txt index f66ad83..f29d1c6 100644 --- a/proposals/186-multiple-orports.txt +++ b/proposals/186-multiple-orports.txt @@ -1,3 +1,4 @@ +``` Filename: 186-multiple-orports.txt Title: Multiple addresses for one OR or bridge Author: Nick Mathewson @@ -283,3 +284,4 @@ TODO: * Come up with a better alternative to running6 for the longterm? +``` diff --git a/proposals/187-allow-client-auth.txt b/proposals/187-allow-client-auth.txt index 67c878e..b461b92 100644 --- a/proposals/187-allow-client-auth.txt +++ b/proposals/187-allow-client-auth.txt @@ -1,3 +1,4 @@ +``` Filename: 187-allow-client-auth.txt Title: Reserve a cell type to allow client authorization Author: Nick Mathewson @@ -129,3 +130,4 @@ Counterargument: it's too late! break their connections only to a couple of alpha versions which one hopes by then will be long-deprecated already. +``` diff --git a/proposals/188-bridge-guards.txt b/proposals/188-bridge-guards.txt index 2a74416..51f7139 100644 --- a/proposals/188-bridge-guards.txt +++ b/proposals/188-bridge-guards.txt @@ -1,3 +1,4 @@ +``` Filename: 188-bridge-guards.txt Title: Bridge Guards and other anti-enumeration defenses Author: Nick Mathewson, Isis Lovecruft @@ -520,3 +521,4 @@ Status: Reserve What fraction of our traffic is bridge traffic? Will this alter our circuit selection weights? +``` diff --git a/proposals/189-authorize-cell.txt b/proposals/189-authorize-cell.txt index 4836c5f..14d9a77 100644 --- a/proposals/189-authorize-cell.txt +++ b/proposals/189-authorize-cell.txt @@ -1,3 +1,4 @@ +``` Filename: 189-authorize-cell.txt Title: AUTHORIZE and AUTHORIZED cells Author: George Kadianakis @@ -138,3 +139,4 @@ Status: Obsolete This proposal does not try to specify how a bridge can avoid detection by an adversary. +``` diff --git a/proposals/190-shared-secret-bridge-authorization.txt b/proposals/190-shared-secret-bridge-authorization.txt index 31c692b..8c9d45b 100644 --- a/proposals/190-shared-secret-bridge-authorization.txt +++ b/proposals/190-shared-secret-bridge-authorization.txt @@ -1,3 +1,4 @@ +``` Filename: 190-shared-secret-bridge-authorization.txt Title: Bridge Client Authorization Based on a Shared Secret Author: George Kadianakis @@ -101,3 +102,4 @@ Notes: This is obsoleted by pluggable transports. Thanks to Nick Mathewson and Robert Ransom for the help and suggestions while writing this proposal. +``` diff --git a/proposals/191-mitm-bridge-detection-resistance.txt b/proposals/191-mitm-bridge-detection-resistance.txt index a4689bb..4faf60d 100644 --- a/proposals/191-mitm-bridge-detection-resistance.txt +++ b/proposals/191-mitm-bridge-detection-resistance.txt @@ -1,3 +1,4 @@ +``` Filename: 191-mitm-bridge-detection-resistance.txt Title: Bridge Detection Resistance against MITM-capable Adversaries Author: George Kadianakis @@ -161,3 +162,4 @@ Status: Obsolete Thanks to Robert Ransom for his great help and suggestions on devising this scheme and writing this proposal! +``` diff --git a/proposals/192-store-bridge-information.txt b/proposals/192-store-bridge-information.txt index fe7d016..4e2dee0 100644 --- a/proposals/192-store-bridge-information.txt +++ b/proposals/192-store-bridge-information.txt @@ -1,3 +1,4 @@ +``` Filename: 192-store-bridge-information.txt Title: Automatically retrieve and store information about bridges Author: Sebastian Hahn @@ -135,3 +136,4 @@ the bridge authority side doesn't need any changes. The new configuration options governing the behaviour of Tor would be incompatible with previous versions, so the torrc needs to be adapted. The state file changes should not affect older versions. +``` diff --git a/proposals/193-safe-cookie-authentication.txt b/proposals/193-safe-cookie-authentication.txt index a3717fe..c97fcbb 100644 --- a/proposals/193-safe-cookie-authentication.txt +++ b/proposals/193-safe-cookie-authentication.txt @@ -1,3 +1,4 @@ +``` Filename: 193-safe-cookie-authentication.txt Title: Safe cookie authentication for Tor controllers Author: Robert Ransom @@ -142,3 +143,4 @@ Rationale: to keep controllers from using the server as an oracle for its own challenges (it could be replaced with a hash function). +``` diff --git a/proposals/194-mnemonic-urls.txt b/proposals/194-mnemonic-urls.txt index aeae092..0dfbfc0 100644 --- a/proposals/194-mnemonic-urls.txt +++ b/proposals/194-mnemonic-urls.txt @@ -1,3 +1,4 @@ +``` Filename: 194-mnemonic-urls.txt Title: Mnemonic .onion URLs Author: Sai, Alex Fink @@ -198,3 +199,4 @@ https://github.com/hellais/Onion-url/blob/master/dev/mnemonic.py [17] http://www.reddit.com/r/technology/comments/ecllk [18] http://wordnet.princeton.edu/wordnet/man2.1/wnstats.7WN.html +``` diff --git a/proposals/195-TLS-normalization-for-024.txt b/proposals/195-TLS-normalization-for-024.txt index bfc8d9a..8b3d8a9 100644 --- a/proposals/195-TLS-normalization-for-024.txt +++ b/proposals/195-TLS-normalization-for-024.txt @@ -1,3 +1,4 @@ +``` Filename: 195-TLS-normalization-for-024.txt Title: TLS certificate normalization for Tor 0.2.4.x Author: Jacob Appelbaum, Gladys Shufflebottom, Nick Mathewson, Tim Wilde @@ -168,3 +169,4 @@ Target: 0.2.4.x plausible way would probably be an implementation bug in OpenSSL or something. +``` diff --git a/proposals/196-transport-control-ports.txt b/proposals/196-transport-control-ports.txt index f4d8a0e..972ddb3 100644 --- a/proposals/196-transport-control-ports.txt +++ b/proposals/196-transport-control-ports.txt @@ -1,3 +1,4 @@ +``` Filename: 196-transport-control-ports.txt Title: Extended ORPort and TransportControlPort Author: George Kadianakis, Nick Mathewson @@ -238,3 +239,4 @@ Implemented-In: 0.2.5.2-alpha control information with the Tor client. The current proposal doesn't yet support this, but we should not add functionality that will prevent it from being possible. +``` diff --git a/proposals/197-postmessage-ipc.txt b/proposals/197-postmessage-ipc.txt index da246b3..e9ae6e6 100644 --- a/proposals/197-postmessage-ipc.txt +++ b/proposals/197-postmessage-ipc.txt @@ -1,3 +1,4 @@ +``` Filename: 197-postmessage-ipc.txt Title: Message-based Inter-Controller IPC Channel Author: Mike Perry @@ -97,3 +98,4 @@ Implementation plan needed. However, Torbutton does not have immediate need for this protocol. +``` diff --git a/proposals/198-restore-clienthello-semantics.txt b/proposals/198-restore-clienthello-semantics.txt index 2a3a560..9ec5aa3 100644 --- a/proposals/198-restore-clienthello-semantics.txt +++ b/proposals/198-restore-clienthello-semantics.txt @@ -1,3 +1,4 @@ +``` Filename: 198-restore-clienthello-semantics.txt Title: Restore semantics of TLS ClientHello Author: Nick Mathewson @@ -188,3 +189,4 @@ Open questions: implications of leaking cell types and circuit IDs to a future compromise. +``` diff --git a/proposals/199-bridgefinder-integration.txt b/proposals/199-bridgefinder-integration.txt index fbabf91..4c806e7 100644 --- a/proposals/199-bridgefinder-integration.txt +++ b/proposals/199-bridgefinder-integration.txt @@ -1,3 +1,4 @@ +``` Filename: 199-bridgefinder-integration.txt Title: Integration of BridgeFinder and BridgeFinderHelper Author: Mike Perry @@ -412,3 +413,4 @@ Security Concerns: BridgeFinder and BridgeFinderHelper encountered. +``` diff --git a/proposals/200-new-create-and-extend-cells.txt b/proposals/200-new-create-and-extend-cells.txt index 1a556dc..52934c9 100644 --- a/proposals/200-new-create-and-extend-cells.txt +++ b/proposals/200-new-create-and-extend-cells.txt @@ -1,3 +1,4 @@ +``` Filename: 200-new-create-and-extend-cells.txt Title: Adding new, extensible CREATE, EXTEND, and related cells Author: Robert Ransom @@ -153,3 +154,4 @@ Bugs: type that would require it. +``` diff --git a/proposals/201-bridge-v3-reqs-stats.txt b/proposals/201-bridge-v3-reqs-stats.txt index e267cba..81503d0 100644 --- a/proposals/201-bridge-v3-reqs-stats.txt +++ b/proposals/201-bridge-v3-reqs-stats.txt @@ -1,3 +1,4 @@ +``` Filename: 201-bridge-v3-reqs-stats.txt Title: Make bridges report statistics on daily v3 network status requests Author: Karsten Loesing @@ -36,3 +37,4 @@ Specification: [0] https://metrics.torproject.org/papers/countingusers-2010-11-30.pdf +``` diff --git a/proposals/202-improved-relay-crypto.txt b/proposals/202-improved-relay-crypto.txt index e9a1fd1..4913f98 100644 --- a/proposals/202-improved-relay-crypto.txt +++ b/proposals/202-improved-relay-crypto.txt @@ -1,3 +1,4 @@ +``` Filename: 202-improved-relay-crypto.txt Title: Two improved relay encryption protocols for Tor cells Author: Nick Mathewson @@ -817,3 +818,4 @@ R. References [23R] The 23 Raccoons, "Analysis of the Relative Severity of Tagging Attacks" http://archives.seul.org/or/dev/Mar-2012/msg00019.html (You'll want to read the rest of the thread too.) +``` diff --git a/proposals/203-https-frontend.txt b/proposals/203-https-frontend.txt index 4bbb271..e9e83c1 100644 --- a/proposals/203-https-frontend.txt +++ b/proposals/203-https-frontend.txt @@ -1,3 +1,4 @@ +``` Filename: 203-https-frontend.txt Title: Avoiding censorship by impersonating an HTTPS server Author: Nick Mathewson @@ -275,3 +276,4 @@ Related work: (high collateral damage). 1. https://trac.torproject.org/projects/tor/wiki/doc/meek +``` diff --git a/proposals/204-hidserv-subdomains.txt b/proposals/204-hidserv-subdomains.txt index 056fbad..aee5727 100644 --- a/proposals/204-hidserv-subdomains.txt +++ b/proposals/204-hidserv-subdomains.txt @@ -1,3 +1,4 @@ +``` Filename: 204-hidserv-subdomains.txt Title: Subdomain support for Hidden Service addresses Author: Alessandro Preite Martinez @@ -27,3 +28,4 @@ Status: Closed simply as "aaaaaaaaaaaaaaaa.onion". +``` diff --git a/proposals/205-local-dnscache.txt b/proposals/205-local-dnscache.txt index d0a0107..b7c8aa0 100644 --- a/proposals/205-local-dnscache.txt +++ b/proposals/205-local-dnscache.txt @@ -1,3 +1,4 @@ +``` Filename: 205-local-dnscache.txt Title: Remove global client-side DNS caching Author: Nick Mathewson @@ -157,3 +158,4 @@ Status: Closed without most of the security issues, if we restrict it to DNSSec-signed answers. +``` diff --git a/proposals/206-directory-sources.txt b/proposals/206-directory-sources.txt index d81b2f6..bdb6481 100644 --- a/proposals/206-directory-sources.txt +++ b/proposals/206-directory-sources.txt @@ -1,3 +1,4 @@ +``` Filename: 206-directory-sources.txt Title: Preconfigured directory sources for bootstrapping Author: Nick Mathewson @@ -87,3 +88,4 @@ Some notes on security: +``` diff --git a/proposals/207-directory-guards.txt b/proposals/207-directory-guards.txt index 8d21d3e..51dcdbf 100644 --- a/proposals/207-directory-guards.txt +++ b/proposals/207-directory-guards.txt @@ -1,3 +1,4 @@ +``` Filename: 207-directory-guards.txt Title: Directory guards Author: Nick Mathewson @@ -52,3 +53,4 @@ Open questions and notes: +``` diff --git a/proposals/208-ipv6-exits-redux.txt b/proposals/208-ipv6-exits-redux.txt index 32fb09c..fd0adfa 100644 --- a/proposals/208-ipv6-exits-redux.txt +++ b/proposals/208-ipv6-exits-redux.txt @@ -1,3 +1,4 @@ +``` Filename: 208-ipv6-exits-redux.txt Title: IPv6 Exits Redux Author: Nick Mathewson @@ -125,3 +126,4 @@ Implemented-In: 0.2.4.7-alpha This makes proposal 205 especially necessary here. +``` diff --git a/proposals/209-path-bias-tuning.txt b/proposals/209-path-bias-tuning.txt index 7d02422..fcdaf78 100644 --- a/proposals/209-path-bias-tuning.txt +++ b/proposals/209-path-bias-tuning.txt @@ -1,3 +1,4 @@ +``` Filename: 209-path-bias-tuning.txt Title: Tuning the Parameters for the Path Bias Defense Author: Mike Perry @@ -286,3 +287,4 @@ Implementation Notes: Differences between proposal and current source 3. https://gitweb.torproject.org/torflow.git/tree/HEAD:/CircuitAnalysis/PathBias 4. https://github.com/meejah/txtorcon/blob/exit_scanner/apps/exit_scanner/failure-rate-scanner.py 5. See 2.4.1 of path-spec.txt for further details on circuit timeout calculations. +``` diff --git a/proposals/210-faster-headless-consensus-bootstrap.txt b/proposals/210-faster-headless-consensus-bootstrap.txt index 380a9ac..3bf8760 100644 --- a/proposals/210-faster-headless-consensus-bootstrap.txt +++ b/proposals/210-faster-headless-consensus-bootstrap.txt @@ -1,3 +1,4 @@ +``` Filename: 210-faster-headless-consensus-bootstrap.txt Title: Faster Headless Consensus Bootstrapping Author: Mike Perry, Tim Wilson-Brown, Peter Palfrader @@ -236,3 +237,4 @@ Reliability Analysis among a larger number of directory mirrors. In addition, if all the authorities are blocked, current clients will inevitably fail, as they do not have a list of directory mirrors. +``` diff --git a/proposals/211-mapaddress-tor-status.txt b/proposals/211-mapaddress-tor-status.txt index 28be3bb..5dbe445 100644 --- a/proposals/211-mapaddress-tor-status.txt +++ b/proposals/211-mapaddress-tor-status.txt @@ -1,3 +1,4 @@ +``` Filename: 211-mapaddress-tor-status.txt Title: Internal Mapaddress for Tor Configuration Testing Author: Mike Perry @@ -143,3 +144,4 @@ Security Considerations 1. http://www.hpenterprisesecurity.com/vulncat/en/vulncat/dotnet/javascript_hijacking_vulnerable_framework.html +``` diff --git a/proposals/212-using-old-consensus.txt b/proposals/212-using-old-consensus.txt index 3fd9839..621466e 100644 --- a/proposals/212-using-old-consensus.txt +++ b/proposals/212-using-old-consensus.txt @@ -1,3 +1,4 @@ +``` Filename: 212-using-old-consensus.txt Title: Increase Acceptable Consensus Age Author: Mike Perry @@ -107,3 +108,4 @@ Implementation Notes still attempting to fetch new ones. 1. https://trac.torproject.org/projects/tor/ticket/7126 +``` diff --git a/proposals/213-remove-stream-sendmes.txt b/proposals/213-remove-stream-sendmes.txt index f2b8b34..18b51f3 100644 --- a/proposals/213-remove-stream-sendmes.txt +++ b/proposals/213-remove-stream-sendmes.txt @@ -1,3 +1,4 @@ +``` Filename: 213-remove-stream-sendmes.txt Title: Remove stream-level sendmes from the design Author: Roger Dingledine @@ -150,3 +151,4 @@ Status: Dead since it lacks any stream-level pushback for streams that are blocking on writes. Nicely done! +``` diff --git a/proposals/214-longer-circids.txt b/proposals/214-longer-circids.txt index b5bd88a..54675ac 100644 --- a/proposals/214-longer-circids.txt +++ b/proposals/214-longer-circids.txt @@ -1,3 +1,4 @@ +``` Filename: 214-longer-circids.txt Title: Allow 4-byte circuit IDs in a new link protocol Author: Nick Mathewson @@ -92,3 +93,4 @@ Implemented-In: 0.2.4.11-alpha implementation in branch "wide_circ_ids" in my public repository. Be afraid! More testing is needed! +``` diff --git a/proposals/215-update-min-consensus-ver.txt b/proposals/215-update-min-consensus-ver.txt index 899a943..c6fba59 100644 --- a/proposals/215-update-min-consensus-ver.txt +++ b/proposals/215-update-min-consensus-ver.txt @@ -1,3 +1,4 @@ +``` Filename: 215-update-min-consensus-ver.txt Title: Let the minimum consensus method change with time Author: Nick Mathewson @@ -93,3 +94,4 @@ Implemented-In: 0.2.6.1-alpha at all." That's harder to program, though. +``` diff --git a/proposals/216-ntor-handshake.txt b/proposals/216-ntor-handshake.txt index f76e81c..a96dd83 100644 --- a/proposals/216-ntor-handshake.txt +++ b/proposals/216-ntor-handshake.txt @@ -1,3 +1,4 @@ +``` Filename: 216-ntor-handshake.txt Title: Improved circuit-creation key exchange Author: Nick Mathewson @@ -211,3 +212,4 @@ Test vectors for HKDF-SHA256: b4dc21d69363e2895321c06184879d94b18f0784 11be70b767c7fc40679a9440a0c95ea83a23efbf +``` diff --git a/proposals/217-ext-orport-auth.txt b/proposals/217-ext-orport-auth.txt index 3734924..6a61a65 100644 --- a/proposals/217-ext-orport-auth.txt +++ b/proposals/217-ext-orport-auth.txt @@ -1,3 +1,4 @@ +``` Filename: 217-ext-orport-auth.txt Title: Tor Extended ORPort Authentication Author: George Kadianakis @@ -175,3 +176,4 @@ http://archives.seul.org/or/announce/Sep-2007/msg00000.html [1]: https://gitweb.torproject.org/torspec.git/blob/79f488c32c43562522e5592f2c19952dc7681a65:/control-spec.txt#l1069 +``` diff --git a/proposals/218-usage-controller-events.txt b/proposals/218-usage-controller-events.txt index 76c64cd..f709550 100644 --- a/proposals/218-usage-controller-events.txt +++ b/proposals/218-usage-controller-events.txt @@ -1,3 +1,4 @@ +``` Filename: 218-usage-controller-events.txt Title: Controller events to better understand connection/circuit usage Author: Rob Jansen, Karsten Loesing @@ -219,3 +220,4 @@ Implemented-In: 0.2.5.2-alpha as small as possible. Still, we should make sure that memory requirements won't grow significantly on busy relays. +``` diff --git a/proposals/219-expanded-dns.txt b/proposals/219-expanded-dns.txt index 7298b78..86b3f1a 100644 --- a/proposals/219-expanded-dns.txt +++ b/proposals/219-expanded-dns.txt @@ -1,3 +1,4 @@ +``` Filename: 219-expanded-dns.txt Title: Support for full DNS and DNSSEC resolution in Tor Authors: Ondrej Mikle @@ -346,3 +347,4 @@ Status: Needs-Revision References: [1] https://www.ietf.org/mail-archive/web/dane/current/msg02823.html +``` diff --git a/proposals/220-ecc-id-keys.txt b/proposals/220-ecc-id-keys.txt index 21b6dac..a6e6b54 100644 --- a/proposals/220-ecc-id-keys.txt +++ b/proposals/220-ecc-id-keys.txt @@ -1,3 +1,4 @@ +``` Filename: 220-ecc-id-keys.txt Title: Migrate server identity keys to Ed25519 Authors: Nick Mathewson @@ -683,3 +684,4 @@ A.6. Related changes keys. This will improve link security, and deliver better fingerprinting resistence. See proposal 179 for an older discussion of this issue. +``` diff --git a/proposals/221-stop-using-create-fast.txt b/proposals/221-stop-using-create-fast.txt index 7903a39..2fa2209 100644 --- a/proposals/221-stop-using-create-fast.txt +++ b/proposals/221-stop-using-create-fast.txt @@ -1,3 +1,4 @@ +``` Filename: 221-stop-using-create-fast.txt Title: Stop using CREATE_FAST Authors: Nick Mathewson @@ -88,3 +89,4 @@ Status: Closed reliable signal, since "FastFirstHopPK 0" is easy enough to type, and the source code is easy enough to edit. Proposal 163 and its successors have better ideas here anyway. +``` diff --git a/proposals/222-remove-client-timestamps.txt b/proposals/222-remove-client-timestamps.txt index b2a2ae9..3af19d1 100644 --- a/proposals/222-remove-client-timestamps.txt +++ b/proposals/222-remove-client-timestamps.txt @@ -1,3 +1,4 @@ +``` Filename: 222-remove-client-timestamps.txt Title: Stop sending client timestamps Authors: Nick Mathewson @@ -248,3 +249,4 @@ Implementors might want to make this feature optional and on-by-default, just in case some higher-level application protocol really does depend on it. ================================================================== +``` diff --git a/proposals/223-ace-handshake.txt b/proposals/223-ace-handshake.txt index 5e333bb..70d34f4 100644 --- a/proposals/223-ace-handshake.txt +++ b/proposals/223-ace-handshake.txt @@ -1,3 +1,4 @@ +``` Filename: 223-ace-handshake.txt Title: Ace: Improved circuit-creation key exchange Author: Esfandiar Mohammadi, Aniket Kate, Michael Backes @@ -110,3 +111,4 @@ See also: http://www.infsec.cs.uni-saarland.de/~mohammadi/ace/ace.html for implementations, academic paper, and benchmarking code. +``` diff --git a/proposals/224-rend-spec-ng.txt b/proposals/224-rend-spec-ng.txt index 1dfe87b..e1cba7f 100644 --- a/proposals/224-rend-spec-ng.txt +++ b/proposals/224-rend-spec-ng.txt @@ -1,3 +1,4 @@ +``` Filename: 224-rend-spec-ng.txt Title: Next-Generation Hidden Services in Tor Author: David Goulet, George Kadianakis, Nick Mathewson @@ -2368,3 +2369,4 @@ Appendix E. Managing authorized client data [CLIENT-AUTH-MGMT] [XXX what happens when people use both the control port interface and the filesystem interface?] +``` diff --git a/proposals/225-strawman-shared-rand.txt b/proposals/225-strawman-shared-rand.txt index f124174..6d71327 100644 --- a/proposals/225-strawman-shared-rand.txt +++ b/proposals/225-strawman-shared-rand.txt @@ -1,3 +1,4 @@ +``` Filename: 225-strawman-shared-rand.txt Title: Strawman proposal: commit-and-reveal shared rng Author: Nick Mathewson @@ -112,3 +113,4 @@ Superseded-by: 250 (TODO: should this be its own spec? If so, does it have to use our regular metaformat or can it use something less sucky?) +``` diff --git a/proposals/226-bridgedb-database-improvements.txt b/proposals/226-bridgedb-database-improvements.txt index dd52ae6..d239a4a 100644 --- a/proposals/226-bridgedb-database-improvements.txt +++ b/proposals/226-bridgedb-database-improvements.txt @@ -1,3 +1,4 @@ +``` Filename: 226-bridgedb-database-improvements.txt Title: "Scalability and Stability Improvements to BridgeDB: Switching to a Distributed Database System and RDBMS" @@ -256,3 +257,4 @@ Status: Reserve [15]: http://redis.io/commands/evalsha [16]: http://www.restmq.com/ [17]: https://www.mediawiki.org/wiki/Redis +``` diff --git a/proposals/227-vote-on-package-fingerprints.txt b/proposals/227-vote-on-package-fingerprints.txt index 8839941..8b0b472 100644 --- a/proposals/227-vote-on-package-fingerprints.txt +++ b/proposals/227-vote-on-package-fingerprints.txt @@ -1,3 +1,4 @@ +``` Filename: 227-vote-on-package-fingerprints.txt Title: Include package fingerprints in consensus documents Author: Nick Mathewson, Mike Perry @@ -145,3 +146,4 @@ Implemented-In: 0.2.6.3-alpha tool. +``` diff --git a/proposals/228-cross-certification-onionkeys.txt b/proposals/228-cross-certification-onionkeys.txt index 64fe5f6..50413d3 100644 --- a/proposals/228-cross-certification-onionkeys.txt +++ b/proposals/228-cross-certification-onionkeys.txt @@ -1,3 +1,4 @@ +``` Filename: 228-cross-certification-onionkeys.txt Title: Cross-certifying identity keys with onion keys Author: Nick Mathewson @@ -166,3 +167,4 @@ C. Implementation notes * An ntor onion key. +``` diff --git a/proposals/229-further-socks5-extensions.txt b/proposals/229-further-socks5-extensions.txt index 5661536..390750f 100644 --- a/proposals/229-further-socks5-extensions.txt +++ b/proposals/229-further-socks5-extensions.txt @@ -1,3 +1,4 @@ +``` Filename: 229-further-socks5-extensions.txt Title: Further SOCKS5 extensions Author: Yawning Angel @@ -252,3 +253,4 @@ Note: These are good ideas, but it's better not to hack SOCKS any further * check it into git * clean text a little, fix redundancy * ask some questions +``` diff --git a/proposals/230-rsa1024-relay-id-migration.txt b/proposals/230-rsa1024-relay-id-migration.txt index 3a997fe..92e72d8 100644 --- a/proposals/230-rsa1024-relay-id-migration.txt +++ b/proposals/230-rsa1024-relay-id-migration.txt @@ -1,3 +1,4 @@ +``` Filename: 230-rsa1024-relay-id-migration.txt Title: How to change RSA1024 relay identity keys Authors: Nick Mathewson @@ -83,3 +84,4 @@ Note: Obsoleted by Ed25519 ID keys; superseded by 240 and 256. the descriptor. +``` diff --git a/proposals/231-migrate-authority-rsa1024-ids.txt b/proposals/231-migrate-authority-rsa1024-ids.txt index 0388cf0..af5c6af 100644 --- a/proposals/231-migrate-authority-rsa1024-ids.txt +++ b/proposals/231-migrate-authority-rsa1024-ids.txt @@ -1,3 +1,4 @@ +``` Filename: 231-migrate-authority-rsa1024-ids.txt Title: Migrating authority RSA1024 identity keys Authors: Nick Mathewson @@ -68,3 +69,4 @@ Note: Obsoleted by Ed25519 ID keys; superseded by 240 and 256. but not include them in votes or the consensus; or if they are included, don't assign them flags that will get them used. +``` diff --git a/proposals/232-pluggable-transports-through-proxy.txt b/proposals/232-pluggable-transports-through-proxy.txt index 03cee07..53b1524 100644 --- a/proposals/232-pluggable-transports-through-proxy.txt +++ b/proposals/232-pluggable-transports-through-proxy.txt @@ -1,3 +1,4 @@ +``` Filename: 232-pluggable-transports-through-proxy.txt Title: Pluggable Transport through SOCKS proxy Author: Arturo Filastò @@ -77,3 +78,4 @@ Specifications: Tor Pluggable Transport communication from its stdout and it is configured to use both a normal proxy and a pluggable transport it should kill the transport proxy. +``` diff --git a/proposals/233-quicken-tor2web-mode.txt b/proposals/233-quicken-tor2web-mode.txt index fd002c0..2821f36 100644 --- a/proposals/233-quicken-tor2web-mode.txt +++ b/proposals/233-quicken-tor2web-mode.txt @@ -1,3 +1,4 @@ +``` Filename: 233-quicken-tor2web-mode.txt Title: Making Tor2Web mode faster Author: Virgil Griffith, Fabio Pietrosanti, Giovanni Pellerano @@ -33,3 +34,4 @@ Status: Rejected [1] Tor2web mode: https://trac.torproject.org/projects/tor/ticket/2553 [2] Self-rendezvous: https://trac.torproject.org/projects/tor/ticket/9685 +``` diff --git a/proposals/234-remittance-addresses.txt b/proposals/234-remittance-addresses.txt index 712c1ed..e49830d 100644 --- a/proposals/234-remittance-addresses.txt +++ b/proposals/234-remittance-addresses.txt @@ -1,3 +1,4 @@ +``` Filename: 234-remittance-addresses.txt Title: Adding remittance field to directory specification Author: Virgil Griffith, Leif Ryge, Rob Jansen @@ -38,3 +39,4 @@ Note: Rejected. People are doing this with ContactInfo lines. (left-most) one of each currency is published in the directory consensus. +``` diff --git a/proposals/235-kill-named-flag.txt b/proposals/235-kill-named-flag.txt index 238315a..17688ed 100644 --- a/proposals/235-kill-named-flag.txt +++ b/proposals/235-kill-named-flag.txt @@ -1,3 +1,4 @@ +``` Filename: 235-kill-named-flag.txt Title: Stop assigning (and eventually supporting) the Named flag Authors: Sebastian Hahn @@ -51,3 +52,4 @@ Status: Closed 4. Open questions None. +``` diff --git a/proposals/236-single-guard-node.txt b/proposals/236-single-guard-node.txt index 7db8a67..50c734b 100644 --- a/proposals/236-single-guard-node.txt +++ b/proposals/236-single-guard-node.txt @@ -1,3 +1,4 @@ +``` Filename: 236-single-guard-node.txt Title: The move to a single guard node Author: George Kadianakis, Nicholas Hopper @@ -304,3 +305,4 @@ References: [6]: https://trac.torproject.org/projects/tor/ticket/9001 [7]: https://trac.torproject.org/projects/tor/ticket/10969 +``` diff --git a/proposals/237-directory-servers-for-all.txt b/proposals/237-directory-servers-for-all.txt index 130bd32..a25352f 100644 --- a/proposals/237-directory-servers-for-all.txt +++ b/proposals/237-directory-servers-for-all.txt @@ -1,3 +1,4 @@ +``` Filename: 237-directory-servers-for-all.txt Title: All relays are directory servers Author: Matthew Finkel @@ -131,3 +132,4 @@ Future Considerations: Is V2Dir a good name for this? It's the name we currently use, but that's a silly reason to continue using it. +``` diff --git a/proposals/238-hs-relay-stats.txt b/proposals/238-hs-relay-stats.txt index 9134003..7b263bd 100644 --- a/proposals/238-hs-relay-stats.txt +++ b/proposals/238-hs-relay-stats.txt @@ -1,3 +1,4 @@ +``` Filename: 238-hs-relay-stats.txt Title: Better hidden service stats from Tor relays Author: George Kadianakis, David Goulet, Karsten Loesing, Aaron Johnson @@ -291,3 +292,4 @@ Status: Closed [ONIONS-LAPLACE-GRAPH]: https://raw.githubusercontent.com/corcra/pioton/master/vis/laplacePDF_mu0.0_b26.67.png https://raw.githubusercontent.com/corcra/pioton/master/vis/laplaceCDF_mu0.0_b26.67.png +``` diff --git a/proposals/239-consensus-hash-chaining.txt b/proposals/239-consensus-hash-chaining.txt index 0c259dd..72738a2 100644 --- a/proposals/239-consensus-hash-chaining.txt +++ b/proposals/239-consensus-hash-chaining.txt @@ -1,3 +1,4 @@ +``` Filename: 239-consensus-hash-chaining.txt Title: Consensus Hash Chaining Author: Nick Mathewson, Andrea Shepard @@ -97,3 +98,4 @@ Status: Open * Clients MAY offer the option to retain previous consensuses in memory only to allow for validation without the potential disk leak. +``` diff --git a/proposals/240-auth-cert-revocation.txt b/proposals/240-auth-cert-revocation.txt index ae7d4b8..1128671 100644 --- a/proposals/240-auth-cert-revocation.txt +++ b/proposals/240-auth-cert-revocation.txt @@ -1,3 +1,4 @@ +``` Filename: 240-auth-cert-revocation.txt Title: Early signing key revocation for directory authorities Author: Nick Mathewson @@ -46,3 +47,4 @@ Status: Open of certificate B, and certificate B revokes the key of certificate A. Instead, I am inclined to say that this is a MUST NOT. +``` diff --git a/proposals/241-suspicious-guard-turnover.txt b/proposals/241-suspicious-guard-turnover.txt index 004feb1..16ccfb4 100644 --- a/proposals/241-suspicious-guard-turnover.txt +++ b/proposals/241-suspicious-guard-turnover.txt @@ -1,3 +1,4 @@ +``` Filename: 241-suspicious-guard-turnover.txt Title: Resisting guard-turnover attacks Author: Aaron Johnson, Nick Mathewson @@ -180,3 +181,4 @@ Appendix B. Acknowledgements Appendix C. Desirable revisions Incorporate ideas from proposal 156. +``` diff --git a/proposals/242-better-families.txt b/proposals/242-better-families.txt index 5f4ac47..c2edb54 100644 --- a/proposals/242-better-families.txt +++ b/proposals/242-better-families.txt @@ -1,3 +1,4 @@ +``` Filename: 242-better-families.txt Title: Better performance and usability for the MyFamily option Author: Nick Mathewson @@ -95,3 +96,4 @@ Superseded-by: 321-happy-families.md It's possible that families are a bad idea entirely; see ticket #6676. +``` diff --git a/proposals/243-hsdir-flag-need-stable.txt b/proposals/243-hsdir-flag-need-stable.txt index 035550b..1ed0245 100644 --- a/proposals/243-hsdir-flag-need-stable.txt +++ b/proposals/243-hsdir-flag-need-stable.txt @@ -1,3 +1,4 @@ +``` Filename: 243-hsdir-flag-need-stable.txt Title: Give out HSDir flag only to relays with Stable flag Author: George Kadianakis @@ -82,3 +83,4 @@ Implemented-in: 0.2.7 With the upcoming keyblinding scheme (#8106) and non-deterministic HSDir selection (#8244), are there any other criteria that we should use when assigning HSDir flags? +``` diff --git a/proposals/244-use-rfc5705-for-tls-binding.txt b/proposals/244-use-rfc5705-for-tls-binding.txt index 41ebfc1..faa148c 100644 --- a/proposals/244-use-rfc5705-for-tls-binding.txt +++ b/proposals/244-use-rfc5705-for-tls-binding.txt @@ -1,3 +1,4 @@ +``` Filename: 244-use-rfc5705-for-tls-binding.txt Title: Use RFC5705 Key Exporting in our AUTHENTICATE calls Author: Nick Mathewson @@ -44,3 +45,4 @@ Implemented-In: 0.3.0.1-alpha * The label string "EXPORTER FOR TOR TLS CLIENT BINDING " + TYPE * The context value equal to the client's Ed25519 identity key * The length 32. +``` diff --git a/proposals/245-tap-out.txt b/proposals/245-tap-out.txt index 4a9afd8..6edc3f9 100644 --- a/proposals/245-tap-out.txt +++ b/proposals/245-tap-out.txt @@ -1,3 +1,4 @@ +``` Filename: 245-tap-out.txt Title: Deprecating and removing the TAP circuit extension protocol Author: Nick Mathewson @@ -94,3 +95,4 @@ Status: Needs-Revision (For reference, the functions to patch would be rend_client_get_random_intro_impl and find_rp_for_intro.) +``` diff --git a/proposals/246-merge-hsdir-and-intro.txt b/proposals/246-merge-hsdir-and-intro.txt index 0a70aea..e9aa780 100644 --- a/proposals/246-merge-hsdir-and-intro.txt +++ b/proposals/246-merge-hsdir-and-intro.txt @@ -1,3 +1,4 @@ +``` Filename: 246-merge-hsdir-and-intro.txt Title: Merging Hidden Service Directories and Introduction Points Author: John Brooks, George Kadianakis @@ -302,3 +303,4 @@ Change history: References: [1] : https://lists.torproject.org/pipermail/tor-dev/2016-January/010203.html +``` diff --git a/proposals/247-hs-guard-discovery.txt b/proposals/247-hs-guard-discovery.txt index a41e025..a42161b 100644 --- a/proposals/247-hs-guard-discovery.txt +++ b/proposals/247-hs-guard-discovery.txt @@ -1,3 +1,4 @@ +``` Filename: 247-hs-guard-discovery.txt Title: Defending Against Guard Discovery Attacks using Vanguards Authors: George Kadianakis and Mike Perry @@ -757,3 +758,4 @@ CDF_table_323() ---------------------- 1. https://onionbalance.readthedocs.org/en/latest/design.html#overview +``` diff --git a/proposals/248-removing-rsa-identities.txt b/proposals/248-removing-rsa-identities.txt index eaabe41..3f05df3 100644 --- a/proposals/248-removing-rsa-identities.txt +++ b/proposals/248-removing-rsa-identities.txt @@ -1,3 +1,4 @@ +``` Filename: 248-removing-rsa-identities.txt Title: Remove all RSA identity keys Authors: Nick Mathewson @@ -86,3 +87,4 @@ Status: Needs-Revision make the phase 2 and phase 3 changes. +``` diff --git a/proposals/249-large-create-cells.txt b/proposals/249-large-create-cells.txt index 4be3e26..fdc80e2 100644 --- a/proposals/249-large-create-cells.txt +++ b/proposals/249-large-create-cells.txt @@ -1,3 +1,4 @@ +``` Filename: 249-large-create-cells.txt Title: Allow CREATE cells with >505 bytes of handshake data Authors: Nick Mathewson, Isis Lovecruft @@ -276,3 +277,4 @@ Appendix B. Acknowledgements This research was supported in part by NSF grants CNS-1111539, CNS-1314637, CNS-1526306, CNS-1619454, and CNS-1640548. +``` diff --git a/proposals/250-commit-reveal-consensus.txt b/proposals/250-commit-reveal-consensus.txt index c9711aa..f9821e9 100644 --- a/proposals/250-commit-reveal-consensus.txt +++ b/proposals/250-commit-reveal-consensus.txt @@ -1,3 +1,4 @@ +``` Filename: 250-commit-reveal-consensus.txt Title: Random Number Generation During Tor Voting Authors: David Goulet, George Kadianakis @@ -640,3 +641,4 @@ References: [RNGMESSAGING]: https://moderncrypto.org/mail-archive/messaging/2015/002032.html +``` diff --git a/proposals/251-netflow-padding.txt b/proposals/251-netflow-padding.txt index 39cc795..c7a2ef4 100644 --- a/proposals/251-netflow-padding.txt +++ b/proposals/251-netflow-padding.txt @@ -1,3 +1,4 @@ +``` Filename: 251-netflow-padding.txt Title: Padding for netflow record resolution reduction Authors: Mike Perry @@ -363,3 +364,4 @@ Appendix A: Padding Cell Timeout Distribution Statistics 13. https://gitweb.torproject.org/torspec.git/tree/proposals/188-bridge-guards.txt 14. http://www.ntop.org/wp-content/uploads/2013/03/nProbe_UserGuide.pdf 15. http://infodoc.alcatel-lucent.com/html/0_add-h-f/93-0073-10-01/7750_SR_OS_Router_Configuration_Guide/Cflowd-CLI.html +``` diff --git a/proposals/252-single-onion.txt b/proposals/252-single-onion.txt index ddd78ea..c9b4c24 100644 --- a/proposals/252-single-onion.txt +++ b/proposals/252-single-onion.txt @@ -1,3 +1,4 @@ +``` Filename: 252-single-onion.txt Title: Single Onion Services Author: John Brooks, Paul Syverson, Roger Dingledine @@ -257,3 +258,4 @@ Superseded-by: 260 Tor circuit. In speech we sometimes use the even briefer, 'two-nion' and 'one-ion' respectively. +``` diff --git a/proposals/253-oob-hmac.txt b/proposals/253-oob-hmac.txt index 9e9e1d7..c7abbd9 100644 --- a/proposals/253-oob-hmac.txt +++ b/proposals/253-oob-hmac.txt @@ -1,3 +1,4 @@ +``` Filename: 253-oob-hmac.txt Title: Out of Band Circuit HMACs Authors: Mike Perry @@ -270,3 +271,4 @@ AES-NI[2]. 1. https://lists.torproject.org/pipermail/tor-dev/2012-March/003347.html 2. http://archives.seul.org/tor/dev/Mar-2015/msg00137.html +``` diff --git a/proposals/254-padding-negotiation.txt b/proposals/254-padding-negotiation.txt index 5ab5341..84b1d1b 100644 --- a/proposals/254-padding-negotiation.txt +++ b/proposals/254-padding-negotiation.txt @@ -1,3 +1,4 @@ +``` Filename: 254-padding-negotiation.txt Title: Padding Negotiation Authors: Mike Perry @@ -416,3 +417,4 @@ close the circuit upon detecting this activity. 1. https://gitweb.torproject.org/torspec.git/tree/proposals/251-netflow-padding.txt 2. https://www.cs.kau.se/pulls/hot/thebasketcase-wtfpad/ 3. https://www.cs.kau.se/pulls/hot/thebasketcase-ape/ +``` diff --git a/proposals/255-hs-load-balancing.txt b/proposals/255-hs-load-balancing.txt index 16657d0..136969d 100644 --- a/proposals/255-hs-load-balancing.txt +++ b/proposals/255-hs-load-balancing.txt @@ -1,3 +1,4 @@ +``` Filename: 255-hs-load-balancing.txt Title: Controller features to allow for load-balancing hidden services Author: Tom van der Woerdt @@ -156,3 +157,4 @@ the controller. The developer of the tor implementation should not have to choose between a round-robin algorithm and something that could pull CPU load averages from a centralized monitoring system. +``` diff --git a/proposals/256-key-revocation.txt b/proposals/256-key-revocation.txt index bbccaf9..e512471 100644 --- a/proposals/256-key-revocation.txt +++ b/proposals/256-key-revocation.txt @@ -1,3 +1,4 @@ +``` Filename: 256-key-revocation.txt Title: Key revocation for relays and authorities Authors: Nick Mathewson @@ -240,3 +241,4 @@ Status: Reserve NOT trust any signing key certificate with a matching key. +``` diff --git a/proposals/257-hiding-authorities.txt b/proposals/257-hiding-authorities.txt index 9c32f90..cdb49d9 100644 --- a/proposals/257-hiding-authorities.txt +++ b/proposals/257-hiding-authorities.txt @@ -1,3 +1,4 @@ +``` Filename: 257-hiding-authorities.txt Title: Refactoring authorities and making them more isolated from the net Authors: Nick Mathewson, Andrea Shepard @@ -211,3 +212,4 @@ A.1. Check-then-start TLS (We shouldn't do this for all the TLS in Tor: only for the cases where we want to restrict the users of a given TLS server.) +``` diff --git a/proposals/258-dirauth-dos.txt b/proposals/258-dirauth-dos.txt index 91ffd37..aa65d79 100644 --- a/proposals/258-dirauth-dos.txt +++ b/proposals/258-dirauth-dos.txt @@ -1,3 +1,4 @@ +``` Filename: 258-dirauth-dos.txt Title: Denial-of-service resistance for directory authorities Author: Andrea Shepard @@ -109,3 +110,4 @@ Status: Dead Widely distributed DoS attacks with many source IPs may still be able to avoid raising any single DIRIND_ONEHOP or DIRIND_DIRECT_CONN counter above threshold. +``` diff --git a/proposals/259-guard-selection.txt b/proposals/259-guard-selection.txt index 30bdcfa..76ee9b5 100644 --- a/proposals/259-guard-selection.txt +++ b/proposals/259-guard-selection.txt @@ -1,3 +1,4 @@ +``` Filename: 259-guard-selection.txt Title: New Guard Selection Behaviour Author: Isis Lovecruft, George Kadianakis @@ -302,3 +303,4 @@ This proposal was made obsolete by proposal #271. -*- coding: utf-8 -*- +``` diff --git a/proposals/260-rend-single-onion.txt b/proposals/260-rend-single-onion.txt index c4f2617..08ca33a 100644 --- a/proposals/260-rend-single-onion.txt +++ b/proposals/260-rend-single-onion.txt @@ -1,3 +1,4 @@ +``` Filename: 260-rend-single-onion.txt Title: Rendezvous Single Onion Services Author: Tim Wilson-Brown, John Brooks, Aaron Johnson, Rob Jansen, George Kadianakis, Paul Syverson, Roger Dingledine @@ -460,3 +461,4 @@ splitting described in section 8. Here are some initial ideas. be more comfortable enabling submission over an onion service, due to the additional security benefits. +``` diff --git a/proposals/261-aez-crypto.txt b/proposals/261-aez-crypto.txt index f9c652e..1a6c108 100644 --- a/proposals/261-aez-crypto.txt +++ b/proposals/261-aez-crypto.txt @@ -1,3 +1,4 @@ +``` Filename: 261-aez-crypto.txt Title: AEZ for relay cryptography Author: Nick Mathewson @@ -280,3 +281,4 @@ A.3. Performance notes: what if we don't have AESNI? Preliminary results are a bit disappointing, though, so I'll need to invetigate alternatives as well.] +``` diff --git a/proposals/262-rekey-circuits.txt b/proposals/262-rekey-circuits.txt index f639d1d..9f933f0 100644 --- a/proposals/262-rekey-circuits.txt +++ b/proposals/262-rekey-circuits.txt @@ -1,3 +1,4 @@ +``` Filename: 262-rekey-circuits.txt Title: Re-keying live circuits with new cryptographic material Author: Nick Mathewson @@ -132,3 +133,4 @@ Status: Reserve in the presence of traffic analysis. +``` diff --git a/proposals/263-ntru-for-pq-handshake.txt b/proposals/263-ntru-for-pq-handshake.txt index 6a3353e..f4d97d4 100644 --- a/proposals/263-ntru-for-pq-handshake.txt +++ b/proposals/263-ntru-for-pq-handshake.txt @@ -1,3 +1,4 @@ +``` Filename: 263-ntru-for-pq-handshake.txt Title: Request to change key exchange protocol for handshake v1.2 Author: John SCHANCK, William WHYTE and Zhenfei ZHANG @@ -313,3 +314,4 @@ This proposal was made obsolete by proposal #269. +``` diff --git a/proposals/264-subprotocol-versions.txt b/proposals/264-subprotocol-versions.txt index bd58125..9ccb803 100644 --- a/proposals/264-subprotocol-versions.txt +++ b/proposals/264-subprotocol-versions.txt @@ -1,3 +1,4 @@ +``` Filename: 264-subprotocol-versions.txt Title: Putting version numbers on the Tor subprotocols Author: Nick Mathewson @@ -547,3 +548,4 @@ A.3. Example integration with other open proposals New Relay protocol version. +``` diff --git a/proposals/265-load-balancing-with-overhead.txt b/proposals/265-load-balancing-with-overhead.txt index 731a5b0..43110ba 100644 --- a/proposals/265-load-balancing-with-overhead.txt +++ b/proposals/265-load-balancing-with-overhead.txt @@ -1,3 +1,4 @@ +``` Filename: 265-load-balancing-with-overhead.txt Title: Load Balancing with Overhead Parameters Authors: Mike Perry @@ -359,3 +360,4 @@ combined = solve( Wme, Wmg, Wgg, Wee) pprint(combined) +``` diff --git a/proposals/266-removing-current-obsolete-clients.txt b/proposals/266-removing-current-obsolete-clients.txt index a4cc75c..c266ce1 100644 --- a/proposals/266-removing-current-obsolete-clients.txt +++ b/proposals/266-removing-current-obsolete-clients.txt @@ -1,3 +1,4 @@ +``` Filename: 266-removing-current-obsolete-clients.txt Title: Removing current obsolete clients from the Tor network Author: Nick Mathewson @@ -259,3 +260,4 @@ B. Next steps. http://meetbot.debian.net/tor-dev/2016/tor-dev.2016-02-12-15.01.log.html has a more full discussion of the above ideas. +``` diff --git a/proposals/267-tor-consensus-transparency.txt b/proposals/267-tor-consensus-transparency.txt index 117b76a..f5ea26e 100644 --- a/proposals/267-tor-consensus-transparency.txt +++ b/proposals/267-tor-consensus-transparency.txt @@ -1,3 +1,4 @@ +``` Filename: 267-tor-consensus-transparency.txt Title: Tor Consensus Transparency Author: Linus Nordberg @@ -361,3 +362,4 @@ C. References [RFC4627] https://tools.ietf.org/html/rfc4627 [rfc6962-bis-12] https://datatracker.ietf.org/doc/draft-ietf-trans-rfc6962-bis/12 [CT] https://https://www.certificate-transparency.org/ +``` diff --git a/proposals/268-guard-selection.txt b/proposals/268-guard-selection.txt index 3417b54..2d50790 100644 --- a/proposals/268-guard-selection.txt +++ b/proposals/268-guard-selection.txt @@ -1,3 +1,4 @@ +``` Filename: 268-guard-selection.txt Title: New Guard Selection Behaviour Author: Isis Lovecruft, George Kadianakis, [Ola Bini] @@ -507,3 +508,4 @@ Status: Obsolete version we can't recognize or it was chosen more than GUARD_LIFETIME ago. -*- coding: utf-8 -*- +``` diff --git a/proposals/269-hybrid-handshake.txt b/proposals/269-hybrid-handshake.txt index 4ab7e19..0f32944 100644 --- a/proposals/269-hybrid-handshake.txt +++ b/proposals/269-hybrid-handshake.txt @@ -1,3 +1,4 @@ +``` Filename: 269-hybrid-handshake.txt Title: Transitionally secure hybrid handshakes Author: John Schanck, William Whyte, Zhenfei Zhang, @@ -420,3 +421,4 @@ A2. Instantiation with NewHope PK := B | A_SEED KEM_ENC(PK) -> NEWHOPE_ENCAPS(PK) KEM_DEC(C, sk) -> NEWHOPE_DECAPS(C, sk) +``` diff --git a/proposals/270-newhope-hybrid-handshake.txt b/proposals/270-newhope-hybrid-handshake.txt index 52f6b50..6ad2e18 100644 --- a/proposals/270-newhope-hybrid-handshake.txt +++ b/proposals/270-newhope-hybrid-handshake.txt @@ -1,3 +1,4 @@ +``` Filename: 270-newhope-hybrid-handshake.txt Title: RebelAlliance: A Post-Quantum Secure Hybrid Handshake Based on NewHope Author: Isis Lovecruft, Peter Schwabe @@ -762,3 +763,4 @@ Depends: prop#220 prop#249 prop#264 prop#270 §C. Test Vectors +``` diff --git a/proposals/271-another-guard-selection.txt b/proposals/271-another-guard-selection.txt index 38da000..dc6ec46 100644 --- a/proposals/271-another-guard-selection.txt +++ b/proposals/271-another-guard-selection.txt @@ -1,3 +1,4 @@ +``` Filename: 271-another-guard-selection.txt Title: Another algorithm for guard selection Author: Isis Lovecruft, George Kadianakis, Ola Bini, Nick Mathewson @@ -887,3 +888,4 @@ TODO. Still non-addressed issues [Section:TODO] Suggestion: Treat it as a preference when adding to {CONFIRMED_GUARDS}, but not otherwise. +``` diff --git a/proposals/272-valid-and-running-by-default.txt b/proposals/272-valid-and-running-by-default.txt index 210a631..4ec056b 100644 --- a/proposals/272-valid-and-running-by-default.txt +++ b/proposals/272-valid-and-running-by-default.txt @@ -1,3 +1,4 @@ +``` Filename: 272-valid-and-running-by-default.txt Title: Listed routers should be Valid, Running, and treated as such Created: 26 Aug 2016 @@ -58,3 +59,4 @@ B. Does this solve the zombie problem? most ancient consensus-understanding Tors stop fetching descriptors and using the network. More testing needed though. +``` diff --git a/proposals/273-exit-relay-pinning.txt b/proposals/273-exit-relay-pinning.txt index 4a83a00..40e5abf 100644 --- a/proposals/273-exit-relay-pinning.txt +++ b/proposals/273-exit-relay-pinning.txt @@ -1,3 +1,4 @@ +``` Filename: 273-exit-relay-pinning.txt Title: Exit relay pinning for web services Author: Philipp Winter, Tobias Pulls, Roya Ensafi, and Nick Feamster @@ -221,3 +222,4 @@ Target: n/a policy over several exit relays, spread over time. In doing so, we are making assumptions on the number of visits the website sees. Is there a better solution that isn't significantly more complex? +``` diff --git a/proposals/274-rotate-onion-keys-less.txt b/proposals/274-rotate-onion-keys-less.txt index 09f5ea5..5625eb3 100644 --- a/proposals/274-rotate-onion-keys-less.txt +++ b/proposals/274-rotate-onion-keys-less.txt @@ -1,3 +1,4 @@ +``` Filename: 274-rotate-onion-keys-less.txt Title: Rotate onion keys less frequently Author: Nick Mathewson @@ -111,3 +112,4 @@ Implemented-In: 0.3.1.1-alpha previous onion key for an additional onion-key-grace-period-days days after it is replaced. +``` diff --git a/proposals/275-md-published-time-is-silly.txt b/proposals/275-md-published-time-is-silly.txt index e08e9a8..1cff90d 100644 --- a/proposals/275-md-published-time-is-silly.txt +++ b/proposals/275-md-published-time-is-silly.txt @@ -1,3 +1,4 @@ +``` Filename: 275-md-published-time-is-silly.txt Title: Stop including meaningful "published" time in microdescriptor consensus Author: Nick Mathewson @@ -126,3 +127,4 @@ Implemented-In: 0.4.8.1-alpha +``` diff --git a/proposals/276-lower-bw-granularity.txt b/proposals/276-lower-bw-granularity.txt index 156639e..bcd80c0 100644 --- a/proposals/276-lower-bw-granularity.txt +++ b/proposals/276-lower-bw-granularity.txt @@ -1,3 +1,4 @@ +``` Filename: 276-lower-bw-granularity.txt Title: Report bandwidth with lower granularity in consensus documents Author: Nick Mathewson @@ -77,3 +78,4 @@ Target: 0.3.1.x-alpha be safe? Would a time-aware smoothing mechanism work better? +``` diff --git a/proposals/277-detect-id-sharing.txt b/proposals/277-detect-id-sharing.txt index 7837615..e19f7ee 100644 --- a/proposals/277-detect-id-sharing.txt +++ b/proposals/277-detect-id-sharing.txt @@ -1,3 +1,4 @@ +``` Filename: 277-detect-id-sharing.txt Title: Detect multiple relay instances running with same ID Author: Nick Mathewson @@ -57,3 +58,4 @@ Target: 0.3.?? +``` diff --git a/proposals/278-directory-compression-scheme-negotiation.txt b/proposals/278-directory-compression-scheme-negotiation.txt index 6f509fc..6d80d39 100644 --- a/proposals/278-directory-compression-scheme-negotiation.txt +++ b/proposals/278-directory-compression-scheme-negotiation.txt @@ -1,3 +1,4 @@ +``` Filename: 278-directory-compression-scheme-negotiation.txt Title: Directory Compression Scheme Negotiation Author: Alexander Færøy @@ -207,3 +208,4 @@ Implemented-In: 0.3.1.1-alpha This research was supported in part by NSF grants CNS-1111539, CNS-1314637, CNS-1526306, CNS-1619454, and CNS-1640548. +``` diff --git a/proposals/279-naming-layer-api.txt b/proposals/279-naming-layer-api.txt index 1a56ecd..1d22662 100644 --- a/proposals/279-naming-layer-api.txt +++ b/proposals/279-naming-layer-api.txt @@ -1,3 +1,4 @@ +``` Filename: 279-naming-layer-api.txt Title: A Name System API for Tor Onion Services Author: George Kadianakis, Yawning Angel, David Goulet @@ -522,3 +523,4 @@ Appendix A.2: Example plugins [PLUGINEXAMPLES] f) OnioNS g) Namecoin/Blockstart +``` diff --git a/proposals/280-privcount-in-tor.txt b/proposals/280-privcount-in-tor.txt index fd5a011..2944419 100644 --- a/proposals/280-privcount-in-tor.txt +++ b/proposals/280-privcount-in-tor.txt @@ -1,3 +1,4 @@ +``` Filename: 280-privcount-in-tor.txt Title: Privacy-Preserving Statistics with Privcount in Tor Author: Nick Mathewson, Tim Wilson-Brown @@ -335,3 +336,4 @@ Superseded-By: 288 want to expand this scheme to work with anonymous data collectors, such as Tor clients. +``` diff --git a/proposals/281-bulk-md-download.txt b/proposals/281-bulk-md-download.txt index c561aa0..8f624d9 100644 --- a/proposals/281-bulk-md-download.txt +++ b/proposals/281-bulk-md-download.txt @@ -1,3 +1,4 @@ +``` Filename: 281-bulk-md-download.txt Title: Downloading microdescriptors in bulk Author: Nick Mathewson @@ -98,3 +99,4 @@ Status: Reserve Caches supporting this download protocol need to advertise support of a new DirCache subprotocol version. +``` diff --git a/proposals/282-remove-named-from-consensus.txt b/proposals/282-remove-named-from-consensus.txt index eeef519..b37e855 100644 --- a/proposals/282-remove-named-from-consensus.txt +++ b/proposals/282-remove-named-from-consensus.txt @@ -1,3 +1,4 @@ +``` Filename: 282-remove-named-from-consensus.txt Title: Remove "Named" and "Unnamed" handling from consensus voting Author: Nick Mathewson @@ -22,3 +23,4 @@ Target: arti-dirauth the negotiated method is M or later, then the Named and Unnamed flags are handled as if any they were any other consensus flags. +``` diff --git a/proposals/283-ipv6-in-micro-consensus.txt b/proposals/283-ipv6-in-micro-consensus.txt index 1d29f78..f5846ef 100644 --- a/proposals/283-ipv6-in-micro-consensus.txt +++ b/proposals/283-ipv6-in-micro-consensus.txt @@ -1,3 +1,4 @@ +``` Filename: 283-ipv6-in-micro-consensus.txt Title: Move IPv6 ORPorts from microdescriptors to the microdesc consensus Author: Tim Wilson-Brown (teor), Nick Mathewson @@ -186,3 +187,4 @@ Ticket: #20916 Consensus-health implements a ReachableIPv6 pseudo-flag for authorities and relays: https://consensus-health.torproject.org/ +``` diff --git a/proposals/284-hsv3-control-port.txt b/proposals/284-hsv3-control-port.txt index 1ff20bb..6bef07a 100644 --- a/proposals/284-hsv3-control-port.txt +++ b/proposals/284-hsv3-control-port.txt @@ -1,3 +1,4 @@ +``` Filename: 284-hsv3-control-port.txt Title: Hidden Service v3 Control Port Author: David Goulet @@ -191,3 +192,4 @@ Status: Closed an "HSAddress". This field is extended to support v3 address: HSAddress = 16*Base32Character / 56*Base32Character / "UNKNOWN" +``` diff --git a/proposals/285-utf-8.txt b/proposals/285-utf-8.txt index cb7bab4..453a8db 100644 --- a/proposals/285-utf-8.txt +++ b/proposals/285-utf-8.txt @@ -1,3 +1,4 @@ +``` Filename: 285-utf-8.txt Title: Directory documents should be standardized as UTF-8 Author: Nick Mathewson @@ -115,3 +116,4 @@ Ticket: https://gitlab.torproject.org/tpo/core/tor/-/issues/40131 * Well-Formed UTF-8 Byte Sequences: Table 3-7, page 126. * Byte order mark: C11, page 83; D94, page 130. * UTF-8 encoding scheme: D96, pages 130. +``` diff --git a/proposals/286-hibernation-api.txt b/proposals/286-hibernation-api.txt index 796cfa8..e76773e 100644 --- a/proposals/286-hibernation-api.txt +++ b/proposals/286-hibernation-api.txt @@ -1,3 +1,4 @@ +``` Filename: 286-hibernation-api.txt Title: Controller APIs for hibernation access on mobile Author: Nick Mathewson @@ -195,3 +196,4 @@ Notes: This proposal was useful for our early thinking, but a simpler StartIdle -- Boolean. If set to 1, Tor begins in IDLE mode. +``` diff --git a/proposals/287-reduce-lifetime.txt b/proposals/287-reduce-lifetime.txt index 06e87d0..e37ead9 100644 --- a/proposals/287-reduce-lifetime.txt +++ b/proposals/287-reduce-lifetime.txt @@ -1,3 +1,4 @@ +``` Filename: 287-reduce-lifetime.txt Title: Reduce circuit lifetime without overloading the network Author: Fernando Fernandez Mancera @@ -103,3 +104,4 @@ Enhancing Tor's Performance using Real-time Traffic Classification [https://www.cypherpunks.ca/~iang/pubs/difftor-ccs.pdf] (It's not exactly about that, but they talk about circuit lifetime and the ten minutes problem a few times. Also it's an interesting paper.) +``` diff --git a/proposals/288-privcount-with-shamir.txt b/proposals/288-privcount-with-shamir.txt index 109c743..7315d4c 100644 --- a/proposals/288-privcount-with-shamir.txt +++ b/proposals/288-privcount-with-shamir.txt @@ -1,3 +1,4 @@ +``` Filename: 288-privcount-with-shamir.txt Title: Privacy-Preserving Statistics with Privcount in Tor (Shamir version) Author: Nick Mathewson, Tim Wilson-Brown, Aaron Johnson @@ -565,3 +566,4 @@ Appendix Z. Remaining client-side uncertainties How data is uploaded to DCs? What to say about persistence on the DC side? +``` diff --git a/proposals/289-authenticated-sendmes.txt b/proposals/289-authenticated-sendmes.txt index e09b882..ebb1e3c 100644 --- a/proposals/289-authenticated-sendmes.txt +++ b/proposals/289-authenticated-sendmes.txt @@ -1,3 +1,4 @@ +``` Filename: 289-authenticated-sendmes.txt Title: Authenticating sendme cells to mitigate bandwidth attacks Author: Rob Jansen, Roger Dingledine, David Goulet @@ -493,3 +494,4 @@ Implemented-In: 0.4.1.1-alpha This research was supported in part by NSF grants CNS-1111539, CNS-1314637, CNS-1526306, CNS-1619454, and CNS-1640548. +``` diff --git a/proposals/290-deprecate-consensus-methods.txt b/proposals/290-deprecate-consensus-methods.txt index 979559c..832426e 100644 --- a/proposals/290-deprecate-consensus-methods.txt +++ b/proposals/290-deprecate-consensus-methods.txt @@ -1,3 +1,4 @@ +``` Filename: 290-deprecate-consensus-methods.txt Title: Continuously update consensus methods Author: Nick Mathewson @@ -75,3 +76,4 @@ B. Client and relay compatibility notes not represent backward-incompatible changes in how they generate their consensuses. +``` diff --git a/proposals/291-two-guard-nodes.txt b/proposals/291-two-guard-nodes.txt index 56424cc..9b89eae 100644 --- a/proposals/291-two-guard-nodes.txt +++ b/proposals/291-two-guard-nodes.txt @@ -1,3 +1,4 @@ +``` Filename: 291-two-guard-nodes.txt Title: The move to two guard nodes Author: Mike Perry @@ -254,3 +255,4 @@ References: 12. https://trac.torproject.org/projects/tor/ticket/10969 13. https://www.freehaven.net/anonbib/cache/wpes12-cogs.pdf 14. https://gitweb.torproject.org/torspec.git/tree/proposals/247-hs-guard-discovery.txt +``` diff --git a/proposals/292-mesh-vanguards.txt b/proposals/292-mesh-vanguards.txt index 5341213..7e579c8 100644 --- a/proposals/292-mesh-vanguards.txt +++ b/proposals/292-mesh-vanguards.txt @@ -1,8 +1,9 @@ +``` Filename: 292-mesh-vanguards.txt Title: Mesh-based vanguards Authors: George Kadianakis and Mike Perry Created: 2018-05-08 -Status: Accepted +Status: Closed Supersedes: 247 0. Motivation @@ -534,3 +535,4 @@ CDF_table_33() ---------------------- 1. https://onionbalance.readthedocs.org/en/latest/design.html#overview +``` diff --git a/proposals/293-know-when-to-publish.txt b/proposals/293-know-when-to-publish.txt index c6fd1aa..2ca4222 100644 --- a/proposals/293-know-when-to-publish.txt +++ b/proposals/293-know-when-to-publish.txt @@ -1,3 +1,4 @@ +``` Filename: 293-know-when-to-publish.txt Title: Other ways for relays to know when to publish Author: Nick Mathewson @@ -59,3 +60,4 @@ Implemented-In: 0.4.0.1-alpha published time in consensus documents accurate -- we can start setting it to some time in the distant future, per proposal 275. +``` diff --git a/proposals/294-tls-1.3.txt b/proposals/294-tls-1.3.txt index c9bae79..e195b2f 100644 --- a/proposals/294-tls-1.3.txt +++ b/proposals/294-tls-1.3.txt @@ -1,3 +1,4 @@ +``` Filename: 294-tls-1.3.txt Title: TLS 1.3 Migration Authors: Isis Lovecruft @@ -338,3 +339,4 @@ A. References https://www.openssl.org/docs/manmaster/man3/SSL_CTX_set1_groups.html [TLS-1.3_SIGNATURE_ALGOS]: https://tlswg.github.io/tls13-spec/draft-ietf-tls-tls13.html#signature-algorithms +``` diff --git a/proposals/295-relay-crypto-with-adl.txt b/proposals/295-relay-crypto-with-adl.txt index 7f8fc42..bf8fd77 100644 --- a/proposals/295-relay-crypto-with-adl.txt +++ b/proposals/295-relay-crypto-with-adl.txt @@ -1,3 +1,4 @@ +``` Filename: 295-relay-crypto-with-adl.txt Title: Using ADL for relay cryptography (solving the crypto-tagging attack) Author: Tomer Ashur, Orr Dunkelman, Atul Luykx @@ -561,3 +562,4 @@ Status: Open No changes are required to the authentication process when the relay cell is sent backwards. +``` diff --git a/proposals/296-expose-bandwidth-files.txt b/proposals/296-expose-bandwidth-files.txt index 908a9ad..c61dc68 100644 --- a/proposals/296-expose-bandwidth-files.txt +++ b/proposals/296-expose-bandwidth-files.txt @@ -1,3 +1,4 @@ +``` Filename: 296-expose-bandwidth-files.txt Title: Have Directory Authorities expose raw bandwidth list files Author: Tom Ritter @@ -84,3 +85,4 @@ Applications that parse the document should follow the bandwidth list file specification in bandwidth-file-spec.txt. If a new bandwidth list format version is added, the applications MAY need to upgrade to that version. +``` diff --git a/proposals/297-safer-protover-shutdowns.txt b/proposals/297-safer-protover-shutdowns.txt index 59fbac1..96d6f62 100644 --- a/proposals/297-safer-protover-shutdowns.txt +++ b/proposals/297-safer-protover-shutdowns.txt @@ -1,3 +1,4 @@ +``` Filename: 297-safer-protover-shutdowns.txt Title: Relaxing the protover-based shutdown rules Author: Nick Mathewson @@ -104,3 +105,4 @@ IMPLEMENTATION NOTE: +``` diff --git a/proposals/298-canonical-families.txt b/proposals/298-canonical-families.txt index a96ab8d..4576d63 100644 --- a/proposals/298-canonical-families.txt +++ b/proposals/298-canonical-families.txt @@ -1,3 +1,4 @@ +``` Filename: 298-canonical-families.txt Title: Putting family lines in canonical form Author: Nick Mathewson @@ -61,3 +62,4 @@ Implemented-In: 0.4.0.1-alpha +``` diff --git a/proposals/299-ip-failure-count.txt b/proposals/299-ip-failure-count.txt index 8c2319f..1078e6d 100644 --- a/proposals/299-ip-failure-count.txt +++ b/proposals/299-ip-failure-count.txt @@ -1,3 +1,4 @@ +``` Filename: 299-ip-failure-count.txt Title: Preferring IPv4 or IPv6 based on IP Version Failure Count Author: Neel Chauhan @@ -222,3 +223,4 @@ Ticket: https://trac.torproject.org/projects/tor/ticket/27491 Thank you teor for aiding me with the implementation of Happy Eyeballs in Tor. This would not have been possible if it weren't for you. +``` diff --git a/proposals/300-walking-onions.txt b/proposals/300-walking-onions.txt index 3df9f44..5ab3a7a 100644 --- a/proposals/300-walking-onions.txt +++ b/proposals/300-walking-onions.txt @@ -1,3 +1,4 @@ +``` Filename: 300-walking-onions.txt Title: Walking Onions: Scaling and Saving Bandwidth Author: Nick Mathewson @@ -511,3 +512,4 @@ A.2. Additional ideas but it might be worth looking at harder. +``` diff --git a/proposals/301-dont-vote-on-package-fingerprints.txt b/proposals/301-dont-vote-on-package-fingerprints.txt index cabf4f7..e4f5c03 100644 --- a/proposals/301-dont-vote-on-package-fingerprints.txt +++ b/proposals/301-dont-vote-on-package-fingerprints.txt @@ -1,3 +1,4 @@ +``` Filename: 301-dont-vote-on-package-fingerprints.txt Title: Don't include package fingerprints in consensus documents Author: Iain R. Learmonth @@ -71,3 +72,4 @@ B. Acknowledgements Thanks to teor and Nick Mathewson for their comments and suggestions on this proposal. +``` diff --git a/proposals/302-padding-machines-for-onion-clients.txt b/proposals/302-padding-machines-for-onion-clients.txt index cd24298..52710b0 100644 --- a/proposals/302-padding-machines-for-onion-clients.txt +++ b/proposals/302-padding-machines-for-onion-clients.txt @@ -1,3 +1,4 @@ +``` Filename: 302-padding-machines-for-onion-clients.txt Title: Hiding onion service clients using padding Author: George Kadianakis, Mike Perry @@ -303,3 +304,4 @@ A. Acknowledgements [1]: "Understanding Tor Usage with Privacy-Preserving Measurement" by Akshaya Mani, T Wilson-Brown, Rob Jansen, Aaron Johnson, and Micah Sherr In Proceedings of the Internet Measurement Conference 2018 (IMC 2018). +``` diff --git a/proposals/303-protover-removal-policy.txt b/proposals/303-protover-removal-policy.txt index 54fcead..c2d5697 100644 --- a/proposals/303-protover-removal-policy.txt +++ b/proposals/303-protover-removal-policy.txt @@ -1,3 +1,4 @@ +``` Filename: 303-protover-removal-policy.txt Title: When and how to remove support for protocol versions Author: Nick Mathewson @@ -60,3 +61,4 @@ Status: Open _recommended_ for clients for at least 18 months. +``` diff --git a/proposals/304-socks5-extending-hs-error-codes.txt b/proposals/304-socks5-extending-hs-error-codes.txt index d8ff7d5..8866fa1 100644 --- a/proposals/304-socks5-extending-hs-error-codes.txt +++ b/proposals/304-socks5-extending-hs-error-codes.txt @@ -1,3 +1,4 @@ +``` Filename: 304-socks5-extending-hs-error-codes.txt Title: Extending SOCKS5 Onion Service Error Codes Author: David Goulet, George Kadianakis @@ -107,3 +108,4 @@ References: [PROP229] https://gitweb.torproject.org/torspec.git/tree/proposals/229-further-socks5-extensions.txt [RFC1928] https://www.ietf.org/rfc/rfc1928.txt +``` diff --git a/proposals/305-establish-intro-dos-defense-extention.txt b/proposals/305-establish-intro-dos-defense-extention.txt index e377f3e..e1b0256 100644 --- a/proposals/305-establish-intro-dos-defense-extention.txt +++ b/proposals/305-establish-intro-dos-defense-extention.txt @@ -1,3 +1,4 @@ +``` Filename: 305-establish-intro-dos-defense-extention.txt Title: ESTABLISH_INTRO Cell DoS Defense Extension Author: David Goulet, George Kadianakis @@ -199,3 +200,4 @@ References: [1] https://lists.torproject.org/pipermail/tor-dev/2019-May/013837.html [2] https://trac.torproject.org/15516 +``` diff --git a/proposals/306-ipv6-happy-eyeballs.txt b/proposals/306-ipv6-happy-eyeballs.txt index c5c9ddf..86a1445 100644 --- a/proposals/306-ipv6-happy-eyeballs.txt +++ b/proposals/306-ipv6-happy-eyeballs.txt @@ -1,3 +1,4 @@ +``` Filename: 306-ipv6-happy-eyeballs.txt Title: A Tor Implementation of IPv6 Happy Eyeballs Author: Neel Chauhan @@ -344,3 +345,4 @@ Ticket: https://trac.torproject.org/projects/tor/ticket/29801 [P299-TRAC]: https://trac.torproject.org/projects/tor/ticket/29801 [TEOR-P306-REP]: https://lists.torproject.org/pipermail/tor-dev/2019-July/013919.html +``` diff --git a/proposals/307-onionbalance-v3.txt b/proposals/307-onionbalance-v3.txt index baaa717..fb44cb3 100644 --- a/proposals/307-onionbalance-v3.txt +++ b/proposals/307-onionbalance-v3.txt @@ -1,3 +1,4 @@ +``` Filename: 307-onionbalance-v3.txt Title: Onion Balance Support for Onion Service v3 Author: Nick Mathewson @@ -183,3 +184,4 @@ A. Acknowledgments in this problem space. This research was supported by NSF grants CNS-1526306 and CNS-1619454. +``` diff --git a/proposals/308-counter-galois-onion.txt b/proposals/308-counter-galois-onion.txt index a57a38e..4714e8d 100644 --- a/proposals/308-counter-galois-onion.txt +++ b/proposals/308-counter-galois-onion.txt @@ -1,3 +1,4 @@ +``` Filename: 308-counter-galois-onion.txt Title: Counter Galois Onion: A New Proposal for Forward-Secure Relay Cryptography Authors: Jean Paul Degabriele, Alessandro Melloni, Martijn Stam @@ -556,3 +557,4 @@ Misuse-Resistant Authenticated Encryption", RFC 8452, April 2019. [ST13] Thomas Shrimpton, R. Seth Terashima, "A Modular Framework for Building Variable-Input Length Tweakable Ciphers", ASIACRYPT 2013. +``` diff --git a/proposals/309-optimistic-socks-in-tor.txt b/proposals/309-optimistic-socks-in-tor.txt index d22a1a0..7f9272d 100644 --- a/proposals/309-optimistic-socks-in-tor.txt +++ b/proposals/309-optimistic-socks-in-tor.txt @@ -1,3 +1,4 @@ +``` Filename: 309-optimistic-socks-in-tor.txt Title: Optimistic SOCKS Data Author: Tom Ritter @@ -76,3 +77,4 @@ Ticket: #5915 References: [RFC1928] https://www.ietf.org/rfc/rfc1928.txt +``` diff --git a/proposals/310-bandaid-on-guard-selection.txt b/proposals/310-bandaid-on-guard-selection.txt index 2001994..94477a1 100644 --- a/proposals/310-bandaid-on-guard-selection.txt +++ b/proposals/310-bandaid-on-guard-selection.txt @@ -1,3 +1,4 @@ +``` Filename: 310-bandaid-on-guard-selection.txt Title: Towards load-balancing in Prop 271 Author: Florentin Rochet, Aaron Johnson et al. @@ -93,3 +94,4 @@ Status: Closed force clients onto compromised guards, using relays that are part of the history if they're still available (in sampled order), and by tracking its size. This should maintain the initial goals of Prop 271. +``` diff --git a/proposals/311-relay-ipv6-reachability.txt b/proposals/311-relay-ipv6-reachability.txt index 11282c3..2ad7f46 100644 --- a/proposals/311-relay-ipv6-reachability.txt +++ b/proposals/311-relay-ipv6-reachability.txt @@ -1,3 +1,4 @@ +``` Filename: 311-relay-ipv6-reachability.txt Title: Tor Relay IPv6 Reachability Author: teor, Nick Mathewson @@ -836,3 +837,4 @@ References: [Tor Specification]: https://gitweb.torproject.org/torspec.git/tree/tor-spec.txt +``` diff --git a/proposals/312-relay-auto-ipv6-addr.txt b/proposals/312-relay-auto-ipv6-addr.txt index 712175e..a82e239 100644 --- a/proposals/312-relay-auto-ipv6-addr.txt +++ b/proposals/312-relay-auto-ipv6-addr.txt @@ -1,3 +1,4 @@ +``` Filename: 312-relay-auto-ipv6-addr.txt Title: Tor Relay Automatic IPv6 Address Discovery Author: teor, Nick Mathewson, s7r @@ -1563,3 +1564,4 @@ References: [Tor Supported Platforms]: https://trac.torproject.org/projects/tor/wiki/org/teams/NetworkTeam/SupportedPlatforms#OSSupportlevels +``` diff --git a/proposals/313-relay-ipv6-stats.txt b/proposals/313-relay-ipv6-stats.txt index ae02a33..c637dad 100644 --- a/proposals/313-relay-ipv6-stats.txt +++ b/proposals/313-relay-ipv6-stats.txt @@ -1,3 +1,4 @@ +``` Filename: 313-relay-ipv6-stats.txt Title: Tor Relay IPv6 Statistics Author: teor, Karsten Loesing, Nick Mathewson @@ -405,3 +406,4 @@ References: [Tor Specification]: https://gitweb.torproject.org/torspec.git/tree/tor-spec.txt +``` diff --git a/proposals/315-update-dir-required-fields.txt b/proposals/315-update-dir-required-fields.txt index 1761a03..875d8a0 100644 --- a/proposals/315-update-dir-required-fields.txt +++ b/proposals/315-update-dir-required-fields.txt @@ -1,3 +1,4 @@ +``` Filename: 315-update-dir-required-fields.txt Title: Updating the list of fields required in directory documents Author: Nick Mathewson @@ -129,3 +130,4 @@ Notes: become required: * pr +``` diff --git a/proposals/316-flashflow.md b/proposals/316-flashflow.md index 8258d76..6755913 100644 --- a/proposals/316-flashflow.md +++ b/proposals/316-flashflow.md @@ -56,7 +56,7 @@ time while Torflow takes days/weeks to assign them their full fair share of bandwidth (especially for non-exits). FlashFlow is more secure than Torflow: FlashFlow allows a relay to inflate its measured capacity by up to 1.33x (configured by a parameter) while Torflow allows weight -inflation by a factor of 89x [0] or even 177x [1]. +inflation by a factor of 89x [\[0\]] or even 177x [\[1\]]. After an overview in section 2 of the planned deployment stages, section 3, 4, and 5 discuss the short, medium, and long term deployment plans in @@ -133,7 +133,7 @@ allow itself to be measured more than twice by a FlashFlow deployment in any time window of this length. Relays should not change this option unless they really know what they're doing. Changing it at the relay will not change how often FlashFlow will attempt to measure the relay. -Possible values are in the range [1 hour, 1 month] inclusive. Default: 1 +Possible values are in the range \[1 hour, 1 month\] inclusive. Default: 1 day. FFBackgroundTrafficPercent: The maximum amount of regular @@ -142,7 +142,7 @@ percent of total traffic (measurement + non-measurement). This parameter is a trade off between having to limit background traffic and limiting how much a relay can inflate its result by handling no background traffic but reporting that it has done so. Possible values -are in the range [0, 99] inclusive. Default: 25 (a maximum inflation +are in the range \[0, 99\] inclusive. Default: 25 (a maximum inflation factor of 1.33). FFMaxMeasurementDuration: The maximum amount of time, in seconds, that @@ -151,7 +151,7 @@ measurement will begin soon and the end of the measurement. If this amount of time passes, the relay shall close all measurement connections and exit its measurement mode. Note this duration includes handshake time, thus it necessarily is larger than the expected actual measurement -duration. Possible values are in the range [10, 120] inclusive. +duration. Possible values are in the range \[10, 120\] inclusive. Default: 45. ### 3.1.2 New Cell Types @@ -219,7 +219,7 @@ error to the coordinator and considers the measurement a failure. It is also a failure if any measurer is unable to open at least half of its circuits with the target. -The payload of MEAS_PARAMS cells [XXX more may need to be added]: +The payload of MEAS_PARAMS cells \[XXX more may need to be added\]: ``` - meas_duration [2 bytes] [1, 600] @@ -231,10 +231,10 @@ meas_duration is the duration, in seconds, that the actual measurement will last. num_measurers is how many link_specifier structs follow containing information on the measurers that the relay should expect. Future versions of FlashFlow and MEAS_PARAMS will use TLS certificates instead of IP addresses. -[XXX probably need diff layout to allow upgrade to TLS certs instead of +\[XXX probably need diff layout to allow upgrade to TLS certs instead of link_specifier structs. probably using ext-type-length-value like teor -suggests] -[XXX want to specify number of conns to expect from each measurer here?] +suggests\] +\[XXX want to specify number of conns to expect from each measurer here?\] MEAS_PARAMS_OK has no payload: it's just padding bytes to make the cell PAYLOAD_LEN (509) bytes long. @@ -245,7 +245,7 @@ The payload of MEAS_ECHO cells: - arbitrary bytes [PAYLOAD_LEN bytes] ``` -The payload of MEAS_BG cells [XXX more for extra info? like CPU usage]: +The payload of MEAS_BG cells \[XXX more for extra info? like CPU usage\]: ``` - second [2 byte] [1, 600] @@ -260,7 +260,7 @@ subsequent cell will increment it by one. sent_bg_bytes is the number of background traffic bytes sent in the last second (since the last MEAS_BG cell). recv_bg_bytes is the same but for received bytes. -The payload of MEAS_ERR cells [XXX need field for more info]: +The payload of MEAS_ERR cells \[XXX need field for more info\]: ``` - err_code [1 byte] [0, 255] @@ -302,8 +302,8 @@ If x is very small, the relay will perform the calculation s.t. x is the number of cells required to produce 10 Mbit/s of measurement traffic, thus ensuring some minimum amount of background traffic is allowed. -[XXX teor suggests in [4] that the number 10 Mbit/s could be derived more -intelligently. E.g. based on AuthDirFastGuarantee or AuthDirGuardBWGuarantee] +\[XXX teor suggests in [\[4\]] that the number 10 Mbit/s could be derived more +intelligently. E.g. based on AuthDirFastGuarantee or AuthDirGuardBWGuarantee\] ## 3.2 FlashFlow Components @@ -317,7 +317,7 @@ values. They are: with the target relay. We suggest s=160 based on the FF paper. - The bandwidth multiplier, m. Given an existing capacity estimate for a relay, z, the coordinator will instruct the measurers to, in - aggregate, send m*z Mbit/s to the target relay. We recommend m=2.25. + aggregate, send m\*z Mbit/s to the target relay. We recommend m=2.25. - The measurement duration, d. Based on the FF paper, we recommend d=30 seconds. @@ -419,7 +419,7 @@ slot for it. It picks slot 3. The coordinator takes the next largest, ``` The coordinator takes the next largest, 250, and randomly picks slot 2. -Slot 2 already has 600 Mbit/s of measurer capacity reserved (300*m); +Slot 2 already has 600 Mbit/s of measurer capacity reserved (300\*m); given just 1000 Mbit/s of total measurer capacity, there is just 400 Mbit/s of spare capacity while this relay requires 500 Mbit/s. There is not enough room in slot 2 for this relay. The coordinator picks a new @@ -473,11 +473,11 @@ v3bw.2020-03-01-04-00-00 v3bw.2020-03-01-05-00-00 ``` -[XXX Either FF should auto-delete old ones, logrotate config should be +\[XXX Either FF should auto-delete old ones, logrotate config should be provided, a script provided, or something to help bwauths not accidentally -fill up their disk] +fill up their disk\] -[XXX What's the approxmiate disk usage for, say, a few years of these?] +\[XXX What's the approxmiate disk usage for, say, a few years of these?\] ### 3.2.2 FlashFlow Measurer @@ -532,7 +532,7 @@ report more than ~33 Mbit/s, FlashFlow limits it to just ~33 Mbit/s.) With r=25%, FlashFlow only allows 1.33x weight inflation. Prior work shows that Torflow allows weight inflation by a factor of 89x -[0] or even 177x [1]. +[\[0\]] or even 177x [\[1\]]. The ratio chosen is a trade-off between impact on background traffic and security: r=50% allows a relay to double its weight but won't impact @@ -578,7 +578,7 @@ supports it. New link- and relay-subprotocol versions will be used by the relay to indicate FF support. E.g. at the time of writing, the next relay subprotocol version is -4 [3]. +4 [\[3\]]. We plan to host a FlashFlow deployment consisting of a FF coordinator and a single FF measurer on a single 1 Gbit/s machine. Data produced by @@ -652,7 +652,7 @@ The following is quoted from Section 4.3 of the FlashFlow paper. ensures that old relays will continue to be measured, with new relays given secondary priority in the order they arrive. -[XXX Teor leaves good ideas in his tor-dev@ post [5], +\[XXX Teor leaves good ideas in his tor-dev@ post [\[5\]], including a good plain language description of what the FF paper quotes says, and a recommendation on which consensus to use when making a new schedule] @@ -665,7 +665,7 @@ time. What specifically to do here is left for medium/long term work. ## 5.3 Experiments - [XXX todo] +\[XXX todo\] ## 5.4 Other Changes/Investigations/Ideas @@ -694,28 +694,32 @@ time. What specifically to do here is left for medium/long term work. background traffic. - What to do about co-located relays. Can they be detected reliably? Should we just add a torrc option a la MyFamily for co-located relays? -- What is the explanation for dennis.jackson's scary graphs in this [2] +- What is the explanation for dennis.jackson's scary graphs in this [\[2\]] ticket? Was it because of the speed test? Why? Will FlashFlow produce the same behavior? -# Citations - -[0] F. Thill. Hidden Service Tracking Detection and Bandwidth Cheating - in Tor Anonymity Network. Master’s thesis, Univ. Luxembourg, 2014. - https://www.cryptolux.org/images/b/bc/Tor_Issues_Thesis_Thill_Fabrice.pdf -[1] A. Johnson, R. Jansen, N. Hopper, A. Segal, and P. Syverson. - PeerFlow: Secure Load Balancing in Tor. Proceedings on Privacy - Enhancing Technologies (PoPETs), 2017(2), April 2017. - https://ohmygodel.com/publications/peerflow-popets2017.pdf -[2] Mike Perry: Graph onionperf and consensus information from Rob's - experiments - https://trac.torproject.org/projects/tor/ticket/33076 -[3] tor-spec.txt Section 9.3 "Relay" Subprotocol versioning - https://gitweb.torproject.org/torspec.git/tree/tor-spec.txt#n2132 -[4] Teor's second respose to FlashFlow proposal - https://lists.torproject.org/pipermail/tor-dev/2020-April/014251.html -[5] Teor's first respose to FlashFlow proposal - https://lists.torproject.org/pipermail/tor-dev/2020-April/014246.html +<!-- # Citations --> + +[\[0\]]: <https://www.cryptolux.org/images/b/bc/Tor_Issues_Thesis_Thill_Fabrice.pdf> +"F. Thill. Hidden Service Tracking Detection and Bandwidth Cheating +in Tor Anonymity Network. Master’s thesis, Univ. Luxembourg, 2014." + +[\[1\]]: <https://ohmygodel.com/publications/peerflow-popets2017.pdf> +"A. Johnson, R. Jansen, N. Hopper, A. Segal, and P. Syverson. +PeerFlow: Secure Load Balancing in Tor. Proceedings on Privacy +Enhancing Technologies (PoPETs), 2017(2), April 2017." + +[\[2\]]: <https://trac.torproject.org/projects/tor/ticket/33076> +"Mike Perry: Graph onionperf and consensus information from Rob's experiments" + +[\[3\]]: ../tor-spec/subprotocol-versioning.md#relay +"Tor Specification: \"Relay\" Subprotocol versioning" + +[\[4\]]: <https://lists.torproject.org/pipermail/tor-dev/2020-April/014251.html> +"Teor's second response to FlashFlow proposal" + +[\[5\]]: <https://lists.torproject.org/pipermail/tor-dev/2020-April/014246.html> +"Teor's first response to FlashFlow proposal" # Appendix A: Save CPU at measurer by not encrypting all MEAS_ECHO cells @@ -745,10 +749,10 @@ measurement. Consider bucket_size is 1000. For the moment ignore cell encryption. -We start at idx=0 and pick an idx in [0, 1000) to record, say 640. At -idx=640 we record the cell. At idx=1000 we choose a new idx in [1000, +We start at idx=0 and pick an idx in \[0, 1000) to record, say 640. At +idx=640 we record the cell. At idx=1000 we choose a new idx in \[1000, 2000) to record, say 1236. At idx=1236 we record the cell. At idx=2000 -we choose a new idx in [2000, 3000). Etc. +we choose a new idx in \[2000, 3000). Etc. There's 2000+ cells in flight and the measurer has recorded two items: @@ -821,7 +825,7 @@ will have (49/50)^(2*146) or 0.2% odds of success, which is quite low. Wanting a <1% chance that a 10 Mbit/s relay can successfully cheat results in a bucket size of approximately 125: -- 10*30 = 300 Mbit of traffic during 30s measurement. 37.5 million +- 10\*30 = 300 Mbit of traffic during 30s measurement. 37.5 million bytes. - 37,500,000 bytes / 514 bytes/cell = ~73,000 cells - bucket_size of 125 cells means 73,000 / 125 = 584 buckets @@ -830,4 +834,3 @@ results in a bucket size of approximately 125: Slower relays can cheat more easily but the amount of extra weight they can obtain is insignificant in absolute terms. Faster relays are essentially unable to cheat. - diff --git a/proposals/317-secure-dns-name-resolution.txt b/proposals/317-secure-dns-name-resolution.txt index ad2b7e6..55db19f 100644 --- a/proposals/317-secure-dns-name-resolution.txt +++ b/proposals/317-secure-dns-name-resolution.txt @@ -1,3 +1,4 @@ +``` Filename: 317-secure-dns-name-resolution.txt Title: Improve security aspects of DNS name resolution Author: Christian Hofer @@ -218,3 +219,4 @@ Performance and scalability: onion service in a performant manner it is always possible to fallback to clearweb nameservers by changing a configuration setting. +``` diff --git a/proposals/323-walking-onions-full.md b/proposals/323-walking-onions-full.md index 86d57b2..192ae48 100644 --- a/proposals/323-walking-onions-full.md +++ b/proposals/323-walking-onions-full.md @@ -1572,9 +1572,9 @@ even, take the lower of the two center votes (the one at position N/2) if `BREAK_EVEN_LOW` is true. Otherwise, take the higher of the two center votes (the one at position N/2 + 1). -For example, the Median(…, even_low: True, type: "uint") of the votes -["String", 2, 111, 6] is 6. The Median(…, even_low: True, type: "uint") -of the votes ["String", 77, 9, 22, "String", 3] is 9. +For example, the `Median(…, even_low: True, type: "uint")` of the votes +`["String", 2, 111, 6]` is 6. The `Median(…, even_low: True, type: "uint")` +of the votes `["String", 77, 9, 22, "String", 3]` is 9. <!-- Section 3.3.4.2 --> <a id='S3.3.4.2'></a> @@ -1681,7 +1681,7 @@ elements that appears in at least `MIN_COUNT` votes. (Note that the input votes may contain duplicate elements. These must be treated as if there were no duplicates: the vote -[1, 1, 1, 1] is the same as the vote [1]. Implementations may want +`[1, 1, 1, 1]` is the same as the vote `[1]`. Implementations may want to preprocess votes by discarding all but one instance of each member.) @@ -1827,10 +1827,12 @@ computed so far), and on the entirety of the set of votes. > our current behavior. Parameters: + `FIELDS` (one or more other locations in the vote) `RULE` (the rule used to combine values) -Encoding +Encoding: + ; This item is "derived from" some other field. DerivedItemOp = { op: "DerivedFrom", @@ -2646,7 +2648,7 @@ corresponding ENDIVERouterData. Because SNIPLocation objects are signed, they must be encoded as "canonical" cbor, according to section 3.9 of RFC 7049. -If R[idx] is {} (the empty map) for any given idx, then no SNIP will be +If `R[idx]` is `{}` (the empty map) for any given idx, then no SNIP will be generated for the SNIPRouterData at that routing index for this index group. <!-- Section 4.2 --> <a id='S4.2'></a> @@ -2839,6 +2841,7 @@ The CREATE2, CREATED2, and EXTENDED2 cells change as follows: These extensions are defined by this proposal: +```text [01] -- `Partial_SNIPRouterData` -- Sent from an extending relay to a target relay. This extension holds one or more fields from the SNIPRouterData that the extending relay is using, @@ -2861,6 +2864,7 @@ These extensions are defined by this proposal: originator does not want a SNIP. Otherwise, the originator does want a SNIP containing the router and the specified index. Other values are unspecified. +``` By default, EXTENDED2 cells are sent with a SNIP iff the EXTENDED2 cell used a `snip_index_pos` link specifier, and CREATED2 cells are @@ -2923,14 +2927,14 @@ following main changes. So the client's message is now: - CLIENT_PK [32 bytes] + CLIENT_PK [32 bytes] And the relay's reply is now: - NODEID [32 bytes] - KEYID [32 bytes] - SERVER_PK [32 bytes] - AUTH [32 bytes] + NODEID [32 bytes] + KEYID [32 bytes] + SERVER_PK [32 bytes] + AUTH [32 bytes] otherwise, all fields are computed as described in tor-spec. @@ -3330,10 +3334,10 @@ relay cells.) > that the relay might not understand. To include the SNIP, the client places it in an extension in the -INTRODUCE cell. The onion key can now be omitted[*], along with +INTRODUCE cell. The onion key can now be omitted\[\*\], along with the link specifiers. -> [*] Technically, we use a zero-length onion key, with a new type +> \[\*\] Technically, we use a zero-length onion key, with a new type > "implicit in SNIP". To know whether the service can recognize this kind of cell, the @@ -3445,10 +3449,10 @@ between updating ENDIVEs under ideal circumstances. # Migrating to Walking Onions This proposal is a major change in the Tor network that will -eventually require the participation of all relays [*], and will make +eventually require the participation of all relays \[\*\], and will make clients who support it distinguishable from clients that don't. -> [*] Technically, the last relay in the path doesn't need support. +> \[\*\] Technically, the last relay in the path doesn't need support. To keep the compatibility issues under control, here is the order in which it should be deployed on the network. @@ -3903,6 +3907,7 @@ guards and/or exits depending on overall balance of resources on the network. Formula: + type: 'weighted', source: { type:'bw', require_flags: ['Valid'], 'bwfield' : ["RM", "mbw"] @@ -3978,10 +3983,10 @@ Formula: ## Appendix H: Choosing good clusters of exit policies With Walking Onions, we cannot easily support all the port -combinations [*] that we currently allow in the "policy summaries" +combinations \[\*\] that we currently allow in the "policy summaries" that we support in microdescriptors. -> [*] How many "short policy summaries" are there? The number would be +> \[\*\] How many "short policy summaries" are there? The number would be > 2^65535, except for the fact today's Tor doesn't permit exit policies to > get maximally long. diff --git a/proposals/324-rtt-congestion-control.txt b/proposals/324-rtt-congestion-control.txt index 4235be8..c601980 100644 --- a/proposals/324-rtt-congestion-control.txt +++ b/proposals/324-rtt-congestion-control.txt @@ -1,3 +1,4 @@ +``` Filename: 324-rtt-congestion-control.txt Title: RTT-based Congestion Control for Tor Author: Mike Perry @@ -2313,3 +2314,4 @@ receive more data. It is sent to tell the sender to resume sending. 34. https://gitlab.torproject.org/tpo/core/tor/-/issues/40642 35. https://gitlab.torproject.org/tpo/network-health/analysis/-/issues/49 +``` diff --git a/proposals/325-packed-relay-cells.md b/proposals/325-packed-relay-cells.md index 7a88840..7d0ffca 100644 --- a/proposals/325-packed-relay-cells.md +++ b/proposals/325-packed-relay-cells.md @@ -57,23 +57,27 @@ concatenated one after another following this format of a relay cell. The first command is the same header format as a normal relay cell detailed in section 6.1 of tor-spec.txt - Relay Command [1 byte] - 'Recognized' [2 bytes] - StreamID [2 bytes] - Digest [4 bytes] - Length [2 bytes] - Data [Length bytes] - RELAY\_MESSAGE - Padding [up to end of cell] +```text +Relay Command [1 byte] +'Recognized' [2 bytes] +StreamID [2 bytes] +Digest [4 bytes] +Length [2 bytes] +Data [Length bytes] +RELAY\_MESSAGE +Padding [up to end of cell] +``` The `RELAY_MESSAGE` can be empty as in no bytes indicating no other messages or set to the following: - Relay Command [1 byte] - StreamID [2 bytes] - Length [2 bytes] - Data [Length bytes] - RELAY\_MESSAGE +```text +Relay Command [1 byte] +StreamID [2 bytes] +Length [2 bytes] +Data [Length bytes] +RELAY\_MESSAGE +``` Note that the Recognized and Digest field are not added to a second relay message, they are solely used for the whole relay cell thus how we @@ -123,10 +127,12 @@ with the default set to use a consensus parameter. The parameter is: - "relay-cell-packing" +```text +"relay-cell-packing" - Boolean: if 1, clients should send packed relay cells. - (Min: 0, Max 1, Default: 0) +Boolean: if 1, clients should send packed relay cells. +(Min: 0, Max 1, Default: 0) +``` To handle migration, first the parameter should be set to 0 and the configuration setting should be "auto". To test the feature, individual @@ -150,17 +156,21 @@ I propose a new relay message format, described here (with `ux` denoting an x-bit bitfield). This format is 2 bytes or 4 bytes, depending on its first bit. - struct relay_header { - u1 stream_id_included; // Is the stream_id included? - u6 relay_command; // as before - u9 relay_data_len; // as before - u8 optional_stream_id[]; // 0 bytes or two bytes. - } +```C +struct relay_header { + u1 stream_id_included; // Is the stream_id included? + u6 relay_command; // as before + u9 relay_data_len; // as before + u8 optional_stream_id[]; // 0 bytes or two bytes. +} +``` Alternatively, you can view the first three fields as a 16-bit value, computed as: - (stream_id_included<<15) | (relay_command << 9) | (relay_data_len). +```C +(stream_id_included<<15) | (relay_command << 9) | (relay_data_len). +``` If the `optional_stream_id` field is not present, then the default value for the `stream_id` is computed as follows. We use stream_id 0 diff --git a/proposals/326-tor-relay-well-known-uri-rfc8615.md b/proposals/326-tor-relay-well-known-uri-rfc8615.md index 075aa6d..8bc705a 100644 --- a/proposals/326-tor-relay-well-known-uri-rfc8615.md +++ b/proposals/326-tor-relay-well-known-uri-rfc8615.md @@ -38,7 +38,7 @@ The verification of listed Tor relay/bridge IDs only succeeds if the claim can b * Each line contains one relay fingerprint. * The file MUST NOT contain fingerprints of Tor bridges (or hashes of bridge fingerprints). For bridges see the file `hashed-bridge-rsa-fingerprint.txt`. * The file may contain comments (starting with #). -* Non-comment lines must be exactly 40 characters long and consist of the following characters [a-fA-F0-9]. +* Non-comment lines must be exactly 40 characters long and consist of the following characters `[a-fA-F0-9]`. * Fingerprints are not case-sensitive. * Each fingerprint MUST appear at most once. * The file MUST not be larger than one MByte. @@ -59,7 +59,7 @@ The RSA SHA1 relay fingerprint can be found in the file named "fingerprint" loca * This file is not relevant for bridges. * Each line contains one public ed25519 master key in its base64 encoded form. * The file may contain comments (starting with #). -* Non-comment lines must be exactly 43 characters long and consist of the following characters [a-zA-z0-9/+]. +* Non-comment lines must be exactly 43 characters long and consist of the following characters `[a-zA-z0-9/+]`. * Each key MUST appear at most once. * The file MUST not be larger than one MByte. * The content MUST be a media type of "text/plain". @@ -80,7 +80,7 @@ The base64 encoded ed25519 public master key can be found in the file named "fin * The file contains one or more SHA1 hashed Tor bridge SHA1 fingerprints operated by the entity in control of this website. * Each line contains one hashed bridge fingerprint. * The file may contain comments (starting with #). -* Non-comment lines must be exactly 40 characters long and consist of the following characters [a-fA-F0-9]. +* Non-comment lines must be exactly 40 characters long and consist of the following characters `[a-fA-F0-9]`. * Hashed fingerprints are not case-sensitive. * Each hashed fingerprint MUST appear at most once. * The file MUST not be larger than one MByte. diff --git a/proposals/327-pow-over-intro.txt b/proposals/327-pow-over-intro.txt index bcaf6f3..d267d3c 100644 --- a/proposals/327-pow-over-intro.txt +++ b/proposals/327-pow-over-intro.txt @@ -1,8 +1,9 @@ +``` Filename: 327-pow-over-intro.txt Title: A First Take at PoW Over Introduction Circuits Author: George Kadianakis, Mike Perry, David Goulet, tevador Created: 2 April 2020 -Status: Finished +Status: Closed 0. Abstract @@ -1207,9 +1208,10 @@ A.2. References [REF_CREDS]: https://lists.torproject.org/pipermail/tor-dev/2020-March/014198.html [REF_TARGET]: https://en.bitcoin.it/wiki/Target [REF_TLS]: https://www.ietf.org/archive/id/draft-nygren-tls-client-puzzles-02.txt - https://tools.ietf.org/id/draft-nir-tls-puzzles-00.html + https://datatracker.ietf.org/doc/html/draft-nir-tls-puzzles-00.html https://tools.ietf.org/html/draft-ietf-ipsecme-ddos-protection-10 [REF_TLS_1]: https://www.ietf.org/archive/id/draft-nygren-tls-client-puzzles-02.txt [REF_TEVADOR_1]: https://lists.torproject.org/pipermail/tor-dev/2020-May/014268.html [REF_TEVADOR_2]: https://lists.torproject.org/pipermail/tor-dev/2020-June/014358.html [REF_TEVADOR_SIM]: https://github.com/mikeperry-tor/scratchpad/blob/master/tor-pow/effort_sim.py#L57 +``` diff --git a/proposals/328-relay-overload-report.md b/proposals/328-relay-overload-report.md index 5901b93..bdb4fec 100644 --- a/proposals/328-relay-overload-report.md +++ b/proposals/328-relay-overload-report.md @@ -41,16 +41,16 @@ state" which can be one or many of the following load metrics: - Any OOM invocation due to memory pressure - Any ntor onionskins are dropped - [Removed in tor-0.4.6.11 and 0.4.7.5-alpha] + \[Removed in tor-0.4.6.11 and 0.4.7.5-alpha\] - A certain ratio of ntor onionskins dropped. - [Added in tor-0.4.6.11 and 0.4.7.5-alpha] + \[Added in tor-0.4.6.11 and 0.4.7.5-alpha\] - TCP port exhaustion - DNS timeout reached (X% of timeouts over Y seconds). - [Removed in tor-0.4.7.3-alpha] + \[Removed in tor-0.4.7.3-alpha\] - CPU utilization of Tor's mainloop CPU core above 90% for 60 sec - [Never implemented] + \[Never implemented\] - Control port overload (too many messages queued) - [Never implemented] + \[Never implemented\] For DNS timeouts, the X and Y are consensus parameters (overload_dns_timeout_scale_percent and overload_dns_timeout_period_secs) diff --git a/proposals/329-traffic-splitting.txt b/proposals/329-traffic-splitting.txt index 2a24f1f..e03d3aa 100644 --- a/proposals/329-traffic-splitting.txt +++ b/proposals/329-traffic-splitting.txt @@ -1,3 +1,4 @@ +``` Filename: 329-traffic-splitting.txt Title: Overcoming Tor's Bottlenecks with Traffic Splitting Author: David Goulet, Mike Perry @@ -1051,3 +1052,4 @@ References: [PROP308] https://gitlab.torproject.org/tpo/core/torspec/-/blob/main/proposals/308-counter-galois-onion.txt +``` diff --git a/proposals/331-res-tokens-for-anti-dos.md b/proposals/331-res-tokens-for-anti-dos.md index a7293e8..448ebe4 100644 --- a/proposals/331-res-tokens-for-anti-dos.md +++ b/proposals/331-res-tokens-for-anti-dos.md @@ -56,20 +56,20 @@ Status: Draft from various types of Token Issuers and then redeem them at the onion service to gain access even when under DoS conditions. - In section [TOKEN_DESIGN], we list our requirements from an anonymous + In section [TOKEN_DESIGN](#token-design), we list our requirements from an anonymous credential scheme and provide a high-level overview of how the Res token scheme works. - In section [PROTOCOL_SPEC], we specify the token issuance and redemption protocols, + In section [PROTOCOL_SPEC](#protocol-spec), we specify the token issuance and redemption protocols, as well as the mathematical operations that need to be conducted for these to work. - In section [TOKEN_ISSUERS], we provide a few examples and guidelines for + In section [TOKEN_ISSUERS](#token-issuers), we provide a few examples and guidelines for various token issuer services that could exist. - In section [DISCUSSION], we provide more use cases for Res tokens as well as + In section [DISCUSSION](#discussion), we provide more use cases for Res tokens as well as future improvements we can conduct to the scheme. -# 3. Design [TOKEN_DESIGN] +# 3. Design \[TOKEN_DESIGN\] {#token-design} In this section we will go over the high-level design of the system, and in the next section we will delve into the lower-level details of the protocol. @@ -109,13 +109,13 @@ Status: Draft - Quick Verification: Onions are already experiencing resource starvation because of the DoS attacks so it's important that the process of - verifying a token should be as quick as possible. In section [TOKEN_PERF] + verifying a token should be as quick as possible. In section [TOKEN_PERF](#token-perf) we will go deeper into this requirement. After careful consideration of the above requirements, we have leaned towards using Blind RSA as the primitive for our tokens, since it's the fastest - scheme by far that also allows public verifiability. See also Appendix B - [BLIND_RSA_PROOF] for a security proof sketch of Blind RSA perfect unlinkability. + scheme by far that also allows public verifiability. See also [Appendix A](#blind-rsa-proof) + for a security proof sketch of Blind RSA perfect unlinkability. ## 3.3. Other security considerations @@ -128,7 +128,7 @@ Status: Draft - Metadata: We want to encode metadata/attributes in the tokens. In particular, we want to encode the destination onion service and an - expiration date. For more information see section [DEST_DIGEST]. For + expiration date. For more information see section [DEST_DIGEST](#dest-digest). For blind RSA tokens this is usually done using "partially blind signatures" but to keep it simple we instead encode the destination directly in the message to be blind-signed and the expiration date using a set of @@ -162,9 +162,9 @@ Status: Draft an adversary breaks an issuance key, she will be able to forge tokens for just a few hours before that key expires. - For more ideas on future schemes and improvements see section [FUTURE_RES]. + For more ideas on future schemes and improvements see section [FUTURE_RES](#future-res). -## 3.5. Token performance requirements [TOKEN_PERF] +## 3.5. Token performance requirements \[TOKEN_PERF\] {#token-perf} As discussed above, verification performance is extremely important in the anti-DoS use case. In this section we provide some concrete numbers on what @@ -188,14 +188,14 @@ Status: Draft For this reason we implemented a basic version of the Res token scheme in Rust and benchmarked the verification and issuance procedure [REF_RES_BENCH]. - We measured that the verification procedure from section [RES_VERIFY] takes + We measured that the verification procedure from section [RES_VERIFY](#res-verify) takes about 0.104 ms, which we believe is a reasonable verification overhead for the purposes of this proposal. - We also measured that the issuance procedure from [RES_ISSUANCE] takes about + We also measured that the issuance procedure from [RES_ISSUANCE](#res-issuance) takes about 0.614 ms. -# 4. Specification [PROTOCOL_SPEC] +# 4. Specification \[PROTOCOL_SPEC\] {#protocol-spec} +--------------+ +------------------+ | Token Issuer | | Onion Service | @@ -270,7 +270,7 @@ Status: Draft onion service in its token store. If not, it needs to acquire some and hence the token issuance protocol commences. -### 4.3.1. Client preparation [DEST_DIGEST] +### 4.3.1. Client preparation \[DEST_DIGEST\] {#dest-digest} Alice first chooses an issuer supported by the onion service depending on her preferences by looking at the consensus and her Tor configuration file for @@ -289,7 +289,7 @@ Status: Draft - 'destination' is the 32-byte ed25519 public identity key of the destination onion - 'salt' is a random 32-byte value, - 3) Alice samples a blinding factor 'r' uniformly at random from [1, N) + 3) Alice samples a blinding factor 'r' uniformly at random from \[1, N) 4) Alice computes: blinded_message = dest_digest * r^e (mod N) @@ -301,7 +301,7 @@ Status: Draft XXX Is the salt needed? Reevaluate. -### 4.3.3. Token Issuance [RES_ISSUANCE] +### 4.3.3. Token Issuance \[RES_ISSUANCE\] {#res-issuance} Alice now initiates contact with the Token Issuer and spends the resources required to get issued a token (e.g. solve a CAPTCHA or a PoW, create an @@ -348,8 +348,9 @@ Status: Draft onion service. To do so, Alice adds an extension to the encrypted portion of the INTRODUCE1 - cell by using the EXTENSIONS field (see [PROCESS_INTRO2] section in - rend-spec-v3.txt). The encrypted portion of the INTRODUCE1 cell only gets + cell by using the EXTENSIONS field (see [PROCESS_INTRO2 section in + rend-spec-v3.txt](../rend-spec/introduction-protocol.md#PROCESS_INTRO2)). + The encrypted portion of the INTRODUCE1 cell only gets read by the onion service and is ignored by the introduction point. We propose a new EXT_FIELD_TYPE value: @@ -365,7 +366,7 @@ Status: Draft SALT [32 bytes] where: - - TOKEN_VERSION is the version of the token ([0x01] for Res tokens) + - TOKEN_VERSION is the version of the token (\[0x01\] for Res tokens) - ISSUER_KEY is the public key of the chosen issuer (truncated to 4 bytes) - DEST_DIGEST is the 'dest_digest' from above - TOKEN is the 'token' from above @@ -381,7 +382,7 @@ Status: Draft XXX maybe with a bit of tweaking we can even use a 1536-bit RSA signature here... -### 4.4.2. Onion service verifies token [RES_VERIFY] +### 4.4.2. Onion service verifies token \[RES_VERIFY\] {#res-verify} Upon receiving an INTRODUCE1 cell with the above extension the service verifies the token. It does so as follows: @@ -408,7 +409,7 @@ Status: Draft considers the token valid and the rest of the onion service protocol carries out as normal. -# 5. Token issuers [TOKEN_ISSUERS] +# 5. Token issuers \[TOKEN_ISSUERS\]{#token-issuers} In this section we go over some example token issuers. While we can have official token issuers that are supported by the Tor directory authorities, @@ -472,7 +473,7 @@ Status: Draft one-show, so the onion service cannot provide a single token that will work for multiple "logins". In the future we can design multi-show credential systems that also have revocation to further facilitate this use case (see - [FUTURE_RES] for more info). + [FUTURE_RES](#future-res) for more info). # 6. User Experience @@ -506,7 +507,7 @@ Status: Draft XXX Actually analyze the above if we think there is merit to listing them -# 8. Discussion [DISCUSSION] +# 8. Discussion \[DISCUSSION\] {#discussion} ## 8.1. Using Res tokens on Exit relays @@ -525,7 +526,7 @@ Status: Draft websites and web services on the public Internet. We hope that this way we will see less websites blocking Tor. -## 8.2. Future improvements to this proposal [FUTURE_RES] +## 8.2. Future improvements to this proposal \[FUTURE_RES\] {#future-res} The Res token scheme is a pragmatic scheme that works for the space/time constraints of this use case but it's far from ideal for the greater future @@ -579,7 +580,7 @@ Status: Draft --- -# Appendix A: RSA Blinding Security Proof [BLIND_RSA_PROOF] +# Appendix A: RSA Blinding Security Proof \[BLIND_RSA_PROOF\] {#blind-rsa-proof} This proof sketch was provided by Michele Orrù: diff --git a/proposals/332-ntor-v3-with-extra-data.md b/proposals/332-ntor-v3-with-extra-data.md index 58a3bf3..10f0dd4 100644 --- a/proposals/332-ntor-v3-with-extra-data.md +++ b/proposals/332-ntor-v3-with-extra-data.md @@ -47,7 +47,7 @@ The client knows: * An optional "verification" string. The relay knows: - * A set of [(b,B)...] "onion key" keypairs. One of them is + * A set of \[(b,B)...\] "onion key" keypairs. One of them is "current", the others are outdated, but still valid. * ID: Its own identity. * A function for computing a server message SM, based on a given @@ -398,7 +398,7 @@ client sends a message, with type `CIRCWINDOW_INC`, containing a two-byte integer equal to `circwindow_inc_dflt`. The relay rejects the message if the value given is outside of the -[`circwindow_inc_min`, `circwindow_inc_max`] range. Otherwise, it +\[`circwindow_inc_min`, `circwindow_inc_max`\] range. Otherwise, it accepts it, and replies with the same message that the client sent. # X.2: Test vectors diff --git a/proposals/333-vanguards-lite.md b/proposals/333-vanguards-lite.md index 8c1ccb9..1e83046 100644 --- a/proposals/333-vanguards-lite.md +++ b/proposals/333-vanguards-lite.md @@ -3,7 +3,7 @@ Filename: 333-vanguards-lite.md Title: Vanguards lite Author: George Kadianakis, Mike Perry Created: 2021-05-20 -Status: Finished +Status: Closed Implemented-In: 0.4.7.1-alpha ``` diff --git a/proposals/334-middle-only-flag.txt b/proposals/334-middle-only-flag.txt index 823e927..c011b6c 100644 --- a/proposals/334-middle-only-flag.txt +++ b/proposals/334-middle-only-flag.txt @@ -1,3 +1,4 @@ +``` Filename: 334-middle-only-flag.txt Title: A Directory Authority Flag To Mark Relays As Middle-only Author: Neel Chauhan @@ -114,3 +115,4 @@ Superseded-by: 335-middle-only-redux.md [2] - https://lists.torproject.org/pipermail/tor-dev/2021-September/014627.html [3] - https://lists.torproject.org/pipermail/tor-dev/2021-September/014630.html +``` diff --git a/proposals/339-udp-over-tor.md b/proposals/339-udp-over-tor.md index 5993bdc..12de0c6 100644 --- a/proposals/339-udp-over-tor.md +++ b/proposals/339-udp-over-tor.md @@ -264,7 +264,7 @@ I'd suggest we do it. 1. We would add a new "`FLAG_UNCONNECTED`" flag for `CONNECT_UDP` messages. -2. We would designate the ANY addresses 0.0.0.0:0 and [::]:0 as permitted in +2. We would designate the ANY addresses 0.0.0.0:0 and \[::\]:0 as permitted in `CONNECT_UDP` messages, and as indicating unconnected sockets. These would be only permitted along with the `FLAG_UNCONNECTED` flag, and not permitted otherwise. diff --git a/proposals/340-packed-and-fragmented.md b/proposals/340-packed-and-fragmented.md index 822ba05..da0cbdb 100644 --- a/proposals/340-packed-and-fragmented.md +++ b/proposals/340-packed-and-fragmented.md @@ -31,7 +31,7 @@ This proposal combines ideas from [proposal 319](./319-wide-everything.md) (fragmentation) and [proposal 325](./325-packed-relay-cells.md) (packed cells). It requires [ntor v3](./332-ntor-v3-with-extra-data.md) and prepares for -[next-generation relay cryptography](./308-counter-galois-onion). +[next-generation relay cryptography](./308-counter-galois-onion.md). Additionally, this proposal has been revised to incorporate another protocol change, and move StreamId from the relay cell header into a new, @@ -51,9 +51,11 @@ formats going on at the same time. The new format for a decrypted relay _cell_ will be: - recognized [2 bytes] - digest [14 bytes] - body [509 - 16 = 493 bytes] +```text +recognized [2 bytes] +digest [14 bytes] +body [509 - 16 = 493 bytes] +``` The `recognized` and `digest` fields are computed as before; the only difference is that they occur _before_ the rest of the cell, and that `digest` diff --git a/proposals/343-rend-caa.txt b/proposals/343-rend-caa.txt index 0859690..1de1355 100644 --- a/proposals/343-rend-caa.txt +++ b/proposals/343-rend-caa.txt @@ -1,3 +1,4 @@ +``` Filename: 343-rend-caa.txt Title: CAA Extensions for the Tor Rendezvous Specification Author: Q Misell <q@as207960.net> @@ -108,4 +109,4 @@ References: [tor-rend-spec-v3] The Tor Project, "Tor Rendezvous Specification - Version 3", - <https://spec.torproject.org/rend-spec-v3>.
\ No newline at end of file + <https://spec.torproject.org/rend-spec-v3>.``` diff --git a/proposals/344-protocol-info-leaks.txt b/proposals/344-protocol-info-leaks.txt index abae2a7..f20ebca 100644 --- a/proposals/344-protocol-info-leaks.txt +++ b/proposals/344-protocol-info-leaks.txt @@ -1,3 +1,4 @@ +``` Filename: 344-protocol-info-leaks.txt Title: Prioritizing Protocol Information Leaks in Tor Author: Mike Perry @@ -1184,3 +1185,4 @@ References: [WFNETSIM] https://petsymposium.org/2023/files/papers/issue4/popets-2023-0125.pdf +``` diff --git a/proposals/345-specs-in-mdbook.md b/proposals/345-specs-in-mdbook.md index 697134d..40c9504 100644 --- a/proposals/345-specs-in-mdbook.md +++ b/proposals/345-specs-in-mdbook.md @@ -3,7 +3,7 @@ Filename: 345-specs-in-mdbook.md Title: Migrating the tor specifications to mdbook Author: Nick Mathewson Created: 2023-10-03 -Status: Open +Status: Closed ``` # Introduction @@ -114,6 +114,7 @@ We can tweak the scripts until we like the that they produce. After running a recent `torspec-converter` on a fairly recent torspec.git, here is how the branch looks: + https://gitlab.torproject.org/nickm/torspec/-/tree/spec_conversion?ref_type=heads And here's the example output when running mdbook on that branch: diff --git a/proposals/346-protovers-again.md b/proposals/346-protovers-again.md new file mode 100644 index 0000000..a18e238 --- /dev/null +++ b/proposals/346-protovers-again.md @@ -0,0 +1,297 @@ +``` +Filename: 346-protovers-again.md +Title: Clarifying and extending the use of protocol versioning +Author: Nick Mathewson +Created: 19 Oct 2023 +Status: Open +``` + +# Introduction + +In proposal 264, we introduced "subprotocol versions" as a way to +independently version different pieces of the Tor protocols, and +communicate which parts of the Tor protocols are supported, +recommended, and required. + +Here we clarify the semantics of individual subprotocol versions, and +describe more ways to use and negotiate them. + +# Semantics: Protocol versions are feature flags + +One issue we left unclarified previously is the relationship between +two different versions of the same subprotocol. That is, if we know +the semantics of (say) `Handshake=7`, can we infer anything about a +relay that supports `Handshake=8`? In particular, can we infer that +it supports all of the same features implied by `Handshake=7`? If we +want to know "does this relay support some feature supported by +`Handshake=7`", must we check whether it supports `Handshake=7`, or +should we check `Handshake=x for any x≥7`? + +In this proposal, we settle the question as follows: subprotocol +versions are flags. They do not have any necessary semantic +relationship between them. + +We reject the `≥` interpretation for several reasons: + * It's tricky to implement. + * It prevents us from ever removing a feature. + * It requires us to implement features in the same order across + all Tor versions. + +## ...but sometimes a flag _is_ a version! + +There _are_ some places in our protocol (notably: directory authority +consensus methods, and channel protocol versions) where there _is_ a +semantic relationship between version numbers. Specifically: "higher +numbers are already better". When parties need to pick a one of +_these_ versions, they always pick the highest version number +supported by enough of them. + +When this kind of _real_ version intersects with the "subprotocol +versions" system, we use the same numbers: + + * `Link` subprotocols correspond one-to-one with the version numbers + sent in a VERSIONS cell. + * `Microdesc` and `Cons` subprotocols correspond to a _subset_ of + the version numbers of consensus methods. + +## How to document subprotocol versions + +When describing a subprotocol, we should be clear what relationship, +if any, exists between its versions and any versions negotiated +elsewhere in the specifications. + +Unless otherwise documented, all versions can be in use _at the same +time_: if only one can exist at once (on a single circuit, a single +document, etc), this must be documented. + +> Implication: This means that we must say (for example) that you +> can't use Link=4 and Link=5 on the same channel. + +# Negotiating protocol versions in circuit handshakes. + +Here we describe a way for a client to opt into features as part of +its circuit handshake, in order to avoid proliferating negotiating +extensions. + +## Binary-encoding protocol versions. + +We assign a one-byte encoding for each protocol version number, +ordered in the same way as in tor-spec. + +| Protocol | Id | +| --- | --- | +| Link | 0 | +| LinkAuth | 1 | +| Relay | 2 | +| DirCache | 3 | +| HSDir | 4 | +| HSIntro | 5 | +| HSRend | 6 | +| Desc | 7 | +| MicroDesc| 8 | +| Cons | 9 | +| Padding | 10 | +| FlowCtrl | 11 | +| Conflux | 12 | +| Datagram | 13 | + +> Note: This is the same encoding used in [walking onions proposal][prop323]. +> It takes its order from the ordering of protocol versions in +> [tor-spec][subprotocol-versioning] and matches up with the values defined in +> for `protocol_type_t` in C tor's `protover.h`. + +## Requesting an opt-in circuit feature + +When a client wants to request a given set of features, it sends an +`ntor_v3` extension containing: + +``` +struct subproto_request { + struct req[..]; // up to end of extension +} + +struct req { + u8 protocol_id; + u8 protovol_version; +} +``` + +> Note 1: The above format does not include any parameters for each +> req. Thus, if we're negotiating an extension that requires a client- +> supplied parameter, it may not be appropriate to use this +> request format. +> +> Note 2: This proposal does not include any relay extension +> acknowledging support. In the case of individual subprotocols, we could +> later say "If this subprotocol is in use, the relay MUST also send +> extension foo". +> +> Note 3: The existence of this extension does not preclude the later +> addition of other extensions to negotiate featuress differently, or +> to do anything else. + +Each `req` entry corresponds to a single subprotocol version. A client +MUST NOT send any `req` entry unless: + * That subprotocol version is advertised by the relay, + * OR that subprotocol version is listed as required for relays in the + current consensus, using `required-relay-protocols`. + +> Note: We say above that a client may request a _required_ subprotocol +> even if the relay does not advertise it. This is what allows +> clients to send a `req` extension to introduction points and +> rendezvous points, even when we do not recognize the relay from the +> consensus. +> +> Note 2: If the intro/rend point does not support a _required_ protocol, +> it should not be on the network, and the client/service should not have +> selected it. + +If a relay receives a `subproto_request` extension for any subprotocol +version that it does not support, it MUST reject the circuit with a +DESTROY cell. + +> Alternatives: we _could_ give the relay the option to +> decline to support an extension, and we _could_ require the +> relay to acknowledge which extensions it is providing. +> We aren't doing that, in the name of simplicity. + +Only certain subprotocol versions need to be negotiated in this way; +they will be explicitly listed as such in our specifications, with +language like "This extension is negotiated as part of the circuit +extension handshake". Other subprotocol versions MUST NOT be listed +in this extension; if they are, the relay SHOULD reject the circuit. + +> Alternative: We _could_ allow the client to list other subprotocols +> that the relay supports which are nonetheless irrelevant to +> the circuit protocol, like `Microdesc`, or ones that don't currently need +> to be negotiated, like `HsRend`. +> +> This is not something we plan to do. + +Currently specified subprotocol versions which can be negotiated using +this extension are: + + * FlowCtrl=2 (congestion control) + * Packed-and-fragmented support (proposal 340) + +The availability of the `subproto_request` extension itself +will be indicated by a new Relay=X flag. When used, it will supplant +several older `ntor_v3` extensions, including: + + * (TODO: list these here, if we find any. I think FlowCtrl has an + extension?) + +That is, if using `subproto_request`, there is no need to send the +(TODO) extensions. + + + +## Making features that can be disabled. + +Sometimes, we will want the ability to make features that can be +enabled or disabled from the consensus. But if we were to make a single +flag that can turn the feature on and off, we'd run into trouble: +after the feature was turned off, every relay would stop providing it +right away, but there would be a delay before clients realized that +the relays had stopped advertising the feature. During this interval, +clients would try to enable the feature, and the relays would reject +their circuits. + +To solve this problem, we need to make features like these controlled +by a _pair_ of consensus parameters: one to disable advertising the +feature, and one to disable the feature itself. To disable a feature, +first the authorities would tell relays to stop advertising it, and +only later tell the relays to stop supporting it. (If we want to +_enable_ a previously disabled feature, we can turn on advertisement +and support at the same time.) + +These parameters would be specified something like this (for a +hypthetical `Relay=33` feature). + + * `support-relay-33`: if set to 1, relays that can provide + `Relay=33` should do so. + * `advertise-relay-33`: if set to 1, relays that are + providing `Relay=33` should include it in their advertised + protocol versions. + +Note: as a safety measure, relays MUST NOT advertise any feature that +they do not support. This is reflected in the descriptions of the +parameters above. + +When we add a new feature of this kind, we should have the +`advertise-*` flag parameter be `1` by default, and probably we should +have `support-*` be `1` by default oo. + +# Subprotocol versions in onion services + +Here we describe how to expand the onion service protocols in order +to better accomodate subprotocol versions. + +## Advertising an onion service's subprotocols + +In its encrypted descriptor (the innermost layer), the onion service +adds a new entry: + + * `"protocols"` - A list of supported subprotocol versions, in the + same format as those listed in a microdescriptor or descriptor. + +Note that this is NOT a complete list of all the subprotocol versions +actually supported by the onion service. Instead, onion services only +advertise a subprotocol version if they support it, _and_ it is +documented in the specs as being supported by onion services. + +> Alternative: I had considered having a mask that would be put in the +> consensus document, telling the onion services which subprotocols +> to advertise. I don't think that's a great idea, however. + +Right now, the following protocols should be advertised: + + - FlowCtrl + - Conflux (?? Doesn't this take parameters? TODO) + - Pow (??? Doesn't this take parameters? If we do it, we need to + allocate a subprotocol for it. TODO) + + +## Negotiating subprotocols with an onion service. + +In the `hs_ntor` handshake sent by the client, we add an encrypted +`subproto_request` extension of the same format, with the same +semantics, as used in the `ntor-v3` handshake. + +This supplants the following: + + - (Congestion Control; Pow? TODO) + + +## Advertising other relays' subprotocols? + +> Alternative: I had previously considered a design where the introduction points +> in the onion service descriptor would be listed along with their +> subprotocols, and the `hs_ntor` handshake would contain the +> subprotocols of the rendezvous point. +> +> I'm rejecting this design for now because it gives the onion service +> and the client too much freedom to lie about relays. In the future, +> the walking onions design would solve this, since the contact +> information for intro and rend points would be authenticated. + +# Appendix + +New numbers to reserve: + + * An extension ID for the `ntor_v3` handshake `subproto_request` + extension. + * An extension ID for the `hs_ntor` handshake `subproto_request` + extension. + * A Relay= subprotocol indicating support for the ntor-v3 and + hs_ntor extensions. + * The numeric encoding of each existing subprotocol, in the table + above. + +# Acknowledgments + +Thanks to David Goulet and Mike Perry for their feedback on earlier +versions of this proposal! + +[prop323]: https://spec.torproject.org/proposals/323-walking-onions-full.html +[subprotocol-versioning]: https://spec.torproject.org/tor-spec/subprotocol-versioning.html diff --git a/proposals/347-domain-separation.md b/proposals/347-domain-separation.md new file mode 100644 index 0000000..3056d10 --- /dev/null +++ b/proposals/347-domain-separation.md @@ -0,0 +1,80 @@ +``` +Filename: 347-domain-separation.md +Title: Domain separation for certificate signing keys +Author: Nick Mathewson +Created: 19 Oct 2023 +Status: Open +``` + +## Our goal + +We'd like to be able to use the "family key" from proposal 321 as a +general purpose signing key, to authenticate other things than the +membership of a family. For example, we might want to have a +challenge/response mechanism where the challenger says, "If you want +to log in as the owner of the account corresponding to this family, +sign the following challenge with your key. Or we might want to +have a message authentication scheme where an operator can +sign a message in a way that proves key ownership. + +We _might_ also like to use relay identity keys or onion service +identitiy keys for the same purpose. + +## The problem + +When we're using a signing key for two purposes, it's important +to perform some kind of domain separation so that documents +signed for one purpose can't be mistaken for documents signed for the +other. + +For example, in the challenge/response example, it would be bad if +the challenger could provide a challenge string that would cause the +signer to inadvertently authenticate an incorrect family. + +These keys are currently used in some places with no +personalization. Their signature format is as described in +`cert-spec.txt`, which says: + +> The signature is created by signing all the fields in +> the certificate up until "SIGNATURE" (that is, signing +> sizeof(ed25519_cert) - 64 bytes). + + + +## One solution + +This one is pretty easy: we would extend `cert-spec` as follows. + + +> ### Using signing keys for other purposes. +> +> Other protocols may introduce other uses for the signing keys +> in these certificates other than those specified here. When +> they do, they MUST ensure that the documents being signed +> cannot be confused with the certificate bodies of this document. +> +> In some existing cases in the Tor protocols, we achieve this by +> specifying an ASCII prefix string that must be prepended to the +> other protocol's signed object before it is signed. +> +> For future protocols, we recommend that this be done by +> specifying that the signing key is to be used to sign a +> cSHAKE digest (or other secure customizable digest) +> of the other protocol's signed object, using +> a customization string unique to the other protocol. + +We would also make this amendment: + +> ### Future versions of this specification +> +> In order to maintain the domain separation that currently exists +> between the signatures on these certificates and other documents +> signed with the same keys, it suffices (for now!) that these +> certificates begin with the version byte `[01]`, whereas +> the other documents are in printable ASCII, which never +> includes `[01]`. +> +> Future versions of this specification SHOULD move to using +> an ed25519-prehash construction, using a customizable hash +> with built-in domain separation. + diff --git a/proposals/348-udp-app-support.md b/proposals/348-udp-app-support.md new file mode 100644 index 0000000..9482a8f --- /dev/null +++ b/proposals/348-udp-app-support.md @@ -0,0 +1,742 @@ +``` +Filename: 348-udp-app-support.md +Title: UDP Application Support in Tor +Author: Micah Elizabeth Scott +Created: December 2023 +Status: Open +``` + +# UDP Application Support in Tor + +## Table of Contents + +- [Introduction](#introduction) + - [History](#history) + - [Scope](#scope) +- [UDP Traffic Models](#udp-traffic-models) + - [User Datagram Protocol (RFC768)](#user-datagram-protocol-rfc768) + - [Socket Layer](#socket-layer) + - [Network Address Translation (NAT)](#network-address-translation-nat) + - [Mapping and Filtering Behaviors](#mapping-and-filtering-behaviors) + - [Common Protocols](#common-protocols) + - [QUIC](#quic) + - [WebRTC](#webrtc) + - [Common Applications](#common-applications) +- [Overview of Possible Solutions](#overview-of-possible-solutions) + - [Datagram Routing](#datagram-routing) + - [Intentional UDP Leak](#intentional-udp-leak) + - [3rd Party Implementations](#3rd-party-implementations) + - [Future Work on Tor](#future-work-on-tor) + - [Tunneling](#tunneling) + - [TURN Encapsulated in Tor Streams](#turn-encapsulated-in-tor-streams) + - [Tor Stream Tunnel to an Exit](#tor-stream-tunnel-to-an-exit) + - [Tor Stream Tunnel to a Rendezvous Point](#tor-stream-tunnel-to-a-rendezvous-point) +- [Specific Designs Using Tor Streams](#specific-designs-using-tor-streams) + - [One Stream per Tunnel (VPN)](#one-stream-per-tunnel-vpn) + - [One Stream per Local Port (TURN)](#one-stream-per-local-port-turn) + - [One Stream per Local Port (Proposal 339)](#one-stream-per-local-port-proposal-339) + - [One Stream per Local Port (NAT Mapping)](#one-stream-per-local-port-nat-mapping) + - [One Stream per Flow](#one-stream-per-flow) + - [Hybrid Mapping and Flow Approach](#hybrid-mapping-and-flow-approach) +- [Risks](#risks) + - [Behavior Regressions](#behavior-regressions) + - [Bandwidth Usage](#bandwidth-usage) + - [Local Port Exhaustion](#local-port-exhaustion) + - [Application Fingerprinting](#application-fingerprinting) + - [Peer-to-Peer Metadata Collection](#peer-to-peer-metadata-collection) + - [Interaction with Other Networks](#interaction-with-other-networks) + - [Traffic Injection](#traffic-injection) + - [Malicious Outgoing Traffic](#malicious-outgoing-traffic) +- [Next Steps](#next-steps) + - [Requiring a Long-Term Datagram Plan](#requiring-a-long-term-datagram-plan) + - [Alternatively, Modular Application-Level Support](#alternatively-modular-application-level-support) + +## Introduction + +This proposal takes a fresh look at the problem of implementing support in Tor for applications which require UDP/IP communication. + +This work is being done with the sponsorship and goals of the [Tor VPN Client for Android project](https://gitlab.torproject.org/groups/tpo/-/milestones/32). + +The proposal begins with a summary of previous work and the specific problem space being addressed. +This leads into an analysis of possible solutions, and finally some possible conclusions about the available development opportunities. + +### History + +There have already been multiple attempts over Tor's history to define some type of UDP extension. + +#### 2006 + +[Proposal 100](https://spec.torproject.org/proposals/100-tor-spec-udp.html) by Marc Liberatore in 2006 suggested a way to "add support for tunneling unreliable datagrams through tor with as few modifications to the protocol as possible." +This proposal suggested extending the existing TLS+TCP protocol with a new DTLS+UDP link mode. +The focus of this work was on a potential way to support unreliable traffic, not necessarily on UDP itself or on UDP applications. + +In proposal 100, a Tor *stream* is used for one pairing of local and remote address and port, copying the technique used by Tor for TCP. +This works for some types of UDP applications, but it's broken by common behaviors like ICE connectivity checks, NAT traversal attempts, or using multiple servers via the same socket. + +No additional large-message fragmentation protocol is defined, so the MTU in proposal 100 is limited to what fits in a single Tor cell. +This value is much too small for most applications. + +It's possible these UDP protocol details would have been elaborated during design, but the proposal hit a snag elsewhere: +there was no agreement on a way to avoid facilitating new attacks against anonymity. + +#### 2014 + +In [a thread on the tor-talk mailing list](https://tor-talk.torproject.narkive.com/Weyl6Vrq/using-udpgw-and-tun2socks-over-tor), Nathan Freitas suggested UDP tunneling over Tor using the [*BadVPN*](https://github.com/ambrop72/badvpn) project's `udpgw` protocol. + +This protocol was never formally documented and is no longer actively maintained, but it was very broadly similar in scope to a [RFC8656](https://www.rfc-editor.org/rfc/rfc8656) TURN relay operating over a TCP transport. + +#### 2018 + +In 2018, Nick Mathewson and Mike Perry wrote a +[summary of the side-channel issues with unreliable transports for Tor](https://research.torproject.org/techreports/side-channel-analysis-2018-11-27.pdf). + +The focus of this document is on the communication between Tor relays, but there is considerable overlap between the attack space explored here and the potential risks of any application-level UDP support. +Attacks that are described here, such as drops and injections, may be applied by malicious exits or some types of third parties even in an implementation using only present-day reliable Tor transports. + +#### 2020 + +[Proposal 339](https://spec.torproject.org/proposals/339-udp-over-tor.html) by Nick Mathewson in 2020 introduced a simpler UDP encapsulation design which had similar stream mapping properties as in proposal 100, but with the unreliable transport omitted. Datagrams are tunneled over a new type of Tor stream using a new type of Tor message. +As a prerequisite, it depends on [proposal 319](https://spec.torproject.org/proposals/319-wide-everything.html) to support messages that may be larger than a cell, extending the MTU to support arbitrarily large UDP datagrams. + +In proposal 339 the property of binding a stream both to a local port and to a remote peer is described in UNIX-style terminology as a *connected socket*. This idea is explored below using alternate terminology from [RFC4787](https://www.rfc-editor.org/rfc/rfc4787), *NAT behavior requirements for UDP*. +The single-peer *connected socket* behavior would be referred to as an *endpoint-dependent mapping* in RFC4787. +This type works fine for client/server apps but precludes the use of NAT traversal for peer-to-peer transfer. + +### Scope + +This proposal aims to allow Tor applications and Tor-based VPNs to provide compatibility with applications that require UDP/IP communications. + +We don't have a specific list of applications that must be supported, but we are currently aiming for broad support of popular applications while still respecting and referencing all applicable Internet standards documents. + +Changes to the structure of the Tor network are out of scope, as are most performance optimizations. We expect to rely on common optimizations to the performance of Tor circuits, rather than looking to make specific changes that optimize for unreliable datagram transmission. + +This document will briefly discuss UDP for onion services below. +It's worth planning for this as a way to evaluate the future design space, but in practice we are not aiming for UDP onion services yet. +This will require changes to most applications that want to use it, as it implies that any media negotiations will need to understand onion addressing in addition to IPv4 and IPv6. + +The allowed subset of UDP traffic is not subject to a single rigid definition. +There are several options discussed below using the RFC4787 framework. + +We require support for DNS clients. Tor currently only supports a limited subset of DNS queries, and it's desirable to support more. This will be analyzed in detail as an application below. DNS is one of very few applications that still rely on fragmented UDP datagrams, though this may not be relevant for us since only servers typically need to control the production of fragments. + +We require support for voice/video telecommunications apps. Even without an underlying transport that supports unreliable datagrams, we expect a tunnel to provide a usable level of compatibility. This design space is very similar to the TURN ([RFC8656](https://www.rfc-editor.org/rfc/rfc8656)) specification, already used very widely for compatibility with networks that filter UDP. See the analysis of specific applications below. + +We require support for peer-to-peer UDP transfer without additional relaying, in apps that use ICE ([RFC8445](https://www.rfc-editor.org/rfc/rfc8445)) or similar connection establishment techniques. +Video calls between two Tor users should transit directly between two exit nodes. +This requires that allocated UDP ports can each communicate with multiple peers: +*endpoint-independent mapping* as described by RFC4787. + +We do not plan to support applications which accept incoming datagrams from previously-unknown peers, for example a DNS server hosted via Tor. +RFC4787 calls this *endpoint-independent filtering*. +It's unnecessary for running peer-to-peer apps, and it facilitates an extremely easy [traffic injection](#traffic-injection) attack. + +## UDP Traffic Models + +To better specify the role of a UDP extension for Tor, this section explores a few frameworks for describing noteworthy subsets of UDP traffic. + +### User Datagram Protocol (RFC768) + +The "User Interface" suggested by [RFC768](https://www.rfc-editor.org/rfc/rfc768) for the protocol is a rough sketch, suggesting that applications have some way to allocate a local port for receiving datagrams and to transmit datagrams with arbitrary headers. + +Despite UDP's simplicity as an application of IP, we do need to be aware of IP features that are typically hidden by TCP's abstraction. + +UDP applications typically try to obtain an awareness of the path MTU, using some type of path MTU discovery (PMTUD) algorithm. +On IPv4, this requires sending packets with the "Don't Fragment" flag set, and measuring when those packets are lost or when ICMP "Fragmentation Needed" replies are seen. + +Note that many applications have their own requirements for path MTU. For example, QUIC and common implementations of WebRTC require an MTU no smaller than 1200 bytes, but they can discover larger MTUs when available. + +### Socket Layer + +In practice the straightforward "User Interface" from RFC768, capable of arbitrary local address, is only available to privileged users. + +BSD-style sockets support UDP via `SOCK_DGRAM`. +UDP is a stateless protocol, but sockets do have state. +Each socket is bound, either explicitly with `bind()` or automatically, to a source IP and port. + +At the API level, a socket is said to be *connected* to a remote `(address, port)` if that address is the default destination. +A *connected* socket will also filter out incoming packets with source addresses different from this default destination. +A socket is considered *unconnected* if `connect()` has not been called. +These sockets have no default destination, and they accept datagrams from any source. + +There does not need to be any particular mapping between the lifetime of these application sockets and any higher-level "connection" the application establishes. +It's better to think of one socket as one allocated local port. +A typical application may allocate only a single port (one socket) for talking to many peers. +Every datagram sent or received on the socket may have a different peer address. + +### Network Address Translation (NAT) + +Much of the real-world complexity in applying UDP comes from defining strategies to detect and overcome the effects of NAT. +As a result, an intimidating quantity of IETF documentation has been written on NAT behavior and on strategies for NAT traversal. + +[RFC4787](https://www.rfc-editor.org/rfc/rfc4787.html) and later [RFC7857](https://www.rfc-editor.org/rfc/rfc7857.html) offer best practices for implementing NAT. These are sometimes referred to as the [BEHAVE-WG](https://datatracker.ietf.org/wg/behave/about/) recommendations, based on the "Behavior Engineering for Hindrance Avoidance" working group behind them. + +[RFC6888](https://www.rfc-editor.org/rfc/rfc6888.html) makes additional recommendations for "carrier grade" NAT systems, where small pools of IP addresses are shared among a much larger number of subscribers. + +[RFC8445](https://www.rfc-editor.org/rfc/rfc8445.html) describes the Interactive Connectivity Establishment (ICE) protocol, which has become a common and recommended application-level technique for building peer-to-peer applications that work through NAT. + +There are multiple fundamental technical issues that NAT presents: + +1. NAT must be stateful in order to route replies back to the correct source. + + This directly conflicts with the stateless nature of UDP itself. + The NAT's mapping lifetime, determined by a timer, will not necessarily match the lifetime of the application-level connection. + This necessitates keep-alive packets in some protocols. + Protocols that allow their binding to expire may be open to a NAT rebinding attack, when a different party acquires access to the NAT's port allocation. + +2. Applications no longer know an address they can be reached at without outside help. + + Chosen port numbers may or may not be used by the NAT. + The effective IP address and port are not knowable without observing from an outside peer. + +3. Filtering and mapping approaches both vary, and it's not generally possible to establish a connection without interactive probing. + + This is the reason ICE exists, but it's also a possible anonymity hazard. + This risk is explored a bit further below in the context of [interaction with other networks](#interaction-with-other-networks). + +We can use the constraints of NAT both to understand application behavior and as an opportunity to model Tor's behavior as a type of NAT. +In fact, Tor's many exit nodes already share similarity with some types of carrier-grade NAT. +Applications will need to assume very little about the IP address their outbound UDP originates on, and we can use that to our advantage in implementing UDP for Tor. + +This body of work is invaluable for understanding the scope of the problem and for defining common terminology. +Let's take inspiration from these documents while also keeping in mind that the analogy between Tor and a NAT is imperfect. +For example, in analyzing Tor as a type of carrier-grade NAT, we may consider the "pooling behavior" defined in RFC4787: the choice of which external addresses map to an internal address. +Tor by necessity must carefully limit how predictable these mappings can ever be, to preserve its anonymity properties. +A literal application of RFC6888 would find trouble in REQ-2 and REQ-9, as well as the various per-subscriber limiting requirements. + +#### Mapping and Filtering Behaviors + +RFC4787 defines a framework for understanding the behavior of NAT by analyzing both its "mapping" and "filtering" behavior separately. +Mappings are the NAT's unit of state tracking. +Filters are layered on top of mappings, potentially rejecting incoming datagrams that don't match an already-expected address. +Both RFC4787 and the demands of peer-to-peer applications make a good case for always using an **Endpoint-Independent Mapping**. + +Choice of filtering strategy is left open by the BEHAVE-WG recommendations. +RFC4787 does not make one single recommendation for all circumstances, +instead it defines three behavior options with different properties: + +- **Endpoint-Independent Filtering** allows incoming datagrams from any peer once a mapping has been established. + + RFC4787 recommends this approach, with the concession that it may not be ideal for all security requirements. + + This technique cannot be safely applied in the context of Tor. + It makes [traffic injection](#traffic-injection) attacks possible from any source address, provided you can guess the UDP port number used at an exit. + It also makes possible clear-net hosting of UDP servers using an exit node's IP, which may have undesirable abuse properties. + + This permissive filter is also incompatible with our proposed mitigation to [local port exhaustion](#local-port-exhaustion) on exit relays. Even with per-circuit rate limiting, an attacker could trivially overwhelm the local port capacity of all combined UDP-capable Tor exits. + + It is still common for present-day applications to *prefer* endpoint-independent filtering, as it allows incoming connections from peers which cannot use STUN or a similar address fixing protocol. + Choosing endpoint-independent filtering would have *some* compatibility benefit, but among modern protocols which use ICE and STUN there would be no improvement. + The cost, on the other hand, would be an uncomfortably low-cost traffic injection attack and additional risks toward exit nodes. + +- **Address-Dependent Filtering** + + This is a permitted alternative according to RFC4787, in which incoming datagrams are allowed from only IP addresses we have previously sent to, but any port on that IP may be the sender. + + The intended benefits of this approach versus the port-dependent filtering below are unclear, and may no longer be relevant. In theory they would be: + + - To support a class of applications that rely on, for a single local port, multiple remote ports achieving filter acceptance status when only one of those ports has been sent a datagram. + We are currently lacking examples of applications in this category. + Any application using ICE should be outside this category, as each port would have its own connectivity check datagrams exchanged in each direction. + + - REQ-8 in RFC4787 claims the existence of a scenario in which this approach facilitates ICE connections with a remote peer that disregards REQ-1 (the peer does not use **Endpoint-Independent Mapping**). It is not clear that this claim is still relevant. + + One security hazard of address-dependent and non-port-dependent filtering, identified in RFC4787, is that a peer on a NAT effectively negates the security benefits of this host filtering. + In fact, this should raise additional red flags as applied to either Tor or carrier grade NAT. + If supporting peer-to-peer applications, it should be commonplace to establish UDP flows between two Tor exit nodes. + When this takes place, non-port-dependent filtering would then allow anyone on Tor to connect via those same nodes and perform traffic injection. + The resulting security properties really become uncomfortably similar to endpoint-independent filtering. + +- **Address- and Port-Dependent Filtering** + + This is the strictest variety of filtering, and it is an allowed alternative under RFC4787. + It provides opportunities for increased security and opportunities for reduced compatibility, both of which in practice may depend on other factors. + + For every application we've analyzed so far, port-dependent filtering is not a problem. + Usage of ICE will open all required filters during the connectivity check phase. + + This is the only type of filtering that provides any barrier at all between cross-circuit traffic injection when the communicating parties are known. + +RFC4787 recommends that filtering style be configurable. +We would like to implement that advice, but only to the extent it can be done safely and meaningfully in the context of an anonymity system. +When possible, it would provide additional compatibility at no mandatory cost to allow applications to optionally request **Address-Dependent Filtering**. +Otherwise, **Address- and Port-Dependent Filtering** is the most appropriate default setting. + +### Common Protocols + +Applications that want to use UDP are increasingly making use of higher-level protocols to avoid creating bespoke solutions for problems like NAT traversal, connection establishment, and reliable delivery. + +This section looks at how these protocols affect Tor's UDP traffic requirements. + +#### QUIC + +[RFC9000](https://www.rfc-editor.org/rfc/rfc9000.html) defines QUIC, a multiplexed secure point-to-point protocol which supports reliable and unreliable delivery. The most common use is as an optional HTTP replacement, especially among Google services. + +QUIC does not normally try to traverse NAT; as an HTTP replacement, the server is expected to have an address reachable without any prior connection setup. + +QUIC provides its own flexible connection lifetimes which may outlive individual network links or NAT mappings. +The intention is to provide transparent roaming as mobile users change networks. +This automated path discovery opens additional opportunities for malicious traffic, for which the RFC also offers mitigations. See *path validation* in section `8.2`, and the additional mitigations from section `9.3`. + +When QUIC is used as an optional upgrade path, we must compare any proposed UDP support against the baseline of a non-upgraded original connection. +In these cases the goal is not a compatibility enhancement but an avoidance of regression. + +In cases where QUIC is used as a primary protocol without TCP fallback, UDP compatibility will be vital. +These applications are currently niche but we expect they may rise in popularity. + +#### WebRTC + +WebRTC is a large collection of protocols tuned to work together for media transport and NAT traversal. +It is increasingly common, both for browser-based telephony and for peer-to-peer data transfer. +Non-browser apps often implement WebRTC as well, for example using [`libwebrtc`](https://github.com/webrtc-sdk/libwebrtc). +Even non-WebRTC apps sometimes have significant overlaps in their technology stacks, due to the independent history of ICE, RTP, and SDP adoption. + +Of particular importance to us, WebRTC uses the Interactive Connection Establishment (ICE) protocol to establish a bidirectional channel between endpoints that may or may not be behind a NAT with unknown configuration. + +Any generalized solution to connection establishment, like ICE, will require sending connectivity test probes. These have an inherent hazard to anonymity: assuming no delays are inserted intentionally, the result is a broadcast of similar traffic across all available network interfaces. This could form a convenient correlation beacon for an attacker attempting to de-anonymize users who use WebRTC over a Tor VPN. This is the risk enumerated below as [*interaction with other networks*](#interaction-with-other-networks). + +See +[RFC8825](https://www.rfc-editor.org/rfc/rfc8825.html) *Overview: Real-Time Protocols for Browser-Based Applications*, +[RFC8445](https://www.rfc-editor.org/rfc/rfc8445.html) *Interactive Connectivity Establishment (ICE): A Protocol for Network Address Translator (NAT) Traversal*, +[RFC8838](https://www.rfc-editor.org/rfc/rfc8838.html) *Trickle ICE: Incremental Provisioning of Candidates for the Interactive Connectivity Establishment (ICE) Protocol*, +[RFC5389](https://www.rfc-editor.org/rfc/rfc5389.html) *Session Traversal Utilities for NAT (STUN)*, +and others. + +### Common Applications + +With applications exhibiting such a wide variety of behaviors, how do we know what to expect from a good implementation? +How do we know which compatibility decisions will be most important to users? +For this it's helpful to look at specific application behaviors. +This is a best-effort analysis conducted at a point in time. +It's not meant to be a definitive reference, think of it as a site survey taken before planning a building. + +In alphabetical order: + +| Application | Type | Protocol features | Current behavior | Expected outcome | +| ---------------------- | -------------- | --------------------------------- | ------------------ | -------------------------------------------------- | +| BitTorrent | File sharing | Many peers per local port | Fails without UDP | Works, new source of nuisance traffic | +| BigBlueButton | Telecom | WebRTC, TURN, TURN-over-TLS | Works | Slight latency improvement | +| Discord | Telecom | Proprietary, client/server | Fails without UDP | Starts working | +| DNS | Infrastructure | Might want IP fragmentation | Limited | Full DNS support, for better and worse | +| FaceTime | Telecom | WebRTC, TURN, TURN-over-TCP | Works | Slight latency improvement | +| Google Meet | Telecom | STUN/TURN, TURN-over-TCP | Works | Slight latency improvement | +| Jitsi (meet.jit.si) | Telecom | WebRTC, TURN-over-TLS, Cloudflare | Fails on Tor | No change, problem does not appear UDP-related | +| Jitsi (docker-compose) | Telecom | WebRTC, centralized STUN only | Fails without UDP | Starts working | +| Linphone | Telecom (SIP) | SIP-over-TLS, STUN, TURN | Fails without UDP | Starts working | +| Signal | Telecom | WebRTC, TURN, TURN-over-TCP | Works | Slight latency improvement | +| Skype | Telecom | P2P, STUN, TURN-over-TLS | Works | Slight latency improvement | +| WhatsApp | Telecom | STUN, TURN-over-TCP. Multi server | Works | Slight latency improvement | +| WiFi Calling | Telecom | IPsec tunnel | Out of scope | Still out of scope | +| Zoom | Telecom | client/server or P2P, UDP/TCP | Works | Slight latency improvement | + +## Overview of Possible Solutions + +This section starts to examine different high-level implementation techniques we could adopt. +Broadly they can be split into *datagram routing* and *tunneling*. + +### Datagram Routing + +These approaches seek to use a network that can directly route datagrams from place to place. These approaches are the most obviously suitable for implementing UDP, but they also form the widest departure from classic Tor. + +#### Intentional UDP Leak + +The simplest approach would be to allow UDP traffic to bypass the anonymity layer. This is an unacceptable loss of anonymity in many cases, given that the client's real IP address is made visible to web application providers. + +In other cases, this is an acceptable or even preferable approach. For example, VPN users may be more concerned with achieving censorship-resistant network connectivity than hiding personal identifiers from application vendors. + +In threat models where application vendors are more trustworthy than the least trustworthy Tor exits, it may be more appropriate to allow direct peer-to-peer connections than to trust Tor exits with unencrypted connection establishment traffic. + +#### 3rd Party Implementations + +Another option would be to use an unrelated anonymizer system for datagram traffic. It's not clear that a suitable system already exists. I2P provides a technical solution for routing anonymized datagrams, but not a Tor-style infrastructure of exit node operators. + +This points to the key weakness of relying on a separate network for UDP: Tor has an especially well-developed community of volunteers running relays. Any UDP solution that is inconvenient for relay operators has little chance of adoption. + +#### Future Work on Tor + +There may be room for future changes to Tor which allow it to somehow transfer and route datagrams directly, without a separate process of establishing circuits and tunnels. +If this is practical it may prove to be the simplest and highest performance route to achieving high quality UDP support in the long term. +A specific design is out of the scope of this document. + +It is worth thinking early about how we can facilitate combinations of approaches. +Even without bringing any new network configurations to Tor, achieving interoperable support for both exit nodes and onion services in a Tor UDP implementation requires some attention to how multiple UDP providers can share protocol responsibilities. +This may warrant the introduction of some additional routing layer. + +### Tunneling + +The approaches in this section add a new construct which does not exist in UDP itself: a point-to-point tunnel between clients and some other location at which they establish the capability to send and receive UDP datagrams. + +Any tunneling approach requires some way to discover tunnel endpoints. +For the best usability and adoption this should come as an extension of Tor's existing process for distributing consensus and representing exit policy. + +In practice, exit policies for UDP will have limited practical amounts of diversity. +VPN implementations will need to know ahead of time which tunnel circuits to build, or they will suffer a significant spike in latency for the first outgoing datagram to a new peer. +Additionally, it's common for UDP port numbers to be randomly assigned. +This would make highly specific Tor exit policies even less useful and even higher overhead than they are with TCP. + +#### TURN Encapsulated in Tor Streams + +The scope of this tunnel is quite similar to the existing TURN relays, used commonly by WebRTC applications to implement fallbacks for clients who cannot find a more direct connection path. + +TURN is defined by [RFC8656](https://www.rfc-editor.org/rfc/rfc8656) as a set of extensions built on the framework from STUN in [RFC8489](https://www.rfc-editor.org/rfc/rfc8489.html). The capabilities are a good match for our needs, offering clients the ability to encapsulate UDP datagrams within a TCP stream, and to allocate local port mappings on the server. + +TURN was designed to be a set of modular and extensible pieces, which might be too distant from Tor's design philosophy of providing single canonical representations. Any adoption of TURN will need to consider the potential for malicious implementations to mark traffic, facilitating de-anonymization attacks. + +TURN has a popular embeddable C-language implementation, [coturn](https://github.com/coturn/coturn), which may be suitable for including alongside or inside C tor. + +#### Tor Stream Tunnel to an Exit + +Most of the discussion on UDP implementation in Tor so far has assumed this approach. Essentially it's the same strategy as TCP exits, but for UDP. When the OP initializes support for UDP, it pre-builds circuits to exits that support required UDP exit policies. These pre-built circuits can then be used as tunnels for UDP datagrams. + +Within this overall approach, there are various ways to assign Tor *streams* for the UDP traffic. This will be considered below. + +#### Tor Stream Tunnel to a Rendezvous Point + +To implement onion services which advertise UDP ports, we can use additional tunnels. +A new type of tunnel could end at a rendezvous point rather than an exit node. +Clients could establish the ability to allocate a temporary virtual datagram mailbox at these rendezvous nodes. + +This leaves more open questions about how outgoing traffic is routed, and which addressing format would be used for the datagram mailbox. +The most immediate challenge in UDP rendezvous would then become application support. Protocols like STUN and ICE deal directly with IPv4 and IPv6 formats in order to advertise a reachable address to their peer. Supporting onion services in WebRTC would require protocol extensions and software modifications for STUN, TURN, ICE, and SDP at minimum. + +UDP-like rendezvous extensions would have limited meaning unless they form part of a long-term strategy to forward datagrams in some new way for enhanced performance or compatibility. Otherwise, application authors might as well stick with Tor's existing reliable circuit rendezvous functionality. + +## Specific Designs Using Tor Streams + +Let's look more closely at Tor *streams*, the multiplexing layer right below circuits. + +Streams have an opaque 16-bit identifier, allocated from the onion proxy (OP) endpoint. +Stream lifetimes are subject to some slight ambiguity still in the Tor spec. +They are always allocated from the OP end but may be destroyed asynchronously by either circuit endpoint. + +We have an opportunity to use this additional existing multiplexing layer to serve a useful function in the new protocol, or we can opt to interact with streams as little as possible in order to keep the protocol features more orthogonal. + +### One Stream per Tunnel (VPN) + +It's possible to transport arbitrary UDP traffic using only a single Tor stream ID within the circuit. In this case, both the source and destination address would need to be represented somehow within an added per-datagram header. + +The most common examples of this approach might be TUN/TAP forwarding over SSH, or any VPN that uses a TCP transport. + +This design might be preferable if 16-bit stream IDs are found to be in short supply, but otherwise we would expect some benefit from using multiple streams IDs and therefore allowing a shorter per-datagram header. + +### One Stream per Local port (TURN) + +With the standard TURN relay protocol, implemented over a TCP transport, +the "allocation" of a relayed port can be performed once per tuple of `(protocol, source IP, source port, destination IP, destination port)`. +Put another way, TURN does not amplify the number of local ports available. +An application that wishes to use two relayed ports will need two separate TCP connections, or in this context two separate Tor stream IDs. + +As with the SSH example above, TURN could be implemented without any new Tor message types. +TURN adds a 4-byte "channel data" header to normal datagrams, to facilitate message framing and peer identification. + +We could choose to implement protocol support for bundling TURN relay functionality with a normal Tor exit node. +In that case, we would want a single new Tor message type: + +- `CONNECT_TURN` + + - Establish a stream as a connection to the exit relay's built-in (or configured) TURN server. + + This would logically be a TURN-over-TCP connection, though it does not need to correspond to any real TCP socket if the TURN server is implemented in-process with tor. + +Note that RFC8656 requires authentication before data can be relayed, which is a good default best practice for the internet perhaps but is the opposite of what Tor is trying to do. We would either deviate from the specification to relax this auth requirement, or provide a way for clients to discover credentials: perhaps by fixing them ahead of time or by including them in the relay descriptor. + +If authentication is used, we must consider the protocol's privacy limitations. +STUN/TURN usernames are conveyed in plaintext unless an additional TLS layer is also in use. +For anonymity, use a randomly generated username. +The [`draft-uberti-behave-turn-rest-00`](https://datatracker.ietf.org/doc/html/draft-uberti-behave-turn-rest-00) document describes a method for generating time-limited credentials, as implemented through `--use-auth-secret` in `coturn`. +Username timestamps should be randomized, to avoid exposing the precise local system clock. + +### One Stream per Local Port (Proposal 339) + +One stream **per socket** was the approach suggested in [Proposal 339](https://spec.torproject.org/proposals/339-udp-over-tor.html) by Nick Mathewson in 2020, defining a per-local-port approach using slightly different terminology than TURN. + +Proposal 339 chooses a different UDP subset to support, but otherwise the main technical distinction vs. TURN's per-local-port stream comes down to the framing layer. +TURN defines an additional header to be used within a TCP-like stream, whereas proposal 339 avoids the two redundant length bytes by defining new Tor message types: `CONNECT_UDP`, `CONNECTED_UDP`, and `DATAGRAM`. + +Similar to TURN, each stream's lifetime would match the lifetime of a local port allocation. +Unlike TURN, there would be a single peer `(remote address, remote port)` allowed per `local port`. +This matches the usage of BSD-style sockets on which `connect()` has completed, but it's incompatible with many of the applications analyzed. +Multiple peers are typically needed for a variety of reasons, like connectivity checks or multi-region servers. + +This approach would be simplest to implement and specify, especially in the existing C tor implementation. +It also unfortunately has very limited compatibility, and no clear path toward incremental upgrades if we wish to improve compatibility later. + +A simple one-to-one mapping between streams and sockets would preclude the optimizations necessary to address [local port exhaustion](#local-port-exhaustion) risks below. Solutions under this design are possible, but only by decoupling logical protocol-level sockets from the ultimate implementation-level sockets and reintroducing much of the complexity that we attempted to avoid by choosing this design. + +### One Stream per Local Port (NAT Mapping) + +We could improve the compatibility of Proposal 339 while retaining its very low per-datagram overhead by updating it to align with terminology and requirements from [RFC4787](https://www.rfc-editor.org/rfc/rfc4787) where practical. + +This approach would use each stream to represent one **endpoint-independent mapping**, for use as a local UDP port in communication with multiple peers. + +A mapping would always be allocated from the OP (client) side. +It could explicitly specify a filtering style, if we wish to allow applications to request non-port-dependent filtering for compatibility. +Each datagram within the stream would still need to be tagged with a peer address/port in some way. + +This approach would involve a single new type of stream, and two new messages that pertain to these *mapping* streams: + +- `NEW_UDP_MAPPING` + + - Always client-to-exit. + - Creates a new mapping, with a specified stream ID. + - Succeeds instantly; no reply is expected, early data is ok. + - Externally-visible local port number is arbitrary, and must be determined through interaction with other endpoints. + - Might contain an IP "don't fragment" flag. + - Might contain a requested filtering mode. + - Lifetime is until circuit teardown or `END` message. + +- `UDP_MAPPING_DATAGRAM` + + - Conveys one datagram on a stream previously defined by `NEW_UDP_MAPPING`. + - Includes peer address (IPv4/IPv6) as well as datagram content. + +This updated approach is more compatible than Proposal 339 but it's not yet as efficient as TURN. +Peers are almost always repeated over the course of a tunnel's lifetime, so the header space needed for addressing could be compressed by keeping some persistent information about peers as well as mappings. + +### One Stream per Flow + +One stream **per flow** has also been suggested. +Specifically, Mike Perry brought this up during our conversations about UDP recently and we spent some time analyzing it from a RFC4787 perspective. +This approach has some interesting properties but also hidden complexity that may ultimately make other options more easily applicable. + +This would assign a stream ID to the tuple consisting of at least `(local port, remote address, remote port)`. Additional flags may be included for features like transmit and receive filtering, IPv4/v6 choice, and IP *Don't Fragment*. + +This has advantages in keeping the datagram cells simple, with no additional IDs beyond the existing circuit ID. +It may also have advantages in DoS-prevention and in privacy analysis. + +Stream lifetimes, in this case, would not have any specific meaning other than the lifetime of the ID itself. +The bundle of flows associated with one local port would still all be limited to the lifetime of a Tor circuit, by scoping the local port identifier to be contained within the lifetime of its circuit. + +It would be necessary to allocate a new stream ID any time a new `(local port, remote address, remote port)` tuple is seen. +This would most commonly happen as a result of a first datagram sent to a new peer, coinciding with the establishment of a NAT-style mapping and the possible allocation of a socket on the exit. + +A less common case needs to be considered too: what if the parameter tuple first occurs on the exit side? +We don't yet have a way to allocate stream IDs from either end of a circuit. +This would need to be considered. +One simple solution would be to statically partition the stream ID space into a portion that can be independently allocated by each side. + +When is this exit-originated circuit ID allocation potentially needed? +It is clearly needed when using **address-dependent filtering**. +An incoming datagram from a previously-unseen peer port is expected to be deliverable, and the exit would need to allocate an ID for it. + +Even with the stricter **address and port-dependent filtering** clients may still be exposed to exit-originated circuit IDs if there are mismatches in the lifetime of the filter and the stream. + +This approach thus requires some attention to either correctly allocating stream IDs on both sides of the circuit, or choosing a filtering strategy and filter/mapping lifetime that does not ever leave stream IDs undefined when expecting incoming datagrams. + +### Hybrid Mapping and Flow Approach + +We can extend the approach above with an optimization that addresses the undesirable space overhead from redundant address headers. +This uses two new types of stream, in order to have streams **per mapping** and **per flow** at the same time. + +The per-mapping stream remains the sole interface for managing the lifetime of a mapped UDP port. Mappings are created explicitly by the client. As an optimization, within the lifetime of a mapping there may exist some number of *flows*, each assigned their own ID. + +This tries to combine the strengths of both approaches, using the lifetime of one stream to define a mapping and to carry otherwise-unbundled traffic while also allowing additional streams to bundle datagrams that would otherwise have repetitive headers. +It avoids the space overhead of a purely **per mapping** approach and avoids the ID allocation and lifetime complexity introduced with **per flow**. + +This approach takes some inspiration from TURN, where commonly used peers will be defined as a "channel" with an especially short header. +Incoming datagrams with no channel can always be represented in the long form, so TURN never has to allocate channels unexpectedly. + +The implementation here could be a strict superset of the **per mapping** implementation, adding new commands for flows while retaining existing behavior for mappings. There would be a total of four new message types: + +- `NEW_UDP_MAPPING` + + - Same as above. + +- `UDP_MAPPING_DATAGRAM` + + - Same as above. + +- `NEW_UDP_FLOW` + + - Allocates a stream ID as a *flow*, given the ID to be allocated and the ID of its parent *mapping* stream. + - Includes a peer address (IPv4/IPv6). + - The *flow* has a lifetime strictly bounded by the outer *mapping*. It is deleted by an explicit `END` or when the mapping is de-allocated for any reason. + +- `UDP_FLOW_DATAGRAM` + + - Datagram contents only, without address. + - Only appears on *flow* streams. + +We must consider the traffic marking opportunities provided when allowing an exit to represent one incoming datagram as either a *flow* or *mapping* datagram. + +It's possible this traffic injection potential is not worse than the baseline amount of injection potential than every UDP protocol presents. See more on [risks](#risks) below. For this hybrid stream approach specifically, there's a limited mitigation available which allows exits only a bounded amount of leaked information per UDP peer: + +Ideally exits may not choose to send a `UDP_MAPPING_DATAGRAM` when they could have sent a `UDP_FLOW_DATAGRAM`. +Sometimes it is genuinely unclear though: an exit may have received this datagram in-between processing `NEW_UDP_MAPPING` and `NEW_UDP_FLOW`. +A partial mitigation would terminate circuits which send a `UDP_MAPPING_DATAGRAM` for a peer that has already been referenced in a `UDP_FLOW_DATAGRAM`. +The exit is thereby given a one-way gate allowing it to switch from using *mapping* datagrams to using *flow* datagrams at some point, but not to switch back and forth repeatedly. + +Mappings that do not request port-specific filtering may always get unexpected `UDP_MAPPING_DATAGRAM`s. Mappings that do use port-specific filtering could make a flow for their only expected peers, then expect to never see `UDP_MAPPING_DATAGRAM`. + +`NEW_UDP_MAPPING` could have an option requiring that only `UDP_FLOW_DATAGRAM` is to be used, never `UDP_MAPPING_DATAGRAM`. +This would remove the potential for ambiguity, but costs in compatibility as it's no longer possible to implement non-port-specific filtering. + +## Risks + +Any proposed UDP support involves significant risks to user privacy and software maintainability. +This section elaborates some of these risks, so they can be compared against expected benefits. + +### Behavior Regressions + +In some applications it is possible that Tor's implementation of a UDP compatibility layer will cause a regression in the ultimate level of performance or security. + +Performance regressions can occur accidentally due to bugs or compatibility glitches. +They may also occur for more fundamental reasons of protocol layering. +For example, the redundant error correction layers when tunneling QUIC over TCP. +These performance degradations are expected to be minor, but there's some unavoidable risk. + +The risk of severe performance or compatibility regressions may be mitigated by giving users a way to toggle UDP support per-application. + +Privacy and security regressions have more severe consequences and they can be much harder to detect. +There are straightforward downgrades, like WebRTC apps that give up TURN-over-TLS for plaintext TURN-over-UDP. +More subtly, the act of centralizing connection establishment traffic in Tor exit nodes can make users an easier target for other attacks. + +### Bandwidth Usage + +We should expect an increase in overall exit bandwidth requirements due to peer-to-peer file sharing applications. + +Current users attempting to use BitTorrent over Tor are hampered by the lack of UDP compatibility. Interoperability with common file-sharing peers would make Tor more appealing to users with a large and sustained appetite for anonymized bandwidth. + +### Local Port Exhaustion + +Exit routers will have a limited number of local UDP ports. In the most constrained scenario, an exit may have a single IP with 16384 or fewer ephemeral ports available. These ports could each be allocated by one client for an unbounded amount of exclusive use. + +In order to enforce high levels of isolation between different subsequent users of the same local UDP port, we may wish to enforce an extended delay between allocations during which nobody may own the port. Effective isolation requires this timer duration to be greater than any timer encountered on a peer or a NAT. In RFC4787's recommendations a NAT's mapping timer must be longer than 2 minutes. Our timer should ideally be *much* longer than 2 minutes. + +An attacker who allocates ports for only this minimum duration of 2 minutes would need to send 136.5 requests per second to achieve sustained use of all available ports. With multiple simultaneous clients this could easily be done while bypassing per-circuit rate limiting. + +The expanded definition of "Port overlapping" from [RFC7857 section 3](https://datatracker.ietf.org/doc/html/rfc7857#section-3), may form at least a partial mitigation: + +> This document clarifies that this port overlapping behavior may be extended to connections originating from different internal source IP addresses and ports as long as their destinations are different. + +This gives us an opportunity for a vast reduction in the number of required ports and file descriptors. +Exit routers can automatically allocate local ports for use with a specific peer when that peer is first added to the client's filter. + +Due to the general requirements of NAT traversal, UDP applications with any NAT support will always need to communicate with a relatively well known server prior to any attempts at peer-to-peer communication. +This early peer could be an entire application server, or it could be a STUN endpoint. +In any case, the identity of this first peer gives us a hint about the set of all potential peers. + +Within the exit router, each local port will track a separate mapping owner for each peer. +When processing that first outgoing datagram, the exit may choose any local port where the specific peer is not taken. +Subsequent outgoing datagrams on the same port may communicate with a different peer, and there's no guarantee all these future peers will be claimed successfully. + +When is this a problem? +An un-claimable peer represents a case where the exact `(local ip, local port, remote ip, remote port)` tuple is in use already for a different mapping in some other Tor stream. +For example, imagine two clients are running different types of telecom apps which are nevertheless inter-compatible and capable of both calling the same peers. +Alternatively, consider the same app but with servers in several regions. +The two apps will begin by communicating with different sets of peers, due to different application servers and different bundled STUN servers. +This is our hint that it's likely appropriate to overlap their local port allocations. +At this point, both of these applications may be successfully sharing a `(local ip, local port)` tuple on the exit. +As soon as one of these apps calls a peer with some `(remote ip, remote port)`, the other app will be unable to contact that specific peer. + +The lack of connectivity may seem like a niche inconvenience, and perhaps that is the extent of the issue. +It seems likely this heuristic could result in a problematic information disclosure under some circumstances, and it deserves closer study. + +### Application Fingerprinting + +UDP applications present an increased surface of plaintext data that may be available for user fingerprinting by malicious exits. + +Exposed values can include short-lived identifiers like STUN usernames. +Typically it will also be possible to determine what type of software is in use, and maybe what version of that software. + +Short-lived identifiers are still quite valuable to attackers, because they may reliably track application sessions across changes to the Tor exit. If longer-lived identifiers exist for any reason, that of course provides a powerful tool for call metadata gathering. + +### Peer-to-Peer Metadata Collection + +One of our goals was to achieve the compatibility and perhaps performance benefits of allowing "peer-to-peer" (in our case really exit-to-exit) UDP connections. +We expect this to enable the subset of applications that lack a fallback path which loops traffic through an app-provided server. + +This goal may be at odds with our privacy requirements. At minimum, a pool of malicious exit nodes could passively collect metadata about these connections as a noisy proxy for call metadata. + +Any additional signals like [application fingerprints](#application-fingerprinting) or [injected](#traffic-injection) markers may be used to enrich this metadata graph, possibly tracking users across sessions or across changes to the tunnel endpoint. + +### Interaction with Other Networks + +Any application using an ICE-like interactive connection establishment scheme will easily leak information across network boundaries if it ever has access to multiple networks at once. + +In applications that are not privacy conscious this is often desired behavior. For example, a video call to someone in your household may typically transit directly over Wifi, decreasing service costs and improving latency. This implies that some type of local identifier accompanies the call signaling info, allowing the devices to find each other's LAN address. + +Privacy-preserving solutions for this use case are still an active area of standardization effort. The [IETF draft on mDNS ICE candidates](https://www.ietf.org/archive/id/draft-ietf-mmusic-mdns-ice-candidates-02.html) proposes one way to accomplish this by generating short-lived unique IDs which are only useful to peers with physical access to the same mDNS services. + +Without special attention to privacy, the typical implementation is to share all available IP addresses and to initiate simultaneous connectivity tests using any IP pairs which cannot be trivially discarded. This applies to ICE as specified, but also to any proprietary protocol which operates in the same design space as ICE. This has multiple issues in a privacy-conscious environment: The IP address disclosures alone can be fatal to anonymity under common threat models. Even if meaningful IP addresses are not disclosed, the timing correlation from connectivity checks can provide confirmation beacons that alert an attacker to some connection between a LAN user and a Tor exit. + +### Traffic Injection + +Some forms of UDP support would have obvious and severe traffic injection vulnerabilities. For example, the very permissive *endpoint-independent filtering* strategy would allow any host on the internet to send datagrams in bulk to all available local ports on a Tor exit in order to map that traffic's effect on any guards they control. + +Any vehicle for malicious parties to mark traffic can be abused to de-anonymize users. Even if there is a more restrictive filtering policy, UDP's lack of sequence numbers make header spoofing attacks considerably easier than in TCP. Third parties capable of routing datagrams to an exit with a spoofed source address could bypass filtering when the communicating parties are known or can be guessed. For example, a malicious superuser at an ISP without egress filtering could send packets with the source IPs set to various common DNS servers and application STUN/TURN servers. If a specific application is being targeted, only the exit node and local port numbers need to be guessed by brute force. + +In case of malicious exit relays, whole datagrams can be inserted and dropped, and datagrams may be padded with additional data, both without any specific knowledge of the application protocol. With specific protocol insights, a malicious relay may make arbitrary edits to plaintext data. + +Of particular interest is the plaintext STUN, TURN, and ICE traffic used by most WebRTC apps. These applications rely on higher-level protocols (SRTP, DTLS) to provide end-to-end encryption and authentication. A compromise at the connection establishment layer would not violate application-level end-to-end security requirements, making it outside the threat model of WebRTC but very much still a concern for Tor. + +These attacks are not fully unique to the proposed UDP support, but UDP may increase exposure. In cases where the application already has a fallback using TURN-over-TLS, the proposal is a clear regression over previous behaviors. Even when comparing plaintext to plaintext, there may be a serious downside to centralizing all connection establishment traffic through a small number of exit IPs. Depending on your threat model, it could very well be more private to allow the UDP traffic to bypass Tor entirely. + +### Malicious Outgoing Traffic + +We can expect UDP compatibility in Tor will give malicious actors additional opportunities to transmit unwanted traffic. + +In general, exit abuse will need to be filtered administratively somehow. +This is not unique to UDP support, and exit relay administration typically involves some type of filtering response tooling that falls outside the scope of Tor itself. + +Exit administrators may choose to modify their exit policy, or to silently drop problematic traffic. +Silent dropping is discouraged in most cases, as Tor prioritizes the accuracy of an exit's advertised policy. +Detailed exit policies have a significant space overhead in the overall Tor consensus document, but it's still seen as a valuable resource for clients during circuit establishment. + +Exit policy filtering may be less useful in UDP than with TCP due to the inconvenient latency spike when establishing a new [tunnel](#tunneling). +Applications that are sensitive to RTT measurements made during connection establishment may fail entirely when the tunnel cannot be pre-built. + +This section lists a few potential hazards, but the real-world impact may be hard to predict owing to a diversity of custom UDP protocols implemented across the internet. + +- Amplification attacks against arbitrary targets + + These are possible only in limited circumstances where the protocol allows an arbitrary reply address, like SIP. +The peer is often at fault for having an overly permissive configuration. +Nevertheless, any of these *easy* amplification targets can be exploited from Tor with little consequence, creating a nuisance for the ultimate target and for exit operators. + +- Amplification attacks against an exit relay + + An amplification peer which doesn't allow arbitrary destinations can still be used to attack the exit relay itself or other users of that relay. +This is essentially the same attack that is possible against any NAT the attacker is behind. + +- Malicious fragmented traffic + + If we allow sending large UDP datagrams over IPv4 without the *Don't Fragment* flag set, we allow attackers to generate fragmented IP datagrams. +This is not itself a problem, but it has historically been a common source of inconsistencies in firewall behavior. + +- Excessive sends to an uninterested peer + + Whereas TCP mandates a successful handshake, UDP will happily send unlimited amounts of traffic to a peer that has never responded. +To prevent denial of service attacks we have an opportunity and perhaps a responsibility to define our supported subset of UDP to include true bidirectional traffic but exclude continued sends to peers who do not respond. + + See also [RFC7675](https://www.rfc-editor.org/rfc/rfc7675.html) and STUN's concept of "Send consent". + +- Excessive number of peers + + We may want to place conservative limits on the maximum number of peers per mapping or per circuit, in order to make bulk scanning of UDP port space less convenient. + + The limit would need to be on peers, not stream IDs as we presently do for TCP. +In this proposal stream IDs are not necessarily meaningful except as a representational choice made by clients. + +## Next Steps + +At this point we have perhaps too many possibilities for how to proceed. We could integrate UDP quite closely with Tor itself or not at all. + +Choosing a route forward is both a risk/benefit tradeoff and a guess about the future of Tor. If we expect that some future version of Tor will provide its own datagram transport, we should plot a course aiming in that direction. If not, we might be better served by keeping compatibility features confined to separate protocol layers when that's practical. + +### Requiring a Long-Term Datagram Plan + +If we choose to add new long-term maintenance burdens to our protocol stack, we should ensure they serve our long-term goals for UDP adoption as well as these shorter-term application compatibility goals. + +This work so far has been done with the assumption that end-to-end datagram support is out of scope. If we intend to proceed down any path which encodes a datagram-specific protocol into Tor proper, we should prioritize additional protocol research and standardization work. + +### Alternatively, Modular Application-Level Support + +Without a clear way to implement fully generic UDP support, we're left with application-level goals. +Different applications may have contrasting needs, and we can only achieve high levels of both compatibility and privacy by delegating some choices to app authors or per-app VPN settings. + +This points to an alternative in which UDP support is excluded from Tor as much as possible while still supporting application requirements. +For example, this could motivate the [TURN Encapsulated in Tor Streams](#turn-encapsulated-in-tor-streams) design, or even simpler designs where TURN servers are maintained independently from the Tor exit infrastructure. + +The next prototyping step we would need at this stage is a version of `onionmasq` extended to support NAT-style UDP mappings using TURN allocations provided through TURN-over-TCP or TURN-over-TLS. +This can be a platform for further application compatibility experiments. +It could potentially become a low-cost minimal implementation of UDP application compatibility, serving to assess which remaining user needs are still unmet. diff --git a/proposals/BY_INDEX.md b/proposals/BY_INDEX.md index 50884b5..a8f1ac4 100644 --- a/proposals/BY_INDEX.md +++ b/proposals/BY_INDEX.md @@ -11,256 +11,259 @@ of these proposals are implemented; some are works in progress; and some will never be implemented. Below are a list of proposals sorted by their proposal number. See -[README.md](/proposals/README.md) for a list of proposals sorted by status. +[BY_STATUS.md](/proposals/BY_STATUS.md) for a list of proposals sorted by status. -* [`000-index.txt`](/proposals/000-index.txt): Index of Tor Proposals [META] -* [`001-process.txt`](/proposals/001-process.txt): The Tor Proposal Process [META] -* [`098-todo.txt`](/proposals/098-todo.txt): Proposals that should be written [OBSOLETE] -* [`099-misc.txt`](/proposals/099-misc.txt): Miscellaneous proposals [OBSOLETE] -* [`100-tor-spec-udp.txt`](/proposals/100-tor-spec-udp.txt): Tor Unreliable Datagram Extension Proposal [DEAD] -* [`101-dir-voting.txt`](/proposals/101-dir-voting.txt): Voting on the Tor Directory System [CLOSED] -* [`102-drop-opt.txt`](/proposals/102-drop-opt.txt): Dropping "opt" from the directory format [CLOSED] -* [`103-multilevel-keys.txt`](/proposals/103-multilevel-keys.txt): Splitting identity key from regularly used signing key [CLOSED] -* [`104-short-descriptors.txt`](/proposals/104-short-descriptors.txt): Long and Short Router Descriptors [CLOSED] -* [`105-handshake-revision.txt`](/proposals/105-handshake-revision.txt): Version negotiation for the Tor protocol [CLOSED] -* [`106-less-tls-constraint.txt`](/proposals/106-less-tls-constraint.txt): Checking fewer things during TLS handshakes [CLOSED] -* [`107-uptime-sanity-checking.txt`](/proposals/107-uptime-sanity-checking.txt): Uptime Sanity Checking [CLOSED] -* [`108-mtbf-based-stability.txt`](/proposals/108-mtbf-based-stability.txt): Base "Stable" Flag on Mean Time Between Failures [CLOSED] -* [`109-no-sharing-ips.txt`](/proposals/109-no-sharing-ips.txt): No more than one server per IP address [CLOSED] -* [`110-avoid-infinite-circuits.txt`](/proposals/110-avoid-infinite-circuits.txt): Avoiding infinite length circuits [CLOSED] -* [`111-local-traffic-priority.txt`](/proposals/111-local-traffic-priority.txt): Prioritizing local traffic over relayed traffic [CLOSED] -* [`112-bring-back-pathlencoinweight.txt`](/proposals/112-bring-back-pathlencoinweight.txt): Bring Back Pathlen Coin Weight [SUPERSEDED] -* [`113-fast-authority-interface.txt`](/proposals/113-fast-authority-interface.txt): Simplifying directory authority administration [SUPERSEDED] -* [`114-distributed-storage.txt`](/proposals/114-distributed-storage.txt): Distributed Storage for Tor Hidden Service Descriptors [CLOSED] -* [`115-two-hop-paths.txt`](/proposals/115-two-hop-paths.txt): Two Hop Paths [DEAD] -* [`116-two-hop-paths-from-guard.txt`](/proposals/116-two-hop-paths-from-guard.txt): Two hop paths from entry guards [DEAD] -* [`117-ipv6-exits.txt`](/proposals/117-ipv6-exits.txt): IPv6 exits [CLOSED] -* [`118-multiple-orports.txt`](/proposals/118-multiple-orports.txt): Advertising multiple ORPorts at once [SUPERSEDED] -* [`119-controlport-auth.txt`](/proposals/119-controlport-auth.txt): New PROTOCOLINFO command for controllers [CLOSED] -* [`120-shutdown-descriptors.txt`](/proposals/120-shutdown-descriptors.txt): Shutdown descriptors when Tor servers stop [DEAD] -* [`121-hidden-service-authentication.txt`](/proposals/121-hidden-service-authentication.txt): Hidden Service Authentication [CLOSED] -* [`122-unnamed-flag.txt`](/proposals/122-unnamed-flag.txt): Network status entries need a new Unnamed flag [CLOSED] -* [`123-autonaming.txt`](/proposals/123-autonaming.txt): Naming authorities automatically create bindings [CLOSED] -* [`124-tls-certificates.txt`](/proposals/124-tls-certificates.txt): Blocking resistant TLS certificate usage [SUPERSEDED] -* [`125-bridges.txt`](/proposals/125-bridges.txt): Behavior for bridge users, bridge relays, and bridge authorities [CLOSED] -* [`126-geoip-reporting.txt`](/proposals/126-geoip-reporting.txt): Getting GeoIP data and publishing usage summaries [CLOSED] -* [`127-dirport-mirrors-downloads.txt`](/proposals/127-dirport-mirrors-downloads.txt): Relaying dirport requests to Tor download site / website [OBSOLETE] -* [`128-bridge-families.txt`](/proposals/128-bridge-families.txt): Families of private bridges [DEAD] -* [`129-reject-plaintext-ports.txt`](/proposals/129-reject-plaintext-ports.txt): Block Insecure Protocols by Default [CLOSED] -* [`130-v2-conn-protocol.txt`](/proposals/130-v2-conn-protocol.txt): Version 2 Tor connection protocol [CLOSED] -* [`131-verify-tor-usage.txt`](/proposals/131-verify-tor-usage.txt): Help users to verify they are using Tor [OBSOLETE] -* [`132-browser-check-tor-service.txt`](/proposals/132-browser-check-tor-service.txt): A Tor Web Service For Verifying Correct Browser Configuration [OBSOLETE] -* [`133-unreachable-ors.txt`](/proposals/133-unreachable-ors.txt): Incorporate Unreachable ORs into the Tor Network [RESERVE] -* [`134-robust-voting.txt`](/proposals/134-robust-voting.txt): More robust consensus voting with diverse authority sets [REJECTED] -* [`135-private-tor-networks.txt`](/proposals/135-private-tor-networks.txt): Simplify Configuration of Private Tor Networks [CLOSED] -* [`136-legacy-keys.txt`](/proposals/136-legacy-keys.txt): Mass authority migration with legacy keys [CLOSED] -* [`137-bootstrap-phases.txt`](/proposals/137-bootstrap-phases.txt): Keep controllers informed as Tor bootstraps [CLOSED] -* [`138-remove-down-routers-from-consensus.txt`](/proposals/138-remove-down-routers-from-consensus.txt): Remove routers that are not Running from consensus documents [CLOSED] -* [`139-conditional-consensus-download.txt`](/proposals/139-conditional-consensus-download.txt): Download consensus documents only when it will be trusted [CLOSED] -* [`140-consensus-diffs.txt`](/proposals/140-consensus-diffs.txt): Provide diffs between consensuses [CLOSED] -* [`141-jit-sd-downloads.txt`](/proposals/141-jit-sd-downloads.txt): Download server descriptors on demand [OBSOLETE] -* [`142-combine-intro-and-rend-points.txt`](/proposals/142-combine-intro-and-rend-points.txt): Combine Introduction and Rendezvous Points [DEAD] -* [`143-distributed-storage-improvements.txt`](/proposals/143-distributed-storage-improvements.txt): Improvements of Distributed Storage for Tor Hidden Service Descriptors [SUPERSEDED] -* [`144-enforce-distinct-providers.txt`](/proposals/144-enforce-distinct-providers.txt): Increase the diversity of circuits by detecting nodes belonging the same provider [OBSOLETE] -* [`145-newguard-flag.txt`](/proposals/145-newguard-flag.txt): Separate "suitable as a guard" from "suitable as a new guard" [SUPERSEDED] -* [`146-long-term-stability.txt`](/proposals/146-long-term-stability.txt): Add new flag to reflect long-term stability [SUPERSEDED] -* [`147-prevoting-opinions.txt`](/proposals/147-prevoting-opinions.txt): Eliminate the need for v2 directories in generating v3 directories [REJECTED] -* [`148-uniform-client-end-reason.txt`](/proposals/148-uniform-client-end-reason.txt): Stream end reasons from the client side should be uniform [CLOSED] -* [`149-using-netinfo-data.txt`](/proposals/149-using-netinfo-data.txt): Using data from NETINFO cells [SUPERSEDED] -* [`150-exclude-exit-nodes.txt`](/proposals/150-exclude-exit-nodes.txt): Exclude Exit Nodes from a circuit [CLOSED] -* [`151-path-selection-improvements.txt`](/proposals/151-path-selection-improvements.txt): Improving Tor Path Selection [CLOSED] -* [`152-single-hop-circuits.txt`](/proposals/152-single-hop-circuits.txt): Optionally allow exit from single-hop circuits [CLOSED] -* [`153-automatic-software-update-protocol.txt`](/proposals/153-automatic-software-update-protocol.txt): Automatic software update protocol [SUPERSEDED] -* [`154-automatic-updates.txt`](/proposals/154-automatic-updates.txt): Automatic Software Update Protocol [SUPERSEDED] -* [`155-four-hidden-service-improvements.txt`](/proposals/155-four-hidden-service-improvements.txt): Four Improvements of Hidden Service Performance [CLOSED] -* [`156-tracking-blocked-ports.txt`](/proposals/156-tracking-blocked-ports.txt): Tracking blocked ports on the client side [SUPERSEDED] -* [`157-specific-cert-download.txt`](/proposals/157-specific-cert-download.txt): Make certificate downloads specific [CLOSED] -* [`158-microdescriptors.txt`](/proposals/158-microdescriptors.txt): Clients download consensus + microdescriptors [CLOSED] -* [`159-exit-scanning.txt`](/proposals/159-exit-scanning.txt): Exit Scanning [INFORMATIONAL] -* [`160-bandwidth-offset.txt`](/proposals/160-bandwidth-offset.txt): Authorities vote for bandwidth offsets in consensus [CLOSED] -* [`161-computing-bandwidth-adjustments.txt`](/proposals/161-computing-bandwidth-adjustments.txt): Computing Bandwidth Adjustments [CLOSED] -* [`162-consensus-flavors.txt`](/proposals/162-consensus-flavors.txt): Publish the consensus in multiple flavors [CLOSED] -* [`163-detecting-clients.txt`](/proposals/163-detecting-clients.txt): Detecting whether a connection comes from a client [SUPERSEDED] -* [`164-reporting-server-status.txt`](/proposals/164-reporting-server-status.txt): Reporting the status of server votes [OBSOLETE] -* [`165-simple-robust-voting.txt`](/proposals/165-simple-robust-voting.txt): Easy migration for voting authority sets [REJECTED] -* [`166-statistics-extra-info-docs.txt`](/proposals/166-statistics-extra-info-docs.txt): Including Network Statistics in Extra-Info Documents [CLOSED] -* [`167-params-in-consensus.txt`](/proposals/167-params-in-consensus.txt): Vote on network parameters in consensus [CLOSED] -* [`168-reduce-circwindow.txt`](/proposals/168-reduce-circwindow.txt): Reduce default circuit window [REJECTED] -* [`169-eliminating-renegotiation.txt`](/proposals/169-eliminating-renegotiation.txt): Eliminate TLS renegotiation for the Tor connection handshake [SUPERSEDED] -* [`170-user-path-config.txt`](/proposals/170-user-path-config.txt): Configuration options regarding circuit building [SUPERSEDED] -* [`171-separate-streams.txt`](/proposals/171-separate-streams.txt): Separate streams across circuits by connection metadata [CLOSED] -* [`172-circ-getinfo-option.txt`](/proposals/172-circ-getinfo-option.txt): GETINFO controller option for circuit information [RESERVE] -* [`173-getinfo-option-expansion.txt`](/proposals/173-getinfo-option-expansion.txt): GETINFO Option Expansion [OBSOLETE] -* [`174-optimistic-data-server.txt`](/proposals/174-optimistic-data-server.txt): Optimistic Data for Tor: Server Side [CLOSED] -* [`175-automatic-node-promotion.txt`](/proposals/175-automatic-node-promotion.txt): Automatically promoting Tor clients to nodes [REJECTED] -* [`176-revising-handshake.txt`](/proposals/176-revising-handshake.txt): Proposed version-3 link handshake for Tor [CLOSED] -* [`177-flag-abstention.txt`](/proposals/177-flag-abstention.txt): Abstaining from votes on individual flags [RESERVE] -* [`178-param-voting.txt`](/proposals/178-param-voting.txt): Require majority of authorities to vote for consensus parameters [CLOSED] -* [`179-TLS-cert-and-parameter-normalization.txt`](/proposals/179-TLS-cert-and-parameter-normalization.txt): TLS certificate and parameter normalization [CLOSED] -* [`180-pluggable-transport.txt`](/proposals/180-pluggable-transport.txt): Pluggable transports for circumvention [CLOSED] -* [`181-optimistic-data-client.txt`](/proposals/181-optimistic-data-client.txt): Optimistic Data for Tor: Client Side [CLOSED] -* [`182-creditbucket.txt`](/proposals/182-creditbucket.txt): Credit Bucket [OBSOLETE] -* [`183-refillintervals.txt`](/proposals/183-refillintervals.txt): Refill Intervals [CLOSED] -* [`184-v3-link-protocol.txt`](/proposals/184-v3-link-protocol.txt): Miscellaneous changes for a v3 Tor link protocol [CLOSED] -* [`185-dir-without-dirport.txt`](/proposals/185-dir-without-dirport.txt): Directory caches without DirPort [SUPERSEDED] -* [`186-multiple-orports.txt`](/proposals/186-multiple-orports.txt): Multiple addresses for one OR or bridge [CLOSED] -* [`187-allow-client-auth.txt`](/proposals/187-allow-client-auth.txt): Reserve a cell type to allow client authorization [CLOSED] -* [`188-bridge-guards.txt`](/proposals/188-bridge-guards.txt): Bridge Guards and other anti-enumeration defenses [RESERVE] -* [`189-authorize-cell.txt`](/proposals/189-authorize-cell.txt): AUTHORIZE and AUTHORIZED cells [OBSOLETE] -* [`190-shared-secret-bridge-authorization.txt`](/proposals/190-shared-secret-bridge-authorization.txt): Bridge Client Authorization Based on a Shared Secret [OBSOLETE] -* [`191-mitm-bridge-detection-resistance.txt`](/proposals/191-mitm-bridge-detection-resistance.txt): Bridge Detection Resistance against MITM-capable Adversaries [OBSOLETE] -* [`192-store-bridge-information.txt`](/proposals/192-store-bridge-information.txt): Automatically retrieve and store information about bridges [OBSOLETE] -* [`193-safe-cookie-authentication.txt`](/proposals/193-safe-cookie-authentication.txt): Safe cookie authentication for Tor controllers [CLOSED] -* [`194-mnemonic-urls.txt`](/proposals/194-mnemonic-urls.txt): Mnemonic .onion URLs [SUPERSEDED] -* [`195-TLS-normalization-for-024.txt`](/proposals/195-TLS-normalization-for-024.txt): TLS certificate normalization for Tor 0.2.4.x [DEAD] -* [`196-transport-control-ports.txt`](/proposals/196-transport-control-ports.txt): Extended ORPort and TransportControlPort [CLOSED] -* [`197-postmessage-ipc.txt`](/proposals/197-postmessage-ipc.txt): Message-based Inter-Controller IPC Channel [REJECTED] -* [`198-restore-clienthello-semantics.txt`](/proposals/198-restore-clienthello-semantics.txt): Restore semantics of TLS ClientHello [CLOSED] -* [`199-bridgefinder-integration.txt`](/proposals/199-bridgefinder-integration.txt): Integration of BridgeFinder and BridgeFinderHelper [OBSOLETE] -* [`200-new-create-and-extend-cells.txt`](/proposals/200-new-create-and-extend-cells.txt): Adding new, extensible CREATE, EXTEND, and related cells [CLOSED] -* [`201-bridge-v3-reqs-stats.txt`](/proposals/201-bridge-v3-reqs-stats.txt): Make bridges report statistics on daily v3 network status requests [RESERVE] -* [`202-improved-relay-crypto.txt`](/proposals/202-improved-relay-crypto.txt): Two improved relay encryption protocols for Tor cells [META] -* [`203-https-frontend.txt`](/proposals/203-https-frontend.txt): Avoiding censorship by impersonating an HTTPS server [OBSOLETE] -* [`204-hidserv-subdomains.txt`](/proposals/204-hidserv-subdomains.txt): Subdomain support for Hidden Service addresses [CLOSED] -* [`205-local-dnscache.txt`](/proposals/205-local-dnscache.txt): Remove global client-side DNS caching [CLOSED] -* [`206-directory-sources.txt`](/proposals/206-directory-sources.txt): Preconfigured directory sources for bootstrapping [CLOSED] -* [`207-directory-guards.txt`](/proposals/207-directory-guards.txt): Directory guards [CLOSED] -* [`208-ipv6-exits-redux.txt`](/proposals/208-ipv6-exits-redux.txt): IPv6 Exits Redux [CLOSED] -* [`209-path-bias-tuning.txt`](/proposals/209-path-bias-tuning.txt): Tuning the Parameters for the Path Bias Defense [OBSOLETE] -* [`210-faster-headless-consensus-bootstrap.txt`](/proposals/210-faster-headless-consensus-bootstrap.txt): Faster Headless Consensus Bootstrapping [SUPERSEDED] -* [`211-mapaddress-tor-status.txt`](/proposals/211-mapaddress-tor-status.txt): Internal Mapaddress for Tor Configuration Testing [RESERVE] -* [`212-using-old-consensus.txt`](/proposals/212-using-old-consensus.txt): Increase Acceptable Consensus Age [NEEDS-REVISION] -* [`213-remove-stream-sendmes.txt`](/proposals/213-remove-stream-sendmes.txt): Remove stream-level sendmes from the design [DEAD] -* [`214-longer-circids.txt`](/proposals/214-longer-circids.txt): Allow 4-byte circuit IDs in a new link protocol [CLOSED] -* [`215-update-min-consensus-ver.txt`](/proposals/215-update-min-consensus-ver.txt): Let the minimum consensus method change with time [CLOSED] -* [`216-ntor-handshake.txt`](/proposals/216-ntor-handshake.txt): Improved circuit-creation key exchange [CLOSED] -* [`217-ext-orport-auth.txt`](/proposals/217-ext-orport-auth.txt): Tor Extended ORPort Authentication [CLOSED] -* [`218-usage-controller-events.txt`](/proposals/218-usage-controller-events.txt): Controller events to better understand connection/circuit usage [CLOSED] -* [`219-expanded-dns.txt`](/proposals/219-expanded-dns.txt): Support for full DNS and DNSSEC resolution in Tor [NEEDS-REVISION] -* [`220-ecc-id-keys.txt`](/proposals/220-ecc-id-keys.txt): Migrate server identity keys to Ed25519 [CLOSED] -* [`221-stop-using-create-fast.txt`](/proposals/221-stop-using-create-fast.txt): Stop using CREATE_FAST [CLOSED] -* [`222-remove-client-timestamps.txt`](/proposals/222-remove-client-timestamps.txt): Stop sending client timestamps [CLOSED] -* [`223-ace-handshake.txt`](/proposals/223-ace-handshake.txt): Ace: Improved circuit-creation key exchange [RESERVE] -* [`224-rend-spec-ng.txt`](/proposals/224-rend-spec-ng.txt): Next-Generation Hidden Services in Tor [CLOSED] -* [`225-strawman-shared-rand.txt`](/proposals/225-strawman-shared-rand.txt): Strawman proposal: commit-and-reveal shared rng [SUPERSEDED] -* [`226-bridgedb-database-improvements.txt`](/proposals/226-bridgedb-database-improvements.txt): "Scalability and Stability Improvements to BridgeDB: Switching to a Distributed Database System and RDBMS" [RESERVE] -* [`227-vote-on-package-fingerprints.txt`](/proposals/227-vote-on-package-fingerprints.txt): Include package fingerprints in consensus documents [CLOSED] -* [`228-cross-certification-onionkeys.txt`](/proposals/228-cross-certification-onionkeys.txt): Cross-certifying identity keys with onion keys [CLOSED] -* [`229-further-socks5-extensions.txt`](/proposals/229-further-socks5-extensions.txt): Further SOCKS5 extensions [REJECTED] -* [`230-rsa1024-relay-id-migration.txt`](/proposals/230-rsa1024-relay-id-migration.txt): How to change RSA1024 relay identity keys [OBSOLETE] -* [`231-migrate-authority-rsa1024-ids.txt`](/proposals/231-migrate-authority-rsa1024-ids.txt): Migrating authority RSA1024 identity keys [OBSOLETE] -* [`232-pluggable-transports-through-proxy.txt`](/proposals/232-pluggable-transports-through-proxy.txt): Pluggable Transport through SOCKS proxy [CLOSED] -* [`233-quicken-tor2web-mode.txt`](/proposals/233-quicken-tor2web-mode.txt): Making Tor2Web mode faster [REJECTED] -* [`234-remittance-addresses.txt`](/proposals/234-remittance-addresses.txt): Adding remittance field to directory specification [REJECTED] -* [`235-kill-named-flag.txt`](/proposals/235-kill-named-flag.txt): Stop assigning (and eventually supporting) the Named flag [CLOSED] -* [`236-single-guard-node.txt`](/proposals/236-single-guard-node.txt): The move to a single guard node [CLOSED] -* [`237-directory-servers-for-all.txt`](/proposals/237-directory-servers-for-all.txt): All relays are directory servers [CLOSED] -* [`238-hs-relay-stats.txt`](/proposals/238-hs-relay-stats.txt): Better hidden service stats from Tor relays [CLOSED] -* [`239-consensus-hash-chaining.txt`](/proposals/239-consensus-hash-chaining.txt): Consensus Hash Chaining [OPEN] -* [`240-auth-cert-revocation.txt`](/proposals/240-auth-cert-revocation.txt): Early signing key revocation for directory authorities [OPEN] -* [`241-suspicious-guard-turnover.txt`](/proposals/241-suspicious-guard-turnover.txt): Resisting guard-turnover attacks [REJECTED] -* [`242-better-families.txt`](/proposals/242-better-families.txt): Better performance and usability for the MyFamily option [SUPERSEDED] -* [`243-hsdir-flag-need-stable.txt`](/proposals/243-hsdir-flag-need-stable.txt): Give out HSDir flag only to relays with Stable flag [CLOSED] -* [`244-use-rfc5705-for-tls-binding.txt`](/proposals/244-use-rfc5705-for-tls-binding.txt): Use RFC5705 Key Exporting in our AUTHENTICATE calls [CLOSED] -* [`245-tap-out.txt`](/proposals/245-tap-out.txt): Deprecating and removing the TAP circuit extension protocol [NEEDS-REVISION] -* [`246-merge-hsdir-and-intro.txt`](/proposals/246-merge-hsdir-and-intro.txt): Merging Hidden Service Directories and Introduction Points [REJECTED] -* [`247-hs-guard-discovery.txt`](/proposals/247-hs-guard-discovery.txt): Defending Against Guard Discovery Attacks using Vanguards [SUPERSEDED] -* [`248-removing-rsa-identities.txt`](/proposals/248-removing-rsa-identities.txt): Remove all RSA identity keys [NEEDS-REVISION] -* [`249-large-create-cells.txt`](/proposals/249-large-create-cells.txt): Allow CREATE cells with >505 bytes of handshake data [SUPERSEDED] -* [`250-commit-reveal-consensus.txt`](/proposals/250-commit-reveal-consensus.txt): Random Number Generation During Tor Voting [CLOSED] -* [`251-netflow-padding.txt`](/proposals/251-netflow-padding.txt): Padding for netflow record resolution reduction [CLOSED] -* [`252-single-onion.txt`](/proposals/252-single-onion.txt): Single Onion Services [SUPERSEDED] -* [`253-oob-hmac.txt`](/proposals/253-oob-hmac.txt): Out of Band Circuit HMACs [DEAD] -* [`254-padding-negotiation.txt`](/proposals/254-padding-negotiation.txt): Padding Negotiation [CLOSED] -* [`255-hs-load-balancing.txt`](/proposals/255-hs-load-balancing.txt): Controller features to allow for load-balancing hidden services [RESERVE] -* [`256-key-revocation.txt`](/proposals/256-key-revocation.txt): Key revocation for relays and authorities [RESERVE] -* [`257-hiding-authorities.txt`](/proposals/257-hiding-authorities.txt): Refactoring authorities and making them more isolated from the net [META] -* [`258-dirauth-dos.txt`](/proposals/258-dirauth-dos.txt): Denial-of-service resistance for directory authorities [DEAD] -* [`259-guard-selection.txt`](/proposals/259-guard-selection.txt): New Guard Selection Behaviour [OBSOLETE] -* [`260-rend-single-onion.txt`](/proposals/260-rend-single-onion.txt): Rendezvous Single Onion Services [FINISHED] -* [`261-aez-crypto.txt`](/proposals/261-aez-crypto.txt): AEZ for relay cryptography [OBSOLETE] -* [`262-rekey-circuits.txt`](/proposals/262-rekey-circuits.txt): Re-keying live circuits with new cryptographic material [RESERVE] -* [`263-ntru-for-pq-handshake.txt`](/proposals/263-ntru-for-pq-handshake.txt): Request to change key exchange protocol for handshake v1.2 [OBSOLETE] -* [`264-subprotocol-versions.txt`](/proposals/264-subprotocol-versions.txt): Putting version numbers on the Tor subprotocols [CLOSED] -* [`265-load-balancing-with-overhead.txt`](/proposals/265-load-balancing-with-overhead.txt): Load Balancing with Overhead Parameters [OPEN] -* [`266-removing-current-obsolete-clients.txt`](/proposals/266-removing-current-obsolete-clients.txt): Removing current obsolete clients from the Tor network [SUPERSEDED] -* [`267-tor-consensus-transparency.txt`](/proposals/267-tor-consensus-transparency.txt): Tor Consensus Transparency [OPEN] -* [`268-guard-selection.txt`](/proposals/268-guard-selection.txt): New Guard Selection Behaviour [OBSOLETE] -* [`269-hybrid-handshake.txt`](/proposals/269-hybrid-handshake.txt): Transitionally secure hybrid handshakes [NEEDS-REVISION] -* [`270-newhope-hybrid-handshake.txt`](/proposals/270-newhope-hybrid-handshake.txt): RebelAlliance: A Post-Quantum Secure Hybrid Handshake Based on NewHope [OBSOLETE] -* [`271-another-guard-selection.txt`](/proposals/271-another-guard-selection.txt): Another algorithm for guard selection [CLOSED] -* [`272-valid-and-running-by-default.txt`](/proposals/272-valid-and-running-by-default.txt): Listed routers should be Valid, Running, and treated as such [CLOSED] -* [`273-exit-relay-pinning.txt`](/proposals/273-exit-relay-pinning.txt): Exit relay pinning for web services [RESERVE] -* [`274-rotate-onion-keys-less.txt`](/proposals/274-rotate-onion-keys-less.txt): Rotate onion keys less frequently [CLOSED] -* [`275-md-published-time-is-silly.txt`](/proposals/275-md-published-time-is-silly.txt): Stop including meaningful "published" time in microdescriptor consensus [CLOSED] -* [`276-lower-bw-granularity.txt`](/proposals/276-lower-bw-granularity.txt): Report bandwidth with lower granularity in consensus documents [DEAD] -* [`277-detect-id-sharing.txt`](/proposals/277-detect-id-sharing.txt): Detect multiple relay instances running with same ID [OPEN] -* [`278-directory-compression-scheme-negotiation.txt`](/proposals/278-directory-compression-scheme-negotiation.txt): Directory Compression Scheme Negotiation [CLOSED] -* [`279-naming-layer-api.txt`](/proposals/279-naming-layer-api.txt): A Name System API for Tor Onion Services [NEEDS-REVISION] -* [`280-privcount-in-tor.txt`](/proposals/280-privcount-in-tor.txt): Privacy-Preserving Statistics with Privcount in Tor [SUPERSEDED] -* [`281-bulk-md-download.txt`](/proposals/281-bulk-md-download.txt): Downloading microdescriptors in bulk [RESERVE] -* [`282-remove-named-from-consensus.txt`](/proposals/282-remove-named-from-consensus.txt): Remove "Named" and "Unnamed" handling from consensus voting [ACCEPTED] -* [`283-ipv6-in-micro-consensus.txt`](/proposals/283-ipv6-in-micro-consensus.txt): Move IPv6 ORPorts from microdescriptors to the microdesc consensus [CLOSED] -* [`284-hsv3-control-port.txt`](/proposals/284-hsv3-control-port.txt): Hidden Service v3 Control Port [CLOSED] -* [`285-utf-8.txt`](/proposals/285-utf-8.txt): Directory documents should be standardized as UTF-8 [ACCEPTED] -* [`286-hibernation-api.txt`](/proposals/286-hibernation-api.txt): Controller APIs for hibernation access on mobile [REJECTED] -* [`287-reduce-lifetime.txt`](/proposals/287-reduce-lifetime.txt): Reduce circuit lifetime without overloading the network [OPEN] -* [`288-privcount-with-shamir.txt`](/proposals/288-privcount-with-shamir.txt): Privacy-Preserving Statistics with Privcount in Tor (Shamir version) [RESERVE] -* [`289-authenticated-sendmes.txt`](/proposals/289-authenticated-sendmes.txt): Authenticating sendme cells to mitigate bandwidth attacks [CLOSED] -* [`290-deprecate-consensus-methods.txt`](/proposals/290-deprecate-consensus-methods.txt): Continuously update consensus methods [META] -* [`291-two-guard-nodes.txt`](/proposals/291-two-guard-nodes.txt): The move to two guard nodes [FINISHED] -* [`292-mesh-vanguards.txt`](/proposals/292-mesh-vanguards.txt): Mesh-based vanguards [ACCEPTED] -* [`293-know-when-to-publish.txt`](/proposals/293-know-when-to-publish.txt): Other ways for relays to know when to publish [CLOSED] -* [`294-tls-1.3.txt`](/proposals/294-tls-1.3.txt): TLS 1.3 Migration [DRAFT] -* [`295-relay-crypto-with-adl.txt`](/proposals/295-relay-crypto-with-adl.txt): Using ADL for relay cryptography (solving the crypto-tagging attack) [OPEN] -* [`296-expose-bandwidth-files.txt`](/proposals/296-expose-bandwidth-files.txt): Have Directory Authorities expose raw bandwidth list files [CLOSED] -* [`297-safer-protover-shutdowns.txt`](/proposals/297-safer-protover-shutdowns.txt): Relaxing the protover-based shutdown rules [CLOSED] -* [`298-canonical-families.txt`](/proposals/298-canonical-families.txt): Putting family lines in canonical form [CLOSED] -* [`299-ip-failure-count.txt`](/proposals/299-ip-failure-count.txt): Preferring IPv4 or IPv6 based on IP Version Failure Count [SUPERSEDED] -* [`300-walking-onions.txt`](/proposals/300-walking-onions.txt): Walking Onions: Scaling and Saving Bandwidth [INFORMATIONAL] -* [`301-dont-vote-on-package-fingerprints.txt`](/proposals/301-dont-vote-on-package-fingerprints.txt): Don't include package fingerprints in consensus documents [CLOSED] -* [`302-padding-machines-for-onion-clients.txt`](/proposals/302-padding-machines-for-onion-clients.txt): Hiding onion service clients using padding [CLOSED] -* [`303-protover-removal-policy.txt`](/proposals/303-protover-removal-policy.txt): When and how to remove support for protocol versions [OPEN] -* [`304-socks5-extending-hs-error-codes.txt`](/proposals/304-socks5-extending-hs-error-codes.txt): Extending SOCKS5 Onion Service Error Codes [CLOSED] -* [`305-establish-intro-dos-defense-extention.txt`](/proposals/305-establish-intro-dos-defense-extention.txt): ESTABLISH_INTRO Cell DoS Defense Extension [CLOSED] -* [`306-ipv6-happy-eyeballs.txt`](/proposals/306-ipv6-happy-eyeballs.txt): A Tor Implementation of IPv6 Happy Eyeballs [OPEN] -* [`307-onionbalance-v3.txt`](/proposals/307-onionbalance-v3.txt): Onion Balance Support for Onion Service v3 [RESERVE] -* [`308-counter-galois-onion.txt`](/proposals/308-counter-galois-onion.txt): Counter Galois Onion: A New Proposal for Forward-Secure Relay Cryptography [SUPERSEDED] -* [`309-optimistic-socks-in-tor.txt`](/proposals/309-optimistic-socks-in-tor.txt): Optimistic SOCKS Data [OPEN] -* [`310-bandaid-on-guard-selection.txt`](/proposals/310-bandaid-on-guard-selection.txt): Towards load-balancing in Prop 271 [CLOSED] -* [`311-relay-ipv6-reachability.txt`](/proposals/311-relay-ipv6-reachability.txt): Tor Relay IPv6 Reachability [ACCEPTED] -* [`312-relay-auto-ipv6-addr.txt`](/proposals/312-relay-auto-ipv6-addr.txt): Tor Relay Automatic IPv6 Address Discovery [ACCEPTED] -* [`313-relay-ipv6-stats.txt`](/proposals/313-relay-ipv6-stats.txt): Tor Relay IPv6 Statistics [ACCEPTED] -* [`314-allow-markdown-proposals.md`](/proposals/314-allow-markdown-proposals.md): Allow Markdown for proposal format [CLOSED] -* [`315-update-dir-required-fields.txt`](/proposals/315-update-dir-required-fields.txt): Updating the list of fields required in directory documents [CLOSED] -* [`316-flashflow.md`](/proposals/316-flashflow.md): FlashFlow: A Secure Speed Test for Tor (Parent Proposal) [DRAFT] -* [`317-secure-dns-name-resolution.txt`](/proposals/317-secure-dns-name-resolution.txt): Improve security aspects of DNS name resolution [NEEDS-REVISION] -* [`318-limit-protovers.md`](/proposals/318-limit-protovers.md): Limit protover values to 0-63 [CLOSED] -* [`319-wide-everything.md`](/proposals/319-wide-everything.md): RELAY_FRAGMENT cells [OBSOLETE] -* [`320-tap-out-again.md`](/proposals/320-tap-out-again.md): Removing TAP usage from v2 onion services [REJECTED] -* [`321-happy-families.md`](/proposals/321-happy-families.md): Better performance and usability for the MyFamily option (v2) [ACCEPTED] -* [`322-dirport-linkspec.md`](/proposals/322-dirport-linkspec.md): Extending link specifiers to include the directory port [OPEN] -* [`323-walking-onions-full.md`](/proposals/323-walking-onions-full.md): Specification for Walking Onions [OPEN] -* [`324-rtt-congestion-control.txt`](/proposals/324-rtt-congestion-control.txt): RTT-based Congestion Control for Tor [FINISHED] -* [`325-packed-relay-cells.md`](/proposals/325-packed-relay-cells.md): Packed relay cells: saving space on small commands [OBSOLETE] -* [`326-tor-relay-well-known-uri-rfc8615.md`](/proposals/326-tor-relay-well-known-uri-rfc8615.md): The "tor-relay" Well-Known Resource Identifier [OPEN] -* [`327-pow-over-intro.txt`](/proposals/327-pow-over-intro.txt): A First Take at PoW Over Introduction Circuits [FINISHED] -* [`328-relay-overload-report.md`](/proposals/328-relay-overload-report.md): Make Relays Report When They Are Overloaded [CLOSED] -* [`329-traffic-splitting.txt`](/proposals/329-traffic-splitting.txt): Overcoming Tor's Bottlenecks with Traffic Splitting [FINISHED] -* [`330-authority-contact.md`](/proposals/330-authority-contact.md): Modernizing authority contact entries [OPEN] -* [`331-res-tokens-for-anti-dos.md`](/proposals/331-res-tokens-for-anti-dos.md): Res tokens: Anonymous Credentials for Onion Service DoS Resilience [DRAFT] -* [`332-ntor-v3-with-extra-data.md`](/proposals/332-ntor-v3-with-extra-data.md): Ntor protocol with extra data, version 3 [CLOSED] -* [`333-vanguards-lite.md`](/proposals/333-vanguards-lite.md): Vanguards lite [FINISHED] -* [`334-middle-only-flag.txt`](/proposals/334-middle-only-flag.txt): A Directory Authority Flag To Mark Relays As Middle-only [SUPERSEDED] -* [`335-middle-only-redux.md`](/proposals/335-middle-only-redux.md): An authority-only design for MiddleOnly [CLOSED] -* [`336-randomize-guard-retries.md`](/proposals/336-randomize-guard-retries.md): Randomized schedule for guard retries [CLOSED] -* [`337-simpler-guard-usability.md`](/proposals/337-simpler-guard-usability.md): A simpler way to decide, "Is this guard usable?" [CLOSED] -* [`338-netinfo-y2038.md`](/proposals/338-netinfo-y2038.md): Use an 8-byte timestamp in NETINFO cells [ACCEPTED] -* [`339-udp-over-tor.md`](/proposals/339-udp-over-tor.md): UDP traffic over Tor [ACCEPTED] -* [`340-packed-and-fragmented.md`](/proposals/340-packed-and-fragmented.md): Packed and fragmented relay messages [OPEN] -* [`341-better-oos.md`](/proposals/341-better-oos.md): A better algorithm for out-of-sockets eviction [OPEN] -* [`342-decouple-hs-interval.md`](/proposals/342-decouple-hs-interval.md): Decoupling hs_interval and SRV lifetime [DRAFT] -* [`343-rend-caa.txt`](/proposals/343-rend-caa.txt): CAA Extensions for the Tor Rendezvous Specification [OPEN] -* [`344-protocol-info-leaks.txt`](/proposals/344-protocol-info-leaks.txt): Prioritizing Protocol Information Leaks in Tor [OPEN] -* [`345-specs-in-mdbook.md`](/proposals/345-specs-in-mdbook.md): Migrating the tor specifications to mdbook [OPEN] +* [`000-index.txt`](/proposals/000-index.txt): Index of Tor Proposals \[META\] +* [`001-process.txt`](/proposals/001-process.txt): The Tor Proposal Process \[META\] +* [`098-todo.txt`](/proposals/098-todo.txt): Proposals that should be written \[OBSOLETE\] +* [`099-misc.txt`](/proposals/099-misc.txt): Miscellaneous proposals \[OBSOLETE\] +* [`100-tor-spec-udp.txt`](/proposals/100-tor-spec-udp.txt): Tor Unreliable Datagram Extension Proposal \[DEAD\] +* [`101-dir-voting.txt`](/proposals/101-dir-voting.txt): Voting on the Tor Directory System \[CLOSED\] +* [`102-drop-opt.txt`](/proposals/102-drop-opt.txt): Dropping "opt" from the directory format \[CLOSED\] +* [`103-multilevel-keys.txt`](/proposals/103-multilevel-keys.txt): Splitting identity key from regularly used signing key \[CLOSED\] +* [`104-short-descriptors.txt`](/proposals/104-short-descriptors.txt): Long and Short Router Descriptors \[CLOSED\] +* [`105-handshake-revision.txt`](/proposals/105-handshake-revision.txt): Version negotiation for the Tor protocol \[CLOSED\] +* [`106-less-tls-constraint.txt`](/proposals/106-less-tls-constraint.txt): Checking fewer things during TLS handshakes \[CLOSED\] +* [`107-uptime-sanity-checking.txt`](/proposals/107-uptime-sanity-checking.txt): Uptime Sanity Checking \[CLOSED\] +* [`108-mtbf-based-stability.txt`](/proposals/108-mtbf-based-stability.txt): Base "Stable" Flag on Mean Time Between Failures \[CLOSED\] +* [`109-no-sharing-ips.txt`](/proposals/109-no-sharing-ips.txt): No more than one server per IP address \[CLOSED\] +* [`110-avoid-infinite-circuits.txt`](/proposals/110-avoid-infinite-circuits.txt): Avoiding infinite length circuits \[CLOSED\] +* [`111-local-traffic-priority.txt`](/proposals/111-local-traffic-priority.txt): Prioritizing local traffic over relayed traffic \[CLOSED\] +* [`112-bring-back-pathlencoinweight.txt`](/proposals/112-bring-back-pathlencoinweight.txt): Bring Back Pathlen Coin Weight \[SUPERSEDED\] +* [`113-fast-authority-interface.txt`](/proposals/113-fast-authority-interface.txt): Simplifying directory authority administration \[SUPERSEDED\] +* [`114-distributed-storage.txt`](/proposals/114-distributed-storage.txt): Distributed Storage for Tor Hidden Service Descriptors \[CLOSED\] +* [`115-two-hop-paths.txt`](/proposals/115-two-hop-paths.txt): Two Hop Paths \[DEAD\] +* [`116-two-hop-paths-from-guard.txt`](/proposals/116-two-hop-paths-from-guard.txt): Two hop paths from entry guards \[DEAD\] +* [`117-ipv6-exits.txt`](/proposals/117-ipv6-exits.txt): IPv6 exits \[CLOSED\] +* [`118-multiple-orports.txt`](/proposals/118-multiple-orports.txt): Advertising multiple ORPorts at once \[SUPERSEDED\] +* [`119-controlport-auth.txt`](/proposals/119-controlport-auth.txt): New PROTOCOLINFO command for controllers \[CLOSED\] +* [`120-shutdown-descriptors.txt`](/proposals/120-shutdown-descriptors.txt): Shutdown descriptors when Tor servers stop \[DEAD\] +* [`121-hidden-service-authentication.txt`](/proposals/121-hidden-service-authentication.txt): Hidden Service Authentication \[CLOSED\] +* [`122-unnamed-flag.txt`](/proposals/122-unnamed-flag.txt): Network status entries need a new Unnamed flag \[CLOSED\] +* [`123-autonaming.txt`](/proposals/123-autonaming.txt): Naming authorities automatically create bindings \[CLOSED\] +* [`124-tls-certificates.txt`](/proposals/124-tls-certificates.txt): Blocking resistant TLS certificate usage \[SUPERSEDED\] +* [`125-bridges.txt`](/proposals/125-bridges.txt): Behavior for bridge users, bridge relays, and bridge authorities \[CLOSED\] +* [`126-geoip-reporting.txt`](/proposals/126-geoip-reporting.txt): Getting GeoIP data and publishing usage summaries \[CLOSED\] +* [`127-dirport-mirrors-downloads.txt`](/proposals/127-dirport-mirrors-downloads.txt): Relaying dirport requests to Tor download site / website \[OBSOLETE\] +* [`128-bridge-families.txt`](/proposals/128-bridge-families.txt): Families of private bridges \[DEAD\] +* [`129-reject-plaintext-ports.txt`](/proposals/129-reject-plaintext-ports.txt): Block Insecure Protocols by Default \[CLOSED\] +* [`130-v2-conn-protocol.txt`](/proposals/130-v2-conn-protocol.txt): Version 2 Tor connection protocol \[CLOSED\] +* [`131-verify-tor-usage.txt`](/proposals/131-verify-tor-usage.txt): Help users to verify they are using Tor \[OBSOLETE\] +* [`132-browser-check-tor-service.txt`](/proposals/132-browser-check-tor-service.txt): A Tor Web Service For Verifying Correct Browser Configuration \[OBSOLETE\] +* [`133-unreachable-ors.txt`](/proposals/133-unreachable-ors.txt): Incorporate Unreachable ORs into the Tor Network \[RESERVE\] +* [`134-robust-voting.txt`](/proposals/134-robust-voting.txt): More robust consensus voting with diverse authority sets \[REJECTED\] +* [`135-private-tor-networks.txt`](/proposals/135-private-tor-networks.txt): Simplify Configuration of Private Tor Networks \[CLOSED\] +* [`136-legacy-keys.txt`](/proposals/136-legacy-keys.txt): Mass authority migration with legacy keys \[CLOSED\] +* [`137-bootstrap-phases.txt`](/proposals/137-bootstrap-phases.txt): Keep controllers informed as Tor bootstraps \[CLOSED\] +* [`138-remove-down-routers-from-consensus.txt`](/proposals/138-remove-down-routers-from-consensus.txt): Remove routers that are not Running from consensus documents \[CLOSED\] +* [`139-conditional-consensus-download.txt`](/proposals/139-conditional-consensus-download.txt): Download consensus documents only when it will be trusted \[CLOSED\] +* [`140-consensus-diffs.txt`](/proposals/140-consensus-diffs.txt): Provide diffs between consensuses \[CLOSED\] +* [`141-jit-sd-downloads.txt`](/proposals/141-jit-sd-downloads.txt): Download server descriptors on demand \[OBSOLETE\] +* [`142-combine-intro-and-rend-points.txt`](/proposals/142-combine-intro-and-rend-points.txt): Combine Introduction and Rendezvous Points \[DEAD\] +* [`143-distributed-storage-improvements.txt`](/proposals/143-distributed-storage-improvements.txt): Improvements of Distributed Storage for Tor Hidden Service Descriptors \[SUPERSEDED\] +* [`144-enforce-distinct-providers.txt`](/proposals/144-enforce-distinct-providers.txt): Increase the diversity of circuits by detecting nodes belonging the same provider \[OBSOLETE\] +* [`145-newguard-flag.txt`](/proposals/145-newguard-flag.txt): Separate "suitable as a guard" from "suitable as a new guard" \[SUPERSEDED\] +* [`146-long-term-stability.txt`](/proposals/146-long-term-stability.txt): Add new flag to reflect long-term stability \[SUPERSEDED\] +* [`147-prevoting-opinions.txt`](/proposals/147-prevoting-opinions.txt): Eliminate the need for v2 directories in generating v3 directories \[REJECTED\] +* [`148-uniform-client-end-reason.txt`](/proposals/148-uniform-client-end-reason.txt): Stream end reasons from the client side should be uniform \[CLOSED\] +* [`149-using-netinfo-data.txt`](/proposals/149-using-netinfo-data.txt): Using data from NETINFO cells \[SUPERSEDED\] +* [`150-exclude-exit-nodes.txt`](/proposals/150-exclude-exit-nodes.txt): Exclude Exit Nodes from a circuit \[CLOSED\] +* [`151-path-selection-improvements.txt`](/proposals/151-path-selection-improvements.txt): Improving Tor Path Selection \[CLOSED\] +* [`152-single-hop-circuits.txt`](/proposals/152-single-hop-circuits.txt): Optionally allow exit from single-hop circuits \[CLOSED\] +* [`153-automatic-software-update-protocol.txt`](/proposals/153-automatic-software-update-protocol.txt): Automatic software update protocol \[SUPERSEDED\] +* [`154-automatic-updates.txt`](/proposals/154-automatic-updates.txt): Automatic Software Update Protocol \[SUPERSEDED\] +* [`155-four-hidden-service-improvements.txt`](/proposals/155-four-hidden-service-improvements.txt): Four Improvements of Hidden Service Performance \[CLOSED\] +* [`156-tracking-blocked-ports.txt`](/proposals/156-tracking-blocked-ports.txt): Tracking blocked ports on the client side \[SUPERSEDED\] +* [`157-specific-cert-download.txt`](/proposals/157-specific-cert-download.txt): Make certificate downloads specific \[CLOSED\] +* [`158-microdescriptors.txt`](/proposals/158-microdescriptors.txt): Clients download consensus + microdescriptors \[CLOSED\] +* [`159-exit-scanning.txt`](/proposals/159-exit-scanning.txt): Exit Scanning \[INFORMATIONAL\] +* [`160-bandwidth-offset.txt`](/proposals/160-bandwidth-offset.txt): Authorities vote for bandwidth offsets in consensus \[CLOSED\] +* [`161-computing-bandwidth-adjustments.txt`](/proposals/161-computing-bandwidth-adjustments.txt): Computing Bandwidth Adjustments \[CLOSED\] +* [`162-consensus-flavors.txt`](/proposals/162-consensus-flavors.txt): Publish the consensus in multiple flavors \[CLOSED\] +* [`163-detecting-clients.txt`](/proposals/163-detecting-clients.txt): Detecting whether a connection comes from a client \[SUPERSEDED\] +* [`164-reporting-server-status.txt`](/proposals/164-reporting-server-status.txt): Reporting the status of server votes \[OBSOLETE\] +* [`165-simple-robust-voting.txt`](/proposals/165-simple-robust-voting.txt): Easy migration for voting authority sets \[REJECTED\] +* [`166-statistics-extra-info-docs.txt`](/proposals/166-statistics-extra-info-docs.txt): Including Network Statistics in Extra-Info Documents \[CLOSED\] +* [`167-params-in-consensus.txt`](/proposals/167-params-in-consensus.txt): Vote on network parameters in consensus \[CLOSED\] +* [`168-reduce-circwindow.txt`](/proposals/168-reduce-circwindow.txt): Reduce default circuit window \[REJECTED\] +* [`169-eliminating-renegotiation.txt`](/proposals/169-eliminating-renegotiation.txt): Eliminate TLS renegotiation for the Tor connection handshake \[SUPERSEDED\] +* [`170-user-path-config.txt`](/proposals/170-user-path-config.txt): Configuration options regarding circuit building \[SUPERSEDED\] +* [`171-separate-streams.txt`](/proposals/171-separate-streams.txt): Separate streams across circuits by connection metadata \[CLOSED\] +* [`172-circ-getinfo-option.txt`](/proposals/172-circ-getinfo-option.txt): GETINFO controller option for circuit information \[RESERVE\] +* [`173-getinfo-option-expansion.txt`](/proposals/173-getinfo-option-expansion.txt): GETINFO Option Expansion \[OBSOLETE\] +* [`174-optimistic-data-server.txt`](/proposals/174-optimistic-data-server.txt): Optimistic Data for Tor: Server Side \[CLOSED\] +* [`175-automatic-node-promotion.txt`](/proposals/175-automatic-node-promotion.txt): Automatically promoting Tor clients to nodes \[REJECTED\] +* [`176-revising-handshake.txt`](/proposals/176-revising-handshake.txt): Proposed version-3 link handshake for Tor \[CLOSED\] +* [`177-flag-abstention.txt`](/proposals/177-flag-abstention.txt): Abstaining from votes on individual flags \[RESERVE\] +* [`178-param-voting.txt`](/proposals/178-param-voting.txt): Require majority of authorities to vote for consensus parameters \[CLOSED\] +* [`179-TLS-cert-and-parameter-normalization.txt`](/proposals/179-TLS-cert-and-parameter-normalization.txt): TLS certificate and parameter normalization \[CLOSED\] +* [`180-pluggable-transport.txt`](/proposals/180-pluggable-transport.txt): Pluggable transports for circumvention \[CLOSED\] +* [`181-optimistic-data-client.txt`](/proposals/181-optimistic-data-client.txt): Optimistic Data for Tor: Client Side \[CLOSED\] +* [`182-creditbucket.txt`](/proposals/182-creditbucket.txt): Credit Bucket \[OBSOLETE\] +* [`183-refillintervals.txt`](/proposals/183-refillintervals.txt): Refill Intervals \[CLOSED\] +* [`184-v3-link-protocol.txt`](/proposals/184-v3-link-protocol.txt): Miscellaneous changes for a v3 Tor link protocol \[CLOSED\] +* [`185-dir-without-dirport.txt`](/proposals/185-dir-without-dirport.txt): Directory caches without DirPort \[SUPERSEDED\] +* [`186-multiple-orports.txt`](/proposals/186-multiple-orports.txt): Multiple addresses for one OR or bridge \[CLOSED\] +* [`187-allow-client-auth.txt`](/proposals/187-allow-client-auth.txt): Reserve a cell type to allow client authorization \[CLOSED\] +* [`188-bridge-guards.txt`](/proposals/188-bridge-guards.txt): Bridge Guards and other anti-enumeration defenses \[RESERVE\] +* [`189-authorize-cell.txt`](/proposals/189-authorize-cell.txt): AUTHORIZE and AUTHORIZED cells \[OBSOLETE\] +* [`190-shared-secret-bridge-authorization.txt`](/proposals/190-shared-secret-bridge-authorization.txt): Bridge Client Authorization Based on a Shared Secret \[OBSOLETE\] +* [`191-mitm-bridge-detection-resistance.txt`](/proposals/191-mitm-bridge-detection-resistance.txt): Bridge Detection Resistance against MITM-capable Adversaries \[OBSOLETE\] +* [`192-store-bridge-information.txt`](/proposals/192-store-bridge-information.txt): Automatically retrieve and store information about bridges \[OBSOLETE\] +* [`193-safe-cookie-authentication.txt`](/proposals/193-safe-cookie-authentication.txt): Safe cookie authentication for Tor controllers \[CLOSED\] +* [`194-mnemonic-urls.txt`](/proposals/194-mnemonic-urls.txt): Mnemonic .onion URLs \[SUPERSEDED\] +* [`195-TLS-normalization-for-024.txt`](/proposals/195-TLS-normalization-for-024.txt): TLS certificate normalization for Tor 0.2.4.x \[DEAD\] +* [`196-transport-control-ports.txt`](/proposals/196-transport-control-ports.txt): Extended ORPort and TransportControlPort \[CLOSED\] +* [`197-postmessage-ipc.txt`](/proposals/197-postmessage-ipc.txt): Message-based Inter-Controller IPC Channel \[REJECTED\] +* [`198-restore-clienthello-semantics.txt`](/proposals/198-restore-clienthello-semantics.txt): Restore semantics of TLS ClientHello \[CLOSED\] +* [`199-bridgefinder-integration.txt`](/proposals/199-bridgefinder-integration.txt): Integration of BridgeFinder and BridgeFinderHelper \[OBSOLETE\] +* [`200-new-create-and-extend-cells.txt`](/proposals/200-new-create-and-extend-cells.txt): Adding new, extensible CREATE, EXTEND, and related cells \[CLOSED\] +* [`201-bridge-v3-reqs-stats.txt`](/proposals/201-bridge-v3-reqs-stats.txt): Make bridges report statistics on daily v3 network status requests \[RESERVE\] +* [`202-improved-relay-crypto.txt`](/proposals/202-improved-relay-crypto.txt): Two improved relay encryption protocols for Tor cells \[META\] +* [`203-https-frontend.txt`](/proposals/203-https-frontend.txt): Avoiding censorship by impersonating an HTTPS server \[OBSOLETE\] +* [`204-hidserv-subdomains.txt`](/proposals/204-hidserv-subdomains.txt): Subdomain support for Hidden Service addresses \[CLOSED\] +* [`205-local-dnscache.txt`](/proposals/205-local-dnscache.txt): Remove global client-side DNS caching \[CLOSED\] +* [`206-directory-sources.txt`](/proposals/206-directory-sources.txt): Preconfigured directory sources for bootstrapping \[CLOSED\] +* [`207-directory-guards.txt`](/proposals/207-directory-guards.txt): Directory guards \[CLOSED\] +* [`208-ipv6-exits-redux.txt`](/proposals/208-ipv6-exits-redux.txt): IPv6 Exits Redux \[CLOSED\] +* [`209-path-bias-tuning.txt`](/proposals/209-path-bias-tuning.txt): Tuning the Parameters for the Path Bias Defense \[OBSOLETE\] +* [`210-faster-headless-consensus-bootstrap.txt`](/proposals/210-faster-headless-consensus-bootstrap.txt): Faster Headless Consensus Bootstrapping \[SUPERSEDED\] +* [`211-mapaddress-tor-status.txt`](/proposals/211-mapaddress-tor-status.txt): Internal Mapaddress for Tor Configuration Testing \[RESERVE\] +* [`212-using-old-consensus.txt`](/proposals/212-using-old-consensus.txt): Increase Acceptable Consensus Age \[NEEDS-REVISION\] +* [`213-remove-stream-sendmes.txt`](/proposals/213-remove-stream-sendmes.txt): Remove stream-level sendmes from the design \[DEAD\] +* [`214-longer-circids.txt`](/proposals/214-longer-circids.txt): Allow 4-byte circuit IDs in a new link protocol \[CLOSED\] +* [`215-update-min-consensus-ver.txt`](/proposals/215-update-min-consensus-ver.txt): Let the minimum consensus method change with time \[CLOSED\] +* [`216-ntor-handshake.txt`](/proposals/216-ntor-handshake.txt): Improved circuit-creation key exchange \[CLOSED\] +* [`217-ext-orport-auth.txt`](/proposals/217-ext-orport-auth.txt): Tor Extended ORPort Authentication \[CLOSED\] +* [`218-usage-controller-events.txt`](/proposals/218-usage-controller-events.txt): Controller events to better understand connection/circuit usage \[CLOSED\] +* [`219-expanded-dns.txt`](/proposals/219-expanded-dns.txt): Support for full DNS and DNSSEC resolution in Tor \[NEEDS-REVISION\] +* [`220-ecc-id-keys.txt`](/proposals/220-ecc-id-keys.txt): Migrate server identity keys to Ed25519 \[CLOSED\] +* [`221-stop-using-create-fast.txt`](/proposals/221-stop-using-create-fast.txt): Stop using CREATE_FAST \[CLOSED\] +* [`222-remove-client-timestamps.txt`](/proposals/222-remove-client-timestamps.txt): Stop sending client timestamps \[CLOSED\] +* [`223-ace-handshake.txt`](/proposals/223-ace-handshake.txt): Ace: Improved circuit-creation key exchange \[RESERVE\] +* [`224-rend-spec-ng.txt`](/proposals/224-rend-spec-ng.txt): Next-Generation Hidden Services in Tor \[CLOSED\] +* [`225-strawman-shared-rand.txt`](/proposals/225-strawman-shared-rand.txt): Strawman proposal: commit-and-reveal shared rng \[SUPERSEDED\] +* [`226-bridgedb-database-improvements.txt`](/proposals/226-bridgedb-database-improvements.txt): "Scalability and Stability Improvements to BridgeDB: Switching to a Distributed Database System and RDBMS" \[RESERVE\] +* [`227-vote-on-package-fingerprints.txt`](/proposals/227-vote-on-package-fingerprints.txt): Include package fingerprints in consensus documents \[CLOSED\] +* [`228-cross-certification-onionkeys.txt`](/proposals/228-cross-certification-onionkeys.txt): Cross-certifying identity keys with onion keys \[CLOSED\] +* [`229-further-socks5-extensions.txt`](/proposals/229-further-socks5-extensions.txt): Further SOCKS5 extensions \[REJECTED\] +* [`230-rsa1024-relay-id-migration.txt`](/proposals/230-rsa1024-relay-id-migration.txt): How to change RSA1024 relay identity keys \[OBSOLETE\] +* [`231-migrate-authority-rsa1024-ids.txt`](/proposals/231-migrate-authority-rsa1024-ids.txt): Migrating authority RSA1024 identity keys \[OBSOLETE\] +* [`232-pluggable-transports-through-proxy.txt`](/proposals/232-pluggable-transports-through-proxy.txt): Pluggable Transport through SOCKS proxy \[CLOSED\] +* [`233-quicken-tor2web-mode.txt`](/proposals/233-quicken-tor2web-mode.txt): Making Tor2Web mode faster \[REJECTED\] +* [`234-remittance-addresses.txt`](/proposals/234-remittance-addresses.txt): Adding remittance field to directory specification \[REJECTED\] +* [`235-kill-named-flag.txt`](/proposals/235-kill-named-flag.txt): Stop assigning (and eventually supporting) the Named flag \[CLOSED\] +* [`236-single-guard-node.txt`](/proposals/236-single-guard-node.txt): The move to a single guard node \[CLOSED\] +* [`237-directory-servers-for-all.txt`](/proposals/237-directory-servers-for-all.txt): All relays are directory servers \[CLOSED\] +* [`238-hs-relay-stats.txt`](/proposals/238-hs-relay-stats.txt): Better hidden service stats from Tor relays \[CLOSED\] +* [`239-consensus-hash-chaining.txt`](/proposals/239-consensus-hash-chaining.txt): Consensus Hash Chaining \[OPEN\] +* [`240-auth-cert-revocation.txt`](/proposals/240-auth-cert-revocation.txt): Early signing key revocation for directory authorities \[OPEN\] +* [`241-suspicious-guard-turnover.txt`](/proposals/241-suspicious-guard-turnover.txt): Resisting guard-turnover attacks \[REJECTED\] +* [`242-better-families.txt`](/proposals/242-better-families.txt): Better performance and usability for the MyFamily option \[SUPERSEDED\] +* [`243-hsdir-flag-need-stable.txt`](/proposals/243-hsdir-flag-need-stable.txt): Give out HSDir flag only to relays with Stable flag \[CLOSED\] +* [`244-use-rfc5705-for-tls-binding.txt`](/proposals/244-use-rfc5705-for-tls-binding.txt): Use RFC5705 Key Exporting in our AUTHENTICATE calls \[CLOSED\] +* [`245-tap-out.txt`](/proposals/245-tap-out.txt): Deprecating and removing the TAP circuit extension protocol \[NEEDS-REVISION\] +* [`246-merge-hsdir-and-intro.txt`](/proposals/246-merge-hsdir-and-intro.txt): Merging Hidden Service Directories and Introduction Points \[REJECTED\] +* [`247-hs-guard-discovery.txt`](/proposals/247-hs-guard-discovery.txt): Defending Against Guard Discovery Attacks using Vanguards \[SUPERSEDED\] +* [`248-removing-rsa-identities.txt`](/proposals/248-removing-rsa-identities.txt): Remove all RSA identity keys \[NEEDS-REVISION\] +* [`249-large-create-cells.txt`](/proposals/249-large-create-cells.txt): Allow CREATE cells with >505 bytes of handshake data \[SUPERSEDED\] +* [`250-commit-reveal-consensus.txt`](/proposals/250-commit-reveal-consensus.txt): Random Number Generation During Tor Voting \[CLOSED\] +* [`251-netflow-padding.txt`](/proposals/251-netflow-padding.txt): Padding for netflow record resolution reduction \[CLOSED\] +* [`252-single-onion.txt`](/proposals/252-single-onion.txt): Single Onion Services \[SUPERSEDED\] +* [`253-oob-hmac.txt`](/proposals/253-oob-hmac.txt): Out of Band Circuit HMACs \[DEAD\] +* [`254-padding-negotiation.txt`](/proposals/254-padding-negotiation.txt): Padding Negotiation \[CLOSED\] +* [`255-hs-load-balancing.txt`](/proposals/255-hs-load-balancing.txt): Controller features to allow for load-balancing hidden services \[RESERVE\] +* [`256-key-revocation.txt`](/proposals/256-key-revocation.txt): Key revocation for relays and authorities \[RESERVE\] +* [`257-hiding-authorities.txt`](/proposals/257-hiding-authorities.txt): Refactoring authorities and making them more isolated from the net \[META\] +* [`258-dirauth-dos.txt`](/proposals/258-dirauth-dos.txt): Denial-of-service resistance for directory authorities \[DEAD\] +* [`259-guard-selection.txt`](/proposals/259-guard-selection.txt): New Guard Selection Behaviour \[OBSOLETE\] +* [`260-rend-single-onion.txt`](/proposals/260-rend-single-onion.txt): Rendezvous Single Onion Services \[FINISHED\] +* [`261-aez-crypto.txt`](/proposals/261-aez-crypto.txt): AEZ for relay cryptography \[OBSOLETE\] +* [`262-rekey-circuits.txt`](/proposals/262-rekey-circuits.txt): Re-keying live circuits with new cryptographic material \[RESERVE\] +* [`263-ntru-for-pq-handshake.txt`](/proposals/263-ntru-for-pq-handshake.txt): Request to change key exchange protocol for handshake v1.2 \[OBSOLETE\] +* [`264-subprotocol-versions.txt`](/proposals/264-subprotocol-versions.txt): Putting version numbers on the Tor subprotocols \[CLOSED\] +* [`265-load-balancing-with-overhead.txt`](/proposals/265-load-balancing-with-overhead.txt): Load Balancing with Overhead Parameters \[OPEN\] +* [`266-removing-current-obsolete-clients.txt`](/proposals/266-removing-current-obsolete-clients.txt): Removing current obsolete clients from the Tor network \[SUPERSEDED\] +* [`267-tor-consensus-transparency.txt`](/proposals/267-tor-consensus-transparency.txt): Tor Consensus Transparency \[OPEN\] +* [`268-guard-selection.txt`](/proposals/268-guard-selection.txt): New Guard Selection Behaviour \[OBSOLETE\] +* [`269-hybrid-handshake.txt`](/proposals/269-hybrid-handshake.txt): Transitionally secure hybrid handshakes \[NEEDS-REVISION\] +* [`270-newhope-hybrid-handshake.txt`](/proposals/270-newhope-hybrid-handshake.txt): RebelAlliance: A Post-Quantum Secure Hybrid Handshake Based on NewHope \[OBSOLETE\] +* [`271-another-guard-selection.txt`](/proposals/271-another-guard-selection.txt): Another algorithm for guard selection \[CLOSED\] +* [`272-valid-and-running-by-default.txt`](/proposals/272-valid-and-running-by-default.txt): Listed routers should be Valid, Running, and treated as such \[CLOSED\] +* [`273-exit-relay-pinning.txt`](/proposals/273-exit-relay-pinning.txt): Exit relay pinning for web services \[RESERVE\] +* [`274-rotate-onion-keys-less.txt`](/proposals/274-rotate-onion-keys-less.txt): Rotate onion keys less frequently \[CLOSED\] +* [`275-md-published-time-is-silly.txt`](/proposals/275-md-published-time-is-silly.txt): Stop including meaningful "published" time in microdescriptor consensus \[CLOSED\] +* [`276-lower-bw-granularity.txt`](/proposals/276-lower-bw-granularity.txt): Report bandwidth with lower granularity in consensus documents \[DEAD\] +* [`277-detect-id-sharing.txt`](/proposals/277-detect-id-sharing.txt): Detect multiple relay instances running with same ID \[OPEN\] +* [`278-directory-compression-scheme-negotiation.txt`](/proposals/278-directory-compression-scheme-negotiation.txt): Directory Compression Scheme Negotiation \[CLOSED\] +* [`279-naming-layer-api.txt`](/proposals/279-naming-layer-api.txt): A Name System API for Tor Onion Services \[NEEDS-REVISION\] +* [`280-privcount-in-tor.txt`](/proposals/280-privcount-in-tor.txt): Privacy-Preserving Statistics with Privcount in Tor \[SUPERSEDED\] +* [`281-bulk-md-download.txt`](/proposals/281-bulk-md-download.txt): Downloading microdescriptors in bulk \[RESERVE\] +* [`282-remove-named-from-consensus.txt`](/proposals/282-remove-named-from-consensus.txt): Remove "Named" and "Unnamed" handling from consensus voting \[ACCEPTED\] +* [`283-ipv6-in-micro-consensus.txt`](/proposals/283-ipv6-in-micro-consensus.txt): Move IPv6 ORPorts from microdescriptors to the microdesc consensus \[CLOSED\] +* [`284-hsv3-control-port.txt`](/proposals/284-hsv3-control-port.txt): Hidden Service v3 Control Port \[CLOSED\] +* [`285-utf-8.txt`](/proposals/285-utf-8.txt): Directory documents should be standardized as UTF-8 \[ACCEPTED\] +* [`286-hibernation-api.txt`](/proposals/286-hibernation-api.txt): Controller APIs for hibernation access on mobile \[REJECTED\] +* [`287-reduce-lifetime.txt`](/proposals/287-reduce-lifetime.txt): Reduce circuit lifetime without overloading the network \[OPEN\] +* [`288-privcount-with-shamir.txt`](/proposals/288-privcount-with-shamir.txt): Privacy-Preserving Statistics with Privcount in Tor (Shamir version) \[RESERVE\] +* [`289-authenticated-sendmes.txt`](/proposals/289-authenticated-sendmes.txt): Authenticating sendme cells to mitigate bandwidth attacks \[CLOSED\] +* [`290-deprecate-consensus-methods.txt`](/proposals/290-deprecate-consensus-methods.txt): Continuously update consensus methods \[META\] +* [`291-two-guard-nodes.txt`](/proposals/291-two-guard-nodes.txt): The move to two guard nodes \[FINISHED\] +* [`292-mesh-vanguards.txt`](/proposals/292-mesh-vanguards.txt): Mesh-based vanguards \[CLOSED\] +* [`293-know-when-to-publish.txt`](/proposals/293-know-when-to-publish.txt): Other ways for relays to know when to publish \[CLOSED\] +* [`294-tls-1.3.txt`](/proposals/294-tls-1.3.txt): TLS 1.3 Migration \[DRAFT\] +* [`295-relay-crypto-with-adl.txt`](/proposals/295-relay-crypto-with-adl.txt): Using ADL for relay cryptography (solving the crypto-tagging attack) \[OPEN\] +* [`296-expose-bandwidth-files.txt`](/proposals/296-expose-bandwidth-files.txt): Have Directory Authorities expose raw bandwidth list files \[CLOSED\] +* [`297-safer-protover-shutdowns.txt`](/proposals/297-safer-protover-shutdowns.txt): Relaxing the protover-based shutdown rules \[CLOSED\] +* [`298-canonical-families.txt`](/proposals/298-canonical-families.txt): Putting family lines in canonical form \[CLOSED\] +* [`299-ip-failure-count.txt`](/proposals/299-ip-failure-count.txt): Preferring IPv4 or IPv6 based on IP Version Failure Count \[SUPERSEDED\] +* [`300-walking-onions.txt`](/proposals/300-walking-onions.txt): Walking Onions: Scaling and Saving Bandwidth \[INFORMATIONAL\] +* [`301-dont-vote-on-package-fingerprints.txt`](/proposals/301-dont-vote-on-package-fingerprints.txt): Don't include package fingerprints in consensus documents \[CLOSED\] +* [`302-padding-machines-for-onion-clients.txt`](/proposals/302-padding-machines-for-onion-clients.txt): Hiding onion service clients using padding \[CLOSED\] +* [`303-protover-removal-policy.txt`](/proposals/303-protover-removal-policy.txt): When and how to remove support for protocol versions \[OPEN\] +* [`304-socks5-extending-hs-error-codes.txt`](/proposals/304-socks5-extending-hs-error-codes.txt): Extending SOCKS5 Onion Service Error Codes \[CLOSED\] +* [`305-establish-intro-dos-defense-extention.txt`](/proposals/305-establish-intro-dos-defense-extention.txt): ESTABLISH_INTRO Cell DoS Defense Extension \[CLOSED\] +* [`306-ipv6-happy-eyeballs.txt`](/proposals/306-ipv6-happy-eyeballs.txt): A Tor Implementation of IPv6 Happy Eyeballs \[OPEN\] +* [`307-onionbalance-v3.txt`](/proposals/307-onionbalance-v3.txt): Onion Balance Support for Onion Service v3 \[RESERVE\] +* [`308-counter-galois-onion.txt`](/proposals/308-counter-galois-onion.txt): Counter Galois Onion: A New Proposal for Forward-Secure Relay Cryptography \[SUPERSEDED\] +* [`309-optimistic-socks-in-tor.txt`](/proposals/309-optimistic-socks-in-tor.txt): Optimistic SOCKS Data \[OPEN\] +* [`310-bandaid-on-guard-selection.txt`](/proposals/310-bandaid-on-guard-selection.txt): Towards load-balancing in Prop 271 \[CLOSED\] +* [`311-relay-ipv6-reachability.txt`](/proposals/311-relay-ipv6-reachability.txt): Tor Relay IPv6 Reachability \[ACCEPTED\] +* [`312-relay-auto-ipv6-addr.txt`](/proposals/312-relay-auto-ipv6-addr.txt): Tor Relay Automatic IPv6 Address Discovery \[ACCEPTED\] +* [`313-relay-ipv6-stats.txt`](/proposals/313-relay-ipv6-stats.txt): Tor Relay IPv6 Statistics \[ACCEPTED\] +* [`314-allow-markdown-proposals.md`](/proposals/314-allow-markdown-proposals.md): Allow Markdown for proposal format \[CLOSED\] +* [`315-update-dir-required-fields.txt`](/proposals/315-update-dir-required-fields.txt): Updating the list of fields required in directory documents \[CLOSED\] +* [`316-flashflow.md`](/proposals/316-flashflow.md): FlashFlow: A Secure Speed Test for Tor (Parent Proposal) \[DRAFT\] +* [`317-secure-dns-name-resolution.txt`](/proposals/317-secure-dns-name-resolution.txt): Improve security aspects of DNS name resolution \[NEEDS-REVISION\] +* [`318-limit-protovers.md`](/proposals/318-limit-protovers.md): Limit protover values to 0-63 \[CLOSED\] +* [`319-wide-everything.md`](/proposals/319-wide-everything.md): RELAY_FRAGMENT cells \[OBSOLETE\] +* [`320-tap-out-again.md`](/proposals/320-tap-out-again.md): Removing TAP usage from v2 onion services \[REJECTED\] +* [`321-happy-families.md`](/proposals/321-happy-families.md): Better performance and usability for the MyFamily option (v2) \[ACCEPTED\] +* [`322-dirport-linkspec.md`](/proposals/322-dirport-linkspec.md): Extending link specifiers to include the directory port \[OPEN\] +* [`323-walking-onions-full.md`](/proposals/323-walking-onions-full.md): Specification for Walking Onions \[OPEN\] +* [`324-rtt-congestion-control.txt`](/proposals/324-rtt-congestion-control.txt): RTT-based Congestion Control for Tor \[FINISHED\] +* [`325-packed-relay-cells.md`](/proposals/325-packed-relay-cells.md): Packed relay cells: saving space on small commands \[OBSOLETE\] +* [`326-tor-relay-well-known-uri-rfc8615.md`](/proposals/326-tor-relay-well-known-uri-rfc8615.md): The "tor-relay" Well-Known Resource Identifier \[OPEN\] +* [`327-pow-over-intro.txt`](/proposals/327-pow-over-intro.txt): A First Take at PoW Over Introduction Circuits \[CLOSED\] +* [`328-relay-overload-report.md`](/proposals/328-relay-overload-report.md): Make Relays Report When They Are Overloaded \[CLOSED\] +* [`329-traffic-splitting.txt`](/proposals/329-traffic-splitting.txt): Overcoming Tor's Bottlenecks with Traffic Splitting \[FINISHED\] +* [`330-authority-contact.md`](/proposals/330-authority-contact.md): Modernizing authority contact entries \[OPEN\] +* [`331-res-tokens-for-anti-dos.md`](/proposals/331-res-tokens-for-anti-dos.md): Res tokens: Anonymous Credentials for Onion Service DoS Resilience \[DRAFT\] +* [`332-ntor-v3-with-extra-data.md`](/proposals/332-ntor-v3-with-extra-data.md): Ntor protocol with extra data, version 3 \[CLOSED\] +* [`333-vanguards-lite.md`](/proposals/333-vanguards-lite.md): Vanguards lite \[CLOSED\] +* [`334-middle-only-flag.txt`](/proposals/334-middle-only-flag.txt): A Directory Authority Flag To Mark Relays As Middle-only \[SUPERSEDED\] +* [`335-middle-only-redux.md`](/proposals/335-middle-only-redux.md): An authority-only design for MiddleOnly \[CLOSED\] +* [`336-randomize-guard-retries.md`](/proposals/336-randomize-guard-retries.md): Randomized schedule for guard retries \[CLOSED\] +* [`337-simpler-guard-usability.md`](/proposals/337-simpler-guard-usability.md): A simpler way to decide, "Is this guard usable?" \[CLOSED\] +* [`338-netinfo-y2038.md`](/proposals/338-netinfo-y2038.md): Use an 8-byte timestamp in NETINFO cells \[ACCEPTED\] +* [`339-udp-over-tor.md`](/proposals/339-udp-over-tor.md): UDP traffic over Tor \[ACCEPTED\] +* [`340-packed-and-fragmented.md`](/proposals/340-packed-and-fragmented.md): Packed and fragmented relay messages \[OPEN\] +* [`341-better-oos.md`](/proposals/341-better-oos.md): A better algorithm for out-of-sockets eviction \[OPEN\] +* [`342-decouple-hs-interval.md`](/proposals/342-decouple-hs-interval.md): Decoupling hs_interval and SRV lifetime \[DRAFT\] +* [`343-rend-caa.txt`](/proposals/343-rend-caa.txt): CAA Extensions for the Tor Rendezvous Specification \[OPEN\] +* [`344-protocol-info-leaks.txt`](/proposals/344-protocol-info-leaks.txt): Prioritizing Protocol Information Leaks in Tor \[OPEN\] +* [`345-specs-in-mdbook.md`](/proposals/345-specs-in-mdbook.md): Migrating the tor specifications to mdbook \[CLOSED\] +* [`346-protovers-again.md`](/proposals/346-protovers-again.md): Clarifying and extending the use of protocol versioning \[OPEN\] +* [`347-domain-separation.md`](/proposals/347-domain-separation.md): Domain separation for certificate signing keys \[OPEN\] +* [`348-udp-app-support.md`](/proposals/348-udp-app-support.md): UDP Application Support in Tor \[OPEN\] diff --git a/proposals/BY_INDEX_template.md b/proposals/BY_INDEX_template.md index d18dc60..2c8d425 100644 --- a/proposals/BY_INDEX_template.md +++ b/proposals/BY_INDEX_template.md @@ -11,6 +11,6 @@ of these proposals are implemented; some are works in progress; and some will never be implemented. Below are a list of proposals sorted by their proposal number. See -[README.md](/proposals/README.md) for a list of proposals sorted by status. +[BY_STATUS.md](/proposals/BY_STATUS.md) for a list of proposals sorted by status. {BY_INDEX} diff --git a/proposals/BY_STATUS.md b/proposals/BY_STATUS.md new file mode 100644 index 0000000..0c265c2 --- /dev/null +++ b/proposals/BY_STATUS.md @@ -0,0 +1,370 @@ +# Tor proposals by status + +<!-- DO NOT EDIT THIS FILE --> + +<!-- If you want to make changes here, make sure to edit the --> +<!-- README_template.md file. The README.md file is generated by the --> +<!-- reindex.py script. --> + +Here we have a set of proposals for changes to the Tor protocol. Some +of these proposals are implemented; some are works in progress; and some +will never be implemented. + +Below are a list of proposals sorted by status. See +[BY_INDEX.md](/proposals/BY_INDEX.md) for a list of proposals sorted by number. + +# Active proposals by status + +## OPEN proposals: under discussion + +These are proposals that we think are likely to be complete, and ripe +for discussion. + +* [`239-consensus-hash-chaining.txt`](/proposals/239-consensus-hash-chaining.txt): Consensus Hash Chaining +* [`240-auth-cert-revocation.txt`](/proposals/240-auth-cert-revocation.txt): Early signing key revocation for directory authorities +* [`265-load-balancing-with-overhead.txt`](/proposals/265-load-balancing-with-overhead.txt): Load Balancing with Overhead Parameters +* [`267-tor-consensus-transparency.txt`](/proposals/267-tor-consensus-transparency.txt): Tor Consensus Transparency +* [`277-detect-id-sharing.txt`](/proposals/277-detect-id-sharing.txt): Detect multiple relay instances running with same ID +* [`287-reduce-lifetime.txt`](/proposals/287-reduce-lifetime.txt): Reduce circuit lifetime without overloading the network +* [`295-relay-crypto-with-adl.txt`](/proposals/295-relay-crypto-with-adl.txt): Using ADL for relay cryptography (solving the crypto-tagging attack) +* [`303-protover-removal-policy.txt`](/proposals/303-protover-removal-policy.txt): When and how to remove support for protocol versions +* [`306-ipv6-happy-eyeballs.txt`](/proposals/306-ipv6-happy-eyeballs.txt): A Tor Implementation of IPv6 Happy Eyeballs +* [`309-optimistic-socks-in-tor.txt`](/proposals/309-optimistic-socks-in-tor.txt): Optimistic SOCKS Data +* [`322-dirport-linkspec.md`](/proposals/322-dirport-linkspec.md): Extending link specifiers to include the directory port +* [`323-walking-onions-full.md`](/proposals/323-walking-onions-full.md): Specification for Walking Onions +* [`326-tor-relay-well-known-uri-rfc8615.md`](/proposals/326-tor-relay-well-known-uri-rfc8615.md): The "tor-relay" Well-Known Resource Identifier +* [`330-authority-contact.md`](/proposals/330-authority-contact.md): Modernizing authority contact entries +* [`340-packed-and-fragmented.md`](/proposals/340-packed-and-fragmented.md): Packed and fragmented relay messages +* [`341-better-oos.md`](/proposals/341-better-oos.md): A better algorithm for out-of-sockets eviction +* [`343-rend-caa.txt`](/proposals/343-rend-caa.txt): CAA Extensions for the Tor Rendezvous Specification +* [`344-protocol-info-leaks.txt`](/proposals/344-protocol-info-leaks.txt): Prioritizing Protocol Information Leaks in Tor +* [`346-protovers-again.md`](/proposals/346-protovers-again.md): Clarifying and extending the use of protocol versioning +* [`347-domain-separation.md`](/proposals/347-domain-separation.md): Domain separation for certificate signing keys +* [`348-udp-app-support.md`](/proposals/348-udp-app-support.md): UDP Application Support in Tor + + +## ACCEPTED proposals: slated for implementation + +These are the proposals that we agree we'd like to implement. They +might or might not have a specific timeframe planned for their +implementation. + +* [`282-remove-named-from-consensus.txt`](/proposals/282-remove-named-from-consensus.txt): Remove "Named" and "Unnamed" handling from consensus voting +* [`285-utf-8.txt`](/proposals/285-utf-8.txt): Directory documents should be standardized as UTF-8 +* [`311-relay-ipv6-reachability.txt`](/proposals/311-relay-ipv6-reachability.txt): Tor Relay IPv6 Reachability +* [`312-relay-auto-ipv6-addr.txt`](/proposals/312-relay-auto-ipv6-addr.txt): Tor Relay Automatic IPv6 Address Discovery +* [`313-relay-ipv6-stats.txt`](/proposals/313-relay-ipv6-stats.txt): Tor Relay IPv6 Statistics +* [`321-happy-families.md`](/proposals/321-happy-families.md): Better performance and usability for the MyFamily option (v2) +* [`338-netinfo-y2038.md`](/proposals/338-netinfo-y2038.md): Use an 8-byte timestamp in NETINFO cells +* [`339-udp-over-tor.md`](/proposals/339-udp-over-tor.md): UDP traffic over Tor + + +## FINISHED proposals: implemented, specs not merged + +These proposals are implemented in some version of Tor; the proposals +themselves still need to be merged into the specifications proper. + +* [`260-rend-single-onion.txt`](/proposals/260-rend-single-onion.txt): Rendezvous Single Onion Services +* [`291-two-guard-nodes.txt`](/proposals/291-two-guard-nodes.txt): The move to two guard nodes +* [`324-rtt-congestion-control.txt`](/proposals/324-rtt-congestion-control.txt): RTT-based Congestion Control for Tor +* [`329-traffic-splitting.txt`](/proposals/329-traffic-splitting.txt): Overcoming Tor's Bottlenecks with Traffic Splitting + + +## META proposals: about the proposal process + +These proposals describe ongoing policies and changes to the proposals +process. + +* [`000-index.txt`](/proposals/000-index.txt): Index of Tor Proposals +* [`001-process.txt`](/proposals/001-process.txt): The Tor Proposal Process +* [`202-improved-relay-crypto.txt`](/proposals/202-improved-relay-crypto.txt): Two improved relay encryption protocols for Tor cells +* [`257-hiding-authorities.txt`](/proposals/257-hiding-authorities.txt): Refactoring authorities and making them more isolated from the net +* [`290-deprecate-consensus-methods.txt`](/proposals/290-deprecate-consensus-methods.txt): Continuously update consensus methods + + +## INFORMATIONAL proposals: not actually specifications + +These proposals describe a process or project, but aren't actually +proposed changes in the Tor specifications. + +* [`159-exit-scanning.txt`](/proposals/159-exit-scanning.txt): Exit Scanning +* [`300-walking-onions.txt`](/proposals/300-walking-onions.txt): Walking Onions: Scaling and Saving Bandwidth + + +# Preliminary proposals + +## DRAFT proposals: incomplete works + +These proposals have been marked as a draft by their author or the editors, +indicating that they aren't yet in a complete form. They're still open for +discussion. + +* [`294-tls-1.3.txt`](/proposals/294-tls-1.3.txt): TLS 1.3 Migration +* [`316-flashflow.md`](/proposals/316-flashflow.md): FlashFlow: A Secure Speed Test for Tor (Parent Proposal) +* [`331-res-tokens-for-anti-dos.md`](/proposals/331-res-tokens-for-anti-dos.md): Res tokens: Anonymous Credentials for Onion Service DoS Resilience +* [`342-decouple-hs-interval.md`](/proposals/342-decouple-hs-interval.md): Decoupling hs_interval and SRV lifetime + + +## NEEDS-REVISION proposals: ideas that we can't implement as-is + +These proposals have some promise, but we can't implement them without +certain changes. + +* [`212-using-old-consensus.txt`](/proposals/212-using-old-consensus.txt): Increase Acceptable Consensus Age +* [`219-expanded-dns.txt`](/proposals/219-expanded-dns.txt): Support for full DNS and DNSSEC resolution in Tor +* [`245-tap-out.txt`](/proposals/245-tap-out.txt): Deprecating and removing the TAP circuit extension protocol +* [`248-removing-rsa-identities.txt`](/proposals/248-removing-rsa-identities.txt): Remove all RSA identity keys +* [`269-hybrid-handshake.txt`](/proposals/269-hybrid-handshake.txt): Transitionally secure hybrid handshakes +* [`279-naming-layer-api.txt`](/proposals/279-naming-layer-api.txt): A Name System API for Tor Onion Services +* [`317-secure-dns-name-resolution.txt`](/proposals/317-secure-dns-name-resolution.txt): Improve security aspects of DNS name resolution + + +## NEEDS-RESEARCH proposals: blocking on research + +These proposals are interesting ideas, but there's more research that would +need to happen before we can know whether to implement them or not, or to +fill in certain details. + +(There are no proposals in this category) + + +# Inactive proposals by status + +## CLOSED proposals: implemented and specified + +These proposals have been implemented in some version of Tor, and the +changes from the proposals have been merged into the specifications as +necessary. + +* [`101-dir-voting.txt`](/proposals/101-dir-voting.txt): Voting on the Tor Directory System +* [`102-drop-opt.txt`](/proposals/102-drop-opt.txt): Dropping "opt" from the directory format +* [`103-multilevel-keys.txt`](/proposals/103-multilevel-keys.txt): Splitting identity key from regularly used signing key +* [`104-short-descriptors.txt`](/proposals/104-short-descriptors.txt): Long and Short Router Descriptors +* [`105-handshake-revision.txt`](/proposals/105-handshake-revision.txt): Version negotiation for the Tor protocol +* [`106-less-tls-constraint.txt`](/proposals/106-less-tls-constraint.txt): Checking fewer things during TLS handshakes +* [`107-uptime-sanity-checking.txt`](/proposals/107-uptime-sanity-checking.txt): Uptime Sanity Checking +* [`108-mtbf-based-stability.txt`](/proposals/108-mtbf-based-stability.txt): Base "Stable" Flag on Mean Time Between Failures +* [`109-no-sharing-ips.txt`](/proposals/109-no-sharing-ips.txt): No more than one server per IP address +* [`110-avoid-infinite-circuits.txt`](/proposals/110-avoid-infinite-circuits.txt): Avoiding infinite length circuits +* [`111-local-traffic-priority.txt`](/proposals/111-local-traffic-priority.txt): Prioritizing local traffic over relayed traffic +* [`114-distributed-storage.txt`](/proposals/114-distributed-storage.txt): Distributed Storage for Tor Hidden Service Descriptors +* [`117-ipv6-exits.txt`](/proposals/117-ipv6-exits.txt): IPv6 exits +* [`119-controlport-auth.txt`](/proposals/119-controlport-auth.txt): New PROTOCOLINFO command for controllers +* [`121-hidden-service-authentication.txt`](/proposals/121-hidden-service-authentication.txt): Hidden Service Authentication +* [`122-unnamed-flag.txt`](/proposals/122-unnamed-flag.txt): Network status entries need a new Unnamed flag +* [`123-autonaming.txt`](/proposals/123-autonaming.txt): Naming authorities automatically create bindings +* [`125-bridges.txt`](/proposals/125-bridges.txt): Behavior for bridge users, bridge relays, and bridge authorities +* [`126-geoip-reporting.txt`](/proposals/126-geoip-reporting.txt): Getting GeoIP data and publishing usage summaries +* [`129-reject-plaintext-ports.txt`](/proposals/129-reject-plaintext-ports.txt): Block Insecure Protocols by Default +* [`130-v2-conn-protocol.txt`](/proposals/130-v2-conn-protocol.txt): Version 2 Tor connection protocol +* [`135-private-tor-networks.txt`](/proposals/135-private-tor-networks.txt): Simplify Configuration of Private Tor Networks +* [`136-legacy-keys.txt`](/proposals/136-legacy-keys.txt): Mass authority migration with legacy keys +* [`137-bootstrap-phases.txt`](/proposals/137-bootstrap-phases.txt): Keep controllers informed as Tor bootstraps +* [`138-remove-down-routers-from-consensus.txt`](/proposals/138-remove-down-routers-from-consensus.txt): Remove routers that are not Running from consensus documents +* [`139-conditional-consensus-download.txt`](/proposals/139-conditional-consensus-download.txt): Download consensus documents only when it will be trusted +* [`140-consensus-diffs.txt`](/proposals/140-consensus-diffs.txt): Provide diffs between consensuses +* [`148-uniform-client-end-reason.txt`](/proposals/148-uniform-client-end-reason.txt): Stream end reasons from the client side should be uniform +* [`150-exclude-exit-nodes.txt`](/proposals/150-exclude-exit-nodes.txt): Exclude Exit Nodes from a circuit +* [`151-path-selection-improvements.txt`](/proposals/151-path-selection-improvements.txt): Improving Tor Path Selection +* [`152-single-hop-circuits.txt`](/proposals/152-single-hop-circuits.txt): Optionally allow exit from single-hop circuits +* [`155-four-hidden-service-improvements.txt`](/proposals/155-four-hidden-service-improvements.txt): Four Improvements of Hidden Service Performance +* [`157-specific-cert-download.txt`](/proposals/157-specific-cert-download.txt): Make certificate downloads specific +* [`158-microdescriptors.txt`](/proposals/158-microdescriptors.txt): Clients download consensus + microdescriptors +* [`160-bandwidth-offset.txt`](/proposals/160-bandwidth-offset.txt): Authorities vote for bandwidth offsets in consensus +* [`161-computing-bandwidth-adjustments.txt`](/proposals/161-computing-bandwidth-adjustments.txt): Computing Bandwidth Adjustments +* [`162-consensus-flavors.txt`](/proposals/162-consensus-flavors.txt): Publish the consensus in multiple flavors +* [`166-statistics-extra-info-docs.txt`](/proposals/166-statistics-extra-info-docs.txt): Including Network Statistics in Extra-Info Documents +* [`167-params-in-consensus.txt`](/proposals/167-params-in-consensus.txt): Vote on network parameters in consensus +* [`171-separate-streams.txt`](/proposals/171-separate-streams.txt): Separate streams across circuits by connection metadata +* [`174-optimistic-data-server.txt`](/proposals/174-optimistic-data-server.txt): Optimistic Data for Tor: Server Side +* [`176-revising-handshake.txt`](/proposals/176-revising-handshake.txt): Proposed version-3 link handshake for Tor +* [`178-param-voting.txt`](/proposals/178-param-voting.txt): Require majority of authorities to vote for consensus parameters +* [`179-TLS-cert-and-parameter-normalization.txt`](/proposals/179-TLS-cert-and-parameter-normalization.txt): TLS certificate and parameter normalization +* [`180-pluggable-transport.txt`](/proposals/180-pluggable-transport.txt): Pluggable transports for circumvention +* [`181-optimistic-data-client.txt`](/proposals/181-optimistic-data-client.txt): Optimistic Data for Tor: Client Side +* [`183-refillintervals.txt`](/proposals/183-refillintervals.txt): Refill Intervals +* [`184-v3-link-protocol.txt`](/proposals/184-v3-link-protocol.txt): Miscellaneous changes for a v3 Tor link protocol +* [`186-multiple-orports.txt`](/proposals/186-multiple-orports.txt): Multiple addresses for one OR or bridge +* [`187-allow-client-auth.txt`](/proposals/187-allow-client-auth.txt): Reserve a cell type to allow client authorization +* [`193-safe-cookie-authentication.txt`](/proposals/193-safe-cookie-authentication.txt): Safe cookie authentication for Tor controllers +* [`196-transport-control-ports.txt`](/proposals/196-transport-control-ports.txt): Extended ORPort and TransportControlPort +* [`198-restore-clienthello-semantics.txt`](/proposals/198-restore-clienthello-semantics.txt): Restore semantics of TLS ClientHello +* [`200-new-create-and-extend-cells.txt`](/proposals/200-new-create-and-extend-cells.txt): Adding new, extensible CREATE, EXTEND, and related cells +* [`204-hidserv-subdomains.txt`](/proposals/204-hidserv-subdomains.txt): Subdomain support for Hidden Service addresses +* [`205-local-dnscache.txt`](/proposals/205-local-dnscache.txt): Remove global client-side DNS caching +* [`206-directory-sources.txt`](/proposals/206-directory-sources.txt): Preconfigured directory sources for bootstrapping +* [`207-directory-guards.txt`](/proposals/207-directory-guards.txt): Directory guards +* [`208-ipv6-exits-redux.txt`](/proposals/208-ipv6-exits-redux.txt): IPv6 Exits Redux +* [`214-longer-circids.txt`](/proposals/214-longer-circids.txt): Allow 4-byte circuit IDs in a new link protocol +* [`215-update-min-consensus-ver.txt`](/proposals/215-update-min-consensus-ver.txt): Let the minimum consensus method change with time +* [`216-ntor-handshake.txt`](/proposals/216-ntor-handshake.txt): Improved circuit-creation key exchange +* [`217-ext-orport-auth.txt`](/proposals/217-ext-orport-auth.txt): Tor Extended ORPort Authentication +* [`218-usage-controller-events.txt`](/proposals/218-usage-controller-events.txt): Controller events to better understand connection/circuit usage +* [`220-ecc-id-keys.txt`](/proposals/220-ecc-id-keys.txt): Migrate server identity keys to Ed25519 +* [`221-stop-using-create-fast.txt`](/proposals/221-stop-using-create-fast.txt): Stop using CREATE_FAST +* [`222-remove-client-timestamps.txt`](/proposals/222-remove-client-timestamps.txt): Stop sending client timestamps +* [`224-rend-spec-ng.txt`](/proposals/224-rend-spec-ng.txt): Next-Generation Hidden Services in Tor +* [`227-vote-on-package-fingerprints.txt`](/proposals/227-vote-on-package-fingerprints.txt): Include package fingerprints in consensus documents +* [`228-cross-certification-onionkeys.txt`](/proposals/228-cross-certification-onionkeys.txt): Cross-certifying identity keys with onion keys +* [`232-pluggable-transports-through-proxy.txt`](/proposals/232-pluggable-transports-through-proxy.txt): Pluggable Transport through SOCKS proxy +* [`235-kill-named-flag.txt`](/proposals/235-kill-named-flag.txt): Stop assigning (and eventually supporting) the Named flag +* [`236-single-guard-node.txt`](/proposals/236-single-guard-node.txt): The move to a single guard node +* [`237-directory-servers-for-all.txt`](/proposals/237-directory-servers-for-all.txt): All relays are directory servers +* [`238-hs-relay-stats.txt`](/proposals/238-hs-relay-stats.txt): Better hidden service stats from Tor relays +* [`243-hsdir-flag-need-stable.txt`](/proposals/243-hsdir-flag-need-stable.txt): Give out HSDir flag only to relays with Stable flag +* [`244-use-rfc5705-for-tls-binding.txt`](/proposals/244-use-rfc5705-for-tls-binding.txt): Use RFC5705 Key Exporting in our AUTHENTICATE calls +* [`250-commit-reveal-consensus.txt`](/proposals/250-commit-reveal-consensus.txt): Random Number Generation During Tor Voting +* [`251-netflow-padding.txt`](/proposals/251-netflow-padding.txt): Padding for netflow record resolution reduction +* [`254-padding-negotiation.txt`](/proposals/254-padding-negotiation.txt): Padding Negotiation +* [`264-subprotocol-versions.txt`](/proposals/264-subprotocol-versions.txt): Putting version numbers on the Tor subprotocols +* [`271-another-guard-selection.txt`](/proposals/271-another-guard-selection.txt): Another algorithm for guard selection +* [`272-valid-and-running-by-default.txt`](/proposals/272-valid-and-running-by-default.txt): Listed routers should be Valid, Running, and treated as such +* [`274-rotate-onion-keys-less.txt`](/proposals/274-rotate-onion-keys-less.txt): Rotate onion keys less frequently +* [`275-md-published-time-is-silly.txt`](/proposals/275-md-published-time-is-silly.txt): Stop including meaningful "published" time in microdescriptor consensus +* [`278-directory-compression-scheme-negotiation.txt`](/proposals/278-directory-compression-scheme-negotiation.txt): Directory Compression Scheme Negotiation +* [`283-ipv6-in-micro-consensus.txt`](/proposals/283-ipv6-in-micro-consensus.txt): Move IPv6 ORPorts from microdescriptors to the microdesc consensus +* [`284-hsv3-control-port.txt`](/proposals/284-hsv3-control-port.txt): Hidden Service v3 Control Port +* [`289-authenticated-sendmes.txt`](/proposals/289-authenticated-sendmes.txt): Authenticating sendme cells to mitigate bandwidth attacks +* [`292-mesh-vanguards.txt`](/proposals/292-mesh-vanguards.txt): Mesh-based vanguards +* [`293-know-when-to-publish.txt`](/proposals/293-know-when-to-publish.txt): Other ways for relays to know when to publish +* [`296-expose-bandwidth-files.txt`](/proposals/296-expose-bandwidth-files.txt): Have Directory Authorities expose raw bandwidth list files +* [`297-safer-protover-shutdowns.txt`](/proposals/297-safer-protover-shutdowns.txt): Relaxing the protover-based shutdown rules +* [`298-canonical-families.txt`](/proposals/298-canonical-families.txt): Putting family lines in canonical form +* [`301-dont-vote-on-package-fingerprints.txt`](/proposals/301-dont-vote-on-package-fingerprints.txt): Don't include package fingerprints in consensus documents +* [`302-padding-machines-for-onion-clients.txt`](/proposals/302-padding-machines-for-onion-clients.txt): Hiding onion service clients using padding +* [`304-socks5-extending-hs-error-codes.txt`](/proposals/304-socks5-extending-hs-error-codes.txt): Extending SOCKS5 Onion Service Error Codes +* [`305-establish-intro-dos-defense-extention.txt`](/proposals/305-establish-intro-dos-defense-extention.txt): ESTABLISH_INTRO Cell DoS Defense Extension +* [`310-bandaid-on-guard-selection.txt`](/proposals/310-bandaid-on-guard-selection.txt): Towards load-balancing in Prop 271 +* [`314-allow-markdown-proposals.md`](/proposals/314-allow-markdown-proposals.md): Allow Markdown for proposal format +* [`315-update-dir-required-fields.txt`](/proposals/315-update-dir-required-fields.txt): Updating the list of fields required in directory documents +* [`318-limit-protovers.md`](/proposals/318-limit-protovers.md): Limit protover values to 0-63 +* [`327-pow-over-intro.txt`](/proposals/327-pow-over-intro.txt): A First Take at PoW Over Introduction Circuits +* [`328-relay-overload-report.md`](/proposals/328-relay-overload-report.md): Make Relays Report When They Are Overloaded +* [`332-ntor-v3-with-extra-data.md`](/proposals/332-ntor-v3-with-extra-data.md): Ntor protocol with extra data, version 3 +* [`333-vanguards-lite.md`](/proposals/333-vanguards-lite.md): Vanguards lite +* [`335-middle-only-redux.md`](/proposals/335-middle-only-redux.md): An authority-only design for MiddleOnly +* [`336-randomize-guard-retries.md`](/proposals/336-randomize-guard-retries.md): Randomized schedule for guard retries +* [`337-simpler-guard-usability.md`](/proposals/337-simpler-guard-usability.md): A simpler way to decide, "Is this guard usable?" +* [`345-specs-in-mdbook.md`](/proposals/345-specs-in-mdbook.md): Migrating the tor specifications to mdbook + + +## RESERVE proposals: saving for later + +These proposals aren't anything we plan to implement soon, but for one +reason or another we think they might be a good idea in the +future. We're keeping them around as a reference in case we someday +confront the problems that they try to solve. + +* [`133-unreachable-ors.txt`](/proposals/133-unreachable-ors.txt): Incorporate Unreachable ORs into the Tor Network +* [`172-circ-getinfo-option.txt`](/proposals/172-circ-getinfo-option.txt): GETINFO controller option for circuit information +* [`177-flag-abstention.txt`](/proposals/177-flag-abstention.txt): Abstaining from votes on individual flags +* [`188-bridge-guards.txt`](/proposals/188-bridge-guards.txt): Bridge Guards and other anti-enumeration defenses +* [`201-bridge-v3-reqs-stats.txt`](/proposals/201-bridge-v3-reqs-stats.txt): Make bridges report statistics on daily v3 network status requests +* [`211-mapaddress-tor-status.txt`](/proposals/211-mapaddress-tor-status.txt): Internal Mapaddress for Tor Configuration Testing +* [`223-ace-handshake.txt`](/proposals/223-ace-handshake.txt): Ace: Improved circuit-creation key exchange +* [`226-bridgedb-database-improvements.txt`](/proposals/226-bridgedb-database-improvements.txt): "Scalability and Stability Improvements to BridgeDB: Switching to a Distributed Database System and RDBMS" +* [`255-hs-load-balancing.txt`](/proposals/255-hs-load-balancing.txt): Controller features to allow for load-balancing hidden services +* [`256-key-revocation.txt`](/proposals/256-key-revocation.txt): Key revocation for relays and authorities +* [`262-rekey-circuits.txt`](/proposals/262-rekey-circuits.txt): Re-keying live circuits with new cryptographic material +* [`273-exit-relay-pinning.txt`](/proposals/273-exit-relay-pinning.txt): Exit relay pinning for web services +* [`281-bulk-md-download.txt`](/proposals/281-bulk-md-download.txt): Downloading microdescriptors in bulk +* [`288-privcount-with-shamir.txt`](/proposals/288-privcount-with-shamir.txt): Privacy-Preserving Statistics with Privcount in Tor (Shamir version) +* [`307-onionbalance-v3.txt`](/proposals/307-onionbalance-v3.txt): Onion Balance Support for Onion Service v3 + + +## SUPERSEDED proposals: replaced by something else + +These proposals were obsoleted by a later proposal before they were +implemented. + +* [`112-bring-back-pathlencoinweight.txt`](/proposals/112-bring-back-pathlencoinweight.txt): Bring Back Pathlen Coin Weight +* [`113-fast-authority-interface.txt`](/proposals/113-fast-authority-interface.txt): Simplifying directory authority administration +* [`118-multiple-orports.txt`](/proposals/118-multiple-orports.txt): Advertising multiple ORPorts at once +* [`124-tls-certificates.txt`](/proposals/124-tls-certificates.txt): Blocking resistant TLS certificate usage +* [`143-distributed-storage-improvements.txt`](/proposals/143-distributed-storage-improvements.txt): Improvements of Distributed Storage for Tor Hidden Service Descriptors +* [`145-newguard-flag.txt`](/proposals/145-newguard-flag.txt): Separate "suitable as a guard" from "suitable as a new guard" +* [`146-long-term-stability.txt`](/proposals/146-long-term-stability.txt): Add new flag to reflect long-term stability +* [`149-using-netinfo-data.txt`](/proposals/149-using-netinfo-data.txt): Using data from NETINFO cells +* [`153-automatic-software-update-protocol.txt`](/proposals/153-automatic-software-update-protocol.txt): Automatic software update protocol +* [`154-automatic-updates.txt`](/proposals/154-automatic-updates.txt): Automatic Software Update Protocol +* [`156-tracking-blocked-ports.txt`](/proposals/156-tracking-blocked-ports.txt): Tracking blocked ports on the client side +* [`163-detecting-clients.txt`](/proposals/163-detecting-clients.txt): Detecting whether a connection comes from a client +* [`169-eliminating-renegotiation.txt`](/proposals/169-eliminating-renegotiation.txt): Eliminate TLS renegotiation for the Tor connection handshake +* [`170-user-path-config.txt`](/proposals/170-user-path-config.txt): Configuration options regarding circuit building +* [`185-dir-without-dirport.txt`](/proposals/185-dir-without-dirport.txt): Directory caches without DirPort +* [`194-mnemonic-urls.txt`](/proposals/194-mnemonic-urls.txt): Mnemonic .onion URLs +* [`210-faster-headless-consensus-bootstrap.txt`](/proposals/210-faster-headless-consensus-bootstrap.txt): Faster Headless Consensus Bootstrapping +* [`225-strawman-shared-rand.txt`](/proposals/225-strawman-shared-rand.txt): Strawman proposal: commit-and-reveal shared rng +* [`242-better-families.txt`](/proposals/242-better-families.txt): Better performance and usability for the MyFamily option +* [`247-hs-guard-discovery.txt`](/proposals/247-hs-guard-discovery.txt): Defending Against Guard Discovery Attacks using Vanguards +* [`249-large-create-cells.txt`](/proposals/249-large-create-cells.txt): Allow CREATE cells with >505 bytes of handshake data +* [`252-single-onion.txt`](/proposals/252-single-onion.txt): Single Onion Services +* [`266-removing-current-obsolete-clients.txt`](/proposals/266-removing-current-obsolete-clients.txt): Removing current obsolete clients from the Tor network +* [`280-privcount-in-tor.txt`](/proposals/280-privcount-in-tor.txt): Privacy-Preserving Statistics with Privcount in Tor +* [`299-ip-failure-count.txt`](/proposals/299-ip-failure-count.txt): Preferring IPv4 or IPv6 based on IP Version Failure Count +* [`308-counter-galois-onion.txt`](/proposals/308-counter-galois-onion.txt): Counter Galois Onion: A New Proposal for Forward-Secure Relay Cryptography +* [`334-middle-only-flag.txt`](/proposals/334-middle-only-flag.txt): A Directory Authority Flag To Mark Relays As Middle-only + + +## DEAD, REJECTED, OBSOLETE proposals: not in our plans + +These proposals are not on-track for discussion or +implementation. Either discussion has stalled out (the proposal is +DEAD), the proposal has been considered and not adopted (the proposal is +REJECTED), or the proposal addresses an issue or a solution that is no +longer relevant (the proposal is OBSOLETE). + +* [`098-todo.txt`](/proposals/098-todo.txt): Proposals that should be written \[OBSOLETE\] +* [`099-misc.txt`](/proposals/099-misc.txt): Miscellaneous proposals \[OBSOLETE\] +* [`100-tor-spec-udp.txt`](/proposals/100-tor-spec-udp.txt): Tor Unreliable Datagram Extension Proposal \[DEAD\] +* [`115-two-hop-paths.txt`](/proposals/115-two-hop-paths.txt): Two Hop Paths \[DEAD\] +* [`116-two-hop-paths-from-guard.txt`](/proposals/116-two-hop-paths-from-guard.txt): Two hop paths from entry guards \[DEAD\] +* [`120-shutdown-descriptors.txt`](/proposals/120-shutdown-descriptors.txt): Shutdown descriptors when Tor servers stop \[DEAD\] +* [`127-dirport-mirrors-downloads.txt`](/proposals/127-dirport-mirrors-downloads.txt): Relaying dirport requests to Tor download site / website \[OBSOLETE\] +* [`128-bridge-families.txt`](/proposals/128-bridge-families.txt): Families of private bridges \[DEAD\] +* [`131-verify-tor-usage.txt`](/proposals/131-verify-tor-usage.txt): Help users to verify they are using Tor \[OBSOLETE\] +* [`132-browser-check-tor-service.txt`](/proposals/132-browser-check-tor-service.txt): A Tor Web Service For Verifying Correct Browser Configuration \[OBSOLETE\] +* [`134-robust-voting.txt`](/proposals/134-robust-voting.txt): More robust consensus voting with diverse authority sets \[REJECTED\] +* [`141-jit-sd-downloads.txt`](/proposals/141-jit-sd-downloads.txt): Download server descriptors on demand \[OBSOLETE\] +* [`142-combine-intro-and-rend-points.txt`](/proposals/142-combine-intro-and-rend-points.txt): Combine Introduction and Rendezvous Points \[DEAD\] +* [`144-enforce-distinct-providers.txt`](/proposals/144-enforce-distinct-providers.txt): Increase the diversity of circuits by detecting nodes belonging the same provider \[OBSOLETE\] +* [`147-prevoting-opinions.txt`](/proposals/147-prevoting-opinions.txt): Eliminate the need for v2 directories in generating v3 directories \[REJECTED\] +* [`164-reporting-server-status.txt`](/proposals/164-reporting-server-status.txt): Reporting the status of server votes \[OBSOLETE\] +* [`165-simple-robust-voting.txt`](/proposals/165-simple-robust-voting.txt): Easy migration for voting authority sets \[REJECTED\] +* [`168-reduce-circwindow.txt`](/proposals/168-reduce-circwindow.txt): Reduce default circuit window \[REJECTED\] +* [`173-getinfo-option-expansion.txt`](/proposals/173-getinfo-option-expansion.txt): GETINFO Option Expansion \[OBSOLETE\] +* [`175-automatic-node-promotion.txt`](/proposals/175-automatic-node-promotion.txt): Automatically promoting Tor clients to nodes \[REJECTED\] +* [`182-creditbucket.txt`](/proposals/182-creditbucket.txt): Credit Bucket \[OBSOLETE\] +* [`189-authorize-cell.txt`](/proposals/189-authorize-cell.txt): AUTHORIZE and AUTHORIZED cells \[OBSOLETE\] +* [`190-shared-secret-bridge-authorization.txt`](/proposals/190-shared-secret-bridge-authorization.txt): Bridge Client Authorization Based on a Shared Secret \[OBSOLETE\] +* [`191-mitm-bridge-detection-resistance.txt`](/proposals/191-mitm-bridge-detection-resistance.txt): Bridge Detection Resistance against MITM-capable Adversaries \[OBSOLETE\] +* [`192-store-bridge-information.txt`](/proposals/192-store-bridge-information.txt): Automatically retrieve and store information about bridges \[OBSOLETE\] +* [`195-TLS-normalization-for-024.txt`](/proposals/195-TLS-normalization-for-024.txt): TLS certificate normalization for Tor 0.2.4.x \[DEAD\] +* [`197-postmessage-ipc.txt`](/proposals/197-postmessage-ipc.txt): Message-based Inter-Controller IPC Channel \[REJECTED\] +* [`199-bridgefinder-integration.txt`](/proposals/199-bridgefinder-integration.txt): Integration of BridgeFinder and BridgeFinderHelper \[OBSOLETE\] +* [`203-https-frontend.txt`](/proposals/203-https-frontend.txt): Avoiding censorship by impersonating an HTTPS server \[OBSOLETE\] +* [`209-path-bias-tuning.txt`](/proposals/209-path-bias-tuning.txt): Tuning the Parameters for the Path Bias Defense \[OBSOLETE\] +* [`213-remove-stream-sendmes.txt`](/proposals/213-remove-stream-sendmes.txt): Remove stream-level sendmes from the design \[DEAD\] +* [`229-further-socks5-extensions.txt`](/proposals/229-further-socks5-extensions.txt): Further SOCKS5 extensions \[REJECTED\] +* [`230-rsa1024-relay-id-migration.txt`](/proposals/230-rsa1024-relay-id-migration.txt): How to change RSA1024 relay identity keys \[OBSOLETE\] +* [`231-migrate-authority-rsa1024-ids.txt`](/proposals/231-migrate-authority-rsa1024-ids.txt): Migrating authority RSA1024 identity keys \[OBSOLETE\] +* [`233-quicken-tor2web-mode.txt`](/proposals/233-quicken-tor2web-mode.txt): Making Tor2Web mode faster \[REJECTED\] +* [`234-remittance-addresses.txt`](/proposals/234-remittance-addresses.txt): Adding remittance field to directory specification \[REJECTED\] +* [`241-suspicious-guard-turnover.txt`](/proposals/241-suspicious-guard-turnover.txt): Resisting guard-turnover attacks \[REJECTED\] +* [`246-merge-hsdir-and-intro.txt`](/proposals/246-merge-hsdir-and-intro.txt): Merging Hidden Service Directories and Introduction Points \[REJECTED\] +* [`253-oob-hmac.txt`](/proposals/253-oob-hmac.txt): Out of Band Circuit HMACs \[DEAD\] +* [`258-dirauth-dos.txt`](/proposals/258-dirauth-dos.txt): Denial-of-service resistance for directory authorities \[DEAD\] +* [`259-guard-selection.txt`](/proposals/259-guard-selection.txt): New Guard Selection Behaviour \[OBSOLETE\] +* [`261-aez-crypto.txt`](/proposals/261-aez-crypto.txt): AEZ for relay cryptography \[OBSOLETE\] +* [`263-ntru-for-pq-handshake.txt`](/proposals/263-ntru-for-pq-handshake.txt): Request to change key exchange protocol for handshake v1.2 \[OBSOLETE\] +* [`268-guard-selection.txt`](/proposals/268-guard-selection.txt): New Guard Selection Behaviour \[OBSOLETE\] +* [`270-newhope-hybrid-handshake.txt`](/proposals/270-newhope-hybrid-handshake.txt): RebelAlliance: A Post-Quantum Secure Hybrid Handshake Based on NewHope \[OBSOLETE\] +* [`276-lower-bw-granularity.txt`](/proposals/276-lower-bw-granularity.txt): Report bandwidth with lower granularity in consensus documents \[DEAD\] +* [`286-hibernation-api.txt`](/proposals/286-hibernation-api.txt): Controller APIs for hibernation access on mobile \[REJECTED\] +* [`319-wide-everything.md`](/proposals/319-wide-everything.md): RELAY_FRAGMENT cells \[OBSOLETE\] +* [`320-tap-out-again.md`](/proposals/320-tap-out-again.md): Removing TAP usage from v2 onion services \[REJECTED\] +* [`325-packed-relay-cells.md`](/proposals/325-packed-relay-cells.md): Packed relay cells: saving space on small commands \[OBSOLETE\] + + + + diff --git a/proposals/README_template.md b/proposals/BY_STATUS_template.md index 55a913d..c8c2963 100644 --- a/proposals/README_template.md +++ b/proposals/BY_STATUS_template.md @@ -11,7 +11,7 @@ of these proposals are implemented; some are works in progress; and some will never be implemented. Below are a list of proposals sorted by status. See -[INDEX.md](/proposals/INDEX.md) for a list of proposals sorted by number. +[BY_INDEX.md](/proposals/BY_INDEX.md) for a list of proposals sorted by number. # Active proposals by status diff --git a/proposals/README.md b/proposals/README.md index 3c7f8fc..abaeebf 100644 --- a/proposals/README.md +++ b/proposals/README.md @@ -1,367 +1,17 @@ -# Tor proposals by status - -<!-- DO NOT EDIT THIS FILE --> - -<!-- If you want to make changes here, make sure to edit the --> -<!-- README_template.md file. The README.md file is generated by the --> -<!-- reindex.py script. --> - -Here we have a set of proposals for changes to the Tor protocol. Some -of these proposals are implemented; some are works in progress; and some -will never be implemented. - -Below are a list of proposals sorted by status. See -[INDEX.md](/proposals/INDEX.md) for a list of proposals sorted by number. - -# Active proposals by status - -## OPEN proposals: under discussion - -These are proposals that we think are likely to be complete, and ripe -for discussion. - -* [`239-consensus-hash-chaining.txt`](/proposals/239-consensus-hash-chaining.txt): Consensus Hash Chaining -* [`240-auth-cert-revocation.txt`](/proposals/240-auth-cert-revocation.txt): Early signing key revocation for directory authorities -* [`265-load-balancing-with-overhead.txt`](/proposals/265-load-balancing-with-overhead.txt): Load Balancing with Overhead Parameters -* [`267-tor-consensus-transparency.txt`](/proposals/267-tor-consensus-transparency.txt): Tor Consensus Transparency -* [`277-detect-id-sharing.txt`](/proposals/277-detect-id-sharing.txt): Detect multiple relay instances running with same ID -* [`287-reduce-lifetime.txt`](/proposals/287-reduce-lifetime.txt): Reduce circuit lifetime without overloading the network -* [`295-relay-crypto-with-adl.txt`](/proposals/295-relay-crypto-with-adl.txt): Using ADL for relay cryptography (solving the crypto-tagging attack) -* [`303-protover-removal-policy.txt`](/proposals/303-protover-removal-policy.txt): When and how to remove support for protocol versions -* [`306-ipv6-happy-eyeballs.txt`](/proposals/306-ipv6-happy-eyeballs.txt): A Tor Implementation of IPv6 Happy Eyeballs -* [`309-optimistic-socks-in-tor.txt`](/proposals/309-optimistic-socks-in-tor.txt): Optimistic SOCKS Data -* [`322-dirport-linkspec.md`](/proposals/322-dirport-linkspec.md): Extending link specifiers to include the directory port -* [`323-walking-onions-full.md`](/proposals/323-walking-onions-full.md): Specification for Walking Onions -* [`326-tor-relay-well-known-uri-rfc8615.md`](/proposals/326-tor-relay-well-known-uri-rfc8615.md): The "tor-relay" Well-Known Resource Identifier -* [`330-authority-contact.md`](/proposals/330-authority-contact.md): Modernizing authority contact entries -* [`340-packed-and-fragmented.md`](/proposals/340-packed-and-fragmented.md): Packed and fragmented relay messages -* [`341-better-oos.md`](/proposals/341-better-oos.md): A better algorithm for out-of-sockets eviction -* [`343-rend-caa.txt`](/proposals/343-rend-caa.txt): CAA Extensions for the Tor Rendezvous Specification -* [`344-protocol-info-leaks.txt`](/proposals/344-protocol-info-leaks.txt): Prioritizing Protocol Information Leaks in Tor -* [`345-specs-in-mdbook.md`](/proposals/345-specs-in-mdbook.md): Migrating the tor specifications to mdbook - - -## ACCEPTED proposals: slated for implementation - -These are the proposals that we agree we'd like to implement. They -might or might not have a specific timeframe planned for their -implementation. - -* [`282-remove-named-from-consensus.txt`](/proposals/282-remove-named-from-consensus.txt): Remove "Named" and "Unnamed" handling from consensus voting -* [`285-utf-8.txt`](/proposals/285-utf-8.txt): Directory documents should be standardized as UTF-8 -* [`292-mesh-vanguards.txt`](/proposals/292-mesh-vanguards.txt): Mesh-based vanguards -* [`311-relay-ipv6-reachability.txt`](/proposals/311-relay-ipv6-reachability.txt): Tor Relay IPv6 Reachability -* [`312-relay-auto-ipv6-addr.txt`](/proposals/312-relay-auto-ipv6-addr.txt): Tor Relay Automatic IPv6 Address Discovery -* [`313-relay-ipv6-stats.txt`](/proposals/313-relay-ipv6-stats.txt): Tor Relay IPv6 Statistics -* [`321-happy-families.md`](/proposals/321-happy-families.md): Better performance and usability for the MyFamily option (v2) -* [`338-netinfo-y2038.md`](/proposals/338-netinfo-y2038.md): Use an 8-byte timestamp in NETINFO cells -* [`339-udp-over-tor.md`](/proposals/339-udp-over-tor.md): UDP traffic over Tor - - -## FINISHED proposals: implemented, specs not merged - -These proposals are implemented in some version of Tor; the proposals -themselves still need to be merged into the specifications proper. - -* [`260-rend-single-onion.txt`](/proposals/260-rend-single-onion.txt): Rendezvous Single Onion Services -* [`291-two-guard-nodes.txt`](/proposals/291-two-guard-nodes.txt): The move to two guard nodes -* [`324-rtt-congestion-control.txt`](/proposals/324-rtt-congestion-control.txt): RTT-based Congestion Control for Tor -* [`327-pow-over-intro.txt`](/proposals/327-pow-over-intro.txt): A First Take at PoW Over Introduction Circuits -* [`329-traffic-splitting.txt`](/proposals/329-traffic-splitting.txt): Overcoming Tor's Bottlenecks with Traffic Splitting -* [`333-vanguards-lite.md`](/proposals/333-vanguards-lite.md): Vanguards lite - - -## META proposals: about the proposal process - -These proposals describe ongoing policies and changes to the proposals -process. - -* [`000-index.txt`](/proposals/000-index.txt): Index of Tor Proposals -* [`001-process.txt`](/proposals/001-process.txt): The Tor Proposal Process -* [`202-improved-relay-crypto.txt`](/proposals/202-improved-relay-crypto.txt): Two improved relay encryption protocols for Tor cells -* [`257-hiding-authorities.txt`](/proposals/257-hiding-authorities.txt): Refactoring authorities and making them more isolated from the net -* [`290-deprecate-consensus-methods.txt`](/proposals/290-deprecate-consensus-methods.txt): Continuously update consensus methods - - -## INFORMATIONAL proposals: not actually specifications - -These proposals describe a process or project, but aren't actually -proposed changes in the Tor specifications. - -* [`159-exit-scanning.txt`](/proposals/159-exit-scanning.txt): Exit Scanning -* [`300-walking-onions.txt`](/proposals/300-walking-onions.txt): Walking Onions: Scaling and Saving Bandwidth - - -# Preliminary proposals - -## DRAFT proposals: incomplete works - -These proposals have been marked as a draft by their author or the editors, -indicating that they aren't yet in a complete form. They're still open for -discussion. - -* [`294-tls-1.3.txt`](/proposals/294-tls-1.3.txt): TLS 1.3 Migration -* [`316-flashflow.md`](/proposals/316-flashflow.md): FlashFlow: A Secure Speed Test for Tor (Parent Proposal) -* [`331-res-tokens-for-anti-dos.md`](/proposals/331-res-tokens-for-anti-dos.md): Res tokens: Anonymous Credentials for Onion Service DoS Resilience -* [`342-decouple-hs-interval.md`](/proposals/342-decouple-hs-interval.md): Decoupling hs_interval and SRV lifetime - - -## NEEDS-REVISION proposals: ideas that we can't implement as-is - -These proposals have some promise, but we can't implement them without -certain changes. - -* [`212-using-old-consensus.txt`](/proposals/212-using-old-consensus.txt): Increase Acceptable Consensus Age -* [`219-expanded-dns.txt`](/proposals/219-expanded-dns.txt): Support for full DNS and DNSSEC resolution in Tor -* [`245-tap-out.txt`](/proposals/245-tap-out.txt): Deprecating and removing the TAP circuit extension protocol -* [`248-removing-rsa-identities.txt`](/proposals/248-removing-rsa-identities.txt): Remove all RSA identity keys -* [`269-hybrid-handshake.txt`](/proposals/269-hybrid-handshake.txt): Transitionally secure hybrid handshakes -* [`279-naming-layer-api.txt`](/proposals/279-naming-layer-api.txt): A Name System API for Tor Onion Services -* [`317-secure-dns-name-resolution.txt`](/proposals/317-secure-dns-name-resolution.txt): Improve security aspects of DNS name resolution - - -## NEEDS-RESEARCH proposals: blocking on research - -These proposals are interesting ideas, but there's more research that would -need to happen before we can know whether to implement them or not, or to -fill in certain details. - -(There are no proposals in this category) - - -# Inactive proposals by status - -## CLOSED proposals: implemented and specified - -These proposals have been implemented in some version of Tor, and the -changes from the proposals have been merged into the specifications as -necessary. - -* [`101-dir-voting.txt`](/proposals/101-dir-voting.txt): Voting on the Tor Directory System -* [`102-drop-opt.txt`](/proposals/102-drop-opt.txt): Dropping "opt" from the directory format -* [`103-multilevel-keys.txt`](/proposals/103-multilevel-keys.txt): Splitting identity key from regularly used signing key -* [`104-short-descriptors.txt`](/proposals/104-short-descriptors.txt): Long and Short Router Descriptors -* [`105-handshake-revision.txt`](/proposals/105-handshake-revision.txt): Version negotiation for the Tor protocol -* [`106-less-tls-constraint.txt`](/proposals/106-less-tls-constraint.txt): Checking fewer things during TLS handshakes -* [`107-uptime-sanity-checking.txt`](/proposals/107-uptime-sanity-checking.txt): Uptime Sanity Checking -* [`108-mtbf-based-stability.txt`](/proposals/108-mtbf-based-stability.txt): Base "Stable" Flag on Mean Time Between Failures -* [`109-no-sharing-ips.txt`](/proposals/109-no-sharing-ips.txt): No more than one server per IP address -* [`110-avoid-infinite-circuits.txt`](/proposals/110-avoid-infinite-circuits.txt): Avoiding infinite length circuits -* [`111-local-traffic-priority.txt`](/proposals/111-local-traffic-priority.txt): Prioritizing local traffic over relayed traffic -* [`114-distributed-storage.txt`](/proposals/114-distributed-storage.txt): Distributed Storage for Tor Hidden Service Descriptors -* [`117-ipv6-exits.txt`](/proposals/117-ipv6-exits.txt): IPv6 exits -* [`119-controlport-auth.txt`](/proposals/119-controlport-auth.txt): New PROTOCOLINFO command for controllers -* [`121-hidden-service-authentication.txt`](/proposals/121-hidden-service-authentication.txt): Hidden Service Authentication -* [`122-unnamed-flag.txt`](/proposals/122-unnamed-flag.txt): Network status entries need a new Unnamed flag -* [`123-autonaming.txt`](/proposals/123-autonaming.txt): Naming authorities automatically create bindings -* [`125-bridges.txt`](/proposals/125-bridges.txt): Behavior for bridge users, bridge relays, and bridge authorities -* [`126-geoip-reporting.txt`](/proposals/126-geoip-reporting.txt): Getting GeoIP data and publishing usage summaries -* [`129-reject-plaintext-ports.txt`](/proposals/129-reject-plaintext-ports.txt): Block Insecure Protocols by Default -* [`130-v2-conn-protocol.txt`](/proposals/130-v2-conn-protocol.txt): Version 2 Tor connection protocol -* [`135-private-tor-networks.txt`](/proposals/135-private-tor-networks.txt): Simplify Configuration of Private Tor Networks -* [`136-legacy-keys.txt`](/proposals/136-legacy-keys.txt): Mass authority migration with legacy keys -* [`137-bootstrap-phases.txt`](/proposals/137-bootstrap-phases.txt): Keep controllers informed as Tor bootstraps -* [`138-remove-down-routers-from-consensus.txt`](/proposals/138-remove-down-routers-from-consensus.txt): Remove routers that are not Running from consensus documents -* [`139-conditional-consensus-download.txt`](/proposals/139-conditional-consensus-download.txt): Download consensus documents only when it will be trusted -* [`140-consensus-diffs.txt`](/proposals/140-consensus-diffs.txt): Provide diffs between consensuses -* [`148-uniform-client-end-reason.txt`](/proposals/148-uniform-client-end-reason.txt): Stream end reasons from the client side should be uniform -* [`150-exclude-exit-nodes.txt`](/proposals/150-exclude-exit-nodes.txt): Exclude Exit Nodes from a circuit -* [`151-path-selection-improvements.txt`](/proposals/151-path-selection-improvements.txt): Improving Tor Path Selection -* [`152-single-hop-circuits.txt`](/proposals/152-single-hop-circuits.txt): Optionally allow exit from single-hop circuits -* [`155-four-hidden-service-improvements.txt`](/proposals/155-four-hidden-service-improvements.txt): Four Improvements of Hidden Service Performance -* [`157-specific-cert-download.txt`](/proposals/157-specific-cert-download.txt): Make certificate downloads specific -* [`158-microdescriptors.txt`](/proposals/158-microdescriptors.txt): Clients download consensus + microdescriptors -* [`160-bandwidth-offset.txt`](/proposals/160-bandwidth-offset.txt): Authorities vote for bandwidth offsets in consensus -* [`161-computing-bandwidth-adjustments.txt`](/proposals/161-computing-bandwidth-adjustments.txt): Computing Bandwidth Adjustments -* [`162-consensus-flavors.txt`](/proposals/162-consensus-flavors.txt): Publish the consensus in multiple flavors -* [`166-statistics-extra-info-docs.txt`](/proposals/166-statistics-extra-info-docs.txt): Including Network Statistics in Extra-Info Documents -* [`167-params-in-consensus.txt`](/proposals/167-params-in-consensus.txt): Vote on network parameters in consensus -* [`171-separate-streams.txt`](/proposals/171-separate-streams.txt): Separate streams across circuits by connection metadata -* [`174-optimistic-data-server.txt`](/proposals/174-optimistic-data-server.txt): Optimistic Data for Tor: Server Side -* [`176-revising-handshake.txt`](/proposals/176-revising-handshake.txt): Proposed version-3 link handshake for Tor -* [`178-param-voting.txt`](/proposals/178-param-voting.txt): Require majority of authorities to vote for consensus parameters -* [`179-TLS-cert-and-parameter-normalization.txt`](/proposals/179-TLS-cert-and-parameter-normalization.txt): TLS certificate and parameter normalization -* [`180-pluggable-transport.txt`](/proposals/180-pluggable-transport.txt): Pluggable transports for circumvention -* [`181-optimistic-data-client.txt`](/proposals/181-optimistic-data-client.txt): Optimistic Data for Tor: Client Side -* [`183-refillintervals.txt`](/proposals/183-refillintervals.txt): Refill Intervals -* [`184-v3-link-protocol.txt`](/proposals/184-v3-link-protocol.txt): Miscellaneous changes for a v3 Tor link protocol -* [`186-multiple-orports.txt`](/proposals/186-multiple-orports.txt): Multiple addresses for one OR or bridge -* [`187-allow-client-auth.txt`](/proposals/187-allow-client-auth.txt): Reserve a cell type to allow client authorization -* [`193-safe-cookie-authentication.txt`](/proposals/193-safe-cookie-authentication.txt): Safe cookie authentication for Tor controllers -* [`196-transport-control-ports.txt`](/proposals/196-transport-control-ports.txt): Extended ORPort and TransportControlPort -* [`198-restore-clienthello-semantics.txt`](/proposals/198-restore-clienthello-semantics.txt): Restore semantics of TLS ClientHello -* [`200-new-create-and-extend-cells.txt`](/proposals/200-new-create-and-extend-cells.txt): Adding new, extensible CREATE, EXTEND, and related cells -* [`204-hidserv-subdomains.txt`](/proposals/204-hidserv-subdomains.txt): Subdomain support for Hidden Service addresses -* [`205-local-dnscache.txt`](/proposals/205-local-dnscache.txt): Remove global client-side DNS caching -* [`206-directory-sources.txt`](/proposals/206-directory-sources.txt): Preconfigured directory sources for bootstrapping -* [`207-directory-guards.txt`](/proposals/207-directory-guards.txt): Directory guards -* [`208-ipv6-exits-redux.txt`](/proposals/208-ipv6-exits-redux.txt): IPv6 Exits Redux -* [`214-longer-circids.txt`](/proposals/214-longer-circids.txt): Allow 4-byte circuit IDs in a new link protocol -* [`215-update-min-consensus-ver.txt`](/proposals/215-update-min-consensus-ver.txt): Let the minimum consensus method change with time -* [`216-ntor-handshake.txt`](/proposals/216-ntor-handshake.txt): Improved circuit-creation key exchange -* [`217-ext-orport-auth.txt`](/proposals/217-ext-orport-auth.txt): Tor Extended ORPort Authentication -* [`218-usage-controller-events.txt`](/proposals/218-usage-controller-events.txt): Controller events to better understand connection/circuit usage -* [`220-ecc-id-keys.txt`](/proposals/220-ecc-id-keys.txt): Migrate server identity keys to Ed25519 -* [`221-stop-using-create-fast.txt`](/proposals/221-stop-using-create-fast.txt): Stop using CREATE_FAST -* [`222-remove-client-timestamps.txt`](/proposals/222-remove-client-timestamps.txt): Stop sending client timestamps -* [`224-rend-spec-ng.txt`](/proposals/224-rend-spec-ng.txt): Next-Generation Hidden Services in Tor -* [`227-vote-on-package-fingerprints.txt`](/proposals/227-vote-on-package-fingerprints.txt): Include package fingerprints in consensus documents -* [`228-cross-certification-onionkeys.txt`](/proposals/228-cross-certification-onionkeys.txt): Cross-certifying identity keys with onion keys -* [`232-pluggable-transports-through-proxy.txt`](/proposals/232-pluggable-transports-through-proxy.txt): Pluggable Transport through SOCKS proxy -* [`235-kill-named-flag.txt`](/proposals/235-kill-named-flag.txt): Stop assigning (and eventually supporting) the Named flag -* [`236-single-guard-node.txt`](/proposals/236-single-guard-node.txt): The move to a single guard node -* [`237-directory-servers-for-all.txt`](/proposals/237-directory-servers-for-all.txt): All relays are directory servers -* [`238-hs-relay-stats.txt`](/proposals/238-hs-relay-stats.txt): Better hidden service stats from Tor relays -* [`243-hsdir-flag-need-stable.txt`](/proposals/243-hsdir-flag-need-stable.txt): Give out HSDir flag only to relays with Stable flag -* [`244-use-rfc5705-for-tls-binding.txt`](/proposals/244-use-rfc5705-for-tls-binding.txt): Use RFC5705 Key Exporting in our AUTHENTICATE calls -* [`250-commit-reveal-consensus.txt`](/proposals/250-commit-reveal-consensus.txt): Random Number Generation During Tor Voting -* [`251-netflow-padding.txt`](/proposals/251-netflow-padding.txt): Padding for netflow record resolution reduction -* [`254-padding-negotiation.txt`](/proposals/254-padding-negotiation.txt): Padding Negotiation -* [`264-subprotocol-versions.txt`](/proposals/264-subprotocol-versions.txt): Putting version numbers on the Tor subprotocols -* [`271-another-guard-selection.txt`](/proposals/271-another-guard-selection.txt): Another algorithm for guard selection -* [`272-valid-and-running-by-default.txt`](/proposals/272-valid-and-running-by-default.txt): Listed routers should be Valid, Running, and treated as such -* [`274-rotate-onion-keys-less.txt`](/proposals/274-rotate-onion-keys-less.txt): Rotate onion keys less frequently -* [`275-md-published-time-is-silly.txt`](/proposals/275-md-published-time-is-silly.txt): Stop including meaningful "published" time in microdescriptor consensus -* [`278-directory-compression-scheme-negotiation.txt`](/proposals/278-directory-compression-scheme-negotiation.txt): Directory Compression Scheme Negotiation -* [`283-ipv6-in-micro-consensus.txt`](/proposals/283-ipv6-in-micro-consensus.txt): Move IPv6 ORPorts from microdescriptors to the microdesc consensus -* [`284-hsv3-control-port.txt`](/proposals/284-hsv3-control-port.txt): Hidden Service v3 Control Port -* [`289-authenticated-sendmes.txt`](/proposals/289-authenticated-sendmes.txt): Authenticating sendme cells to mitigate bandwidth attacks -* [`293-know-when-to-publish.txt`](/proposals/293-know-when-to-publish.txt): Other ways for relays to know when to publish -* [`296-expose-bandwidth-files.txt`](/proposals/296-expose-bandwidth-files.txt): Have Directory Authorities expose raw bandwidth list files -* [`297-safer-protover-shutdowns.txt`](/proposals/297-safer-protover-shutdowns.txt): Relaxing the protover-based shutdown rules -* [`298-canonical-families.txt`](/proposals/298-canonical-families.txt): Putting family lines in canonical form -* [`301-dont-vote-on-package-fingerprints.txt`](/proposals/301-dont-vote-on-package-fingerprints.txt): Don't include package fingerprints in consensus documents -* [`302-padding-machines-for-onion-clients.txt`](/proposals/302-padding-machines-for-onion-clients.txt): Hiding onion service clients using padding -* [`304-socks5-extending-hs-error-codes.txt`](/proposals/304-socks5-extending-hs-error-codes.txt): Extending SOCKS5 Onion Service Error Codes -* [`305-establish-intro-dos-defense-extention.txt`](/proposals/305-establish-intro-dos-defense-extention.txt): ESTABLISH_INTRO Cell DoS Defense Extension -* [`310-bandaid-on-guard-selection.txt`](/proposals/310-bandaid-on-guard-selection.txt): Towards load-balancing in Prop 271 -* [`314-allow-markdown-proposals.md`](/proposals/314-allow-markdown-proposals.md): Allow Markdown for proposal format -* [`315-update-dir-required-fields.txt`](/proposals/315-update-dir-required-fields.txt): Updating the list of fields required in directory documents -* [`318-limit-protovers.md`](/proposals/318-limit-protovers.md): Limit protover values to 0-63 -* [`328-relay-overload-report.md`](/proposals/328-relay-overload-report.md): Make Relays Report When They Are Overloaded -* [`332-ntor-v3-with-extra-data.md`](/proposals/332-ntor-v3-with-extra-data.md): Ntor protocol with extra data, version 3 -* [`335-middle-only-redux.md`](/proposals/335-middle-only-redux.md): An authority-only design for MiddleOnly -* [`336-randomize-guard-retries.md`](/proposals/336-randomize-guard-retries.md): Randomized schedule for guard retries -* [`337-simpler-guard-usability.md`](/proposals/337-simpler-guard-usability.md): A simpler way to decide, "Is this guard usable?" - - -## RESERVE proposals: saving for later - -These proposals aren't anything we plan to implement soon, but for one -reason or another we think they might be a good idea in the -future. We're keeping them around as a reference in case we someday -confront the problems that they try to solve. - -* [`133-unreachable-ors.txt`](/proposals/133-unreachable-ors.txt): Incorporate Unreachable ORs into the Tor Network -* [`172-circ-getinfo-option.txt`](/proposals/172-circ-getinfo-option.txt): GETINFO controller option for circuit information -* [`177-flag-abstention.txt`](/proposals/177-flag-abstention.txt): Abstaining from votes on individual flags -* [`188-bridge-guards.txt`](/proposals/188-bridge-guards.txt): Bridge Guards and other anti-enumeration defenses -* [`201-bridge-v3-reqs-stats.txt`](/proposals/201-bridge-v3-reqs-stats.txt): Make bridges report statistics on daily v3 network status requests -* [`211-mapaddress-tor-status.txt`](/proposals/211-mapaddress-tor-status.txt): Internal Mapaddress for Tor Configuration Testing -* [`223-ace-handshake.txt`](/proposals/223-ace-handshake.txt): Ace: Improved circuit-creation key exchange -* [`226-bridgedb-database-improvements.txt`](/proposals/226-bridgedb-database-improvements.txt): "Scalability and Stability Improvements to BridgeDB: Switching to a Distributed Database System and RDBMS" -* [`255-hs-load-balancing.txt`](/proposals/255-hs-load-balancing.txt): Controller features to allow for load-balancing hidden services -* [`256-key-revocation.txt`](/proposals/256-key-revocation.txt): Key revocation for relays and authorities -* [`262-rekey-circuits.txt`](/proposals/262-rekey-circuits.txt): Re-keying live circuits with new cryptographic material -* [`273-exit-relay-pinning.txt`](/proposals/273-exit-relay-pinning.txt): Exit relay pinning for web services -* [`281-bulk-md-download.txt`](/proposals/281-bulk-md-download.txt): Downloading microdescriptors in bulk -* [`288-privcount-with-shamir.txt`](/proposals/288-privcount-with-shamir.txt): Privacy-Preserving Statistics with Privcount in Tor (Shamir version) -* [`307-onionbalance-v3.txt`](/proposals/307-onionbalance-v3.txt): Onion Balance Support for Onion Service v3 - - -## SUPERSEDED proposals: replaced by something else - -These proposals were obsoleted by a later proposal before they were -implemented. - -* [`112-bring-back-pathlencoinweight.txt`](/proposals/112-bring-back-pathlencoinweight.txt): Bring Back Pathlen Coin Weight -* [`113-fast-authority-interface.txt`](/proposals/113-fast-authority-interface.txt): Simplifying directory authority administration -* [`118-multiple-orports.txt`](/proposals/118-multiple-orports.txt): Advertising multiple ORPorts at once -* [`124-tls-certificates.txt`](/proposals/124-tls-certificates.txt): Blocking resistant TLS certificate usage -* [`143-distributed-storage-improvements.txt`](/proposals/143-distributed-storage-improvements.txt): Improvements of Distributed Storage for Tor Hidden Service Descriptors -* [`145-newguard-flag.txt`](/proposals/145-newguard-flag.txt): Separate "suitable as a guard" from "suitable as a new guard" -* [`146-long-term-stability.txt`](/proposals/146-long-term-stability.txt): Add new flag to reflect long-term stability -* [`149-using-netinfo-data.txt`](/proposals/149-using-netinfo-data.txt): Using data from NETINFO cells -* [`153-automatic-software-update-protocol.txt`](/proposals/153-automatic-software-update-protocol.txt): Automatic software update protocol -* [`154-automatic-updates.txt`](/proposals/154-automatic-updates.txt): Automatic Software Update Protocol -* [`156-tracking-blocked-ports.txt`](/proposals/156-tracking-blocked-ports.txt): Tracking blocked ports on the client side -* [`163-detecting-clients.txt`](/proposals/163-detecting-clients.txt): Detecting whether a connection comes from a client -* [`169-eliminating-renegotiation.txt`](/proposals/169-eliminating-renegotiation.txt): Eliminate TLS renegotiation for the Tor connection handshake -* [`170-user-path-config.txt`](/proposals/170-user-path-config.txt): Configuration options regarding circuit building -* [`185-dir-without-dirport.txt`](/proposals/185-dir-without-dirport.txt): Directory caches without DirPort -* [`194-mnemonic-urls.txt`](/proposals/194-mnemonic-urls.txt): Mnemonic .onion URLs -* [`210-faster-headless-consensus-bootstrap.txt`](/proposals/210-faster-headless-consensus-bootstrap.txt): Faster Headless Consensus Bootstrapping -* [`225-strawman-shared-rand.txt`](/proposals/225-strawman-shared-rand.txt): Strawman proposal: commit-and-reveal shared rng -* [`242-better-families.txt`](/proposals/242-better-families.txt): Better performance and usability for the MyFamily option -* [`247-hs-guard-discovery.txt`](/proposals/247-hs-guard-discovery.txt): Defending Against Guard Discovery Attacks using Vanguards -* [`249-large-create-cells.txt`](/proposals/249-large-create-cells.txt): Allow CREATE cells with >505 bytes of handshake data -* [`252-single-onion.txt`](/proposals/252-single-onion.txt): Single Onion Services -* [`266-removing-current-obsolete-clients.txt`](/proposals/266-removing-current-obsolete-clients.txt): Removing current obsolete clients from the Tor network -* [`280-privcount-in-tor.txt`](/proposals/280-privcount-in-tor.txt): Privacy-Preserving Statistics with Privcount in Tor -* [`299-ip-failure-count.txt`](/proposals/299-ip-failure-count.txt): Preferring IPv4 or IPv6 based on IP Version Failure Count -* [`308-counter-galois-onion.txt`](/proposals/308-counter-galois-onion.txt): Counter Galois Onion: A New Proposal for Forward-Secure Relay Cryptography -* [`334-middle-only-flag.txt`](/proposals/334-middle-only-flag.txt): A Directory Authority Flag To Mark Relays As Middle-only - - -## DEAD, REJECTED, OBSOLETE proposals: not in our plans - -These proposals are not on-track for discussion or -implementation. Either discussion has stalled out (the proposal is -DEAD), the proposal has been considered and not adopted (the proposal is -REJECTED), or the proposal addresses an issue or a solution that is no -longer relevant (the proposal is OBSOLETE). - -* [`098-todo.txt`](/proposals/098-todo.txt): Proposals that should be written [OBSOLETE] -* [`099-misc.txt`](/proposals/099-misc.txt): Miscellaneous proposals [OBSOLETE] -* [`100-tor-spec-udp.txt`](/proposals/100-tor-spec-udp.txt): Tor Unreliable Datagram Extension Proposal [DEAD] -* [`115-two-hop-paths.txt`](/proposals/115-two-hop-paths.txt): Two Hop Paths [DEAD] -* [`116-two-hop-paths-from-guard.txt`](/proposals/116-two-hop-paths-from-guard.txt): Two hop paths from entry guards [DEAD] -* [`120-shutdown-descriptors.txt`](/proposals/120-shutdown-descriptors.txt): Shutdown descriptors when Tor servers stop [DEAD] -* [`127-dirport-mirrors-downloads.txt`](/proposals/127-dirport-mirrors-downloads.txt): Relaying dirport requests to Tor download site / website [OBSOLETE] -* [`128-bridge-families.txt`](/proposals/128-bridge-families.txt): Families of private bridges [DEAD] -* [`131-verify-tor-usage.txt`](/proposals/131-verify-tor-usage.txt): Help users to verify they are using Tor [OBSOLETE] -* [`132-browser-check-tor-service.txt`](/proposals/132-browser-check-tor-service.txt): A Tor Web Service For Verifying Correct Browser Configuration [OBSOLETE] -* [`134-robust-voting.txt`](/proposals/134-robust-voting.txt): More robust consensus voting with diverse authority sets [REJECTED] -* [`141-jit-sd-downloads.txt`](/proposals/141-jit-sd-downloads.txt): Download server descriptors on demand [OBSOLETE] -* [`142-combine-intro-and-rend-points.txt`](/proposals/142-combine-intro-and-rend-points.txt): Combine Introduction and Rendezvous Points [DEAD] -* [`144-enforce-distinct-providers.txt`](/proposals/144-enforce-distinct-providers.txt): Increase the diversity of circuits by detecting nodes belonging the same provider [OBSOLETE] -* [`147-prevoting-opinions.txt`](/proposals/147-prevoting-opinions.txt): Eliminate the need for v2 directories in generating v3 directories [REJECTED] -* [`164-reporting-server-status.txt`](/proposals/164-reporting-server-status.txt): Reporting the status of server votes [OBSOLETE] -* [`165-simple-robust-voting.txt`](/proposals/165-simple-robust-voting.txt): Easy migration for voting authority sets [REJECTED] -* [`168-reduce-circwindow.txt`](/proposals/168-reduce-circwindow.txt): Reduce default circuit window [REJECTED] -* [`173-getinfo-option-expansion.txt`](/proposals/173-getinfo-option-expansion.txt): GETINFO Option Expansion [OBSOLETE] -* [`175-automatic-node-promotion.txt`](/proposals/175-automatic-node-promotion.txt): Automatically promoting Tor clients to nodes [REJECTED] -* [`182-creditbucket.txt`](/proposals/182-creditbucket.txt): Credit Bucket [OBSOLETE] -* [`189-authorize-cell.txt`](/proposals/189-authorize-cell.txt): AUTHORIZE and AUTHORIZED cells [OBSOLETE] -* [`190-shared-secret-bridge-authorization.txt`](/proposals/190-shared-secret-bridge-authorization.txt): Bridge Client Authorization Based on a Shared Secret [OBSOLETE] -* [`191-mitm-bridge-detection-resistance.txt`](/proposals/191-mitm-bridge-detection-resistance.txt): Bridge Detection Resistance against MITM-capable Adversaries [OBSOLETE] -* [`192-store-bridge-information.txt`](/proposals/192-store-bridge-information.txt): Automatically retrieve and store information about bridges [OBSOLETE] -* [`195-TLS-normalization-for-024.txt`](/proposals/195-TLS-normalization-for-024.txt): TLS certificate normalization for Tor 0.2.4.x [DEAD] -* [`197-postmessage-ipc.txt`](/proposals/197-postmessage-ipc.txt): Message-based Inter-Controller IPC Channel [REJECTED] -* [`199-bridgefinder-integration.txt`](/proposals/199-bridgefinder-integration.txt): Integration of BridgeFinder and BridgeFinderHelper [OBSOLETE] -* [`203-https-frontend.txt`](/proposals/203-https-frontend.txt): Avoiding censorship by impersonating an HTTPS server [OBSOLETE] -* [`209-path-bias-tuning.txt`](/proposals/209-path-bias-tuning.txt): Tuning the Parameters for the Path Bias Defense [OBSOLETE] -* [`213-remove-stream-sendmes.txt`](/proposals/213-remove-stream-sendmes.txt): Remove stream-level sendmes from the design [DEAD] -* [`229-further-socks5-extensions.txt`](/proposals/229-further-socks5-extensions.txt): Further SOCKS5 extensions [REJECTED] -* [`230-rsa1024-relay-id-migration.txt`](/proposals/230-rsa1024-relay-id-migration.txt): How to change RSA1024 relay identity keys [OBSOLETE] -* [`231-migrate-authority-rsa1024-ids.txt`](/proposals/231-migrate-authority-rsa1024-ids.txt): Migrating authority RSA1024 identity keys [OBSOLETE] -* [`233-quicken-tor2web-mode.txt`](/proposals/233-quicken-tor2web-mode.txt): Making Tor2Web mode faster [REJECTED] -* [`234-remittance-addresses.txt`](/proposals/234-remittance-addresses.txt): Adding remittance field to directory specification [REJECTED] -* [`241-suspicious-guard-turnover.txt`](/proposals/241-suspicious-guard-turnover.txt): Resisting guard-turnover attacks [REJECTED] -* [`246-merge-hsdir-and-intro.txt`](/proposals/246-merge-hsdir-and-intro.txt): Merging Hidden Service Directories and Introduction Points [REJECTED] -* [`253-oob-hmac.txt`](/proposals/253-oob-hmac.txt): Out of Band Circuit HMACs [DEAD] -* [`258-dirauth-dos.txt`](/proposals/258-dirauth-dos.txt): Denial-of-service resistance for directory authorities [DEAD] -* [`259-guard-selection.txt`](/proposals/259-guard-selection.txt): New Guard Selection Behaviour [OBSOLETE] -* [`261-aez-crypto.txt`](/proposals/261-aez-crypto.txt): AEZ for relay cryptography [OBSOLETE] -* [`263-ntru-for-pq-handshake.txt`](/proposals/263-ntru-for-pq-handshake.txt): Request to change key exchange protocol for handshake v1.2 [OBSOLETE] -* [`268-guard-selection.txt`](/proposals/268-guard-selection.txt): New Guard Selection Behaviour [OBSOLETE] -* [`270-newhope-hybrid-handshake.txt`](/proposals/270-newhope-hybrid-handshake.txt): RebelAlliance: A Post-Quantum Secure Hybrid Handshake Based on NewHope [OBSOLETE] -* [`276-lower-bw-granularity.txt`](/proposals/276-lower-bw-granularity.txt): Report bandwidth with lower granularity in consensus documents [DEAD] -* [`286-hibernation-api.txt`](/proposals/286-hibernation-api.txt): Controller APIs for hibernation access on mobile [REJECTED] -* [`319-wide-everything.md`](/proposals/319-wide-everything.md): RELAY_FRAGMENT cells [OBSOLETE] -* [`320-tap-out-again.md`](/proposals/320-tap-out-again.md): Removing TAP usage from v2 onion services [REJECTED] -* [`325-packed-relay-cells.md`](/proposals/325-packed-relay-cells.md): Packed relay cells: saving space on small commands [OBSOLETE] +# Proposals for changes in the Tor protocols +This "book" is a list of proposals that people have made over the +years, (dating back to 2007) for protocol changes in Tor. +Some of these proposals are already implemented or rejected; +others are under active discussion. +If you're looking for a specific proposal, you can find it, +by filename, in the summary bar on the left, or at +[this index](./BY_INDEX.md). You can also see a list of Tor protocols +by their status at [`BY_STATUS.md`](./BY_STATUS.md). +For information on creating a new proposal, you would ideally look at +[`001-process.txt`](./001-process.txt). That file is a bit out-of-date, though, and you +should probably just contact the developers. +* <a href="..">Back to the Tor specifications</a> diff --git a/proposals/SUMMARY.md b/proposals/SUMMARY.md new file mode 100644 index 0000000..df02273 --- /dev/null +++ b/proposals/SUMMARY.md @@ -0,0 +1,264 @@ +# Summary + +[Tor Proposals](README.md) +[All proposals, by number](BY_INDEX.md) +[All proposals, by status](BY_STATUS.md) + +# Proposals + + - [`000-index`](./000-index.txt): Index of Tor Proposals (META) + - [`001-process`](./001-process.txt): The Tor Proposal Process (META) + - [`098-todo`](./098-todo.txt): Proposals that should be written (OBSOLETE) + - [`099-misc`](./099-misc.txt): Miscellaneous proposals (OBSOLETE) + - [`100-tor-spec-udp`](./100-tor-spec-udp.txt): Tor Unreliable Datagram Extension Proposal (DEAD) + - [`101-dir-voting`](./101-dir-voting.txt): Voting on the Tor Directory System (CLOSED) + - [`102-drop-opt`](./102-drop-opt.txt): Dropping "opt" from the directory format (CLOSED) + - [`103-multilevel-keys`](./103-multilevel-keys.txt): Splitting identity key from regularly used signing key (CLOSED) + - [`104-short-descriptors`](./104-short-descriptors.txt): Long and Short Router Descriptors (CLOSED) + - [`105-handshake-revision`](./105-handshake-revision.txt): Version negotiation for the Tor protocol (CLOSED) + - [`106-less-tls-constraint`](./106-less-tls-constraint.txt): Checking fewer things during TLS handshakes (CLOSED) + - [`107-uptime-sanity-checking`](./107-uptime-sanity-checking.txt): Uptime Sanity Checking (CLOSED) + - [`108-mtbf-based-stability`](./108-mtbf-based-stability.txt): Base "Stable" Flag on Mean Time Between Failures (CLOSED) + - [`109-no-sharing-ips`](./109-no-sharing-ips.txt): No more than one server per IP address (CLOSED) + - [`110-avoid-infinite-circuits`](./110-avoid-infinite-circuits.txt): Avoiding infinite length circuits (CLOSED) + - [`111-local-traffic-priority`](./111-local-traffic-priority.txt): Prioritizing local traffic over relayed traffic (CLOSED) + - [`112-bring-back-pathlencoinweight`](./112-bring-back-pathlencoinweight.txt): Bring Back Pathlen Coin Weight (SUPERSEDED) + - [`113-fast-authority-interface`](./113-fast-authority-interface.txt): Simplifying directory authority administration (SUPERSEDED) + - [`114-distributed-storage`](./114-distributed-storage.txt): Distributed Storage for Tor Hidden Service Descriptors (CLOSED) + - [`115-two-hop-paths`](./115-two-hop-paths.txt): Two Hop Paths (DEAD) + - [`116-two-hop-paths-from-guard`](./116-two-hop-paths-from-guard.txt): Two hop paths from entry guards (DEAD) + - [`117-ipv6-exits`](./117-ipv6-exits.txt): IPv6 exits (CLOSED) + - [`118-multiple-orports`](./118-multiple-orports.txt): Advertising multiple ORPorts at once (SUPERSEDED) + - [`119-controlport-auth`](./119-controlport-auth.txt): New PROTOCOLINFO command for controllers (CLOSED) + - [`120-shutdown-descriptors`](./120-shutdown-descriptors.txt): Shutdown descriptors when Tor servers stop (DEAD) + - [`121-hidden-service-authentication`](./121-hidden-service-authentication.txt): Hidden Service Authentication (CLOSED) + - [`122-unnamed-flag`](./122-unnamed-flag.txt): Network status entries need a new Unnamed flag (CLOSED) + - [`123-autonaming`](./123-autonaming.txt): Naming authorities automatically create bindings (CLOSED) + - [`124-tls-certificates`](./124-tls-certificates.txt): Blocking resistant TLS certificate usage (SUPERSEDED) + - [`125-bridges`](./125-bridges.txt): Behavior for bridge users, bridge relays, and bridge authorities (CLOSED) + - [`126-geoip-reporting`](./126-geoip-reporting.txt): Getting GeoIP data and publishing usage summaries (CLOSED) + - [`127-dirport-mirrors-downloads`](./127-dirport-mirrors-downloads.txt): Relaying dirport requests to Tor download site / website (OBSOLETE) + - [`128-bridge-families`](./128-bridge-families.txt): Families of private bridges (DEAD) + - [`129-reject-plaintext-ports`](./129-reject-plaintext-ports.txt): Block Insecure Protocols by Default (CLOSED) + - [`130-v2-conn-protocol`](./130-v2-conn-protocol.txt): Version 2 Tor connection protocol (CLOSED) + - [`131-verify-tor-usage`](./131-verify-tor-usage.txt): Help users to verify they are using Tor (OBSOLETE) + - [`132-browser-check-tor-service`](./132-browser-check-tor-service.txt): A Tor Web Service For Verifying Correct Browser Configuration (OBSOLETE) + - [`133-unreachable-ors`](./133-unreachable-ors.txt): Incorporate Unreachable ORs into the Tor Network (RESERVE) + - [`134-robust-voting`](./134-robust-voting.txt): More robust consensus voting with diverse authority sets (REJECTED) + - [`135-private-tor-networks`](./135-private-tor-networks.txt): Simplify Configuration of Private Tor Networks (CLOSED) + - [`136-legacy-keys`](./136-legacy-keys.txt): Mass authority migration with legacy keys (CLOSED) + - [`137-bootstrap-phases`](./137-bootstrap-phases.txt): Keep controllers informed as Tor bootstraps (CLOSED) + - [`138-remove-down-routers-from-consensus`](./138-remove-down-routers-from-consensus.txt): Remove routers that are not Running from consensus documents (CLOSED) + - [`139-conditional-consensus-download`](./139-conditional-consensus-download.txt): Download consensus documents only when it will be trusted (CLOSED) + - [`140-consensus-diffs`](./140-consensus-diffs.txt): Provide diffs between consensuses (CLOSED) + - [`141-jit-sd-downloads`](./141-jit-sd-downloads.txt): Download server descriptors on demand (OBSOLETE) + - [`142-combine-intro-and-rend-points`](./142-combine-intro-and-rend-points.txt): Combine Introduction and Rendezvous Points (DEAD) + - [`143-distributed-storage-improvements`](./143-distributed-storage-improvements.txt): Improvements of Distributed Storage for Tor Hidden Service Descriptors (SUPERSEDED) + - [`144-enforce-distinct-providers`](./144-enforce-distinct-providers.txt): Increase the diversity of circuits by detecting nodes belonging the same provider (OBSOLETE) + - [`145-newguard-flag`](./145-newguard-flag.txt): Separate "suitable as a guard" from "suitable as a new guard" (SUPERSEDED) + - [`146-long-term-stability`](./146-long-term-stability.txt): Add new flag to reflect long-term stability (SUPERSEDED) + - [`147-prevoting-opinions`](./147-prevoting-opinions.txt): Eliminate the need for v2 directories in generating v3 directories (REJECTED) + - [`148-uniform-client-end-reason`](./148-uniform-client-end-reason.txt): Stream end reasons from the client side should be uniform (CLOSED) + - [`149-using-netinfo-data`](./149-using-netinfo-data.txt): Using data from NETINFO cells (SUPERSEDED) + - [`150-exclude-exit-nodes`](./150-exclude-exit-nodes.txt): Exclude Exit Nodes from a circuit (CLOSED) + - [`151-path-selection-improvements`](./151-path-selection-improvements.txt): Improving Tor Path Selection (CLOSED) + - [`152-single-hop-circuits`](./152-single-hop-circuits.txt): Optionally allow exit from single-hop circuits (CLOSED) + - [`153-automatic-software-update-protocol`](./153-automatic-software-update-protocol.txt): Automatic software update protocol (SUPERSEDED) + - [`154-automatic-updates`](./154-automatic-updates.txt): Automatic Software Update Protocol (SUPERSEDED) + - [`155-four-hidden-service-improvements`](./155-four-hidden-service-improvements.txt): Four Improvements of Hidden Service Performance (CLOSED) + - [`156-tracking-blocked-ports`](./156-tracking-blocked-ports.txt): Tracking blocked ports on the client side (SUPERSEDED) + - [`157-specific-cert-download`](./157-specific-cert-download.txt): Make certificate downloads specific (CLOSED) + - [`158-microdescriptors`](./158-microdescriptors.txt): Clients download consensus + microdescriptors (CLOSED) + - [`159-exit-scanning`](./159-exit-scanning.txt): Exit Scanning (INFORMATIONAL) + - [`160-bandwidth-offset`](./160-bandwidth-offset.txt): Authorities vote for bandwidth offsets in consensus (CLOSED) + - [`161-computing-bandwidth-adjustments`](./161-computing-bandwidth-adjustments.txt): Computing Bandwidth Adjustments (CLOSED) + - [`162-consensus-flavors`](./162-consensus-flavors.txt): Publish the consensus in multiple flavors (CLOSED) + - [`163-detecting-clients`](./163-detecting-clients.txt): Detecting whether a connection comes from a client (SUPERSEDED) + - [`164-reporting-server-status`](./164-reporting-server-status.txt): Reporting the status of server votes (OBSOLETE) + - [`165-simple-robust-voting`](./165-simple-robust-voting.txt): Easy migration for voting authority sets (REJECTED) + - [`166-statistics-extra-info-docs`](./166-statistics-extra-info-docs.txt): Including Network Statistics in Extra-Info Documents (CLOSED) + - [`167-params-in-consensus`](./167-params-in-consensus.txt): Vote on network parameters in consensus (CLOSED) + - [`168-reduce-circwindow`](./168-reduce-circwindow.txt): Reduce default circuit window (REJECTED) + - [`169-eliminating-renegotiation`](./169-eliminating-renegotiation.txt): Eliminate TLS renegotiation for the Tor connection handshake (SUPERSEDED) + - [`170-user-path-config`](./170-user-path-config.txt): Configuration options regarding circuit building (SUPERSEDED) + - [`171-separate-streams`](./171-separate-streams.txt): Separate streams across circuits by connection metadata (CLOSED) + - [`172-circ-getinfo-option`](./172-circ-getinfo-option.txt): GETINFO controller option for circuit information (RESERVE) + - [`173-getinfo-option-expansion`](./173-getinfo-option-expansion.txt): GETINFO Option Expansion (OBSOLETE) + - [`174-optimistic-data-server`](./174-optimistic-data-server.txt): Optimistic Data for Tor: Server Side (CLOSED) + - [`175-automatic-node-promotion`](./175-automatic-node-promotion.txt): Automatically promoting Tor clients to nodes (REJECTED) + - [`176-revising-handshake`](./176-revising-handshake.txt): Proposed version-3 link handshake for Tor (CLOSED) + - [`177-flag-abstention`](./177-flag-abstention.txt): Abstaining from votes on individual flags (RESERVE) + - [`178-param-voting`](./178-param-voting.txt): Require majority of authorities to vote for consensus parameters (CLOSED) + - [`179-TLS-cert-and-parameter-normalization`](./179-TLS-cert-and-parameter-normalization.txt): TLS certificate and parameter normalization (CLOSED) + - [`180-pluggable-transport`](./180-pluggable-transport.txt): Pluggable transports for circumvention (CLOSED) + - [`181-optimistic-data-client`](./181-optimistic-data-client.txt): Optimistic Data for Tor: Client Side (CLOSED) + - [`182-creditbucket`](./182-creditbucket.txt): Credit Bucket (OBSOLETE) + - [`183-refillintervals`](./183-refillintervals.txt): Refill Intervals (CLOSED) + - [`184-v3-link-protocol`](./184-v3-link-protocol.txt): Miscellaneous changes for a v3 Tor link protocol (CLOSED) + - [`185-dir-without-dirport`](./185-dir-without-dirport.txt): Directory caches without DirPort (SUPERSEDED) + - [`186-multiple-orports`](./186-multiple-orports.txt): Multiple addresses for one OR or bridge (CLOSED) + - [`187-allow-client-auth`](./187-allow-client-auth.txt): Reserve a cell type to allow client authorization (CLOSED) + - [`188-bridge-guards`](./188-bridge-guards.txt): Bridge Guards and other anti-enumeration defenses (RESERVE) + - [`189-authorize-cell`](./189-authorize-cell.txt): AUTHORIZE and AUTHORIZED cells (OBSOLETE) + - [`190-shared-secret-bridge-authorization`](./190-shared-secret-bridge-authorization.txt): Bridge Client Authorization Based on a Shared Secret (OBSOLETE) + - [`191-mitm-bridge-detection-resistance`](./191-mitm-bridge-detection-resistance.txt): Bridge Detection Resistance against MITM-capable Adversaries (OBSOLETE) + - [`192-store-bridge-information`](./192-store-bridge-information.txt): Automatically retrieve and store information about bridges (OBSOLETE) + - [`193-safe-cookie-authentication`](./193-safe-cookie-authentication.txt): Safe cookie authentication for Tor controllers (CLOSED) + - [`194-mnemonic-urls`](./194-mnemonic-urls.txt): Mnemonic .onion URLs (SUPERSEDED) + - [`195-TLS-normalization-for-024`](./195-TLS-normalization-for-024.txt): TLS certificate normalization for Tor 0.2.4.x (DEAD) + - [`196-transport-control-ports`](./196-transport-control-ports.txt): Extended ORPort and TransportControlPort (CLOSED) + - [`197-postmessage-ipc`](./197-postmessage-ipc.txt): Message-based Inter-Controller IPC Channel (REJECTED) + - [`198-restore-clienthello-semantics`](./198-restore-clienthello-semantics.txt): Restore semantics of TLS ClientHello (CLOSED) + - [`199-bridgefinder-integration`](./199-bridgefinder-integration.txt): Integration of BridgeFinder and BridgeFinderHelper (OBSOLETE) + - [`200-new-create-and-extend-cells`](./200-new-create-and-extend-cells.txt): Adding new, extensible CREATE, EXTEND, and related cells (CLOSED) + - [`201-bridge-v3-reqs-stats`](./201-bridge-v3-reqs-stats.txt): Make bridges report statistics on daily v3 network status requests (RESERVE) + - [`202-improved-relay-crypto`](./202-improved-relay-crypto.txt): Two improved relay encryption protocols for Tor cells (META) + - [`203-https-frontend`](./203-https-frontend.txt): Avoiding censorship by impersonating an HTTPS server (OBSOLETE) + - [`204-hidserv-subdomains`](./204-hidserv-subdomains.txt): Subdomain support for Hidden Service addresses (CLOSED) + - [`205-local-dnscache`](./205-local-dnscache.txt): Remove global client-side DNS caching (CLOSED) + - [`206-directory-sources`](./206-directory-sources.txt): Preconfigured directory sources for bootstrapping (CLOSED) + - [`207-directory-guards`](./207-directory-guards.txt): Directory guards (CLOSED) + - [`208-ipv6-exits-redux`](./208-ipv6-exits-redux.txt): IPv6 Exits Redux (CLOSED) + - [`209-path-bias-tuning`](./209-path-bias-tuning.txt): Tuning the Parameters for the Path Bias Defense (OBSOLETE) + - [`210-faster-headless-consensus-bootstrap`](./210-faster-headless-consensus-bootstrap.txt): Faster Headless Consensus Bootstrapping (SUPERSEDED) + - [`211-mapaddress-tor-status`](./211-mapaddress-tor-status.txt): Internal Mapaddress for Tor Configuration Testing (RESERVE) + - [`212-using-old-consensus`](./212-using-old-consensus.txt): Increase Acceptable Consensus Age (NEEDS-REVISION) + - [`213-remove-stream-sendmes`](./213-remove-stream-sendmes.txt): Remove stream-level sendmes from the design (DEAD) + - [`214-longer-circids`](./214-longer-circids.txt): Allow 4-byte circuit IDs in a new link protocol (CLOSED) + - [`215-update-min-consensus-ver`](./215-update-min-consensus-ver.txt): Let the minimum consensus method change with time (CLOSED) + - [`216-ntor-handshake`](./216-ntor-handshake.txt): Improved circuit-creation key exchange (CLOSED) + - [`217-ext-orport-auth`](./217-ext-orport-auth.txt): Tor Extended ORPort Authentication (CLOSED) + - [`218-usage-controller-events`](./218-usage-controller-events.txt): Controller events to better understand connection/circuit usage (CLOSED) + - [`219-expanded-dns`](./219-expanded-dns.txt): Support for full DNS and DNSSEC resolution in Tor (NEEDS-REVISION) + - [`220-ecc-id-keys`](./220-ecc-id-keys.txt): Migrate server identity keys to Ed25519 (CLOSED) + - [`221-stop-using-create-fast`](./221-stop-using-create-fast.txt): Stop using CREATE_FAST (CLOSED) + - [`222-remove-client-timestamps`](./222-remove-client-timestamps.txt): Stop sending client timestamps (CLOSED) + - [`223-ace-handshake`](./223-ace-handshake.txt): Ace: Improved circuit-creation key exchange (RESERVE) + - [`224-rend-spec-ng`](./224-rend-spec-ng.txt): Next-Generation Hidden Services in Tor (CLOSED) + - [`225-strawman-shared-rand`](./225-strawman-shared-rand.txt): Strawman proposal: commit-and-reveal shared rng (SUPERSEDED) + - [`226-bridgedb-database-improvements`](./226-bridgedb-database-improvements.txt): "Scalability and Stability Improvements to BridgeDB: Switching to a Distributed Database System and RDBMS" (RESERVE) + - [`227-vote-on-package-fingerprints`](./227-vote-on-package-fingerprints.txt): Include package fingerprints in consensus documents (CLOSED) + - [`228-cross-certification-onionkeys`](./228-cross-certification-onionkeys.txt): Cross-certifying identity keys with onion keys (CLOSED) + - [`229-further-socks5-extensions`](./229-further-socks5-extensions.txt): Further SOCKS5 extensions (REJECTED) + - [`230-rsa1024-relay-id-migration`](./230-rsa1024-relay-id-migration.txt): How to change RSA1024 relay identity keys (OBSOLETE) + - [`231-migrate-authority-rsa1024-ids`](./231-migrate-authority-rsa1024-ids.txt): Migrating authority RSA1024 identity keys (OBSOLETE) + - [`232-pluggable-transports-through-proxy`](./232-pluggable-transports-through-proxy.txt): Pluggable Transport through SOCKS proxy (CLOSED) + - [`233-quicken-tor2web-mode`](./233-quicken-tor2web-mode.txt): Making Tor2Web mode faster (REJECTED) + - [`234-remittance-addresses`](./234-remittance-addresses.txt): Adding remittance field to directory specification (REJECTED) + - [`235-kill-named-flag`](./235-kill-named-flag.txt): Stop assigning (and eventually supporting) the Named flag (CLOSED) + - [`236-single-guard-node`](./236-single-guard-node.txt): The move to a single guard node (CLOSED) + - [`237-directory-servers-for-all`](./237-directory-servers-for-all.txt): All relays are directory servers (CLOSED) + - [`238-hs-relay-stats`](./238-hs-relay-stats.txt): Better hidden service stats from Tor relays (CLOSED) + - [`239-consensus-hash-chaining`](./239-consensus-hash-chaining.txt): Consensus Hash Chaining (OPEN) + - [`240-auth-cert-revocation`](./240-auth-cert-revocation.txt): Early signing key revocation for directory authorities (OPEN) + - [`241-suspicious-guard-turnover`](./241-suspicious-guard-turnover.txt): Resisting guard-turnover attacks (REJECTED) + - [`242-better-families`](./242-better-families.txt): Better performance and usability for the MyFamily option (SUPERSEDED) + - [`243-hsdir-flag-need-stable`](./243-hsdir-flag-need-stable.txt): Give out HSDir flag only to relays with Stable flag (CLOSED) + - [`244-use-rfc5705-for-tls-binding`](./244-use-rfc5705-for-tls-binding.txt): Use RFC5705 Key Exporting in our AUTHENTICATE calls (CLOSED) + - [`245-tap-out`](./245-tap-out.txt): Deprecating and removing the TAP circuit extension protocol (NEEDS-REVISION) + - [`246-merge-hsdir-and-intro`](./246-merge-hsdir-and-intro.txt): Merging Hidden Service Directories and Introduction Points (REJECTED) + - [`247-hs-guard-discovery`](./247-hs-guard-discovery.txt): Defending Against Guard Discovery Attacks using Vanguards (SUPERSEDED) + - [`248-removing-rsa-identities`](./248-removing-rsa-identities.txt): Remove all RSA identity keys (NEEDS-REVISION) + - [`249-large-create-cells`](./249-large-create-cells.txt): Allow CREATE cells with >505 bytes of handshake data (SUPERSEDED) + - [`250-commit-reveal-consensus`](./250-commit-reveal-consensus.txt): Random Number Generation During Tor Voting (CLOSED) + - [`251-netflow-padding`](./251-netflow-padding.txt): Padding for netflow record resolution reduction (CLOSED) + - [`252-single-onion`](./252-single-onion.txt): Single Onion Services (SUPERSEDED) + - [`253-oob-hmac`](./253-oob-hmac.txt): Out of Band Circuit HMACs (DEAD) + - [`254-padding-negotiation`](./254-padding-negotiation.txt): Padding Negotiation (CLOSED) + - [`255-hs-load-balancing`](./255-hs-load-balancing.txt): Controller features to allow for load-balancing hidden services (RESERVE) + - [`256-key-revocation`](./256-key-revocation.txt): Key revocation for relays and authorities (RESERVE) + - [`257-hiding-authorities`](./257-hiding-authorities.txt): Refactoring authorities and making them more isolated from the net (META) + - [`258-dirauth-dos`](./258-dirauth-dos.txt): Denial-of-service resistance for directory authorities (DEAD) + - [`259-guard-selection`](./259-guard-selection.txt): New Guard Selection Behaviour (OBSOLETE) + - [`260-rend-single-onion`](./260-rend-single-onion.txt): Rendezvous Single Onion Services (FINISHED) + - [`261-aez-crypto`](./261-aez-crypto.txt): AEZ for relay cryptography (OBSOLETE) + - [`262-rekey-circuits`](./262-rekey-circuits.txt): Re-keying live circuits with new cryptographic material (RESERVE) + - [`263-ntru-for-pq-handshake`](./263-ntru-for-pq-handshake.txt): Request to change key exchange protocol for handshake v1.2 (OBSOLETE) + - [`264-subprotocol-versions`](./264-subprotocol-versions.txt): Putting version numbers on the Tor subprotocols (CLOSED) + - [`265-load-balancing-with-overhead`](./265-load-balancing-with-overhead.txt): Load Balancing with Overhead Parameters (OPEN) + - [`266-removing-current-obsolete-clients`](./266-removing-current-obsolete-clients.txt): Removing current obsolete clients from the Tor network (SUPERSEDED) + - [`267-tor-consensus-transparency`](./267-tor-consensus-transparency.txt): Tor Consensus Transparency (OPEN) + - [`268-guard-selection`](./268-guard-selection.txt): New Guard Selection Behaviour (OBSOLETE) + - [`269-hybrid-handshake`](./269-hybrid-handshake.txt): Transitionally secure hybrid handshakes (NEEDS-REVISION) + - [`270-newhope-hybrid-handshake`](./270-newhope-hybrid-handshake.txt): RebelAlliance: A Post-Quantum Secure Hybrid Handshake Based on NewHope (OBSOLETE) + - [`271-another-guard-selection`](./271-another-guard-selection.txt): Another algorithm for guard selection (CLOSED) + - [`272-valid-and-running-by-default`](./272-valid-and-running-by-default.txt): Listed routers should be Valid, Running, and treated as such (CLOSED) + - [`273-exit-relay-pinning`](./273-exit-relay-pinning.txt): Exit relay pinning for web services (RESERVE) + - [`274-rotate-onion-keys-less`](./274-rotate-onion-keys-less.txt): Rotate onion keys less frequently (CLOSED) + - [`275-md-published-time-is-silly`](./275-md-published-time-is-silly.txt): Stop including meaningful "published" time in microdescriptor consensus (CLOSED) + - [`276-lower-bw-granularity`](./276-lower-bw-granularity.txt): Report bandwidth with lower granularity in consensus documents (DEAD) + - [`277-detect-id-sharing`](./277-detect-id-sharing.txt): Detect multiple relay instances running with same ID (OPEN) + - [`278-directory-compression-scheme-negotiation`](./278-directory-compression-scheme-negotiation.txt): Directory Compression Scheme Negotiation (CLOSED) + - [`279-naming-layer-api`](./279-naming-layer-api.txt): A Name System API for Tor Onion Services (NEEDS-REVISION) + - [`280-privcount-in-tor`](./280-privcount-in-tor.txt): Privacy-Preserving Statistics with Privcount in Tor (SUPERSEDED) + - [`281-bulk-md-download`](./281-bulk-md-download.txt): Downloading microdescriptors in bulk (RESERVE) + - [`282-remove-named-from-consensus`](./282-remove-named-from-consensus.txt): Remove "Named" and "Unnamed" handling from consensus voting (ACCEPTED) + - [`283-ipv6-in-micro-consensus`](./283-ipv6-in-micro-consensus.txt): Move IPv6 ORPorts from microdescriptors to the microdesc consensus (CLOSED) + - [`284-hsv3-control-port`](./284-hsv3-control-port.txt): Hidden Service v3 Control Port (CLOSED) + - [`285-utf-8`](./285-utf-8.txt): Directory documents should be standardized as UTF-8 (ACCEPTED) + - [`286-hibernation-api`](./286-hibernation-api.txt): Controller APIs for hibernation access on mobile (REJECTED) + - [`287-reduce-lifetime`](./287-reduce-lifetime.txt): Reduce circuit lifetime without overloading the network (OPEN) + - [`288-privcount-with-shamir`](./288-privcount-with-shamir.txt): Privacy-Preserving Statistics with Privcount in Tor (Shamir version) (RESERVE) + - [`289-authenticated-sendmes`](./289-authenticated-sendmes.txt): Authenticating sendme cells to mitigate bandwidth attacks (CLOSED) + - [`290-deprecate-consensus-methods`](./290-deprecate-consensus-methods.txt): Continuously update consensus methods (META) + - [`291-two-guard-nodes`](./291-two-guard-nodes.txt): The move to two guard nodes (FINISHED) + - [`292-mesh-vanguards`](./292-mesh-vanguards.txt): Mesh-based vanguards (CLOSED) + - [`293-know-when-to-publish`](./293-know-when-to-publish.txt): Other ways for relays to know when to publish (CLOSED) + - [`294-tls-1.3`](./294-tls-1.3.txt): TLS 1.3 Migration (DRAFT) + - [`295-relay-crypto-with-adl`](./295-relay-crypto-with-adl.txt): Using ADL for relay cryptography (solving the crypto-tagging attack) (OPEN) + - [`296-expose-bandwidth-files`](./296-expose-bandwidth-files.txt): Have Directory Authorities expose raw bandwidth list files (CLOSED) + - [`297-safer-protover-shutdowns`](./297-safer-protover-shutdowns.txt): Relaxing the protover-based shutdown rules (CLOSED) + - [`298-canonical-families`](./298-canonical-families.txt): Putting family lines in canonical form (CLOSED) + - [`299-ip-failure-count`](./299-ip-failure-count.txt): Preferring IPv4 or IPv6 based on IP Version Failure Count (SUPERSEDED) + - [`300-walking-onions`](./300-walking-onions.txt): Walking Onions: Scaling and Saving Bandwidth (INFORMATIONAL) + - [`301-dont-vote-on-package-fingerprints`](./301-dont-vote-on-package-fingerprints.txt): Don't include package fingerprints in consensus documents (CLOSED) + - [`302-padding-machines-for-onion-clients`](./302-padding-machines-for-onion-clients.txt): Hiding onion service clients using padding (CLOSED) + - [`303-protover-removal-policy`](./303-protover-removal-policy.txt): When and how to remove support for protocol versions (OPEN) + - [`304-socks5-extending-hs-error-codes`](./304-socks5-extending-hs-error-codes.txt): Extending SOCKS5 Onion Service Error Codes (CLOSED) + - [`305-establish-intro-dos-defense-extention`](./305-establish-intro-dos-defense-extention.txt): ESTABLISH_INTRO Cell DoS Defense Extension (CLOSED) + - [`306-ipv6-happy-eyeballs`](./306-ipv6-happy-eyeballs.txt): A Tor Implementation of IPv6 Happy Eyeballs (OPEN) + - [`307-onionbalance-v3`](./307-onionbalance-v3.txt): Onion Balance Support for Onion Service v3 (RESERVE) + - [`308-counter-galois-onion`](./308-counter-galois-onion.txt): Counter Galois Onion: A New Proposal for Forward-Secure Relay Cryptography (SUPERSEDED) + - [`309-optimistic-socks-in-tor`](./309-optimistic-socks-in-tor.txt): Optimistic SOCKS Data (OPEN) + - [`310-bandaid-on-guard-selection`](./310-bandaid-on-guard-selection.txt): Towards load-balancing in Prop 271 (CLOSED) + - [`311-relay-ipv6-reachability`](./311-relay-ipv6-reachability.txt): Tor Relay IPv6 Reachability (ACCEPTED) + - [`312-relay-auto-ipv6-addr`](./312-relay-auto-ipv6-addr.txt): Tor Relay Automatic IPv6 Address Discovery (ACCEPTED) + - [`313-relay-ipv6-stats`](./313-relay-ipv6-stats.txt): Tor Relay IPv6 Statistics (ACCEPTED) + - [`314-allow-markdown-proposals`](./314-allow-markdown-proposals.md): Allow Markdown for proposal format (CLOSED) + - [`315-update-dir-required-fields`](./315-update-dir-required-fields.txt): Updating the list of fields required in directory documents (CLOSED) + - [`316-flashflow`](./316-flashflow.md): FlashFlow: A Secure Speed Test for Tor (Parent Proposal) (DRAFT) + - [`317-secure-dns-name-resolution`](./317-secure-dns-name-resolution.txt): Improve security aspects of DNS name resolution (NEEDS-REVISION) + - [`318-limit-protovers`](./318-limit-protovers.md): Limit protover values to 0-63 (CLOSED) + - [`319-wide-everything`](./319-wide-everything.md): RELAY_FRAGMENT cells (OBSOLETE) + - [`320-tap-out-again`](./320-tap-out-again.md): Removing TAP usage from v2 onion services (REJECTED) + - [`321-happy-families`](./321-happy-families.md): Better performance and usability for the MyFamily option (v2) (ACCEPTED) + - [`322-dirport-linkspec`](./322-dirport-linkspec.md): Extending link specifiers to include the directory port (OPEN) + - [`323-walking-onions-full`](./323-walking-onions-full.md): Specification for Walking Onions (OPEN) + - [`324-rtt-congestion-control`](./324-rtt-congestion-control.txt): RTT-based Congestion Control for Tor (FINISHED) + - [`325-packed-relay-cells`](./325-packed-relay-cells.md): Packed relay cells: saving space on small commands (OBSOLETE) + - [`326-tor-relay-well-known-uri-rfc8615`](./326-tor-relay-well-known-uri-rfc8615.md): The "tor-relay" Well-Known Resource Identifier (OPEN) + - [`327-pow-over-intro`](./327-pow-over-intro.txt): A First Take at PoW Over Introduction Circuits (CLOSED) + - [`328-relay-overload-report`](./328-relay-overload-report.md): Make Relays Report When They Are Overloaded (CLOSED) + - [`329-traffic-splitting`](./329-traffic-splitting.txt): Overcoming Tor's Bottlenecks with Traffic Splitting (FINISHED) + - [`330-authority-contact`](./330-authority-contact.md): Modernizing authority contact entries (OPEN) + - [`331-res-tokens-for-anti-dos`](./331-res-tokens-for-anti-dos.md): Res tokens: Anonymous Credentials for Onion Service DoS Resilience (DRAFT) + - [`332-ntor-v3-with-extra-data`](./332-ntor-v3-with-extra-data.md): Ntor protocol with extra data, version 3 (CLOSED) + - [`333-vanguards-lite`](./333-vanguards-lite.md): Vanguards lite (CLOSED) + - [`334-middle-only-flag`](./334-middle-only-flag.txt): A Directory Authority Flag To Mark Relays As Middle-only (SUPERSEDED) + - [`335-middle-only-redux`](./335-middle-only-redux.md): An authority-only design for MiddleOnly (CLOSED) + - [`336-randomize-guard-retries`](./336-randomize-guard-retries.md): Randomized schedule for guard retries (CLOSED) + - [`337-simpler-guard-usability`](./337-simpler-guard-usability.md): A simpler way to decide, "Is this guard usable?" (CLOSED) + - [`338-netinfo-y2038`](./338-netinfo-y2038.md): Use an 8-byte timestamp in NETINFO cells (ACCEPTED) + - [`339-udp-over-tor`](./339-udp-over-tor.md): UDP traffic over Tor (ACCEPTED) + - [`340-packed-and-fragmented`](./340-packed-and-fragmented.md): Packed and fragmented relay messages (OPEN) + - [`341-better-oos`](./341-better-oos.md): A better algorithm for out-of-sockets eviction (OPEN) + - [`342-decouple-hs-interval`](./342-decouple-hs-interval.md): Decoupling hs_interval and SRV lifetime (DRAFT) + - [`343-rend-caa`](./343-rend-caa.txt): CAA Extensions for the Tor Rendezvous Specification (OPEN) + - [`344-protocol-info-leaks`](./344-protocol-info-leaks.txt): Prioritizing Protocol Information Leaks in Tor (OPEN) + - [`345-specs-in-mdbook`](./345-specs-in-mdbook.md): Migrating the tor specifications to mdbook (CLOSED) + - [`346-protovers-again`](./346-protovers-again.md): Clarifying and extending the use of protocol versioning (OPEN) + - [`347-domain-separation`](./347-domain-separation.md): Domain separation for certificate signing keys (OPEN) + - [`348-udp-app-support`](./348-udp-app-support.md): UDP Application Support in Tor (OPEN) + + + diff --git a/proposals/SUMMARY_template.md b/proposals/SUMMARY_template.md new file mode 100644 index 0000000..3ca7f6e --- /dev/null +++ b/proposals/SUMMARY_template.md @@ -0,0 +1,11 @@ +# Summary + +[Tor Proposals](README.md) +[All proposals, by number](BY_INDEX.md) +[All proposals, by status](BY_STATUS.md) + +# Proposals + +{SUMMARY_TABLE} + + diff --git a/proposals/proposal-status.txt b/proposals/proposal-status.txt index 8a709bc..af522d1 100644 --- a/proposals/proposal-status.txt +++ b/proposals/proposal-status.txt @@ -1,3 +1,4 @@ +``` [Last updated 9 Sep 2015] This is an update to the Tor proposal status overview. I last sent one @@ -553,3 +554,4 @@ again to remind me! this writing) and which parts of our current, more idiosyncratic, uses of TLS can be removed. +``` diff --git a/spec/README.md b/spec/README.md new file mode 100644 index 0000000..920486d --- /dev/null +++ b/spec/README.md @@ -0,0 +1,46 @@ +# Tor specifications + +These specifications describe how Tor works. They try to present +Tor's protocols in sufficient detail to allow the reader to implement +a compatible implementation of Tor without ever having to read the Tor +source code. + +They were once a separate set of text files, but in late 2023 we +migrated to use [`mdbook`](https://rust-lang.github.io/mdBook/). +We're in the process of updating these documents to improve their quality. + +This is a living document: we are always changing and improving them +in order to make them easier and more accurate, and to improve the +quality of the Tor protocols. They are maintained as a set of +documents in a +[gitlab repository](https://gitlab.torproject.org/tpo/core/torspec/); +you can use that repository to see their history. + +Additionally, the <a href="./proposals/index.html">proposals</a> +directory holds our design proposals. These include historical +documents that have now been merged into the main specifications, and +new proposals that are still under discussion. Of particular +interrest are the +<a href="./proposals/BY_STATUS.html#finished-proposals-implemented-specs-not-merged">`FINISHED` +Tor proposals</a>: They are the ones that have been implemented, but +not yet merged into these documents. + +## Getting started + +There's a lot of material here, and it's not always as well organized +as we like. We have broken it into a few major sections. + +For a table of contents, click on the menu icon to the top-left of +your browser scene. You should probably start by reading [the core +tor protocol specification](./tor-spec), which describes how our +protocol works. After that, you should be able to jump around to +the topics that interest you most. The introduction of each top-level +chapter should provide an introduction. +______________________________________________________________________ + +[git repository]: https://gitlab.torproject.org/tpo/core/torspec/ +[heading ids]: https://github.com/raphlinus/pulldown-cmark/blob/master/specs/heading_attrs.txt +[mdbook documentation]: https://rust-lang.github.io/mdBook/format/summary.html +[`spec/tor-spec/flow-control.md`]: https://gitlab.torproject.org/tpo/core/torspec/-/blob/main/spec/tor-spec/flow-control.md?ref_type=heads +[`spec` directory]: https://gitlab.torproject.org/tpo/core/torspec/-/tree/main/spec?ref_type=heads +[`summary.md`]: https://gitlab.torproject.org/tpo/core/torspec/-/raw/main/spec/SUMMARY.md?ref_type=heads diff --git a/spec/STYLE.md b/spec/STYLE.md new file mode 100644 index 0000000..8111084 --- /dev/null +++ b/spec/STYLE.md @@ -0,0 +1,189 @@ +# Tor specifications: style and usage notes + +## Audience + +The primary intended audiences are: + + * Programmers: + implementors of the Tor Protocols. + This includes both maintainers of existing implementations, + and people writing new implementations. + + * Researchers: + people analysing the security of the Tor Protocols, + including both academic research and + practical security investigation. + + * Expert users who wish to fully understand + the operation of their Tor software. + This includes users of clients and relays. + + * Directory authority operators, + and others with a critical technical role + in the integrity of the Tor network. + + +## Scope and intentions + +These notes apply to our specifications. When possible, they should +also apply to proposals, to make proposals easier to merge into our +specifications when they are done. + +As of 2023, our existing specifications have been written without any +real style guidelines, so you should not expect to find these +guidelines to apply well to all of the documents as you read them. +Instead, these guidelines are for documentation going forward. + +These notes are not terribly well organized. We should improve them +over time. They are meant to be a living document. + + + +## Other sources + +There are a number of other style guides used in Tor. We should +follow these as well. If they do not suit our needs, we should try to +get them changed. + +* [Community team guidelines](https://gitlab.torproject.org/tpo/community/team/-/issues/5) +* [Tor project glossary](https://support.torproject.org/glossary/) +* [Specifications glossary](./glossary.md) + +(Please add more if you know about them!) + +As we refine the guidelines in this file, we should attempt to get +them upstreamed to more project-wide guides, if they are suitable. + +## Starting notes + +We are moving towards using +[semantic newlines](https://rhodesmill.org/brandon/2012/one-sentence-per-line/): +put each thought, or sentence, or subclause, on its own line, +in the Markdown source text. +(This is not yet applied consistently, +and we won't reject contributions that don't write that way.) + +## Vocabulary + +We use these terms freely: + * Channel + * Circuit + * Stream + +We try not to say "connection" without qualification: There are too +many things in Tor that can be called a "connection". + +Similarly, don't say "session" without qualification except when it is +clear from context what we mean. + +Prefer "relay" to "node" or "server". + +Prefer "service" and "client" when talking about onion services and +their users. + +Refer to arti as arti and the C tor implementation as "the C tor +implementation" on first mention. Subsequently you can call it `tor` +or "C tor". + +Avoid "AP" and "OP" and "OR"; those are not in current usage. + +## Documenting keys + +TODO: Explain our new key documentation conventions, as used +[here](./rend-spec/protocol-overview.html#imd-key-menagerie). + +## Documentating data encodings + +We have two competing legacy schemes for documenting data encodings. +One of them is an ad-hoc format the looks like this: +``` + * FIELD_1 [4 bytes] + * FIELD_2 [1 byte] +``` + +The other is a somewhat C-like format based on the +[`trunnel` format](https://gitlab.torproject.org/tpo/core/trunnel/-/blob/main/doc/trunnel.md?ref_type=heads). +It looks like this: +``` +struct message { + u32 field_1; + u8 field_2; +} +``` + +Neither of these is really great. We should find something better. + +## Writing explanations + +> When you are writing an explanation in the middle of a bunch of +> normative text, it is a good idea to put it in quoted text, like +> this. + + +## Managing links + +We're in the early stages of this spec organization, but we should +still be thinking about long term maintainability. + +Please think about how to keep links working in the long term. +If you are going to add a link to a file, make sure that the file's +name is reasonable. Before you rename a file, consider adding +a redirect from the file's old name. (See the mdbook documentation +for more information about how.) + +<div id="mdbook-heading-anchors"> + +If you want to link to a specific section within a file, +make sure that the section has a defined anchor that makes sense. +The syntax to define heading ids in mdbook looks like this: + +`## Heading with a long title that you want shorter name for { #shortname }` + +</div> + +If you need to change a heading, make sure that you keep its +`id` the same as it was before, so that links will still work. + +Finally, when you're looking for specific sections (e.g., to fix +references that say "See section 5.2.3") you can look for the HTML +anchors that our conversion process added. For example, +if you want to find `dir-spec.txt` section 2.1.3, look for +the anchor that says `<a id="dir-spec.txt-2.1.3"></a>`. + +## Specific Markdown syntax + +### Define link fragment targets with `<span id="...">` (usually) + +To manually make an target for a `#`-prefixed link fragment, +prefer `<span id="fragment">Text</span>`, +to `<a id=...>Text</a>`. +This works around mdbook mistakenly styling +`<a>` without `href` +as if it were a clickable link. + +(Of course it is often better to +make the referenced text a section +and +[use the mdbook explicit anchor syntax](#mdbook-heading-anchors).) + +You may need to make sure you have some other text +on the same line as the `<span>` +to avoid mdbook thinking you were writing +a whole paragraph of raw HTML. + +Sometimes you may wish to use `<div id="...">` and `</div>` +(which must usually be placed as paragraphs of their own). + +Example: +``` +<span id="lemons-lemonade">LEMONS +are a sour fruit, sometimes used for making +lemonade</span> + +(later) + +Various fruit may be found, including [lemons](#lemons-lemonade). +``` + +It is OK to use an *empty* `a` element: `<a id=....></a>`. +Many existing section anchors are done this way. diff --git a/spec/SUMMARY.md b/spec/SUMMARY.md new file mode 100644 index 0000000..ade9586 --- /dev/null +++ b/spec/SUMMARY.md @@ -0,0 +1,195 @@ +# Summary + +[About these specifications](./README.md) + +# Introduction + +- [A short introduction to Tor](./intro/index.md) + - [Notation and conventions](./intro/conventions.md) + +# The core Tor protocol + +- [`Tor Protocol Specification`](./tor-spec/index.md) + - [Preliminaries](./tor-spec/preliminaries.md) + - [Relay keys and identities](./tor-spec/relay-keys.md) + - [Channels](./tor-spec/channels.md) + - [Negotiating and initializing channels](./tor-spec/negotiating-channels.md) + - [Obsolete channel negotiation handshakes](./tor-spec/obsolete-channels.md) + - [Cells (messages on channels)](./tor-spec/cell-packet-format.md) + - [Circuit management](./tor-spec/circuit-management.md) + - [CREATE and CREATED cells](./tor-spec/create-created-cells.md) + - [Setting circuit keys](./tor-spec/setting-circuit-keys.md) + - [Creating circuits](./tor-spec/creating-circuits.md) + - [Tearing down circuits](./tor-spec/tearing-down-circuits.md) + - [Routing relay cells](./tor-spec/routing-relay-cells.md) + - [Handling relay_early cells](./tor-spec/relay-early.md) + - [Application connections and stream management](./tor-spec/streams.md) + - [Relay cells](./tor-spec/relay-cells.md) + - [Opening streams and transferring data](./tor-spec/opening-streams.md) + - [Closing streams](./tor-spec/closing-streams.md) + - [Remote hostname lookup](./tor-spec/remote-hostname-lookup.md) + - [Flow control](./tor-spec/flow-control.md) + - [Subprotocol versioning](./tor-spec/subprotocol-versioning.md) + - [Certificates in Tor](./cert-spec.md) +- [`Tor directory protocol, version 3`](./dir-spec/index.md) + - [Outline](./dir-spec/outline.md) + - [Router operation and formats](./dir-spec/router-operation-formats.md) + - [Uploading server descriptors and extra-info documents](./dir-spec/uploading-relay-documents.md) + - [Server descriptor format](./dir-spec/server-descriptor-format.md) + - [Extra-info document format](./dir-spec/extra-info-document-format.md) + - [Nonterminals in server descriptors](./dir-spec/nonterminals-server-descriptors.md) + - [Directory authority operation and formats](./dir-spec/directory-authority-operation-formats.md) + - [Creating key certificates](./dir-spec/creating-key-certificates.md) + - [Accepting server descriptor and extra-info document uploads](./dir-spec/accepting-relay-documents.md) + - [Computing microdescriptors](./dir-spec/computing-microdescriptors.md) + - [Exchanging votes](./dir-spec/exchanging-votes.md) + - [Vote and consensus status document formats](./dir-spec/consensus-formats.md) + - [Assigning flags in a vote](./dir-spec/assigning-flags-vote.md) + - [Serving bandwidth list files](./dir-spec/serving-bandwidth-list-files.md) + - [Downloading information from other directory authorities](./dir-spec/downloading-from-other-auths.md) + - [Computing a consensus from a set of votes](./dir-spec/computing-consensus.md) + - [Exchanging detached signatures](./dir-spec/exchanging-detached-signatures.md) + - [Publishing the signed consensus](./dir-spec/publishing-consensus.md) + - [Directory cache operation](./dir-spec/directory-cache-operation.md) + - [Client operation](./dir-spec/client-operation.md) + - [Standards compliance](./dir-spec/standards-compliance.md) + - [Consensus-negotiation timeline.](./dir-spec/consensus-negotiation-timeline.md) + - [General-use HTTP URLs](./dir-spec/general-use-http-urls.md) + - [Converting a curve25519 public key to an ed25519 public key](./dir-spec/converting-to-ed25519.md) + - [Inferring missing proto lines.](./dir-spec/inferring-missing-proto-lines.md) + - [Limited ed diff format](./dir-spec/limited-ed-diff-format.md) + - [`Tor Shared Random Subsystem Specification`](./srv-spec/index.md) + - [Introduction](./srv-spec/introduction.md) + - [Overview](./srv-spec/overview.md) + - [Protocol](./srv-spec/protocol.md) + - [Specification \[SPEC\]](./srv-spec/specification.md) + - [Security Analysis](./srv-spec/security-analysis.md) + - [Discussion](./srv-spec/discussion.md) + - [Acknowledgements](./srv-spec/acknowledgements.md) +- [`Tor Path Specification`](./path-spec/index.md) + - [General operation](./path-spec/general-operation.md) + - [Building circuits](./path-spec/building-circuits.md) + - [When we build](./path-spec/when-we-build.md) + - [Path selection and constraints](./path-spec/path-selection-constraints.md) + - [Cannibalizing circuits](./path-spec/cannibalizing-circuits.md) + - [Learning when to give up ("timeout") on circuit construction](./path-spec/learning-timeouts.md) + - [Handling failure](./path-spec/handling-failure.md) + - [Attaching streams to circuits](./path-spec/attaching-streams-to-circuits.md) + - [Hidden-service related circuits](./path-spec/hidden-service-related-circuits.md) + - [Guard nodes](./path-spec/guard-nodes.md) + - [Server descriptor purposes](./path-spec/server-descriptor-purposes.md) + - [Detecting route manipulation by Guard nodes (Path Bias)](./path-spec/detecting-route-manipulation.md) +- [`Tor Guard Specification`](./guard-spec/index.md) + - [State instances](./guard-spec/state-instances.md) + - [Circuit Creation, Entry Guard Selection (1000 foot view)](./guard-spec/guard-selection/index.md) + - [The algorithm.](./guard-spec/algorithm.md) + - [Appendices](./guard-spec/appendices.md) +- [`Tor Vanguards Specification`](./vanguards-spec/index.md) + - [Full Vanguards](./vanguards-spec/full-vanguards.md) + - [Vanguards-Lite](./vanguards-spec/vanguards-lite.md) + - [Path Construction](./vanguards-spec/path-construction.md) + - [Statistical Analysis](./vanguards-spec/vanguards-stats.md) +- [`Tor Padding Specification`](./padding-spec/index.md) + - [Overview](./padding-spec/overview.md) + - [Connection-level padding](./padding-spec/connection-level-padding.md) + - [Circuit-level padding](./padding-spec/circuit-level-padding.md) + - [Acknowledgments](./padding-spec/acknowledgments.md) +- [`Preventing Denial-Of-Service`](./dos-spec/index.md) + - [Overview](./dos-spec/overview.md) + - [Memory exhaustion](./dos-spec/memory-exhaustion.md) + +# Additional behaviors for clients + +- [`Tor's extensions to the SOCKS protocol`](./socks-extensions.md) +- [`Special Hostnames in Tor`](./address-spec.md) + +# Onion services + +- [`Tor Rendezvous Specification - Version 3`](./rend-spec/index.md) + - [Hidden services: overview and preliminaries.](./rend-spec/overview.md) + - [Protocol overview](./rend-spec/protocol-overview.md) + - [Generating and publishing hidden service descriptors \[HSDIR\]](./rend-spec/hsdesc.md) + - [Deriving blinded keys and subcredentials \[SUBCRED\]](./rend-spec/deriving-keys.md) + - [Publishing shared random values \[PUB-SHAREDRANDOM\]](./rend-spec/shared-random.md) + - [Hidden service descriptors: outer wrapper \[DESC-OUTER\]](./rend-spec/hsdesc-outer.md) + - [Hidden service descriptors: encryption format \[HS-DESC-ENC\]](./rend-spec/hsdesc-encrypt.md) + - [The introduction protocol \[INTRO-PROTOCOL\]](./rend-spec/introduction-protocol.md) + - [The rendezvous protocol](./rend-spec/rendezvous-protocol.md) + - [Encrypting data between client and host](./rend-spec/encrypting-user-data.md) + - [Encoding onion addresses \[ONIONADDRESS\]](./rend-spec/encoding-onion-addresses.md) + - [Managing streams](./rend-spec/managing-streams.md) + - [References](./rend-spec/references.md) + - [Appendix A: Signature scheme with key blinding \[KEYBLIND\]](./rend-spec/keyblinding-scheme.md) + - [Appendix B: Selecting nodes \[PICKNODES\]](./rend-spec/selecting-nodes-picknodes.md) + - [Appendix C: Recommendations for searching for vanity .onions \[VANITY\]](./rend-spec/vanity-onions.md) + - [Appendix D: (removed)]() + - [Appendix E: Reserved numbers](./rend-spec/reserved-numbers.md) + - [Appendix F: Hidden service directory format \[HIDSERVDIR-FORMAT\]](./rend-spec/fs-contents.md) + - [Appendix G: Managing authorized client data \[CLIENT-AUTH-MGMT\]](./rend-spec/client-authorization.md) + - [Appendix F: Two methods for managing revision counters.](./rend-spec/revision-counter-mgt.md) + - [Appendix G: Text vectors](./rend-spec/text-vectors.md) +- [`Proof of Work for onion service introduction`](./hspow-spec/index.md) + - [Motivation](./hspow-spec/motivation.md) + - [Common protocol](./hspow-spec/common-protocol.md) + - [Version 1, Equi-X and Blake2b](./hspow-spec/v1-equix.md) + - [Analysis and discussion](./hspow-spec/analysis-discussion.md) + +# Anticensorship tools and protocols + +- [`BridgeDB specification`](./bridgedb-spec.md) +- [`Extended ORPort for pluggable transports`](./ext-orport-spec.md) +- [`Pluggable Transport Specification (Version 1)`](./pt-spec/index.md) + - [Introduction](./pt-spec/introduction.md) + - [Architecture Overview](./pt-spec/architecture-overview.md) + - [Specification](./pt-spec/specification.md) + - [Pluggable Transport Naming](./pt-spec/naming.md) + - [Pluggable Transport Configuration Environment Variables](./pt-spec/configuration-environment.md) + - [Pluggable Transport To Parent Process Communication](./pt-spec/ipc.md) + - [Pluggable Transport Shutdown](./pt-spec/shutdown.md) + - [Pluggable Transport Client Per-Connection Arguments](./pt-spec/per-connection-args.md) + - [Anonymity Considerations](./pt-spec/anonymity-considerations.md) + - [References](./pt-spec/references.md) + - [Acknowledgments](./pt-spec/acknowledgments.md) + - [Appendix A: Example Client Pluggable Transport Session](./pt-spec/example-client-session.md) + - [Appendix B: Example Server Pluggable Transport Session](./pt-spec/example-server-session.md) + +# For C Tor only + +- [`The Tor Control Protocol`](./control-spec/index.md) + - [Protocol outline](./control-spec/protocol-outline.md) + - [Message format](./control-spec/message-format.md) + - [Commands](./control-spec/commands.md) + - [Replies](./control-spec/replies.md) + - [Implementation notes](./control-spec/implementation-notes.md) +- [`How Tor Version Numbers Work`](./version-spec.md) + +# Less commonly needed file formats + +- [`Tor Bandwidth File Format`](./bandwidth-file-spec/index.md) + - [Scope and preliminaries](./bandwidth-file-spec/scope-preliminaries.md) + - [Format details](./bandwidth-file-spec/format-details.md) + - [Definitions](./bandwidth-file-spec/definitions.md) + - [Header List format](./bandwidth-file-spec/header-list-format.md) + - [Relay Line format](./bandwidth-file-spec/relay-line-format.md) + - [Implementation details](./bandwidth-file-spec/implementation-details.md) + - [Sample data](./bandwidth-file-spec/sample-data.md) + - [Scaling bandwidths](./bandwidth-file-spec/scaling-bandwidths.md) + +# Implementation details + +- [`Tor Directory List Format`](./dir-list-spec.md) + +# Reserved names and numbers + +- [`Tor network parameters`](./param-spec.md) +- [`SSH protocol extensions`](./ssh-protocols.md) + +# Unfinished + +- [`Glossary`](./glossary.md) + +# Maintenance and and editing of the Tor Specifications + +- [`About the Tor Specifications documents`](./back-matter.md) +- [`Style guide`](./STYLE.md) +- [`Permalinks`](./permalinks.md) diff --git a/spec/address-spec.md b/spec/address-spec.md new file mode 100644 index 0000000..f3b0e3c --- /dev/null +++ b/spec/address-spec.md @@ -0,0 +1,101 @@ +# Special Hostnames in Tor + +<a id="address-spec.txt-1"></a> + +## Overview + +Most of the time, Tor treats user-specified hostnames as opaque: When +the user connects to \<www.torproject.org>, Tor picks an exit node and uses +that node to connect to "www.torproject.org". Some hostnames, however, +can be used to override Tor's default behavior and circuit-building +rules. + +These hostnames can be passed to Tor as the address part of a SOCKS4a or +SOCKS5 request. If the application is connected to Tor using an IP-only +method (such as SOCKS4, TransPort, or NATDPort), these hostnames can be +substituted for certain IP addresses using the MapAddress configuration +option or the MAPADDRESS control command. + +<a id="address-spec.txt-2"></a> + +## .exit + +```text + SYNTAX: [hostname].[name-or-digest].exit + [name-or-digest].exit +``` + +Hostname is a valid hostname; \[name-or-digest\] is either the nickname of a +Tor node or the hex-encoded digest of that node's public key. + +When Tor sees an address in this format, it uses the specified hostname as +the exit node. If no "hostname" component is given, Tor defaults to the +published IPv4 address of the exit node. + +It is valid to try to resolve hostnames, and in fact upon success Tor +will cache an internal mapaddress of the form +"www.google.com.foo.exit=64.233.161.99.foo.exit" to speed subsequent +lookups. + +The .exit notation is disabled by default as of Tor 0.2.2.1-alpha, due +to potential application-level attacks. + +```text + EXAMPLES: + www.example.com.exampletornode.exit + + Connect to www.example.com from the node called "exampletornode". + + exampletornode.exit + + Connect to the published IP address of "exampletornode" using + "exampletornode" as the exit. +``` + +<a id="address-spec.txt-3"></a> + +## .onion + +```text + SYNTAX: [digest].onion + [ignored].[digest].onion +``` + +Version 2 addresses (deprecated since 0.4.6.1-alpha), the digest is the first +eighty bits of a SHA1 hash of the identity key for a hidden service, encoded +in base32. + +Version 3 addresses, the digest is defined as: + +```text + onion_address = base32(PUBKEY | CHECKSUM | VERSION) + CHECKSUM = H(".onion checksum" | PUBKEY | VERSION)[:2] + + where: + - PUBKEY is the 32 bytes ed25519 master pubkey of the onion service. + - VERSION is a one byte version field (default value '\x03') + - ".onion checksum" is a constant string + - H is SHA3-256 + - CHECKSUM is truncated to two bytes before inserting it in onion_address +``` + +When Tor sees an address in this format, it tries to look up and connect to +the specified onion service. See rend-spec-v3.txt for full details. + +The "ignored" portion of the address is intended for use in vhosting, and +is supported in Tor 0.2.4.10-alpha and later. + +<a id="address-spec.txt-4"></a> + +## .noconnect + +SYNTAX: \[string\].noconnect + +When Tor sees an address in this format, it immediately closes the +connection without attaching it to any circuit. This is useful for +controllers that want to test whether a given application is indeed +using the same instance of Tor that they're controlling. + +This feature was added in Tor 0.1.2.4-alpha, and taken out in Tor +0.2.2.1-alpha over fears that it provided another avenue for detecting +Tor users via application-level web tricks. diff --git a/spec/back-matter.md b/spec/back-matter.md new file mode 100644 index 0000000..90ddea2 --- /dev/null +++ b/spec/back-matter.md @@ -0,0 +1,70 @@ +# About the Tor Specifications documents + +The canonical, official, versions of these documents are on the +[Tor Specifications website](https://spec.torproject.org/) +maintained by the [Tor Project](https://www.torproject.org/). + +Only the Tor Specifications themselves are approved. +The [Proposals](../proposals/) are, by their nature, drafts. + +When linking to the Specifications, +consider using one of the links advertised in the +[Table of Permalinks](permalinks.html). + +## Source code + +The Specifications and Proposals are maintained by the Tor Project +in a +[gitlab repository](https://gitlab.torproject.org/tpo/core/torspec/). + +Corrections and clarifications are welcome. +To propose a change to the Tor protocol, use the +[Proposals process](proposals/001-process.txt) + +## Building + +The documents are in Markdown and formatted with +[mdbook](https://rust-lang.github.io/mdBook/). +To build the formatted HTML: + +```text,ignore +cargo install mdbook +git clone https://gitlab.torproject.org/tpo/core/torspec/ +cd torspec +bin/build_html +``` + +The output is then in `html/`. + +## Source code structure, and output webtree + +There are two mdbook books here: + + * **The Tor Specifications**: source code in `specs/`, + formatted output in `html/`. + + * **Proposals**: source code in `proposals/`, + formatted output in `html/proposals/`. + +Each book's source files are listed, +and the chapter defined, +in its `SUMMARY.md`. +The format is pretty restrictive; +see the +[mdbook documentation](https://rust-lang.github.io/mdBook/format/summary.html). + +## Editing advice + +To edit these specs, clone the +[git repository](https://gitlab.torproject.org/tpo/core/torspec/) +and edit the +appropriate file in the `spec` directory. These files will match +the URLs of their corresponding pages, so if you want to edit +`tor-spec/flow-control.html`, +you'll be looking for a file +called `spec/tor-spec/flow-control.md`. + +We have started a [style guide](./STYLE.md) for writing new parts of +this spec; as of 2023 it is quite preliminary. You should feel free to +edit it! + diff --git a/spec/bandwidth-file-spec/definitions.md b/spec/bandwidth-file-spec/definitions.md new file mode 100644 index 0000000..c01b407 --- /dev/null +++ b/spec/bandwidth-file-spec/definitions.md @@ -0,0 +1,55 @@ +<a id="bandwidth-file-spec.txt-2.1"></a> + +# Definitions + +The following nonterminals are defined in Tor directory protocol +sections 1.2., 2.1.1., 2.1.3.: + +```text + bool + Int + SP (space) + NL (newline) + KeywordChar + ArgumentChar + nickname + hexdigest (a '$', followed by 40 hexadecimal characters + ([A-Fa-f0-9])) + + Nonterminal defined section 2 of version-spec.txt [4]: + + version_number + + We define the following nonterminals: + + Line ::= ArgumentChar* NL + RelayLine ::= KeyValue (SP KeyValue)* NL + HeaderLine ::= KeyValue NL + KeyValue ::= Key "=" Value + Key ::= (KeywordChar | "_")+ + Value ::= ArgumentCharValue+ + ArgumentCharValue ::= any printing ASCII character except NL and SP. + Terminator ::= "=====" or "====" + Generators SHOULD use a 5-character terminator. + Timestamp ::= Int + Bandwidth ::= Int + MasterKey ::= a base64-encoded Ed25519 public key, with + padding characters omitted. + DateTime ::= "YYYY-MM-DDTHH:MM:SS", as in ISO 8601 + CountryCode ::= Two capital ASCII letters ([A-Z]{2}), as defined in + ISO 3166-1 alpha-2 plus "ZZ" to denote unknown country + (eg the destination is in a Content Delivery Network). + CountryCodeList ::= One or more CountryCode(s) separated by a comma + ([A-Z]{2}(,[A-Z]{2})*). +``` + +Note that key_value and value are defined in Tor directory protocol +with different formats to KeyValue and Value here. + +Tor versions earlier than 0.3.5.1-alpha require all lines in the file +to be 510 characters or less. The previous limit was 254 characters in +Tor 0.2.6.2-alpha and earlier. Parsers MAY ignore longer Lines. + +Note that directory authorities are only supported on the two most +recent stable Tor versions, so we expect that line limits will be +removed after Tor 0.4.0 is released in 2019. diff --git a/spec/bandwidth-file-spec/format-details.md b/spec/bandwidth-file-spec/format-details.md new file mode 100644 index 0000000..c5de6f3 --- /dev/null +++ b/spec/bandwidth-file-spec/format-details.md @@ -0,0 +1,11 @@ +<a id="bandwidth-file-spec.txt-2"></a> + +# Format details + +```text + The Bandwidth File MUST contain the following sections: + - Header List (exactly once), which is a partially ordered list of + - Header Lines (one or more times), then + - Relay Lines (zero or more times), in an arbitrary order. + If it does not contain these sections, parsers SHOULD ignore the file. +``` diff --git a/spec/bandwidth-file-spec/header-list-format.md b/spec/bandwidth-file-spec/header-list-format.md new file mode 100644 index 0000000..206ff0c --- /dev/null +++ b/spec/bandwidth-file-spec/header-list-format.md @@ -0,0 +1,422 @@ +<a id="bandwidth-file-spec.txt-2.2"></a> + +# Header List format + +It consists of a Timestamp line and zero or more HeaderLines. + +All the header lines MUST conform to the HeaderLine format, except +the first Timestamp line. + +The Timestamp line is not a HeaderLine to keep compatibility with +the legacy Bandwidth File format. + +Some header Lines MUST appear in specific positions, as documented +below. All other Lines can appear in any order. + +If a parser does not recognize any extra material in a header Line, +the Line MUST be ignored. + +If a header Line does not conform to this format, the Line SHOULD be +ignored by parsers. + +It consists of: + +Timestamp NL + +\[At start, exactly once.\] + +The Unix Epoch time in seconds of the most recent generator bandwidth +result. + +If the generator implementation has multiple threads or +subprocesses which can fail independently, it SHOULD take the most +recent timestamp from each thread and use the oldest value. This +ensures all the threads continue running. + +If there are threads that do not run continuously, they SHOULD be +excluded from the timestamp calculation. + +If there are no recent results, the generator MUST NOT generate a new +file. + +It does not follow the KeyValue format for backwards compatibility +with version 1.0.0. + +"version" version_number NL + +\[In second position, zero or one time.\] + +The specification document format version. +It uses semantic versioning \[5\]. + +This Line was added in version 1.1.0 of this specification. + +Version 1.0.0 documents do not contain this Line, and the +version_number is considered to be "1.0.0". + +"software" Value NL + +\[Zero or one time.\] + +The name of the software that created the document. + +This Line was added in version 1.1.0 of this specification. + +Version 1.0.0 documents do not contain this Line, and the software +is considered to be "torflow". + +"software_version" Value NL + +\[Zero or one time.\] + +The version of the software that created the document. +The version may be a version_number, a git commit, or some other +version scheme. + +This Line was added in version 1.1.0 of this specification. + +"file_created" DateTime NL + +\[Zero or one time.\] + +The date and time timestamp in ISO 8601 format and UTC time zone +when the file was created. + +This Line was added in version 1.1.0 of this specification. + +"generator_started" DateTime NL + +\[Zero or one time.\] + +The date and time timestamp in ISO 8601 format and UTC time zone +when the generator started. + +This Line was added in version 1.1.0 of this specification. + +"earliest_bandwidth" DateTime NL + +\[Zero or one time.\] + +The date and time timestamp in ISO 8601 format and UTC time zone +when the first relay bandwidth was obtained. + +This Line was added in version 1.1.0 of this specification. + +"latest_bandwidth" DateTime NL + +\[Zero or one time.\] + +The date and time timestamp in ISO 8601 format and UTC time zone +of the most recent generator bandwidth result. + +This time MUST be identical to the initial Timestamp line. + +This duplicate value is included to make the format easier for people +to read. + +This Line was added in version 1.1.0 of this specification. + +"number_eligible_relays" Int NL + +\[Zero or one time.\] + +The number of relays that have enough measurements to be +included in the bandwidth file. + +This Line was added in version 1.2.0 of this specification. + +"minimum_percent_eligible_relays" Int NL + +\[Zero or one time.\] + +The percentage of relays in the consensus that SHOULD be +included in every generated bandwidth file. + +If this threshold is not reached, format versions 1.3.0 and earlier +SHOULD NOT contain any relays. (Bandwidth files always include a +header.) + +Format versions 1.4.0 and later SHOULD include all the relays for +diagnostic purposes, even if this threshold is not reached. But these +relays SHOULD be marked so that Tor does not vote on them. +See section 1.4 for details. + +The minimum percentage is 60% in Torflow, so sbws uses +60% as the default. + +This Line was added in version 1.2.0 of this specification. + +"number_consensus_relays" Int NL + +\[Zero or one time.\] + +The number of relays in the consensus. + +This Line was added in version 1.2.0 of this specification. + +"percent_eligible_relays" Int NL + +\[Zero or one time.\] + +The number of eligible relays, as a percentage of the number +of relays in the consensus. + +```text + This line SHOULD be equal to: + (number_eligible_relays * 100.0) / number_consensus_relays + to the number of relays in the consensus to include in this file. + + This Line was added in version 1.2.0 of this specification. + + "minimum_number_eligible_relays" Int NL + + [Zero or one time.] +``` + +The minimum number of relays that SHOULD be included in the bandwidth +file. See minimum_percent_eligible_relays for details. + +```text + This line SHOULD be equal to: + number_consensus_relays * (minimum_percent_eligible_relays / 100.0) + + This Line was added in version 1.2.0 of this specification. + + "scanner_country" CountryCode NL + + [Zero or one time.] + + The country, as in political geolocation, where the generator is run. + + This Line was added in version 1.2.0 of this specification. + + "destinations_countries" CountryCodeList NL + + [Zero or one time.] +``` + +The country, as in political geolocation, or countries where the +destination Web server(s) are located. +The destination Web Servers serve the data that the generator retrieves +to measure the bandwidth. + +This Line was added in version 1.2.0 of this specification. + +"recent_consensus_count" Int NL + +\[Zero or one time.\]. + +The number of the different consensuses seen in the last data_period +days. (data_period is 5 by default.) + +```text + Assuming that Tor clients fetch a consensus every 1-2 hours, + and that the data_period is 5 days, the Value of this Key SHOULD be + between: + data_period * 24 / 2 = 60 + data_period * 24 = 120 + + This Line was added in version 1.4.0 of this specification. + + "recent_priority_list_count" Int NL + + [Zero or one time.] +``` + +The number of times that a list with a subset of relays prioritized +to be measured has been created in the last data_period days. +(data_period is 5 by default.) + +```text + In 2019, with 7000 relays in the network, the Value of this Key SHOULD be + approximately: + data_period * 24 / 1.5 = 80 + Being 1.5 the approximate number of hours it takes to measure a + priority list of 7000 * 0.05 (350) relays, when the fraction of relays + in a priority list is the 5% (0.05). + + This Line was added in version 1.4.0 of this specification. + + "recent_priority_relay_count" Int NL + + [Zero or one time.] +``` + +The number of relays that has been in in the list of relays prioritized +to be measured in the last data_period days. (data_period is 5 by +default.) + +```text + In 2019, with 7000 relays in the network, the Value of this Key SHOULD be + approximately: + 80 * (7000 * 0.05) = 28000 + Being 0.05 (5%) the fraction of relays in a priority list and 80 + the approximate number of priority lists (see + "recent_priority_list_count"). + + This Line was added in version 1.4.0 of this specification. + + "recent_measurement_attempt_count" Int NL + + [Zero or one time.] +``` + +The number of times that any relay has been queued to be measured +in the last data_period days. (data_period is 5 by default.) + +In 2019, with 7000 relays in the network, the Value of this Key SHOULD be +approximately the same as "recent_priority_relay_count", +assuming that there is one attempt to measure a relay for each relay that +has been prioritized unless there are system, network or implementation +issues. + +This Line was added in version 1.4.0 of this specification and removed +in version 1.5.0. + +"recent_measurement_failure_count" Int NL + +\[Zero or one time.\] + +The number of times that the scanner attempted to measure a relay in +the last data_period days (5 by default), but the relay has not been +measured because of system, network or implementation issues. + +This Line was added in version 1.4.0 of this specification. + +"recent_measurements_excluded_error_count" Int NL + +\[Zero or one time.\] + +The number of relays that have no successful measurements in the last +data_period days (5 by default). + +(See the note in section 1.4, version 1.4.0, about excluded relays.) + +This Line was added in version 1.4.0 of this specification. + +"recent_measurements_excluded_near_count" Int NL + +\[Zero or one time.\] + +The number of relays that have some successful measurements in the last +data_period days (5 by default), but all those measurements were +performed in a period of time that was too short (by default 1 day). + +(See the note in section 1.4, version 1.4.0, about excluded relays.) + +This Line was added in version 1.4.0 of this specification. + +"recent_measurements_excluded_old_count" Int NL + +\[Zero or one time.\] + +The number of relays that have some successful measurements, but all +those measurements are too old (more than 5 days, by default). + +Excludes relays that are already counted in +recent_measurements_excluded_near_count. + +(See the note in section 1.4, version 1.4.0, about excluded relays.) + +This Line was added in version 1.4.0 of this specification. + +"recent_measurements_excluded_few_count" Int NL + +\[Zero or one time.\] + +The number of relays that don't have enough recent successful +measurements. (Fewer than 2 measurements in the last 5 days, by +default). + +Excludes relays that are already counted in +recent_measurements_excluded_near_count and +recent_measurements_excluded_old_count. + +(See the note in section 1.4, version 1.4.0, about excluded relays.) + +This Line was added in version 1.4.0 of this specification. + +"time_to_report_half_network" Int NL + +\[Zero or one time.\] + +The time in seconds that it would take to report measurements about the +half of the network, given the number of eligible relays and the time +it took in the last days (5 days, by default). + +(See the note in section 1.4, version 1.4.0, about excluded relays.) + +This Line was added in version 1.4.0 of this specification. + +"tor_version" version_number NL + +\[Zero or one time.\] + +The Tor version of the Tor process controlled by the generator. + +This Line was added in version 1.4.0 of this specification. + +"mu" Int NL + +\[Zero or one time.\] + +The network stream bandwidth average calculated as explained in B4.2. + +This Line was added in version 1.7.0 of this specification. + +"muf" Int NL + +\[Zero or one time.\] + +The network stream bandwidth average filtered calculated as explained in +B4.2. + +This Line was added in version 1.7.0 of this specification. + +KeyValue NL + +\[Zero or more times.\] + +"dirauth_nickname" NL + +\[Zero or one time.\] + +The dirauth's nickname which publishes this V3BandwidthsFile. + +This Line was added in version 1.8.0 of this specification. + +There MUST NOT be multiple KeyValue header Lines with the same key. +If there are, the parser SHOULD choose an arbitrary Line. + +If a parser does not recognize a Keyword in a KeyValue Line, it +MUST be ignored. + +Future format versions may include additional KeyValue header Lines. +Additional header Lines will be accompanied by a minor version +increment. + +Implementations MAY add additional header Lines as needed. This +specification SHOULD be updated to avoid conflicting meanings for +the same header keys. + +Parsers MUST NOT rely on the order of these additional Lines. + +Additional header Lines MUST NOT use any keywords specified in the +relay measurements format. +If there are, the parser MAY ignore conflicting keywords. + +Terminator NL + +\[Zero or one time.\] + +The Header List section ends with a Terminator. + +In version 1.0.0, Header List ends when the first relay bandwidth +is found conforming to the next section. + +Implementations of version 1.1.0 and later SHOULD use a 5-character +terminator. + +Tor 0.4.0.1-alpha and later look for a 5-character terminator, +or the first relay bandwidth line. sbws versions 0.1.0 to 1.0.2 +used a 4-character terminator, this bug was fixed in 1.0.3. diff --git a/spec/bandwidth-file-spec/implementation-details.md b/spec/bandwidth-file-spec/implementation-details.md new file mode 100644 index 0000000..dbc02cc --- /dev/null +++ b/spec/bandwidth-file-spec/implementation-details.md @@ -0,0 +1,398 @@ +<a id="bandwidth-file-spec.txt-2.4"></a> + +# Implementation details + +<a id="bandwidth-file-spec.txt-2.4.1"></a> + +## Writing bandwidth files atomically { #write-atomically } + +To avoid inconsistent reads, implementations SHOULD write bandwidth files +atomically. If the file is transferred from another host, it SHOULD be +written to a temporary path, then renamed to the V3BandwidthsFile path. + +sbws versions 0.7.0 and later write the bandwidth file to an archival +location, create a temporary symlink to that location, then atomically rename +the symlink +to the configured V3BandwidthsFile path. + +Torflow does not write bandwidth files atomically. + +<a id="bandwidth-file-spec.txt-2.4.2"></a> + +## Additional KeyValue pair definitions { #key-value-pairs } + +KeyValue pairs in RelayLines that current implementations generate. + +<a id="bandwidth-file-spec.txt-2.4.2.1"></a> + +### Simple Bandwidth Scanner { #sbws } + +sbws RelayLines contain these keys: + +"node_id" hexdigest + +As above. + +"bw" Bandwidth + +As above. + +"nick" nickname + +\[Exactly once.\] + +The relay nickname. + +Torflow also has a "nick" KeyValue. + +"rtt" Int + +\[Zero or one time.\] + +The Round Trip Time in milliseconds to obtain 1 byte of data. + +This KeyValue was added in version 1.1.0 of this specification. +It became optional in version 1.3.0 or 1.4.0 of this specification. + +"time" DateTime + +\[Exactly once.\] + +The date and time timestamp in ISO 8601 format and UTC time zone +when the last bandwidth was obtained. + +This KeyValue was added in version 1.1.0 of this specification. +The Torflow equivalent is "measured_at". + +"success" Int + +\[Zero or one time.\] + +The number of times that the bandwidth measurements for this relay were +successful. + +This KeyValue was added in version 1.1.0 of this specification. + +"error_circ" Int + +\[Zero or one time.\] + +The number of times that the bandwidth measurements for this relay +failed because of circuit failures. + +This KeyValue was added in version 1.1.0 of this specification. +The Torflow equivalent is "circ_fail". + +"error_stream" Int + +\[Zero or one time.\] + +The number of times that the bandwidth measurements for this relay +failed because of stream failures. + +This KeyValue was added in version 1.1.0 of this specification. + +"error_destination" Int + +\[Zero or one time.\] + +The number of times that the bandwidth measurements for this relay +failed because the destination Web server was not available. + +This KeyValue was added in version 1.4.0 of this specification. + +"error_second_relay" Int + +\[Zero or one time.\] + +The number of times that the bandwidth measurements for this relay +failed because sbws could not find a second relay for the test circuit. + +This KeyValue was added in version 1.4.0 of this specification. + +"error_misc" Int + +\[Zero or one time.\] + +The number of times that the bandwidth measurements for this relay +failed because of other reasons. + +This KeyValue was added in version 1.1.0 of this specification. + +"bw_mean" Int + +\[Zero or one time.\] + +The measured bandwidth mean for this relay in bytes per second. + +This KeyValue was added in version 1.2.0 of this specification. + +"bw_median" Int + +\[Zero or one time.\] + +The measured bandwidth median for this relay in bytes per second. + +This KeyValue was added in version 1.2.0 of this specification. + +"desc_bw_avg" Int + +\[Zero or one time.\] + +The descriptor average bandwidth for this relay in bytes per second. + +This KeyValue was added in version 1.2.0 of this specification. + +"desc_bw_obs_last" Int + +\[Zero or one time.\] + +The last descriptor observed bandwidth for this relay in bytes per +second. + +This KeyValue was added in version 1.2.0 of this specification. + +"desc_bw_obs_mean" Int + +\[Zero or one time.\] + +The descriptor observed bandwidth mean for this relay in bytes per +second. + +This KeyValue was added in version 1.2.0 of this specification. + +"desc_bw_bur" Int + +\[Zero or one time.\] + +The descriptor burst bandwidth for this relay in bytes per +second. + +This KeyValue was added in version 1.2.0 of this specification. + +"consensus_bandwidth" Int + +\[Zero or one time.\] + +The consensus bandwidth for this relay in bytes per second. + +This KeyValue was added in version 1.2.0 of this specification. + +"consensus_bandwidth_is_unmeasured" Bool + +\[Zero or one time.\] + +If the consensus bandwidth for this relay was not obtained from +three or more bandwidth authorities, this KeyValue is True or +False otherwise. + +This KeyValue was added in version 1.2.0 of this specification. + +"relay_in_recent_consensus_count" Int + +\[Zero or one time.\] + +The number of times this relay was found in a consensus in the +last data_period days. (Unless otherwise stated, data_period is +5 by default.) + +This KeyValue was added in version 1.4.0 of this specification. + +"relay_recent_priority_list_count" Int + +\[Zero or one time.\] + +The number of times this relay has been prioritized to be measured +in the last data_period days. + +This KeyValue was added in version 1.4.0 of this specification. + +"relay_recent_measurement_attempt_count" Int + +\[Zero or one time.\] + +The number of times this relay was tried to be measured in the +last data_period days. + +This KeyValue was added in version 1.4.0 of this specification. + +"relay_recent_measurement_failure_count" Int + +\[Zero or one time.\] + +The number of times this relay was tried to be measured in the +last data_period days, but it was not possible to obtain a +measurement. + +This KeyValue was added in version 1.4.0 of this specification. + +"relay_recent_measurements_excluded_error_count" Int + +\[Zero or one time.\] + +The number of recent relay measurement attempts that failed. +Measurements are recent if they are in the last data_period days +(5 by default). + +(See the note in section 1.4, version 1.4.0, about excluded relays.) + +This KeyValue was added in version 1.4.0 of this specification. + +"relay_recent_measurements_excluded_near_count" Int + +\[Zero or one time.\] + +When all of a relay's recent successful measurements were performed in +a period of time that was too short (by default 1 day), the relay is +excluded. This KeyValue contains the number of recent successful +measurements for the relay that were ignored for this reason. + +(See the note in section 1.4, version 1.4.0, about excluded relays.) + +This KeyValue was added in version 1.4.0 of this specification. + +"relay_recent_measurements_excluded_old_count" Int + +\[Zero or one time.\] + +The number of successful measurements for this relay that are too old +(more than data_period days, 5 by default). + +Excludes measurements that are already counted in +relay_recent_measurements_excluded_near_count. + +(See the note in section 1.4, version 1.4.0, about excluded relays.) + +This KeyValue was added in version 1.4.0 of this specification. + +"relay_recent_measurements_excluded_few_count" Int + +\[Zero or one time.\] + +The number of successful measurements for this relay that were ignored +because the relay did not have enough successful measurements (fewer +than 2, by default). + +Excludes measurements that are already counted in +relay_recent_measurements_excluded_near_count or +relay_recent_measurements_excluded_old_count. + +(See the note in section 1.4, version 1.4.0, about excluded relays.) + +This KeyValue was added in version 1.4.0 of this specification. + +"under_min_report" bool + +\[Zero or one time.\] + +If the value is 1, there are not enough eligible relays in the +bandwidth file, and Tor bandwidth authorities MAY NOT vote on this +relay. (Current Tor versions do not change their behaviour based on +the "under_min_report" key.) + +If the value is 0 or the KeyValue is not present, there are enough +relays in the bandwidth file. + +Because Tor versions released before April 2019 (see section 1.4. for +the full list of versions) ignore "vote=0", generator implementations +MUST NOT change the bandwidths for under_min_report relays. Using the +same bw value makes authorities that do not understand "vote=0" +or "under_min_report=1" produce votes that don't change relay weights +too much. It also avoids flapping when the reporting threshold is +reached. + +This KeyValue was added in version 1.4.0 of this specification. + +"unmeasured" bool + +\[Zero or one time.\] + +If the value is 1, this relay was not successfully measured and +Tor bandwidth authorities MAY NOT vote on this relay. +(Current Tor versions do not change their behaviour based on +the "unmeasured" key.) + +If the value is 0 or the KeyValue is not present, this relay +was successfully measured. + +Because Tor versions released before April 2019 (see section 1.4. for +the full list of versions) ignore "vote=0", generator implementations +MUST set "bw=1" for unmeasured relays. Using the minimum bw value +makes authorities that do not understand "vote=0" or "unmeasured=1" +produce votes that don't change relay weights too much. + +This KeyValue was added in version 1.4.0 of this specification. + +"vote" bool + +\[Zero or one time.\] + +If the value is 0, Tor directory authorities SHOULD ignore the relay's +entry in the bandwidth file. They SHOULD vote for the relay the same +way they would vote for a relay that is not present in the file. + +This MAY be the case when this relay was not successfully measured but +it is included in the Bandwidth File, to diagnose why they were not +measured. + +If the value is 1 or the KeyValue is not present, Tor directory +authorities MUST use the relay's bw value in any votes for that relay. + +Implementations MUST also set "bw=1" for unmeasured relays. +But they MUST NOT change the bw for under_min_report relays. +(See the explanations under "unmeasured" and "under_min_report" +for more details.) + +This KeyValue was added in version 1.4.0 of this specification. + +"xoff_recv" Int + +\[Zero or one time.\] + +The number of times this relay received `XOFF_RECV` stream events while +being measured in the last data_period days. + +This KeyValue was added in version 1.6.0 of this specification. + +"xoff_sent" Int + +\[Zero or one time.\] + +The number of times this relay received `XOFF_SENT` stream events while +being measured in the last data_period days. + +This KeyValue was added in version 1.6.0 of this specification. + +"r_strm" Float + +\[Zero or one time.\] + +The stream ratio of this relay calculated as explained in B4.3. + +This KeyValue was added in version 1.7.0 of this specification. + +"r_strm_filt" Float + +\[Zero or one time.\] + +The filtered stream ratio of this relay calculated as explained in B4.3. + +This KeyValue was added in version 1.7.0 of this specification. + +<a id="bandwidth-file-spec.txt-2.4.2.2"></a> + +### Torflow + +Torflow RelayLines include node_id and bw, and other KeyValue pairs \[2\]. + +References: + + +1. <https://gitlab.torproject.org/tpo/network-health/torflow> +2. <https://gitlab.torproject.org/tpo/network-health/torflow/-/blob/main/NetworkScanners/BwAuthority/README.spec.txt?ref_type=heads#L332> + The Torflow specification is outdated, and does not match the current + implementation. See section A.1. for the format produced by Torflow. +3. [The Tor Directory Protocol](../dir-spec) +4. [How Tor Version Numbers Work In Tor](../version-spec.md) +5. <https://semver.org/> +``` diff --git a/spec/bandwidth-file-spec/index.md b/spec/bandwidth-file-spec/index.md new file mode 100644 index 0000000..86563fa --- /dev/null +++ b/spec/bandwidth-file-spec/index.md @@ -0,0 +1,18 @@ +# Tor Bandwidth File Format + +```text + juga + teor +``` + +This document describes the format of Tor's Bandwidth File, version +1.0.0 and later. + +It is a new specification for the existing bandwidth file format, +which we call version 1.0.0. It also specifies new format versions +1.1.0 and later, which are backwards compatible with 1.0.0 parsers. + +Since Tor version 0.2.4.12-alpha, the directory authorities use +the Bandwidth File file called "V3BandwidthsFile" generated by +Torflow \[1\]. The details of this format are described in Torflow's +README.spec.txt. We also summarise the format in this specification. diff --git a/spec/bandwidth-file-spec/relay-line-format.md b/spec/bandwidth-file-spec/relay-line-format.md new file mode 100644 index 0000000..31e1d9b --- /dev/null +++ b/spec/bandwidth-file-spec/relay-line-format.md @@ -0,0 +1,129 @@ +<a id="bandwidth-file-spec.txt-2.3"></a> + +# Relay Line format { #relay-line } + +It consists of zero or more RelayLines containing relay ids and +bandwidths. The relays and their KeyValues are in arbitrary order. + +There MUST NOT be multiple KeyValue pairs with the same key in the same +RelayLine. If there are, the parser SHOULD choose an arbitrary Value. + +There MUST NOT be multiple RelayLines per relay identity (node_id or +master_key_ed25519). If there are, parsers SHOULD issue a warning. +Parers MAY reject the file, choose an arbitrary RelayLine, or ignore +both RelayLines. + +If a parser does not recognize any extra material in a RelayLine, +the extra material MUST be ignored. + +Each RelayLine includes the following KeyValue pairs: + +"node_id" hexdigest + +\[Exactly once.\] + +The fingerprint for the relay's RSA identity key. + +```text + Note: In bandwidth files read by Tor versions earlier than + 0.3.4.1-alpha, node_id MUST NOT be at the end of the Line. + These authority versions are no longer supported. +``` + +Current Tor versions ignore master_key_ed25519, so node_id MUST be +present in each relay Line. + +Implementations of version 1.1.0 and later SHOULD include both node_id +and master_key_ed25519. Parsers SHOULD accept Lines that contain at +least one of them. + +From version 1.9.0 of this specification, "node_id" does not longer need to +start with the dollar sign before the hexdigit. + +"master_key_ed25519" MasterKey + +\[Zero or one time.\] + +The relays's master Ed25519 key, base64 encoded, +without trailing "="s, to avoid ambiguity with KeyValue "=" +character. + +This KeyValue pair SHOULD be present, see the note under node_id. + +This KeyValue was added in version 1.1.0 of this specification. + +"bw" Bandwidth + +\[Exactly once.\] + +The bandwidth of this relay in kilobytes per second. + +No Zero Bandwidths: +Tor accepts zero bandwidths, but they trigger bugs in older Tor +implementations. Therefore, implementations SHOULD NOT produce zero +bandwidths. Instead, they SHOULD use one as their minimum bandwidth. +If there are zero bandwidths, the parser MAY ignore them. + +Bandwidth Aggregation: +Multiple measurements can be aggregated using an averaging scheme, +such as a mean, median, or decaying average. + +Bandwidth Scaling: +Torflow scales bandwidths to kilobytes per second. Other +implementations SHOULD use kilobytes per second for their initial +bandwidth scaling. + +If different implementations or configurations are used in votes for +the same network, their measurements MAY need further scaling. See +Appendix B for information about scaling, and one possible scaling +method. + +MaxAdvertisedBandwidth: +Bandwidth generators MUST limit the relays' measured bandwidth based +on the MaxAdvertisedBadwidth. +A relay's MaxAdvertisedBandwidth limits the bandwidth-avg in its +descriptor. bandwidth-avg is the minimum of MaxAdvertisedBandwidth, +BandwidthRate, RelayBandwidthRate, BandwidthBurst, and +RelayBandwidthBurst. +Therefore, generators MUST limit a relay's measured bandwidth to its +descriptor's bandwidth-avg. This limit needs to be implemented in the +generator, because generators may scale consensus weights before +sending them to Tor. +Generators SHOULD NOT limit measured bandwidths based on descriptors' +bandwidth-observed, because that penalises new relays. + +sbws limits the relay's measured bandwidth to the bandwidth-avg +advertised. + +Torflow partitions relays based on their bandwidth. For unmeasured +relays, Torflow uses the minimum of all descriptor bandwidths, +including bandwidth-avg (MaxAdvertisedBandwidth) and +bandwidth-observed. Then Torflow measures the relays in each partition +against each other, which implicitly limits a relay's measured +bandwidth to the bandwidths of similar relays. + +Torflow also generates consensus weights based on the ratio between the +measured bandwidth and the minimum of all descriptor bandwidths (at the +time of the measurement). So when an operator reduces the +MaxAdvertisedBandwidth for a relay, Torflow reduces that relay's +measured bandwidth. + +KeyValue + +\[Zero or more times.\] + +Future format versions may include additional KeyValue pairs on a +RelayLine. +Additional KeyValue pairs will be accompanied by a minor version +increment. + +Implementations MAY add additional relay KeyValue pairs as needed. +This specification SHOULD be updated to avoid conflicting meanings +for the same Keywords. + +Parsers MUST NOT rely on the order of these additional KeyValue +pairs. + +Additional KeyValue pairs MUST NOT use any keywords specified in the +header format. +If there are, the parser MAY ignore conflicting keywords. diff --git a/spec/bandwidth-file-spec/sample-data.md b/spec/bandwidth-file-spec/sample-data.md new file mode 100644 index 0000000..1a689fd --- /dev/null +++ b/spec/bandwidth-file-spec/sample-data.md @@ -0,0 +1,139 @@ +<a id="bandwidth-file-spec.txt-A"></a> + +# Sample data + +The following has not been obtained from any real measurement. + +<a id="bandwidth-file-spec.txt-A.1"></a> + +## Generated by Torflow { #torflow } + +This an example version 1.0.0 document: + +```text +1523911758 +node_id=$68A483E05A2ABDCA6DA5A3EF8DB5177638A27F80 bw=760 nick=Test measured_at=1523911725 updated_at=1523911725 pid_error=4.11374090719 pid_error_sum=4.11374090719 pid_bw=57136645 pid_delta=2.12168374577 circ_fail=0.2 scanner=/filepath +node_id=$96C15995F30895689291F455587BD94CA427B6FC bw=189 nick=Test2 measured_at=1523911623 updated_at=1523911623 pid_error=3.96703337994 pid_error_sum=3.96703337994 pid_bw=47422125 pid_delta=2.65469736988 circ_fail=0.0 scanner=/filepath +``` + +<a id="bandwidth-file-spec.txt-A.2"></a> + +## Generated by sbws version 0.1.0 { #sbws-010 } + +```text +1523911758 +version=1.1.0 +software=sbws +software_version=0.1.0 +latest_bandwidth=2018-04-16T20:49:18 +file_created=2018-04-16T21:49:18 +generator_started=2018-04-16T15:13:25 +earliest_bandwidth=2018-04-16T15:13:26 +==== + +bw=380 error_circ=0 error_misc=0 error_stream=1 master_key_ed25519=YaqV4vbvPYKucElk297eVdNArDz9HtIwUoIeo0+cVIpQ nick=Test node_id=$68A483E05A2ABDCA6DA5A3EF8DB5177638A27F80 rtt=380 success=1 time=2018-05-08T16:13:26 +bw=189 error_circ=0 error_misc=0 error_stream=0 master_key_ed25519=a6a+dZadrQBtfSbmQkP7j2ardCmLnm5NJ4ZzkvDxbo0I nick=Test2 node_id=$96C15995F30895689291F455587BD94CA427B6FC rtt=378 success=1 time=2018-05-08T16:13:36 +``` + +<a id="bandwidth-file-spec.txt-A.3"></a> + +## Generated by sbws version 1.0.3 { #sbws-103 } + +````text +1523911758 +version=1.2.0 +latest_bandwidth=2018-04-16T20:49:18 +file_created=2018-04-16T21:49:18 +generator_started=2018-04-16T15:13:25 +earliest_bandwidth=2018-04-16T15:13:26 +minimum_number_eligible_relays=3862 +minimum_percent_eligible_relays=60 +number_consensus_relays=6436 +number_eligible_relays=6000 +percent_eligible_relays=93 +software=sbws +software_version=1.0.3 +===== + +bw=38000 bw_mean=1127824 bw_median=1180062 desc_bw_avg=1073741824 desc_bw_obs_last=17230879 desc_bw_obs_mean=14732306 error_circ=0 error_misc=0 error_stream=1 master_key_ed25519=YaqV4vbvPYKucElk297eVdNArDz9HtIwUoIeo0+cVIpQ nick=Test node_id=$68A483E05A2ABDCA6DA5A3EF8DB5177638A27F80 rtt=380 success=1 time=2018-05-08T16:13:26 +bw=1 bw_mean=199162 bw_median=185675 desc_bw_avg=409600 desc_bw_obs_last=836165 desc_bw_obs_mean=858030 error_circ=0 error_misc=0 error_stream=0 master_key_ed25519=a6a+dZadrQBtfSbmQkP7j2ardCmLnm5NJ4ZzkvDxbo0I nick=Test2 node_id=$96C15995F30895689291F455587BD94CA427B6FC rtt=378 success=1 time=2018-05-08T16:13:36 +`` + +<a id="bandwidth-file-spec.txt-A.3.1"></a> + +### When there are not enough eligible measured relays { #sbws-103-not-enough-measured } + +```text +1540496079 +version=1.2.0 +earliest_bandwidth=2018-10-20T19:35:52 +file_created=2018-10-25T19:35:03 +generator_started=2018-10-25T11:42:56 +latest_bandwidth=2018-10-25T19:34:39 +minimum_number_eligible_relays=3862 +minimum_percent_eligible_relays=60 +number_consensus_relays=6436 +number_eligible_relays=2960 +percent_eligible_relays=46 +software=sbws +software_version=1.0.3 +===== +```` + +<a id="bandwidth-file-spec.txt-A.4"></a> + +## Headers generated by sbws version 1.0.4 { #sbws-104 } + +```text +1523911758 +version=1.2.0 +latest_bandwidth=2018-04-16T20:49:18 +destinations_countries=TH,ZZ +file_created=2018-04-16T21:49:18 +generator_started=2018-04-16T15:13:25 +earliest_bandwidth=2018-04-16T15:13:26 +minimum_number_eligible_relays=3862 +minimum_percent_eligible_relays=60 +number_consensus_relays=6436 +number_eligible_relays=6000 +percent_eligible_relays=93 +scanner_country=SN +software=sbws +software_version=1.0.4 +===== +``` + +<a id="bandwidth-file-spec.txt-A.5"></a> + +## Generated by sbws version 1.1.0 { #sbws-110 } + +```text +1523911758 +version=1.4.0 +latest_bandwidth=2018-04-16T20:49:18 +destinations_countries=TH,ZZ +file_created=2018-04-16T21:49:18 +generator_started=2018-04-16T15:13:25 +earliest_bandwidth=2018-04-16T15:13:26 +minimum_number_eligible_relays=3862 +minimum_percent_eligible_relays=60 +number_consensus_relays=6436 +number_eligible_relays=6000 +percent_eligible_relays=93 +recent_measurement_attempt_count=6243 +recent_measurement_failure_count=732 +recent_measurements_excluded_error_count=969 +recent_measurements_excluded_few_count=3946 +recent_measurements_excluded_near_count=90 +recent_measurements_excluded_old_count=0 +recent_priority_list_count=20 +recent_priority_relay_count=6243 +scanner_country=SN +software=sbws +software_version=1.1.0 +time_to_report_half_network=57273 +===== + +bw=1 error_circ=1 error_destination=0 error_misc=0 error_second_relay=0 error_stream=0 master_key_ed25519=J3HQ24kOQWac3L1xlFLp7gY91qkb5NuKxjj1BhDi+m8 nick=snap269 node_id=$DC4D609F95A52614D1E69C752168AF1FCAE0B05F relay_recent_measurement_attempt_count=3 relay_recent_measurements_excluded_error_count=1 relay_recent_measurements_excluded_near_count=3 relay_recent_consensus_count=3 relay_recent_priority_list_count=3 success=3 time=2019-03-16T18:20:57 unmeasured=1 vote=0 +bw=1 error_circ=0 error_destination=0 error_misc=0 error_second_relay=0 error_stream=2 master_key_ed25519=h6ZB1E1yBFWIMloUm9IWwjgaPXEpL5cUbuoQDgdSDKg nick=relay node_id=$C4544F9E209A9A9B99591D548B3E2822236C0503 relay_recent_measurement_attempt_count=3 relay_recent_measurements_excluded_error_count=2 relay_recent_measurements_excluded_few_count=1 relay_recent_consensus_count=3 relay_recent_priority_list_count=3 success=1 time=2019-03-17T06:50:58 unmeasured=1 vote=0 +``` diff --git a/spec/bandwidth-file-spec/scaling-bandwidths.md b/spec/bandwidth-file-spec/scaling-bandwidths.md new file mode 100644 index 0000000..65bb317 --- /dev/null +++ b/spec/bandwidth-file-spec/scaling-bandwidths.md @@ -0,0 +1,132 @@ +<a id="bandwidth-file-spec.txt-B"></a> + +# Scaling bandwidths + +<a id="bandwidth-file-spec.txt-B.1"></a> + +## Scaling requirements + +```text + Tor accepts zero bandwidths, but they trigger bugs in older Tor + implementations. Therefore, scaling methods SHOULD perform the + following checks: + * If the total bandwidth is zero, all relays should be given equal + bandwidths. + * If the scaled bandwidth is zero, it should be rounded up to one. +``` + +Initial experiments indicate that scaling may not be needed for +torflow and sbws, because their measured bandwidths are similar +enough already. + +<a id="bandwidth-file-spec.txt-B.2"></a> + +## A linear scaling method { #linear-method } + +If scaling is required, here is a simple linear bandwidth scaling +method, which ensures that all bandwidth votes contain approximately +the same total bandwidth: + +```text + 1. Calculate the relay quota by dividing the total measured bandwidth + in all votes, by the number of relays with measured bandwidth + votes. In the public tor network, this is approximately 7500 as of + April 2018. The quota should be a consensus parameter, so it can be + adjusted for all generators on the network. + + 2. Calculate a vote quota by multiplying the relay quota by the number + of relays this bandwidth authority has measured + bandwidths for. + + 3. Calculate a scaling factor by dividing the vote quota by the + total unscaled measured bandwidth in this bandwidth + authority's upcoming vote. + + 4. Multiply each unscaled measured bandwidth by the scaling + factor. +``` + +Now, the total scaled bandwidth in the upcoming vote is +approximately equal to the quota. + +<a id="bandwidth-file-spec.txt-B.3"></a> + +## Quota changes + +If all generators are using scaling, the quota can be gradually +reduced or increased as needed. Smaller quotas decrease the size +of uncompressed consensuses, and may decrease the size of +consensus diffs and compressed consensuses. But if the relay +quota is too small, some relays may be over- or under-weighted. + +<a id="bandwidth-file-spec.txt-B.4"></a> + +## Torflow aggregation + +Torflow implements two methods to compute the bandwidth values from the +(stream) bandwidth measurements: with and without PID control feedback. +The method described here is without PID control (see Torflow +specification, section 2.2). + +In the following sections, the relays' measured bandwidth refer to the +ones that this bandwidth authority has measured for the relays that +would be included in the next bandwidth authority's upcoming vote. + +```text + 1. Calculate the filtered bandwidth for each relay: + - choose the relay's measurements (`bw_j`) that are equal or greater + than the mean of the measurements for this relay + - calculate the mean of those measurements + + In pseudocode: + + bw_filt_i = mean(max(mean(bw_j), bw_j)) + + 2. Calculate network averages: + - calculate the filtered average by dividing the sum of all the + relays' filtered bandwidth by the number of relays that have been + measured (`n`), ie, calculate the mean average of the relays' + filtered bandwidth. + - calculate the stream average by dividing the sum of all the + relays' measured bandwidth by the number of relays that have been + measured (`n`), ie, calculate the mean average or the relays' + measured bandwidth. + + In pseudocode: + + bw_avg_filt_ = bw_filt_i / n + bw_avg_strm = bw_i / n + + 3. Calculate ratios for each relay: + - calculate the filtered ratio by dividing each relay filtered + bandwidth by the filtered average + - calculate the stream ratio by dividing each relay measured + bandwidth by the stream average + + In pseudocode: +``` + +r_filt_i = bw_filt_i / bw_avg_filt +r_strm_i = bw_i / bw_avg_strm + +```text + 4. Calculate the final ratio for each relay: + The final ratio is the larger between the filtered bandwidth's and the + stream bandwidth's ratio. + + In pseudocode: + + r_i = max(r_filt_i, r_strm_i) + + 5. Calculate the scaled bandwidth for each relay: + The most recent descriptor observed bandwidth (`bw_obs_i`) is + multiplied by the ratio + + In pseudocode: + + bw_new_i = r_i * bw_obs_i +``` + +\<\<In this way, the resulting network status consensus bandwidth +values are effectively re-weighted proportional to how much faster +the node was as compared to the rest of the network.>> diff --git a/spec/bandwidth-file-spec/scope-preliminaries.md b/spec/bandwidth-file-spec/scope-preliminaries.md new file mode 100644 index 0000000..7eb1419 --- /dev/null +++ b/spec/bandwidth-file-spec/scope-preliminaries.md @@ -0,0 +1,85 @@ +<a id="bandwidth-file-spec.txt-1"></a> + +# Scope and preliminaries + +The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL +NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and +"OPTIONAL" in this document are to be interpreted as described in +RFC 2119. + +<a id="bandwidth-file-spec.txt-1.2"></a> + +## Acknowledgements + +The original bandwidth generator (Torflow) and format was +created by mike. Teor suggested to write this specification while +contributing on pastly's new bandwidth generator implementation. + +This specification was revised after feedback from: + +Nick Mathewson (nickm) +Iain Learmonth (irl) + +<a id="bandwidth-file-spec.txt-1.3"></a> + +## Outline + +The Tor directory protocol (dir-spec.txt \[3\]) sections 3.4.1 +and 3.4.2, use the term bandwidth measurements, to refer to what +here is called Bandwidth File. + +A Bandwidth File contains information on relays' bandwidth +capacities and is produced by bandwidth generators, previously known +as bandwidth scanners. + +<a id="bandwidth-file-spec.txt-1.4"></a> + +## Format Versions + +1.0.0 - The legacy Bandwidth File format + +```text + 1.1.0 - Adds a header containing information about the bandwidth + file. Document the sbws and Torflow relay line keys. + + 1.2.0 - If there are not enough eligible relays, the bandwidth file + SHOULD contain a header, but no relays. (To match Torflow's + existing behaviour.) + + Adds scanner and destination countries to the header. + Adds new KeyValue Lines to the Header List section with + statistics about the number of relays included in the file. + Adds new KeyValues to Relay Bandwidth Lines, with different + bandwidth values (averages and descriptor bandwidths). + + 1.4.0 - Adds monitoring KeyValues to the header and relay lines. + + RelayLines for excluded relays MAY be present in the bandwidth + file for diagnostic reasons. Similarly, if there are not enough + eligible relays, the bandwidth file MAY contain all known relays. + + Diagnostic relay lines SHOULD be marked with vote=0, and + Tor SHOULD NOT use their bandwidths in its votes. + + Also adds Tor version. + 1.5.0 - Removes "recent_measurement_attempt_count" KeyValue. + 1.6.0 - Adds congestion control stream events KeyValues. + 1.7.0 - Adds ratios KeyValues to the relay lines and network averages + KeyValues to the header. + 1.8.0 - Adds "dirauth_nickname" KeyValue to the header. + 1.9.0 - Allows "node_id" KeyValue without the dollar sign at the start of the + hexdigit. + All Tor versions can consume format version 1.0.0. +``` + +All Tor versions can consume format version 1.1.0 and later, +but Tor versions earlier than 0.3.5.1-alpha warn if the header +contains any KeyValue lines after the Timestamp. + +```text + Tor versions 0.4.0.3-alpha, 0.3.5.8, 0.3.4.11, and earlier do not + understand "vote=0". Instead, they will vote for the actual bandwidths + that sbws puts in diagnostic relay lines: + * 1 for relays with "unmeasured=1", and + * the relay's measured and scaled bandwidth when "under_min_report=1". +``` diff --git a/spec/bridgedb-spec.md b/spec/bridgedb-spec.md new file mode 100644 index 0000000..b12fcfb --- /dev/null +++ b/spec/bridgedb-spec.md @@ -0,0 +1,443 @@ +# BridgeDB specification + +<a id="bridgedb-spec.txt-0"></a> + +This document specifies how BridgeDB processes bridge descriptor files +to learn about new bridges, maintains persistent assignments of bridges +to distributors, and decides which bridges to give out upon user +requests. + +Some of the decisions here may be suboptimal: this document is meant to +specify current behavior as of August 2013, not to specify ideal +behavior. + +<a id="bridgedb-spec.txt-1"></a> + +## Importing bridge network statuses and bridge descriptors { #importing } + +BridgeDB learns about bridges by parsing bridge network statuses, +bridge descriptors, and extra info documents as specified in Tor's +directory protocol. BridgeDB parses one bridge network status file +first and at least one bridge descriptor file and potentially one extra +info file afterwards. + +BridgeDB scans its files on sighup. + +BridgeDB does not validate signatures on descriptors or networkstatus +files: the operator needs to make sure that these documents have come +from a Tor instance that did the validation for us. + +<a id="bridgedb-spec.txt-1.1"></a> + +### Parsing bridge network statuses { #parsing-network-status } + +Bridge network status documents contain the information of which bridges +are known to the bridge authority and which flags the bridge authority +assigns to them. +We expect bridge network statuses to contain at least the following two +lines for every bridge in the given order (format fully specified in Tor's +directory protocol): + +```text + "r" SP nickname SP identity SP digest SP publication SP IP SP ORPort + SP DirPort NL + "a" SP address ":" port NL (no more than 8 instances) + "s" SP Flags NL +``` + +BridgeDB parses the identity and the publication timestamp from the "r" +line, the OR address(es) and ORPort(s) from the "a" line(s), and the +assigned flags from the "s" line, specifically checking the assignment +of the "Running" and "Stable" flags. +BridgeDB memorizes all bridges that have the Running flag as the set of +running bridges that can be given out to bridge users. +BridgeDB memorizes assigned flags if it wants to ensure that sets of +bridges given out should contain at least a given number of bridges +with these flags. + +<a id="bridgedb-spec.txt-1.2"></a> + +### Parsing bridge descriptors { #parsing-bridge-descriptors } + +BridgeDB learns about a bridge's most recent IP address and OR port +from parsing bridge descriptors. +In theory, both IP address and OR port of a bridge are also contained +in the "r" line of the bridge network status, so there is no mandatory +reason for parsing bridge descriptors. But the functionality described +in this section is still implemented in case we need data from the +bridge descriptor in the future. + +Bridge descriptor files may contain one or more bridge descriptors. +We expect a bridge descriptor to contain at least the following lines in +the stated order: + +```text + "@purpose" SP purpose NL + "router" SP nickname SP IP SP ORPort SP SOCKSPort SP DirPort NL + "published" SP timestamp + ["opt" SP] "fingerprint" SP fingerprint NL + "router-signature" NL Signature NL +``` + +BridgeDB parses the purpose, IP, ORPort, nickname, and fingerprint +from these lines. +BridgeDB skips bridge descriptors if the fingerprint is not contained +in the bridge network status parsed earlier or if the bridge does not +have the Running flag. +BridgeDB discards bridge descriptors which have a different purpose +than "bridge". BridgeDB can be configured to only accept descriptors +with another purpose or not discard descriptors based on purpose at +all. +BridgeDB memorizes the IP addresses and OR ports of the remaining +bridges. +If there is more than one bridge descriptor with the same fingerprint, +BridgeDB memorizes the IP address and OR port of the most recently +parsed bridge descriptor. +If BridgeDB does not find a bridge descriptor for a bridge contained in +the bridge network status parsed before, it does not add that bridge +to the set of bridges to be given out to bridge users. + +<a id="bridgedb-spec.txt-1.3"></a> + +### Parsing extra-info documents { #parsing-extra-info } + +BridgeDB learns if a bridge supports a pluggable transport by parsing +extra-info documents. +Extra-info documents contain the name of the bridge (but only if it is +named), the bridge's fingerprint, the type of pluggable transport(s) it +supports, and the IP address and port number on which each transport +listens, respectively. + +Extra-info documents may contain zero or more entries per bridge. We expect +an extra-info entry to contain the following lines in the stated order: + +```text + "extra-info" SP name SP fingerprint NL + "transport" SP transport SP IP ":" PORT ARGS NL +``` + +BridgeDB parses the fingerprint, transport type, IP address, port and any +arguments that are specified on these lines. BridgeDB skips the name. If +the fingerprint is invalid, BridgeDB skips the entry. BridgeDB memorizes +the transport type, IP address, port number, and any arguments that are be +provided and then it assigns them to the corresponding bridge based on the +fingerprint. Arguments are comma-separated and are of the form k=v,k=v. +Bridges that do not have an associated extra-info entry are not invalid. + +<a id="bridgedb-spec.txt-2"></a> + +## Assigning bridges to distributors { #assigning-to-distributors } + +A "distributor" is a mechanism by which bridges are given (or not +given) to clients. The current distributors are "email", "https", +and "unallocated". + +BridgeDB assigns bridges to distributors based on an HMAC hash of the +bridge's ID and a secret and makes these assignments persistent. +Persistence is achieved by using a database to map node ID to +distributor. +Each bridge is assigned to exactly one distributor (including +the "unallocated" distributor). +BridgeDB may be configured to support only a non-empty subset of the +distributors specified in this document. +BridgeDB may be configured to use different probabilities for assigning +new bridges to distributors. +BridgeDB does not change existing assignments of bridges to +distributors, even if probabilities for assigning bridges to +distributors change or distributors are disabled entirely. + +<a id="bridgedb-spec.txt-3"></a> + +## Giving out bridges upon requests { #distributing } + +Upon receiving a client request, a BridgeDB distributor provides a +subset of the bridges assigned to it. +BridgeDB only gives out bridges that are contained in the most recently +parsed bridge network status and that have the Running flag set (see +Section 1). +BridgeDB may be configured to give out a different number of bridges +(typically 4) depending on the distributor. +BridgeDB may define an arbitrary number of rules. These rules may +specify the criteria by which a bridge is selected. Specifically, +the available rules restrict the IP address version, OR port number, +transport type, bridge relay flag, or country in which the bridge +should not be blocked. + +<a id="bridgedb-spec.txt-4"></a> + +## Selecting bridges to be given out based on IP addresses { #ip-based } + +```text + BridgeDB may be configured to support one or more distributors which + gives out bridges based on the requestor's IP address. Currently, this + is how the HTTPS distributor works. + The goal is to avoid handing out all the bridges to users in a similar + IP space and time. + +> Someone else should look at proposals/ideas/old/xxx-bridge-disbursement +> to see if this section is missing relevant pieces from it. -KL + + BridgeDB fixes the set of bridges to be returned for a defined time + period. + BridgeDB considers all IP addresses coming from the same /24 network + as the same IP address and returns the same set of bridges. From here on, + this non-unique address will be referred to as the IP address's 'area'. + BridgeDB divides the IP address space equally into a small number of + +> Note, changed term from "areas" to "disjoint clusters" -MF + + disjoint clusters (typically 4) and returns different results for requests + coming from addresses that are placed into different clusters. + +> I found that BridgeDB is not strict in returning only bridges for a +> given area. If a ring is empty, it considers the next one. Is this +> expected behavior? -KL +> +> This does not appear to be the case, anymore. If a ring is empty, then +> BridgeDB simply returns an empty set of bridges. -MF +> +> I also found that BridgeDB does not make the assignment to areas +> persistent in the database. So, if we change the number of rings, it +> will assign bridges to other rings. I assume this is okay? -KL + + BridgeDB maintains a list of proxy IP addresses and returns the same + set of bridges to requests coming from these IP addresses. + The bridges returned to proxy IP addresses do not come from the same + set as those for the general IP address space. +``` + +BridgeDB can be configured to include bridge fingerprints in replies +along with bridge IP addresses and OR ports. +BridgeDB can be configured to display a CAPTCHA which the user must solve +prior to returning the requested bridges. + +The current algorithm is as follows. An IP-based distributor splits +the bridges uniformly into a set of "rings" based on an HMAC of their +ID. Some of these rings are "area" rings for parts of IP space; some +are "category" rings for categories of IPs (like proxies). When a +client makes a request from an IP, the distributor first sees whether +the IP is in one of the categories it knows. If so, the distributor +returns an IP from the category rings. If not, the distributor +maps the IP into an "area" (that is, a /24), and then uses an HMAC to +map the area to one of the area rings. + +When the IP-based distributor determines from which area ring it is handing +out bridges, it identifies which rules it will use to choose appropriate +bridges. Using this information, it searches its cache of rings for one +that already adheres to the criteria specified in this request. If one +exists, then BridgeDB maps the current "epoch" (N-hour period) and the +IP's area (/24) to a point on the ring based on HMAC, and hands out +bridges at that point. If a ring does not already exist which satisfies this +request, then a new ring is created and filled with bridges that fulfill +the requirements. This ring is then used to select bridges as described. + +"Mapping X to Y based on an HMAC" above means one of the following: + +```text + - We keep all of the elements of Y in some order, with a mapping + from all 160-bit strings to positions in Y. + - We take an HMAC of X using some fixed string as a key to get a + 160-bit value. We then map that value to the next position of Y. +``` + +When giving out bridges based on a position in a ring, BridgeDB first +looks at flag requirements and port requirements. For example, +BridgeDB may be configured to "Give out at least L bridges with port +443, and at least M bridges with Stable, and at most N bridges +total." To do this, BridgeDB combines to the results: + +```text + - The first L bridges in the ring after the position that have the + port 443, and + - The first M bridges in the ring after the position that have the + flag stable and that it has not already decided to give out, and + - The first N-L-M bridges in the ring after the position that it + has not already decided to give out. + + After BridgeDB selects appropriate bridges to return to the requestor, it + then prioritises the ordering of them in a list so that as many criteria + are fulfilled as possible within the first few bridges. This list is then + truncated to N bridges, if possible. N is currently defined as a + piecewise function of the number of bridges in the ring such that: + + / + | 1, if len(ring) < 20 + | + N = | 2, if 20 <= len(ring) <= 100 + | + | 3, if 100 <= len(ring) + \ + + The bridges in this sublist, containing no more than N bridges, are the + bridges returned to the requestor. +``` + +<a id="bridgedb-spec.txt-5"></a> + +## Selecting bridges to be given out based on email addresses + +```text + BridgeDB can be configured to support one or more distributors that are + giving out bridges based on the requestor's email address. Currently, + this is how the email distributor works. + The goal is to bootstrap based on one or more popular email service's + sybil prevention algorithms. + +> Someone else should look at proposals/ideas/old/xxx-bridge-disbursement +> to see if this section is missing relevant pieces from it. -KL + + BridgeDB rejects email addresses containing other characters than the + ones that RFC2822 allows. + BridgeDB may be configured to reject email addresses containing other + characters it might not process correctly. + +> I don't think we do this, is it worthwhile? -MF + + BridgeDB rejects email addresses coming from other domains than a + configured set of permitted domains. + BridgeDB normalizes email addresses by removing "." characters and by + removing parts after the first "+" character. + BridgeDB can be configured to discard requests that do not have the + value "pass" in their X-DKIM-Authentication-Result header or does not + have this header. The X-DKIM-Authentication-Result header is set by + the incoming mail stack that needs to check DKIM authentication. + + BridgeDB does not return a new set of bridges to the same email address + until a given time period (typically a few hours) has passed. + +> Why don't we fix the bridges we give out for a global 3-hour time period +> like we do for IP addresses? This way we could avoid storing email +> addresses. -KL +> +> The 3-hour value is probably much too short anyway. If we take longer +> time values, then people get new bridges when bridges show up, as +> opposed to then we decide to reset the bridges we give them. (Yes, this +> problem exists for the IP distributor). -NM +> +> I'm afraid I don't fully understand what you mean here. Can you +> elaborate? -KL +> +> Assuming an average churn rate, if we use short time periods, then a +> requestor will receive new bridges based on rate-limiting and will (likely) +> eventually work their way around the ring; eventually exhausting all bridges +> available to them from this distributor. If we use a longer time period, +> then each time the period expires there will be more bridges in the ring +> thus reducing the likelihood of all bridges being blocked and increasing +> the time and effort required to enumerate all bridges. (This is my +> understanding, not from Nick) -MF +> +> Also, we presently need the cache to prevent replays and because if a user +> sent multiple requests with different criteria in each then we would leak +> additional bridges otherwise. -MF + + BridgeDB can be configured to include bridge fingerprints in replies + along with bridge IP addresses and OR ports. + BridgeDB can be configured to sign all replies using a PGP signing key. + BridgeDB periodically discards old email-address-to-bridge mappings. + BridgeDB rejects too frequent email requests coming from the same + normalized address. +``` + +To map previously unseen email addresses to a set of bridges, BridgeDB +proceeds as follows: + +```text + - It normalizes the email address as above, by stripping out dots, + removing all of the localpart after the +, and putting it all + in lowercase. (Example: "John.Doe+bridges@example.COM" becomes + "johndoe@example.com".) + - It maps an HMAC of the normalized address to a position on its ring + of bridges. + - It hands out bridges starting at that position, based on the + port/flag requirements, as specified at the end of section 4. + + See section 4 for the details of how bridges are selected from the ring + and returned to the requestor. +``` + +<a id="bridgedb-spec.txt-6"></a> + +## Selecting unallocated bridges to be stored in file buckets { #unallocated-buckets } + +> Kaner should have a look at this section. -NM + +```text + BridgeDB can be configured to reserve a subset of bridges and not give + them out via one of the distributors. + BridgeDB assigns reserved bridges to one or more file buckets of fixed + sizes and write these file buckets to disk for manual distribution. + BridgeDB ensures that a file bucket always contains the requested + number of running bridges. + If the requested number of bridges in a file bucket is reduced or the + file bucket is not required anymore, the unassigned bridges are + returned to the reserved set of bridges. + If a bridge stops running, BridgeDB replaces it with another bridge + from the reserved set of bridges. + +> I'm not sure if there's a design bug in file buckets. What happens if +> we add a bridge X to file bucket A, and X goes offline? We would add +> another bridge Y to file bucket A. OK, but what if A comes back? We +> cannot put it back in file bucket A, because it's full. Are we going to +> add it to a different file bucket? Doesn't that mean that most bridges +> will be contained in most file buckets over time? -KL +> +> This should be handled the same as if the file bucket is reduced in size. +> If X returns, then it should be added to the appropriate distributor. -MF +``` + +<a id="bridgedb-spec.txt-7"></a> + +## Displaying Bridge Information { #formatting } + +After bridges are selected using one of the methods described in +Sections 4 - 6, they are output in one of two formats. Bridges are +formatted as: + +`<address:port> NL` + +Pluggable transports are formatted as: + +`<transportname> SP <address:port> [SP arglist] NL` + +where arglist is an optional space-separated list of key-value pairs in +the form of k=v. + +Previously, each line was prepended with the "bridge" keyword, such as + +`"bridge" SP <address:port> NL` + +`"bridge" SP <transportname> SP <address:port> [SP arglist] NL` + +> We don't do this anymore because Vidalia and TorLauncher don't expect it. +> See the commit message for b70347a9c5fd769c6d5d0c0eb5171ace2999a736. + +<a id="bridgedb-spec.txt-8"></a> + +## Writing bridge assignments for statistics + +BridgeDB can be configured to write bridge assignments to disk for +statistical analysis. +The start of a bridge assignment is marked by the following line: + +"bridge-pool-assignment" SP YYYY-MM-DD HH:MM:SS NL + +YYYY-MM-DD HH:MM:SS is the time, in UTC, when BridgeDB has completed +loading new bridges and assigning them to distributors. + +For every running bridge there is a line with the following format: + +fingerprint SP distributor (SP key "=" value)\* NL + +The distributor is one out of "email", "https", or "unallocated". + +Both "email" and "https" distributors support adding keys for "port", +"flag" and "transport". Respectively, the port number, flag name, and +transport types are the values. These are used to indicate that +a bridge matches certain port, flag, transport criteria of requests. + +The "https" distributor also allows the key "ring" with a number as +value to indicate to which IP address area the bridge is returned. + +The "unallocated" distributor allows the key "bucket" with the file +bucket name as value to indicate which file bucket a bridge is assigned +to. diff --git a/spec/cert-spec.md b/spec/cert-spec.md new file mode 100644 index 0000000..873c258 --- /dev/null +++ b/spec/cert-spec.md @@ -0,0 +1,252 @@ +# Certificates in Tor + +This document describes a certificate formats that Tor uses for +its Ed25519 internal certificates, +and discusses how that format is labeled and encoded. + +This format is not the only certificate format that Tor uses. +For the certificates that authorities use +for their signing keys, +see ["Creating key certificates"](dir-spec/creating-key-certificates.md). + +Additionally, Tor uses TLS, which depends on X.509 certificates. + +> The certificates in this document were first introduced in +> proposal 220, and were first supported by Tor in Tor version +> 0.2.7.2-alpha. + +<a id="cert-spec.txt-1.1"></a> + +## Signing + +All signatures here, unless otherwise specified, +are computed using an Ed25519 key. + +In order to future-proof the format, +before signing anything, +the signed document is prefixed with a personalization string, +which will be different in each case. + +<a id="cert-spec.txt-2"></a> + +## Document formats + +### X.509 certificates {#x509} + +Describing this format is out of scope +for the Tor specifications. + +<a id="cert-spec.txt-2.1"></a> + +### Ed25519 Certificates {#ed-certs} + +When generating a signing key, +we also generate a certificate for it. +These representation for this certificate is: + +| Field | Size | Description | +| ---------------- | ---- | ----------------------------------- | +| `VERSION` | 1 | The version of this format | +| `CERT_TYPE` | 1 | [Purpose and meaning of the cert](#list-cert-types) | +| `EXPIRATION_DATE`| 4 | When the cert becomes invalid | +| `CERT_KEY_TYPE` | 1 | [Type of `CERTIFIED_KEY`](#list-key-types) | +| `CERTIFIED_KEY` | 32 | Certified key, or its digest | +| `N_EXTENSIONS` | 1 | Number of extensions | +| `N_EXTENSIONS` times: | | | +| - `ExtLen` | 2 | Length of encoded extension body | +| - `ExtType` | 1 | [Type of extension](#list-ext-types)| +| - `ExtFlags` | 1 | Control interpretation of extension | +| - `ExtData` | `ExtLen` | Encoded extension body | +| SIGNATURE | 64 | Signature of all previous fields | + + +The `VERSION` field holds the value `[01]`. + +The `CERT_TYPE` field holds a value depending on the type of certificate. +(See ["Certificate types"](#list-cert-types).) + +The `CERTIFIED_KEY` field is an Ed25519 public key +if CERT_KEY_TYPE is `[01]`, or a digest of some other key type +depending on the value of CERT_KEY_TYPE. +(See ["List of certified key types"](#list-key-types).) + +The `EXPIRATION_DATE` is a date, given in **hours** since the epoch, +after which this certificate isn't valid. + +> (A four-byte date here will work fine until 10136 A.D.) + +The `ExtFlags` field holds flags. Only one flag is currently defined: + +- **1**: `AFFECTS_VALIDATION`. + If this flag is present, + then the extension affects whether the certificate is valid; + implementations MUST NOT accept the certificate as valid + unless they recognize the `ExtType` + and accept the extension as valid. + +The interpretation of `ExtBody` depends on the `ExtType` field. +See ["Recognized extensions"](#extensions) below. + +It is an error for an extension to be truncated; +such a certificate is invalid. + +Before processing any certificate, +parties SHOULD know which key it is supposed to be signed by, +and then check the signature. + +The signature is created by signing all the fields in the certificate +up until but not including `SIGNATURE`. + + +<a id="cert-spec.txt-2.2"></a> + +### Recognized extensions {#extensions} + +<a id="cert-spec.txt-2.2.1"></a> + +#### Signed-with-ed25519-key extension \[type 04\] { #signed-with-ed25519 } + +In several places, +it's desirable to bundle the signing key +along with the certificate. +We do so with this extension. + +With this extension: +- `ExtLen` is 32. +- `ExtData is a 32-byte Ed25519 public key. + +When this extension is present, +it MUST match the key used to sign the certificate. + +<a id="cert-spec.txt-2.3"></a> + +### RSA→Ed25519 cross-certificate { #rsa-cross-cert } + +In one place, +we have a binary certificate that signs an Ed25519 key +using a legacy 1024-bit RSA key. +Its format is: + +| Field | Size | Description | +| ----------------- | ---- | ----------- | +| `ED25519_KEY` | 32 | The subject key +| `EXPIRATION_DATE` | 4 | When the cert becomes invalid | +| `SIGLEN` | 1 | Length of RSA signature. | +| `SIGNATURE` | `SIGLEN` | RSA Signature | + +Just as with the +[Ed25519 certificates above](#ed-certs), +the `EXPIRATION_DATE` field is a number of **hours** +since the epoch. + +As elsewhere, +the RSA signature is generated using RSA-PKCSv1 padding, +with hash algorithm OIDs omitted. + +The signature is computed on the SHA256 hash of +`PREFIX | FIELDS`, +where `PREFIX` is the string +`"Tor TLS RSA/Ed25519 cross-certificate"` +(without any terminating NUL), +and `FIELDS` is all other fields in the certificate +(other than the signature itself). + +<a id="cert-spec.txt-A.1"></a> + +## Certificate types (CERT_TYPE field) { #list-cert-types } + +This table shows values of the `CERT_TYPE` field in Ed, +as well as values of the `CertType` field +used in a [`CERTS` cell](./tor-spec/negotiating-channels.md#CERTS-cells) +during channel negotiation. + +> You might ned to scroll this table to view it all. +> +> We'll try to fix this once we have a better grip on our mdbook CSS. + +| Type | Mnemonic | Format | Subject | Signing key | Reference | Notes | +|------| ------------- | ------ | ----------------------- | ------------ | --------- | ----- | +|`[01]`| `TLS_LINK_X509` | [X.509]| [`KP_legacy_conn_tls`] | [`KS_relayid_rsa`] | [Legacy channel negotiation] | Obsolete | +|`[02]`| `RSA_ID_X509` | [X.509]| [`KP_relayid_rsa`] | [`KS_relayid_rsa`] | [Legacy channel negotiation] | Obsolete | +|`[03]`| `LINK_AUTH_X509` | [X.509]| [`KP_legacy_linkauth_rsa`]|[`KS_relayid_rsa`] | [Legacy channel negotiation] | Obsolete | +|`[04]`| `IDENTITY_V_SIGNING` |[Ed]| [`KP_relaysign_ed`] | [`KS_relayid_ed`] | [Online signing keys] | | +|`[05]`| `SIGNING_V_TLS_CERT` |[Ed]| A TLS certificate | [`KS_relaysign_ed`] | [CERTS cells] | | +|`[06]`| `SIGNING_V_LINK_AUTH`|[Ed]| [`KP_link_ed`] | [`KS_relaysign_ed`] | [CERTS cells] | | +|`[07]`| `RSA_ID_V_IDENTITY` |[Rsa]|[`KP_relayid_ed`] | [`KS_relayid_rsa`] | [CERTS cells] | | +|`[08]`| `BLINDED_ID_V_SIGNING`|[Ed]|[`KP_hs_desc_sign`] | [`KS_hs_blind_id`] | [HsDesc (outer)] | | +|`[09]`| `HS_IP_V_SIGNING` |[Ed]| [`KP_hs_ipt_sid`] | [`KS_hs_desc_sign`] | [HsDesc (`auth-key`)] | Backwards, see [note 1](#note-1) | +|`[0A]`| `NTOR_CC_IDENTITY` |[Ed]| [`KP_relayid_ed`] | [`EdCvt`]`(`[`KS_ntor`]`)` | [ntor cross-cert] | | +|`[0B]`| `HS_IP_CC_SIGNING` |[Ed]| [`KP_hss_ntor`] | [`KS_hs_desc_sign`] | [HsDesc (`enc-key-cert`)] | Backwards, see [note 1](#note-1) | + +[X.509]: #x509 +[Rsa]: #rsa-cross-cert +[Ed]: #ed-certs +[`KP_legacy_conn_tls`]: ./tor-spec/relay-keys.md#legacy_conn_tls +[`KP_legacy_linkauth_rsa`]: ./tor-spec/relay-keys.md#legacy_linkauth_rsa +[`KP_relayid_rsa`]: ./tor-spec/relay-keys.md#relayid_rsa +[`KP_relaysign_ed`]: ./tor-spec/relay-keys.md#relaysign_ed +[`KP_relayid_ed`]: ./tor-spec/relay-keys.md#relayid_ed +[`KP_link_ed`]: ./tor-spec/relay-keys.md#link_ed +[`KS_legacy_conn_tls`]: ./tor-spec/relay-keys.md#legacy_conn_tls +[`KS_relayid_rsa`]: ./tor-spec/relay-keys.md#relayid_rsa +[`KS_relaysign_ed`]: ./tor-spec/relay-keys.md#relaysign_ed +[`KS_relayid_ed`]: ./tor-spec/relay-keys.md#relayid_ed +[`KS_ntor`]: ./tor-spec/relay-keys.md#ntor +[`KS_link_ed`]: ./tor-spec/relay-keys.md#link_ed +[`KP_hs_desc_sign`]: ./rend-spec/protocol-overview.md#hs_desc_sign +[`KS_hs_desc_sign`]: ./rend-spec/protocol-overview.md#hs_desc_sign +[`KP_hs_ipt_sid`]: ./rend-spec/protocol-overview.md#hs_ipt_sid +[`KP_hss_ntor`]: ./rend-spec/protocol-overview.md#hss_ntor +[`KS_hs_blind_id`]: ./rend-spec/protocol-overview.md#hs_blind_id +[Legacy channel negotiation]: ./tor-spec/obsolete-channels.md +[Online signing keys]: ./tor-spec/relay-keys.md#online-signing +[CERTS cells]: ./tor-spec/negotiating-channels.md#CERTS-cells +[`EdCvt`]: ./dir-spec/converting-to-ed25519.md +[HsDesc (outer)]: ./rend-spec/hsdesc-outer.md#descriptor-signing-key-cert +[HsDesc (`auth-key`)]: ./rend-spec/hsdesc-encrypt.md#auth-key +[HsDesc (`enc-key-cert`)]: ./rend-spec/hsdesc-encrypt.md#enc-key-cert +[ntor cross-cert]: ./dir-spec/server-descriptor-format.md#ntor-onion-key-crosscert + + +<span id="note-1">Note 1: +The certificate types +[`[09] HS_IP_V_SIGNING`][HsDesc (`auth-key`)] +and +[`[0B] HS_IP_CC_SIGNING`][HsDesc (`enc-key-cert`)] +were implemented incorrectly, and now cannot be changed. +Their signing keys and subject keys, as implemented, +are given in the table. +They were originally meant to be the inverse of this order. +</span> + + +<a id="cert-spec.txt-A.2"></a> + +## List of extension types { #list-ext-types } + +- `[04]` - [signed-with-ed25519-key](#signed-with-ed25519) + +<a id="cert-spec.txt-A.3"></a> + +## List of signature prefixes { #list-sig-prefixes } + +We describe various documents as being signed with a prefix. Here +are those prefixes: + +"Tor router descriptor signature v1" (see dir-spec.txt) + +<a id="cert-spec.txt-A.4"></a> + +## List of certified key types (CERT_KEY_TYPE field) { #list-key-types } + +- `[01]`: ed25519 key +- `[02]`: SHA256 hash of an RSA key. (Not currently used.) +- `[03]`: SHA256 hash of an X.509 certificate. (Used with certificate + type 5.) + +> (NOTE: Up till 0.4.5.1-alpha, +> all versions of Tor have incorrectly used +> `[01]` for all types of certified key. +> Implementations SHOULD allow "01" in this position, +> and infer the actual key type from the `CERT_TYPE` field. + diff --git a/spec/control-spec/commands.md b/spec/control-spec/commands.md new file mode 100644 index 0000000..d087473 --- /dev/null +++ b/spec/control-spec/commands.md @@ -0,0 +1,1910 @@ +<a id="control-spec.txt-3"></a> + +# Commands + +All commands are case-insensitive, but most keywords are case-sensitive. + +<a id="control-spec.txt-3.1"></a> + +## SETCONF + +Change the value of one or more configuration variables. The syntax is: + +```text + "SETCONF" 1*(SP keyword ["=" value]) CRLF + value = String / QuotedString +``` + +Tor behaves as though it had just read each of the key-value pairs +from its configuration file. Keywords with no corresponding values have +their configuration values reset to 0 or NULL (use RESETCONF if you want +to set it back to its default). SETCONF is all-or-nothing: if there +is an error in any of the configuration settings, Tor sets none of them. + +Tor responds with a "250 OK" reply on success. +If some of the listed keywords can't be found, Tor replies with a +"552 Unrecognized option" message. Otherwise, Tor responds with a +"513 syntax error in configuration values" reply on syntax error, or a +"553 impossible configuration setting" reply on a semantic error. + +Some configuration options (e.g. "Bridge") take multiple values. Also, +some configuration keys (e.g. for hidden services and for entry +guard lists) form a context-sensitive group where order matters (see +GETCONF below). In these cases, setting _any_ of the options in a +SETCONF command is taken to reset all of the others. For example, +if two ORListenAddress values are configured, and a SETCONF command +arrives containing a single ORListenAddress value, the new command's +value replaces the two old values. + +Sometimes it is not possible to change configuration options solely by +issuing a series of SETCONF commands, because the value of one of the +configuration options depends on the value of another which has not yet +been set. Such situations can be overcome by setting multiple configuration +options with a single SETCONF command (e.g. SETCONF ORPort=443 +ORListenAddress=9001). + +<a id="control-spec.txt-3.2"></a> + +## RESETCONF + +Remove all settings for a given configuration option entirely, assign +its default value (if any), and then assign the String provided. +Typically the String is left empty, to simply set an option back to +its default. The syntax is: + +"RESETCONF" 1\*(SP keyword \["=" String\]) CRLF + +Otherwise it behaves like SETCONF above. + +<a id="control-spec.txt-3.3"></a> + +## GETCONF + +Request the value of zero or more configuration variable(s). +The syntax is: + +"GETCONF" \*(SP keyword) CRLF + +If all of the listed keywords exist in the Tor configuration, Tor replies +with a series of reply lines of the form: + +250 keyword=value + +If any option is set to a 'default' value semantically different from an +empty string, Tor may reply with a reply line of the form: + +250 keyword + +Value may be a raw value or a quoted string. Tor will try to use unquoted +values except when the value could be misinterpreted through not being +quoted. (Right now, Tor supports no such misinterpretable values for +configuration options.) + +If some of the listed keywords can't be found, Tor replies with a +"552 unknown configuration keyword" message. + +If an option appears multiple times in the configuration, all of its +key-value pairs are returned in order. + +If no keywords were provided, Tor responds with "250 OK" message. + +Some options are context-sensitive, and depend on other options with +different keywords. These cannot be fetched directly. Currently there +is only one such option: clients should use the "HiddenServiceOptions" +virtual keyword to get all HiddenServiceDir, HiddenServicePort, +HiddenServiceVersion, and HiddenserviceAuthorizeClient option settings. + +<a id="control-spec.txt-3.4"></a> + +## SETEVENTS + +Request the server to inform the client about interesting events. The +syntax is: + +"SETEVENTS" \[SP "EXTENDED"\] \*(SP EventCode) CRLF + +EventCode = 1\*(ALPHA / "\_") (see section 4.1.x for event types) + +Any events _not_ listed in the SETEVENTS line are turned off; thus, sending +SETEVENTS with an empty body turns off all event reporting. + +The server responds with a "250 OK" reply on success, and a "552 +Unrecognized event" reply if one of the event codes isn't recognized. (On +error, the list of active event codes isn't changed.) + +If the flag string "EXTENDED" is provided, Tor may provide extra +information with events for this connection; see 4.1 for more information. +NOTE: All events on a given connection will be provided in extended format, +or none. +NOTE: "EXTENDED" was first supported in Tor 0.1.1.9-alpha; it is +always-on in Tor 0.2.2.1-alpha and later. + +Each event is described in more detail in Section 4.1. + +<a id="control-spec.txt-3.5"></a> + +## AUTHENTICATE + +Sent from the client to the server. The syntax is: + +"AUTHENTICATE" \[ SP 1\*HEXDIG / QuotedString \] CRLF + +This command is used to authenticate to the server. The provided string is +one of the following: + +```text + * (For the HASHEDPASSWORD authentication method; see 3.21) + The original password represented as a QuotedString. + + * (For the COOKIE is authentication method; see 3.21) + The contents of the cookie file, formatted in hexadecimal + + * (For the SAFECOOKIE authentication method; see 3.21) + The HMAC based on the AUTHCHALLENGE message, in hexadecimal. +``` + +The server responds with "250 OK" on success or "515 Bad authentication" if +the authentication cookie is incorrect. Tor closes the connection on an +authentication failure. + +The authentication token can be specified as either a quoted ASCII string, +or as an unquoted hexadecimal encoding of that same string (to avoid escaping +issues). + +For information on how the implementation securely stores authentication +information on disk, see section 5.1. + +Before the client has authenticated, no command other than +PROTOCOLINFO, AUTHCHALLENGE, AUTHENTICATE, or QUIT is valid. If the +controller sends any other command, or sends a malformed command, or +sends an unsuccessful AUTHENTICATE command, or sends PROTOCOLINFO or +AUTHCHALLENGE more than once, Tor sends an error reply and closes +the connection. + +To prevent some cross-protocol attacks, the AUTHENTICATE command is still +required even if all authentication methods in Tor are disabled. In this +case, the controller should just send "AUTHENTICATE" CRLF. + +(Versions of Tor before 0.1.2.16 and 0.2.0.4-alpha did not close the +connection after an authentication failure.) + +<a id="control-spec.txt-3.6"></a> + +## SAVECONF + +Sent from the client to the server. The syntax is: + +"SAVECONF" \[SP "FORCE"\] CRLF + +Instructs the server to write out its config options into its torrc. Server +returns "250 OK" if successful, or "551 Unable to write configuration +to disk" if it can't write the file or some other error occurs. + +If the %include option is used on torrc, SAVECONF will not write the +configuration to disk. If the flag string "FORCE" is provided, the +configuration will be overwritten even if %include is used. Using %include +on defaults-torrc does not affect SAVECONF. (Introduced in 0.3.1.1-alpha.) + +See also the "getinfo config-text" command, if the controller wants +to write the torrc file itself. + +See also the "getinfo config-can-saveconf" command, to tell if the FORCE +flag will be required. (Also introduced in 0.3.1.1-alpha.) + +<a id="control-spec.txt-3.7"></a> + +## SIGNAL + +Sent from the client to the server. The syntax is: + +"SIGNAL" SP Signal CRLF + +```text + Signal = "RELOAD" / "SHUTDOWN" / "DUMP" / "DEBUG" / "HALT" / + "HUP" / "INT" / "USR1" / "USR2" / "TERM" / "NEWNYM" / + "CLEARDNSCACHE" / "HEARTBEAT" / "ACTIVE" / "DORMANT" + + The meaning of the signals are: + + RELOAD -- Reload: reload config items. + SHUTDOWN -- Controlled shutdown: if server is an OP, exit immediately. + If it's an OR, close listeners and exit after + ShutdownWaitLength seconds. + DUMP -- Dump stats: log information about open connections and + circuits. + DEBUG -- Debug: switch all open logs to loglevel debug. + HALT -- Immediate shutdown: clean up and exit now. + CLEARDNSCACHE -- Forget the client-side cached IPs for all hostnames. + NEWNYM -- Switch to clean circuits, so new application requests + don't share any circuits with old ones. Also clears + the client-side DNS cache. (Tor MAY rate-limit its + response to this signal.) + HEARTBEAT -- Make Tor dump an unscheduled Heartbeat message to log. + DORMANT -- Tell Tor to become "dormant". A dormant Tor will + try to avoid CPU and network usage until it receives + user-initiated network request. (Don't use this + on relays or hidden services yet!) + ACTIVE -- Tell Tor to stop being "dormant", as if it had received + a user-initiated network request. +``` + +The server responds with "250 OK" if the signal is recognized (or simply +closes the socket if it was asked to close immediately), or "552 +Unrecognized signal" if the signal is unrecognized. + +Note that not all of these signals have POSIX signal equivalents. The +ones that do are as below. You may also use these POSIX names for the +signal that have them. + +```text + RELOAD: HUP + SHUTDOWN: INT + HALT: TERM + DUMP: USR1 + DEBUG: USR2 + + [SIGNAL DORMANT and SIGNAL ACTIVE were added in 0.4.0.1-alpha.] +``` + +<a id="control-spec.txt-3.8"></a> + +## MAPADDRESS + +Sent from the client to the server. The syntax is: + +"MAPADDRESS" 1\*(Address "=" Address SP) CRLF + +The first address in each pair is an "original" address; the second is a +"replacement" address. The client sends this message to the server in +order to tell it that future SOCKS requests for connections to the original +address should be replaced with connections to the specified replacement +address. If the addresses are well-formed, and the server is able to +fulfill the request, the server replies with a 250 message: + +```text + 250-OldAddress1=NewAddress1 + 250 OldAddress2=NewAddress2 +``` + +containing the source and destination addresses. If request is +malformed, the server replies with "512 syntax error in command +argument". If the server can't fulfill the request, it replies with +"451 resource exhausted". + +The client may decline to provide a body for the original address, and +instead send a special null address ("0.0.0.0" for IPv4, "::0" for IPv6, or +"." for hostname), signifying that the server should choose the original +address itself, and return that address in the reply. The server +should ensure that it returns an element of address space that is unlikely +to be in actual use. If there is already an address mapped to the +destination address, the server may reuse that mapping. + +If the original address is already mapped to a different address, the old +mapping is removed. If the original address and the destination address +are the same, the server removes any mapping in place for the original +address. + +Example: + +```text + C: MAPADDRESS 1.2.3.4=torproject.org + S: 250 1.2.3.4=torproject.org + + C: GETINFO address-mappings/control + S: 250-address-mappings/control=1.2.3.4 torproject.org NEVER + S: 250 OK + + C: MAPADDRESS 1.2.3.4=1.2.3.4 + S: 250 1.2.3.4=1.2.3.4 + + C: GETINFO address-mappings/control + S: 250-address-mappings/control= + S: 250 OK +``` + +{Note: This feature is designed to be used to help Tor-ify applications +that need to use SOCKS4 or hostname-less SOCKS5. There are three +approaches to doing this: + +```text + 1. Somehow make them use SOCKS4a or SOCKS5-with-hostnames instead. + 2. Use tor-resolve (or another interface to Tor's resolve-over-SOCKS + feature) to resolve the hostname remotely. This doesn't work + with special addresses like x.onion or x.y.exit. + 3. Use MAPADDRESS to map an IP address to the desired hostname, and then + arrange to fool the application into thinking that the hostname + has resolved to that IP. + + This functionality is designed to help implement the 3rd approach.} +``` + +Mappings set by the controller last until the Tor process exits: +they never expire. If the controller wants the mapping to last only +a certain time, then it must explicitly un-map the address when that +time has elapsed. + +MapAddress replies MAY contain mixed status codes. + +Example: + +```text + C: MAPADDRESS xxx=@@@ 0.0.0.0=bogus1.google.com + S: 512-syntax error: invalid address '@@@' + S: 250 127.199.80.246=bogus1.google.com +``` + +<a id="control-spec.txt-3.9"></a> + +## GETINFO + +Sent from the client to the server. The syntax is as for GETCONF: + +"GETINFO" 1\*(SP keyword) CRLF + +Unlike GETCONF, this message is used for data that are not stored in the Tor +configuration file, and that may be longer than a single line. On success, +one ReplyLine is sent for each requested value, followed by a final 250 OK +ReplyLine. If a value fits on a single line, the format is: + +```text + 250-keyword=value + If a value must be split over multiple lines, the format is: + + 250+keyword= + value + . + The server sends a 551 or 552 error on failure. + + Recognized keys and their values include: + + "version" -- The version of the server's software, which MAY include the + name of the software, such as "Tor 0.0.9.4". The name of the software, + if absent, is assumed to be "Tor". + + "config-file" -- The location of Tor's configuration file ("torrc"). + + "config-defaults-file" -- The location of Tor's configuration + defaults file ("torrc.defaults"). This file gets parsed before + torrc, and is typically used to replace Tor's default + configuration values. [First implemented in 0.2.3.9-alpha.] + + "config-text" -- The contents that Tor would write if you send it + a SAVECONF command, so the controller can write the file to + disk itself. [First implemented in 0.2.2.7-alpha.] + + "exit-policy/default" -- The default exit policy lines that Tor will + *append* to the ExitPolicy config option. + + "exit-policy/reject-private/default" -- The default exit policy lines + that Tor will *prepend* to the ExitPolicy config option when + ExitPolicyRejectPrivate is 1. + + "exit-policy/reject-private/relay" -- The relay-specific exit policy + lines that Tor will *prepend* to the ExitPolicy config option based + on the current values of ExitPolicyRejectPrivate and + ExitPolicyRejectLocalInterfaces. These lines are based on the public + addresses configured in the torrc and present on the relay's + interfaces. Will send 552 error if the server is not running as + onion router. Will send 551 on internal error which may be transient. + + "exit-policy/ipv4" + "exit-policy/ipv6" + "exit-policy/full" -- This OR's exit policy, in IPv4-only, IPv6-only, or + all-entries flavors. Handles errors in the same way as "exit-policy/ + reject-private/relay" does. + + "desc/id/<OR identity>" or "desc/name/<OR nickname>" -- the latest + server descriptor for a given OR. (Note that modern Tor clients + do not download server descriptors by default, but download + microdescriptors instead. If microdescriptors are enabled, you'll + need to use "md" instead.) + + "md/all" -- all known microdescriptors for the entire Tor network. + Each microdescriptor is terminated by a newline. + [First implemented in 0.3.5.1-alpha] + + "md/id/<OR identity>" or "md/name/<OR nickname>" -- the latest + microdescriptor for a given OR. Empty if we have no microdescriptor for + that OR (because we haven't downloaded one, or it isn't in the + consensus). [First implemented in 0.2.3.8-alpha.] + + "desc/download-enabled" -- "1" if we try to download router descriptors; + "0" otherwise. [First implemented in 0.3.2.1-alpha] + + "md/download-enabled" -- "1" if we try to download microdescriptors; + "0" otherwise. [First implemented in 0.3.2.1-alpha] + + "dormant" -- A nonnegative integer: zero if Tor is currently active and + building circuits, and nonzero if Tor has gone idle due to lack of use + or some similar reason. [First implemented in 0.2.3.16-alpha] + + "desc-annotations/id/<OR identity>" -- outputs the annotations string + (source, timestamp of arrival, purpose, etc) for the corresponding + descriptor. [First implemented in 0.2.0.13-alpha.] + + "extra-info/digest/<digest>" -- the extrainfo document whose digest (in + hex) is <digest>. Only available if we're downloading extra-info + documents. + + "ns/id/<OR identity>" or "ns/name/<OR nickname>" -- the latest router + status info (v3 directory style) for a given OR. Router status + info is as given in dir-spec.txt, and reflects the latest + consensus opinion about the + router in question. Like directory clients, controllers MUST + tolerate unrecognized flags and lines. The published date and + descriptor digest are those believed to be best by this Tor, + not necessarily those for a descriptor that Tor currently has. + [First implemented in 0.1.2.3-alpha.] + [In 0.2.0.9-alpha this switched from v2 directory style to v3] + + "ns/all" -- Router status info (v3 directory style) for all ORs we + that the consensus has an opinion about, joined by newlines. + [First implemented in 0.1.2.3-alpha.] + [In 0.2.0.9-alpha this switched from v2 directory style to v3] + + "ns/purpose/<purpose>" -- Router status info (v3 directory style) + for all ORs of this purpose. Mostly designed for /ns/purpose/bridge + queries. + [First implemented in 0.2.0.13-alpha.] + [In 0.2.0.9-alpha this switched from v2 directory style to v3] + [In versions before 0.4.1.1-alpha we set the Running flag on + bridges when /ns/purpose/bridge is accessed] + [In 0.4.1.1-alpha we set the Running flag on bridges when the + bridge networkstatus file is written to disk] + + "desc/all-recent" -- the latest server descriptor for every router that + Tor knows about. (See md note about "desc/id" and "desc/name" above.) + + "network-status" -- [Deprecated in 0.3.1.1-alpha, removed + in 0.4.5.1-alpha.] + + "address-mappings/all" + "address-mappings/config" + "address-mappings/cache" + "address-mappings/control" -- a \r\n-separated list of address + mappings, each in the form of "from-address to-address expiry". + The 'config' key returns those address mappings set in the + configuration; the 'cache' key returns the mappings in the + client-side DNS cache; the 'control' key returns the mappings set + via the control interface; the 'all' target returns the mappings + set through any mechanism. + Expiry is formatted as with ADDRMAP events, except that "expiry" is + always a time in UTC or the string "NEVER"; see section 4.1.7. + First introduced in 0.2.0.3-alpha. + + "addr-mappings/*" -- as for address-mappings/*, but without the + expiry portion of the value. Use of this value is deprecated + since 0.2.0.3-alpha; use address-mappings instead. + + "address" -- the best guess at our external IP address. If we + have no guess, return a 551 error. (Added in 0.1.2.2-alpha) + + "address/v4" + "address/v6" + the best guess at our respective external IPv4 or IPv6 address. + If we have no guess, return a 551 error. (Added in 0.4.5.1-alpha) + + "fingerprint" -- the contents of the fingerprint file that Tor + writes as a relay, or a 551 if we're not a relay currently. + (Added in 0.1.2.3-alpha) + + "circuit-status" + A series of lines as for a circuit status event. Each line is of + the form described in section 4.1.1, omitting the initial + "650 CIRC ". Note that clients must be ready to accept additional + arguments as described in section 4.1. + + "stream-status" + A series of lines as for a stream status event. Each is of the form: + StreamID SP StreamStatus SP CircuitID SP Target CRLF + + "orconn-status" + A series of lines as for an OR connection status event. In Tor + 0.1.2.2-alpha with feature VERBOSE_NAMES enabled and in Tor + 0.2.2.1-alpha and later by default, each line is of the form: + LongName SP ORStatus CRLF + + In Tor versions 0.1.2.2-alpha through 0.2.2.1-alpha with feature + VERBOSE_NAMES turned off and before version 0.1.2.2-alpha, each line + is of the form: + ServerID SP ORStatus CRLF + + "entry-guards" + A series of lines listing the currently chosen entry guards, if any. + In Tor 0.1.2.2-alpha with feature VERBOSE_NAMES enabled and in Tor + 0.2.2.1-alpha and later by default, each line is of the form: + LongName SP Status [SP ISOTime] CRLF + + In Tor versions 0.1.2.2-alpha through 0.2.2.1-alpha with feature + VERBOSE_NAMES turned off and before version 0.1.2.2-alpha, each line + is of the form: + ServerID2 SP Status [SP ISOTime] CRLF + ServerID2 = Nickname / 40*HEXDIG + + The definition of Status is the same for both: + Status = "up" / "never-connected" / "down" / + "unusable" / "unlisted" + + [From 0.1.1.4-alpha to 0.1.1.10-alpha, entry-guards was called + "helper-nodes". Tor still supports calling "helper-nodes", but it + is deprecated and should not be used.] + + [Older versions of Tor (before 0.1.2.x-final) generated 'down' instead + of unlisted/unusable. Between 0.1.2.x-final and 0.2.6.3-alpha, + 'down' was never generated.] + + [XXXX ServerID2 differs from ServerID in not prefixing fingerprints + with a $. This is an implementation error. It would be nice to add + the $ back in if we can do so without breaking compatibility.] + + "traffic/read" -- Total bytes read (downloaded). + + "traffic/written" -- Total bytes written (uploaded). + + "uptime" -- Uptime of the Tor daemon (in seconds). Added in + 0.3.5.1-alpha. + + "accounting/enabled" + "accounting/hibernating" + "accounting/bytes" + "accounting/bytes-left" + "accounting/interval-start" + "accounting/interval-wake" + "accounting/interval-end" + Information about accounting status. If accounting is enabled, + "enabled" is 1; otherwise it is 0. The "hibernating" field is "hard" + if we are accepting no data; "soft" if we're accepting no new + connections, and "awake" if we're not hibernating at all. The "bytes" + and "bytes-left" fields contain (read-bytes SP write-bytes), for the + start and the rest of the interval respectively. The 'interval-start' + and 'interval-end' fields are the borders of the current interval; the + 'interval-wake' field is the time within the current interval (if any) + where we plan[ned] to start being active. The times are UTC. + + "config/names" + A series of lines listing the available configuration options. Each is + of the form: + OptionName SP OptionType [ SP Documentation ] CRLF + OptionName = Keyword + OptionType = "Integer" / "TimeInterval" / "TimeMsecInterval" / + "DataSize" / "Float" / "Boolean" / "Time" / "CommaList" / + "Dependent" / "Virtual" / "String" / "LineList" + Documentation = Text + Note: The incorrect spelling "Dependant" was used from the time this key + was introduced in Tor 0.1.1.4-alpha until it was corrected in Tor + 0.3.0.2-alpha. It is recommended that clients accept both spellings. + + "config/defaults" + A series of lines listing default values for each configuration + option. Options which don't have a valid default don't show up + in the list. Introduced in Tor 0.2.4.1-alpha. + OptionName SP OptionValue CRLF + OptionName = Keyword + OptionValue = Text + + "info/names" + A series of lines listing the available GETINFO options. Each is of + one of these forms: + OptionName SP Documentation CRLF + OptionPrefix SP Documentation CRLF + OptionPrefix = OptionName "/*" + The OptionPrefix form indicates a number of options beginning with the + prefix. So if "config/*" is listed, other options beginning with + "config/" will work, but "config/*" itself is not an option. + + "events/names" + A space-separated list of all the events supported by this version of + Tor's SETEVENTS. + + "features/names" + A space-separated list of all the features supported by this version + of Tor's USEFEATURE. + + "signal/names" + A space-separated list of all the values supported by the SIGNAL + command. + + "ip-to-country/ipv4-available" + "ip-to-country/ipv6-available" + "1" if the relevant geoip or geoip6 database is present; "0" otherwise. + This field was added in Tor 0.3.2.1-alpha. + + "ip-to-country/*" + Maps IP addresses to 2-letter country codes. For example, + "GETINFO ip-to-country/18.0.0.1" should give "US". + + "process/pid" -- Process id belonging to the main tor process. + "process/uid" -- User id running the tor process, -1 if unknown (this is + unimplemented on Windows, returning -1). + "process/user" -- Username under which the tor process is running, + providing an empty string if none exists (this is unimplemented on + Windows, returning an empty string). + "process/descriptor-limit" -- Upper bound on the file descriptor limit, -1 + if unknown + + "dir/status-vote/current/consensus" [added in Tor 0.2.1.6-alpha] + "dir/status-vote/current/consensus-microdesc" [added in Tor 0.4.3.1-alpha] + "dir/status/authority" + "dir/status/fp/<F>" + "dir/status/fp/<F1>+<F2>+<F3>" + "dir/status/all" + "dir/server/fp/<F>" + "dir/server/fp/<F1>+<F2>+<F3>" + "dir/server/d/<D>" + "dir/server/d/<D1>+<D2>+<D3>" + "dir/server/authority" + "dir/server/all" + A series of lines listing directory contents, provided according to the + specification for the URLs listed in Section 4.4 of dir-spec.txt. Note + that Tor MUST NOT provide private information, such as descriptors for + routers not marked as general-purpose. When asked for 'authority' + information for which this Tor is not authoritative, Tor replies with + an empty string. + + Note that, as of Tor 0.2.3.3-alpha, Tor clients don't download server + descriptors anymore, but microdescriptors. So, a "551 Servers + unavailable" reply to all "GETINFO dir/server/*" requests is actually + correct. If you have an old program which absolutely requires server + descriptors to work, try setting UseMicrodescriptors 0 or + FetchUselessDescriptors 1 in your client's torrc. + + "status/circuit-established" + "status/enough-dir-info" + "status/good-server-descriptor" + "status/accepted-server-descriptor" + "status/..." + These provide the current internal Tor values for various Tor + states. See Section 4.1.10 for explanations. (Only a few of the + status events are available as getinfo's currently. Let us know if + you want more exposed.) + "status/reachability-succeeded/or" + 0 or 1, depending on whether we've found our ORPort reachable. + "status/reachability-succeeded/dir" + 0 or 1, depending on whether we've found our DirPort reachable. + 1 if there is no DirPort, and therefore no need for a reachability + check. + "status/reachability-succeeded" + "OR=" ("0"/"1") SP "DIR=" ("0"/"1") + Combines status/reachability-succeeded/*; controllers MUST ignore + unrecognized elements in this entry. + "status/bootstrap-phase" + Returns the most recent bootstrap phase status event + sent. Specifically, it returns a string starting with either + "NOTICE BOOTSTRAP ..." or "WARN BOOTSTRAP ...". Controllers should + use this getinfo when they connect or attach to Tor to learn its + current bootstrap state. + "status/version/recommended" + List of currently recommended versions. + "status/version/current" + Status of the current version. One of: new, old, unrecommended, + recommended, new in series, obsolete, unknown. + "status/clients-seen" + A summary of which countries we've seen clients from recently, + formatted the same as the CLIENTS_SEEN status event described in + Section 4.1.14. This GETINFO option is currently available only + for bridge relays. + "status/fresh-relay-descs" + Provides fresh server and extra-info descriptors for our relay. Note + this is *not* the latest descriptors we've published, but rather what we + would generate if we needed to make a new descriptor right now. + + "net/listeners/*" + + A quoted, space-separated list of the locations where Tor is listening + for connections of the specified type. These can contain IPv4 + network address... + + "127.0.0.1:9050" "127.0.0.1:9051" + + ... or local unix sockets... + + "unix:/home/my_user/.tor/socket" + + ... or IPv6 network addresses: + + "[2001:0db8:7000:0000:0000:dead:beef:1234]:9050" + + [New in Tor 0.2.2.26-beta.] + + "net/listeners/or" + + Listeners for OR connections. Talks Tor protocol as described in + tor-spec.txt. + + "net/listeners/dir" + + Listeners for Tor directory protocol, as described in dir-spec.txt. + + "net/listeners/socks" + + Listeners for onion proxy connections that talk SOCKS4/4a/5 protocol. + + "net/listeners/trans" + + Listeners for transparent connections redirected by firewall, such as + pf or netfilter. + + "net/listeners/natd" + + Listeners for transparent connections redirected by natd. + + "net/listeners/dns" + + Listeners for a subset of DNS protocol that Tor network supports. + + "net/listeners/control" + + Listeners for Tor control protocol, described herein. + + "net/listeners/extor" + + Listeners corresponding to Extended ORPorts for integration with + pluggable transports. See proposals 180 and 196. + + "net/listeners/httptunnel" + + Listeners for onion proxy connections that leverage HTTP CONNECT + tunnelling. + + [The extor and httptunnel lists were added in 0.3.2.12, 0.3.3.10, and + 0.3.4.6-rc.] + + "dir-usage" + A newline-separated list of how many bytes we've served to answer + each type of directory request. The format of each line is: + Keyword 1*SP Integer 1*SP Integer + where the first integer is the number of bytes written, and the second + is the number of requests answered. + + [This feature was added in Tor 0.2.2.1-alpha, and removed in + Tor 0.2.9.1-alpha. Even when it existed, it only provided + useful output when the Tor client was built with either the + INSTRUMENT_DOWNLOADS or RUNNING_DOXYGEN compile-time options.] + + "bw-event-cache" + A space-separated summary of recent BW events in chronological order + from oldest to newest. Each event is represented by a comma-separated + tuple of "R,W", R is the number of bytes read, and W is the number of + bytes written. These entries each represent about one second's worth + of traffic. + [New in Tor 0.2.6.3-alpha] + + "consensus/valid-after" + "consensus/fresh-until" + "consensus/valid-until" + Each of these produces an ISOTime describing part of the lifetime of + the current (valid, accepted) consensus that Tor has. + [New in Tor 0.2.6.3-alpha] + + "hs/client/desc/id/<ADDR>" + Prints the content of the hidden service descriptor corresponding to + the given <ADDR> which is an onion address without the ".onion" part. + The client's cache is queried to find the descriptor. The format of + the descriptor is described in section 1.3 of the rend-spec.txt + document. + + If <ADDR> is unrecognized or if not found in the cache, a 551 error is + returned. + + [New in Tor 0.2.7.1-alpha] + [HS v3 support added 0.3.3.1-alpha] + + "hs/service/desc/id/<ADDR>" + Prints the content of the hidden service descriptor corresponding to + the given <ADDR> which is an onion address without the ".onion" part. + The service's local descriptor cache is queried to find the descriptor. + The format of the descriptor is described in section 1.3 of the + rend-spec.txt document. + + If <ADDR> is unrecognized or if not found in the cache, a 551 error is + returned. + + [New in Tor 0.2.7.2-alpha] + [HS v3 support added 0.3.3.1-alpha] + + "onions/current" + "onions/detached" + A newline-separated list of the Onion ("Hidden") Services created + via the "ADD_ONION" command. The 'current' key returns Onion Services + belonging to the current control connection. The 'detached' key + returns Onion Services detached from the parent control connection + (as in, belonging to no control connection). + The format of each line is: + HSAddress + [New in Tor 0.2.7.1-alpha.] + [HS v3 support added 0.3.3.1-alpha] + + "network-liveness" + The string "up" or "down", indicating whether we currently believe the + network is reachable. + + "downloads/" + The keys under downloads/ are used to query download statuses; they all + return either a sequence of newline-terminated hex encoded digests, or + a "serialized download status" as follows: + + SerializedDownloadStatus = + -- when do we plan to next attempt to download this object? + "next-attempt-at" SP ISOTime CRLF + -- how many times have we failed since the last success? + "n-download-failures" SP UInt CRLF + -- how many times have we tried to download this? + "n-download-attempts" SP UInt CRLF + -- according to which schedule rule will we download this? + "schedule" SP DownloadSchedule CRLF + -- do we want to fetch this from an authority, or will any cache do? + "want-authority" SP DownloadWantAuthority CRLF + -- do we increase our download delay whenever we fail to fetch this, + -- or whenever we attempt fetching this? + "increment-on" SP DownloadIncrementOn CRLF + -- do we increase the download schedule deterministically, or at + -- random? + "backoff" SP DownloadBackoff CRLF + [ + -- with an exponential backoff, where are we in the schedule? + "last-backoff-position" Uint CRLF + -- with an exponential backoff, what was our last delay? + "last-delay-used UInt CRLF + ] + + where + + DownloadSchedule = + "DL_SCHED_GENERIC" / "DL_SCHED_CONSENSUS" / "DL_SCHED_BRIDGE" + DownloadWantAuthority = + "DL_WANT_ANY_DIRSERVER" / "DL_WANT_AUTHORITY" + DownloadIncrementOn = + "DL_SCHED_INCREMENT_FAILURE" / "DL_SCHED_INCREMENT_ATTEMPT" + DownloadBackoff = + "DL_SCHED_DETERMINISTIC" / "DL_SCHED_RANDOM_EXPONENTIAL" + + The optional last two lines must be present if DownloadBackoff is + "DL_SCHED_RANDOM_EXPONENTIAL" and must be absent if DownloadBackoff + is "DL_SCHED_DETERMINISTIC". + + In detail, the keys supported are: + + "downloads/networkstatus/ns" + The SerializedDownloadStatus for the NS-flavored consensus for + whichever bootstrap state Tor is currently in. + + "downloads/networkstatus/ns/bootstrap" + The SerializedDownloadStatus for the NS-flavored consensus at + bootstrap time, regardless of whether we are currently bootstrapping. + + "downloads/networkstatus/ns/running" + + The SerializedDownloadStatus for the NS-flavored consensus when + running, regardless of whether we are currently bootstrapping. + + "downloads/networkstatus/microdesc" + The SerializedDownloadStatus for the microdesc-flavored consensus for + whichever bootstrap state Tor is currently in. + + "downloads/networkstatus/microdesc/bootstrap" + The SerializedDownloadStatus for the microdesc-flavored consensus at + bootstrap time, regardless of whether we are currently bootstrapping. + + "downloads/networkstatus/microdesc/running" + The SerializedDownloadStatus for the microdesc-flavored consensus when + running, regardless of whether we are currently bootstrapping. + + "downloads/cert/fps" + + A newline-separated list of hex-encoded digests for authority + certificates for which we have download status available. + + "downloads/cert/fp/<Fingerprint>" + A SerializedDownloadStatus for the default certificate for the + identity digest <Fingerprint> returned by the downloads/cert/fps key. + + "downloads/cert/fp/<Fingerprint>/sks" + A newline-separated list of hex-encoded signing key digests for the + authority identity digest <Fingerprint> returned by the + downloads/cert/fps key. + + "downloads/cert/fp/<Fingerprint>/<SKDigest>" + A SerializedDownloadStatus for the certificate for the identity + digest <Fingerprint> returned by the downloads/cert/fps key and signing + key digest <SKDigest> returned by the downloads/cert/fp/<Fingerprint>/ + sks key. + + "downloads/desc/descs" + A newline-separated list of hex-encoded router descriptor digests + [note, not identity digests - the Tor process may not have seen them + yet while downloading router descriptors]. If the Tor process is not + using a NS-flavored consensus, a 551 error is returned. + + "downloads/desc/<Digest>" + A SerializedDownloadStatus for the router descriptor with digest + <Digest> as returned by the downloads/desc/descs key. If the Tor + process is not using a NS-flavored consensus, a 551 error is returned. + + "downloads/bridge/bridges" + A newline-separated list of hex-encoded bridge identity digests. If + the Tor process is not using bridges, a 551 error is returned. + + "downloads/bridge/<Digest>" + A SerializedDownloadStatus for the bridge descriptor with identity + digest <Digest> as returned by the downloads/bridge/bridges key. If + the Tor process is not using bridges, a 551 error is returned. + + "sr/current" + "sr/previous" + The current or previous shared random value, as received in the + consensus, base-64 encoded. An empty value means that either + the consensus has no shared random value, or Tor has no consensus. + + "current-time/local" + "current-time/utc" + The current system or UTC time, as returned by the system, in ISOTime2 + format. (Introduced in 0.3.4.1-alpha.) + + "stats/ntor/requested" + "stats/ntor/assigned" + The NTor circuit onion handshake rephist values which are requested or + assigned. (Introduced in 0.4.5.1-alpha) + + "stats/tap/requested" + "stats/tap/assigned" + The TAP circuit onion handshake rephist values which are requested or + assigned. (Introduced in 0.4.5.1-alpha) + + "config-can-saveconf" + 0 or 1, depending on whether it is possible to use SAVECONF without the + FORCE flag. (Introduced in 0.3.1.1-alpha.) + + "limits/max-mem-in-queues" + The amount of memory that Tor's out-of-memory checker will allow + Tor to allocate (in places it can see) before it starts freeing memory + and killing circuits. See the MaxMemInQueues option for more + details. Unlike the option, this value reflects Tor's actual limit, and + may be adjusted depending on the available system memory rather than on + the MaxMemInQueues option. (Introduced in 0.2.5.4-alpha) + + Examples: + + C: GETINFO version desc/name/moria1 + S: 250+desc/name/moria= + S: [Descriptor for moria] + S: . + S: 250-version=Tor 0.1.1.0-alpha-cvs + S: 250 OK +``` + +<a id="control-spec.txt-3.10"></a> + +## EXTENDCIRCUIT + +Sent from the client to the server. The format is: + +```text + "EXTENDCIRCUIT" SP CircuitID + [SP ServerSpec *("," ServerSpec)] + [SP "purpose=" Purpose] CRLF +``` + +This request takes one of two forms: either the CircuitID is zero, in +which case it is a request for the server to build a new circuit, +or the CircuitID is nonzero, in which case it is a request for the +server to extend an existing circuit with that ID according to the +specified path. + +If the CircuitID is 0, the controller has the option of providing +a path for Tor to use to build the circuit. If it does not provide +a path, Tor will select one automatically from high capacity nodes +according to path-spec.txt. + +If CircuitID is 0 and "purpose=" is specified, then the circuit's +purpose is set. Two choices are recognized: "general" and +"controller". If not specified, circuits are created as "general". + +If the request is successful, the server sends a reply containing a +message body consisting of the CircuitID of the (maybe newly created) +circuit. The syntax is "250" SP "EXTENDED" SP CircuitID CRLF. + +<a id="control-spec.txt-3.11"></a> + +## SETCIRCUITPURPOSE + +Sent from the client to the server. The format is: + +"SETCIRCUITPURPOSE" SP CircuitID SP "purpose=" Purpose CRLF + +This changes the circuit's purpose. See EXTENDCIRCUIT above for details. + +<a id="control-spec.txt-3.12"></a> + +## SETROUTERPURPOSE + +Sent from the client to the server. The format is: + +"SETROUTERPURPOSE" SP NicknameOrKey SP Purpose CRLF + +This changes the descriptor's purpose. See +POSTDESCRIPTOR below +for details. + +NOTE: This command was disabled and made obsolete as of Tor +0.2.0.8-alpha. It doesn't exist anymore, and is listed here only for +historical interest. + +<a id="control-spec.txt-3.13"></a> + +## ATTACHSTREAM + +Sent from the client to the server. The syntax is: + +"ATTACHSTREAM" SP StreamID SP CircuitID \[SP "HOP=" HopNum\] CRLF + +This message informs the server that the specified stream should be +associated with the specified circuit. Each stream may be associated with +at most one circuit, and multiple streams may share the same circuit. +Streams can only be attached to completed circuits (that is, circuits that +have sent a circuit status 'BUILT' event or are listed as built in a +GETINFO circuit-status request). + +If the circuit ID is 0, responsibility for attaching the given stream is +returned to Tor. + +If HOP=HopNum is specified, Tor will choose the HopNumth hop in the +circuit as the exit node, rather than the last node in the circuit. +Hops are 1-indexed; generally, it is not permitted to attach to hop 1. + +Tor responds with "250 OK" if it can attach the stream, 552 if the +circuit or stream didn't exist, 555 if the stream isn't in an +appropriate state to be attached (e.g. it's already open), or 551 if +the stream couldn't be attached for another reason. + +{Implementation note: Tor will close unattached streams by itself, +roughly two minutes after they are born. Let the developers know if +that turns out to be a problem.} + +{Implementation note: By default, Tor automatically attaches streams to +circuits itself, unless the configuration variable +"\_\_LeaveStreamsUnattached" is set to "1". Attempting to attach streams +via TC when "\_\_LeaveStreamsUnattached" is false may cause a race between +Tor and the controller, as both attempt to attach streams to circuits.} + +{Implementation note: You can try to attachstream to a stream that +has already sent a connect or resolve request but hasn't succeeded +yet, in which case Tor will detach the stream from its current circuit +before proceeding with the new attach request.} + +<a id="control-spec.txt-3.14"></a> + +## POSTDESCRIPTOR + +Sent from the client to the server. The syntax is: + +```text + "+POSTDESCRIPTOR" [SP "purpose=" Purpose] [SP "cache=" Cache] + CRLF Descriptor CRLF "." CRLF +``` + +This message informs the server about a new descriptor. If Purpose is +specified, it must be either "general", "controller", or "bridge", +else we return a 552 error. The default is "general". + +If Cache is specified, it must be either "no" or "yes", else we +return a 552 error. If Cache is not specified, Tor will decide for +itself whether it wants to cache the descriptor, and controllers +must not rely on its choice. + +The descriptor, when parsed, must contain a number of well-specified +fields, including fields for its nickname and identity. + +If there is an error in parsing the descriptor, the server must send a +"554 Invalid descriptor" reply. If the descriptor is well-formed but +the server chooses not to add it, it must reply with a 251 message +whose body explains why the server was not added. If the descriptor +is added, Tor replies with "250 OK". + +<a id="control-spec.txt-3.15"></a> + +## REDIRECTSTREAM + +Sent from the client to the server. The syntax is: + +"REDIRECTSTREAM" SP StreamID SP Address \[SP Port\] CRLF + +Tells the server to change the exit address on the specified stream. If +Port is specified, changes the destination port as well. No remapping +is performed on the new provided address. + +To be sure that the modified address will be used, this event must be sent +after a new stream event is received, and before attaching this stream to +a circuit. + +Tor replies with "250 OK" on success. + +<a id="control-spec.txt-3.16"></a> + +## CLOSESTREAM + +Sent from the client to the server. The syntax is: + +"CLOSESTREAM" SP StreamID SP Reason \*(SP Flag) CRLF + +Tells the server to close the specified stream. The reason should be one +of the Tor RELAY_END reasons given in tor-spec.txt, as a decimal. Flags is +not used currently; Tor servers SHOULD ignore unrecognized flags. Tor may +hold the stream open for a while to flush any data that is pending. + +Tor replies with "250 OK" on success, or a 512 if there aren't enough +arguments, or a 552 if it doesn't recognize the StreamID or reason. + +<a id="control-spec.txt-3.17"></a> + +## CLOSECIRCUIT + +The syntax is: + +```text + "CLOSECIRCUIT" SP CircuitID *(SP Flag) CRLF + Flag = "IfUnused" +``` + +Tells the server to close the specified circuit. If "IfUnused" is +provided, do not close the circuit unless it is unused. + +Other flags may be defined in the future; Tor SHOULD ignore unrecognized +flags. + +Tor replies with "250 OK" on success, or a 512 if there aren't enough +arguments, or a 552 if it doesn't recognize the CircuitID. + +<a id="control-spec.txt-3.18"></a> + +## QUIT + +Tells the server to hang up on this controller connection. This command +can be used before authenticating. + +<a id="control-spec.txt-3.19"></a> + +## USEFEATURE + +Adding additional features to the control protocol sometimes will break +backwards compatibility. Initially such features are added into Tor and +disabled by default. USEFEATURE can enable these additional features. + +The syntax is: + +```text + "USEFEATURE" *(SP FeatureName) CRLF + FeatureName = 1*(ALPHA / DIGIT / "_" / "-") + + Feature names are case-insensitive. +``` + +Once enabled, a feature stays enabled for the duration of the connection +to the controller. A new connection to the controller must be opened to +disable an enabled feature. + +Features are a forward-compatibility mechanism; each feature will eventually +become a standard part of the control protocol. Once a feature becomes part +of the protocol, it is always-on. Each feature documents the version it was +introduced as a feature and the version in which it became part of the +protocol. + +Tor will ignore a request to use any feature that is always-on. Tor will give +a 552 error in response to an unrecognized feature. + +EXTENDED_EVENTS + +```text + Same as passing 'EXTENDED' to SETEVENTS; this is the preferred way to + request the extended event syntax. + + This feature was first introduced in 0.1.2.3-alpha. It is always-on + and part of the protocol in Tor 0.2.2.1-alpha and later. + + VERBOSE_NAMES + + Replaces ServerID with LongName in events and GETINFO results. LongName + provides a Fingerprint for all routers, an indication of Named status, + and a Nickname if one is known. LongName is strictly more informative + than ServerID, which only provides either a Fingerprint or a Nickname. + + This feature was first introduced in 0.1.2.2-alpha. It is always-on and + part of the protocol in Tor 0.2.2.1-alpha and later. +``` + +<a id="control-spec.txt-3.20"></a> + +## RESOLVE + +The syntax is + +```text + "RESOLVE" *Option *Address CRLF + Option = "mode=reverse" + Address = a hostname or IPv4 address +``` + +This command launches a remote hostname lookup request for every specified +request (or reverse lookup if "mode=reverse" is specified). Note that the +request is done in the background: to see the answers, your controller will +need to listen for ADDRMAP events; see 4.1.7 below. + +\[Added in Tor 0.2.0.3-alpha\] + +<a id="control-spec.txt-3.21"></a> + +## PROTOCOLINFO + +The syntax is: + +"PROTOCOLINFO" \*(SP PIVERSION) CRLF + +The server reply format is: + +"250-PROTOCOLINFO" SP PIVERSION CRLF \*InfoLine "250 OK" CRLF + +InfoLine = AuthLine / VersionLine / OtherLine + +```text + AuthLine = "250-AUTH" SP "METHODS=" AuthMethod *("," AuthMethod) + *(SP "COOKIEFILE=" AuthCookieFile) CRLF + VersionLine = "250-VERSION" SP "Tor=" TorVersion OptArguments CRLF + + AuthMethod = + "NULL" / ; No authentication is required + "HASHEDPASSWORD" / ; A controller must supply the original password + "COOKIE" / ; ... or supply the contents of a cookie file + "SAFECOOKIE" ; ... or prove knowledge of a cookie file's contents + + AuthCookieFile = QuotedString + TorVersion = QuotedString + + OtherLine = "250-" Keyword OptArguments CRLF + + PIVERSION: 1*DIGIT +``` + +This command tells the controller what kinds of authentication are +supported. + +Tor MAY give its InfoLines in any order; controllers MUST ignore InfoLines +with keywords they do not recognize. Controllers MUST ignore extraneous +data on any InfoLine. + +PIVERSION is there in case we drastically change the syntax one day. For +now it should always be "1". Controllers MAY provide a list of the +protocolinfo versions they support; Tor MAY select a version that the +controller does not support. + +AuthMethod is used to specify one or more control authentication +methods that Tor currently accepts. + +AuthCookieFile specifies the absolute path and filename of the +authentication cookie that Tor is expecting and is provided iff the +METHODS field contains the method "COOKIE" and/or "SAFECOOKIE". +Controllers MUST handle escape sequences inside this string. + +All authentication cookies are 32 bytes long. Controllers MUST NOT +use the contents of a non-32-byte-long file as an authentication +cookie. + +If the METHODS field contains the method "SAFECOOKIE", every +AuthCookieFile must contain the same authentication cookie. + +The COOKIE authentication method exposes the user running a +controller to an unintended information disclosure attack whenever +the controller has greater filesystem read access than the process +that it has connected to. (Note that a controller may connect to a +process other than Tor.) It is almost never safe to use, even if +the controller's user has explicitly specified which filename to +read an authentication cookie from. For this reason, the COOKIE +authentication method has been deprecated and will be removed from +a future version of Tor. + +The VERSION line contains the Tor version. + +\[Unlike other commands besides AUTHENTICATE, PROTOCOLINFO may be used (but +only once!) before AUTHENTICATE.\] + +\[PROTOCOLINFO was not supported before Tor 0.2.0.5-alpha.\] + +<a id="control-spec.txt-3.22"></a> + +## LOADCONF + +The syntax is: + +"+LOADCONF" CRLF ConfigText CRLF "." CRLF + +This command allows a controller to upload the text of a config file +to Tor over the control port. This config file is then loaded as if +it had been read from disk. + +\[LOADCONF was added in Tor 0.2.1.1-alpha.\] + +<a id="control-spec.txt-3.23"></a> + +## TAKEOWNERSHIP + +The syntax is: + +"TAKEOWNERSHIP" CRLF + +This command instructs Tor to shut down when this control +connection is closed. This command affects each control connection +that sends it independently; if multiple control connections send +the TAKEOWNERSHIP command to a Tor instance, Tor will shut down when +any of those connections closes. + +(As of Tor 0.2.5.2-alpha, Tor does not wait a while for circuits to +close when shutting down because of an exiting controller. If you +want to ensure a clean shutdown--and you should!--then send "SIGNAL +SHUTDOWN" and wait for the Tor process to close.) + +This command is intended to be used with the +\_\_OwningControllerProcess configuration option. A controller that +starts a Tor process which the user cannot easily control or stop +should 'own' that Tor process: + +```text + * When starting Tor, the controller should specify its PID in an + __OwningControllerProcess on Tor's command line. This will + cause Tor to poll for the existence of a process with that PID, + and exit if it does not find such a process. (This is not a + completely reliable way to detect whether the 'owning + controller' is still running, but it should work well enough in + most cases.) + + * Once the controller has connected to Tor's control port, it + should send the TAKEOWNERSHIP command along its control + connection. At this point, *both* the TAKEOWNERSHIP command and + the __OwningControllerProcess option are in effect: Tor will + exit when the control connection ends *and* Tor will exit if it + detects that there is no process with the PID specified in the + __OwningControllerProcess option. + + * After the controller has sent the TAKEOWNERSHIP command, it + should send "RESETCONF __OwningControllerProcess" along its + control connection. This will cause Tor to stop polling for the + existence of a process with its owning controller's PID; Tor + will still exit when the control connection ends. + + [TAKEOWNERSHIP was added in Tor 0.2.2.28-beta.] +``` + +<a id="control-spec.txt-3.24"></a> + +## AUTHCHALLENGE + +The syntax is: + +```text + "AUTHCHALLENGE" SP "SAFECOOKIE" + SP ClientNonce + CRLF + + ClientNonce = 2*HEXDIG / QuotedString +``` + +This command is used to begin the authentication routine for the +SAFECOOKIE method of authentication. + +If the server accepts the command, the server reply format is: + +```text + "250 AUTHCHALLENGE" + SP "SERVERHASH=" ServerHash + SP "SERVERNONCE=" ServerNonce + CRLF + + ServerHash = 64*64HEXDIG + ServerNonce = 64*64HEXDIG +``` + +The ClientNonce, ServerHash, and ServerNonce values are +encoded/decoded in the same way as the argument passed to the +AUTHENTICATE command. ServerNonce MUST be 32 bytes long. + +ServerHash is computed as: + +```text + HMAC-SHA256("Tor safe cookie authentication server-to-controller hash", + CookieString | ClientNonce | ServerNonce) + + (with the HMAC key as its first argument) +``` + +After a controller sends a successful AUTHCHALLENGE command, the +next command sent on the connection must be an AUTHENTICATE command, +and the only authentication string which that AUTHENTICATE command +will accept is: + +```text + HMAC-SHA256("Tor safe cookie authentication controller-to-server hash", + CookieString | ClientNonce | ServerNonce) +``` + +\[Unlike other commands besides AUTHENTICATE, AUTHCHALLENGE may be +used (but only once!) before AUTHENTICATE.\] + +\[AUTHCHALLENGE was added in Tor 0.2.3.13-alpha.\] + +<a id="control-spec.txt-3.25"></a> + +## DROPGUARDS + +The syntax is: + +"DROPGUARDS" CRLF + +Tells the server to drop all guard nodes. Do not invoke this command +lightly; it can increase vulnerability to tracking attacks over time. + +Tor replies with "250 OK" on success. + +\[DROPGUARDS was added in Tor 0.2.5.2-alpha.\] + +<a id="control-spec.txt-3.26"></a> + +## HSFETCH + +The syntax is: + +```text + "HSFETCH" SP (HSAddress / "v" Version "-" DescId) + *[SP "SERVER=" Server] CRLF + + HSAddress = 16*Base32Character / 56*Base32Character + Version = "2" / "3" + DescId = 32*Base32Character + Server = LongName +``` + +This command launches hidden service descriptor fetch(es) for the given +HSAddress or DescId. + +HSAddress can be version 2 or version 3 addresses. DescIDs can only be +version 2 IDs. Version 2 addresses consist of 16*Base32Character and +version 3 addresses consist of 56*Base32Character. + +If a DescId is specified, at least one Server MUST also be provided, +otherwise a 512 error is returned. If no DescId and Server(s) are specified, +it behaves like a normal Tor client descriptor fetch. If one or more +Server are given, they are used instead triggering a fetch on each of them +in parallel. + +The caching behavior when fetching a descriptor using this command is +identical to normal Tor client behavior. + +Details on how to compute a descriptor id (DescId) can be found in +rend-spec.txt section 1.3. + +If any values are unrecognized, a 513 error is returned and the command is +stopped. On success, Tor replies "250 OK" then Tor MUST eventually follow +this with both a HS_DESC and HS_DESC_CONTENT events with the results. If +SERVER is specified then events are emitted for each location. + +Examples are: + +```text + C: HSFETCH v2-gezdgnbvgy3tqolbmjrwizlgm5ugs2tl + SERVER=9695DFC35FFEB861329B9F1AB04C46397020CE31 + S: 250 OK + + C: HSFETCH ajkhdsfuygaesfaa + S: 250 OK + + C: HSFETCH vww6ybal4bd7szmgncyruucpgfkqahzddi37ktceo3ah7ngmcopnpyyd + S: 250 OK +``` + +\[HSFETCH was added in Tor 0.2.7.1-alpha\] +\[HS v3 support added 0.4.1.1-alpha\] + +<a id="control-spec.txt-3.27"></a> + +## ADD_ONION + +The syntax is: + +```text + "ADD_ONION" SP KeyType ":" KeyBlob + [SP "Flags=" Flag *("," Flag)] + [SP "MaxStreams=" NumStreams] + 1*(SP "Port=" VirtPort ["," Target]) + *(SP "ClientAuth=" ClientName [":" ClientBlob]) CRLF + *(SP "ClientAuthV3=" V3Key) CRLF + + KeyType = + "NEW" / ; The server should generate a key of algorithm KeyBlob + "RSA1024" / ; The server should use the 1024 bit RSA key provided + in as KeyBlob (v2). + "ED25519-V3"; The server should use the ed25519 v3 key provided in as + KeyBlob (v3). + + KeyBlob = + "BEST" / ; The server should generate a key using the "best" + supported algorithm (KeyType == "NEW"). + [As of 0.4.2.3-alpha, ED25519-V3 is used] + "RSA1024" / ; The server should generate a 1024 bit RSA key + (KeyType == "NEW") (v2). + "ED25519-V3"; The server should generate an ed25519 private key + (KeyType == "NEW") (v3). + String ; A serialized private key (without whitespace) + + Flag = + "DiscardPK" / ; The server should not include the newly generated + private key as part of the response. + "Detach" / ; Do not associate the newly created Onion Service + to the current control connection. + "BasicAuth" / ; Client authorization is required using the "basic" + method (v2 only). + "V3Auth" / ; Version 3 client authorization is required (v3 only). + + "NonAnonymous" /; Add a non-anonymous Single Onion Service. Tor + checks this flag matches its configured hidden + service anonymity mode. + "MaxStreamsCloseCircuit"; Close the circuit is the maximum streams + allowed is reached. + + NumStreams = A value between 0 and 65535 which is used as the maximum + streams that can be attached on a rendezvous circuit. Setting + it to 0 means unlimited which is also the default behavior. + + VirtPort = The virtual TCP Port for the Onion Service (As in the + HiddenServicePort "VIRTPORT" argument). + + Target = The (optional) target for the given VirtPort (As in the + optional HiddenServicePort "TARGET" argument). + + ClientName = An identifier 1 to 16 characters long, using only + characters in A-Za-z0-9+-_ (no spaces) (v2 only). + + ClientBlob = Authorization data for the client, in an opaque format + specific to the authorization method (v2 only). + + V3Key = The client's base32-encoded x25519 public key, using only the key + part of rend-spec-v3.txt section G.1.2 (v3 only). + + The server reply format is: + + "250-ServiceID=" ServiceID CRLF + ["250-PrivateKey=" KeyType ":" KeyBlob CRLF] + *("250-ClientAuth=" ClientName ":" ClientBlob CRLF) + "250 OK" CRLF + + ServiceID = The Onion Service address without the trailing ".onion" + suffix +``` + +Tells the server to create a new Onion ("Hidden") Service, with the +specified private key and algorithm. If a KeyType of "NEW" is selected, +the server will generate a new keypair using the selected algorithm. +The "Port" argument's VirtPort and Target values have identical +semantics to the corresponding HiddenServicePort configuration values. + +The server response will only include a private key if the server was +requested to generate a new keypair, and also the "DiscardPK" flag was +not specified. (Note that if "DiscardPK" flag is specified, there is no +way to recreate the generated keypair and the corresponding Onion +Service at a later date). + +If client authorization is enabled using the "BasicAuth" flag (which is v2 +only), the service will not be accessible to clients without valid +authorization data (configured with the "HidServAuth" option). The list of +authorized clients is specified with one or more "ClientAuth" parameters. +If "ClientBlob" is not specified for a client, a new credential will be +randomly generated and returned. + +Tor instances can either be in anonymous hidden service mode, or +non-anonymous single onion service mode. All hidden services on the same +tor instance have the same anonymity. To guard against unexpected loss +of anonymity, Tor checks that the ADD_ONION "NonAnonymous" flag matches +the current hidden service anonymity mode. The hidden service anonymity +mode is configured using the Tor options HiddenServiceSingleHopMode and +HiddenServiceNonAnonymousMode. If both these options are 1, the +"NonAnonymous" flag must be provided to ADD_ONION. If both these options +are 0 (the Tor default), the flag must NOT be provided. + +Once created the new Onion Service will remain active until either the +Onion Service is removed via "DEL_ONION", the server terminates, or the +control connection that originated the "ADD_ONION" command is closed. +It is possible to override disabling the Onion Service on control +connection close by specifying the "Detach" flag. + +It is the Onion Service server application's responsibility to close +existing client connections if desired after the Onion Service is +removed. + +(The KeyBlob format is left intentionally opaque, however for "RSA1024" +keys it is currently the Base64 encoded DER representation of a PKCS#1 +RSAPrivateKey, with all newlines removed. For a "ED25519-V3" key is +the Base64 encoding of the concatenation of the 32-byte ed25519 secret +scalar in little-endian and the 32-byte ed25519 PRF secret.) + +\[Note: The ED25519-V3 format is not the same as, e.g., SUPERCOP +ed25519/ref, which stores the concatenation of the 32-byte ed25519 +hash seed concatenated with the 32-byte public key, and which derives +the secret scalar and PRF secret by expanding the hash seed with +SHA-512. Our key blinding scheme is incompatible with storing +private keys as seeds, so we store the secret scalar alongside the +PRF secret, and just pay the cost of recomputing the public key when +importing an ED25519-V3 key.\] + +Examples: + +```text + C: ADD_ONION NEW:BEST Flags=DiscardPK Port=80 + S: 250-ServiceID=exampleoniont2pqglbny66wpovyvao3ylc23eileodtevc4b75ikpad + S: 250 OK + + C: ADD_ONION RSA1024:[Blob Redacted] Port=80,192.168.1.1:8080 + S: 250-ServiceID=sampleonion12456 + S: 250 OK + + C: ADD_ONION NEW:BEST Port=22 Port=80,8080 + S: 250-ServiceID=sampleonion4t2pqglbny66wpovyvao3ylc23eileodtevc4b75ikpad + S: 250-PrivateKey=ED25519-V3:[Blob Redacted] + S: 250 OK + + C: ADD_ONION NEW:RSA1024 Flags=DiscardPK,BasicAuth Port=22 + ClientAuth=alice:[Blob Redacted] ClientAuth=bob + S: 250-ServiceID=testonion1234567 + S: 250-ClientAuth=bob:[Blob Redacted] + S: 250 OK + + C: ADD_ONION NEW:ED25519-V3 ClientAuthV3=[Blob Redacted] Port=22 + S: 250-ServiceID=n35etu3yjxrqjpntmfziom5sjwspoydchmelc4xleoy4jk2u4lziz2yd + S: 250-ClientAuthV3=[Blob Redacted] + S: 250 OK + + Examples with Tor in anonymous onion service mode: + + C: ADD_ONION NEW:BEST Flags=DiscardPK Port=22 + S: 250-ServiceID=exampleoniont2pqglbny66wpovyvao3ylc23eileodtevc4b75ikpad + S: 250 OK + + C: ADD_ONION NEW:BEST Flags=DiscardPK,NonAnonymous Port=22 + S: 512 Tor is in anonymous hidden service mode + + Examples with Tor in non-anonymous onion service mode: + + C: ADD_ONION NEW:BEST Flags=DiscardPK Port=22 + S: 512 Tor is in non-anonymous hidden service mode + + C: ADD_ONION NEW:BEST Flags=DiscardPK,NonAnonymous Port=22 + S: 250-ServiceID=exampleoniont2pqglbny66wpovyvao3ylc23eileodtevc4b75ikpad + S: 250 OK +``` + +\[ADD_ONION was added in Tor 0.2.7.1-alpha.\] +\[MaxStreams and MaxStreamsCloseCircuit were added in Tor 0.2.7.2-alpha\] +\[ClientAuth was added in Tor 0.2.9.1-alpha. It is v2 only.\] +\[NonAnonymous was added in Tor 0.2.9.3-alpha.\] +\[HS v3 support added 0.3.3.1-alpha\] +\[ClientV3Auth support added 0.4.6.1-alpha\] + +<a id="control-spec.txt-3.28"></a> + +## DEL_ONION + +The syntax is: + +"DEL_ONION" SP ServiceID CRLF + +```text + ServiceID = The Onion Service address without the trailing ".onion" + suffix +``` + +Tells the server to remove an Onion ("Hidden") Service, that was +previously created via an "ADD_ONION" command. It is only possible to +remove Onion Services that were created on the same control connection +as the "DEL_ONION" command, and those that belong to no control +connection in particular (The "Detach" flag was specified at creation). + +If the ServiceID is invalid, or is neither owned by the current control +connection nor a detached Onion Service, the server will return a 552. + +It is the Onion Service server application's responsibility to close +existing client connections if desired after the Onion Service has been +removed via "DEL_ONION". + +Tor replies with "250 OK" on success, or a 512 if there are an invalid +number of arguments, or a 552 if it doesn't recognize the ServiceID. + +\[DEL_ONION was added in Tor 0.2.7.1-alpha.\] +\[HS v3 support added 0.3.3.1-alpha\] + +<a id="control-spec.txt-3.29"></a> + +## HSPOST + +The syntax is: + +```text + "+HSPOST" *[SP "SERVER=" Server] [SP "HSADDRESS=" HSAddress] + CRLF Descriptor CRLF "." CRLF + + Server = LongName + HSAddress = 56*Base32Character + Descriptor = The text of the descriptor formatted as specified + in rend-spec.txt section 1.3. +``` + +The "HSAddress" key is optional and only applies for v3 descriptors. A 513 +error is returned if used with v2. + +This command launches a hidden service descriptor upload to the specified +HSDirs. If one or more Server arguments are provided, an upload is triggered +on each of them in parallel. If no Server options are provided, it behaves +like a normal HS descriptor upload and will upload to the set of responsible +HS directories. + +If any value is unrecognized, a 552 error is returned and the command is +stopped. If there is an error in parsing the descriptor, the server +must send a "554 Invalid descriptor" reply. + +On success, Tor replies "250 OK" then Tor MUST eventually follow +this with a HS_DESC event with the result for each upload location. + +Examples are: + +```text + C: +HSPOST SERVER=9695DFC35FFEB861329B9F1AB04C46397020CE31 + [DESCRIPTOR] + . + S: 250 OK + + [HSPOST was added in Tor 0.2.7.1-alpha] +``` + +<a id="control-spec.txt-3.30"></a> + +## ONION_CLIENT_AUTH_ADD + +The syntax is: + +```text + "ONION_CLIENT_AUTH_ADD" SP HSAddress + SP KeyType ":" PrivateKeyBlob + [SP "ClientName=" Nickname] + [SP "Flags=" TYPE] CRLF + + HSAddress = 56*Base32Character + KeyType = "x25519" is the only one supported right now + PrivateKeyBlob = base64 encoding of x25519 key +``` + +Tells the connected Tor to add client-side v3 client auth credentials for the +onion service with "HSAddress". The "PrivateKeyBlob" is the x25519 private +key that should be used for this client, and "Nickname" is an optional +nickname for the client. + +FLAGS is a comma-separated tuple of flags for this new client. For now, the +currently supported flags are: + +```text + "Permanent" - This client's credentials should be stored in the filesystem. + If this is not set, the client's credentials are ephemeral + and stored in memory. +``` + +If client auth credentials already existed for this service, replace them +with the new ones. + +If Tor has cached onion service descriptors that it has been unable to +decrypt in the past (due to lack of client auth credentials), attempt to +decrypt those descriptors as soon as this command succeeds. + +On success, "250 OK" is returned. Otherwise, the following error codes exist: + +```text + 251 - Client auth credentials for this onion service already existed and replaced. + 252 - Added client auth credentials and successfully decrypted a cached descriptor. + 451 - We reached authorized client capacity + 512 - Syntax error in "HSAddress", or "PrivateKeyBlob" or "Nickname" + 551 - Client with with this "Nickname" already exists + 552 - Unrecognized KeyType + + [ONION_CLIENT_AUTH_ADD was added in Tor 0.4.3.1-alpha] +``` + +<a id="control-spec.txt-3.31"></a> + +## ONION_CLIENT_AUTH_REMOVE + +The syntax is: + +"ONION_CLIENT_AUTH_REMOVE" SP HSAddress + +KeyType = "x25519" is the only one supported right now + +Tells the connected Tor to remove the client-side v3 client auth credentials +for the onion service with "HSAddress". + +On success "250 OK" is returned. Otherwise, the following error codes exist: + +```text + 512 - Syntax error in "HSAddress". + 251 - Client credentials for "HSAddress" did not exist. + + [ONION_CLIENT_AUTH_REMOVE was added in Tor 0.4.3.1-alpha] +``` + +<a id="control-spec.txt-3.32"></a> + +## ONION_CLIENT_AUTH_VIEW + +The syntax is: + +"ONION_CLIENT_AUTH_VIEW" \[SP HSAddress\] CRLF + +Tells the connected Tor to list all the stored client-side v3 client auth +credentials for "HSAddress". If no "HSAddress" is provided, list all the +stored client-side v3 client auth credentials. + +The server reply format is: + +```text + "250-ONION_CLIENT_AUTH_VIEW" [SP HSAddress] CRLF + *("250-CLIENT" SP HSAddress SP KeyType ":" PrivateKeyBlob + [SP "ClientName=" Nickname] + [SP "Flags=" FLAGS] CRLF) + "250 OK" CRLF + + HSAddress = The onion address under which this credential is stored + KeyType = "x25519" is the only one supported right now + PrivateKeyBlob = base64 encoding of x25519 key +``` + +"Nickname" is an optional nickname for this client, which can be set either +through the ONION_CLIENT_AUTH_ADD command, or it's the filename of this +client if the credentials are stored in the filesystem. + +FLAGS is a comma-separated field of flags for this client, the currently +supported flags are: + +"Permanent" - This client's credentials are stored in the filesystem. + +On success "250 OK" is returned. Otherwise, the following error codes exist: + +512 - Syntax error in "HSAddress". + +\[ONION_CLIENT_AUTH_VIEW was added in Tor 0.4.3.1-alpha\] + +<a id="control-spec.txt-3.33"></a> + +## DROPOWNERSHIP + +The syntax is: + +"DROPOWNERSHIP" CRLF + +This command instructs Tor to relinquish ownership of its control +connection. As such tor will not shut down when this control +connection is closed. + +This method is idempotent. If the control connection does not +already have ownership this method returns successfully, and +does nothing. + +The controller can call TAKEOWNERSHIP again to re-establish +ownership. + +\[DROPOWNERSHIP was added in Tor 0.4.0.0-alpha\] + +<a id="control-spec.txt-3.34"></a> + +## DROPTIMEOUTS + +```text + The syntax is: + "DROPTIMEOUTS" CRLF +``` + +Tells the server to drop all circuit build times. Do not invoke this command +lightly; it can increase vulnerability to tracking attacks over time. + +Tor replies with "250 OK" on success. Tor also emits the BUILDTIMEOUT_SET +RESET event right after this "250 OK". + +\[DROPTIMEOUTS was added in Tor 0.4.5.0-alpha.\] diff --git a/spec/control-spec/implementation-notes.md b/spec/control-spec/implementation-notes.md new file mode 100644 index 0000000..97ac811 --- /dev/null +++ b/spec/control-spec/implementation-notes.md @@ -0,0 +1,721 @@ +<a id="control-spec.txt-5"></a> + +# Implementation notes + +<a id="control-spec.txt-5.1"></a> + +## Authentication + +If the control port is open and no authentication operation is enabled, Tor +trusts any local user that connects to the control port. This is generally +a poor idea. + +If the 'CookieAuthentication' option is true, Tor writes a "magic +cookie" file named "control_auth_cookie" into its data directory (or +to another file specified in the 'CookieAuthFile' option). To +authenticate, the controller must demonstrate that it can read the +contents of the cookie file: + +- Current versions of Tor support cookie authentication + +```text + using the "COOKIE" authentication method: the controller sends the + contents of the cookie file, encoded in hexadecimal. This + authentication method exposes the user running a controller to an + unintended information disclosure attack whenever the controller + has greater filesystem read access than the process that it has + connected to. (Note that a controller may connect to a process + other than Tor.) It is almost never safe to use, even if the + controller's user has explicitly specified which filename to read + an authentication cookie from. For this reason, the COOKIE + authentication method has been deprecated and will be removed from + Tor before some future version of Tor. + + * 0.2.2.x versions of Tor starting with 0.2.2.36, and all versions of + + Tor after 0.2.3.12-alpha, support cookie authentication using the + "SAFECOOKIE" authentication method, which discloses much less + information about the contents of the cookie file. +``` + +If the 'HashedControlPassword' option is set, it must contain the salted +hash of a secret password. The salted hash is computed according to the +S2K algorithm in RFC 2440 (OpenPGP), and prefixed with the s2k specifier. +This is then encoded in hexadecimal, prefixed by the indicator sequence +"16:". Thus, for example, the password 'foo' could encode to: + +```text + 16:660537E3E1CD49996044A3BF558097A981F539FEA2F9DA662B4626C1C2 + ++++++++++++++++**^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + salt hashed value + indicator +``` + +You can generate the salt of a password by calling + +```text + 'tor --hash-password <password>' +``` + +or by using the example code in the Python and Java controller libraries. +To authenticate under this scheme, the controller sends Tor the original +secret that was used to generate the password, either as a quoted string +or encoded in hexadecimal. + +<a id="control-spec.txt-5.2"></a> + +## Don't let the buffer get too big { #buffer-size } + +With old versions of Tor (before 0.2.0.16-alpha), if you ask for +lots of events, and 16MB of them queue up on the buffer, the Tor +process will close the socket. + +Newer Tor versions do not have this 16 MB buffer limit. However, +if you leave huge numbers of events unread, Tor may still run out +of memory, so you should still be careful about buffer size. + +<a id="control-spec.txt-5.3"></a> + +## Backward compatibility with v0 control protocol { #v0-compat } + +The 'version 0' control protocol was replaced in Tor 0.1.1.x. Support +was removed in Tor 0.2.0.x. Every non-obsolete version of Tor now +supports the version 1 control protocol. + +For backward compatibility with the "version 0" control protocol, +Tor used to check whether the third octet of the first command is zero. +(If it was, Tor assumed that version 0 is in use.) + +This compatibility was removed in Tor 0.1.2.16 and 0.2.0.4-alpha. + +<a id="control-spec.txt-5.4"></a> + +## Tor config options for use by controllers { #special-config-options } + +Tor provides a few special configuration options for use by controllers. +These options are not saved to disk by SAVECONF. Most can be set and +examined by the SETCONF and GETCONF commands, but some (noted below) can +only be given in a torrc file or on the command line. + +Generally, these options make Tor unusable by disabling a portion of Tor's +normal operations. Unless a controller provides replacement functionality +to fill this gap, Tor will not correctly handle user requests. + +\_\_AllDirActionsPrivate + +```text + If true, Tor will try to launch all directory operations through + anonymous connections. (Ordinarily, Tor only tries to anonymize + requests related to hidden services.) This option will slow down + directory access, and may stop Tor from working entirely if it does not + yet have enough directory information to build circuits. + + (Boolean. Default: "0".) + + __DisablePredictedCircuits + + If true, Tor will not launch preemptive "general-purpose" circuits for + streams to attach to. (It will still launch circuits for testing and + for hidden services.) + + (Boolean. Default: "0".) + + __LeaveStreamsUnattached + + If true, Tor will not automatically attach new streams to circuits; + instead, the controller must attach them with ATTACHSTREAM. If the + controller does not attach the streams, their data will never be routed. + + (Boolean. Default: "0".) + + __HashedControlSessionPassword + + As HashedControlPassword, but is not saved to the torrc file by + SAVECONF. Added in Tor 0.2.0.20-rc. + + __ReloadTorrcOnSIGHUP + + If this option is true (the default), we reload the torrc from disk + every time we get a SIGHUP (from the controller or via a signal). + Otherwise, we don't. This option exists so that controllers can keep + their options from getting overwritten when a user sends Tor a HUP for + some other reason (for example, to rotate the logs). + + (Boolean. Default: "1") + + __OwningControllerProcess + + If this option is set to a process ID, Tor will periodically check + whether a process with the specified PID exists, and exit if one + does not. Added in Tor 0.2.2.28-beta. This option's intended use + is documented in section 3.23 with the related TAKEOWNERSHIP + command. + + Note that this option can only specify a single process ID, unlike + the TAKEOWNERSHIP command which can be sent along multiple control + connections. + + (String. Default: unset.) + + __OwningControllerFD + + If this option is a valid socket, Tor will start with an open control + connection on this socket. Added in Tor 0.3.3.1-alpha. + + This socket will be an owning controller, as if it had already called + TAKEOWNERSHIP. It will be automatically authenticated. This option + should only be used by other programs that are starting Tor. + + This option cannot be changed via SETCONF; it must be set in a torrc or + via the command line. + + (Integer. Default: -1.) + + __DisableSignalHandlers + + If this option is set to true during startup, then Tor will not install + any signal handlers to watch for POSIX signals. The SIGNAL controller + command will still work. + + This option is meant for embedding Tor inside another process, when + the controlling process would rather handle signals on its own. + + This option cannot be changed via SETCONF; it must be set in a torrc or + via the command line. + + (Boolean. Default: 0.) +``` + +<a id="control-spec.txt-5.5"></a> + +## Phases from the Bootstrap status event { #bootstrap-phases } + +```text + [For the bootstrap phases reported by Tor prior to 0.4.0.x, see + Section 5.6.] +``` + +This section describes the various bootstrap phases currently reported +by Tor. Controllers should not assume that the percentages and tags +listed here will continue to match up, or even that the tags will stay +in the same order. Some phases might also be skipped (not reported) +if the associated bootstrap step is already complete, or if the phase +no longer is necessary. Only "starting" and "done" are guaranteed to +exist in all future versions. + +Current Tor versions enter these phases in order, monotonically. +Future Tors MAY revisit earlier phases, for example, if the network +fails. + +<a id="control-spec.txt-5.5.1"></a> + +### Overview of Bootstrap reporting { #bootstrap-overview } + +Bootstrap phases can be viewed as belonging to one of three stages: + +1. Initial connection to a Tor relay or bridge +1. Obtaining directory information +1. Building an application circuit + +Tor doesn't specifically enter Stage 1; that is a side effect of +other actions that Tor is taking. Tor could be making a connection +to a fallback directory server, or it could be making a connection +to a guard candidate. Either one counts as Stage 1 for the purposes +of bootstrap reporting. + +Stage 2 might involve Tor contacting directory servers, or it might +involve reading cached directory information from a previous +session. Large parts of Stage 2 might be skipped if there is already +enough cached directory information to build circuits. Tor will +defer reporting progress in Stage 2 until Stage 1 is complete. + +Tor defers this reporting because Tor can already have enough +directory information to build circuits, yet not be able to connect +to a relay. Without that deferral, a user might misleadingly see Tor +stuck at a large amount of progress when something as fundamental as +making a TCP connection to any relay is failing. + +Tor also doesn't specifically enter Stage 3; that is a side effect +of Tor building circuits for some purpose or other. In a typical +client, Tor builds predicted circuits to provide lower latency for +application connection requests. In Stage 3, Tor might make new +connections to relays or bridges that it did not connect to in Stage +1\. + +<a id="control-spec.txt-5.5.2"></a> + +### Phases in Bootstrap Stage 1 { #bootstrap-stage1 } + +Phase 0: +tag=starting summary="Starting" + +Tor starts out in this phase. + +```text + Phase 1: + tag=conn_pt summary="Connecting to pluggable transport" + [This phase is new in 0.4.0.x] +``` + +Tor is making a TCP connection to the transport plugin for a +pluggable transport. Tor will use this pluggable transport to make +its first connection to a bridge. + +```text + Phase 2: + tag=conn_done_pt summary="Connected to pluggable transport" + [New in 0.4.0.x] +``` + +Tor has completed its TCP connection to the transport plugin for the +pluggable transport. + +```text + Phase 3: + tag=conn_proxy summary="Connecting to proxy" + [New in 0.4.0.x] +``` + +Tor is making a TCP connection to a proxy to make its first +connection to a relay or bridge. + +```text + Phase 4: + tag=conn_done_proxy summary="Connected to proxy" + [New in 0.4.0.x] +``` + +Tor has completed its TCP connection to a proxy to make its first +connection to a relay or bridge. + +```text + Phase 5: + tag=conn summary="Connecting to a relay" + [New in 0.4.0.x; prior versions of Tor had a "conn_dir" phase that + sometimes but not always corresponded to connecting to a directory server] +``` + +Tor is making its first connection to a relay. This might be through +a pluggable transport or proxy connection that Tor has already +established. + +```text + Phase 10: + tag=conn_done summary="Connected to a relay" + [New in 0.4.0.x] + + Tor has completed its first connection to a relay. + + Phase 14: + tag=handshake summary="Handshaking with a relay" + [New in 0.4.0.x; prior versions of Tor had a "handshake_dir" phase] + + Tor is in the process of doing a TLS handshake with a relay. + + Phase 15: + tag=handshake_done summary="Handshake with a relay done" + [New in 0.4.0.x] + + Tor has completed its TLS handshake with a relay. +``` + +<a id="control-spec.txt-5.5.3"></a> + +### Phases in Bootstrap Stage 2 { #bootstrap-stage2 } + +```text + Phase 20: + tag=onehop_create summary="Establishing an encrypted directory connection" + [prior to 0.4.0.x, this was numbered 15] +``` + +Once TLS is finished with a relay, Tor will send a CREATE_FAST cell +to establish a one-hop circuit for retrieving directory information. +It will remain in this phase until it receives the CREATED_FAST cell +back, indicating that the circuit is ready. + +```text + Phase 25: + tag=requesting_status summary="Asking for networkstatus consensus" + [prior to 0.4.0.x, this was numbered 20] +``` + +Once we've finished our one-hop circuit, we will start a new stream +for fetching the networkstatus consensus. We'll stay in this phase +until we get the RELAY_CONNECTED message back, indicating that we've +established a directory connection. + +```text + Phase 30: + tag=loading_status summary="Loading networkstatus consensus" + [prior to 0.4.0.x, this was numbered 25] +``` + +Once we've established a directory connection, we will start fetching +the networkstatus consensus document. This could take a while; this +phase is a good opportunity for using the "progress" keyword to indicate +partial progress. + +This phase could stall if the directory server we picked doesn't +have a copy of the networkstatus consensus so we have to ask another, +or it does give us a copy but we don't find it valid. + +Phase 40: +tag=loading_keys summary="Loading authority key certs" + +Sometimes when we've finished loading the networkstatus consensus, +we find that we don't have all the authority key certificates for the +keys that signed the consensus. At that point we put the consensus we +fetched on hold and fetch the keys so we can verify the signatures. + +Phase 45 +tag=requesting_descriptors summary="Asking for relay descriptors" + +Once we have a valid networkstatus consensus and we've checked all +its signatures, we start asking for relay descriptors. We stay in this +phase until we have received a RELAY_CONNECTED message in response to +a request for descriptors. + +```text + [Some versions of Tor (starting with 0.2.6.2-alpha but before + 0.4.0.x): Tor could report having internal paths only; see Section + 5.6] +``` + +Phase 50: +tag=loading_descriptors summary="Loading relay descriptors" + +We will ask for relay descriptors from several different locations, +so this step will probably make up the bulk of the bootstrapping, +especially for users with slow connections. We stay in this phase until +we have descriptors for a significant fraction of the usable relays +listed in the networkstatus consensus (this can be between 25% and 95% +depending on Tor's configuration and network consensus parameters). +This phase is also a good opportunity to use the "progress" keyword to +indicate partial steps. + +```text + [Some versions of Tor (starting with 0.2.6.2-alpha but before + 0.4.0.x): Tor could report having internal paths only; see Section + 5.6] + + Phase 75: + tag=enough_dirinfo summary="Loaded enough directory info to build + circuits" + [New in 0.4.0.x; previously, Tor would misleadingly report the + "conn_or" tag once it had enough directory info.] +``` + +<a id="control-spec.txt-5.5.4"></a> + +### Phases in Bootstrap Stage 3 { #bootstrap-stage3 } + +```text + Phase 76: + tag=ap_conn_pt summary="Connecting to pluggable transport to build + circuits" + [New in 0.4.0.x] +``` + +This is similar to conn_pt, except for making connections to +additional relays or bridges that Tor needs to use to build +application circuits. + +```text + Phase 77: + tag=ap_conn_done_pt summary="Connected to pluggable transport to build circuits" + [New in 0.4.0.x] +``` + +This is similar to conn_done_pt, except for making connections to +additional relays or bridges that Tor needs to use to build +application circuits. + +```text + Phase 78: + tag=ap_conn_proxy summary="Connecting to proxy to build circuits" + [New in 0.4.0.x] +``` + +This is similar to conn_proxy, except for making connections to +additional relays or bridges that Tor needs to use to build +application circuits. + +```text + Phase 79: + tag=ap_conn_done_proxy summary="Connected to proxy to build circuits" + [New in 0.4.0.x] +``` + +This is similar to conn_done_proxy, except for making connections to +additional relays or bridges that Tor needs to use to build +application circuits. + +```text + Phase 80: + tag=ap_conn summary="Connecting to a relay to build circuits" + [New in 0.4.0.x] +``` + +This is similar to conn, except for making connections to additional +relays or bridges that Tor needs to use to build application +circuits. + +```text + Phase 85: + tag=ap_conn_done summary="Connected to a relay to build circuits" + [New in 0.4.0.x] +``` + +This is similar to conn_done, except for making connections to +additional relays or bridges that Tor needs to use to build +application circuits. + +```text + Phase 89: + tag=ap_handshake summary="Finishing handshake with a relay to build circuits" + [New in 0.4.0.x] +``` + +This is similar to handshake, except for making connections to +additional relays or bridges that Tor needs to use to build +application circuits. + +```text + Phase 90: + tag=ap_handshake_done summary="Handshake finished with a relay to build circuits" + [New in 0.4.0.x] +``` + +This is similar to handshake_done, except for making connections to +additional relays or bridges that Tor needs to use to build +application circuits. + +```text + Phase 95: + tag=circuit_create summary="Establishing a[n internal] Tor circuit" + [prior to 0.4.0.x, this was numbered 90] +``` + +Once we've finished our TLS handshake with the first hop of a circuit, +we will set about trying to make some 3-hop circuits in case we need them +soon. + +```text + [Some versions of Tor (starting with 0.2.6.2-alpha but before + 0.4.0.x): Tor could report having internal paths only; see Section + 5.6] +``` + +Phase 100: +tag=done summary="Done" + +A full 3-hop circuit has been established. Tor is ready to handle +application connections now. + +```text + [Some versions of Tor (starting with 0.2.6.2-alpha but before + 0.4.0.x): Tor could report having internal paths only; see Section + 5.6] +``` + +<a id="control-spec.txt-5.6"></a> + +## Bootstrap phases reported by older versions of Tor { #bootstrap-obsolete } + +These phases were reported by Tor older than 0.4.0.x. For newer +versions of Tor, see Section 5.5. + +```text + [Newer versions of Tor (0.2.6.2-alpha and later): + If the consensus contains Exits (the typical case), Tor will build both + exit and internal circuits. When bootstrap completes, Tor will be ready + to handle an application requesting an exit circuit to services like the + World Wide Web. +``` + +If the consensus does not contain Exits, Tor will only build internal +circuits. In this case, earlier statuses will have included "internal" +as indicated above. When bootstrap completes, Tor will be ready to handle +an application requesting an internal circuit to hidden services at +".onion" addresses. + +If a future consensus contains Exits, exit circuits may become available.\] + +Phase 0: +tag=starting summary="Starting" + +Tor starts out in this phase. + +Phase 5: +tag=conn_dir summary="Connecting to directory server" + +Tor sends this event as soon as Tor has chosen a directory server -- +e.g. one of the authorities if bootstrapping for the first time or +after a long downtime, or one of the relays listed in its cached +directory information otherwise. + +Tor will stay at this phase until it has successfully established +a TCP connection with some directory server. Problems in this phase +generally happen because Tor doesn't have a network connection, or +because the local firewall is dropping SYN packets. + +Phase 10: +tag=handshake_dir summary="Finishing handshake with directory server" + +This event occurs when Tor establishes a TCP connection with a relay or +authority used as a directory server (or its https proxy if it's using +one). Tor remains in this phase until the TLS handshake with the relay +or authority is finished. + +Problems in this phase generally happen because Tor's firewall is +doing more sophisticated MITM attacks on it, or doing packet-level +keyword recognition of Tor's handshake. + +Phase 15: +tag=onehop_create summary="Establishing an encrypted directory connection" + +Once TLS is finished with a relay, Tor will send a CREATE_FAST cell +to establish a one-hop circuit for retrieving directory information. +It will remain in this phase until it receives the CREATED_FAST cell +back, indicating that the circuit is ready. + +Phase 20: +tag=requesting_status summary="Asking for networkstatus consensus" + +Once we've finished our one-hop circuit, we will start a new stream +for fetching the networkstatus consensus. We'll stay in this phase +until we get the RELAY_CONNECTED message back, indicating that we've +established a directory connection. + +Phase 25: +tag=loading_status summary="Loading networkstatus consensus" + +Once we've established a directory connection, we will start fetching +the networkstatus consensus document. This could take a while; this +phase is a good opportunity for using the "progress" keyword to indicate +partial progress. + +This phase could stall if the directory server we picked doesn't +have a copy of the networkstatus consensus so we have to ask another, +or it does give us a copy but we don't find it valid. + +Phase 40: +tag=loading_keys summary="Loading authority key certs" + +Sometimes when we've finished loading the networkstatus consensus, +we find that we don't have all the authority key certificates for the +keys that signed the consensus. At that point we put the consensus we +fetched on hold and fetch the keys so we can verify the signatures. + +```text + Phase 45 + tag=requesting_descriptors summary="Asking for relay descriptors + [ for internal paths]" +``` + +Once we have a valid networkstatus consensus and we've checked all +its signatures, we start asking for relay descriptors. We stay in this +phase until we have received a RELAY_CONNECTED message in response to +a request for descriptors. + +```text + [Newer versions of Tor (0.2.6.2-alpha and later): + If the consensus contains Exits (the typical case), Tor will ask for + descriptors for both exit and internal paths. If not, Tor will only ask + for descriptors for internal paths. In this case, this status will + include "internal" as indicated above.] + + Phase 50: + tag=loading_descriptors summary="Loading relay descriptors[ for internal + paths]" +``` + +We will ask for relay descriptors from several different locations, +so this step will probably make up the bulk of the bootstrapping, +especially for users with slow connections. We stay in this phase until +we have descriptors for a significant fraction of the usable relays +listed in the networkstatus consensus (this can be between 25% and 95% +depending on Tor's configuration and network consensus parameters). +This phase is also a good opportunity to use the "progress" keyword to +indicate partial steps. + +```text + [Newer versions of Tor (0.2.6.2-alpha and later): + If the consensus contains Exits (the typical case), Tor will download + descriptors for both exit and internal paths. If not, Tor will only + download descriptors for internal paths. In this case, this status will + include "internal" as indicated above.] +``` + +Phase 80: +tag=conn_or summary="Connecting to the Tor network\[ internally\]" + +Once we have a valid consensus and enough relay descriptors, we choose +entry guard(s) and start trying to build some circuits. This step +is similar to the "conn_dir" phase above; the only difference is +the context. + +If a Tor starts with enough recent cached directory information, +its first bootstrap status event will be for the conn_or phase. + +```text + [Newer versions of Tor (0.2.6.2-alpha and later): + If the consensus contains Exits (the typical case), Tor will build both + exit and internal circuits. If not, Tor will only build internal circuits. + In this case, this status will include "internal(ly)" as indicated above.] + + Phase 85: + tag=handshake_or summary="Finishing handshake with first hop[ of internal + circuit]" +``` + +This phase is similar to the "handshake_dir" phase, but it gets reached +if we finish a TCP connection to a Tor relay and we have already reached +the "conn_or" phase. We'll stay in this phase until we complete a TLS +handshake with a Tor relay. + +```text + [Newer versions of Tor (0.2.6.2-alpha and later): + If the consensus contains Exits (the typical case), Tor may be finishing + a handshake with the first hop if either an exit or internal circuit. In + this case, it won't specify which type. If the consensus contains no Exits, + Tor will only build internal circuits. In this case, this status will + include "internal" as indicated above.] +``` + +Phase 90: +tag=circuit_create summary="Establishing a\[n internal\] Tor circuit" + +Once we've finished our TLS handshake with the first hop of a circuit, +we will set about trying to make some 3-hop circuits in case we need them +soon. + +```text + [Newer versions of Tor (0.2.6.2-alpha and later): + If the consensus contains Exits (the typical case), Tor will build both + exit and internal circuits. If not, Tor will only build internal circuits. + In this case, this status will include "internal" as indicated above.] +``` + +Phase 100: +tag=done summary="Done" + +A full 3-hop circuit has been established. Tor is ready to handle +application connections now. + +```text + [Newer versions of Tor (0.2.6.2-alpha and later): + If the consensus contains Exits (the typical case), Tor will build both + exit and internal circuits. At this stage, Tor will be ready to handle + an application requesting an exit circuit to services like the World + Wide Web. +``` + +If the consensus does not contain Exits, Tor will only build internal +circuits. In this case, earlier statuses will have included "internal" +as indicated above. At this stage, Tor will be ready to handle an +application requesting an internal circuit to hidden services at ".onion" +addresses. + +If a future consensus contains Exits, exit circuits may become available.\] diff --git a/spec/control-spec/index.md b/spec/control-spec/index.md new file mode 100644 index 0000000..8ce17b3 --- /dev/null +++ b/spec/control-spec/index.md @@ -0,0 +1,24 @@ +# TC: A Tor control protocol (Version 1) + +<a id="control-spec.txt-0"></a> + +## Scope + +This document describes an implementation-specific protocol that is used +for other programs (such as frontend user-interfaces) to communicate with a +locally running Tor process. It is not part of the Tor onion routing +protocol. + +This protocol replaces version 0 of TC, which is now deprecated. For +reference, TC is described in "control-spec-v0.txt". Implementors are +recommended to avoid using TC directly, but instead to use a library that +can easily be updated to use the newer protocol. (Version 0 is used by Tor +versions 0.1.0.x; the protocol in this document only works with Tor +versions in the 0.1.1.x series and later.) + +```text + The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL + NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and + "OPTIONAL" in this document are to be interpreted as described in + RFC 2119. +``` diff --git a/spec/control-spec/message-format.md b/spec/control-spec/message-format.md new file mode 100644 index 0000000..a9af669 --- /dev/null +++ b/spec/control-spec/message-format.md @@ -0,0 +1,185 @@ +<a id="control-spec.txt-2"></a> + +# Message format + +<a id="control-spec.txt-2.1"></a> + +## Description format + +The message formats listed below use ABNF as described in RFC 2234. +The protocol itself is loosely based on SMTP (see RFC 2821). + +We use the following nonterminals from RFC 2822: atom, qcontent + +We define the following general-use nonterminals: + +QuotedString = DQUOTE \*qcontent DQUOTE + +There are explicitly no limits on line length. All 8-bit characters +are permitted unless explicitly disallowed. In QuotedStrings, +backslashes and quotes must be escaped; other characters need not be +escaped. + +Wherever CRLF is specified to be accepted from the controller, Tor MAY also +accept LF. Tor, however, MUST NOT generate LF instead of CRLF. +Controllers SHOULD always send CRLF. + +<a id="control-spec.txt-2.1.1"></a> + +### Notes on an escaping bug + +CString = DQUOTE \*qcontent DQUOTE + +Note that although these nonterminals have the same grammar, they +are interpreted differently. In a QuotedString, a backslash +followed by any character represents that character. But +in a CString, the escapes "\\n", "\\t", "\\r", and the octal escapes +"\\0" ... "\\377" represent newline, tab, carriage return, and the +256 possible octet values respectively. + +The use of CString in this document reflects a bug in Tor; +they should have been QuotedString instead. In the future, they +may migrate to use QuotedString instead. If they do, the +QuotedString implementation will never place a backslash before a +"n", "t", "r", or digit, to ensure that old controllers don't get +confused. + +For future-proofing, controller implementors MAY use the following +rules to be compatible with buggy Tor implementations and with +future ones that implement the spec as intended: + +```text + Read \n \t \r and \0 ... \377 as C escapes. + Treat a backslash followed by any other character as that character. +``` + +Currently, many of the QuotedString instances below that Tor +outputs are in fact CStrings. We intend to fix this in future +versions of Tor, and document which ones were broken. (See +bugtracker ticket #14555 for a bit more information.) + +Note that this bug exists only in strings generated by Tor for the +Tor controller; Tor should parse input QuotedStrings from the +controller correctly. + +<a id="control-spec.txt-2.2"></a> + +## Commands from controller to Tor { #commands } + +```text + Command = Keyword OptArguments CRLF / "+" Keyword OptArguments CRLF CmdData + Keyword = 1*ALPHA + OptArguments = [ SP *(SP / VCHAR) ] +``` + +A command is either a single line containing a Keyword and arguments, or a +multiline command whose initial keyword begins with +, and whose data +section ends with a single "." on a line of its own. (We use a special +character to distinguish multiline commands so that Tor can correctly parse +multi-line commands that it does not recognize.) Specific commands and +their arguments are described below in section 3. + +<a id="control-spec.txt-2.3"></a> + +## Replies from Tor to the controller { #replies } + +```text + Reply = SyncReply / AsyncReply + SyncReply = *(MidReplyLine / DataReplyLine) EndReplyLine + AsyncReply = *(MidReplyLine / DataReplyLine) EndReplyLine + + MidReplyLine = StatusCode "-" ReplyLine + DataReplyLine = StatusCode "+" ReplyLine CmdData + EndReplyLine = StatusCode SP ReplyLine + ReplyLine = [ReplyText] CRLF + ReplyText = XXXX + StatusCode = 3DIGIT +``` + +Unless specified otherwise, multiple lines in a single reply from +Tor to the controller are guaranteed to share the same status +code. Specific replies are mentioned below in section 3, and +described more fully in section 4. + +\[Compatibility note: versions of Tor before 0.2.0.3-alpha sometimes +generate AsyncReplies of the form "\*(MidReplyLine / DataReplyLine)". +This is incorrect, but controllers that need to work with these +versions of Tor should be prepared to get multi-line AsyncReplies with +the final line (usually "650 OK") omitted.\] + +<a id="control-spec.txt-2.4"></a> + +## General-use tokens { #tokens } + +; CRLF means, "the ASCII Carriage Return character (decimal value 13) +; followed by the ASCII Linefeed character (decimal value 10)." +CRLF = CR LF + +; How a controller tells Tor about a particular OR. There are four +; possible formats: +; $Fingerprint -- The router whose identity key hashes to the fingerprint. +; This is the preferred way to refer to an OR. +; $Fingerprint~Nickname -- The router whose identity key hashes to the +; given fingerprint, but only if the router has the given nickname. +; $Fingerprint=Nickname -- The router whose identity key hashes to the +; given fingerprint, but only if the router is Named and has the given +; nickname. +; Nickname -- The Named router with the given nickname, or, if no such +; router exists, any router whose nickname matches the one given. +; This is not a safe way to refer to routers, since Named status +; could under some circumstances change over time. +; +; The tokens that implement the above follow: + +ServerSpec = LongName / Nickname +LongName = Fingerprint \[ "~" Nickname \] + +; For tors older than 0.3.1.3-alpha, LongName may have included an equal +; sign ("=") in lieu of a tilde ("~"). The presence of an equal sign +; denoted that the OR possessed the "Named" flag: + +LongName = Fingerprint \[ ( "=" / "~" ) Nickname \] + +Fingerprint = "$" 40*HEXDIG +NicknameChar = "a"-"z" / "A"-"Z" / "0" - "9" +Nickname = 1*19 NicknameChar + +; What follows is an outdated way to refer to ORs. +; Feature VERBOSE_NAMES replaces ServerID with LongName in events and +; GETINFO results. VERBOSE_NAMES can be enabled starting in Tor version +; 0.1.2.2-alpha and it is always-on in 0.2.2.1-alpha and later. +ServerID = Nickname / Fingerprint + +; Unique identifiers for streams or circuits. Currently, Tor only +; uses digits, but this may change +StreamID = 1*16 IDChar +CircuitID = 1*16 IDChar +ConnID = 1*16 IDChar +QueueID = 1*16 IDChar +IDChar = ALPHA / DIGIT + +Address = ip4-address / ip6-address / hostname (XXXX Define these) + +; A "CmdData" section is a sequence of octets concluded by the terminating +; sequence CRLF "." CRLF. The terminating sequence may not appear in the +; body of the data. Leading periods on lines in the data are escaped with +; an additional leading period as in RFC 2821 section 4.5.2. +CmdData = *DataLine "." CRLF +DataLine = CRLF / "." 1*LineItem CRLF / NonDotItem *LineItem CRLF +LineItem = NonCR / 1*CR NonCRLF +NonDotItem = NonDotCR / 1\*CR NonCRLF + +; ISOTime, ISOTime2, and ISOTime2Frac are time formats as specified in +; ISO8601. +; example ISOTime: "2012-01-11 12:15:33" +; example ISOTime2: "2012-01-11T12:15:33" +; example ISOTime2Frac: "2012-01-11T12:15:33.51" +IsoDatePart = 4*DIGIT "-" 2*DIGIT "-" 2*DIGIT +IsoTimePart = 2*DIGIT ":" 2*DIGIT ":" 2*DIGIT +ISOTime = IsoDatePart " " IsoTimePart +ISOTime2 = IsoDatePart "T" IsoTimePart +ISOTime2Frac = IsoTime2 \[ "." 1\*DIGIT \] + +; Numbers +LeadingDigit = "1" - "9" +UInt = LeadingDigit \*Digit diff --git a/spec/control-spec/protocol-outline.md b/spec/control-spec/protocol-outline.md new file mode 100644 index 0000000..0329b1d --- /dev/null +++ b/spec/control-spec/protocol-outline.md @@ -0,0 +1,71 @@ +<a id="control-spec.txt-1"></a> + +# Protocol outline + +TC is a bidirectional message-based protocol. It assumes an underlying +stream for communication between a controlling process (the "client" +or "controller") and a Tor process (or "server"). The stream may be +implemented via TCP, TLS-over-TCP, a Unix-domain socket, or so on, +but it must provide reliable in-order delivery. For security, the +stream should not be accessible by untrusted parties. + +In TC, the client and server send typed messages to each other over the +underlying stream. The client sends "commands" and the server sends +"replies". + +By default, all messages from the server are in response to messages from +the client. Some client requests, however, will cause the server to send +messages to the client indefinitely far into the future. Such +"asynchronous" replies are marked as such. + +Servers respond to messages in the order messages are received. + +<a id="control-spec.txt-1.1"></a> + +## Forward-compatibility + +This is an evolving protocol; new client and server behavior will be +allowed in future versions. To allow new backward-compatible behavior +on behalf of the client, we may add new commands and allow existing +commands to take new arguments in future versions. To allow new +backward-compatible server behavior, we note various places below +where servers speaking a future version of this protocol may insert +new data, and note that clients should/must "tolerate" unexpected +elements in these places. There are two ways that we do this: + +- Adding a new field to a message: + +```text + For example, we might say "This message has three space-separated + fields; clients MUST tolerate more fields." This means that a + client MUST NOT crash or otherwise fail to parse the message or + other subsequent messages when there are more than three fields, and + that it SHOULD function at least as well when more fields are + provided as it does when it only gets the fields it accepts. The + most obvious way to do this is by ignoring additional fields; the + next-most-obvious way is to report additional fields verbatim to the + user, perhaps as part of an expert UI. + + * Adding a new possible value to a list of alternatives: + + For example, we might say "This field will be OPEN, CLOSED, or + CONNECTED. Clients MUST tolerate unexpected values." This means + that a client MUST NOT crash or otherwise fail to parse the message + or other subsequent messages when there are unexpected values, and + that it SHOULD try to handle the rest of the message as well as it + can. The most obvious way to do this is by pretending that each + list of alternatives has an additional "unrecognized value" element, + and mapping any unrecognized values to that element; the + next-most-obvious way is to create a separate "unrecognized value" + element for each unrecognized value. + + Clients SHOULD NOT "tolerate" unrecognized alternatives by + pretending that the message containing them is absent. For example, + a stream closed for an unrecognized reason is nevertheless closed, + and should be reported as such. + + (If some list of alternatives is given, and there isn't an explicit + statement that clients must tolerate unexpected values, clients still + must tolerate unexpected values. The only exception would be if there + were an explicit statement that no future values will ever be added.) +``` diff --git a/spec/control-spec/replies.md b/spec/control-spec/replies.md new file mode 100644 index 0000000..6fce9b1 --- /dev/null +++ b/spec/control-spec/replies.md @@ -0,0 +1,1794 @@ +<a id="control-spec.txt-4"></a> + +# Replies + +Reply codes follow the same 3-character format as used by SMTP, with the +first character defining a status, the second character defining a +subsystem, and the third designating fine-grained information. + +The TC protocol currently uses the following first characters: + +```text + 2yz Positive Completion Reply + The command was successful; a new request can be started. + + 4yz Temporary Negative Completion reply + The command was unsuccessful but might be reattempted later. + + 5yz Permanent Negative Completion Reply + The command was unsuccessful; the client should not try exactly + that sequence of commands again. + + 6yz Asynchronous Reply + Sent out-of-order in response to an earlier SETEVENTS command. + + The following second characters are used: + + x0z Syntax + Sent in response to ill-formed or nonsensical commands. + + x1z Protocol + Refers to operations of the Tor Control protocol. + + x5z Tor + Refers to actual operations of Tor system. + + The following codes are defined: + + 250 OK + 251 Operation was unnecessary + [Tor has declined to perform the operation, but no harm was done.] + + 451 Resource exhausted + + 500 Syntax error: protocol + + 510 Unrecognized command + 511 Unimplemented command + 512 Syntax error in command argument + 513 Unrecognized command argument + 514 Authentication required + 515 Bad authentication + + 550 Unspecified Tor error + + 551 Internal error + [Something went wrong inside Tor, so that the client's + request couldn't be fulfilled.] + + 552 Unrecognized entity + [A configuration key, a stream ID, circuit ID, event, + mentioned in the command did not actually exist.] + + 553 Invalid configuration value + [The client tried to set a configuration option to an + incorrect, ill-formed, or impossible value.] + + 554 Invalid descriptor + + 555 Unmanaged entity + + 650 Asynchronous event notification +``` + +Unless specified to have specific contents, the human-readable messages +in error replies should not be relied upon to match those in this document. + +<a id="control-spec.txt-4.1"></a> + +## Asynchronous events + +These replies can be sent after a corresponding SETEVENTS command has been +received. They will not be interleaved with other Reply elements, but they +can appear between a command and its corresponding reply. For example, +this sequence is possible: + +```text + C: SETEVENTS CIRC + S: 250 OK + C: GETCONF SOCKSPORT ORPORT + S: 650 CIRC 1000 EXTENDED moria1,moria2 + S: 250-SOCKSPORT=9050 + S: 250 ORPORT=0 + + But this sequence is disallowed: + + C: SETEVENTS CIRC + S: 250 OK + C: GETCONF SOCKSPORT ORPORT + S: 250-SOCKSPORT=9050 + S: 650 CIRC 1000 EXTENDED moria1,moria2 + S: 250 ORPORT=0 +``` + +Clients MUST tolerate more arguments in an asynchronous reply than +expected, and MUST tolerate more lines in an asynchronous reply than +expected. For instance, a client that expects a CIRC message like: + +650 CIRC 1000 EXTENDED moria1,moria2 + +must tolerate: + +```text + 650-CIRC 1000 EXTENDED moria1,moria2 0xBEEF + 650-EXTRAMAGIC=99 + 650 ANONYMITY=high +``` + +If clients receives extended events (selected by USEFEATUERE +EXTENDED_EVENTS in Tor 0.1.2.2-alpha..Tor-0.2.1.x, and always-on in +Tor 0.2.2.x and later), then each event line as specified below may be +followed by additional arguments and additional lines. Additional +lines will be of the form: + +"650" ("-"/" ") KEYWORD \["=" ARGUMENTS\] CRLF + +Additional arguments will be of the form + +SP KEYWORD \["=" ( QuotedString / * NonSpDquote ) \] + +Clients MUST tolerate events with arguments and keywords they do not +recognize, and SHOULD process those events as if any unrecognized +arguments and keywords were not present. + +Clients SHOULD NOT depend on the order of keyword=value arguments, +and SHOULD NOT depend on there being no new keyword=value arguments +appearing between existing keyword=value arguments, though as of this +writing (Jun 2011) some do. Thus, extensions to this protocol should +add new keywords only after the existing keywords, until all +controllers have been fixed. At some point this "SHOULD NOT" might +become a "MUST NOT". + +<a id="control-spec.txt-4.1.1"></a> + +### Circuit status changed { #CIRC } + +The syntax is: + +```text + "650" SP "CIRC" SP CircuitID SP CircStatus [SP Path] + [SP "BUILD_FLAGS=" BuildFlags] [SP "PURPOSE=" Purpose] + [SP "HS_STATE=" HSState] [SP "REND_QUERY=" HSAddress] + [SP "TIME_CREATED=" TimeCreated] + [SP "REASON=" Reason [SP "REMOTE_REASON=" Reason]] + [SP "SOCKS_USERNAME=" EscapedUsername] + [SP "SOCKS_PASSWORD=" EscapedPassword] + [SP "HS_POW=" HSPoW ] + CRLF + + CircStatus = + "LAUNCHED" / ; circuit ID assigned to new circuit + "BUILT" / ; all hops finished, can now accept streams + "GUARD_WAIT" / ; all hops finished, waiting to see if a + ; circuit with a better guard will be usable. + "EXTENDED" / ; one more hop has been completed + "FAILED" / ; circuit closed (was not built) + "CLOSED" ; circuit closed (was built) + + Path = LongName *("," LongName) + ; In Tor versions 0.1.2.2-alpha through 0.2.2.1-alpha with feature + ; VERBOSE_NAMES turned off and before version 0.1.2.2-alpha, Path + ; is as follows: + ; Path = ServerID *("," ServerID) + + BuildFlags = BuildFlag *("," BuildFlag) + BuildFlag = "ONEHOP_TUNNEL" / "IS_INTERNAL" / + "NEED_CAPACITY" / "NEED_UPTIME" + + Purpose = "GENERAL" / "HS_CLIENT_INTRO" / "HS_CLIENT_REND" / + "HS_SERVICE_INTRO" / "HS_SERVICE_REND" / "TESTING" / + "CONTROLLER" / "MEASURE_TIMEOUT" / + "HS_VANGUARDS" / "PATH_BIAS_TESTING" / + "CIRCUIT_PADDING" + + HSState = "HSCI_CONNECTING" / "HSCI_INTRO_SENT" / "HSCI_DONE" / + "HSCR_CONNECTING" / "HSCR_ESTABLISHED_IDLE" / + "HSCR_ESTABLISHED_WAITING" / "HSCR_JOINED" / + "HSSI_CONNECTING" / "HSSI_ESTABLISHED" / + "HSSR_CONNECTING" / "HSSR_JOINED" + + HSPoWType = "v1" + HSPoWEffort = 1*DIGIT + HSPoW = HSPoWType "," HSPoWEffort + + EscapedUsername = QuotedString + EscapedPassword = QuotedString + + HSAddress = 16*Base32Character / 56*Base32Character + Base32Character = ALPHA / "2" / "3" / "4" / "5" / "6" / "7" + + TimeCreated = ISOTime2Frac + Seconds = 1*DIGIT + Microseconds = 1*DIGIT + + Reason = "NONE" / "TORPROTOCOL" / "INTERNAL" / "REQUESTED" / + "HIBERNATING" / "RESOURCELIMIT" / "CONNECTFAILED" / + "OR_IDENTITY" / "OR_CONN_CLOSED" / "TIMEOUT" / + "FINISHED" / "DESTROYED" / "NOPATH" / "NOSUCHSERVICE" / + "MEASUREMENT_EXPIRED" + + The path is provided only when the circuit has been extended at least one + hop. + + The "BUILD_FLAGS" field is provided only in versions 0.2.3.11-alpha + and later. Clients MUST accept build flags not listed above. + Build flags are defined as follows: + + ONEHOP_TUNNEL (one-hop circuit, used for tunneled directory conns) + IS_INTERNAL (internal circuit, not to be used for exiting streams) + NEED_CAPACITY (this circuit must use only high-capacity nodes) + NEED_UPTIME (this circuit must use only high-uptime nodes) + + The "PURPOSE" field is provided only in versions 0.2.1.6-alpha and + later, and only if extended events are enabled (see 3.19). Clients + MUST accept purposes not listed above. Purposes are defined as + follows: + + GENERAL (circuit for AP and/or directory request streams) + HS_CLIENT_INTRO (HS client-side introduction-point circuit) + HS_CLIENT_REND (HS client-side rendezvous circuit; carries AP streams) + HS_SERVICE_INTRO (HS service-side introduction-point circuit) + HS_SERVICE_REND (HS service-side rendezvous circuit) + TESTING (reachability-testing circuit; carries no traffic) + CONTROLLER (circuit built by a controller) + MEASURE_TIMEOUT (circuit being kept around to see how long it takes) + HS_VANGUARDS (circuit created ahead of time when using + HS vanguards, and later repurposed as needed) + PATH_BIAS_TESTING (circuit used to probe whether our circuits are + being deliberately closed by an attacker) + CIRCUIT_PADDING (circuit that is being held open to disguise its + true close time) + + The "HS_STATE" field is provided only for hidden-service circuits, + and only in versions 0.2.3.11-alpha and later. Clients MUST accept + hidden-service circuit states not listed above. Hidden-service + circuit states are defined as follows: + + HSCI_* (client-side introduction-point circuit states) + HSCI_CONNECTING (connecting to intro point) + HSCI_INTRO_SENT (sent INTRODUCE1; waiting for reply from IP) + HSCI_DONE (received reply from IP relay; closing) + + HSCR_* (client-side rendezvous-point circuit states) + HSCR_CONNECTING (connecting to or waiting for reply from RP) + HSCR_ESTABLISHED_IDLE (established RP; waiting for introduction) + HSCR_ESTABLISHED_WAITING (introduction sent to HS; waiting for rend) + HSCR_JOINED (connected to HS) + + HSSI_* (service-side introduction-point circuit states) + HSSI_CONNECTING (connecting to intro point) + HSSI_ESTABLISHED (established intro point) + + HSSR_* (service-side rendezvous-point circuit states) + HSSR_CONNECTING (connecting to client's rend point) + HSSR_JOINED (connected to client's RP circuit) + + The "SOCKS_USERNAME" and "SOCKS_PASSWORD" fields indicate the credentials + that were used by a SOCKS client to connect to Tor's SOCKS port and + initiate this circuit. (Streams for SOCKS clients connected with different + usernames and/or passwords are isolated on separate circuits if the + IsolateSOCKSAuth flag is active; see Proposal 171.) [Added in Tor + 0.4.3.1-alpha.] + + The "REND_QUERY" field is provided only for hidden-service-related + circuits, and only in versions 0.2.3.11-alpha and later. Clients + MUST accept hidden service addresses in formats other than that + specified above. [Added in Tor 0.4.3.1-alpha.] + + The "TIME_CREATED" field is provided only in versions 0.2.3.11-alpha and + later. TIME_CREATED is the time at which the circuit was created or + cannibalized. [Added in Tor 0.4.3.1-alpha.] + + The "REASON" field is provided only for FAILED and CLOSED events, and only + if extended events are enabled (see 3.19). Clients MUST accept reasons + not listed above. [Added in Tor 0.4.3.1-alpha.] Reasons are as given in + tor-spec.txt, except for: + + NOPATH (Not enough nodes to make circuit) + MEASUREMENT_EXPIRED (As "TIMEOUT", except that we had left the circuit + open for measurement purposes to see how long it + would take to finish.) + IP_NOW_REDUNDANT (Closing a circuit to an introduction point that + has become redundant, since some other circuit + opened in parallel with it has succeeded.) + + The "REMOTE_REASON" field is provided only when we receive a DESTROY cell or + RELAY_TRUNCATE message, and only if extended events are enabled. It contains the + actual reason given by the remote OR for closing the circuit. Clients MUST + accept reasons not listed above. Reasons are as listed in tor-spec.txt. + [Added in Tor 0.4.3.1-alpha.] +``` + +<a id="control-spec.txt-4.1.2"></a> + +### Stream status changed { #STREAM } + +The syntax is: + +```text + "650" SP "STREAM" SP StreamID SP StreamStatus SP CircuitID SP Target + [SP "REASON=" Reason [ SP "REMOTE_REASON=" Reason ]] + [SP "SOURCE=" Source] [ SP "SOURCE_ADDR=" Address ":" Port ] + [SP "PURPOSE=" Purpose] [SP "SOCKS_USERNAME=" EscapedUsername] + [SP "SOCKS_PASSWORD=" EscapedPassword] + [SP "CLIENT_PROTOCOL=" ClientProtocol] [SP "NYM_EPOCH=" NymEpoch] + [SP "SESSION_GROUP=" SessionGroup] [SP "ISO_FIELDS=" IsoFields] + CRLF + + StreamStatus = + "NEW" / ; New request to connect + "NEWRESOLVE" / ; New request to resolve an address + "REMAP" / ; Address re-mapped to another + "SENTCONNECT" / ; Sent a connect message along a circuit + "SENTRESOLVE" / ; Sent a resolve message along a circuit + "SUCCEEDED" / ; Received a reply; stream established + "FAILED" / ; Stream failed and not retriable + "CLOSED" / ; Stream closed + "DETACHED" / ; Detached from circuit; still retriable + "CONTROLLER_WAIT" ; Waiting for controller to use ATTACHSTREAM + ; (new in 0.4.5.1-alpha) + "XOFF_SENT" ; XOFF has been sent for this stream + ; (new in 0.4.7.5-alpha) + "XOFF_RECV" ; XOFF has been received for this stream + ; (new in 0.4.7.5-alpha) + "XON_SENT" ; XON has been sent for this stream + ; (new in 0.4.7.5-alpha) + "XON_RECV" ; XON has been received for this stream + ; (new in 0.4.7.5-alpha) + + Target = TargetAddress ":" Port + Port = an integer from 0 to 65535 inclusive + TargetAddress = Address / "(Tor_internal)" + + EscapedUsername = QuotedString + EscapedPassword = QuotedString + + ClientProtocol = + "SOCKS4" / + "SOCKS5" / + "TRANS" / + "NATD" / + "DNS" / + "HTTPCONNECT" / + "UNKNOWN" + + NymEpoch = a nonnegative integer + SessionGroup = an integer + + IsoFields = a comma-separated list of IsoField values + + IsoField = + "CLIENTADDR" / + "CLIENTPORT" / + "DESTADDR" / + "DESTPORT" / + the name of a field that is valid for STREAM events +``` + +The circuit ID designates which circuit this stream is attached to. If +the stream is unattached, the circuit ID "0" is given. The target +indicates the address which the stream is meant to resolve or connect to; +it can be "(Tor_internal)" for a virtual stream created by the Tor program +to talk to itself. + +```text + Reason = "MISC" / "RESOLVEFAILED" / "CONNECTREFUSED" / + "EXITPOLICY" / "DESTROY" / "DONE" / "TIMEOUT" / + "NOROUTE" / "HIBERNATING" / "INTERNAL"/ "RESOURCELIMIT" / + "CONNRESET" / "TORPROTOCOL" / "NOTDIRECTORY" / "END" / + "PRIVATE_ADDR" + + The "REASON" field is provided only for FAILED, CLOSED, and DETACHED + events, and only if extended events are enabled (see 3.19). Clients MUST + accept reasons not listed above. Reasons are as given in tor-spec.txt, + except for: + + END (We received a RELAY_END message from the other side of this + stream.) + PRIVATE_ADDR (The client tried to connect to a private address like + 127.0.0.1 or 10.0.0.1 over Tor.) + [XXXX document more. -NM] + + The "REMOTE_REASON" field is provided only when we receive a RELAY_END + message, and only if extended events are enabled. It contains the actual + reason given by the remote OR for closing the stream. Clients MUST accept + reasons not listed above. Reasons are as listed in tor-spec.txt. + + "REMAP" events include a Source if extended events are enabled: + + Source = "CACHE" / "EXIT" + + Clients MUST accept sources not listed above. "CACHE" is given if + the Tor client decided to remap the address because of a cached + answer, and "EXIT" is given if the remote node we queried gave us + the new address as a response. + + The "SOURCE_ADDR" field is included with NEW and NEWRESOLVE events if + extended events are enabled. It indicates the address and port + that requested the connection, and can be (e.g.) used to look up the + requesting program. + + Purpose = "DIR_FETCH" / "DIR_UPLOAD" / "DNS_REQUEST" / + "USER" / "DIRPORT_TEST" + + The "PURPOSE" field is provided only for NEW and NEWRESOLVE events, and + only if extended events are enabled (see 3.19). Clients MUST accept + purposes not listed above. The purposes above are defined as: + + "DIR_FETCH" -- This stream is generated internally to Tor for + fetching directory information. + "DIR_UPLOAD" -- An internal stream for uploading information to + a directory authority. + "DIRPORT_TEST" -- A stream we're using to test our own directory + port to make sure it's reachable. + "DNS_REQUEST" -- A user-initiated DNS request. + "USER" -- This stream is handling user traffic, OR it's internal + to Tor, but it doesn't match one of the purposes above. + + The "SOCKS_USERNAME" and "SOCKS_PASSWORD" fields indicate the credentials + that were used by a SOCKS client to connect to Tor's SOCKS port and + initiate this stream. (Streams for SOCKS clients connected with different + usernames and/or passwords are isolated on separate circuits if the + IsolateSOCKSAuth flag is active; see Proposal 171.) + + The "CLIENT_PROTOCOL" field indicates the protocol that was used by a client + to initiate this stream. (Streams for clients connected with different + protocols are isolated on separate circuits if the IsolateClientProtocol + flag is active.) Controllers MUST tolerate unrecognized client protocols. + + The "NYM_EPOCH" field indicates the nym epoch that was active when a client + initiated this stream. The epoch increments when the NEWNYM signal is + received. (Streams with different nym epochs are isolated on separate + circuits.) + + The "SESSION_GROUP" field indicates the session group of the listener port + that a client used to initiate this stream. By default, the session group is + different for each listener port, but this can be overridden for a listener + via the "SessionGroup" option in torrc. (Streams with different session + groups are isolated on separate circuits.) + + The "ISO_FIELDS" field indicates the set of STREAM event fields for which + stream isolation is enabled for the listener port that a client used to + initiate this stream. The special values "CLIENTADDR", "CLIENTPORT", + "DESTADDR", and "DESTPORT", if their correspondingly named fields are not + present, refer to the Address and Port components of the "SOURCE_ADDR" and + Target fields. +``` + +<a id="control-spec.txt-4.1.3"></a> + +### OR Connection status changed { #ORCONN } + +The syntax is: + +```text + "650" SP "ORCONN" SP (LongName / Target) SP ORStatus [ SP "REASON=" + Reason ] [ SP "NCIRCS=" NumCircuits ] [ SP "ID=" ConnID ] CRLF + + ORStatus = "NEW" / "LAUNCHED" / "CONNECTED" / "FAILED" / "CLOSED" + + ; In Tor versions 0.1.2.2-alpha through 0.2.2.1-alpha with feature + ; VERBOSE_NAMES turned off and before version 0.1.2.2-alpha, OR + ; Connection is as follows: + "650" SP "ORCONN" SP (ServerID / Target) SP ORStatus [ SP "REASON=" + Reason ] [ SP "NCIRCS=" NumCircuits ] CRLF +``` + +NEW is for incoming connections, and LAUNCHED is for outgoing +connections. CONNECTED means the TLS handshake has finished (in +either direction). FAILED means a connection is being closed that +hasn't finished its handshake, and CLOSED is for connections that +have handshaked. + +A LongName or ServerID is specified unless it's a NEW connection, in +which case we don't know what server it is yet, so we use Address:Port. + +If extended events are enabled (see 3.19), optional reason and +circuit counting information is provided for CLOSED and FAILED +events. + +```text + Reason = "MISC" / "DONE" / "CONNECTREFUSED" / + "IDENTITY" / "CONNECTRESET" / "TIMEOUT" / "NOROUTE" / + "IOERROR" / "RESOURCELIMIT" / "PT_MISSING" + + NumCircuits counts both established and pending circuits. + + The ORStatus values are as follows: + + NEW -- We have received a new incoming OR connection, and are starting + the server-side handshake. + LAUNCHED -- We have launched a new outgoing OR connection, and are + starting the client-side handshake. + CONNECTED -- The OR connection has been connected and the handshake is + done. + FAILED -- Our attempt to open the OR connection failed. + CLOSED -- The OR connection closed in an unremarkable way. + + The Reason values for closed/failed OR connections are: + + DONE -- The OR connection has shut down cleanly. + CONNECTREFUSED -- We got an ECONNREFUSED while connecting to the target + OR. + IDENTITY -- We connected to the OR, but found that its identity was + not what we expected. + CONNECTRESET -- We got an ECONNRESET or similar IO error from the + connection with the OR. + TIMEOUT -- We got an ETIMEOUT or similar IO error from the connection + with the OR, or we're closing the connection for being idle for too + long. + NOROUTE -- We got an ENOTCONN, ENETUNREACH, ENETDOWN, EHOSTUNREACH, or + similar error while connecting to the OR. + IOERROR -- We got some other IO error on our connection to the OR. + RESOURCELIMIT -- We don't have enough operating system resources (file + descriptors, buffers, etc) to connect to the OR. + PT_MISSING -- No pluggable transport was available. + MISC -- The OR connection closed for some other reason. + + [First added ID parameter in 0.2.5.2-alpha] +``` + +<a id="control-spec.txt-4.1.4"></a> + +### Bandwidth used in the last second { #BW } + +The syntax is: + +```text + "650" SP "BW" SP BytesRead SP BytesWritten *(SP Type "=" Num) CRLF + BytesRead = 1*DIGIT + BytesWritten = 1*DIGIT + Type = "DIR" / "OR" / "EXIT" / "APP" / ... + Num = 1*DIGIT +``` + +BytesRead and BytesWritten are the totals. \[In a future Tor version, +we may also include a breakdown of the connection types that used +bandwidth this second (not implemented yet).\] + +<a id="control-spec.txt-4.1.5"></a> + +### Log messages { #LOG } + +The syntax is: + +"650" SP Severity SP ReplyText CRLF + +or + +"650+" Severity CRLF Data 650 SP "OK" CRLF + +Severity = "DEBUG" / "INFO" / "NOTICE" / "WARN"/ "ERR" + +Some low-level logs may be sent from signal handlers, so their destination +logs must be signal-safe. These low-level logs include backtraces, +logging function errors, and errors in code called by logging functions. +Signal-safe logs are never sent as control port log events. + +Control port message trace debug logs are never sent as control port log +events, to avoid modifying control output when debugging. + +<a id="control-spec.txt-4.1.6"></a> + +### New descriptors available { #NEWDESC } + +This event is generated when new router descriptors (not microdescs or +extrainfos or anything else) are received. + +Syntax: + +```text + "650" SP "NEWDESC" 1*(SP LongName) CRLF + ; In Tor versions 0.1.2.2-alpha through 0.2.2.1-alpha with feature + ; VERBOSE_NAMES turned off and before version 0.1.2.2-alpha, it + ; is as follows: + "650" SP "NEWDESC" 1*(SP ServerID) CRLF +``` + +<a id="control-spec.txt-4.1.7"></a> + +### New Address mapping { #ADDRMAP } + +These events are generated when a new address mapping is entered in +Tor's address map cache, or when the answer for a RESOLVE command is +found. Entries can be created by a successful or failed DNS lookup, +a successful or failed connection attempt, a RESOLVE command, +a MAPADDRESS command, the AutomapHostsOnResolve feature, or the +TrackHostExits feature. + +Syntax: + +```text + "650" SP "ADDRMAP" SP Address SP NewAddress SP Expiry + [SP "error=" ErrorCode] [SP "EXPIRES=" UTCExpiry] [SP "CACHED=" Cached] + [SP "STREAMID=" StreamId] CRLF + + NewAddress = Address / "<error>" + Expiry = DQUOTE ISOTime DQUOTE / "NEVER" + + ErrorCode = "yes" / "internal" / "Unable to launch resolve request" + UTCExpiry = DQUOTE IsoTime DQUOTE + + Cached = DQUOTE "YES" DQUOTE / DQUOTE "NO" DQUOTE + StreamId = DQUOTE StreamId DQUOTE +``` + +Error and UTCExpiry are only provided if extended events are enabled. +The values for Error are mostly useless. Future values will be +chosen to match 1\*(ALNUM / "\_"); the "Unable to launch resolve request" +value is a bug in Tor before 0.2.4.7-alpha. + +Expiry is expressed as the local time (rather than UTC). This is a bug, +left in for backward compatibility; new code should look at UTCExpiry +instead. (If Expiry is "NEVER", UTCExpiry is omitted.) + +Cached indicates whether the mapping will be stored until it expires, or if +it is just a notification in response to a RESOLVE command. + +StreamId is the global stream identifier of the stream or circuit from which +the address was resolved. + +<a id="control-spec.txt-4.1.8"></a> + +### Descriptors uploaded to us in our role as authoritative dirserver { #AUTHDIR_NEWDESCS} + +\[NOTE: This feature was removed in Tor 0.3.2.1-alpha.\] + +Tor generates this event when it's a directory authority, and +somebody has just uploaded a server descriptor. + +Syntax: + +```text + "650" "+" "AUTHDIR_NEWDESCS" CRLF Action CRLF Message CRLF + Descriptor CRLF "." CRLF "650" SP "OK" CRLF + Action = "ACCEPTED" / "DROPPED" / "REJECTED" + Message = Text +``` + +The Descriptor field is the text of the server descriptor; the Action +field is "ACCEPTED" if we're accepting the descriptor as the new +best valid descriptor for its router, "REJECTED" if we aren't taking +the descriptor and we're complaining to the uploading relay about +it, and "DROPPED" if we decide to drop the descriptor without +complaining. The Message field is a human-readable string +explaining why we chose the Action. (It doesn't contain newlines.) + +<a id="control-spec.txt-4.1.9"></a> + +### Our descriptor changed { #DESCCHANGED } + +Syntax: + +"650" SP "DESCCHANGED" CRLF + +\[First added in 0.1.2.2-alpha.\] + +<a id="control-spec.txt-4.1.10"></a> + +### Status events { #STATUS } + +Status events (STATUS_GENERAL, STATUS_CLIENT, and STATUS_SERVER) are sent +based on occurrences in the Tor process pertaining to the general state of +the program. Generally, they correspond to log messages of severity Notice +or higher. They differ from log messages in that their format is a +specified interface. + +Syntax: + +```text + "650" SP StatusType SP StatusSeverity SP StatusAction + [SP StatusArguments] CRLF + + StatusType = "STATUS_GENERAL" / "STATUS_CLIENT" / "STATUS_SERVER" + StatusSeverity = "NOTICE" / "WARN" / "ERR" + StatusAction = 1*ALPHA + StatusArguments = StatusArgument *(SP StatusArgument) + StatusArgument = StatusKeyword '=' StatusValue + StatusKeyword = 1*(ALNUM / "_") + StatusValue = 1*(ALNUM / '_') / QuotedString + + StatusAction is a string, and StatusArguments is a series of + keyword=value pairs on the same line. Values may be space-terminated + strings, or quoted strings. + + These events are always produced with EXTENDED_EVENTS and + VERBOSE_NAMES; see the explanations in the USEFEATURE section + for details. + + Controllers MUST tolerate unrecognized actions, MUST tolerate + unrecognized arguments, MUST tolerate missing arguments, and MUST + tolerate arguments that arrive in any order. + + Each event description below is accompanied by a recommendation for + controllers. These recommendations are suggestions only; no controller + is required to implement them. +``` + +Compatibility note: versions of Tor before 0.2.0.22-rc incorrectly +generated "STATUS_SERVER" as "STATUS_SEVER". To be compatible with those +versions, tools should accept both. + +Actions for STATUS_GENERAL events can be as follows: + +```text + CLOCK_JUMPED + "TIME=NUM" + Tor spent enough time without CPU cycles that it has closed all + its circuits and will establish them anew. This typically + happens when a laptop goes to sleep and then wakes up again. It + also happens when the system is swapping so heavily that Tor is + starving. The "time" argument specifies the number of seconds Tor + thinks it was unconscious for (or alternatively, the number of + seconds it went back in time). + + This status event is sent as NOTICE severity normally, but WARN + severity if Tor is acting as a server currently. + + {Recommendation for controller: ignore it, since we don't really + know what the user should do anyway. Hm.} + + DANGEROUS_VERSION + "CURRENT=version" + "REASON=NEW/OBSOLETE/UNRECOMMENDED" + "RECOMMENDED=\"version, version, ...\"" + Tor has found that directory servers don't recommend its version of + the Tor software. RECOMMENDED is a comma-and-space-separated string + of Tor versions that are recommended. REASON is NEW if this version + of Tor is newer than any recommended version, OBSOLETE if + this version of Tor is older than any recommended version, and + UNRECOMMENDED if some recommended versions of Tor are newer and + some are older than this version. (The "OBSOLETE" reason was called + "OLD" from Tor 0.1.2.3-alpha up to and including 0.2.0.12-alpha.) + + {Controllers may want to suggest that the user upgrade OLD or + UNRECOMMENDED versions. NEW versions may be known-insecure, or may + simply be development versions.} + + TOO_MANY_CONNECTIONS + "CURRENT=NUM" + Tor has reached its ulimit -n or whatever the native limit is on file + descriptors or sockets. CURRENT is the number of sockets Tor + currently has open. The user should really do something about + this. The "current" argument shows the number of connections currently + open. + + {Controllers may recommend that the user increase the limit, or + increase it for them. Recommendations should be phrased in an + OS-appropriate way and automated when possible.} + + BUG + "REASON=STRING" + Tor has encountered a situation that its developers never expected, + and the developers would like to learn that it happened. Perhaps + the controller can explain this to the user and encourage her to + file a bug report? + + {Controllers should log bugs, but shouldn't annoy the user in case a + bug appears frequently.} + + CLOCK_SKEW + SKEW="+" / "-" SECONDS + MIN_SKEW="+" / "-" SECONDS. + SOURCE="DIRSERV:" IP ":" Port / + "NETWORKSTATUS:" IP ":" Port / + "OR:" IP ":" Port / + "CONSENSUS" + If "SKEW" is present, it's an estimate of how far we are from the + time declared in the source. (In other words, if we're an hour in + the past, the value is -3600.) "MIN_SKEW" is present, it's a lower + bound. If the source is a DIRSERV, we got the current time from a + connection to a dirserver. If the source is a NETWORKSTATUS, we + decided we're skewed because we got a v2 networkstatus from far in + the future. If the source is OR, the skew comes from a NETINFO + cell from a connection to another relay. If the source is + CONSENSUS, we decided we're skewed because we got a networkstatus + consensus from the future. + + {Tor should send this message to controllers when it thinks the + skew is so high that it will interfere with proper Tor operation. + Controllers shouldn't blindly adjust the clock, since the more + accurate source of skew info (DIRSERV) is currently + unauthenticated.} + + BAD_LIBEVENT + "METHOD=" libevent method + "VERSION=" libevent version + "BADNESS=" "BROKEN" / "BUGGY" / "SLOW" + "RECOVERED=" "NO" / "YES" + Tor knows about bugs in using the configured event method in this + version of libevent. "BROKEN" libevents won't work at all; + "BUGGY" libevents might work okay; "SLOW" libevents will work + fine, but not quickly. If "RECOVERED" is YES, Tor managed to + switch to a more reliable (but probably slower!) libevent method. + + {Controllers may want to warn the user if this event occurs, though + generally it's the fault of whoever built the Tor binary and there's + not much the user can do besides upgrade libevent or upgrade the + binary.} + + DIR_ALL_UNREACHABLE + Tor believes that none of the known directory servers are + reachable -- this is most likely because the local network is + down or otherwise not working, and might help to explain for the + user why Tor appears to be broken. + + {Controllers may want to warn the user if this event occurs; further + action is generally not possible.} + + Actions for STATUS_CLIENT events can be as follows: + + BOOTSTRAP + "PROGRESS=" num + "TAG=" Keyword + "SUMMARY=" String + ["WARNING=" String] + ["REASON=" Keyword] + ["COUNT=" num] + ["RECOMMENDATION=" Keyword] + ["HOST=" QuotedString] + ["HOSTADDR=" QuotedString] + + Tor has made some progress at establishing a connection to the + Tor network, fetching directory information, or making its first + circuit; or it has encountered a problem while bootstrapping. This + status event is especially useful for users with slow connections + or with connectivity problems. + + "Progress" gives a number between 0 and 100 for how far through + the bootstrapping process we are. "Summary" is a string that can + be displayed to the user to describe the *next* task that Tor + will tackle, i.e., the task it is working on after sending the + status event. "Tag" is a string that controllers can use to + recognize bootstrap phases, if they want to do something smarter + than just blindly displaying the summary string; see Section 5 + for the current tags that Tor issues. + + The StatusSeverity describes whether this is a normal bootstrap + phase (severity notice) or an indication of a bootstrapping + problem (severity warn). + + For bootstrap problems, we include the same progress, tag, and + summary values as we would for a normal bootstrap event, but we + also include "warning", "reason", "count", and "recommendation" + key/value combos. The "count" number tells how many bootstrap + problems there have been so far at this phase. The "reason" + string lists one of the reasons allowed in the ORCONN event. The + "warning" argument string with any hints Tor has to offer about + why it's having troubles bootstrapping. + + The "reason" values are long-term-stable controller-facing tags to + identify particular issues in a bootstrapping step. The warning + strings, on the other hand, are human-readable. Controllers + SHOULD NOT rely on the format of any warning string. Currently + the possible values for "recommendation" are either "ignore" or + "warn" -- if ignore, the controller can accumulate the string in + a pile of problems to show the user if the user asks; if warn, + the controller should alert the user that Tor is pretty sure + there's a bootstrapping problem. + + The "host" value is the identity digest (in hex) of the node we're + trying to connect to; the "hostaddr" is an address:port combination, + where 'address' is an ipv4 or ipv6 address. + + Currently Tor uses recommendation=ignore for the first + nine bootstrap problem reports for a given phase, and then + uses recommendation=warn for subsequent problems at that + phase. Hopefully this is a good balance between tolerating + occasional errors and reporting serious problems quickly. + + ENOUGH_DIR_INFO + Tor now knows enough network-status documents and enough server + descriptors that it's going to start trying to build circuits now. + [Newer versions of Tor (0.2.6.2-alpha and later): + If the consensus contains Exits (the typical case), Tor will build + both exit and internal circuits. If not, Tor will only build internal + circuits.] + + {Controllers may want to use this event to decide when to indicate + progress to their users, but should not interrupt the user's browsing + to tell them so.} + + NOT_ENOUGH_DIR_INFO + We discarded expired statuses and server descriptors to fall + below the desired threshold of directory information. We won't + try to build any circuits until ENOUGH_DIR_INFO occurs again. + + {Controllers may want to use this event to decide when to indicate + progress to their users, but should not interrupt the user's browsing + to tell them so.} + + CIRCUIT_ESTABLISHED + Tor is able to establish circuits for client use. This event will + only be sent if we just built a circuit that changed our mind -- + that is, prior to this event we didn't know whether we could + establish circuits. + + {Suggested use: controllers can notify their users that Tor is + ready for use as a client once they see this status event. [Perhaps + controllers should also have a timeout if too much time passes and + this event hasn't arrived, to give tips on how to troubleshoot. + On the other hand, hopefully Tor will send further status events + if it can identify the problem.]} + + CIRCUIT_NOT_ESTABLISHED + "REASON=" "EXTERNAL_ADDRESS" / "DIR_ALL_UNREACHABLE" / "CLOCK_JUMPED" + We are no longer confident that we can build circuits. The "reason" + keyword provides an explanation: which other status event type caused + our lack of confidence. + + {Controllers may want to use this event to decide when to indicate + progress to their users, but should not interrupt the user's browsing + to do so.} + [Note: only REASON=CLOCK_JUMPED is implemented currently.] + + CONSENSUS_ARRIVED + Tor has received and validated a new consensus networkstatus. + (This event can be delayed a little while after the consensus + is received, if Tor needs to fetch certificates.) + + DANGEROUS_PORT + "PORT=" port + "RESULT=" "REJECT" / "WARN" + A stream was initiated to a port that's commonly used for + vulnerable-plaintext protocols. If the Result is "reject", we + refused the connection; whereas if it's "warn", we allowed it. + + {Controllers should warn their users when this occurs, unless they + happen to know that the application using Tor is in fact doing so + correctly (e.g., because it is part of a distributed bundle). They + might also want some sort of interface to let the user configure + their RejectPlaintextPorts and WarnPlaintextPorts config options.} + + DANGEROUS_SOCKS + "PROTOCOL=" "SOCKS4" / "SOCKS5" + "ADDRESS=" IP:port + A connection was made to Tor's SOCKS port using one of the SOCKS + approaches that doesn't support hostnames -- only raw IP addresses. + If the client application got this address from gethostbyname(), + it may be leaking target addresses via DNS. + + {Controllers should warn their users when this occurs, unless they + happen to know that the application using Tor is in fact doing so + correctly (e.g., because it is part of a distributed bundle).} + + SOCKS_UNKNOWN_PROTOCOL + "DATA=string" + A connection was made to Tor's SOCKS port that tried to use it + for something other than the SOCKS protocol. Perhaps the user is + using Tor as an HTTP proxy? The DATA is the first few characters + sent to Tor on the SOCKS port. + + {Controllers may want to warn their users when this occurs: it + indicates a misconfigured application.} + + SOCKS_BAD_HOSTNAME + "HOSTNAME=QuotedString" + Some application gave us a funny-looking hostname. Perhaps + it is broken? In any case it won't work with Tor and the user + should know. + + {Controllers may want to warn their users when this occurs: it + usually indicates a misconfigured application.} + + Actions for STATUS_SERVER can be as follows: + + EXTERNAL_ADDRESS + "ADDRESS=IP" + "HOSTNAME=NAME" + "METHOD=CONFIGURED/CONFIGURED_ORPORT/DIRSERV/RESOLVED/ + INTERFACE/GETHOSTNAME" + Our best idea for our externally visible IP has changed to 'IP'. If + 'HOSTNAME' is present, we got the new IP by resolving 'NAME'. If the + method is 'CONFIGURED', the IP was given verbatim as the Address + configuration option. If the method is 'CONFIGURED_ORPORT', the IP was + given verbatim in the ORPort configuration option. If the method is + 'RESOLVED', we resolved the Address configuration option to get the IP. + If the method is 'GETHOSTNAME', we resolved our hostname to get the IP. + If the method is 'INTERFACE', we got the address of one of our network + interfaces to get the IP. If the method is 'DIRSERV', a directory + server told us a guess for what our IP might be. + + {Controllers may want to record this info and display it to the user.} + + CHECKING_REACHABILITY + "ORADDRESS=IP:port" + "DIRADDRESS=IP:port" + We're going to start testing the reachability of our external OR port + or directory port. + + {This event could affect the controller's idea of server status, but + the controller should not interrupt the user to tell them so.} + + REACHABILITY_SUCCEEDED + "ORADDRESS=IP:port" + "DIRADDRESS=IP:port" + We successfully verified the reachability of our external OR port or + directory port (depending on which of ORADDRESS or DIRADDRESS is + given.) + + {This event could affect the controller's idea of server status, but + the controller should not interrupt the user to tell them so.} + + GOOD_SERVER_DESCRIPTOR + We successfully uploaded our server descriptor to at least one + of the directory authorities, with no complaints. + + {Originally, the goal of this event was to declare "every authority + has accepted the descriptor, so there will be no complaints + about it." But since some authorities might be offline, it's + harder to get certainty than we had thought. As such, this event + is equivalent to ACCEPTED_SERVER_DESCRIPTOR below. Controllers + should just look at ACCEPTED_SERVER_DESCRIPTOR and should ignore + this event for now.} + + SERVER_DESCRIPTOR_STATUS + "STATUS=" "LISTED" / "UNLISTED" + We just got a new networkstatus consensus, and whether we're in + it or not in it has changed. Specifically, status is "listed" + if we're listed in it but previous to this point we didn't know + we were listed in a consensus; and status is "unlisted" if we + thought we should have been listed in it (e.g. we were listed in + the last one), but we're not. + + {Moving from listed to unlisted is not necessarily cause for + alarm. The relay might have failed a few reachability tests, + or the Internet might have had some routing problems. So this + feature is mainly to let relay operators know when their relay + has successfully been listed in the consensus.} + + [Not implemented yet. We should do this in 0.2.2.x. -RD] + + NAMESERVER_STATUS + "NS=addr" + "STATUS=" "UP" / "DOWN" + "ERR=" message + One of our nameservers has changed status. + + {This event could affect the controller's idea of server status, but + the controller should not interrupt the user to tell them so.} + + NAMESERVER_ALL_DOWN + All of our nameservers have gone down. + + {This is a problem; if it happens often without the nameservers + coming up again, the user needs to configure more or better + nameservers.} + + DNS_HIJACKED + Our DNS provider is providing an address when it should be saying + "NOTFOUND"; Tor will treat the address as a synonym for "NOTFOUND". + + {This is an annoyance; controllers may want to tell admins that their + DNS provider is not to be trusted.} + + DNS_USELESS + Our DNS provider is giving a hijacked address instead of well-known + websites; Tor will not try to be an exit node. + + {Controllers could warn the admin if the relay is running as an + exit node: the admin needs to configure a good DNS server. + Alternatively, this happens a lot in some restrictive environments + (hotels, universities, coffeeshops) when the user hasn't registered.} + + BAD_SERVER_DESCRIPTOR + "DIRAUTH=addr:port" + "REASON=string" + A directory authority rejected our descriptor. Possible reasons + include malformed descriptors, incorrect keys, highly skewed clocks, + and so on. + + {Controllers should warn the admin, and try to cope if they can.} + + ACCEPTED_SERVER_DESCRIPTOR + "DIRAUTH=addr:port" + A single directory authority accepted our descriptor. + // actually notice + + {This event could affect the controller's idea of server status, but + the controller should not interrupt the user to tell them so.} + + REACHABILITY_FAILED + "ORADDRESS=IP:port" + "DIRADDRESS=IP:port" + We failed to connect to our external OR port or directory port + successfully. + + {This event could affect the controller's idea of server status. The + controller should warn the admin and suggest reasonable steps to take.} + + HIBERNATION_STATUS + "STATUS=" "AWAKE" | "SOFT" | "HARD" + Our bandwidth based accounting status has changed, and we are now + relaying traffic/rejecting new connections/hibernating. + + {This event could affect the controller's idea of server status. The + controller MAY inform the admin, though presumably the accounting was + explicitly enabled for a reason.} + + [This event was added in tor 0.2.9.0-alpha.] +``` + +<a id="control-spec.txt-4.1.11"></a> + +### Our set of guard nodes has changed { #GUARD } + +Syntax: + +```text + "650" SP "GUARD" SP Type SP Name SP Status ... CRLF + Type = "ENTRY" + Name = ServerSpec + (Identifies the guard affected) + Status = "NEW" | "UP" | "DOWN" | "BAD" | "GOOD" | "DROPPED" +``` + +The ENTRY type indicates a guard used for connections to the Tor +network. + +The Status values are: + +```text + "NEW" -- This node was not previously used as a guard; now we have + picked it as one. + "DROPPED" -- This node is one we previously picked as a guard; we + no longer consider it to be a member of our guard list. + "UP" -- The guard now seems to be reachable. + "DOWN" -- The guard now seems to be unreachable. + "BAD" -- Because of flags set in the consensus and/or values in the + configuration, this node is now unusable as a guard. + "BAD_L2" -- This layer2 guard has expired or got removed from the + consensus. This node is removed from the layer2 guard set. + "GOOD" -- Because of flags set in the consensus and/or values in the + configuration, this node is now usable as a guard. + + Controllers must accept unrecognized types and unrecognized statuses. +``` + +<a id="control-spec.txt-4.1.12"></a> + +### Network status has changed { #NS } + +Syntax: + +"650" "+" "NS" CRLF 1\*NetworkStatus "." CRLF "650" SP "OK" CRLF + +The event is used whenever our local view of a relay status changes. +This happens when we get a new v3 consensus (in which case the entries +we see are a duplicate of what we see in the NEWCONSENSUS event, +below), but it also happens when we decide to mark a relay as up or +down in our local status, for example based on connection attempts. + +\[First added in 0.1.2.3-alpha\] + +<a id="control-spec.txt-4.1.13"></a> + +### Bandwidth used on an application stream { #STREAM_BW } + +The syntax is: + +```text + "650" SP "STREAM_BW" SP StreamID SP BytesWritten SP BytesRead SP + Time CRLF + BytesWritten = 1*DIGIT + BytesRead = 1*DIGIT + Time = ISOTime2Frac +``` + +BytesWritten and BytesRead are the number of bytes written and read +by the application since the last STREAM_BW event on this stream. + +Note that from Tor's perspective, *reading* a byte on a stream means +that the application *wrote* the byte. That's why the order of "written" +vs "read" is opposite for stream_bw events compared to bw events. + +The Time field is provided only in versions 0.3.2.1-alpha and later. It +records when Tor created the bandwidth event. + +These events are generated about once per second per stream; no events +are generated for streams that have not written or read. These events +apply only to streams entering Tor (such as on a SOCKSPort, TransPort, +or so on). They are not generated for exiting streams. + +<a id="control-spec.txt-4.1.14"></a> + +### Per-country client stats { #CLIENTS_SEEN } + +The syntax is: + +```text + "650" SP "CLIENTS_SEEN" SP TimeStarted SP CountrySummary SP + IPVersions CRLF +``` + +We just generated a new summary of which countries we've seen clients +from recently. The controller could display this for the user, e.g. +in their "relay" configuration window, to give them a sense that they +are actually being useful. + +Currently only bridge relays will receive this event, but once we figure +out how to sufficiently aggregate and sanitize the client counts on +main relays, we might start sending these events in other cases too. + +TimeStarted is a quoted string indicating when the reported summary +counts from (in UTCS). + +The CountrySummary keyword has as its argument a comma-separated, +possibly empty set of "countrycode=count" pairs. For example (without +linebreak), +650-CLIENTS_SEEN TimeStarted="2008-12-25 23:50:43" +CountrySummary=us=16,de=8,uk=8 + +The IPVersions keyword has as its argument a comma-separated set of +"protocol-family=count" pairs. For example, +IPVersions=v4=16,v6=40 + +Note that these values are rounded, not exact. The rounding +algorithm is specified in the description of "geoip-client-origins" +in dir-spec.txt. + +<a id="control-spec.txt-4.1.15"></a> + +### New consensus networkstatus has arrived { #NEWCONSENSUS } + +The syntax is: + +```text + "650" "+" "NEWCONSENSUS" CRLF 1*NetworkStatus "." CRLF "650" SP + "OK" CRLF +``` + +A new consensus networkstatus has arrived. We include NS-style lines for +every relay in the consensus. NEWCONSENSUS is a separate event from the +NS event, because the list here represents every usable relay: so any +relay *not* mentioned in this list is implicitly no longer recommended. + +\[First added in 0.2.1.13-alpha\] + +<a id="control-spec.txt-4.1.16"></a> + +### New circuit buildtime has been set { #BUILDTIMEOUT_SET } + +The syntax is: + +```text + "650" SP "BUILDTIMEOUT_SET" SP Type SP "TOTAL_TIMES=" Total SP + "TIMEOUT_MS=" Timeout SP "XM=" Xm SP "ALPHA=" Alpha SP + "CUTOFF_QUANTILE=" Quantile SP "TIMEOUT_RATE=" TimeoutRate SP + "CLOSE_MS=" CloseTimeout SP "CLOSE_RATE=" CloseRate + CRLF + Type = "COMPUTED" / "RESET" / "SUSPENDED" / "DISCARD" / "RESUME" + Total = Integer count of timeouts stored + Timeout = Integer timeout in milliseconds + Xm = Estimated integer Pareto parameter Xm in milliseconds + Alpha = Estimated floating point Paredo parameter alpha + Quantile = Floating point CDF quantile cutoff point for this timeout + TimeoutRate = Floating point ratio of circuits that timeout + CloseTimeout = How long to keep measurement circs in milliseconds + CloseRate = Floating point ratio of measurement circuits that are closed +``` + +A new circuit build timeout time has been set. If Type is "COMPUTED", +Tor has computed the value based on historical data. If Type is "RESET", +initialization or drastic network changes have caused Tor to reset +the timeout back to the default, to relearn again. If Type is +"SUSPENDED", Tor has detected a loss of network connectivity and has +temporarily changed the timeout value to the default until the network +recovers. If type is "DISCARD", Tor has decided to discard timeout +values that likely happened while the network was down. If type is +"RESUME", Tor has decided to resume timeout calculation. + +The Total value is the count of circuit build times Tor used in +computing this value. It is capped internally at the maximum number +of build times Tor stores (NCIRCUITS_TO_OBSERVE). + +The Timeout itself is provided in milliseconds. Internally, Tor rounds +this value to the nearest second before using it. + +\[First added in 0.2.2.7-alpha\] + +<a id="control-spec.txt-4.1.17"></a> + +### Signal received { #SIGNAL } + +The syntax is: + +"650" SP "SIGNAL" SP Signal CRLF + +Signal = "RELOAD" / "DUMP" / "DEBUG" / "NEWNYM" / "CLEARDNSCACHE" + +A signal has been received and actions taken by Tor. The meaning of each +signal, and the mapping to Unix signals, is as defined in section 3.7. +Future versions of Tor MAY generate signals other than those listed here; +controllers MUST be able to accept them. + +If Tor chose to ignore a signal (such as NEWNYM), this event will not be +sent. Note that some options (like ReloadTorrcOnSIGHUP) may affect the +semantics of the signals here. + +Note that the HALT (SIGTERM) and SHUTDOWN (SIGINT) signals do not currently +generate any event. + +\[First added in 0.2.3.1-alpha\] + +<a id="control-spec.txt-4.1.18"></a> + +### Configuration changed { #CONF_CHANGED } + +The syntax is: + +StartReplyLine \*(MidReplyLine) EndReplyLine + +```text + StartReplyLine = "650-CONF_CHANGED" CRLF + MidReplyLine = "650-" KEYWORD ["=" VALUE] CRLF + EndReplyLine = "650 OK" +``` + +Tor configuration options have changed (such as via a SETCONF or RELOAD +signal). KEYWORD and VALUE specify the configuration option that was changed. +Undefined configuration options contain only the KEYWORD. + +<a id="control-spec.txt-4.1.19"></a> + +### Circuit status changed slightly { #CIRC_MINOR } + +The syntax is: + +```text + "650" SP "CIRC_MINOR" SP CircuitID SP CircEvent [SP Path] + [SP "BUILD_FLAGS=" BuildFlags] [SP "PURPOSE=" Purpose] + [SP "HS_STATE=" HSState] [SP "REND_QUERY=" HSAddress] + [SP "TIME_CREATED=" TimeCreated] + [SP "OLD_PURPOSE=" Purpose [SP "OLD_HS_STATE=" HSState]] CRLF + + CircEvent = + "PURPOSE_CHANGED" / ; circuit purpose or HS-related state changed + "CANNIBALIZED" ; circuit cannibalized + + Clients MUST accept circuit events not listed above. +``` + +The "OLD_PURPOSE" field is provided for both PURPOSE_CHANGED and +CANNIBALIZED events. The "OLD_HS_STATE" field is provided whenever +the "OLD_PURPOSE" field is provided and is a hidden-service-related +purpose. + +Other fields are as specified in section 4.1.1 above. + +\[First added in 0.2.3.11-alpha\] + +<a id="control-spec.txt-4.1.20"></a> + +### Pluggable transport launched { #TRANSPORT_LAUNCHED } + +The syntax is: + +```text + "650" SP "TRANSPORT_LAUNCHED" SP Type SP Name SP TransportAddress SP Port + Type = "server" | "client" + Name = The name of the pluggable transport + TransportAddress = An IPv4 or IPv6 address on which the pluggable + transport is listening for connections + Port = The TCP port on which it is listening for connections. + + A pluggable transport called 'Name' of type 'Type' was launched + successfully and is now listening for connections on 'Address':'Port'. +``` + +<a id="control-spec.txt-4.1.21"></a> + +### Bandwidth used on an OR or DIR or EXIT connection { #CONN_BW } + +The syntax is: + +```text + "650" SP "CONN_BW" SP "ID=" ConnID SP "TYPE=" ConnType + SP "READ=" BytesRead SP "WRITTEN=" BytesWritten CRLF + + ConnType = "OR" / ; Carrying traffic within the tor network. This can + either be our own (client) traffic or traffic we're + relaying within the network. + "DIR" / ; Fetching tor descriptor data, or transmitting + descriptors we're mirroring. + "EXIT" ; Carrying traffic between the tor network and an + external destination. + + BytesRead = 1*DIGIT + BytesWritten = 1*DIGIT + + Controllers MUST tolerate unrecognized connection types. +``` + +BytesWritten and BytesRead are the number of bytes written and read +by Tor since the last CONN_BW event on this connection. + +These events are generated about once per second per connection; no +events are generated for connections that have not read or written. +These events are only generated if TestingTorNetwork is set. + +\[First added in 0.2.5.2-alpha\] + +<a id="control-spec.txt-4.1.22"></a> + +### Bandwidth used by all streams attached to a circuit { #CIRC_BW } + +The syntax is: + +```text + "650" SP "CIRC_BW" SP "ID=" CircuitID SP "READ=" BytesRead SP + "WRITTEN=" BytesWritten SP "TIME=" Time SP + "DELIVERED_READ=" DeliveredBytesRead SP + "OVERHEAD_READ=" OverheadBytesRead SP + "DELIVERED_WRITTEN=" DeliveredBytesWritten SP + "OVERHEAD_WRITTEN=" OverheadBytesWritten SP + "SS=" SlowStartState SP + "CWND=" CWNDCells SP + "RTT=" RTTMilliseconds SP + "MIN_RTT=" RTTMilliseconds CRLF + BytesRead = 1*DIGIT + BytesWritten = 1*DIGIT + OverheadBytesRead = 1*DIGIT + OverheadBytesWritten = 1*DIGIT + DeliveredBytesRead = 1*DIGIT + DeliveredBytesWritten = 1*DIGIT + SlowStartState = 0 or 1 + CWNDCells = 1*DIGIT + RTTMilliseconds= 1*DIGIT + Time = ISOTime2Frac +``` + +BytesRead and BytesWritten are the number of bytes read and written +on this circuit since the last CIRC_BW event. These bytes have not +necessarily been validated by Tor, and can include invalid cells, +dropped cells, and ignored cells (such as padding cells). These +values include the relay headers, but not circuit headers. + +Circuit data that has been validated and processed by Tor is further +broken down into two categories: delivered payloads and overhead. +DeliveredBytesRead and DeliveredBytesWritten are the total relay cell +payloads transmitted since the last CIRC_BW event, not counting relay +cell headers or circuit headers. OverheadBytesRead and +OverheadBytesWritten are the extra unused bytes at the end of each +cell in order for it to be the fixed CELL_LEN bytes long. + +The sum of DeliveredBytesRead and OverheadBytesRead MUST be less than +BytesRead, and the same is true for their written counterparts. This +sum represents the total relay cell bytes on the circuit that +have been validated by Tor, not counting relay headers and cell headers. +Subtracting this sum (plus relay cell headers) from the BytesRead +(or BytesWritten) value gives the byte count that Tor has decided to +reject due to protocol errors, or has otherwise decided to ignore. + +The Time field is provided only in versions 0.3.2.1-alpha and later. It +records when Tor created the bandwidth event. + +The SS, CWND, RTT, and MIN_RTT fields are present only if the circuit +has negotiated congestion control to an onion service or Exit hop (any +intermediate leaky pipe congestion control hops are not examined here). +SS provides an indication if the circuit is in slow start (1), or not (0). +CWND is the size of the congestion window in terms of number of cells. +RTT is the N_EWMA smoothed current RTT value, and MIN_RTT is the minimum +RTT value of the circuit. The SS and CWND fields apply only to the +upstream direction of the circuit. The slow start state and CWND values +of the other endpoint may be different. + +These events are generated about once per second per circuit; no events +are generated for circuits that had no attached stream writing or +reading. + +\[First added in 0.2.5.2-alpha\] + +\[DELIVERED_READ, OVERHEAD_READ, DELIVERED_WRITTEN, and OVERHEAD_WRITTEN +were added in Tor 0.3.4.0-alpha\] + +\[SS, CWND, RTT, and MIN_RTT were added in Tor 0.4.7.5-alpha\] + +<a id="control-spec.txt-4.1.23"></a> + +### Per-circuit cell stats { #CELL_STATS } + +The syntax is: + +```text + "650" SP "CELL_STATS" + [ SP "ID=" CircuitID ] + [ SP "InboundQueue=" QueueID SP "InboundConn=" ConnID ] + [ SP "InboundAdded=" CellsByType ] + [ SP "InboundRemoved=" CellsByType SP + "InboundTime=" MsecByType ] + [ SP "OutboundQueue=" QueueID SP "OutboundConn=" ConnID ] + [ SP "OutboundAdded=" CellsByType ] + [ SP "OutboundRemoved=" CellsByType SP + "OutboundTime=" MsecByType ] CRLF + CellsByType, MsecByType = CellType ":" 1*DIGIT + 0*( "," CellType ":" 1*DIGIT ) + CellType = 1*( "a" - "z" / "0" - "9" / "_" ) + + Examples are: + + 650 CELL_STATS ID=14 OutboundQueue=19403 OutboundConn=15 + OutboundAdded=create_fast:1,relay_early:2 + OutboundRemoved=create_fast:1,relay_early:2 + OutboundTime=create_fast:0,relay_early:0 + 650 CELL_STATS InboundQueue=19403 InboundConn=32 + InboundAdded=relay:1,created_fast:1 + InboundRemoved=relay:1,created_fast:1 + InboundTime=relay:0,created_fast:0 + OutboundQueue=6710 OutboundConn=18 + OutboundAdded=create:1,relay_early:1 + OutboundRemoved=create:1,relay_early:1 + OutboundTime=create:0,relay_early:0 +``` + +ID is the locally unique circuit identifier that is only included if the +circuit originates at this node. + +Inbound and outbound refer to the direction of relay cell flow through the +circuit which is either to origin (inbound) or from origin (outbound). + +InboundQueue and OutboundQueue are identifiers of the inbound and +outbound circuit queues of this circuit. These identifiers are only +unique per OR connection. OutboundQueue is chosen by this node and +matches InboundQueue of the next node in the circuit. + +InboundConn and OutboundConn are locally unique IDs of inbound and +outbound OR connection. OutboundConn does not necessarily match +InboundConn of the next node in the circuit. + +InboundQueue and InboundConn are not present if the circuit originates +at this node. OutboundQueue and OutboundConn are not present if the +circuit (currently) ends at this node. + +InboundAdded and OutboundAdded are total number of cells by cell type +added to inbound and outbound queues. Only present if at least one cell +was added to a queue. + +InboundRemoved and OutboundRemoved are total number of cells by +cell type processed from inbound and outbound queues. InboundTime and +OutboundTime are total waiting times in milliseconds of all processed +cells by cell type. Only present if at least one cell was removed from +a queue. + +These events are generated about once per second per circuit; no +events are generated for circuits that have not added or processed any +cell. These events are only generated if TestingTorNetwork is set. + +\[First added in 0.2.5.2-alpha\] + +<a id="control-spec.txt-4.1.24"></a> + +### Token buckets refilled { #TB_EMPTY } + +The syntax is: + +```text + "650" SP "TB_EMPTY" SP BucketName [ SP "ID=" ConnID ] SP + "READ=" ReadBucketEmpty SP "WRITTEN=" WriteBucketEmpty SP + "LAST=" LastRefill CRLF + + BucketName = "GLOBAL" / "RELAY" / "ORCONN" + ReadBucketEmpty = 1*DIGIT + WriteBucketEmpty = 1*DIGIT + LastRefill = 1*DIGIT + + Examples are: + + 650 TB_EMPTY ORCONN ID=16 READ=0 WRITTEN=0 LAST=100 + 650 TB_EMPTY GLOBAL READ=93 WRITTEN=93 LAST=100 + 650 TB_EMPTY RELAY READ=93 WRITTEN=93 LAST=100 +``` + +This event is generated when refilling a previously empty token +bucket. BucketNames "GLOBAL" and "RELAY" keywords are used for the +global or relay token buckets, BucketName "ORCONN" is used for the +token buckets of an OR connection. Controllers MUST tolerate +unrecognized bucket names. + +ConnID is only included if the BucketName is "ORCONN". + +If both global and relay buckets and/or the buckets of one or more OR +connections run out of tokens at the same time, multiple separate +events are generated. + +ReadBucketEmpty (WriteBucketEmpty) is the time in millis that the read +(write) bucket was empty since the last refill. LastRefill is the +time in millis since the last refill. + +If a bucket went negative and if refilling tokens didn't make it go +positive again, there will be multiple consecutive TB_EMPTY events for +each refill interval during which the bucket contained zero tokens or +less. In such a case, ReadBucketEmpty or WriteBucketEmpty are capped +at LastRefill in order not to report empty times more than once. + +These events are only generated if TestingTorNetwork is set. + +\[First added in 0.2.5.2-alpha\] + +<a id="control-spec.txt-4.1.25"></a> + +### HiddenService descriptors { #HS_DESC } + +The syntax is: + +```text + "650" SP "HS_DESC" SP Action SP HSAddress SP AuthType SP HsDir + [SP DescriptorID] [SP "REASON=" Reason] [SP "REPLICA=" Replica] + [SP "HSDIR_INDEX=" HSDirIndex] + + Action = "REQUESTED" / "UPLOAD" / "RECEIVED" / "UPLOADED" / "IGNORE" / + "FAILED" / "CREATED" + HSAddress = 16*Base32Character / 56*Base32Character / "UNKNOWN" + AuthType = "NO_AUTH" / "BASIC_AUTH" / "STEALTH_AUTH" / "UNKNOWN" + HsDir = LongName / Fingerprint / "UNKNOWN" + DescriptorID = 32*Base32Character / 43*Base64Character + Reason = "BAD_DESC" / "QUERY_REJECTED" / "UPLOAD_REJECTED" / "NOT_FOUND" / + "UNEXPECTED" / "QUERY_NO_HSDIR" / "QUERY_RATE_LIMITED" + Replica = 1*DIGIT + HSDirIndex = 64*HEXDIG + + These events will be triggered when required HiddenService descriptor is + not found in the cache and a fetch or upload with the network is performed. + + If the fetch was triggered with only a DescriptorID (using the HSFETCH + command for instance), the HSAddress only appears in the Action=RECEIVED + since there is no way to know the HSAddress from the DescriptorID thus + the value will be "UNKNOWN". + + If we already had the v0 descriptor, the newly fetched v2 descriptor + will be ignored and a "HS_DESC" event with "IGNORE" action will be + generated. + + For HsDir, LongName is always preferred. If HsDir cannot be found in node + list at the time event is sent, Fingerprint will be used instead. + + If Action is "FAILED", Tor SHOULD send Reason field as well. Possible + values of Reason are: + - "BAD_DESC" - descriptor was retrieved, but found to be unparsable. + - "QUERY_REJECTED" - query was rejected by HS directory. + - "UPLOAD_REJECTED" - descriptor was rejected by HS directory. + - "NOT_FOUND" - HS descriptor with given identifier was not found. + - "UNEXPECTED" - nature of failure is unknown. + - "QUERY_NO_HSDIR" - No suitable HSDir were found for the query. + - "QUERY_RATE_LIMITED" - query for this service is rate-limited + + For "QUERY_NO_HSDIR" or "QUERY_RATE_LIMITED", the HsDir will be set to + "UNKNOWN" which was introduced in tor 0.3.1.0-alpha and 0.4.1.0-alpha + respectively. + + If Action is "CREATED", Tor SHOULD send Replica field as well. The Replica + field contains the replica number of the generated descriptor. The Replica + number is specified in rend-spec.txt section 1.3 and determines the + descriptor ID of the descriptor. + + For hidden service v3, the following applies: + + The "HSDIR_INDEX=" is an optional field that is only for version 3 + which contains the computed index of the HsDir the descriptor was + uploaded to or fetched from. + + The "DescriptorID" key is the descriptor blinded key used for the index + value at the "HsDir". + + The "REPLICA=" field is not used for the "CREATED" event because v3 + doesn't use the replica number in the descriptor ID computation. + + Because client authentication is not yet implemented, the "AuthType" + field is always "NO_AUTH". + + [HS v3 support added 0.3.3.1-alpha] +``` + +<a id="control-spec.txt-4.1.26"></a> + +### HiddenService descriptors content { #HS_DESC_CONTENT } + +The syntax is: + +```text + "650" "+" "HS_DESC_CONTENT" SP HSAddress SP DescId SP HsDir CRLF + Descriptor CRLF "." CRLF "650" SP "OK" CRLF + + HSAddress = 16*Base32Character / 56*Base32Character / "UNKNOWN" + DescId = 32*Base32Character / 32*Base64Character + HsDir = LongName / "UNKNOWN" + Descriptor = The text of the descriptor formatted as specified in + rend-spec.txt section 1.3 (v2) or rend-spec-v3.txt + section 2.4 (v3) or empty string on failure. +``` + +This event is triggered when a successfully fetched HS descriptor is +received. The text of that descriptor is then replied. If the HS_DESC +event is enabled, it is replied just after the RECEIVED action. + +If a fetch fails, the Descriptor is an empty string and HSAddress is set +to "UNKNOWN". The HS_DESC event should be used to get more information on +the failed request. + +If the fetch fails for the QUERY_NO_HSDIR or QUERY_RATE_LIMITED reason from +the HS_DESC event, the HsDir is set to "UNKNOWN". This was introduced in +0.3.1.0-alpha and 0.4.1.0-alpha respectively. + +It's expected to receive a reply relatively fast as in it's the time it +takes to fetch something over the Tor network. This can be between a +couple of seconds up to 60 seconds (not a hard limit). But, in any cases, +this event will reply either the descriptor's content or an empty one. + +\[HS_DESC_CONTENT was added in Tor 0.2.7.1-alpha\] +\[HS v3 support added 0.3.3.1-alpha\] + +<a id="control-spec.txt-4.1.27"></a> + +### Network liveness has changed { #NETWORK_LIVENESS } + +Syntax: + +```text + "650" SP "NETWORK_LIVENESS" SP Status CRLF + Status = "UP" / ; The network now seems to be reachable. + "DOWN" / ; The network now seems to be unreachable. + + Controllers MUST tolerate unrecognized status types. + + [NETWORK_LIVENESS was added in Tor 0.2.7.2-alpha] +``` + +<a id="control-spec.txt-4.1.28"></a> + +### Pluggable Transport Logs { #PT_LOG } + +Syntax: + +"650" SP "PT_LOG" SP PT=Program SP Message + +```text + Program = The program path as defined in the *TransportPlugin + configuration option. Tor accepts relative and full path. + Message = The log message that the PT sends back to the tor parent + process minus the "LOG" string prefix. Formatted as + specified in pt-spec.txt section "3.3.4. Pluggable + Transport Log Message". + + This event is triggered when tor receives a log message from the PT. + + Example: + + PT (obfs4): LOG SEVERITY=debug MESSAGE="Connected to bridge A" + + the resulting control port event would be: + + Tor: 650 PT_LOG PT=/usr/bin/obs4proxy SEVERITY=debug MESSAGE="Connected to bridge A" + + [PT_LOG was added in Tor 0.4.0.1-alpha] +``` + +<a id="control-spec.txt-4.1.29"></a> + +### Pluggable Transport Status { #PT_STATUS } + +Syntax: + +"650" SP "PT_STATUS" SP PT=Program SP TRANSPORT=Transport SP Message + +```text + Program = The program path as defined in the *TransportPlugin + configuration option. Tor accepts relative and full path. + Transport = This value indicates a hint on what the PT is such as the + name or the protocol used for instance. + Message = The status message that the PT sends back to the tor parent + process minus the "STATUS" string prefix. Formatted as + specified in pt-spec.txt section "3.3.5 Pluggable + Transport Status Message". + + This event is triggered when tor receives a log message from the PT. + + Example: + + PT (obfs4): STATUS TRANSPORT=obfs4 CONNECT=Success + + the resulting control port event would be: + + Tor: 650 PT_STATUS PT=/usr/bin/obs4proxy TRANSPORT=obfs4 CONNECT=Success + + [PT_STATUS was added in Tor 0.4.0.1-alpha] +``` diff --git a/spec/dir-list-spec.md b/spec/dir-list-spec.md new file mode 100644 index 0000000..6bf9466 --- /dev/null +++ b/spec/dir-list-spec.md @@ -0,0 +1,568 @@ +# Tor Directory List Format + +```text + + Tim Wilson-Brown (teor) +``` + +<a id="dir-list-spec.txt-1"></a> + +## Scope and Preliminaries + +This document describes the format of Tor's directory lists, which are +compiled and hard-coded into the tor binary. There is currently one +list: the fallback directory mirrors. This list is also parsed by other +libraries, like stem and metrics-lib. Alternate Tor implementations can +use this list to bootstrap from the latest public Tor directory +information. + +The FallbackDir feature was introduced by proposal 210, and was first +supported by Tor in Tor version 0.2.4.7-alpha. The first hard-coded +list was shipped in 0.2.8.1-alpha. + +The hard-coded fallback directory list is located in the tor source +repository at: + +`src/app/config/fallback_dirs.inc` + +In Tor 0.3.4 and earlier, the list is located at: + +`src/or/fallback_dirs.inc` + +This document describes version 2.0.0 and later of the directory list +format. + +Legacy, semi-structured versions of the fallback list were released with +Tor 0.2.8.1-alpha through Tor 0.3.1.9. We call this format version 1. +Stem and Relay Search have parsers for this legacy format. + +<a id="dir-list-spec.txt-1.1"></a> + +### Format Overview + +A directory list is a C code fragment containing an array of C string +constants. Each double-quoted C string constant is a valid torrc +FallbackDir entry. Each entry contains various data fields. + +Directory lists do not include the C array's declaration, or the array's +terminating NULL. Entries in directory lists do not include the +FallbackDir torrc option. These are handled by the including C code. + +Directory lists also include C-style comments and whitespace. The +presence of whitespace may be significant, but the amount of whitespace +is never significant. The type of whitespace is not significant to the +C compiler or Tor C string parser. However, other parsers MAY rely on +the distinction between newlines and spaces. (And that the only +whitespace characters in the list are newlines and spaces.) + +The directory entry C string constants are split over multiple lines for +readability. Structured C-style comments are used to provide additional +data fields. This information is not used by Tor, but may be of interest +to other libraries. + +The order of directory entries and data fields is not significant, +except where noted below. + +<a id="dir-list-spec.txt-1.2"></a> + +### Acknowledgements + +The original fallback directory script and format was created by +weasel. The current script uses code written by gsathya & karsten. + +This specification was revised after feedback from Damian Johnson ("atagar") +and Iain R. Learmonth ("irl"). + + +<a id="dir-list-spec.txt-1.3"></a> + +### Format Versions + +The directory list format uses semantic versioning: <https://semver.org> + +```text + In particular: + * major versions are used for incompatible changes, like + removing non-optional fields + * minor versions are used for compatible changes, like adding + fields + * patch versions are for bug fixes, like fixing an + incorrectly-formatted Summary item + + 1.0.0 - The legacy fallback directory list format + + 2.0.0 - Adds name and extrainfo structured comments, and section separator + comments to make the list easier to parses. Also adds a source list + comment to the header. + + 3.0.0 - Modifies the format of the source list comment. +``` + +<a id="dir-list-spec.txt-1.4"></a> + +### Future Plans + +Tor also has an auth_dirs.inc file, but it is not yet in this format. +Tor uses slightly different formats for authorities and fallback +directory mirrors, so we will need to make some changes to tor so that +it parses this format. (We will also need to add authority-specific +information to this format.) See #24818 for details. + +We want to add a torrc option so operators can opt-in their relays as +fallback directory mirrors. This gives us a signed opt-in confirmation. +(We can also continue to accept whitelist entries, and do other checks.) +We need to write a short proposal, and make some changes to tor and the +fallback update script. See #24839 for details. + +<a id="dir-list-spec.txt-2"></a> + +## Format Details + +Directory lists contain the following sections: + + - List Header (exactly once) + - List Generation (exactly once, may be empty) + - Directory Entry (zero or more times) + +Each section (or entry) ends with a separator. + +<a id="dir-list-spec.txt-2.1"></a> + +### Nonterminals + +The following nonterminals are defined in the Onionoo details document +specification: + + * dir_address + * fingerprint + * nickname + +See <https://metrics.torproject.org/onionoo.html#details> + +The following nonterminals are defined in the "Tor directory protocol" +specification in dir-spec.txt: + +```text + Keyword + ArgumentChar + NL (newline) + SP (space) + bool (must not be confused with Onionoo's JSON "boolean") +``` + +We derive the following nonterminals from Onionoo and dir-spec.txt: + +``` + ipv4_or_port ::= port from an IPv4 or_addresses item + + The ipv4_or_port is the port part of an IPv4 address from the + Onionoo or_addresses list. + + ipv6_or_address ::= an IPv6 or_addresses item + + The ipv6_or_address is an IPv6 address and port from the Onionoo + or_addresses list. The address MAY be in the canonical RFC 5952 + IPv6 address format. + + A key-value pair: + + value ::= Zero or more ArgumentChar, excluding the following strings: + * a double quotation mark (DQUOTE), and + * the C comment terminators ("/*" and "*/"). + + Note that the C++ comment ("//") and equals sign ("=") are + not excluded, because they are reserved for future use in + base64 values. + + key_value ::= Keyword "=" value + + We also define these additional nonterminals: + + number ::= An optional negative sign ("-"), followed by one or more + numeric characters ([0-9]), with an optional decimal part + (".", followed by one or more numeric characters). + + separator ::= "/*" SP+ "=====" SP+ "*/" +``` + +<a id="dir-list-spec.txt-2.2"></a> + +### List Header + +The list header consists of a number of key-value pairs, embedded in +C-style comments. + +<a id="dir-list-spec.txt-2.2.1"></a> + +#### List Header Format + +```text + + "/*" SP+ "type=" Keyword SP+ "*/" SP\* NL + + [At start, exactly once.] + + The type of directory entries in the list. Parsers SHOULD exit with + an error if this is not the first line of the list, or if the value + is anything other than "fallback". + + "/*" SP+ "version=" version_number SP+ "*/" SP* NL + + [In second position, exactly once.] + + The version of the directory list format. + + version_number is a semantic version, see the "Format Versions" + section for details. + + Version 1.0.0 represents the undocumented, legacy fallback list + format(s). Version 2.0.0 and later are documented by this + specification. + + "/*" SP+ "timestamp=" number SP+ "*/" SP* NL + + [Exactly once.] + + A positive integer that indicates when this directory list was + generated. This timestamp is guaranteed to increase for every + version 2.0.0 and later directory list. + + The current timestamp format is YYYYMMDDHHMMSS, as an integer. + + "/*" SP+ "source=" Keyword ("," Keyword)* SP+ "*/" SP* NL + + [Zero or one time.] + + A list of the sources of the directory entries in the list. + + As of version 3.0.0, the possible sources are: + * "offer-list" - the fallback_offer_list file in the fallback-scripts + repository. + * "descriptor" - one or more signed descriptors, each containing an + "offer-fallback-dir" line. This feature will be + implemented in ticket #24839. + * "fallback" - a fallback_dirs.inc file from a tor repository. + Used in check_existing mode. + + Before #24839 is implemented, the default is "offer-list". During the + transition to signed offers, it will be "descriptor,offer-list". + Afterwards, it will be "descriptor". + + In version 2.0.0, only one source name was allowed after "source=", + and the deprecated "whitelist" source name was used instead of + "offer-list". + + This line was added in version 2.0.0 of this specification. The format + of this line was modified in version 3.0.0 of this specification. + + "/*" SP+ key_value SP+ "*/" SP* NL + + [Zero or more times.] + + Future releases may include additional header fields. Parsers MUST NOT + rely on the order of these additional fields. Additional header fields + will be accompanied by a minor version increment. + + separator SP* NL + + The list header ends with the section separator. +``` + +<a id="dir-list-spec.txt-2.3"></a> + +### List Generation + +The list generation information consists of human-readable prose +describing the content and origin of this directory list. It is contained +in zero or more C-style comments, and may contain multi-line comments and +uncommented C code. + +In particular, this section may contain C-style comments that contain +an equals ("=") character. It may also be entirely empty. + +Future releases may arbitrarily change the content of this section. +Parsers MUST NOT rely on a version increment when the format changes. + +<a id="dir-list-spec.txt-2.3.1"></a> + +#### List Generation Format + +In general, parsers MUST NOT rely on the format of this section. + +Parsers MAY rely on the following details: + +The list generation section MUST NOT be a valid directory entry. + +The list generation summary MUST end with a section separator: + +separator SP\* NL + +There MUST NOT be any section separators in the list generation +section, other than the terminating section separator. + +<a id="dir-list-spec.txt-2.4"></a> + +### Directory Entry + +A directory entry consists of a C string constant, and one or more +C-style comments. The C string constant is a valid argument to the +DirAuthority or FallbackDir torrc option. The section also contains +additional key-value fields in C-style comments. + +The list of fallback entries does not include the directory +authorities: they are in a separate list. (The Tor implementation combines +these lists after parsing them, and applies the DirAuthorityFallbackRate +to their weights.) + +<a id="dir-list-spec.txt-2.4.1"></a> + +#### Directory Entry Format + +```text + If a directory entry does not conform to this format, the entry SHOULD + be ignored by parsers. + + DQUOTE dir_address SP+ "orport=" ipv4_or_port SP+ + "id=" fingerprint DQUOTE SP* NL + + [At start, exactly once, on a single line.] + + This line consists of the following fields: + + dir_address + + An IPv4 address and DirPort for this directory, as defined by + Onionoo. In this format version, all IPv4 addresses and DirPorts + are guaranteed to be non-zero. (For IPv4 addresses, this means + that they are not equal to "0.0.0.0".) + + ipv4_or_port + + An IPv4 ORPort for this directory, derived from Onionoo. In this + format version, all IPv4 ORPorts are guaranteed to be non-zero. + + fingerprint + + The relay fingerprint of this directory, as defined by Onionoo. + All relay fingerprints are guaranteed to have one or more non-zero + digits. + + Note: + + Each double-quoted C string line that occurs after the first line, + starts with space inside the quotes. This is a requirement of the + Tor implementation. + + DQUOTE SP+ "ipv6=" ipv6_or_address DQUOTE SP* NL + + [Zero or one time.] + + The IPv6 address and ORPort for this directory, as defined by + Onionoo. If present, IPv6 addresses and ORPorts are guaranteed to be + non-zero. (For IPv6 addresses, this means that they are not equal to + "[::]".) + + DQUOTE SP+ "weight=" number DQUOTE SP* NL + + [Zero or one time.] + + A non-negative, real-numbered weight for this directory. + The default fallback weight is 1.0, and the default + DirAuthorityFallbackRate is 1.0 in legacy Tor versions, and 0.1 in + recent Tor versions. + + weight was removed in version 2.0.0, but is documented because it + may be of interest to libraries implementing Tor's fallback + behaviour. + + DQUOTE SP+ key_value DQUOTE SP* NL + + [Zero or more times.] + + Future releases may include additional data fields in double-quoted + C string constants. Parsers MUST NOT rely on the order of these + additional fields. Additional data fields will be accompanied by a + minor version increment. + + "/*" SP+ "nickname=" nickname* SP+ "*/" SP* NL + + [Exactly once.] + + The nickname for this directory, as defined by Onionoo. An + empty nickname indicates that the nickname is unknown. + + The first fallback list in the 2.0.0 format had nickname lines, but + they were all empty. + + "/*" SP+ "extrainfo=" bool SP+ "*/" SP* NL + + [Exactly once.] + + An integer flag that indicates whether this directory caches + extra-info documents. Set to 1 if the directory claimed that it + cached extra-info documents in its descriptor when the list was + created. 0 indicates that it did not, or its descriptor was not + available. + + The first fallback list in the 2.0.0 format had extrainfo lines, but + they were all zero. + + "/*" SP+ key_value SP+ "*/" SP* NL + + [Zero or more times.] + + Future releases may include additional data fields in C-style + comments. Parsers MUST NOT rely on the order of these additional + fields. Additional data fields will be accompanied by a minor version + increment. + + separator SP* NL + + [Exactly once.] + + Each directory entry ends with the section separator. + + "," SP* NL + + [Exactly once.] + + The comma terminates the C string constant. (Multiple C string + constants separated by whitespace or comments are coalesced by + the C compiler.) +``` + +<a id="dir-list-spec.txt-3"></a> + +## Usage Considerations + +This section contains recommended library behaviours. It does not affect +the format of directory lists. + +<a id="dir-list-spec.txt-3.1"></a> + +### Caching + +The fallback list typically changes once every 6-12 months. The data in +the list represents the state of the fallback directory entries when the +list was created. Fallbacks can and do change their details over time. + +Libraries SHOULD parse and cache the most recent version of these lists +during their build or release processes. Libraries MUST NOT retrieve the +lists by default every time they are deployed or executed. + +The latest fallback list can be retrieved from: + +<https://gitlab.torproject.org/tpo/core/tor/-/raw/main/src/app/config/fallback_dirs.inc?ref_type=heads> + +Libraries MUST NOT rely on the availability of the server that hosts +these lists. + +The list can also be retrieved using: + +`git clone https://gitlab.torproject.org/tpo/core/tor/` + +If you just want the latest list, you may wish to perform a shallow +clone. + +<a id="dir-list-spec.txt-3.2"></a> + +### Retrieving Directory Information + +Some libraries retrieve directory documents directly from the Tor +Directory Authorities. The directory authorities are designed to support +Tor relay and client bootstrap, and MAY choose to rate-limit library +access. Libraries MAY provide a user-agent in their requests, if they +are not intended to support anonymous operation. (User agents are a +fingerprinting vector.) + +Libraries SHOULD consider the potential load on the authorities, and +whether other sources can meet their needs. + +Libraries that require high-uptime availability of Tor directory +information should investigate the following options: + +```text + * OnionOO: https://metrics.torproject.org/onionoo.html + * Third-party OnionOO mirrors are also available + * CollecTor: https://collector.torproject.org/ + * Fallback Directory Mirrors +``` + +Onionoo and CollecTor are typically updated every hour on a regular +schedule. Fallbacks update their own directory information at random +intervals, see dir-spec for details. + +<a id="dir-list-spec.txt-3.3"></a> + +### Fallback Reliability + +The fallback list is typically regenerated when the fallback failure +rate exceeds 25%. Libraries SHOULD NOT rely on any particular fallback +being available, or some proportion of fallbacks being available. + +Libraries that use fallbacks MAY wish to query an authority after a +few fallback queries fail. For example, Tor clients try 3-4 fallbacks +before trying an authority. + +<a id="dir-list-spec.txt-A.1"></a> + +### Sample Data + +A sample version 2.0.0 fallback list is available here: + +<https://trac.torproject.org/projects/tor/raw-attachment/ticket/22759/fallback_dirs_new_format_version.4.inc> + +A sample transitional version 2.0.0 fallback list is available here: + +<https://raw.githubusercontent.com/teor2345/tor/fallback-format-2-v4/src/or/fallback_dirs.inc> + +<a id="dir-list-spec.txt-A.1.1"></a> + +#### Sample Fallback List Header + +```c +/*type=fallback */ +/* version=2.0.0 */ +/* =====*/ +``` + +<a id="dir-list-spec.txt-A.1.2"></a> + +#### Sample Fallback List Generation + +```c +/*Whitelist & blacklist excluded 1326 of 1513 candidates. */ +/* Checked IPv4 DirPorts served a consensus within 15.0s. */ +/* +Final Count: 151 (Eligible 187, Target 392 (1963* 0.20), Max 200) +Excluded: 36 (Same Operator 27, Failed/Skipped Download 9, Excess 0) +Bandwidth Range: 1.3 - 40.0 MByte/s +*/ +/* +Onionoo Source: details Date: 2017-05-16 07:00:00 Version: 4.0 +URL: https:onionoo.torproject.orgdetails?fields=fingerprint%2Cnickname%2Ccontact%2Clast_changed_address_or_port%2Cconsensus_weight%2Cadvertised_bandwidth%2Cor_addresses%2Cdir_address%2Crecommended_version%2Cflags%2Ceffective_family%2Cplatform&flag=V2Dir&type=relay&last_seen_days=-0&first_seen_days=30- +*/ +/* +Onionoo Source: uptime Date: 2017-05-16 07:00:00 Version: 4.0 +URL: https:onionoo.torproject.orguptime?first_seen_days=30-&flag=V2Dir&type=relay&last_seen_days=-0 +*/ +/* ===== \*/ +``` + +<a id="dir-list-spec.txt-A.1.3"></a> + +#### Sample Fallback Entries + +```c +"176.10.104.240:80 orport=443 id=0111BA9B604669E636FFD5B503F382A4B7AD6E80" +/*nickname=foo */ +/* extrainfo=1 */ +/* ===== */ +, +"5.9.110.236:9030 orport=9001 id=0756B7CD4DFC8182BE23143FAC0642F515182CEB" +" ipv6=\[2a01:4f8:162:51e2::2\]:9001" +/* nickname= */ +/* extrainfo=0 */ +/* =====*/ +, +```
\ No newline at end of file diff --git a/spec/dir-spec/accepting-relay-documents.md b/spec/dir-spec/accepting-relay-documents.md new file mode 100644 index 0000000..ffcf7c7 --- /dev/null +++ b/spec/dir-spec/accepting-relay-documents.md @@ -0,0 +1,52 @@ +<a id="dir-spec.txt-3.2"></a> + +# Accepting server descriptor and extra-info document uploads + +When a router posts a signed descriptor to a directory authority, the +authority first checks whether it is well-formed and correctly +self-signed. If it is, the authority next verifies that the nickname +in question is not already assigned to a router with a different +public key. +Finally, the authority MAY check that the router is not blacklisted +because of its key, IP, or another reason. + +An authority also keeps a record of all the Ed25519/RSA1024 +identity key pairs that it has seen before. It rejects any +descriptor that has a known Ed/RSA identity key that it has +already seen accompanied by a different RSA/Ed identity key +in an older descriptor. + +At a future date, authorities will begin rejecting all +descriptors whose RSA key was previously accompanied by an +Ed25519 key, if the descriptor does not list an Ed25519 key. + +At a future date, authorities will begin rejecting all descriptors +that do not list an Ed25519 key. + +If the descriptor passes these tests, and the authority does not already +have a descriptor for a router with this public key, it accepts the +descriptor and remembers it. + +If the authority _does_ have a descriptor with the same public key, the +newly uploaded descriptor is remembered if its publication time is more +recent than the most recent old descriptor for that router, and either: + +```text + - There are non-cosmetic differences between the old descriptor and the + new one. + - Enough time has passed between the descriptors' publication times. + (Currently, 2 hours.) +``` + +Differences between server descriptors are "non-cosmetic" if they would be +sufficient to force an upload as described in section 2.1 above. + +Note that the "cosmetic difference" test only applies to uploaded +descriptors, not to descriptors that the authority downloads from other +authorities. + +When a router posts a signed extra-info document to a directory authority, +the authority again checks it for well-formedness and correct signature, +and checks that its matches the extra-info-digest in some router +descriptor that it believes is currently useful. If so, it accepts it and +stores it and serves it as requested. If not, it drops it. diff --git a/spec/dir-spec/assigning-flags-vote.md b/spec/dir-spec/assigning-flags-vote.md new file mode 100644 index 0000000..7eb3849 --- /dev/null +++ b/spec/dir-spec/assigning-flags-vote.md @@ -0,0 +1,172 @@ +<a id="dir-spec.txt-3.4.2"></a> + +# Assigning flags in a vote + +(This section describes how directory authorities choose which status +flags to apply to routers. Later directory authorities MAY do things +differently, so long as clients keep working well. Clients MUST NOT +depend on the exact behaviors in this section.) + +In the below definitions, a router is considered "active" if it is +running, valid, and not hibernating. + +When we speak of a router's bandwidth in this section, we mean either +its measured bandwidth, or its advertised bandwidth. If a sufficient +threshold (configurable with MinMeasuredBWsForAuthToIgnoreAdvertised, +500 by default) of routers have measured bandwidth values, then the +authority bases flags on _measured_ bandwidths, and treats nodes with +non-measured bandwidths as if their bandwidths were zero. Otherwise, +it uses measured bandwidths for nodes that have them, and advertised +bandwidths for other nodes. + +When computing thresholds based on percentiles of nodes, an authority +only considers nodes that are active, that have not been +omitted as a sybil (see below), and whose bandwidth is at least +4 KB. Nodes that don't meet these criteria do not influence any +threshold calculations (including calculation of stability and uptime +and bandwidth thresholds) and also do not have their Exit status +change. + +"Valid" -- a router is 'Valid' if it is running a version of Tor not +known to be broken, and the directory authority has not blacklisted +it as suspicious. + +```text + "Named" -- + "Unnamed" -- Directory authorities no longer assign these flags. + They were once used to determine whether a relay's nickname was + canonically linked to its public key. +``` + +"Running" -- A router is 'Running' if the authority managed to connect to +it successfully within the last 45 minutes on all its published ORPorts. +Authorities check reachability on: + +```text + * the IPv4 ORPort in the "r" line, and + * the IPv6 ORPort considered for the "a" line, if: + * the router advertises at least one IPv6 ORPort, and + * AuthDirHasIPv6Connectivity 1 is set on the authority. +``` + +A minority of voting authorities that set AuthDirHasIPv6Connectivity will +drop unreachable IPv6 ORPorts from the full consensus. Consensus method 27 +in 0.3.3.x puts IPv6 ORPorts in the microdesc consensus, so that +authorities can drop unreachable IPv6 ORPorts from all consensus flavors. +Consensus method 28 removes IPv6 ORPorts from microdescriptors. + +"Stable" -- A router is 'Stable' if it is active, and either its Weighted +MTBF is at least the median for known active routers or its Weighted MTBF +corresponds to at least 7 days. Routers are never called Stable if they are +running a version of Tor known to drop circuits stupidly. (0.1.1.10-alpha +through 0.1.1.16-rc are stupid this way.) + +To calculate weighted MTBF, compute the weighted mean of the lengths +of all intervals when the router was observed to be up, weighting +intervals by $\\alpha^n$, where $n$ is the amount of time that has +passed since the interval ended, and $\\alpha$ is chosen so that +measurements over approximately one month old no longer influence the +weighted MTBF much. + +\[XXXX what happens when we have less than 4 days of MTBF info.\] + +"Exit" -- A router is called an 'Exit' iff it allows exits to at +least one /8 address space on each of ports 80 and 443. (Up until +Tor version 0.3.2, the flag was assigned if relays exit to at least +two of the ports 80, 443, and 6667.) + +"Fast" -- A router is 'Fast' if it is active, and its bandwidth is either in +the top 7/8ths for known active routers or at least 100KB/s. + +"Guard" -- A router is a possible Guard if all of the following apply: + +```text + - It is Fast, + - It is Stable, + - Its Weighted Fractional Uptime is at least the median for "familiar" + active routers, + - It is "familiar", + - Its bandwidth is at least AuthDirGuardBWGuarantee (if set, 2 MB by + default), OR its bandwidth is among the 25% fastest relays, + - It qualifies for the V2Dir flag as described below (this + constraint was added in 0.3.3.x, because in 0.3.0.x clients + started avoiding guards that didn't also have the V2Dir flag). +``` + +To calculate weighted fractional uptime, compute the fraction +of time that the router is up in any given day, weighting so that +downtime and uptime in the past counts less. + +A node is 'familiar' if 1/8 of all active nodes have appeared more +recently than it, OR it has been around for a few weeks. + +"Authority" -- A router is called an 'Authority' if the authority +generating the network-status document believes it is an authority. + +"V2Dir" -- A router supports the v2 directory protocol or higher if it has +an open directory port OR a tunnelled-dir-server line in its router +descriptor, and it is running a version of the directory +protocol that supports the functionality clients need. (Currently, every +supported version of Tor supports the functionality that clients need, +but some relays might set "DirCache 0" or set really low rate limiting, +making them unqualified to be a directory mirror, i.e. they will omit +the tunnelled-dir-server line from their descriptor.) + +"HSDir" -- A router is a v2 hidden service directory if it stores and +serves v2 hidden service descriptors, has the Stable and Fast flag, and the +authority believes that it's been up for at least 96 hours (or the current +value of MinUptimeHidServDirectoryV2). + +"MiddleOnly" -- An authority should vote for this flag if it believes +that a relay is unsuitable for use except as a middle relay. When +voting for this flag, the authority should also vote against "Exit", +"Guard", "HsDir", and "V2Dir". When voting for this flag, if the +authority votes on the "BadExit" flag, the authority should vote in +favor of "BadExit". (This flag was added in 0.4.7.2-alpha.) + +"NoEdConsensus" -- authorities should not vote on this flag; it is +produced as part of the consensus for consensus method 22 or later. + +"StaleDesc" -- authorities should vote to assign this flag if the +published time on the descriptor is over 18 hours in the past. (This flag +was added in 0.4.0.1-alpha.) + +"Sybil" -- authorities SHOULD NOT accept more than 2 relays on a single IP. +If this happens, the authority _should_ vote for the excess relays, but +should omit the Running or Valid flags and instead should assign the "Sybil" +flag. When there are more than 2 (or AuthDirMaxServersPerAddr) relays to +choose from, authorities should first prefer authorities to non-authorities, +then prefer Running to non-Running, and then prefer high-bandwidth to +low-bandwidth relays. In this comparison, measured bandwidth is used unless +it is not present for a router, in which case advertised bandwidth is used. + +Thus, the network-status vote includes all non-blacklisted, +non-expired, non-superseded descriptors. + +The bandwidth in a "w" line should be taken as the best estimate +of the router's actual capacity that the authority has. For now, +this should be the lesser of the observed bandwidth and bandwidth +rate limit from the server descriptor. It is given in kilobytes +per second, and capped at some arbitrary value (currently 10 MB/s). + +The Measured= keyword on a "w" line vote is currently computed +by multiplying the previous published consensus bandwidth by the +ratio of the measured average node stream capacity to the network +average. If 3 or more authorities provide a Measured= keyword for +a router, the authorities produce a consensus containing a "w" +Bandwidth= keyword equal to the median of the Measured= votes. + +As a special case, if the "w" line in a vote is about a relay with the +Authority flag, it should not include a Measured= keyword. The goal is +to leave such relays marked as Unmeasured, so they can reserve their +attention for authority-specific activities. "w" lines for votes about +authorities may include the bandwidth authority's measurement using +a different keyword, e.g. MeasuredButAuthority=, so it can still be +reported and recorded for posterity. + +The ports listed in a "p" line should be taken as those ports for +which the router's exit policy permits 'most' addresses, ignoring any +accept not for all addresses, ignoring all rejects for private +netblocks. "Most" addresses are permitted if no more than 2^25 +IPv4 addresses (two /8 networks) were blocked. The list is encoded +as described in section 3.8.2. diff --git a/spec/dir-spec/client-operation.md b/spec/dir-spec/client-operation.md new file mode 100644 index 0000000..a59fae4 --- /dev/null +++ b/spec/dir-spec/client-operation.md @@ -0,0 +1,241 @@ +<a id="dir-spec.txt-5"></a> + +# Client operation + +Every Tor that is not a directory server (that is, those that do +not have a DirPort set) implements this section. + +<a id="dir-spec.txt-5.1"></a> + +## Downloading network-status documents { #download-ns } + +Each client maintains a list of directory authorities. Insofar as +possible, clients SHOULD all use the same list. + +```text + [Newer versions of Tor (0.2.8.1-alpha and later): + Each client also maintains a list of default fallback directory mirrors + (fallbacks). Each released version of Tor MAY have a different list, + depending on the mirrors that satisfy the fallback directory criteria at + release time.] +``` + +Clients try to have a live consensus network-status document at all times. +A network-status document is "live" if the time in its valid-after field +has passed, and the time in its valid-until field has not passed. + +When a client has no consensus network-status document, it downloads it +from a randomly chosen fallback directory mirror or authority. Clients +prefer fallbacks to authorities, trying them earlier and more frequently. +In all other cases, the client downloads from caches randomly chosen from +among those believed to be V3 directory servers. (This information comes +from the network-status documents.) + +After receiving any response client MUST discard any network-status +documents that it did not request. + +On failure, the client waits briefly, then tries that network-status +document again from another cache. The client does not build circuits +until it has a live network-status consensus document, and it has +descriptors for a significant proportion of the routers that it believes +are running (this is configurable using torrc options and consensus +parameters). + +```text + [Newer versions of Tor (0.2.6.2-alpha and later): + If the consensus contains Exits (the typical case), Tor will build both + exit and internal circuits. When bootstrap completes, Tor will be ready + to handle an application requesting an exit circuit to services like the + World Wide Web. +``` + +If the consensus does not contain Exits, Tor will only build internal +circuits. In this case, earlier statuses will have included "internal" +as indicated above. When bootstrap completes, Tor will be ready to handle +an application requesting an internal circuit to hidden services at +".onion" addresses. + +If a future consensus contains Exits, exit circuits may become available.\] + +(Note: clients can and should pick caches based on the network-status +information they have: once they have first fetched network-status info +from an authority or fallback, they should not need to go to the authority +directly again, and should only choose the fallback at random, based on its +consensus weight in the current consensus.) + +To avoid swarming the caches whenever a consensus expires, the +clients download new consensuses at a randomly chosen time after the +caches are expected to have a fresh consensus, but before their +consensus will expire. (This time is chosen uniformly at random from +the interval between the time 3/4 into the first interval after the +consensus is no longer fresh, and 7/8 of the time remaining after +that before the consensus is invalid.) + +```text + [For example, if a client has a consensus that became valid at 1:00, + and is fresh until 2:00, and expires at 4:00, that client will fetch + a new consensus at a random time between 2:45 and 3:50, since 3/4 + of the one-hour interval is 45 minutes, and 7/8 of the remaining 75 + minutes is 65 minutes.] +``` + +Clients may choose to download the microdescriptor consensus instead +of the general network status consensus. In that case they should use +the same update strategy as for the normal consensus. They should not +download more than one consensus flavor. + +When a client does not have a live consensus, it will generally use the +most recent consensus it has if that consensus is "reasonably live". A +"reasonably live" consensus is one that expired less than 24 hours ago. + +<a id="dir-spec.txt-5.2"></a> + +## Downloading server descriptors or microdescriptors { #download-desc } + +Clients try to have the best descriptor for each router. A descriptor is +"best" if: + +- It is listed in the consensus network-status document. + +Periodically (currently every 10 seconds) clients check whether there are +any "downloadable" descriptors. A descriptor is downloadable if: + +```text + - It is the "best" descriptor for some router. + - The descriptor was published at least 10 minutes in the past. + (This prevents clients from trying to fetch descriptors that the + mirrors have probably not yet retrieved and cached.) + - The client does not currently have it. + - The client is not currently trying to download it. + - The client would not discard it immediately upon receiving it. + - The client thinks it is running and valid (see section 5.4.1 below). +``` + +If at least 16 known routers have downloadable descriptors, or if +enough time (currently 10 minutes) has passed since the last time the +client tried to download descriptors, it launches requests for all +downloadable descriptors. + +When downloading multiple server descriptors, the client chooses multiple +mirrors so that: + +```text + - At least 3 different mirrors are used, except when this would result + in more than one request for under 4 descriptors. + - No more than 128 descriptors are requested from a single mirror. + - Otherwise, as few mirrors as possible are used. + After choosing mirrors, the client divides the descriptors among them + randomly. +``` + +After receiving any response the client MUST discard any descriptors that +it did not request. + +When a descriptor download fails, the client notes it, and does not +consider the descriptor downloadable again until a certain amount of time +has passed. (Currently 0 seconds for the first failure, 60 seconds for the +second, 5 minutes for the third, 10 minutes for the fourth, and 1 day +thereafter.) Periodically (currently once an hour) clients reset the +failure count. + +Clients retain the most recent descriptor they have downloaded for each +router so long as it is listed in the consensus. If it is not listed, +they keep it so long as it is not too old (currently, ROUTER_MAX_AGE=48 +hours) and no better router descriptor has been downloaded for the same +relay. Caches retain descriptors until they are at least +OLD_ROUTER_DESC_MAX_AGE=5 days old. + +Clients which chose to download the microdescriptor consensus instead +of the general consensus must download the referenced microdescriptors +instead of server descriptors. Clients fetch and cache +microdescriptors preemptively from dir mirrors when starting up, like +they currently fetch descriptors. After bootstrapping, clients only +need to fetch the microdescriptors that have changed. + +When a client gets a new microdescriptor consensus, it looks to see if +there are any microdescriptors it needs to learn, and launches a request +for them. + +Clients maintain a cache of microdescriptors along with metadata like +when it was last referenced by a consensus, and which identity key +it corresponds to. They keep a microdescriptor until it hasn't been +mentioned in any consensus for a week. Future clients might cache them +for longer or shorter times. + +<a id="dir-spec.txt-5.3"></a> + +## Downloading extra-info documents { #download-extra } + +Any client that uses extra-info documents should implement this +section. + +Note that generally, clients don't need extra-info documents. + +Periodically, the Tor instance checks whether it is missing any extra-info +documents: in other words, if it has any server descriptors with an +extra-info-digest field that does not match any of the extra-info +documents currently held. If so, it downloads whatever extra-info +documents are missing. Clients try to download from caches. +We follow the same splitting and back-off rules as in section 5.2. + +<a id="dir-spec.txt-5.5"></a> + +## Retrying failed downloads + +This section applies to caches as well as to clients. + +When a client fails to download a resource (a consensus, a router +descriptor, a microdescriptor, etc) it waits for a certain amount of +time before retrying the download. To determine the amount of time +to wait, clients use a randomized exponential backoff algorithm. +(Specifically, they use a variation of the "decorrelated jitter" +algorithm from +<https://aws.amazon.com/blogs/architecture/exponential-backoff-and-jitter/> .) + +The specific formula used to compute the 'i+1'th delay is: + +```text + Delay_{i+1} = MIN(cap, random_between(lower_bound, upper_bound))) + where upper_bound = MAX(lower_bound+1, Delay_i * 3) + lower_bound = MAX(1, base_delay). +``` + +The value of 'cap' is set to INT_MAX; the value of 'base_delay' +depends on what is being downloaded, whether the client is fully +bootstrapped, how the client is configured, and where it is +downloading from. Current base_delay values are: + +```text + Consensus objects, as a non-bridge cache: + 0 (TestingServerConsensusDownloadInitialDelay) + + Consensus objects, as a client or bridge that has bootstrapped: + 0 (TestingClientConsensusDownloadInitialDelay) + + Consensus objects, as a client or bridge that is bootstrapping, + when connecting to an authority because no "fallback" caches are + known: + 0 (ClientBootstrapConsensusAuthorityOnlyDownloadInitialDelay) + + Consensus objects, as a client or bridge that is bootstrapping, + when "fallback" caches are known but connecting to an authority + anyway: + 6 (ClientBootstrapConsensusAuthorityDownloadInitialDelay) + + Consensus objects, as a client or bridge that is bootstrapping, + when downloading from a "fallback" cache. + 0 (ClientBootstrapConsensusFallbackDownloadInitialDelay) + + Bridge descriptors, as a bridge-using client when at least one bridge + is usable: + 10800 (TestingBridgeDownloadInitialDelay) + + Bridge descriptors, otherwise: + 0 (TestingBridgeBootstrapDownloadInitialDelay) + + Other objects, as cache or authority: + 0 (TestingServerDownloadInitialDelay) + + Other objects, as client: + 0 (TestingClientDownloadInitialDelay) +``` diff --git a/spec/dir-spec/computing-consensus.md b/spec/dir-spec/computing-consensus.md new file mode 100644 index 0000000..329fd45 --- /dev/null +++ b/spec/dir-spec/computing-consensus.md @@ -0,0 +1,683 @@ +# Computing a consensus from a set of votes { #computing-consensus } + +Given a set of votes, authorities compute the contents of the consensus. + +The consensus status, along with as many signatures as the server +currently knows (see section 3.10 below), should be available at + +`http://<hostname>/tor/status-vote/next/consensus.z` + +The contents of the consensus document are as follows: + +The "valid-after", "valid-until", and "fresh-until" times are taken as +the median of the respective values from all the votes. + +The times in the "voting-delay" line are taken as the median of the +VoteSeconds and DistSeconds times in the votes. + +Known-flags is the union of all flags known by any voter. + +Entries are given on the "params" line for every keyword on which a +majority of authorities (total authorities, not just those +participating in this vote) voted on, or if at least three +authorities voted for that parameter. The values given are the +low-median of all votes on that keyword. + +(In consensus methods 7 to 11 inclusive, entries were given on +the "params" line for every keyword on which *any* authority voted, +the value given being the low-median of all votes on that keyword.) + +```text + "client-versions" and "server-versions" are sorted in ascending + order; A version is recommended in the consensus if it is recommended + by more than half of the voting authorities that included a + client-versions or server-versions lines in their votes. +``` + +With consensus methods 19 through 33, a package line is generated for a +given PACKAGENAME/VERSION pair if at least three authorities list such a +package in their votes. (Call these lines the "input" lines for +PACKAGENAME.) The consensus will contain every "package" line that is +listed verbatim by more than half of the authorities listing a line for +the PACKAGENAME/VERSION pair, and no others. + +The authority item groups (dir-source, contact, fingerprint, +vote-digest) are taken from the votes of the voting +authorities. These groups are sorted by the digests of the +authorities identity keys, in ascending order. If the consensus +method is 3 or later, a dir-source line must be included for +every vote with legacy-key entry, using the legacy-key's +fingerprint, the voter's ordinary nickname with the string +"-legacy" appended, and all other fields as from the original +vote's dir-source line. + +```text + A router status entry: + * is included in the result if some router status entry with the same + identity is included by more than half of the authorities (total + authorities, not just those whose votes we have). + (Consensus method earlier than 21) + + * is included according to the rules in section 3.8.0.1 and + 3.8.0.2 below. (Consensus method 22 or later) + + * For any given RSA identity digest, we include at most + one router status entry. + + * For any given Ed25519 identity, we include at most one router + status entry. + + * A router entry has a flag set if that is included by more than half + of the authorities who care about that flag. + + * Two router entries are "the same" if they have the same + (descriptor digest, published time, nickname, IP, ports> tuple. + We choose the tuple for a given router as whichever tuple appears + for that router in the most votes. We break ties first in favor of + the more recently published, then in favor of smaller server + descriptor digest. + + [ + * The Named flag appears if it is included for this routerstatus by + _any_ authority, and if all authorities that list it list the same + nickname. However, if consensus-method 2 or later is in use, and + any authority calls this identity/nickname pair Unnamed, then + this routerstatus does not get the Named flag. + + * If consensus-method 2 or later is in use, the Unnamed flag is + set for a routerstatus if any authorities have voted for a different + identities to be Named with that nickname, or if any authority + lists that nickname/ID pair as Unnamed. + + (With consensus-method 1, Unnamed is set like any other flag.) + + [But note that authorities no longer vote for the Named flag, + and the above two bulletpoints are now irrelevant.] + ] + + * The version is given as whichever version is listed by the most + voters, with ties decided in favor of more recent versions. + + * If consensus-method 4 or later is in use, then routers that + do not have the Running flag are not listed at all. + + * If consensus-method 5 or later is in use, then the "w" line + is generated using a low-median of the bandwidth values from + the votes that included "w" lines for this router. + + * If consensus-method 5 or later is in use, then the "p" line + is taken from the votes that have the same policy summary + for the descriptor we are listing. (They should all be the + same. If they are not, we pick the most commonly listed + one, breaking ties in favor of the lexicographically larger + vote.) The port list is encoded as specified in section 3.8.2. + + * If consensus-method 6 or later is in use and if 3 or more + authorities provide a Measured= keyword in their votes for + a router, the authorities produce a consensus containing a + Bandwidth= keyword equal to the median of the Measured= votes. + + * If consensus-method 7 or later is in use, the params line is + included in the output. + + * If the consensus method is under 11, bad exits are considered as + possible exits when computing bandwidth weights. Otherwise, if + method 11 or later is in use, any router that is determined to get + the BadExit flag doesn't count when we're calculating weights. + + * If consensus method 12 or later is used, only consensus + parameters that more than half of the total number of + authorities voted for are included in the consensus. + + [ As of 0.2.6.1-alpha, authorities no longer advertise or negotiate + any consensus methods lower than 13. ] + + * If consensus method 13 or later is used, microdesc consensuses + omit any router for which no microdesc was agreed upon. + + * If consensus method 14 or later is used, the ns consensus and + microdescriptors may include an "a" line for each router, listing + an IPv6 OR port. + + * If consensus method 15 or later is used, microdescriptors + include "p6" lines including IPv6 exit policies. + + * If consensus method 16 or later is used, ntor-onion-key + are included in microdescriptors + + * If consensus method 17 or later is used, authorities impose a + maximum on the Bandwidth= values that they'll put on a 'w' + line for any router that doesn't have at least 3 measured + bandwidth values in votes. They also add an "Unmeasured=1" + flag to such 'w' lines. + + * If consensus method 18 or later is used, authorities include + "id" lines in microdescriptors. This method adds RSA ids. + + * If consensus method 19 or later is used, authorities may include + "package" lines in consensuses. + + * If consensus method 20 or later is used, authorities may include + GuardFraction information in microdescriptors. + + * If consensus method 21 or later is used, authorities may include + an "id" line for ed25519 identities in microdescriptors. + + [ As of 0.2.8.2-alpha, authorities no longer advertise or negotiate + consensus method 21, because it contains bugs. ] + + * If consensus method 22 or later is used, and the votes do not + produce a majority consensus about a relay's Ed25519 key (see + 3.8.0.1 below), the consensus must include a NoEdConsensus flag on + the "s" line for every relay whose listed Ed key does not reflect + consensus. + + * If consensus method 23 or later is used, authorities include + shared randomness protocol data on their votes and consensus. + + * If consensus-method 24 or later is in use, then routers that + do not have the Valid flag are not listed at all. + + [ As of 0.3.4.1-alpha, authorities no longer advertise or negotiate + any consensus methods lower than 25. ] + + * If consensus-method 25 or later is in use, then we vote + on recommended-protocols and required-protocols lines in the + consensus. We also include protocols lines in routerstatus + entries. + + * If consensus-method 26 or later is in use, then we initialize + bandwidth weights to 1 in our calculations, to avoid + division-by-zero errors on unusual networks. + + * If consensus method 27 or later is used, the microdesc consensus + may include an "a" line for each router, listing an IPv6 OR port. + + [ As of 0.4.3.1-alpha, authorities no longer advertise or negotiate + any consensus methods lower than 28. ] + + * If consensus method 28 or later is used, microdescriptors no longer + include "a" lines. + + * If consensus method 29 or later is used, microdescriptor "family" + lines are canonicalized to improve compression. + + * If consensus method 30 or later is used, the base64 encoded + ntor-onion-key does not include the trailing = sign. + + * If consensus method 31 or later is used, authorities parse the + "bwweightscale" and "maxunmeasuredbw" parameters correctly when + computing votes. + + * If consensus method 32 or later is used, authorities handle the + "MiddleOnly" flag specially when computing a consensus. When the + voters agree to include "MiddleOnly" in a routerstatus, they + automatically remove "Exit", "Guard", "V2Dir", and "HSDir". If + the BadExit flag is included in the consensus, they automatically + add it to the routerstatus. + + * If consensus method 33 or later is used, and the consensus + flavor is "microdesc", then the "Publication" field in the "r" + line is set to "2038-01-01 00:00:00". + + * If consensus method 34 or later is used, the consensus + does not include any "package" lines. +``` + +The signatures at the end of a consensus document are sorted in +ascending order by identity digest. + +All ties in computing medians are broken in favor of the smaller or +earlier item. + +<a id="dir-spec.txt-3.8.0.1"></a> + +## Deciding which Ids to include { #choosing-relay-ids } + +This sorting algorithm is used for consensus-method 22 and later. + +```text + First, consider each listing by tuple of <Ed,Rsa> identities, where 'Ed' + may be "None" if the voter included "id ed25519 none" to indicate that + the authority knows what ed25519 identities are, and thinks that the RSA + key doesn't have one. + + For each such <Ed, RSA> tuple that is listed by more than half of the + total authorities (not just total votes), include it. (It is not + possible for any other <id-Ed, id-RSA'> to have as many votes.) If more + than half of the authorities list a single <Ed,Rsa> pair of this type, we + consider that Ed key to be "consensus"; see description of the + NoEdConsensus flag. + + Log any other id-RSA values corresponding to an id-Ed we included, and any + other id-Ed values corresponding to an id-RSA we included. + + For each <id-RSA> that is not yet included, if it is listed by more than + half of the total authorities, and we do not already have it listed with + some <id-Ed>, include it, but do not consider its Ed identity canonical. +``` + +<a id="dir-spec.txt-3.8.0.2"></a> + +### Deciding which descriptors to include { #choosing-relay-descs } + +Deciding which descriptors to include. + +A tuple belongs to an `<id-RSA, id-Ed>` identity if it is a new tuple that +matches both ID parts, or if it is an old tuple (one with no Ed opinion) +that matches the RSA part. A tuple belongs to an `<id-RSA>` identity if its +RSA identity matches. + +A tuple matches another tuple if all the fields that are present in both +tuples are the same. + +For every included identity, consider the tuples belonging to that +identity. Group them into sets of matching tuples. Include the tuple +that matches the largest set, breaking ties in favor of the most recently +published, and then in favor of the smaller server descriptor digest. + +<a id="dir-spec.txt-3.8.1"></a> + +## Forward compatibility { #consensus-method-list } + +Future versions of Tor will need to include new information in the +consensus documents, but it is important that all authorities (or at least +half) generate and sign the same signed consensus. + +To achieve this, authorities list in their votes their supported methods +for generating consensuses from votes. Later methods will be assigned +higher numbers. Currently specified methods: + +```text + "1" -- The first implemented version. + "2" -- Added support for the Unnamed flag. + "3" -- Added legacy ID key support to aid in authority ID key rollovers + "4" -- No longer list routers that are not running in the consensus + "5" -- adds support for "w" and "p" lines. + "6" -- Prefers measured bandwidth values rather than advertised + "7" -- Provides keyword=integer pairs of consensus parameters + "8" -- Provides microdescriptor summaries + "9" -- Provides weights for selecting flagged routers in paths + "10" -- Fixes edge case bugs in router flag selection weights + "11" -- Don't consider BadExits when calculating bandwidth weights + "12" -- Params are only included if enough auths voted for them + "13" -- Omit router entries with missing microdescriptors. + "14" -- Adds support for "a" lines in ns consensuses and microdescriptors. + "15" -- Adds support for "p6" lines. + "16" -- Adds ntor keys to microdescriptors + "17" -- Adds "Unmeasured=1" flags to "w" lines + "18" -- Adds 'id' to microdescriptors. + "19" -- Adds "package" lines to consensuses + "20" -- Adds GuardFraction information to microdescriptors. + "21" -- Adds Ed25519 keys to microdescriptors. + "22" -- Instantiates Ed25519 voting algorithm correctly. + "23" -- Adds shared randomness protocol data. + "24" -- No longer lists routers that are not Valid in the consensus. + "25" -- Vote on recommended-protocols and required-protocols. + "26" -- Initialize bandwidth weights to 1 to avoid division-by-zero. + "27" -- Adds support for "a" lines in microdescriptor consensuses. + "28" -- Removes "a" lines from microdescriptors. + "29" -- Canonicalizes families in microdescriptors. + "30" -- Removes padding from ntor-onion-key. + "31" -- Uses correct parsing for bwweightscale and maxunmeasuredbw + when computing weights + "32" -- Adds special handling for MiddleOnly flag. + "33" -- Sets "publication" field in microdesc consensus "r" lines + to a meaningless value. + "34" -- Removes "package" lines from consensus. +``` + +Before generating a consensus, an authority must decide which consensus +method to use. To do this, it looks for the highest version number +supported by more than 2/3 of the authorities voting. If it supports this +method, then it uses it. Otherwise, it falls back to the newest consensus +method that it supports (which will probably not result in a sufficiently +signed consensus). + +All authorities MUST support method 25; authorities SHOULD support +more recent methods as well. Authorities SHOULD NOT support or +advertise support for any method before 25. Clients MAY assume that +they will never see a current valid signed consensus for any method +before method 25. + +(The consensuses generated by new methods must be parsable by +implementations that only understand the old methods, and must not cause +those implementations to compromise their anonymity. This is a means for +making changes in the contents of consensus; not for making +backward-incompatible changes in their format.) + +The following methods have incorrect implementations; authorities SHOULD +NOT advertise support for them: + +"21" -- Did not correctly enable support for ed25519 key collation. + +<a id="dir-spec.txt-3.8.2"></a> + +## Encoding port lists + +Whether the summary shows the list of accepted ports or the list of +rejected ports depends on which list is shorter (has a shorter string +representation). In case of ties we choose the list of accepted +ports. As an exception to this rule an allow-all policy is +represented as "accept 1-65535" instead of "reject " and a reject-all +policy is similarly given as "reject 1-65535". + +Summary items are compressed, that is instead of "80-88,89-100" there +only is a single item of "80-100", similarly instead of "20,21" a +summary will say "20-21". + +Port lists are sorted in ascending order. + +The maximum allowed length of a policy summary (including the "accept " +or "reject ") is 1000 characters. If a summary exceeds that length we +use an accept-style summary and list as much of the port list as is +possible within these 1000 bytes. \[XXXX be more specific.\] + +<a id="dir-spec.txt-3.8.3"></a> + +## Computing Bandwidth Weights + +Let weight_scale = 10000, or the value of the "bwweightscale" parameter. +(Before consensus method 31 there was a bug in parsing bwweightscale, so +that if there were any consensus parameters after it alphabetically, it +would always be treated as 10000. A similar bug existed for +"maxunmeasuredbw".) + +Starting with consensus method 26, G, M, E, and D are initialized to 1 and +T to 4. Prior consensus methods initialize them all to 0. With this change, +test tor networks that are small or new are much more likely to produce +bandwidth-weights in their consensus. The extra bandwidth has a negligible +impact on the bandwidth weights in the public tor network. + +Let G be the total bandwidth for Guard-flagged nodes. +Let M be the total bandwidth for non-flagged nodes. +Let E be the total bandwidth for Exit-flagged nodes. +Let D be the total bandwidth for Guard+Exit-flagged nodes. +Let T = G+M+E+D + +Let Wgd be the weight for choosing a Guard+Exit for the guard position. +Let Wmd be the weight for choosing a Guard+Exit for the middle position. +Let Wed be the weight for choosing a Guard+Exit for the exit position. + +Let Wme be the weight for choosing an Exit for the middle position. +Let Wmg be the weight for choosing a Guard for the middle position. + +Let Wgg be the weight for choosing a Guard for the guard position. +Let Wee be the weight for choosing an Exit for the exit position. + +Balanced network conditions then arise from solutions to the following +system of equations: + +Wgg*G + Wgd*D == M + Wmd*D + Wme*E + Wmg*G (guard bw = middle bw) +Wgg*G + Wgd*D == Wee*E + Wed*D (guard bw = exit bw) +Wed*D + Wmd*D + Wgd*D == D (aka: Wed+Wmd+Wdg = weight_scale) +Wmg*G + Wgg*G == G (aka: Wgg = weight_scale-Wmg) +Wme*E + Wee*E == E (aka: Wee = weight_scale-Wme) + +We are short 2 constraints with the above set. The remaining constraints +come from examining different cases of network load. The following +constraints are used in consensus method 10 and above. There are another +incorrect and obsolete set of constraints used for these same cases in +consensus method 9. For those, see dir-spec.txt in Tor 0.2.2.10-alpha +to 0.2.2.16-alpha. + +Case 1: E >= T/3 && G >= T/3 (Neither Exit nor Guard Scarce) + +In this case, the additional two constraints are: Wmg == Wmd, +Wed == 1/3. + +```text + This leads to the solution: + Wgd = weight_scale/3 + Wed = weight_scale/3 + Wmd = weight_scale/3 + Wee = (weight_scale*(E+G+M))/(3*E) + Wme = weight_scale - Wee + Wmg = (weight_scale*(2*G-E-M))/(3*G) + Wgg = weight_scale - Wmg + + Case 2: E < T/3 && G < T/3 (Both are scarce) +``` + +Let R denote the more scarce class (Rare) between Guard vs Exit. +Let S denote the less scarce class. + +Subcase a: R+D \< S + +In this subcase, we simply devote all of D bandwidth to the +scarce class. + +```text + Wgg = Wee = weight_scale + Wmg = Wme = Wmd = 0; + if E < G: + Wed = weight_scale + Wgd = 0 + else: + Wed = 0 + Wgd = weight_scale + + Subcase b: R+D >= S +``` + +In this case, if M \<= T/3, we have enough bandwidth to try to achieve +a balancing condition. + +Add constraints Wgg = weight_scale, Wmd == Wgd to maximize bandwidth in +the guard position while still allowing exits to be used as middle nodes: + +Wee = (weight_scale\*(E - G + M))/E +Wed = (weight_scale\*(D - 2*E + 4*G - 2*M))/(3*D) +Wme = (weight_scale\*(G-M))/E +Wmg = 0 +Wgg = weight_scale +Wmd = (weight_scale - Wed)/2 +Wgd = (weight_scale - Wed)/2 + +If this system ends up with any values out of range (ie negative, or +above weight_scale), use the constraints Wgg == weight_scale and Wee == +weight_scale, since both those positions are scarce: + +```text + Wgg = weight_scale + Wee = weight_scale + Wed = (weight_scale*(D - 2*E + G + M))/(3*D) + Wmd = (weight_Scale*(D - 2*M + G + E))/(3*D) + Wme = 0 + Wmg = 0 + Wgd = weight_scale - Wed - Wmd + + If M > T/3, then the Wmd weight above will become negative. Set it to 0 + in this case: + Wmd = 0 + Wgd = weight_scale - Wed + + Case 3: One of E < T/3 or G < T/3 + + Let S be the scarce class (of E or G). + + Subcase a: (S+D) < T/3: + if G=S: + Wgg = Wgd = weight_scale; + Wmd = Wed = Wmg = 0; + // Minor subcase, if E is more scarce than M, + // keep its bandwidth in place. + if (E < M) Wme = 0; + else Wme = (weight_scale*(E-M))/(2*E); + Wee = weight_scale-Wme; + if E=S: + Wee = Wed = weight_scale; + Wmd = Wgd = Wme = 0; + // Minor subcase, if G is more scarce than M, + // keep its bandwidth in place. + if (G < M) Wmg = 0; + else Wmg = (weight_scale*(G-M))/(2*G); + Wgg = weight_scale-Wmg; + + Subcase b: (S+D) >= T/3 + if G=S: + Add constraints Wgg = weight_scale, Wmd == Wed to maximize bandwidth + in the guard position, while still allowing exits to be + used as middle nodes: + Wgg = weight_scale + Wgd = (weight_scale*(D - 2*G + E + M))/(3*D) + Wmg = 0 + Wee = (weight_scale*(E+M))/(2*E) + Wme = weight_scale - Wee + Wmd = (weight_scale - Wgd)/2 + Wed = (weight_scale - Wgd)/2 + if E=S: + Add constraints Wee == weight_scale, Wmd == Wgd to maximize bandwidth + in the exit position: + Wee = weight_scale; + Wed = (weight_scale*(D - 2*E + G + M))/(3*D); + Wme = 0; + Wgg = (weight_scale*(G+M))/(2*G); + Wmg = weight_scale - Wgg; + Wmd = (weight_scale - Wed)/2; + Wgd = (weight_scale - Wed)/2; +``` + +To ensure consensus, all calculations are performed using integer math +with a fixed precision determined by the bwweightscale consensus +parameter (defaults at 10000, Min: 1, Max: INT32_MAX). (See note above +about parsing bug in bwweightscale before consensus method 31.) + +For future balancing improvements, Tor clients support 11 additional weights +for directory requests and middle weighting. These weights are currently +set at weight_scale, with the exception of the following groups of +assignments: + +Directory requests use middle weights: + +Wbd=Wmd, Wbg=Wmg, Wbe=Wme, Wbm=Wmm + +Handle bridges and strange exit policies: + +Wgm=Wgg, Wem=Wee, Weg=Wed + +<a id="dir-spec.txt-3.9"></a> + +## Computing consensus flavors + +Consensus flavors are variants of the consensus that clients can choose +to download and use instead of the unflavored consensus. The purpose +of a consensus flavor is to remove or replace information in the +unflavored consensus without forcing clients to download information +they would not use anyway. + +Directory authorities can produce and serve an arbitrary number of +flavors of the same consensus. A downside of creating too many new +flavors is that clients will be distinguishable based on which flavor +they download. A new flavor should not be created when adding a field +instead wouldn't be too onerous. + +Examples for consensus flavors include: + +```text + - Publishing hashes of microdescriptors instead of hashes of + full descriptors (see section 3.9.2). + - Including different digests of descriptors, instead of the + perhaps-soon-to-be-totally-broken SHA1. +``` + +Consensus flavors are derived from the unflavored consensus once the +voting process is complete. This is to avoid consensus synchronization +problems. + +Every consensus flavor has a name consisting of a sequence of one +or more alphanumeric characters and dashes. For compatibility, +the original (unflavored) consensus type is called "ns". + +The supported consensus flavors are defined as part of the +authorities' consensus method. + +All consensus flavors have in common that their first line is +"network-status-version" where version is 3 or higher, and the flavor +is a string consisting of alphanumeric characters and dashes: + +"network-status-version" SP version \[SP flavor\] NL + +<a id="dir-spec.txt-3.9.1"></a> + +### ns consensus + +The ns consensus flavor is equivalent to the unflavored consensus. +When the flavor is omitted from the "network-status-version" line, +it should be assumed to be "ns". Some implementations may explicitly +state that the flavor is "ns" when generating consensuses, but should +accept consensuses where the flavor is omitted. + +<a id="dir-spec.txt-3.9.2"></a> + +### Microdescriptor consensus + +The microdescriptor consensus is a consensus flavor that contains +microdescriptor hashes instead of descriptor hashes and that omits +exit-policy summaries which are contained in microdescriptors. The +microdescriptor consensus was designed to contain elements that are +small and frequently changing. Clients use the information in the +microdescriptor consensus to decide which servers to fetch information +about and which servers to fetch information from. + +The microdescriptor consensus is based on the unflavored consensus with +the exceptions as follows: + +"network-status-version" SP version SP "microdesc" NL + +\[At start, exactly once.\] + +The flavor name of a microdescriptor consensus is "microdesc". + +Changes to router status entries are as follows: + +```text + "r" SP nickname SP identity SP publication SP IP SP ORPort + SP DirPort NL + + [At start, exactly once.] + + Similar to "r" lines in section 3.4.1, but without the digest element. + + "a" SP address ":" port NL + + [Any number] + + Identical to the "r" lines in section 3.4.1. +``` + +(Only included when the vote is generated with consensus-method 14 +or later, and the consensus is generated with consensus-method 27 or +later.) + +"p" ... NL + +\[At most once\] + +Not currently generated. + +Exit policy summaries are contained in microdescriptors and +therefore omitted in the microdescriptor consensus. + +"m" SP digest NL + +\[Exactly once.\*\] + +"digest" is the base64 of the SHA256 hash of the router's +microdescriptor with trailing =s omitted. For a given router +descriptor digest and consensus method there should only be a +single microdescriptor digest in the "m" lines of all votes. +If different votes have different microdescriptor digests for +the same descriptor digest and consensus method, at least one +of the authorities is broken. If this happens, the microdesc +consensus should contain whichever microdescriptor digest is +most common. If there is no winner, we break ties in the favor +of the lexically earliest. + +\[\*Before consensus method 13, this field was sometimes erroneously +omitted.\] + +Additionally, a microdescriptor consensus SHOULD use the sha256 digest +algorithm for its signatures. + +<a id="dir-spec.txt-3.10"></a> diff --git a/spec/dir-spec/computing-microdescriptors.md b/spec/dir-spec/computing-microdescriptors.md new file mode 100644 index 0000000..655c66a --- /dev/null +++ b/spec/dir-spec/computing-microdescriptors.md @@ -0,0 +1,161 @@ +<a id="dir-spec.txt-3.3"></a> + +# Computing microdescriptors + +Microdescriptors are a stripped-down version of server descriptors +generated by the directory authorities which may additionally contain +authority-generated information. Microdescriptors contain only the +most relevant parts that clients care about. Microdescriptors are +expected to be relatively static and only change about once per week. +Microdescriptors do not contain any information that clients need to +use to decide which servers to fetch information about, or which +servers to fetch information from. + +Microdescriptors are a straight transform from the server descriptor +and the consensus method. Microdescriptors have no header or footer. +Microdescriptors are identified by the hash of its concatenated +elements without a signature by the router. Microdescriptors do not +contain any version information, because their version is determined +by the consensus method. + +Starting with consensus method 8, microdescriptors contain the +following elements taken from or based on the server descriptor. Order +matters here, because different directory authorities must be able to +transform a given server descriptor and consensus method into the exact +same microdescriptor. + +"onion-key" NL a public key in PEM format + +\[Exactly once, at start\] +\[No extra arguments\] + +The "onion-key" element as specified in section 2.1.1. + +When generating microdescriptors for consensus method 30 or later, +the trailing = sign must be absent. For consensus method 29 or +earlier, the trailing = sign must be present. + +"ntor-onion-key" SP base-64-encoded-key NL + +\[Exactly once\] + +The "ntor-onion-key" element as specified in section 2.1.1. + +(Only included when generating microdescriptors for +consensus-method 16 or later.) + +\[Before Tor 0.4.5.1-alpha, this field was optional.\] + +"a" SP address ":" port NL + +\[Any number\] + +Additional advertised addresses for the OR. + +Present currently only if the OR advertises at least one IPv6 +address; currently, the first address is included and all others are +omitted. Any other IPv4 or IPv6 addresses should be ignored. + +Address and port are as for "or-address" as specified in +section 2.1.1. + +(Only included when generating microdescriptors for +consensus-methods 14 to 27.) + +"family" names NL + +\[At most once\] + +The "family" element as specified in section 2.1.1. + +When generating microdescriptors for consensus method 29 or later, +the following canonicalization algorithm is applied to improve +compression: + +```text + For all entries of the form $hexid=name or $hexid~name, + remove the =name or ~name portion. + + Remove all entries of the form $hexid, where hexid is not + 40 hexadecimal characters long. + + If an entry is a valid nickname, put it into lower case. + + If an entry is a valid $hexid, put it into upper case. + + If there are any entries, add a single $hexid entry for + the relay in question, so that it is a member of its own + family. + + Sort all entries in lexical order. + + Remove duplicate entries. +``` + +(Note that if an entry is not of the form "nickname", "$hexid", +"$hexid=nickname" or "$hexid~nickname", then it will be unchanged: +this is what makes the algorithm forward-compatible.) + +"p" SP ("accept" / "reject") SP PortList NL + +\[At most once.\] + +The exit-policy summary as specified in sections 3.4.1 and 3.8.2. + +\[With microdescriptors, clients don't learn exact exit policies: +clients can only guess whether a relay accepts their request, try the +BEGIN request, and might get end-reason-exit-policy if they guessed +wrong, in which case they'll have to try elsewhere.\] + +\[In consensus methods before 5, this line was omitted.\] + +"p6" SP ("accept" / "reject") SP PortList NL + +\[At most once\] + +The IPv6 exit policy summary as specified in sections 3.4.1 and +3.8.2. A missing "p6" line is equivalent to "p6 reject 1-65535". + +(Only included when generating microdescriptors for +consensus-method 15 or later.) + +"id" SP "rsa1024" SP base64-encoded-identity-digest NL + +\[At most once\] + +The node identity digest (as described in tor-spec.txt), base64 +encoded, without trailing =s. This line is included to prevent +collisions between microdescriptors. + +Implementations SHOULD ignore these lines: they are +added to microdescriptors only to prevent collisions. + +(Only included when generating microdescriptors for +consensus-method 18 or later.) + +"id" SP "ed25519" SP base64-encoded-ed25519-identity NL + +\[At most once\] + +The node's master Ed25519 identity key, base64 encoded, +without trailing =s. + +All implementations MUST ignore this key for any microdescriptor +whose corresponding entry in the consensus includes the +'NoEdConsensus' flag. + +(Only included when generating microdescriptors for +consensus-method 21 or later.) + +"id" SP keytype ... NL + +\[At most once per distinct keytype.\] + +Implementations MUST ignore "id" lines with unrecognized +key-types in place of "rsa1024" or "ed25519" + +(Note that with microdescriptors, clients do not learn the RSA identity of +their routers: they only learn a hash of the RSA identity key. This is +all they need to confirm the actual identity key when doing a TLS +handshake, and all they need to put the identity key digest in their +CREATE cells.) diff --git a/spec/dir-spec/consensus-formats.md b/spec/dir-spec/consensus-formats.md new file mode 100644 index 0000000..7a6cd46 --- /dev/null +++ b/spec/dir-spec/consensus-formats.md @@ -0,0 +1,706 @@ +<a id="dir-spec.txt-3.4.1"></a> + +# Vote and consensus status document formats + +Votes and consensuses are more strictly formatted than other documents +in this specification, since different authorities must be able to +generate exactly the same consensus given the same set of votes. + +The procedure for deciding when to generate vote and consensus status +documents are described in section 1.4 on the voting timeline. + +Status documents contain a preamble, an authority section, a list of +router status entries, and one or more footer signature, in that order. + +Unlike other formats described above, a SP in these documents must be a +single space character (hex 20). + +Some items appear only in votes, and some items appear only in +consensuses. Unless specified, items occur in both. + +The preamble contains the following items. They SHOULD occur in the +order given here: + +"network-status-version" SP version NL + +\[At start, exactly once.\] + +A document format version. For this specification, the version is +"3". + +"vote-status" SP type NL + +\[Exactly once.\] + +The status MUST be "vote" or "consensus", depending on the type of +the document. + +"consensus-methods" SP IntegerList NL + +\[At most once for votes; does not occur in consensuses.\] + +A space-separated list of supported methods for generating +consensuses from votes. See section 3.8.1 for details. Absence of +the line means that only method "1" is supported. + +"consensus-method" SP Integer NL + +\[At most once for consensuses; does not occur in votes.\] +\[No extra arguments\] + +See section 3.8.1 for details. + +(Only included when the vote is generated with consensus-method 2 or +later.) + +"published" SP YYYY-MM-DD SP HH:MM:SS NL + +\[Exactly once for votes; does not occur in consensuses.\] + +The publication time for this status document (if a vote). + +"valid-after" SP YYYY-MM-DD SP HH:MM:SS NL + +\[Exactly once.\] + +The start of the Interval for this vote. Before this time, the +consensus document produced from this vote is not officially in +use. + +(Note that because of propagation delays, clients and relays +may see consensus documents that are up to `DistSeconds` +earlier than this time, and should not warn about them.) + +See section 1.4 for voting timeline information. + +"fresh-until" SP YYYY-MM-DD SP HH:MM:SS NL + +\[Exactly once.\] + +The time at which the next consensus should be produced; before this +time, there is no point in downloading another consensus, since there +won't be a new one. See section 1.4 for voting timeline information. + +"valid-until" SP YYYY-MM-DD SP HH:MM:SS NL + +\[Exactly once.\] + +The end of the Interval for this vote. After this time, all +clients should try to find a more recent consensus. See section 1.4 +for voting timeline information. + +In practice, clients continue to use the consensus for up to 24 hours +after it is no longer valid, if no more recent consensus can be +downloaded. + +"voting-delay" SP VoteSeconds SP DistSeconds NL + +\[Exactly once.\] + +VoteSeconds is the number of seconds that we will allow to collect +votes from all authorities; DistSeconds is the number of seconds +we'll allow to collect signatures from all authorities. See +section 1.4 for voting timeline information. + +"client-versions" SP VersionList NL + +\[At most once.\] + +A comma-separated list of recommended Tor versions for client +usage, in ascending order. The versions are given as defined by +version-spec.txt. If absent, no opinion is held about client +versions. + +"server-versions" SP VersionList NL + +\[At most once.\] + +A comma-separated list of recommended Tor versions for relay +usage, in ascending order. The versions are given as defined by +version-spec.txt. If absent, no opinion is held about server +versions. + +"package" SP PackageName SP Version SP URL SP DIGESTS NL + +\[Any number of times.\] + +For this element: + +```text + PACKAGENAME = NONSPACE + VERSION = NONSPACE + URL = NONSPACE + DIGESTS = DIGEST | DIGESTS SP DIGEST + DIGEST = DIGESTTYPE "=" DIGESTVAL + NONSPACE = one or more non-space printing characters + DIGESTVAL = DIGESTTYPE = one or more non-space printing characters + other than "=". +``` + +Indicates that a package called "package" of version VERSION may be +found at URL, and its digest as computed with DIGESTTYPE is equal to +DIGESTVAL. In consensuses, these lines are sorted lexically by +"PACKAGENAME VERSION" pairs, and DIGESTTYPES must appear in ascending +order. A consensus must not contain the same "PACKAGENAME VERSION" +more than once. If a vote contains the same "PACKAGENAME VERSION" +more than once, all but the last is ignored. + +Included in consensuses only for methods 19-33. Earlier methods +did not include this; method 34 removed it. + +"known-flags" SP FlagList NL + +\[Exactly once.\] + +A space-separated list of all of the flags that this document +might contain. A flag is "known" either because the authority +knows about them and might set them (if in a vote), or because +enough votes were counted for the consensus for an authoritative +opinion to have been formed about their status. + +"flag-thresholds" SP Thresholds NL + +\[At most once for votes; does not occur in consensuses.\] + +```text + A space-separated list of the internal performance thresholds + that the directory authority had at the moment it was forming + a vote. + + The metaformat is: + Thresholds = Threshold | Threshold SP Thresholds + Threshold = ThresholdKey '=' ThresholdVal + ThresholdKey = (KeywordChar | "_") + + ThresholdVal = [0-9]+("."[0-9]+)? "%"? + + Commonly used Thresholds at this point include: + + "stable-uptime" -- Uptime (in seconds) required for a relay + to be marked as stable. + + "stable-mtbf" -- MTBF (in seconds) required for a relay to be + marked as stable. + + "enough-mtbf" -- Whether we have measured enough MTBF to look + at stable-mtbf instead of stable-uptime. + + "fast-speed" -- Bandwidth (in bytes per second) required for + a relay to be marked as fast. + + "guard-wfu" -- WFU (in seconds) required for a relay to be + marked as guard. + + "guard-tk" -- Weighted Time Known (in seconds) required for a + relay to be marked as guard. + + "guard-bw-inc-exits" -- If exits can be guards, then all guards + must have a bandwidth this high. + + "guard-bw-exc-exits" -- If exits can't be guards, then all guards + must have a bandwidth this high. + + "ignoring-advertised-bws" -- 1 if we have enough measured bandwidths + that we'll ignore the advertised bandwidth + claims of routers without measured bandwidth. +``` + +"recommended-client-protocols" SP Entries NL +"recommended-relay-protocols" SP Entries NL +"required-client-protocols" SP Entries NL +"required-relay-protocols" SP Entries NL + +\[At most once for each.\] + +The "proto" element as specified in section 2.1.1. + +To vote on these entries, a protocol/version combination is included +only if it is listed by a majority of the voters. + +These lines should be voted on. A majority of votes is sufficient to +make a protocol un-supported. A supermajority of authorities (2/3) +are needed to make a protocol required. The required protocols +should not be torrc-configurable, but rather should be hardwired in +the Tor code. + +The tor-spec.txt section 9 details how a relay and a client should +behave when they encounter these lines in the consensus. + +"params" SP \[Parameters\] NL + +\[At most once\] + +Parameter ::= Keyword '=' Int32 +Int32 ::= A decimal integer between -2147483648 and 2147483647. +Parameters ::= Parameter | Parameters SP Parameter + +The parameters list, if present, contains a space-separated list of +case-sensitive key-value pairs, sorted in lexical order by their +keyword (as ASCII byte strings). Each parameter has its own meaning. + +(Only included when the vote is generated with consensus-method 7 or +later.) + +See param-spec.txt for a list of parameters and their meanings. + +"shared-rand-previous-value" SP NumReveals SP Value NL + +\[At most once\] + +NumReveals ::= An integer greater or equal to 0. +Value ::= Base64-encoded-data + +The shared_random_value that was generated during the second-to-last +shared randomness protocol run. For example, if this document was +created on the 5th of November, this field carries the shared random +value generated during the protocol run of the 3rd of November. + +See section \[SRCALC\] of srv-spec.txt for instructions on how to compute +this value, and see section \[CONS\] for why we include old shared random +values in votes and consensus. + +Value is the actual shared random value encoded in base64. It will +be exactly 256 bits long. NumReveals is the number of commits used +to generate this SRV. + +"shared-rand-current-value" SP NumReveals SP Value NL + +\[At most once\] + +NumReveals ::= An integer greater or equal to 0. +Value ::= Base64-encoded-data + +The shared_random_value that was generated during the latest shared +randomness protocol run. For example, if this document was created on +the 5th of November, this field carries the shared random value +generated during the protocol run of the 4th of November + +See section \[SRCALC\] of srv-spec.txt for instructions on how to compute +this value given the active commits. + +Value is the actual shared random value encoded in base64. It will +be exactly 256 bits long. NumReveals is the number of commits used to +generate this SRV. + +"bandwidth-file-headers" SP KeyValues NL + +\[At most once for votes; does not occur in consensuses.\] + +KeyValues ::= "" | KeyValue | KeyValues SP KeyValue +KeyValue ::= Keyword '=' Value +Value ::= ArgumentCharValue+ +ArgumentCharValue ::= any printing ASCII character except NL and SP. + +The headers from the bandwidth file used to generate this vote. +The bandwidth file headers are described in bandwidth-file-spec.txt. + +If an authority is not configured with a V3BandwidthsFile, this line +SHOULD NOT appear in its vote. + +If an authority is configured with a V3BandwidthsFile, but parsing +fails, this line SHOULD appear in its vote, but without any headers. + +First-appeared: Tor 0.3.5.1-alpha. + +"bandwidth-file-digest" 1\*(SP algorithm "=" digest) NL + +\[At most once for votes; does not occur in consensuses.\] + +A digest of the bandwidth file used to generate this vote. +"algorithm" is the name of the hash algorithm producing "digest", +which can be "sha256" or another algorithm. "digest" is the +base64 encoding of the hash of the bandwidth file, with trailing =s +omitted. + +If an authority is not configured with a V3BandwidthsFile, this line +SHOULD NOT appear in its vote. + +If an authority is configured with a V3BandwidthsFile, but parsing +fails, this line SHOULD appear in its vote, with the digest(s) of the +unparseable file. + +First-appeared: Tor 0.4.0.4-alpha + +The authority section of a vote contains the following items, followed +in turn by the authority's current key certificate: + +```text + "dir-source" SP nickname SP identity SP address SP IP SP dirport SP + orport NL + + [Exactly once, at start] +``` + +Describes this authority. The nickname is a convenient identifier +for the authority. The identity is an uppercase hex fingerprint of +the authority's current (v3 authority) identity key. The address is +the server's hostname. The IP is the server's current IP address, +and dirport is its current directory port. The orport is the +port at that address where the authority listens for OR +connections. + +"contact" SP string NL + +\[Exactly once\] + +An arbitrary string describing how to contact the directory +server's administrator. Administrators should include at least an +email address and a PGP fingerprint. + +"legacy-dir-key" SP FINGERPRINT NL + +\[At most once\] + +Lists a fingerprint for an obsolete _identity_ key still used +by this authority to keep older clients working. This option +is used to keep key around for a little while in case the +authorities need to migrate many identity keys at once. +(Generally, this would only happen because of a security +vulnerability that affected multiple authorities, like the +Debian OpenSSL RNG bug of May 2008.) + +"shared-rand-participate" NL + +\[At most once\] + +Denotes that the directory authority supports and can participate in the +shared random protocol. + +"shared-rand-commit" SP Version SP AlgName SP Identity SP Commit \[SP Reveal\] NL + +\[Any number of times\] + +Version ::= An integer greater or equal to 0. +AlgName ::= 1\*(ALPHA / DIGIT / "\_" / "-") +Identity ::= 40\* HEXDIG +Commit ::= Base64-encoded-data +Reveal ::= Base64-encoded-data + +Denotes a directory authority commit for the shared randomness +protocol, containing the commitment value and potentially also the +reveal value. See sections \[COMMITREVEAL\] and \[VALIDATEVALUES\] of +srv-spec.txt on how to generate and validate these values. + +Version is the current shared randomness protocol version. AlgName is +the hash algorithm that is used (e.g. "sha3-256") and Identity is the +authority's SHA1 v3 identity fingerprint. Commit is the encoded +commitment value in base64. Reveal is optional and if it's set, it +contains the reveal value in base64. + +If a vote contains multiple commits from the same authority, the +receiver MUST only consider the first commit listed. + +"shared-rand-previous-value" SP NumReveals SP Value NL + +\[At most once\] + +See shared-rand-previous-value description above. + +"shared-rand-current-value" SP NumReveals SP Value NL + +\[At most once\] + +See shared-rand-current-value description above. + +The authority section of a consensus contains groups of the following items, +in the order given, with one group for each authority that contributed to +the consensus, with groups sorted by authority identity digest: + +```text + "dir-source" SP nickname SP identity SP address SP IP SP dirport SP + orport NL + + [Exactly once, at start] + + As in the authority section of a vote. + + "contact" SP string NL + + [Exactly once.] + + As in the authority section of a vote. + + "vote-digest" SP digest NL + + [Exactly once.] +``` + +A digest of the vote from the authority that contributed to this +consensus, as signed (that is, not including the signature). +(Hex, upper-case.) + +For each "legacy-dir-key" in the vote, there is an additional "dir-source" +line containing that legacy key's fingerprint, the authority's nickname +with "-legacy" appended, and all other fields as in the main "dir-source" +line for that authority. These "dir-source" lines do not have +corresponding "contact" or "vote-digest" entries. + +Each router status entry contains the following items. Router status +entries are sorted in ascending order by identity digest. + +```text + "r" SP nickname SP identity SP digest SP publication SP IP SP ORPort + SP DirPort NL + + [At start, exactly once.] +``` + +"Nickname" is the OR's nickname. "Identity" is a hash of its +identity key, encoded in base64, with trailing equals sign(s) +removed. "Digest" is a hash of its most recent descriptor as +signed (that is, not including the signature) by the RSA identity +key (see section 1.3.), encoded in base64. + +"Publication" was once the publication time of the router's most +recent descriptor, in the form YYYY-MM-DD HH:MM:SS, in UTC. Now +it is only used in votes, and may be set to a fixed value in +consensus documents. Implementations SHOULD ignore this value +in non-vote documents. + +"IP" is its current IP address; ORPort is its current OR port, +"DirPort" is its current directory port, or "0" for "none". + +"a" SP address ":" port NL + +\[Any number\] + +The first advertised IPv6 address for the OR, if it is reachable. + +Present only if the OR advertises at least one IPv6 address, and the +authority believes that the first advertised address is reachable. +Any other IPv4 or IPv6 addresses should be ignored. + +Address and port are as for "or-address" as specified in +section 2.1.1. + +(Only included when the vote or consensus is generated with +consensus-method 14 or later.) + +"s" SP Flags NL + +\[Exactly once.\] + +A series of space-separated status flags, in lexical order (as ASCII +byte strings). Currently documented flags are: + +```text + "Authority" if the router is a directory authority. + "BadExit" if the router is believed to be useless as an exit node + (because its ISP censors it, because it is behind a restrictive + proxy, or for some similar reason). + "Exit" if the router is more useful for building + general-purpose exit circuits than for relay circuits. The + path building algorithm uses this flag; see path-spec.txt. + "Fast" if the router is suitable for high-bandwidth circuits. + "Guard" if the router is suitable for use as an entry guard. + "HSDir" if the router is considered a v2 hidden service directory. + "MiddleOnly" if the router is considered unsuitable for + usage other than as a middle relay. Clients do not need + to handle this option, since when it is present, the authorities + will automatically vote against flags that would make the router + usable in other positions. (Since 0.4.7.2-alpha.) + "NoEdConsensus" if any Ed25519 key in the router's descriptor or + microdescriptor does not reflect authority consensus. + "Stable" if the router is suitable for long-lived circuits. + "StaleDesc" if the router should upload a new descriptor because + the old one is too old. + "Running" if the router is currently usable over all its published + ORPorts. (Authorities ignore IPv6 ORPorts unless configured to + check IPv6 reachability.) Relays without this flag are omitted + from the consensus, and current clients (since 0.2.9.4-alpha) + assume that every listed relay has this flag. + "Valid" if the router has been 'validated'. Clients before + 0.2.9.4-alpha would not use routers without this flag by + default. Currently, relays without this flag are omitted + from the consensus, and current (post-0.2.9.4-alpha) clients + assume that every listed relay has this flag. + "V2Dir" if the router implements the v2 directory protocol or + higher. + + "v" SP version NL + + [At most once.] +``` + +The version of the Tor protocol that this relay is running. If +the value begins with "Tor" SP, the rest of the string is a Tor +version number, and the protocol is "The Tor protocol as supported +by the given version of Tor." Otherwise, if the value begins with +some other string, Tor has upgraded to a more sophisticated +protocol versioning system, and the protocol is "a version of the +Tor protocol more recent than any we recognize." + +Directory authorities SHOULD omit version strings they receive from +descriptors if they would cause "v" lines to be over 128 characters +long. + +"pr" SP Entries NL + +\[At most once.\] + +The "proto" family element as specified in section 2.1.1. + +During voting, authorities copy these lines immediately below the "v" +lines. When a descriptor does not contain a "proto" entry, the +authorities should reconstruct it using the approach described below +in section D. They are included in the consensus using the same rules +as currently used for "v" lines, if a sufficiently late consensus +method is in use. + +"w" SP "Bandwidth=" INT \[SP "Measured=" INT\] \[SP "Unmeasured=1"\] NL + +\[At most once.\] + +An estimate of the bandwidth of this relay, in an arbitrary +unit (currently kilobytes per second). Used to weight router +selection. See section 3.4.2 for details on how the value of +Bandwidth is determined in a consensus. + +Additionally, the Measured= keyword is present in votes by +participating bandwidth measurement authorities to indicate +a measured bandwidth currently produced by measuring stream +capacities. It does not occur in consensuses. + +'Bandwidth=' and 'Measured=' values must be between 0 and +2^32 - 1 inclusive. + +The "Unmeasured=1" value is included in consensuses generated +with method 17 or later when the 'Bandwidth=' value is not +based on a threshold of 3 or more measurements for this relay. + +Other weighting keywords may be added later. +Clients MUST ignore keywords they do not recognize. + +"p" SP ("accept" / "reject") SP PortList NL + +\[At most once.\] + +PortList = PortOrRange +PortList = PortList "," PortOrRange +PortOrRange = INT "-" INT / INT + +A list of those ports that this router supports (if 'accept') +or does not support (if 'reject') for exit to "most +addresses". + +"m" SP methods 1\*(SP algorithm "=" digest) NL + +\[Any number, only in votes.\] + +Microdescriptor hashes for all consensus methods that an authority +supports and that use the same microdescriptor format. "methods" +is a comma-separated list of the consensus methods that the +authority believes will produce "digest". "algorithm" is the name +of the hash algorithm producing "digest", which can be "sha256" or +something else, depending on the consensus "methods" supporting +this algorithm. "digest" is the base64 encoding of the hash of +the router's microdescriptor with trailing =s omitted. + +```text + "id" SP "ed25519" SP ed25519-identity NL + "id" SP "ed25519" SP "none" NL + [vote only, at most once] + + "stats" SP [KeyValues] NL + + [At most once. Vote only] +``` + +KeyValue ::= Keyword '=' Number +Number ::= \[0-9\]+("."\[0-9\]+)? +KeyValues ::= KeyValue | KeyValues SP KeyValue + +Line containing various statistics that an authority has computed for +this relay. Each stats is represented as a key + value. Reported keys +are: + +```text + "wfu" - Weighted Fractional Uptime + "tk" - Weighted Time Known + "mtbf" - Mean Time Between Failure (stability) + + (As of tor-0.4.6.1-alpha) +``` + +The footer section is delineated in all votes and consensuses supporting +consensus method 9 and above with the following: + +"directory-footer" NL +\[No extra arguments\] + +It contains two subsections, a bandwidths-weights line and a +directory-signature. (Prior to consensus method 9, footers only contained +directory-signatures without a 'directory-footer' line or +bandwidth-weights.) + +The bandwidths-weights line appears At Most Once for a consensus. It does +not appear in votes. + +"bandwidth-weights" \[SP Weights\] NL + +Weight ::= Keyword '=' Int32 +Int32 ::= A decimal integer between -2147483648 and 2147483647. +Weights ::= Weight | Weights SP Weight + +List of optional weights to apply to router bandwidths during path +selection. They are sorted in lexical order (as ASCII byte strings) and +values are divided by the consensus' "bwweightscale" param. Definition +of our known entries are... + +```text + Wgg - Weight for Guard-flagged nodes in the guard position + Wgm - Weight for non-flagged nodes in the guard Position + Wgd - Weight for Guard+Exit-flagged nodes in the guard Position + + Wmg - Weight for Guard-flagged nodes in the middle Position + Wmm - Weight for non-flagged nodes in the middle Position + Wme - Weight for Exit-flagged nodes in the middle Position + Wmd - Weight for Guard+Exit flagged nodes in the middle Position + + Weg - Weight for Guard flagged nodes in the exit Position + Wem - Weight for non-flagged nodes in the exit Position + Wee - Weight for Exit-flagged nodes in the exit Position + Wed - Weight for Guard+Exit-flagged nodes in the exit Position + + Wgb - Weight for BEGIN_DIR-supporting Guard-flagged nodes + Wmb - Weight for BEGIN_DIR-supporting non-flagged nodes + Web - Weight for BEGIN_DIR-supporting Exit-flagged nodes + Wdb - Weight for BEGIN_DIR-supporting Guard+Exit-flagged nodes + + Wbg - Weight for Guard flagged nodes for BEGIN_DIR requests + Wbm - Weight for non-flagged nodes for BEGIN_DIR requests + Wbe - Weight for Exit-flagged nodes for BEGIN_DIR requests + Wbd - Weight for Guard+Exit-flagged nodes for BEGIN_DIR requests + + These values are calculated as specified in section 3.8.3. +``` + +The signature contains the following item, which appears Exactly Once +for a vote, and At Least Once for a consensus. + +```text + "directory-signature" [SP Algorithm] SP identity SP signing-key-digest + NL Signature +``` + +This is a signature of the status document, with the initial item +"network-status-version", and the signature item +"directory-signature", using the signing key. (In this case, we take +the hash through the _space_ after directory-signature, not the +newline: this ensures that all authorities sign the same thing.) +"identity" is the hex-encoded digest of the authority identity key of +the signing authority, and "signing-key-digest" is the hex-encoded +digest of the current authority signing key of the signing authority. + +The Algorithm is one of "sha1" or "sha256" if it is present; +implementations MUST ignore directory-signature entries with an +unrecognized Algorithm. "sha1" is the default, if no Algorithm is +given. The algorithm describes how to compute the hash of the +document before signing it. + +"ns"-flavored consensus documents must contain only sha1 signatures. +Votes and microdescriptor documents may contain other signature +types. Note that only one signature from each authority should be +"counted" as meaning that the authority has signed the consensus. + +(Tor clients before 0.2.3.x did not understand the 'algorithm' +field.) diff --git a/spec/dir-spec/consensus-negotiation-timeline.md b/spec/dir-spec/consensus-negotiation-timeline.md new file mode 100644 index 0000000..59f9ced --- /dev/null +++ b/spec/dir-spec/consensus-negotiation-timeline.md @@ -0,0 +1,16 @@ +<a id="dir-spec.txt-A"></a> + +# Consensus-negotiation timeline + +```text + Period begins: this is the Published time. + Everybody sends votes + Reconciliation: everybody tries to fetch missing votes. + consensus may exist at this point. + End of voting period: + everyone swaps signatures. + Now it's okay for caches to download + Now it's okay for clients to download. + + Valid-after/valid-until switchover +``` diff --git a/spec/dir-spec/converting-to-ed25519.md b/spec/dir-spec/converting-to-ed25519.md new file mode 100644 index 0000000..57f9d1d --- /dev/null +++ b/spec/dir-spec/converting-to-ed25519.md @@ -0,0 +1,77 @@ +<a id="dir-spec.txt-C"></a> + +# Converting a curve25519 public key to an ed25519 public key + +Given an X25519 key, that is, an affine point (u,v) on the +Montgomery curve defined by + +bv^2 = u(u^2 + au +1) + +where + +```text + a = 486662 + b = 1 +``` + +and comprised of the compressed form (i.e. consisting of only the +u-coordinate), we can retrieve the y-coordinate of the affine point +(x,y) on the twisted Edwards form of the curve defined by + +-x^2 + y^2 = 1 + d x^2 y^2 + +where + +d = - 121665/121666 + +by computing + +y = (u-1)/(u+1). + +and then we can apply the usual curve25519 twisted Edwards point +decompression algorithm to find _an_ x-coordinate of an affine +twisted Edwards point to check signatures with. Signing keys for +ed25519 are compressed curve points in twisted Edwards form (so a +y-coordinate and the sign of the x-coordinate), and X25519 keys are +compressed curve points in Montgomery form (i.e. a u-coordinate). + +However, note that compressed point in Montgomery form neglects to +encode what the sign of the corresponding twisted Edwards +x-coordinate would be. Thus, we need the sign of the x-coordinate +to do this operation; otherwise, we'll have two possible +x-coordinates that might have correspond to the ed25519 public key. + +To get the sign, the easiest way is to take the corresponding +private key, feed it to the ed25519 public key generation +algorithm, and see what the sign is. + +\[Recomputing the sign bit from the private key every time sounds +rather strange and inefficient to me… —isis\] + +Note that in addition to its coordinates, an expanded Ed25519 private key +also has a 32-byte random value, "prefix", used to compute internal `r` +values in the signature. For security, this prefix value should be +derived deterministically from the curve25519 key. The Tor +implementation derives it as `SHA512(private_key | STR)[0..32]`, where +STR is the nul-terminated string: + +"Derive high part of ed25519 key from curve25519 key\\0" + +On the client side, where there is no access to the curve25519 private +keys, one may use the curve25519 public key's Montgomery u-coordinate to +recover the Montgomery v-coordinate by computing the right-hand side of +the Montgomery curve equation: + +bv^2 = u(u^2 + au +1) + +where + +```text + a = 486662 + b = 1 +``` + +Then, knowing the intended sign of the Edwards x-coordinate, one +may recover said x-coordinate by computing: + +x = (u/v) * sqrt(-a - 2) diff --git a/spec/dir-spec/creating-key-certificates.md b/spec/dir-spec/creating-key-certificates.md new file mode 100644 index 0000000..fc3326f --- /dev/null +++ b/spec/dir-spec/creating-key-certificates.md @@ -0,0 +1,90 @@ +<a id="dir-spec.txt-3.1"></a> + +# Creating key certificates + +Key certificates consist of the following items: + +"dir-key-certificate-version" version NL + +\[At start, exactly once.\] + +Determines the version of the key certificate. MUST be "3" for +the protocol described in this document. Implementations MUST +reject formats they don't understand. + +```text + "dir-address" IPPort NL + [At most once] + + An IP:Port for this authority's directory port. + + "fingerprint" fingerprint NL + + [Exactly once.] +``` + +Hexadecimal encoding without spaces based on the authority's +identity key. + +"dir-identity-key" NL a public key in PEM format + +\[Exactly once.\] +\[No extra arguments\] + +The long-term authority identity key for this authority. This key +SHOULD be at least 2048 bits long; it MUST NOT be shorter than +1024 bits. + +"dir-key-published" YYYY-MM-DD HH:MM:SS NL + +\[Exactly once.\] + +The time (in UTC) when this document and corresponding key were +last generated. + +Implementations SHOULD reject certificates that are published +too far in the future, though they MAY tolerate some clock skew. + +"dir-key-expires" YYYY-MM-DD HH:MM:SS NL + +\[Exactly once.\] + +A time (in UTC) after which this key is no longer valid. + +Implementations SHOULD reject expired certificates, though they +MAY tolerate some clock skew. + +"dir-signing-key" NL a key in PEM format + +\[Exactly once.\] +\[No extra arguments\] + +The directory server's public signing key. This key MUST be at +least 1024 bits, and MAY be longer. + +"dir-key-crosscert" NL CrossSignature NL + +\[Exactly once.\] +\[No extra arguments\] + +CrossSignature is a signature, made using the certificate's signing +key, of the digest of the PKCS1-padded hash of the certificate's +identity key. For backward compatibility with broken versions of the +parser, we wrap the base64-encoded signature in -----BEGIN ID +SIGNATURE---- and -----END ID SIGNATURE----- tags. Implementations +MUST allow the "ID " portion to be omitted, however. + +Implementations MUST verify that the signature is a correct signature +of the hash of the identity key using the signing key. + +"dir-key-certification" NL Signature NL + +\[At end, exactly once.\] +\[No extra arguments\] + +A document signature as documented in section 1.3, using the +initial item "dir-key-certificate-version" and the final item +"dir-key-certification", signed with the authority identity key. + +Authorities MUST generate a new signing key and corresponding +certificate before the key expires. diff --git a/spec/dir-spec/directory-authority-operation-formats.md b/spec/dir-spec/directory-authority-operation-formats.md new file mode 100644 index 0000000..bd487f6 --- /dev/null +++ b/spec/dir-spec/directory-authority-operation-formats.md @@ -0,0 +1,10 @@ +<a id="dir-spec.txt-3"></a> + +# Directory authority operation and formats + +Every authority has two keys used in this protocol: a signing key, and +an authority identity key. (Authorities also have a router identity +key used in their role as a router and by earlier versions of the +directory protocol.) The identity key is used from time to time to +sign new key certificates using new signing keys; it is very sensitive. +The signing key is used to sign key certificates and status documents. diff --git a/spec/dir-spec/directory-cache-operation.md b/spec/dir-spec/directory-cache-operation.md new file mode 100644 index 0000000..14f9a9c --- /dev/null +++ b/spec/dir-spec/directory-cache-operation.md @@ -0,0 +1,167 @@ +<a id="dir-spec.txt-4"></a> + +# Directory cache operation + +All directory caches implement this section, except as noted. + +<a id="dir-spec.txt-4.1"></a> + +## Downloading consensus status documents from directory authorities { #download-ns-from-auth } + +All directory caches try to keep a recent +network-status consensus document to serve to clients. A cache ALWAYS +downloads a network-status consensus if any of the following are true: + +- The cache has no consensus document. +- The cache's consensus document is no longer valid. + +Otherwise, the cache downloads a new consensus document at a randomly +chosen time in the first half-interval after its current consensus +stops being fresh. (This time is chosen at random to avoid swarming +the authorities at the start of each period. The interval size is +inferred from the difference between the valid-after time and the +fresh-until time on the consensus.) + +```text + [For example, if a cache has a consensus that became valid at 1:00, + and is fresh until 2:00, that cache will fetch a new consensus at + a random time between 2:00 and 2:30.] +``` + +Directory caches also fetch consensus flavors from the authorities. +Caches check the correctness of consensus flavors, but do not check +anything about an unrecognized consensus document beyond its digest and +length. Caches serve all consensus flavors from the same locations as +the directory authorities. + +<a id="dir-spec.txt-4.2"></a> + +## Downloading server descriptors from directory authorities { #download-desc-from-auth } + +Periodically (currently, every 10 seconds), directory caches check +whether there are any specific descriptors that they do not have and that +they are not currently trying to download. Caches identify these +descriptors by hash in the recent network-status consensus documents. + +If so, the directory cache launches requests to the authorities for these +descriptors. + +If one of these downloads fails, we do not try to download that descriptor +from the authority that failed to serve it again unless we receive a newer +network-status consensus that lists the same descriptor. + +Directory caches must potentially cache multiple descriptors for each +router. Caches must not discard any descriptor listed by any recent +consensus. If there is enough space to store additional descriptors, +caches SHOULD try to hold those which clients are likely to download the +most. (Currently, this is judged based on the interval for which each +descriptor seemed newest.) + +\[XXXX define recent\] + +<a id="dir-spec.txt-4.3"></a> + +## Downloading microdescriptors from directory authorities { #download-md-from-auth } + +Directory mirrors should fetch, cache, and serve each microdescriptor +from the authorities. + +The microdescriptors with base64 hashes `<D1>`, `<D2>`, `<D3>` are available +at: + +`http://<hostname>/tor/micro/d/<D1>-<D2>-<D3>[.z]` + +`<Dn>` are base64 encoded with trailing =s omitted for size and for +consistency with the microdescriptor consensus format. -s are used +instead of +s to separate items, since the + character is used in +base64 encoding. + +Directory mirrors should check to make sure that the microdescriptors +they're about to serve match the right hashes (either the hashes from +the fetch URL or the hashes from the consensus, respectively). + +(NOTE: Due to squid proxy url limitations at most 92 microdescriptor hashes +can be retrieved in a single request.) + +<a id="dir-spec.txt-4.4"></a> + +## Downloading extra-info documents from directory authorities { #download-ei-from-auth } + +Any cache that chooses to cache extra-info documents should implement this +section. + +Periodically, the Tor instance checks whether it is missing any extra-info +documents: in other words, if it has any server descriptors with an +extra-info-digest field that does not match any of the extra-info +documents currently held. If so, it downloads whatever extra-info +documents are missing. Caches download from authorities. We follow the +same splitting and back-off rules as in section 4.2. + +<a id="dir-spec.txt-4.5"></a> + +## Consensus diffs { #diffs } + +Instead of downloading an entire consensus, clients may download +a "diff" document containing an ed-style diff from a previous +consensus document. Caches (and authorities) make these diffs as +they learn about new consensuses. To do so, they must store a +record of older consensuses. + +(Support for consensus diffs was added in 0.3.1.1-alpha, and is +advertised with the DirCache protocol version "2" or later.) + +<a id="dir-spec.txt-4.5.1"></a> + +### Consensus diff format { #diff-format } + +Consensus diffs are formatted as follows: + +The first line is "network-status-diff-version 1" NL + +The second line is + +"hash" SP FromDigest SP ToDigest NL + +where FromDigest is the hex-encoded SHA3-256 digest of the _signed +part_ of the consensus that the diff should be applied to, and +ToDigest is the hex-encoded SHA3-256 digest of the _entire_ +consensus resulting from applying the diff. (See 3.4.1 for +information on that part of a consensus is signed.) + +The third and subsequent lines encode the diff from FromDigest to +ToDigest in a limited subset of the ed diff format, as specified +in appendix E. + +<a id="dir-spec.txt-4.5.2"></a> + +### Serving and requesting diffs { #diff-requests } + +When downloading the current consensus, a client may include an +HTTP header of the form + +X-Or-Diff-From-Consensus: HASH1, HASH2, ... + +where the HASH values are hex-encoded SHA3-256 digests of the +_signed part_ of one or more consensuses that the client knows +about. + +If a cache knows a consensus diff from one of those consensuses +to the most recent consensus of the requested flavor, it may +send that diff instead of the specified consensus. + +Caches also serve diffs from the URIs: + +```text +/tor/status-vote/current/consensus/diff/<HASH>/<FPRLIST>.z +/tor/status-vote/current/consensus-<FLAVOR>/diff/<HASH>/<FPRLIST>.z +``` + +where FLAVOR is the consensus flavor, defaulting to "ns", and +FPRLIST is +-separated list of recognized authority identity +fingerprints as in appendix B. + +<a id="dir-spec.txt-4.6"></a> + +## Retrying failed downloads { #retry-as-cache } + +See section 5.5 below; it applies to caches as well as clients. diff --git a/spec/dir-spec/downloading-from-other-auths.md b/spec/dir-spec/downloading-from-other-auths.md new file mode 100644 index 0000000..8f474e9 --- /dev/null +++ b/spec/dir-spec/downloading-from-other-auths.md @@ -0,0 +1,54 @@ +# Downloading information from other directory authorities + +## Downloading missing certificates from other directory authorities { #download-missing-certs } + +XXX when to download certificates. + +<a id="dir-spec.txt-3.6"></a> + +## Downloading server descriptors from other directory authorities { #download-missing-descs } + +Periodically (currently, every 10 seconds), directory authorities check +whether there are any specific descriptors that they do not have and that +they are not currently trying to download. +Authorities identify them by hash in vote (if publication date is more +recent than the descriptor we currently have). + +\[XXXX need a way to fetch descriptors ahead of the vote? v2 status docs can +do that for now.\] + +If so, the directory authority launches requests to the authorities for these +descriptors, such that each authority is only asked for descriptors listed +in its most recent vote. If more +than one authority lists the descriptor, we choose which to ask at random. + +If one of these downloads fails, we do not try to download that descriptor +from the authority that failed to serve it again unless we receive a newer +network-status (consensus or vote) from that authority that lists the same +descriptor. + +```text + Directory authorities must potentially cache multiple descriptors for each + router. Authorities must not discard any descriptor listed by any recent + consensus. If there is enough space to store additional descriptors, + authorities SHOULD try to hold those which clients are likely to download the + most. (Currently, this is judged based on the interval for which each + descriptor seemed newest.) +[XXXX define recent] +``` + +Authorities SHOULD NOT download descriptors for routers that they would +immediately reject for reasons listed in section 3.2. + +<a id="dir-spec.txt-3.7"></a> + +## Downloading extra-info documents from other directory authorities { #download-missing-extrainfo } + +Periodically, an authority checks whether it is missing any extra-info +documents: in other words, if it has any server descriptors with an +extra-info-digest field that does not match any of the extra-info +documents currently held. If so, it downloads whatever extra-info +documents are missing. We follow the same splitting and back-off rules +as in section 3.6. + +<a id="dir-spec.txt-3.8"></a> diff --git a/spec/dir-spec/exchanging-detached-signatures.md b/spec/dir-spec/exchanging-detached-signatures.md new file mode 100644 index 0000000..7a8495e --- /dev/null +++ b/spec/dir-spec/exchanging-detached-signatures.md @@ -0,0 +1,77 @@ +# Exchanging detached signatures + +Once an authority has computed and signed a consensus network status, it +should send its detached signature to each other authority in an HTTP POST +request to the URL: + +`http://<hostname>/tor/post/consensus-signature` + +\[XXX Note why we support push-and-then-pull.\] + +All of the detached signatures it knows for consensus status should be +available at: + +`http://<hostname>/tor/status-vote/next/consensus-signatures.z` + +Assuming full connectivity, every authority should compute and sign the +same consensus including any flavors in each period. Therefore, it +isn't necessary to download the consensus or any flavors of it computed +by each authority; instead, the authorities only push/fetch each +others' signatures. A "detached signature" document contains items as +follows: + +"consensus-digest" SP Digest NL + +\[At start, at most once.\] + +The digest of the consensus being signed. + +"valid-after" SP YYYY-MM-DD SP HH:MM:SS NL +"fresh-until" SP YYYY-MM-DD SP HH:MM:SS NL +"valid-until" SP YYYY-MM-DD SP HH:MM:SS NL + +\[As in the consensus\] + +"additional-digest" SP flavor SP algname SP digest NL + +\[Any number.\] + +For each supported consensus flavor, every directory authority +adds one or more "additional-digest" lines. "flavor" is the name +of the consensus flavor, "algname" is the name of the hash +algorithm that is used to generate the digest, and "digest" is the +hex-encoded digest. + +The hash algorithm for the microdescriptor consensus flavor is +defined as SHA256 with algname "sha256". + +```text + "additional-signature" SP flavor SP algname SP identity SP + signing-key-digest NL signature. + + [Any number.] +``` + +For each supported consensus flavor and defined digest algorithm, +every directory authority adds an "additional-signature" line. +"flavor" is the name of the consensus flavor. "algname" is the +name of the algorithm that was used to hash the identity and +signing keys, and to compute the signature. "identity" is the +hex-encoded digest of the authority identity key of the signing +authority, and "signing-key-digest" is the hex-encoded digest of +the current authority signing key of the signing authority. + +The "sha256" signature format is defined as the RSA signature of +the OAEP+-padded SHA256 digest of the item to be signed. When +checking signatures, the signature MUST be treated as valid if the +signature material begins with SHA256(document), so that other +data can get added later. +\[To be honest, I didn't fully understand the previous paragraph +and only copied it from the proposals. Review carefully. -KL\] + +"directory-signature" + +\[As in the consensus; the signature object is the same as in the +consensus document.\] + +<a id="dir-spec.txt-3.11"></a> diff --git a/spec/dir-spec/exchanging-votes.md b/spec/dir-spec/exchanging-votes.md new file mode 100644 index 0000000..924aa67 --- /dev/null +++ b/spec/dir-spec/exchanging-votes.md @@ -0,0 +1,53 @@ +<a id="dir-spec.txt-3.4"></a> + +# Exchanging votes + +Authorities divide time into Intervals. Authority administrators SHOULD +try to all pick the same interval length, and SHOULD pick intervals that +are commonly used divisions of time (e.g., 5 minutes, 15 minutes, 30 +minutes, 60 minutes, 90 minutes). Voting intervals SHOULD be chosen to +divide evenly into a 24-hour day. + +Authorities SHOULD act according to interval and delays in the +latest consensus. Lacking a latest consensus, they SHOULD default to a +30-minute Interval, a 5 minute VotingDelay, and a 5 minute DistDelay. + +Authorities MUST take pains to ensure that their clocks remain accurate +within a few seconds. (Running NTP is usually sufficient.) + +The first voting period of each day begins at 00:00 (midnight) UTC. If +the last period of the day would be truncated by one-half or more, it is +merged with the second-to-last period. + +An authority SHOULD publish its vote immediately at the start of each voting +period (minus VoteSeconds+DistSeconds). It does this by making it +available at + +`http://<hostname>/tor/status-vote/next/authority.z` + +and sending it in an HTTP POST request to each other authority at the URL + +`http://<hostname>/tor/post/vote` + +If, at the start of the voting period, minus DistSeconds, an authority +does not have a current statement from another authority, the first +authority downloads the other's statement. + +Once an authority has a vote from another authority, it makes it available +at + +`http://<hostname>/tor/status-vote/next/<fp>.z` + +where `<fp>` is the fingerprint of the other authority's identity key. +And at + +`http://<hostname>/tor/status-vote/next/d/<d>.z` + +where `<d>` is the digest of the vote document. + +Also, once an authority receives a vote from another authority, it +examines it for new descriptors and fetches them from that authority. +This may be the only way for an authority to hear about relays that didn't +publish their descriptor to all authorities, and, while it's too late +for the authority to include relays in its current vote, it can include +them in its next vote. See section 3.6 below for details. diff --git a/spec/dir-spec/extra-info-document-format.md b/spec/dir-spec/extra-info-document-format.md new file mode 100644 index 0000000..0defe70 --- /dev/null +++ b/spec/dir-spec/extra-info-document-format.md @@ -0,0 +1,591 @@ +<a id="dir-spec.txt-2.1.2"></a> + +# Extra-info document format + +Extra-info documents consist of the following items: + +```text + "extra-info" Nickname Fingerprint NL + [At start, exactly once.] +``` + +Identifies what router this is an extra-info descriptor for. +Fingerprint is encoded in hex (using upper-case letters), with +no spaces. + +```text + "identity-ed25519" + [As in router descriptors] + + "published" YYYY-MM-DD HH:MM:SS NL + + [Exactly once.] +``` + +The time, in UTC, when this document (and its corresponding router +descriptor if any) was generated. It MUST match the published time +in the corresponding server descriptor. + +```text + "read-history" YYYY-MM-DD HH:MM:SS (NSEC s) NUM,NUM,NUM,NUM,NUM... NL + [At most once.] + "write-history" YYYY-MM-DD HH:MM:SS (NSEC s) NUM,NUM,NUM,NUM,NUM... NL + [At most once.] +``` + +Declare how much bandwidth the OR has used recently. Usage is divided +into intervals of NSEC seconds. The YYYY-MM-DD HH:MM:SS field +defines the end of the most recent interval. The numbers are the +number of bytes used in the most recent intervals, ordered from +oldest to newest. + +These fields include both IPv4 and IPv6 traffic. + +```text + "ipv6-read-history" YYYY-MM-DD HH:MM:SS (NSEC s) NUM,NUM,NUM... NL + [At most once] + "ipv6-write-history" YYYY-MM-DD HH:MM:SS (NSEC s) NUM,NUM,NUM... NL + [At most once] +``` + +Declare how much bandwidth the OR has used recently, on IPv6 +connections. See "read-history" and "write-history" for full details. + +```text + "geoip-db-digest" Digest NL + [At most once.] +``` + +SHA1 digest of the IPv4 GeoIP database file that is used to +resolve IPv4 addresses to country codes. + +```text + "geoip6-db-digest" Digest NL + [At most once.] +``` + +SHA1 digest of the IPv6 GeoIP database file that is used to +resolve IPv6 addresses to country codes. + +("geoip-start-time" YYYY-MM-DD HH:MM:SS NL) +("geoip-client-origins" CC=NUM,CC=NUM,... NL) + +Only generated by bridge routers (see blocking.pdf), and only +when they have been configured with a geoip database. +Non-bridges SHOULD NOT generate these fields. Contains a list +of mappings from two-letter country codes (CC) to the number +of clients that have connected to that bridge from that +country (approximate, and rounded up to the nearest multiple of 8 +in order to hamper traffic analysis). A country is included +only if it has at least one address. The time in +"geoip-start-time" is the time at which we began collecting geoip +statistics. + +"geoip-start-time" and "geoip-client-origins" have been replaced by +"bridge-stats-end" and "bridge-ips" in 0.2.2.4-alpha. The +reason is that the measurement interval with "geoip-stats" as +determined by subtracting "geoip-start-time" from "published" could +have had a variable length, whereas the measurement interval in +0.2.2.4-alpha and later is set to be exactly 24 hours long. In +order to clearly distinguish the new measurement intervals from +the old ones, the new keywords have been introduced. + +```text + "bridge-stats-end" YYYY-MM-DD HH:MM:SS (NSEC s) NL + [At most once.] +``` + +YYYY-MM-DD HH:MM:SS defines the end of the included measurement +interval of length NSEC seconds (86400 seconds by default). + +A "bridge-stats-end" line, as well as any other "bridge-\*" line, +is only added when the relay has been running as a bridge for at +least 24 hours. + +```text + "bridge-ips" CC=NUM,CC=NUM,... NL + [At most once.] +``` + +List of mappings from two-letter country codes to the number of +unique IP addresses that have connected from that country to the +bridge and which are no known relays, rounded up to the nearest +multiple of 8. + +```text + "bridge-ip-versions" FAM=NUM,FAM=NUM,... NL + [At most once.] +``` + +List of unique IP addresses that have connected to the bridge +per protocol family. + +```text + "bridge-ip-transports" PT=NUM,PT=NUM,... NL + [At most once.] +``` + +List of mappings from pluggable transport names to the number +of unique IP addresses that have connected using that +pluggable transport. Unobfuscated connections are counted +using the reserved pluggable transport name "`<OR>`" (without +quotes). If we received a connection from a transport proxy +but we couldn't figure out the name of the pluggable +transport, we use the reserved pluggable transport name +"`<??>`". + +("`<OR>`" and "`<??>`" are reserved because normal pluggable +transport names MUST match the following regular expression: +"`[a-zA-Z_][a-zA-Z0-9_]*`" ) + +The pluggable transport name list is sorted into lexically +ascending order. + +If no clients have connected to the bridge yet, we only write +"bridge-ip-transports" to the stats file. + +```text + "dirreq-stats-end" YYYY-MM-DD HH:MM:SS (NSEC s) NL + [At most once.] +``` + +YYYY-MM-DD HH:MM:SS defines the end of the included measurement +interval of length NSEC seconds (86400 seconds by default). + +A "dirreq-stats-end" line, as well as any other "dirreq-\*" line, +is only added when the relay has opened its Dir port and after 24 +hours of measuring directory requests. + +```text + "dirreq-v2-ips" CC=NUM,CC=NUM,... NL + [At most once.] + "dirreq-v3-ips" CC=NUM,CC=NUM,... NL + [At most once.] +``` + +List of mappings from two-letter country codes to the number of +unique IP addresses that have connected from that country to +request a v2/v3 network status, rounded up to the nearest multiple +of 8. Only those IP addresses are counted that the directory can +answer with a 200 OK status code. (Note here and below: current Tor +versions, as of 0.2.5.2-alpha, no longer cache or serve v2 +networkstatus documents.) + +```text + "dirreq-v2-reqs" CC=NUM,CC=NUM,... NL + [At most once.] + "dirreq-v3-reqs" CC=NUM,CC=NUM,... NL + [At most once.] +``` + +List of mappings from two-letter country codes to the number of +requests for v2/v3 network statuses from that country, rounded up +to the nearest multiple of 8. Only those requests are counted that +the directory can answer with a 200 OK status code. + +```text + "dirreq-v2-share" NUM% NL + [At most once.] + "dirreq-v3-share" NUM% NL + [At most once.] +``` + +The share of v2/v3 network status requests that the directory +expects to receive from clients based on its advertised bandwidth +compared to the overall network bandwidth capacity. Shares are +formatted in percent with two decimal places. Shares are +calculated as means over the whole 24-hour interval. + +```text + "dirreq-v2-resp" status=NUM,... NL + [At most once.] + "dirreq-v3-resp" status=NUM,... NL + [At most once.] +``` + +List of mappings from response statuses to the number of requests +for v2/v3 network statuses that were answered with that response +status, rounded up to the nearest multiple of 4. Only response +statuses with at least 1 response are reported. New response +statuses can be added at any time. The current list of response +statuses is as follows: + +```text + "ok": a network status request is answered; this number + corresponds to the sum of all requests as reported in + "dirreq-v2-reqs" or "dirreq-v3-reqs", respectively, before + rounding up. + "not-enough-sigs: a version 3 network status is not signed by a + sufficient number of requested authorities. + "unavailable": a requested network status object is unavailable. + "not-found": a requested network status is not found. + "not-modified": a network status has not been modified since the + If-Modified-Since time that is included in the request. + "busy": the directory is busy. + + "dirreq-v2-direct-dl" key=NUM,... NL + [At most once.] + "dirreq-v3-direct-dl" key=NUM,... NL + [At most once.] + "dirreq-v2-tunneled-dl" key=NUM,... NL + [At most once.] + "dirreq-v3-tunneled-dl" key=NUM,... NL + [At most once.] +``` + +List of statistics about possible failures in the download process +of v2/v3 network statuses. Requests are either "direct" +HTTP-encoded requests over the relay's directory port, or +"tunneled" requests using a BEGIN_DIR relay message over the relay's OR +port. The list of possible statistics can change, and statistics +can be left out from reporting. The current list of statistics is +as follows: + +Successful downloads and failures: + +```text + "complete": a client has finished the download successfully. + "timeout": a download did not finish within 10 minutes after + starting to send the response. + "running": a download is still running at the end of the + measurement period for less than 10 minutes after starting to + send the response. + + Download times: + + "min", "max": smallest and largest measured bandwidth in B/s. + "d[1-4,6-9]": 1st to 4th and 6th to 9th decile of measured + bandwidth in B/s. For a given decile i, i/10 of all downloads + had a smaller bandwidth than di, and (10-i)/10 of all downloads + had a larger bandwidth than di. + "q[1,3]": 1st and 3rd quartile of measured bandwidth in B/s. One + fourth of all downloads had a smaller bandwidth than q1, one + fourth of all downloads had a larger bandwidth than q3, and the + remaining half of all downloads had a bandwidth between q1 and + q3. + "md": median of measured bandwidth in B/s. Half of the downloads + had a smaller bandwidth than md, the other half had a larger + bandwidth than md. + + "dirreq-read-history" YYYY-MM-DD HH:MM:SS (NSEC s) NUM,NUM,NUM... NL + [At most once] + "dirreq-write-history" YYYY-MM-DD HH:MM:SS (NSEC s) NUM,NUM,NUM... NL + [At most once] +``` + +Declare how much bandwidth the OR has spent on answering directory +requests. Usage is divided into intervals of NSEC seconds. The +YYYY-MM-DD HH:MM:SS field defines the end of the most recent +interval. The numbers are the number of bytes used in the most +recent intervals, ordered from oldest to newest. + +```text + "entry-stats-end" YYYY-MM-DD HH:MM:SS (NSEC s) NL + [At most once.] +``` + +YYYY-MM-DD HH:MM:SS defines the end of the included measurement +interval of length NSEC seconds (86400 seconds by default). + +An "entry-stats-end" line, as well as any other "entry-\*" +line, is first added after the relay has been running for at least +24 hours. + +```text + "entry-ips" CC=NUM,CC=NUM,... NL + [At most once.] +``` + +List of mappings from two-letter country codes to the number of +unique IP addresses that have connected from that country to the +relay and which are no known other relays, rounded up to the +nearest multiple of 8. + +```text + "cell-stats-end" YYYY-MM-DD HH:MM:SS (NSEC s) NL + [At most once.] +``` + +YYYY-MM-DD HH:MM:SS defines the end of the included measurement +interval of length NSEC seconds (86400 seconds by default). + +A "cell-stats-end" line, as well as any other "cell-\*" line, +is first added after the relay has been running for at least 24 +hours. + +```text + "cell-processed-cells" NUM,...,NUM NL + [At most once.] +``` + +Mean number of processed cells per circuit, subdivided into +deciles of circuits by the number of cells they have processed in +descending order from loudest to quietest circuits. + +```text + "cell-queued-cells" NUM,...,NUM NL + [At most once.] +``` + +Mean number of cells contained in queues by circuit decile. These +means are calculated by 1) determining the mean number of cells in +a single circuit between its creation and its termination and 2) +calculating the mean for all circuits in a given decile as +determined in "cell-processed-cells". Numbers have a precision of +two decimal places. + +Note that this statistic can be inaccurate for circuits that had +queued cells at the start or end of the measurement interval. + +```text + "cell-time-in-queue" NUM,...,NUM NL + [At most once.] +``` + +Mean time cells spend in circuit queues in milliseconds. Times are +calculated by 1) determining the mean time cells spend in the +queue of a single circuit and 2) calculating the mean for all +circuits in a given decile as determined in +"cell-processed-cells". + +Note that this statistic can be inaccurate for circuits that had +queued cells at the start or end of the measurement interval. + +```text + "cell-circuits-per-decile" NUM NL + [At most once.] +``` + +Mean number of circuits that are included in any of the deciles, +rounded up to the next integer. + +```text + "conn-bi-direct" YYYY-MM-DD HH:MM:SS (NSEC s) BELOW,READ,WRITE,BOTH NL + [At most once] +``` + +Number of connections, split into 10-second intervals, that are +used uni-directionally or bi-directionally as observed in the NSEC +seconds (usually 86400 seconds) before YYYY-MM-DD HH:MM:SS. Every +10 seconds, we determine for every connection whether we read and +wrote less than a threshold of 20 KiB (BELOW), read at least 10 +times more than we wrote (READ), wrote at least 10 times more than +we read (WRITE), or read and wrote more than the threshold, but +not 10 times more in either direction (BOTH). After classifying a +connection, read and write counters are reset for the next +10-second interval. + +This measurement includes both IPv4 and IPv6 connections. + +```text + "ipv6-conn-bi-direct" YYYY-MM-DD HH:MM:SS (NSEC s) BELOW,READ,WRITE,BOTH NL + [At most once] +``` + +Number of IPv6 connections that are used uni-directionally or +bi-directionally. See "conn-bi-direct" for more details. + +```text + "exit-stats-end" YYYY-MM-DD HH:MM:SS (NSEC s) NL + [At most once.] +``` + +YYYY-MM-DD HH:MM:SS defines the end of the included measurement +interval of length NSEC seconds (86400 seconds by default). + +An "exit-stats-end" line, as well as any other "exit-\*" line, is +first added after the relay has been running for at least 24 hours +and only if the relay permits exiting (where exiting to a single +port and IP address is sufficient). + +```text + "exit-kibibytes-written" port=N,port=N,... NL + [At most once.] + "exit-kibibytes-read" port=N,port=N,... NL + [At most once.] +``` + +List of mappings from ports to the number of kibibytes that the +relay has written to or read from exit connections to that port, +rounded up to the next full kibibyte. Relays may limit the +number of listed ports and subsume any remaining kibibytes under +port "other". + +```text + "exit-streams-opened" port=N,port=N,... NL + [At most once.] +``` + +List of mappings from ports to the number of opened exit streams +to that port, rounded up to the nearest multiple of 4. Relays may +limit the number of listed ports and subsume any remaining opened +streams under port "other". + +```text + "hidserv-stats-end" YYYY-MM-DD HH:MM:SS (NSEC s) NL + [At most once.] + "hidserv-v3-stats-end" YYYY-MM-DD HH:MM:SS (NSEC s) NL + [At most once.] +``` + +YYYY-MM-DD HH:MM:SS defines the end of the included measurement +interval of length NSEC seconds (86400 seconds by default). + +A "hidserv-stats-end" line, as well as any other "hidserv-\*" line, +is first added after the relay has been running for at least 24 +hours. + +(Introduced in tor-0.4.6.1-alpha) + +```text + "hidserv-rend-relayed-cells" SP NUM SP key=val SP key=val ... NL + [At most once.] + "hidserv-rend-v3-relayed-cells" SP NUM SP key=val SP key=val ... NL + [At most once.] +``` + +Approximate number of relay cells seen in either direction on a +circuit after receiving and successfully processing a RENDEZVOUS1 +cell. + +The original measurement value is obfuscated in several steps: +first, it is rounded up to the nearest multiple of 'bin_size' +which is reported in the key=val part of this line; second, a +(possibly negative) noise value is added to the result of the +first step by randomly sampling from a Laplace distribution with +mu = 0 and b = (delta_f / epsilon) with 'delta_f' and 'epsilon' +being reported in the key=val part, too; third, the result of the +previous obfuscation steps is truncated to the next smaller +integer and included as 'NUM'. Note that the overall reported +value can be negative. + +(Introduced in tor-0.4.6.1-alpha) + +```text + "hidserv-dir-onions-seen" SP NUM SP key=val SP key=val ... NL + [At most once.] + "hidserv-dir-v3-onions-seen" SP NUM SP key=val SP key=val ... NL + [At most once.] +``` + +Approximate number of unique hidden-service identities seen in +descriptors published to and accepted by this hidden-service +directory. + +The original measurement value is obfuscated in the same way as +the 'NUM' value reported in "hidserv-rend-relayed-cells", but +possibly with different parameters as reported in the key=val part +of this line. Note that the overall reported value can be +negative. + +(Introduced in tor-0.4.6.1-alpha) + +```text + "transport" transportname address:port [arglist] NL + [Any number.] +``` + +Signals that the router supports the 'transportname' pluggable +transport in IP address 'address' and TCP port 'port'. A single +descriptor MUST not have more than one transport line with the +same 'transportname'. + +Pluggable transports are only relevant to bridges, but these entries +can appear in non-bridge relays as well. + +```text + "padding-counts" YYYY-MM-DD HH:MM:SS (NSEC s) key=NUM key=NUM ... NL + [At most once.] +``` + +YYYY-MM-DD HH:MM:SS defines the end of the included measurement +interval of length NSEC seconds (86400 seconds by default). Counts +are reset to 0 at the end of this interval. + +The keyword list is currently as follows: + +```text + bin-size + - The current rounding value for cell count fields (10000 by + default) + write-drop + - The number of RELAY_DROP messages this relay sent + write-pad + - The number of PADDING cells this relay sent + write-total + - The total number of cells this relay cent + read-drop + - The number of RELAY_DROP messages this relay received + read-pad + - The number of PADDING cells this relay received + read-total + - The total number of cells this relay received + enabled-read-pad + - The number of PADDING cells this relay received on + connections that support padding + enabled-read-total + - The total number of cells this relay received on connections + that support padding + enabled-write-pad + - The total number of cells this relay received on connections + that support padding + enabled-write-total + - The total number of cells sent by this relay on connections + that support padding + max-chanpad-timers + - The maximum number of timers that this relay scheduled for + padding in the previous NSEC interval + + "overload-ratelimits" SP version SP YYYY-MM-DD SP HH:MM:SS + SP rate-limit SP burst-limit + SP read-overload-count SP write-overload-count NL + [At most once.] + + Indicates that a bandwidth limit was exhausted for this relay. +``` + +The "rate-limit" and "burst-limit" are the raw values from the +BandwidthRate and BandwidthBurst found in the torrc configuration file. + +The "{read|write}-overload-count" are the counts of how many times the +reported limits of burst/rate were exhausted and thus the maximum +between the read and write count occurrences. To make the counter more +meaningful and to avoid multiple connections saturating the counter +when a relay is overloaded, we only increment it once a minute. + +The 'version' field is set to '1' for now. + +(Introduced in tor-0.4.6.1-alpha) + +```text + "overload-fd-exhausted" SP version YYYY-MM-DD HH:MM:SS NL + [At most once.] +``` + +Indicates that a file descriptor exhaustion was experienced by this +relay. + +The timestamp indicates that the maximum was reached between the +timestamp and the "published" timestamp of the document. + +This overload field should remain in place for 72 hours since last +triggered. If the limits are reached again in this period, the +timestamp is updated, and this 72 hour period restarts. + +The 'version' field is set to '1' for the initial implementation which +detects fd exhaustion only when a socket open fails. + +(Introduced in tor-0.4.6.1-alpha) + +```text + "router-sig-ed25519" + [As in router descriptors] + + "router-signature" NL Signature NL + [At end, exactly once.] + [No extra arguments] +``` + +A document signature as documented in section 1.3, using the +initial item "extra-info" and the final item "router-signature", +signed with the router's identity key. diff --git a/spec/dir-spec/general-use-http-urls.md b/spec/dir-spec/general-use-http-urls.md new file mode 100644 index 0000000..98ea7a5 --- /dev/null +++ b/spec/dir-spec/general-use-http-urls.md @@ -0,0 +1,133 @@ +<a id="dir-spec.txt-B"></a> + +# General-use HTTP URLs + +"Fingerprints" in these URLs are base16-encoded SHA1 hashes. + +The most recent v3 consensus should be available at: + +`http://<hostname>/tor/status-vote/current/consensus.z` + +Similarly, the v3 microdescriptor consensus should be available at: + +`http://<hostname>/tor/status-vote/current/consensus-microdesc.z` + +Starting with Tor version 0.2.1.1-alpha is also available at: + +`http://<hostname>/tor/status-vote/current/consensus/<F1>+<F2>+<F3>.z` + +(NOTE: Due to squid proxy url limitations at most 96 fingerprints can be +retrieved in a single request.) + +Where F1, F2, etc. are authority identity fingerprints the client trusts. +Servers will only return a consensus if more than half of the requested +authorities have signed the document, otherwise a 404 error will be sent +back. The fingerprints can be shortened to a length of any multiple of +two, using only the leftmost part of the encoded fingerprint. Tor uses +3 bytes (6 hex characters) of the fingerprint. + +Clients SHOULD sort the fingerprints in ascending order. Server MUST +accept any order. + +Clients SHOULD use this format when requesting consensus documents from +directory authority servers and from caches running a version of Tor +that is known to support this URL format. + +A concatenated set of all the current key certificates should be available +at: + +`http://<hostname>/tor/keys/all.z` + +The key certificate for this server should be available at: + +`http://<hostname>/tor/keys/authority.z` + +The key certificate for an authority whose authority identity fingerprint +is `<F>` should be available at: + +`http://<hostname>/tor/keys/fp/<F>.z` + +The key certificate whose signing key fingerprint is `<F>` should be +available at: + +`http://<hostname>/tor/keys/sk/<F>.z` + +The key certificate whose identity key fingerprint is `<F>` and whose signing +key fingerprint is `<S>` should be available at: + +`http://<hostname>/tor/keys/fp-sk/<F>-<S>.z` + +(As usual, clients may request multiple certificates using: + +`http://<hostname>/tor/keys/fp-sk/<F1>-<S1>+<F2>-<S2>.z` ) + +\[The above fp-sk format was not supported before Tor 0.2.1.9-alpha.\] + +The most recent descriptor for a server whose identity key has a +fingerprint of `<F>` should be available at: + +`http://<hostname>/tor/server/fp/<F>.z` + +The most recent descriptors for servers with identity fingerprints +`<F1>`, `<F2>`,`<F3>` should be available at: + +`http://<hostname>/tor/server/fp/<F1>+<F2>+<F3>.z` + +(NOTE: Due to squid proxy url limitations at most 96 fingerprints can be +retrieved in a single request. + +Implementations SHOULD NOT download descriptors by identity key +fingerprint. This allows a corrupted server (in collusion with a cache) to +provide a unique descriptor to a client, and thereby partition that client +from the rest of the network.) + +The server descriptor with (descriptor) digest `<D>` (in hex) should be +available at: + +`http://<hostname>/tor/server/d/<D>.z` + +The most recent descriptors with digests `<D1>`, `<D2>`, `<D3>` should be +available at: + +`http://<hostname>/tor/server/d/<D1>+<D2>+<D3>.z` + +The most recent descriptor for this server should be at: + +`http://<hostname>/tor/server/authority.z` + +This is used for authorities, and also if a server is configured +as a bridge. The official Tor implementations (starting at +0.1.1.x) use this resource to test whether a server's own DirPort +is reachable. It is also useful for debugging purposes. + +A concatenated set of the most recent descriptors for all known servers +should be available at: + +`http://<hostname>/tor/server/all.z` + +Extra-info documents are available at the URLS: + +```text + http://<hostname>/tor/extra/d/... + http://<hostname>/tor/extra/fp/... + http://<hostname>/tor/extra/all[.z] + http://<hostname>/tor/extra/authority[.z] +``` + +(These work like the `/tor/server/` URLs: they support fetching extra-info +documents by their digest, by the fingerprint of their servers, +or all at once. When serving by fingerprint, we serve the +extra-info that corresponds to the descriptor we would serve by +that fingerprint. Only directory authorities of version +0.2.0.1-alpha or later are guaranteed to support the first +three classes of URLs. Caches may support them, and MUST +support them if they have advertised "caches-extra-info".) + +For debugging, directories SHOULD expose non-compressed objects at +URLs like the above, but without the final ".z". If the client uses +Accept-Encodings header, it should override the presence or absence +of the ".z" (see section 6.1). + +Clients SHOULD use upper case letters (A-F) when base16-encoding +fingerprints. Servers MUST accept both upper and lower case fingerprints +in requests. diff --git a/spec/dir-spec/index.md b/spec/dir-spec/index.md new file mode 100644 index 0000000..626ded0 --- /dev/null +++ b/spec/dir-spec/index.md @@ -0,0 +1,119 @@ +# Tor directory protocol, version 3 + +<a id="dir-spec.txt-0"></a> + +This directory protocol is used by Tor version 0.2.0.x-alpha and later. +See dir-spec-v1.txt for information on the protocol used up to the +0.1.0.x series, and dir-spec-v2.txt for information on the protocol +used by the 0.1.1.x and 0.1.2.x series. + +This document merges and supersedes the following proposals: + +- 101 Voting on the Tor Directory System +- 103 Splitting identity key from regularly used signing key +- 104 Long and Short Router Descriptors + +XXX timeline +XXX fill in XXXXs + +The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL +NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and +"OPTIONAL" in this document are to be interpreted as described in +RFC 2119. + +<a id="dir-spec.txt-0.1"></a> + +## History + +The earliest versions of Onion Routing shipped with a list of known +routers and their keys. When the set of routers changed, users needed to +fetch a new list. + +### The Version 1 Directory protocol { #v1-protocol } + +Early versions of Tor (0.0.2) introduced "Directory authorities": servers +that served signed "directory" documents containing a list of signed +"server descriptors", along with short summary of the status of each +router. Thus, clients could get up-to-date information on the state of +the network automatically, and be certain that the list they were getting +was attested by a trusted directory authority. + +Later versions (0.0.8) added directory caches, which download +directories from the authorities and serve them to clients. Non-caches +fetch from the caches in preference to fetching from the authorities, thus +distributing bandwidth requirements. + +Also added during the version 1 directory protocol were "router status" +documents: short documents that listed only the up/down status of the +routers on the network, rather than a complete list of all the +descriptors. Clients and caches would fetch these documents far more +frequently than they would fetch full directories. + +### The Version 2 Directory Protocol { #v2-protocol } + +During the Tor 0.1.1.x series, Tor revised its handling of directory +documents in order to address two major problems: + +```text + * Directories had grown quite large (over 1MB), and most directory + downloads consisted mainly of server descriptors that clients + already had. + + * Every directory authority was a trust bottleneck: if a single + directory authority lied, it could make clients believe for a time + an arbitrarily distorted view of the Tor network. (Clients + trusted the most recent signed document they downloaded.) Thus, + adding more authorities would make the system less secure, not + more. +``` + +To address these, we extended the directory protocol so that +authorities now published signed "network status" documents. Each +network status listed, for every router in the network: a hash of its +identity key, a hash of its most recent descriptor, and a summary of +what the authority believed about its status. Clients would download +the authorities' network status documents in turn, and believe +statements about routers iff they were attested to by more than half of +the authorities. + +Instead of downloading all server descriptors at once, clients +downloaded only the descriptors that they did not have. Descriptors +were indexed by their digests, in order to prevent malicious caches +from giving different versions of a server descriptor to different +clients. + +Routers began working harder to upload new descriptors only when their +contents were substantially changed. + +<a id="dir-spec.txt-0.2"></a> + +### Goals of the version 3 protocol { #v3-goals } + +Version 3 of the Tor directory protocol tries to solve the following +issues: + +```text + * A great deal of bandwidth used to transmit server descriptors was + used by two fields that are not actually used by Tor routers + (namely read-history and write-history). We save about 60% by + moving them into a separate document that most clients do not + fetch or use. + + * It was possible under certain perverse circumstances for clients + to download an unusual set of network status documents, thus + partitioning themselves from clients who have a more recent and/or + typical set of documents. Even under the best of circumstances, + clients were sensitive to the ages of the network status documents + they downloaded. Therefore, instead of having the clients + correlate multiple network status documents, we have the + authorities collectively vote on a single consensus network status + document. + + * The most sensitive data in the entire network (the identity keys + of the directory authorities) needed to be stored unencrypted so + that the authorities can sign network-status documents on the fly. + Now, the authorities' identity keys are stored offline, and used + to certify medium-term signing keys that can be rotated. +``` + + diff --git a/spec/dir-spec/inferring-missing-proto-lines.md b/spec/dir-spec/inferring-missing-proto-lines.md new file mode 100644 index 0000000..7468a20 --- /dev/null +++ b/spec/dir-spec/inferring-missing-proto-lines.md @@ -0,0 +1,15 @@ +<a id="dir-spec.txt-D"></a> + +# Inferring missing proto lines + +The directory authorities no longer allow versions of Tor before +0.2.4.18-rc. But right now, there is no version of Tor in the consensus +before 0.2.4.19. Therefore, we should disallow versions of Tor earlier +than 0.2.4.19, so that we can have the protocol list for all current Tor +versions include: + +Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1-2 Link=1-4 +LinkAuth=1 Microdesc=1-2 Relay=1-2 + +For Desc, Microdesc and Cons, Tor versions before 0.2.7.stable should be +taken to only support version 1. diff --git a/spec/dir-spec/limited-ed-diff-format.md b/spec/dir-spec/limited-ed-diff-format.md new file mode 100644 index 0000000..f6a2d61 --- /dev/null +++ b/spec/dir-spec/limited-ed-diff-format.md @@ -0,0 +1,39 @@ +<a id="dir-spec.txt-E"></a> + +# Limited ed diff format + +We support the following format for consensus diffs. It's a +subset of the ed diff format, but clients MUST NOT accept other +ed commands. + +We support the following ed commands, each on a line by itself: + +```text + - "<n1>d" Delete line n1 + - "<n1>,<n2>d" Delete lines n1 through n2, inclusive + - "<n1>,$d" Delete line n1 through the end of the file, inclusive. + - "<n1>c" Replace line n1 with the following block + - "<n1>,<n2>c" Replace lines n1 through n2, inclusive, with the + following block. + - "<n1>a" Append the following block after line n1. +``` + +Note that line numbers always apply to the file after all previous +commands have already been applied. Note also that line numbers +are 1-indexed. + +The commands MUST apply to the file from back to front, such that +lines are only ever referred to by their position in the original +file. + +If there are any directory signatures on the original document, the +first command MUST be a "`<n1>,$d`" form to remove all of the directory +signatures. Using this format ensures that the client will +successfully apply the diff even if they have an unusual encoding for +the signatures. + +The replace and append command take blocks. These blocks are simply +appended to the diff after the line with the command. A line with +just a period (".") ends the block (and is not part of the lines +to add). Note that it is impossible to insert a line with just +a single dot. diff --git a/spec/dir-spec/nonterminals-server-descriptors.md b/spec/dir-spec/nonterminals-server-descriptors.md new file mode 100644 index 0000000..e2240ae --- /dev/null +++ b/spec/dir-spec/nonterminals-server-descriptors.md @@ -0,0 +1,32 @@ +<a id="dir-spec.txt-2.1.3"></a> + +# Nonterminals in server descriptors + +```text + nickname ::= between 1 and 19 alphanumeric characters ([A-Za-z0-9]), + case-insensitive. + hexdigest ::= a '$', followed by 40 hexadecimal characters + ([A-Fa-f0-9]). [Represents a relay by the digest of its identity + key.] +``` + +exitpattern ::= addrspec ":" portspec +portspec ::= "\*" | port | port "-" port +port ::= an integer between 1 and 65535, inclusive. + +```text + [Some implementations incorrectly generate ports with value 0. + Implementations SHOULD accept this, and SHOULD NOT generate it. + Connections to port 0 are never permitted.] +``` + +addrspec ::= "\*" | ip4spec | ip6spec +ipv4spec ::= ip4 | ip4 "/" num_ip4_bits | ip4 "/" ip4mask +ip4 ::= an IPv4 address in dotted-quad format +ip4mask ::= an IPv4 mask in dotted-quad format +num_ip4_bits ::= an integer between 0 and 32 +ip6spec ::= ip6 | ip6 "/" num_ip6_bits +ip6 ::= an IPv6 address, surrounded by square brackets. +num_ip6_bits ::= an integer between 0 and 128 + +bool ::= "0" | "1" diff --git a/spec/dir-spec/outline.md b/spec/dir-spec/outline.md new file mode 100644 index 0000000..de969b9 --- /dev/null +++ b/spec/dir-spec/outline.md @@ -0,0 +1,255 @@ +<a id="dir-spec.txt-1"></a> + +# Outline + +There is a small set (say, around 5-10) of semi-trusted directory +authorities. A default list of authorities is shipped with the Tor +software. Users can change this list, but are encouraged not to do so, +in order to avoid partitioning attacks. + +Every authority has a very-secret, long-term "Authority Identity Key". +This is stored encrypted and/or offline, and is used to sign "key +certificate" documents. Every key certificate contains a medium-term +(3-12 months) "authority signing key", that is used by the authority to +sign other directory information. (Note that the authority identity +key is distinct from the router identity key that the authority uses +in its role as an ordinary router.) + +Routers periodically upload signed "routers descriptors" to the +directory authorities describing their keys, capabilities, and other +information. Routers may also upload signed "extra-info documents" +containing information that is not required for the Tor protocol. +Directory authorities serve server descriptors indexed by router +identity, or by hash of the descriptor. + +Routers may act as directory caches to reduce load on the directory +authorities. They announce this in their descriptors. + +Periodically, each directory authority generates a view of +the current descriptors and status for known routers. They send a +signed summary of this view (a "status vote") to the other +authorities. The authorities compute the result of this vote, and sign +a "consensus status" document containing the result of the vote. + +Directory caches download, cache, and re-serve consensus documents. + +Clients, directory caches, and directory authorities all use consensus +documents to find out when their list of routers is out-of-date. +(Directory authorities also use vote statuses.) If it is, they download +any missing server descriptors. Clients download missing descriptors +from caches; caches and authorities download from authorities. +Descriptors are downloaded by the hash of the descriptor, not by the +relay's identity key: this prevents directory servers from attacking +clients by giving them descriptors nobody else uses. + +All directory information is uploaded and downloaded with HTTP. + +<a id="dir-spec.txt-1.1"></a> + +## What's different from version 2? { #changes-since-v2 } + +Clients used to download multiple network status documents, +corresponding roughly to "status votes" above. They would compute the +result of the vote on the client side. + +Authorities used to sign documents using the same private keys they used +for their roles as routers. This forced them to keep these extremely +sensitive keys in memory unencrypted. + +All of the information in extra-info documents used to be kept in the +main descriptors. + +<a id="dir-spec.txt-1.2"></a> + +## Document meta-format { #metaformat } + +Server descriptors, directories, and running-routers documents all obey the +following lightweight extensible information format. + +The highest level object is a Document, which consists of one or more +Items. Every Item begins with a KeywordLine, followed by zero or more +Objects. A KeywordLine begins with a Keyword, optionally followed by +whitespace and more non-newline characters, and ends with a newline. A +Keyword is a sequence of one or more characters in the set \[A-Za-z0-9-\], +but may not start with -. +An Object is a block of encoded data in pseudo-Privacy-Enhanced-Mail (PEM) +style format: that is, lines of encoded data MAY be wrapped by inserting +an ascii linefeed ("LF", also called newline, or "NL" here) character +(cf. RFC 4648 §3.1). When line wrapping, implementations MUST wrap lines +at 64 characters. Upon decoding, implementations MUST ignore and discard +all linefeed characters. + +More formally: + +``` +NL = The ascii LF character (hex value 0x0a). +Document ::= (Item | NL)+ +Item ::= KeywordLine Object? +KeywordLine ::= Keyword (WS Argument)*NL +Keyword = KeywordStart KeywordChar* +KeywordStart ::= 'A' ... 'Z' | 'a' ... 'z' | '0' ... '9' +KeywordChar ::= KeywordStart | '-' +Argument := ArgumentChar+ +ArgumentChar ::= any graphical printing ASCII character. +WS = (SP | TAB)+ +Object ::= BeginLine Base64-encoded-data EndLine +BeginLine ::= "-----BEGIN " Keyword (" " Keyword)*"-----" NL +EndLine ::= "-----END " Keyword (" " Keyword)* "-----" NL +``` + +A Keyword may not be `-----BEGIN`. + +The BeginLine and EndLine of an Object must use the same keyword. + +When interpreting a Document, software MUST ignore any KeywordLine that +starts with a keyword it doesn't recognize; future implementations MUST NOT +require current clients to understand any KeywordLine not currently +described. + +Other implementations that want to extend Tor's directory format MAY +introduce their own items. The keywords for extension items SHOULD start +with the characters "x-" or "X-", to guarantee that they will not conflict +with keywords used by future versions of Tor. + +In our document descriptions below, we tag Items with a multiplicity in +brackets. Possible tags are: + +```text + "At start, exactly once": These items MUST occur in every instance of + the document type, and MUST appear exactly once, and MUST be the + first item in their documents. + + "Exactly once": These items MUST occur exactly one time in every + instance of the document type. + + "At end, exactly once": These items MUST occur in every instance of + the document type, and MUST appear exactly once, and MUST be the + last item in their documents. + + "At most once": These items MAY occur zero or one times in any + instance of the document type, but MUST NOT occur more than once. + + "Any number": These items MAY occur zero, one, or more times in any + instance of the document type. + + "Once or more": These items MUST occur at least once in any instance + of the document type, and MAY occur more. +``` + +For forward compatibility, each item MUST allow extra arguments at the +end of the line unless otherwise noted. So if an item's description below +is given as: + +"thing" int int int NL + +then implementations SHOULD accept this string as well: + +"thing 5 9 11 13 16 12" NL + +but not this string: + +"thing 5" NL + +and not this string: + +```text + "thing 5 10 thing" NL + . +``` + +Whenever an item DOES NOT allow extra arguments, we will tag it with +"no extra arguments". + +<a id="dir-spec.txt-1.3"></a> + +## Signing documents { #signing } + +Every signable document below is signed in a similar manner, using a +given "Initial Item", a final "Signature Item", a digest algorithm, and +a signing key. + +The Initial Item must be the first item in the document. + +The Signature Item has the following format: + +`<signature item keyword> [arguments] NL SIGNATURE NL` + +The "SIGNATURE" Object contains a signature (using the signing key) of +the PKCS#1 1.5 padded digest of the entire document, taken from the +beginning of the Initial item, through the newline after the Signature +Item's keyword and its arguments. + +The signature does not include the algorithmIdentifier specified in PKCS #1. + +Unless specified otherwise, the digest algorithm is SHA-1. + +All documents are invalid unless signed with the correct signing key. + +The "Digest" of a document, unless stated otherwise, is its digest *as +signed by this signature scheme*. + +<a id="dir-spec.txt-1.4"></a> + +## Voting timeline + +Every consensus document has a "valid-after" (VA) time, a "fresh-until" +(FU) time and a "valid-until" (VU) time. VA MUST precede FU, which MUST +in turn precede VU. Times are chosen so that every consensus will be +"fresh" until the next consensus becomes valid, and "valid" for a while +after. At least 3 consensuses should be valid at any given time. + +The timeline for a given consensus is as follows: + +VA-DistSeconds-VoteSeconds: The authorities exchange votes. Each authority +uploads their vote to all other authorities. + +VA-DistSeconds-VoteSeconds/2: The authorities try to download any +votes they don't have. + +Authorities SHOULD also reject any votes that other authorities try to +upload after this time. (0.4.4.1-alpha was the first version to reject votes +in this way.) + +Note: Refusing late uploaded votes minimizes the chance of a consensus +split, particular when authorities are under bandwidth pressure. If an +authority is struggling to upload its vote, and finally uploads to a +fraction of authorities after this period, they will compute a consensus +different from the others. By refusing uploaded votes after this time, +we increase the likelihood that most authorities will use the same vote +set. + +Rejecting late uploaded votes does not fix the problem entirely. If +some authorities are able to download a specific vote, but others fail +to do so, then there may still be a consensus split. However, this +change does remove one common cause of consensus splits. + +VA-DistSeconds: The authorities calculate the consensus and exchange +signatures. (This is the earliest point at which anybody can +possibly get a given consensus if they ask for it.) + +VA-DistSeconds/2: The authorities try to download any signatures +they don't have. + +VA: All authorities have a multiply signed consensus. + +```text + VA ... FU: Caches download the consensus. (Note that since caches have + no way of telling what VA and FU are until they have downloaded + the consensus, they assume that the present consensus's VA is + equal to the previous one's FU, and that its FU is one interval after + that.) + + FU: The consensus is no longer the freshest consensus. + + FU ... (the current consensus's VU): Clients download the consensus. + (See note above: clients guess that the next consensus's FU will be + two intervals after the current VA.) +``` + +VU: The consensus is no longer valid; clients should continue to try to +download a new consensus if they have not done so already. + +VU + 24 hours: Clients will no longer use the consensus at all. + +VoteSeconds and DistSeconds MUST each be at least 20 seconds; FU-VA and +VU-FU MUST each be at least 5 minutes. diff --git a/spec/dir-spec/publishing-consensus.md b/spec/dir-spec/publishing-consensus.md new file mode 100644 index 0000000..6f9563c --- /dev/null +++ b/spec/dir-spec/publishing-consensus.md @@ -0,0 +1,38 @@ +# Publishing the signed consensus { #publishing-signed-consensus } + +The voting period ends at the valid-after time. If the consensus has +been signed by a majority of authorities, these documents are made +available at + +`http://<hostname>/tor/status-vote/current/consensus.z` + +and + +`http://<hostname>/tor/status-vote/current/consensus-signatures.z` + +```text + [XXX current/consensus-signatures is not currently implemented, as it + is not used in the voting protocol.] + + [XXX possible future features include support for downloading old + consensuses.] + + The other vote documents are analogously made available under + +http://<hostname>/tor/status-vote/current/authority.z +http://<hostname>/tor/status-vote/current/<fp>.z +http://<hostname>/tor/status-vote/current/d/<d>.z +http://<hostname>/tor/status-vote/current/bandwidth.z +``` + +once the voting period ends, regardless of the number of signatures. + +The authorities serve another consensus of each flavor "F" from the +locations + +```text +/tor/status-vote/(current|next)/consensus-F.z. and +/tor/status-vote/(current|next)/consensus-F/<FP1>+....z. +``` + +The standard URLs for bandwidth list files first-appeared in Tor 0.3.5. diff --git a/spec/dir-spec/router-operation-formats.md b/spec/dir-spec/router-operation-formats.md new file mode 100644 index 0000000..477e92d --- /dev/null +++ b/spec/dir-spec/router-operation-formats.md @@ -0,0 +1,7 @@ +<a id="dir-spec.txt-2"></a> + +# Router operation and formats + +This section describes how relays must behave when publishing their +information to the directory authorities, and the formats that they +use to do so. diff --git a/spec/dir-spec/server-descriptor-format.md b/spec/dir-spec/server-descriptor-format.md new file mode 100644 index 0000000..ddd7fb2 --- /dev/null +++ b/spec/dir-spec/server-descriptor-format.md @@ -0,0 +1,509 @@ +<a id="dir-spec.txt-2.1.1"></a> + +# Server descriptor format + +Server descriptors consist of the following items. + +In lines that take multiple arguments, extra arguments SHOULD be +accepted and ignored. Many of the nonterminals below are defined in +section 2.1.3. + +Note that many versions of Tor will generate an extra newline at the +end of their descriptors. Implementations MUST tolerate one or +more blank lines at the end of a single descriptor or a list of +concatenated descriptors. New implementations SHOULD NOT generate +such blank lines. + +"router" nickname address ORPort SOCKSPort DirPort NL + +\[At start, exactly once.\] + +Indicates the beginning of a server descriptor. "nickname" must be a +valid router nickname as specified in section 2.1.3. "address" must +be an IPv4 +address in dotted-quad format. The last three numbers indicate the +TCP ports at which this OR exposes functionality. ORPort is a port at +which this OR accepts TLS connections for the main OR protocol; +SOCKSPort is deprecated and should always be 0; and DirPort is the +port at which this OR accepts directory-related HTTP connections. If +any port is not supported, the value 0 is given instead of a port +number. (At least one of DirPort and ORPort SHOULD be set; +authorities MAY reject any descriptor with both DirPort and ORPort of +0.) + +```text + "identity-ed25519" NL "-----BEGIN ED25519 CERT-----" NL certificate + "-----END ED25519 CERT-----" NL +``` + +\[Exactly once, in second position in document.\] +\[No extra arguments\] + +The certificate is a base64-encoded Ed25519 certificate (see +cert-spec.txt) with terminating =s removed. When this element +is present, it MUST appear as the first or second element in +the router descriptor. + +The certificate has CERT_TYPE of \[04\]. It must include a +signed-with-ed25519-key extension (see cert-spec.txt, +section 2.2.1), so that we can extract the master identity key. + +\[Before Tor 0.4.5.1-alpha, this field was optional.\] + +"master-key-ed25519" SP MasterKey NL + +\[Exactly once\] + +Contains the base-64 encoded ed25519 master key as a single +argument. If it is present, it MUST match the identity key +in the identity-ed25519 entry. + +\[Before Tor 0.4.5.1-alpha, this field was optional.\] + +"bandwidth" bandwidth-avg bandwidth-burst bandwidth-observed NL + +\[Exactly once\] + +Estimated bandwidth for this router, in bytes per second. The +"average" bandwidth is the volume per second that the OR is willing to +sustain over long periods; the "burst" bandwidth is the volume that +the OR is willing to sustain in very short intervals. The "observed" +value is an estimate of the capacity this relay can handle. The +relay remembers the max bandwidth sustained output over any ten +second period in the past 5 days, and another sustained input. The +"observed" value is the lesser of these two numbers. + +Tor versions released before 2018 only kept bandwidth-observed for one +day. These versions are no longer supported or recommended. + +"platform" string NL + +\[At most once\] + +A human-readable string describing the system on which this OR is +running. This MAY include the operating system, and SHOULD include +the name and version of the software implementing the Tor protocol. + +"published" YYYY-MM-DD HH:MM:SS NL + +\[Exactly once\] + +The time, in UTC, when this descriptor (and its corresponding +extra-info document if any) was generated. + +"fingerprint" fingerprint NL + +\[At most once\] + +A fingerprint (a HASH_LEN-byte of asn1 encoded public key, encoded in +hex, with a single space after every 4 characters) for this router's +identity key. A descriptor is considered invalid (and MUST be +rejected) if the fingerprint line does not match the public key. + +```text + [We didn't start parsing this line until Tor 0.1.0.6-rc; it should + be marked with "opt" until earlier versions of Tor are obsolete.] + + "hibernating" bool NL + + [At most once] +``` + +If the value is 1, then the Tor relay was hibernating when the +descriptor was published, and shouldn't be used to build circuits. + +```text + [We didn't start parsing this line until Tor 0.1.0.6-rc; it should be + marked with "opt" until earlier versions of Tor are obsolete.] + + "uptime" number NL + + [At most once] + + The number of seconds that this OR process has been running. + + "onion-key" NL a public key in PEM format +``` + +\[Exactly once\] +\[No extra arguments\] + +This key is used to encrypt CREATE cells for this OR. The key MUST be +accepted for at least 1 week after any new key is published in a +subsequent descriptor. It MUST be 1024 bits. + +The key encoding is the encoding of the key as a PKCS#1 RSAPublicKey +structure, encoded in base64, and wrapped in "-----BEGIN RSA PUBLIC +KEY-----" and "-----END RSA PUBLIC KEY-----". + +"onion-key-crosscert" NL a RSA signature in PEM format. + +\[Exactly once\] +\[No extra arguments\] + +This element contains an RSA signature, generated using the +onion-key, of the following: + +```text + A SHA1 hash of the RSA identity key, + i.e. RSA key from "signing-key" (see below) [20 bytes] + The Ed25519 identity key, + i.e. Ed25519 key from "master-key-ed25519" [32 bytes] +``` + +If there is no Ed25519 identity key, or if in some future version +there is no RSA identity key, the corresponding field must be +zero-filled. + +Parties verifying this signature MUST allow additional data +beyond the 52 bytes listed above. + +This signature proves that the party creating the descriptor +had control over the secret key corresponding to the +onion-key. + +\[Before Tor 0.4.5.1-alpha, this field was optional whenever +identity-ed25519 was absent.\] + +"ntor-onion-key" base-64-encoded-key + +\[Exactly once\] + +A curve25519 public key used for the ntor circuit extended +handshake. It's the standard encoding of the OR's curve25519 +public key, encoded in base 64. The trailing '=' sign MAY be +omitted from the base64 encoding. The key MUST be accepted +for at least 1 week after any new key is published in a +subsequent descriptor. + +\[Before Tor 0.4.5.1-alpha, this field was optional.\] + +<a id="ntor-onion-key-crosscert"></a> +```text + "ntor-onion-key-crosscert" SP Bit NL + "-----BEGIN ED25519 CERT-----" NL certificate + "-----END ED25519 CERT-----" NL +``` + +\[Exactly once\] +\[No extra arguments\] + +A signature created with the ntor-onion-key, using the +certificate format documented in cert-spec.txt, with type +\[0a\]. The signed key here is the master identity key. + +Bit must be "0" or "1". It indicates the sign of the ed25519 +public key corresponding to the ntor onion key. If Bit is "0", +then implementations MUST guarantee that the x-coordinate of +the resulting ed25519 public key is positive. Otherwise, if +Bit is "1", then the sign of the x-coordinate MUST be negative. + +To compute the ed25519 public key corresponding to a curve25519 +key, and for further explanation on key formats, see appendix C. + +This signature proves that the party creating the descriptor +had control over the secret key corresponding to the +ntor-onion-key. + +\[Before Tor 0.4.5.1-alpha, this field was optional whenever +identity-ed25519 was absent.\] + +"signing-key" NL a public key in PEM format + +\[Exactly once\] +\[No extra arguments\] + +The OR's long-term RSA identity key. It MUST be 1024 bits. + +The encoding is as for "onion-key" above. + +"accept" exitpattern NL +"reject" exitpattern NL + +\[Any number\] + +These lines describe an "exit policy": the rules that an OR follows +when deciding whether to allow a new stream to a given address. The +'exitpattern' syntax is described below. There MUST be at least one +such entry. The rules are considered in order; if no rule matches, +the address will be accepted. For clarity, the last such entry SHOULD +be accept *:* or reject *:*. + +"ipv6-policy" SP ("accept" / "reject") SP PortList NL + +\[At most once.\] + +An exit-policy summary as specified in sections 3.4.1 and 3.8.2, +summarizing +the router's rules for connecting to IPv6 addresses. A missing +"ipv6-policy" line is equivalent to "ipv6-policy reject +1-65535". + +"overload-general" SP version SP YYYY-MM-DD HH:MM:SS NL + +\[At most once.\] + +Indicates that a relay has reached an "overloaded state" which can be +one or many of the following load metrics: + +```text + - Any OOM invocation due to memory pressure + - Any ntor onionskins are dropped + - TCP port exhaustion +``` + +The timestamp is when at least one metrics was detected. It should always +be at the hour and thus, as an example, "2020-01-10 13:00:00" is an +expected timestamp. Because this is a binary state, if the line is +present, we consider that it was hit at the very least once somewhere +between the provided timestamp and the "published" timestamp of the +document which is when the document was generated. + +The overload-general line should remain in place for 72 hours since last +triggered. If the limits are reached again in this period, the timestamp +is updated, and this 72 hour period restarts. + +The 'version' field is set to '1' for now. + +```text + (Introduced in tor-0.4.6.1-alpha, but moved from extra-info to general + descriptor in tor-0.4.6.2-alpha) + + "router-sig-ed25519" SP Signature NL + + [Exactly once.] +``` + +It MUST be the next-to-last element in the descriptor, appearing +immediately before the RSA signature. It MUST contain an Ed25519 +signature of a SHA256 digest of the entire document. This digest is +taken from the first character up to and including the first space +after the "router-sig-ed25519" string. Before computing the digest, +the string "Tor router descriptor signature v1" is prefixed to the +document. + +The signature is encoded in Base64, with terminating =s removed. + +The signing key in the identity-ed25519 certificate MUST +be the one used to sign the document. + +\[Before Tor 0.4.5.1-alpha, this field was optional whenever +identity-ed25519 was absent.\] + +"router-signature" NL Signature NL + +\[At end, exactly once\] +\[No extra arguments\] + +The "SIGNATURE" object contains a signature of the PKCS1-padded +hash of the entire server descriptor, taken from the beginning of the +"router" line, through the newline after the "router-signature" line. +The server descriptor is invalid unless the signature is performed +with the router's identity key. + +"contact" info NL + +\[At most once\] + +Describes a way to contact the relay's administrator, preferably +including an email address and a PGP key fingerprint. + +"bridge-distribution-request" SP Method NL + +\[At most once, bridges only.\] + +The "Method" describes how a Bridge address is distributed by +BridgeDB. Recognized methods are: "none", "any", "https", "email", +"moat". If set to "none", BridgeDB will avoid distributing your bridge +address. If set to "any", BridgeDB will choose how to distribute your +bridge address. Choosing any of the other methods will tell BridgeDB to +distribute your bridge via a specific method: + +```text + - "https" specifies distribution via the web interface at + https://bridges.torproject.org; + - "email" specifies distribution via the email autoresponder at + bridges@torproject.org; + - "moat" specifies distribution via an interactive menu inside Tor + Browser; and + + Potential future "Method" specifiers must be as follows: + Method = (KeywordChar | "_") + +``` + +All bridges SHOULD include this line. Non-bridges MUST NOT include +it. + +BridgeDB SHOULD treat unrecognized Method values as if they were +"none". + +(Default: "any") + +\[This line was introduced in 0.3.2.3-alpha, with a minimal backport +to 0.2.5.16, 0.2.8.17, 0.2.9.14, 0.3.0.13, 0.3.1.9, and later.\] + +"family" names NL + +\[At most once\] + +'Names' is a space-separated list of relay nicknames or +hexdigests. If two ORs list one another in their "family" entries, +then OPs should treat them as a single OR for the purpose of path +selection. + +For example, if node A's descriptor contains "family B", and node B's +descriptor contains "family A", then node A and node B should never +be used on the same circuit. + +```text + "read-history" YYYY-MM-DD HH:MM:SS (NSEC s) NUM,NUM,NUM,NUM,NUM... NL + [At most once] + "write-history" YYYY-MM-DD HH:MM:SS (NSEC s) NUM,NUM,NUM,NUM,NUM... NL + [At most once] +``` + +(These fields once appeared in router descriptors, but have +appeared in extra-info descriptors since 0.2.0.x.) + +"eventdns" bool NL + +\[At most once\] + +Declare whether this version of Tor is using the newer enhanced +dns logic. Versions of Tor with this field set to false SHOULD NOT +be used for reverse hostname lookups. + +```text + [This option is obsolete. All Tor current relays should be presumed + to have the evdns backend.] + + "caches-extra-info" NL +``` + +\[At most once.\] +\[No extra arguments\] + +Present only if this router is a directory cache that provides +extra-info documents. + +\[Versions before 0.2.0.1-alpha don't recognize this\] + +"extra-info-digest" SP sha1-digest \[SP sha256-digest\] NL + +\[At most once\] + +"sha1-digest" is a hex-encoded SHA1 digest (using upper-case characters) +of the router's extra-info document, as signed in the router's +extra-info (that is, not including the signature). (If this field is +absent, the router is not uploading a corresponding extra-info +document.) + +"sha256-digest" is a base64-encoded SHA256 digest of the extra-info +document. Unlike the "sha1-digest", this digest is calculated over the +entire document, including the signature. This difference is due to +a long-lived bug in the tor implementation that it would be difficult +to roll out an incremental fix for, not a design choice. Future digest +algorithms specified should not include the signature in the data used +to compute the digest. + +\[Versions before 0.2.7.2-alpha did not include a SHA256 digest.\] +\[Versions before 0.2.0.1-alpha don't recognize this field at all.\] + +"hidden-service-dir" NL + +\[At most once.\] + +Present only if this router stores and serves hidden service +descriptors. This router supports the descriptor versions declared +in the HSDir "proto" entry. If there is no "proto" entry, this +router supports version 2 descriptors. + +```text + "protocols" SP "Link" SP LINK-VERSION-LIST SP "Circuit" SP + CIRCUIT-VERSION-LIST NL + + [At most once.] +``` + +An obsolete list of protocol versions, superseded by the "proto" +entry. This list was never parsed, and has not been emitted +since Tor 0.2.9.4-alpha. New code should neither generate nor +parse this line. + +"allow-single-hop-exits" NL + +\[At most once.\] +\[No extra arguments\] + +Present only if the router allows single-hop circuits to make exit +connections. Most Tor relays do not support this: this is +included for specialized controllers designed to support perspective +access and such. This is obsolete in tor version >= 0.3.1.0-alpha. + +"or-address" SP ADDRESS ":" PORT NL + +\[Any number\] + +ADDRESS = IP6ADDR | IP4ADDR +IPV6ADDR = an ipv6 address, surrounded by square brackets. +IPV4ADDR = an ipv4 address, represented as a dotted quad. +PORT = a number between 1 and 65535 inclusive. + +An alternative for the address and ORPort of the "router" line, but with +two added capabilities: + +```text + * or-address can be either an IPv4 or IPv6 address + * or-address allows for multiple ORPorts and addresses +``` + +A descriptor SHOULD NOT include an or-address line that does nothing but +duplicate the address:port pair from its "router" line. + +The ordering of or-address lines and their PORT entries matter because +Tor MAY accept a limited number of address/port pairs. As of +Tor 0.2.3.x only the first address/port pair is advertised and used. + +"tunnelled-dir-server" NL + +\[At most once.\] +\[No extra arguments\] + +```text + Present if the router accepts "tunneled" directory requests using a + BEGIN_DIR relay message over the router's OR port. + (Added in 0.2.8.1-alpha. Before this, Tor relays accepted + tunneled directory requests only if they had a DirPort open, + or if they were bridges.) + + "proto" SP Entries NL + + [Exactly once.] +``` + +Entries = +Entries = Entry +Entries = Entry SP Entries + +Entry = Keyword "=" Values + +Values = +Values = Value +Values = Value "," Values + +Value = Int +Value = Int "-" Int + +Int = NON_ZERO_DIGIT +Int = Int DIGIT + +Each 'Entry' in the "proto" line indicates that the Tor relay supports +one or more versions of the protocol in question. Entries should be +sorted by keyword. Values should be numerically ascending within each +entry. (This implies that there should be no overlapping ranges.) +Ranges should be represented as compactly as possible. Ints must be no +larger than 63. + +This field was first added in Tor 0.2.9.x. + +\[Before Tor 0.4.5.1-alpha, this field was optional.\] diff --git a/spec/dir-spec/serving-bandwidth-list-files.md b/spec/dir-spec/serving-bandwidth-list-files.md new file mode 100644 index 0000000..6860a6b --- /dev/null +++ b/spec/dir-spec/serving-bandwidth-list-files.md @@ -0,0 +1,30 @@ +<a id="dir-spec.txt-3.4.3"></a> + +# Serving bandwidth list files { #serving-bwlist } + +If an authority has used a bandwidth list file to generate a vote +document it SHOULD make it available at + +`http://<hostname>/tor/status-vote/next/bandwidth.z` + +at the start of each voting period. + +It MUST NOT attempt to send its bandwidth list file in a HTTP POST to +other authorities and it SHOULD NOT make bandwidth list files from other +authorities available. + +If an authority makes this file available, it MUST be the bandwidth file +used to create the vote document available at + +`http://<hostname>/tor/status-vote/next/authority.z` + +To avoid inconsistent reads, authorities SHOULD only read the bandwidth +file once per voting period. Further processing and serving SHOULD use a +cached copy. + +The bandwidth list format is described in bandwidth-file-spec.txt. + +The standard URLs for bandwidth list files first-appeared in +Tor 0.4.0.4-alpha. + +<a id="dir-spec.txt-3.5"></a> diff --git a/spec/dir-spec/standards-compliance.md b/spec/dir-spec/standards-compliance.md new file mode 100644 index 0000000..2d4e525 --- /dev/null +++ b/spec/dir-spec/standards-compliance.md @@ -0,0 +1,81 @@ +<a id="dir-spec.txt-6"></a> + +# Standards compliance + +All clients and servers MUST support HTTP 1.0. Clients and servers MAY +support later versions of HTTP as well. + +<a id="dir-spec.txt-6.1"></a> + +## HTTP headers + +Servers SHOULD set Content-Encoding to the algorithm used to compress the +document(s) being served. Recognized algorithms are: + +```text + - "identity" -- RFC2616 section 3.5 + - "deflate" -- RFC2616 section 3.5 + - "gzip" -- RFC2616 section 3.5 + - "x-zstd" -- The zstandard compression algorithm (www.zstd.net) + - "x-tor-lzma" -- The lzma compression algorithm, with a "preset" + value no higher than 6. +``` + +Clients SHOULD use Accept-Encoding on most directory requests to indicate +which of the above compression algorithms they support. If they omit it +(as Tor clients did before 0.3.1.1-alpha), then the server should serve +only "deflate" or "identity" encoded documents, based on the presence or +absence of the ".z" suffix on the requested URL. + +Note that for anonymous directory requests (that is, requests made over +multi-hop circuits, like those for onion service lookups) implementations +SHOULD NOT advertise any Accept-Encoding values other than deflate. To do +so would be to create a fingerprinting opportunity. + +When receiving multiple documents, clients MUST accept compressed +concatenated documents and concatenated compressed documents as +equivalent. + +Servers MAY set the Content-Length: header. When they do, it should +match the number of compressed bytes that they are sending. + +Servers MAY include an X-Your-Address-Is: header, whose value is the +apparent IP address of the client connecting to them (as a dotted quad). +For directory connections tunneled over a BEGIN_DIR stream, servers SHOULD +report the IP from which the circuit carrying the BEGIN_DIR stream reached +them. + +Servers SHOULD disable caching of multiple network statuses or multiple +server descriptors. Servers MAY enable caching of single descriptors, +single network statuses, the list of all server descriptors, a v1 +directory, or a v1 running routers document. XXX mention times. + +<a id="dir-spec.txt-6.2"></a> + +## HTTP status codes + +Tor delivers the following status codes. Some were chosen without much +thought; other code SHOULD NOT rely on specific status codes yet. + +```text + 200 -- the operation completed successfully + -- the user requested statuses or serverdescs, and none of the ones we + requested were found (0.2.0.4-alpha and earlier). + + 304 -- the client specified an if-modified-since time, and none of the + requested resources have changed since that time. + + 400 -- the request is malformed, or + -- the URL is for a malformed variation of one of the URLs we support, + or + -- the client tried to post to a non-authority, or + -- the authority rejected a malformed posted document, or + + 404 -- the requested document was not found. + -- the user requested statuses or serverdescs, and none of the ones + requested were found (0.2.0.5-alpha and later). + + 503 -- we are declining the request in order to save bandwidth + -- user requested some items that we ordinarily generate or store, + but we do not have any available. +``` diff --git a/spec/dir-spec/uploading-relay-documents.md b/spec/dir-spec/uploading-relay-documents.md new file mode 100644 index 0000000..da34943 --- /dev/null +++ b/spec/dir-spec/uploading-relay-documents.md @@ -0,0 +1,43 @@ +<a id="dir-spec.txt-2.1"></a> + +# Uploading server descriptors and extra-info documents + +ORs SHOULD generate a new server descriptor and a new extra-info +document whenever any of the following events have occurred: + +```text + - A period of time (18 hrs by default) has passed since the last + time a descriptor was generated. + + - A descriptor field other than bandwidth or uptime has changed. + + - Its uptime is less than 24h and bandwidth has changed by a factor of 2 + from the last time a descriptor was generated, and at least a given + interval of time (3 hours by default) has passed since then. + + - Its uptime has been reset (by restarting). + + - It receives a networkstatus consensus in which it is not listed. + + - It receives a networkstatus consensus in which it is listed + with the StaleDesc flag. + + [XXX this list is incomplete; see router_differences_are_cosmetic() + in routerlist.c for others] +``` + +ORs SHOULD NOT publish a new server descriptor or extra-info document +if none of the above events have occurred and not much time has passed +(12 hours by default). + +Tor versions older than 0.3.5.1-alpha ignore uptime when checking for +bandwidth changes. + +After generating a descriptor, ORs upload them to every directory +authority they know, by posting them (in order) to the URL + +http://<hostname:port>/tor/ + +Server descriptors may not exceed 20,000 bytes in length; extra-info +documents may not exceed 50,000 bytes in length. If they do, the +authorities SHOULD reject them. diff --git a/spec/dos-spec/index.md b/spec/dos-spec/index.md new file mode 100644 index 0000000..3645935 --- /dev/null +++ b/spec/dos-spec/index.md @@ -0,0 +1,7 @@ +# Denial-of-service prevention mechanisms in Tor + +This document covers the strategy, motivation, and implementation for denial-of-service mitigation systems designed into Tor. + +The older `dos-spec` document is now the [Memory exhaustion](./memory-exhaustion.md) section here. + +An in-depth description of the proof of work mechanism for onion services, originally [proposal 327](../../proposals/327-pow-over-intro.txt), is now in the [Proof of Work for onion service introduction](../hspow-spec/index.md) spec.
\ No newline at end of file diff --git a/dos-spec.md b/spec/dos-spec/memory-exhaustion.md index 04470b2..14c1f6b 100644 --- a/dos-spec.md +++ b/spec/dos-spec/memory-exhaustion.md @@ -1,28 +1,10 @@ -# Denial-of-service prevention mechanisms in Tor +# Memory exhaustion { #oom } -This document is incomplete; it describes some mechanisms that Tor -uses to avoid different kinds of denial-of-service attacks. +Memory exhaustion is a broad issue with many underlying causes. The Tor protocol requires clients, onion services, relays, and authorities to store various kind of information in buffers and caches. But an attacker can use these buffers and queues to exhaust the memory of the a targeted Tor process, and force the operating system to kill that process. -## Handling low-memory conditions +With this in mind, any Tor implementation (especially one that runs as a relay or onion service) must take steps to prevent memory-based denial-of-service attacks. -(See also `tor-spec.txt`, section 8.1.) - -The Tor protocol requires clients, onion services, relays, and -authorities to store various kind of information in buffers and -caches. But an attacker can use these buffers and queues to queues -to exhaust the memory of the a targeted Tor process, and force the -operating system to kill that process. - -Worse still, the ability to kill targeted Tor instances can be used -to facilitate traffic analysis. (For example, see -[the "Sniper Attack" paper](https://www.freehaven.net/anonbib/#sniper14) -by Jansen, Tschorsch, Johnson, and Scheuermann. - -With this in mind, any Tor implementation—especially one that -runs as a relay or onion service—must take steps to prevent -memory-based denial-of-service attacks. - -### Detecting low memory +## Detecting low memory { #oom-detection } The easiest way to notice you're out of memory would, in theory, be getting an error when you try to allocate more. Unfortunately, some @@ -40,18 +22,19 @@ or derived from a fraction of the total amount of system RAM. As of Tor 0.4.7.x, the MaxMemInQueues mechanism tracks the following kinds of allocation: - * Cells queued on circuits. - * Per-connection read or write buffers. - * On-the-fly compression or decompression state. - * Half-open stream records. - * Cached onion service descriptors (hsdir only). - * Cached DNS resolves (relay only). - * GEOIP-based usage activity statistics. + +- Relay cells queued on circuits. +- Per-connection read or write buffers. +- On-the-fly compression or decompression state. +- Half-open stream records. +- Cached onion service descriptors (hsdir only). +- Cached DNS resolves (relay only). +- GEOIP-based usage activity statistics. Note that directory caches aren't counted, since those are stored on disk and accessed via mmap. -### Responding to low memory +### Responding to low memory { #oom-response } If our allocations exceed MaxMemInQueues, then we take the following steps to reduce our memory allocation. @@ -76,7 +59,7 @@ oldest data. (For example, a circuit on which a single cell has been queued for 5 minutes would be freed before a circuit where 100 cells have been queued for 5 seconds.) "Data queued on a circuit" includes all data that we could drop if the circuit were destroyed: -not only the cells on the circuit's cell queue, but also any bytes +not only the cells on the circuit's relay cell queue, but also any bytes queued in buffers associated with streams or half-stream records attached to the circuit. @@ -86,12 +69,10 @@ rule, according to the age of their oldest queued data. Upon freeing a circuit, a "DESTROY cell" must be sent in both directions. -### Reporting low memory. +### Reporting low memory { #oom-reporting } We define a "low threshold" equal to 3/4 of MaxMemInQueues. Every time our memory usage is above the low threshold, we record ourselves as being "under memory pressure". (This is not currently reported.) - - diff --git a/spec/dos-spec/overview.md b/spec/dos-spec/overview.md new file mode 100644 index 0000000..81b7e35 --- /dev/null +++ b/spec/dos-spec/overview.md @@ -0,0 +1,78 @@ +# Overview + +As a public and anonymous network, Tor is open to many types of denial-of-service attempts. It's necessary to constantly develop a variety of defenses that mitigate specific types of attacks. + +These mitigations are expected to improve network availability, but DoS mitigation is also important for limiting the avenues an attacker could use to perform active attacks on anonymity. For example, the ability to kill targeted Tor instances can be used to facilitate traffic analysis. See the ["Sniper Attack" paper](https://www.freehaven.net/anonbib/#sniper14) by Jansen, Tschorsch, Johnson, and Scheuermann. + +The attack and defense environment changes over time. +Expect that this document is an attempt to describe the current state of things, but that it may not be complete. + +The defenses here are organized by the type of resource under contention. These can be physical resources ([Memory](#memory), [CPU](#cpu), [Bandwidth](#bandwidth)) or protocol resources ([Channels](#channels), [Circuits](#circuits), [Introductions](#hs-intro)). + +In practice there are always overlaps between these resource types. +Connecting to an onion service, for example, puts some strain on every resource type here. + +## Physical resources + +### Memory {#memory} + +[Memory exhaustion](./memory-exhaustion.md) is both one of the most serious denial-of-service avenues and the subject of the most fully developed defense mechanisms so far. We track overall memory use and free the most disposable objects first when usage is over threshold. + +### CPU {#cpu} + +The available CPU time on a router can be exhausted, assuming the implementation is not capable of processing network input at line rate in all circumstances. +This is especially problematic in the single-threaded C implementation. +Certain expensive operations like circuit extension handshakes are deferred to a thread pool, but time on the main thread is still a precious resource. + +We currently don't directly monitor and respond to CPU usage. +Instead C Tor relies on limits for protocol resources, like circuits extensions and onion service introductions, that are associated with this CPU load. + +### Bandwidth {#bandwidth} + +Relay operators can place hard limits on total bandwidth using the `Bandwidth` or `RelayBandwidth` options. These options can help relay operators avoid bandwidth peaks on their network, however they aren't designed as denial of service prevention mechanisms. + +Beyond just shaving off harmful bandwidth peaks it's important that normal service is not disrupted too much, and especially not disrupted in a targetable way. +To approximate this goal we rely on [flow control](../tor-spec/flow-control.md) and fair dequeueing of relay cells. + +## Protocol resources + +### Channels {#channels} + +All channels to some extent are a limited resource, but we focus specifically on preventing floods of incoming TLS connections. + +Excessive incoming TLS connections consume memory as well as limited network and operating system resources. +Excessive incoming connections typically signal a low-effort denial of service attack. + +The C Tor implementation establishes limits on both the number of concurrent connections per IP address and the rate of new connections, using the `DoSConnection` family of configuration options and their corresponding consensus parameters. + +### Circuits {#circuits} + +Excessive circuit creation can impact the entire path of that circuit, so it's important to reject these attacks any time they can be identified. Ideally we reject them as early as possible, before they have fully built the circuit. + +Because of Tor's anonymity, most affected nodes experience the circuit flood as coming from every direction. The guard position, however, has a chance to notice specific peers that are creating too many circuits. + +The C Tor implementation limits the acceptable rate of circuit creation per client IP address using the `DoSCircuit` configuration options and their corresponding consensus parameters. + +### Onion service introductions {#hs-intro} + +Flooding an onion service with introduction attempts causes significant network load. In addition to the CPU, memory, and bandwidth load experienced by the introduction point and the service, all involved relays experience a circuit creation flood. + +We have two types of onion service DoS mitigations currently. Both are optional, enabled as needed by individual onion servce operators. + +#### Mitigation by rate limiting {#hs-intro-rate} + +Introduction attempts can be rate-limited by each introduction point, at the request of the service. + +This defense is configured by an operator using the `HiddenServiceEnableIntroDos` configuration options. Services use the [introduction DoS extension](../rend-spec/introduction-protocol.html#EST_INTRO_DOS_EXT) to communicate these settings to each introduction point. + +#### Mitigation using proof of work {#hs-intro-pow} + +A short non-interactive computational puzzle can be solved with each connection attempt. Requests provided by the client will be entered into a queue prioritized by their puzzle solution's effort score. Requests are processed by the service at a limited rate, which can be adjusted to a value within the server's capabilities. + +Based on the queue behavior, servers will continuously provide an updated effort suggestion. +Queue backlogs cause the effort to rise, and an idle server will cause the effort to decay. +If the queue is never overfull the effort decays to zero, asking clients not to include a proof-of-work solution at all. + +We may support multiple cryptographic algorithms for this puzzle in the future, but currently we support one type. It's called `v1` in our protocol, and it's based on the Equi-X algorithm developed for this purpose. See the document on [Proof of Work for onion service introduction](../hspow-spec/index.md). + +This defense is configured by an operator using the `HiddenServicePoW` configuration options. Additionally, it requires both the client and the onion service to be compiled with the `pow` module (and `--enable-gpl` mode) available. Despite this non-default build setting, proof of work *is* available through common packagers like the Tor Browser and Debian. diff --git a/spec/ext-orport-spec.md b/spec/ext-orport-spec.md new file mode 100644 index 0000000..1341630 --- /dev/null +++ b/spec/ext-orport-spec.md @@ -0,0 +1,254 @@ +# Extended ORPort for pluggable transports + +```text + George Kadianakis, Nick Mathewson +``` + +<a id="ext-orport-spec.txt-1"></a> + +## Overview + +This document describes the "Extended ORPort" protocol, a wrapper +around Tor's ordinary ORPort protocol for use by bridges that +support pluggable transports. It provides a way for server-side PTs +and bridges to exchange additional information before beginning +the actual OR connection. + +See `tor-spec.txt` for information on the regular OR protocol, and +`pt-spec.txt` for information on pluggable transports. + +This protocol was originally proposed in proposal 196, and +extended with authentication in proposal 217. + +<a id="ext-orport-spec.txt-2"></a> + +## Establishing a connection and authenticating { #establishing } + +When a client (that is to say, a server-side pluggable transport) +connects to an Extended ORPort, the server sends: + +```text + AuthTypes [variable] + EndAuthTypes [1 octet] + + Where, + + + AuthTypes are the authentication schemes that the server supports + for this session. They are multiple concatenated 1-octet values that + take values from 1 to 255. + + EndAuthTypes is the special value 0. +``` + +The client reads the list of supported authentication schemes, +chooses one, and sends it back: + +AuthType \[1 octet\] + +Where, + +```text + + AuthType is the authentication scheme that the client wants to use + for this session. A valid authentication type takes values from 1 to + 255. A value of 0 means that the client did not like the + authentication types offered by the server. +``` + +If the client sent an AuthType of value 0, or an AuthType that the +server does not support, the server MUST close the connection. + +<a id="ext-orport-spec.txt-2.1"></a> + +### Authentication type: SAFE_COOKIE { #SAFE_COOKIE } + +We define one authentication type: SAFE_COOKIE. Its AuthType +value is 1. It is based on the client proving to the bridge that +it can access a given "cookie" file on disk. The purpose of +authentication is to defend against cross-protocol attacks. + +If the Extended ORPort is enabled, Tor should regenerate the cookie +file on startup and store it in +$DataDirectory/extended_orport_auth_cookie. + +The location of the cookie can be overridden by using the +configuration file parameter ExtORPortCookieAuthFile, which is +defined as: + +`ExtORPortCookieAuthFile <path>` + +where `<path>` is a filesystem path. + +<a id="ext-orport-spec.txt-2.1.2"></a> + +#### Cookie-file format { #SAFE_COOKIE_file } + +The format of the cookie-file is: + +```text + StaticHeader [32 octets] + Cookie [32 octets] + + Where, + + StaticHeader is the following string: + "! Extended ORPort Auth Cookie !\x0a" + + Cookie is the shared-secret. During the SAFE_COOKIE protocol, the + cookie is called CookieString. +``` + +Extended ORPort clients MUST make sure that the StaticHeader is +present in the cookie file, before proceeding with the +authentication protocol. + +<a id="ext-orport-spec.txt-2.1.3"></a> + +#### SAFE_COOKIE Protocol specification { #SAFE_COOKIE_spec } + +A client that performs the SAFE_COOKIE handshake begins by sending: + +```text +ClientNonce [32 octets] +``` + +Where, + +- ClientNonce is 32 octets of random data. + +Then, the server replies with: + +```text + ServerHash [32 octets] + ServerNonce [32 octets] + + Where, + + ServerHash is computed as: + HMAC-SHA256(CookieString, + "ExtORPort authentication server-to-client hash" | ClientNonce | ServerNonce) + + ServerNonce is 32 random octets. +``` + +Upon receiving that data, the client computes ServerHash, and +validates it against the ServerHash provided by the server. + +If the server-provided ServerHash is invalid, the client MUST +terminate the connection. + +Otherwise the client replies with: + +```text +ClientHash [32 octets] + + Where, + + ClientHash is computed as: + HMAC-SHA256(CookieString, + "ExtORPort authentication client-to-server hash" | ClientNonce | ServerNonce) +``` + +Upon receiving that data, the server computes ClientHash, and +validates it against the ClientHash provided by the client. + +Finally, the server replies with: + +Status \[1 octet\] + +```text + Where, + + Status is 1 if the authentication was successful. If the + authentication failed, Status is 0. +``` + +<a id="ext-orport-spec.txt-3"></a> + +## The extended ORPort protocol { #ext_orport_protocol} + +Once a connection is established and authenticated, the parties +communicate with the protocol described here. + +<a id="ext-orport-spec.txt-3.1"></a> + +### Protocol + +The extended server port protocol is as follows: + +```text + COMMAND [2 bytes, big-endian] + BODYLEN [2 bytes, big-endian] + BODY [BODYLEN bytes] + + Commands sent from the transport proxy to the bridge are: + + [0x0000] DONE: There is no more information to give. The next + bytes sent by the transport will be those tunneled over it. + (body ignored) + + [0x0001] USERADDR: an address:port string that represents the + client's address. + + [0x0002] TRANSPORT: a string of the name of the pluggable + transport currently in effect on the connection. + + Replies sent from tor to the proxy are: + + [0x1000] OKAY: Send the user's traffic. (body ignored) + + [0x1001] DENY: Tor would prefer not to get more traffic from + this address for a while. (body ignored) + + [0x1002] CONTROL: (Not used) + + Parties MUST ignore command codes that they do not understand. +``` + +If the server receives a recognized command that does not parse, it +MUST close the connection to the client. + +<a id="ext-orport-spec.txt-3.2"></a> + +### Command descriptions { #ext-orport-commands} + +<a id="ext-orport-spec.txt-3.2.1"></a> + +#### USERADDR + +```text + An ASCII string holding the TCP/IP address of the client of the + pluggable transport proxy. A Tor bridge SHOULD use that address to + collect statistics about its clients. Recognized formats are: + 1.2.3.4:5678 + [1:2::3:4]:5678 +``` + +(Current Tor versions may accept other formats, but this is a bug: +transports MUST NOT send them.) + +The string MUST not be NUL-terminated. + +<a id="ext-orport-spec.txt-3.2.2"></a> + +#### TRANSPORT + +An ASCII string holding the name of the pluggable transport used by +the client of the pluggable transport proxy. A Tor bridge that +supports multiple transports SHOULD use that information to collect +statistics about the popularity of individual pluggable transports. + +The string MUST not be NUL-terminated. + +Pluggable transport names are C-identifiers and Tor MUST check them +for correctness. + +<a id="ext-orport-spec.txt-4"></a> + +## Security Considerations + +Extended ORPort or TransportControlPort do _not_ provide link +confidentiality, authentication or integrity. Sensitive data, like +cryptographic material, should not be transferred through them. + +An attacker with superuser access is able to sniff network traffic, +and capture TransportControlPort identifiers and any data passed +through those ports. + +Tor SHOULD issue a warning if the bridge operator tries to bind +Extended ORPort to a non-localhost address. + +Pluggable transport proxies SHOULD issue a warning if they are +instructed to connect to a non-localhost Extended ORPort. diff --git a/spec/glossary.md b/spec/glossary.md new file mode 100644 index 0000000..aa68699 --- /dev/null +++ b/spec/glossary.md @@ -0,0 +1,238 @@ +# Glossary + +The Tor Project + +This document aims to specify terms, notations, and phrases related +to Tor, as used in the Tor specification documents and other documentation. + +This glossary is not a design document; it is only a reference. + +This glossary is a work-in-progress; double-check its definitions before +citing them authoritatively. ;) + +<a id="glossary.txt-0"></a> + +## Preliminaries + +The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL +NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and +"OPTIONAL" in this document are to be interpreted as described in +RFC 2119. + +<a id="glossary.txt-1.0"></a> + +## Commonly used Tor configuration terms { #configuration } + +ORPort - Onion Router Port +DirPort - Directory Port + +<a id="glossary.txt-2.0"></a> + +## Tor network components { #network-compoennts } + +<a id="glossary.txt-2.1"></a> + +## Relays, aka OR (onion router) { #relay} + +\[Style guide: prefer the term "Relay"\] + +<a id="glossary.txt-2.1.1"></a> + +### Specific roles { #roles } + +Exit relay: The final hop in an exit circuit before traffic leaves +the Tor network to connect to external servers. + +Non-exit relay: Relays that send and receive traffic only to +other Tor relays and Tor clients. + +Entry relay: The first hop in a Tor circuit. Can be either a guard +relay or a bridge, depending on the client's configuration. + +Guard relay: A relay that a client uses as its entry for a longer +period of time. Guard relays are rotated more slowly to prevent +attacks that can come from being exposed to too many guards. + +Bridge: A relay intentionally not listed in the public Tor +consensus, with the purpose of circumventing entities (such as +governments or ISPs) seeking to block clients from using Tor. +Currently, bridges are used only as entry relays. + +Directory cache: A relay that downloads cached directory information +from the directory authorities and serves it to clients on demand. +Any relay will act as a directory cache, if its bandwidth is high enough. + +Rendezvous point: A relay connecting a client to a hidden service. +Each party builds a three-hop circuit, meeting at the +rendezvous point. + +<a id="glossary.txt-2.2"></a> + +## Client, aka OP (onion proxy) { #client } + +\[Style: the "OP" and "onion proxy" terms are deprecated.\] + +<a id="glossary.txt-2.3"></a> + +## Authorities { #authorities } + +Directory Authority: Nine total in the Tor network, operated by +trusted individuals. Directory authorities define and serve the +consensus document, defining the "state of the network." This document +contains a "router status" section for every relay currently +in the network. Directory authorities also serve router descriptors, +extra info documents, microdescriptors, and the microdescriptor consensus. + +Bridge Authority: One total. Similar in responsibility to directory +authorities, but for bridges. + +Fallback directory mirror: One of a list of directory caches distributed +with the Tor software. (When a client first connects to the network, and +has no directory information, it asks a fallback directory. From then on, +the client can ask any directory cache that's listed in the directory +information it has.) + +<a id="glossary.txt-2.4"></a> + +## Hidden Service { #hidden-service } + +A hidden service is a server that will only accept incoming +connections via the hidden service protocol. Connection +initiators will not be able to learn the IP address of the hidden +service, allowing the hidden service to receive incoming connections, +serve content, etc, while preserving its location anonymity. + +<a id="glossary.txt-2.5"></a> + +## Circuit + +An established path through the network, where cryptographic keys +are negotiated using the ntor protocol or TAP (Tor Authentication +Protocol (deprecated)) with each hop. Circuits can differ in length +depending on their purpose. See also Leaky Pipe Topology. + +Origin Circuit - + +Exit Circuit: A circuit which connects clients to destinations +outside the Tor network. For example, if a client wanted to visit +duckduckgo.com, this connection would require an exit circuit. + +Internal Circuit: A circuit whose traffic never leaves the Tor +network. For example, a client could connect to a hidden service via +an internal circuit. + +<a id="glossary.txt-2.6"></a> + +## Edge connection + +```text +2.7. Consensus: The state of the Tor network, published every hour, + decided by a vote from the network's directory authorities. Clients + fetch the consensus from directory authorities, fallback + directories, or directory caches. + +2.8. Descriptor: Each descriptor represents information about one + relay in the Tor network. The descriptor includes the relay's IP + address, public keys, and other data. Relays send + descriptors to directory authorities, who vote and publish a + summary of them in the network consensus. +``` + +<a id="glossary.txt-3.0"></a> + +## Tor network protocols + +<a id="glossary.txt-3.1"></a> + +## Messages and cells + +Cell: A message sent over a channel. Every cell has an associated +command. A cell may be fixed-length or variable-length, depending on +its command. Cells are sometimes referred to by their command types: +for example, a cell whose command is `DESTROY` is called a +DESTROY cell. + +Relay cell: A cell that tells a relay or client about instructions sent +over a circuit. The command of a relay cell may be `RELAY` or `RELAY_EARLY`. +If we need to refer to a cell whose command is specifically `RELAY`, +we call it a "RELAY" cell. + +Enveloped relay message: The results of decrypting a relay cell: +a relay message plus an associated (optional) StreamID. +(If the StreamID is not present, or zero, +then the relay message is addressed to the circuit itself +rather than to any particular stream on the circuit.) + +Relay message: A message sent over a circuit to an individual stream, +or to the circuit itself. Relay messages are sometimes referred to +by their command types: for example, a message whose command is +`DATA` is sometimes called a DATA message. +Sometimes, relay messages are just called "Messages" +if no ambiguity would result. + +> Note that when [prop340](proposals/340-packed-and-fragmented.md) is implemented, +> the relationship between relay cells and (enveloped) relay messages +> will no longer be 1:1. + +## Link handshake + +The link handshake establishes the TLS connection over which two +Tor participants will send Tor cells. This handshake also +authenticates the participants to each other, possibly using Tor +cells. + +<a id="glossary.txt-3.2"></a> + +## Circuit handshake + +Circuit handshakes establish the hop-by-hop onion encryption +that clients use to tunnel their application traffic. The +client does a pairwise key establishment handshake with each +individual relay in the circuit. For every hop except the +first, these handshakes tunnel through existing hops in the +circuit. Each cell type in this protocol also has a newer +version (with a "2" suffix), e.g., CREATE2. + +CREATE cell: First part of a handshake, sent by the initiator. + +CREATED cell: Second part of a handshake, sent by the responder. + +EXTEND message: (also known as a RELAY_EXTEND message) First part of a +handshake, tunneled through an existing circuit. The last relay +in the circuit so far will decrypt this cell and send the +payload in a CREATED cell to the chosen next hop relay. + +EXTENDED cell: (also known as a RELAY_EXTENDED message) Second part +of a handshake, tunneled through an existing circuit. The last +relay in the circuit so far receives the CREATED cell from the +new last hop relay and encrypts the payload in an EXTENDED message +to tunnel back to the client. + +Onion skin: The body of a CREATE/CREATE2 cell or an EXTEND/EXTEND2 message. +It contains the first part of the TAP or ntor key establishment +handshake. + +<a id="glossary.txt-3.3"></a> + +## Hidden Service Protocol + +<a id="glossary.txt-3.4"></a> + +## Directory Protocol + +<a id="glossary.txt-4.0"></a> + +## General network definitions + +Leaky Pipe Topology: The ability for the origin of a circuit to address +relay cells to be addressed to any hop in the path of a circuit. In Tor, +the destination hop is determined by using the 'recognized' field of relay +cells. + +Stream: A single application-level connection or request, multiplexed over +a Tor circuit. A 'Stream' can currently carry the contents of a TCP +connection, a DNS request, or a Tor directory request. + +Channel: A pairwise connection between two Tor relays, or between a +client and a relay. Circuits are multiplexed over Channels. All +channels are currently implemented as TLS connections. diff --git a/spec/guard-spec/algorithm.md b/spec/guard-spec/algorithm.md new file mode 100644 index 0000000..ba1c725 --- /dev/null +++ b/spec/guard-spec/algorithm.md @@ -0,0 +1,672 @@ +<a id="guard-spec.txt-4"></a> + +# The algorithm + +<a id="guard-spec.txt-4.0"></a> + +## The guards listed in the current consensus. {#GUARDS} + +By {set:GUARDS} we mean the set of all guards in the current +consensus that are usable for all circuits and directory +requests. (They must have the flags: Stable, Fast, V2Dir, Guard.) + +**Rationale** + +We require all guards to have the flags that we potentially need +from any guard, so that all guards are usable for all circuits. + +<a id="guard-spec.txt-4.1"></a> + +## The Sampled Guard Set. {#SAMPLED} + +We maintain a set, {set:SAMPLED_GUARDS}, that persists across +invocations of Tor. It is a subset of the nodes ordered by a sample idx that +we have seen listed as a guard in the consensus at some point. +For each such guard, we record persistently: + +```text + - {pvar:ADDED_ON_DATE}: The date on which it was added to + sampled_guards. + + We set this value to a point in the past, using + RAND(now, {GUARD_LIFETIME}/10). See + Appendix [RANDOM] below. + + - {pvar:ADDED_BY_VERSION}: The version of Tor that added it to + sampled_guards. + + - {pvar:IS_LISTED}: Whether it was listed as a usable Guard in + the _most recent_ consensus we have seen. + + - {pvar:FIRST_UNLISTED_AT}: If IS_LISTED is false, the publication date + of the earliest consensus in which this guard was listed such that we + have not seen it listed in any later consensus. Otherwise "None." + We randomize this to a point in the past, based on + RAND(added_at_time, {REMOVE_UNLISTED_GUARDS_AFTER} / 5) +``` + +For each guard in {SAMPLED_GUARDS}, we also record this data, +non-persistently: + +```text + - {tvar:last_tried_connect}: A 'last tried to connect at' + time. Default 'never'. + + - {tvar:is_reachable}: an "is reachable" tristate, with + possible values { <state:yes>, <state:no>, <state:maybe> }. + Default '<maybe>.' + + [Note: "yes" is not strictly necessary, but I'm + making it distinct from "maybe" anyway, to make our + logic clearer. A guard is "maybe" reachable if it's + worth trying. A guard is "yes" reachable if we tried + it and succeeded.] + + [Note 2: This variable is, in fact, a combination + of different context-sensitive variables depending + on the _purpose_ for which we are selecting a guard. + When we are selecing a guard for an ordinary + circuit, we look at the regular {is_reachable}. + But when we are selecting the guard for a one-hop + directory circuit, we also look at an instance + of {is_reachable} that tracks whether + downloads of the types we are making have failed + recently.] + + - {tvar:failing_since}: The first time when we failed to + connect to this guard. Defaults to "never". Reset to + "never" when we successfully connect to this guard. + + - {tvar:is_pending} A "pending" flag. This indicates that we + are trying to build an exploratory circuit through the + guard, and we don't know whether it will succeed. + + - {tvar:pending_since}: A timestamp. Set whenever we set + {tvar:is_pending} to true; cleared whenever we set + {tvar:is_pending} to false. NOTE +``` + +We require that {SAMPLED_GUARDS} contain at least +{MIN_FILTERED_SAMPLE} guards from the consensus (if possible), +but not more than {MAX_SAMPLE_THRESHOLD} of the number of guards +in the consensus, and not more than {MAX_SAMPLE_SIZE} in total. +(But if the maximum would be smaller than {MIN_FILTERED_SAMPLE}, we +set the maximum at {MIN_FILTERED_SAMPLE}.) + +To add a new guard to {SAMPLED_GUARDS}, pick an entry at random from +({GUARDS} - {SAMPLED_GUARDS}), according to the path selection rules. + +We remove an entry from {SAMPLED_GUARDS} if: + +```text + * We have a live consensus, and {IS_LISTED} is false, and + {FIRST_UNLISTED_AT} is over {REMOVE_UNLISTED_GUARDS_AFTER} + days in the past. + + OR + + * We have a live consensus, and {ADDED_ON_DATE} is over + {GUARD_LIFETIME} ago, *and* {CONFIRMED_ON_DATE} is either + "never", or over {GUARD_CONFIRMED_MIN_LIFETIME} ago. +``` + +Note that {SAMPLED_GUARDS} does not depend on our configuration. +It is possible that we can't actually connect to any of these +guards. + +**Rationale** + +The {SAMPLED_GUARDS} set is meant to limit the total number of +guards that a client will connect to in a given period. The +upper limit on its size prevents us from considering too many +guards. + +The first expiration mechanism is there so that our +{SAMPLED_GUARDS} list does not accumulate so many dead +guards that we cannot add new ones. + +The second expiration mechanism makes us rotate our guards slowly +over time. + +Ordering the {SAMPLED_GUARDS} set in the order in which we sampled those +guards and picking guards from that set according to this ordering improves +load-balancing. It is closer to offer the expected usage of the guard nodes +as per the path selection rules. + +The ordering also improves on another objective of this proposal: trying to +resist an adversary pushing clients over compromised guards, since the +adversary would need the clients to exhaust all their initial +{SAMPLED_GUARDS} set before having a chance to use a newly deployed +adversary node. + +<a id="guard-spec.txt-4.2"></a> + +## The Usable Sample {#FILTERED} + +We maintain another set, {set:FILTERED_GUARDS}, that does not +persist. It is derived from: + +```text + - {SAMPLED_GUARDS} + - our current configuration, + - the path bias information. +``` + +A guard is a member of {set:FILTERED_GUARDS} if and only if all +of the following are true: + +```text + - It is a member of {SAMPLED_GUARDS}, with {IS_LISTED} set to + true. + - It is not disabled because of path bias issues. + - It is not disabled because of ReachableAddresses policy, + the ClientUseIPv4 setting, the ClientUseIPv6 setting, + the FascistFirewall setting, or some other + option that prevents using some addresses. + - It is not disabled because of ExcludeNodes. + - It is a bridge if UseBridges is true; or it is not a + bridge if UseBridges is false. + - Is included in EntryNodes if EntryNodes is set and + UseBridges is not. (But see 2.B above). +``` + +We have an additional subset, {set:USABLE_FILTERED_GUARDS}, which +is defined to be the subset of {FILTERED_GUARDS} where +{is_reachable} is `<yes>` or `<maybe>`. + +We try to maintain a requirement that {USABLE_FILTERED_GUARDS} +contain at least {MIN_FILTERED_SAMPLE} elements: + +```text + Whenever we are going to sample from {USABLE_FILTERED_GUARDS}, + and it contains fewer than {MIN_FILTERED_SAMPLE} elements, we + add new elements to {SAMPLED_GUARDS} until one of the following + is true: + + * {USABLE_FILTERED_GUARDS} is large enough, + OR + * {SAMPLED_GUARDS} is at its maximum size. + + ** Rationale ** +``` + +These filters are applied _after_ sampling: if we applied them +before the sampling, then our sample would reflect the set of +filtering restrictions that we had in the past. + +<a id="guard-spec.txt-4.3"></a> + +## The confirmed-guard list. {#CONFIRMED} + +\[formerly USED_GUARDS\] + +We maintain a persistent ordered list, {list:CONFIRMED_GUARDS}. +It contains guards that we have used before, in our preference +order of using them. It is a subset of {SAMPLED_GUARDS}. For +each guard in this list, we store persistently: + +- {pvar:IDENTITY} Its fingerprint. + +```text + - {pvar:CONFIRMED_ON_DATE} When we added this guard to + {CONFIRMED_GUARDS}. + + Randomized to a point in the past as RAND(now, {GUARD_LIFETIME}/10). +``` + +We append new members to {CONFIRMED_GUARDS} when we mark a circuit +built through a guard as "for user traffic." + +Whenever we remove a member from {SAMPLED_GUARDS}, we also remove +it from {CONFIRMED_GUARDS}. + +```text + [Note: You can also regard the {CONFIRMED_GUARDS} list as a + total ordering defined over a subset of {SAMPLED_GUARDS}.] +``` + +Definition: we call Guard A "higher priority" than another Guard B +if, when A and B are both reachable, we would rather use A. We +define priority as follows: + +```text + * Every guard in {CONFIRMED_GUARDS} has a higher priority + than every guard not in {CONFIRMED_GUARDS}. + + * Among guards in {CONFIRMED_GUARDS}, the one appearing earlier + on the {CONFIRMED_GUARDS} list has a higher priority. + + * Among guards that do not appear in {CONFIRMED_GUARDS}, + {is_pending}==true guards have higher priority. + + * Among those, the guard with earlier {last_tried_connect} time + has higher priority. + + * Finally, among guards that do not appear in + {CONFIRMED_GUARDS} with {is_pending==false}, all have equal + priority. + + ** Rationale ** +``` + +We add elements to this ordering when we have actually used them +for building a usable circuit. We could mark them at some other +time (such as when we attempt to connect to them, or when we +actually connect to them), but this approach keeps us from +committing to a guard before we actually use it for sensitive +traffic. + +<a id="guard-spec.txt-4.4"></a> + +## The Primary guards {#PRIMARY} + +We keep a run-time non-persistent ordered list of +{list:PRIMARY_GUARDS}. It is a subset of {FILTERED_GUARDS}. It +contains {N_PRIMARY_GUARDS} elements. + +To compute primary guards, take the ordered intersection of +{CONFIRMED_GUARDS} and {FILTERED_GUARDS}, and take the first +{N_PRIMARY_GUARDS} elements. If there are fewer than +{N_PRIMARY_GUARDS} elements, append additional elements to +PRIMARY_GUARDS chosen from ({FILTERED_GUARDS} - {CONFIRMED_GUARDS}), +ordered in "sample order" (that is, by {ADDED_ON_DATE}). + +Once an element has been added to {PRIMARY_GUARDS}, we do not remove it +until it is replaced by some element from {CONFIRMED_GUARDS}. +That is: if a non-primary guard becomes confirmed and not every primary +guard is confirmed, then the list of primary guards list is regenerated, +first from the confirmed guards (as before), and then from any +non-confirmed primary guards. + +Note that {PRIMARY_GUARDS} do not have to be in +{USABLE_FILTERED_GUARDS}: they might be unreachable. + +**Rationale** + +These guards are treated differently from other guards. If one of +them is usable, then we use it right away. For other guards +{FILTERED_GUARDS}, if it's usable, then before using it we might +first double-check whether perhaps one of the primary guards is +usable after all. + +<a id="guard-spec.txt-4.5"></a> + +## Retrying guards. {#RETRYING} + +(We run this process as frequently as needed. It can be done once +a second, or just-in-time.) + +If a primary sampled guard's {is_reachable} status is `<no>`, then +we decide whether to update its {is_reachable} status to `<maybe>` +based on its {last_tried_connect} time, its {failing_since} time, +and the {PRIMARY_GUARDS_RETRY_SCHED} schedule. + +If a non-primary sampled guard's {is_reachable} status is `<no>`, then +we decide whether to update its {is_reachable} status to `<maybe>` +based on its {last_tried_connect} time, its {failing_since} time, +and the {GUARDS_RETRY_SCHED} schedule. + +**Rationale** + +An observation that a guard has been 'unreachable' only lasts for +a given amount of time, since we can't infer that it's unreachable +now from the fact that it was unreachable a few minutes ago. + +## Circuit status {#CIRC-STATUS} + +Sometimes the guard selection algorithm will return a guard that we +would always be happy to use; but in other cases, the guard +selection algorithm can return a guard that we shouldn't use +without gathering additional information. + +From the point of view of guard selection, +every circuit we build is considered to be in one of these states: + + * `<state:usable_on_completion>` + * `<state:usable_if_no_better_guard>` + * `<state:waiting_for_better_guard>` + * `<state:complete>` + +You may only attach streams to `<complete>` circuits. +(Additionally, you may only send RENDEZVOUS messages, ESTABLISH_INTRO +messages, and INTRODUCE messages on `<complete>` circuits.) + +The per-circuit state machine is: + + * New circuits are `<usable_on_completion>` or + `<usable_if_no_better_guard>`. + + * A `<usable_on_completion>` circuit may become `<complete>`, or may + fail. + + * A `<usable_if_no_better_guard>` circuit may become + `<usable_on_completion>`; may become `<waiting_for_better_guard>`; or may + fail. + + * A `<waiting_for_better_guard>` circuit will become `<complete>`, or will + be closed, or will fail. + + * A `<complete>` circuit remains `<complete>` until it fails or is + closed. + +```mermaid +stateDiagram-v2 + [*] --> usable_on_completion + [*] --> usable_if_no_better_guard + usable_on_completion --> complete + usable_on_completion --> failed + usable_if_no_better_guard --> usable_on_completion + usable_if_no_better_guard --> waiting_for_better_guard + usable_if_no_better_guard --> failed + waiting_for_better_guard --> complete + waiting_for_better_guard --> failed + waiting_for_better_guard --> closed + complete --> failed + complete --> closed + failed --> [*] + closed --> [*] +``` +<!-- TODO: The above diagram does not yet render. Fix that. --> + +Each of these transitions is described in sections below. + +<a id="guard-spec.txt-4.6"></a> + +## Selecting guards for circuits. \[Section:SELECTING\] {#SELECTING} + +Now that we have described the various lists of guards, we can explain how +guards are chosen for each circuit. + +We keep, as global transient state: + + * {tvar:last_time_on_internet} -- the last time at which we + successfully used a circuit or connected to a guard. At + startup we set this to "infinitely far in the past." + +As an input to the algorithm, we take a list of _restrictions_. +These may include specific relays or families that we need to avoid. + +Here is the algorithm. It is given as a series of sub-algorithms, +in decreasing order of preference from best case to worst case. +When we want to build a circuit, and we need to pick a guard: + + * In the base case, if any entry in PRIMARY_GUARDS has {is_reachable} status of + `<maybe>` or `<yes>`, consider only such guards. Call them the + "reachable primary guards". + + Start by considering the the first {NUM_USABLE_PRIMARY_GUARDS} (or + {NUM_USABLE_PRIMARY_DIRECTORY_GUARDS}) reachable primary guards. + Remove any guards that do not follow our path selection + restrictions, to build a temporary list. If that temporary list contains at + least one guard, choose a guard from that list uniformly at + random. + + If the temporary list contains no guards, return the first guard + from our reachable primary guard that does obey the path restriction. + + When selecting a guard according to this approach, its circuit + is `<usable_on_completion>`. + + \[Note: We do not use {is_pending} on primary guards, since we + are willing to try to build multiple circuits through them + before we know for sure whether they work, and since we will + not use any non-primary guards until we are sure that the + primary guards are all down. (XX is this good?)\] + + * Otherwise, if the ordered intersection of {CONFIRMED_GUARDS} + and {USABLE_FILTERED_GUARDS} is nonempty, return the first + entry in that intersection that has {is_pending} set to + false. Set its value of {is_pending} to true, + and set its {pending_since} to the current time. + The circuit + is now `<usable_if_no_better_guard>`. (If all entries have + {is_pending} true, pick the first one.) + + * Otherwise, if there is no such entry, select a member from + {USABLE_FILTERED_GUARDS} in sample order. Set its {is_pending} field to + true, and set its {pending_since} to the current time. + The circuit is `<usable_if_no_better_guard>`. + + * Otherwise, in the worst case, if USABLE_FILTERED_GUARDS is empty, we have exhausted + all the sampled guards. In this case we proceed by marking all guards + as `<maybe>` reachable so that we can keep on trying circuits. + +Whenever we select a guard for a new circuit attempt, we update the +{last_tried_connect} time for the guard to 'now.' + +In some cases (for example, when we need a certain directory feature, +or when we need to avoid using a certain exit as a guard), we need to +restrict the guards that we use for a single circuit. When this happens, we +remember the restrictions that applied when choosing the guard for +that circuit, since we will need them later (see \[UPDATE_WAITING\].). + +**Rationale** + +We're getting to the core of the algorithm here. Our main goals are to +make sure that + + 1. If it's possible to use a primary guard, we do. + 2. We probably use the first primary guard. + +So we only try non-primary guards if we're pretty sure that all +the primary guards are down, and we only try a given primary guard +if the earlier primary guards seem down. + +When we _do_ try non-primary guards, however, we only build one +circuit through each, to give it a chance to succeed or fail. If +ever such a circuit succeeds, we don't use it until we're pretty +sure that it's the best guard we're getting. (see below). + +\[XXX timeout.\] + +<a id="guard-spec.txt-4.7"></a> + +## When a circuit fails. {#ON_FAIL} + +When a circuit fails in a way that makes us conclude that a guard +is not reachable, we take the following steps: + +```text + * Set the guard's {is_reachable} status to <no>. If it had + {is_pending} set to true, we make it non-pending and clear + {pending_since}. + + * Close the circuit, of course. (This removes it from + consideration by the algorithm in [UPDATE_WAITING].) + + * Update the list of waiting circuits. (See [UPDATE_WAITING] + below.) +``` + +\[Note: the existing Tor logic will cause us to create more +circuits in response to some of these steps; and also see +\[ON_CONSENSUS\].\] + +\[Note 2: In the case of a one-hop circuit made for a directory +request, it is possible for the request to fail _after_ the circuit +is built: for example, if we ask for the latest consensus and we are +told "404". In this case, we mark the appropriate directory-specific +`{is_reachable}` instance for that guard to `<no>`.\] + +> C tor implements the above "note 2" by treating requests for +> directory guards for as if they had an +> extra type of restriction, rather than a separate instance of +> `{is_reachable}`. (For more on restrictions, see ["Selecting +> Guards"](#SELECTING) above.) This requires the C tor +> impementation to special-case this restriction type, so that +> it is treated the same way as an `{is_reachable}` variable. + + +**Rationale** + +See \[SELECTING\] above for rationale. + +<a id="guard-spec.txt-4.8"></a> + +## When a circuit succeeds {#ON_SUCCESS} + +When a circuit succeeds in a way that makes us conclude that a +guard _was_ reachable, we take these steps: + +```text + * We set its {is_reachable} status to <yes>. + * We set its {failing_since} to "never". + * If the guard was {is_pending}, we clear the {is_pending} flag + and set {pending_since} to false. + * If the guard was not a member of {CONFIRMED_GUARDS}, we add + it to the end of {CONFIRMED_GUARDS}. + + * If this circuit was <usable_on_completion>, this circuit is + now <complete>. You may attach streams to this circuit, + and use it for hidden services. + + * If this circuit was <usable_if_no_better_guard>, it is now + <waiting_for_better_guard>. You may not yet attach streams to it. + Then check whether the {last_time_on_internet} is more than + {INTERNET_LIKELY_DOWN_INTERVAL} seconds ago: + + * If it is, then mark all {PRIMARY_GUARDS} as "maybe" + reachable. + + * If it is not, update the list of waiting circuits. (See + [UPDATE_WAITING] below) +``` + +\[Note: the existing Tor logic will cause us to create more +circuits in response to some of these steps; and see +\[ON_CONSENSUS\].\] + +**Rationale** + +See \[SELECTING\] above for rationale. + +<a id="guard-spec.txt-4.9"></a> + +## Updating the list of waiting circuits {#UPDATE_WAITING} + +We run this procedure whenever it's possible that a +`<waiting_for_better_guard>` circuit might be ready to be called +`<complete>`. + +```text + * If any circuit C1 is <waiting_for_better_guard>, AND: + * All primary guards have reachable status of <no>. + * There is no circuit C2 that "blocks" C1. + Then, upgrade C1 to <complete>. + + Definition: In the algorithm above, C2 "blocks" C1 if: + * C2 obeys all the restrictions that C1 had to obey, AND + * C2 has higher priority than C1, AND + * Either C2 is <complete>, or C2 is <waiting_for_better_guard>, + or C2 has been <usable_if_no_better_guard> for no more than + {NONPRIMARY_GUARD_CONNECT_TIMEOUT} seconds. + + We run this procedure periodically: + + * If any circuit stays in <waiting_for_better_guard> + for more than {NONPRIMARY_GUARD_IDLE_TIMEOUT} seconds, + time it out. + + **Rationale** +``` + +If we open a connection to a guard, we might want to use it +immediately (if we're sure that it's the best we can do), or we +might want to wait a little while to see if some other circuit +which we like better will finish. + +When we mark a circuit `<complete>`, we don't close the +lower-priority circuits immediately: we might decide to use +them after all if the `<complete>` circuit goes down before +{NONPRIMARY_GUARD_IDLE_TIMEOUT} seconds. + +<a id="guard-spec.txt-4.9.1"></a> + +### Without a list of waiting circuits {#NO_CIRCLIST} + +As an alternative to the section \[SECTION:UPDATE_WAITING\] above, +this section presents a new way to maintain guard status +independently of tracking individual circuit status. This +formulation gives a result equivalent or similar to the approach +above, but simplifies the necessary communications between the +guard and circuit subsystems. + +As before, when all primary guards are Unreachable, we need to +try non-primary guards. We select the first such guard (in +preference order) that is neither Unreachable nor Pending. +Whenever we give out such a guard, if the guard's status is +Unknown, then we call that guard "Pending" with its {is_pending} +flag, until the attempt to use it succeeds or fails. We remember +when the guard became Pending with the {pending_since variable}. + +After completing a circuit, the implementation must check whether +its guard is usable. A guard's usability status may be "usable", +"unusable", or "unknown". A guard is usable according to +these rules: + +1. Primary guards are always usable. + +1. Non-primary guards are usable _for a given circuit_ if every + guard earlier in the preference list is either unsuitable for + that circuit (e.g. because of family restrictions), or marked as + Unreachable, or has been pending for at least + `{NONPRIMARY_GUARD_CONNECT_TIMEOUT}`. + + Non-primary guards are not usable _for a given circuit_ if some + guard earlier in the preference list is suitable for the circuit + _and_ Reachable. + + Non-primary guards are unusable if they have not become + usable after `{NONPRIMARY_GUARD_IDLE_TIMEOUT}` seconds. + +1. If a circuit's guard is not usable or unusable immediately, the + circuit is not discarded; instead, it is kept (but not used) until the + guard becomes usable or unusable. + +<a id="guard-spec.txt-4.10"></a> + +## Whenever we get a new consensus. {#ON_CONSENSUS} + +We update {GUARDS}. + +For every guard in {SAMPLED_GUARDS}, we update {IS_LISTED} and +{FIRST_UNLISTED_AT}. + +\[\*\*\] We remove entries from {SAMPLED_GUARDS} if appropriate, +according to the sampled-guards expiration rules. If they were +in {CONFIRMED_GUARDS}, we also remove them from +{CONFIRMED_GUARDS}. + +We recompute {FILTERED_GUARDS}, and everything that derives from +it, including {USABLE_FILTERED_GUARDS}, and {PRIMARY_GUARDS}. + +(Whenever one of the configuration options that affects the +filter is updated, we repeat the process above, starting at the +\[\*\*\] line.) + +```text +4.11. Deciding whether to generate a new circuit. + [Section:NEW_CIRCUIT_NEEDED] +``` + +We generate a new circuit when we don't have +enough circuits either built or in-progress to handle a given +stream, or an expected stream. + +For the purpose of this rule, we say that `<waiting_for_better_guard>` +circuits are neither built nor in-progress; that `<complete>` +circuits are built; and that the other states are in-progress. + +```text +4.12. When we are missing descriptors. + [Section:MISSING_DESCRIPTORS] +``` + +We need either a router descriptor or a microdescriptor in order +to build a circuit through a guard. If we do not have such a +descriptor for a guard, we can still use the guard for one-hop +directory fetches, but not for longer circuits. + +(Also, when we are missing descriptors for our first +{NUM_USABLE_PRIMARY_GUARDS} primary guards, we don't build +circuits at all until we have fetched them.) diff --git a/spec/guard-spec/appendices.md b/spec/guard-spec/appendices.md new file mode 100644 index 0000000..3aaa4e3 --- /dev/null +++ b/spec/guard-spec/appendices.md @@ -0,0 +1,152 @@ +<a id="guard-spec.txt-A"></a> + +# Appendices + +<a id="guard-spec.txt-A.0"></a> + +## Acknowledgements + +This research was supported in part by NSF grants CNS-1111539, +CNS-1314637, CNS-1526306, CNS-1619454, and CNS-1640548. + +<a id="guard-spec.txt-A.1"></a> + +## Parameters with suggested values. {#PARAM_VALS} + +(All suggested values chosen arbitrarily) + +{param:MAX_SAMPLE_THRESHOLD} -- 20% + +{param:MAX_SAMPLE_SIZE} -- 60 + +{param:GUARD_LIFETIME} -- 120 days + +```text + {param:REMOVE_UNLISTED_GUARDS_AFTER} -- 20 days + [previously ENTRY_GUARD_REMOVE_AFTER] + + {param:MIN_FILTERED_SAMPLE} -- 20 + + {param:N_PRIMARY_GUARDS} -- 3 + + {param:PRIMARY_GUARDS_RETRY_SCHED} + + We recommend the following schedule, which is the one + used in Arti: + + -- Use the "decorrelated-jitter" algorithm from "dir-spec.txt" + section 5.5 where `base_delay` is 30 seconds and `cap` + is 6 hours. + + This legacy schedule is the one used in C tor: + + -- every 10 minutes for the first six hours, + -- every 90 minutes for the next 90 hours, + -- every 4 hours for the next 3 days, + -- every 9 hours thereafter. + + {param:GUARDS_RETRY_SCHED} -- + + We recommend the following schedule, which is the one + used in Arti: + + -- Use the "decorrelated-jitter" algorithm from "dir-spec.txt" + section 5.5 where `base_delay` is 10 minutes and `cap` + is 36 hours. + + This legacy schedule is the one used in C tor: + + -- every hour for the first six hours, + -- every 4 hours for the 90 hours, + -- every 18 hours for the next 3 days, + -- every 36 hours thereafter. + + {param:INTERNET_LIKELY_DOWN_INTERVAL} -- 10 minutes + + {param:NONPRIMARY_GUARD_CONNECT_TIMEOUT} -- 15 seconds + + {param:NONPRIMARY_GUARD_IDLE_TIMEOUT} -- 10 minutes + + {param:MEANINGFUL_RESTRICTION_FRAC} -- .2 + + {param:EXTREME_RESTRICTION_FRAC} -- .01 + + {param:GUARD_CONFIRMED_MIN_LIFETIME} -- 60 days + + {param:NUM_USABLE_PRIMARY_GUARDS} -- 1 + + {param:NUM_USABLE_PRIMARY_DIRECTORY_GUARDS} -- 3 +``` + +<a id="guard-spec.txt-A.2"></a> + +## Random values {#RANDOM} + +Frequently, we want to randomize the expiration time of something +so that it's not easy for an observer to match it to its start +time. We do this by randomizing its start date a little, so that +we only need to remember a fixed expiration interval. + +By RAND(now, INTERVAL) we mean a time between now and INTERVAL in +the past, chosen uniformly at random. + +<a id="guard-spec.txt-A.5"></a> + +## Persistent state format {#persistent-state} + +The persistent state format doesn't need to be part of this +specification, since different implementations can do it +differently. Nonetheless, here's the one Tor uses: + +The "state" file contains one Guard entry for each sampled guard +in each instance of the guard state (see section 2). The value +of this Guard entry is a set of space-separated K=V entries, +where K contains any nonspace character except =, and V contains +any nonspace characters. + +Implementations must retain any unrecognized K=V entries for a +sampled guard when they regenerate the state file. + +The order of K=V entries is not allowed to matter. + +Recognized fields (values of K) are: + +```text + "in" -- the name of the guard state instance that this + sampled guard is in. If a sampled guard is in two guard + states instances, it appears twice, with a different "in" + field each time. Required. + + "rsa_id" -- the RSA id digest for this guard, encoded in + hex. Required. + + "bridge_addr" -- If the guard is a bridge, its configured address and + port (this can be the ORPort or a pluggable transport port). Optional. + + "nickname" -- the guard's nickname, if any. Optional. + + "sampled_on" -- the date when the guard was sampled. Required. + + "sampled_by" -- the Tor version that sampled this guard. + Optional. + + "unlisted_since" -- the date since which the guard has been + unlisted. Optional. + + "listed" -- 0 if the guard is not listed; 1 if it is. Required. + + "confirmed_on" -- date when the guard was + confirmed. Optional. + + "confirmed_idx" -- position of the guard in the confirmed + list. Optional. + + "pb_use_attempts", "pb_use_successes", "pb_circ_attempts", + "pb_circ_successes", "pb_successful_circuits_closed", + "pb_collapsed_circuits", "pb_unusable_circuits", + "pb_timeouts" -- state for the circuit path bias algorithm, + given in decimal fractions. Optional. +``` + +All dates here are given as a (spaceless) ISO8601 combined date +and time in UTC (e.g., 2016-11-29T19:39:31). diff --git a/spec/guard-spec/guard-selection/index.md b/spec/guard-spec/guard-selection/index.md new file mode 100644 index 0000000..c3738d4 --- /dev/null +++ b/spec/guard-spec/guard-selection/index.md @@ -0,0 +1,72 @@ +<a id="guard-spec.txt-3"></a> + +# Circuit Creation, Entry Guard Selection (1000 foot view) {#1000-foot} + +A circuit in Tor is a path through the network connecting a client to +its destination. At a high-level, a three-hop exit circuit will look +like this: + +Client \<-> Entry Guard \<-> Middle Node \<-> Exit Node \<-> Destination + +Entry guards are the only nodes which a client will connect to +directly. Exit relays are the nodes by which traffic exits the +Tor network in order to connect to an external destination. + +## Path selection + +For any multi-hop circuit, at least one entry guard and middle node(s) are +required. An exit node is required if traffic will exit the Tor +network. Depending on its configuration, a relay listed in a +consensus could be used for any of these roles. However, this +specification defines how entry guards specifically should be selected and +managed, as opposed to middle or exit nodes. + +### Managing entry guards + +At a high level, a relay listed in a consensus will move through the +following states in the process from initial selection to eventual +usage as an entry guard: + +```text + relays listed in consensus + | + sampled + | | + confirmed filtered + | | | + primary usable_filtered +``` + +Relays listed in the latest consensus can be sampled for guard usage +if they have the "Guard" flag. Sampling is random but weighted by +a measured bandwidth multiplied by bandwidth-weights (Wgg if guard only, +Wgd if guard+exit flagged). + +Once a path is built and a circuit established using this guard, it +is marked as confirmed. Until this point, guards are first sampled +and then filtered based on information such as our current +configuration (see SAMPLED and FILTERED sections) and later marked as +usable_filtered if the guard is not primary but can be reached. + +It is always preferable to use a primary guard when building a new +circuit in order to reduce guard churn; only on failure to connect to +existing primary guards will new guards be used. + +### Middle and exit node selection + +Middle nodes are selected at random from relays listed in the latest +consensus, weighted by bandwidth and bandwidth-weights. Exit nodes are +chosen similarly but restricted to relays with a sufficiently permissive +exit policy. + +## Circuit Building + +Once a path is chosen, Tor will use this path to build a new circuit. + +If the circuit is built successfully, Tor will either use it +immediately, or Tor will wait for a circuit with a more preferred +guard if there's a good chance that it will be able to make one. + +If the circuit fails in a way that makes us conclude that a guard +is not reachable, the guard is marked as unreachable, the circuit is +closed, and waiting circuits are updated. diff --git a/spec/guard-spec/index.md b/spec/guard-spec/index.md new file mode 100644 index 0000000..e9167bf --- /dev/null +++ b/spec/guard-spec/index.md @@ -0,0 +1,47 @@ +# Tor Guard Specification + +<a id="guard-spec.txt-1"></a> + +## Introduction and motivation + +Tor uses entry guards to prevent an attacker who controls some +fraction of the network from observing a fraction of every user's +traffic. If users chose their entries and exits uniformly at +random from the list of servers every time they build a circuit, +then an adversary who had (k/N) of the network would deanonymize +F=(k/N)^2 of all circuits... and after a given user had built C +circuits, the attacker would see them at least once with +probability 1-(1-F)^C. With large C, the attacker would get a +sample of every user's traffic with probability 1. + +To prevent this from happening, Tor clients choose a small number +of guard nodes (e.g. 3). These guard nodes are the only +nodes that the client will connect to directly. If they are not +compromised, the user's paths are not compromised. + +This specification outlines Tor's guard housekeeping algorithm, +which tries to meet the following goals: + +```text + - Heuristics and algorithms for determining how and which guards + are chosen should be kept as simple and easy to understand as + possible. + + - Clients in censored regions or who are behind a fascist + firewall who connect to the Tor network should not experience + any significant disadvantage in terms of reachability or + usability. + + - Tor should make a best attempt at discovering the most + appropriate behavior, with as little user input and + configuration as possible. + + - Tor clients should discover usable guards without too much + delay. + + - Tor clients should resist (to the extent possible) attacks + that try to force them onto compromised guards. + + - Should maintain the load-balancing offered by the path selection + algorithm +``` diff --git a/spec/guard-spec/state-instances.md b/spec/guard-spec/state-instances.md new file mode 100644 index 0000000..5449648 --- /dev/null +++ b/spec/guard-spec/state-instances.md @@ -0,0 +1,49 @@ +<a id="guard-spec.txt-2"></a> + +# State instances + +In the algorithm below, we describe a set of persistent and +non-persistent state variables. These variables should be +treated as an object, of which multiple instances can exist. + +In particular, we specify the use of three particular instances: + +A. UseBridges + +```text + If UseBridges is set, then we replace the {GUARDS} set in + [Sec:GUARDS] below with the list of configured + bridges. We maintain a separate persistent instance of + {SAMPLED_GUARDS} and {CONFIRMED_GUARDS} and other derived + values for the UseBridges case. + + In this case, we impose no upper limit on the sample size. + + B. EntryNodes / ExcludeNodes / Reachable*Addresses / + FascistFirewall / ClientUseIPv4=0 + + If one of the above options is set, and UseBridges is not, + then we compare the fraction of usable guards in the consensus + to the total number of guards in the consensus. + + If this fraction is less than {MEANINGFUL_RESTRICTION_FRAC}, + we use a separate instance of the state. + + (While Tor is running, we do not change back and forth between + the separate instance of the state and the default instance + unless the fraction of usable guards is 5% higher than, or 5% + lower than, {MEANINGFUL_RESTRICTION_FRAC}. This prevents us + from flapping back and forth between instances if we happen to + hit {MEANINGFUL_RESTRICTION_FRAC} exactly. + + If this fraction is less than {EXTREME_RESTRICTION_FRAC}, we use a + separate instance of the state, and warn the user. + + [TODO: should we have a different instance for each set of heavily + restricted options?] + + C. Default + + If neither of the above variant-state instances is used, + we use a default instance. +``` diff --git a/spec/hspow-spec/analysis-discussion.md b/spec/hspow-spec/analysis-discussion.md new file mode 100644 index 0000000..6585d3f --- /dev/null +++ b/spec/hspow-spec/analysis-discussion.md @@ -0,0 +1,416 @@ +# Analysis and discussion + +*Warning*: Take all the PoW performance numbers on this page with a large grain of salt. Most of this is based on very early analysis that has not been updated for the current state of implementation. + +For current performance numbers on a specific piece of hardware, please run `cargo bench` from the [`equix/bench`](https://gitlab.torproject.org/tpo/core/arti/-/tree/main/crates/equix/bench) crate within [Arti](https://gitlab.torproject.org/tpo/core/arti/). This framework tests both the C and Rust implementations side-by-side. + +## Attacker strategies {#attacker-strategies} + +To design a protocol and choose its parameters, we first need to understand a few high-level attacker strategies to see what we are fighting against. + +### Overwhelm PoW verification ("top half") {#attack-top-half} + +A basic attack here is the adversary spamming with bogus INTRODUCE messages so that the service does not have computing capacity to even verify the proof-of-work. This adversary tries to overwhelm the procedure in the [`v1` verification algorithm](./v1-equix.md#service-verify) section. + +That's why we need the PoW algorithm to have a cheap verification time so that this attack is not possible: we explore this PoW parameter below in the section on [PoW verification](#pow-tuning-verification). + +### Overwhelm rendezvous capacity ("bottom half") {#attack-bottom-half} + +Given the way [the introduction queue](./common-protocol.md#intro-queue) works, a very effective strategy for the attacker is to totally overwhelm the queue processing by sending more high-effort introductions than the onion service can handle at any given tick. +This adversary tries to overwhelm the process of [handling queued introductions](./common-protocol.md#handling-queue). + +To do so, the attacker would have to send at least 20 high-effort INTRODUCE messages every 100ms, where high-effort is a PoW which is above the estimated level of ["the motivated user"](./motivation.md#user-profiles). + +An easier attack for the adversary, is the same strategy but with INTRODUCE messages that are all above the comfortable level of ["the standard user"](./motivation.md#user-profiles). +This would block out all standard users and only allow motivated users to pass. + +### Hybrid overwhelm strategy {#attack-hybrid} + +If both the top- and bottom- halves are processed by the same thread, this opens up the possibility for a "hybrid" attack. +Given the performance figures for the bottom half (0.31 ms/req.) and the top half (5.5 ms/req.), the attacker can optimally deny service by submitting 91 high-effort requests and 1520 invalid requests per second. +This will completely saturate the main loop because: + +```text + 0.31*(1520+91) ~ 0.5 sec. + 5.5*91 ~ 0.5 sec. +``` + +This attack only has half the bandwidth requirement of a [top-half attack](#attack-top-half) and half the compute requirement of a [bottom-half attack](#attack-bottom-half).. + +Alternatively, the attacker can adjust the ratio between invalid and high-effort requests depending on their bandwidth and compute capabilities. + +### Gaming the effort control logic {#attack-effort} + +Another way to beat this system is for the attacker to game the [effort control logic](./common-protocol.md#effort-control). Essentially, there are two attacks that we are trying to avoid: + +- Attacker sets descriptor suggested-effort to a very high value effectively making it impossible for most clients to produce a PoW token in a reasonable timeframe. +- Attacker sets descriptor suggested-effort to a very small value so that most clients aim for a small value while the attacker comfortably launches an [bottom-half attack](#attack-bottom-half) using medium effort PoW (see [this post by tevador on tor-dev from May 2020](https://lists.torproject.org/pipermail/tor-dev/2020-May/014268.html)). + +### Precomputed PoW attack {#attack-precomputed} + +The attacker may precompute many valid PoW nonces and submit them all at once before the current seed expires, overwhelming the service temporarily even using a single computer. +The current scheme gives the attackers 4 hours to launch this attack since each seed lasts 2 hours and the service caches two seeds. + +An attacker with this attack might be aiming to DoS the service for a limited amount of time, or to cause an [effort control attack](#attack-effort). + +## Parameter tuning {#parameter-tuning} + +There are various parameters in this PoW system that need to be tuned: + +We first start by tuning the time it takes to verify a PoW token. +We do this first because it's fundamental to the performance of onion services and can turn into a DoS vector of its own. We will do this tuning in a way that's agnostic to the chosen PoW function. + +We previously considered the concept of a nonzero starting difficulty setting. This analysis still references such a concept, even though the currently recommended implementation uses a starting effort of zero. (We now expect early increases in effort during an attack to be driven primarily by client retry behavior.) + +At the end of this section we will estimate the resources that an attacker needs to overwhelm the onion service, the resources that the service needs to verify introduction requests, and the resources that legitimate clients need to get to the onion service. + +### PoW verification {#pow-tuning-verification} + +Verifying a PoW token is the first thing that a service does when it receives an INTRODUCE2 message. Our current implementation is described by the [`v1` verification algorithm](./v1-equix.md#service-verify) specification. + +Verification time is a critical performance parameter. Actual times can be measured by `cargo bench` now, and the verification speeds we achieve are more like 50-120 microseconds. The specific numbers below are dated, but the analysys below is preserved as an illustration of the design space we are optimizing within. + +To defend against a [top-half attack](#attack-top-half) it's important that we can quickly perform all the steps in-between receiving an introduction request over the network and adding it to our effort-prioritized queue. + +All time spent verifying PoW adds overhead to the already existing "top half" part of handling an INTRODUCE message. +Hence we should be careful to add minimal overhead here. + +During our [performance measurements on tor](#tor-measurements) we learned that the "top half" takes about 0.26 msecs in average, without doing any sort of PoW verification. +Using that value we compute the following table, that describes the number of requests we can queue per second (aka times we can perform the "top half" process) for different values of PoW verification time: + +| PoW Verification Time | Total "top half" time | Requests Queued per second +| --------------------- | --------------------- | ----------------------- +| 0 msec | 0.26 msec | 3846 +| 1 msec | 1.26 msec | 793 +| 2 msec | 2.26 msec | 442 +| 3 msec | 3.26 msec | 306 +| 4 msec | 4.26 msec | 234 +| 5 msec | 5.26 msec | 190 +| 6 msec | 6.26 msec | 159 +| 7 msec | 7.26 msec | 137 +| 8 msec | 8.26 msec | 121 +| 9 msec | 9.26 msec | 107 +| 10 msec | 10.26 msec | 97 + +Here is how you can read the table above: + +- For a PoW function with a 1ms verification time, an attacker needs to send 793 dummy introduction requests per second to succeed in a [top-half attack](#attack-top-half). +- For a PoW function with a 2ms verification time, an attacker needs to send 442 dummy requests per second to succeed in a [top-half attack](#attack-top-half). +- For a PoW function with a 10ms verification time, an attacker needs to send 97 dummy requests per second to succeed in a [top-half attack](#attack-top-half). + +Whether an attacker can succeed at that depends on the attacker's resources, but also on the network's capacity. + +Our purpose here is to have the smallest PoW verification overhead possible that also allows us to achieve all our other goals. + +Note that the table above is simply the result of a naive multiplication and does not take into account all the auxiliary overheads that happen every second like the time to invoke the mainloop, the bottom-half processes, or pretty much anything other than the "top-half" processing. + +During our measurements the time to handle introduction requests dominates any other action time: +There might be events that require a long processing time, but these are pretty infrequent (like uploading a new HS descriptor) and hence over a long time they smooth out. +Hence extrapolating the total introduction requests queued per second based on a single "top half" time seems like good enough to get some initial intuition. +That said, the values of "Requests queued per second" from the table above, are likely much smaller than displayed above because of all the auxiliary overheads. + +### PoW difficulty analysis {#pow-difficulty-analysis} + +The difficulty setting of our PoW basically dictates how difficult it should be to get a success in our PoW system. +An attacker who can get many successes per second can pull a successful [bottom-half attack](#attack-bottom-half) against our system. + +In classic PoW systems, "success" is defined as getting a hash output below the "target". +However, since our system is dynamic, we define "success" as an abstract high-effort computation. + +The original analysis here concluded that we still need a starting difficulty setting that will be used for bootstrapping the system. +The client and attacker can still aim higher or lower but for UX purposes and for analysis purposes it was useful to define a starting difficulty, to minimize retries by clients. + +In current use it was found that an effort of 1 makes a fine minimum, so we don't normally have a concept of minimum effort. Consider the actual "minimum effort" in `v1` now to simply be the expected runtime of one single Equi-X solve. + +#### Analysis based on adversary power {#pow-difficulty-adversary} + +In this section we will try to do an analysis of PoW difficulty without using any sort of Tor-related or PoW-related benchmark numbers. + +We created the table (see `[REF_TABLE]`) below which shows how much time a legitimate client with a single machine should expect to burn before they get a single success. + +The x-axis is how many successes we want the attacker to be able to do per second: +the more successes we allow the adversary, the more they can overwhelm our introduction queue. +The y-axis is how many machines the adversary has in her disposal, ranging from just 5 to 1000. + +```text + =============================================================== + | Expected Time (in seconds) Per Success For One Machine | + =========================================================================== + | | + | Attacker Succeses 1 5 10 20 30 50 | + | per second | + | | + | 5 5 1 0 0 0 0 | + | 50 50 10 5 2 1 1 | + | 100 100 20 10 5 3 2 | + | Attacker 200 200 40 20 10 6 4 | + | Boxes 300 300 60 30 15 10 6 | + | 400 400 80 40 20 13 8 | + | 500 500 100 50 25 16 10 | + | 1000 1000 200 100 50 33 20 | + | | + ============================================================================ +``` + +Here is how you can read the table above: + +- If an adversary has a botnet with 1000 boxes, and we want to limit her to 1 success per second, then a legitimate client with a single box should be expected to spend 1000 seconds getting a single success. +- If an adversary has a botnet with 1000 boxes, and we want to limit her to 5 successes per second, then a legitimate client with a single box should be expected to spend 200 seconds getting a single success. +- If an adversary has a botnet with 500 boxes, and we want to limit her to 5 successes per second, then a legitimate client with a single box should be expected to spend 100 seconds getting a single success. +- If an adversary has access to 50 boxes, and we want to limit her to 5 successes per second, then a legitimate client with a single box should be expected to spend 10 seconds getting a single success. +- If an adversary has access to 5 boxes, and we want to limit her to 5 successes per second, then a legitimate client with a single box should be expected to spend 1 seconds getting a single success. + +With the above table we can create some profiles for starting values of our PoW difficulty. + +#### Analysis based on Tor's performance {#pow-difficulty-tor} + +To go deeper here, we can use the [performance measurements on tor](#tor-measurements) to get a more specific intuition on the starting difficulty. +In particular, we learned that completely handling an introduction request takes 5.55 msecs in average. +Using that value, we can compute the following table, that describes the number of introduction requests we can handle per second for different values of PoW verification: + +| PoW Verification Time | Total time to handle request | Requests handled per second +| --------------------- | ---------------------------- | ------------------------ +| 0 msec | 5.55 msec | 180.18 +| 1 msec | 6.55 msec | 152.67 +| 2 msec | 7.55 msec | 132.45 +| 3 msec | 8.55 msec | 116.96 +| 4 msec | 9.55 mesc | 104.71 +| 5 msec | 10.55 msec | 94.79 +| 6 msec | 11.55 msec | 86.58 +| 7 msec | 12.55 msec | 79.68 +| 8 msec | 13.55 msec | 73.80 +| 9 msec | 14.55 msec | 68.73 +| 10 msec | 15.55 msec | 64.31 + +Here is how you can read the table above: + +- For a PoW function with a 1ms verification time, an attacker needs to send 152 high-effort introduction requests per second to succeed in a [bottom-half attack](#attack-bottom-half) attack. +- For a PoW function with a 10ms verification time, an attacker needs to send 64 high-effort introduction requests per second to succeed in a [bottom-half attack](#attack-bottom-half) attack. + +We can use this table to specify a starting difficulty that won't allow our target adversary to succeed in an [bottom-half attack](#attack-bottom-half) attack. + +Note that in practice verification times are much lower; the scale of the above table does not match the current implementation's reality. + +## User experience {#ux} + +This proposal has user facing UX consequences. + +When the client first attempts a pow, it can note how long iterations of the hash function take, and then use this to determine an estimation of the duration of the PoW. +This estimation could be communicated via the control port or other mechanism, such that the browser could display how long the PoW is expected to take on their device. +If the device is a mobile platform, and this time estimation is large, it could recommend that the user try from a desktop machine. + +## Future work {#future-work} + +### Incremental improvements to this proposal + +There are various improvements that can be done in this proposal, and while we are trying to keep this `v1` version simple, we need to keep the design extensible so that we build more features into it. In particular: + +- End-to-end introduction ACKs + + This proposal suffers from various UX issues because there is no end-to-end + mechanism for an onion service to inform the client about its introduction + request. + If we had end-to-end introduction ACKs many of the problems seen in [client-side effort estimation](./common-protocol.md#client-effort) would be alleviated. + The problem here is that end-to-end ACKs require modifications on the introduction point code and a network update which is a lengthy process. + +- Multithreading scheduler + + Our scheduler is pretty limited by the fact that Tor has a single-threaded design. + If we improve our multithreading support we could handle a much greater amount of introduction requests per second. + +### Future designs {#future-designs} + +This is just the beginning in DoS defences for Tor and there are various future designs and schemes that we can investigate. Here is a brief summary of these: + +- "More advanced PoW schemes" -- + We could use more advanced memory-hard PoW schemes like MTP-argon2 or Itsuku to make it even harder for adversaries to create successful PoWs. Unfortunately these schemes have much bigger proof sizes, and they won't fit in INTRODUCE1 messages. See #31223 for more details. + +- "Third-party anonymous credentials" -- + We can use anonymous credentials and a third-party token issuance server on the clearnet to issue tokens based on PoW or CAPTCHA and then use those tokens to get access to the service. See `[REF_CREDS]` for more details. + +- "PoW + Anonymous Credentials" -- + We can make a hybrid of the above ideas where we present a hard puzzle to the user when connecting to the onion service, and if they solve it we then give the user a bunch of anonymous tokens that can be used in the future. + This can all happen between the client and the service without a need for a third party. + +All of the above approaches are much more complicated than the `v1` design, and hence we want to start easy before we get into more serious projects. +The current implementation requires complexity within the Equi-X implementation but its impact on the overall tor network can be relatively simple. + +## Environment {#environment} + +This algorithm shares a broad concept, proof of work, with some notoriously power hungry and wasteful software. We love the environment, and we too are concerned with how proof of work schemes typically waste huge amounts of energy by doing useless hash iterations. + +Nevertheless, there are some massive differences in both the scale and the dynamics of what we are doing here: we are performing fairly small amounts of computation, and it's used as part of a scheme to disincentivize attacks entirely. If we do our job well, people stop computing these proof-of-work functions entirely and find something else to attack. + +We think we aren't making a bad situation worse: DoS attacks on the Tor network are already happening and attackers are already burning energy to carry them out. +As we see in the [denial-of-service overview](../dos-spec/overview.md#hs-intro), attacks on onion services are in a position to cause downstream resource consumption of nearly every type. +Each relay involved experiences increased CPU load from the circuit floods they process. +We think that asking legitimate clients to carry out PoW computations doesn't affect the equation too much, since an attacker right now can very quickly use the same resources that hundreds of legitimate clients do in a whole day. + +We hope to make things better: The hope is that systems like this will make the DoS actors go away and hence the PoW system will not be used. +As long as DoS is happening there will be a waste of energy, but if we manage to demotivate them with technical means, the network as a whole will less wasteful. +Also see [The DoS Catch-22](./motivation.md#catch22). + +## Acknowledgements {#acknowledgements} + +Thanks a lot to tevador for the various improvements to the proposal and for helping us understand and tweak the RandomX scheme. + +Thanks to Solar Designer for the help in understanding the current PoW landscape, the various approaches we could take, and teaching us a few neat tricks. + +## Scheduler implementation for C tor {#tor-scheduler} + +This section describes how we will implement this proposal in the "tor" software (little-t tor). + +The following should be read as if tor is an onion service and thus the end point of all inbound data. + +### The Main Loop {#tor-main-loop} + +Tor uses libevent for its mainloop. +For network I/O operations, a mainloop event is used to inform tor if it can read on a certain socket, or a connection object in tor. + +From there, this event will empty the connection input buffer (inbuf) by extracting and processing a request at a time. +The mainloop is single threaded and thus each request is handled sequentially. + +Processing an INTRODUCE2 message at the onion service means a series of operations (in order): + +1. Unpack relay cell from inbuf to local buffer. +2. Decrypt cell (AES operations). +3. Parse relay message and process it depending on its RELAY_COMMAND. +4. INTRODUCE2 handling which means building a rendezvous circuit: + - Path selection + - Launch circuit to first hop. +5. Return to mainloop event which essentially means back to step (1). + +Tor will read at most 32 messages out of the inbuf per mainloop round. + +### Requirements for PoW {#tor-pow-queue} + +With this proposal, in order to prioritize requests by the amount of PoW work +it has done, requests can *not* be processed sequentially as described above. + +Thus, we need a way to queue a certain number of requests, prioritize them and then process some request(s) from the top of the queue (that is, the requests that have done the most PoW effort). + +We thus require a new request processing flow that is *not* compatible with current tor design. The elements are: + +- Validate PoW and place requests in a priority queue of introduction requests ([the introduction queue](./common-protocol.md#intro-queue)). +- Defer "bottom half" request processing for after requests have been queued into the priority queue. + +### Proposed scheduler {#tor-scheduler} + +The intuitive way to address the [queueing requirements](#tor-pow-queue) above would be to do this simple and naive approach: + +1. Mainloop: Empty inbuf of introduction requests into priority queue +2. Process all requests in pqueue +3. Goto (1) + +However, we are worried that handling all those requests before returning to the mainloop opens possibilities of attack by an adversary since the priority queue is not gonna be kept up to date while we process all those requests. +This means that we might spend lots of time dealing with introductions that don't deserve it. + +We thus propose to split the INTRODUCE2 handling into two different steps: "top half" and "bottom half" process. + +#### Top half and bottom half {#top-half-bottom-half} + +The top half process is responsible for queuing introductions into the priority queue as follows: + +1. Unpack cell from inbuf to local buffer. +2. Decrypt cell (AES operations). +3. Parse INTRODUCE2 message and validate PoW. +4. Return to mainloop event which essentially means step (1). + +The top-half basically does all operations from the [main loop](#tor-main-loop) section above, excepting (4). + +An then, the bottom-half process is responsible for handling introductions and doing rendezvous. +To achieve this we introduce a new mainloop event to process the priority queue _after_ the top-half event has completed. +This new event would do these operations sequentially: + +1. Pop INTRODUCE2 message from priority queue. +2. Parse and process INTRODUCE2 message. +3. End event and yield back to mainloop. + +#### Scheduling the bottom half process {#sched-bottom-half} + +The question now becomes: when should the "bottom half" event get triggered from the mainloop? + +We propose that this event is scheduled in when the network I/O event queues at least 1 request into the priority queue. Then, as long as it has a request in the queue, it would re-schedule itself for immediate execution meaning at the next mainloop round, it would execute again. + +The idea is to try to empty the queue as fast as it can in order to provide a fast response time to an introduction request but always leave a chance for more requests to appear between request processing by yielding back to the mainloop. +With this we are aiming to always have the most up-to-date version of the priority queue when we are completing introductions: +this way we are prioritizing clients that spent a lot of time and effort completing their PoW. + +If the size of the queue drops to 0, it stops scheduling itself in order to not create a busy loop. +The network I/O event will re-schedule it in time. + +Notice that the proposed solution will make the service handle 1 single introduction request at every main loop event. +However, when we do performance measurements we might learn that it's preferable to bump the number of requests in the future from 1 to N where N <= 32. + +## Performance measurements + +This section will detail the performance measurements we've done on `tor.git` for handling an INTRODUCE2 message and then a discussion on how much more CPU time we can add (for PoW validation) before it badly degrades our performance. + +### Tor measurements {#tor-measurements} + +In this section we will derive measurement numbers for the "top half" and "bottom half" parts of handling an introduction request. + +These measurements have been done on tor.git at commit +`80031db32abebaf4d0a91c01db258fcdbd54a471`. + +We've measured several set of actions of the INTRODUCE2 message handling process on Intel(R) Xeon(R) CPU E5-2650 v4. +Our service was accessed by an array of clients that sent introduction requests for a period of 60 seconds. + +1. Full Mainloop Event + + We start by measuring the full time it takes for a mainloop event to process an inbuf containing INTRODUCE2 messages. The mainloop event processed 2.42 messages per invocation on average during our measurements. + + ```text + Total measurements: 3279 + + Min: 0.30 msec - 1st Q.: 5.47 msec - Median: 5.91 msec + Mean: 13.43 msec - 3rd Q.: 16.20 msec - Max: 257.95 msec + ``` + +2. INTRODUCE2 message processing (bottom-half) + + We also measured how much time the "bottom half" part of the process takes. + That's the heavy part of processing an introduction request as seen in step (4) of the [main loop](#tor-main-loop) section above: + + ```text + Total measurements: 7931 + + Min: 0.28 msec - 1st Q.: 5.06 msec - Median: 5.33 msec + Mean: 5.29 msec - 3rd Q.: 5.57 msec - Max: 14.64 msec + ``` + +3. Connection data read (top half) + + Now that we have the above pieces, we can use them to measure just the "top half" part of the procedure. + That's when bytes are taken from the connection inbound buffer and parsed into an INTRODUCE2 message where basic validation is done. + + There is an average of 2.42 INTRODUCE2 messages per mainloop event and so we divide that by the full mainloop event mean time to get the time for one message. + From that we subtract the "bottom half" mean time to get how much the "top half" takes: + + ```text + => 13.43 / (7931 / 3279) = 5.55 + => 5.55 - 5.29 = 0.26 + + Mean: 0.26 msec + ``` + +To summarize, during our measurements the average number of INTRODUCE2 messages a mainloop event processed is ~2.42 messages (7931 messages for 3279 mainloop invocations). + +This means that, taking the mean of mainloop event times, it takes ~5.55msec (13.43/2.42) to completely process an INTRODUCE2 messages. +Then if we look deeper we see that the "top half" of INTRODUCE2 message processing takes 0.26 msec in average, whereas the "bottom half" takes around 5.33 msec. + +The heavyness of the "bottom half" is to be expected since that's where 95% of the total work takes place: in particular the rendezvous path selection and circuit launch. + +## References + +```text + [REF_EQUIX]: https://github.com/tevador/equix + https://github.com/tevador/equix/blob/master/devlog.md + [REF_TABLE]: The table is based on the script below plus some manual editing for readability: + https://gist.github.com/asn-d6/99a936b0467b0cef88a677baaf0bbd04 + [REF_BOTNET]: https://media.kasperskycontenthub.com/wp-content/uploads/sites/43/2009/07/01121538/ynam_botnets_0907_en.pdf + [REF_CREDS]: https://lists.torproject.org/pipermail/tor-dev/2020-March/014198.html + [REF_TARGET]: https://en.bitcoin.it/wiki/Target + [REF_TEVADOR_2]: https://lists.torproject.org/pipermail/tor-dev/2020-June/014358.html + [REF_TEVADOR_SIM]: https://github.com/mikeperry-tor/scratchpad/blob/master/tor-pow/effort_sim.py#L57 +``` diff --git a/spec/hspow-spec/common-protocol.md b/spec/hspow-spec/common-protocol.md new file mode 100644 index 0000000..cd98643 --- /dev/null +++ b/spec/hspow-spec/common-protocol.md @@ -0,0 +1,203 @@ +# Common protocol + +We have made an effort to split the design of the proof-of-work subsystem into an algorithm-specific piece that can be upgraded, and a core protocol that provides queueing and effort adjustment. + +Currently there is only one versioned subprotocol defined: +- [Version 1, Equi-X and Blake2b](./v1-equix.md) + +## Overview + +```text + +----------------------------------+ + | Onion Service | + +-------+ INTRO1 +-----------+ INTRO2 +--------+ | + |Client |-------->|Intro Point|------->| PoW |-----------+ | + +-------+ +-----------+ |Verifier| | | + +--------+ | | + | | | + | | | + | +----------v---------+ | + | |Intro Priority Queue| | + +---------+--------------------+---+ + | | | + Rendezvous | | | + circuits | | | + v v v +``` + +The proof-of-work scheme specified in this document takes place during the [introduction phase of the onion service protocol](../rend-spec/introduction-protocol.md). + +The system described in this proposal is not meant to be on all the time, and it can be entirely disabled for services that do not experience DoS attacks. + +When the subsystem is enabled, suggested effort is continuously adjusted and the computational puzzle can be bypassed entirely when the effort reaches zero. +In these cases, the proof-of-work subsystem can be dormant but still provide the necessary parameters for clients to voluntarily provide effort in order to get better placement in the priority queue. + +The protocol involves the following major steps: + +1. Service encodes PoW parameters in descriptor: `pow-params` in the [second layer plaintext format](../rend-spec/hsdesc-encrypt.md#second-layer-plaintext). +2. Client fetches descriptor and begins solving. Currently this must use the [`v1` solver algorithm](../hspow-spec/v1-equix.md#client-solver). +3. Client finishes solving and sends results using the [proof-of-work extension to INTRODUCE1](../rend-spec/introduction-protocol.md#INTRO1_POW_EXT). +4. Service verifies the proof and queues an introduction based on proven effort. This currently uses the [`v1` verify algorithm](../hspow-spec/v1-equix.md#service-verify) only. +5. Requests are continuously drained from the queue, highest effort first, subject to multiple constraints on speed. See below for more on [handling queued requests](#handling-queue). + +## Replay protection {#replay-protection} + +The service MUST NOT accept introduction requests with the same (seed, nonce) tuple. +For this reason a replay protection mechanism must be employed. + +The simplest way is to use a hash table to check whether a (seed, nonce) tuple has been used before for the active duration of a seed. +Depending on how long a seed stays active this might be a viable solution with reasonable memory/time overhead. + +If there is a worry that we might get too many introductions during the lifetime of a seed, we can use a Bloom filter or similar as our replay cache mechanism. A probabilistic filter means that we will potentially flag some connections as replays even if they are not, with this false positive probability increasing as the number of entries increase. With the right parameter tuning this probability should be negligible, and dropped requests will be retried by the client. + +## The introduction queue {#intro-queue} + +When proof-of-work is enabled for a service, that service diverts all incoming introduction requests to a priority queue system rather than handling them immediately. + +### Adding introductions to the introduction queue {#add-queue} + +When PoW is enabled and an introduction request includes a verified proof, the service queues each request in a data structure sorted by effort. Requests including no proof at all MUST be assigned an effort of zero. Requests with a proof that fails to verify MUST be rejected and not enqueued. + +Services MUST check whether the queue is overfull when adding to it, not just when processing requests. +Floods of low-effort and zero-effort introductions need to be efficiently discarded when the queue is growing faster than it's draining. + +The C implementation chooses a maximum number of queued items based on its configured dequeue rate limit multiplied by the circuit timeout. +In effect, items past this threshold are expected not to be reachable by the time they will timeout. +When this limit is exceeded, the queue experiences a mass trim event where the lowest effort half of all items are discarded. + +### Handling queued introductions {#handling-queue} + +When deciding which introduction request to consider next, the service chooses the highest available effort. When efforts are equivalent, the oldest queued request is chosen. + +The service should handle introductions only by pulling from the introduction queue. +We call this part of introduction handling the "bottom half" because most of the computation happens in this stage. + +For more on how we expect such a system to work in Tor, see the [scheduler analysis and discussion](./analysis-discussion.md#tor-scheduler) section. + +## Effort control {#effort-control} + +### Overall strategy for effort determination {#effort-strategy} + +Denial-of-service is a dynamic problem where the attacker's capabilities constantly change, and hence we want our proof-of-work system to be dynamic and not stuck with a static difficulty setting. +Instead of forcing clients to go below a static target configured by the service operator, we ask clients to "bid" using their PoW effort. +Effectively, a client gets higher priority the higher effort they put into their proof-of-work. +Clients automatically increase their bid when retrying, and services regularly offer a suggested starting point based on the recent queue status. + +[Motivated users](./motivation.md#user-profiles) can spend a high amount of effort in their PoW computation, which should guarantee access to the service given reasonable adversary models. + +An effective effort control algorithm will improve reachability and UX by suggesting values that reduce overall service load to tolerable values while also leaving users with a tolerable overall delay. + +The service starts with a default suggested-effort value of 0, which keeps the PoW defenses dormant until we notice signs of queue overload. + +The entire process of determining effort can be thought of as a set of multiple coupled feedback loops. +Clients perform their own effort adjustments via [timeout retry](#client-timeout) atop a base effort suggested by the service. +That suggestion incorporates the service's control adjustments atop a base effort calculated using a sum of currently-queued client effort. + +Each feedback loop has an opportunity to cover different time scales. +Clients can make adjustments at every single circuit creation request, whereas services are limited by the extra load that frequent updates would place on HSDir nodes. + +In the combined client/service system these client-side increases are expected to provide the most effective quick response to an emerging DoS attack. +After early clients increase the effort using timeouts, later clients benefit from the service detecting this increased queued effort and publishing a larger suggested effort. + +Effort increases and decreases both have a cost. +Increasing effort will make the service more expensive to contact, +and decreasing effort makes new requests likely to become backlogged behind older requests. +The steady state condition is preferable to either of these side-effects, but ultimately it's expected that the control loop always oscillates to some degree. + +### Service-side effort control {#service-effort} + +Services keep an internal suggested effort target which updates on a regular periodic timer in response to measurements made on queue behavior in the previous period. +These internal effort changes can optionally trigger client-visible [descriptor changes](#service-effort-update) when the difference is great enough to warrant republication to the [HSDir](../rend-spec/hsdesc.md). + +This evaluation and update period is referred to as `HS_UPDATE_PERIOD`. +The service-side effort control loop takes inspiration from TCP congestion control's additive increase / multiplicative decrease approach, but unlike a typical AIMD this algorithm is fixed-rate and doesn't update immediately in response to events. + +TODO: `HS_UPDATE_PERIOD` is hardcoded to 300 (5 minutes) currently, but it should be configurable in some way. +Is it more appropriate to use the service's torrc here or a consensus parameter? + +#### Per-period service state {#service-effort-periodic} + +During each update period, the service maintains some state: + +1. `TOTAL_EFFORT`, a sum of all effort values for rendezvous requests that were successfully validated and enqueued. +2. `REND_HANDLED`, a count of rendezvous requests that were actually launched. Requests that made it to dequeueing but were too old to launch by then are not included. +3. `HAD_QUEUE`, a flag which is set if at any time in the update period we saw the priority queue filled with more than a minimum amount of work, greater than we would expect to process in approximately 1/4 second using the configured dequeue rate. +4. `MAX_TRIMMED_EFFORT`, the largest observed single request effort that we discarded during the period. Requests are discarded either due to age (timeout) or during culling events that discard the bottom half of the entire queue when it's too full. + +#### Service AIMD conditions {#service-effort-aimd} + +At the end of each period, the service may decide to increase effort, decrease effort, or make no changes, based on these accumulated state values: + +1. If `MAX_TRIMMED_EFFORT` > our previous internal `suggested_effort`, always INCREASE. + Requests that follow our latest advice are being dropped. +2. If the `HAD_QUEUE` flag was set and the queue still contains at least one item with effort >= our previous internal `suggested_effort`, INCREASE. + Even if we haven't yet reached the point of dropping requests, this signal indicates that our latest suggestion isn't high enough and requests will build up in the queue. +3. If neither condition 1 or 2 are taking place and the queue is below a level we would expect to process in approximately 1/4 second, choose to DECREASE. +4. If none of these conditions match, the `suggested_effort` is unchanged. + +When we INCREASE, the internal `suggested_effort` is increased to either its previous value + 1, or (`TOTAL_EFFORT` / `REND_HANDLED`), whichever is larger. + +When we DECREASE, the internal `suggested_effort` is scaled by 2/3rds. + +Over time, this will continue to decrease our effort suggestion any time the service is fully processing its request queue. +If the queue stays empty, the effort suggestion decreases to zero and clients should no longer submit a proof-of-work solution with their first connection attempt. + +It's worth noting that the `suggested_effort` is not a hard limit to the efforts that are accepted by the service, and it's only meant to serve as a guideline for clients to reduce the number of unsuccessful requests that get to the service. +When [adding requests to the queue](#add-queue), services do accept valid solutions with efforts higher or lower than the published values from `pow-params`. + +#### Updating descriptor with new suggested effort {#service-effort-update} + +The service descriptors may be updated for multiple reasons including introduction point rotation common to all v3 onion services, scheduled seed rotations like the one described for [`v1` parameters](./v1-equix.md#parameter-descriptor), and updates to the effort suggestion. +Even though the internal effort value updates on a regular timer, we avoid propagating those changes into the descriptor and the HSDir hosts unless there is a significant change. + +If the PoW params otherwise match but the seed has changed by less than 15 percent, services SHOULD NOT upload a new descriptor. + +### Client-side effort control {#client-effort} + +Clients are responsible for making their own effort adjustments in response to connection trouble, to allow the system a chance to react before the service has published new effort values. +This is an important tool to uphold UX expectations without relying on excessively frequent updates through the HSDir. + +TODO: This is the weak link in user experience for our current implementation. The C tor implementation does not detect and retry onion service connections as reliably as we would like. Currently our best strategy to improve retry behavior is the Arti rewrite. + +#### Failure ambiguity {#client-failure-ambiguity} + +The first challenge in reacting to failure, in our case, is to even accurately and quickly understand when a failure has occurred. + +This proposal introduces a bunch of new ways where a legitimate client can fail to reach the onion service. +Furthermore, there is currently no end-to-end way for the onion service to inform the client that the introduction failed. +The INTRODUCE_ACK message is not end-to-end (it's from the introduction point to the client) and hence it does not allow the service to inform the client that the rendezvous is never gonna occur. + +From the client's perspective there's no way to attribute this failure to the service itself rather than the introduction point, so error accounting is performed separately for each introduction-point. +Prior mechanisms will discard an introduction point that's required too many retries. + +#### Clients handling timeouts {#client-timeout} + +Alice can fail to reach the onion service if her introduction request gets trimmed off the priority queue when [enqueueing new requests](#add-queue), or if the service does not get through its priority queue in time and the connection times out. + +This section presents a heuristic method for the client getting service even in such scenarios. + +If the rendezvous request times out, the client SHOULD fetch a new descriptor for the service to make sure that it's using the right suggested-effort for the PoW and the right PoW seed. +If the fetched descriptor includes a new suggested effort or seed, it should first retry the request with these parameters. + +TODO: This is not actually implemented yet, but we should do it. +How often should clients at most try to fetch new descriptors? +Determined by a consensus parameter? +This change will also allow clients to retry effectively in cases where the service has just been reconfigured to enable PoW defenses. + +Every time the client retries the connection, it will count these failures per-introduction-point. These counts of previous retries are combined with the service's `suggested_effort` when calculating the actual effort to spend on any individual request to a service that advertises PoW support, even when the currently advertised `suggested_effort` is zero. + +On each retry, the client modifies its solver effort: + +1. If the effort is below `CLIENT_POW_EFFORT_DOUBLE_UNTIL` (= 1000) it will be doubled. +2. Otherwise, multiply the effort by `CLIENT_POW_RETRY_MULTIPLIER` (= 1.5). +3. Constrain the effort to no less than `CLIENT_MIN_RETRY_POW_EFFORT` (= 8). Note that this limit is specific to retries only. Clients may use a lower effort for their first connection attempt. +3. Apply the maximum effort limit [described below](#client-limits). + +#### Client-imposed effort limits {#client-limits} + +There isn't a practical upper limit on effort defined by the protocol itself, but clients may choose a maximum effort limit to enforce. +It may be desirable to do this in some cases to improve responsiveness, but the main reason for this limit currently is as a workaround for weak cancellation support in our implementation. + +Effort values used for both initial connections and retries are currently limited to no greater than `CLIENT_MAX_POW_EFFORT` (= 10000). + +TODO: This hardcoded limit should be replaced by timed limits and/or an unlimited solver with robust cancellation. This is [issue 40787](https://gitlab.torproject.org/tpo/core/tor/-/issues/40787) in C tor. diff --git a/spec/hspow-spec/index.md b/spec/hspow-spec/index.md new file mode 100644 index 0000000..14db248 --- /dev/null +++ b/spec/hspow-spec/index.md @@ -0,0 +1,5 @@ +# Proof of Work for onion service introduction + +The overall denial-of-service prevention strategies in Tor are described in the [Denial-of-service prevention mechanisms in Tor](../dos-spec/index.md) document. This document describes one specific mitigation, the proof-of-work client puzzle for onion service introduction. + +This was originally [proposal 327, A First Take at PoW Over Introduction Circuits](../proposals/327-pow-over-intro.txt) authored by George Kadianakis, Mike Perry, David Goulet, and tevador. diff --git a/spec/hspow-spec/motivation.md b/spec/hspow-spec/motivation.md new file mode 100644 index 0000000..8fffe3f --- /dev/null +++ b/spec/hspow-spec/motivation.md @@ -0,0 +1,85 @@ +# Motivation + +See the [denial-of-service overview](../dos-spec/overview.md) for the big-picture view. +Here we are focusing on a mitigation for attacks on one specific resource: onion service introductions. + +Attackers can generate low-effort floods of introductions which cause the onion service and all involved relays to perform a disproportionate amount of work, leading to a denial-of-service opportunity. +This proof-of-work scheme intends to make introduction floods unattractive to attackers, reducing the network-wide impact of this activity. + +Previous to this work, our attempts at limiting the impact of introduction flooding DoS attacks on onion services has been focused on horizontal scaling with Onionbalance, optimizing the CPU usage of Tor and applying rate limiting. +While these measures move the goalpost forward, a core problem with onion service DoS is that building rendezvous circuits is a costly procedure both for the service and for the network. + +For more information on the limitations of rate-limiting when defending against DDoS, see [`draft-nygren-tls-client-puzzles-02`](https://www.ietf.org/archive/id/draft-nygren-tls-client-puzzles-02.txt). + +If we ever hope to have truly reachable global onion services, we need to make it harder for attackers to overload the service with introduction requests. +This proposal achieves this by allowing onion services to specify an optional dynamic proof-of-work scheme that its clients need to participate in if they want to get served. + +With the right parameters, this proof-of-work scheme acts as a gatekeeper to block amplification attacks by attackers while letting legitimate clients through. + +## Related work {#related-work} + +For a similar concept, see the three internet drafts that have been proposed for defending against TLS-based DDoS attacks using client puzzles: + +- [`draft-nygren-tls-client-puzzles-02`](https://www.ietf.org/archive/id/draft-nygren-tls-client-puzzles-02.txt) +- [`draft-nir-tls-puzzles-00`](https://www.ietf.org/archive/id/draft-nir-tls-puzzles-00.txt) +- [`draft-ietf-ipsecme-ddos-protection-10`](https://tools.ietf.org/html/draft-ietf-ipsecme-ddos-protection-10) + +## Threat model + +### Attacker profiles {#attacker-profiles} + +This mitigation is written to thwart specific attackers. The current protocol is not intended to defend against all and every DoS attack on the Internet, but there are adversary models we can defend against. + +Let's start with some adversary profiles: + +- "The script-kiddie" + + The script-kiddie has a single computer and pushes it to its limits. + Perhaps it also has a VPS and a pwned server. + We are talking about an attacker with total access to 10 GHz of CPU and 10 GB of RAM. + We consider the total cost for this attacker to be zero $. + +- "The small botnet" + + The small botnet is a bunch of computers lined up to do an introduction flooding attack. + Assuming 500 medium-range computers, we are talking about an attacker with total access to 10 THz of CPU and 10 TB of RAM. + We consider the upfront cost for this attacker to be about $400. + +- "The large botnet" + + The large botnet is a serious operation with many thousands of computers organized to do this attack. + Assuming 100k medium-range computers, we are talking about an attacker with total access to 200 THz of CPU and 200 TB of RAM. + The upfront cost for this attacker is about $36k. + +We hope that this proposal can help us defend against the script-kiddie attacker and small botnets. +To defend against a large botnet we would need more tools at our disposal (see the [discussion on future designs](./analysis-discussion.md#future-designs)). + +### User profiles {#user-profiles} + +We have attackers and we have users. Here are a few user profiles: + +- "The standard web user" + + This is a standard laptop/desktop user who is trying to browse the web. + They don't know how these defences work and they don't care to configure or tweak them. + If the site doesn't load, they are gonna close their browser and be sad at Tor. + They run a 2GHz computer with 4GB of RAM. + +- "The motivated user" + + This is a user that really wants to reach their destination. + They don't care about the journey; they just want to get there. + They know what's going on; they are willing to make their computer do expensive multi-minute PoW computations to get where they want to be. + +- "The mobile user" + + This is a motivated user on a mobile phone. + Even tho they want to read the news article, they don't have much leeway on stressing their machine to do more computation. + +We hope that this proposal will allow the motivated user to always connect where they want to connect to, and also give more chances to the other user groups to reach the destination. + +### The DoS Catch-22 {#catch22} + +This proposal is not perfect and it does not cover all the use cases. +Still, we think that by covering some use cases and giving reachability to the people who really need it, we will severely demotivate the attackers from continuing the DoS attacks and hence stop the DoS threat all together. +Furthermore, by increasing the cost to launch a DoS attack, a big class of DoS attackers will disappear from the map, since the expected ROI will decrease. diff --git a/spec/hspow-spec/v1-equix.md b/spec/hspow-spec/v1-equix.md new file mode 100644 index 0000000..025af48 --- /dev/null +++ b/spec/hspow-spec/v1-equix.md @@ -0,0 +1,176 @@ +# Onion service proof-of-work: Version 1, Equi-X and Blake2b + +## Implementations {#implementations} + +For our `v1` proof-of-work function we use the Equi-X asymmetric client puzzle algorithm by tevador. +The concept and the C implementation were developed specifically for our use case by tevador, based on a survey of existing work and an analysis of Tor's requirements. + +- [Original Equi-X source repository](https://github.com/tevador/equix) +- [Development log](https://github.com/tevador/equix/blob/master/devlog.md) + +Equi-X is an asymmetric PoW function based on Equihash<60,3>, using HashX as the underlying layer. +It features lightning fast verification speed, and also aims to minimize the asymmetry between CPU and GPU. +Furthermore, it's designed for this particular use-case and hence cryptocurrency miners are not incentivized to make optimized ASICs for it. + +At this point there is no formal specification for Equi-X or the underlying HashX function. +We have two actively maintained implementations of both components, which we subject to automated cross-compatibility and fuzz testing: + +- A fork of tevador's implementation is maintained within the C tor repository. + + This is the [`src/ext/equix` subdirectory](https://gitlab.torproject.org/tpo/core/tor/-/tree/main/src/ext/equix). + Currently this contains important fixes for security, portability, and testability which have not been merged upstream! + This implementation is released under the LGPL license. + When `tor` is built with the required `--enable-gpl` option this code will be statically linked. + +- As part of Arti, a new Rust re-implementation was written based loosely on tevador's original. + + This is the [`equix` crate](https://tpo.pages.torproject.net/core/doc/rust/equix/index.html). + This implementation currently has somewhat lower verification performance than the original but otherwise offers equivalent features. + +## Algorithm overview {#overview} + +The overall scheme consists of several layers that provide different pieces of this functionality: + +1. At the lowest layers, Blake2b and siphash are used as hashing and PRNG algorithms that are well suited to common 64-bit CPUs. +2. A custom hash function family, HashX, randomizes its implementation for each new seed value. + These functions are tuned to utilize the pipelined integer performance on a modern 64-bit CPU. + This layer provides the strongest ASIC resistance, since a hardware reimplementation would need to include a CPU-like pipelined execution unit to keep up. +3. The Equi-X layer itself builds on HashX and adds an algorithmic puzzle that's designed to be strongly asymmetric and to require RAM to solve efficiently. +4. The PoW protocol itself builds on this Equi-X function with a particular construction of the challenge input and particular constraints on the allowed Blake2b hash of the solution. + This layer provides a linearly adjustable effort that we can verify. +5. At this point, all further layers are part of the [common protocol](./common-protocol.md). Above the level of individual PoW handshakes, the client and service form a closed-loop system that adjusts the effort of future handshakes. + +Equi-X itself provides two functions that will be used in this proposal: +- `equix_solve`(`challenge`) which solves a puzzle instance, returning a variable number of solutions per invocation depending on the specific challenge value. +- `equix_verify`(`challenge`, `solution`) which verifies a puzzle solution quickly. + Verification still depends on executing the HashX function, but far fewer times than when searching for a solution. + +For the purposes of this proposal, all cryptographic algorithms are assumed to produce and consume byte strings, even if internally they operate on some other data type like 64-bit words. +This is conventionally little endian order for Blake2b, which contrasts with Tor's typical use of big endian. +HashX itself is configured with an 8-byte output but its input is a single 64-bit word of undefined byte order, of which only the low 16 bits are used by Equi-X in its solution output. +We treat Equi-X solution arrays as byte arrays using their packed little endian 16-bit representation. + +## Linear effort adjustment {#effort} + +The underlying Equi-X puzzle has an approximately fixed computational cost. +Adjustable effort comes from the construction of the overlying Blake2b layer, which requires clients to test a variable number of Equi-X solutions in order to find answers which also satisfy this layer's effort constraint. + +It's common for proof-of-work systems to define an exponential effort function based on a particular number of leading zero bits or equivalent. +For the benefit of our effort control system, it's quite useful if we have a linear scale instead. We use the first 32 bits of a hashed version of the Equi-X solution as a uniformly distributed random value. + +Conceptually we could define a function: +```text +unsigned effort(uint8_t *token) +``` +which takes as its argument a hashed solution, interprets it as a bitstring, and returns the quotient of dividing a bitstring of 1s by it. + +So for example: +```text +effort(00000001100010101101) = 11111111111111111111 + / 00000001100010101101 +``` +or the same in decimal: +```text +effort(6317) = 1048575 / 6317 = 165. +``` + +In practice we can avoid even having to perform this division, performing just one multiply instead to see if a request's claimed effort is supported by the smallness of the resulting 32-bit hash prefix. +This assumes we send the desired effort explicitly as part of each PoW solution. +We do want to force clients to pick a specific effort before looking for a solution, otherwise a client could opportunistically claim a very large effort any time a lucky hash prefix comes up. +Thus the effort is communicated explicitly in our protocol, and it forms part of the concatenated Equi-X challenge. + +## Parameter descriptor {#parameter-descriptor} + +This whole protocol starts with the service encoding its parameters in a `pow-params` line within the 'encrypted' (inner) part of the v3 descriptor. The [second layer plaintext format](../rend-spec/hsdesc-encrypt.md#second-layer-plaintext) describes it canonically. The parameters offered are: +- `type`, always `v1` for the algorithm described here +- `seed-b64`, a periodically updated 32-byte random seed, base64 encoded +- `suggested-effort`, the latest output from the [service-side effort controller](./common-protocol.md#service-effort) +- `expiration-time`, a timestamp when we plan to replace the seed. + +Seed expiration and rotation allows used nonces to expire from the anti-replay memory. +At every seed rotation, a new expiration time is chosen uniformly at random from the recommended range: +- At the earliest, 105 minutes in the future +- At the latest, 2 hours in the future (15 minutes later) + +The service SHOULD refresh its seed when expiration-time passes. +The service SHOULD keep its previous seed in memory and accept PoWs using it to avoid race-conditions with clients that have an old seed. +The service SHOULD avoid generating two consequent seeds that have a common 4 bytes prefix; see the usage of seed headings below in the [introduction extension](#intro-ext). + +## Client computes a solution {#client-solver} + +If a client receives a descriptor with `pow-params`, it should assume that the service is prepared to receive PoW solutions as part of the introduction protocol. + +The client parses the descriptor and extracts the PoW parameters. +It makes sure that the `expiration-time` has not expired. +If it has, the descriptor may be out of date. +Clients SHOULD fetch a fresh descriptor if the descriptor is stale and the seed is expired. + +Inputs to the solver: + +1. Effort `E`, the [client-side effort choice](./common-protocol.md#client-effort) made based on the server's `suggested-effort` and the client's connection attempt history. This is a 32-bit unsigned integer. +2. Constant personalization string `P`, equal to the following nul-terminated ASCII text: `"Tor hs intro v1\0"`. +3. Identity string `ID`, a 32-byte value unique to the specific onion service. This is the blinded public ID key `KP_hs_blind_id`. +4. Seed `C`, a 32-byte random value decoded from `seed-b64` above. +5. Initial nonce `N`, a 16-byte value generated using a secure random generator. + +The solver itself is iterative; the following steps are repeated until they succeed: + +1. Construct the *challenge string* by concatenating `P || ID || C || N || htonl(E)`. +2. Calculate a candidate proof `S` by passing this challenge to Equi-X. + + `S = equix_solve(P || ID || C || N || htonl(E))` +3. Calculate a 32-bit check value by interpreting a 32-bit Blake2b hash of the concatenated challenge and solution as an integer in network byte order. + + `R = ntohl(blake2b_32(P || ID || C || N || htonl(E) || S))` +4. Check if 32-bit multiplication of `R * E` would overflow + + If `R * E` overflows (the result would be greater than `UINT32_MAX`) the solver must retry with another nonce value. The client interprets N as a 16-byte little-endian integer, increments it by 1, and goes back to step 1. + + If there is no overflow (the result is less than or equal to `UINT32_MAX`) this is a valid solution. The client can submit final nonce `N`, effort `E`, the first 4 bytes of seed `C`, and proof `S`. + +Note that the Blake2b hash includes the output length parameter in its initial state vector, so a `blake2b_32` is not equivalent to the prefix of a `blake2b_512`. +We calculate the 32-bit Blake2b specifically, and interpret it in network byte order as an unsigned integer. + +At the end of the above procedure, the client should have calculated a proof `S` and final nonce `N` that satisfies both the Equi-X proof conditions and the Blake2b effort test. +The time taken, on average, is linearly proportional with the target effort `E` parameter. + +The algorithm as described is suitable for single-threaded computation. +Optionally, a client may choose multiple nonces and attempt several solutions in parallel on separate CPU cores. +The specific choice of nonce is entirely up to the client, so parallelization choices like this do not impact the network protocol's interoperability at all. + +## Client sends its proof in an INTRO1 extension {#intro-ext} + +Now that the client has an answer to the puzzle it's time to encode it into an INTRODUCE1 message. +To do so the client adds an extension to the encrypted portion of the INTRODUCE1 message by using the EXTENSIONS field. The encrypted portion of the INTRODUCE1 message only gets read by the onion service and is ignored by the introduction point. + +This extension includes the chosen nonce and effort in full, as well as the actual Equi-X proof. +Clients provide only the first 4 bytes of the seed, enough to disambiguate between multiple recent seeds offered by the service. + +This format is defined canonically as the [proof-of-work extension to INTRODUCE1](../rend-spec/introduction-protocol.md#INTRO1_POW_EXT). + +## Service verifies PoW and handles the introduction {#service-verify} + +When a service receives an INTRODUCE1 with the `PROOF_OF_WORK` extension, it should check its configuration on whether proof-of-work is enabled on the service. +If it's not enabled, the extension SHOULD BE ignored. +If enabled, even if the suggested effort is currently zero, the service follows the procedure detailed in this section. + +If the service requires the `PROOF_OF_WORK` extension but received an INTRODUCE1 message without any embedded proof-of-work, the service SHOULD consider this message as a zero-effort introduction for the purposes of the [priority queue](./common-protocol.md#intro-queue). + +To verify the client's proof-of-work the service MUST do the following steps: + +1. Find a valid seed `C` that starts with `POW_SEED`. + Fail if no such seed exists. +2. Fail if `N = POW_NONCE` is present in the [replay protection data structure](./common-protocol.md#replay-protection). +3. Construct the *challenge string* as above by concatenating `P || ID || C || N || htonl(E)`. In this case, `E` and `N` are values provided by the client. +4. Calculate `R = ntohl(blake2b_32(P || ID || C || N || htonl(E) || S))`, as above +5. Fail if the the effort test overflows (`R * E > UINT32_MAX`). +6. Fail if Equi-X reports that the proof `S` is malformed or not applicable (`equix_verify(P || ID || C || N || htonl(E), S) != EQUIX_OK`) +7. If both the Blake2b and Equi-X tests pass, the request can be enqueued with priority `E`. + +It's a minor performance optimization for services to compute the effort test before invoking `equix_verify`. +Blake2b verification is cheaper than Equi-X verification, so this ordering slightly raises the minimum effort required to perform a [top-half attack](./analysis-discussion.md#attack-top-half). + +If any of these steps fail the service MUST ignore this introduction request and abort the protocol. + +In this document we call the above steps the "top half" of introduction handling. +If all the steps of the "top half" have passed, then the circuit is added to the [introduction queue](./common-protocol.md#intro-queue). diff --git a/spec/intro/conventions.md b/spec/intro/conventions.md new file mode 100644 index 0000000..8a92521 --- /dev/null +++ b/spec/intro/conventions.md @@ -0,0 +1,112 @@ +# Notation and conventions + +These conventions apply, +at least in theory, +to all of the specification documents +unless stated otherwise. + +> Remember, our specification documents +> were once a collection of separate text files, +> written separately +> and edited over the course of years. +> +> While we are trying (as of 2023) +> to edit them into consistency, +> you should be aware that these conventions +> are not now followed uniformly everywhere. + +## MUST, SHOULD, and so on {#rfc2119} + +The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL +NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and +"OPTIONAL" in this document are to be interpreted as described in +[RFC 2119](https://datatracker.ietf.org/doc/html/rfc2119). + +## Data lengths {#data-lengths} + +Unless otherwise stated, +all lengths are given as a number of 8-bit bytes. + +> All bytes are 8 bits long. +> We sometimes call them "octets"; +> the terms as used here are interchangeable. + +When referring to longer lengths, +we use [SI binary prefixes](https://en.wikipedia.org/wiki/Binary_prefix) +(as in "kibibytes", "mebibytes", and so on) +to refer unambiguously to increments of 1024<sup>X</sup> bytes. + +> If you encounter a reference +> to "kilobytes", "megabytes", or so on, +> you cannot safely infer whether the author intended +> a decimal (1000<sup>N</sup>) or binary (1024<sup>N</sup>) interpretation. +> In these cases, it is better to revise the specifications. + +<a id="tor-spec.txt-0.1.1"></a> + +## Integer encoding {#integers} + +Unless otherwise stated, +all multi-byte integers are encoded +in big-endian ("network") order. + +> For example, 4660 (0x1234), +> when encoded as a two-byte integer, +> is the byte 0x12 followed by the byte 0x34. (\[12 34\]) +> +> When encoded as a four-byte integer, +> it is the byte 0x00, the byte 0x00, the byte 0x12, and the byte 0x34. +> (\[00 00 12 34\]). + +## Binary-as-text encodings {#binascii} + +When we refer to "base64", "base32", or "base16", +we mean the encodings described in +[RFC 4648](https://datatracker.ietf.org/doc/html/rfc4648), +with the following notes: + +- In base32, we never insert linefeeds in base32, + and we omit trailing `=` padding characters. +- In base64, + we _sometimes_ omit trailing `=` padding characters, + and we do not insert linefeeds unless explicitly noted. +- We do not insert any other whitespace, + except as specifically noted. + +Base 16 and base 32 are case-insensitive. +Unless otherwise stated, +implementations should accept any cases, +and should produce a single uniform case. + +We sometimes refer to base16 as "hex" or "hexadecimal". + +> Note that as of 2023, in some places, the specs are not always +> explicit about: +> +> - which base64 strings are multiline +> - which base32 strings and base16 strings +> should be generated in what case. +> +> This is something we should correct. + +## Notation {#notation} + +### Operations on byte strings {#ops} + +* `A | B` represents the concatenation of two binary strings `A` and `B`. + +### Binary literals {#binary-literals} + +When we write a series of one-byte hexadecimal literals +in square brackets, +it represents a multi-byte binary string. + +> For example, +> `[6f 6e 69 6f 6e 20 72 6f 75 74 69 6e 67]` +> is a 13-byte sequence representing the unterminated ASCII string, +> `onion routing`. + + + + + diff --git a/spec/intro/index.md b/spec/intro/index.md new file mode 100644 index 0000000..894d7e0 --- /dev/null +++ b/spec/intro/index.md @@ -0,0 +1,140 @@ +# A short introduction to Tor {#tor-intro} + +### Basic functionality {#basics} + +Tor is a distributed overlay network designed to anonymize +low-latency TCP-based applications +such as web browsing, secure shell, and instant messaging. +The network is built of a number of servers, called **relays** +(also called "onion routers" or "ORs" in some older documentation). + +To connect to the network, +a client needs to download an up-to-date signed directory +of the relays on the network. +These directory documents are generated and signed +by a set of semi-trusted **directory authority** servers, +and are cached by the relays themselves. +(If a client does not yet have a directory, +it finds a cache by looking at a list of stable cache locations, +distributed along with its source code.) + +> For more information on the directory subsystem, +> see the [directory protocol specification](../dir-spec). + +After the client knows the relays on the network, +it can pick a relay and open a [**channel**](../tor-spec/channels.md) +to one of these relays. +A channel is an encrypted reliable non-anonymous transport +between a client and a relay or a relay and a relay, +used to transmit messages called [**cells**](../tor-spec/cell-packet-format.md). +(Under the hood, a channel is just a TLS connection over TCP, +with a specified encoding for cells.) + +To anonymize its traffic, +a client chooses a **path**—a sequence of relays on the network— +and opens a channel to the first relay on the path +(if it does not already have a channel open to that relay). +The client then uses that channel to build +a multi-hop cryptographic structure +called a [**circuit**](../tor-spec/circuit-management.md). +A circuit is built over a sequence of relays (typically three). +Every relay in the circuit knows its precessor and successor, +but no other relays in the circuit. +Many circuits can be multiplexed over a single channel. + +> For more information on how paths are selected, +> see the [path specification](../path-spec). +> The first hop on a path, +> also called a **guard node**, +> has complicated rules for its selection; +> for more on those, see the [guard specification](../guard-spec). + +Once a circuit exists, +the client can use it to exchange fixed-length +[**relay cells**](../tor-spec/relay-cells.md) +with any relay on the circuit. +These relay cells are wrapped in multiple layers of encryption: +as part of building the circuit, +the client [negotiates](../tor-spec/create-created-cells.md) +a separate set of symmetric keys +with each relay on the circuit. +Each relay removes (or adds) +a [single layer of encryption](../tor-spec/routing-relay-cells.md) +for each relay cell before passing it on. + +A client uses these relay cells +to exchange [**relay messages**](../tor-spec/relay-cells.md) with relays on a circuit. +These "relay messages" in turn are used +to actually deliver traffic over the network. +In the [simplest use case](../tor-spec/opening-streams.md), +the client sends a `BEGIN` message +to tell the last relay on the circuit +(called the **exit node**) +to create a new session, or **stream**, +and associate that stream +with a new TCP connection to a target host. +The exit node replies with a `CONNECTED` message +to say that the TCP connection has succeeded. +Then the client and the exit exchange `DATA` messages +to represent the contents of the anonymized stream. + +> Note that as of 2023, +> the specifications do not perfectly distinguish +> between relay cells and relay messages. +> This is because, until recently, +> there was a 1-to-1 relationship between the two: +> every relay cell held a single relay message. +> As [proposal 340](../proposals/340-packed-and-fragmented.md) is implemented, +> we will revise the specifications +> for improved clarify on this point. + +Other kinds of relay messages can be used +for more advanced functionality. + +<!-- TODO: I'm not so sure about the vocabulary in this part. --> + +Using a system called **conflux** +a client can build multiple circuits to the _same_ exit node, +and associate those circuits within a **conflux set**. +Once this is done, +relay messages can be sent over _either_ circuit in the set, +depending on capacity and performance. + +> For more on conflux, +> which has been integrated into the C tor implementation, +> but not yet (as of 2023) into this document, +> see [proposal 329](../proposals/329-traffic-splitting.txt). + +### Advanced topics: Onion services and responder anonymity {#onions} + +In addition to _initiating_ anonymous communications, +clients can also arrange to _receive_ communications +without revealing their identity or location. +This is called **responder anonymity**, +and the mechanism Tor uses to achieve it +is called **onion services** +(or "hidden services" or "rendezvous services" +in some older documentation). + +> For the details on onion services, +> see the [Tor Rendezvous Specification](../rend-spec). + +### Advanced topics: Censorship resistence {#anticensorship} + +In some places, Tor is censored. +Typically, censors do this by blocking connections +to the addresses of the known Tor relays, +and by blocking traffic that resembles Tor. + +To resist this censorship, +some Tor relays, called **bridges**, +are unlisted in the public directory: +their addresses are distributed by [other means](../bridgedb-spec.md). +(To distinguish ordinary published relays from bridges, +we sometimes call them **public relays**.) + +Additionally, Tor clients and bridges can use extension programs, +called [**pluggable transports**](../pt-spec), +that obfuscate their traffic to make it harder to detect. + + diff --git a/spec/padding-spec/acknowledgments.md b/spec/padding-spec/acknowledgments.md new file mode 100644 index 0000000..d3bd64d --- /dev/null +++ b/spec/padding-spec/acknowledgments.md @@ -0,0 +1,28 @@ +<a id="padding-spec.txt-A"></a> + +# Acknowledgments + +This research was supported in part by NSF grants CNS-1111539, +CNS-1314637, CNS-1526306, CNS-1619454, and CNS-1640548. + +```text +1. https://en.wikipedia.org/wiki/NetFlow +2. http://infodoc.alcatel-lucent.com/html/0_add-h-f/93-0073-10-01/7750_SR_OS_Router_Configuration_Guide/Cflowd-CLI.html +3. http://www.cisco.com/en/US/docs/ios/12_3t/netflow/command/reference/nfl_a1gt_ps5207_TSD_Products_Command_Reference_Chapter.html#wp1185203 +4. http://www.cisco.com/c/en/us/support/docs/switches/catalyst-6500-series-switches/70974-netflow-catalyst6500.html#opconf +5. https://www.juniper.net/techpubs/software/erx/junose60/swconfig-routing-vol1/html/ip-jflow-stats-config4.html#560916 +6. http://www.jnpr.net/techpubs/en_US/junos15.1/topics/reference/configuration-statement/flow-active-timeout-edit-forwarding-options-po.html +7. http://www.jnpr.net/techpubs/en_US/junos15.1/topics/reference/configuration-statement/flow-active-timeout-edit-forwarding-options-po.html +8. http://www.h3c.com/portal/Technical_Support___Documents/Technical_Documents/Switches/H3C_S9500_Series_Switches/Command/Command/H3C_S9500_CM-Release1648%5Bv1.24%5D-System_Volume/200901/624854_1285_0.htm#_Toc217704193 +9. http://docs-legacy.fortinet.com/fgt/handbook/cli52_html/FortiOS%205.2%20CLI/config_system.23.046.html +10. http://wiki.mikrotik.com/wiki/Manual:IP/Traffic_Flow +11. https://metrics.torproject.org/dirbytes.html +12. http://freehaven.net/anonbib/cache/murdoch-pet2007.pdf +13. https://spec.torproject.org/proposals/188-bridge-guards.html +14. http://www.ntop.org/wp-content/uploads/2013/03/nProbe_UserGuide.pdf +15. http://arxiv.org/pdf/1512.00524 +16. https://www.cs.kau.se/pulls/hot/thebasketcase-ape/ +17. https://github.com/torproject/tor/tree/master/doc/HACKING/CircuitPaddingDevelopment.md +18. https://www.usenix.org/node/190967 + https://blog.torproject.org/technical-summary-usenix-fingerprinting-paper +``` diff --git a/spec/padding-spec/circuit-level-padding.md b/spec/padding-spec/circuit-level-padding.md new file mode 100644 index 0000000..85f9137 --- /dev/null +++ b/spec/padding-spec/circuit-level-padding.md @@ -0,0 +1,290 @@ +<a id="padding-spec.txt-3"></a> + +# Circuit-level padding {#circuit-level-padding} + +The circuit padding system in Tor is an extension of the WTF-PAD +event-driven state machine design\[15\]. At a high level, this design places +one or more padding state machines at the client, and one or more padding +state machines at a relay, on each circuit. + +State transition and histogram generation has been generalized to be fully +programmable, and probability distribution support was added to support more +compact representations like APE\[16\]. Additionally, packet count limits, +rate limiting, and circuit application conditions have been added. + +At present, Tor uses this system to deploy two pairs of circuit padding +machines, to obscure differences between the setup phase of client-side +onion service circuits, up to the first 10 relay cells. + +This specification covers only the resulting behavior of these padding +machines, and thus does not cover the state machine implementation details or +operation. For full details on using the circuit padding system to develop +future padding defenses, see the research developer documentation\[17\]. + +<a id="padding-spec.txt-3.1"></a> + +## Circuit Padding Negotiation {#negotiation} + +Circuit padding machines are advertised as "Padding" subprotocol versions +(see tor-spec.txt Section 9). The onion service circuit padding machines are +advertised as "Padding=2". + +Because circuit padding machines only become active at certain points in +circuit lifetime, and because more than one padding machine may be active at +any given point in circuit lifetime, there is also a PADDING_NEGOTIATE +message and a PADDING_NEGOTIATED message. These are relay commands 41 and 42 respectively, +with relay headers as per section 6.1 of tor-spec.txt. + +The fields in the body of a PADDING_NEGOTIATE message are +as follows: + +```text + const CIRCPAD_COMMAND_STOP = 1; + const CIRCPAD_COMMAND_START = 2; + + const CIRCPAD_RESPONSE_OK = 1; + const CIRCPAD_RESPONSE_ERR = 2; + + const CIRCPAD_MACHINE_CIRC_SETUP = 1; + + struct circpad_negotiate { + u8 version IN [0]; + u8 command IN [CIRCPAD_COMMAND_START, CIRCPAD_COMMAND_STOP]; + + u8 machine_type IN [CIRCPAD_MACHINE_CIRC_SETUP]; + + u8 unused; // Formerly echo_request + + u32 machine_ctr; + }; +``` + +When a client wants to start a circuit padding machine, it first checks that +the desired destination hop advertises the appropriate subprotocol version for +that machine. It then sends a PADDING_NEGOTIATE message to that hop with +command=CIRCPAD_COMMAND_START, and machine_type=CIRCPAD_MACHINE_CIRC_SETUP (for +the circ setup machine, the destination hop is the second hop in the +circuit). The machine_ctr is the count of which machine instance this is on +the circuit. It is used to disambiguate shutdown requests. + +When a relay receives a PADDING_NEGOTIATE message, it checks that it supports +the requested machine, and sends a PADDING_NEGOTIATED message, which is formatted +in the data payload of a relay cell with command number 42 (see tor-spec.txt +section 6.1), as follows: + +```text + struct circpad_negotiated { + u8 version IN [0]; + u8 command IN [CIRCPAD_COMMAND_START, CIRCPAD_COMMAND_STOP]; + u8 response IN [CIRCPAD_RESPONSE_OK, CIRCPAD_RESPONSE_ERR]; + + u8 machine_type IN [CIRCPAD_MACHINE_CIRC_SETUP]; + + u32 machine_ctr; + }; +``` + +If the machine is supported, the response field will contain +CIRCPAD_RESPONSE_OK. If it is not, it will contain CIRCPAD_RESPONSE_ERR. + +Either side may send a CIRCPAD_COMMAND_STOP to shut down the padding machines +(clients MUST only send circpad_negotiate, and relays MUST only send +circpad_negotiated for this purpose). + +If the machine_ctr does not match the current machine instance count +on the circuit, the command is ignored. + +<a id="padding-spec.txt-3.2"></a> + +## Circuit Padding Machine Message Management { #machine-msg-mgt } + +Clients MAY send padding cells towards the relay before receiving the +circpad_negotiated response, to allow for outbound cover traffic before +negotiation completes. + +Clients MAY send another PADDING_NEGOTIATE message before receiving the +circpad_negotiated response, to allow for rapid machine changes. + +Relays MUST NOT send padding cells or PADDING_NEGOTIATE messages unless a +padding machine is active. Any padding cells or padding-related messages +that arrive at the client +from unexpected relay sources are protocol violations, and clients MAY +immediately tear down such circuits to avoid side channel risk. + +<a id="padding-spec.txt-3.3"></a> + +## Obfuscating client-side onion service circuit setup { #hiding-circ-setup } + +The circuit padding currently deployed in Tor attempts to hide client-side +onion service circuit setup. Service-side setup is not covered, because doing +so would involve significantly more overhead, and/or require interaction with +the application layer. + +The approach taken aims to make client-side introduction and rendezvous +circuits match the cell direction sequence and cell count of 3 hop general +circuits used for normal web traffic, for the first 10 cells only. The +lifespan of introduction circuits is also made to match the lifespan +of general circuits. + +Note that inter-arrival timing is not obfuscated by this defense. + +<a id="padding-spec.txt-3.3.1"></a> + +### Common general circuit construction sequences { #circ-setup-sequences} + +Most general Tor circuits used to surf the web or download directory +information start with the following 6-cell relay cell sequence (cells +surrounded in \[brackets\] are outgoing, the others are incoming): + +\[EXTEND2\] -> EXTENDED2 -> \[EXTEND2\] -> EXTENDED2 -> \[BEGIN\] -> CONNECTED + +When this is done, the client has established a 3-hop circuit and also opened +a stream to the other end. Usually after this comes a series of DATA message that +either fetches pages, establishes an SSL connection or fetches directory +information: + +\[DATA\] -> \[DATA\] -> DATA -> DATA...(inbound cells continue) + +The above stream of 10 relay cells defines the grand majority of general +circuits that come out of Tor browser during our testing, and it's what we use +to make introduction and rendezvous circuits blend in. + +Please note that in this section we only investigate relay cells and not +connection-level cells like CREATE/CREATED or AUTHENTICATE/etc. that are used +during the link-layer handshake. The rationale is that connection-level cells +depend on the type of guard used and are not an effective fingerprint for a +network/guard-level adversary. + +<a id="padding-spec.txt-3.3.2"></a> + +### Client-side onion service introduction circuit obfuscation { #hiding-intro } + +Two circuit padding machines work to hide client-side introduction circuits: +one machine at the origin, and one machine at the second hop of the circuit. +Each machine sends padding towards the other. The padding from the origin-side +machine terminates at the second hop and does not get forwarded to the actual +introduction point. + +From Section 3.3.1 above, most general circuits have the following initial +relay cell sequence (outgoing cells marked in \[brackets\]): + +```text + [EXTEND2] -> EXTENDED2 -> [EXTEND2] -> EXTENDED2 -> [BEGIN] -> CONNECTED + -> [DATA] -> [DATA] -> DATA -> DATA...(inbound data cells continue) + + Whereas normal introduction circuits usually look like: + + [EXTEND2] -> EXTENDED2 -> [EXTEND2] -> EXTENDED2 -> [EXTEND2] -> EXTENDED2 + -> [INTRO1] -> INTRODUCE_ACK +``` + +This means that up to the sixth cell (first line of each sequence above), +both general and intro circuits have identical cell sequences. After that +we want to mimic the second line sequence of + +-> \[DATA\] -> \[DATA\] -> DATA -> DATA...(inbound data cells continue) + +We achieve this by starting padding INTRODUCE1 has been sent. With padding +negotiation cells, in the common case of the second line looks like: + +-> \[INTRO1\] -> \[PADDING_NEGOTIATE\] -> PADDING_NEGOTIATED -> INTRO_ACK + +Then, the middle node will send between INTRO_MACHINE_MINIMUM_PADDING (7) and +INTRO_MACHINE_MAXIMUM_PADDING (10) cells, to match the "...(inbound data cells +continue)" portion of the trace (aka the rest of an HTTPS response body). + +We also set a special flag which keeps the circuit open even after the +introduction is performed. With this feature the circuit will stay alive for +the same duration as normal web circuits before they expire (usually 10 +minutes). + +<a id="padding-spec.txt-3.3.3"></a> + +### Client-side rendezvous circuit hiding { #hiding-rendezvous } + +Following a similar argument as for intro circuits, we are aiming for padded +rendezvous circuits to blend in with the initial cell sequence of general +circuits which usually look like this: + +```text + [EXTEND2] -> EXTENDED2 -> [EXTEND2] -> EXTENDED2 -> [BEGIN] -> CONNECTED + -> [DATA] -> [DATA] -> DATA -> DATA...(incoming cells continue) + + Whereas normal rendezvous circuits usually look like: + + [EXTEND2] -> EXTENDED2 -> [EXTEND2] -> EXTENDED2 -> [EST_REND] -> REND_EST + -> REND2 -> [BEGIN] +``` + +This means that up to the sixth cell (the first line), both general and +rend circuits have identical cell sequences. + +After that we want to mimic a \[DATA\] -> \[DATA\] -> DATA -> DATA sequence. + +With padding negotiation right after the REND_ESTABLISHED, the sequence +becomes: + +```text + [EXTEND2] -> EXTENDED2 -> [EXTEND2] -> EXTENDED2 -> [EST_REND] -> REND_EST + -> [PADDING_NEGOTIATE] -> [DROP] -> PADDING_NEGOTIATED -> DROP... + + After which normal application DATA-bearing cells continue on the circuit. +``` + +Hence this way we make rendezvous circuits look like general circuits up +till the end of the circuit setup. + +After that our machine gets deactivated, and we let the actual rendezvous +circuit shape the traffic flow. Since rendezvous circuits usually imitate +general circuits (their purpose is to surf the web), we can expect that they +will look alike. + +<a id="padding-spec.txt-3.3.4"></a> + +### Circuit setup machine overhead { #setup-overhead } + +For the intro circuit case, we see that the origin-side machine just sends a +single PADDING_NEGOTIATE message, whereas the origin-side machine sends a +PADDING_NEGOTIATED message and between 7 to 10 DROP cells. This means that the +average overhead of this machine is 11 padding cells per introduction circuit. + +For the rend circuit case, this machine is quite light. Both sides send 2 +padding cells, for a total of 4 padding cells. + +<a id="padding-spec.txt-3.4"></a> + +## Circuit padding consensus parameters { #consenus-parameters } + +The circuit padding system has a handful of consensus parameters that can +either disable circuit padding entirely, or rate limit the total overhead +at relays and clients. + +```text + * circpad_padding_disabled + - If set to 1, no circuit padding machines will negotiate, and all + current padding machines will cease padding immediately. + - Default: 0 + + * circpad_padding_reduced + - If set to 1, only circuit padding machines marked as "reduced"/"low + overhead" will be used. (Currently no such machines are marked + as "reduced overhead"). + - Default: 0 + + * circpad_global_allowed_cells + - This is the number of padding cells that must be sent before + the 'circpad_global_max_padding_percent' parameter is applied. + - Default: 0 + + * circpad_global_max_padding_percent + - This is the maximum ratio of padding cells to total cells, specified + as a percent. If the global ratio of padding cells to total cells + across all circuits exceeds this percent value, no more padding is sent + until the ratio becomes lower. 0 means no limit. + - Default: 0 + + * circpad_max_circ_queued_cells + - This is the maximum number of cells that can be in the circuitmux queue + before padding stops being sent on that circuit. + - Default: CIRCWINDOW_START_MAX (1000) +``` diff --git a/spec/padding-spec/connection-level-padding.md b/spec/padding-spec/connection-level-padding.md new file mode 100644 index 0000000..f3c0fb7 --- /dev/null +++ b/spec/padding-spec/connection-level-padding.md @@ -0,0 +1,289 @@ +<a id="padding-spec.txt-2"></a> + +# Connection-level padding + +<a id="padding-spec.txt-2.1"></a> + +## Background + +Tor clients and relays make use of PADDING to reduce the resolution of +connection-level metadata retention by ISPs and surveillance infrastructure. + +Such metadata retention is implemented by Internet routers in the form of +Netflow, jFlow, Netstream, or IPFIX records. These records are emitted by +gateway routers in a raw form and then exported (often over plaintext) to a +"collector" that either records them verbatim, or reduces their granularity +further\[1\]. + +Netflow records and the associated data collection and retention tools are +very configurable, and have many modes of operation, especially when +configured to handle high throughput. However, at ISP scale, per-flow records +are very likely to be employed, since they are the default, and also provide +very high resolution in terms of endpoint activity, second only to full packet +and/or header capture. + +Per-flow records record the endpoint connection 5-tuple, as well as the +total number of bytes sent and received by that 5-tuple during a particular +time period. They can store additional fields as well, but it is primarily +timing and bytecount information that concern us. + +When configured to provide per-flow data, routers emit these raw flow +records periodically for all active connections passing through them +based on two parameters: the "active flow timeout" and the "inactive +flow timeout". + +The "active flow timeout" causes the router to emit a new record +periodically for every active TCP session that continuously sends data. The +default active flow timeout for most routers is 30 minutes, meaning that a +new record is created for every TCP session at least every 30 minutes, no +matter what. This value can be configured from 1 minute to 60 minutes on +major routers. + +The "inactive flow timeout" is used by routers to create a new record if a +TCP session is inactive for some number of seconds. It allows routers to +avoid the need to track a large number of idle connections in memory, and +instead emit a separate record only when there is activity. This value +ranges from 10 seconds to 600 seconds on common routers. It appears as +though no routers support a value lower than 10 seconds. + +For reference, here are default values and ranges (in parenthesis when +known) for common routers, along with citations to their manuals. + +Some routers speak other collection protocols than Netflow, and in the +case of Juniper, use different timeouts for these protocols. Where this +is known to happen, it has been noted. + +```text + Inactive Timeout Active Timeout + Cisco IOS[3] 15s (10-600s) 30min (1-60min) + Cisco Catalyst[4] 5min 32min + Juniper (jFlow)[5] 15s (10-600s) 30min (1-60min) + Juniper (Netflow)[6,7] 60s (10-600s) 30min (1-30min) + H3C (Netstream)[8] 60s (60-600s) 30min (1-60min) + Fortinet[9] 15s 30min + MicroTik[10] 15s 30min + nProbe[14] 30s 120s + Alcatel-Lucent[2] 15s (10-600s) 30min (1-600min) +``` + +The combination of the active and inactive netflow record timeouts allow us +to devise a low-cost padding defense that causes what would otherwise be +split records to "collapse" at the router even before they are exported to +the collector for storage. So long as a connection transmits data before the +"inactive flow timeout" expires, then the router will continue to count the +total bytes on that flow before finally emitting a record at the "active +flow timeout". + +This means that for a minimal amount of padding that prevents the "inactive +flow timeout" from expiring, it is possible to reduce the resolution of raw +per-flow netflow data to the total amount of bytes send and received in a 30 +minute window. This is a vast reduction in resolution for HTTP, IRC, XMPP, +SSH, and other intermittent interactive traffic, especially when all +user traffic in that time period is multiplexed over a single connection +(as it is with Tor). + +Though flow measurement in principle can be bidirectional (counting cells +sent in both directions between a pair of IPs) or unidirectional (counting +only cells sent from one IP to another), we assume for safety that all +measurement is unidirectional, and so traffic must be sent by both parties +in order to prevent record splitting. + +<a id="padding-spec.txt-2.2"></a> + +## Implementation + +Tor clients currently maintain one TLS connection to their Guard node to +carry actual application traffic, and make up to 3 additional connections to +other nodes to retrieve directory information. + +We pad only the client's connection to the Guard node, and not any other +connection. We treat Bridge node connections to the Tor network as client +connections, and pad them, but otherwise not pad between normal relays. + +Both clients and Guards will maintain a timer for all application (ie: +non-directory) TLS connections. Every time a padding packet sent by an +endpoint, that endpoint will sample a timeout value from +the max(X,X) distribution described in Section 2.3. The default +range is from 1.5 seconds to 9.5 seconds time range, subject to consensus +parameters as specified in Section 2.6. + +(The timing is randomized to avoid making it obvious which cells are +padding.) + +If another cell is sent for any reason before this timer expires, the timer +is reset to a new random value. + +If the connection remains inactive until the timer expires, a +single PADDING cell will be sent on that connection (which will +also start a new timer). + +In this way, the connection will only be padded in a given direction in +the event that it is idle in that direction, and will always transmit a +packet before the minimum 10 second inactive timeout. + +(In practice, an implementation may not be able to determine when, +exactly, a cell is sent on a given channel. For example, even though the +cell has been given to the kernel via a call to `send(2)`, the kernel may +still be buffering that cell. In cases such as these, implementations +should use a reasonable proxy for the time at which a cell is sent: for +example, when the cell is queued. If this strategy is used, +implementations should try to observe the innermost (closest to the wire) +queue that they practically can, and if this queue is already nonempty, +padding should not be scheduled until after the queue does become empty.) + +<a id="padding-spec.txt-2.3"></a> + +## Padding Cell Timeout Distribution Statistics { #distribution-statistics } + +To limit the amount of padding sent, instead of sampling each endpoint +timeout uniformly, we instead sample it from max(X,X), where X is +uniformly distributed. + +If X is a random variable uniform from 0..R-1 (where R=high-low), then the +random variable Y = max(X,X) has Prob(Y == i) = (2.0*i + 1)/(R*R). + +Then, when both sides apply timeouts sampled from Y, the resulting +bidirectional padding packet rate is now a third random variable: +Z = min(Y,Y). + +The distribution of Z is slightly bell-shaped, but mostly flat around the +mean. It also turns out that Exp\[Z\] ~= Exp\[X\]. Here's a table of average +values for each random variable: + +```text + R Exp[X] Exp[Z] Exp[min(X,X)] Exp[Y=max(X,X)] + 2000 999.5 1066 666.2 1332.8 + 3000 1499.5 1599.5 999.5 1999.5 + 5000 2499.5 2666 1666.2 3332.8 + 6000 2999.5 3199.5 1999.5 3999.5 + 7000 3499.5 3732.8 2332.8 4666.2 + 8000 3999.5 4266.2 2666.2 5332.8 + 10000 4999.5 5328 3332.8 6666.2 + 15000 7499.5 7995 4999.5 9999.5 + 20000 9900.5 10661 6666.2 13332.8 +``` + +<a id="padding-spec.txt-2.4"></a> + +## Maximum overhead bounds + +With the default parameters and the above distribution, we expect a +padded connection to send one padding cell every 5.5 seconds. This +averages to 103 bytes per second full duplex (~52 bytes/sec in each +direction), assuming a 512 byte cell and 55 bytes of TLS+TCP+IP headers. +For a client connection that remains otherwise idle for its expected +~50 minute lifespan (governed by the circuit available timeout plus a +small additional connection timeout), this is about 154.5KB of overhead +in each direction (309KB total). + +With 2.5M completely idle clients connected simultaneously, 52 bytes per +second amounts to 130MB/second in each direction network-wide, which is +roughly the current amount of Tor directory traffic\[11\]. Of course, our +2.5M daily users will neither be connected simultaneously, nor entirely +idle, so we expect the actual overhead to be much lower than this. + +<a id="padding-spec.txt-2.5"></a> + +## Reducing or Disabling Padding via Negotiation { #negotiation } + +To allow mobile clients to either disable or reduce their padding overhead, +the PADDING_NEGOTIATE cell (tor-spec.txt section 7.2) may be sent from +clients to relays. This cell is used to instruct relays to cease sending +padding. + +If the client has opted to use reduced padding, it continues to send +padding cells sampled from the range \[9000,14000\] milliseconds (subject to +consensus parameter alteration as per Section 2.6), still using the +Y=max(X,X) distribution. Since the padding is now unidirectional, the +expected frequency of padding cells is now governed by the Y distribution +above as opposed to Z. For a range of 5000ms, we can see that we expect to +send a padding packet every 9000+3332.8 = 12332.8ms. We also half the +circuit available timeout from ~50min down to ~25min, which causes the +client's OR connections to be closed shortly there after when it is idle, +thus reducing overhead. + +These two changes cause the padding overhead to go from 309KB per one-time-use +Tor connection down to 69KB per one-time-use Tor connection. For continual +usage, the maximum overhead goes from 103 bytes/sec down to 46 bytes/sec. + +If a client opts to completely disable padding, it sends a +PADDING_NEGOTIATE to instruct the relay not to pad, and then does not +send any further padding itself. + +Currently, clients negotiate padding only when a channel is created, +immediately after sending their NETINFO cell. Recipients SHOULD, however, +accept padding negotiation messages at any time. + +If a client which previously negotiated reduced, or disabled, padding, and +wishes to re-enable default padding (ie padding according to the consensus +parameters), it SHOULD send PADDING_NEGOTIATE START with zero in the +ito_low_ms and ito_high_ms fields. (It therefore SHOULD NOT copy the values +from its own established consensus into the PADDING_NEGOTIATE cell.) +This avoids the client needing to send updated padding negotiations if the +consensus parameters should change. The recipient's clamping of the timing +parameters will cause the recipient to use its notion of the consensus +parameters. + +Clients and bridges MUST reject padding negotiation messages from relays, +and close the channel if they receive one. + +<a id="padding-spec.txt-2.6"></a> + +## Consensus Parameters Governing Behavior { #consensus-parameters } + +Connection-level padding is controlled by the following consensus parameters: + +```text + * nf_ito_low + - The low end of the range to send padding when inactive, in ms. + - Default: 1500 + + * nf_ito_high + - The high end of the range to send padding, in ms. + - Default: 9500 + - If nf_ito_low == nf_ito_high == 0, padding will be disabled. + + * nf_ito_low_reduced + - For reduced padding clients: the low end of the range to send padding + when inactive, in ms. + - Default: 9000 + + * nf_ito_high_reduced + - For reduced padding clients: the high end of the range to send padding, + in ms. + - Default: 14000 + + * nf_conntimeout_clients + - The number of seconds to keep never-used circuits opened and + available for clients to use. Note that the actual client timeout is + randomized uniformly from this value to twice this value. + - The number of seconds to keep idle (not currently used) canonical + channels are open and available. (We do this to ensure a sufficient + time duration of padding, which is the ultimate goal.) + - This value is also used to determine how long, after a port has been + used, we should attempt to keep building predicted circuits for that + port. (See path-spec.txt section 2.1.1.) This behavior was + originally added to work around implementation limitations, but it + serves as a reasonable default regardless of implementation. + - For all use cases, reduced padding clients use half the consensus + value. + - Implementations MAY mark circuits held open past the reduced padding + quantity (half the consensus value) as "not to be used for streams", + to prevent their use from becoming a distinguisher. + - Default: 1800 + + * nf_pad_before_usage + - If set to 1, OR connections are padded before the client uses them + for any application traffic. If 0, OR connections are not padded + until application data begins. + - Default: 1 + + * nf_pad_relays + - If set to 1, we also pad inactive relay-to-relay connections + - Default: 0 + + * nf_conntimeout_relays + - The number of seconds that idle relay-to-relay connections are kept + open. + - Default: 3600 +``` diff --git a/spec/padding-spec/index.md b/spec/padding-spec/index.md new file mode 100644 index 0000000..70726a7 --- /dev/null +++ b/spec/padding-spec/index.md @@ -0,0 +1,11 @@ +# Tor Padding Specification + +Mike Perry, George Kadianakis + +Note: This is an attempt to specify Tor as currently implemented. Future +versions of Tor will implement improved algorithms. + +This document tries to cover how Tor chooses to use cover traffic to obscure +various traffic patterns from external and internal observers. Other +implementations MAY take other approaches, but implementors should be aware of +the anonymity and load-balancing implications of their choices. diff --git a/spec/padding-spec/overview.md b/spec/padding-spec/overview.md new file mode 100644 index 0000000..594b9d4 --- /dev/null +++ b/spec/padding-spec/overview.md @@ -0,0 +1,28 @@ +<a id="padding-spec.txt-1"></a> + +# Overview + +Tor supports two classes of cover traffic: connection-level padding, and +circuit-level padding. + +Connection-level padding uses the PADDING cell command for cover +traffic, where as circuit-level padding uses the DROP relay +command. PADDING cells are single-hop only and can be differentiated from +normal traffic by Tor relays ("internal" observers), but not by entities +monitoring Tor OR connections ("external" observers). + +DROP relay messages are multi-hop, and is not visible to intermediate Tor +relays, because the relay command field is covered by circuit layer +encryption. Moreover, Tor's 'recognized' field allows DROP messages +to be sent to any intermediate node in a circuit (as per Section +6.1 of tor-spec.txt). + +Tor uses both connection level and circuit level padding. Connection +level padding is described in section 2. Circuit level padding is +described in section 3. + +The circuit-level padding system is completely orthogonal to the +connection-level padding. The connection-level padding system regards +circuit-level padding as normal data traffic, and hence the connection-level +padding system will not add any additional overhead while the circuit-level +padding system is actively padding. diff --git a/spec/param-spec.md b/spec/param-spec.md new file mode 100644 index 0000000..cd0af5e --- /dev/null +++ b/spec/param-spec.md @@ -0,0 +1,619 @@ +# Tor network parameters + +This file lists the recognized parameters that can appear on the "params" +line of a directory consensus. + +<a id="param-spec.txt-1"></a> + +## Network protocol parameters {#network-protocol} + +"circwindow" -- the default package window that circuits should be +established with. It started out at 1000 DATA-bearing relay cells, +but some research +indicates that a lower value would mean fewer cells in transit in the +network at any given time. +Min: 100, Max: 1000, Default: 1000 +First-appeared: Tor 0.2.1.20 + +"UseOptimisticData" -- If set to zero, clients by default shouldn't try +to send optimistic data to servers until they have received a +CONNECTED message. +Min: 0, Max: 1, Default: 1 +First-appeared: 0.2.3.3-alpha +Default was 0 before: 0.2.9.1-alpha +Removed in 0.4.5.1-alpha; now always on. + +"usecreatefast" -- Used to control whether clients use the CREATE_FAST +handshake on the first hop of their circuits. +Min: 0, Max: 1. Default: 1. +First-appeared: 0.2.4.23, 0.2.5.2-alpha +Removed in 0.4.5.1-alpha; now always off. + +"min_paths_for_circs_pct" -- A percentage threshold that determines +whether clients believe they have enough directory information to +build circuits. This value applies to the total fraction of +bandwidth-weighted paths that the client could build; see +path-spec.txt for more information. +Min: 25, Max: 95, Default: 60 +First-appeared: 0.2.4 + +"ExtendByEd25519ID" -- If true, clients should include Ed25519 +identities for relays when generating EXTEND2 messages. +Min: 0. Max: 1. Default: 0. +First-appeared: 0.3.0 + +"sendme_emit_min_version" -- Minimum SENDME version that can be sent. +Min: 0. Max: 255. Default 0. +First appeared: 0.4.1.1-alpha. + +"sendme_accept_min_version" -- Minimum SENDME version that is accepted. +Min: 0. Max: 255. Default 0. +First appeared: 0.4.1.1-alpha. + +"allow-network-reentry" -- If true, the Exit relays allow connections that +are exiting the network to re-enter. If false, any exit connections going +to a relay ORPort or an authority ORPort and DirPort is denied and the +stream is terminated. +Min: 0. Max: 1. Default: 0 +First appeared: 0.4.5.1-alpha. + +<a id="param-spec.txt-2"></a> + +## Performance-tuning parameters {#performance-tuning} + +"CircuitPriorityHalflifeMsec" -- the halflife parameter used when +weighting which circuit will send the next relay cell. Obeyed by Tor +0.2.2.10-alpha and later. (Versions of Tor between 0.2.2.7-alpha and +0.2.2.10-alpha recognized a "CircPriorityHalflifeMsec" parameter, but +mishandled it badly.) +Min: 1, Max: 2147483647 (INT32_MAX), Default: 30000. +First-appeared: Tor 0.2.2.11-alpha + +```text + "perconnbwrate" and "perconnbwburst" -- if set, each relay sets up a + separate token bucket for every client OR connection, and rate limits + that connection independently. Typically left unset, except when used for + performance experiments around trac entry 1750. Only honored by relays + running Tor 0.2.2.16-alpha and later. (Note that relays running + 0.2.2.7-alpha through 0.2.2.14-alpha looked for bwconnrate and + bwconnburst, but then did the wrong thing with them; see bug 1830 for + details.) + Min: 1, Max: 2147483647 (INT32_MAX), Default: (user setting of + BandwidthRate/BandwidthBurst). + First-appeared: 0.2.2.7-alpha + Removed-in: 0.2.2.16-alpha +``` + +"NumNTorsPerTAP" -- When balancing ntor and TAP requests at relays, +how many ntor handshakes should we perform for each TAP handshake? +Min: 1. Max: 100000. Default: 10. +First-appeared: 0.2.4.17-rc + +"circ_max_cell_queue_size" -- This parameter determines the maximum +number of relay cells allowed per circuit queue. +Min: 1000. Max: 2147483647 (INT32_MAX). Default: 50000. +First-appeared: 0.3.3.6-rc. + +"KISTSchedRunInterval" -- How frequently should the "KIST" scheduler +run in order to decide which data to write to the network? Value in +units of milliseconds. +Min: 2. Max: 100. Default: 2 +First appeared: 0.3.2 + +"KISTSchedRunIntervalClient" -- How frequently should the "KIST" scheduler +run in order to decide which data to write to the network, on clients? Value +in units of milliseconds. The client value needs to be much lower than +the relay value. +Min: 2. Max: 100. Default: 2. +First appeared: 0.4.8.2 + +<a id="param-spec.txt-3"></a> + +## Voting-related parameters {#voting} + +"bwweightscale" -- Value that bandwidth-weights are divided by. If not +present then this defaults to 10000. +Min: 1 +First-appeared: 0.2.2.10-alpha + +"maxunmeasuredbw" -- Used by authorities during voting with method 17 or +later. The maximum value to give for any Bandwidth= entry for a router +that isn't based on at least three measurements. + +(Note: starting in version 0.4.6.1-alpha +there was a bug where Tor authorities would instead look at +a parameter called "maxunmeasurdbw", without the "e". +This bug was fixed in 0.4.9.1-alpha and in 0.4.8.8. +Until all relays are running a fixed version, then either this parameter +must not be set, or it must be set to the same value for both +spellings.) + +First-appeared: 0.2.4.11-alpha + +"FastFlagMinThreshold", "FastFlagMaxThreshold" -- lowest and highest +allowable values for the cutoff for routers that should get the Fast +flag. This is used during voting to prevent the threshold for getting +the Fast flag from being too low or too high. +FastFlagMinThreshold: Min: 4. Max: INT32_MAX: Default: 4. +FastFlagMaxThreshold: Min: -. Max: INT32_MAX: Default: INT32_MAX +First-appeared: 0.2.3.11-alpha + +"AuthDirNumSRVAgreements" -- Minimum number of agreeing directory +authority votes required for a fresh shared random value to be written in +the consensus (this rule only applies on the first commit round of the +shared randomness protocol). +Min: 1. Max: INT32_MAX. Default: 2/3 of the total number of +dirauth. + +<a id="param-spec.txt-4"></a> + +## Circuit-build-timeout parameters {#cbt} + +"cbtdisabled", "cbtnummodes", "cbtrecentcount", "cbtmaxtimeouts", +"cbtmincircs", "cbtquantile", "cbtclosequantile", "cbttestfreq", +"cbtmintimeout", "cbtlearntimeout", "cbtmaxopencircs", and +"cbtinitialtimeout" -- see "2.4.5. Consensus parameters governing +behavior" in path-spec.txt for a series of circuit build time related +consensus parameters. + +<a id="param-spec.txt-5"></a> + +## Directory-related parameters {#directory} + +"max-consensus-age-to-cache-for-diff" -- Determines how much +consensus history (in hours) relays should try to cache in order to +serve diffs. (min 0, max 8192, default 72) + +"try-diff-for-consensus-newer-than" -- This parameter determines how +old a consensus can be (in hours) before a client should no longer +try to find a diff for it. (min 0, max 8192, default 72) + +<a id="param-spec.txt-6"></a> + +## Pathbias parameters {#pathbias} + +"pb_mincircs", "pb_noticepct", "pb_warnpct", "pb_extremepct", +"pb_dropguards", "pb_scalecircs", "pb_scalefactor", +"pb_multfactor", "pb_minuse", "pb_noticeusepct", +"pb_extremeusepct", "pb_scaleuse" -- DOCDOC + +<a id="param-spec.txt-7"></a> + +## Relay behavior parameters {#relay-behavior} + +"refuseunknownexits" -- if set to one, exit relays look at the previous +hop of circuits that ask to open an exit stream, and refuse to exit if +they don't recognize it as a relay. The goal is to make it harder for +people to use them as one-hop proxies. See trac entry 1751 for details. +Min: 0, Max: 1 +First-appeared: 0.2.2.17-alpha + +<a id="onion-key-rotation-days"></a> + +"onion-key-rotation-days" -- (min 1, max 90, default 28) + +"onion-key-grace-period-days" -- (min 1, max +onion-key-rotation-days, default 7) + +Every relay should list each onion key it generates for +onion-key-rotation-days days after generating it, and then +replace it. Relays should continue to accept their most recent +previous onion key for an additional onion-key-grace-period-days +days after it is replaced. (Introduced in 0.3.1.1-alpha; +prior versions of tor hardcoded both of these values to 7 days.) + +"AllowNonearlyExtend" -- If true, permit EXTEND/EXTEND2 requests that are not inside +RELAY_EARLY cells. +Min: 0. Max: 1. Default: 0. +First-appeared: 0.2.3.11-alpha + +"overload_dns_timeout_scale_percent" -- This value is a percentage of how +many DNS timeout over N seconds we accept before reporting the overload +general state. It is scaled by a factor of 1000 in order to be able to +represent decimal point. As an example, a value of 1000 means 1%. +Min: 0. Max: 100000. Default: 1000. +First-appeared: 0.4.6.8 +Deprecated: 0.4.7.3-alpha-dev + +"overload_dns_timeout_period_secs" -- This value is the period in seconds +of the DNS timeout measurements (the N in the +"overload_dns_timeout_scale_percent" parameter). For this amount of +seconds, we will gather DNS statistics and at the end, we'll do an +assessment on the overload general signal with regards to DNS timeouts. +Min: 0. Max: 2147483647. Default: 600 +First-appeared: 0.4.6.8 +Deprecated: 0.4.7.3-alpha-dev + +"overload_onionskin_ntor_scale_percent" -- This value is a percentage of +how many onionskin ntor drop over N seconds we accept before reporting the +overload general state. It is scaled by a factor of 1000 in order to be +able to represent decimal point. As an example, a value of 1000 means 1%. +Min: 0. Max: 100000. Default: 1000. +First-appeared: 0.4.7.5-alpha + +"overload_onionskin_ntor_period_secs" -- This value is the period in +seconds of the onionskin ntor overload measurements (the N in the +"overload_onionskin_ntor_scale_percent" parameter). For this amount of +seconds, we will gather onionskin ntor statistics and at the end, we'll do +an assessment on the overload general signal. +Min: 0. Max: 2147483647. Default: 21600 (6 hours) +First-appeared: 0.4.7.5-alpha + +"assume-reachable" -- If true, relays should publish descriptors +even when they cannot make a connection to their IPv4 ORPort. +Min: 0. Max: 1. Default: 0. +First appeared: 0.4.5.1-alpha. + +"assume-reachable-ipv6" -- If true, relays should publish +descriptors even when they cannot make a connection to their IPv6 +ORPort. +Min: 0. Max: 1. Default: 0. +First appeared: 0.4.5.1-alpha. + +"exit_dns_timeout" -- The time in milliseconds an Exit sets libevent to +wait before it considers the DNS timed out. The corresponding libevent +option is "timeout:". +Min: 1. Max: 120000. Default: 1000 (1sec) +First appeared: 0.4.7.5-alpha. + +"exit_dns_num_attempts" -- How many attempts _after the first_ should an +Exit should try a timing-out DNS query before calling it hopeless? (Each of +these attempts will wait for "exit_dns_timeout" independently). The +corresponding libevent option is "attempts:". +Min: 0. Max: 255. Default: 2 +First appeared: 0.4.7.5-alpha. + +<a id="param-spec.txt-8"></a> + +## V3 onion service parameters {#onion-service} + +"hs_intro_min_introduce2", "hs_intro_max_introduce2" -- +Minimum/maximum amount of INTRODUCE2 messages allowed per circuit +before rotation (actual amount picked at random between these two +values). +Min: 0. Max: INT32_MAX. Defaults: 16384, 32768. + +"hs_intro_min_lifetime", "hs_intro_max_lifetime" -- Minimum/maximum +lifetime in seconds that a service should keep an intro point for +(actual lifetime picked at random between these two values). +Min: 0. Max: INT32_MAX. Defaults: 18 hours, 24 hours. + +"hs_intro_num_extra" -- Number of extra intro points a service is +allowed to open. This concept comes from proposal #155. +Min: 0. Max: 128. Default: 2. + +"hsdir_interval" -- The length of a time period, _in minutes_. See +rend-spec-v3.txt section \[TIME-PERIODS\]. +Min: 30. Max: 14400. Default: 1440. + +"hsdir_n_replicas" -- Number of HS descriptor replicas. +Min: 1. Max: 16. Default: 2. + +"hsdir_spread_fetch" -- Total number of HSDirs per replica a tor +client should select to try to fetch a descriptor. +Min: 1. Max: 128. Default: 3. + +"hsdir_spread_store" -- Total number of HSDirs per replica a service +will upload its descriptor to. +Min: 1. Max: 128. Default: 4 + +"HSV3MaxDescriptorSize" -- Maximum descriptor size (in bytes). +Min: 1. Max: INT32_MAX. Default: 50000 + +"hs_service_max_rdv_failures" -- This parameter determines the +maximum number of rendezvous attempt an HS service can make per +introduction. +Min 1. Max 10. Default 2. +First-appeared: 0.3.3.0-alpha. + +<a id="HiddenServiceEnableIntroDoSDefense"></a> +"HiddenServiceEnableIntroDoSDefense" -- This parameter makes +introduction points +start using a rate-limiting defense if they support it. +Introduction points use this value when no +[`DOS_PARAMS` extension] is sent in the ESTABLISH_INTRO message. +Min: 0. Max: 1. Default: 0. +First appeared: 0.4.2.1-alpha. + +<a id="HiddenServiceEnableIntroDoSBurstPerSec"></a> +"HiddenServiceEnableIntroDoSBurstPerSec" -- Default maximum burst +rate to be used +for token bucket for the introduction point rate-limiting. +Introduction points use this value when no +[`DOS_PARAMS` extension] is sent in the ESTABLISH_INTRO message. +Min: 0. Max: INT32_MAX. Default: 200 +First appeared: 0.4.2.1-alpha. + +> Note that the above parameter is slightly misnamed: +> a burst is not meaningfully "per second". + +<a id="HiddenServiceEnableIntroDoSRatePerSec"></a> +"HiddenServiceEnableIntroDoSRatePerSec" -- Default maximum rate per second +to be used +for token bucket for the introduction point rate-limiting. +Introduction points use this value when no +[`DOS_PARAMS` extension] is sent. +Min: 0. Max: INT32_MAX. Default: 25 +First appeared: 0.4.2.1-alpha. + +[`DOS_PARAMS` extension]: ./rend-spec/introduction-protocol.md#DOS_PARAMS + +<a id="param-spec.txt-9"></a> + +## Denial-of-service parameters {#dos} + +Denial of Service mitigation parameters. Introduced in 0.3.3.2-alpha: + +"DoSCircuitCreationEnabled" -- Enable the circuit creation DoS +mitigation. + +"DoSCircuitCreationMinConnections" -- Minimum threshold of +concurrent connections before a client address can be flagged as +executing a circuit creation DoS + +"DoSCircuitCreationRate" -- Allowed circuit creation rate per second +per client IP address once the minimum concurrent connection +threshold is reached. + +"DoSCircuitCreationBurst" -- The allowed circuit creation burst per +client IP address once the minimum concurrent connection threshold +is reached. + +```text + "DoSCircuitCreationDefenseType" -- Defense type applied to a + detected client address for the circuit creation mitigation. + 1: No defense. + 2: Refuse circuit creation for the length of + "DoSCircuitCreationDefenseTimePeriod". +``` + +"DoSCircuitCreationDefenseTimePeriod" -- The base time period that +the DoS defense is activated for. + +"DoSConnectionEnabled" -- Enable the connection DoS mitigation. + +"DoSConnectionMaxConcurrentCount" -- The maximum threshold of +concurrent connection from a client IP address. + +```text + "DoSConnectionDefenseType" -- Defense type applied to a detected + client address for the connection mitigation. Possible values are: + 1: No defense. + 2: Immediately close new connections. +``` + +"DoSRefuseSingleHopClientRendezvous" -- Refuse establishment of +rendezvous points for single hop clients. + +"DoSStreamCreationEnabled" -- Enable the stream creation DoS +mitigation. +First appeared: 0.4.9.0-alpha-dev. + +"DoSStreamCreationRate" -- Allowed stream creation rate per second +per circuit. +First appeared: 0.4.9.0-alpha-dev. + +"DoSStreamCreationBurst" -- The allowed stream creation burst per +circuit. +First appeared: 0.4.9.0-alpha-dev. + +```text + "DoSStreamCreationDefenseType" -- Defense type applied to a + stream for the stream creation mitigation. + 1: No defense. + 2: Reject the stream or resolve request. + 3: Close the underlying circuit. + First appeared: 0.4.9.0-alpha-dev. +``` + +<a id="param-spec.txt-10"></a> + +## Padding-related parameters {#padding} + +"circpad_max_circ_queued_cells" -- The circuitpadding module will +stop sending more padding relay cells if more than this many cells are in +the circuit queue a given circuit. +Min: 0. Max: 50000. Default 1000. +First appeared: 0.4.0.3-alpha. + +"circpad_global_allowed_cells" -- This is the number of padding relay cells +that must be sent before the 'circpad_global_max_padding_percent' +parameter is applied. +Min: 0. Max: 65535. Default: 0 + +"circpad_global_max_padding_pct" -- This is the maximum ratio of +padding relay cells to total relay cells, specified as a percent. If the global +ratio of padding cells to total cells across all circuits exceeds +this percent value, no more padding is sent until the ratio becomes +lower. 0 means no limit. +Min: 0. Max: 100. Default: 0 + +"circpad_padding_disabled" -- If set to 1, no circuit padding machines +will negotiate, and all current padding machines will cease padding +immediately. +Min: 0. Max: 1. Default: 0 + +"circpad_padding_reduced" -- If set to 1, only circuit padding +machines marked as "reduced"/"low overhead" will be used. +(Currently no such machines are marked as "reduced overhead"). +Min: 0. Max: 1. Default: 0 + +"nf_conntimeout_clients" + - The number of seconds to keep never-used circuits opened and + available for clients to use. Note that the actual client timeout is + randomized uniformly from this value to twice this value. + - The number of seconds to keep idle (not currently used) canonical + channels are open and available. (We do this to ensure a sufficient + time duration of padding, which is the ultimate goal.) + - This value is also used to determine how long, after a port has been + used, we should attempt to keep building predicted circuits for that + port. (See path-spec.txt section 2.1.1.) This behavior was + originally added to work around implementation limitations, but it + serves as a reasonable default regardless of implementation. + - For all use cases, reduced padding clients use half the consensus + value. + - Implementations MAY mark circuits held open past the reduced padding + quantity (half the consensus value) as "not to be used for streams", + to prevent their use from becoming a distinguisher. +Min: 60. Max: 86400. Default: 1800 + +"nf_conntimeout_relays" -- The number of seconds that idle +relay-to-relay connections are kept open. +Min: 60. Max: 604800. Default: 3600 + +"nf_ito_low" -- The low end of the range to send padding when +inactive, in ms. +Min: 0. Max: 60000. Default: 1500 + +"nf_ito_high" -- The high end of the range to send padding, in ms. +If nf_ito_low == nf_ito_high == 0, padding will be disabled. +Min: nf_ito_low. Max: 60000. Default: 9500 + +"nf_ito_low_reduced" -- For reduced padding clients: the low +end of the range to send padding when inactive, in ms. +Min: 0. Max: 60000. Default: 9000 + +"nf_ito_high_reduced" -- For reduced padding clients: the high +end of the range to send padding, in ms. +Min: nf_ito_low_reduced. Max: 60000. Default: 14000 + +"nf_pad_before_usage" -- If set to 1, OR connections are padded +before the client uses them for any application traffic. If 0, +OR connections are not padded until application data begins. +Min: 0. Max: 1. Default: 1 + +"nf_pad_relays" -- If set to 1, we also pad inactive +relay-to-relay connections. +Min: 0. Max: 1. Default: 0 + +"nf_pad_single_onion" -- DOCDOC + +<a id="param-spec.txt-11"></a> + +## Guard-related parameters + +(See guard-spec.txt for more information on the vocabulary used here.) + +"UseGuardFraction" -- If true, clients use `GuardFraction` +information from the consensus in order to decide how to weight +guards when picking them. +Min: 0. Max: 1. Default: 0. +First appeared: 0.2.6 + +"guard-lifetime-days" -- Controls guard lifetime. If an unconfirmed +guard has been sampled more than this many days ago, it should be +removed from the guard sample. +Min: 1. Max: 3650. Default: 120. +First appeared: 0.3.0 + +"guard-confirmed-min-lifetime-days" -- Controls confirmed guard +lifetime: if a guard was confirmed more than this many days ago, it +should be removed from the guard sample. +Min: 1. Max: 3650. Default: 60. +First appeared: 0.3.0 + +"guard-internet-likely-down-interval" -- If Tor has been unable to +build a circuit for this long (in seconds), assume that the internet +connection is down, and treat guard failures as unproven. +Min: 1. Max: INT32_MAX. Default: 600. +First appeared: 0.3.0 + +"guard-max-sample-size" -- Largest number of guards that clients +should try to collect in their sample. +Min: 1. Max: INT32_MAX. Default: 60. +First appeared: 0.3.0 + +"guard-max-sample-threshold-percent" -- Largest bandwidth-weighted +fraction of guards that clients should try to collect in their +sample. +Min: 1. Max: 100. Default: 20. +First appeared: 0.3.0 + +"guard-meaningful-restriction-percent" -- If the client has +configured tor to exclude so many guards that the available guard +bandwidth is less than this percentage of the total, treat the guard +sample as "restricted", and keep it in a separate sample. +Min: 1. Max: 100. Default: 20. +First appeared: 0.3.0 + +"guard-extreme-restriction-percent" -- Warn the user if they have +configured tor to exclude so many guards that the available guard +bandwidth is less than this percentage of the total. +Min: 1. Max: 100. Default: 1. +First appeared: 0.3.0. MAX was INT32_MAX, which would have no meaningful +effect. MAX lowered to 100 in 0.4.7. + +"guard-min-filtered-sample-size" -- If fewer than this number of +guards is available in the sample after filtering out unusable +guards, the client should try to add more guards to the sample (if +allowed). +Min: 1. Max: INT32_MAX. Default: 20. +First appeared: 0.3.0 + +"guard-n-primary-guards" -- The number of confirmed guards that the +client should treat as "primary guards". +Min: 1. Max: INT32_MAX. Default: 3. +First appeared: 0.3.0 + +```text + "guard-n-primary-guards-to-use", "guard-n-primary-dir-guards-to-use" + -- number of primary guards and primary directory guards that the + client should be willing to use in parallel. Other primary guards + won't get used unless the earlier ones are down. + "guard-n-primary-guards-to-use": + Min 1, Max INT32_MAX: Default: 1. + "guard-n-primary-dir-guards-to-use" + Min 1, Max INT32_MAX: Default: 3. + First appeared: 0.3.0 +``` + +"guard-nonprimary-guard-connect-timeout" -- When trying to confirm +nonprimary guards, if a guard doesn't answer for more than this long +in seconds, treat lower-priority guards as usable. +Min: 1. Max: INT32_MAX. Default: 15 +First appeared: 0.3.0 + +"guard-nonprimary-guard-idle-timeout" -- When trying to confirm +nonprimary guards, if a guard doesn't answer for more than this long +in seconds, treat it as down. +Min: 1. Max: INT32_MAX. Default: 600 +First appeared: 0.3.0 + +"guard-remove-unlisted-guards-after-days" -- If a guard has been +unlisted in the consensus for at least this many days, remove it +from the sample. +Min: 1. Max: 3650. Default: 20. +First appeared: 0.3.0 + +<a id="param-spec.txt-X"></a> + +## Obsolete parameters {#obsolete} + +"NumDirectoryGuards", "NumEntryGuards" -- Number of guard nodes +clients should use by default. If NumDirectoryGuards is 0, we +default to NumEntryGuards. +NumDirectoryGuards: Min: 0. Max: 10. Default: 0 +NumEntryGuards: Min: 1. Max: 10. Default: 3 +First-appeared: 0.2.4.23, 0.2.5.6-alpha +Removed in: 0.3.0 + +"GuardLifetime" -- Duration for which clients should choose guard +nodes, in seconds. +Min: 30 days. Max: 1826 days. Default: 60 days. +First-appeared: 0.2.4.12-alpha +Removed in: 0.3.0. + +"UseNTorHandshake" -- If true, then versions of Tor that support +NTor will prefer to use it by default. +Min: 0, Max: 1. Default: 1. +First-appeared: 0.2.4.8-alpha +Removed in: 0.2.9. + +"Support022HiddenServices" -- Used to implement a mass switch-over +from sending timestamps to hidden services by default to sending no +timestamps at all. If this option is absent, or is set to 1, +clients with the default configuration send timestamps; otherwise, +they do not. +Min: 0, Max: 1. Default: 1. +First-appeared: 0.2.4.18-rc +Removed in: 0.2.6 diff --git a/spec/path-spec/attaching-streams-to-circuits.md b/spec/path-spec/attaching-streams-to-circuits.md new file mode 100644 index 0000000..585885b --- /dev/null +++ b/spec/path-spec/attaching-streams-to-circuits.md @@ -0,0 +1,19 @@ +<a id="path-spec.txt-3"></a> + +# Attaching streams to circuits + +When a circuit that might support a request is built, Tor tries to attach +the request's stream to the circuit and sends a BEGIN, BEGIN_DIR, +or RESOLVE relay +cell as appropriate. If the request completes unsuccessfully, Tor +considers the reason given in the CLOSE relay cell. \[XXX yes, and?\] + +After a request has remained unattached for SocksTimeout (2 minutes +by default), Tor abandons the attempt and signals an error to the +client as appropriate (e.g., by closing the SOCKS connection). + +XXX Timeouts and when Tor auto-retries. + +- What stream-end-reasons are appropriate for retrying. + +If no reply to BEGIN/RESOLVE, then the stream will timeout and fail. diff --git a/spec/path-spec/building-circuits.md b/spec/path-spec/building-circuits.md new file mode 100644 index 0000000..dfb1332 --- /dev/null +++ b/spec/path-spec/building-circuits.md @@ -0,0 +1,9 @@ +<a id="path-spec.txt-2"></a> + +# Building circuits + +Here we describe a number of rules for building circuits: under what +circumstances we do so, how we choose the paths for them, when we give +up on an in-progress circuits, and what we do when circuit +construction fails. + diff --git a/spec/path-spec/cannibalizing-circuits.md b/spec/path-spec/cannibalizing-circuits.md new file mode 100644 index 0000000..3987839 --- /dev/null +++ b/spec/path-spec/cannibalizing-circuits.md @@ -0,0 +1,15 @@ +<a id="path-spec.txt-2.3"></a> + +# Cannibalizing circuits + +If we need a circuit and have a clean one already established, in +some cases we can adapt the clean circuit for our new +purpose. Specifically, + +For hidden service interactions, we can "cannibalize" a clean internal +circuit if one is available, so we don't need to build those circuits +from scratch on demand. + +We can also cannibalize clean circuits when the client asks to exit +at a given node -- either via the ".exit" notation or because the +destination is running at the same location as an exit node. diff --git a/spec/path-spec/detecting-route-manipulation.md b/spec/path-spec/detecting-route-manipulation.md new file mode 100644 index 0000000..76be4d1 --- /dev/null +++ b/spec/path-spec/detecting-route-manipulation.md @@ -0,0 +1,202 @@ +<a id="path-spec.txt-7"></a> + +# Detecting route manipulation by Guard nodes (Path Bias) {#pathbias} + +The Path Bias defense is designed to defend against a type of route +capture where malicious Guard nodes deliberately fail or choke circuits +that extend to non-colluding Exit nodes to maximize their network +utilization in favor of carrying only compromised traffic. + +In the extreme, the attack allows an adversary that carries c/n +of the network capacity to deanonymize c/n of the network +connections, breaking the O((c/n)^2) property of Tor's original +threat model. It also allows targeted attacks aimed at monitoring +the activity of specific users, bridges, or Guard nodes. + +There are two points where path selection can be manipulated: +during construction, and during usage. Circuit construction +can be manipulated by inducing circuit failures during circuit +extend steps, which causes the Tor client to transparently retry +the circuit construction with a new path. Circuit usage can be +manipulated by abusing the stream retry features of Tor (for +example by withholding stream attempt responses from the client +until the stream timeout has expired), at which point the tor client +will also transparently retry the stream on a new path. + +The defense as deployed therefore makes two independent sets of +measurements of successful path use: one during circuit construction, +and one during circuit usage. + +The intended behavior is for clients to ultimately disable the use +of Guards responsible for excessive circuit failure of either type +(for the parameters to do this, see ["Parameterization"](#parameters) below); +however known issues with the Tor network currently +restrict the defense to being informational only at this stage +(see ["Known barriers to enforcement"](#barriers)). + +<a id="path-spec.txt-7.1"></a> + +## Measuring path construction success rates {#construction-success-rate} + +Clients maintain two counts for each of their guards: a count of the +number of times a circuit was extended to at least two hops through that +guard, and a count of the number of circuits that successfully complete +through that guard. The ratio of these two numbers is used to determine +a circuit success rate for that Guard. + +[Circuit build timeouts](./learning-timeouts.md) +are counted as construction failures if the +circuit fails to complete before the 95% "right-censored" timeout +interval, not the 80% timeout condition. + +If a circuit closes prematurely after construction but before being +requested to close by the client, this is counted as a failure. + +<a id="path-spec.txt-7.2"></a> + +## Measuring path usage success rates {#usage-success-rate} + +Clients maintain two usage counts for each of their guards: a count +of the number of usage attempts, and a count of the number of +successful usages. + +A usage attempt means any attempt to attach a stream to a circuit. + +Usage success status is temporarily recorded by state flags on circuits. +Guard usage success counts are not incremented until circuit close. A +circuit is marked as successfully used if we receive a properly +recognized RELAY cell on that circuit that was expected for the current +circuit purpose. + +If subsequent stream attachments fail or time out, the successfully used +state of the circuit is cleared, causing it once again to be regarded +as a usage attempt only. + +Upon close by the client, all circuits that are still marked as usage +attempts are probed using a RELAY_BEGIN cell constructed with a +destination of the form 0.a.b.c:25, where a.b.c is a 24 bit random +nonce. If we get a RELAY_COMMAND_END in response matching our nonce, +the circuit is counted as successfully used. + +If any unrecognized RELAY cells arrive after the probe has been sent, +the circuit is counted as a usage failure. + +If the stream failure reason codes DESTROY, TORPROTOCOL, or INTERNAL +are received in response to any stream attempt, such circuits are not +probed and are declared usage failures. + +Prematurely closed circuits are not probed, and are counted as usage +failures. + +<a id="path-spec.txt-7.3"></a> + +## Scaling success counts {#scaling} + +To provide a moving average of recent Guard activity while +still preserving the ability to verify correctness, we periodically +"scale" the success counts by multiplying them by a scale factor +between 0 and 1.0. + +Scaling is performed when either usage or construction attempt counts +exceed a parametrized value. + +To avoid error due to scaling during circuit construction and use, +currently open circuits are subtracted from the usage counts before +scaling, and added back after scaling. + +<a id="path-spec.txt-7.4"></a> + +## Parametrization {#parameters} + +The following consensus parameters tune various aspects of the +defense. + +```text + pb_mincircs + Default: 150 + Min: 5 + Effect: This is the minimum number of circuits that must complete + at least 2 hops before we begin evaluating construction rates. + + pb_noticepct + Default: 70 + Min: 0 + Max: 100 + Effect: If the circuit success rate falls below this percentage, + we emit a notice log message. + + pb_warnpct + Default: 50 + Min: 0 + Max: 100 + Effect: If the circuit success rate falls below this percentage, + we emit a warn log message. + + pb_extremepct + Default: 30 + Min: 0 + Max: 100 + Effect: If the circuit success rate falls below this percentage, + we emit a more alarmist warning log message. If + pb_dropguard is set to 1, we also disable the use of the + guard. + + pb_dropguards + Default: 0 + Min: 0 + Max: 1 + Effect: If the circuit success rate falls below pb_extremepct, + when pb_dropguard is set to 1, we disable use of that + guard. + + pb_scalecircs + Default: 300 + Min: 10 + Effect: After this many circuits have completed at least two hops, + Tor performs the scaling described in + ["Scaling success counts"](#scaling). + + pb_multfactor and pb_scalefactor + Default: 1/2 + Min: 0.0 + Max: 1.0 + Effect: The double-precision result obtained from + pb_multfactor/pb_scalefactor is multiplied by our current + counts to scale them. + + pb_minuse + Default: 20 + Min: 3 + Effect: This is the minimum number of circuits that we must attempt to + use before we begin evaluating construction rates. + + pb_noticeusepct + Default: 80 + Min: 3 + Effect: If the circuit usage success rate falls below this percentage, + we emit a notice log message. + + pb_extremeusepct + Default: 60 + Min: 3 + Effect: If the circuit usage success rate falls below this percentage, + we emit a warning log message. We also disable the use of the + guard if pb_dropguards is set. + + pb_scaleuse + Default: 100 + Min: 10 + Effect: After we have attempted to use this many circuits, + Tor performs the scaling described in + ["Scaling success counts"](#scaling). +``` + +<a id="path-spec.txt-7.5"></a> + +## Known barriers to enforcement {#barriers} + +Due to intermittent CPU overload at relays, the normal rate of +successful circuit completion is highly variable. The Guard-dropping +version of the defense is unlikely to be deployed until the ntor +circuit handshake is enabled, or the nature of CPU overload induced +failure is better understood. diff --git a/spec/path-spec/general-operation.md b/spec/path-spec/general-operation.md new file mode 100644 index 0000000..721ab3d --- /dev/null +++ b/spec/path-spec/general-operation.md @@ -0,0 +1,97 @@ +<a id="path-spec.txt-1"></a> + +# General operation + +Tor begins building circuits as soon as it has +[enough directory information](./when-we-build.md) to do so. +Some circuits are +built preemptively because we expect to need them later (for user +traffic), and some are built because of immediate need (for user traffic +that no current circuit can handle, for testing the network or our +reachability, and so on). + +```text + [Newer versions of Tor (0.2.6.2-alpha and later): + If the consensus contains Exits (the typical case), Tor will build both + exit and internal circuits. When bootstrap completes, Tor will be ready + to handle an application requesting an exit circuit to services like the + World Wide Web. +``` + +If the consensus does not contain Exits, Tor will only build internal +circuits. In this case, earlier statuses will have included "internal" +as indicated above. When bootstrap completes, Tor will be ready to handle +an application requesting an internal circuit to hidden services at +".onion" addresses. + +If a future consensus contains Exits, exit circuits may become available.\] + +When a client application creates a new stream (by opening a SOCKS +connection or launching a resolve request), we attach it to an appropriate +open circuit if one exists, or wait if an appropriate circuit is +in-progress. We launch a new circuit only +if no current circuit can handle the request. We rotate circuits over +time to avoid some profiling attacks. + +To build a circuit, we choose all the nodes we want to use, and then +construct the circuit. Sometimes, when we want a circuit that ends at a +given hop, and we have an appropriate unused circuit, we "cannibalize" the +existing circuit and extend it to the new terminus. + +These processes are described in more detail below. + +This document describes Tor's automatic path selection logic only; path +selection can be overridden by a controller (with the EXTENDCIRCUIT and +ATTACHSTREAM commands). Paths constructed through these means may +violate some constraints given below. + +<a id="path-spec.txt-1.1"></a> + +## Terminology + +A "path" is an ordered sequence of nodes, not yet built as a circuit. + +A "clean" circuit is one that has not yet been used for any traffic. + +A "fast" or "stable" or "valid" node is one that has the 'Fast' or +'Stable' or 'Valid' flag +set respectively, based on our current directory information. A "fast" +or "stable" circuit is one consisting only of "fast" or "stable" nodes. + +In an "exit" circuit, the final node is chosen based on waiting stream +requests if any, and in any case it avoids nodes with exit policy of +"reject *:*". An "internal" circuit, on the other hand, is one where +the final node is chosen just like a middle node (ignoring its exit +policy). + +A "request" is a client-side stream or DNS resolve that needs to be +served by a circuit. + +A "pending" circuit is one that we have started to build, but which has +not yet completed. + +A circuit or path "supports" a request if it is okay to use the +circuit/path to fulfill the request, according to the rules given below. +A circuit or path "might support" a request if some aspect of the request +is unknown (usually its target IP), but we believe the path probably +supports the request according to the rules given below. + +<a id="path-spec.txt-1.2"></a> + +## A relay's bandwidth {#bandwidth} + +Old versions of Tor did not report bandwidths in network status +documents, so clients had to learn them from the routers' advertised +relay descriptors. + +For versions of Tor prior to 0.2.1.17-rc, everywhere below where we +refer to a relay's "bandwidth", we mean its clipped advertised +bandwidth, computed by taking the smaller of the 'rate' and +'observed' arguments to the "bandwidth" element in the relay's +descriptor. If a router's advertised bandwidth is greater than +MAX_BELIEVABLE_BANDWIDTH (currently 10 MB/s), we clipped to that +value. + +For more recent versions of Tor, we take the bandwidth value declared +in the consensus, and fall back to the clipped advertised bandwidth +only if the consensus does not have bandwidths listed. diff --git a/spec/path-spec/guard-nodes.md b/spec/path-spec/guard-nodes.md new file mode 100644 index 0000000..af5750c --- /dev/null +++ b/spec/path-spec/guard-nodes.md @@ -0,0 +1,44 @@ +<a id="path-spec.txt-5"></a> + +# Guard nodes + +We use Guard nodes (also called "helper nodes" in the research +literature) to prevent certain profiling attacks. For an overview of +our Guard selection algorithm -- which has grown rather complex -- see +guard-spec.txt. + +<a id="path-spec.txt-5.1"></a> + +## How consensus bandwidth weights factor into entry guard selection {#bw-and-guards} + +When weighting a list of routers for choosing an entry guard, the following +consensus parameters (from the "bandwidth-weights" line) apply: + +```text + Wgg - Weight for Guard-flagged nodes in the guard position + Wgm - Weight for non-flagged nodes in the guard Position + Wgd - Weight for Guard+Exit-flagged nodes in the guard Position + Wgb - Weight for BEGIN_DIR-supporting Guard-flagged nodes + Wmb - Weight for BEGIN_DIR-supporting non-flagged nodes + Web - Weight for BEGIN_DIR-supporting Exit-flagged nodes + Wdb - Weight for BEGIN_DIR-supporting Guard+Exit-flagged nodes +``` + +Please see "bandwidth-weights" in §3.4.1 of dir-spec.txt for more in depth +descriptions of these parameters. + +If a router has been marked as both an entry guard and an exit, then we +prefer to use it more, with our preference for doing so (roughly) linearly +increasing w.r.t. the router's non-guard bandwidth and bandwidth weight +(calculated without taking the guard flag into account). From proposal +236: + +| +| Let Wpf denote the weight from the 'bandwidth-weights' line a +| client would apply to N for position p if it had the guard +| flag, Wpn the weight if it did not have the guard flag, and B the +| measured bandwidth of N in the consensus. Then instead of choosing +| N for position p proportionally to Wpf*B or Wpn*B, clients should +| choose N proportionally to F*Wpf*B + (1-F)*Wpn*B. + +where F is the weight as calculated using the above parameters. diff --git a/spec/path-spec/handling-failure.md b/spec/path-spec/handling-failure.md new file mode 100644 index 0000000..8db9245 --- /dev/null +++ b/spec/path-spec/handling-failure.md @@ -0,0 +1,19 @@ +<a id="path-spec.txt-2.5"></a> + +# Handling failure + +If an attempt to extend a circuit fails (either because the first create +failed or a subsequent extend failed) then the circuit is torn down and is +no longer pending. (XXXX really?) Requests that might have been +supported by the pending circuit thus become unsupported, and a new +circuit needs to be constructed. + +If a stream "begin" attempt fails with an EXITPOLICY error, we +decide that the exit node's exit policy is not correctly advertised, +so we treat the exit node as if it were a non-exit until we retrieve +a fresh descriptor for it. + +Excessive amounts of either type of failure can indicate an +attack on anonymity. +See [discussion of path bias detection](./detecting-route-manipulation.md) +for how excessive failure is handled. diff --git a/spec/path-spec/hidden-service-related-circuits.md b/spec/path-spec/hidden-service-related-circuits.md new file mode 100644 index 0000000..fc6e30d --- /dev/null +++ b/spec/path-spec/hidden-service-related-circuits.md @@ -0,0 +1,5 @@ +<a id="path-spec.txt-4"></a> + +# Hidden-service related circuits + +XXX Tracking expected hidden service use (client-side and hidserv-side) diff --git a/spec/path-spec/index.md b/spec/path-spec/index.md new file mode 100644 index 0000000..55e5df8 --- /dev/null +++ b/spec/path-spec/index.md @@ -0,0 +1,21 @@ +# Tor Path Specification + +```text + Roger Dingledine + Nick Mathewson +``` + +Note: This is an attempt to specify Tor as currently implemented. Future +versions of Tor will implement improved algorithms. + +This document tries to cover how Tor chooses to build circuits and assign +streams to circuits. Other implementations MAY take other approaches, but +implementors should be aware of the anonymity and load-balancing implications +of their choices. + +THIS SPEC ISN'T DONE YET. + +The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL +NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and +"OPTIONAL" in this document are to be interpreted as described in +RFC 2119. diff --git a/spec/path-spec/learning-timeouts.md b/spec/path-spec/learning-timeouts.md new file mode 100644 index 0000000..e98bac3 --- /dev/null +++ b/spec/path-spec/learning-timeouts.md @@ -0,0 +1,305 @@ +<a id="path-spec.txt-2.4"></a> + +# Learning when to give up ("timeout") on circuit construction {#timing-out} + +Since version 0.2.2.8-alpha, Tor clients attempt to learn when to give +up on circuits based on network conditions. + +<a id="path-spec.txt-2.4.1"></a> + +## Distribution choice + +Based on studies of build times, we found that the distribution of +circuit build times appears to be a Frechet distribution (and a multi-modal +Frechet distribution, if more than one guard or bridge is used). However, +estimators and quantile functions of the Frechet distribution are difficult +to work with and slow to converge. So instead, since we are only interested +in the accuracy of the tail, clients approximate the tail of the multi-modal +distribution with a single Pareto curve. + +<a id="path-spec.txt-2.4.2"></a> + +## How much data to record {#observations} + +From our observations, the minimum number of circuit build times for a +reasonable fit appears to be on the order of 100. However, to keep a +good fit over the long term, clients store 1000 most recent circuit build +times in a circular array. + +These build times only include the times required to build three-hop +circuits, and the times required to build the first three hops of circuits +with more than three hops. Circuits of fewer than three hops are not +recorded, and hops past the third are not recorded. + +The Tor client should build test circuits at a rate of one every 'cbttestfreq' +(10 seconds) until 'cbtmincircs' (100 circuits) are built, with a maximum of +'cbtmaxopencircs' (default: 10) circuits open at once. This allows a fresh +Tor to have a CircuitBuildTimeout estimated within 30 minutes after install +or network change +(see [Detecting Changing Network Conditions](#changes-in-network) below.) + +Timeouts are stored on disk in a histogram of 10ms bin width, the same +width used to calculate the Xm value above. The timeouts recorded in the +histogram must be shuffled after being read from disk, to preserve a +proper expiration of old values after restart. + +Thus, some build time resolution is lost during restart. Implementations may +choose a different persistence mechanism than this histogram, but be aware +that build time binning is still needed for parameter estimation. + +<a id="path-spec.txt-2.4.3"></a> + +## Parameter estimation + +Once 'cbtmincircs' build times are recorded, Tor clients update the +distribution parameters and recompute the timeout every circuit completion +(though +[see below](#changes-in-network) +for when to pause and reset timeout due to +too many circuits timing out). + +Tor clients calculate the parameters for a Pareto distribution fitting the +data using the maximum likelihood estimator. For derivation, see: +<https://en.wikipedia.org/wiki/Pareto_distribution#Estimation_of_parameters> + +Because build times are not a true Pareto distribution, we alter how Xm is +computed. In a max likelihood estimator, the mode of the distribution is +used directly as Xm. + +Instead of using the mode of discrete build times directly, Tor clients +compute the Xm parameter using the weighted average of the midpoints +of the 'cbtnummodes' (10) most frequently occurring 10ms histogram bins. +Ties are broken in favor of earlier bins (that is, in favor of bins +corresponding to shorter build times). + +(The use of 10 modes was found to minimize error from the selected +cbtquantile, with 10ms bins for quantiles 60-80, compared to many other +heuristics). + +To avoid ln(1.0+epsilon) precision issues, use log laws to rewrite the +estimator for 'alpha' as the sum of logs followed by subtraction, rather +than multiplication and division: + +`alpha = n/(Sum_n{ln(MAX(Xm, x_i))} - n\*ln(Xm))` + +In this, n is the total number of build times that have completed, x_i is +the ith recorded build time, and Xm is the modes of x_i as above. + +All times below Xm are counted as having the Xm value via the MAX(), +because in Pareto estimators, Xm is supposed to be the lowest value. +However, since clients use mode averaging to estimate Xm, there can be +values below our Xm. Effectively, the Pareto estimator then treats that +everything smaller than Xm happened at Xm. One can also see that if +clients did not do this, alpha could underflow to become negative, which +results in an exponential curve, not a Pareto probability distribution. + +The timeout itself is calculated by using the Pareto Quantile function (the +inverted CDF) to give us the value on the CDF such that 80% of the mass +of the distribution is below the timeout value (parameter 'cbtquantile'). + +The Pareto Quantile Function (inverse CDF) is: + +`F(q) = Xm/((1.0-q)^(1.0/alpha))` + +Thus, clients obtain the circuit build timeout for 3-hop circuits by +computing: + +`timeout_ms = F(0.8) # 'cbtquantile' == 0.8` + +With this, we expect that the Tor client will accept the fastest 80% of the +total number of paths on the network. + +Clients obtain the circuit close time to completely abandon circuits as: + +`close_ms = F(0.99) # 'cbtclosequantile' == 0.99` + +To avoid waiting an unreasonably long period of time for circuits that +simply have relays that are down, Tor clients cap timeout_ms at the max +build time actually observed so far, and cap close_ms at twice this max, +but at least 60 seconds: + +```text + timeout_ms = MIN(timeout_ms, max_observed_timeout) + close_ms = MAX(MIN(close_ms, 2*max_observed_timeout), 'cbtinitialtimeout') +``` + +<a id="path-spec.txt-2.4.3"></a> + +## Calculating timeouts thresholds for circuits of different lengths {#different-lengths} + +The timeout_ms and close_ms estimates above are good only for 3-hop +circuits, since only 3-hop circuits are recorded in the list of build +times. + +To calculate the appropriate timeouts and close timeouts for circuits of +other lengths, the client multiples the timeout_ms and close_ms values +by a scaling factor determined by the number of communication hops +needed to build their circuits: + +```text +timeout_ms\[hops=n\] = timeout_ms * Actions(N) / Actions(3) + +close_ms\[hops=n\] = close_ms * Actions(N) / Actions(3) +``` + +where `Actions(N) = N * (N + 1) / 2.` + +To calculate timeouts for operations other than circuit building, +the client should add X to Actions(N) for every round-trip communication +required with the Xth hop. + +<a id="path-spec.txt-2.4.4"></a> + +## How to record timeouts {#recording-timeouts} + +Pareto estimators begin to lose their accuracy if the tail is omitted. +Hence, Tor clients actually calculate two timeouts: a usage timeout, and a +close timeout. + +Circuits that pass the usage timeout are marked as measurement circuits, +and are allowed to continue to build until the close timeout corresponding +to the point 'cbtclosequantile' (default 99) on the Pareto curve, or 60 +seconds, whichever is greater. + +The actual completion times for these measurement circuits should be +recorded. + +Implementations should completely abandon a circuit and ignore the circuit +if the total build time exceeds the close threshold. Such closed circuits +should be ignored, as this typically means one of the relays in the path is +offline. + +<a id="path-spec.txt-2.4.5"></a> + +## Detecting Changing Network Conditions {#changes-in-network} + +Tor clients attempt to detect both network connectivity loss and drastic +changes in the timeout characteristics. + +To detect changing network conditions, clients keep a history of +the timeout or non-timeout status of the past 'cbtrecentcount' circuits +(20 circuits) that successfully completed at least one hop. If more than +90% of these circuits timeout, the client discards all buildtimes history, +resets the timeout to 'cbtinitialtimeout' (60 seconds), and then begins +recomputing the timeout. + +If the timeout was already at least `cbtinitialtimeout`, +the client doubles the timeout. + +The records here (of how many circuits succeeded or failed among the most +recent 'cbrrecentcount') are not stored as persistent state. On reload, +we start with a new, empty state. + +<a id="path-spec.txt-2.4.6"></a> + +## Consensus parameters governing behavior {#parameters} + +Clients that implement circuit build timeout learning should obey the +following consensus parameters that govern behavior, in order to allow +us to handle bugs or other emergent behaviors due to client circuit +construction. If these parameters are not present in the consensus, +the listed default values should be used instead. + +```text + cbtdisabled + Default: 0 + Min: 0 + Max: 1 + Effect: If 1, all CircuitBuildTime learning code should be + disabled and history should be discarded. For use in + emergency situations only. + + cbtnummodes + Default: 10 + Min: 1 + Max: 20 + Effect: This value governs how many modes to use in the weighted + average calculation of Pareto parameter Xm. Selecting Xm as the + average of multiple modes improves accuracy of the Pareto tail + for quantile cutoffs from 60-80% (see cbtquantile). + + cbtrecentcount + Default: 20 + Min: 3 + Max: 1000 + Effect: This is the number of circuit build outcomes (success vs + timeout) to keep track of for the following option. + + cbtmaxtimeouts + Default: 18 + Min: 3 + Max: 10000 + Effect: When this many timeouts happen in the last 'cbtrecentcount' + circuit attempts, the client should discard all of its + history and begin learning a fresh timeout value. + + Note that if this parameter's value is greater than the value + of 'cbtrecentcount', then the history will never be + discarded because of this feature. + + cbtmincircs + Default: 100 + Min: 1 + Max: 10000 + Effect: This is the minimum number of circuits to build before + computing a timeout. + + Note that if this parameter's value is higher than 1000 (the + number of time observations that a client keeps in its + circular buffer), circuit build timeout calculation is + effectively disabled, and the default timeouts are used + indefinitely. + + cbtquantile + Default: 80 + Min: 10 + Max: 99 + Effect: This is the position on the quantile curve to use to set the + timeout value. It is a percent (10-99). + + cbtclosequantile + Default: 99 + Min: Value of cbtquantile parameter + Max: 99 + Effect: This is the position on the quantile curve to use to set the + timeout value to use to actually close circuits. It is a + percent (0-99). + + cbttestfreq + Default: 10 + Min: 1 + Max: 2147483647 (INT32_MAX) + Effect: Describes how often in seconds to build a test circuit to + gather timeout values. Only applies if less than 'cbtmincircs' + have been recorded. + + cbtmintimeout + Default: 10 + Min: 10 + Max: 2147483647 (INT32_MAX) + Effect: This is the minimum allowed timeout value in milliseconds. + + cbtinitialtimeout + Default: 60000 + Min: Value of cbtmintimeout + Max: 2147483647 (INT32_MAX) + Effect: This is the timeout value to use before we have enough data + to compute a timeout, in milliseconds. If we do not have + enough data to compute a timeout estimate (see cbtmincircs), + then we use this interval both for the close timeout and the + abandon timeout. + + cbtlearntimeout + Default: 180 + Min: 10 + Max: 60000 + Effect: This is how long idle circuits will be kept open while cbt is + learning a new timeout value. + + cbtmaxopencircs + Default: 10 + Min: 0 + Max: 14 + Effect: This is the maximum number of circuits that can be open at + at the same time during the circuit build time learning phase. +``` diff --git a/spec/path-spec/path-selection-constraints.md b/spec/path-spec/path-selection-constraints.md new file mode 100644 index 0000000..8cac4fc --- /dev/null +++ b/spec/path-spec/path-selection-constraints.md @@ -0,0 +1,146 @@ +<a id="path-spec.txt-2.2"></a> + +# Path selection and constraints + +We choose the path for each new circuit before we build it, +based on our current directory information. +(Clients and relays use the latest directory information they have; +directory authorities use their own opinions.) + +We choose the +exit node first, followed by the other nodes in the circuit, front to +back. (In other words, for a 3-hop circuit, we first pick hop 3, +then hop 1, then hop 2.) + +## Universal constraints + +All paths we generate obey the following +constraints: + +- We do not choose the same router twice for the same path. +- We do not choose any router in the same family as another in the same + path. (Two routers are in the same family if each one lists the other + in the "family" entries of its descriptor.) +- We do not choose more than one router in a given network range, + which defaults to /16 for IPv4 and /32 for IPv6. + (C Tor overrides this with `EnforceDistinctSubnets`; + Arti overrides this with `ipv[46]_subnet_family_prefix`.) +- The first node must be a Guard (see + discussion [below](./guard-nodes.md) and in the + [guard specification](../guard-spec)). +- XXXX Choosing the length + +## Special-purpose constraints + +Additionally, we may be building circuits with one or more requests in +mind. Each kind of request puts certain constraints on paths. + +Most circuits need to be "Fast". +For these, we only choose nodes with the `Fast` flag. +For non-"fast" circuits, nodes without the `Fast` flag are eligible. + +- TODO document which circuits (do not) need to be Fast. + +Similarly, some circuits need to be "Stable". +For these, we only choose nodes with the Stable flag. + +- All service-side introduction circuits and all rendezvous paths + should be Stable. +- All connection requests for connections that we think will need to + stay open a long time require Stable circuits. Currently, Tor decides + this by examining the request's target port, and comparing it to a + list of "long-lived" ports. (Default: 21, 22, 706, 1863, 5050, + 5190, 5222, 5223, 6667, 6697, 8300.) + +<a id="path-spec.txt-2.2.1"></a> + +## Weighting node selection + +For all circuits, we weight node selection according to router bandwidth. + +We also weight the bandwidth of Exit and Guard flagged nodes depending on +the fraction of total bandwidth that they make up and depending upon the +position they are being selected for. + +These weights are published in the consensus, and are computed as described +in +["Computing Bandwidth Weights"](../dir-spec/computing-consensus.md#computing-bandwidth-weights) +in the directory specification. +They are: + +```text + Wgg - Weight for Guard-flagged nodes in the guard position + Wgm - Weight for non-flagged nodes in the guard Position + Wgd - Weight for Guard+Exit-flagged nodes in the guard Position + + Wmg - Weight for Guard-flagged nodes in the middle Position + Wmm - Weight for non-flagged nodes in the middle Position + Wme - Weight for Exit-flagged nodes in the middle Position + Wmd - Weight for Guard+Exit flagged nodes in the middle Position + + Weg - Weight for Guard flagged nodes in the exit Position + Wem - Weight for non-flagged nodes in the exit Position + Wee - Weight for Exit-flagged nodes in the exit Position + Wed - Weight for Guard+Exit-flagged nodes in the exit Position + + Wgb - Weight for BEGIN_DIR-supporting Guard-flagged nodes + Wmb - Weight for BEGIN_DIR-supporting non-flagged nodes + Web - Weight for BEGIN_DIR-supporting Exit-flagged nodes + Wdb - Weight for BEGIN_DIR-supporting Guard+Exit-flagged nodes + + Wbg - Weight for Guard+Exit-flagged nodes for BEGIN_DIR requests + Wbm - Weight for Guard+Exit-flagged nodes for BEGIN_DIR requests + Wbe - Weight for Guard+Exit-flagged nodes for BEGIN_DIR requests + Wbd - Weight for Guard+Exit-flagged nodes for BEGIN_DIR requests +``` + +If any of those weights is malformed or not present in a consensus, +clients proceed with the regular path selection algorithm setting +the weights to the default value of 10000. + + +## Choosing an exit + +If we know what IP address we want to connect to, we can +trivially tell whether a given router will support it by simulating +its declared exit policy. + +(DNS resolve requests are only sent to relays whose +exit policy is not equivalent to "reject *:*".) + +Because we often connect to addresses of the form hostname:port, we do not +always know the target IP address when we select an exit node. In these +cases, we need to pick an exit node that "might support" connections to a +given address port with an unknown address. An exit node "might support" +such a connection if any clause that accepts any connections to that port +precedes all clauses (if any) that reject all connections to that port. + +Unless requested to do so by the user, we never choose an exit node +flagged as "BadExit" by more than half of the authorities who advertise +themselves as listing bad exits. + +<a id="path-spec.txt-2.2.2"></a> + +## User configuration + +Users can alter the default behavior for path selection with configuration +options. + +```text + - If "ExitNodes" is provided, then every request requires an exit node on + the ExitNodes list. (If a request is supported by no nodes on that list, + and StrictExitNodes is false, then Tor treats that request as if + ExitNodes were not provided.) + + - "EntryNodes" and "StrictEntryNodes" behave analogously. + + - If a user tries to connect to or resolve a hostname of the form + <target>.<servername>.exit, the request is rewritten to a request for + <target>, and the request is only supported by the exit whose nickname + or fingerprint is <servername>. + + - When set, "HSLayer2Nodes" and "HSLayer3Nodes" relax Tor's path + restrictions to allow nodes in the same /16 and node family to reappear + in the path. They also allow the guard node to be chosen as the RP, IP, + and HSDIR, and as the hop before those positions. +``` diff --git a/spec/path-spec/server-descriptor-purposes.md b/spec/path-spec/server-descriptor-purposes.md new file mode 100644 index 0000000..3a7dca0 --- /dev/null +++ b/spec/path-spec/server-descriptor-purposes.md @@ -0,0 +1,19 @@ +<a id="path-spec.txt-6"></a> + +# Server descriptor purposes + +There are currently three "purposes" supported for server descriptors: +general, controller, and bridge. Most descriptors are of type general +-- these are the ones listed in the consensus, and the ones fetched +and used in normal cases. + +Controller-purpose descriptors are those delivered by the controller +and labelled as such: they will be kept around (and expire like +normal descriptors), and they can be used by the controller in its +CIRCUITEXTEND commands. Otherwise they are ignored by Tor when it +chooses paths. + +Bridge-purpose descriptors are for routers that are used as bridges. See +doc/design-paper/blocking.pdf for more design explanation, or proposal +125 for specific details. Currently bridge descriptors are used in place +of normal entry guards, for Tor clients that have UseBridges enabled. diff --git a/spec/path-spec/when-we-build.md b/spec/path-spec/when-we-build.md new file mode 100644 index 0000000..7130978 --- /dev/null +++ b/spec/path-spec/when-we-build.md @@ -0,0 +1,177 @@ +<a id="path-spec.txt-2.1"></a> + +# When we build + +<a id="path-spec.txt-2.1.0"></a> + +## We don't build circuits until we have enough directory info + +There's a class of possible attacks where our directory servers +only give us information about the relays that they would like us +to use. To prevent this attack, we don't build multi-hop +circuits +(including +[preemptive circuits](#preemptive), +[on-demand circuits(#on-demand), +[onion-service circuits](#onion-service)] +or [self-testing testing circuits](#self-test)) +for real traffic +until we have enough directory information to be +reasonably confident this attack isn't being done to us. + +Here, "enough" directory information is defined as: + +```text + * Having a consensus that's been valid at some point in the + last REASONABLY_LIVE_TIME interval (24 hours). + + * Having enough descriptors that we could build at least some + fraction F of all bandwidth-weighted paths, without taking + ExitNodes/EntryNodes/etc into account. + + (F is set by the PathsNeededToBuildCircuits option, + defaulting to the 'min_paths_for_circs_pct' consensus + parameter, with a final default value of 60%.) + + * Having enough descriptors that we could build at least some + fraction F of all bandwidth-weighted paths, _while_ taking + ExitNodes/EntryNodes/etc into account. + + (F is as above.) + + * Having a descriptor for every one of the first + NUM_USABLE_PRIMARY_GUARDS guards among our primary guards. (see + guard-spec.txt) +``` + +We define the "fraction of bandwidth-weighted paths" as the product of +these three fractions. + +```text + * The fraction of descriptors that we have for nodes with the Guard + flag, weighted by their bandwidth for the guard position. + * The fraction of descriptors that we have for all nodes, + weighted by their bandwidth for the middle position. + * The fraction of descriptors that we have for nodes with the Exit + flag, weighted by their bandwidth for the exit position. +``` + +If the consensus has zero weighted bandwidth for a given kind of +relay (Guard, Middle, or Exit), Tor instead uses the fraction of relays +for which it has the descriptor (not weighted by bandwidth at all). + +If the consensus lists zero exit-flagged relays, Tor instead uses the +fraction of middle relays. + +<a id="path-spec.txt-2.1.1"></a> + +## Clients build circuits preemptively {#preemptive} + +When running as a client, Tor tries to maintain at least a certain +number of clean circuits, so that new streams can be handled +quickly. To increase the likelihood of success, Tor tries to +predict what circuits will be useful by choosing from among nodes +that support the ports we have used in the recent past (by default +one hour). Specifically, on startup Tor tries to maintain one clean +fast exit circuit that allows connections to port 80, and at least +two fast clean stable internal circuits in case we get a resolve +request or hidden service request (at least three if we _run_ a +hidden service). + +After that, Tor will adapt the circuits that it preemptively builds +based on the requests it sees from the user: it tries to have two fast +clean exit circuits available for every port seen within the past hour +(each circuit can be adequate for many predicted ports -- it doesn't +need two separate circuits for each port), and it tries to have the +above internal circuits available if we've seen resolves or hidden +service activity within the past hour. If there are 12 or more clean +circuits open, it doesn't open more even if it has more predictions. + +Only stable circuits can "cover" a port that is listed in the +LongLivedPorts config option. Similarly, hidden service requests +to ports listed in LongLivedPorts make us create stable internal +circuits. + +Note that if there are no requests from the user for an hour, Tor +will predict no use and build no preemptive circuits. + +The Tor client SHOULD NOT store its list of predicted requests to a +persistent medium. + +<a id="path-spec.txt-2.1.2"></a> + +## Clients build circuits on demand {#on-demand} + +Additionally, when a client request exists that no circuit (built or +pending) might support, we create a new circuit to support the request. +For exit connections, we pick an exit node that will handle the +most pending requests (choosing arbitrarily among ties), launch a +circuit to end there, and repeat until every unattached request +might be supported by a pending or built circuit. For internal +circuits, we pick an arbitrary acceptable path, repeating as needed. + +Clients consider a circuit to become "dirty" as soon as a stream is +attached to it, or some other request is performed over the circuit. +If a circuit has been "dirty" for at least MaxCircuitDirtiness seconds, +new circuits may not be attached to it. + +In some cases we can reuse an already established circuit if it's +clean; see ["cannibalizing circuits"](./cannibalizing-circuits.md) + +for details. + +<a id="path-spec.txt-2.1.3"></a> + +## Relays build circuits for testing reachability and bandwidth {#self-test} + +Tor relays test reachability of their ORPort once they have +successfully built a circuit (on startup and whenever their IP address +changes). They build an ordinary fast internal circuit with themselves +as the last hop. As soon as any testing circuit succeeds, the Tor +relay decides it's reachable and is willing to publish a descriptor. + +We launch multiple testing circuits (one at a time), until we +have NUM_PARALLEL_TESTING_CIRC (4) such circuits open. Then we +do a "bandwidth test" by sending a certain number of relay drop +cells down each circuit: BandwidthRate * 10 / CELL_NETWORK_SIZE +total cells divided across the four circuits, but never more than +CIRCWINDOW_START (1000) cells total. This exercises both outgoing and +incoming bandwidth, and helps to jumpstart the observed bandwidth +(see dir-spec.txt). + +Tor relays also test reachability of their DirPort once they have +established a circuit, but they use an ordinary exit circuit for +this purpose. + +<a id="path-spec.txt-2.1.4"></a> + +## Hidden-service circuits {#onion-service} + +See section 4 below. + +<a id="path-spec.txt-2.1.5"></a> + +## Rate limiting of failed circuits + +If we fail to build a circuit N times in a X second period +(see ["Handling failure"](./handling-failure.md) +for how this works), we stop building circuits until the X seconds +have elapsed. +XXXX + +<a id="path-spec.txt-2.1.6"></a> + +## When to tear down circuits + +Clients should tear down circuits (in general) only when those circuits +have no streams on them. Additionally, clients should tear-down +stream-less circuits only under one of the following conditions: + +```text + - The circuit has never had a stream attached, and it was created too + long in the past (based on CircuitsAvailableTimeout or + cbtlearntimeout, depending on timeout estimate status). + + - The circuit is dirty (has had a stream attached), and it has been + dirty for at least MaxCircuitDirtiness. +``` diff --git a/spec/permalinks.md b/spec/permalinks.md new file mode 100644 index 0000000..052d48f --- /dev/null +++ b/spec/permalinks.md @@ -0,0 +1,57 @@ +# Permalinks + +These URLs at `spec.toprorject.org` are intended to be +long-term permalinks. + +<!-- BEGIN AUTO-GENERATED REDIRECTS --> +<dl> +<dt><code>/address-spec</code></dt> +<dd><a href="https://spec.torproject.org/address-spec"><code>https://spec.torproject.org/address-spec</code> (Special Hostnames in Tor)</a></dt> +<dt><code>/bandwidth-file-spec</code></dt> +<dd><a href="https://spec.torproject.org/bandwidth-file-spec"><code>https://spec.torproject.org/bandwidth-file-spec</code> (Directory Authority Bandwidth File spec)</a></dt> +<dt><code>/bridgedb-spec</code></dt> +<dd><a href="https://spec.torproject.org/bridgedb-spec"><code>https://spec.torproject.org/bridgedb-spec</code> (BridgeDB specification)</a></dt> +<dt><code>/cert-spec</code></dt> +<dd><a href="https://spec.torproject.org/cert-spec"><code>https://spec.torproject.org/cert-spec</code> (Ed25519 certificates in Tor)</a></dt> +<dt><code>/collector-protocol</code></dt> +<dd><a href="https://gitlab.torproject.org/tpo/network-health/metrics/collector/-/blob/master/src/main/resources/docs/PROTOCOL?ref_type=heads"><code>https://gitlab.torproject.org/tpo/network-health/metrics/collector/-/blob/master/src/main/resources/docs/PROTOCOL?ref_type=heads</code> (Protocol of CollecTor's File Structure)</a></dt> +<dt><code>/control-spec</code></dt> +<dd><a href="https://spec.torproject.org/control-spec"><code>https://spec.torproject.org/control-spec</code> (Tor control protocol, version 1)</a></dt> +<dt><code>/dir-spec</code></dt> +<dd><a href="https://spec.torproject.org/dir-spec"><code>https://spec.torproject.org/dir-spec</code> (Tor directory protocol, version 3)</a></dt> +<dt><code>/dir-list-spec</code></dt> +<dd><a href="https://spec.torproject.org/dir-list-spec"><code>https://spec.torproject.org/dir-list-spec</code> (Tor Directory List file format)</a></dt> +<dt><code>/ext-orport-spec</code></dt> +<dd><a href="https://spec.torproject.org/ext-orport-spec"><code>https://spec.torproject.org/ext-orport-spec</code> (Extended ORPort for pluggable transports)</a></dt> +<dt><code>/gettor-spec</code></dt> +<dd><a href="https://gitlab.torproject.org/tpo/core/torspec/-/raw/main/attic/text_formats/gettor-spec.txt?ref_type=heads"><code>https://gitlab.torproject.org/tpo/core/torspec/-/raw/main/attic/text_formats/gettor-spec.txt?ref_type=heads</code> (GetTor specification)</a></dt> +<dt><code>/padding-spec</code></dt> +<dd><a href="https://spec.torproject.org/padding-spec"><code>https://spec.torproject.org/padding-spec</code> (Tor Padding Specification)</a></dt> +<dt><code>/path-spec</code></dt> +<dd><a href="https://spec.torproject.org/path-spec"><code>https://spec.torproject.org/path-spec</code> (Tor Path Specification)</a></dt> +<dt><code>/pt-spec</code></dt> +<dd><a href="https://spec.torproject.org/pt-spec"><code>https://spec.torproject.org/pt-spec</code> (Tor Pluggable Transport Specification, version 1)</a></dt> +<dt><code>/rend-spec</code></dt> +<dd><a href="https://spec.torproject.org/rend-spec"><code>https://spec.torproject.org/rend-spec</code> (Tor Onion Service Rendezvous Specification, latest version)</a></dt> +<dt><code>/rend-spec-v2</code></dt> +<dd><a href="https://gitlab.torproject.org/tpo/core/torspec/-/blob/main/attic/rend-spec-v2.txt?ref_type=heads"><code>https://gitlab.torproject.org/tpo/core/torspec/-/blob/main/attic/rend-spec-v2.txt?ref_type=heads</code> (Tor Onion Service Rendezvous Specification, Version 2 (Obsolete))</a></dt> +<dt><code>/rend-spec-v3</code></dt> +<dd><a href="https://spec.torproject.org/rend-spec"><code>https://spec.torproject.org/rend-spec</code> (Tor Onion Service Rendezvous Specification, Version 3 (Latest))</a></dt> +<dt><code>/socks-extensions</code></dt> +<dd><a href="https://spec.torproject.org/socks-extensions"><code>https://spec.torproject.org/socks-extensions</code> (Tor's extensions to the SOCKS protocol)</a></dt> +<dt><code>/srv-spec</code></dt> +<dd><a href="https://spec.torproject.org/srv-spec"><code>https://spec.torproject.org/srv-spec</code> (Tor Shared Random Subsystem Specification)</a></dt> +<dt><code>/tor-fw-helper-spec</code></dt> +<dd><a href="https://gitlab.torproject.org/tpo/core/torspec/-/blob/main/attic/tor-fw-helper-spec.txt?ref_type=heads"><code>https://gitlab.torproject.org/tpo/core/torspec/-/blob/main/attic/tor-fw-helper-spec.txt?ref_type=heads</code> (Tor's (little) Firewall Helper specification)</a></dt> +<dt><code>/tor-spec</code></dt> +<dd><a href="https://spec.torproject.org/tor-spec"><code>https://spec.torproject.org/tor-spec</code> (Tor Protocol Specification)</a></dt> +<dt><code>/torbrowser-design</code></dt> +<dd><a href="https://2019.www.torproject.org/projects/torbrowser/design/"><code>https://2019.www.torproject.org/projects/torbrowser/design/</code> (The Design and Implementation of the Tor Browser)</a></dt> +<dt><code>/version-spec</code></dt> +<dd><a href="https://spec.torproject.org/version-spec"><code>https://spec.torproject.org/version-spec</code> (How Tor Version Numbers Work)</a></dt> +<dt><code>/tor-design</code></dt> +<dd><a href="https://svn.torproject.org/svn/projects/design-paper/tor-design.pdf"><code>https://svn.torproject.org/svn/projects/design-paper/tor-design.pdf</code> (Tor: The Second-Generation Onion Router)</a></dt> +<dt><code>/walking-onions</code></dt> +<dd><a href="https://spec.torproject.org/proposals/323-walking-onions-full.html"><code>https://spec.torproject.org/proposals/323-walking-onions-full.html</code> (Walking Onions specifications)</a></dt> +</dl> +<!-- END AUTO-GENERATED REDIRECTS --> diff --git a/spec/pt-spec/acknowledgments.md b/spec/pt-spec/acknowledgments.md new file mode 100644 index 0000000..92d373a --- /dev/null +++ b/spec/pt-spec/acknowledgments.md @@ -0,0 +1,6 @@ +<a id="pt-spec.txt-6"></a> + +# Acknowledgments + +This specification draws heavily from prior versions done by Jacob +Appelbaum, Nick Mathewson, and George Kadianakis. diff --git a/spec/pt-spec/anonymity-considerations.md b/spec/pt-spec/anonymity-considerations.md new file mode 100644 index 0000000..ce3a54e --- /dev/null +++ b/spec/pt-spec/anonymity-considerations.md @@ -0,0 +1,23 @@ +<a id="pt-spec.txt-4"></a> + +# Anonymity Considerations + +When designing and implementing a Pluggable Transport, care +should be taken to preserve the privacy of clients and to avoid +leaking personally identifying information. + +Examples of client related considerations are: + +- Not logging client IP addresses to disk. + +- Not leaking DNS addresses except when necessary. + +```text + - Ensuring that "TOR_PT_PROXY"'s "fail closed" behavior is + implemented correctly. +``` + +Additionally, certain obfuscation mechanisms rely on information +such as the server IP address/port being confidential, so clients +also need to take care to preserve server side information +confidential when applicable. diff --git a/spec/pt-spec/architecture-overview.md b/spec/pt-spec/architecture-overview.md new file mode 100644 index 0000000..ef4a6a4 --- /dev/null +++ b/spec/pt-spec/architecture-overview.md @@ -0,0 +1,37 @@ +<a id="pt-spec.txt-2"></a> + +# Architecture Overview + +```text + +------------+ +---------------------------+ + | Client App +-- Local Loopback --+ PT Client (SOCKS Proxy) +--+ + +------------+ +---------------------------+ | + | + Public Internet (Obfuscated/Transformed traffic) ==> | + | + +------------+ +---------------------------+ | + | Server App +-- Local Loopback --+ PT Server (Reverse Proxy) +--+ + +------------+ +---------------------------+ +``` + +On the client's host, the PT Client software exposes a SOCKS proxy +\[RFC1928\] to the client application, and obfuscates or otherwise +transforms traffic before forwarding it to the server's host. + +On the server's host, the PT Server software exposes a reverse proxy +that accepts connections from PT Clients, and handles reversing the +obfuscation/transformation applied to traffic, before forwarding it +to the actual server software. An optional lightweight protocol +exists to facilitate communicating connection meta-data that would +otherwise be lost such as the source IP address and port +\[EXTORPORT\]. + +All PT instances are configured by the respective parent process via +a set of standardized environment variables (3.2) that are set at +launch time, and report status information back to the parent via +writing output in a standardized format to stdout (3.3). + +Each invocation of a PT MUST be either a client OR a server. + +All PT client forward proxies MUST support either SOCKS 4 or SOCKS 5, +and SHOULD prefer SOCKS 5 over SOCKS 4. diff --git a/spec/pt-spec/configuration-environment.md b/spec/pt-spec/configuration-environment.md new file mode 100644 index 0000000..f2f0eb5 --- /dev/null +++ b/spec/pt-spec/configuration-environment.md @@ -0,0 +1,258 @@ +<a id="pt-spec.txt-3.2"></a> + +# Pluggable Transport Configuration Environment Variables {#envvars} + +All Pluggable Transport proxy instances are configured by their +parent process at launch time via a set of well defined +environment variables. + +The "TOR_PT\_" prefix is used for namespacing reasons and does not +indicate any relations to Tor, except for the origins of this +specification. + +<a id="pt-spec.txt-3.2.1"></a> + +## Common Environment Variables {#common} + +When launching either a client or server Pluggable Transport proxy, +the following common environment variables MUST be set. + +"TOR_PT_MANAGED_TRANSPORT_VER" + +Specifies the versions of the Pluggable Transport specification +the parent process supports, delimited by commas. All PTs MUST +accept any well-formed list, as long as a compatible version is +present. + +Valid versions MUST consist entirely of non-whitespace, +non-comma printable ASCII characters. + +The version of the Pluggable Transport specification as of this +document is "1". + +Example: + +TOR_PT_MANAGED_TRANSPORT_VER=1,1a,2b,this_is_a_valid_ver + +"TOR_PT_STATE_LOCATION" + +Specifies an absolute path to a directory where the PT is +allowed to store state that will be persisted across +invocations. The directory is not required to exist when +the PT is launched, however PT implementations SHOULD be +able to create it as required. + +PTs MUST only store files in the path provided, and MUST NOT +create or modify files elsewhere on the system. + +Example: + +TOR_PT_STATE_LOCATION=/var/lib/tor/pt_state/ + +"TOR_PT_EXIT_ON_STDIN_CLOSE" + +Specifies that the parent process will close the PT proxy's +standard input (stdin) stream to indicate that the PT proxy +should gracefully exit. + +PTs MUST NOT treat a closed stdin as a signal to terminate +unless this environment variable is set to "1". + +PTs SHOULD treat stdin being closed as a signal to gracefully +terminate if this environment variable is set to "1". + +Example: + +TOR_PT_EXIT_ON_STDIN_CLOSE=1 + +"TOR_PT_OUTBOUND_BIND_ADDRESS_V4" + +Specifies an IPv4 IP address that the PT proxy SHOULD use as source address for +outgoing IPv4 IP packets. This feature allows people with multiple network +interfaces to specify explicitly which interface they prefer the PT proxy to +use. + +If this value is unset or empty, the PT proxy MUST use the default source +address for outgoing connections. + +This setting MUST be ignored for connections to +loopback addresses (127.0.0.0/8). + +Example: + +TOR_PT_OUTBOUND_BIND_ADDRESS_V4=203.0.113.4 + +"TOR_PT_OUTBOUND_BIND_ADDRESS_V6" + +Specifies an IPv6 IP address that the PT proxy SHOULD use as source address for +outgoing IPv6 IP packets. This feature allows people with multiple network +interfaces to specify explicitly which interface they prefer the PT proxy to +use. + +If this value is unset or empty, the PT proxy MUST use the default source +address for outgoing connections. + +This setting MUST be ignored for connections to the loopback address (\[::1\]). + +IPv6 addresses MUST always be wrapped in square brackets. + +Example:: + +TOR_PT_OUTBOUND_BIND_ADDRESS_V6=\[2001:db8::4\] + +<a id="pt-spec.txt-3.2.2"></a> + +## Pluggable Transport Client Environment Variables {#client} + +Client-side Pluggable Transport forward proxies are configured +via the following environment variables. + +"TOR_PT_CLIENT_TRANSPORTS" + +Specifies the PT protocols the client proxy should initialize, +as a comma separated list of PT names. + +PTs SHOULD ignore PT names that it does not recognize. + +Parent processes MUST set this environment variable when +launching a client-side PT proxy instance. + +Example: + +TOR_PT_CLIENT_TRANSPORTS=obfs2,obfs3,obfs4 + +"TOR_PT_PROXY" + +Specifies an upstream proxy that the PT MUST use when making +outgoing network connections. It is a URI \[RFC3986\] of the +format: + +`<proxy_type>://[<user_name>[:<password>][@]<ip>:<port>`. + +The "TOR_PT_PROXY" environment variable is OPTIONAL and +MUST be omitted if there is no need to connect via an +upstream proxy. + +Examples: + +```text + TOR_PT_PROXY=socks5://tor:test1234@198.51.100.1:8000 + TOR_PT_PROXY=socks4a://198.51.100.2:8001 + TOR_PT_PROXY=http://198.51.100.3:443 +``` + +<a id="pt-spec.txt-3.2.3"></a> + +## Pluggable Transport Server Environment Variables {#server} + +Server-side Pluggable Transport reverse proxies are configured +via the following environment variables. + +"TOR_PT_SERVER_TRANSPORTS" + +Specifies the PT protocols the server proxy should initialize, +as a comma separated list of PT names. + +PTs SHOULD ignore PT names that it does not recognize. + +Parent processes MUST set this environment variable when +launching a server-side PT reverse proxy instance. + +Example: + +TOR_PT_SERVER_TRANSPORTS=obfs3,scramblesuit + +"TOR_PT_SERVER_TRANSPORT_OPTIONS" + +Specifies per-PT protocol configuration directives, as a +semicolon-separated list of `<key>:<value>` pairs, where `<key>` +is a PT name and `<value>` is a k=v string value with options +that are to be passed to the transport. + +Colons, semicolons, and backslashes MUST be +escaped with a backslash. + +If there are no arguments that need to be passed to any of +PT transport protocols, "TOR_PT_SERVER_TRANSPORT_OPTIONS" +MAY be omitted. + +Example: + +TOR_PT_SERVER_TRANSPORT_OPTIONS=scramblesuit:key=banana;automata:rule=110;automata:depth=3 + +```text + Will pass to 'scramblesuit' the parameter 'key=banana' and to + 'automata' the arguments 'rule=110' and 'depth=3'. + + "TOR_PT_SERVER_BINDADDR" +``` + +A comma separated list of `<key>-<value>` pairs, where `<key>` is +a PT name and `<value>` is the `<address>:<port>` on which it +should listen for incoming client connections. + +The keys holding transport names MUST be in the same order as +they appear in "TOR_PT_SERVER_TRANSPORTS". + +The `<address>` MAY be a locally scoped address as long as port +forwarding is done externally. + +The `<address>:<port>` combination MUST be an IP address +supported by `bind()`, and MUST NOT be a host name. + +Applications MUST NOT set more than one `<address>:<port>` pair +per PT name. + +If there is no specific `<address>:<port>` combination to be +configured for any transports, "TOR_PT_SERVER_BINDADDR" MAY +be omitted. + +Example: + +TOR_PT_SERVER_BINDADDR=obfs3-198.51.100.1:1984,scramblesuit-127.0.0.1:4891 + +"TOR_PT_ORPORT" + +Specifies the destination that the PT reverse proxy should forward +traffic to after transforming it as appropriate, as an +`<address>:<port>`. + +Connections to the destination specified via "TOR_PT_ORPORT" +MUST only contain application payload. If the parent process +requires the actual source IP address of client connections +(or other metadata), it should set "TOR_PT_EXTENDED_SERVER_PORT" +instead. + +Example: + +TOR_PT_ORPORT=127.0.0.1:9001 + +"TOR_PT_EXTENDED_SERVER_PORT" + +Specifies the destination that the PT reverse proxy should +forward traffic to, via the Extended ORPort protocol \[EXTORPORT\] +as an `<address>:<port>`. + +The Extended ORPort protocol allows the PT reverse proxy to +communicate per-connection metadata such as the PT name and +client IP address/port to the parent process. + +If the parent process does not support the ExtORPort protocol, +it MUST set "TOR_PT_EXTENDED_SERVER_PORT" to an empty string. + +Example: + +TOR_PT_EXTENDED_SERVER_PORT=127.0.0.1:4200 + +"TOR_PT_AUTH_COOKIE_FILE" + +Specifies an absolute filesystem path to the Extended ORPort +authentication cookie, required to communicate with the +Extended ORPort specified via "TOR_PT_EXTENDED_SERVER_PORT". + +If the parent process is not using the ExtORPort protocol for +incoming traffic, "TOR_PT_AUTH_COOKIE_FILE" MUST be omitted. + +Example: + +TOR_PT_AUTH_COOKIE_FILE=/var/lib/tor/extended_orport_auth_cookie diff --git a/spec/pt-spec/example-client-session.md b/spec/pt-spec/example-client-session.md new file mode 100644 index 0000000..2f20f5d --- /dev/null +++ b/spec/pt-spec/example-client-session.md @@ -0,0 +1,19 @@ +<a id="pt-spec.txt-A"></a> + +# Appendix A: Example Client Pluggable Transport Session + +Environment variables: + +TOR_PT_MANAGED_TRANSPORT_VER=1 +TOR_PT_STATE_LOCATION=/var/lib/tor/pt_state/ +TOR_PT_EXIT_ON_STDIN_CLOSE=1 +TOR_PT_PROXY=socks5://127.0.0.1:8001 +TOR_PT_CLIENT_TRANSPORTS=obfs3,obfs4 + +Messages the PT Proxy writes to stdin: + +VERSION 1 +PROXY DONE +CMETHOD obfs3 socks5 127.0.0.1:32525 +CMETHOD obfs4 socks5 127.0.0.1:37347 +CMETHODS DONE diff --git a/spec/pt-spec/example-server-session.md b/spec/pt-spec/example-server-session.md new file mode 100644 index 0000000..3fc3962 --- /dev/null +++ b/spec/pt-spec/example-server-session.md @@ -0,0 +1,18 @@ +<a id="pt-spec.txt-B"></a> + +# Appendix B: Example Server Pluggable Transport Session + +Environment variables: + +TOR_PT_MANAGED_TRANSPORT_VER=1 +TOR_PT_STATE_LOCATION=/var/lib/tor/pt_state +TOR_PT_EXIT_ON_STDIN_CLOSE=1 +TOR_PT_SERVER_TRANSPORTS=obfs3,obfs4 +TOR_PT_SERVER_BINDADDR=obfs3-198.51.100.1:1984 + +Messages the PT Proxy writes to stdin: + +VERSION 1 +SMETHOD obfs3 198.51.100.1:1984 +SMETHOD obfs4 198.51.100.1:43734 ARGS:cert=HszPy3vWfjsESCEOo9ZBkRv6zQ/1mGHzc8arF0y2SpwFr3WhsMu8rK0zyaoyERfbz3ddFw,iat-mode=0 +SMETHODS DONE diff --git a/spec/pt-spec/index.md b/spec/pt-spec/index.md new file mode 100644 index 0000000..64b999e --- /dev/null +++ b/spec/pt-spec/index.md @@ -0,0 +1,12 @@ +# Pluggable Transport Specification (Version 1) + +## Abstract + +Pluggable Transports (PTs) are a generic mechanism for the rapid +development and deployment of censorship circumvention, +based around the idea of modular sub-processes that transform +traffic to defeat censors. + +This document specifies the sub-process startup, shutdown, +and inter-process communication mechanisms required to utilize +PTs. diff --git a/spec/pt-spec/introduction.md b/spec/pt-spec/introduction.md new file mode 100644 index 0000000..796468e --- /dev/null +++ b/spec/pt-spec/introduction.md @@ -0,0 +1,27 @@ +<a id="pt-spec.txt-1"></a> + +# Introduction + +This specification describes a way to decouple protocol-level +obfuscation from an application's client/server code, in a manner +that promotes rapid development of obfuscation/circumvention +tools and promotes reuse beyond the scope of the Tor Project's +efforts in that area. + +This is accomplished by utilizing helper sub-processes that +implement the necessary forward/reverse proxy servers that handle +the censorship circumvention, with a well defined and +standardized configuration and management interface. + +Any application code that implements the interfaces as specified +in this document will be able to use all spec compliant Pluggable +Transports. + +<a id="pt-spec.txt-1.1"></a> + +## Requirements Notation + +The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL +NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and +"OPTIONAL" in this document are to be interpreted as described in +\[RFC2119\]. diff --git a/spec/pt-spec/ipc.md b/spec/pt-spec/ipc.md new file mode 100644 index 0000000..d419fba --- /dev/null +++ b/spec/pt-spec/ipc.md @@ -0,0 +1,303 @@ +<a id="pt-spec.txt-3.3"></a> + +# Pluggable Transport To Parent Process Communication + +All Pluggable Transport Proxies communicate to the parent process +via writing NL-terminated lines to stdout. The line metaformat is: + +```text +<Line> ::= <Keyword> <OptArgs> <NL> +<Keyword> ::= <KeywordChar> | <Keyword> <KeywordChar> +<KeywordChar> ::= <any US-ASCII alphanumeric, dash, and underscore> +<OptArgs> ::= <Args>* +<Args> ::= <SP> <ArgChar> | <Args> <ArgChar> +<ArgChar> ::= <any US-ASCII character but NUL or NL> +<SP> ::= <US-ASCII whitespace symbol (32)> +<NL> ::= <US-ASCII newline (line feed) character (10)> +``` + +The parent process MUST ignore lines received from PT proxies with +unknown keywords. + +<a id="pt-spec.txt-3.3.1"></a> + +## Common Messages + +When a PT proxy first starts up, it must determine which version +of the Pluggable Transports Specification to use to configure +itself. + +It does this via the "TOR_PT_MANAGED_TRANSPORT_VER" (3.2.1) +environment variable which contains all of the versions supported +by the application. + +Upon determining the version to use, or lack thereof, the PT +proxy responds with one of two messages. + +`VERSION-ERROR <ErrorMessage>` + +The "VERSION-ERROR" message is used to signal that there was +no compatible Pluggable Transport Specification version +present in the "TOR_PT_MANAGED_TRANSPORT_VER" list. + +The `<ErrorMessage>` SHOULD be set to "no-version" for +historical reasons but MAY be set to a useful error message +instead. + +PT proxies MUST terminate after outputting a "VERSION-ERROR" +message. + +Example: + +`VERSION-ERROR no-version` + +`VERSION <ProtocolVersion>` + +The "VERSION" message is used to signal the Pluggable Transport +Specification version (as in "TOR_PT_MANAGED_TRANSPORT_VER") +that the PT proxy will use to configure its transports and +communicate with the parent process. + +The version for the environment values and reply messages +specified by this document is "1". + +PT proxies MUST either report an error and terminate, or output +a "VERSION" message before moving on to client/server proxy +initialization and configuration. + +Example: + +VERSION 1 + +After version negotiation has been completed the PT proxy must +then validate that all of the required environment variables are +provided, and that all of the configuration values supplied are +well formed. + +At any point, if there is an error encountered related to +configuration supplied via the environment variables, it MAY +respond with an error message and terminate. + +`ENV-ERROR <ErrorMessage>` + +The "ENV-ERROR" message is used to signal the PT proxy's +failure to parse the configuration environment variables (3.2). + +The `<ErrorMessage>` SHOULD consist of a useful error message +that can be used to diagnose and correct the root cause of +the failure. + +PT proxies MUST terminate after outputting a "ENV-ERROR" +message. + +Example: + +ENV-ERROR No TOR_PT_AUTH_COOKIE_FILE when TOR_PT_EXTENDED_SERVER_PORT set + +<a id="pt-spec.txt-3.3.2"></a> + +## Pluggable Transport Client Messages {#client-messages} + +After negotiating the Pluggable Transport Specification version, +PT client proxies MUST first validate "TOR_PT_PROXY" (3.2.2) if +it is set, before initializing any transports. + +Assuming that an upstream proxy is provided, PT client proxies +MUST respond with a message indicating that the proxy is valid, +supported, and will be used OR a failure message. + +PROXY DONE + +The "PROXY DONE" message is used to signal the PT proxy's +acceptance of the upstream proxy specified by "TOR_PT_PROXY". + +`PROXY-ERROR <ErrorMessage>` + +The "PROXY-ERROR" message is used to signal that the upstream +proxy is malformed/unsupported or otherwise unusable. + +PT proxies MUST terminate immediately after outputting a +"PROXY-ERROR" message. + +Example: + +PROXY-ERROR SOCKS 4 upstream proxies unsupported. + +After the upstream proxy (if any) is configured, PT clients then +iterate over the requested transports in "TOR_PT_CLIENT_TRANSPORTS" +and initialize the listeners. + +For each transport initialized, the PT proxy reports the listener +status back to the parent via messages to stdout. + +`CMETHOD <transport> <'socks4','socks5'> <address:port>` + +The "CMETHOD" message is used to signal that a requested +PT transport has been launched, the protocol which the parent +should use to make outgoing connections, and the IP address +and port that the PT transport's forward proxy is listening on. + +Example: + +`CMETHOD trebuchet socks5 127.0.0.1:19999` + +`CMETHOD-ERROR <transport> <ErrorMessage>` + +The "CMETHOD-ERROR" message is used to signal that +requested PT transport was unable to be launched. + +Example: + +CMETHOD-ERROR trebuchet no rocks available + +Once all PT transports have been initialized (or have failed), the +PT proxy MUST send a final message indicating that it has finished +initializing. + +CMETHODS DONE + +The "CMETHODS DONE" message signals that the PT proxy has +finished initializing all of the transports that it is capable +of handling. + +Upon sending the "CMETHODS DONE" message, the PT proxy +initialization is complete. + +Notes: + +```text + - Unknown transports in "TOR_PT_CLIENT_TRANSPORTS" are ignored + entirely, and MUST NOT result in a "CMETHOD-ERROR" message. + Thus it is entirely possible for a given PT proxy to + immediately output "CMETHODS DONE". + + - Parent processes MUST handle "CMETHOD"/"CMETHOD-ERROR" + messages in any order, regardless of ordering in + "TOR_PT_CLIENT_TRANSPORTS". +``` + +<a id="pt-spec.txt-3.3.3"></a> + +## Pluggable Transport Server Messages {#server-messages} + +PT server reverse proxies iterate over the requested transports +in "TOR_PT_CLIENT_TRANSPORTS" and initialize the listeners. + +For each transport initialized, the PT proxy reports the listener +status back to the parent via messages to stdout. + +`SMETHOD <transport> <address:port> [options]` + +The "SMETHOD" message is used to signal that a requested +PT transport has been launched, the protocol which will be +used to handle incoming connections, and the IP address and +port that clients should use to reach the reverse-proxy. + +If there is a specific <address:port> provided for a given +PT transport via "TOR_PT_SERVER_BINDADDR", the transport +MUST be initialized using that as the server address. + +The OPTIONAL 'options' field is used to pass additional +per-transport information back to the parent process. + +The currently recognized 'options' are: + +`ARGS:[<Key>=<Value>,]+[<Key>=<Value>]` + +```text + The "ARGS" option is used to pass additional key/value + formatted information that clients will require to use + the reverse proxy. + + Equal signs and commas MUST be escaped with a backslash. + + Tor: The ARGS are included in the transport line of the + Bridge's extra-info document. + + Examples: + + SMETHOD trebuchet 198.51.100.1:19999 + SMETHOD rot_by_N 198.51.100.1:2323 ARGS:N=13 + + SMETHOD-ERROR <transport> <ErrorMessage> +``` + +The "SMETHOD-ERROR" message is used to signal that +requested PT transport reverse proxy was unable to be +launched. + +Example: + +SMETHOD-ERROR trebuchet no cows available + +Once all PT transports have been initialized (or have failed), the +PT proxy MUST send a final message indicating that it has finished +initializing. + +SMETHODS DONE + +The "SMETHODS DONE" message signals that the PT proxy has +finished initializing all of the transports that it is capable +of handling. + +Upon sending the "SMETHODS DONE" message, the PT proxy +initialization is complete. + +<a id="pt-spec.txt-3.3.4"></a> + +## Pluggable Transport Log Message {#log-messages} + +This message is for a client or server PT to be able to signal back to the +parent process via stdout or stderr any log messages. + +A log message can be any kind of messages (human readable) that the PT +sends back so the parent process can gather information about what is going +on in the child process. It is not intended for the parent process to parse +and act accordingly but rather a message used for plain logging. + +For example, the tor daemon logs those messages at the Severity level and +sends them onto the control port using the PT_LOG (see control-spec.txt) +event so any third party can pick them up for debugging. + +The format of the message: + +`LOG SEVERITY=Severity MESSAGE=Message` + +The SEVERITY value indicate at which logging level the message applies. +The accepted values for `<Severity>` are: error, warning, notice, info, debug + +The MESSAGE value is a human readable string formatted by the PT. The +`<Message>` contains the log message which can be a String or CString (see +section 2 in control-spec.txt). + +Example: + +`LOG SEVERITY=debug MESSAGE="Connected to bridge A"` + +<a id="pt-spec.txt-3.3.5"></a> + +## Pluggable Transport Status Message {#status-messages} + +This message is for a client or server PT to be able to signal back to the +parent process via stdout or stderr any status messages. + +The format of the message: + +`STATUS TRANSPORT=Transport <K_1>=<V_1> [<K_2>=<V_2> ...]` + +The TRANSPORT value indicates a hint on what the PT is such has the name or +the protocol used for instance. As an example, obfs4proxy would use +"obfs4". Thus, the Transport value can be anything the PT itself defines +and it can be a String or CString (see section 2 in control-spec.txt). + +The `<K_n>=<V_n>` values are specific to the PT and there has to be at least +one. They are messages that reflects the status that the PT wants to +report. `<V_n>` can be a String or CString. + +Examples (fictional): + +```text +STATUS TRANSPORT=obfs4 ADDRESS=198.51.100.123:1234 CONNECT=Success +STATUS TRANSPORT=obfs4 ADDRESS=198.51.100.222:2222 CONNECT=Failed FINGERPRINT=<Fingerprint> ERRSTR="Connection refused" +STATUS TRANSPORT=trebuchet ADDRESS=198.51.100.15:443 PERCENT=42 +``` diff --git a/spec/pt-spec/naming.md b/spec/pt-spec/naming.md new file mode 100644 index 0000000..f86c5e3 --- /dev/null +++ b/spec/pt-spec/naming.md @@ -0,0 +1,13 @@ +<a id="pt-spec.txt-3.1"></a> + +# Pluggable Transport Naming + +Pluggable Transport names serve as unique identifiers, and every +PT MUST have a unique name. + +PT names MUST be valid C identifiers. PT names MUST begin with +a letter or underscore, and the remaining characters MUST be +ASCII letters, numbers or underscores. No length limit is +imposted. + +PT names MUST satisfy the regular expression "`[a-zA-Z_][a-zA-Z0-9_]*`". diff --git a/spec/pt-spec/per-connection-args.md b/spec/pt-spec/per-connection-args.md new file mode 100644 index 0000000..e1a133a --- /dev/null +++ b/spec/pt-spec/per-connection-args.md @@ -0,0 +1,38 @@ +<a id="pt-spec.txt-3.5"></a> + +# Pluggable Transport Client Per-Connection Arguments + +Certain PT transport protocols require that the client provides +per-connection arguments when making outgoing connections. On +the server side, this is handled by the "ARGS" optional argument +as part of the "SMETHOD" message. + +On the client side, arguments are passed via the authentication +fields that are part of the SOCKS protocol. + +First the "`<Key>=<Value>`" formatted arguments MUST be escaped, +such that all backslash, equal sign, and semicolon characters +are escaped with a backslash. + +Second, all of the escaped are concatenated together. + +Example: + +shared-secret=rahasia;secrets-file=/tmp/blob + +Lastly the arguments are transmitted when making the outgoing +connection using the authentication mechanism specific to the +SOCKS protocol version. + +```text + - In the case of SOCKS 4, the concatenated argument list is + transmitted in the "USERID" field of the "CONNECT" request. + + - In the case of SOCKS 5, the parent process must negotiate + "Username/Password" authentication [RFC1929], and transmit + the arguments encoded in the "UNAME" and "PASSWD" fields. +``` + +If the encoded argument list is less than 255 bytes in +length, the "PLEN" field must be set to "1" and the "PASSWD" +field must contain a single NUL character. diff --git a/spec/pt-spec/references.md b/spec/pt-spec/references.md new file mode 100644 index 0000000..e94d41d --- /dev/null +++ b/spec/pt-spec/references.md @@ -0,0 +1,22 @@ +<a id="pt-spec.txt-5"></a> + +# References + +```text + [RFC2119] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, March 1997. + + [RFC1928] Leech, M., Ganis, M., Lee, Y., Kuris, R., + Koblas, D., Jones, L., "SOCKS Protocol Version 5", + RFC 1928, March 1996. + + [EXTORPORT] Kadianakis, G., Mathewson, N., "Extended ORPort and + TransportControlPort", Tor Proposal 196, March 2012. + + [RFC3986] Berners-Lee, T., Fielding, R., Masinter, L., "Uniform + Resource Identifier (URI): Generic Syntax", RFC 3986, + January 2005. + + [RFC1929] Leech, M., "Username/Password Authentication for + SOCKS V5", RFC 1929, March 1996. +``` diff --git a/spec/pt-spec/shutdown.md b/spec/pt-spec/shutdown.md new file mode 100644 index 0000000..aeba498 --- /dev/null +++ b/spec/pt-spec/shutdown.md @@ -0,0 +1,32 @@ +<a id="pt-spec.txt-3.4"></a> + +# Pluggable Transport Shutdown + +The recommended way for Pluggable Transport using applications and +Pluggable Transports to handle graceful shutdown is as follows. + +```text + - (Parent) Set "TOR_PT_EXIT_ON_STDIN_CLOSE" (3.2.1) when + launching the PT proxy, to indicate that stdin will be used + for graceful shutdown notification. + + - (Parent) When the time comes to terminate the PT proxy: + + 1. Close the PT proxy's stdin. + 2. Wait for a "reasonable" amount of time for the PT to exit. + 3. Attempt to use OS specific mechanisms to cause graceful + PT shutdown (eg: 'SIGTERM') + 4. Use OS specific mechanisms to force terminate the PT + (eg: 'SIGKILL', 'ProccessTerminate()'). + + - PT proxies SHOULD monitor stdin, and exit gracefully when + it is closed, if the parent supports that behavior. + + - PT proxies SHOULD handle OS specific mechanisms to gracefully + terminate (eg: Install a signal handler on 'SIGTERM' that + causes cleanup and a graceful shutdown if able). + + - PT proxies SHOULD attempt to detect when the parent has + terminated (eg: via detecting that its parent process ID has + changed on U*IX systems), and gracefully terminate. +``` diff --git a/spec/pt-spec/specification.md b/spec/pt-spec/specification.md new file mode 100644 index 0000000..7a8822b --- /dev/null +++ b/spec/pt-spec/specification.md @@ -0,0 +1,52 @@ +<a id="pt-spec.txt-3"></a> + +# Specification + +Pluggable Transport proxies follow the following workflow +throughout their lifespan. + +```text + 1) Parent process sets the required environment values (3.2) + and launches the PT proxy as a sub-process (fork()/exec()). + + 2) The PT Proxy determines the versions of the PT specification + supported by the parent"TOR_PT_MANAGED_TRANSPORT_VER" (3.2.1) + + 2.1) If there are no compatible versions, the PT proxy + writes a "VERSION-ERROR" message (3.3.1) to stdout and + terminates. + + 2.2) If there is a compatible version, the PT proxy writes + a "VERSION" message (3.3.1) to stdout. + + 3) The PT Proxy parses the rest of the environment values. + + 3.1) If the environment values are malformed, or otherwise + invalid, the PT proxy writes a "ENV-ERROR" message + (3.3.1) to stdout and terminates. + + 3.2) Determining if it is a client side forward proxy or + a server side reverse proxy can be done via examining + the "TOR_PT_CLIENT_TRANSPORTS" and "TOR_PT_SERVER_TRANSPORTS" + environment variables. + + 4) (Client only) If there is an upstream proxy specified via + "TOR_PT_PROXY" (3.2.2), the PT proxy validates the URI + provided. + + 4.1) If the upstream proxy is unusable, the PT proxy writes + a "PROXY-ERROR" message (3.3.2) to stdout and + terminates. + + 4.2) If there is a supported and well-formed upstream proxy + the PT proxy writes a "PROXY DONE" message (3.3.2) to + stdout. + + 5) The PT Proxy initializes the transports and reports the + status via stdout (3.3.2, 3.3.3) + + 6) The PT Proxy forwards and transforms traffic as appropriate. + + 7) Upon being signaled to terminate by the parent process (3.4), + the PT Proxy gracefully shuts down. +``` diff --git a/spec/rend-spec/client-authorization.md b/spec/rend-spec/client-authorization.md new file mode 100644 index 0000000..1cffce4 --- /dev/null +++ b/spec/rend-spec/client-authorization.md @@ -0,0 +1,105 @@ +<a id="rend-spec-v3.txt-G"></a> + +# Appendix G: Managing authorized client data \[CLIENT-AUTH-MGMT\] + +Hidden services and clients can configure their authorized client data either +using the torrc, or using the control port. This section presents a suggested +scheme for configuring client authorization. Please see appendix +\[HIDSERVDIR-FORMAT\] for more information about relevant hidden service files. + +(NOTE: client authorization is implemented as of 0.3.5.1-alpha.) + +G.1. Configuring client authorization using torrc + +G.1.1. Hidden Service side configuration + +```text + A hidden service that wants to enable client authorization, needs to + populate the "authorized_clients/" directory of its HiddenServiceDir + directory with the ".auth" files of its authorized clients. + + When Tor starts up with a configured onion service, Tor checks its + <HiddenServiceDir>/authorized_clients/ directory for ".auth" files, and if + any recognized and parseable such files are found, then client + authorization becomes activated for that service. + + G.1.2. Service-side bookkeeping + + This section contains more details on how onion services should be keeping + track of their client ".auth" files. + + For the "descriptor" authentication type, the ".auth" file MUST contain + the x25519 public key of that client. Here is a suggested file format: + + <auth-type>:<key-type>:<base32-encoded-public-key> + + Here is an an example: + + descriptor:x25519:OM7TGIVRYMY6PFX6GAC6ATRTA5U6WW6U7A4ZNHQDI6OVL52XVV2Q + + Tor SHOULD ignore lines it does not recognize. + Tor SHOULD ignore files that don't use the ".auth" suffix. + + G.1.3. Client side configuration + + A client who wants to register client authorization data for onion + services needs to add the following line to their torrc to indicate the + directory which hosts ".auth_private" files containing client-side + credentials for onion services: + + ClientOnionAuthDir <DIR> + + The <DIR> contains a file with the suffix ".auth_private" for each onion + service the client is authorized with. Tor should scan the directory for + ".auth_private" files to find which onion services require client + authorization from this client. + + For the "descriptor" auth-type, a ".auth_private" file contains the + private x25519 key: + + <onion-address>:descriptor:x25519:<base32-encoded-privkey> + + The keypair used for client authorization is created by a third party tool + for which the public key needs to be transferred to the service operator + in a secure out-of-band way. The third party tool SHOULD add appropriate + headers to the private key file to ensure that users won't accidentally + give out their private key. + + G.2. Configuring client authorization using the control port + + G.2.1. Service side + + A hidden service also has the option to configure authorized clients + using the control port. The idea is that hidden service operators can use + controller utilities that manage their access control instead of using + the filesystem to register client keys. + + Specifically, we require a new control port command ADD_ONION_CLIENT_AUTH + which is able to register x25519/ed25519 public keys tied to a specific + authorized client. + [XXX figure out control port command format] + + Hidden services who use the control port interface for client auth need + to perform their own key management. + + G.2.2. Client side + + There should also be a control port interface for clients to register + authorization data for hidden services without having to use the + torrc. It should allow both generation of client authorization private + keys, and also to import client authorization data provided by a hidden + service + + This way, Tor Browser can present "Generate client auth keys" and "Import + client auth keys" dialogs to users when they try to visit a hidden service + that is protected by client authorization. + + Specifically, we require two new control port commands: + IMPORT_ONION_CLIENT_AUTH_DATA + GENERATE_ONION_CLIENT_AUTH_DATA + which import and generate client authorization data respectively. + + [XXX how does key management work here?] + [XXX what happens when people use both the control port interface and the + filesystem interface?] +``` diff --git a/spec/rend-spec/deriving-keys.md b/spec/rend-spec/deriving-keys.md new file mode 100644 index 0000000..69eeee6 --- /dev/null +++ b/spec/rend-spec/deriving-keys.md @@ -0,0 +1,425 @@ +<a id="rend-spec-v3.txt-2.1"></a> + +# Deriving blinded keys and subcredentials {#SUBCRED} + +In each time period (see \[TIME-PERIODS\] for a definition of time +periods), a hidden service host uses a different blinded private key +to sign its directory information, and clients use a different +blinded public key as the index for fetching that information. + +For a candidate for a key derivation method, see Appendix \[KEYBLIND\]. + +Additionally, clients and hosts derive a subcredential for each +period. Knowledge of the subcredential is needed to decrypt hidden +service descriptors for each period and to authenticate with the +hidden service host in the introduction process. Unlike the +credential, it changes each period. Knowing the subcredential, even +in combination with the blinded private key, does not enable the +hidden service host to derive the main credential--therefore, it is +safe to put the subcredential on the hidden service host while +leaving the hidden service's private key offline. + +The subcredential for a period is derived as: + +```text +N_hs_subcred = H("subcredential" | N_hs_cred | blinded-public-key). +``` + +In the above formula, credential corresponds to: + +```text +N_hs_cred = H("credential" | public-identity-key) +``` + +where `public-identity-key` is the public identity master key of the hidden +service. + +# Locating, uploading, and downloading hidden service descriptors {#HASHRING} + +To avoid attacks where a hidden service's descriptor is easily +targeted for censorship, we store them at different directories over +time, and use shared random values to prevent those directories from +being predictable far in advance. + +Which Tor servers hosts a hidden service depends on: + +```text + * the current time period, + * the daily subcredential, + * the hidden service directories' public keys, + * a shared random value that changes in each time period, + shared_random_value. + * a set of network-wide networkstatus consensus parameters. + (Consensus parameters are integer values voted on by authorities + and published in the consensus documents, described in + dir-spec.txt, section 3.3.) + + Below we explain in more detail. +``` + +<a id="rend-spec-v3.txt-2.2.1"></a> + +## Dividing time into periods {#TIME-PERIODS} + +To prevent a single set of hidden service directory from becoming a +target by adversaries looking to permanently censor a hidden service, +hidden service descriptors are uploaded to different locations that +change over time. + +The length of a "time period" is controlled by the consensus +parameter 'hsdir-interval', and is a number of minutes between 30 and +14400 (10 days). The default time period length is 1440 (one day). + +Time periods start at the Unix epoch (Jan 1, 1970), and are computed by +taking the number of minutes since the epoch and dividing by the time +period. However, we want our time periods to start at a regular offset +from the SRV voting schedule, so we subtract a "rotation time offset" +of 12 voting periods from the number of minutes since the epoch, before +dividing by the time period (effectively making "our" epoch start at Jan +1, 1970 12:00UTC when the voting period is 1 hour.) + +Example: If the current time is 2016-04-13 11:15:01 UTC, making the seconds +since the epoch 1460546101, and the number of minutes since the epoch +24342435\. We then subtract the "rotation time offset" of 12\*60 minutes from +the minutes since the epoch, to get 24341715. If the current time period +length is 1440 minutes, by doing the division we see that we are currently +in time period number 16903. + +Specifically, time period #16903 began 16903\*1440\*60 + (12\*60\*60) seconds +after the epoch, at 2016-04-12 12:00 UTC, and ended at 16904\*1440\*60 + +(12\*60\*60) seconds after the epoch, at 2016-04-13 12:00 UTC. + +<a id="rend-spec-v3.txt-2.2.2"></a> + +## When to publish a hidden service descriptor {#WHEN-HSDESC} + +Hidden services periodically publish their descriptor to the responsible +HSDirs. The set of responsible HSDirs is determined as specified in +\[WHERE-HSDESC\]. + +Specifically, every time a hidden service publishes its descriptor, it also +sets up a timer for a random time between 60 minutes and 120 minutes in the +future. When the timer triggers, the hidden service needs to publish its +descriptor again to the responsible HSDirs for that time period. +\[TODO: Control republish period using a consensus parameter?\] + +<a id="rend-spec-v3.txt-2.2.2.1"></a> + +### Overlapping descriptors {#OVERLAPPING-DESCS} + +Hidden services need to upload multiple descriptors so that they can be +reachable to clients with older or newer consensuses than them. Services +need to upload their descriptors to the HSDirs *before* the beginning of +each upcoming time period, so that they are readily available for clients to +fetch them. Furthermore, services should keep uploading their old descriptor +even after the end of a time period, so that they can be reachable by +clients that still have consensuses from the previous time period. + +Hence, services maintain two active descriptors at every point. Clients on +the other hand, don't have a notion of overlapping descriptors, and instead +always download the descriptor for the current time period and shared random +value. It's the job of the service to ensure that descriptors will be +available for all clients. See section \[FETCHUPLOADDESC\] for how this is +achieved. + +\[TODO: What to do when we run multiple hidden services in a single host?\] + +<a id="rend-spec-v3.txt-2.2.3"></a> + +## Where to publish a hidden service descriptor {#WHERE-HSDESC} + +This section specifies how the HSDir hash ring is formed at any given +time. Whenever a time value is needed (e.g. to get the current time period +number), we assume that clients and services use the valid-after time from +their latest live consensus. + +The following consensus parameters control where a hidden service +descriptor is stored; + +```text + hsdir_n_replicas = an integer in range [1,16] with default value 2. + hsdir_spread_fetch = an integer in range [1,128] with default value 3. + hsdir_spread_store = an integer in range [1,128] with default value 4. + (Until 0.3.2.8-rc, the default was 3.) +``` + +To determine where a given hidden service descriptor will be stored +in a given period, after the blinded public key for that period is +derived, the uploading or downloading party calculates: + +```text + for replicanum in 1...hsdir_n_replicas: + hs_service_index(replicanum) = H("store-at-idx" | + blinded_public_key | + INT_8(replicanum) | + INT_8(period_length) | + INT_8(period_num) ) +``` + +where blinded_public_key is specified in section \[KEYBLIND\], period_length +is the length of the time period in minutes, and period_num is calculated +using the current consensus "valid-after" as specified in section +\[TIME-PERIODS\]. + +Then, for each node listed in the current consensus with the HSDir flag, +we compute a directory index for that node as: + +```text + hs_relay_index(node) = H("node-idx" | node_identity | + shared_random_value | + INT_8(period_num) | + INT_8(period_length) ) +``` + +where shared_random_value is the shared value generated by the authorities +in section \[PUB-SHAREDRANDOM\], and node_identity is the ed25519 identity +key of the node. + +Finally, for replicanum in 1...hsdir_n_replicas, the hidden service +host uploads descriptors to the first hsdir_spread_store nodes whose +indices immediately follow hs_service_index(replicanum). If any of those +nodes have already been selected for a lower-numbered replica of the +service, any nodes already chosen are disregarded (i.e. skipped over) +when choosing a replica's hsdir_spread_store nodes. + +When choosing an HSDir to download from, clients choose randomly from +among the first hsdir_spread_fetch nodes after the indices. (Note +that, in order to make the system better tolerate disappearing +HSDirs, hsdir_spread_fetch may be less than hsdir_spread_store.) +Again, nodes from lower-numbered replicas are disregarded when +choosing the spread for a replica. + +<a id="rend-spec-v3.txt-2.2.4"></a> + +## Using time periods and SRVs to fetch/upload HS descriptors {#FETCHUPLOADDESC} + +Hidden services and clients need to make correct use of time periods (TP) +and shared random values (SRVs) to successfully fetch and upload +descriptors. Furthermore, to avoid problems with skewed clocks, both clients +and services use the 'valid-after' time of a live consensus as a way to take +decisions with regards to uploading and fetching descriptors. By using the +consensus times as the ground truth here, we minimize the desynchronization +of clients and services due to system clock. Whenever time-based decisions +are taken in this section, assume that they are consensus times and not +system times. + +As \[PUB-SHAREDRANDOM\] specifies, consensuses contain two shared random +values (the current one and the previous one). Hidden services and clients +are asked to match these shared random values with descriptor time periods +and use the right SRV when fetching/uploading descriptors. This section +attempts to precisely specify how this works. + +Let's start with an illustration of the system: + +```text + +------------------------------------------------------------------+ + | | + | 00:00 12:00 00:00 12:00 00:00 12:00 | + | SRV#1 TP#1 SRV#2 TP#2 SRV#3 TP#3 | + | | + | $==========|-----------$===========|-----------$===========| | + | | + | | + +------------------------------------------------------------------+ + + Legend: [TP#1 = Time Period #1] + [SRV#1 = Shared Random Value #1] + ["$" = descriptor rotation moment] +``` + +<a id="rend-spec-v3.txt-2.2.4.1"></a> + +### Client behavior for fetching descriptors {#CLIENTFETCH} + +And here is how clients use TPs and SRVs to fetch descriptors: + +Clients always aim to synchronize their TP with SRV, so they always want to +use TP#N with SRV#N: To achieve this wrt time periods, clients always use +the current time period when fetching descriptors. Now wrt SRVs, if a client +is in the time segment between a new time period and a new SRV (i.e. the +segments drawn with "-") it uses the current SRV, else if the client is in a +time segment between a new SRV and a new time period (i.e. the segments +drawn with "="), it uses the previous SRV. + +Example: + +```text ++------------------------------------------------------------------+ +| | +| 00:00 12:00 00:00 12:00 00:00 12:00 | +| SRV#1 TP#1 SRV#2 TP#2 SRV#3 TP#3 | +| | +| $==========|-----------$===========|-----------$===========| | +| ^ ^ | +| C1 C2 | ++------------------------------------------------------------------+ +``` + +If a client (C1) is at 13:00 right after TP#1, then it will use TP#1 and +SRV#1 for fetching descriptors. Also, if a client (C2) is at 01:00 right +after SRV#2, it will still use TP#1 and SRV#1. + +<a id="rend-spec-v3.txt-2.2.4.2"></a> + +### Service behavior for uploading descriptors {#SERVICEUPLOAD} + +As discussed above, services maintain two active descriptors at any time. We +call these the "first" and "second" service descriptors. Services rotate +their descriptor every time they receive a consensus with a valid_after time +past the next SRV calculation time. They rotate their descriptors by +discarding their first descriptor, pushing the second descriptor to the +first, and rebuilding their second descriptor with the latest data. + +Services like clients also employ a different logic for picking SRV and TP +values based on their position in the graph above. Here is the logic: + +<a id="rend-spec-v3.txt-2.2.4.2.1"></a> + +#### First descriptor upload logic {#FIRSTDESCUPLOAD} + +Here is the service logic for uploading its first descriptor: + +When a service is in the time segment between a new time period a new SRV +(i.e. the segments drawn with "-"), it uses the previous time period and +previous SRV for uploading its first descriptor: that's meant to cover +for clients that have a consensus that is still in the previous time period. + +Example: Consider in the above illustration that the service is at 13:00 +right after TP#1. It will upload its first descriptor using TP#0 and SRV#0. +So if a client still has a 11:00 consensus it will be able to access it +based on the client logic above. + +Now if a service is in the time segment between a new SRV and a new time +period (i.e. the segments drawn with "=") it uses the current time period +and the previous SRV for its first descriptor: that's meant to cover clients +with an up-to-date consensus in the same time period as the service. + +Example: + +```text ++------------------------------------------------------------------+ +| | +| 00:00 12:00 00:00 12:00 00:00 12:00 | +| SRV#1 TP#1 SRV#2 TP#2 SRV#3 TP#3 | +| | +| $==========|-----------$===========|-----------$===========| | +| ^ | +| S | ++------------------------------------------------------------------+ +``` + +Consider that the service is at 01:00 right after SRV#2: it will upload its +first descriptor using TP#1 and SRV#1. + +<a id="rend-spec-v3.txt-2.2.4.2.2"></a> + +#### Second descriptor upload logic {#SECONDDESCUPLOAD} + +Here is the service logic for uploading its second descriptor: + +When a service is in the time segment between a new time period a new SRV +(i.e. the segments drawn with "-"), it uses the current time period and +current SRV for uploading its second descriptor: that's meant to cover for +clients that have an up-to-date consensus on the same TP as the service. + +Example: Consider in the above illustration that the service is at 13:00 +right after TP#1: it will upload its second descriptor using TP#1 and SRV#1. + +Now if a service is in the time segment between a new SRV and a new time +period (i.e. the segments drawn with "=") it uses the next time period and +the current SRV for its second descriptor: that's meant to cover clients +with a newer consensus than the service (in the next time period). + +Example: + +```text ++------------------------------------------------------------------+ +| | +| 00:00 12:00 00:00 12:00 00:00 12:00 | +| SRV#1 TP#1 SRV#2 TP#2 SRV#3 TP#3 | +| | +| $==========|-----------$===========|-----------$===========| | +| ^ | +| S | ++------------------------------------------------------------------+ +``` + +Consider that the service is at 01:00 right after SRV#2: it will upload its +second descriptor using TP#2 and SRV#2. + +<a id="rend-spec-v3.txt-2.2.4.3"></a> + +### Directory behavior for handling descriptor uploads \[DIRUPLOAD\] + +Upon receiving a hidden service descriptor publish request, directories MUST +check the following: + +```text + * The outer wrapper of the descriptor can be parsed according to + [DESC-OUTER] + * The version-number of the descriptor is "3" + * If the directory has already cached a descriptor for this hidden service, + the revision-counter of the uploaded descriptor must be greater than the + revision-counter of the cached one + * The descriptor signature is valid +``` + +If any of these basic validity checks fails, the directory MUST reject the +descriptor upload. + +NOTE: Even if the descriptor passes the checks above, its first and second +layers could still be invalid: directories cannot validate the encrypted +layers of the descriptor, as they do not have access to the public key of the +service (required for decrypting the first layer of encryption), or the +necessary client credentials (for decrypting the second layer). + +<a id="rend-spec-v3.txt-2.2.5"></a> + +## Expiring hidden service descriptors {#EXPIRE-DESC} + +Hidden services set their descriptor's "descriptor-lifetime" field to 180 +minutes (3 hours). Hidden services ensure that their descriptor will remain +valid in the HSDir caches, by republishing their descriptors periodically as +specified in \[WHEN-HSDESC\]. + +Hidden services MUST also keep their introduction circuits alive for as long +as descriptors including those intro points are valid (even if that's after +the time period has changed). + +<a id="rend-spec-v3.txt-2.2.6"></a> + +## URLs for anonymous uploading and downloading {#urls} + +Hidden service descriptors conforming to this specification are uploaded +with an HTTP POST request to the URL `/tor/hs/<version>/publish` relative to +the hidden service directory's root, and downloaded with an HTTP GET +request for the URL `/tor/hs/<version>/<z>` where `<z>` is a base64 encoding of +the hidden service's blinded public key and `<version>` is the protocol +version which is "3" in this case. + +These requests must be made anonymously, on circuits not used for +anything else. + +<a id="rend-spec-v3.txt-2.2.7"></a> + +## Client-side validation of onion addresses {#addr-validation} + +When a Tor client receives a prop224 onion address from the user, it +MUST first validate the onion address before attempting to connect or +fetch its descriptor. If the validation fails, the client MUST +refuse to connect. + +As part of the address validation, Tor clients should check that the +underlying ed25519 key does not have a torsion component. If Tor accepted +ed25519 keys with torsion components, attackers could create multiple +equivalent onion addresses for a single ed25519 key, which would map to the +same service. We want to avoid that because it could lead to phishing +attacks and surprising behaviors (e.g. imagine a browser plugin that blocks +onion addresses, but could be bypassed using an equivalent onion address +with a torsion component). + +The right way for clients to detect such fraudulent addresses (which should +only occur malevolently and never naturally) is to extract the ed25519 +public key from the onion address and multiply it by the ed25519 group order +and ensure that the result is the ed25519 identity element. For more +details, please see \[TORSION-REFS\]. diff --git a/spec/rend-spec/encoding-onion-addresses.md b/spec/rend-spec/encoding-onion-addresses.md new file mode 100644 index 0000000..be8daf7 --- /dev/null +++ b/spec/rend-spec/encoding-onion-addresses.md @@ -0,0 +1,28 @@ +<a id="rend-spec-v3.txt-6"></a> + +# Encoding onion addresses \[ONIONADDRESS\] + +The onion address of a hidden service includes its identity public key, a +version field and a basic checksum. All this information is then base32 +encoded as shown below: + +```text + onion_address = base32(PUBKEY | CHECKSUM | VERSION) + ".onion" + CHECKSUM = H(".onion checksum" | PUBKEY | VERSION)[:2] + + where: + - PUBKEY is the 32 bytes ed25519 master pubkey of the hidden service. + - VERSION is a one byte version field (default value '\x03') + - ".onion checksum" is a constant string + - CHECKSUM is truncated to two bytes before inserting it in onion_address + + Here are a few example addresses: + + pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd.onion + sp3k262uwy4r2k3ycr5awluarykdpag6a7y33jxop4cs2lu5uz5sseqd.onion + xa4r2iadxm55fbnqgwwi5mymqdcofiu3w6rpbtqn7b2dyn7mgwj64jyd.onion +``` + +> For historical notes and rationales about this encoding, +> see [this discussion thread](https://lists.torproject.org/pipermail/tor-dev/2017-January/011816.html). + diff --git a/spec/rend-spec/encrypting-user-data.md b/spec/rend-spec/encrypting-user-data.md new file mode 100644 index 0000000..460f71e --- /dev/null +++ b/spec/rend-spec/encrypting-user-data.md @@ -0,0 +1,12 @@ +<a id="rend-spec-v3.txt-5"></a> + +# Encrypting data between client and host + +A successfully completed handshake, as embedded in the +INTRODUCE/RENDEZVOUS messages, gives the client and hidden service host +a shared set of keys Kf, Kb, Df, Db, which they use for sending +end-to-end traffic encryption and authentication as in the regular +Tor relay encryption protocol, applying encryption with these keys +before other encryption, and decrypting with these keys before other +decryption. The client encrypts with Kf and decrypts with Kb; the +service host does the opposite. diff --git a/spec/rend-spec/fs-contents.md b/spec/rend-spec/fs-contents.md new file mode 100644 index 0000000..f5c06f4 --- /dev/null +++ b/spec/rend-spec/fs-contents.md @@ -0,0 +1,30 @@ +<a id="rend-spec-v3.txt-F"></a> + +# Appendix F: Hidden service directory format \[HIDSERVDIR-FORMAT\] + +This appendix section specifies the contents of the HiddenServiceDir directory: + +- "hostname" \[FILE\] + +This file contains the onion address of the onion service. + +- "private_key_ed25519" \[FILE\] + +This file contains the private master ed25519 key of the onion service. +\[TODO: Offline keys\] + +```text + - "./authorized_clients/" [DIRECTORY] + "./authorized_clients/alice.auth" [FILE] + "./authorized_clients/bob.auth" [FILE] + "./authorized_clients/charlie.auth" [FILE] +``` + +If client authorization is enabled, this directory MUST contain a ".auth" +file for each authorized client. Each such file contains the public key of +the respective client. The files are transmitted to the service operator by +the client. + +See section \[CLIENT-AUTH-MGMT\] for more details and the format of the client file. + +(NOTE: client authorization is implemented as of 0.3.5.1-alpha.) diff --git a/spec/rend-spec/hsdesc-encrypt.md b/spec/rend-spec/hsdesc-encrypt.md new file mode 100644 index 0000000..3e91172 --- /dev/null +++ b/spec/rend-spec/hsdesc-encrypt.md @@ -0,0 +1,517 @@ +<a id="rend-spec-v3.txt-2.5"></a> + +# Hidden service descriptors: encryption format {#HS-DESC-ENC} + +Hidden service descriptors are protected by two layers of encryption. +Clients need to decrypt both layers to connect to the hidden service. + +The first layer of encryption provides confidentiality against entities who +don't know the public key of the hidden service (e.g. HSDirs), while the +second layer of encryption is only useful when client authorization is enabled +and protects against entities that do not possess valid client credentials. + +<a id="rend-spec-v3.txt-2.5.1"></a> + +## First layer of encryption {#HS-DESC-FIRST-LAYER} + +The first layer of HS descriptor encryption is designed to protect +descriptor confidentiality against entities who don't know the public +identity key of the hidden service. + +<a id="rend-spec-v3.txt-2.5.1.1"></a> + +### First layer encryption logic {#first-layer-logic} + +The encryption keys and format for the first layer of encryption are +generated as specified in \[HS-DESC-ENCRYPTION-KEYS\] with customization +parameters: + +```text + SECRET_DATA = blinded-public-key + STRING_CONSTANT = "hsdir-superencrypted-data" +``` + +The encryption scheme in \[HS-DESC-ENCRYPTION-KEYS\] uses the service +credential which is derived from the public identity key (see \[SUBCRED\]) to +ensure that only entities who know the public identity key can decrypt the +first descriptor layer. + +The ciphertext is placed on the "superencrypted" field of the descriptor. + +Before encryption the plaintext is padded with NUL bytes to the nearest +multiple of 10k bytes. + +<a id="rend-spec-v3.txt-2.5.1.2"></a> + +### First layer plaintext format {#first-layer-plaintext} + +After clients decrypt the first layer of encryption, they need to parse the +plaintext to get to the second layer ciphertext which is contained in the +"encrypted" field. + +If client auth is enabled, the hidden service generates a fresh +descriptor_cookie key (`N_hs_desc_enc`, 32 random bytes) and encrypts +it using each authorized client's identity x25519 key. Authorized +clients can use the descriptor cookie (`N_hs_desc_enc`) to decrypt +the second (inner) layer of encryption. Our encryption scheme +requires the hidden service to also generate an ephemeral x25519 +keypair for each new descriptor. + +If client auth is disabled, fake data is placed in each of the fields below +to obfuscate whether client authorization is enabled. + +Here are all the supported fields: + +"desc-auth-type" SP type NL + +\[Exactly once\] + +```text + This field contains the type of authorization used to protect the + descriptor. The only recognized type is "x25519" and specifies the + encryption scheme described in this section. + + If client authorization is disabled, the value here should be "x25519". + + "desc-auth-ephemeral-key" SP KP_hs_desc_ephem NL + + [Exactly once] + + This field contains `KP_hss_desc_enc`, an ephemeral x25519 public + key generated by the hidden service and encoded in base64. The key + is used by the encryption scheme below. + + If client authorization is disabled, the value here should be a fresh + x25519 pubkey that will remain unused. + + "auth-client" SP client-id SP iv SP encrypted-cookie + + [At least once] + + When client authorization is enabled, the hidden service inserts an + "auth-client" line for each of its authorized clients. If client + authorization is disabled, the fields here can be populated with random + data of the right size (that's 8 bytes for 'client-id', 16 bytes for 'iv' + and 16 bytes for 'encrypted-cookie' all encoded with base64). + + When client authorization is enabled, each "auth-client" line + contains the descriptor cookie `N_hs_desc_enc` encrypted to each + individual client. We assume that each authorized client possesses + a pre-shared x25519 keypair (`KP_hsc_desc_enc`) which is used to + decrypt the descriptor cookie. + + We now describe the descriptor cookie encryption scheme. Here is what + the hidden service computes: + + SECRET_SEED = x25519(KS_hs_desc_ephem, KP_hsc_desc_enc) + KEYS = KDF(N_hs_subcred | SECRET_SEED, 40) + CLIENT-ID = fist 8 bytes of KEYS + COOKIE-KEY = last 32 bytes of KEYS + + Here is a description of the fields in the "auth-client" line: + + - The "client-id" field is CLIENT-ID from above encoded in base64. + + - The "iv" field is 16 random bytes encoded in base64. + + - The "encrypted-cookie" field contains the descriptor cookie ciphertext + as follows and is encoded in base64: + encrypted-cookie = STREAM(iv, COOKIE-KEY) XOR N_hs_desc_enc. + + See section [FIRST-LAYER-CLIENT-BEHAVIOR] for the client-side logic of + how to decrypt the descriptor cookie. + + "encrypted" NL encrypted-string + + [Exactly once] + + An encrypted blob containing the second layer ciphertext, whose format is + discussed in [HS-DESC-SECOND-LAYER] below. The blob is base64 encoded + and enclosed in -----BEGIN MESSAGE---- and ----END MESSAGE---- wrappers. + + Compatibility note: The C Tor implementation does not include a final + newline when generating this first-layer-plaintext section; other + implementations MUST accept this section even if it is missing its final + newline. Other implementations MAY generate this section without a final + newline themselves, to avoid being distinguishable from C tor. +``` + +<a id="rend-spec-v3.txt-2.5.1.3"></a> + +### Client behavior {#FIRST-LAYER-CLIENT-BEHAVIOR} + +```text + The goal of clients at this stage is to decrypt the "encrypted" field as + described in [HS-DESC-SECOND-LAYER]. + + If client authorization is enabled, authorized clients need to extract the + descriptor cookie to proceed with decryption of the second layer as + follows: + + An authorized client parsing the first layer of an encrypted descriptor, + extracts the ephemeral key from "desc-auth-ephemeral-key" and calculates + CLIENT-ID and COOKIE-KEY as described in the section above using their + x25519 private key. The client then uses CLIENT-ID to find the right + "auth-client" field which contains the ciphertext of the descriptor + cookie. The client then uses COOKIE-KEY and the iv to decrypt the + descriptor_cookie, which is used to decrypt the second layer of descriptor + encryption as described in [HS-DESC-SECOND-LAYER]. +``` + +<a id="rend-spec-v3.txt-2.5.1.4"></a> + +### Hiding client authorization data {#hiding-client-auth} + +```text + Hidden services should avoid leaking whether client authorization is + enabled or how many authorized clients there are. + + Hence even when client authorization is disabled, the hidden service adds + fake "desc-auth-type", "desc-auth-ephemeral-key" and "auth-client" lines to + the descriptor, as described in [HS-DESC-FIRST-LAYER]. + + The hidden service also avoids leaking the number of authorized clients by + adding fake "auth-client" entries to its descriptor. Specifically, + descriptors always contain a number of authorized clients that is a + multiple of 16 by adding fake "auth-client" entries if needed. + [XXX consider randomization of the value 16] + + Clients MUST accept descriptors with any number of "auth-client" lines as + long as the total descriptor size is within the max limit of 50k (also + controlled with a consensus parameter). +``` + +<a id="rend-spec-v3.txt-2.5.2"></a> + +## Second layer of encryption {#HS-DESC-SECOND-LAYER} + +The second layer of descriptor encryption is designed to protect descriptor +confidentiality against unauthorized clients. If client authorization is +enabled, it's encrypted using the descriptor_cookie, and contains needed +information for connecting to the hidden service, like the list of its +introduction points. + +If client authorization is disabled, then the second layer of HS encryption +does not offer any additional security, but is still used. + +<a id="rend-spec-v3.txt-2.5.2.1"></a> + +### Second layer encryption keys {#second-layer-keys} + +The encryption keys and format for the second layer of encryption are +generated as specified in \[HS-DESC-ENCRYPTION-KEYS\] with customization +parameters as follows: + +```text + SECRET_DATA = blinded-public-key | descriptor_cookie + STRING_CONSTANT = "hsdir-encrypted-data" + + If client authorization is disabled the 'descriptor_cookie' field is left blank. + + The ciphertext is placed on the "encrypted" field of the descriptor. +``` + +<a id="rend-spec-v3.txt-2.5.2.2"></a> + +### Second layer plaintext format {#second-layer-plaintext} + +After decrypting the second layer ciphertext, clients can finally learn the +list of intro points etc. The plaintext has the following format: + +```text +"create2-formats" SP formats NL + +\[Exactly once\] + + A space-separated list of integers denoting CREATE2 cell HTYPEs + (handshake types) that the server recognizes. Must include at least + ntor as described in tor-spec.txt. See tor-spec section 5.1 for a list + of recognized handshake types. +``` + +```text + "intro-auth-required" SP types NL + + [At most once] + + A space-separated list of introduction-layer authentication types; see + section [INTRO-AUTH] for more info. A client that does not support at + least one of these authentication types will not be able to contact the + host. Recognized types are: 'ed25519'. +``` + +```text + "single-onion-service" + + [At most once] + + If present, this line indicates that the service is a Single Onion + Service (see prop260 for more details about that type of service). This + field has been introduced in 0.3.0 meaning 0.2.9 service don't include + this. +``` + +```text + "pow-params" SP type SP seed-b64 SP suggested-effort + SP expiration-time NL + + [At most once per "type"] + + If present, this line provides parameters for an optional proof-of-work + client puzzle. A client that supports an offered scheme can include a + corresponding solution in its introduction request to improve priority + in the service's processing queue. + + Only version 1 is currently defined. + Other versions may have a different format. + Introduced in tor-0.4.8.1-alpha. + + type: The type of PoW system used. We call the one specified here "v1". + + seed-b64: A random seed that should be used as the input to the PoW + hash function. Should be 32 random bytes encoded in base64 + without trailing padding. + + suggested-effort: An unsigned integer specifying an effort value that + clients should aim for when contacting the service. Can be + zero to mean that PoW is available but not currently + suggested for a first connection attempt. + + expiration-time: A timestamp in "YYYY-MM-DDTHH:MM:SS" format (iso time + with no space) after which the above seed expires and + is no longer valid as the input for PoW. +``` + +Followed by zero or more introduction points as follows (see section +\[NUM_INTRO_POINT\] below for accepted values): + +```text + "introduction-point" SP link-specifiers NL + + [Exactly once per introduction point at start of introduction + point section] + + The link-specifiers is a base64 encoding of a link specifier + block in the format described in [BUILDING-BLOCKS] above. + + As of 0.4.1.1-alpha, services include both IPv4 and IPv6 link + specifiers in descriptors. All available addresses SHOULD be + included in the descriptor, regardless of the address that the + onion service actually used to connect/extend to the intro + point. + + The client SHOULD NOT reject any LSTYPE fields which it doesn't + recognize; instead, it should use them verbatim in its EXTEND + request to the introduction point. + + The client SHOULD perform the basic validity checks on the link + specifiers in the descriptor, described in `tor-spec.txt` + section 5.1.2. These checks SHOULD NOT leak + detailed information about the client's version, configuration, + or consensus. (See 3.3 for service link specifier handling.) + + When connecting to the introduction point, the client SHOULD send + this list of link specifiers verbatim, in the same order as given + here. + + The client MAY reject the list of link specifiers if it is + inconsistent with relay information from the directory, but SHOULD + NOT modify it. +``` + +```text + "onion-key" SP "ntor" SP key NL + + [Exactly once per introduction point] + + The key is a base64 encoded curve25519 public key which is the onion + key of the introduction point Tor node used for the ntor handshake + when a client extends to it. +``` + +```text + "onion-key" SP KeyType SP key.. NL + + [Any number of times] + + Implementations should accept other types of onion keys using this + syntax (where "KeyType" is some string other than "ntor"); + unrecognized key types should be ignored. +``` + +<a id="auth-key"></a> +```text + "auth-key" NL certificate NL + + [Exactly once per introduction point] + + The certificate is a proposal 220 certificate wrapped in + "-----BEGIN ED25519 CERT-----". It contains the introduction + point authentication key (`KP_hs_ipt_sid`), signed by + the descriptor signing key (`KP_hs_desc_sign`). The + certificate type must be [09], and the signing key extension + is mandatory. + + NOTE: This certificate was originally intended to be + constructed the other way around: the signing and signed keys + are meant to be reversed. However, C tor implemented it + backwards, and other implementations now need to do the same + in order to conform. (Since this section is inside the + descriptor, which is _already_ signed by `KP_hs_desc_sign`, + the verification aspect of this certificate serves no point in + its current form.) +``` + +```text + "enc-key" SP "ntor" SP key NL + + [Exactly once per introduction point] + + The key is a base64 encoded curve25519 public key used to encrypt + the introduction request to service. (`KP_hss_ntor`) +``` + +```text + "enc-key" SP KeyType SP key.. NL + + [Any number of times] + + Implementations should accept other types of onion keys using this + syntax (where "KeyType" is some string other than "ntor"); + unrecognized key types should be ignored. +``` + +<a id="enc-key-cert"></a> +```text + "enc-key-cert" NL certificate NL + + [Exactly once per introduction point] + + Cross-certification of the encryption key using the descriptor + signing key. + + For "ntor" keys, certificate is a proposal 220 certificate + wrapped in "-----BEGIN ED25519 CERT-----" armor. + + The subject + key is the the ed25519 equivalent of a curve25519 public + encryption key (`KP_hss_ntor`), with the ed25519 key + derived using the process in proposal 228 appendix A, + and its sign bit set to zero. + + The + signing key is the descriptor signing key (`KP_hs_desc_sign`). + The certificate type must be [0B], and the signing-key + extension is mandatory. + + NOTE: As with "auth-key", this certificate was intended to be + constructed the other way around. However, for compatibility + with C tor, implementations need to construct it this way. It + serves even less point than "auth-key", however, since the + encryption key `KP_hss_ntor` is already available from + the `enc-key` entry. + + ALSO NOTE: Setting the sign bit of the subject key + to zero makes the subjected unusable for verification; + this is also a mistake preserved for compatiblility with + C tor. + + "legacy-key" NL key NL + + [None or at most once per introduction point] + [This field is obsolete and should never be generated; it + is included for historical reasons only.] + + The key is an ASN.1 encoded RSA public key in PEM format used for a + legacy introduction point as described in [LEGACY_EST_INTRO]. + + This field is only present if the introduction point only supports + legacy protocol (v2) that is <= 0.2.9 or the protocol version value + "HSIntro 3". + + "legacy-key-cert" NL certificate NL + + [None or at most once per introduction point] + [This field is obsolete and should never be generated; it + is included for historical reasons only.] + + MUST be present if "legacy-key" is present. + + The certificate is a proposal 220 RSA->Ed cross-certificate wrapped + in "-----BEGIN CROSSCERT-----" armor, cross-certifying the RSA + public key found in "legacy-key" using the descriptor signing key. +``` + +To remain compatible with future revisions to the descriptor format, +clients should ignore unrecognized lines in the descriptor. +Other encryption and authentication key formats are allowed; clients +should ignore ones they do not recognize. + +Clients who manage to extract the introduction points of the hidden service +can proceed with the introduction protocol as specified in \[INTRO-PROTOCOL\]. + +Compatibility note: At least some versions of OnionBalance do not include +a final newline when generating this inner plaintext section; other +implementations MUST accept this section even if it is missing its final +newline. + +<a id="rend-spec-v3.txt-2.5.3"></a> + +## Deriving hidden service descriptor encryption keys {#HS-DESC-ENCRYPTION-KEYS} + +In this section we present the generic encryption format for hidden service +descriptors. We use the same encryption format in both encryption layers, +hence we introduce two customization parameters SECRET_DATA and +STRING_CONSTANT which vary between the layers. + +The SECRET_DATA parameter specifies the secret data that are used during +encryption key generation, while STRING_CONSTANT is merely a string constant +that is used as part of the KDF. + +Here is the key generation logic: + +```text + SALT = 16 bytes from H(random), changes each time we rebuild the + descriptor even if the content of the descriptor hasn't changed. + (So that we don't leak whether the intro point list etc. changed) + + secret_input = SECRET_DATA | N_hs_subcred | INT_8(revision_counter) + + keys = KDF(secret_input | salt | STRING_CONSTANT, S_KEY_LEN + S_IV_LEN + MAC_KEY_LEN) + + SECRET_KEY = first S_KEY_LEN bytes of keys + SECRET_IV = next S_IV_LEN bytes of keys + MAC_KEY = last MAC_KEY_LEN bytes of keys + + The encrypted data has the format: + + SALT hashed random bytes from above [16 bytes] + ENCRYPTED The ciphertext [variable] + MAC D_MAC of both above fields [32 bytes] + + The final encryption format is ENCRYPTED = STREAM(SECRET_IV,SECRET_KEY) XOR Plaintext . + + Where D_MAC = H(mac_key_len | MAC_KEY | salt_len | SALT | ENCRYPTED) + and + mac_key_len = htonll(len(MAC_KEY)) + and + salt_len = htonll(len(SALT)). +``` + +<a id="rend-spec-v3.txt-2.5.4"></a> + +## Number of introduction points {#NUM_INTRO_POINT} + +This section defines how many introduction points an hidden service +descriptor can have at minimum, by default and the maximum: + +Minimum: 0 - Default: 3 - Maximum: 20 + +A value of 0 would means that the service is still alive but doesn't want +to be reached by any client at the moment. Note that the descriptor size +increases considerably as more introduction points are added. + +The reason for a maximum value of 20 is to give enough scalability to tools +like OnionBalance to be able to load balance up to 120 servers (20 x 6 +HSDirs) but also in order for the descriptor size to not overwhelmed hidden +service directories with user defined values that could be gigantic. diff --git a/spec/rend-spec/hsdesc-outer.md b/spec/rend-spec/hsdesc-outer.md new file mode 100644 index 0000000..f67fc6c --- /dev/null +++ b/spec/rend-spec/hsdesc-outer.md @@ -0,0 +1,83 @@ +<a id="rend-spec-v3.txt-2.4"></a> + +# Hidden service descriptors: outer wrapper \[DESC-OUTER\] + +The format for a hidden service descriptor is as follows, using the +meta-format from dir-spec.txt. + +"hs-descriptor" SP version-number NL + +\[At start, exactly once.\] + +```text + The version-number is a 32 bit unsigned integer indicating the version + of the descriptor. Current version is "3". + + "descriptor-lifetime" SP LifetimeMinutes NL + + [Exactly once] + + The lifetime of a descriptor in minutes. An HSDir SHOULD expire the + hidden service descriptor at least LifetimeMinutes after it was + uploaded. + + The LifetimeMinutes field can take values between 30 and 720 (12 + hours). +``` + +<a id="descriptor-signing-key-cert"></a> +```text + "descriptor-signing-key-cert" NL certificate NL + + [Exactly once.] + + The 'certificate' field contains a certificate in the format from + proposal 220, wrapped with "-----BEGIN ED25519 CERT-----". The + certificate cross-certifies the short-term descriptor signing key with + the blinded public key. The certificate type must be [08], and the + blinded public key must be present as the signing-key extension. +``` + +```text + "revision-counter" SP Integer NL + + [Exactly once.] + + The revision number of the descriptor. If an HSDir receives a + second descriptor for a key that it already has a descriptor for, + it should retain and serve the descriptor with the higher + revision-counter. + + (Checking for monotonically increasing revision-counter values + prevents an attacker from replacing a newer descriptor signed by + a given key with a copy of an older version.) + + Implementations MUST be able to parse 64-bit values for these + counters. +``` + +```text + "superencrypted" NL encrypted-string + + [Exactly once.] + + An encrypted blob, whose format is discussed in [HS-DESC-ENC] below. The + blob is base64 encoded and enclosed in -----BEGIN MESSAGE---- and + ----END MESSAGE---- wrappers. (The resulting document does not end with + a newline character.) +``` + +```text + "signature" SP signature NL + + [exactly once, at end.] + + A signature of all previous fields, using the signing key in the + descriptor-signing-key-cert line, prefixed by the string "Tor onion + service descriptor sig v3". We use a separate key for signing, so that + the hidden service host does not need to have its private blinded key + online. +``` + +HSDirs accept hidden service descriptors of up to 50k bytes (a consensus +parameter should also be introduced to control this value). diff --git a/spec/rend-spec/hsdesc.md b/spec/rend-spec/hsdesc.md new file mode 100644 index 0000000..8b1d75a --- /dev/null +++ b/spec/rend-spec/hsdesc.md @@ -0,0 +1,8 @@ +<a id="rend-spec-v3.txt-2"></a> + +# Generating and publishing hidden service descriptors \[HSDIR\] + +Hidden service descriptors follow the same metaformat as other Tor +directory objects. They are published anonymously to Tor servers with the +HSDir flag, HSDir=2 protocol version and tor version >= 0.3.0.8 (because a +bug was fixed in this version). diff --git a/spec/rend-spec/index.md b/spec/rend-spec/index.md new file mode 100644 index 0000000..47e65aa --- /dev/null +++ b/spec/rend-spec/index.md @@ -0,0 +1,9 @@ +# Tor Rendezvous Specification - Version 3 + +This document specifies how the hidden service version 3 protocol works. This +text used to be proposal 224-rend-spec-ng.txt. + +This document describes a proposed design and specification for +hidden services in Tor version 0.2.5.x or later. It's a replacement +for the current rend-spec.txt, rewritten for clarity and for improved +design. diff --git a/spec/rend-spec/introduction-protocol.md b/spec/rend-spec/introduction-protocol.md new file mode 100644 index 0000000..cb7debf --- /dev/null +++ b/spec/rend-spec/introduction-protocol.md @@ -0,0 +1,778 @@ +<a id="rend-spec-v3.txt-3"></a> + +# The introduction protocol {#INTRO-PROTOCOL} + +The introduction protocol proceeds in three steps. + +First, a hidden service host builds an anonymous circuit to a Tor +node and registers that circuit as an introduction point. + +Single Onion Services attempt to build a non-anonymous single-hop circuit, +but use an anonymous 3-hop circuit if: + +```text + * the intro point is on an address that is configured as unreachable via + a direct connection, or + * the initial attempt to connect to the intro point over a single-hop + circuit fails, and they are retrying the intro point connection. + + [After 'First' and before 'Second', the hidden service publishes its + introduction points and associated keys, and the client fetches + them as described in section [HSDIR] above.] +``` + +Second, a client builds an anonymous circuit to the introduction +point, and sends an introduction request. + +Third, the introduction point relays the introduction request along +the introduction circuit to the hidden service host, and acknowledges +the introduction request to the client. + +<a id="rend-spec-v3.txt-3.1"></a> + +## Registering an introduction point {#REG_INTRO_POINT} + +<a id="rend-spec-v3.txt-3.1.1"></a> + +### Extensible ESTABLISH_INTRO protocol {#EST_INTRO} + +When a hidden service is establishing a new introduction point, it +sends an ESTABLISH_INTRO message with the following contents: + +```text + AUTH_KEY_TYPE [1 byte] + AUTH_KEY_LEN [2 bytes] + AUTH_KEY [AUTH_KEY_LEN bytes] + N_EXTENSIONS [1 byte] + N_EXTENSIONS times: + EXT_FIELD_TYPE [1 byte] + EXT_FIELD_LEN [1 byte] + EXT_FIELD [EXT_FIELD_LEN bytes] + HANDSHAKE_AUTH [MAC_LEN bytes] + SIG_LEN [2 bytes] + SIG [SIG_LEN bytes] +``` + +The AUTH_KEY_TYPE field indicates the type of the introduction point +authentication key and the type of the MAC to use in +HANDSHAKE_AUTH. Recognized types are: + +```text + [00, 01] -- Reserved for legacy introduction messages; see + [LEGACY_EST_INTRO below] + [02] -- Ed25519; SHA3-256. +``` + +The AUTH_KEY_LEN field determines the length of the AUTH_KEY +field. The AUTH_KEY field contains the public introduction point +authentication key, KP_hs_ipt_sid. + +The EXT_FIELD_TYPE, EXT_FIELD_LEN, EXT_FIELD entries are reserved for +extensions to the introduction protocol. Extensions with +unrecognized EXT_FIELD_TYPE values must be ignored. +(`EXT_FIELD_LEN` may be zero, in which case EXT_FIELD is absent.) + +```text + Unless otherwise specified in the documentation for an extension type: + * Each extension type SHOULD be sent only once in a message. + * Parties MUST ignore any occurrences all occurrences of an extension + with a given type after the first such occurrence. + * Extensions SHOULD be sent in numerically ascending order by type. + (The above extension sorting and multiplicity rules are only defaults; + they may be overridden in the descriptions of individual extensions.) +``` + +The following extensions are currently defined: + +| `EXT_FIELD_TYPE` | Name | +| ---------------- | -------------- | +| `[01]` | [`DOS_PARAMS`] | + +[`DOS_PARAMS`]: #DOS_PARAMS + +The HANDSHAKE_AUTH field contains the MAC of all earlier fields in +the message using as its key the shared per-circuit material ("KH") +generated during the circuit extension protocol; see tor-spec.txt +section 5.2, "Setting circuit keys". It prevents replays of +ESTABLISH_INTRO messages. + +SIG_LEN is the length of the signature. + +SIG is a signature, using AUTH_KEY, of all contents of the message, up +to but not including SIG_LEN and SIG. These contents are prefixed +with the string "Tor establish-intro cell v1". + +> (Note that this string is _sic_; +> it predates our efforts to distinguish cells from relay messages.) + +Upon receiving an ESTABLISH_INTRO message, a Tor node first decodes the +key and the signature, and checks the signature. The node must reject +the ESTABLISH_INTRO message and destroy the circuit in these cases: + +```text + * If the key type is unrecognized + * If the key is ill-formatted + * If the signature is incorrect + * If the HANDSHAKE_AUTH value is incorrect + + * If the circuit is already a rendezvous circuit. + * If the circuit is already an introduction circuit. + [TODO: some scalability designs fail there.] + * If the key is already in use by another circuit. +``` + +Otherwise, the node must associate the key with the circuit, for use +later in INTRODUCE1 messages. + + + +<a id="rend-spec-v3.txt-3.1.1.1"></a> + +#### Denial-of-Service defense extension (DOS\_PARAMS) {#EST_INTRO_DOS_EXT} + +<a id="DOS_PARAMS"></a> +The `DOS_PARAMS` extension +in ESTABLISH_INTRO +is used to send Denial-of-Service (DoS) parameters to +the introduction point in order for it to apply them for the introduction +circuit. + +This is for the [rate limiting DoS mitigation](../dos-spec/overview.md#hs-intro-rate) specifically. + +The `EXT_FIELD_TYPE` value for the `DOS_PARAMS` extension is `[01]`. + +The content is defined as follows: + +| Field | Size | Description | +| ----------------- | ---- | -------------------- | +| `N_PARAMS` | 1 | Number of parameters | +| `N_PARAMS` times: | | | +| - PARAM_TYPE | 1 | Identifier for a parameter | +| - PARAM_VALUE | 8 | Integer value | + +Recognized values for `PARAM_TYPE` in this extension are: + +| `PARAM_TYPE` | Name | Min | Max | +| ----------- | -------------------------------- | --- | ---------- | +| `[01]` | [`DOS_INTRODUCE2_RATE_PER_SEC`] | 0 | 0x7fffffff | +| `[02]` | [`DOS_INTRODUCE2_BURST_PER_SEC`] | 0 | 0x7fffffff | + +[`DOS_INTRODUCE2_RATE_PER_SEC`]: #DOS_INTRODUCE2_RATE_PER_SEC +[`DOS_INTRODUCE2_BURST_PER_SEC`]: #DOS_INTRODUCE2_BURST_PER_SEC + +Together, these parameters configure a token bucket +that determines how many INTRODUCE2 messages +the introduction point may send to the service. + +<span id="DOS_INTRODUCE2_RATE_PER_SEC"> +The `DOS_INTRODUCE2_RATE_PER_SEC` parameter defines the maximum +average rate of messages; +</span> +<span id="DOS_INTRODUCE2_BURST_PER_SEC"> +The `DOS_INTRODUCE2_BURST_PER_SEC` parameter defines the largest +allowable burst of messages +(that is, the size of the token bucket). +</span> + +> Technically speaking, the `BURST` parameter is misnamed +> in that it is not actually "per second": +> only a _rate_ has an associated time. + +If either of these parameters is set to 0, +the defense is disabled, +and the introduction point should ignore the other parameter. + +If the burst is lower than the rate, +the introduction point SHOULD ignore the extension. + +> Using this extension extends the payload of the ESTABLISH_INTRO message by 19 +> bytes bringing it from 134 bytes to 155 bytes. + +When this extension is not _sent_, +introduction points use default settings +taken from taken from the consensus parameters +[HiddenServiceEnableIntroDoSDefense](../param-spec.md#HiddenServiceEnableIntroDoSDefense), +[HiddenServiceEnableIntroDoSRatePerSec](../param-spec.md#HiddenServiceEnableIntroDoSRatePerSec), +and +[HiddenServiceEnableIntroDoSBurstPerSec](../param-spec.md#HiddenServiceEnableIntroDoSBurstPerSec). + +This extension can only be used with relays supporting the protocol version +["HSIntro=5"](../tor-spec/subprotocol-versioning.md#HSIntro). + +Introduced in tor-0.4.2.1-alpha. + +<a id="rend-spec-v3.txt-3.1.2"></a> + +## Registering an introduction point on a legacy Tor node {#LEGACY_EST_INTRO} + +> This section is obsolete and refers to a workaround for now-obsolete Tor +> relay versions. It is included for historical reasons. + +Tor nodes should also support an older version of the ESTABLISH_INTRO +message, first documented in rend-spec.txt. New hidden service hosts +must use this format when establishing introduction points at older +Tor nodes that do not support the format above in \[EST_INTRO\]. + +In this older protocol, an ESTABLISH_INTRO message contains: + +```text + KEY_LEN [2 bytes] + KEY [KEY_LEN bytes] + HANDSHAKE_AUTH [20 bytes] + SIG [variable, up to end of relay payload] + + The KEY_LEN variable determines the length of the KEY field. +``` + +The KEY field is the ASN1-encoded legacy RSA public key that was also +included in the hidden service descriptor. + +The HANDSHAKE_AUTH field contains the SHA1 digest of (KH | "INTRODUCE"). + +The SIG field contains an RSA signature, using PKCS1 padding, of all +earlier fields. + +Older versions of Tor always use a 1024-bit RSA key for these introduction +authentication keys. + +<a id="rend-spec-v3.txt-3.1.3"></a> + +### Acknowledging establishment of introduction point {#INTRO_ESTABLISHED} + +After setting up an introduction circuit, the introduction point reports its +status back to the hidden service host with an INTRO_ESTABLISHED message. + +The INTRO_ESTABLISHED message has the following contents: + +```text + N_EXTENSIONS [1 byte] + N_EXTENSIONS times: + EXT_FIELD_TYPE [1 byte] + EXT_FIELD_LEN [1 byte] + EXT_FIELD [EXT_FIELD_LEN bytes] +``` + +Older versions of Tor send back an empty INTRO_ESTABLISHED message instead. +Services must accept an empty INTRO_ESTABLISHED message from a legacy relay. +\[The above paragraph is obsolete and refers to a workaround for +now-obsolete Tor relay versions. It is included for historical reasons.\] + +The same rules for multiplicity, ordering, and handling unknown types +apply to the extension fields here as described \[EST_INTRO\] above. + +<a id="rend-spec-v3.txt-3.2"></a> + +## Sending an INTRODUCE1 message to the introduction point {#SEND_INTRO1} + +In order to participate in the introduction protocol, a client must +know the following: + +```text + * An introduction point for a service. + * The introduction authentication key for that introduction point. + * The introduction encryption key for that introduction point. +``` + +The client sends an INTRODUCE1 message to the introduction point, +containing an identifier for the service, an identifier for the +encryption key that the client intends to use, and an opaque blob to +be relayed to the hidden service host. + +In reply, the introduction point sends an INTRODUCE_ACK message back to +the client, either informing it that its request has been delivered, +or that its request will not succeed. + +If the INTRODUCE_ACK message indicates success, +the client SHOULD close the circuit to the introduction point, +and not use it for anything else. +If the INTRODUCE_ACK message indicates failure, +the client MAY try a different introduction point. +It MAY reach the different introduction point +either by extending its introduction circuit an additional hop, +or by building a new introduction circuit. + +```text + [TODO: specify what tor should do when receiving a malformed message. Drop it? + Kill circuit? This goes for all possible messages.] +``` + +<a id="rend-spec-v3.txt-3.2.1"></a> + +### Extensible INTRODUCE1 message format {#FMT_INTRO1} + +When a client is connecting to an introduction point, INTRODUCE1 messages +should be of the form: + +```text + LEGACY_KEY_ID [20 bytes] + AUTH_KEY_TYPE [1 byte] + AUTH_KEY_LEN [2 bytes] + AUTH_KEY [AUTH_KEY_LEN bytes] + N_EXTENSIONS [1 byte] + N_EXTENSIONS times: + EXT_FIELD_TYPE [1 byte] + EXT_FIELD_LEN [1 byte] + EXT_FIELD [EXT_FIELD_LEN bytes] + ENCRYPTED [Up to end of relay payload] +``` + +The `ENCRYPTED` field is described in the \[PROCESS_INTRO2\] section. + +AUTH_KEY_TYPE is defined as in \[EST_INTRO\]. Currently, the only value of +AUTH_KEY_TYPE for this message is an Ed25519 public key \[02\]. + +The LEGACY_KEY_ID field is used to distinguish between legacy and new style +INTRODUCE1 messages. In new style INTRODUCE1 messages, LEGACY_KEY_ID is 20 zero +bytes. Upon receiving an INTRODUCE1 messages, the introduction point checks the +LEGACY_KEY_ID field. If LEGACY_KEY_ID is non-zero, the INTRODUCE1 message +should be handled as a legacy INTRODUCE1 message by the intro point. + +Upon receiving a INTRODUCE1 message, the introduction point checks +whether AUTH_KEY matches the introduction point authentication key for an +active introduction circuit. If so, the introduction point sends an +INTRODUCE2 message with exactly the same contents to the service, and sends an +INTRODUCE_ACK response to the client. + +(Note that the introduction point does not "clean up" the +INTRODUCE1 message that it retransmits. Specifically, it does not +change the order or multiplicity of the extensions sent by the +client.) + +The same rules for multiplicity, ordering, and handling unknown types +apply to the extension fields here as described \[EST_INTRO\] above. + +#### Proof-of-work extension to INTRODUCE1 {#INTRO1_POW_EXT} + +This extension can be used to optionally attach a proof of work to the introduction request. +The proof must be calculated using unique parameters appropriate for this specific service. +An acceptable proof will raise the priority of this introduction request according to the proof's verified computational effort. + +This is for the [proof-of-work DoS mitigation](../dos-spec/overview.md#hs-intro-pow), described in depth by the [Proof of Work for onion service introduction](../hspow-spec/index.md) specification. + +If used, it needs to be encoded within the N_EXTENSIONS field of the +ESTABLISH_INTRO message defined in the previous section. The content is +defined as follows: + +EXT_FIELD_TYPE: + +\[02\] -- `PROOF_OF_WORK` + +```text +The EXT_FIELD content format is: + + POW_VERSION [1 byte] + POW_NONCE [16 bytes] + POW_EFFORT [4 bytes] + POW_SEED [4 bytes] + POW_SOLUTION [16 bytes] + +where: + +POW_VERSION is 1 for the protocol specified here +POW_NONCE is the nonce value chosen by the client's solver +POW_EFFORT is the effort value chosen by the client, + as a 32-bit integer in network byte order +POW_SEED identifies which seed was in use, by its first 4 bytes +POW_SOLUTION is a matching proof computed by the client's solver +``` + +Only version 1 is currently defined. +Other versions may have a different format. +A correctly functioning client only submits solutions with a version and seed which were advertised by the server and have not yet expired. +An extension with an unknown version or expired seed is suspicious and SHOULD result in introduction failure. + +This will increase the INTRODUCE1 payload size by 43 bytes since the extension type and length is 2 extra bytes, the N_EXTENSIONS field is always present and currently set to 0 and the EXT_FIELD is 41 bytes. +According to ticket #33650, INTRODUCE1 messages currently have more than 200 bytes available. + +Introduced in tor-0.4.8.1-alpha. + +<a id="rend-spec-v3.txt-3.2.2"></a> + +### INTRODUCE_ACK message format. {#INTRO_ACK} + +An INTRODUCE_ACK message has the following fields: + +```text + STATUS [2 bytes] + N_EXTENSIONS [1 bytes] + N_EXTENSIONS times: + EXT_FIELD_TYPE [1 byte] + EXT_FIELD_LEN [1 byte] + EXT_FIELD [EXT_FIELD_LEN bytes] + + Recognized status values are: + + [00 00] -- Success: message relayed to hidden service host. + [00 01] -- Failure: service ID not recognized + [00 02] -- Bad message format + [00 03] -- Can't relay message to service +``` + +The same rules for multiplicity, ordering, and handling unknown types +apply to the extension fields here as described \[EST_INTRO\] above. + +<a id="rend-spec-v3.txt-3.3"></a> + +## Processing an INTRODUCE2 message at the hidden service. {#PROCESS_INTRO2} + +Upon receiving an INTRODUCE2 message, the hidden service host checks whether +the AUTH_KEY or LEGACY_KEY_ID field matches the keys for this +introduction circuit. + +The service host then checks whether it has received a message with these +contents or rendezvous cookie before. If it has, it silently drops it as a +replay. (It must maintain a replay cache for as long as it accepts messages +with the same encryption key. Note that the encryption format below should +be non-malleable.) + +If the message is not a replay, it decrypts the ENCRYPTED field, +establishes a shared key with the client, and authenticates the whole +contents of the message as having been unmodified since they left the +client. There may be multiple ways of decrypting the ENCRYPTED field, +depending on the chosen type of the encryption key. Requirements for +an introduction handshake protocol are described in +\[INTRO-HANDSHAKE-REQS\]. We specify one below in section +\[NTOR-WITH-EXTRA-DATA\]. + +The decrypted plaintext must have the form: + +```text + RENDEZVOUS_COOKIE [20 bytes] + N_EXTENSIONS [1 byte] + N_EXTENSIONS times: + EXT_FIELD_TYPE [1 byte] + EXT_FIELD_LEN [1 byte] + EXT_FIELD [EXT_FIELD_LEN bytes] + ONION_KEY_TYPE [1 bytes] + ONION_KEY_LEN [2 bytes] + ONION_KEY [ONION_KEY_LEN bytes] + NSPEC (Number of link specifiers) [1 byte] + NSPEC times: + LSTYPE (Link specifier type) [1 byte] + LSLEN (Link specifier length) [1 byte] + LSPEC (Link specifier) [LSLEN bytes] + PAD (optional padding) [up to end of plaintext] +``` + +Upon processing this plaintext, the hidden service makes sure that +any required authentication is present in the extension fields, and +then extends a rendezvous circuit to the node described in the LSPEC +fields, using the ONION_KEY to complete the extension. As mentioned +in \[BUILDING-BLOCKS\], the "TLS-over-TCP, IPv4" and "Legacy node +identity" specifiers must be present. + +As of 0.4.1.1-alpha, clients include both IPv4 and IPv6 link specifiers +in INTRODUCE1 messages. All available addresses SHOULD be included in the +message, regardless of the address that the client actually used to extend +to the rendezvous point. + +The hidden service should handle invalid or unrecognised link specifiers +the same way as clients do in section 2.5.2.2. In particular, services +SHOULD perform basic validity checks on link specifiers, and SHOULD NOT +reject unrecognised link specifiers, to avoid information leaks. +The list of link specifiers received here SHOULD either be rejected, or +sent verbatim when extending to the rendezvous point, in the same order +received. + +The service MAY reject the list of link specifiers if it is +inconsistent with relay information from the directory, but SHOULD +NOT modify it. + +The ONION_KEY_TYPE field is: + +\[01\] NTOR: ONION_KEY is 32 bytes long. + +The ONION_KEY field describes the onion key that must be used when +extending to the rendezvous point. It must be of a type listed as +supported in the hidden service descriptor. + +The PAD field should be filled with zeros; its size should be chosen +so that the INTRODUCE2 message occupies a fixed maximum size, in +order to hide the length of the encrypted data. (This maximum size is +490, since we assume that a future Tor implementations will implement +proposal 340 and thus lower the number of bytes that can be contained +in a single relay message.) Note also that current versions of Tor +only pad the INTRODUCE2 message up to 246 bytes. + +Upon receiving a well-formed INTRODUCE2 message, the hidden service host +will have: + +```text + * The information needed to connect to the client's chosen + rendezvous point. + * The second half of a handshake to authenticate and establish a + shared key with the hidden service client. + * A set of shared keys to use for end-to-end encryption. +``` + +The same rules for multiplicity, ordering, and handling unknown types +apply to the extension fields here as described \[EST_INTRO\] above. + +### INTRODUCE1 Extensions + +The following sections details the currently supported or reserved extensions +of an `INTRODUCE1` message. + +#### Congestion Control + +This is used to request that the rendezvous circuit with the service be +configured with congestion control. + + EXT_FIELD_TYPE: + + \[01\] -- Congestion Control Request. + +This field is has zero payload length. Its presence signifies that the client +wants to use congestion control. The client MUST NOT set this field, or use +ntorv3, if the service did not list "2" in the `FlowCtrl` line in the +descriptor. The client SHOULD NOT provide this field if the consensus parameter +'cc_alg' is 0. + +The service MUST ignore any unknown fields. + +#### Proof-of-Work (PoW) + +This is used to send the work done by the client to the service. + + EXT_FIELD_TYPE: + + \[02\] -- Proof Of Work. + + EXT_FIELD content format is: + + POW_VERSION [1 byte] + POW_NONCE [16 bytes] + POW_EFFORT [4 bytes] + POW_SEED [4 bytes] + POW_SOLUTION [16 bytes] + + where: + + POW_VERSION is 1 for the protocol specified in this proposal + POW_NONCE is the nonce 'N' from the section above + POW_EFFORT is the 32-bit integer effort value, in network byte order + POW_SEED is the first 4 bytes of the seed used + + +When a service receives an INTRODUCE1 with the extension, it should check its +configuration on whether proof-of-work is enabled on the service. If it's not +enabled, the extension SHOULD be ignored. If enabled, even if the suggested +effort is currently zero, the service follows the procedure detailed in +prop327. + +If the service requires the PROOF_OF_WORK extension but received an INTRODUCE1 +message without any embedded proof-of-work, the service SHOULD consider this message +as a zero-effort introduction for the purposes of the priority queue (see +section \[INTRO_QUEUE\] of prop327). + +(TODO: We should have a proof-of-work.md to fold in prop327. For now, just +point to the proposal.) + +#### Subprotocol Request + +\[RESERVED\] + + EXT_FIELD_TYPE: + + \[03\] -- Subprotocol Request + +<a id="rend-spec-v3.txt-3.3.1"></a> + +### Introduction handshake encryption requirements {#INTRO-HANDSHAKE-REQS} + +When decoding the encrypted information in an INTRODUCE2 message, a +hidden service host must be able to: + +```text + * Decrypt additional information included in the INTRODUCE2 message, + to include the rendezvous token and the information needed to + extend to the rendezvous point. + + * Establish a set of shared keys for use with the client. + + * Authenticate that the message has not been modified since the client + generated it. +``` + +Note that the old TAP-derived protocol of the previous hidden service +design achieved the first two requirements, but not the third. + +```text +3.3.2. Example encryption handshake: ntor with extra data + [NTOR-WITH-EXTRA-DATA] + + [TODO: relocate this] +``` + +This is a variant of the ntor handshake (see tor-spec.txt, section +5.1.4; see proposal 216; and see "Anonymity and one-way +authentication in key-exchange protocols" by Goldberg, Stebila, and +Ustaoglu). + +It behaves the same as the ntor handshake, except that, in addition +to negotiating forward secure keys, it also provides a means for +encrypting non-forward-secure data to the server (in this case, to +the hidden service host) as part of the handshake. + +Notation here is as in section 5.1.4 of tor-spec.txt, which defines +the ntor handshake. + +The PROTOID for this variant is "tor-hs-ntor-curve25519-sha3-256-1". +We also use the following tweak values: + +```text + t_hsenc = PROTOID | ":hs_key_extract" + t_hsverify = PROTOID | ":hs_verify" + t_hsmac = PROTOID | ":hs_mac" + m_hsexpand = PROTOID | ":hs_key_expand" +``` + +To make an INTRODUCE1 message, the client must know a public encryption +key B for the hidden service on this introduction circuit. The client +generates a single-use keypair: + +x,X = KEYGEN() + +and computes: + +```text + intro_secret_hs_input = EXP(B,x) | AUTH_KEY | X | B | PROTOID + info = m_hsexpand | N_hs_subcred + hs_keys = KDF(intro_secret_hs_input | t_hsenc | info, S_KEY_LEN+MAC_LEN) + ENC_KEY = hs_keys[0:S_KEY_LEN] + MAC_KEY = hs_keys[S_KEY_LEN:S_KEY_LEN+MAC_KEY_LEN] + + and sends, as the ENCRYPTED part of the INTRODUCE1 message: + + CLIENT_PK [PK_PUBKEY_LEN bytes] + ENCRYPTED_DATA [Padded to length of plaintext] + MAC [MAC_LEN bytes] +``` + +Substituting those fields into the INTRODUCE1 message body format +described in \[FMT_INTRO1\] above, we have + +```text + LEGACY_KEY_ID [20 bytes] + AUTH_KEY_TYPE [1 byte] + AUTH_KEY_LEN [2 bytes] + AUTH_KEY [AUTH_KEY_LEN bytes] + N_EXTENSIONS [1 bytes] + N_EXTENSIONS times: + EXT_FIELD_TYPE [1 byte] + EXT_FIELD_LEN [1 byte] + EXT_FIELD [EXT_FIELD_LEN bytes] + ENCRYPTED: + CLIENT_PK [PK_PUBKEY_LEN bytes] + ENCRYPTED_DATA [Padded to length of plaintext] + MAC [MAC_LEN bytes] +``` + +(This format is as documented in \[FMT_INTRO1\] above, except that here +we describe how to build the ENCRYPTED portion.) + +Here, the encryption key plays the role of B in the regular ntor +handshake, and the AUTH_KEY field plays the role of the node ID. +The CLIENT_PK field is the public key X. The ENCRYPTED_DATA field is +the message plaintext, encrypted with the symmetric key ENC_KEY. The +MAC field is a MAC of all of the message from the AUTH_KEY through the +end of ENCRYPTED_DATA, using the MAC_KEY value as its key. + +To process this format, the hidden service checks PK_VALID(CLIENT_PK) +as necessary, and then computes ENC_KEY and MAC_KEY as the client did +above, except using EXP(CLIENT_PK,b) in the calculation of +intro_secret_hs_input. The service host then checks whether the MAC is +correct. If it is invalid, it drops the message. Otherwise, it computes +the plaintext by decrypting ENCRYPTED_DATA. + +The hidden service host now completes the service side of the +extended ntor handshake, as described in tor-spec.txt section 5.1.4, +with the modified PROTOID as given above. To be explicit, the hidden +service host generates a keypair of y,Y = KEYGEN(), and uses its +introduction point encryption key 'b' to compute: + +```text + intro_secret_hs_input = EXP(X,b) | AUTH_KEY | X | B | PROTOID + info = m_hsexpand | N_hs_subcred + hs_keys = KDF(intro_secret_hs_input | t_hsenc | info, S_KEY_LEN+MAC_LEN) + HS_DEC_KEY = hs_keys[0:S_KEY_LEN] + HS_MAC_KEY = hs_keys[S_KEY_LEN:S_KEY_LEN+MAC_KEY_LEN] + + (The above are used to check the MAC and then decrypt the + encrypted data.) + + rend_secret_hs_input = EXP(X,y) | EXP(X,b) | AUTH_KEY | B | X | Y | PROTOID + NTOR_KEY_SEED = MAC(rend_secret_hs_input, t_hsenc) + verify = MAC(rend_secret_hs_input, t_hsverify) + auth_input = verify | AUTH_KEY | B | Y | X | PROTOID | "Server" + AUTH_INPUT_MAC = MAC(auth_input, t_hsmac) + + (The above are used to finish the ntor handshake.) + + The server's handshake reply is: + + SERVER_PK Y [PK_PUBKEY_LEN bytes] + AUTH AUTH_INPUT_MAC [MAC_LEN bytes] +``` + +These fields will be sent to the client in a RENDEZVOUS1 message using the +HANDSHAKE_INFO element (see \[JOIN_REND\]). + +The hidden service host now also knows the keys generated by the +handshake, which it will use to encrypt and authenticate data +end-to-end between the client and the server. These keys are as +computed in tor-spec.txt section 5.1.4, except that instead of using +AES-128 and SHA1 for this hop, we use AES-256 and SHA3-256. + +<a id="rend-spec-v3.txt-3.4"></a> + +## Authentication during the introduction phase. {#INTRO-AUTH} + +Hidden services may restrict access only to authorized users. +One mechanism to do so is the credential mechanism, where only users who +know the credential for a hidden service may connect at all. + +There is one defined authentication type: `ed25519`. + +<a id="rend-spec-v3.txt-3.4.1"></a> + +### Ed25519-based authentication `ed25519` {#ed25519-auth} + +(NOTE: This section is not implemented by Tor. It is likely +that we would want to change its design substantially before +deploying any implementation. At the very least, we would +want to bind these extensions to a single onion service, to +prevent replays. We might also want to look for ways to limit +the number of keys a user needs to have.) + +To authenticate with an Ed25519 private key, the user must include an +extension field in the encrypted part of the INTRODUCE1 message with an +EXT_FIELD_TYPE type of \[02\] and the contents: + +```text + Nonce [16 bytes] + Pubkey [32 bytes] + Signature [64 bytes] +``` + +Nonce is a random value. Pubkey is the public key that will be used +to authenticate. \[TODO: should this be an identifier for the public +key instead?\] Signature is the signature, using Ed25519, of: + +```text + "hidserv-userauth-ed25519" + Nonce (same as above) + Pubkey (same as above) + AUTH_KEY (As in the INTRODUCE1 message) +``` + +The hidden service host checks this by seeing whether it recognizes +and would accept a signature from the provided public key. If it +would, then it checks whether the signature is correct. If it is, +then the correct user has authenticated. + +Replay prevention on the whole message is sufficient to prevent replays +on the authentication. + +Users SHOULD NOT use the same public key with multiple hidden +services. diff --git a/spec/rend-spec/keyblinding-scheme.md b/spec/rend-spec/keyblinding-scheme.md new file mode 100644 index 0000000..b69a578 --- /dev/null +++ b/spec/rend-spec/keyblinding-scheme.md @@ -0,0 +1,104 @@ +<a id="rend-spec-v3.txt-A"></a> + +# Appendix A: Signature scheme with key blinding {#KEYBLIND} + +<a id="rend-spec-v3.txt-A.1"></a> + +## Key derivation overview {#overview} + +As described in \[IMD:DIST\] and \[SUBCRED\] above, we require a "key +blinding" system that works (roughly) as follows: + +There is a master keypair (sk, pk). + +```text + Given the keypair and a nonce n, there is a derivation function + that gives a new blinded keypair (sk_n, pk_n). This keypair can + be used for signing. + + Given only the public key and the nonce, there is a function + that gives pk_n. + + Without knowing pk, it is not possible to derive pk_n; without + knowing sk, it is not possible to derive sk_n. + + It's possible to check that a signature was made with sk_n while + knowing only pk_n. + + Someone who sees a large number of blinded public keys and + signatures made using those public keys can't tell which + signatures and which blinded keys were derived from the same + master keypair. + + You can't forge signatures. + + [TODO: Insert a more rigorous definition and better references.] +``` + +<a id="rend-spec-v3.txt-A.2"></a> + +## Tor's key derivation scheme {#scheme} + +We propose the following scheme for key blinding, based on Ed25519. + +(This is an ECC group, so remember that scalar multiplication is the +trapdoor function, and it's defined in terms of iterated point +addition. See the Ed25519 paper \[Reference ED25519-REFS\] for a fairly +clear writeup.) + +Let B be the ed25519 basepoint as found in section 5 of \[ED25519-B-REF\]: + +```text + B = (15112221349535400772501151409588531511454012693041857206046113283949847762202, + 46316835694926478169428394003475163141307993866256225615783033603165251855960) +``` + +Assume B has prime order l, so lB=0. Let a master keypair be written as +(a,A), where a is the private key and A is the public key (A=aB). + +To derive the key for a nonce N and an optional secret s, compute the +blinding factor like this: + +```text + h = H(BLIND_STRING | A | s | B | N) + BLIND_STRING = "Derive temporary signing key" | INT_1(0) + N = "key-blind" | INT_8(period-number) | INT_8(period_length) + B = "(1511[...]2202, 4631[...]5960)" + + then clamp the blinding factor 'h' according to the ed25519 spec: + + h[0] &= 248; + h[31] &= 63; + h[31] |= 64; + + and do the key derivation as follows: + + private key for the period: + + a' = h a mod l + RH' = SHA-512(RH_BLIND_STRING | RH)[:32] + RH_BLIND_STRING = "Derive temporary signing key hash input" + + public key for the period: + + A' = h A = (ha)B +``` + +Generating a signature of M: given a deterministic random-looking r +(see EdDSA paper), take R=rB, S=r+hash(R,A',M)ah mod l. Send signature +(R,S) and public key A'. + +Verifying the signature: Check whether SB = R+hash(R,A',M)A'. + +```text + (If the signature is valid, + SB = (r + hash(R,A',M)ah)B + = rB + (hash(R,A',M)ah)B + = R + hash(R,A',M)A' ) + + This boils down to regular Ed25519 with key pair (a', A'). +``` + +See \[KEYBLIND-REFS\] for an extensive discussion on this scheme and +possible alternatives. Also, see \[KEYBLIND-PROOF\] for a security +proof of this scheme. diff --git a/spec/rend-spec/managing-streams.md b/spec/rend-spec/managing-streams.md new file mode 100644 index 0000000..9eb1e82 --- /dev/null +++ b/spec/rend-spec/managing-streams.md @@ -0,0 +1,33 @@ + +# Managing streams + +## Sending BEGIN messages { #send-begin } + +In order to open a new stream to an onion service, +the client sends a BEGIN message on an established rendezvous circuit. + +When sending a BEGIN message to an onion service, +a client should use an empty string as the target address, +and not set any flags on the begin message. + +> For example, to open a connection to `<some_addr>.onion` +> on port 443, a client would send a BEGIN message with +> the address:port string of `":443"`, and a `FLAGS` value of 0. +> The 0-values `FLAGS` would not be encoded, according to +> the instructions for [encoding BEGIN messages](../tor-spec/opening-streams.md#opening). + +## Receiving BEGIN messages { #receive-begin } + +When a service receives a BEGIN message, it should check its port, +_and ignore all other fields in the begin message_, including its +address and flags. + +If a service chooses to reject a BEGIN message, it should typically +destroy the circuit entirely to prevent port scanning, +resource exhaustion, and other undesirable behaviors. +But if it rejects the BEGIN without destroy the circuit, +it should send back an `END` message with the `DONE` reason, +to avoid leaking any further information. + +If the service chooses to accept the BEGIN message, +it should send back a CONNECTED message with an empty body. diff --git a/spec/rend-spec/overview.md b/spec/rend-spec/overview.md new file mode 100644 index 0000000..5ccddd4 --- /dev/null +++ b/spec/rend-spec/overview.md @@ -0,0 +1,315 @@ +<a id="rend-spec-v3.txt-0"></a> + +# Hidden services: overview and preliminaries + +Hidden services aim to provide responder anonymity for bidirectional +stream-based communication on the Tor network. Unlike regular Tor +connections, where the connection initiator receives anonymity but +the responder does not, hidden services attempt to provide +bidirectional anonymity. + +Participants: + +Operator -- A person running a hidden service + +```text + Host, "Server" -- The Tor software run by the operator to provide + a hidden service. + + User -- A person contacting a hidden service. + + Client -- The Tor software running on the User's computer + + Hidden Service Directory (HSDir) -- A Tor node that hosts signed + statements from hidden service hosts so that users can make + contact with them. + + Introduction Point -- A Tor node that accepts connection requests + for hidden services and anonymously relays those requests to the + hidden service. + + Rendezvous Point -- A Tor node to which clients and servers + connect and which relays traffic between them. +``` + +<a id="rend-spec-v3.txt-0.1"></a> + +## Improvements over previous versions {#improvements} + +Here is a list of improvements of this proposal over the legacy hidden +services: + +a) Better crypto (replaced SHA1/DH/RSA1024 with SHA3/ed25519/curve25519) +b) Improved directory protocol leaking less to directory servers. +c) Improved directory protocol with smaller surface for targeted attacks. +d) Better onion address security against impersonation. +e) More extensible introduction/rendezvous protocol. +f) Offline keys for onion services +g) Advanced client authorization + +<a id="rend-spec-v3.txt-0.2"></a> + +## Notation and vocabulary {#notation} + +Unless specified otherwise, all multi-octet integers are big-endian. + +We write sequences of bytes in two ways: + +```text + 1. A sequence of two-digit hexadecimal values in square brackets, + as in [AB AD 1D EA]. + + 2. A string of characters enclosed in quotes, as in "Hello". The + characters in these strings are encoded in their ascii + representations; strings are NOT nul-terminated unless + explicitly described as NUL terminated. + + We use the words "byte" and "octet" interchangeably. + + We use the vertical bar | to denote concatenation. +``` + +We use INT_N(val) to denote the network (big-endian) encoding of the +unsigned integer "val" in N bytes. For example, INT_4(1337) is \[00 00 +05 39\]. Values are truncated like so: val % (2 ^ (N * 8)). For example, +INT_4(42) is 42 % 4294967296 (32 bit). + +<a id="rend-spec-v3.txt-0.3"></a> + +## Cryptographic building blocks {#cryptography} + +This specification uses the following cryptographic building blocks: + +```text + * A pseudorandom number generator backed by a strong entropy source. + The output of the PRNG should always be hashed before being posted on + the network to avoid leaking raw PRNG bytes to the network + (see [PRNG-REFS]). + + * A stream cipher STREAM(iv, k) where iv is a nonce of length + S_IV_LEN bytes and k is a key of length S_KEY_LEN bytes. + + * A public key signature system SIGN_KEYGEN()->seckey, pubkey; + SIGN_SIGN(seckey,msg)->sig; and SIGN_CHECK(pubkey, sig, msg) -> + { "OK", "BAD" }; where secret keys are of length SIGN_SECKEY_LEN + bytes, public keys are of length SIGN_PUBKEY_LEN bytes, and + signatures are of length SIGN_SIG_LEN bytes. + + This signature system must also support key blinding operations + as discussed in appendix [KEYBLIND] and in section [SUBCRED]: + SIGN_BLIND_SECKEY(seckey, blind)->seckey2 and + SIGN_BLIND_PUBKEY(pubkey, blind)->pubkey2 . + + * A public key agreement system "PK", providing + PK_KEYGEN()->seckey, pubkey; PK_VALID(pubkey) -> {"OK", "BAD"}; + and PK_HANDSHAKE(seckey, pubkey)->output; where secret keys are + of length PK_SECKEY_LEN bytes, public keys are of length + PK_PUBKEY_LEN bytes, and the handshake produces outputs of + length PK_OUTPUT_LEN bytes. + + * A cryptographic hash function H(d), which should be preimage and + collision resistant. It produces hashes of length HASH_LEN + bytes. + + * A cryptographic message authentication code MAC(key,msg) that + produces outputs of length MAC_LEN bytes. + + * A key derivation function KDF(message, n) that outputs n bytes. + + As a first pass, I suggest: + + * Instantiate STREAM with AES256-CTR. + + * Instantiate SIGN with Ed25519 and the blinding protocol in + [KEYBLIND]. + + * Instantiate PK with Curve25519. + + * Instantiate H with SHA3-256. + + * Instantiate KDF with SHAKE-256. + + * Instantiate MAC(key=k, message=m) with H(k_len | k | m), + where k_len is htonll(len(k)). +``` + +When we need a particular MAC key length below, we choose +MAC_KEY_LEN=32 (256 bits). + +For legacy purposes, we specify compatibility with older versions of +the Tor introduction point and rendezvous point protocols. These used +RSA1024, DH1024, AES128, and SHA1, as discussed in +rend-spec.txt. + +As in \[proposal 220\], all signatures are generated not over strings +themselves, but over those strings prefixed with a distinguishing +value. + +<a id="rend-spec-v3.txt-0.4"></a> + +## Protocol building blocks {#BUILDING-BLOCKS} + +In sections below, we need to transmit the locations and identities +of Tor nodes. We do so in the link identification format used by +EXTEND2 messages in the Tor protocol. + +```text + NSPEC (Number of link specifiers) [1 byte] + NSPEC times: + LSTYPE (Link specifier type) [1 byte] + LSLEN (Link specifier length) [1 byte] + LSPEC (Link specifier) [LSLEN bytes] +``` + +Link specifier types are as described in tor-spec.txt. Every set of +link specifiers SHOULD include at minimum specifiers of type \[00\] +(TLS-over-TCP, IPv4), \[02\] (legacy node identity) and \[03\] (ed25519 +identity key). Sets of link specifiers without these three types +SHOULD be rejected. + +As of 0.4.1.1-alpha, Tor includes both IPv4 and IPv6 link specifiers +in v3 onion service protocol link specifier lists. All available +addresses SHOULD be included as link specifiers, regardless of the +address that Tor actually used to connect/extend to the remote relay. + +We also incorporate Tor's circuit extension handshakes, as used in +the CREATE2 and CREATED2 cells described in tor-spec.txt. In these +handshakes, a client who knows a public key for a server sends a +message and receives a message from that server. Once the exchange is +done, the two parties have a shared set of forward-secure key +material, and the client knows that nobody else shares that key +material unless they control the secret key corresponding to the +server's public key. + +<a id="rend-spec-v3.txt-0.5"></a> + +## Assigned relay message types {#relay-cell-types} + +These relay message types are reserved for use in the hidden service +protocol. + +32 -- RELAY_COMMAND_ESTABLISH_INTRO + +```text + Sent from hidden service host to introduction point; + establishes introduction point. Discussed in + [REG_INTRO_POINT]. + + 33 -- RELAY_COMMAND_ESTABLISH_RENDEZVOUS + + Sent from client to rendezvous point; creates rendezvous + point. Discussed in [EST_REND_POINT]. + + 34 -- RELAY_COMMAND_INTRODUCE1 + + Sent from client to introduction point; requests + introduction. Discussed in [SEND_INTRO1] + + 35 -- RELAY_COMMAND_INTRODUCE2 + + Sent from introduction point to hidden service host; requests + introduction. Same format as INTRODUCE1. Discussed in + [FMT_INTRO1] and [PROCESS_INTRO2] + + 36 -- RELAY_COMMAND_RENDEZVOUS1 + + Sent from hidden service host to rendezvous point; + attempts to join host's circuit to + client's circuit. Discussed in [JOIN_REND] + + 37 -- RELAY_COMMAND_RENDEZVOUS2 + + Sent from rendezvous point to client; + reports join of host's circuit to + client's circuit. Discussed in [JOIN_REND] + + 38 -- RELAY_COMMAND_INTRO_ESTABLISHED + + Sent from introduction point to hidden service host; + reports status of attempt to establish introduction + point. Discussed in [INTRO_ESTABLISHED] + + 39 -- RELAY_COMMAND_RENDEZVOUS_ESTABLISHED + + Sent from rendezvous point to client; acknowledges + receipt of ESTABLISH_RENDEZVOUS message. Discussed in + [EST_REND_POINT] + + 40 -- RELAY_COMMAND_INTRODUCE_ACK + + Sent from introduction point to client; acknowledges + receipt of INTRODUCE1 message and reports success/failure. + Discussed in [INTRO_ACK] +``` + +<a id="rend-spec-v3.txt-0.6"></a> + +## Acknowledgments + +This design includes ideas from many people, including + +```text + Christopher Baines, + Daniel J. Bernstein, + Matthew Finkel, + Ian Goldberg, + George Kadianakis, + Aniket Kate, + Tanja Lange, + Robert Ransom, + Roger Dingledine, + Aaron Johnson, + Tim Wilson-Brown ("teor"), + special (John Brooks), + s7r +``` + +It's based on Tor's original hidden service design by Roger +Dingledine, Nick Mathewson, and Paul Syverson, and on improvements to +that design over the years by people including + +```text + Tobias Kamm, + Thomas Lauterbach, + Karsten Loesing, + Alessandro Preite Martinez, + Robert Ransom, + Ferdinand Rieger, + Christoph Weingarten, + Christian Wilms, +``` + +We wouldn't be able to do any of this work without good attack +designs from researchers including + +```text + Alex Biryukov, + Lasse Øverlier, + Ivan Pustogarov, + Paul Syverson, + Ralf-Philipp Weinmann, + + See [ATTACK-REFS] for their papers. + + Several of these ideas have come from conversations with + + Christian Grothoff, + Brian Warner, + Zooko Wilcox-O'Hearn, +``` + +And if this document makes any sense at all, it's thanks to +editing help from + +```text + Matthew Finkel, + George Kadianakis, + Peter Palfrader, + Tim Wilson-Brown ("teor"), +``` + +\[XXX Acknowledge the huge bunch of people working on 8106.\] +\[XXX Acknowledge the huge bunch of people working on 8244.\] + +Please forgive me if I've missed you; please forgive me if I've +misunderstood your best ideas here too. diff --git a/spec/rend-spec/protocol-overview.md b/spec/rend-spec/protocol-overview.md new file mode 100644 index 0000000..a50b3d1 --- /dev/null +++ b/spec/rend-spec/protocol-overview.md @@ -0,0 +1,348 @@ +<a id="rend-spec-v3.txt-1"></a> + +# Protocol overview + +In this section, we outline the hidden service protocol. This section +omits some details in the name of simplicity; those are given more +fully below, when we specify the protocol in more detail. + +<a id="rend-spec-v3.txt-1.1"></a> + +## View from 10,000 feet {#10000-feet} + +A hidden service host prepares to offer a hidden service by choosing +several Tor nodes to serve as its introduction points. It builds +circuits to those nodes, and tells them to forward introduction +requests to it using those circuits. + +Once introduction points have been picked, the host builds a set of +documents called "hidden service descriptors" (or just "descriptors" +for short) and uploads them to a set of HSDir nodes. These documents +list the hidden service's current introduction points and describe +how to make contact with the hidden service. + +When a client wants to connect to a hidden service, it first chooses +a Tor node at random to be its "rendezvous point" and builds a +circuit to that rendezvous point. If the client does not have an +up-to-date descriptor for the service, it contacts an appropriate +HSDir and requests such a descriptor. + +The client then builds an anonymous circuit to one of the hidden +service's introduction points listed in its descriptor, and gives the +introduction point an introduction request to pass to the hidden +service. This introduction request includes the target rendezvous +point and the first part of a cryptographic handshake. + +Upon receiving the introduction request, the hidden service host +makes an anonymous circuit to the rendezvous point and completes the +cryptographic handshake. The rendezvous point connects the two +circuits, and the cryptographic handshake gives the two parties a +shared key and proves to the client that it is indeed talking to the +hidden service. + +Once the two circuits are joined, the client can use Tor RELAY cells +to deliver relay messages +to the server. RELAY_BEGIN messages open streams to an external process +or processes configured by the server; RELAY_DATA messages are used to +communicate data on those streams, and so forth. + +<a id="rend-spec-v3.txt-1.2"></a> + +## In more detail: naming hidden services {#NAMING} + +A hidden service's name is its long term master identity key. This is +encoded as a hostname by encoding the entire key in Base 32, including a +version byte and a checksum, and then appending the string ".onion" at the +end. The result is a 56-character domain name. + +(This is a change from older versions of the hidden service protocol, +where we used an 80-bit truncated SHA1 hash of a 1024 bit RSA key.) + +The names in this format are distinct from earlier names because of +their length. An older name might look like: + +```text + unlikelynamefora.onion + yyhws9optuwiwsns.onion + + And a new name following this specification might look like: + + l5satjgud6gucryazcyvyvhuxhr74u6ygigiuyixe3a6ysis67ororad.onion + + Please see section [ONIONADDRESS] for the encoding specification. +``` + +<a id="rend-spec-v3.txt-1.3"></a> + +## In more detail: Access control {#IMD:AC} + +Access control for a hidden service is imposed at multiple points through +the process above. Furthermore, there is also the option to impose +additional client authorization access control using pre-shared secrets +exchanged out-of-band between the hidden service and its clients. + +The first stage of access control happens when downloading HS descriptors. +Specifically, in order to download a descriptor, clients must know which +blinded signing key was used to sign it. (See the next section for more info +on key blinding.) + +To learn the introduction points, clients must decrypt the body of the +hidden service descriptor. To do so, clients must know the _unblinded_ +public key of the service, which makes the descriptor unusable by entities +without that knowledge (e.g. HSDirs that don't know the onion address). + +Also, if optional client authorization is enabled, hidden service +descriptors are superencrypted using each authorized user's identity x25519 +key, to further ensure that unauthorized entities cannot decrypt it. + +In order to make the introduction point send a rendezvous request to the +service, the client needs to use the per-introduction-point authentication +key found in the hidden service descriptor. + +The final level of access control happens at the server itself, which may +decide to respond or not respond to the client's request depending on the +contents of the request. The protocol is extensible at this point: at a +minimum, the server requires that the client demonstrate knowledge of the +contents of the encrypted portion of the hidden service descriptor. If +optional client authorization is enabled, the service may additionally +require the client to prove knowledge of a pre-shared private key. + +<a id="rend-spec-v3.txt-1.4"></a> + +## In more detail: Distributing hidden service descriptors. {#IMD:DIST} + +Periodically, hidden service descriptors become stored at different +locations to prevent a single directory or small set of directories +from becoming a good DoS target for removing a hidden service. + +For each period, the Tor directory authorities agree upon a +collaboratively generated random value. (See section 2.3 for a +description of how to incorporate this value into the voting +practice; generating the value is described in other proposals, +including \[SHAREDRANDOM-REFS\].) That value, combined with hidden service +directories' public identity keys, determines each HSDir's position +in the hash ring for descriptors made in that period. + +Each hidden service's descriptors are placed into the ring in +positions based on the key that was used to sign them. Note that +hidden service descriptors are not signed with the services' public +keys directly. Instead, we use a key-blinding system \[KEYBLIND\] to +create a new key-of-the-day for each hidden service. Any client that +knows the hidden service's public identity key can derive these blinded +signing keys for a given period. It should be impossible to derive +the blinded signing key lacking that knowledge. + +This is achieved using two nonces: + +```text + * A "credential", derived from the public identity key KP_hs_id. + N_hs_cred. + + * A "subcredential", derived from the credential N_hs_cred + and information which various with the current time period. + N_hs_subcred. +``` + +The body of each descriptor is also encrypted with a key derived from +the public signing key. + +To avoid a "thundering herd" problem where every service generates +and uploads a new descriptor at the start of each period, each +descriptor comes online at a time during the period that depends on +its blinded signing key. The keys for the last period remain valid +until the new keys come online. + +<a id="rend-spec-v3.txt-1.5"></a> + +## In more detail: Scaling to multiple hosts {#imd-scaling} + +This design is compatible with our current approaches for scaling hidden +services. Specifically, hidden service operators can use onionbalance to +achieve high availability between multiple nodes on the HSDir +layer. Furthermore, operators can use proposal 255 to load balance their +hidden services on the introduction layer. See \[SCALING-REFS\] for further +discussions on this topic and alternative designs. + +```text +1.6. In more detail: Backward compatibility with older hidden service + protocols +``` + +This design is incompatible with the clients, server, and hsdir node +protocols from older versions of the hidden service protocol as +described in rend-spec.txt. On the other hand, it is designed to +enable the use of older Tor nodes as rendezvous points and +introduction points. + +<a id="rend-spec-v3.txt-1.7"></a> + +## In more detail: Keeping crypto keys offline {#imd-offline-keys} + +In this design, a hidden service's secret identity key may be +stored offline. It's used only to generate blinded signing keys, +which are used to sign descriptor signing keys. + +In order to operate a hidden service, the operator can generate in +advance a number of blinded signing keys and descriptor signing +keys (and their credentials; see \[DESC-OUTER\] and \[HS-DESC-ENC\] +below), and their corresponding descriptor encryption keys, and +export those to the hidden service hosts. + +As a result, in the scenario where the Hidden Service gets +compromised, the adversary can only impersonate it for a limited +period of time (depending on how many signing keys were generated +in advance). + +It's important to not send the private part of the blinded signing +key to the Hidden Service since an attacker can derive from it the +secret master identity key. The secret blinded signing key should +only be used to create credentials for the descriptor signing keys. + +(NOTE: although the protocol allows them, offline keys are not +implemented as of 0.3.2.1-alpha.) + +<a id="rend-spec-v3.txt-1.8"></a> + +## In more detail: Encryption Keys And Replay Resistance {#imd-encryption-keys} + +To avoid replays of an introduction request by an introduction point, +a hidden service host must never accept the same request +twice. Earlier versions of the hidden service design used an +authenticated timestamp here, but including a view of the current +time can create a problematic fingerprint. (See proposal 222 for more +discussion.) + +<a id="rend-spec-v3.txt-1.9"></a> + +## In more detail: A menagerie of keys {#imd-key-menagerie} + +\[In the text below, an "encryption keypair" is roughly "a keypair you +can do Diffie-Hellman with" and a "signing keypair" is roughly "a +keypair you can do ECDSA with."\] + +Public/private keypairs defined in this document: + +<a id="hs_id"></a> +```text + Master (hidden service) identity key -- A master signing keypair + used as the identity for a hidden service. This key is long + term and not used on its own to sign anything; it is only used + to generate blinded signing keys as described in [KEYBLIND] + and [SUBCRED]. The public key is encoded in the ".onion" + address according to [NAMING]. + KP_hs_id, KS_hs_id. +``` + +<a id="hs_blind_id"></a> +```text + Blinded signing key -- A keypair derived from the identity key, + used to sign descriptor signing keys. It changes periodically for + each service. Clients who know a 'credential' consisting of the + service's public identity key and an optional secret can derive + the public blinded identity key for a service. This key is used + as an index in the DHT-like structure of the directory system + (see [SUBCRED]). + KP_hs_blind_id, KS_hs_blind_id. + +``` + +<a id="hs_desc_sign"></a> +```text + Descriptor signing key -- A key used to sign hidden service + descriptors. This is signed by blinded signing keys. Unlike + blinded signing keys and master identity keys, the secret part + of this key must be stored online by hidden service hosts. The + public part of this key is included in the unencrypted section + of HS descriptors (see [DESC-OUTER]). + KP_hs_desc_sign, KS_hs_desc_sign. +``` + +<a id="hs_ipt_sid"></a> +```text + Introduction point authentication key -- A short-term signing + keypair used to identify a hidden service's session at a given + introduction point. The service makes a fresh keypair for each + introduction point; these are used to sign the request that a + hidden service host makes when establishing an introduction + point, so that clients who know the public component of this key + can get their introduction requests sent to the right + service. No keypair is ever used with more than one introduction + point. (previously called a "service key" in rend-spec.txt) + KP_hs_ipt_sid, KS_hs_ipt_sid + ("hidden service introduction point session id"). +``` + +<a id="hss_ntor"></a> +```text + Introduction point encryption key -- A short-term encryption + keypair used when establishing connections via an introduction + point. Plays a role analogous to Tor nodes' onion keys. The service + makes a fresh keypair for each introduction point. + KP_hss_ntor, KS_hss_ntor. +``` + +<a id="hss_desc_enc"></a> +```text + Ephemeral descriptor encryption key -- A short-lived encryption + keypair made by the service, and used to encrypt the inner layer + of hidden service descriptors when client authentication is in + use. + KP_hss_desc_enc, KS_hss_desc_enc +``` + +```text + Nonces defined in this document: + + N_hs_desc_enc -- a nonce used to derive keys to decrypt the inner + encryption layer of hidden service descriptors. This is + sometimes also called a "descriptor cookie". + + Public/private keypairs defined elsewhere: + + Onion key -- Short-term encryption keypair (KS_ntor, KP_ntor). + + (Node) identity key (KP_relayid). + + Symmetric key-like things defined elsewhere: + + KH from circuit handshake -- An unpredictable value derived as + part of the Tor circuit extension handshake, used to tie a request + to a particular circuit. +``` + +<a id="rend-spec-v3.txt-1.9.1"></a> + +### In even more detail: Client authorization keys {#CLIENT-AUTH} + +When client authorization is enabled, each authorized client of a hidden +service has two more asymmetric keypairs which are shared with the hidden +service. An entity without those keys is not able to use the hidden +service. Throughout this document, we assume that these pre-shared keys are +exchanged between the hidden service and its clients in a secure out-of-band +fashion. + +Specifically, each authorized client possesses: + +```text + - An x25519 keypair used to compute decryption keys that allow the client to + decrypt the hidden service descriptor. See [HS-DESC-ENC]. This is + the client's counterpart to KP_hss_desc_enc. + KP_hsc_desc_enc, KS_hsd_desc_enc. + + - An ed25519 keypair which allows the client to compute signatures which + prove to the hidden service that the client is authorized. These + signatures are inserted into the INTRODUCE1 message, and without them the + introduction to the hidden service cannot be completed. See [INTRO-AUTH]. + KP_hsc_intro_auth, KS_hsc_intro_auth. +``` + +The right way to exchange these keys is to have the client generate keys and +send the corresponding public keys to the hidden service out-of-band. An +easier but less secure way of doing this exchange would be to have the +hidden service generate the keypairs and pass the corresponding private keys +to its clients. See section \[CLIENT-AUTH-MGMT\] for more details on how these +keys should be managed. + +\[TODO: Also specify stealth client authorization.\] + +(NOTE: client authorization is implemented as of 0.3.5.1-alpha.) diff --git a/spec/rend-spec/references.md b/spec/rend-spec/references.md new file mode 100644 index 0000000..6488ebe --- /dev/null +++ b/spec/rend-spec/references.md @@ -0,0 +1,89 @@ +<a id="rend-spec-v3.txt-7"></a> + +# References + +How can we improve the HSDir unpredictability design proposed in +\[SHAREDRANDOM\]? See these references for discussion. + +```text +[SHAREDRANDOM-REFS]: + https://gitweb.torproject.org/torspec.git/tree/proposals/250-commit-reveal-consensus.txt + https://trac.torproject.org/projects/tor/ticket/8244 +``` + + +Scaling hidden services is hard. There are on-going discussions that +you might be able to help with: + +```text +[SCALING-REFS]: + https://lists.torproject.org/pipermail/tor-dev/2013-October/005556.html +``` + +How can hidden service addresses become memorable while retaining +their self-authenticating and decentralized nature? See +these references for some proposals; many more are possible. + +```text +[HUMANE-HSADDRESSES-REFS]: + https://gitweb.torproject.org/torspec.git/blob/HEAD:/proposals/ideas/xxx-onion-nyms.txt + http://archives.seul.org/or/dev/Dec-2011/msg00034.html +``` + +Hidden Services are pretty slow. Both because of the lengthy setup +procedure and because the final circuit has 6 hops. How can we make +the Hidden Service protocol faster? See these references for some +suggestions. + +```text +[PERFORMANCE-REFS]: + "Improving Efficiency and Simplicity of Tor circuit + establishment and hidden services" by Overlier, L., and + P. Syverson + + [TODO: Need more here! Do we have any? :( ] +``` + +Other references: + +```text +[KEYBLIND-REFS]: + https://trac.torproject.org/projects/tor/ticket/8106 + https://lists.torproject.org/pipermail/tor-dev/2012-September/004026.html + +[KEYBLIND-PROOF]: + https://lists.torproject.org/pipermail/tor-dev/2013-December/005943.html + +[ATTACK-REFS]: + "Trawling for Tor Hidden Services: Detection, Measurement, + Deanonymization" by Alex Biryukov, Ivan Pustogarov, + Ralf-Philipp Weinmann + + "Locating Hidden Servers" by Lasse Øverlier and Paul + Syverson + +[ED25519-REFS]: + "High-speed high-security signatures" by Daniel + J. Bernstein, Niels Duif, Tanja Lange, Peter Schwabe, and + Bo-Yin Yang. http://cr.yp.to/papers.html#ed25519 + +[ED25519-B-REF]: + https://tools.ietf.org/html/draft-josefsson-eddsa-ed25519-03#section-5: + +[PRNG-REFS]: + http://projectbullrun.org/dual-ec/ext-rand.html + https://lists.torproject.org/pipermail/tor-dev/2015-November/009954.html + +[SRV-TP-REFS]: + https://lists.torproject.org/pipermail/tor-dev/2016-April/010759.html + +[VANITY-REFS]: + https://github.com/Yawning/horse25519 + +[ONIONADDRESS-REFS]: + https://lists.torproject.org/pipermail/tor-dev/2017-January/011816.html + +[TORSION-REFS]: + https://lists.torproject.org/pipermail/tor-dev/2017-April/012164.html + https://getmonero.org/2017/05/17/disclosure-of-a-major-bug-in-cryptonote-based-currencies.html +``` diff --git a/spec/rend-spec/rendezvous-protocol.md b/spec/rend-spec/rendezvous-protocol.md new file mode 100644 index 0000000..2b14934 --- /dev/null +++ b/spec/rend-spec/rendezvous-protocol.md @@ -0,0 +1,138 @@ +<a id="rend-spec-v3.txt-4"></a> + +# The rendezvous protocol + +Before connecting to a hidden service, the client first builds a +circuit to an arbitrarily chosen Tor node (known as the rendezvous +point), and sends an ESTABLISH_RENDEZVOUS message. The hidden service +later connects to the same node and sends a RENDEZVOUS message. Once +this has occurred, the relay forwards the contents of the RENDEZVOUS +message to the client, and joins the two circuits together. + +Single Onion Services attempt to build a non-anonymous single-hop circuit, +but use an anonymous 3-hop circuit if: + +```text + * the rend point is on an address that is configured as unreachable via + a direct connection, or + * the initial attempt to connect to the rend point over a single-hop + circuit fails, and they are retrying the rend point connection. +``` + +<a id="rend-spec-v3.txt-4.1"></a> + +## Establishing a rendezvous point {#EST_REND_POINT} + +The client sends the rendezvous point a RELAY_COMMAND_ESTABLISH_RENDEZVOUS +message containing a 20-byte value. + +RENDEZVOUS_COOKIE \[20 bytes\] + +Rendezvous points MUST ignore any extra bytes in an +ESTABLISH_RENDEZVOUS message. (Older versions of Tor did not.) + +The rendezvous cookie is an arbitrary 20-byte value, chosen randomly +by the client. The client SHOULD choose a new rendezvous cookie for +each new connection attempt. If the rendezvous cookie is already in +use on an existing circuit, the rendezvous point should reject it and +destroy the circuit. + +Upon receiving an ESTABLISH_RENDEZVOUS message, the rendezvous point associates +the cookie with the circuit on which it was sent. It replies to the client +with an empty RENDEZVOUS_ESTABLISHED message to indicate success. Clients MUST +ignore any extra bytes in a RENDEZVOUS_ESTABLISHED message. + +The client MUST NOT use the circuit which sent the message for any +purpose other than rendezvous with the given location-hidden service. + +The client should establish a rendezvous point BEFORE trying to +connect to a hidden service. + +<a id="rend-spec-v3.txt-4.2"></a> + +## Joining to a rendezvous point {#JOIN_REND} + +To complete a rendezvous, the hidden service host builds a circuit to +the rendezvous point and sends a RENDEZVOUS1 message containing: + +```text + RENDEZVOUS_COOKIE [20 bytes] + HANDSHAKE_INFO [variable; depends on handshake type + used.] +``` + +where RENDEZVOUS_COOKIE is the cookie suggested by the client during the +introduction (see \[PROCESS_INTRO2\]) and HANDSHAKE_INFO is defined in +\[NTOR-WITH-EXTRA-DATA\]. + +If the cookie matches the rendezvous cookie set on any +not-yet-connected circuit on the rendezvous point, the rendezvous +point connects the two circuits, and sends a RENDEZVOUS2 message to the +client containing the HANDSHAKE_INFO field of the RENDEZVOUS1 message. + +Upon receiving the RENDEZVOUS2 message, the client verifies that HANDSHAKE_INFO +correctly completes a handshake. To do so, the client parses SERVER_PK from +HANDSHAKE_INFO and reverses the final operations of section +\[NTOR-WITH-EXTRA-DATA\] as shown here: + +```text + rend_secret_hs_input = EXP(Y,x) | EXP(B,x) | AUTH_KEY | B | X | Y | PROTOID + NTOR_KEY_SEED = MAC(rend_secret_hs_input, t_hsenc) + verify = MAC(rend_secret_hs_input, t_hsverify) + auth_input = verify | AUTH_KEY | B | Y | X | PROTOID | "Server" + AUTH_INPUT_MAC = MAC(auth_input, t_hsmac) +``` + +Finally the client verifies that the received AUTH field of HANDSHAKE_INFO +is equal to the computed AUTH_INPUT_MAC. + +Now both parties use the handshake output to derive shared keys for use on +the circuit as specified in the section below: + +<a id="rend-spec-v3.txt-4.2.1"></a> + +### Key expansion + +The hidden service and its client need to derive crypto keys from the +NTOR_KEY_SEED part of the handshake output. To do so, they use the KDF +construction as follows: + +K = KDF(NTOR_KEY_SEED | m_hsexpand, HASH_LEN *2 + S_KEY_LEN* 2) + +The first HASH_LEN bytes of K form the forward digest Df; the next HASH_LEN +bytes form the backward digest Db; the next S_KEY_LEN bytes form Kf, and the +final S_KEY_LEN bytes form Kb. Excess bytes from K are discarded. + +Subsequently, the rendezvous point passes RELAY cells, unchanged, from each +of the two circuits to the other. When Alice's OP sends RELAY cells along +the circuit, it authenticates with Df, and encrypts them with the Kf, then +with all of the keys for the ORs in Alice's side of the circuit; and when +Alice's OP receives RELAY cells from the circuit, it decrypts them with the +keys for the ORs in Alice's side of the circuit, then decrypts them with Kb, +and checks integrity with Db. Bob's OP does the same, with Kf and Kb +interchanged. + +\[TODO: Should we encrypt HANDSHAKE_INFO as we did INTRODUCE2 +contents? It's not necessary, but it could be wise. Similarly, we +should make it extensible.\] + +<a id="rend-spec-v3.txt-4.3"></a> + +## Using legacy hosts as rendezvous points {#legacy-rend-hosts} + +\[This section is obsolete and refers to a workaround for now-obsolete Tor +relay versions. It is included for historical reasons.\] + +The behavior of ESTABLISH_RENDEZVOUS is unchanged from older versions +of this protocol, except that relays should now ignore unexpected +bytes at the end. + +Old versions of Tor required that RENDEZVOUS message payloads be exactly +168 bytes long. All shorter rendezvous payloads should be padded to +this length with random bytes, to make them difficult to distinguish from +older protocols at the rendezvous point. + +Relays older than 0.2.9.1 should not be used for rendezvous points by next +generation onion services because they enforce too-strict length checks to +RENDEZVOUS messages. Hence the "HSRend" protocol from proposal#264 should be +used to select relays for rendezvous points. diff --git a/spec/rend-spec/reserved-numbers.md b/spec/rend-spec/reserved-numbers.md new file mode 100644 index 0000000..70c9e36 --- /dev/null +++ b/spec/rend-spec/reserved-numbers.md @@ -0,0 +1,17 @@ +<a id="rend-spec-v3.txt-E"></a> + +# Appendix E: Reserved numbers + +We reserve these certificate type values for Ed25519 certificates: + +```text + [08] short-term descriptor signing key, signed with blinded + public key. (Section 2.4) + [09] intro point authentication key, cross-certifying the descriptor + signing key. (Section 2.5) + [0B] ed25519 key derived from the curve25519 intro point encryption key, + cross-certifying the descriptor signing key. (Section 2.5) + + Note: The value "0A" is skipped because it's reserved for the onion key + cross-certifying ntor identity key from proposal 228. +``` diff --git a/spec/rend-spec/revision-counter-mgt.md b/spec/rend-spec/revision-counter-mgt.md new file mode 100644 index 0000000..f5c9a9c --- /dev/null +++ b/spec/rend-spec/revision-counter-mgt.md @@ -0,0 +1,103 @@ +<a id="rend-spec-v3.txt-F"></a> + +# Appendix F: Two methods for managing revision counters + +Implementations MAY generate revision counters in any way they please, +so long as they are monotonically increasing over the lifetime of each +blinded public key. But to avoid fingerprinting, implementors SHOULD +choose a strategy also used by other Tor implementations. Here we +describe two, and additionally list some strategies that implementors +should NOT use. + +## F.1. Increment-on-generation {#increment-on-generation} + +This is the simplest strategy, and the one used by Tor through at +least version 0.3.4.0-alpha. + +Whenever using a new blinded key, the service records the +highest revision counter it has used with that key. When generating +a descriptor, the service uses the smallest non-negative number +higher than any number it has already used. + +In other words, the revision counters under this system start fresh +with each blinded key as 0, 1, 2, 3, and so on. + +## F.2. Encrypted time in period {#encrypted-time} + +This scheme is what we recommend for situations when multiple +service instances need to +[coordinate their revision counters](#use-case), +without an actual coordination mechanism. + +Let T be the number of seconds that have elapsed +since the start time of the SRV protocol run +that corresponds to this descriptor, +plus 1. (T must be at least 1.) + +Let S be a per-time-period secret that all the service providers share. +(C tor and arti use `S = KS_hs_blind_id`; +when `KS_hs_blind_id` is not available, +implementations may use `S = KS_hs_desc_sign`.) + +Let K be an AES-256 key, generated as +``` +K = H("rev-counter-generation" | S) +``` + +Use `K`, and AES in counter mode with IV=0, to generate a stream of +`T * 2` bytes. Consider these bytes as a sequence of T 16-bit +little-endian words. Add these words. + +Let the sum of these words, plus T, be the revision counter. + +> (We include T in the sum so that every increment in T adds at least +> one to the output.) + +Cryptowiki attributes roughly this scheme to G. Bebek in: + +> G. Bebek. Anti-tamper database research: Inference control +> techniques. Technical Report EECS 433 Final Report, Case +> Western Reserve University, November 2002. + +Although we believe it is suitable for use in this application, it +is not a perfect order-preserving encryption algorithm (and all +order-preserving encryption has weaknesses). Please think twice +before using it for anything else. + +(This scheme can be optimized pretty easily by caching the encryption of +`X*1`, `X*2`, `X*3`, etc for some well chosen `X`.) + +For a slow reference implementation +that can generate test vectors, +see `src/test/ope_ref.py` in the +Tor source repository. + +> <h6 id="use-case">Note:</h6> +> +> Some onion service operators have historically relied upon +> the behavior of this OPE scheme to provide +> a kind of ersatz load-balancing: +> they run multiple onion services, +> each with the the same `KP_hs_id`. +> The onion services choose different introduction points, +> and race each other to publish their HSDescs. +> +> It's probably better to use +> [Onionbalance](https://onionbalance.readthedocs.io/en/latest/) +> or a similar tool for this purpose. +> Nonetheless, onion services implemenentations +> MAY choose to implement this particular OPE scheme exactly +> in order to provide interoperability with C tor +> in this use case. + +## F.X. Some revision-counter strategies to avoid {#avoid} + +Though it might be tempting, implementations SHOULD NOT use the +current time or the current time within the period directly as their +revision counter -- doing so leaks their view of the current time, +which can be used to link the onion service to other services run on +the same host. + +Similarly, implementations SHOULD NOT let the revision counter +increase forever without resetting it -- doing so links the service +across changes in the blinded public key. diff --git a/spec/rend-spec/selecting-nodes-picknodes.md b/spec/rend-spec/selecting-nodes-picknodes.md new file mode 100644 index 0000000..8b0973c --- /dev/null +++ b/spec/rend-spec/selecting-nodes-picknodes.md @@ -0,0 +1,10 @@ +<a id="rend-spec-v3.txt-B"></a> + +# Appendix B: Selecting nodes \[PICKNODES\] + +Picking introduction points +Picking rendezvous points +Building paths +Reusing circuits + +(TODO: This needs a writeup) diff --git a/spec/rend-spec/shared-random.md b/spec/rend-spec/shared-random.md new file mode 100644 index 0000000..e6b2452 --- /dev/null +++ b/spec/rend-spec/shared-random.md @@ -0,0 +1,49 @@ +<a id="rend-spec-v3.txt-2.3"></a> + +# Publishing shared random values {#PUB-SHAREDRANDOM} + +Our design for limiting the predictability of HSDir upload locations +relies on a shared random value (SRV) that isn't predictable in advance or +too influenceable by an attacker. The authorities must run a protocol +to generate such a value at least once per hsdir period. Here we +describe how they publish these values; the procedure they use to +generate them can change independently of the rest of this +specification. For more information see \[SHAREDRANDOM-REFS\]. + +According to proposal 250, we add two new lines in consensuses: + +```text + "shared-rand-previous-value" SP NUM_REVEALS SP VALUE NL + "shared-rand-current-value" SP NUM_REVEALS SP VALUE NL +``` + +<a id="rend-spec-v3.txt-2.3.1"></a> + +## Client behavior in the absence of shared random values {#client-disaster} + +If the previous or current shared random value cannot be found in a +consensus, then Tor clients and services need to generate their own random +value for use when choosing HSDirs. + +To do so, Tor clients and services use: + +SRV = H("shared-random-disaster" | INT_8(period_length) | INT_8(period_num)) + +where period_length is the length of a time period in minutes, +rounded down; period_num is calculated as specified in +\[TIME-PERIODS\] for the wanted shared random value that could not be +found originally. + +<a id="rend-spec-v3.txt-2.3.2"></a> + +## Hidden services and changing shared random values {#service-problems} + +It's theoretically possible that the consensus shared random values will +change or disappear in the middle of a time period because of directory +authorities dropping offline or misbehaving. + +To avoid client reachability issues in this rare event, hidden services +should use the new shared random values to find the new responsible HSDirs +and upload their descriptors there. + +XXX How long should they upload descriptors there for? diff --git a/spec/rend-spec/text-vectors.md b/spec/rend-spec/text-vectors.md new file mode 100644 index 0000000..eadaee2 --- /dev/null +++ b/spec/rend-spec/text-vectors.md @@ -0,0 +1,101 @@ +<a id="rend-spec-v3.txt-G"></a> + +# Appendix G: Text vectors + +G.1. Test vectors for hs-ntor / NTOR-WITH-EXTRA-DATA + +```text + Here is a set of test values for the hs-ntor handshake, called + [NTOR-WITH-EXTRA-DATA] in this document. They were generated by + instrumenting Tor's code to dump the values for an INTRODUCE/RENDEZVOUS + handshake, and then by running that code on a Chutney network. + + We assume an onion service with: + + KP_hs_ipd_sid = 34E171E4358E501BFF21ED907E96AC6B + FEF697C779D040BBAF49ACC30FC5D21F + KP_hss_ntor = 8E5127A40E83AABF6493E41F142B6EE3 + 604B85A3961CD7E38D247239AFF71979 + KS_hss_ntor = A0ED5DBF94EEB2EDB3B514E4CF6ABFF6 + 022051CC5F103391F1970A3FCD15296A + N_hs_subcred = 0085D26A9DEBA252263BF0231AEAC59B + 17CA11BAD8A218238AD6487CBAD68B57 + + The client wants to make in INTRODUCE request. It generates + the following header (everything before the ENCRYPTED portion) + of its INTRODUCE1 message: + + H = 000000000000000000000000000000000000000002002034E171E4358E501BFF + 21ED907E96AC6BFEF697C779D040BBAF49ACC30FC5D21F00 + + It generates the following plaintext body to encrypt. (This + is the "decrypted plaintext body" from [PROCESS_INTRO2]. + + P = 6BD364C12638DD5C3BE23D76ACA05B04E6CE932C0101000100200DE6130E4FCA + C4EDDA24E21220CC3EADAE403EF6B7D11C8273AC71908DE565450300067F0000 + 0113890214F823C4F8CC085C792E0AEE0283FE00AD7520B37D0320728D5DF39B + 7B7077A0118A900FF4456C382F0041300ACF9C58E51C392795EF870000000000 + 0000000000000000000000000000000000000000000000000000000000000000 + 000000000000000000000000000000000000000000000000000000000000 + + (Note! This should in fact be padded to be longer; when these + test vectors were generated, the target INTRODUCE1 length in C + Tor was needlessly short.) + + The client now begins the hs-ntor handshake. It generates + a curve25519 keypair: + + x = 60B4D6BF5234DCF87A4E9D7487BDF3F4 + A69B6729835E825CA29089CFDDA1E341 + X = BF04348B46D09AED726F1D66C618FDEA + 1DE58E8CB8B89738D7356A0C59111D5D + + Then it calculates: + + ENC_KEY = 9B8917BA3D05F3130DACCE5300C3DC27 + F6D012912F1C733036F822D0ED238706 + MAC_KEY = FC4058DA59D4DF61E7B40985D122F502 + FD59336BC21C30CAF5E7F0D4A2C38FD5 + + With these, it encrypts the plaintext body P with ENC_KEY, getting + an encrypted value C. It computes MAC(MAC_KEY, H | X | C), + getting a MAC value M. It then assembles the final INTRODUCE1 + body as H | X | C | M: + + 000000000000000000000000000000000000000002002034E171E4358E501BFF + 21ED907E96AC6BFEF697C779D040BBAF49ACC30FC5D21F00BF04348B46D09AED + 726F1D66C618FDEA1DE58E8CB8B89738D7356A0C59111D5DADBECCCB38E37830 + 4DCC179D3D9E437B452AF5702CED2CCFEC085BC02C4C175FA446525C1B9D5530 + 563C362FDFFB802DAB8CD9EBC7A5EE17DA62E37DEEB0EB187FBB48C63298B0E8 + 3F391B7566F42ADC97C46BA7588278273A44CE96BC68FFDAE31EF5F0913B9A9C + 7E0F173DBC0BDDCD4ACB4C4600980A7DDD9EAEC6E7F3FA3FC37CD95E5B8BFB3E + 35717012B78B4930569F895CB349A07538E42309C993223AEA77EF8AEA64F25D + DEE97DA623F1AEC0A47F150002150455845C385E5606E41A9A199E7111D54EF2 + D1A51B7554D8B3692D85AC587FB9E69DF990EFB776D8 +``` + +Later the service receives that body in an INTRODUCE2 message. It +processes it according to the hs-ntor handshake, and recovers +the client's plaintext P. To continue the hs-ntor handshake, +the service chooses a curve25519 keypair: + +```text + y = 68CB5188CA0CD7924250404FAB54EE13 + 92D3D2B9C049A2E446513875952F8F55 + Y = 8FBE0DB4D4A9C7FF46701E3E0EE7FD05 + CD28BE4F302460ADDEEC9E93354EE700 + + From this and the client's input, it computes: + + AUTH_INPUT_MAC = 4A92E8437B8424D5E5EC279245D5C72B + 25A0327ACF6DAF902079FCB643D8B208 + NTOR_KEY_SEED = 4D0C72FE8AFF35559D95ECC18EB5A368 + 83402B28CDFD48C8A530A5A3D7D578DB +``` + +The service sends back Y | AUTH_INPUT_MAC in its RENDEZVOUS1 message +body. From these, the client finishes the handshake, validates +AUTH_INPUT_MAC, and computes the same NTOR_KEY_SEED. + +Now that both parties have the same NTOR_KEY_SEED, they can derive +the shared key material they will use for their circuit. diff --git a/spec/rend-spec/vanity-onions.md b/spec/rend-spec/vanity-onions.md new file mode 100644 index 0000000..f07d108 --- /dev/null +++ b/spec/rend-spec/vanity-onions.md @@ -0,0 +1,45 @@ +<a id="rend-spec-v3.txt-C"></a> + +# Appendix C: Recommendations for searching for vanity .onions \[VANITY\] + +EDITORIAL NOTE: The author thinks that it's silly to brute-force the +keyspace for a key that, when base-32 encoded, spells out the name of +your website. It also feels a bit dangerous to me. If you train your +users to connect to + +llamanymityx4fi3l6x2gyzmtmgxjyqyorj9qsb5r543izcwymle.onion + +I worry that you're making it easier for somebody to trick them into +connecting to + +llamanymityb4sqi0ta0tsw6uovyhwlezkcrmczeuzdvfauuemle.onion + +Nevertheless, people are probably going to try to do this, so here's a +decent algorithm to use. + +To search for a public key with some criterion X: + +Generate a random (sk,pk) pair. + +While pk does not satisfy X: + +```text + Add the number 8 to sk + Add the point 8*B to pk + + Return sk, pk. +``` + +We add 8 and 8\*B, rather than 1 and B, so that sk is always a valid +Curve25519 private key, with the lowest 3 bits equal to 0. + +This algorithm is safe \[source: djb, personal communication\] \[TODO: +Make sure I understood correctly!\] so long as only the final (sk,pk) +pair is used, and all previous values are discarded. + +To parallelize this algorithm, start with an independent (sk,pk) pair +generated for each independent thread, and let each search proceed +independently. + +See \[VANITY-REFS\] for a reference implementation of this vanity .onion +search scheme. diff --git a/spec/socks-extensions.md b/spec/socks-extensions.md new file mode 100644 index 0000000..6fbfff5 --- /dev/null +++ b/spec/socks-extensions.md @@ -0,0 +1,184 @@ +# Tor's extensions to the SOCKS protocol + +<a id="socks-extensions.txt-1"></a> + +## Overview + +The SOCKS protocol provides a generic interface for TCP proxies. Client +software connects to a SOCKS server via TCP, and requests a TCP connection +to another address and port. The SOCKS server establishes the connection, +and reports success or failure to the client. After the connection has +been established, the client application uses the TCP stream as usual. + +Tor supports SOCKS4 as defined in \[1\], SOCKS4A as defined in \[2\], and +SOCKS5 as defined in \[3\] and \[4\]. + +The stickiest issue for Tor in supporting clients, in practice, is forcing +DNS lookups to occur at the OR side: if clients do their own DNS lookup, +the DNS server can learn which addresses the client wants to reach. +SOCKS4 supports addressing by IPv4 address; SOCKS4A is a kludge on top of +SOCKS4 to allow addressing by hostname; SOCKS5 supports IPv4, IPv6, and +hostnames. + +<a id="socks-extensions.txt-1.1"></a> + +### Extent of support + +Tor supports the SOCKS4, SOCKS4A, and SOCKS5 standards, except as follows: + +BOTH: + +- The BIND command is not supported. + +SOCKS4,4A: + +- SOCKS4 usernames are used to implement stream isolation. + +```text + SOCKS5: + - The (SOCKS5) "UDP ASSOCIATE" command is not supported. + - SOCKS5 BIND command is not supported. + - IPv6 is not supported in CONNECT commands. + - SOCKS5 GSSAPI subnegotiation is not supported. + - The "NO AUTHENTICATION REQUIRED" (SOCKS5) authentication method [00] is + supported; and as of Tor 0.2.3.2-alpha, the "USERNAME/PASSWORD" (SOCKS5) + authentication method [02] is supported too, and used as a method to + implement stream isolation. As an extension to support some broken clients, + we allow clients to pass "USERNAME/PASSWORD" authentication message to us + even if no authentication was selected. Furthermore, we allow + username/password fields of this message to be empty. This technically + violates RFC1929 [4], but ensures interoperability with somewhat broken + SOCKS5 client implementations. + - Custom reply error code. The "REP" fields, as per the RFC[3], has + unassigned values which are used to describe Tor internal errors. See + ExtendedErrors in the tor.1 man page for more details. It is only sent + back if this SocksPort flag is set. +``` + +(For more information on stream isolation, see IsolateSOCKSAuth on the Tor +manpage.) + +<a id="socks-extensions.txt-2"></a> + +## Name lookup + +As an extension to SOCKS4A and SOCKS5, Tor implements a new command value, +"RESOLVE" \[F0\]. When Tor receives a "RESOLVE" SOCKS command, it initiates +a remote lookup of the hostname provided as the target address in the SOCKS +request. The reply is either an error (if the address couldn't be +resolved) or a success response. In the case of success, the address is +stored in the portion of the SOCKS response reserved for remote IP address. + +(We support RESOLVE in SOCKS4 too, even though it is unnecessary.) + +For SOCKS5 only, we support reverse resolution with a new command value, +"RESOLVE_PTR" \[F1\]. In response to a "RESOLVE_PTR" SOCKS5 command with +an IPv4 address as its target, Tor attempts to find the canonical +hostname for that IPv4 record, and returns it in the "server bound +address" portion of the reply. +(This command was not supported before Tor 0.1.2.2-alpha.) + +<a id="socks-extensions.txt-3"></a> + +## Other command extensions + +Tor 0.1.2.4-alpha added a new command value: "CONNECT_DIR" \[F2\]. +In this case, Tor will open an encrypted direct TCP connection to the +directory port of the Tor server specified by address:port (the port +specified should be the ORPort of the server). It uses a one-hop tunnel +and a "BEGIN_DIR" relay message to accomplish this secure connection. + +The F2 command value was removed in Tor 0.2.0.10-alpha in favor of a +new use_begindir flag in edge_connection_t. + +<a id="socks-extensions.txt-4"></a> + +## HTTP-resistance + +Tor checks the first byte of each SOCKS request to see whether it looks +more like an HTTP request (that is, it starts with a "G", "H", or "P"). If +so, Tor returns a small webpage, telling the user that his/her browser is +misconfigured. This is helpful for the many users who mistakenly try to +use Tor as an HTTP proxy instead of a SOCKS proxy. + +<a id="socks-extensions.txt-5"></a> + +## Optimistic data + +Tor allows SOCKS clients to send connection data before Tor has sent a +SOCKS response. When using an exit node that supports "optimistic data", +Tor will send such data to the server without waiting to see whether the +connection attempt succeeds. This behavior can save a single round-trip +time when starting connections with a protocol where the client speaks +first (like HTTP). Clients that do this must be ready to hear that +their connection has succeeded or failed _after_ they have sent the +data. + +<a id="socks-extensions.txt-6"></a> + +## Extended error codes + +We define a set of additional extension error codes that can be returned +by our SOCKS implementation in response to failed onion service +connections. + +(In the C Tor implementation, these error codes can be disabled +via the ExtendedErrors flag. In Arti, these error codes are enabled +whenever onion services are.) + +- X'F0' Onion Service Descriptor Can Not be Found + +```text + The requested onion service descriptor can't be found on the hashring + and thus not reachable by the client. + + * X'F1' Onion Service Descriptor Is Invalid + + The requested onion service descriptor can't be parsed or signature + validation failed. + + * X'F2' Onion Service Introduction Failed + + Client failed to introduce to the service meaning the descriptor was + found but the service is not anymore at the introduction points. The + service has likely changed its descriptor or is not running. + + * X'F3' Onion Service Rendezvous Failed + + Client failed to rendezvous with the service which means that the client + is unable to finalize the connection. + + * X'F4' Onion Service Missing Client Authorization + + Tor was able to download the requested onion service descriptor but is + unable to decrypt its content because it is missing client authorization + information for it. + + * X'F5' Onion Service Wrong Client Authorization + + Tor was able to download the requested onion service descriptor but is + unable to decrypt its content using the client authorization information + it has. This means the client access were revoked. + + * X'F6' Onion Service Invalid Address + + The given .onion address is invalid. In one of these cases this + error is returned: address checksum doesn't match, ed25519 public + key is invalid or the encoding is invalid. + + * X'F7' Onion Service Introduction Timed Out + + Similar to X'F2' code but in this case, all introduction attempts + have failed due to a time out. +``` + +(Note that not all of the above error codes are currently returned +by Arti as of August 2023.) + +```text +References: + [1] http://en.wikipedia.org/wiki/SOCKS#SOCKS4 + [2] http://en.wikipedia.org/wiki/SOCKS#SOCKS4a + [3] SOCKS5: RFC 1928 https://www.ietf.org/rfc/rfc1928.txt + [4] RFC 1929: https://www.ietf.org/rfc/rfc1929.txt +``` diff --git a/spec/srv-spec/acknowledgements.md b/spec/srv-spec/acknowledgements.md new file mode 100644 index 0000000..edeb53e --- /dev/null +++ b/spec/srv-spec/acknowledgements.md @@ -0,0 +1,29 @@ +<a id="srv-spec.txt-7"></a> + +# Acknowledgements + +Thanks to everyone who has contributed to this design with feedback and +discussion. + +Thanks go to arma, ioerror, kernelcorn, nickm, s7r, Sebastian, teor, weasel +and everyone else! + +References: + +```text +[RANDOM-REFS]: + http://projectbullrun.org/dual-ec/ext-rand.html + https://lists.torproject.org/pipermail/tor-dev/2015-November/009954.html + +[RNGMESSAGING]: + https://moderncrypto.org/mail-archive/messaging/2015/002032.html + +[HOPPER]: + https://lists.torproject.org/pipermail/tor-dev/2014-January/006053.html + +[UNICORN]: + https://eprint.iacr.org/2015/366.pdf + +[VDFS]: + https://eprint.iacr.org/2018/601.pdf +``` diff --git a/spec/srv-spec/discussion.md b/spec/srv-spec/discussion.md new file mode 100644 index 0000000..59d26df --- /dev/null +++ b/spec/srv-spec/discussion.md @@ -0,0 +1,42 @@ +<a id="srv-spec.txt-6"></a> + +# Discussion + +<a id="srv-spec.txt-6.1"></a> + +## Why the added complexity from proposal 225? {#why-complexity} + +The complexity difference between this proposal and prop225 is in part +because prop225 doesn't specify how the shared random value gets to the +clients. This proposal spends lots of effort specifying how the two shared +random values can always be readily accessible to clients. + +<a id="srv-spec.txt-6.2"></a> + +## Why do you do a commit-and-reveal protocol in 24 rounds? {#why-rounds} + +The reader might be wondering why we span the protocol over the course of a +whole day (24 hours), when only 3 rounds would be sufficient to generate a +shared random value. + +We decided to do it this way, because we piggyback on the Tor voting +protocol which also happens every hour. + +We could instead only do the shared randomness protocol from 21:00 to 00:00 +every day. Or to do it multiple times a day. + +However, we decided that since the shared random value needs to be in every +consensus anyway, carrying the commitments/reveals as well will not be a +big problem. Also, this way we give more chances for a failing dirauth to +recover and rejoin the protocol. + +<a id="srv-spec.txt-6.3"></a> + +## Why can't we recover if the 00:00UTC consensus fails? {#why-no-recovery} + +If the 00:00UTC consensus fails, there will be no shared random value for +the whole day. In theory, we could recover by calculating the shared +randomness of the day at 01:00UTC instead. However, the engineering issues +with adding such recovery logic are too great. For example, it's not easy +for an authority who just booted to learn whether a specific consensus +failed to be created. diff --git a/spec/srv-spec/index.md b/spec/srv-spec/index.md new file mode 100644 index 0000000..cb954ef --- /dev/null +++ b/spec/srv-spec/index.md @@ -0,0 +1,4 @@ +# Tor Shared Random Subsystem Specification + +This document specifies how the commit-and-reveal shared random subsystem of +Tor works. This text used to be proposal 250-commit-reveal-consensus.txt. diff --git a/spec/srv-spec/introduction.md b/spec/srv-spec/introduction.md new file mode 100644 index 0000000..37d6473 --- /dev/null +++ b/spec/srv-spec/introduction.md @@ -0,0 +1,30 @@ +<a id="srv-spec.txt-1"></a> + +# Introduction + +<a id="srv-spec.txt-1.1"></a> + +## Motivation + +For the next generation hidden services project, we need the Tor network to +produce a fresh random value every day in such a way that it cannot be +predicted in advance or influenced by an attacker. + +Currently we need this random value to make the HSDir hash ring +unpredictable (#8244), which should resolve a wide class of hidden service +DoS attacks and should make it harder for people to gauge the popularity +and activity of target hidden services. Furthermore this random value can +be used by other systems in need of fresh global randomness like +Tor-related protocols (e.g. OnioNS) or even non-Tor-related (e.g. warrant +canaries). + +<a id="srv-spec.txt-1.2"></a> + +## Previous work + +Proposal 225 specifies a commit-and-reveal protocol that can be run as an +external script and have the results be fed to the directory authorities. +However, directory authority operators feel unsafe running a third-party +script that opens TCP ports and accepts connections from the Internet. +Hence, this proposal aims to embed the commit-and-reveal idea in the Tor +voting process which should make it smoother to deploy and maintain. diff --git a/spec/srv-spec/overview.md b/spec/srv-spec/overview.md new file mode 100644 index 0000000..b409e6c --- /dev/null +++ b/spec/srv-spec/overview.md @@ -0,0 +1,164 @@ +<a id="srv-spec.txt-2"></a> + +# Overview + +This proposal alters the Tor consensus protocol such that a random number is +generated every midnight by the directory authorities during the regular voting +process. The distributed random generator scheme is based on the +commit-and-reveal technique. + +The proposal also specifies how the final shared random value is embedded +in consensus documents so that clients who need it can get it. + +<a id="srv-spec.txt-2.1"></a> + +## Introduction to our commit-and-reveal protocol {#commit-and-reveal} + +Every day, before voting for the consensus at 00:00UTC each authority +generates a new random value and keeps it for the whole day. The authority +cryptographically hashes the random value and calls the output its +"commitment" value. The original random value is called the "reveal" value. + +The idea is that given a reveal value you can cryptographically confirm that +it corresponds to a given commitment value (by hashing it). However given a +commitment value you should not be able to derive the underlying reveal +value. The construction of these values is specified in section \[COMMITREVEAL\]. + +<a id="srv-spec.txt-2.1"></a> + +## Ten thousand feet view of the protocol {#10000-feet} + +Our commit-and-reveal protocol aims to produce a fresh shared random value +(denoted shared_random_value here and elsewhere) every day at 00:00UTC. The +final fresh random value is embedded in the consensus document at that +time. + +Our protocol has two phases and uses the hourly voting procedure of Tor. +Each phase lasts 12 hours, which means that 12 voting rounds happen in +between. In short, the protocol works as follows: + +Commit phase: + +```text + Starting at 00:00UTC and for a period of 12 hours, authorities every + hour include their commitment in their votes. They also include any + received commitments from other authorities, if available. + + Reveal phase: + + At 12:00UTC, the reveal phase starts and lasts till the end of the + protocol at 00:00UTC. In this stage, authorities must reveal the value + they committed to in the previous phase. The commitment and revealed + values from other authorities, when available, are also added to the + vote. + + Shared Randomness Calculation: + + At 00:00UTC, the shared random value is computed from the agreed + revealed values and added to the consensus. + + This concludes the commit-and-reveal protocol every day at 00:00UTC. +``` + +<a id="srv-spec.txt-2.3"></a> + +## How we use the consensus {#CONS} + +The produced shared random values need to be readily available to +clients. For this reason we include them in the consensus documents. + +Every hour the consensus documents need to include the shared random value +of the day, as well as the shared random value of the previous day. That's +because either of these values might be needed at a given time for a Tor +client to access a hidden service according to section \[TIME-OVERLAP\] of +proposal 224. This means that both of these two values need to be included +in votes as well. + +Hence, consensuses need to include: + +```text + (a) The shared random value of the current time period. + (b) The shared random value of the previous time period. +``` + +For this, a new SR consensus method will be needed to indicate which +authorities support this new protocol. + +<a id="srv-spec.txt-2.3.1"></a> + +### Inserting Shared Random Values in the consensus {#inserting} + +After voting happens, we need to be careful on how we pick which shared +random values (SRV) to put in the consensus, to avoid breaking the consensus +because of authorities having different views of the commit-and-reveal +protocol (because maybe they missed some rounds of the protocol). + +For this reason, authorities look at the received votes before creating a +consensus and employ the following logic: + +```text + - First of all, they make sure that the agreed upon consensus method is + above the SR consensus method. + + - Authorities include an SRV in the consensus if and only if the SRV has + been voted by at least the majority of authorities. + + - For the consensus at 00:00UTC, authorities include an SRV in the consensus + if and only if the SRV has been voted by at least AuthDirNumAgreements + authorities (where AuthDirNumAgreements is a newly introduced consensus + parameter). +``` + +Authorities include in the consensus the most popular SRV that also +satisfies the above constraints. Otherwise, no SRV should be included. + +The above logic is used to make it harder to break the consensus by natural +partioning causes. + +We use the AuthDirNumAgreements consensus parameter to enforce that a +_supermajority_ of dirauths supports the SR protocol during SRV creation, so +that even if a few of those dirauths drop offline in the middle of the run +the SR protocol does not get disturbed. We go to extra lengths to ensure +this because changing SRVs in the middle of the day has terrible +reachability consequences for hidden service clients. + +<a id="srv-spec.txt-2.4"></a> + +## Persistent State of the Protocol {#STATE} + +A directory authority needs to keep a persistent state on disk of the on +going protocol run. This allows an authority to join the protocol seamlessly +in the case of a reboot. + +During the commitment phase, it is populated with the commitments of all +authorities. Then during the reveal phase, the reveal values are also +stored in the state. + +As discussed previously, the shared random values from the current and +previous time period must also be present in the state at all times if they +are available. + +<a id="srv-spec.txt-2.5"></a> + +## Protocol Illustration {#illustration} + +An illustration for better understanding the protocol can be found here: + +<https://people.torproject.org/~asn/hs_notes/shared_rand.jpg> + +It reads left-to-right. + +The illustration displays what the authorities (A_1, A_2, A_3) put in their +votes. A chain 'A_1 -> c_1 -> r_1' denotes that authority A_1 committed to +the value c_1 which corresponds to the reveal value r_1. + +The illustration depicts only a few rounds of the whole protocol. It starts +with the first three rounds of the commit phase, then it jumps to the last +round of the commit phase. It continues with the first two rounds of the +reveal phase and then it jumps to the final round of the protocol run. It +finally shows the first round of the commit phase of the next protocol run +(00:00UTC) where the final Shared Random Value is computed. In our fictional +example, the SRV was computed with 3 authority contributions and its value +is "a56fg39h". + +We advice you to revisit this after you have read the whole document. diff --git a/spec/srv-spec/protocol.md b/spec/srv-spec/protocol.md new file mode 100644 index 0000000..67aa493 --- /dev/null +++ b/spec/srv-spec/protocol.md @@ -0,0 +1,165 @@ +<a id="srv-spec.txt-3"></a> + +# Protocol + +In this section we give a detailed specification of the protocol. We +describe the protocol participants' logic and the messages they send. The +encoding of the messages is specified in the next section (\[SPEC\]). + +Now we go through the phases of the protocol: + +<a id="srv-spec.txt-3.1"></a> + +## Commitment Phase {#COMMITMENTPHASE} + +The commit phase lasts from 00:00UTC to 12:00UTC. + +During this phase, an authority commits a value in its vote and +saves it to the permanent state as well. + +Authorities also save any received authoritative commits by other authorities +in their permanent state. We call a commit by Alice "authoritative" if it was +included in Alice's vote. + +<a id="srv-spec.txt-3.1.1"></a> + +### Voting During Commitment Phase {#commitment-voting} + +During the commit phase, each authority includes in its votes: + +```text + - The commitment value for this protocol run. + - Any authoritative commitments received from other authorities. + - The two previous shared random values produced by the protocol (if any). +``` + +The commit phase lasts for 12 hours, so authorities have multiple chances to +commit their values. An authority MUST NOT commit a second value during a +subsequent round of the commit phase. + +If an authority publishes a second commitment value in the same commit +phase, only the first commitment should be taken in account by other +authorities. Any subsequent commitments MUST be ignored. + +<a id="srv-spec.txt-3.1.2"></a> + +### Persistent State During Commitment Phase {#STATECOMMIT} + +During the commitment phase, authorities save in their persistent state the +authoritative commits they have received from each authority. Only one commit +per authority must be considered trusted and active at a given time. + +<a id="srv-spec.txt-3.2"></a> + +## Reveal Phase {#reveal-phase} + +The reveal phase lasts from 12:00UTC to 00:00UTC. + +Now that the commitments have been agreed on, it's time for authorities to +reveal their random values. + +<a id="srv-spec.txt-3.2.1"></a> + +### Voting During Reveal Phase {#reveal-voting} + +During the reveal phase, each authority includes in its votes: + +```text + - Its reveal value that was previously committed in the commit phase. + - All the commitments and reveals received from other authorities. + - The two previous shared random values produced by the protocol (if any). +``` + +The set of commitments have been decided during the commitment +phase and must remain the same. If an authority tries to change its +commitment during the reveal phase or introduce a new commitment, +the new commitment MUST be ignored. + +<a id="srv-spec.txt-3.2.2"></a> + +### Persistent State During Reveal Phase {#STATEREVEAL} + +During the reveal phase, authorities keep the authoritative commits from the +commit phase in their persistent state. They also save any received reveals +that correspond to authoritative commits and are valid (as specified in +\[VALIDATEVALUES\]). + +An authority that just received a reveal value from another authority's vote, +MUST wait till the next voting round before including that reveal value in +its votes. + +<a id="srv-spec.txt-3.3"></a> + +## Shared Random Value Calculation At 00:00UTC {#midnight-utc} + +Finally, at 00:00UTC every day, authorities compute a fresh shared random +value and this value must be added to the consensus so clients can use it. + +Authorities calculate the shared random value using the reveal values in +their state as specified in subsection \[SRCALC\]. + +Authorities at 00:00UTC start including this new shared random value in +their votes, replacing the one from two protocol runs ago. Authorities also +start including this new shared random value in the consensus as well. + +Apart from that, authorities at 00:00UTC proceed voting normally as they +would in the first round of the commitment phase (section \[COMMITMENTPHASE\]). + +<a id="srv-spec.txt-3.3.1"></a> + +### Shared Randomness Calculation {#SRCALC} + +An authority that wants to derive the shared random value SRV, should use +the appropriate reveal values for that time period and calculate SRV as +follows. + +HASHED_REVEALS = H(ID_a | R_a | ID_b | R_b | ..) + +```text + SRV = SHA3-256("shared-random" | INT_8(REVEAL_NUM) | INT_4(VERSION) | + HASHED_REVEALS | PREVIOUS_SRV) +``` + +where the ID_a value is the identity key fingerprint of authority 'a' and R_a +is the corresponding reveal value of that authority for the current period. + +Also, REVEAL_NUM is the number of revealed values in this construction, +VERSION is the protocol version number and PREVIOUS_SRV is the previous +shared random value. If no previous shared random value is known, then +PREVIOUS_SRV is set to 32 NUL (\\x00) bytes. + +To maintain consistent ordering in HASHED_REVEALS, all the ID_a | R_a pairs +are ordered based on the R_a value in ascending order. + +<a id="srv-spec.txt-3.4"></a> + +## Bootstrapping Procedure {#bootstrapping} + +As described in \[CONS\], two shared random values are required for the HSDir +overlay periods to work properly as specified in proposal 224. Hence +clients MUST NOT use the randomness of this system till it has bootstrapped +completely; that is, until two shared random values are included in a +consensus. This should happen after three 00:00UTC consensuses have been +produced, which takes 48 hours. + +<a id="srv-spec.txt-3.5"></a> + +## Rebooting Directory Authorities {#REBOOT} + +The shared randomness protocol must be able to support directory +authorities who leave or join in the middle of the protocol execution. + +An authority that commits in the Commitment Phase and then leaves MUST have +stored its reveal value on disk so that it continues participating in the +protocol if it returns before or during the Reveal Phase. The reveal value +MUST be stored timestamped to avoid sending it on wrong protocol runs. + +An authority that misses the Commitment Phase cannot commit anymore, so it's +unable to participate in the protocol for that run. Same goes for an +authority that misses the Reveal phase. Authorities who do not participate in +the protocol SHOULD still carry commits and reveals of others in their vote. + +Finally, authorities MUST implement their persistent state in such a way that they +will never commit two different values in the same protocol run, even if they +have to reboot in the middle (assuming that their persistent state file is +kept). A suggested way to structure the persistent state is found at \[STATEFORMAT\]. diff --git a/spec/srv-spec/security-analysis.md b/spec/srv-spec/security-analysis.md new file mode 100644 index 0000000..d3346f9 --- /dev/null +++ b/spec/srv-spec/security-analysis.md @@ -0,0 +1,104 @@ +<a id="srv-spec.txt-5"></a> + +# Security Analysis + +<a id="srv-spec.txt-5.1"></a> + +## Security of commit-and-reveal and future directions {#sec-commit-and-reveal} + +The security of commit-and-reveal protocols is well understood, and has +certain flaws. Basically, the protocol is insecure to the extent that an +adversary who controls b of the authorities gets to choose among 2^b +outcomes for the result of the protocol. However, an attacker who is not a +dirauth should not be able to influence the outcome at all. + +We believe that this system offers sufficient security especially compared +to the current situation. More secure solutions require much more advanced +crypto and more complex protocols so this seems like an acceptable solution +for now. + +Here are some examples of possible future directions: + +- Schemes based on threshold signatures (e.g. see \[HOPPER\]) +- Unicorn scheme by Lenstra et al. \[UNICORN\] +- Schemes based on Verifiable Delay Functions \[VDFS\] + +For more alternative approaches on collaborative random number generation +also see the discussion at \[RNGMESSAGING\]. + +<a id="srv-spec.txt-5.2"></a> + +## Predicting the shared random value during reveal phase {#sec-predicting} + +The reveal phase lasts 12 hours, and most authorities will send their +reveal value on the first round of the reveal phase. This means that an +attacker can predict the final shared random value about 12 hours before +it's generated. + +This does not pose a problem for the HSDir hash ring, since we impose an +higher uptime restriction on HSDir nodes, so 12 hours predictability is not +an issue. + +Any other protocols using the shared random value from this system should +be aware of this property. + +<a id="srv-spec.txt-5.3"></a> + +## Partition attacks {#sec-partition} + +This design is not immune to certain partition attacks. We believe they +don't offer much gain to an attacker as they are very easy to detect and +difficult to pull off since an attacker would need to compromise a directory +authority at the very least. Also, because of the byzantine general problem, +it's very hard (even impossible in some cases) to protect against all such +attacks. Nevertheless, this section describes all possible partition attack +and how to detect them. + +<a id="srv-spec.txt-5.3.1"></a> + +### Partition attacks during commit phase {#sec-partition-commit} + +A malicious directory authority could send only its commit to one single +authority which results in that authority having an extra commit value for +the shared random calculation that the others don't have. Since the +consensus needs majority, this won't affect the final SRV value. However, +the attacker, using this attack, could remove a single directory authority +from the consensus decision at 24:00 when the SRV is computed. + +An attacker could also partition the authorities by sending two different +commitment values to different authorities during the commit phase. + +All of the above is fairly easy to detect. Commitment values in the vote +coming from an authority should NEVER be different between authorities. If +so, this means an attack is ongoing or very bad bug (highly unlikely). + +<a id="srv-spec.txt-5.3.2"></a> + +### Partition attacks during reveal phase {#sec-partition-reveal} + +Let's consider Alice, a malicious directory authority. Alice could wait +until the last reveal round, and reveal its value to half of the +authorities. That would partition the authorities into two sets: the ones +who think that the shared random value should contain this new reveal, and +the rest who don't know about it. This would result in a tie and two +different shared random value. + +A similar attack is possible. For example, two rounds before the end of the +reveal phase, Alice could advertise her reveal value to only half of the +dirauths. This way, in the last reveal phase round, half of the dirauths +will include that reveal value in their votes and the others will not. In +the end of the reveal phase, half of the dirauths will calculate a +different shared randomness value than the others. + +We claim that this attack is not particularly fruitful: Alice ends up +having two shared random values to choose from which is a fundamental +problem of commit-and-reveal protocols as well (since the last person can +always abort or reveal). The attacker can also sabotage the consensus, but +there are other ways this can be done with the current voting system. + +Furthermore, we claim that such an attack is very noisy and detectable. +First of all, it requires the authority to sabotage two consensuses which +will cause quite some noise. Furthermore, the authority needs to send +different votes to different auths which is detectable. Like the commit +phase attack, the detection here is to make sure that the commitment values +in a vote coming from an authority are always the same for each authority. diff --git a/spec/srv-spec/specification.md b/spec/srv-spec/specification.md new file mode 100644 index 0000000..8118f6d --- /dev/null +++ b/spec/srv-spec/specification.md @@ -0,0 +1,166 @@ +<a id="srv-spec.txt-4"></a> + +# Specification {#spec} + +<a id="srv-spec.txt-4.1"></a> + +## Voting + +This section describes how commitments, reveals and SR values are encoded in +votes. We describe how to encode both the authority's own +commitments/reveals and also the commitments/reveals received from the other +authorities. Commitments and reveals share the same line, but reveals are +optional. + +Participating authorities need to include the line: + +"shared-rand-participate" + +in their votes to announce that they take part in the protocol. + +<a id="srv-spec.txt-4.1.1"></a> + +### Computing commitments and reveals {#COMMITREVEAL} + +A directory authority that wants to participate in this protocol needs to +create a new pair of commitment/reveal values for every protocol +run. Authorities SHOULD generate a fresh pair of such values right before the +first commitment phase of the day (at 00:00UTC). + +The value REVEAL is computed as follows: + +REVEAL = base64-encode( TIMESTAMP || H(RN) ) + +```text + where RN is the SHA3 hashed value of a 256-bit random value. We hash the + random value to avoid exposing raw bytes from our PRNG to the network (see + [RANDOM-REFS]). + + TIMESTAMP is an 8-bytes network-endian time_t value. Authorities SHOULD + set TIMESTAMP to the valid-after time of the vote document they first plan + to publish their commit into (so usually at 00:00UTC, except if they start + up in a later commit round). + + The value COMMIT is computed as follows: + + COMMIT = base64-encode( TIMESTAMP || H(REVEAL) ) +``` + +<a id="srv-spec.txt-4.1.2"></a> + +### Validating commitments and reveals {#VALIDATEVALUES} + +Given a COMMIT message and a REVEAL message it should be possible to verify +that they indeed correspond. To do so, the client extracts the random value +H(RN) from the REVEAL message, hashes it, and compares it with the H(H(RN)) +from the COMMIT message. We say that the COMMIT and REVEAL messages +correspond, if the comparison was successful. + +Participants MUST also check that corresponding COMMIT and REVEAL values +have the same timestamp value. + +Authorities should ignore reveal values during the Reveal Phase that don't +correspond to commit values published during the Commitment Phase. + +<a id="srv-spec.txt-4.1.4"></a> + +### Encoding commit/reveal values in votes {#COMMITVOTE} + +An authority puts in its vote the commitments and reveals it has produced and +seen from the other authorities. To do so, it includes the following in its +votes: + +"shared-rand-commit" SP VERSION SP ALGNAME SP IDENTITY SP COMMIT \[SP REVEAL\] NL + +where VERSION is the version of the protocol the commit was created with. +IDENTITY is the authority's SHA1 identity fingerprint and COMMIT is the +encoded commit \[COMMITREVEAL\]. Authorities during the reveal phase can +also optionally include an encoded reveal value REVEAL. There MUST be only +one line per authority else the vote is considered invalid. Finally, the +ALGNAME is the hash algorithm that should be used to compute COMMIT and +REVEAL which is "sha3-256" for version 1. + +<a id="srv-spec.txt-4.1.5"></a> + +### Shared Random Value {#SRVOTE} + +Authorities include a shared random value (SRV) in their votes using the +following encoding for the previous and current value respectively: + +```text + "shared-rand-previous-value" SP NUM_REVEALS SP VALUE NL + "shared-rand-current-value" SP NUM_REVEALS SP VALUE NL +``` + +where VALUE is the actual shared random value encoded in hex (computed as +specified in section \[SRCALC\]. NUM_REVEALS is the number of reveal values +used to generate this SRV. + +To maintain consistent ordering, the shared random values of the previous +period should be listed before the values of the current period. + +<a id="srv-spec.txt-4.2"></a> + +## Encoding Shared Random Values in the consensus {#SRCONSENSUS} + +Authorities insert the two active shared random values in the consensus +following the same encoding format as in \[SRVOTE\]. + +<a id="srv-spec.txt-4.3"></a> + +## Persistent state format {#STATEFORMAT} + +As a way to keep ground truth state in this protocol, an authority MUST +keep a persistent state of the protocol. The next sub-section suggest a +format for this state which is the same as the current state file format. + +It contains a preamble, a commitment and reveal section and a list of +shared random values. + +The preamble (or header) contains the following items. They MUST occur in +the order given here: + +"Version" SP version NL + +\[At start, exactly once.\] + +A document format version. For this specification, version is "1". + +"ValidUntil" SP YYYY-MM-DD SP HH:MM:SS NL + +\[Exactly once\] + +```text + After this time, this state is expired and shouldn't be used nor + trusted. The validity time period is till the end of the current + protocol run (the upcoming noon). +``` + +The following details the commitment and reveal section. They are encoded +the same as in the vote. This makes it easier for implementation purposes. + +"Commit" SP version SP algname SP identity SP commit \[SP reveal\] NL + +\[Exactly once per authority\] + +The values are the same as detailed in section \[COMMITVOTE\]. + +This line is also used by an authority to store its own value. + +Finally is the shared random value section. + +"SharedRandPreviousValue" SP num_reveals SP value NL + +\[At most once\] + +```text + This is the previous shared random value agreed on at the previous + period. The fields are the same as in section [SRVOTE]. + + "SharedRandCurrentValue" SP num_reveals SP value NL + + [At most once] + + This is the latest shared random value. The fields are the same as in + section [SRVOTE]. +``` diff --git a/spec/ssh-protocols.md b/spec/ssh-protocols.md new file mode 100644 index 0000000..bcc0a3a --- /dev/null +++ b/spec/ssh-protocols.md @@ -0,0 +1,298 @@ +# Tor Project SSH protocol extensions + +The +[SSH protocol](https://datatracker.ietf.org/doc/html/rfc4251) +provides various extension facilities. + +The Tor Project has defined some extensions, +using the +[domain-name-based extension facility](https://datatracker.ietf.org/doc/html/rfc4250#section-4.6.1). +The Tor Project uses names ending `@spec.torproject.org`. + +| Id(s) | Namespace | Summary | Specification<br>link (retrieved at) | +|----------|--------------|----------|-------------------------------------| +| **`ed25519-expaneded@`** | [Public key algorithm](https://datatracker.ietf.org/doc/html/rfc4250#section-4.11.3) (in SSH/OpenSSH key file) | Expanded ed25519 private key | [Arti keystore types] +| **`x25519@`** | [Public key algorithm](https://datatracker.ietf.org/doc/html/rfc4250#section-4.11.3) (in SSH/OpenSSH key file) | X25519 keys | [Arti keystore types] | + +## Registration process + +New entries may be added to this table +after peer review by the Tor Project developers, +via +[gitlab](https://gitlab.torproject.org/tpo/core/torspec) +merge request. + +The specification links may be to external documents, +not managed as part of the Tor Specifications. +Or, they may be links to specific sections of the Tor Specifications, +or to Proposals. +External links should be dated, for ease of future reference. + +Ideally, before a protocol is deployed, +its specification should be transferred to the Tor Specifications +(and the link in the table adjusted). + +## Interpretation + +This section uses +the notation and conventions from +[`PROTOCOL.key`](https://cvsweb.openbsd.org/src/usr.bin/ssh/PROTOCOL.key?annotate=HEAD)) +and +SSH including [RFC4251 s5](https://datatracker.ietf.org/doc/html/rfc4251#section-5)), +not those from the rest of the Tor Specifications. + +### Interpreting the table + +For example, the row for `x25519@` indicates that: + + * The Tor Project has assigned `x25519@spec.torproject.org` + * In the namespace of public key algorithms - + see [RFC4250 table 4.11.3](https://datatracker.ietf.org/doc/html/rfc4250#section-4.11.3), + but only when found within an SSH/OpenSSH format key file. + * The meaning of this name is summarised as "X25519 private key" + * The full details can be found at the linked text, + which is part of the Arti keystore section, below. + +The registered names resemble email addresses, +but they are **not email addresses** +and mail to them will not be delivered. + +For further information, consult the linked specifications. + +## SSH key types for the Arti keystore + +[Arti keystore types]: #ssh-key-types-for-the-arti-keystore + +The +[Arti keystore](https://tpo.pages.torproject.net/core/doc/rust/tor_keymgr/index.html) +stores private keys +in OpenSSH key format +(and, sometimes, public keys, in SSH format). +But it needs to store some key types that are +not used in the SSH protocols. +So the following key types are defined. + +These are in the namespace of Public Key Algorithms +([RFC4250 4.11.3](https://datatracker.ietf.org/doc/html/rfc4250#section-4.11.3)) +but they are only meaningful in +OpenSSH format private key files +([OpenSSH `PROTOCOL.key` document](https://cvsweb.openbsd.org/src/usr.bin/ssh/PROTOCOL.key?annotate=HEAD)) +and +SSH public key files +([RFC4716 3.4](https://datatracker.ietf.org/doc/html/rfc4716#section-3.4)). + +In each case we specify/reference + * the name of the "public key algorithm" (RFC4250 4.11.3), + * the underlying cryptographic algorithm(s), + * the public key data + ("key/certificate data" in + [RFC4253 6.6](https://datatracker.ietf.org/doc/html/rfc4253#section-6.6)), + see [Encoding of the public key data](#encoding-of-the-public-key-data) + * the private key data + (see [Encoding of the private key data](#encoding-of-the-private-key-data)) + +### Encoding of the public key data + +OpenSSH `PROTOCOL.key` does not clearly state +the contents of the `publickey1`/`publickey2`/`publickeyN` fields +in the outer (unencrypted) section (`PROTOCOL.key` s1), +so we state it here. + +Each `publickey` consists of the encoded public key +as per +[RFC4253 6.6](https://datatracker.ietf.org/doc/html/rfc4253#section-6.6) +(under "Certificates and public keys are encoded as follows"). + +So the overall format of this part of the file is: + +``` + uint32 number of keys, N + string publickey1, where the contained binary data is: + string public key algorithm name (RFC4250 table 4.11.3) + byte[] public key data (algorithm-specific) + ... keys 2 to N-1 inclusive, each as for publickey1 ... + string publickeyN (as for publickey1) +``` + +### Encoding of the private key data + +OpenSSH `PROTOCOL.key` defines +the representation of the private key data as, simply: +"using the same rules as used for SSH agent". +However, +no specific section is referred to; +the SSH agent protocol is only available as +an Internet Draft +[draft-miller-ssh-agent-04](https://datatracker.ietf.org/doc/html/draft-miller-ssh-agent-04); +and, the actual encoding used by OpenSSH is hard to relate +to that document. +So we document our understanding here. + +The contents of each `privatekey1`/`privatekey2`/`privatekeyN` is: + +``` + string public key algorithm name (RFC4250 table 4.11.3) + byte[] public key data (algorithm-specific) + byte[] private key data (algorithm-specific) +``` + +> Note that this depends on the reader knowing the public key algorithm. +> The public key data must be self-delimiting, +> since the file format doesn't provide a separate length field. +> Although here we discuss only algorithms whose +> public key data, and private key data, +> are each a single `string`, that is not always the case: +> for example, +> the `ssh-rsa` algorithm's key data formats are sequences of `mpint`s +> without any surrounding overall length. +> +> Note also that the encrypted section does not separately state +> the number of keys or their total length. +> The reader must keep reading until it encounters either the end, +> or something that looks like padding (starting with a byte 0x01). + +### `x25519@spec.torproject.org` + +These refer to keys for +X25519, +ie, Diffie-Hellman on +Curve25519, +as per +[RFC7748 6.1](https://datatracker.ietf.org/doc/html/rfc7748#section-6.1). +and +[s5](https://datatracker.ietf.org/doc/html/rfc7748#section-5). + +The public key data is: +``` + string wrapper for the following fixed-length data: + byte[32] the u-coordinate encoded as u[] from RFC7748 s5 +``` + +The private key data is: +``` + string wrapper for the following fixed-length data: + byte[32] the scalar k encoded according to RFC7748 s5 +``` + +k MUST be stored as the true scalar value. +So if the private key was generated from 32 random bytes +according to the procedure described in RFC7748 s5 +"in order to decode 32 random bytes as an integer scalar". +the value stored MUST be the "clamped" form: +that is, the value +*after* the transformation. +If a stored value is a byte string which doesn't represent +a valid scalar according to RFC7748 s5 +(i.e. an "unclamped" value) +it SHOULD be rejected; +if it is not rejected, +it MUST NOT be used unchanged, but MUST instead be clamped. + +Keys whose `string` wrapper is not of the expected length MUST be rejected. + +> <span id="useless-string">The +> `string` wrapper is useless, +> but the same wrapper approach is used in official SSH for +> ed25519 public keys ([RFC8709 +> s4](https://datatracker.ietf.org/doc/html/rfc8709#name-public-key-format)). +> and for ed25519 private keys in the SSH agent protocol +> ([draft-miller-ssh-agent-04 4.2.3](https://datatracker.ietf.org/doc/html/draft-miller-ssh-agent-04#section-4.2.3)). +> We do the same here for consistency +> (and implementation convenience).</span> + +> X25519 keys are +> [interconvertible with ed25519 keys](dir-spec/converting-to-ed25519.md). +> So, it would be possible to store the ed25519 form instead, +> and convert on load/save. +> However, we use a different key type +> to avoid needing conversions during load/save, and +> to avoid key type punning and accidental key misuse: +> using the same key material for different algorithms is a poor idea. + +### `ed25519-expanded@spec.torproject.org` + +These refer to the expanded form of private keys for +ed25519 +([RFC8032](https://datatracker.ietf.org/doc/html/rfc8032)). + +This key type appears within OpenSSH private key files. +When it does, the `ed25519-expanded@spec.torproject.org` +algorithm name is used for +the private key (`PROTOCOL.key` section 3, `privatekey1` etc.) +but also for +the public key (`PROTOCOL.key` section 1, `publickey1` etc.). + +> In `PROTOCOL.key` we interpret the requirement that there be +> "matching" public and private keys +> to include the requirement that +> the public key algorithm name strings must be the same. + +> In the Arti keystore a private key file +> whose filename ends with `.ed25519_private` +> may contain either +> a standard ed25519 keypair with SSH type `ed25519` +> or +> an `ed25519-expanded@spec.torproject.org` keypair. + +`ed25519-expanded@spec.torproject.org` +SHOULD NOT appear in RFC4716 *public* key files. +Software which is aware of this key type +MUST NOT generate such public key files +and SHOULD reject them on loading. +(Software handling keys in a type-agnostic manner +MAY, and probably will, process such files without complaint.) + +> These rules are because +> public keys should always be advertised as `ed25519` +> even if the private key is only available as `ed25519-expanded@`: +> this avoids leaking information about the key generation process +> to relying parties, +> and simplifies certification and verification. + +> Arti will provide a utility to convert +> anomalous RFC4716 public key files +> containing keys declared to be of type +> `ed25519-expanded@spec.torproject.org` +> to fully conforming files containg `ed25519` keys. +> In other circumstances Arti will reject such anomalous files. + +The public key data is the same as for the official `ed25519` type: +``` + string wrapper for the following fixed-length data: + byte[32] the actual public key, ENC(A) from RFC8032 3.2 +``` +(Reference: [RFC8032 3.2](https://datatracker.ietf.org/doc/html/rfc8032#section-3.2).) + +The private key data is as follows: +``` + string wrapper for the following fixed-length data: + byte[32] ENC(s) as per RFC8032 3.2, "expanded secret key" + byte[32] `h_b...h_(2b-1)` as per RFC8032 3.3, "separation nonce" +``` +(References: `ENC(s)` in [RFC8032 3.2](https://datatracker.ietf.org/doc/html/rfc8032#section-3.2); +`h_b || ... || h_(2b-1)` as per [RFC8032 3.3](https://datatracker.ietf.org/doc/html/rfc8032#section-3.3).) + +Keys whose `string` wrapper is not of the expected length MUST be rejected. + +> [As with `x25519`, the `string` wrapper is useless](#useless-string). +> We adopt it here for the same reasons. + +This private key format does not provide a way to convey or establish +a corresponding (unexpanded, standard) ed25519 private key `k`. + +> The ed25519 standards define the private key for ed25519 +> as a 32-byte secret `k`. +> `k` is then hashed and processed into parameters used for signing, +> and the public key for verification. +> The `ed25519-expanded@spec.torproject.org` can represent private keys +> which can be used with the ed25519 signature algorithm, +> and for which a corresponding working public key is known, +> but for which there is no known value of `k` +> (the "real ed25519" private key). +> Allowing such keys can be convenient when +> one wishes to find private keys whose public keys have particular patterns, +> for example when trying to find a `.onion` domain +> for a [Tor Hidden Service](rend-spec/). +> This format is also used where +> [blinded ed25519 keys](rend-spec/keyblinding-scheme.md) +> need to be stored. diff --git a/spec/tor-spec/cell-packet-format.md b/spec/tor-spec/cell-packet-format.md new file mode 100644 index 0000000..8f2174d --- /dev/null +++ b/spec/tor-spec/cell-packet-format.md @@ -0,0 +1,192 @@ +<a id="tor-spec.txt-3"></a> + +# Cells (messages on channels) {#cell-packet-format} + +The basic unit of communication on a Tor channel is a "cell". + +Once a TLS connection is established, +the two parties send cells to each other. +Cells are sent serially, one after another. + +Cells may be sent embedded in TLS records of any size, +or divided across TLS records, +but the framing of TLS records MUST NOT leak information +about the type or contents of the cells. + +Most cells are of fixed length, +with the actual length depending +on the negotiated link protocol on the channel. +Below we designate the negotiated protocol as `v`. + +As an exception, [`VERSIONS` cells] are always sent with `v = 0`, +since no version has yet been negotiated. + +<span id="fixed-length-cell">A fixed-length cell has this format:</span> + +| Field | Size in bytes | Notes | +|-----------|-------------------|-------| +| `CircID` | [`CIRCID_LEN(v)`] | | +| `Command` | 1 | | +| `Payload` | [`PAYLOAD_LEN`] | Padded to fit | + +The value of `CIRCID_LEN` depends on the negotiated link protocol. + +Some cells have variable length; +the length of these cells is encoded in their header. + +A variable-length cell has this format: + +| Field | Size in bytes | Notes | +|-----------|-----------------|-------| +| `CircID` | `CIRCID_LEN(v)` | | +| `Command` | 1 | | +| `Length` | 2 | A big-endian integer | +| `Payload` | `Length` | | + +[`CIRCID_LEN(v)`]: ./preliminaries.md#msg-len +[`PAYLOAD_LEN`]: ./preliminaries.md#msg-len +[`VERSIONS` cells]: ./negotiating-channels.md#VERSIONS-cells + +Fixed-length and variable-length cells are distinguished +based on the value of their Command field: + +- Command 7 (`VERSIONS`) is variable-length. +- Every other command less than 128 denotes a fixed-length cell. +- Every command greater than or equal to 128 denotes + a variable-length cell. + +> Historical note: +> +> On version 1 connections, all cells were fixed-length. +> +> On version 2 connections, +> only the `VERSIONS` command was variable-length, +> and all others were fixed-length. +> +> These link protocols are obsolete, +> and implementations SHOULD NOT support them. + +## Interpreting the fields: CircID {#circid} + +The `CircID` field determines which [circuit](./circuit-management.md), +if any, the cell is associated with. +If the cell is not associated with any circuit, +its `CircID` is set to 0. + +> Note that a CircID is a channel-local identifier. +> +> A single multi-hop circuit +> will have a different CircID +> on every channel that is used to transmit its data. + +## Interpreting the fields: Command {#command} + +The `Command` field of a fixed-length cell +holds one of the following values: + +| Value |C|P| Identifier | Description | +|-------|-|-|---------------------|--------------------------------------- | +| 0 |N| | `PADDING` | [Link Padding][PADDING] | +| 1 |Y| | `CREATE` | [Create circuit][CREATE] (deprecated) | +| 2 |Y| | `CREATED` | [Acknowledge CREATE][CREATED] (deprecated) | +| 3 |Y| | `RELAY` | [End-to-end data][RELAY] | +| 4 |Y| | `DESTROY` | [Destroy circuit][DESTROY] | +| 5 |Y| | `CREATE_FAST` | [Create circuit, no public key][CREATE_FAST] | +| 6 |Y| | `CREATED_FAST` | [Acknowledge CREATE_FAST][CREATED_FAST] | +| 8 |N| | `NETINFO` | [Time and address info][NETINFO] | +| 9 |Y| | `RELAY_EARLY` | [End-to-end data; limited][RELAY_EARLY] | +| 10 |Y| | `CREATE2` | [Create circuit][CREATE2] | +| 11 |Y| | `CREATED2` | [Acknowledge CREATED2][CREATED2] | +| 12 |Y|5| `PADDING_NEGOTIATE` | [Padding negotiation][PADDING_NEGOTIATE] | + + +[PADDING]: ./flow-control.md#link-padding +[CREATE]: ./create-created-cells.md#CREATE +[CREATED]: ./create-created-cells.md#CREATE +[RELAY]: ./routing-relay-cells.md#routing-relay-cells +[DESTROY]: ./tearing-down-circuits.md#tearing-down-circuits +[CREATE_FAST]: ./create-created-cells.md#create_fast +[CREATED_FAST]: ./create-created-cells.md#create_fast +[NETINFO]: ./negotiating-channels.md#NETINFO-cells +[RELAY_EARLY]: ./relay-early.md#handling-relay-early-cells +[CREATE2]: ./create-created-cells.md#CREATE +[CREATED2]: ./create-created-cells.md#CREATE +[PADDING_NEGOTIATE]: ./flow-control.md#link-padding + +The variable-length `Command` values are: + +| Value |C|P| Identifier | Description | +|-------|-|-|------------------|----------------------------------------| +| 7 |N| | `VERSIONS` | [Negotiate link protocol][VERSIONS] | +| 128 |N| | `VPADDING` | [Variable-length padding][VPADDING] | +| 129 |N| | `CERTS` | [Certificates][CERTS] | +| 130 |N| | `AUTH_CHALLENGE` | [Challenge value][AUTH_CHALLENGE] | +| 131 |N| | `AUTHENTICATE` | [Authenticate initiator][AUTHENTICATE] | +| 132 |N|n/a| `AUTHORIZE` | (Reserved) | + +[VERSIONS]: ./negotiating-channels.md#VERSIONS-cells +[VPADDING]: ./flow-control.md#link-padding +[CERTS]: ./negotiating-channels.md#CERTS-cells +[AUTH_CHALLENGE]: ./negotiating-channels.md#AUTH-CHALLENGE-cells +[AUTHENTICATE]: ./negotiating-channels.md#AUTHENTICATE-cells + +In the tables above, +**C**=Y indicates that a command must have a nonzero CircId, and +**C**=N indicates that a command must have a zero CircId. +Where given, **P** is the first link protocol version +to support a command. +Commands with no value given for **P** +are supported at least in link protocols 3 and above. + +No other command values are allowed. +Implementations SHOULD NOT send undefined command values. +Upon receiving an unrecognized command, +an implementation MAY silently drop the cell +and MAY terminate the channel with an error. + +> Extensibility note: +> +> When we add new cell command types, +> we define a new link protocol version +> to indicate support for that command. +> +> Therefore, implementations can now safely assume +> that other correct implementations +> will never send them an unrecognized cell command. +> +> Historically, before the link protocol was not versioned, +> implementations would drop cells with unrecognized commands, +> under the assumption that the command was sent +> by a more up-to-date version of Tor. + +## Interpreting the fields: Payload {#payload} + +The interpretation of Payload depends on the cell's command. +see the links in the command descriptions above +for more information on each command. + +## Padding fixed-length cell payloads {#payload-padding} + +Often, the amount of information to be sent +in a fixed-length cell +is less than [`PAYLOAD_LEN`] bytes. +When this happens, +the sender MUST fill the unused part of the payload +with zero-valued bytes. + +Recipients MUST ignore padding bytes. + +> [RELAY] and [RELAY_EARLY] cell payloads +> contain encrypted data, +> and are always full +> from the point of the view of the channel layer. +> +> The _plaintext_ of these cells' contents may be padded; +> this uses a [different mechanism](./relay-cells.md#relay-cell-padding) +> and does not interact with channel payload padding. + +Variable-length cells never have extra space, +so there is no need to pad their payloads. +Unless otherwise specified, +variable-length cells have no padding. + diff --git a/spec/tor-spec/channels.md b/spec/tor-spec/channels.md new file mode 100644 index 0000000..652b931 --- /dev/null +++ b/spec/tor-spec/channels.md @@ -0,0 +1,91 @@ +<a id="tor-spec.txt-2"></a> + +# Channels {#channels} + +A channel is a direct encrypted connection +between two Tor relays, +or between a client and a relay. + +Channels are implemented as +[TLS](https://www.rfc-editor.org/rfc/rfc8446) sessions over TCP. + +Clients and relays may both open new channels; +only a relay may be the recipient of a channel. + +> Historical note: +> in some older documentation, +> channels were sometimes called "connections". +> This proved to be confusing, +> and we are trying not to use the term. + +As part of establishing a channel, +the responding relay will always prove cryptographic ownership +of one or more [**relay identities**](./relay-keys.md), +using a [handshake](./negotiating-channels.md) +that combines TLS facilities +and a series of Tor cells. + +<span id="does-initiator-authenticate"> +As part of this handshake, +the initiator MAY also prove cryptographic ownership +of its own relay identities, +if it has any: +public relays SHOULD prove their identities when they initiate a channel, +whereas clients and bridges SHOULD NOT do so. +</span> + +Parties should usually reuse an existing channel +rather than opening new a channel to the same relay. +There are exceptions here; +we discuss them more below. <!-- TODO: add anchor link --> + +To open a channel, +a client or relay must know the IP address and port +of the target relay. +(This is sometimes called +the "OR address" or "OR port" for the relay.) +In most cases, the participant will also know +one or more expected identities for the target relay, +and will reject the channel +if the target relay cannot cryptographically prove +ownership of those identities. + +> (When initiating a connection, if a reasonably live consensus is +> available, then the expected identity key is taken from that +> consensus. But when initiating a connection otherwise, the expected +> identity key is the one given in the hard-coded authority or +> fallback list. Finally, when creating a connection because of an +> EXTEND/EXTEND2 message, the expected identity key is the one given in +> the message.) + +<!-- TODO: Integrate that paragraph better, or put it in a better place. --> + +Opening a channel is multi-step process: + +1. The initiator opens + a new TLS session with certain properties, + and the responding relay checks and enforces those properties. +2. Both parties + exchange cells over this TLS session + in order to establish their identity or identities. +3. Both parties verify that the identities that they received + are the ones that they expected. + (If any expected key is missing or not as expected, + the party MUST close the connection.) + +Once this is done, the channel is Open, +and regular cells can be exchanged. + +## Channel lifetime + +Channels are not permanent. +Either side MAY close a channel if there are no circuits running on it +and an amount of time (KeepalivePeriod, defaults to 5 minutes) +has passed since the last time any traffic was transmitted over it. +Clients SHOULD +also hold a TLS connection with no circuits open, if it is likely that a +circuit will be built soon using that connection. + +<!-- TODO: Give a forward-reference to explain predicted circuits + and how we decide that a channel is "likely" to be used. --> + diff --git a/spec/tor-spec/circuit-management.md b/spec/tor-spec/circuit-management.md new file mode 100644 index 0000000..d7691a3 --- /dev/null +++ b/spec/tor-spec/circuit-management.md @@ -0,0 +1,6 @@ +<a id="tor-spec.txt-5"></a> + +# Circuit management{#circuit-management} + +This section describes how circuits are created, and how they operate +once they are constructed. diff --git a/spec/tor-spec/closing-streams.md b/spec/tor-spec/closing-streams.md new file mode 100644 index 0000000..6bbcaeb --- /dev/null +++ b/spec/tor-spec/closing-streams.md @@ -0,0 +1,100 @@ +<a id="tor-spec.txt-6.3"></a> + +# Closing streams{#closing-streams} + +When an anonymized TCP connection is closed, or an edge node +encounters error on any stream, it sends a 'RELAY_END' message along the +circuit (if possible) and closes the TCP connection immediately. If +an edge node receives a 'RELAY_END' message for any stream, it closes +the TCP connection completely, and sends nothing more along the +circuit for that stream. + +The payload of a RELAY_END message begins with a single 'reason' byte to +describe why the stream is closing. For some reasons, it contains +additional data (depending on the reason.) The values are: + +```text + 1 -- REASON_MISC (catch-all for unlisted reasons) + 2 -- REASON_RESOLVEFAILED (couldn't look up hostname) + 3 -- REASON_CONNECTREFUSED (remote host refused connection) [*] + 4 -- REASON_EXITPOLICY (OR refuses to connect to host or port) + 5 -- REASON_DESTROY (Circuit is being destroyed) + 6 -- REASON_DONE (Anonymized TCP connection was closed) + 7 -- REASON_TIMEOUT (Connection timed out, or OR timed out + while connecting) + 8 -- REASON_NOROUTE (Routing error while attempting to + contact destination) + 9 -- REASON_HIBERNATING (OR is temporarily hibernating) + 10 -- REASON_INTERNAL (Internal error at the OR) + 11 -- REASON_RESOURCELIMIT (OR has no resources to fulfill request) + 12 -- REASON_CONNRESET (Connection was unexpectedly reset) + 13 -- REASON_TORPROTOCOL (Sent when closing connection because of + Tor protocol violations.) + 14 -- REASON_NOTDIRECTORY (Client sent RELAY_BEGIN_DIR to a + non-directory relay.) + + [*] Older versions of Tor also send this reason when connections are + reset. +``` + +OPs and ORs MUST accept reasons not on the above list, since future +versions of Tor may provide more fine-grained reasons. + +For most reasons, the format of RELAY_END is: + +Reason \[1 byte\] + +For REASON_EXITPOLICY, the format of RELAY_END is: + +```text + Reason [1 byte] + IPv4 or IPv6 address [4 bytes or 16 bytes] + TTL [4 bytes] +``` + +(If the TTL is absent, it should be treated as if it were 0xffffffff. +If the address is absent or is the wrong length, the RELAY_END message +should be processed anyway.) + +Tors SHOULD NOT send any reason except REASON_MISC for a stream that they +have originated. + +Implementations SHOULD accept empty RELAY_END messages, and treat them +as if they specified REASON_MISC. + +Upon receiving a RELAY_END message, the recipient may be sure that no further +messages will arrive on that stream, and can treat such messages as a protocol +violation. + +After sending a RELAY_END message, the sender needs to give the recipient +time to receive that message. In the meantime, the sender SHOULD remember +how many messages of which types (CONNECTED, SENDME, DATA) it would have +accepted on that stream, and SHOULD kill the circuit if it receives more +than permitted. + +--- \[The rest of this section describes unimplemented functionality.\] + +Because TCP connections can be half-open, we follow an equivalent +to TCP's FIN/FIN-ACK/ACK protocol to close streams. + +An exit (or onion service) connection can have a TCP stream in one of +three states: 'OPEN', 'DONE_PACKAGING', and 'DONE_DELIVERING'. For the +purposes of modeling transitions, we treat 'CLOSED' as a fourth state, +although connections in this state are not, in fact, tracked by the +onion router. + +A stream begins in the 'OPEN' state. Upon receiving a 'FIN' from +the corresponding TCP connection, the edge node sends a 'RELAY_FIN' +message along the circuit and changes its state to 'DONE_PACKAGING'. +Upon receiving a 'RELAY_FIN' message, an edge node sends a 'FIN' to +the corresponding TCP connection (e.g., by calling +shutdown(SHUT_WR)) and changing its state to 'DONE_DELIVERING'. + +When a stream in already in 'DONE_DELIVERING' receives a 'FIN', it +also sends a 'RELAY_FIN' along the circuit, and changes its state +to 'CLOSED'. When a stream already in 'DONE_PACKAGING' receives a +'RELAY_FIN' message, it sends a 'FIN' and changes its state to +'CLOSED'. + +If an edge node encounters an error on any stream, it sends a +'RELAY_END' message (if possible) and closes the stream immediately. diff --git a/spec/tor-spec/create-created-cells.md b/spec/tor-spec/create-created-cells.md new file mode 100644 index 0000000..a9b7a12 --- /dev/null +++ b/spec/tor-spec/create-created-cells.md @@ -0,0 +1,620 @@ +<a id="tor-spec.txt-5.1"></a> + +# CREATE and CREATED cells{#CREATE} + +Users set up circuits incrementally, one hop at a time. To create a +new circuit, OPs send a CREATE/CREATE2 cell to the first node, with +the first half of an authenticated handshake; that node responds with +a CREATED/CREATED2 cell with the second half of the handshake. To +extend a circuit past the first hop, the OP sends an EXTEND/EXTEND2 +relay message (see [EXTEND and EXTENDED messages](#EXTEND) which instructs the last +node in the circuit to send a CREATE/CREATE2 cell to extend the circuit. + +There are two kinds of CREATE and CREATED cells: The older +"CREATE/CREATED" format, and the newer "CREATE2/CREATED2" format. The +newer format is extensible by design; the older one is not. + +A CREATE2 cell contains: + +| Field | Description | Size | +| ----- | ----------- | ---- | +| `HTYPE` | Client Handshake Type | 2 bytes | +| `HLEN` | Client Handshake Data Len | 2 bytes | +| `HDATA` | Client Handshake Data | `HLEN` bytes | + +A CREATED2 cell contains: + +| Field | Description | Size | +| ----- | ----------- | ---- | +| `HLEN` | Server Handshake Data Len | 2 bytes | +| `HDATA` | Server Handshake Data | `HLEN` bytes | + + +Recognized HTYPEs (handshake types) are: + +| Value | Description | +| ----- | ----------- | +| 0x0000 | TAP -- the original Tor handshake; see [The "TAP" handshake](#TAP) | +| 0x0001 | reserved | +| 0x0002 | ntor -- the ntor+curve25519+sha256 handshake; see [The "ntor" handshake](#ntor) | +| 0x0003 | ntor-v3 -- ntor extended with extra data; see [The "ntor-v3" handshake](#ntor-v3) | + +The format of a CREATE cell is one of the following: + +| Field | Description | Size | +| ----- | ----------- | ---- | +| `HDATA` | Client Handshake Data | `TAP_C_HANDSHAKE_LEN` bytes | + +or + +| Field | Description | Size | +| ----- | ----------- | ---- | +| `HTAG` | Client Handshake Type Tag | 16 bytes | +| `HDATA` | Client Handshake Data | `TAP_C_HANDSHAKE_LEN`-16 bytes | + +The first format is equivalent to a CREATE2 cell with HTYPE of 'tap' +and length of `TAP_C_HANDSHAKE_LEN`. The second format is a way to +encapsulate new handshake types into the old CREATE cell format for +migration. See ["EXTEND and EXTENDED messages"](#EXTEND) below. Recognized HTAG values are: + +| Value | Description | +| ----- | ----------- | +| 'ntorNTORntorNTOR' | ntor | + +The format of a CREATED cell is: + +| Field | Description | Size | +| ----- | ----------- | ---- | +| `HDATA` | Server Handshake Data | `TAP_S_HANDSHAKE_LEN` bytes | + +(It's equivalent to a CREATED2 cell with length of `TAP_S_HANDSHAKE_LEN`.) + +As usual with DH, `x` and `y` MUST be generated randomly. + +In general, clients SHOULD use CREATE whenever they are using the TAP +handshake, and CREATE2 otherwise. Clients SHOULD NOT send the +second format of CREATE cells (the one with the handshake type tag) +to a server directly. + +Servers always reply to a successful CREATE with a CREATED, and to a +successful CREATE2 with a CREATED2. On failure, a server sends a +DESTROY cell to tear down the circuit. + +\[CREATE2 is handled by Tor 0.2.4.7-alpha and later.\] + +<a id="tor-spec.txt-5.1.1"></a> + +## Choosing circuit IDs in create cells {#choosing-circid} + +The CircID for a CREATE/CREATE2 cell is a nonzero integer, selected +by the node (OP or OR) that sends the CREATE/CREATED2 cell. +Depending on the link protocol version, there are certain rules for +choosing the value of CircID which MUST be obeyed, as implementations +MAY decide to refuse in case of a violation. In link protocol 3 or +lower, CircIDs are 2 bytes long; in protocol 4 or higher, CircIDs are +4 bytes long. + +In link protocol version 3 or lower, the nodes choose from only one +half of the possible values based on the ORs' public identity keys, +in order to avoid collisions. If the sending node has a lower key, +it chooses a CircID with an MSB of 0; otherwise, it chooses a CircID +with an MSB of 1. (Public keys are compared numerically by modulus.) +A client with no public key MAY choose any CircID it wishes, since +clients never need to process CREATE/CREATE2 cells. + +In link protocol version 4 or higher, whichever node initiated the +connection MUST set its MSB to 1, and whichever node didn't initiate +the connection MUST set its MSB to 0. + +The CircID value 0 is specifically reserved for cells that do not +belong to any circuit: CircID 0 MUST not be used for circuits. No +other CircID value, including 0x8000 or 0x80000000, is reserved. + +Existing Tor implementations choose their CircID values at random from +among the available unused values. To avoid distinguishability, new +implementations should do the same. Implementations MAY give up and stop +attempting to build new circuits on a channel, if a certain number of +randomly chosen CircID values are all in use (today's Tor stops after 64). + +<a id="tor-spec.txt-5.1.2"></a> + +## EXTEND and EXTENDED messagess {#EXTEND} + +To extend an existing circuit, the client sends an EXTEND or EXTEND2 +message, in a RELAY_EARLY cell, to the last node in the circuit. + +An EXTEND2 message's relay payload contains: + +| Field | Description | Size | +| ----- | ----------- | ---- | +| `NSPEC` | Number of link specifiers | 1 byte +| `NSPEC` times: | +| - `LSTYPE` | Link specifier type | 1 byte +| - `LSLEN` | Link specifier length | 1 byte +| - `LSPEC` | Link specifier | LSLEN bytes +| `HTYPE` | Client Handshake Type | 2 bytes +| `HLEN` | Client Handshake Data Len | 2 bytes +| `HDATA` | Client Handshake Data | HLEN bytes + +Link specifiers describe the next node in the circuit and how to +connect to it. Recognized specifiers are: + +| Value | Description +| ----- | ----------- +| \[00\] | TLS-over-TCP, IPv4 address. A four-byte IPv4 address plus two-byte ORPort. +| \[01\] | TLS-over-TCP, IPv6 address. A sixteen-byte IPv6 address plus two-byte ORPort. +| \[02\] | Legacy identity. A 20-byte SHA1 identity fingerprint. At most one may be listed. +| \[03\] | Ed25519 identity. A 32-byte Ed25519 identity fingerprint. At most one may be listed. + +Nodes MUST ignore unrecognized specifiers, and MUST accept multiple +instances of specifiers other than 'legacy identity' and +'Ed25519 identity'. (Nodes SHOULD reject link specifier lists +that include multiple instances of either one of those specifiers.) + +For purposes of indistinguishability, implementations SHOULD send +these link specifiers, if using them, in this order: \[00\], \[02\], \[03\], +\[01\]. + +The relay payload for an EXTEND relay message consists of: + +| Field | Size +| ----- | ---- +| Address | 4 bytes +| Port | 2 bytes +| Onion skin | `TAP_C_HANDSHAKE_LEN` bytes +| Identity fingerprint | `HASH_LEN` bytes + +The "legacy identity" and "identity fingerprint" fields are the +SHA1 hash of the PKCS#1 ASN1 encoding of the next onion router's +identity (signing) key. (See ["Preliminaries » Ciphers"](./preliminaries.md#ciphers)) +The "Ed25519 identity" +field is the Ed25519 identity key of the target node. Including +this key information allows the extending OR verify that it is +indeed connected to the correct target OR, and prevents certain +man-in-the-middle attacks. + +Extending ORs MUST check _all_ provided identity keys (if they +recognize the format), and and MUST NOT extend the circuit if the +target OR did not prove its ownership of any such identity key. +If only one identity key is provided, but the extending OR knows +the other (from directory information), then the OR SHOULD also +enforce the key in the directory. + +If an extending OR has a channel with a given Ed25519 ID and RSA +identity, and receives a request for that Ed25519 ID and a +different RSA identity, it SHOULD NOT attempt to make another +connection: it should just fail and DESTROY the circuit. + +The client MAY include multiple IPv4 or IPv6 link specifiers in an +EXTEND message; current OR implementations only consider the first +of each type. + +After checking relay identities, extending ORs generate a +CREATE/CREATE2 cell from the contents of the EXTEND/EXTEND2 message. +See [Creating circuits](./creating-circuits.md#creating-circuits) +for details. + +The payload of an EXTENDED message is the same as the payload of a +CREATED cell. + +The payload of an EXTENDED2 message is the same as the payload of a +CREATED2 cell. + +\[Support for EXTEND2/EXTENDED2 was added in Tor 0.2.4.8-alpha.\] + +Clients SHOULD use the EXTEND format whenever sending a TAP +handshake, and MUST use it whenever the EXTEND message will be handled +by a node running a version of Tor too old to support EXTEND2. In +other cases, clients SHOULD use EXTEND2. + +When generating an EXTEND2 message, clients SHOULD include the target's +Ed25519 identity whenever the target has one, and whenever the +target supports LinkAuth subprotocol version "3". (See [LinkAuth](./subprotocol-versioning.md#link-auth)). + +When encoding a non-TAP handshake in an EXTEND message, clients SHOULD +use the format with 'client handshake type tag'. + +<a id="tor-spec.txt-5.1.3"></a> + +## The "TAP" handshake {#TAP} + +This handshake uses Diffie-Hellman in Z<sub>p</sub> and RSA to compute a set of +shared keys which the client knows are shared only with a particular +server, and the server knows are shared with whomever sent the +original handshake (or with nobody at all). It's not very fast and +not very good. (See Goldberg's "On the Security of the Tor +Authentication Protocol".) + +Define `TAP_C_HANDSHAKE_LEN` as `DH_LEN+KEY_LEN+KP_PAD_LEN`. +Define `TAP_S_HANDSHAKE_LEN` as `DH_LEN+HASH_LEN`. + +The payload for a CREATE cell is an 'onion skin', which consists of +the first step of the DH handshake data (also known as `g^x`). This +value is encrypted using the "legacy hybrid encryption" algorithm +(see ["Preliminaries » A bad hybrid encryption algorithm..."](./preliminaries.md#legacy-hybrid-encryption)) +to the server's onion key, giving a client handshake: + +| Field | Size +| ----- | ---- +| KP-encrypted: +| - Padding | `KP_PAD_LEN` bytes +| - Symmetric key | `KEY_LEN` bytes +| - First part of `g^x` | `KP_ENC_LEN-KP_PAD_LEN-KEY_LEN` bytes +| Symmetrically encrypted +| - Second part of `g^x` | `DH_LEN-(KP_ENC_LEN-KP_PAD_LEN-KEY_LEN)` bytes + +The payload for a CREATED cell, or the relay payload for an +EXTENDED message, contains: + +| Field | Size +| ----- | ---- +| DH data (`g^y`) | `DH_LEN` bytes +| Derivative key data (`KH`) | `HASH_LEN` bytes (see ["Setting Circuit Keys"](./setting-circuit-keys.md)) + + +Once the handshake between the OP and an OR is completed, both can +now calculate `g^xy` with ordinary DH. Before computing `g^xy`, both parties +MUST verify that the received `g^x` or `g^y` value is not degenerate; +that is, it must be strictly greater than `1` and strictly less than `p-1` +where `p` is the DH modulus. Implementations MUST NOT complete a handshake +with degenerate keys. Implementations MUST NOT discard other "weak" +`g^x` values. + +(Discarding degenerate keys is critical for security; if bad keys +are not discarded, an attacker can substitute the OR's CREATED +cell's `g^y` with `0` or `1`, thus creating a known `g^xy` and impersonating +the OR. Discarding other keys may allow attacks to learn bits of +the private key.) + +Once both parties have `g^xy`, they derive their shared circuit keys +and 'derivative key data' value via the +[KDF-TOR function](./setting-circuit-keys.md#kdf-tor). + +<a id="tor-spec.txt-5.1.4"></a> + +## The "ntor" handshake {#ntor} + +This handshake uses a set of DH handshakes to compute a set of +shared keys which the client knows are shared only with a particular +server, and the server knows are shared with whomever sent the +original handshake (or with nobody at all). Here we use the +"curve25519" group and representation as specified in "Curve25519: +new Diffie-Hellman speed records" by D. J. Bernstein. + +\[The ntor handshake was added in Tor 0.2.4.8-alpha.\] + +In this section, define: + +```text +H(x,t) as HMAC_SHA256 with message x and key t. +H_LENGTH = 32. +ID_LENGTH = 20. +G_LENGTH = 32 +PROTOID = "ntor-curve25519-sha256-1" +t_mac = PROTOID | ":mac" +t_key = PROTOID | ":key_extract" +t_verify = PROTOID | ":verify" +G = The preferred base point for curve25519 ([9]) +KEYGEN() = The curve25519 key generation algorithm, returning + a private/public keypair. +m_expand = PROTOID | ":key_expand" +KEYID(A) = A +EXP(a, b) = The ECDH algorithm for establishing a shared secret. +``` + +To perform the handshake, the client needs to know an identity key +digest for the server, and an ntor onion key (a curve25519 public +key) for that server. Call the ntor onion key `B`. The client +generates a temporary keypair: + +```text +x,X = KEYGEN() +``` + +and generates a client-side handshake with contents: + +| Field | Value | Size +| ----- | ----- | ---- +| `NODEID` | Server identity digest | `ID_LENGTH` bytes +| `KEYID` | KEYID(B) | `H_LENGTH` bytes +| `CLIENT_KP` | X | `G_LENGTH` bytes + +The server generates a keypair of `y,Y = KEYGEN()`, and uses its ntor +private key `b` to compute: + +```text +secret_input = EXP(X,y) | EXP(X,b) | ID | B | X | Y | PROTOID +KEY_SEED = H(secret_input, t_key) +verify = H(secret_input, t_verify) +auth_input = verify | ID | B | Y | X | PROTOID | "Server" +``` + +The server's handshake reply is: + +| Field | Value | Size +| ----- | ----- | ---- +| `SERVER_KP` | `Y` | `G_LENGTH` bytes +| `AUTH` | `H(auth_input, t_mac)` | `H_LENGTH` bytes + +The client then checks `Y` is in G<sup>*</sup> \[see NOTE below\], and computes + +```text +secret_input = EXP(Y,x) | EXP(B,x) | ID | B | X | Y | PROTOID +KEY_SEED = H(secret_input, t_key) +verify = H(secret_input, t_verify) +auth_input = verify | ID | B | Y | X | PROTOID | "Server" +``` + +The client verifies that `AUTH == H(auth_input, t_mac)`. + +Both parties check that none of the `EXP()` operations produced the +point at infinity. \[NOTE: This is an adequate replacement for +checking `Y` for group membership, if the group is curve25519.\] + +Both parties now have a shared value for `KEY_SEED`. They expand this +into the keys needed for the Tor relay protocol, using the KDF +described in ["KDF-RFC5869"](./setting-circuit-keys.md#kdf-rfc5869) +and the tag `m_expand`. + +<a id="tor-spec.txt-5.1.4.1"></a> + +### The "ntor-v3" handshake {#ntor-v3} + +This handshake extends the ntor handshake to include support +for extra data transmitted as part of the handshake. Both +the client and the server can transmit extra data; in both cases, +the extra data is encrypted, but only server data receives +forward secrecy. + +To advertise support for this handshake, servers advertise the +"Relay=4" subprotocol version. To select it, clients use the +'ntor-v3' HTYPE value in their CREATE2 cells. + +In this handshake, we define: + +```text +PROTOID = "ntor3-curve25519-sha3_256-1" +t_msgkdf = PROTOID | ":kdf_phase1" +t_msgmac = PROTOID | ":msg_mac" +t_key_seed = PROTOID | ":key_seed" +t_verify = PROTOID | ":verify" +t_final = PROTOID | ":kdf_final" +t_auth = PROTOID | ":auth_final" + +`ENCAP(s)` -- an encapsulation function. We define this +as `htonll(len(s)) | s`. (Note that `len(ENCAP(s)) = len(s) + 8`). + +`PARTITION(s, n1, n2, n3, ...)` -- a function that partitions a +bytestring `s` into chunks of length `n1`, `n2`, `n3`, and so +on. Extra data is put into a final chunk. If `s` is not long +enough, the function fails. + +H(s, t) = SHA3_256(ENCAP(t) | s) +MAC(k, msg, t) = SHA3_256(ENCAP(t) | ENCAP(k) | s) +KDF(s, t) = SHAKE_256(ENCAP(t) | s) +ENC(k, m) = AES_256_CTR(k, m) + +EXP(pk,sk), KEYGEN: defined as in curve25519 + +DIGEST_LEN = MAC_LEN = MAC_KEY_LEN = ENC_KEY_LEN = PUB_KEY_LEN = 32 + +ID_LEN = 32 (representing an ed25519 identity key) + +For any tag "t_foo": + H_foo(s) = H(s, t_foo) + MAC_foo(k, msg) = MAC(k, msg, t_foo) + KDF_foo(s) = KDF(s, t_foo) +``` + +Other notation is as in the [ntor description above](#ntor). + +The client begins by knowing: + +```text +B, ID -- The curve25519 onion key and Ed25519 ID of the server that it + wants to use. +CM -- A message it wants to send as part of its handshake. +VER -- An optional shared verification string: +``` + +The client computes: + +```text +x,X = KEYGEN() +Bx = EXP(B,x) +secret_input_phase1 = Bx | ID | X | B | PROTOID | ENCAP(VER) +phase1_keys = KDF_msgkdf(secret_input_phase1) +(ENC_K1, MAC_K1) = PARTITION(phase1_keys, ENC_KEY_LEN, MAC_KEY_LEN) +encrypted_msg = ENC(ENC_K1, CM) +msg_mac = MAC_msgmac(MAC_K1, ID | B | X | encrypted_msg) +``` + + The client then sends, as its CREATE handshake: + +| Field | Value | Size +| ----- | ----- | ---- +| `NODEID` | `ID` | `ID_LEN` bytes +| `KEYID` | `B` | `PUB_KEY_LEN` bytes +| `CLIENT_PK` | `X` | `PUB_KEY_LEN` bytes +| `MSG` | `encrypted_msg` | `len(CM)` bytes +| `MAC` | `msg_mac` | `MAC_LEN` bytes + +The client remembers `x`, `X`, `B`, `ID`, `Bx`, and `msg_mac`. + +When the server receives this handshake, it checks whether `NODEID` is as +expected, and looks up the `(b,B)` keypair corresponding to `KEYID`. If the +keypair is missing or the `NODEID` is wrong, the handshake fails. + +Now the relay uses `X=CLIENT_PK` to compute: + +```text +Xb = EXP(X,b) +secret_input_phase1 = Xb | ID | X | B | PROTOID | ENCAP(VER) +phase1_keys = KDF_msgkdf(secret_input_phase1) +(ENC_K1, MAC_K1) = PARTITION(phase1_keys, ENC_KEY_LEN, MAC_KEY_LEN) + +expected_mac = MAC_msgmac(MAC_K1, ID | B | X | MSG) +``` + +If `expected_mac` is not `MAC`, the handshake fails. Otherwise +the relay computes `CM` as: + +```text +CM = DEC(MSG, ENC_K1) +``` + +The relay then checks whether `CM` is well-formed, and in response +composes `SM`, the reply that it wants to send as part of the +handshake. It then generates a new ephemeral keypair: + +```text +y,Y = KEYGEN() +``` + +and computes the rest of the handshake: + +```text +Xy = EXP(X,y) +secret_input = Xy | Xb | ID | B | X | Y | PROTOID | ENCAP(VER) +ntor_key_seed = H_key_seed(secret_input) +verify = H_verify(secret_input) + +RAW_KEYSTREAM = KDF_final(ntor_key_seed) +(ENC_KEY, KEYSTREAM) = PARTITION(RAW_KEYSTREAM, ENC_KEY_LKEN, ...) + +encrypted_msg = ENC(ENC_KEY, SM) + +auth_input = verify | ID | B | Y | X | MAC | ENCAP(encrypted_msg) | + PROTOID | "Server" +AUTH = H_auth(auth_input) +``` + +The relay then sends as its CREATED handshake: + +| Field | Value | Size +| ----- | ----- | ---- +| `Y` | `Y` | `PUB_KEY_LEN` bytes +| `AUTH` | `AUTH` | `DIGEST_LEN` bytes +| `MSG` | `encrypted_msg` | `len(SM)` bytes, up to end of the message + +Upon receiving this handshake, the client computes: + +```text +Yx = EXP(Y, x) +secret_input = Yx | Bx | ID | B | X | Y | PROTOID | ENCAP(VER) +ntor_key_seed = H_key_seed(secret_input) +verify = H_verify(secret_input) + +auth_input = verify | ID | B | Y | X | MAC | ENCAP(MSG) | + PROTOID | "Server" +AUTH_expected = H_auth(auth_input) +``` + +If `AUTH_expected` is equal to `AUTH`, then the handshake has +succeeded. The client can then calculate: + +```text +RAW_KEYSTREAM = KDF_final(ntor_key_seed) +(ENC_KEY, KEYSTREAM) = PARTITION(RAW_KEYSTREAM, ENC_KEY_LKEN, ...) + +SM = DEC(ENC_KEY, MSG) +``` + +SM is the message from the relay, and the client uses KEYSTREAM to +generate the shared secrets for the newly created circuit. + +Now both parties share the same KEYSTREAM, and can use it to generate +their circuit keys. + +<a id="tor-spec.txt-5.1.5"></a> + +## CREATE_FAST/CREATED_FAST cells {#create_fast} + +When creating a one-hop circuit, the OP has already +established the OR's identity and negotiated a secret key using TLS. +Because of this, it is not necessary for the OP to perform the +public key operations to create a circuit. +In this case, the OP MAY send a CREATE_FAST cell instead of a CREATE cell. +The OR responds with a CREATED_FAST cell, and the circuit is +created. + +A CREATE_FAST cell contains: + +| Field | Size +| ----- | ---- +| Key material (`X`) | `HASH_LEN` bytes + +A CREATED_FAST cell contains: + +| Field | Size +| ----- | ---- +| Key material (`Y`) | `HASH_LEN` bytes +| Derivative key data | `HASH_LEN` bytes (See [KDF-TOR](./setting-circuit-keys.md#kdf-tor)) + +The values of `X` and `Y` must be generated randomly. + +Once both parties have `X` and `Y`, they derive their shared circuit keys +and 'derivative key data' value via the [KDF-TOR function](./setting-circuit-keys.md#kdf-tor). + +Parties SHOULD NOT use CREATE_FAST except for creating one-hop circuits. + +<a id="tor-spec.txt-5.1.6"></a> + +## Additional data in CREATE/CREATED cells {#additional-data} + +Some handshakes (currently ntor-v3 defined above) allow the client or the +relay to send additional data as part of the handshake. When used in a +CREATE/CREATED handshake, this additional data must have the following +format: + +| Field | Size +| ----- | ---- +| `N_EXTENSIONS` | one byte +| `N_EXTENSIONS` times: | +| - `EXT_FIELD_TYPE` | one byte +| - `EXT_FIELD_LEN` | one byte +| - `EXT_FIELD` | `EXT_FIELD_LEN` bytes + +(`EXT_FIELD_LEN` may be zero, in which case `EXT_FIELD` is absent.) + +All parties MUST reject messages that are not well-formed per the +rules above. + +We do not specify specific TYPE semantics here; we leave those for +other proposals and specifications. + +Parties MUST ignore extensions with `EXT_FIELD_TYPE` bodies they do not +recognize. + +Unless otherwise specified in the documentation for an extension type: + * Each extension type SHOULD be sent only once in a message. + * Parties MUST ignore any occurrences all occurrences of an extension + with a given type after the first such occurrence. + * Extensions SHOULD be sent in numerically ascending order by type. + +(The above extension sorting and multiplicity rules are only defaults; +they may be overridden in the description of individual extensions.) + +Currently supported extensions are: + + * 1 -- `CC_FIELD_REQUEST` \[Client to server\] + + Contains an empty payload. Signifies that the client + wants to use the extended congestion control described + in [proposal 324]. + + * 2 -- `CC_FIELD_RESPONSE` \[Server to client\] + + Indicates that the relay will use the congestion control + of [proposal 324], as requested by the client. One byte + in length: + + `sendme_inc [1 byte]` + + * 3 -- Subprotocol Request \[Client to Server\] + + (RESERVED) Tells the endpoint what protocol version to use on the + circuit ([proposal 346]). + +[proposal 324]: ../proposals/324-rtt-congestion-control.txt +[proposal 346]: ../proposals/346-protovers-again.md diff --git a/spec/tor-spec/creating-circuits.md b/spec/tor-spec/creating-circuits.md new file mode 100644 index 0000000..10e23b4 --- /dev/null +++ b/spec/tor-spec/creating-circuits.md @@ -0,0 +1,102 @@ +<a id="tor-spec.txt-5.3"></a> + +# Creating circuits{#creating-circuits} + +When creating a circuit through the network, the circuit creator +(OP) performs the following steps: + +1. Choose an onion router as an end node (R_N): + * N MAY be 1 for non-anonymous directory mirror, introduction point, + or service rendezvous connections. + * N SHOULD be 3 or more for anonymous connections. + Some end nodes accept streams (see ["Opening streams"](./opening-streams.md)), + others are introduction or rendezvous points (see the [Rendezvous Spec](../rend-spec/index.md)). + +2. Choose a chain of (N-1) onion routers (R_1...R_N-1) to constitute + the path, such that no router appears in the path twice. + +3. If not already connected to the first router in the chain, + open a new connection to that router. + +4. Choose a circID not already in use on the connection with the + first router in the chain; send a CREATE/CREATE2 cell along + the connection, to be received by the first onion router. + +5. Wait until a CREATED/CREATED2 cell is received; finish the + handshake and extract the forward key Kf_1 and the backward + key Kb_1. + +6. For each subsequent onion router R (R_2 through R_N), extend + the circuit to R. + +To extend the circuit by a single onion router R_M, the OP performs +these steps: + +1. Create an onion skin, encrypted to R_M's public onion key. + +2. Send the onion skin in a relay EXTEND/EXTEND2 message along + the circuit (see + ["EXTEND and EXTENDED messages"](./create-created-cells.md#EXTEND) + and ["Routing relay cells"](./routing-relay-cells.md#routing-relay-cells)). + +3. When a relay EXTENDED/EXTENDED2 message is received, verify KH, + and calculate the shared keys. The circuit is now extended. + +When an onion router receives an EXTEND relay message, it sends a CREATE +cell to the next onion router, with the enclosed onion skin as its +payload. + +When an onion router receives an EXTEND2 relay message, it sends a CREATE2 +cell to the next onion router, with the enclosed HLEN, HTYPE, and HDATA +as its payload. The initiating onion router chooses some circID not yet +used on the connection between the two onion routers. (But see section +["Choosing circuit IDs in create cells"](./create-created-cells.md#choosing-circid)) + +As special cases, if the EXTEND/EXTEND2 message includes a legacy identity, or +identity fingerprint of all zeroes, or asks to extend back to the relay +that sent the extend cell, the circuit will fail and be torn down. + +Ed25519 identity keys are not required in EXTEND2 messages, so all zero +keys SHOULD be accepted. If the extending relay knows the ed25519 key from +the consensus, it SHOULD also check that key. (See +[EXTEND and EXTENDED message](./create-created-cells.md#EXTEND)) + +If an EXTEND2 message contains the ed25519 key of the relay that sent the +EXTEND2 message, the circuit will fail and be torn down. + +When an onion router receives a CREATE/CREATE2 cell, if it already has a +circuit on the given connection with the given circID, it drops the +cell. Otherwise, after receiving the CREATE/CREATE2 cell, it completes +the specified handshake, and replies with a CREATED/CREATED2 cell. + +Upon receiving a CREATED/CREATED2 cell, an onion router packs it payload +into an [EXTENDED/EXTENDED2](./create-created-cells.md#EXTEND) relay message, and sends +that message up the circuit. Upon receiving the EXTENDED/EXTENDED2 relay +message, the OP can retrieve the handshake material. + +(As an optimization, OR implementations may delay processing onions +until a break in traffic allows time to do so without harming +network latency too greatly.) + +<a id="tor-spec.txt-5.3.1"></a> + +## Canonical connections{#canonical-connections} + +It is possible for an attacker to launch a man-in-the-middle attack +against a connection by telling OR Alice to extend to OR Bob at some +address X controlled by the attacker. The attacker cannot read the +encrypted traffic, but the attacker is now in a position to count all +bytes sent between Alice and Bob (assuming Alice was not already +connected to Bob.) + +To prevent this, when an OR gets an extend request, it SHOULD use an +existing OR connection if the ID matches, and ANY of the following +conditions hold: + + - The IP matches the requested IP. + - The OR knows that the IP of the connection it's using is canonical + because it was listed in the NETINFO cell. + +ORs SHOULD NOT check the IPs that are listed in the server descriptor. +Trusting server IPs makes it easier to covertly impersonate a relay, after +stealing its keys. diff --git a/spec/tor-spec/flow-control.md b/spec/tor-spec/flow-control.md new file mode 100644 index 0000000..8c3bcce --- /dev/null +++ b/spec/tor-spec/flow-control.md @@ -0,0 +1,199 @@ +<a id="tor-spec.txt-7"></a> + +# Flow control{#flow-control} + +<a id="tor-spec.txt-7.1"></a> + +## Link throttling + +Each client or relay should do appropriate bandwidth throttling to +keep its user happy. + +Communicants rely on TCP's default flow control to push back when they +stop reading. + +The mainline Tor implementation uses token buckets (one for reads, +one for writes) for the rate limiting. + +Since 0.2.0.x, Tor has let the user specify an additional pair of +token buckets for "relayed" traffic, so people can deploy a Tor relay +with strict rate limiting, but also use the same Tor as a client. To +avoid partitioning concerns we combine both classes of traffic over a +given OR connection, and keep track of the last time we read or wrote +a high-priority (non-relayed) cell. If it's been less than N seconds +(currently N=30), we give the whole connection high priority, else we +give the whole connection low priority. We also give low priority +to reads and writes for connections that are serving directory +information. See [proposal 111] for details. + +[proposal 111]: ../proposals/111-local-traffic-priority.txt + +<a id="tor-spec.txt-7.2"></a> + +## Link padding{#link-padding} + +Link padding can be created by sending PADDING or VPADDING cells +along the connection; relay messages of type "DROP" can be used for +long-range padding. The payloads of PADDING cells, VPADDING cells, or DROP +message are filled with padding bytes. +See [Cell Packet format](./cell-packet-format.md#cell-packet-format). + +If the link protocol is version 5 or higher, link level padding is +enabled as per padding-spec.txt. On these connections, clients may +negotiate the use of padding with a PADDING_NEGOTIATE command +whose format is as follows: + +```text + Version [1 byte] + Command [1 byte] + ito_low_ms [2 bytes] + ito_high_ms [2 bytes] +``` + +Currently, only version 0 of this cell is defined. In it, the command +field is either 1 (stop padding) or 2 (start padding). For the start +padding command, a pair of timeout values specifying a low and a high +range bounds for randomized padding timeouts may be specified as unsigned +integer values in milliseconds. The ito_low_ms field should not be lower +than the current consensus parameter value for nf_ito_low (default: +1500). The ito_high_ms field should not be lower than ito_low_ms. +(If any party receives an out-of-range value, they clamp it so +that it is in-range.) + +For the stop padding command, the timeout fields should be sent as +zero (to avoid client distinguishability) and ignored by the recipient. + +For more details on padding behavior, see padding-spec.txt. + +<a id="tor-spec.txt-7.3"></a> + +## Circuit-level flow control + +To control a circuit's bandwidth usage, each OR keeps track of two +'windows', consisting of how many DATA-bearing relay cells it is allowed to +originate or willing to consume. + +(For the purposes of flow control, +we call a relay cell "DATA-bearing" +if it holds a DATA relay message. +Note that this design does _not_ limit relay cells that don't contain +a DATA message; +this limitation may be addressed in the future.) + +These two windows are respectively named: the package window (packaged for +transmission) and the deliver window (delivered for local streams). + +Because of our leaky-pipe topology, every relay on the circuit has a pair +of windows, and the OP has a pair of windows for every relay on the +circuit. +These windows apply only to _originated_ and _consumed_ cells. +They do not, however, apply to _relayed_ cells, +and a relay +that is never used for streams will never decrement its windows or cause the +client to decrement a window. + +Each 'window' value is initially set based on the consensus parameter +'circwindow' in the directory (see dir-spec.txt), or to 1000 +DATA-bearing relay cells if +no 'circwindow' value is given. In each direction, cells that are not +RELAY_DATA cells do not affect the window. + +An OR or OP (depending on the stream direction) sends a RELAY_SENDME message +to indicate that it is willing to receive more DATA-bearing cells when its deliver +window goes down below a full increment (100). For example, if the window +started at 1000, it should send a RELAY_SENDME when it reaches 900. + +When an OR or OP receives a RELAY_SENDME, it increments its package window +by a value of 100 (circuit window increment) and proceeds to sending the +remaining DATA-bearing cells. + +If a package window reaches 0, the OR or OP stops reading from TCP +connections for all streams on the corresponding circuit, and sends no more +DATA-bearing cells until receiving a RELAY_SENDME message. + +If a deliver window goes below 0, the circuit should be torn down. + +Starting with tor-0.4.1.1-alpha, authenticated SENDMEs are supported +(version 1, see below). This means that both the OR and OP need to remember +the rolling digest of the relay cell that precedes (triggers) a RELAY_SENDME. +This can be known if the package window gets to a multiple of the circuit +window increment (100). + +When the RELAY_SENDME version 1 arrives, it will contain a digest that MUST +match the one remembered. This represents a proof that the end point of the +circuit saw the sent relay cells. On failure to match, the circuit should be torn +down. + +To ensure unpredictability, random bytes should be added to at least one +RELAY_DATA cell within one increment window. In other word, +at every 100 data-bearing cells (increment), +random bytes should be introduced in at least one cell. + +<a id="tor-spec.txt-7.3.1"></a> + +### SENDME Message Format + +A circuit-level RELAY_SENDME message always has its StreamID=0. + +An OR or OP must obey these two consensus parameters in order to know which +version to emit and accept. + +```text + 'sendme_emit_min_version': Minimum version to emit. + 'sendme_accept_min_version': Minimum version to accept. +``` + +If a RELAY_SENDME version is received that is below the minimum accepted +version, the circuit should be closed. + +The RELAY_SENDME payload contains the following: + +```text + VERSION [1 byte] + DATA_LEN [2 bytes] + DATA [DATA_LEN bytes] +``` + +The VERSION tells us what is expected in the DATA section of length +DATA_LEN and how to handle it. The recognized values are: + +0x00: The rest of the payload should be ignored. + +0x01: Authenticated SENDME. The DATA section MUST contain: + +DIGEST \[20 bytes\] + +```text + If the DATA_LEN value is less than 20 bytes, the message should be + dropped and the circuit closed. If the value is more than 20 bytes, + then the first 20 bytes should be read to get the DIGEST value. + + The DIGEST is the rolling digest value from the DATA-bearing relay cell that + immediately preceded (triggered) this RELAY_SENDME. This value is + matched on the other side from the previous cell sent that the OR/OP + must remember. + + (Note that if the digest in use has an output length greater than 20 + bytes—as is the case for the hop of an onion service rendezvous + circuit created by the hs_ntor handshake—we truncate the digest + to 20 bytes here.) +``` + +If the VERSION is unrecognized or below the minimum accepted version (taken +from the consensus), the circuit should be torn down. + +<a id="tor-spec.txt-7.4"></a> + +## Stream-level flow control + +Edge nodes use RELAY_SENDME messages to implement end-to-end flow +control for individual connections across circuits. Similarly to +circuit-level flow control, edge nodes begin with a window of +DATA-bearing cells +(500) per stream, and increment the window by a fixed value (50) +upon receiving a RELAY_SENDME message. Edge nodes initiate RELAY_SENDME +messages when both a) the window is \<= 450, and b) there are less than +ten cell payloads remaining to be flushed at that edge. + +Stream-level RELAY_SENDME messages are distinguished by having nonzero +StreamID. They are still empty; the body still SHOULD be ignored. diff --git a/spec/tor-spec/index.md b/spec/tor-spec/index.md new file mode 100644 index 0000000..219ea1d --- /dev/null +++ b/spec/tor-spec/index.md @@ -0,0 +1,12 @@ +# Tor Protocol Specification + +Note: This document aims to specify Tor as currently implemented, though it +may take it a little time to become fully up to date. Future versions of Tor +may implement improved protocols, and compatibility is not guaranteed. +We may or may not remove compatibility notes for other obsolete versions of +Tor as they become obsolete. + +This specification is not a design document; most design criteria +are not examined. For more information on why Tor acts as it does, +see tor-design.pdf. + diff --git a/spec/tor-spec/negotiating-channels.md b/spec/tor-spec/negotiating-channels.md new file mode 100644 index 0000000..4c1ae1c --- /dev/null +++ b/spec/tor-spec/negotiating-channels.md @@ -0,0 +1,516 @@ +<a id="tor-spec.txt-4"></a> + +# Negotiating and initializing channels {#negotiating} + +Here we describe the primary TLS behavior +used by Tor relays and clients to create a new channel. +There are older versions of these handshakes, +which we describe in [another section](./obsolete-channels.md). + +In brief: + - The initiator starts the handshake + by [opening a TLS connection](#tls). + - Both parties send a [VERSIONS](#VERSIONS-cells) + to negotiate the protocol version to use. + - The responder sends a + [CERTS cell](#CERTS-cells) to give the + initiator the certificates it needs to learn the responder's + identity, an [AUTH_CHALLENGE cell](#AUTH-CHALLENGE-cells) + that the initiator must include + as part of its answer if it chooses to authenticate, and a + [NETINFO cell](#NETINFO-cells) + to establish clock skew and IP addresses. + - The initiator checks whether the CERTS cell is correct, + and decides whether to authenticate. + - If the initiator + [is not authenticating itself](./channels.md#does-initiator-authenticate), + it sends a [NETINFO cell](#NETINFO-cells). + - If the initiator + [is authenticating itself](./channels.md#does-initiator-authenticate), + it sends a [CERTS cell](#CERTS-cells), + an [AUTHENTICATE cell](#AUTHENTICATE-cells), + a [NETINFO cell](#NETINFO-cells). + +When this handshake is in use, +the first cell must be VERSIONS, VPADDING, or AUTHORIZE, +and no other cell type is allowed to intervene besides those specified, +except for VPADDING cells. + +(The AUTHORIZE cell type is reserved for future use +by scanning-resistance designs. +It is not specified here.) + +```mermaid +sequenceDiagram + Initiator --> Responder: TLS Handshake + + Note over Initiator,Responder: The rest is encrypted + + Initiator ->> Responder: VERSIONS + Responder ->> Initiator: VERSIONS, CERTS, AUTH_CHALLENGE, NETINFO + + opt if the initiator is authenticating + Initiator ->> Responder: CERTS, AUTHENTICATE + end + + Initiator ->> Responder: NETINFO +``` + + + +## The TLS handshake {#tls} + +<span id="in-protocol">The +initiator must send a ciphersuite list containing +at least one ciphersuite other than +[those listed in the obsolete v1 handshake](./obsolete-channels.md#v1-ciphersuite-list). +</span> + +> This is trivially achieved +> by using any modern TLS implementation, +> and most implementations will not need to worry about it. +> +> This requirement distinguishes the current protocol +> (sometimes called the "in-protocol" or "v3" handshake) +> from the obsolete v1 protocol. + +<a id="tor-spec.txt-2.2"></a> + +### TLS security considerations + +(Standard TLS security guarantees apply; +this is not a comprehensive guide.) + +Implementations SHOULD NOT allow TLS session resumption -- it can +exacerbate some attacks (e.g. the "Triple Handshake" attack from +Feb 2013), and it plays havoc with forward secrecy guarantees. + +Implementations SHOULD NOT allow TLS compression -- although we don't +know a way to apply a CRIME-style attack to current Tor directly, +it's a waste of resources. + +<a id="tor-spec.txt-4.1"></a> + +## Negotiating versions with VERSIONS cells {#VERSIONS-cells} + +There are multiple instances of the Tor channel protocol. + +Once the TLS handshake is complete, +both parties send a VERSIONS cell +to negotiate which one they will use. + +The payload in a VERSIONS cell is a series of big-endian two-byte +integers. +Both parties MUST select as the link protocol version the +highest number contained both in the VERSIONS cell they sent and in the +VERSIONS cell they received. +If they have no such version in common, +they cannot communicate and MUST close the connection. +Either party MUST +close the connection if the VERSIONS cell is not well-formed (for example, +if the payload contains an odd number of bytes). + +Any VERSIONS cells sent after the first VERSIONS cell MUST be ignored. +(To be interpreted correctly, later VERSIONS cells MUST have a CIRCID_LEN +matching the version negotiated with the first VERSIONS cell.) + +> (The +> [obsolete v1 channel protocol](./obsolete-channels.md#v1-handshake) +> does note VERSIONS cells. +> Implementations MUST NOT list version 1 in their VERSIONS cells. +> The +> [obsolete v2 channel protocol](./obsolete-channels.md#v2-handshake) +> can only be used after renegotiation; +> implementations MUST NOT list version 2 in their VERSIONS cells +> unless they have renegotiated the TLS session.) + +The currently specified [Link](./subprotocol-versioning.md#link) protocols are: + +| Version | Description +| ------- | ----------- +| 1 | (Obsolete) The ["certs up front"](./obsolete-channels.md#v1-handshake) handshake. +| 2 | (Obsolete) Uses the [renegotiation-based handshake](./obsolete-channels.md#v2-handshake). Introduces variable-length cells. +| 3 | Uses the [current ("in-protocol") handshake](#in-protocol). Must always be advertised. +| 4 | Increases circuit ID width to 4 bytes. +| 5 | Adds support for [link padding](../padding-spec) and negotiation. + + +<a id="tor-spec.txt-4.2"></a> + +## CERTS cells {#CERTS-cells} + +The CERTS cell describes the keys +that a Tor instance is claiming to have, +and provides certificates to authenticate that those keys +belong, ultimately, to one or more +[identity keys](./relay-keys.md#identity). + +CERTS is a variable-length cell. Its payload format is: + +| Field | Size | Description | +| ----- | ---- | ------------------------------ | +| N | 1 | Number of certificates in cell | +| N times: | | | +| - CertType | 1 | Type of certificate | +| - CertLen | 2 | Length of "Certificate" field | +| - Certificate | CertLen | Encoded certificate | + +Any extra octets at the end of a CERTS cell MUST be ignored. + +The CertType field determines +the format of the certificate, +and the roles of its keys within the Tor protcol. +Recognized values are defined in +["Certificate types (CERT_TYPE field)"](../cert-spec.md#list-cert-types). + +A CERTS cell MUST have no more than one certificate of any CertType. + +### Authenticating the responder from its CERTS {#auth-responder} + +The responder's CERTS cell is as follows: + +- The CERTS cell contains exactly one CertType 4 Ed25519 + `IDENTITY_V_SIGNING_CERT`. + - This cert must be self-signed; + the signing key must be included in a + ["signed-with-ed25519-key" extension](../cert-spec.md#signed-with-ed25519) + extension. + This signing key is `KP_relayid_ed`. + The subject key is `KP_relaysign_ed`. +- The CERTS cell contains exactly one CertType 5 Ed25519 + `SIGNING_V_TLS_CERT` certificate. + - This cert must be signed with `KP_relaysign_ed`. + Its subject must be the SHA-256 digest + of the TLS certificate + that was presented curing the TLS handshake. +- All of the certs above must be correctly signed, and not expired. + +The initiator must check all of the above. +If this is successful +the initiator knows that the responder +has the identity `KP_relayid_ed`. + +> The responder's CERTS cell is meant to prove +> that the responder posses one or more +> [relay identities](./relay-keys.md#identity). +> It does this by containing certificate chains +> from each relay identity key +> to the TLS certificate presented during the TLS handshake. + +> The responder's ownership of that TLS certificate +> was already proven during the TLS hadnshake itself. + +### Validating an initiator's CERTS {#validate-initiator-certs} + +When required +by [other parts of this specification](./channels.md#does-initiator-authenticate); +to prove its identity, +the initiator must provide a CERTS cell. + +> Recall that +> [not all initiators authenticate themselves](./channels.md#does-initiator-authenticate); +> bridges and clients do not prove their identity. + +The initiator's CERTS cell must conform to the rules +for the responder's CERTS cell (see above, +exchanging "initiator" and "responder") +except that: + +**Instead** of containg a `SIGNING_V_TLS_CERT`, + +- The CERTS cell contains exactly one CertType 6 + `SIGNING_V_LINK_AUTH` certificate. + - This certificate must be signed with `KP_relayid_ed`. + (Its subject key is deemed to be `KP_link_ed`.) +- All of the certs above must be correctly signed, and not expired. + +The responder must check all of the CERTS cell's properties +(as stated here, and in the previous section). +If this is successful +**and** +the initiator later sends a valid +[AUTHENTICATE cell](#AUTHENTICATE-cells), +then the initiator has ownership of the presented `KP_relayid_ed`. + +> Note that +> the CERTS cell is _not_ yet sufficent to authenticate the channel, +> until AUTHENTICATE is received: +> unlike the responder, +> the initiator is not required to present a TLS certificate +> during the TLS handshake. +> Therefore, the initiator has +> no meaningful `SIGNING_V_TLS_CERT` certificate. +> +> Therefore, instead, the initiator's CERTS cell +> proves a chain from the initiator's relay identities +> to a "link authentication" key. +> This key is later used to sign an "authentication challenge", +> and bind it to the channel. + +### Authenticating an RSA identity (#auth-RSA) + +After processing a CERTS cell +to find the other party's +`KP_relayid_ed` Ed25519 identity key, +a Tor instance MAY *additionally* check the CERTS cell +to find the other party's +`KP_relayid_rsa` legacy RSA identity key. + +A party with a given `KP_relayid_ed` identity key +also has a given `KP_relayid_rsa` legacy identity key +when all of the following are true. +(A party MUST NOT conclude that an RSA identity key +is associated with a channel +without checking these properties.) + +- The CERTS cell contains exactly one CertType 2 + `RSA_ID_X509` certificate. + - This must be a self-signed certificate containing a 1024-bit RSA key; + that key's exponent must be 65537. + That key is `KP_relayid_rsa`. +- The CERTS cell contains exactly one CertType 7 + `RSA_ID_V_IDENTITY` certificate. + - This certificate must be signed with `KP_relayid_rsa`. + - This certificate's subject key must be the same + as an already-authenticated `KP_relayid_ed`. +- All of the certs above must be correctly signed, + not expired, + and not before their `validAfter` dates. + +If the above tests all pass, +then any relay which can prove it has the the identity `KP_relayid_ed` +also has the legacy identity `KP_relayid_rsa`. + + +<a id="tor-spec.txt-4.3"></a> + +## AUTH_CHALLENGE cells{#AUTH-CHALLENGE-cells} + +An AUTH_CHALLENGE cell is a variable-length cell with the following +fields: + +| Field | Size +| ----- | ---- +| Challenge | 32 octets +| N_Methods | 2 octets +| Methods | 2 * N_Methods octets + +It is sent from the responder to the initiator. +Initiators MUST ignore unexpected bytes at the end of the cell. +Responders MUST generate every challenge independently. + +The Challenge field is +a [randomly generated](./preliminaries.md#random-values) +binary string that the initiator must sign (a hash of) +as part of their [AUTHENTICATE cell](#AUTHENTICATE-cells). + +The Methods are a list of authentication methods +that the responder will accept. +These methods are defined: + +| Type | Method | +| --------- | ------ | +| `[00 01]` | [RSA-SHA256-TLSSecret] (Obsolete) | +| `[00 02]` | (Historical, never implemented) | +| `[00 03]` | [Ed25519-SHA256-RFC5705] | + + +[RSA-SHA256-TLSSecret]: ./obsolete-channels.md#RSA-SHA256-TLSSecret +[Ed25519-SHA256-RFC5705]: #Ed25519-SHA256-RFC5705 + +<a id="tor-spec.txt-4.4"></a> + +## AUTHENTICATE cells{#AUTHENTICATE-cells} + +To authenticate, an initiator MUST +it respond to the AUTH_CHALLENGE cell +with a CERTS cell and an AUTHENTICATE cell. + +> Recall that initiators are +> [not always required to authenticate](./channels.md#does-initiator-authenticate). +> +> ([As discussed above](#validate-initiator-certs), +> the initiator's CERTS cell differs slightly +> from what a responder would send.) + +An AUTHENTICATE cell contains the following: + +| Field | Size +| ----- | ---- +| AuthType | 2 +| AuthLen | 2 +| Authentication | AuthLen + +Responders MUST ignore extra bytes at the end of an AUTHENTICATE +cell. + +The `AuthType` value corresponds to one of the +authentication methods. +The initiator MUST NOT send an AUTHENTICATE cell +whose AuthType was not contained +in the responder's AUTH_CHALLENGE. + +An initiator MUST NOT send an AUTHENTICATE +cell before it has verified the certificates +presented in the responder's CERTS cell, +and authenticated the responder. + +<a id="tor-spec.txt-4.4.2"></a> + +### Link authentication type 3: Ed25519-SHA256-RFC5705 {#Ed25519-SHA256-RFC5705} + +If AuthType is `[00 03]`, +meaning "Ed25519-SHA256-RFC5705", +the Authentication field of the AUTHENTICATE cell is as follows + +Modified values and new fields below are marked with asterisks. + +| Field | Size | Summary | +| --------- | ---- | ----------- | +| `TYPE` | 8 | The nonterminated string `AUTH0003` | +| `CID` | 32 | `SHA256(KP_relayid_rsa)` for initiator | +| `SID` | 32 | `SHA256(KP_relayid_rsa)` for responder | +| `CID_ED` | 32 | `KP_relayid_ed` for initiator | +| `SID_ED` | 32 | `KP_relayid_ed` for responder | +| `SLOG` | 32 | Responder log digest, SHA256 | +| `CLOG` | 32 | Initiator log digest, SHA256 | +| `SCERT` | 32 | SHA256 of responder's TLS certificate | +| `TLSSECRETS`|32 | RFC5705 information | +| `RAND` | 24 | [Random bytes] | +| `SIG` | 64 | Ed25519 signature | + + +- The `TYPE` string distinguishes this authentication document from others. + It must be the nonterminated 8-byte string `AUTH0003`. +- For `CID` and `SID`, the SHA256 digest of an RSA key + is computed as the SHA256 digest of its asn.1 encoding. +- The `SLOG` field is computed + as the SHA256 digest + of all bytes sent within the TLS channel up to and including + the AUTH_CHALLENGE cell. + - This includes the VERSIONS cell, + the CERTS cell, + the AUTH_CHALLENGE cell, + and any padding cells. +- The `CLOG` field is computed + as the SHA256 digest + of all bytes sent within the TLS channel up to but not including + the AUTHENTICATE cell. + - This includes the VERSIONS cell, + the CERTS cell, and any padding cells. +- The `SCERT` field holds the SHA256 digest + of the X.509 certificate presented by the responder + as part of the TLS negotiation. +- The `TLSSECRETS` field is computed + as the output of a Keying Material Exporter function + on the TLS section. + - The parameters for this exporter are: + - Label string: "EXPORTER FOR TOR TLS CLIENT BINDING AUTH0003" + - Context value: The initiator's `KP_relayid_ed`. + - Length: 32. + - For keying material exporters on TLS 1.3, + see [RFC 8446 Section 7.5]. + - For keying material exporters on older TLS versions, + see [RFC5705]. +- The `RAND` field is a uniform squence of [Random bytes]. +- The `SIG` field is an Ed25519 signature + of all earlier members in the Authentication + (from `TYPE` through `RAND`) + using `KS_link_ed`. + + +[Random bytes]: ./preliminaries.md#random-values +[RFC 8446 Section 7.5]: https://datatracker.ietf.org/doc/html/rfc8446#section-7.5 +[RFC5705]: https://datatracker.ietf.org/doc/html/rfc5705. + + +To check an AUTHENTICATE cell, +a responder checks that all fields from TYPE through TLSSECRETS +contain their unique correct values as described above, +and then verifies the signature. +The responder MUST ignore any extra bytes in the signed data after +the RAND field. + +<!-- TODO: We should consider removing that last sentence. --> + +<a id="tor-spec.txt-4.5"></a> + +## NETINFO cells {#NETINFO-cells} + +To finish the handshake, +each party sends the other +a NETINFO cell. + +The cell's payload is: + +| Field | Description | Size +| ----- | ----------- | ---- +| TIME | Timestamp | 4 bytes +| OTHERADDR: | Other party's address | +| - ATYPE | Address type | 1 byte +| - ALEN | Address length | 1 byte +| - AVAL | Address value | ALEN bytes +| NMYADDR | Number of this party's addresses | 1 byte +| NMYADDR times: | | +| - ATYPE | Address type | 1 byte +| - ALEN | Address length | 1 byte +| - AVAL | Address value | ALEN bytes + +Recognized address types (ATYPE) are: + +| ATYPE | Description +| ----- | ----------- +| 0x04 | IPv4 +| 0x06 | IPv6 + +Implementations SHOULD ignore addresses with unrecognized types. + +ALEN MUST be 4 when ATYPE is 0x04 (IPv4) +and 16 when ATYPE is 0x06 (IPv6). +If the ALEN value is wrong for the given ATYPE value, then +the provided address should be ignored. + +The `OTHERADDR` field SHOULD be set to the actual IP address +observed for the other party. + +> (This is typically the address passed to `connect()` +> when acting as the channel initiator, +> or the address received from `accept()` +> when acting as the channel responder.) + +In the `ATYPE`/`ALEN`/`AVAL` fields, +relays SHOULD send the addresses that they have advertised +in their router descriptors. +Bridges and clients SHOULD send none of their own addresses. + +For the `TIME` field, +relays send a (big-endian) integer +holding the number of seconds since the Unix epoch. +Clients SHOULD send `[00 00 00 00]` as their timestamp, +to avoid fingerprinting. + +> See [proposal 338](../proposals/338-netinfo-y2038.md) +> for a proposal to extend the timestamp to 8 bytes. + +Implementations MUST ignore unexpected bytes at the end of the NETINFO cell. + +### Using information from NETINFO cells {#using-netinfo} + +Implementations MAY use the timestamp value to help decide if their +clocks are skewed. + +Initiators MAY use "other OR's address" field +to help learn which address their connections may be originating from, +if they do not know it; +and to learn whether the peer will treat the current connection as +canonical. +(See [Canonical connections](./creating-circuits.md#canonical-connections)) + +Implementations SHOULD NOT trust these values unconditionally, +especially when they come from non-authorities, +since the other party can lie about the time +or the IP addresses it sees. + +Initiators SHOULD use "this OR's address" to make sure +that they have connected to another OR at its +[canonical address](./creating-circuits.md#canonical-connections). diff --git a/spec/tor-spec/obsolete-channels.md b/spec/tor-spec/obsolete-channels.md new file mode 100644 index 0000000..0bd47f8 --- /dev/null +++ b/spec/tor-spec/obsolete-channels.md @@ -0,0 +1,269 @@ +# Obsolete channel handshakes + +These handshake variants are no longer in use. +Channel initiators MUST NOT send them. +Relays MAY detect and reject them. + +> If you are experienced with TLS, +> you will find some aspects of this handshake +> strange or obfuscated. +> Several historical factors led to its current state. +> +> First, before the development of +> [pluggable transports](../pt-spec/), +> Tor tried to avoid censorship by mimicking the behavior +> of a web client negotiating HTTPS with a web server. +> If we wanted a secure option that was not in common use, +> we had to hide our use of that option. +> +> Second, prior to the introduction of +> [TLS 1.3](https://datatracker.ietf.org/doc/html/rfc8446), +> many more aspects of the handshake +> (such as the number and nature of certificates sent by each party) +> were sent in the clear, +> and were easy to distinguish. +> +> Third, prior to the introduction of TLS 1.3, +> there was no good encrypted signalling mechanism +> that a client could use to declare +> how it wanted the rest of the TLS handshake to proceed. +> Thus, we wound up using the client's list +> of supported ciphersuites +> to send a signal about which variation of the handshake is in use. + + +### Version 1, or "certificates up front" {#v1-handshake} + +With this obsolete handshake, +the responding relay proves ownership of an RSA identity (`KP_relayid_rsa`), +and the initiator also proves ownership of an RSA identity. + +(If the initiator does not have an RSA identity to prove, +it invents one and throws it away afterwards.) + +To select this handshake, +the initiator starts a TLS handshake +containing no ciphersuites other than these: + +<a id="v1-ciphersuite-list"></a> +```text + TLS_DHE_RSA_WITH_AES_256_CBC_SHA + TLS_DHE_RSA_WITH_AES_128_CBC_SHA + SSL_DHE_RSA_WITH_3DES_EDE_CBC_SHA +``` + +> Note that because of this list, +> it is impossible to use this obsolete handshake +> with TLS 1.3. + + +As part of the TLS handshake, +the initiator sends a two-certificate chain, +consisting of an X.509 certificate +for its short-term connection public key (`KP_legacy_conn_tls`) +signed by `KP_relayid_rsa`, and +a second self-signed X.509 certificate containing `KP_relayid_rsa`. +The responder sends a similar +certificate chain. + +Once the TLS handshake is done, +both parties validate each other's certificate chains. +If they are valid, +then the connection is Open, +and both parties may start exchanging [cells](./cell-packet-format.md). + +## Version 2, or "renegotiation" {#v2-handshake} + +In "renegotiation" (a.k.a. "the v2 handshake"), +the connection initiator selects at least one ciphersuite +not in the [list above](#v1-ciphersuite-list). +The initiator sends no certificates, and +the responder sends a single connection certificate in return. + +(If the responder sends a certificate chain, +the initiator assumes that it only knows about the v1 handshake.) + +Once this initial TLS handshake is complete, +the initiator renegotiates the session. +During the renegotiation, +each party sends a two-certificate chain +as in the ["certificates up front"](#v1-handshake) handshake above. + +When this handshake is used, +both parties immediately +send a VERSIONS cell, and after negotiating a link +protocol version (which will be 2), each sends a NETINFO cell +to confirm their addresses and timestamps. +At that point, the channel is Open. +No other intervening cell types are allowed. + +## Indicating support for the in-protocol handshake + +When the in-protocol handshake was new, +we placed a set of constraints on the certificate that the responder would send +to indicate that it supported the v3 handshake. + +Specifically, if at least one of these properties +was true of the responders's certificate, +the initiator could be sure that the responder supported +the in-protocol handshake: + +- The certificate is self-signed +- Some component other than "commonName" is set in the subject or + issuer DN of the certificate. +- The commonName of the subject or issuer of the certificate ends + with a suffix other than ".net". +- The certificate's public key modulus is longer than 1024 bits. + +Otherwise, the initiator would assume +that only the v2 protocol was in use. + +## Fixed ciphersuite list {#fixed-cipehrsuite-list} + +For a long time, clients would advertise +a certain "fixed ciphersuite list" +regardless of whether they actually supported those ciphers. + +That list is: + +```text + TLS1_ECDHE_ECDSA_WITH_AES_256_CBC_SHA + TLS1_ECDHE_RSA_WITH_AES_256_CBC_SHA + TLS1_DHE_RSA_WITH_AES_256_SHA + TLS1_DHE_DSS_WITH_AES_256_SHA + TLS1_ECDH_RSA_WITH_AES_256_CBC_SHA + TLS1_ECDH_ECDSA_WITH_AES_256_CBC_SHA + TLS1_RSA_WITH_AES_256_SHA + TLS1_ECDHE_ECDSA_WITH_RC4_128_SHA + TLS1_ECDHE_ECDSA_WITH_AES_128_CBC_SHA + TLS1_ECDHE_RSA_WITH_RC4_128_SHA + TLS1_ECDHE_RSA_WITH_AES_128_CBC_SHA + TLS1_DHE_RSA_WITH_AES_128_SHA + TLS1_DHE_DSS_WITH_AES_128_SHA + TLS1_ECDH_RSA_WITH_RC4_128_SHA + TLS1_ECDH_RSA_WITH_AES_128_CBC_SHA + TLS1_ECDH_ECDSA_WITH_RC4_128_SHA + TLS1_ECDH_ECDSA_WITH_AES_128_CBC_SHA + SSL3_RSA_RC4_128_MD5 + SSL3_RSA_RC4_128_SHA + TLS1_RSA_WITH_AES_128_SHA + TLS1_ECDHE_ECDSA_WITH_DES_192_CBC3_SHA + TLS1_ECDHE_RSA_WITH_DES_192_CBC3_SHA + SSL3_EDH_RSA_DES_192_CBC3_SHA + SSL3_EDH_DSS_DES_192_CBC3_SHA + TLS1_ECDH_RSA_WITH_DES_192_CBC3_SHA + TLS1_ECDH_ECDSA_WITH_DES_192_CBC3_SHA + SSL3_RSA_FIPS_WITH_3DES_EDE_CBC_SHA + SSL3_RSA_DES_192_CBC3_SHA + [*] The "extended renegotiation is supported" ciphersuite, 0x00ff, is + not counted when checking the list of ciphersuites. +``` + +When encountering this list, +a responder would not select any ciphersuites +besides the mandatory-to-implement +TLS_DHE_RSA_WITH_AES_256_CBC_SHA, TLS_DHE_RSA_WITH_AES_128_CBC_SHA, +and SSL_DHE_RSA_WITH_3DES_EDE_CBC_SHA. + +Clients no longer report ciphers that they do not support. + + +## Legacy CERTS authentication: Responder has RSA Identity only {#certs-responder-legacy} + +A Tor relay that has only an RSA identity key (`KP_relayid_rsa`) +and not an Ed25519 identity key (`KP_relayid_ed`) +will present a different set of certificates in its CERTS cell. + +(Relays like this are no longer supported; +all relays must now have Ed25519 identities.) + +To authenticate a responder as having only an RSA identity, +the initiator would verify the following: + +- The CERTS cell contains exactly one CertType 2 `RSA_ID_X509` certificate. + - This must be a self-signed certificate containing a 1024-bit RSA key; + that key's exponent must be 65537. + That key is `KP_relayid_rsa`. +- The CERTS cell contains exactly one CertType 1 `TLS_LINK_X509` certificate. + - It must be signed with `KP_relayid_rsa`. + - Its subject key must be the same + as `KP_legacy_conn_tls` + (the key used to negotiate the TLS connection). +- All of the certs above must be correctly signed, + not expired, + and not before their `validAfter` dates. + +### Legacy CERTS authentication: Initiator has RSA Identity only {#certs-initiator-legacy} + + +As discussed in +["Validating an initiator's CERTS"](./negotiating-channels.md#validate-initiator-certs), +the initiator of the v3 handshake does not present a TLS certificate. + +Therefore, to process an initiator's CERTS cell, +the responder would have to procede as for a responder's certificates, +[as described above](#certs-responder-legacy), +except that **instead** of checking for a `TLS_LINK_X509` certificate, +it would need to verify that: + +- The CERTS cell contains exactly one CertType 3 + `LINK_AUTH_X509` certificate. + - This certificate must be signed with `KP_relayid_rsa`. + Its subject key is deemed to be `KP_legacy_linkauth_rsa`. +- All of the certs above must be correctly signed, + not expired, + and not before their `validAfter` dates. + + +<a id="tor-spec.txt-4.4.1"></a> + +## Link authentication type 1: RSA-SHA256-TLSSecret {#RSA-SHA256-TLSSecret} + +This is an obsolete authentication method +used before RFC5705 support was ubiquitous. +It is nearly the same as +[Ed25519-SHA256-RFC5705](./negotiating-channels.md#Ed25519-SHA256-RFC5705), +but lacks support for Ed25519, +and does not use keying material exporters +(which were not widely supported at the time it as used. + +If AuthType is `[00 01]` (meaning "RSA-SHA256-TLSSecret"), +then the authentication field of the AUTHENTICATE +cell contains the following: + +| Field | Size | Description | +| --------- | ---- | ----------- | +| `TYPE` | 8 | The nonterminated string `AUTH0001` | +| `CID` | 32 | `SHA256(KP_relayid_rsa)` for initiator | +| `SID` | 32 | `SHA256(KP_relayid_rsa)` for responder | +| `SLOG` | 32 | SHA256 of responder transcript | +| `CLOG` | 32 | SHA256 of initiator transcript | +| `SCERT` | 32 | SHA256 of responder's TLS certificate | +| `TLSSECRETS`|32 | An ad-hoc HMAC output | +| `RAND` | 24 | [Random bytes] | +| `SIG` | Variable | RSA signature | + + +Notes are as for [Ed25519-SHA256-RFC5705], +except as follows: + + +- The `TLSSECRETS` fields holds a SHA256 HMAC, + using the TLS master secret as the secret key, + of the following concatenated fields: + - `client_random`, as sent in the TLS Client Hello + - `server_random`, as sent in the TLS Server Hello + - the NUL terminated ASCII string: + `"Tor V3 handshake TLS cross-certification"` +* The `SIG` fields holds an RSA signature of a SHA256 hash + of all the previous fields + (that is, `TYPE` through `RAND`), + using the initiator's `KS_legacy_linkauth_rsa`. + This field extends through the end of the AUTHENTICATE cell. + +[Random bytes]: ./preliminaries.md#random-values +[Ed25519-SHA256-RFC5705]: ./negotiating-channels.md#Ed25519-SHA256-RFC5705 + +Responders MUST NOT accept this AuthType if the initiator has +claimed to have an Ed25519 identity. + diff --git a/spec/tor-spec/opening-streams.md b/spec/tor-spec/opening-streams.md new file mode 100644 index 0000000..8e63a62 --- /dev/null +++ b/spec/tor-spec/opening-streams.md @@ -0,0 +1,141 @@ +<a id="tor-spec.txt-6.2"></a> + +# Opening streams and transmitting data + +## Opening a new stream: The begin/connected handshake {#opening} + +To open a new anonymized TCP connection, the OP chooses an open +circuit to an exit that may be able to connect to the destination +address, selects an arbitrary StreamID not yet used on that circuit, +and constructs a RELAY_BEGIN message with a payload encoding the address +and port of the destination host. The payload format is: + +```text + ADDRPORT [nul-terminated string] + FLAGS [4 bytes, optional] + + ADDRPORT is made of ADDRESS | ':' | PORT | [00] +``` + +where ADDRESS can be a DNS hostname, or an IPv4 address in +dotted-quad format, or an IPv6 address surrounded by square brackets; +and where PORT is a decimal integer between 1 and 65535, inclusive. + +The ADDRPORT string SHOULD be sent in lower case, to avoid +fingerprinting. Implementations MUST accept strings in any case. + +The FLAGS value has one or more of the following bits set, where +"bit 1" is the LSB of the 32-bit value, and "bit 32" is the MSB. +(Remember that +[all integers in Tor are big-endian](../intro/conventions.md), +so +the MSB of a 4-byte value is the MSB of the first byte, and the LSB +of a 4-byte value is the LSB of its last byte.) + +If FLAGS is absent, its value is 0. Whenever 0 would be sent for +FLAGS, FLAGS is omitted from the message payload. + +```text + bit meaning + 1 -- IPv6 okay. We support learning about IPv6 addresses and + connecting to IPv6 addresses. + 2 -- IPv4 not okay. We don't want to learn about IPv4 addresses + or connect to them. + 3 -- IPv6 preferred. If there are both IPv4 and IPv6 addresses, + we want to connect to the IPv6 one. (By default, we connect + to the IPv4 address.) + 4..32 -- Reserved. Current clients MUST NOT set these. Servers + MUST ignore them. +``` + +Upon receiving this message, the exit node resolves the address as +necessary, and opens a new TCP connection to the target port. If the +address cannot be resolved, or a connection can't be established, the +exit node replies with a RELAY_END message. (See +["Closing streams"](./closing-streams.md#closing-streams).) +Otherwise, the exit node replies with a RELAY_CONNECTED message, whose +payload is in one of the following formats: + +```text + The IPv4 address to which the connection was made [4 octets] + A number of seconds (TTL) for which the address may be cached [4 octets] + + or + + Four zero-valued octets [4 octets] + An address type (6) [1 octet] + The IPv6 address to which the connection was made [16 octets] + A number of seconds (TTL) for which the address may be cached [4 octets] +``` + +Implementations MUST accept either of these formats, +and MUST also accept an empty RELAY_CONNECTED message body. + +Implmentations MAY ignore the address value, +and MAY choose not to cache it. +If an implementation chooses to cache the address, +it SHOULD NOT reuse that address with any other circuit. + +> The reason not to cache an address +> is that the exit might have lied about the actual address of the host, +> or might have given us a unique address to identify us in the future. + +\[Tor exit nodes before 0.1.2.0 set the TTL field to a fixed value. Later +versions set the TTL to the last value seen from a DNS server, and expire +their own cached entries after a fixed interval. This prevents certain +attacks.\] + +## Transmitting data {#transmitting} + +Once a connection has been established, the OP and exit node +package stream data in RELAY_DATA message, and upon receiving such +messages, echo their contents to the corresponding TCP stream. + +If the exit node does not support optimistic data (i.e. its +version number is before 0.2.3.1-alpha), then the OP MUST wait +for a RELAY_CONNECTED message before sending any data. If the exit +node supports optimistic data (i.e. its version number is +0.2.3.1-alpha or later), then the OP MAY send RELAY_DATA messages +immediately after sending the RELAY_BEGIN message (and before +receiving either a RELAY_CONNECTED or RELAY_END message). + +RELAY_DATA messages sent to unrecognized streams are dropped. If +the exit node supports optimistic data, then RELAY_DATA messages it +receives on streams which have seen RELAY_BEGIN but have not yet +been replied to with a RELAY_CONNECTED or RELAY_END are queued. +If the stream creation succeeds with a RELAY_CONNECTED, the queue +is processed immediately afterwards; if the stream creation fails +with a RELAY_END, the contents of the queue are deleted. + +Relay RELAY_DROP messages are long-range dummies; upon receiving such +a message, the OR or OP must drop it. + +<a id="tor-spec.txt-6.2.1"></a> + +## Opening a directory stream + +If a Tor relay is a directory server, it should respond to a +RELAY_BEGIN_DIR message as if it had received a BEGIN message requesting a +connection to its directory port. RELAY_BEGIN_DIR message ignore exit +policy, since the stream is local to the Tor process. + +Directory servers may be: + +- authoritative directories (RELAY_BEGIN_DIR, usually non-anonymous), +- bridge authoritative directories (RELAY_BEGIN_DIR, anonymous), +- directory mirrors (RELAY_BEGIN_DIR, usually non-anonymous), +- onion service directories (RELAY_BEGIN_DIR, anonymous). + +If the Tor relay is not running a directory service, it should respond +with a REASON_NOTDIRECTORY RELAY_END message. + +Clients MUST generate an all-zero payload for RELAY_BEGIN_DIR message, +and relays MUST ignore the payload. + +In response to a RELAY_BEGIN_DIR message, relays respond either with a +RELAY_CONNECTED message on success, or a RELAY_END message on failure. They +MUST send a RELAY_CONNECTED message all-zero payload, and clients MUST ignore +the payload. + +\[RELAY_BEGIN_DIR was not supported before Tor 0.1.2.2-alpha; clients +SHOULD NOT send it to routers running earlier versions of Tor.\] diff --git a/spec/tor-spec/preliminaries.md b/spec/tor-spec/preliminaries.md new file mode 100644 index 0000000..e69d47e --- /dev/null +++ b/spec/tor-spec/preliminaries.md @@ -0,0 +1,140 @@ +<a id="tor-spec.txt-0"></a> + +# Preliminaries + +<a id="tor-spec.txt-0.1"></a> + +## Notation and encoding{#notation-and-encoding} + +```text + KP -- a public key for an asymmetric cipher. + KS -- a private key for an asymmetric cipher. + K -- a key for a symmetric cipher. + N -- a "nonce", a random value, usually deterministically chosen + from other inputs using hashing. +``` + +H(m) -- a cryptographic hash of m. + +## Security parameters + +Tor uses a stream cipher, a public-key cipher, the Diffie-Hellman +protocol, and a hash function. + +KEY_LEN -- the length of the stream cipher's key, in bytes. + +```text + KP_ENC_LEN -- the length of a public-key encrypted message, in bytes. + KP_PAD_LEN -- the number of bytes added in padding for public-key + encryption, in bytes. (The largest number of bytes that can be encrypted + in a single public-key operation is therefore KP_ENC_LEN-KP_PAD_LEN.) + + DH_LEN -- the number of bytes used to represent a member of the + Diffie-Hellman group. + DH_SEC_LEN -- the number of bytes used in a Diffie-Hellman private key (x). + + HASH_LEN -- the length of the hash function's output, in bytes. + +``` + +## Message lengths {#msg-len} + +Some message lengths are fixed in the Tor protocol. +We give them here. +Some of these message lengths depend +on the version of the Tor link protocol in use: +for these, the link protocol is denoted in this table with `v`. + +| Name | Length in bytes | Meaning | +| ---- | --------------- | ------- | +| `PAYLOAD_LEN` | 509 | The longest payload for a [fixed-length cell](./cell-packet-format.md#fixed-length-cell). | +| `CIRCID_LEN(v)`, `v` < 4 | 2 | The length of a [circuit ID](./cell-packet-format.md#circid) | +| `CIRCID_LEN(v)`, `v` ≥ 4 | 4 | | +| `CELL_LEN(v)`, `v` < 4 | 512 | The length of a [fixed-length cell](./cell-packet-format.md). | +| `CELL_LEN(v)`, `v` ≥ 4 | 514 | | + +Note that for all `v`, `CELL_LEN(v) = 1 + CIRCID_LEN(v) + PAYLOAD_LEN`. + +<a id="tor-spec.txt-0.3"></a> + +## Ciphers{#ciphers} + +These are the ciphers we use _unless otherwise specified_. Several of +them are deprecated for new use. + +For a stream cipher, unless otherwise specified, we use 128-bit AES in +counter mode, with an IV of all 0 bytes. (We also require AES256.) + +For a public-key cipher, unless otherwise specified, we use RSA with +1024-bit keys and a fixed exponent of 65537. We use OAEP-MGF1 +padding, with SHA-1 as its digest function. We leave the optional +"Label" parameter unset. (For OAEP padding, see +ftp://ftp.rsasecurity.com/pub/pkcs/pkcs-1/pkcs-1v2-1.pdf) + +We also use the Curve25519 group and the Ed25519 signature format in +several places. + +For Diffie-Hellman, unless otherwise specified, we use a generator +(g) of 2. For the modulus (p), we use the 1024-bit safe prime from +rfc2409 section 6.2 whose hex representation is: + +```text + "FFFFFFFFFFFFFFFFC90FDAA22168C234C4C6628B80DC1CD129024E08" + "8A67CC74020BBEA63B139B22514A08798E3404DDEF9519B3CD3A431B" + "302B0A6DF25F14374FE1356D6D51C245E485B576625E7EC6F44C42E9" + "A637ED6B0BFF5CB6F406B7EDEE386BFB5A899FA5AE9F24117C4B1FE6" + "49286651ECE65381FFFFFFFFFFFFFFFF" +``` + +As an optimization, implementations SHOULD choose DH private keys (x) of +320 bits. Implementations that do this MUST never use any DH key more +than once. +\[May other implementations reuse their DH keys?? -RD\] +\[Probably not. Conceivably, you could get away with changing DH keys once +per second, but there are too many oddball attacks for me to be +comfortable that this is safe. -NM\] + +For a hash function, unless otherwise specified, we use SHA-1. + +KEY_LEN=16. +DH_LEN=128; DH_SEC_LEN=40. +KP_ENC_LEN=128; KP_PAD_LEN=42. +HASH_LEN=20. + +We also use SHA256 and SHA3-256 in some places. + +When we refer to "the hash of a public key", unless otherwise +specified, we mean the SHA-1 hash of the DER encoding of an ASN.1 RSA +public key (as specified in PKCS.1). + +<span id="random-values"> +All "random" values MUST be generated with a cryptographically +strong pseudorandom number generator seeded from a strong entropy +source, unless otherwise noted. +All "random" values MUST selected uniformly at random from the +universe of possible values, unless otherwise noted. +</span> + +<a id="tor-spec.txt-0.4"></a> + +## A bad hybrid encryption algorithm, for legacy purposes {#legacy-hybrid-encryption} + +Some specifications will refer to the "legacy hybrid encryption" of a +byte sequence M with a public key KP. It is computed as follows: + +```text + 1. If the length of M is no more than KP_ENC_LEN-KP_PAD_LEN, + pad and encrypt M with KP. + 2. Otherwise, generate a KEY_LEN byte random key K. + Let M1 = the first KP_ENC_LEN-KP_PAD_LEN-KEY_LEN bytes of M, + and let M2 = the rest of M. + Pad and encrypt K|M1 with KP. Encrypt M2 with our stream cipher, + using the key K. Concatenate these encrypted values. +``` + +Note that this "hybrid encryption" approach does not prevent +an attacker from adding or removing bytes to the end of M. It also +allows attackers to modify the bytes not covered by the OAEP -- +see Goldberg's PET2006 paper for details. Do not use it as the basis +for new protocols! Also note that as used in Tor's protocols, case 1 +never occurs. diff --git a/spec/tor-spec/relay-cells.md b/spec/tor-spec/relay-cells.md new file mode 100644 index 0000000..346899b --- /dev/null +++ b/spec/tor-spec/relay-cells.md @@ -0,0 +1,175 @@ +<a id="tor-spec.txt-6.1"></a> + +# Relay cells {#relay-cells} + +Within a circuit, the OP and the end node use the contents of +relay cells to tunnel end-to-end commands and TCP connections +("Streams") across circuits. End-to-end commands can be initiated +by either edge; streams are initiated by the OP. + +End nodes that accept streams may be: + +- exit relays (RELAY_BEGIN, anonymous), +- directory servers (RELAY_BEGIN_DIR, anonymous or non-anonymous), +- onion services (RELAY_BEGIN, anonymous via a rendezvous point). + +The payload of each unencrypted relay cell consists of an +enveloped relay message, encoded as follows: + +| Field | Size +| ----- | ---- +| Relay command | 1 byte +| 'Recognized' | 2 bytes +| StreamID | 2 bytes +| Digest | 4 bytes +| Length | 2 bytes +| Data | Length bytes +| Padding | PAYLOAD_LEN - 11 - Length bytes + +> TODO: When we implement [prop340](../proposals/340-packed-and-fragmented.md), +> we should clarify which parts of the above are about +> the relay cell, and which are the enveloped message. + +The relay commands are: + +| Command | Identifier | Direction | Control? +| ------- | ---------- | --------- | -------- +| 1 | RELAY_BEGIN | forward | +| 2 | RELAY_DATA | forward or backward | +| 3 | RELAY_END | forward or backward | +| 4 | RELAY_CONNECTED | backward | +| 5 | RELAY_SENDME | forward or backward | sometimes control +| 6 | RELAY_EXTEND | forward | control +| 7 | RELAY_EXTENDED | backward | control +| 8 | RELAY_TRUNCATE | forward | control +| 9 | RELAY_TRUNCATED | backward | control +| 10 | RELAY_DROP | forward or backward | control +| 11 | RELAY_RESOLVE | forward | +| 12 | RELAY_RESOLVED | backward | +| 13 | RELAY_BEGIN_DIR | forward | +| 14 | RELAY_EXTEND2 | forward | control +| 15 | RELAY_EXTENDED2 | backward | control +| 16..18 | Reserved for UDP; Not yet in use, see [prop339]. +| 19..22 | Reserved for Conflux, see [prop329]. +| 32..40 | Used for hidden services; see the [rendezvous spec]. +| 41..42 | Used for circuit padding; see ["Circuit-level padding"] in the padding spec. +| 43 | XON (See Sec 4 of [prop324]) | forward or backward | +| 44 | XOFF (See Sec 4 of [prop324]) | forward or backward | + +[prop324]: ../proposals/324-rtt-congestion-control.txt +[prop329]: ../proposals/329-traffic-splitting.md +[prop339]: ../proposals/339-udp-over-tor.md +[rendezvous spec]: ../rend-spec/index.md +["Circuit-level padding"]: ../padding-spec/circuit-level-padding.md#circuit-level-padding + +Commands labelled as "forward" must only be sent by the originator +of the circuit. Commands labelled as "backward" must only be sent by +other nodes in the circuit back to the originator. Commands marked +as either can be sent either by the originator or other nodes. + +The 'recognized' field is used as a simple indication that the cell +is still encrypted. It is an optimization to avoid calculating +expensive digests for every cell. When sending cells, the unencrypted +'recognized' MUST be set to zero. + +When receiving and decrypting cells the 'recognized' will always be +zero if we're the endpoint that the cell is destined for. For cells +that we should relay, the 'recognized' field will usually be nonzero, +but will accidentally be zero with P=2^-16. + +When handling a relay cell, if the 'recognized' in field in a +decrypted relay payload is zero, the 'digest' field is computed as +the first four bytes of the running digest of all the bytes that have +been destined for this hop of the circuit or originated from this hop +of the circuit, seeded from Df or Db respectively (obtained in +[Setting circuit keys](./setting-circuit-keys.md#setting-circuit-keys)), +and including this relay cell's entire payload +(taken with the digest field set to zero). Note that these digests +_do_ include the padding bytes at the end of the cell, not only those up +to "Len". If the digest is correct, the cell is considered "recognized" +for the purposes of decryption (see +[Routing relay cells](./routing-relay-cells.md#routing-relay-cells)). + +(The digest does not include any bytes from relay cells that do +not start or end at this hop of the circuit. That is, it does not +include forwarded data. Therefore if 'recognized' is zero but the +digest does not match, the running digest at that node should +not be updated, and the cell should be forwarded on.) + +All relay messages pertaining to the same tunneled stream have the same +stream ID. StreamIDs are chosen arbitrarily by the OP. No stream +may have a StreamID of zero. Rather, relay messages that affect the +entire circuit rather than a particular stream use a StreamID of zero +-- they are marked in the table above as "\[control\]" style +cells. (Sendme cells are marked as "sometimes control" because they +can include a StreamID or not depending on their purpose -- see +[Flow control](./flow-control.md#flow-control).) + +The 'Length' field of a relay cell contains the number of bytes in +the relay payload which contain real payload data. The remainder of +the unencrypted payload is padded with padding bytes. Implementations +handle padding bytes of unencrypted relay cells as they do padding +bytes for other cell types; see [Cell Packet format](./cell-packet-format.md#cell-packet-format). + +<span id="relay-cell-padding">The +'Padding' field is used to make relay cell contents unpredictable, to +avoid certain attacks (see +[proposal 289](../proposals/289-authenticated-sendmes.txt) +for rationale). Implementations +SHOULD fill this field with four zero-valued bytes, followed by as many +random bytes as will fit. (If there are fewer than 4 bytes for padding, +then they should all be filled with zero.</span> + +Implementations MUST NOT rely on the contents of the 'Padding' field. + +If the relay cell is recognized but the relay command is not +understood, the cell must be dropped and ignored. Its contents +still count with respect to the digests and flow control windows, though. + +<a id="tor-spec.txt-6.1.1"></a> + +## Calculating the 'Digest' field {#digest-field} + +The 'Digest' field itself serves the purpose to check if a cell has been +fully decrypted, that is, all onion layers have been removed. Having a +single field, namely 'Recognized' is not sufficient, as outlined above. + +When ENCRYPTING a relay cell, an implementation does the following: + +```text +# Encode the cell in binary (recognized and digest set to zero) +tmp = cmd + [0, 0] + stream_id + [0, 0, 0, 0] + length + data + padding + +# Update the digest with the encoded data +digest_state = hash_update(digest_state, tmp) +digest = hash_calculate(digest_state) + +# The encoded data is the same as above with the digest field not being +# zero anymore +encoded = cmd + [0, 0] + stream_id + digest[0..4] + length + data + + padding + +# Now we can encrypt the cell by adding the onion layers ... +``` + + When DECRYPTING a relay cell, an implementation does the following: + +```text +decrypted = decrypt(cell) + +# Replace the digest field in decrypted by zeros +tmp = decrypted[0..5] + [0, 0, 0, 0] + decrypted[9..] + +# Update the digest field with the decrypted data and its digest field +# set to zero +digest_state = hash_update(digest_state, tmp) +digest = hash_calculate(digest_state) + +if digest[0..4] == decrypted[5..9] + # The cell has been fully decrypted ... +``` + +The caveat itself is that only the binary data with the digest bytes set to +zero are being taken into account when calculating the running digest. The +final plain-text cells (with the digest field set to its actual value) are +not taken into the running digest. diff --git a/spec/tor-spec/relay-early.md b/spec/tor-spec/relay-early.md new file mode 100644 index 0000000..fd7dbf4 --- /dev/null +++ b/spec/tor-spec/relay-early.md @@ -0,0 +1,20 @@ +<a id="tor-spec.txt-5.6"></a> + +# Handling RELAY_EARLY cells{#handling-relay-early-cells} + +A RELAY_EARLY cell is designed to limit the length any circuit can reach. +When an OR receives a RELAY_EARLY cell, and the next node in the circuit +is speaking v2 of the link protocol or later, the OR relays the cell as a +RELAY_EARLY cell. Otherwise, older Tors will relay it as a RELAY cell. + +If a node ever receives more than 8 RELAY_EARLY cells on a given +outbound circuit, it SHOULD close the circuit. If it receives any +inbound RELAY_EARLY cells, it MUST close the circuit immediately. + +When speaking v2 of the link protocol or later, clients MUST only send +EXTEND/EXTEND2 message inside RELAY_EARLY cells. Clients SHOULD send the first +~8 relay cells that are not targeted at the first hop of any circuit as +RELAY_EARLY cells too, in order to partially conceal the circuit length. + +\[Starting with Tor 0.2.3.11-alpha, relays should reject any +EXTEND/EXTEND2 cell not received in a RELAY_EARLY cell.\] diff --git a/spec/tor-spec/relay-keys.md b/spec/tor-spec/relay-keys.md new file mode 100644 index 0000000..25ac3ed --- /dev/null +++ b/spec/tor-spec/relay-keys.md @@ -0,0 +1,175 @@ +<a id="tor-spec.txt-1.1"></a> + +# Relay keys and identities + +Every Tor relay has multiple public/private keypairs, +with different lifetimes and purposes. +We explain them here. + +Each key here has an English name +(like "Ed25519 identity key") +and an unambiguous identifier +(like `KP_relayid_ed`). + +In an identifier, +a `KP_` prefix denotes a public key, +and a `KS_` prefix denotes the corresponding secret key. + +> For historical reasons or reasons of space, +> you will sometimes encounter +> multiple English names for the same key, +> or shortened versions of that name. +> The identifier for a key, however, +> should always be unique and unambiguous. + +For security reasons, **all keys MUST be distinct**: +the same key or keypair should never be used +for separate roles within the Tor protocol suite, +unless specifically stated. +For example, +a relay's identity key `KP_relayid_ed` +MUST NOT also be used +as its medium-term signing key `KP_relaysign_ed`. + +## Identity keys {#identity} + +An **identity key** is a long-lived key +that uniquely identifies a relay. +Two relays with the same set of identity keys +are considered to be the same; +any relay that changes its identity key +is considered to have become a different relay. + +An identity keypair's lifetime +is the same as the lifetime of the relay. + +Two identity keys are currently defined: + +- <span id="relayid_ed">`KP_relayid_ed`, `KS_relayid_ed`:</span> + An "ed25519 identity key", + also sometimes called a "master identity key". + + This is an Ed25519 key. + This key never expires. + It is used for only one purpose: + signing the `KP_relaysign_ed` key, + which is used to sign + other important keys and objects. + +- <span id="relayid_rsa">`KP_relayid_rsa`, `KS_relayid_rsa`:</span> + A _legacy_ "RSA identity key". + + This is an RSA key. + It never expires. + It is always 1024 bits long, + and (as discussed [above](./preliminaries.md#ciphers)) + its exponent must be 65537. + It is used to sign directory documents and certificates. + +Note that because the legacy RSA identity key is so short, +it should not be assumed secure against an attacker. +It exists for legacy purposes only. +When authenticating a relay, +a failure to prove an expected RSA identity +is sufficient evidence of a _failure_ to authenticate, +but a successful proof of an RSA identity is not sufficient +to establish a relay's identity. +Parties SHOULD NOT use the RSA identity on its own. + +We write `KP_relayid` to refer to a key which is either +`KP_relayid_rsa` or `KP_relayid_ed`. + +## Online signing keys {#online-signing} + +Since Tor's design tries to support +keeping the high-value Ed25519 relay identity key offline, +we need a corresponding key that can be used for online signing: + +- <span id="relaysign_ed">`KP_relaysign_ed`, `KS_relaysign_ed`:</span> + A medium-term Ed25519 "signing" key. + This key is signed by the identity key `KP_relayid_ed`, + and must be kept online. + A new one should be generated periodically. + It signs nearly everything else, + including directory objects, + and certificates for other keys. + +When this key is generated, +it needs to be signed with the `KP_relayid_ed` key, +producing a [certificate of type `IDENTITY_V_SIGNING`](../cert-spec.md). +The `KP_relayid_ed` key is not used for anything else. + + +## Circuit extension keys + +Each relay has one or more **circuit extension keys** +(also called "onion keys"). +When [creating](./create-created-cells.md) +or [extending](./create-created-cells.md#EXTEND) +a circuit, +a client uses this key to perform +a [one-way authenticated key exchange][cacr2011-goldberg] +with the target relay. +If the recipient does not have the correct private key, +the handshake will fail. + +[cacr2011-goldberg]: https://cacr.uwaterloo.ca/techreports/2011/cacr2011-11.pdf + +Circuit extension keys have moderate lifetimes, +on the order of weeks. +They are published as part of the directory protocol, +and relays SHOULD accept handshakes for a while +after publishing any new key. +(The exact durations for these are set via +[a set of network parameters](../param-spec.md#onion-key-rotation-days).) + +There are two current kinds of circuit extension keys: + +- <span id="ntor">`KP_ntor`, `KS_ntor`</span>: + A curve25519 key + used for the [`ntor`](./create-created-cells.md#ntor) + and [`ntorv3`](./create-created-cells.md#ntor-v3) + circuit extension handshakes. + +- <span id="onion_tap">`KP_onion_tap`, `KS_onion_tap`</span>: + A 1024 bit RSA key + used for the obsolete [`TAP`](./create-created-cells.md#ntor) + circuit extension handshake. + + +## Channel authentication {#auth} + +There are other keys that relays use to authenticate +as part of their [channel negotiation handshakes](./channels.md). + +These keys are authenticated with other, longer lived keys. +Relays MAY rotate them as often as they like, +and SHOULD rotate them frequently—typically, at least once a day. + +- <span id="link_ed">`KP_link_ed`, `KS_link_ed`</span>. + A short-term Ed25519 "link authentication" key, used to authenticate + the link handshake: see + ["Negotiating and initializing channels"](./negotiating-channels.md#negotiating). + This key is signed by the "signing" key, and should be regenerated frequently. + +### Legacy channel authentication {#auth-legacy} + +These key types were used in +[older versions](./obsolete-channels.md) +of the channel negotiation handshakes. + +- <span id="legacy_linkauth_rsa">`KP_legacy_linkauth_rsa`, `KS_legacy_linkauth_rsa`</span>: + A 1024-bit RSA key, used to authenticate the link handshake. + (No longer used in modern Tor.) + It played a role similar to `KP_link_ed`. + + +As a convenience, to describe legacy versions of the link handshake, +we give a name to the public key used for the TLS handshake itself: + +- <span id="legacy_conn_tls">`KP_legacy_conn_tls`, `KS_legacy_conn_tls`</span>: + A short term key used to for TLS connections. + (No longer used in modern Tor.) + This was another name for the server's TLS key, + which at the time was required to be an RSA key. + It was used in some legacy handshake versions. diff --git a/spec/tor-spec/remote-hostname-lookup.md b/spec/tor-spec/remote-hostname-lookup.md new file mode 100644 index 0000000..2793411 --- /dev/null +++ b/spec/tor-spec/remote-hostname-lookup.md @@ -0,0 +1,43 @@ +<a id="tor-spec.txt-6.4"></a> + +# Remote hostname lookup + +To find the address associated with a hostname, the OP sends a +RELAY_RESOLVE message containing the hostname to be resolved with a NUL +terminating byte. (For a reverse lookup, the OP sends a RELAY_RESOLVE +message containing an in-addr.arpa address.) The OR replies with a +RELAY_RESOLVED message containing any number of answers. Each answer is +of the form: + +```text + Type (1 octet) + Length (1 octet) + Value (variable-width) + TTL (4 octets) + "Length" is the length of the Value field. + "Type" is one of: + + 0x00 -- Hostname + 0x04 -- IPv4 address + 0x06 -- IPv6 address + 0xF0 -- Error, transient + 0xF1 -- Error, nontransient + + If any answer has a type of 'Error', then no other answer may be + given. + + The 'Value' field encodes the answer: + IP addresses are given in network order. + Hostnames are given in standard DNS order ("www.example.com") + and not NUL-terminated. + The content of Errors is currently ignored. Relays currently + set it to the string "Error resolving hostname" with no + terminating NUL. Implementations MUST ignore this value. + + For backward compatibility, if there are any IPv4 answers, one of those + must be given as the first answer. + + The RELAY_RESOLVE messge must use a nonzero, distinct streamID; the + corresponding RELAY_RESOLVED message must use the same streamID. No stream + is actually created by the OR when resolving the name. +``` diff --git a/spec/tor-spec/routing-relay-cells.md b/spec/tor-spec/routing-relay-cells.md new file mode 100644 index 0000000..a9e3c49 --- /dev/null +++ b/spec/tor-spec/routing-relay-cells.md @@ -0,0 +1,99 @@ +<a id="tor-spec.txt-5.5"></a> + +# Routing relay cells{#routing-relay-cells} + +<a id="tor-spec.txt-5.5.1"></a> + +## Circuit ID Checks + +When a node wants to send a RELAY or RELAY_EARLY cell, it checks the cell's +circID and determines whether the corresponding circuit along that +connection is still open. If not, the node drops the cell. + +When a node receives a RELAY or RELAY_EARLY cell, it checks the cell's +circID and determines whether it has a corresponding circuit along +that connection. If not, the node drops the cell. + +> Here and elsewhere, we refer to RELAY and RELAY_EARLY cells +> collectively as "relay cells". + +<a id="tor-spec.txt-5.5.2"></a> + +## Forward Direction + +The forward direction is the direction that CREATE/CREATE2 cells +are sent. + +<a id="tor-spec.txt-5.5.2.1"></a> + +### Routing from the Origin + +When a relay cell is sent from an OP, the OP encrypts the payload +with the stream cipher as follows: + +```text +OP sends relay cell: + For I=N...1, where N is the destination node: + Encrypt with Kf_I. + Transmit the encrypted cell to node 1. +``` + +<a id="tor-spec.txt-5.5.2.2"></a> + +### Relaying Forward at Onion Routers + +When a forward relay cell is received by an OR, it decrypts the payload +with the stream cipher, as follows: + +```text +'Forward' relay cell: + Use Kf as key; decrypt. +``` + +The OR then decides whether it recognizes the relay cell, by +inspecting the payload as described in [Relay cells](./relay-cells.md#relay-cells). If the OR +recognizes the cell, it processes the contents of the relay cell. +Otherwise, it passes the decrypted relay cell along the circuit if +the circuit continues. If the OR at the end of the circuit +encounters an unrecognized relay cell, an error has occurred: the OR +sends a DESTROY cell to tear down the circuit. + +For more information, see +[Application connections and stream management](./streams.md#application-connections-and-stream-management). + +<a id="tor-spec.txt-5.5.3"></a> + +## Backward Direction + +The backward direction is the opposite direction from +CREATE/CREATE2 cells. + +<a id="tor-spec.txt-5.5.3.1"></a> + +### Relaying Backward at Onion Routers + +When a backward relay cell is received by an OR, it encrypts the payload +with the stream cipher, as follows: + +```text +'Backward' relay cell: + Use Kb as key; encrypt. +``` + +<a id="tor-spec.txt-5.5.3"></a> + +## Routing to the Origin + +When a relay cell arrives at an OP, the OP decrypts the payload +with the stream cipher as follows: + +```text +OP receives relay cell from node 1: + For I=1...N, where N is the final node on the circuit: + Decrypt with Kb_I. + If the payload is recognized (see [1]), then: + The sending node is I. + Stop and process the payload. +``` + +\[1\]: ["Relay cells"](./relay-cells.md#relay-cells) diff --git a/spec/tor-spec/setting-circuit-keys.md b/spec/tor-spec/setting-circuit-keys.md new file mode 100644 index 0000000..200862f --- /dev/null +++ b/spec/tor-spec/setting-circuit-keys.md @@ -0,0 +1,62 @@ +<a id="tor-spec.txt-5.2"></a> + +# Setting circuit keys{#setting-circuit-keys} + +<a id="tor-spec.txt-5.2.1"></a> + +## KDF-TOR{#kdf-tor} + +This key derivation function is used by the TAP and CREATE_FAST +handshakes, and in the current hidden service protocol. It shouldn't +be used for new functionality. + +If the TAP handshake is used to extend a circuit, both parties +base their key material on K0=g^xy, represented as a big-endian unsigned +integer. + +If CREATE_FAST is used, both parties base their key material on +K0=X|Y. + +From the base key material K0, they compute KEY_LEN*2+HASH_LEN*3 bytes of +derivative key data as + +K = H(K0 | \[00\]) | H(K0 | \[01\]) | H(K0 | \[02\]) | ... + +The first HASH_LEN bytes of K form KH; the next HASH_LEN form the forward +digest Df; the next HASH_LEN 41-60 form the backward digest Db; the next +KEY_LEN 61-76 form Kf, and the final KEY_LEN form Kb. Excess bytes from K +are discarded. + +KH is used in the handshake response to demonstrate knowledge of the +computed shared key. Df is used to seed the integrity-checking hash +for the stream of data going from the OP to the OR, and Db seeds the +integrity-checking hash for the data stream from the OR to the OP. Kf +is used to encrypt the stream of data going from the OP to the OR, and +Kb is used to encrypt the stream of data going from the OR to the OP. + +<a id="tor-spec.txt-5.2.2"></a> + +## KDF-RFC5869{#kdf-rfc5869} + +For newer KDF needs, Tor uses the key derivation function HKDF from +RFC5869, instantiated with SHA256. (This is due to a construction +from Krawczyk.) The generated key material is: + +K = K_1 | K_2 | K_3 | ... + +```text + Where H(x,t) is HMAC_SHA256 with value x and key t + and K_1 = H(m_expand | INT8(1) , KEY_SEED ) + and K_(i+1) = H(K_i | m_expand | INT8(i+1) , KEY_SEED ) + and m_expand is an arbitrarily chosen value, + and INT8(i) is a octet with the value "i". +``` + +In RFC5869's vocabulary, this is HKDF-SHA256 with info == m_expand, +salt == t_key, and IKM == secret_input. + +When used in the ntor handshake, the first HASH_LEN bytes form the +forward digest Df; the next HASH_LEN form the backward digest Db; the +next KEY_LEN form Kf, the next KEY_LEN form Kb, and the final +DIGEST_LEN bytes are taken as a nonce to use in the place of KH in the +hidden service protocol. Excess bytes from K are discarded. diff --git a/spec/tor-spec/streams.md b/spec/tor-spec/streams.md new file mode 100644 index 0000000..4d96048 --- /dev/null +++ b/spec/tor-spec/streams.md @@ -0,0 +1,7 @@ +<a id="tor-spec.txt-6"></a> + +# Application connections and stream management{#application-connections-and-stream-management} + +This section describes how clients use relay messages to communicate with +exit nodes, and how use this communication channel to send and receive +application data. diff --git a/spec/tor-spec/subprotocol-versioning.md b/spec/tor-spec/subprotocol-versioning.md new file mode 100644 index 0000000..5f43c71 --- /dev/null +++ b/spec/tor-spec/subprotocol-versioning.md @@ -0,0 +1,321 @@ +<a id="tor-spec.txt-9"></a> + +# Subprotocol versioning + +This section specifies the Tor subprotocol versioning. They are broken down +into different types with their current version numbers. Any new version +number should be added to this section. + +The dir-spec.txt details how those versions are encoded. See the +"proto"/"pr" line in a descriptor and the "recommended-relay-protocols", +"required-relay-protocols", "recommended-client-protocols" and +"required-client-protocols" lines in the vote/consensus format. + +Here are the rules a relay and client should follow when encountering a +protocol list in the consensus: + + - When a relay lacks a protocol listed in recommended-relay-protocols, + it should warn its operator that the relay is obsolete. + + - When a relay lacks a protocol listed in required-relay-protocols, it + should warn its operator as above. If the consensus is newer than the + date when the software was released or scheduled for release, it must + not attempt to join the network. + + - When a client lacks a protocol listed in recommended-client-protocols, + it should warn the user that the client is obsolete. + + - When a client lacks a protocol listed in required-client-protocols, + it should warn the user as above. If the consensus is newer than the + date when the software was released, it must not connect to the + network. This implements a "safe forward shutdown" mechanism for + zombie clients. + + - If a client or relay has a cached consensus telling it that a given + protocol is required, and it does not implement that protocol, it + SHOULD NOT try to fetch a newer consensus. + +Software release dates SHOULD be automatically updated as part of the +release process, to prevent forgetting to move them forward. Software +release dates MAY be manually adjusted by maintainers if necessary. + +Starting in version 0.2.9.4-alpha, the initial required protocols for +clients that we will Recommend and Require are: + +```text +Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 Link=4 +LinkAuth=1 Microdesc=1-2 Relay=2 +``` + +For relays we will Require: + +```text +Cons=1 Desc=1 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 Link=3-4 +LinkAuth=1 Microdesc=1 Relay=1-2 +``` + +For relays, we will additionally Recommend all protocols which we +recommend for clients. + +<a id="tor-spec.txt-9.1"></a> + +## "Link" + +The "link" protocols are those used by clients and relays to initiate and +receive OR connections and to handle cells on OR connections. The "link" +protocol versions correspond 1:1 to those versions. + +Two Tor instances can make a connection to each other only if they have at +least one link protocol in common. + +The current "link" versions are: "1" through "5". See +[Negotiating versions with VERSIONS cells](./negotiating-channels.md#VERSIONS-cells) for more +information. All current Tor versions support "1-3"; versions from +0.2.4.11-alpha and on support "1-4"; versions from 0.3.1.1-alpha and on +support "1-5". Eventually we will drop "1" and "2". + +<a id="tor-spec.txt-9.2"></a> + +## "LinkAuth" {#link-auth} + +LinkAuth protocols correspond to varieties of AUTHENTICATE cells used for +the v3+ link protocols. + +Current versions are: + + * "1" is the RSA link authentication described in [Link authentication type 1: RSA-SHA256-TLSSecret](./obsolete-channels.md#RSA-SHA256-TLSSecret). + + * "2" is unused, and reserved by [proposal 244]. + + * "3" is the ed25519 link authentication described in [Link authentication type 3: Ed25519-SHA256-RFC5705](./negotiating-channels.md#Ed25519-SHA256-RFC5705). + +<a id="tor-spec.txt-9.3"></a> + +## "Relay" + +The "relay" protocols are those used to handle CREATE/CREATE2 +cells, and those that handle the various relay messages received +after a CREATE/CREATE2 cell. (Except, relay cells used to manage +introduction and rendezvous points are managed with the "HSIntro" +and "HSRend" protocols respectively.) + +Current versions are as follows. + + * "1" -- supports the TAP key exchange, with all features in Tor 0.2.3. + Support for CREATE and CREATED and CREATE_FAST and CREATED_FAST + and EXTEND and EXTENDED. + + * "2" -- supports the ntor key exchange, and all features in Tor + 0.2.4.19. Includes support for CREATE2 and CREATED2 and + EXTEND2 and EXTENDED2. + + Relay=2 has limited IPv6 support: + + * Clients might not include IPv6 ORPorts in EXTEND2 messages. + * Relays (and bridges) might not initiate IPv6 connections in + response to EXTEND2 messages containing IPv6 ORPorts, even if they + are configured with an IPv6 ORPort. + + However, relays support accepting inbound connections to their IPv6 + ORPorts. And they might extend circuits via authenticated IPv6 + connections to other relays. + + * "3" -- relays support extending over IPv6 connections in response to an + EXTEND2 message containing an IPv6 ORPort. + + Bridges might not extend over IPv6, because they try to imitate + client behaviour. + + A successful IPv6 extend requires: + * Relay subprotocol version 3 (or later) on the extending relay, + * an IPv6 ORPort on the extending relay, + * an IPv6 ORPort for the accepting relay in the EXTEND2 message, and + * an IPv6 ORPort on the accepting relay. + (Because different tor instances can have different views of the + network, these checks should be done when the path is selected. + Extending relays should only check local IPv6 information, before + attempting the extend.) + + When relays receive an EXTEND2 message containing both an IPv4 and an + IPv6 ORPort, and there is no existing authenticated connection with + the target relay, the extending relay may choose between IPv4 and + IPv6 at random. The extending relay might not try the other address, + if the first connection fails. + + As is the case with other subprotocol versions, tor advertises, + recommends, or requires support for this protocol version, regardless + of its current configuration. + + In particular: + * relays without an IPv6 ORPort, and + * tor instances that are not relays, + have the following behaviour, regardless of their configuration: + * advertise support for "Relay=3" in their descriptor + (if they are a relay, bridge, or directory authority), and + * react to consensuses recommending or requiring support for + "Relay=3". + + This subprotocol version is described in + [proposal 311], and implemented in Tor 0.4.5.1-alpha. + + * "4" -- support the ntorv3 (version 3) key exchange and all features in + 0.4.7.3-alpha. This adds a new CREATE2 cell type. See [proposal 332] + and [The "ntor-v3" handshake](./create-created-cells.md#ntor-v3) for more details. + + * "5" -- \[RESERVED\] support the ntorv3 subprotocol request extension + ([proposal 346]) + allowing a client to request what features to be used on a circuit. + +<a id="tor-spec.txt-9.4"></a> + +## "HSIntro" {#HSIntro} + +The "HSIntro" protocol handles introduction points. + + * "3" -- supports authentication as of [proposal 121] in Tor 0.2.1.6-alpha. + + * "4" -- support ed25519 authentication keys which is defined by the HS v3 + protocol as part of [proposal 224] in Tor 0.3.0.4-alpha. + + * "5" -- support ESTABLISH_INTRO message DoS parameters extension for onion + service version 3 only in Tor 0.4.2.1-alpha. + +<a id="tor-spec.txt-9.5"></a> + +## "HSRend" + +The "HSRend" protocol handles rendezvous points. + + * "1" -- supports all features in Tor 0.0.6. + + * "2" -- supports RENDEZVOUS2 messages of arbitrary length as long as they + have 20 bytes of cookie in Tor 0.2.9.1-alpha. + +<a id="tor-spec.txt-9.6"></a> + +## "HSDir" + +The "HSDir" protocols are the set of hidden service document types that can +be uploaded to, understood by, and downloaded from a tor relay, and the set +of URLs available to fetch them. + + * "1" -- supports all features in Tor 0.2.0.10-alpha. + + * "2" -- support ed25519 blinded keys request which is defined by the HS v3 + protocol as part of [proposal 224] in Tor 0.3.0.4-alpha. + +<a id="tor-spec.txt-9.7"></a> + +## "DirCache" + +The "DirCache" protocols are the set of documents available for download +from a directory cache via BEGIN_DIR, and the set of URLs available to +fetch them. (This excludes URLs for hidden service objects.) + + * "1" -- supports all features in Tor 0.2.4.19. + + * "2" -- adds support for consensus diffs in Tor 0.3.1.1-alpha. + +<a id="tor-spec.txt-9.8"></a> + +## "Desc" + +Describes features present or absent in descriptors. + +Most features in descriptors don't require a "Desc" update -- only those +that need to someday be required. For example, someday clients will need +to understand ed25519 identities. + + * "1" -- supports all features in Tor 0.2.4.19. + + * "2" -- cross-signing with onion-keys, signing with ed25519 +identities. + +<a id="tor-spec.txt-9.9"></a> + +## "Microdesc" + +Describes features present or absent in microdescriptors. + +Most features in descriptors don't require a "MicroDesc" update -- only +those that need to someday be required. These correspond more or less with +consensus methods. + + * "1" -- consensus methods 9 through 20. + + * "2" -- consensus method 21 (adds ed25519 keys to microdescs). + +<a id="tor-spec.txt-9.10"></a> + +## "Cons" + +Describes features present or absent in consensus documents. + +Most features in consensus documents don't require a "Cons" update -- only +those that need to someday be required. + +These correspond more or less with consensus methods. + + * "1" -- consensus methods 9 through 20. + + * "2" -- consensus method 21 (adds ed25519 keys to microdescs). + +<a id="tor-spec.txt-9.11"></a> + +## "Padding" + +Describes the padding capabilities of the relay. + + * "1" -- \[DEFUNCT\] Relay supports circuit-level padding. This version MUST NOT + be used as it was also enabled in relays that don't actually support + circuit-level padding. Advertised by Tor versions from + tor-0.4.0.1-alpha and only up to and including tor-0.4.1.4-rc. + + * "2" -- Relay supports the HS circuit setup padding machines ([proposal 302]). + Advertised by Tor versions from tor-0.4.1.5 and onwards. + +<a id="tor-spec.txt-9.12"></a> + +## "FlowCtrl" + +Describes the flow control protocol at the circuit and stream level. If +there is no FlowCtrl advertised, tor supports the unauthenticated flow +control features (version 0). + + * "1" -- supports authenticated circuit level SENDMEs as of [proposal 289] in + Tor 0.4.1.1-alpha. + + * "2" -- supports congestion control by the Exits which implies a new SENDME + format and algorithm. See [proposal 324] for more details. Advertised + in tor 0.4.7.3-alpha. + +## "Conflux" + +Describes the communications mechanisms used to bundle circuits together, +in order to split traffic across multiple paths. + +> TODO: This is not yet described here. For details see +> [proposal 329]. + +<a id="tor-spec.txt-9.13"></a> + +## "Datagram" + +Describes the UDP protocol capabilities of a relay. + + * "1" -- \[RESERVED\] supports UDP by an Exit as in the relay command + CONNECT_UDP, CONNECTED_UDP and DATAGRAM. See [proposal 339] for more details. + (Not yet advertised, reserved) + +[proposal 121]: ../proposals/121-hidden-service-authentication.txt +[proposal 224]: ../proposals/224-rend-spec-ng.txt +[proposal 244]: ../proposals/244-use-rfc5705-for-tls-binding.txt +[proposal 289]: ../proposals/289-authenticated-sendmes.txt +[proposal 302]: ../proposals/302-padding-machines-for-onion-clients.txt +[proposal 311]: ../proposals/311-relay-ipv6-reachability.txt +[proposal 324]: ../proposals/324-rtt-congestion-control.txt +[proposal 329]: ../proposals/329-traffic-splitting.txt +[proposal 332]: ../proposals/332-ntor-v3-with-extra-data.md +[proposal 339]: ../proposals/339-udp-over-tor.md +[proposal 346]: ../proposals/346-protovers-again.md + diff --git a/spec/tor-spec/tearing-down-circuits.md b/spec/tor-spec/tearing-down-circuits.md new file mode 100644 index 0000000..fadd1fc --- /dev/null +++ b/spec/tor-spec/tearing-down-circuits.md @@ -0,0 +1,108 @@ +<a id="tor-spec.txt-5.4"></a> + +# Tearing down circuits{#tearing-down-circuits} + +Circuits are torn down when an unrecoverable error occurs along +the circuit, or when all streams on a circuit are closed and the +circuit's intended lifetime is over. + +ORs SHOULD also tear down circuits which attempt to create: + +- streams with RELAY_BEGIN, or +- rendezvous points with ESTABLISH_RENDEZVOUS, + ending at the first hop. Letting Tor be used as a single hop proxy makes + exit and rendezvous nodes a more attractive target for compromise. + +ORs MAY use multiple methods to check if they are the first hop: + +```text + * If an OR sees a circuit created with CREATE_FAST, the OR is sure to be + the first hop of a circuit. + * If an OR is the responder, and the initiator: + * did not authenticate the link, or + * authenticated with a key that is not in the consensus, + then the OR is probably the first hop of a circuit (or the second hop of + a circuit via a bridge relay). + + Circuits may be torn down either completely or hop-by-hop. +``` + +To tear down a circuit completely, an OR or OP sends a DESTROY +cell to the adjacent nodes on that circuit, using the appropriate +direction's circID. + +Upon receiving an outgoing DESTROY cell, an OR frees resources +associated with the corresponding circuit. If it's not the end of +the circuit, it sends a DESTROY cell for that circuit to the next OR +in the circuit. If the node is the end of the circuit, then it tears +down any associated edge connections (see +[Calculating the 'Digest' field](./relay-cells.md#digest-field)). + +After a DESTROY cell has been processed, an OR ignores all data or +DESTROY cells for the corresponding circuit. + +To tear down part of a circuit, the OP may send a RELAY_TRUNCATE message +signaling a given OR (Stream ID zero). That OR sends a DESTROY +cell to the next node in the circuit, and replies to the OP with a +RELAY_TRUNCATED message. + +\[Note: If an OR receives a TRUNCATE message and it has any relay cells +still queued on the circuit for the next node it will drop them +without sending them. This is not considered conformant behavior, +but it probably won't get fixed until a later version of Tor. Thus, +clients SHOULD NOT send a TRUNCATE message to a node running any current +version of Tor if a) they have sent relay cells through that node, +and b) they aren't sure whether those cells have been sent on yet.\] + +```text + When an unrecoverable error occurs along one a circuit, the nodes + must report it as follows: + * If possible, send a DESTROY cell to ORs _away_ from the client. + * If possible, send *either* a DESTROY cell towards the client, or + a RELAY_TRUNCATED cell towards the client. +``` + +Current versions of Tor do not reuse truncated RELAY_TRUNCATED +circuits: An OP, upon receiving a RELAY_TRUNCATED, will send +forward a DESTROY cell in order to entirely tear down the circuit. +Because of this, we recommend that relays should send DESTROY +towards the client, not RELAY_TRUNCATED. + +```text + NOTE: + In tor versions before 0.4.5.13, 0.4.6.11 and 0.4.7.9, relays would + handle an inbound DESTROY by sending the client a RELAY_TRUNCATED + message. Beginning with those versions, relays now propagate + DESTROY cells in either direction, in order to tell every + intermediary ORs to stop queuing data on the circuit. The earlier + behavior created queuing pressure on the intermediary ORs. +``` + +The payload of a DESTROY cell or RELAY_TRUNCATED message contains a single +octet, describing the reason that the circuit was +closed. RELAY_TRUNCATED message, and DESTROY cells sent \_towards the +client, should contain the actual reason from the list of error codes +below. Reasons in DESTROY cell SHOULD NOT be propagated downward or +upward, due to potential side channel risk: An OR receiving a DESTROY +command should use the DESTROYED reason for its next cell. An OP +should always use the NONE reason for its own DESTROY cells. + +The error codes are: + +```text + 0 -- NONE (No reason given.) + 1 -- PROTOCOL (Tor protocol violation.) + 2 -- INTERNAL (Internal error.) + 3 -- REQUESTED (A client sent a TRUNCATE command.) + 4 -- HIBERNATING (Not currently operating; trying to save bandwidth.) + 5 -- RESOURCELIMIT (Out of memory, sockets, or circuit IDs.) + 6 -- CONNECTFAILED (Unable to reach relay.) + 7 -- OR_IDENTITY (Connected to relay, but its OR identity was not + as expected.) + 8 -- CHANNEL_CLOSED (The OR connection that was carrying this circuit + died.) + 9 -- FINISHED (The circuit has expired for being dirty or old.) + 10 -- TIMEOUT (Circuit construction took too long) + 11 -- DESTROYED (The circuit was destroyed w/o client TRUNCATE) + 12 -- NOSUCHSERVICE (Request for unknown hidden service) +``` diff --git a/spec/vanguards-spec/full-vanguards.md b/spec/vanguards-spec/full-vanguards.md new file mode 100644 index 0000000..a3a9448 --- /dev/null +++ b/spec/vanguards-spec/full-vanguards.md @@ -0,0 +1,133 @@ +# Full Vanguards + +Full Vanguards is intended for use by long-lived onion services, which intend +to remain in operation for longer than one month. + +Full Vanguards achieves this longer expected duration by having +two layers of additional fixed relays, of different rotation periods. + +The rotation period of the first vanguard layer (layer 2 guards) is chosen +such that it requires an extremely long and persistent Sybil attack, or a +coercion/compromise attack. + +The rotation period of the second vanguard layer (layer 3 guards) is chosen to +be small enough to force the adversary to perform a Sybil attack against this +layer, rather than attempting to coerce these relays. + +## Threat model, Assumptions, and Goals + +Consider an adversary with the following powers: + +```text + - Can launch a Sybil guard discovery attack against any position of a + rendezvous circuit. The slower the rotation period of their target + position, the longer the attack takes. Similarly, the higher the + percentage of the network is controlled by them, the faster the + attack runs. + + - Can compromise additional relays on the network, but this compromise + takes time and potentially even coercive action, and also carries risk + of discovery. +``` + +We also make the following assumptions about the types of attacks: + + 1. A Sybil attack is observable by both people monitoring the network + for large numbers of new relays, as well as vigilant hidden service + operators. It will require large amounts of traffic sent towards the + hidden service over many many test circuits. + + 2. A Sybil attack requires either a protocol side channel or an application-layer + timing side channel in order to determine successful placement next + to the relay that the adversary is attempting to discover. When Tor's internal + protocol side channels are dealt with, this will be both observable and + controllable at the Application Layer, by operators. + + 3. The adversary is strongly disincentivized from compromising additional + relays that may prove useless, as active compromise attempts are even more + risky for the adversary than a Sybil attack in terms of being noticed. In + other words, the adversary is unlikely to attempt to compromise or coerce + additional relays that are in use for only a short period of time. + +Given this threat model, our security parameters were selected so that the +first two layers of guards should take a very long period of time to attack +using a Sybil guard discovery attack and hence require a relay compromise +attack. + +On the other hand, the outermost layer of guards (the third layer) should +rotate fast enough to _require_ a Sybil attack. If the adversary were to +attempt to compromise or coerce these relays after they are discovered, their +rotation times should be fast enough that the adversary has a very high +probability of them no longer being in use. + +# Design + +When a hidden service picks its guard relays, it also picks an +additional NUM_LAYER2_GUARDS-sized set of middle relays for its +`second_guard_set`, as well as a NUM_LAYER3_GUARDS-sized set of +middle relays for its `third_guard_set`. + +When a hidden service needs to establish a circuit to an HSDir, +introduction point or a rendezvous point, it uses relays from +`second_guard_set` as the second hop of the circuit and relays from +`third_guard_set` as third hop of the circuit. + +A hidden service rotates relays from the 'second_guard_set' at a uniformly +random time between MIN_SECOND_GUARD_LIFETIME hours and +MAX_SECOND_GUARD_LIFETIME hours, chosen for each layer 2 relay. +Implementations MAY expose this layer as an explicit configuration option to +pin specific relays, similar to how a user is allowed to pin specific Guards. + +A hidden service rotates relays from the 'third_guard_set' at a random time +between MIN_THIRD_GUARD_LIFETIME and MAX_THIRD_GUARD_LIFETIME hours, as +weighted by the [max(X,X) +distribution](./vanguards-stats.md#MaxDist), chosen for each +relay. This skewed distribution was chosen so that there is some probability +of a very short rotation period, to deter compromise/coercion, but biased +towards the longer periods, in favor of a somewhat lengthy Sybil attack. For +this reason, users SHOULD NOT be encouraged to pin this layer. + +Each relay's rotation time is tracked independently, to avoid disclosing +the rotation times of the primary and second-level guards. + +The selected vanguards and their rotation timestamp MUST be persisted to disk. + +## Parameterization + +We set NUM_LAYER2_GUARDS to 4 relays and NUM_LAYER3_GUARDS to 6 relays. + +We set MIN_SECOND_GUARD_LIFETIME to 30 days, and MAX_SECOND_GUARD_LIFETIME to +60 days inclusive, for an average rotation rate of 45 days, using a uniform +distribution. This range was chosen to average out to half of the Guard +rotation period; there is no strong motivation for it otherwise, other than to +be long. In fact, it could be set as long as the Guard rotation, and longer +periods MAY be provided as a configuration parameter. + +From the [Sybil rotation table](./vanguards-stats.md#SybilTable) in [statistical +analysis](./vanguards-stats.md), with NUM_LAYER2_GUARDS=4, it +can be seen that this means that the Sybil attack on layer3 will complete with +50% chance in 18\*45 days (2.2 years) for the 1% adversary, 180 days for the +5% adversary, and 90 days for the 10% adversary, with a 45 day average +rotation period. + +If this range is set equal to the Guard rotation period (90 days), the 50% +probability of Sybil success requires 18\*90 days (4.4 years) for the 1% +adversary, 4\*90 days (1 year) for the 5% adversary, and 2\*90 days (6 months) +for the 10% adversary. + +We set MIN_THIRD_GUARD_LIFETIME to 1 hour, and MAX_THIRD_GUARD_LIFETIME to 48 +hours inclusive, for an average rotation rate of 31.5 hours, using the +[max(X,X) distribution](./vanguards-stats.md#MaxDist). +(Again, this wide range and bias is used to discourage the adversary from +exclusively performing coercive attacks, as opposed to mounting the Sybil +attack, so increasing it substantially is not recommended). + +From the [Sybil rotation +table](./vanguards-stats.md#SybilTable) in [statistical +analysis](./vanguards-stats.md), with NUM_LAYER3_GUARDS=6, it +can be seen that this means that the Sybil attack on layer3 will complete with +50% chance in 9\*31.5 hours (15.75 days) for the 1% adversary, ~4 days for the +5% adversary, and 2.62 days for the 10% adversary. + +See the [statistical analysis](./vanguards-stats.md) for more +analysis on these constants. diff --git a/spec/vanguards-spec/index.md b/spec/vanguards-spec/index.md new file mode 100644 index 0000000..3d75065 --- /dev/null +++ b/spec/vanguards-spec/index.md @@ -0,0 +1,144 @@ +# Tor Vanguards Specification + +<a id="vanguards-spec.txt-1"></a> + +## Introduction and motivation + +A guard discovery attack allows attackers to determine the guard relay of a +Tor client. The hidden service protocol provides an attack vector for a guard +discovery attack since anyone can force an HS to construct a 3-hop circuit to +a relay, and repeat this process until one of the adversary's middle relays +eventually ends up chosen in a circuit. These attacks are also possible to +perform against clients, by causing an application to make repeated +connections to multiple unique onion services. + +The adversary must use a protocol side channel to confirm that their relay was +chosen in this position (see [Proposal +#344](../proposals/344-protocol-info-leaks.txt)), and then learns the guard +relay of a client, or service. + +When a successful guard discovery attack is followed with compromise or +coercion of the guard relay, the onion service or onion client can be +deanonymized. Alternatively, Netflow analytics data purchase can be (and has +been) used for this deanonymization, without interacting with the Guard relay +directly (see again [Proposal #344](../proposals/344-protocol-info-leaks.txt)). + +This specification assumes that Tor protocol side channels have 100% accuracy +and are undetectable, for simplicity in [reasoning about expected attack +times](./vanguards-stats.md). Presently, such 100% accurate +side channels exist in silent form, in the Tor Protocol itself. + +As work on addressing Tor's protocol side channels progresses, these attacks +will require application-layer activity that can be monitored and managed by +service operators, as opposed to silent and unobservable side channel activity +via the Tor Protocol itself. Application-layer side channels are also expected +to have less accuracy than native Tor protocol side channels, due to the +possibility of false positives caused by similar application activity +elsewhere on the Tor network. Despite this, we will preserve the original +assumption of 100% accuracy, for simplicity of explanation. + +## Overview + +In this specification, we specify two forms of a multi-layered Guard system: +one for long-lived services, called Full Vanguards, and one for onion clients +and short-lived services, called Vanguards-Lite. + +Both approaches use a mesh topology, where circuits can be created from any +relay in a preceding layer to any relay in a subsequent layer. + +The core difference between these two mechanisms is that Full Vanguards has +two additional layers of fixed vanguard relays, which results in longer path +lengths and higher latency. In contrast, Vanguards-Lite has only one +additional layer of fixed vanguard relays, and preserves the original path +lengths in use by onion services. Thus, Full Vanguards comes with a +performance cost, where as Vanguards-Lite does not. The rotation periods +of the two approaches also differ. + +Vanguards-Lite MUST be the default for all onion service and onion client +activity; Full Vanguards SHOULD be available as an optional configuration +option for services. + +Neither system applies to Exit activity. + +### Terminology + +Tor's original guards are called First Layer Guards. + +The first layer of vanguards is at the second hop, and is called the +Second Layer Guards. + +The second layer of vanguards is at the third hop, and is called the Third +Layer Guards. + +### Visualizing Full Vanguards + +Full Vanguards pins these two middle positions into a mesh topology, where any +relay in a layer can be used in that position in a circuit, as follows: + +```text + -> vanguard_2A + -> vanguard_3A + -> guard_1A -> vanguard_2B -> vanguard_3B + HS -> vanguard_3C + -> guard_1B -> vanguard_2C -> vanguard_3D + -> vanguard_3E + -> vanguard_2D -> vanguard_3F +``` + +Additionally, to avoid trivial discovery of the third layer, and to minimize +linkability, we insert an extra middle relay after the third layer guard for +client side intro and hsdir circuits, and service-side rendezvous circuits. +This means that the set of paths for Client (C) and Service (S) side look like +this: + +```text + Client hsdir: C - G - L2 - L3 - M - HSDIR + Client intro: C - G - L2 - L3 - M - I + Client rend: C - G - L2 - L3 - R + Service hsdir: S - G - L2 - L3 - HSDIR + Service intro: S - G - L2 - L3 - I + Service rend: S - G - L2 - L3 - M - R +``` + +### Visualizing Vanguards-Lite + +Vanguards-Lite uses only one layer of vanguards: + +```text + -> vanguard_2A + + -> guard_1A -> vanguard_2B + HS + -> guard_1B -> vanguard_2C + + -> vanguard_2D +``` + +This yields shorter path lengths, of the following form: + +```text + Client hsdir: C -> G -> L2 -> M -> HSDir + Client intro: C -> G -> L2 -> M -> Intro + Client rend: C -> G -> L2 -> Rend + Service hsdir: C -> G -> L2 -> M -> HSDir + Service intro: C -> G -> L2 -> M -> Intro + Service rend: C -> G -> L2 -> M -> Rend +``` + +## Alternatives + +An alternative to vanguards for client activity is to restrict the number of +onion services that a Tor client is allowed to connect to, in a certain period +of time. This defense was explored in [Onion Not +Found](https://petsymposium.org/2022/files/papers/issue1/popets-2022-0026.pdf). + +We have opted not to deploy this defense, for three reasons: + + 1. It does not generalize to the service-side of onion services + 2. Setting appropriate rate limits on the number of onion service content +elements on a page for Tor Browser is difficult. Sites like Facebook use +multiple onion domains for various content elements on a single page. + 3. It is even more difficult to limit the number of service connections for + arbitrary applications, such as cryptocurrency wallets, mining, and other + distributed apps deployed on top of onion services that connect to multiple + services (such as Ricochet). diff --git a/spec/vanguards-spec/path-construction.md b/spec/vanguards-spec/path-construction.md new file mode 100644 index 0000000..521629e --- /dev/null +++ b/spec/vanguards-spec/path-construction.md @@ -0,0 +1,53 @@ +# Path Construction + +Both vanguards systems use a mesh topology: this means that circuits select +a hop from each layer independently, allowing paths from any relay in a +layer to any relay in the next layer. + +## Selecting Relays + +Vanguards relays are selected from relays with the Stable and Fast flags. + +Tor replaces a vanguard whenever it is no longer listed in the most +recent consensus, with the goal that we will always have the right +number of vanguards ready to be used. + +For implementation reasons, we also replace a vanguard if it loses +the Fast or Stable flag, because the path selection logic wants middle +nodes to have those flags when it's building preemptive vanguard-using +circuits. + +The design doesn't have to be this way: we might instead have chosen to keep +vanguards in our list as long as possible, and continue to use them even if +they have lost some flags. This tradeoff is similar to the one in [Bug +#17773](https://bugs.torproject.org/17773), about whether to continue using +Entry Guards if they lose the Guard flag -- and Tor's current choice is "no, +rotate" for that case too. + +## Path Restriction Changes + +Path restrictions, as well as the ordering of their application, are currently +extremely problematic, resulting in information leaks with this topology. +Until they are reworked, we disable many of them for onion service circuits. + +In particular, we allow the following: + 1. Nodes from the same /16 and same family for any/all hops in a circuit + 2. Guard nodes can be chosen for RP/IP/HSDIR + 3. Guard nodes can be chosen for hop before RP/IP/HSDIR. + +The first change prevents the situation where paths cannot be built if two +layers all share the same subnet and/or node family, or if a layer consists of +only one family and that family is the same as the RP/IP/HSDIR. It also +prevents the the use of a different entry guard based on the family or subnet +of the IP, HSDIR, or RP. (Alternatives of this permissive behavior are +possible: For example, each layer could ensure that it does not consist solely +of the same family or /16, but this property cannot easily apply to conflux +circuits). + +The second change prevents an adversary from forcing the use of a different +entry guard by enumerating all guard-flagged nodes as the RP. This change is +important once onion services support conflux. + +The third change prevents an adversary from learning the guard node by way +of noticing which nodes were not chosen for the hop before it. This change is +important once onion services support conflux. diff --git a/spec/vanguards-spec/vanguards-lite.md b/spec/vanguards-spec/vanguards-lite.md new file mode 100644 index 0000000..877a5ad --- /dev/null +++ b/spec/vanguards-spec/vanguards-lite.md @@ -0,0 +1,35 @@ +# Vanguards-Lite + +Vanguards-Lite is meant for short-lived onion services such as OnionShare, as +well as for onion client activity. + +It is designed for clients and services that expect to have an uptime of no +longer than 1 month. + +## Design + +Because it is for short-lived activity, its rotation times are chosen with the +Sybil adversary in mind, using the [max(X,X) skewed +distribution](./vanguards-stats.md#MaxDist). + +We let NUM_LAYER2_GUARDS=4. We also introduce a consensus parameter +`guard-hs-l2-number` that controls the number of layer2 guards (with a +maximum of 19 layer2 guards). + +No third layer of guards is used. + +We don't write guards on disk. This means that the guard topology resets +when tor restarts. + +## Rotation Period + +The Layer2 lifetime uses the max(x,x) distribution with a minimum of one day +and maximum of 22 days. This makes the average lifetime of approximately two +weeks. Significant extensions of this lifetime are not recommended, as they +will shift the balance in favor of coercive attacks. + +From the [Sybil Rotation Table](./vanguards-stats.md#SybilTable), +with NUM_LAYER2_GUARDS=4 it can be seen that this means that the Sybil attack +on Layer2 will complete with 50% chance in 18\*14 days (252 days) for the 1% +adversary, 4\*14 days (two months) for the 5% adversary, and 2\*14 days (one +month) for the 10% adversary. diff --git a/spec/vanguards-spec/vanguards-stats.md b/spec/vanguards-spec/vanguards-stats.md new file mode 100644 index 0000000..445f2f0 --- /dev/null +++ b/spec/vanguards-spec/vanguards-stats.md @@ -0,0 +1,195 @@ +# Vanguard Rotation Statistics + +## Sybil rotation counts for a given number of Guards {#SybilTable} + +The probability of Sybil success for Guard discovery can be modeled as +the probability of choosing 1 or more malicious middle nodes for a +sensitive circuit over some period of time. + +```text + P(At least 1 bad middle) = 1 - P(All Good Middles) + = 1 - P(One Good middle)^(num_middles) + = 1 - (1 - c/n)^(num_middles) +``` + +c/n is the adversary compromise percentage + +In the case of Vanguards, num_middles is the number of Guards you rotate +through in a given time period. This is a function of the number of vanguards +in that position (v), as well as the number of rotations (r). + +```text + P(At least one bad middle) = 1 - (1 - c/n)^(v*r) +``` + +Here's detailed tables in terms of the number of rotations required for +a given Sybil success rate for certain number of guards. + +```text + 1.0% Network Compromise: + Sybil Success One Two Three Four Five Six Eight Nine Ten Twelve Sixteen + 10% 11 6 4 3 3 2 2 2 2 1 1 + 15% 17 9 6 5 4 3 3 2 2 2 2 + 25% 29 15 10 8 6 5 4 4 3 3 2 + 50% 69 35 23 18 14 12 9 8 7 6 5 + 60% 92 46 31 23 19 16 12 11 10 8 6 + 75% 138 69 46 35 28 23 18 16 14 12 9 + 85% 189 95 63 48 38 32 24 21 19 16 12 + 90% 230 115 77 58 46 39 29 26 23 20 15 + 95% 299 150 100 75 60 50 38 34 30 25 19 + 99% 459 230 153 115 92 77 58 51 46 39 29 + + 5.0% Network Compromise: + Sybil Success One Two Three Four Five Six Eight Nine Ten Twelve Sixteen + 10% 3 2 1 1 1 1 1 1 1 1 1 + 15% 4 2 2 1 1 1 1 1 1 1 1 + 25% 6 3 2 2 2 1 1 1 1 1 1 + 50% 14 7 5 4 3 3 2 2 2 2 1 + 60% 18 9 6 5 4 3 3 2 2 2 2 + 75% 28 14 10 7 6 5 4 4 3 3 2 + 85% 37 19 13 10 8 7 5 5 4 4 3 + 90% 45 23 15 12 9 8 6 5 5 4 3 + 95% 59 30 20 15 12 10 8 7 6 5 4 + 99% 90 45 30 23 18 15 12 10 9 8 6 + + 10.0% Network Compromise: + Sybil Success One Two Three Four Five Six Eight Nine Ten Twelve Sixteen + 10% 2 1 1 1 1 1 1 1 1 1 1 + 15% 2 1 1 1 1 1 1 1 1 1 1 + 25% 3 2 1 1 1 1 1 1 1 1 1 + 50% 7 4 3 2 2 2 1 1 1 1 1 + 60% 9 5 3 3 2 2 2 1 1 1 1 + 75% 14 7 5 4 3 3 2 2 2 2 1 + 85% 19 10 7 5 4 4 3 3 2 2 2 + 90% 22 11 8 6 5 4 3 3 3 2 2 + 95% 29 15 10 8 6 5 4 4 3 3 2 + 99% 44 22 15 11 9 8 6 5 5 4 3 +``` + +The rotation counts in these tables were generated with: + +## Skewed Rotation Distribution {#MaxDist} + +In order to skew the distribution of the third layer guard towards +higher values, we use max(X,X) for the distribution, where X is a +random variable that takes on values from the uniform distribution. + +Here's a table of expectation (arithmetic means) for relevant +ranges of X (sampled from 0..N-1). The table was generated with the +following python functions: + +```text + + def ProbMinXX(N, i): return (2.0*(N-i)-1)/(N*N) + def ProbMaxXX(N, i): return (2.0*i+1)/(N*N) + + def ExpFn(N, ProbFunc): + exp = 0.0 + for i in range(N): exp += i*ProbFunc(N, i) + return exp +``` + +The current choice for second-layer Vanguards-Lite guards is noted with **, +and the current choice for third-layer Full Vanguards is noted with ***. + +```text + Range Min(X,X) Max(X,X) + 22 6.84 14.16** + 23 7.17 14.83 + 24 7.51 15.49 + 25 7.84 16.16 + 26 8.17 16.83 + 27 8.51 17.49 + 28 8.84 18.16 + 29 9.17 18.83 + 30 9.51 19.49 + 31 9.84 20.16 + 32 10.17 20.83 + 33 10.51 21.49 + 34 10.84 22.16 + 35 11.17 22.83 + 36 11.50 23.50 + 37 11.84 24.16 + 38 12.17 24.83 + 39 12.50 25.50 + 40 12.84 26.16 + 40 12.84 26.16 + 41 13.17 26.83 + 42 13.50 27.50 + 43 13.84 28.16 + 44 14.17 28.83 + 45 14.50 29.50 + 46 14.84 30.16 + 47 15.17 30.83 + 48 15.50 31.50*** +``` + +The Cumulative Density Function (CDF) tells us the probability that a +guard will no longer be in use after a given number of time units have +passed. + +Because the Sybil attack on the third node is expected to complete at any +point in the second node's rotation period with uniform probability, if we +want to know the probability that a second-level Guard node will still be in +use after t days, we first need to compute the probability distribution of +the rotation duration of the second-level guard at a uniformly random point +in time. Let's call this P(R=r). + +For P(R=r), the probability of the rotation duration depends on the selection +probability of a rotation duration, and the fraction of total time that +rotation is likely to be in use. This can be written as: + +```text + P(R=r) = ProbMaxXX(X=r)*r / \sum_{i=1}^N ProbMaxXX(X=i)*i +``` + + or in Python: + +```text + def ProbR(N, r, ProbFunc=ProbMaxXX): + return ProbFunc(N, r)*r/ExpFn(N, ProbFunc) +``` + +For the full CDF, we simply sum up the fractional probability density for +all rotation durations. For rotation durations less than t days, we add the +entire probability mass for that period to the density function. For +durations d greater than t days, we take the fraction of that rotation +period's selection probability and multiply it by t/d and add it to the +density. In other words: + +```text + def FullCDF(N, t, ProbFunc=ProbR): + density = 0.0 + for d in range(N): + if t >= d: density += ProbFunc(N, d) + # The +1's below compensate for 0-indexed arrays: + else: density += ProbFunc(N, d)*(float(t+1))/(d+1) + return density +``` + +Computing this yields the following distribution for our current parameters: + +```text + t P(SECOND_ROTATION <= t) + 1 0.03247 + 2 0.06494 + 3 0.09738 + 4 0.12977 + 5 0.16207 + 10 0.32111 + 15 0.47298 + 20 0.61353 + 25 0.73856 + 30 0.84391 + 35 0.92539 + 40 0.97882 + 45 1.00000 +``` + +This CDF tells us that for the second-level Guard rotation, the +adversary can expect that 3.3% of the time, their third-level Sybil +attack will provide them with a second-level guard node that has only +1 day remaining before it rotates. 6.5% of the time, there will +be only 2 day or less remaining, and 9.7% of the time, 3 days or less. + +Note that this distribution is still a day-resolution approximation. diff --git a/spec/version-spec.md b/spec/version-spec.md new file mode 100644 index 0000000..ab2abc2 --- /dev/null +++ b/spec/version-spec.md @@ -0,0 +1,87 @@ +# How Tor Version Numbers Work + +<a id="version-spec.txt-1"></a> + +## The Old Way + +Before 0.1.0, versions were of the format: + +MAJOR.MINOR.MICRO(status(PATCHLEVEL))?(-cvs)? + +where MAJOR, MINOR, MICRO, and PATCHLEVEL are numbers, status is one +of "pre" (for an alpha release), "rc" (for a release candidate), or +"." for a release. As a special case, "a.b.c" was equivalent to +"a.b.c.0". We compare the elements in order (major, minor, micro, +status, patchlevel, cvs), with "cvs" preceding non-cvs. + +We would start each development branch with a final version in mind: +say, "0.0.8". Our first pre-release would be "0.0.8pre1", followed by +(for example) "0.0.8pre2-cvs", "0.0.8pre2", "0.0.8pre3-cvs", +"0.0.8rc1", "0.0.8rc2-cvs", and "0.0.8rc2". Finally, we'd release +0.0.8. The stable CVS branch would then be versioned "0.0.8.1-cvs", +and any eventual bugfix release would be "0.0.8.1". + +<a id="version-spec.txt-2"></a> + +## The New Way + +Starting at 0.1.0.1-rc, versions are of the format: + +`MAJOR.MINOR.MICRO[.PATCHLEVEL][-STATUS_TAG][ (EXTRA_INFO)]*` + +The stuff in parentheses is optional. As before, MAJOR, MINOR, MICRO, +and PATCHLEVEL are numbers, with an absent number equivalent to 0. +All versions should be distinguishable purely by those four +numbers. + +The STATUS_TAG is purely informational, and lets you know how +stable we think the release is: "alpha" is pretty unstable; "rc" is a +release candidate; and no tag at all means that we have a final +release. If the tag ends with "-cvs" or "-dev", you're looking at a +development snapshot that came after a given release. If we *do* +encounter two versions that differ only by status tag, we compare them +lexically. The STATUS_TAG can't contain whitespace. + +The EXTRA_INFO is also purely informational, often containing information +about the SCM commit this version came from. It is surrounded by parentheses +and can't contain whitespace. Unlike the STATUS_TAG this never impacts the way +that versions should be compared. EXTRA_INFO may appear any number of +times. Tools should generally not parse EXTRA_INFO entries. + +Now, we start each development branch with (say) 0.1.1.1-alpha. The +patchlevel increments consistently as the status tag changes, for +example, as in: 0.1.1.2-alpha, 0.1.1.3-alpha, 0.1.1.4-rc, 0.1.1.5-rc. +Eventually, we release 0.1.1.6. The next patch release is 0.1.1.7. + +Between these releases, CVS is versioned with a -cvs tag: after +0.1.1.1-alpha comes 0.1.1.1-alpha-cvs, and so on. But starting with +0.1.2.1-alpha-dev, we switched to SVN and started using the "-dev" +suffix instead of the "-cvs" suffix. + +<a id="version-spec.txt-3"></a> + +## Version status + +```text + Sometimes we need to determine whether a Tor version is obsolete, + experimental, or neither, based on a list of recommended versions. The + logic is as follows: + + * If a version is listed on the recommended list, then it is + "recommended". + + * If a version is newer than every recommended version, that version + is "experimental" or "new". + + * If a version is older than every recommended version, it is + "obsolete" or "old". + + * The first three components (major,minor,micro) of a version number + are its "release series". If a version has other recommended + versions with the same release series, and the version is newer + than all such recommended versions, but it is not newer than + _every_ recommended version, then the version is "new in series". + + * Finally, if none of the above conditions hold, then the version is + "un-recommended." +``` |
