diff options
75 files changed, 799 insertions, 372 deletions
diff --git a/.travis.yml b/.travis.yml index bda1f323e5..8e258aef26 100644 --- a/.travis.yml +++ b/.travis.yml @@ -216,7 +216,7 @@ script: ## We run `make check` because that's what https://jenkins.torproject.org does. - if [[ "$DISTCHECK" == "" && "$TEST_STEM" == "" ]]; then make check; fi ## Diagnostic for bug 29437: kill stem if it hangs for 15 minutes - - if [[ "$TEST_STEM" != "" ]]; then timelimit -p -t 540 -T 30 make src/app/tor test-stem; fi + - if [[ "$TEST_STEM" != "" ]]; then make src/app/tor; timelimit -p -t 540 -s USR1 -T 30 -S ABRT python3 "$STEM_SOURCE_DIR"/run_tests.py --tor src/app/tor --integ --log notice --target RUN_ALL; fi - if [[ "$DISTCHECK" != "" && "$TEST_STEM" == "" ]]; then make distcheck DISTCHECK_CONFIGURE_FLAGS="$CONFIGURE_FLAGS"; fi ## If this build was one that produced coverage, upload it. - if [[ "$COVERAGE_OPTIONS" != "" ]]; then coveralls -b . --exclude src/test --exclude src/trunnel --gcov-options '\-p' || echo "Coverage failed"; fi @@ -230,6 +230,7 @@ after_failure: ## `make distcheck` puts it somewhere different. - if [[ "$DISTCHECK" != "" ]]; then make show-distdir-testlog || echo "make failed"; fi - if [[ "$DISTCHECK" != "" ]]; then make show-distdir-core || echo "make failed"; fi + - if [[ "$TEST_STEM" != "" ]]; then cat "$STEM_SOURCE_DIR"/test/data/tor_log || echo "cat failed"; fi before_cache: ## Delete all gcov files. diff --git a/Makefile.am b/Makefile.am index de11696965..827cf3dc9b 100644 --- a/Makefile.am +++ b/Makefile.am @@ -226,7 +226,7 @@ shellcheck: if command -v shellcheck; then \ find $(top_srcdir)/scripts/ -name "*.sh" -exec shellcheck {} +; \ if [ -d "$(top_srcdir)/scripts/test" ]; then \ - shellcheck $(top_srcdir)/scripts/test/cov-diff $(top_builddir)/scripts/test/coverage; \ + shellcheck $(top_srcdir)/scripts/test/cov-diff $(top_srcdir)/scripts/test/coverage; \ fi; \ fi diff --git a/changes/bug24490 b/changes/bug24490 new file mode 100644 index 0000000000..cf9281c878 --- /dev/null +++ b/changes/bug24490 @@ -0,0 +1,5 @@ + o Minor bugfixes (bridge authority): + - We set bridges as running when we dump the bridge status to a file. + Previously, we set bridges as running in a GETINFO controller, but + these shouldn't modify vital data structures. Fixes bug 24490; + bugfix on 0.2.0.13-alpha. Patch by Neel Chauhan diff --git a/changes/bug27199 b/changes/bug27199 new file mode 100644 index 0000000000..f9d2a422f9 --- /dev/null +++ b/changes/bug27199 @@ -0,0 +1,3 @@ + o Minor bugfixes (rust): + - Abort on panic in all build profiles, instead of potentially unwinding + into C code. Fixes bug 27199; bugfix on 0.3.3.1-alpha. diff --git a/changes/bug28269 b/changes/bug28269 new file mode 100644 index 0000000000..bdfe9e1aae --- /dev/null +++ b/changes/bug28269 @@ -0,0 +1,7 @@ + o Minor bugfixes (onion services): + - If we are launching repeated HSFETCH queries and are rate-limited, + we introduce a new controller response QUERY_RATE_LIMITED instead + of QUERY_NO_HSDIR, while keeping the latter for when onion service + directories are missing a descriptor. Previously, we returned + QUERY_NO_HSDIR for both cases. Fixes bug 28269; bugfix on + 0.3.1.1-alpha. Patch by Neel Chauhan diff --git a/changes/bug29613 b/changes/bug29613 new file mode 100644 index 0000000000..e966973255 --- /dev/null +++ b/changes/bug29613 @@ -0,0 +1,5 @@ + o Minor bugfixes (relay): + - If we are are a relay and have IPv6Exit to 1 while ExitRelay is + auto, we act as if ExitRelay is 1. Previously, we ignored IPv6Exit + if ExitRelay was 0 or auto. Fixes bug 29613; bugfix on 0.3.5.1-alpha. + Patch by Neel Chauhan. diff --git a/changes/bug30001 b/changes/bug30001 new file mode 100644 index 0000000000..52e58872ef --- /dev/null +++ b/changes/bug30001 @@ -0,0 +1,7 @@ + o Minor features (testing): + - Use the approx_time() function when setting the "Expires" header + in directory replies, to make them more testable. Needed for + ticket 30001. + o Minor bug fixes (testing): + - Check the time in the "Expires" header with approx_time(). + Fixes bug 30001; bugfix on 0.4.0.4-rc. diff --git a/changes/bug30151 b/changes/bug30151 new file mode 100644 index 0000000000..8ac9a320a0 --- /dev/null +++ b/changes/bug30151 @@ -0,0 +1,5 @@ + o Minor bugfixes (tor-resolve): + - Fix a memory leak in tor-resolve that could happen if Tor gave it a + malformed SOCKS response. (Memory leaks in tor-resolve don't actually + matter, but it's good to fix them anyway.) Fixes bug 30151; bugfix on + 0.4.0.1-alpha. diff --git a/changes/bug30189 b/changes/bug30189 new file mode 100644 index 0000000000..f8c932a5f9 --- /dev/null +++ b/changes/bug30189 @@ -0,0 +1,4 @@ + o Minor bugfixes (compilation, unusual configuration): + - Avoid failures when building with ALL_BUGS_ARE_FAILED due to + missing declarations of abort(), and prevent other such failures + in the future. Fixes bug 30189; bugfix on 0.3.4.1-alpha. diff --git a/changes/bug30190 b/changes/bug30190 new file mode 100644 index 0000000000..e2352c3b9c --- /dev/null +++ b/changes/bug30190 @@ -0,0 +1,3 @@ + o Minor bugfixes (lib): + do not log a warning for OpenSSL versions that should be compatible + Fixes bug 30190; bugfix on 0.2.4.2-alpha diff --git a/changes/bug30263 b/changes/bug30263 new file mode 100644 index 0000000000..ba81c1b8a1 --- /dev/null +++ b/changes/bug30263 @@ -0,0 +1,3 @@ + o Minor bugfixes (shellcheck): + - Stop looking for scripts in the build directory during + "make shellcheck". Fixes bug 30263; bugfix on 0.4.0.1-alpha. diff --git a/changes/diagnostic_28223_redux b/changes/diagnostic_28223_redux new file mode 100644 index 0000000000..0d7499832e --- /dev/null +++ b/changes/diagnostic_28223_redux @@ -0,0 +1,4 @@ + o Minor features (diagnostic): + - Add more diagnostic log messages in an attempt to solve + the issue of NUL bytes appearing in a microdescriptor cache. + Related to ticket 28223. diff --git a/changes/ticket27821 b/changes/ticket27821 new file mode 100644 index 0000000000..158f308fbf --- /dev/null +++ b/changes/ticket27821 @@ -0,0 +1,3 @@ + o Minor features (HTTP tunnel): + - Return an informative web page when the HTTPTunnelPort is used as an + HTTP proxy. Closes ticket 27821, patch by "eighthave". diff --git a/changes/ticket29732 b/changes/ticket29732 new file mode 100644 index 0000000000..bb72361c48 --- /dev/null +++ b/changes/ticket29732 @@ -0,0 +1,5 @@ + o Minor features (testing): + - Tor's unit test code now contains a standard set of functions to + replace the PRNG with a deterministic or reproducible version for + testing. Previously, various tests implemented this in various ways. + Implements ticket 29732. diff --git a/changes/ticket30033 b/changes/ticket30033 new file mode 100644 index 0000000000..3f66d049c8 --- /dev/null +++ b/changes/ticket30033 @@ -0,0 +1,4 @@ + o Minor features (developer tooling): + - Call pre-commit git hook from pre-push hook to make sure we're + running documentation and code style checks before pushing to remote + git repository. Implements feature 30033. diff --git a/changes/ticket30051 b/changes/ticket30051 new file mode 100644 index 0000000000..87b6d7611f --- /dev/null +++ b/changes/ticket30051 @@ -0,0 +1,5 @@ + o Minor features (developer tooling): + - Call practracker from pre-push and pre-commit git hooks to let a + developer know if they made any code style violations in their last + commit. This should help preventing code style violations appearing + upstream. Closes ticket 30051. diff --git a/changes/ticket30075 b/changes/ticket30075 new file mode 100644 index 0000000000..288abd7674 --- /dev/null +++ b/changes/ticket30075 @@ -0,0 +1,3 @@ + o Removed features: + - Remove the obsolete script at contrib/dist/tor.sh.in. Resolves issue + 30075. diff --git a/changes/ticket30077 b/changes/ticket30077 new file mode 100644 index 0000000000..9be014730e --- /dev/null +++ b/changes/ticket30077 @@ -0,0 +1,2 @@ + o Code simplification and refactoring (shell scripts): + - Fix shellcheck warnings in fuzz_multi.sh. Resolves issue 30077. diff --git a/changes/ticket30114 b/changes/ticket30114 new file mode 100644 index 0000000000..a80f7f4dcf --- /dev/null +++ b/changes/ticket30114 @@ -0,0 +1,3 @@ + o Minor features (git scripts): + - In git-pull-all.sh, also fetch the latest tor-github pull requests. + Implements ticket 30114. diff --git a/changes/ticket30117 b/changes/ticket30117 new file mode 100644 index 0000000000..5b6e6dabf7 --- /dev/null +++ b/changes/ticket30117 @@ -0,0 +1,4 @@ + o Testing (continuous integration): + - In Travis, tell timelimit to use stem's backtrace signals. And launch + python directly from timelimit, so python receives the signals from + timelimit, rather than make. Closes ticket 30117. diff --git a/changes/ticket30149 b/changes/ticket30149 new file mode 100644 index 0000000000..a21687ac2f --- /dev/null +++ b/changes/ticket30149 @@ -0,0 +1,3 @@ + o Code simplification and refactoring: + - Add several assertions in an attempt to fix some Coverity warnings. + Closes ticket 30149. diff --git a/changes/ticket30176 b/changes/ticket30176 new file mode 100644 index 0000000000..da23760ce5 --- /dev/null +++ b/changes/ticket30176 @@ -0,0 +1,4 @@ + o Minor features (defense in depth): + - In smartlist_remove_keeporder(), set any pointers that become + unused to NULL, in case a bug causes them to be used later. Closes + ticket 30176. Patch from Tobias Stoeckmann. diff --git a/changes/ticket30234 b/changes/ticket30234 new file mode 100644 index 0000000000..5a0076bad2 --- /dev/null +++ b/changes/ticket30234 @@ -0,0 +1,2 @@ + o Testing (continuous integration): + - In Travis, show stem's tor log after failure. Closes ticket 30234. diff --git a/changes/ticket30261 b/changes/ticket30261 new file mode 100644 index 0000000000..e4a2643c88 --- /dev/null +++ b/changes/ticket30261 @@ -0,0 +1,4 @@ + o Documentation: + - Document how to find git commits and tags for bug fixes in + CodingStandards.md. And update some changes file documentation. + Closes ticket 30261. diff --git a/configure.ac b/configure.ac index 0b80669f03..3ea578bbba 100644 --- a/configure.ac +++ b/configure.ac @@ -2459,7 +2459,6 @@ AC_CONFIG_FILES([ config.rust contrib/dist/suse/tor.sh contrib/operator-tools/tor.logrotate - contrib/dist/tor.sh contrib/dist/torctl contrib/dist/tor.service src/config/torrc.sample diff --git a/contrib/dist/tor.sh.in b/contrib/dist/tor.sh.in deleted file mode 100644 index 92f890681f..0000000000 --- a/contrib/dist/tor.sh.in +++ /dev/null @@ -1,123 +0,0 @@ -#!/bin/sh -# -# tor The Onion Router -# -# Startup/shutdown script for tor. This is a wrapper around torctl; -# torctl does the actual work in a relatively system-independent, or at least -# distribution-independent, way, and this script deals with fitting the -# whole thing into the conventions of the particular system at hand. -# This particular script is written for Red Hat/Fedora Linux, and may -# also work on Mandrake, but not SuSE. -# -# These next couple of lines "declare" tor for the "chkconfig" program, -# originally from SGI, used on Red Hat/Fedora and probably elsewhere. -# -# chkconfig: 2345 90 10 -# description: Onion Router - A low-latency anonymous proxy -# - -PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin -DAEMON=/usr/sbin/tor -NAME=tor -DESC="tor daemon" -TORPIDDIR=/var/run/tor -TORPID=$TORPIDDIR/tor.pid -WAITFORDAEMON=60 -ARGS="" - -# Library functions -if [ -f /etc/rc.d/init.d/functions ]; then - . /etc/rc.d/init.d/functions -elif [ -f /etc/init.d/functions ]; then - . /etc/init.d/functions -fi - -TORCTL=@BINDIR@/torctl - -# torctl will use these environment variables -TORUSER=@TORUSER@ -export TORUSER - -if [ -x /bin/su ] ; then - SUPROG=/bin/su -elif [ -x /sbin/su ] ; then - SUPROG=/sbin/su -elif [ -x /usr/bin/su ] ; then - SUPROG=/usr/bin/su -elif [ -x /usr/sbin/su ] ; then - SUPROG=/usr/sbin/su -else - SUPROG=/bin/su -fi - -# Raise ulimit based on number of file descriptors available (thanks, Debian) - -if [ -r /proc/sys/fs/file-max ]; then - system_max=`cat /proc/sys/fs/file-max` - if [ "$system_max" -gt "80000" ] ; then - MAX_FILEDESCRIPTORS=32768 - elif [ "$system_max" -gt "40000" ] ; then - MAX_FILEDESCRIPTORS=16384 - elif [ "$system_max" -gt "10000" ] ; then - MAX_FILEDESCRIPTORS=8192 - else - MAX_FILEDESCRIPTORS=1024 - cat << EOF - -Warning: Your system has very few filedescriptors available in total. - -Maybe you should try raising that by adding 'fs.file-max=100000' to your -/etc/sysctl.conf file. Feel free to pick any number that you deem appropriate. -Then run 'sysctl -p'. See /proc/sys/fs/file-max for the current value, and -file-nr in the same directory for how many of those are used at the moment. - -EOF - fi -else - MAX_FILEDESCRIPTORS=8192 -fi - -NICE="" - -case "$1" in - - start) - if [ -n "$MAX_FILEDESCRIPTORS" ]; then - echo -n "Raising maximum number of filedescriptors (ulimit -n) to $MAX_FILEDESCRIPTORS" - if ulimit -n "$MAX_FILEDESCRIPTORS" ; then - echo "." - else - echo ": FAILED." - fi - fi - - action $"Starting tor:" $TORCTL start - RETVAL=$? - ;; - - stop) - action $"Stopping tor:" $TORCTL stop - RETVAL=$? - ;; - - restart) - action $"Restarting tor:" $TORCTL restart - RETVAL=$? - ;; - - reload) - action $"Reloading tor:" $TORCTL reload - RETVAL=$? - ;; - - status) - $TORCTL status - RETVAL=$? - ;; - - *) - echo "Usage: $0 (start|stop|restart|reload|status)" - RETVAL=1 -esac - -exit $RETVAL diff --git a/contrib/include.am b/contrib/include.am index 742bc58163..9f4775632c 100644 --- a/contrib/include.am +++ b/contrib/include.am @@ -4,7 +4,6 @@ EXTRA_DIST+= \ contrib/client-tools/torify \ contrib/dist/rc.subr \ contrib/dist/suse/tor.sh.in \ - contrib/dist/tor.sh \ contrib/dist/torctl \ contrib/dist/tor.service.in \ contrib/operator-tools/tor-exit-notice.html \ diff --git a/doc/HACKING/CodingStandards.md b/doc/HACKING/CodingStandards.md index 4f229348e4..74db2a39a3 100644 --- a/doc/HACKING/CodingStandards.md +++ b/doc/HACKING/CodingStandards.md @@ -110,12 +110,41 @@ it's a bugfix, mention what bug it fixes and when the bug was introduced. To find out which Git tag the change was introduced in, you can use `git describe --contains <sha1 of commit>`. -If at all possible, try to create this file in the same commit where you are -making the change. Please give it a distinctive name that no other branch will -use for the lifetime of your change. To verify the format of the changes file, -you can use `make check-changes`. This is run automatically as part of -`make check` -- if it fails, we must fix it before we release. These -checks are implemented in `scripts/maint/lintChanges.py`. +If you don't know the commit, you can search the git diffs (-S) for the first +instance of the feature (--reverse). + +For example, for #30224, we wanted to know when the bridge-distribution-request +feature was introduced into Tor: + $ git log -S bridge-distribution-request --reverse + commit ebab521525 + Author: Roger Dingledine <arma@torproject.org> + Date: Sun Nov 13 02:39:16 2016 -0500 + + Add new BridgeDistribution config option + + $ git describe --contains ebab521525 + tor-0.3.2.3-alpha~15^2~4 + +If you need to know all the Tor versions that contain a commit, use: + $ git tag --contains 9f2efd02a1 | sort -V + tor-0.2.5.16 + tor-0.2.8.17 + tor-0.2.9.14 + tor-0.2.9.15 + ... + tor-0.3.0.13 + tor-0.3.1.9 + tor-0.3.1.10 + ... + +If at all possible, try to create the changes file in the same commit where +you are making the change. Please give it a distinctive name that no other +branch will use for the lifetime of your change. We usually use "ticketNNNNN" +or "bugNNNNN", where NNNNN is the ticket number. To verify the format of the +changes file, you can use `make check-changes`. This is run automatically as +part of `make check` -- if it fails, we must fix it as soon as possible, so +that our CI passes. These checks are implemented in +`scripts/maint/lintChanges.py`. Changes file style guide: * Changes files begin with " o Header (subheading):". The header diff --git a/doc/tor.1.txt b/doc/tor.1.txt index f992172405..cbbc3515bb 100644 --- a/doc/tor.1.txt +++ b/doc/tor.1.txt @@ -1935,13 +1935,14 @@ is non-zero): exit according to the ExitPolicy option, the ReducedExitPolicy option, or the default ExitPolicy (if no other exit policy option is specified). + + - If ExitRelay is set to 0, no traffic is allowed to - exit, and the ExitPolicy and ReducedExitPolicy options are ignored. + + If ExitRelay is set to 0, no traffic is allowed to exit, and the + ExitPolicy, ReducedExitPolicy, and IPv6Exit options are ignored. + + - If ExitRelay is set to "auto", then Tor checks the ExitPolicy and - ReducedExitPolicy options. If either is set, Tor behaves as if ExitRelay - were set to 1. If neither exit policy option is set, Tor behaves as if - ExitRelay were set to 0. (Default: auto) + If ExitRelay is set to "auto", then Tor checks the ExitPolicy, + ReducedExitPolicy, and IPv6Exit options. If at least one of these options + is set, Tor behaves as if ExitRelay were set to 1. If none of these exit + policy options are set, Tor behaves as if ExitRelay were set to 0. + (Default: auto) [[ExitPolicy]] **ExitPolicy** __policy__,__policy__,__...__:: Set an exit policy for this server. Each policy is of the form @@ -2136,8 +2137,9 @@ is non-zero): (Default: 0) [[IPv6Exit]] **IPv6Exit** **0**|**1**:: - If set, and we are an exit node, allow clients to use us for IPv6 - traffic. (Default: 0) + If set, and we are an exit node, allow clients to use us for IPv6 traffic. + When this option is set and ExitRelay is auto, we act as if ExitRelay + is 1. (Default: 0) [[MaxOnionQueueDelay]] **MaxOnionQueueDelay** __NUM__ [**msec**|**second**]:: If we have more onionskins queued for processing than we can process in diff --git a/scripts/git/git-pull-all.sh b/scripts/git/git-pull-all.sh index 0a4898a111..5d1d58e4bf 100755 --- a/scripts/git/git-pull-all.sh +++ b/scripts/git/git-pull-all.sh @@ -174,6 +174,19 @@ function fetch_origin fi } +# Fetch tor-github pull requests. No arguments. +function fetch_tor_github +{ + local cmd="git fetch tor-github" + printf " %s Fetching tor-github..." "$MARKER" + if [ $DRY_RUN -eq 0 ]; then + msg=$( eval "$cmd" 2>&1 ) + validate_ret $? "$msg" + else + printf "\\n %s\\n" "${IWTH}$cmd${CNRM}" + fi +} + ############### # Entry point # ############### @@ -188,8 +201,11 @@ while getopts "n" opt; do esac done -# First, fetch the origin. +# First, fetch tor-github. goto_repo "$ORIGIN_PATH" +fetch_tor_github + +# Then, fetch the origin. fetch_origin # Go over all configured worktree. diff --git a/scripts/git/pre-commit.git-hook b/scripts/git/pre-commit.git-hook index 65fa99f4c4..b285776c04 100755 --- a/scripts/git/pre-commit.git-hook +++ b/scripts/git/pre-commit.git-hook @@ -12,7 +12,7 @@ cd "$workdir" || exit 1 set -e -if [ ! -z "ls ./changes/*" ]; then +if [ -n "$(ls ./changes/)" ]; then python scripts/maint/lintChanges.py ./changes/* fi @@ -26,7 +26,7 @@ if [ -d src/lib ]; then src/test/*.[ch] \ src/test/*/*.[ch] \ src/tools/*.[ch] -elif [ -d src/common]; then +elif [ -d src/common ]; then # This was the layout before 0.3.5 perl scripts/maint/checkSpace.pl -C \ src/common/*/*.[ch] \ @@ -39,3 +39,7 @@ fi if test -e scripts/maint/checkIncludes.py; then python scripts/maint/checkIncludes.py fi + +if [ -e scripts/maint/practracker/practracker.py ]; then + python3 ./scripts/maint/practracker/practracker.py "$workdir" +fi diff --git a/scripts/git/pre-push.git-hook b/scripts/git/pre-push.git-hook index e7a72efa08..740180d6f6 100755 --- a/scripts/git/pre-push.git-hook +++ b/scripts/git/pre-push.git-hook @@ -27,6 +27,19 @@ ref_is_upstream_branch() { fi } +workdir=$(git rev-parse --show-toplevel) +if [ -x "$workdir/.git/hooks/pre-commit" ]; then + if ! "$workdir"/.git/hooks/pre-commit; then + exit 1 + fi +fi + +if [ -e scripts/maint/practracker/practracker.py ]; then + if ! python3 ./scripts/maint/practracker/practracker.py "$workdir"; then + exit 1 + fi +fi + # shellcheck disable=SC2034 while read -r local_ref local_sha remote_ref remote_sha do diff --git a/scripts/maint/practracker/exceptions.txt b/scripts/maint/practracker/exceptions.txt index 438c1c582a..677e7f0d42 100644 --- a/scripts/maint/practracker/exceptions.txt +++ b/scripts/maint/practracker/exceptions.txt @@ -102,7 +102,7 @@ problem function-size /src/core/or/circuituse.c:circuit_get_open_circ_or_launch( problem function-size /src/core/or/circuituse.c:connection_ap_handshake_attach_circuit() 244 problem function-size /src/core/or/command.c:command_process_create_cell() 156 problem function-size /src/core/or/command.c:command_process_relay_cell() 132 -problem file-size /src/core/or/connection_edge.c 4550 +problem file-size /src/core/or/connection_edge.c 4575 problem include-count /src/core/or/connection_edge.c 64 problem function-size /src/core/or/connection_edge.c:connection_ap_expire_beginning() 117 problem function-size /src/core/or/connection_edge.c:connection_ap_handshake_rewrite() 192 @@ -117,7 +117,7 @@ problem include-count /src/core/or/connection_or.c 51 problem function-size /src/core/or/connection_or.c:connection_or_group_set_badness_() 105 problem function-size /src/core/or/connection_or.c:connection_or_client_learned_peer_id() 144 problem function-size /src/core/or/connection_or.c:connection_or_compute_authenticate_cell_body() 235 -problem file-size /src/core/or/policies.c 3163 +problem file-size /src/core/or/policies.c 3171 problem function-size /src/core/or/policies.c:policy_summarize() 107 problem function-size /src/core/or/protover.c:protover_all_supported() 116 problem file-size /src/core/or/relay.c 3173 @@ -187,7 +187,7 @@ problem function-size /src/feature/dirclient/dirclient.c:handle_response_fetch_c problem function-size /src/feature/dircommon/consdiff.c:gen_ed_diff() 204 problem function-size /src/feature/dircommon/consdiff.c:apply_ed_diff() 159 problem function-size /src/feature/dirparse/authcert_parse.c:authority_cert_parse_from_string() 182 -problem function-size /src/feature/dirparse/microdesc_parse.c:microdescs_parse_from_string() 154 +problem function-size /src/feature/dirparse/microdesc_parse.c:microdescs_parse_from_string() 169 problem function-size /src/feature/dirparse/ns_parse.c:routerstatus_parse_entry_from_string() 286 problem function-size /src/feature/dirparse/ns_parse.c:networkstatus_verify_bw_weights() 389 problem function-size /src/feature/dirparse/ns_parse.c:networkstatus_parse_vote_from_string() 638 @@ -218,7 +218,7 @@ problem include-count /src/feature/nodelist/networkstatus.c 61 problem function-size /src/feature/nodelist/networkstatus.c:networkstatus_check_consensus_signature() 176 problem function-size /src/feature/nodelist/networkstatus.c:networkstatus_set_current_consensus() 293 problem function-size /src/feature/nodelist/node_select.c:router_pick_directory_server_impl() 123 -problem function-size /src/feature/nodelist/node_select.c:compute_weighted_bandwidths() 205 +problem function-size /src/feature/nodelist/node_select.c:compute_weighted_bandwidths() 206 problem function-size /src/feature/nodelist/node_select.c:router_pick_trusteddirserver_impl() 114 problem function-size /src/feature/nodelist/nodelist.c:compute_frac_paths_available() 193 problem file-size /src/feature/nodelist/routerlist.c 3234 @@ -287,5 +287,5 @@ problem function-size /src/lib/tls/tortls_openssl.c:tor_tls_context_new() 171 problem function-size /src/lib/tls/x509_nss.c:tor_tls_create_certificate_internal() 126 problem function-size /src/tools/tor-gencert.c:parse_commandline() 111 problem function-size /src/tools/tor-resolve.c:build_socks5_resolve_request() 104 -problem function-size /src/tools/tor-resolve.c:do_resolve() 173 +problem function-size /src/tools/tor-resolve.c:do_resolve() 175 problem function-size /src/tools/tor-resolve.c:main() 112 diff --git a/src/config/torrc.sample.in b/src/config/torrc.sample.in index c2ae707e93..9d514e6bda 100644 --- a/src/config/torrc.sample.in +++ b/src/config/torrc.sample.in @@ -174,13 +174,11 @@ ## Uncomment this if you want your relay to be an exit, with the default ## exit policy (or whatever exit policy you set below). -## (If ReducedExitPolicy or ExitPolicy are set, relays are exits. -## If neither exit policy option is set, relays are non-exits.) +## (If ReducedExitPolicy, ExitPolicy, or IPv6Exit are set, relays are exits. +## If none of these options are set, relays are non-exits.) #ExitRelay 1 ## Uncomment this if you want your relay to allow IPv6 exit traffic. -## You must also set ExitRelay, ReducedExitPolicy, or ExitPolicy to make your -## relay into an exit. ## (Relays do not allow any exit traffic by default.) #IPv6Exit 1 diff --git a/src/core/or/connection_edge.c b/src/core/or/connection_edge.c index 071a8c91ed..33ba723971 100644 --- a/src/core/or/connection_edge.c +++ b/src/core/or/connection_edge.c @@ -2810,6 +2810,31 @@ connection_ap_process_natd(entry_connection_t *conn) return connection_ap_rewrite_and_attach_if_allowed(conn, NULL, NULL); } +static const char HTTP_CONNECT_IS_NOT_AN_HTTP_PROXY_MSG[] = + "HTTP/1.0 405 Method Not Allowed\r\n" + "Content-Type: text/html; charset=iso-8859-1\r\n\r\n" + "<html>\n" + "<head>\n" + "<title>This is an HTTP CONNECT tunnel, not an full HTTP Proxy</title>\n" + "</head>\n" + "<body>\n" + "<h1>This is an HTTP CONNECT tunnel, not an HTTP proxy.</h1>\n" + "<p>\n" + "It appears you have configured your web browser to use this Tor port as\n" + "an HTTP proxy.\n" + "</p><p>\n" + "This is not correct: This port is configured as a CONNECT tunnel, not\n" + "an HTTP proxy. Please configure your client accordingly. You can also\n" + "use HTTPS, then the client should automatically use HTTP CONNECT." + "</p>\n" + "<p>\n" + "See <a href=\"https://www.torproject.org/documentation.html\">" + "https://www.torproject.org/documentation.html</a> for more " + "information.\n" + "</p>\n" + "</body>\n" + "</html>\n"; + /** Called on an HTTP CONNECT entry connection when some bytes have arrived, * but we have not yet received a full HTTP CONNECT request. Try to parse an * HTTP CONNECT request from the connection's inbuf. On success, set up the @@ -2850,7 +2875,7 @@ connection_ap_process_http_connect(entry_connection_t *conn) tor_assert(command); tor_assert(addrport); if (strcasecmp(command, "connect")) { - errmsg = "HTTP/1.0 405 Method Not Allowed\r\n\r\n"; + errmsg = HTTP_CONNECT_IS_NOT_AN_HTTP_PROXY_MSG; goto err; } diff --git a/src/core/or/policies.c b/src/core/or/policies.c index a6d66d36de..f59894ea8f 100644 --- a/src/core/or/policies.c +++ b/src/core/or/policies.c @@ -1164,6 +1164,15 @@ authdir_policy_badexit_address(uint32_t addr, uint16_t port) #define REJECT(arg) \ STMT_BEGIN *msg = tor_strdup(arg); goto err; STMT_END +/** Check <b>or_options</b> to determine whether or not we are using the + * default options for exit policy. Return true if so, false otherwise. */ +static int +policy_using_default_exit_options(const or_options_t *or_options) +{ + return (or_options->ExitPolicy == NULL && or_options->ExitRelay == -1 && + or_options->ReducedExitPolicy == 0 && or_options->IPv6Exit == 0); +} + /** Config helper: If there's any problem with the policy configuration * options in <b>options</b>, return -1 and set <b>msg</b> to a newly * allocated description of the error. Else return 0. */ @@ -1182,9 +1191,8 @@ validate_addr_policies(const or_options_t *options, char **msg) static int warned_about_nonexit = 0; - if (public_server_mode(options) && - !warned_about_nonexit && options->ExitPolicy == NULL && - options->ExitRelay == -1 && options->ReducedExitPolicy == 0) { + if (public_server_mode(options) && !warned_about_nonexit && + policy_using_default_exit_options(options)) { warned_about_nonexit = 1; log_notice(LD_CONFIG, "By default, Tor does not run as an exit relay. " "If you want to be an exit relay, " @@ -2141,9 +2149,9 @@ policies_parse_exit_policy_from_options(const or_options_t *or_options, int rv = 0; /* Short-circuit for non-exit relays, or for relays where we didn't specify - * ExitPolicy or ReducedExitPolicy and ExitRelay is auto. */ - if (or_options->ExitRelay == 0 || (or_options->ExitPolicy == NULL && - or_options->ExitRelay == -1 && or_options->ReducedExitPolicy == 0)) { + * ExitPolicy or ReducedExitPolicy or IPv6Exit and ExitRelay is auto. */ + if (or_options->ExitRelay == 0 || + policy_using_default_exit_options(or_options)) { append_exit_policy_string(result, "reject *4:*"); append_exit_policy_string(result, "reject *6:*"); return 0; diff --git a/src/feature/client/circpathbias.c b/src/feature/client/circpathbias.c index 1743ab5a81..e6af649ba7 100644 --- a/src/feature/client/circpathbias.c +++ b/src/feature/client/circpathbias.c @@ -176,6 +176,7 @@ pathbias_get_scale_threshold(const or_options_t *options) static double pathbias_get_scale_ratio(const or_options_t *options) { + (void) options; /* * The scale factor is the denominator for our scaling * of circuit counts for our path bias window. @@ -185,7 +186,8 @@ pathbias_get_scale_ratio(const or_options_t *options) */ int denominator = networkstatus_get_param(NULL, "pb_scalefactor", 2, 2, INT32_MAX); - (void) options; + tor_assert(denominator > 0); + /** * The mult factor is the numerator for our scaling * of circuit counts for our path bias window. It diff --git a/src/feature/control/fmt_serverstatus.c b/src/feature/control/fmt_serverstatus.c index a1ddd2119a..d224a1d234 100644 --- a/src/feature/control/fmt_serverstatus.c +++ b/src/feature/control/fmt_serverstatus.c @@ -66,11 +66,9 @@ list_server_status_v1(smartlist_t *routers, char **router_status_out, smartlist_t *rs_entries; time_t now = time(NULL); time_t cutoff = now - ROUTER_MAX_AGE_TO_PUBLISH; - const or_options_t *options = get_options(); /* We include v2 dir auths here too, because they need to answer * controllers. Eventually we'll deprecate this whole function; * see also networkstatus_getinfo_by_purpose(). */ - int authdir = authdir_mode_publishes_statuses(options); tor_assert(router_status_out); rs_entries = smartlist_new(); @@ -78,10 +76,7 @@ list_server_status_v1(smartlist_t *routers, char **router_status_out, SMARTLIST_FOREACH_BEGIN(routers, routerinfo_t *, ri) { const node_t *node = node_get_by_id(ri->cache_info.identity_digest); tor_assert(node); - if (authdir) { - /* Update router status in routerinfo_t. */ - dirserv_set_router_is_running(ri, now); - } + if (for_controller) { char name_buf[MAX_VERBOSE_NICKNAME_LEN+2]; char *cp = name_buf; diff --git a/src/feature/dirauth/bwauth.c b/src/feature/dirauth/bwauth.c index 1cfd8119df..e60c8b86bd 100644 --- a/src/feature/dirauth/bwauth.c +++ b/src/feature/dirauth/bwauth.c @@ -199,9 +199,32 @@ dirserv_get_credible_bandwidth_kb(const routerinfo_t *ri) } /** - * Read the measured bandwidth list file, apply it to the list of - * vote_routerstatus_t and store all the headers in <b>bw_file_headers</b>. + * Read the measured bandwidth list <b>from_file</b>: + * - store all the headers in <b>bw_file_headers</b>, + * - apply bandwidth lines to the list of vote_routerstatus_t in + * <b>routerstatuses</b>, + * - cache bandwidth lines for dirserv_get_bandwidth_for_router(), + * - expire old entries in the measured bandwidth cache, and + * - store the DIGEST_SHA256 of the contents of the file in <b>digest_out</b>. + * * Returns -1 on error, 0 otherwise. + * + * If the file can't be read, or is empty: + * - <b>bw_file_headers</b> is empty, + * - <b>routerstatuses</b> is not modified, + * - the measured bandwidth cache is not modified, and + * - <b>digest_out</b> is the zero-byte digest. + * + * Otherwise, if there is an error later in the file: + * - <b>bw_file_headers</b> contains all the headers up to the error, + * - <b>routerstatuses</b> is updated with all the relay lines up to the error, + * - the measured bandwidth cache is updated with all the relay lines up to + * the error, + * - if the timestamp is valid and recent, old entries in the measured + * bandwidth cache are expired, and + * - <b>digest_out</b> is the digest up to the first read error (if any). + * The digest is taken over all the readable file contents, even if the + * file is outdated or unparseable. */ int dirserv_read_measured_bandwidths(const char *from_file, @@ -223,15 +246,12 @@ dirserv_read_measured_bandwidths(const char *from_file, size_t n = 0; crypto_digest_t *digest = crypto_digest256_new(DIGEST_SHA256); - /* Initialise line, so that we can't possibly run off the end. */ - if (fp == NULL) { log_warn(LD_CONFIG, "Can't open bandwidth file at configured location: %s", from_file); goto err; } - /* If fgets fails, line is either unmodified, or indeterminate. */ if (tor_getline(&line,&n,fp) <= 0) { log_warn(LD_DIRSERV, "Empty bandwidth file"); goto err; @@ -345,6 +365,9 @@ dirserv_read_measured_bandwidths(const char *from_file, * the header block yet. If we encounter an incomplete bw line, return -1 but * don't warn since there could be additional header lines coming. If we * encounter a proper bw line, return 0 (and we got past the headers). + * + * If the line contains "vote=0", stop parsing it, and return -1, so that the + * line is ignored during voting. */ STATIC int measured_bw_line_parse(measured_bw_line_t *out, const char *orig_line, diff --git a/src/feature/dirauth/voteflags.c b/src/feature/dirauth/voteflags.c index 0a53c588d6..4040f162fa 100644 --- a/src/feature/dirauth/voteflags.c +++ b/src/feature/dirauth/voteflags.c @@ -29,6 +29,7 @@ #include "feature/nodelist/node_st.h" #include "feature/nodelist/routerinfo_st.h" +#include "feature/nodelist/routerlist_st.h" #include "feature/nodelist/vote_routerstatus_st.h" #include "lib/container/order.h" @@ -658,3 +659,20 @@ dirserv_set_routerstatus_testing(routerstatus_t *rs) rs->is_hs_dir = 0; } } + +/** Use dirserv_set_router_is_running() to set bridges as running if they're + * reachable. + * + * This function is called from set_bridge_running_callback() when running as + * a bridge authority. + */ +void +dirserv_set_bridges_running(time_t now) +{ + routerlist_t *rl = router_get_routerlist(); + + SMARTLIST_FOREACH_BEGIN(rl->routers, routerinfo_t *, ri) { + if (ri->purpose == ROUTER_PURPOSE_BRIDGE) + dirserv_set_router_is_running(ri, now); + } SMARTLIST_FOREACH_END(ri); +} diff --git a/src/feature/dirauth/voteflags.h b/src/feature/dirauth/voteflags.h index cca6f53746..18b29a5183 100644 --- a/src/feature/dirauth/voteflags.h +++ b/src/feature/dirauth/voteflags.h @@ -25,6 +25,8 @@ void set_routerstatus_from_routerinfo(routerstatus_t *rs, void dirserv_compute_performance_thresholds(digestmap_t *omit_as_sybil); +void dirserv_set_bridges_running(time_t now); + #ifdef VOTEFLAGS_PRIVATE /** Any descriptor older than this age causes the authorities to set the * StaleDesc flag. */ diff --git a/src/feature/dircache/dircache.c b/src/feature/dircache/dircache.c index 06bd298aca..1b36f716f4 100644 --- a/src/feature/dircache/dircache.c +++ b/src/feature/dircache/dircache.c @@ -124,7 +124,7 @@ write_http_response_header_impl(dir_connection_t *conn, ssize_t length, long cache_lifetime) { char date[RFC1123_TIME_LEN+1]; - time_t now = time(NULL); + time_t now = approx_time(); buf_t *buf = buf_new_with_capacity(1024); tor_assert(conn); diff --git a/src/feature/dirparse/microdesc_parse.c b/src/feature/dirparse/microdesc_parse.c index 3b11e65ca0..22cc1e272e 100644 --- a/src/feature/dirparse/microdesc_parse.c +++ b/src/feature/dirparse/microdesc_parse.c @@ -160,7 +160,22 @@ microdescs_parse_from_string(const char *s, const char *eos, if (tokenize_string(area, s, start_of_next_microdesc, tokens, microdesc_token_table, flags)) { - log_warn(LD_DIR, "Unparseable microdescriptor"); + const char *location; + switch (where) { + case SAVED_NOWHERE: + location = "download or generated string"; + break; + case SAVED_IN_CACHE: + location = "cache"; + break; + case SAVED_IN_JOURNAL: + location = "journal"; + break; + default: + location = "unknown location"; + break; + } + log_warn(LD_DIR, "Unparseable microdescriptor found in %s", location); goto next; } diff --git a/src/feature/hs/hs_client.c b/src/feature/hs/hs_client.c index 38b9646c17..7aec6d80bb 100644 --- a/src/feature/hs/hs_client.c +++ b/src/feature/hs/hs_client.c @@ -424,7 +424,7 @@ pick_hsdir_v3(const ed25519_public_key_t *onion_identity_pk) /* Pick an HSDir from the responsible ones. The ownership of * responsible_hsdirs is given to this function so no need to free it. */ - hsdir_rs = hs_pick_hsdir(responsible_hsdirs, base64_blinded_pubkey); + hsdir_rs = hs_pick_hsdir(responsible_hsdirs, base64_blinded_pubkey, NULL); return hsdir_rs; } diff --git a/src/feature/hs/hs_common.c b/src/feature/hs/hs_common.c index b2227432d2..d4736c2862 100644 --- a/src/feature/hs/hs_common.c +++ b/src/feature/hs/hs_common.c @@ -1589,20 +1589,25 @@ hs_purge_last_hid_serv_requests(void) /** Given the list of responsible HSDirs in <b>responsible_dirs</b>, pick the * one that we should use to fetch a descriptor right now. Take into account * previous failed attempts at fetching this descriptor from HSDirs using the - * string identifier <b>req_key_str</b>. + * string identifier <b>req_key_str</b>. We return whether we are rate limited + * into *<b>is_rate_limited_out</b> if it is not NULL. * * Steals ownership of <b>responsible_dirs</b>. * * Return the routerstatus of the chosen HSDir if successful, otherwise return * NULL if no HSDirs are worth trying right now. */ routerstatus_t * -hs_pick_hsdir(smartlist_t *responsible_dirs, const char *req_key_str) +hs_pick_hsdir(smartlist_t *responsible_dirs, const char *req_key_str, + bool *is_rate_limited_out) { smartlist_t *usable_responsible_dirs = smartlist_new(); const or_options_t *options = get_options(); routerstatus_t *hs_dir; time_t now = time(NULL); int excluded_some; + bool rate_limited = false; + int rate_limited_count = 0; + int responsible_dirs_count = smartlist_len(responsible_dirs); tor_assert(req_key_str); @@ -1622,6 +1627,7 @@ hs_pick_hsdir(smartlist_t *responsible_dirs, const char *req_key_str) if (last + hs_hsdir_requery_period(options) >= now || !node || !node_has_preferred_descriptor(node, 0)) { SMARTLIST_DEL_CURRENT(responsible_dirs, dir); + rate_limited_count++; continue; } if (!routerset_contains_node(options->ExcludeNodes, node)) { @@ -1629,6 +1635,10 @@ hs_pick_hsdir(smartlist_t *responsible_dirs, const char *req_key_str) } } SMARTLIST_FOREACH_END(dir); + if (rate_limited_count > 0 || responsible_dirs_count > 0) { + rate_limited = rate_limited_count == responsible_dirs_count; + } + excluded_some = smartlist_len(usable_responsible_dirs) < smartlist_len(responsible_dirs); @@ -1640,9 +1650,10 @@ hs_pick_hsdir(smartlist_t *responsible_dirs, const char *req_key_str) smartlist_free(responsible_dirs); smartlist_free(usable_responsible_dirs); if (!hs_dir) { + const char *warn_str = (rate_limited) ? "we are rate limited." : + "we requested them all recently without success"; log_info(LD_REND, "Could not pick one of the responsible hidden " - "service directories, because we requested them all " - "recently without success."); + "service directories, because %s.", warn_str); if (options->StrictNodes && excluded_some) { log_warn(LD_REND, "Could not pick a hidden service directory for the " "requested hidden service: they are all either down or " @@ -1654,6 +1665,10 @@ hs_pick_hsdir(smartlist_t *responsible_dirs, const char *req_key_str) hs_lookup_last_hid_serv_request(hs_dir, req_key_str, now, 1); } + if (is_rate_limited_out != NULL) { + *is_rate_limited_out = rate_limited; + } + return hs_dir; } diff --git a/src/feature/hs/hs_common.h b/src/feature/hs/hs_common.h index abf39fa431..3009780d90 100644 --- a/src/feature/hs/hs_common.h +++ b/src/feature/hs/hs_common.h @@ -241,7 +241,8 @@ void hs_get_responsible_hsdirs(const struct ed25519_public_key_t *blinded_pk, int use_second_hsdir_index, int for_fetching, smartlist_t *responsible_dirs); routerstatus_t *hs_pick_hsdir(smartlist_t *responsible_dirs, - const char *req_key_str); + const char *req_key_str, + bool *is_rate_limited_out); time_t hs_hsdir_requery_period(const or_options_t *options); time_t hs_lookup_last_hid_serv_request(routerstatus_t *hs_dir, diff --git a/src/feature/nodelist/microdesc.c b/src/feature/nodelist/microdesc.c index b4f05b63a0..36922561a0 100644 --- a/src/feature/nodelist/microdesc.c +++ b/src/feature/nodelist/microdesc.c @@ -70,6 +70,8 @@ struct microdesc_cache_t { }; static microdesc_cache_t *get_microdesc_cache_noload(void); +static void warn_if_nul_found(const char *inp, size_t len, int64_t offset, + const char *activity); /** Helper: computes a hash of <b>md</b> to place it in a hash table. */ static inline unsigned int @@ -223,6 +225,8 @@ dump_microdescriptor(int fd, microdesc_t *md, size_t *annotation_len_out) } md->off = tor_fd_getpos(fd); + warn_if_nul_found(md->body, md->bodylen, (int64_t) md->off, + "dumping a microdescriptor"); written = write_all_to_fd(fd, md->body, md->bodylen); if (written != (ssize_t)md->bodylen) { written = written < 0 ? 0 : written; @@ -482,6 +486,27 @@ microdesc_cache_clear(microdesc_cache_t *cache) cache->bytes_dropped = 0; } +static void +warn_if_nul_found(const char *inp, size_t len, int64_t offset, + const char *activity) +{ + const char *nul_found = memchr(inp, 0, len); + if (BUG(nul_found)) { + log_warn(LD_BUG, "Found unexpected NUL while %s, offset %"PRId64 + "at position %"TOR_PRIuSZ"/%"TOR_PRIuSZ".", + activity, offset, (nul_found - inp), len); + const char *start_excerpt_at, *eos = inp + len; + if ((nul_found - inp) >= 16) + start_excerpt_at = nul_found - 16; + else + start_excerpt_at = inp; + size_t excerpt_len = MIN(32, eos - start_excerpt_at); + char tmp[65]; + base16_encode(tmp, sizeof(tmp), start_excerpt_at, excerpt_len); + log_warn(LD_BUG, " surrounding string: %s", tmp); + } +} + /** Reload the contents of <b>cache</b> from disk. If it is empty, load it * for the first time. Return 0 on success, -1 on failure. */ int @@ -499,6 +524,7 @@ microdesc_cache_reload(microdesc_cache_t *cache) mm = cache->cache_content = tor_mmap_file(cache->cache_fname); if (mm) { + warn_if_nul_found(mm->data, mm->size, 0, "scanning microdesc cache"); added = microdescs_add_to_cache(cache, mm->data, mm->data+mm->size, SAVED_IN_CACHE, 0, -1, NULL); if (added) { @@ -511,6 +537,8 @@ microdesc_cache_reload(microdesc_cache_t *cache) RFTS_IGNORE_MISSING, &st); if (journal_content) { cache->journal_len = (size_t) st.st_size; + warn_if_nul_found(journal_content, cache->journal_len, 0, + "reading microdesc journal"); added = microdescs_add_to_cache(cache, journal_content, journal_content+st.st_size, SAVED_IN_JOURNAL, 0, -1, NULL); diff --git a/src/feature/nodelist/networkstatus.c b/src/feature/nodelist/networkstatus.c index ee22e16323..acc2f0af26 100644 --- a/src/feature/nodelist/networkstatus.c +++ b/src/feature/nodelist/networkstatus.c @@ -2381,7 +2381,6 @@ networkstatus_getinfo_by_purpose(const char *purpose_string, time_t now) smartlist_t *statuses; const uint8_t purpose = router_purpose_from_string(purpose_string); routerstatus_t rs; - const int bridge_auth = authdir_mode_bridge(get_options()); if (purpose == ROUTER_PURPOSE_UNKNOWN) { log_info(LD_DIR, "Unrecognized purpose '%s' when listing router statuses.", @@ -2398,9 +2397,6 @@ networkstatus_getinfo_by_purpose(const char *purpose_string, time_t now) continue; if (ri->purpose != purpose) continue; - /* TODO: modifying the running flag in a getinfo is a bad idea */ - if (bridge_auth && ri->purpose == ROUTER_PURPOSE_BRIDGE) - dirserv_set_router_is_running(ri, now); /* then generate and write out status lines for each of them */ set_routerstatus_from_routerinfo(&rs, node, ri, now, 0); smartlist_add(statuses, networkstatus_getinfo_helper_single(&rs)); @@ -2412,11 +2408,12 @@ networkstatus_getinfo_by_purpose(const char *purpose_string, time_t now) return answer; } -/** Write out router status entries for all our bridge descriptors. */ +/** Write out router status entries for all our bridge descriptors. Here, we + * also mark routers as running. */ void networkstatus_dump_bridge_status_to_file(time_t now) { - char *status = networkstatus_getinfo_by_purpose("bridge", now); + char *status; char *fname = NULL; char *thresholds = NULL; char *published_thresholds_and_status = NULL; @@ -2425,6 +2422,9 @@ networkstatus_dump_bridge_status_to_file(time_t now) char fingerprint[FINGERPRINT_LEN+1]; char *fingerprint_line = NULL; + dirserv_set_bridges_running(now); + status = networkstatus_getinfo_by_purpose("bridge", now); + if (me && crypto_pk_get_fingerprint(me->identity_pkey, fingerprint, 0) >= 0) { tor_asprintf(&fingerprint_line, "fingerprint %s\n", fingerprint); diff --git a/src/feature/nodelist/node_select.c b/src/feature/nodelist/node_select.c index e31abb247f..93ddb066d4 100644 --- a/src/feature/nodelist/node_select.c +++ b/src/feature/nodelist/node_select.c @@ -585,6 +585,7 @@ compute_weighted_bandwidths(const smartlist_t *sl, } weight_scale = networkstatus_get_weight_scale_param(NULL); + tor_assert(weight_scale >= 1); if (rule == WEIGHT_FOR_GUARD) { Wg = networkstatus_get_bw_weight(NULL, "Wgg", -1); diff --git a/src/feature/relay/onion_queue.c b/src/feature/relay/onion_queue.c index 696905cf5e..c37745cf33 100644 --- a/src/feature/relay/onion_queue.c +++ b/src/feature/relay/onion_queue.c @@ -212,10 +212,12 @@ num_ntors_per_tap(void) #define MIN_NUM_NTORS_PER_TAP 1 #define MAX_NUM_NTORS_PER_TAP 100000 - return networkstatus_get_param(NULL, "NumNTorsPerTAP", - DEFAULT_NUM_NTORS_PER_TAP, - MIN_NUM_NTORS_PER_TAP, - MAX_NUM_NTORS_PER_TAP); + int result = networkstatus_get_param(NULL, "NumNTorsPerTAP", + DEFAULT_NUM_NTORS_PER_TAP, + MIN_NUM_NTORS_PER_TAP, + MAX_NUM_NTORS_PER_TAP); + tor_assert(result > 0); + return result; } /** Choose which onion queue we'll pull from next. If one is empty choose diff --git a/src/feature/rend/rendclient.c b/src/feature/rend/rendclient.c index 5a8b234544..f84d221b1a 100644 --- a/src/feature/rend/rendclient.c +++ b/src/feature/rend/rendclient.c @@ -469,16 +469,19 @@ directory_get_from_hs_dir(const char *desc_id, /* Automatically pick an hs dir if none given. */ if (!rs_hsdir) { + bool rate_limited = false; + /* Determine responsible dirs. Even if we can't get all we want, work with * the ones we have. If it's empty, we'll notice in hs_pick_hsdir(). */ smartlist_t *responsible_dirs = smartlist_new(); hid_serv_get_responsible_directories(responsible_dirs, desc_id); - hs_dir = hs_pick_hsdir(responsible_dirs, desc_id_base32); + hs_dir = hs_pick_hsdir(responsible_dirs, desc_id_base32, &rate_limited); if (!hs_dir) { /* No suitable hs dir can be found, stop right now. */ - control_event_hsv2_descriptor_failed(rend_query, NULL, - "QUERY_NO_HSDIR"); + const char *query_response = (rate_limited) ? "QUERY_RATE_LIMITED" : + "QUERY_NO_HSDIR"; + control_event_hsv2_descriptor_failed(rend_query, NULL, query_response); control_event_hs_descriptor_content(rend_data_get_address(rend_query), desc_id_base32, NULL, NULL); return 0; diff --git a/src/lib/crypt_ops/crypto_openssl_mgt.c b/src/lib/crypt_ops/crypto_openssl_mgt.c index 60e4ea795e..c97815f9a4 100644 --- a/src/lib/crypt_ops/crypto_openssl_mgt.c +++ b/src/lib/crypt_ops/crypto_openssl_mgt.c @@ -213,6 +213,14 @@ crypto_openssl_early_init(void) !strcmp(version_str, OPENSSL_VERSION_TEXT)) { log_info(LD_CRYPTO, "OpenSSL version matches version from headers " "(%lx: %s).", version_num, version_str); + } else if ((version_num & 0xffff0000) == + (OPENSSL_VERSION_NUMBER & 0xffff0000)) { + log_notice(LD_CRYPTO, + "We compiled with OpenSSL %lx: %s and we " + "are running with OpenSSL %lx: %s. " + "These two versions should be binary compatible.", + (unsigned long)OPENSSL_VERSION_NUMBER, OPENSSL_VERSION_TEXT, + version_num, version_str); } else { log_warn(LD_CRYPTO, "OpenSSL version from headers does not match the " "version we're running with. If you get weird crashes, that " diff --git a/src/lib/crypt_ops/crypto_rand.h b/src/lib/crypt_ops/crypto_rand.h index c51d6a4480..528f238fa5 100644 --- a/src/lib/crypt_ops/crypto_rand.h +++ b/src/lib/crypt_ops/crypto_rand.h @@ -92,6 +92,10 @@ void crypto_rand_fast_shutdown(void); #if defined(TOR_UNIT_TESTS) /* Used for white-box testing */ size_t crypto_fast_rng_get_bytes_used_per_stream(void); +/* For deterministic prng implementations */ +void crypto_fast_rng_disable_reseed(crypto_fast_rng_t *rng); +/* To override the prng for testing. */ +crypto_fast_rng_t *crypto_replace_thread_fast_rng(crypto_fast_rng_t *rng); #endif #ifdef CRYPTO_RAND_PRIVATE diff --git a/src/lib/crypt_ops/crypto_rand_fast.c b/src/lib/crypt_ops/crypto_rand_fast.c index 01817c618f..b71ade81bd 100644 --- a/src/lib/crypt_ops/crypto_rand_fast.c +++ b/src/lib/crypt_ops/crypto_rand_fast.c @@ -95,8 +95,13 @@ CTASSERT(KEY_BITS == 128 || KEY_BITS == 192 || KEY_BITS == 256); struct crypto_fast_rng_t { /** How many more fills does this buffer have before we should mix - * in the output of crypto_rand()? */ - uint16_t n_till_reseed; + * in the output of crypto_strongest_rand()? + * + * This value may be negative if unit tests are enabled. If so, it + * indicates that we should never mix in extra data from + * crypto_strongest_rand(). + */ + int16_t n_till_reseed; /** How many bytes are remaining in cbuf.bytes? */ uint16_t bytes_left; #ifdef CHECK_PID @@ -181,6 +186,18 @@ crypto_fast_rng_new_from_seed(const uint8_t *seed) return result; } +#ifdef TOR_UNIT_TESTS +/** + * Unit tests only: prevent a crypto_fast_rng_t from ever mixing in more + * entropy. + */ +void +crypto_fast_rng_disable_reseed(crypto_fast_rng_t *rng) +{ + rng->n_till_reseed = -1; +} +#endif + /** * Helper: create a crypto_cipher_t object from SEED_LEN bytes of * input. The first KEY_LEN bytes are used as the stream cipher's key, @@ -193,6 +210,26 @@ cipher_from_seed(const uint8_t *seed) } /** + * Helper: mix additional entropy into <b>rng</b> by using our XOF to mix the + * old value for the seed with some additional bytes from + * crypto_strongest_rand(). + **/ +static void +crypto_fast_rng_add_entopy(crypto_fast_rng_t *rng) +{ + crypto_xof_t *xof = crypto_xof_new(); + crypto_xof_add_bytes(xof, rng->buf.seed, SEED_LEN); + { + uint8_t seedbuf[SEED_LEN]; + crypto_strongest_rand(seedbuf, SEED_LEN); + crypto_xof_add_bytes(xof, seedbuf, SEED_LEN); + memwipe(seedbuf, 0, SEED_LEN); + } + crypto_xof_squeeze_bytes(xof, rng->buf.seed, SEED_LEN); + crypto_xof_free(xof); +} + +/** * Helper: refill the seed bytes and output buffer of <b>rng</b>, using * the input seed bytes as input (key and IV) for the stream cipher. * @@ -202,22 +239,19 @@ cipher_from_seed(const uint8_t *seed) static void crypto_fast_rng_refill(crypto_fast_rng_t *rng) { - if (rng->n_till_reseed-- == 0) { - /* It's time to reseed the RNG. We'll do this by using our XOF to mix the - * old value for the seed with some additional bytes from - * crypto_strongest_rand(). */ - crypto_xof_t *xof = crypto_xof_new(); - crypto_xof_add_bytes(xof, rng->buf.seed, SEED_LEN); - { - uint8_t seedbuf[SEED_LEN]; - crypto_strongest_rand(seedbuf, SEED_LEN); - crypto_xof_add_bytes(xof, seedbuf, SEED_LEN); - memwipe(seedbuf, 0, SEED_LEN); - } - crypto_xof_squeeze_bytes(xof, rng->buf.seed, SEED_LEN); - crypto_xof_free(xof); - + rng->n_till_reseed--; + if (rng->n_till_reseed == 0) { + /* It's time to reseed the RNG. */ + crypto_fast_rng_add_entopy(rng); rng->n_till_reseed = RESEED_AFTER; + } else if (rng->n_till_reseed < 0) { +#ifdef TOR_UNIT_TESTS + /* Reseeding is disabled for testing; never do it on this prng. */ + rng->n_till_reseed = -1; +#else + /* If testing is disabled, this shouldn't be able to become negative. */ + tor_assert_unreached(); +#endif } /* Now fill rng->buf with output from our stream cipher, initialized from * that seed value. */ @@ -363,6 +397,20 @@ destroy_thread_fast_rng(void) tor_threadlocal_set(&thread_rng, NULL); } +#ifdef TOR_UNIT_TESTS +/** + * Replace the current thread's rng with <b>rng</b>. For use by the + * unit tests only. Returns the previous thread rng. + **/ +crypto_fast_rng_t * +crypto_replace_thread_fast_rng(crypto_fast_rng_t *rng) +{ + crypto_fast_rng_t *old_rng = tor_threadlocal_get(&thread_rng); + tor_threadlocal_set(&thread_rng, rng); + return old_rng; +} +#endif + /** * Initialize the global thread-local key that will be used to keep track * of per-thread fast RNG instances. Called from the crypto subsystem's diff --git a/src/lib/fdio/fdio.c b/src/lib/fdio/fdio.c index 6c87af791d..078af6a9ba 100644 --- a/src/lib/fdio/fdio.c +++ b/src/lib/fdio/fdio.c @@ -17,12 +17,16 @@ #ifdef _WIN32 #include <windows.h> #endif +#ifdef HAVE_SYS_TYPES_H +#include <sys/types.h> +#endif #include "lib/fdio/fdio.h" #include "lib/cc/torint.h" #include "lib/err/torerr.h" #include <stdlib.h> +#include <stdio.h> /** @{ */ /** Some old versions of Unix didn't define constants for these values, diff --git a/src/lib/log/util_bug.c b/src/lib/log/util_bug.c index 65ab7bc9c6..76b97c1a08 100644 --- a/src/lib/log/util_bug.c +++ b/src/lib/log/util_bug.c @@ -19,6 +19,7 @@ #include "lib/string/printf.h" #include <string.h> +#include <stdlib.h> #ifdef TOR_UNIT_TESTS static void (*failed_assertion_cb)(void) = NULL; @@ -159,6 +160,19 @@ tor_bug_occurred_(const char *fname, unsigned int line, #endif } +/** + * Call the abort() function to kill the current process with a fatal + * error. + * + * (This is a separate function so that we declare it in util_bug.h without + * including stdlib in all the users of util_bug.h) + **/ +void +tor_abort_(void) +{ + abort(); +} + #ifdef _WIN32 /** Take a filename and return a pointer to its final element. This * function is called on __FILE__ to fix a MSVC nit where __FILE__ diff --git a/src/lib/log/util_bug.h b/src/lib/log/util_bug.h index 63c5309c98..2e220b7286 100644 --- a/src/lib/log/util_bug.h +++ b/src/lib/log/util_bug.h @@ -106,7 +106,7 @@ } else { \ tor_assertion_failed_(SHORT_FILE__, __LINE__, __func__, #expr, \ fmt, ##__VA_ARGS__); \ - abort(); \ + tor_abort_(); \ } STMT_END #endif /* defined(TOR_UNIT_TESTS) && defined(DISABLE_ASSERTS_IN_UNIT_TESTS) */ @@ -114,7 +114,7 @@ STMT_BEGIN { \ tor_assertion_failed_(SHORT_FILE__, __LINE__, __func__, \ "line should be unreached", NULL); \ - abort(); \ + tor_abort_(); \ } STMT_END /* Non-fatal bug assertions. The "unreached" variants mean "this line should @@ -149,7 +149,7 @@ #define BUG(cond) \ (ASSERT_PREDICT_UNLIKELY_(cond) ? \ (tor_assertion_failed_(SHORT_FILE__,__LINE__,__func__,"!("#cond")"), \ - abort(), 1) \ + tor_abort_(), 1) \ : 0) #elif defined(TOR_UNIT_TESTS) && defined(DISABLE_ASSERTS_IN_UNIT_TESTS) #define tor_assert_nonfatal_unreached() STMT_NIL @@ -246,6 +246,8 @@ void tor_bug_occurred_(const char *fname, unsigned int line, const char *func, const char *expr, int once, const char *fmt, ...); +void tor_abort_(void) ATTR_NORETURN; + #ifdef _WIN32 #define SHORT_FILE__ (tor_fix_source_file(__FILE__)) const char *tor_fix_source_file(const char *fname); diff --git a/src/lib/math/prob_distr.h b/src/lib/math/prob_distr.h index 2eb935e4a8..8fccf8d015 100644 --- a/src/lib/math/prob_distr.h +++ b/src/lib/math/prob_distr.h @@ -53,7 +53,7 @@ struct dist { * We define this conditionally to suppress false positives from * Coverity, which gets confused by the sizeof business. */ -#ifdef __COVERITY___ +#ifdef __COVERITY__ #define TYPE_CHECK_OBJ(OPS, OBJ, TYPE) 0 #else #define TYPE_CHECK_OBJ(OPS, OBJ, TYPE) \ diff --git a/src/lib/smartlist_core/smartlist_core.c b/src/lib/smartlist_core/smartlist_core.c index 5947e76271..6b0a305a93 100644 --- a/src/lib/smartlist_core/smartlist_core.c +++ b/src/lib/smartlist_core/smartlist_core.c @@ -177,6 +177,8 @@ smartlist_remove_keeporder(smartlist_t *sl, const void *element) sl->list[i++] = sl->list[j]; } } + memset(sl->list + sl->num_used, 0, + sizeof(void *) * (num_used_orig - sl->num_used)); } /** If <b>sl</b> is nonempty, remove and return the final element. Otherwise, diff --git a/src/rust/Cargo.toml b/src/rust/Cargo.toml index 83f9629660..de8693ea33 100644 --- a/src/rust/Cargo.toml +++ b/src/rust/Cargo.toml @@ -10,6 +10,17 @@ members = [ "tor_util", ] +# Can remove panic="abort" when this issue is fixed: +# https://github.com/rust-lang/rust/issues/52652 +[profile.dev] +panic = "abort" + [profile.release] debug = true panic = "abort" + +[profile.test] +panic = "abort" + +[profile.bench] +panic = "abort" diff --git a/src/test/fuzz/fuzz_multi.sh b/src/test/fuzz/fuzz_multi.sh index b4a17ed8cb..406ab498d9 100755 --- a/src/test/fuzz/fuzz_multi.sh +++ b/src/test/fuzz/fuzz_multi.sh @@ -1,3 +1,5 @@ +#!/bin/sh + MEMLIMIT_BYTES=21990500990976 N_CPUS=1 @@ -6,9 +8,9 @@ if [ $# -ge 1 ]; then shift fi -FILTER=echo +FILTER="echo" -for i in `seq -w "$N_CPUS"`; do +for i in $(seq -w "$N_CPUS"); do if [ "$i" -eq 1 ]; then if [ "$N_CPUS" -eq 1 ]; then INSTANCE="" diff --git a/src/test/include.am b/src/test/include.am index 497aa320a4..022cdbe035 100644 --- a/src/test/include.am +++ b/src/test/include.am @@ -89,6 +89,7 @@ src_test_test_SOURCES += \ src/test/log_test_helpers.c \ src/test/hs_test_helpers.c \ src/test/rend_test_helpers.c \ + src/test/rng_test_helpers.c \ src/test/test.c \ src/test/test_accounting.c \ src/test/test_addr.c \ @@ -211,6 +212,7 @@ endif src_test_test_slow_SOURCES = if UNITTESTS_ENABLED src_test_test_slow_SOURCES += \ + src/test/rng_test_helpers.c \ src/test/test_slow.c \ src/test/test_crypto_slow.c \ src/test/test_process_slow.c \ @@ -319,6 +321,7 @@ noinst_HEADERS+= \ src/test/hs_test_helpers.h \ src/test/log_test_helpers.h \ src/test/rend_test_helpers.h \ + src/test/rng_test_helpers.h \ src/test/test.h \ src/test/ptr_helpers.h \ src/test/test_helpers.h \ diff --git a/src/test/rng_test_helpers.c b/src/test/rng_test_helpers.c new file mode 100644 index 0000000000..262d380bda --- /dev/null +++ b/src/test/rng_test_helpers.c @@ -0,0 +1,226 @@ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file rng_test_helpers.c + * \brief Helpers for overriding PRNGs during unit tests. + * + * We define two PRNG overrides: a "reproducible PRNG" where the seed is + * chosen randomly but the stream can be replayed later on in case a bug is + * found, and a "deterministic PRNG" where the seed is fixed in the unit + * tests. + * + * Obviously, this code is testing-only. + */ + +#include "orconfig.h" +#include "core/or/or.h" + +#include "lib/crypt_ops/crypto_rand.h" + +#include "test/rng_test_helpers.h" + +#ifndef TOR_UNIT_TESTS +#error "No. Never link this code into Tor proper." +#endif + +/** + * True iff the RNG is currently replaced. Prevents double-replacement. + **/ +static bool rng_is_replaced = false; + +/** + * Mutex to protect deterministic prng. + * + * Note that if you actually _use_ the prng from two threads at the same time, + * the results will probably be nondeterministic anyway. + */ +static tor_mutex_t *rng_mutex = NULL; + +/** + * Cached old value for the thread prng. + **/ +static crypto_fast_rng_t *stored_fast_rng = NULL; + +/** replacement for crypto_strongest_rand that delegates to crypto_rand. */ +static void +mock_crypto_strongest_rand(uint8_t *out, size_t len) +{ + crypto_rand((char *)out, len); +} + +/* This is the seed of the deterministic randomness. */ +static uint8_t rng_seed[16]; +static crypto_xof_t *rng_xof = NULL; + +/** + * Print the seed for our PRNG to stdout. We use this when we're + **/ +void +testing_dump_reproducible_rng_seed(void) +{ + printf("\n" + "Seed: %s\n", + hex_str((const char*)rng_seed, sizeof(rng_seed))); +} + +/** Produce deterministic randomness for the stochastic tests using the global + * rng_xof output. + * + * This function produces deterministic data over multiple calls iff it's + * called in the same call order with the same 'n' parameter. + * If not, outputs will deviate. */ +static void +crypto_rand_deterministic(char *out, size_t n) +{ + tor_assert(rng_xof); + tor_mutex_acquire(rng_mutex); + crypto_xof_squeeze_bytes(rng_xof, (uint8_t*)out, n); + tor_mutex_release(rng_mutex); +} + +/** + * Implementation helper: override our crypto_rand() PRNG with a given seed of + * length <b>seed_len</b>. Overlong seeds are truncated; short ones are + * padded. + **/ +static void +enable_deterministic_rng_impl(const uint8_t *seed, size_t seed_len) +{ + tor_assert(!rng_is_replaced); + tor_assert(crypto_rand == crypto_rand__real); + + memset(rng_seed, 0, sizeof(rng_seed)); + memcpy(rng_seed, seed, MIN(seed_len, sizeof(rng_seed))); + + rng_mutex = tor_mutex_new(); + + crypto_xof_free(rng_xof); + rng_xof = crypto_xof_new(); + crypto_xof_add_bytes(rng_xof, rng_seed, sizeof(rng_seed)); + MOCK(crypto_rand, crypto_rand_deterministic); + MOCK(crypto_strongest_rand_, mock_crypto_strongest_rand); + + uint8_t fast_rng_seed[CRYPTO_FAST_RNG_SEED_LEN]; + memset(fast_rng_seed, 0xff, sizeof(fast_rng_seed)); + memcpy(fast_rng_seed, rng_seed, MIN(sizeof(rng_seed), + sizeof(fast_rng_seed))); + crypto_fast_rng_t *fast_rng = crypto_fast_rng_new_from_seed(fast_rng_seed); + crypto_fast_rng_disable_reseed(fast_rng); + stored_fast_rng = crypto_replace_thread_fast_rng(fast_rng); + + rng_is_replaced = true; +} + +/** + * Replace our get_thread_fast_rng(), crypto_rand() and + * crypto_strongest_rand() prngs with a variant that generates all of its + * output deterministically from a randomly chosen seed. In the event of an + * error, you can log the seed later on with + * testing_dump_reproducible_rng_seed. + **/ +void +testing_enable_reproducible_rng(void) +{ + uint8_t seed[16]; + crypto_rand((char*)seed, sizeof(seed)); + enable_deterministic_rng_impl(seed, sizeof(seed)); +} + +/** + * Replace our get_thread_fast_rng(), crypto_rand() and + * crypto_strongest_rand() prngs with a variant that generates all of its + * output deterministically from a fixed seed. This variant is mainly useful + * for cases when we don't want coverage to change between runs. + * + * USAGE NOTE: Test correctness SHOULD NOT depend on the specific output of + * this "rng". If you need a specific output, use + * testing_enable_prefilled_rng() instead. + **/ +void +testing_enable_deterministic_rng(void) +{ + static const uint8_t quotation[] = + "What will it be? A tree? A weed? " + "Each one is started from a seed."; // -- Mary Ann Hoberman + enable_deterministic_rng_impl(quotation, sizeof(quotation)); +} + +static uint8_t *prefilled_rng_buffer = NULL; +static size_t prefilled_rng_buflen; +static size_t prefilled_rng_idx; + +/** + * crypto_rand() replacement that returns canned data. + **/ +static void +crypto_rand_prefilled(char *out, size_t n) +{ + tor_mutex_acquire(rng_mutex); + while (n) { + size_t n_to_copy = MIN(prefilled_rng_buflen - prefilled_rng_idx, n); + memcpy(out, prefilled_rng_buffer + prefilled_rng_idx, n_to_copy); + out += n_to_copy; + n -= n_to_copy; + prefilled_rng_idx += n_to_copy; + + if (prefilled_rng_idx == prefilled_rng_buflen) { + prefilled_rng_idx = 0; + } + } + tor_mutex_release(rng_mutex); +} + +/** + * Replace our crypto_rand() and crypto_strongest_rand() prngs with a variant + * that yields output from a buffer. If it reaches the end of the buffer, it + * starts over. + * + * Note: the get_thread_fast_rng() prng is not replaced by this; we'll need + * more code to support that. + **/ +void +testing_enable_prefilled_rng(const void *buffer, size_t buflen) +{ + tor_assert(buflen > 0); + rng_mutex = tor_mutex_new(); + + prefilled_rng_buffer = tor_memdup(buffer, buflen); + prefilled_rng_buflen = buflen; + prefilled_rng_idx = 0; + + MOCK(crypto_rand, crypto_rand_prefilled); + MOCK(crypto_strongest_rand_, mock_crypto_strongest_rand); +} + +/** + * Reset the position in the prefilled RNG buffer to the start. + */ +void +testing_prefilled_rng_reset(void) +{ + tor_mutex_acquire(rng_mutex); + prefilled_rng_idx = 0; + tor_mutex_release(rng_mutex); +} + +/** + * Undo the overrides for our PRNG. To be used at the end of testing. + * + * Note that this function should be safe to call even if the rng has not + * yet been replaced. + **/ +void +testing_disable_rng_override(void) +{ + crypto_xof_free(rng_xof); + tor_free(prefilled_rng_buffer); + UNMOCK(crypto_rand); + UNMOCK(crypto_strongest_rand_); + tor_mutex_free(rng_mutex); + + crypto_fast_rng_t *rng = crypto_replace_thread_fast_rng(stored_fast_rng); + crypto_fast_rng_free(rng); + + rng_is_replaced = false; +} diff --git a/src/test/rng_test_helpers.h b/src/test/rng_test_helpers.h new file mode 100644 index 0000000000..907099450d --- /dev/null +++ b/src/test/rng_test_helpers.h @@ -0,0 +1,26 @@ +/* Copyright (c) 2017-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_RNG_TEST_HELPERS_H +#define TOR_RNG_TEST_HELPERS_H + +#include "core/or/or.h" + +void testing_enable_deterministic_rng(void); +void testing_enable_reproducible_rng(void); +void testing_enable_prefilled_rng(const void *buffer, size_t buflen); + +void testing_prefilled_rng_reset(void); + +void testing_disable_rng_override(void); + +#define testing_disable_reproducible_rng() \ + testing_disable_rng_override() +#define testing_disable_deterministic_rng() \ + testing_disable_rng_override() +#define testing_disable_prefilled_rng() \ + testing_disable_rng_override() + +void testing_dump_reproducible_rng_seed(void); + +#endif /* !defined(TOR_RNG_TEST_HELPERS_H) */ diff --git a/src/test/test-network.sh b/src/test/test-network.sh index 4d56e83806..5ef995f1a4 100755 --- a/src/test/test-network.sh +++ b/src/test/test-network.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/bin/sh # This script calls the equivalent script in chutney/tools @@ -18,32 +18,14 @@ ECHO="${ECHO:-echo}" # Output is prefixed with the name of the script myname=$(basename "$0") -# Save the arguments before we destroy them -# This might not preserve arguments with spaces in them -ORIGINAL_ARGS=( "$@" ) - # We need to find CHUTNEY_PATH, so that we can call the version of this script # in chutney/tools with the same arguments. We also need to respect --quiet. -until [ -z "$1" ] -do - case "$1" in - --chutney-path) - CHUTNEY_PATH="$2" - shift - ;; - --tor-path) - TOR_DIR="$2" - shift - ;; - --quiet) - ECHO=true - ;; - *) - # maybe chutney's test-network.sh can handle it - ;; - esac - shift -done +CHUTNEY_PATH=$(echo "$@" | awk -F '--chutney-path ' '{sub(" .*","",$2); print $2}') +TOR_DIR=$(echo "$@" | awk -F '--tor-dir ' '{sub(" .*","",$2); print $2}') + +if echo "$@" | grep -e "--quiet" > /dev/null; then + ECHO=true +fi # optional: $TOR_DIR is the tor build directory # it's used to find the location of tor binaries @@ -99,7 +81,7 @@ if [ -d "$CHUTNEY_PATH" ] && [ -x "$TEST_NETWORK" ]; then # this may fail if some arguments have spaces in them # if so, set CHUTNEY_PATH before calling test-network.sh, and spaces # will be handled correctly - exec "$TEST_NETWORK" "${ORIGINAL_ARGS[@]}" # $ORIGINAL_ARGS + exec "$TEST_NETWORK" "$@" else $ECHO "$myname: Could not find tools/test-network.sh in CHUTNEY_PATH." $ECHO "$myname: Please update your chutney using 'git pull'." diff --git a/src/test/test.c b/src/test/test.c index fbc30fb64e..be5cb12b1e 100644 --- a/src/test/test.c +++ b/src/test/test.c @@ -12,6 +12,7 @@ #include "lib/crypt_ops/crypto_dh.h" #include "lib/crypt_ops/crypto_rand.h" #include "app/config/or_state_st.h" +#include "test/rng_test_helpers.h" #include <stdio.h> #ifdef HAVE_FCNTL_H @@ -354,18 +355,6 @@ test_onion_queues(void *arg) tor_free(onionskin); } -static crypto_cipher_t *crypto_rand_aes_cipher = NULL; - -// Mock replacement for crypto_rand: Generates bytes from a provided AES_CTR -// cipher in <b>crypto_rand_aes_cipher</b>. -static void -crypto_rand_deterministic_aes(char *out, size_t n) -{ - tor_assert(crypto_rand_aes_cipher); - memset(out, 0, n); - crypto_cipher_crypt_inplace(crypto_rand_aes_cipher, out, n); -} - static void test_circuit_timeout(void *arg) { @@ -397,8 +386,7 @@ test_circuit_timeout(void *arg) // Use a deterministic RNG here, or else we'll get nondeterministic // coverage in some of the circuitstats functions. - MOCK(crypto_rand, crypto_rand_deterministic_aes); - crypto_rand_aes_cipher = crypto_cipher_new("xyzzyplughplover"); + testing_enable_deterministic_rng(); circuitbuild_running_unit_tests(); #define timeout0 (build_time_t)(30*1000.0) @@ -534,8 +522,8 @@ test_circuit_timeout(void *arg) circuit_build_times_free_timeouts(&final); or_state_free(state); teardown_periodic_events(); - UNMOCK(crypto_rand); - crypto_cipher_free(crypto_rand_aes_cipher); + + testing_disable_deterministic_rng(); } /** Test encoding and parsing of rendezvous service descriptors. */ diff --git a/src/test/test_addr.c b/src/test/test_addr.c index fb8df5f0fb..3a1a7b6997 100644 --- a/src/test/test_addr.c +++ b/src/test/test_addr.c @@ -11,6 +11,7 @@ #include "feature/client/addressmap.h" #include "test/log_test_helpers.h" #include "lib/net/resolve.h" +#include "test/rng_test_helpers.h" #ifdef HAVE_SYS_UN_H #include <sys/un.h> @@ -945,27 +946,6 @@ test_virtaddrmap(void *data) ; } -static const char *canned_data = NULL; -static size_t canned_data_len = 0; - -/* Mock replacement for crypto_rand() that returns canned data from - * canned_data above. */ -static void -crypto_canned(char *ptr, size_t n) -{ - if (canned_data_len) { - size_t to_copy = MIN(n, canned_data_len); - memcpy(ptr, canned_data, to_copy); - canned_data += to_copy; - canned_data_len -= to_copy; - n -= to_copy; - ptr += to_copy; - } - if (n) { - crypto_rand_unmocked(ptr, n); - } -} - static void test_virtaddrmap_persist(void *data) { @@ -973,6 +953,8 @@ test_virtaddrmap_persist(void *data) const char *a, *b, *c; tor_addr_t addr; char *ones = NULL; + const char *canned_data; + size_t canned_data_len; addressmap_init(); @@ -991,7 +973,7 @@ test_virtaddrmap_persist(void *data) "1234567890" // the second call returns this. "abcdefghij"; // the third call returns this. canned_data_len = 30; - MOCK(crypto_rand, crypto_canned); + testing_enable_prefilled_rng(canned_data, canned_data_len); a = addressmap_register_virtual_address(RESOLVED_TYPE_HOSTNAME, tor_strdup("quuxit.baz")); @@ -1001,9 +983,9 @@ test_virtaddrmap_persist(void *data) tt_assert(b); tt_str_op(a, OP_EQ, "gezdgnbvgy3tqojq.virtual"); tt_str_op(b, OP_EQ, "mfrggzdfmztwq2lk.virtual"); + testing_disable_prefilled_rng(); // Now try something to get us an ipv4 address - UNMOCK(crypto_rand); tt_int_op(0,OP_EQ, parse_virtual_addr_network("192.168.0.0/16", AF_INET, 0, NULL)); a = addressmap_register_virtual_address(RESOLVED_TYPE_IPV4, @@ -1020,22 +1002,23 @@ test_virtaddrmap_persist(void *data) // Try some canned entropy and verify all the we discard duplicates, // addresses that end with 0, and addresses that end with 255. - MOCK(crypto_rand, crypto_canned); canned_data = "\x01\x02\x03\x04" // okay "\x01\x02\x03\x04" // duplicate "\x03\x04\x00\x00" // bad ending 1 "\x05\x05\x00\xff" // bad ending 2 "\x05\x06\x07\xf0"; // okay canned_data_len = 20; + testing_enable_prefilled_rng(canned_data, canned_data_len); + a = addressmap_register_virtual_address(RESOLVED_TYPE_IPV4, tor_strdup("wumble.onion")); b = addressmap_register_virtual_address(RESOLVED_TYPE_IPV4, tor_strdup("wumpus.onion")); tt_str_op(a, OP_EQ, "192.168.3.4"); tt_str_op(b, OP_EQ, "192.168.7.240"); + testing_disable_prefilled_rng(); // Now try IPv6! - UNMOCK(crypto_rand); tt_int_op(0,OP_EQ, parse_virtual_addr_network("1010:F000::/20", AF_INET6, 0, NULL)); a = addressmap_register_virtual_address(RESOLVED_TYPE_IPV6, @@ -1051,7 +1034,7 @@ test_virtaddrmap_persist(void *data) tt_assert(!strcmpstart(b, "[1010:f")); // Try IPv6 with canned entropy, to make sure we detect duplicates. - MOCK(crypto_rand, crypto_canned); + canned_data = "acanthopterygian" // okay "cinematographist" // okay "acanthopterygian" // duplicate @@ -1060,6 +1043,8 @@ test_virtaddrmap_persist(void *data) "cinematographist" // duplicate "coadministration"; // okay canned_data_len = 16 * 7; + testing_enable_prefilled_rng(canned_data, canned_data_len); + a = addressmap_register_virtual_address(RESOLVED_TYPE_IPV6, tor_strdup("wuffle.baz")); b = addressmap_register_virtual_address(RESOLVED_TYPE_IPV6, @@ -1072,9 +1057,11 @@ test_virtaddrmap_persist(void *data) // Try address exhaustion: make sure we can actually fail if we // get too many already-existing addresses. + testing_disable_prefilled_rng(); canned_data_len = 128*1024; canned_data = ones = tor_malloc(canned_data_len); memset(ones, 1, canned_data_len); + testing_enable_prefilled_rng(canned_data, canned_data_len); // There is some chance this one will fail if a previous random // allocation gave out the address already. a = addressmap_register_virtual_address(RESOLVED_TYPE_IPV4, @@ -1091,7 +1078,7 @@ test_virtaddrmap_persist(void *data) expect_single_log_msg_containing("Ran out of virtual addresses!"); done: - UNMOCK(crypto_rand); + testing_disable_prefilled_rng(); tor_free(ones); addressmap_free_all(); teardown_capture_of_logs(); diff --git a/src/test/test_containers.c b/src/test/test_containers.c index 7892a08853..67ba457975 100644 --- a/src/test/test_containers.c +++ b/src/test/test_containers.c @@ -1006,6 +1006,10 @@ test_container_smartlist_remove(void *arg) tt_ptr_op(smartlist_get(sl, 1), OP_EQ, &array[2]); tt_ptr_op(smartlist_get(sl, 2), OP_EQ, &array[1]); tt_ptr_op(smartlist_get(sl, 3), OP_EQ, &array[2]); + /* Ordinary code should never look at this pointer; we're doing it here + * to make sure that we really cleared the pointer we removed. + */ + tt_ptr_op(sl->list[4], OP_EQ, NULL); done: smartlist_free(sl); diff --git a/src/test/test_dir_handle_get.c b/src/test/test_dir_handle_get.c index c3a17e7309..e57bd02584 100644 --- a/src/test/test_dir_handle_get.c +++ b/src/test/test_dir_handle_get.c @@ -2526,7 +2526,7 @@ test_dir_handle_get_status_vote_next_bandwidth(void* data) /* Check cache lifetime */ char expbuf[RFC1123_TIME_LEN+1]; - time_t now = time(NULL); + time_t now = approx_time(); /* BANDWIDTH_CACHE_LIFETIME is defined in dircache.c. */ format_rfc1123_time(expbuf, (time_t)(now + 30*60)); char *expires = NULL; diff --git a/src/test/test_extorport.c b/src/test/test_extorport.c index f5d16af921..cfdd11d161 100644 --- a/src/test/test_extorport.c +++ b/src/test/test_extorport.c @@ -18,6 +18,7 @@ #include "test/test.h" #include "test/test_helpers.h" +#include "test/rng_test_helpers.h" #ifdef HAVE_SYS_STAT_H #include <sys/stat.h> @@ -303,16 +304,6 @@ test_ext_or_cookie_auth(void *arg) } static void -crypto_rand_return_tse_str(char *to, size_t n) -{ - if (n != 32) { - TT_FAIL(("Asked for %d bytes, not 32", (int)n)); - return; - } - memcpy(to, "te road There is always another ", 32); -} - -static void test_ext_or_cookie_auth_testvec(void *arg) { char *reply=NULL, *client_hash=NULL; @@ -326,7 +317,7 @@ test_ext_or_cookie_auth_testvec(void *arg) memcpy(ext_or_auth_cookie, "Gliding wrapt in a brown mantle," , 32); ext_or_auth_cookie_is_set = 1; - MOCK(crypto_rand, crypto_rand_return_tse_str); + testing_enable_prefilled_rng("te road There is always another ", 32); tt_int_op(0, OP_EQ, handle_client_auth_nonce(client_nonce, 32, &client_hash, &reply, @@ -351,7 +342,7 @@ test_ext_or_cookie_auth_testvec(void *arg) "33b3cd77ff79bd80c2074bbf438119a2"); done: - UNMOCK(crypto_rand); + testing_disable_prefilled_rng(); tor_free(reply); tor_free(client_hash); tor_free(mem_op_hex_tmp); @@ -414,9 +405,9 @@ do_ext_or_handshake(or_connection_t *conn) CONTAINS("\x01\x00", 2); WRITE("\x01", 1); WRITE("But when I look ahead up the whi", 32); - MOCK(crypto_rand, crypto_rand_return_tse_str); + testing_enable_prefilled_rng("te road There is always another ", 32); tt_int_op(0, OP_EQ, connection_ext_or_process_inbuf(conn)); - UNMOCK(crypto_rand); + testing_disable_prefilled_rng(); tt_int_op(TO_CONN(conn)->state, OP_EQ, EXT_OR_CONN_STATE_AUTH_WAIT_CLIENT_HASH); CONTAINS("\xec\x80\xed\x6e\x54\x6d\x3b\x36\xfd\xfc\x22\xfe\x13\x15\x41\x6b" @@ -481,9 +472,9 @@ test_ext_or_handshake(void *arg) tt_int_op(0, OP_EQ, connection_ext_or_process_inbuf(conn)); /* send the rest of the nonce. */ WRITE("ahead up the whi", 16); - MOCK(crypto_rand, crypto_rand_return_tse_str); + testing_enable_prefilled_rng("te road There is always another ", 32); tt_int_op(0, OP_EQ, connection_ext_or_process_inbuf(conn)); - UNMOCK(crypto_rand); + testing_disable_prefilled_rng(); /* We should get the right reply from the server. */ CONTAINS("\xec\x80\xed\x6e\x54\x6d\x3b\x36\xfd\xfc\x22\xfe\x13\x15\x41\x6b" "\x02\x9f\x1a\xde\x76\x10\xd9\x10\x87\x8b\x62\xee\xb7\x40\x38\x21" @@ -582,7 +573,7 @@ test_ext_or_handshake(void *arg) done: UNMOCK(connection_write_to_buf_impl_); - UNMOCK(crypto_rand); + testing_disable_prefilled_rng(); if (conn) connection_free_minimal(TO_CONN(conn)); #undef CONTAINS diff --git a/src/test/test_hs.c b/src/test/test_hs.c index aeb3387471..5d3327c777 100644 --- a/src/test/test_hs.c +++ b/src/test/test_hs.c @@ -323,6 +323,16 @@ test_hs_desc_event(void *arg) tt_str_op(received_msg,OP_EQ, expected_msg); tor_free(received_msg); + /* test HSDir rate limited */ + rend_query.auth_type = REND_NO_AUTH; + control_event_hsv2_descriptor_failed(&rend_query.base_, NULL, + "QUERY_RATE_LIMITED"); + expected_msg = "650 HS_DESC FAILED "STR_HS_ADDR" NO_AUTH " \ + "UNKNOWN REASON=QUERY_RATE_LIMITED\r\n"; + tt_assert(received_msg); + tt_str_op(received_msg,OP_EQ, expected_msg); + tor_free(received_msg); + /* Test invalid content with no HSDir fingerprint. */ char *exp_msg; control_event_hs_descriptor_content(rend_query.onion_address, diff --git a/src/test/test_hs_cache.c b/src/test/test_hs_cache.c index 48e8d3b8c4..2187c2be39 100644 --- a/src/test/test_hs_cache.c +++ b/src/test/test_hs_cache.c @@ -244,6 +244,7 @@ helper_fetch_desc_from_hsdir(const ed25519_public_key_t *blinded_key) /* Simulate an HTTP GET request to the HSDir */ conn = dir_connection_new(AF_INET); + tt_assert(conn); tor_addr_from_ipv4h(&conn->base_.addr, 0x7f000001); TO_CONN(conn)->linked = 1;/* Pretend the conn is encrypted :) */ retval = directory_handle_command_get(conn, hsdir_query_str, diff --git a/src/test/test_hs_descriptor.c b/src/test/test_hs_descriptor.c index 86965d7d66..5a3fd46dbe 100644 --- a/src/test/test_hs_descriptor.c +++ b/src/test/test_hs_descriptor.c @@ -21,6 +21,7 @@ #include "test/hs_test_helpers.h" #include "test/test_helpers.h" #include "test/log_test_helpers.h" +#include "test/rng_test_helpers.h" #ifdef HAVE_CFLAG_WOVERLENGTH_STRINGS DISABLE_GCC_WARNING(overlength-strings) @@ -30,13 +31,6 @@ DISABLE_GCC_WARNING(overlength-strings) #include "test_hs_descriptor.inc" ENABLE_GCC_WARNING(overlength-strings) -/* Mock function to fill all bytes with 1 */ -static void -mock_crypto_strongest_rand(uint8_t *out, size_t out_len) -{ - memset(out, 1, out_len); -} - /* Test certificate encoding put in a descriptor. */ static void test_cert_encoding(void *arg) @@ -799,7 +793,7 @@ test_build_authorized_client(void *arg) client_pubkey_b16, strlen(client_pubkey_b16)); - MOCK(crypto_strongest_rand_, mock_crypto_strongest_rand); + testing_enable_prefilled_rng("\x01", 1); hs_desc_build_authorized_client(subcredential, &client_auth_pk, &auth_ephemeral_sk, @@ -815,7 +809,7 @@ test_build_authorized_client(void *arg) done: tor_free(desc_client); tor_free(mem_op_hex_tmp); - UNMOCK(crypto_strongest_rand_); + testing_disable_prefilled_rng(); } struct testcase_t hs_descriptor[] = { diff --git a/src/test/test_prob_distr.c b/src/test/test_prob_distr.c index 37cfdae7d9..747c3d98e6 100644 --- a/src/test/test_prob_distr.c +++ b/src/test/test_prob_distr.c @@ -33,6 +33,7 @@ #include "lib/math/prob_distr.h" #include "lib/math/fp.h" #include "lib/crypt_ops/crypto_rand.h" +#include "test/rng_test_helpers.h" #include <float.h> #include <math.h> @@ -1117,49 +1118,14 @@ test_psi_dist_sample(const struct dist *dist) } } -/* This is the seed of the deterministic randomness */ -static uint8_t rng_seed[16]; -static crypto_xof_t *rng_xof = NULL; - -/** Initialize the seed of the deterministic randomness. */ -static void -init_deterministic_rand(void) -{ - crypto_rand((char*)rng_seed, sizeof(rng_seed)); - crypto_xof_free(rng_xof); - rng_xof = crypto_xof_new(); - crypto_xof_add_bytes(rng_xof, rng_seed, sizeof(rng_seed)); -} - -static void -teardown_deterministic_rand(void) -{ - crypto_xof_free(rng_xof); -} - static void dump_seed(void) { printf("\n" "NOTE: This is a stochastic test, and we expect it to fail from\n" "time to time, with some low probability. If you see it fail more\n" - "than one trial in 100, though, please tell us.\n\n" - "Seed: %s\n", - hex_str((const char*)rng_seed, sizeof(rng_seed))); -} - -/** Produce deterministic randomness for the stochastic tests using the global - * deterministic_rand_counter seed - * - * This function produces deterministic data over multiple calls iff it's - * called in the same call order with the same 'n' parameter (which is the - * case for the psi test). If not, outputs will deviate. */ -static void -crypto_rand_deterministic(char *out, size_t n) -{ - /* Use a XOF to squeeze bytes out of that silly counter */ - tor_assert(rng_xof); - crypto_xof_squeeze_bytes(rng_xof, (uint8_t*)out, n); + "than one trial in 100, though, please tell us.\n\n"); + testing_dump_reproducible_rng_seed(); } static void @@ -1199,8 +1165,7 @@ test_stochastic_uniform(void *arg) }; bool ok = true, tests_failed = true; - init_deterministic_rand(); - MOCK(crypto_rand, crypto_rand_deterministic); + testing_enable_reproducible_rng(); ok &= test_psi_dist_sample(&uniform01.base); ok &= test_psi_dist_sample(&uniform_pos.base); @@ -1217,8 +1182,7 @@ test_stochastic_uniform(void *arg) if (tests_failed) { dump_seed(); } - teardown_deterministic_rand(); - UNMOCK(crypto_rand); + testing_disable_reproducible_rng(); } static bool @@ -1288,8 +1252,7 @@ test_stochastic_genpareto(void *arg) bool tests_failed = true; (void) arg; - init_deterministic_rand(); - MOCK(crypto_rand, crypto_rand_deterministic); + testing_enable_reproducible_rng(); ok = test_stochastic_genpareto_impl(0, 1, -0.25); tt_assert(ok); @@ -1312,8 +1275,7 @@ test_stochastic_genpareto(void *arg) if (tests_failed) { dump_seed(); } - teardown_deterministic_rand(); - UNMOCK(crypto_rand); + testing_disable_reproducible_rng(); } static void @@ -1324,8 +1286,7 @@ test_stochastic_geometric(void *arg) (void) arg; - init_deterministic_rand(); - MOCK(crypto_rand, crypto_rand_deterministic); + testing_enable_reproducible_rng(); ok = test_stochastic_geometric_impl(0.1); tt_assert(ok); @@ -1342,8 +1303,7 @@ test_stochastic_geometric(void *arg) if (tests_failed) { dump_seed(); } - teardown_deterministic_rand(); - UNMOCK(crypto_rand); + testing_disable_reproducible_rng(); } static void @@ -1353,8 +1313,7 @@ test_stochastic_logistic(void *arg) bool tests_failed = true; (void) arg; - init_deterministic_rand(); - MOCK(crypto_rand, crypto_rand_deterministic); + testing_enable_reproducible_rng(); ok = test_stochastic_logistic_impl(0, 1); tt_assert(ok); @@ -1371,8 +1330,7 @@ test_stochastic_logistic(void *arg) if (tests_failed) { dump_seed(); } - teardown_deterministic_rand(); - UNMOCK(crypto_rand); + testing_disable_reproducible_rng(); } static void @@ -1382,8 +1340,7 @@ test_stochastic_log_logistic(void *arg) bool tests_failed = true; (void) arg; - init_deterministic_rand(); - MOCK(crypto_rand, crypto_rand_deterministic); + testing_enable_reproducible_rng(); ok = test_stochastic_log_logistic_impl(1, 1); tt_assert(ok); @@ -1400,8 +1357,7 @@ test_stochastic_log_logistic(void *arg) if (tests_failed) { dump_seed(); } - teardown_deterministic_rand(); - UNMOCK(crypto_rand); + testing_disable_reproducible_rng(); } static void @@ -1411,8 +1367,7 @@ test_stochastic_weibull(void *arg) bool tests_failed = true; (void) arg; - init_deterministic_rand(); - MOCK(crypto_rand, crypto_rand_deterministic); + testing_enable_reproducible_rng(); ok = test_stochastic_weibull_impl(1, 0.5); tt_assert(ok); @@ -1431,7 +1386,7 @@ test_stochastic_weibull(void *arg) if (tests_failed) { dump_seed(); } - teardown_deterministic_rand(); + testing_disable_reproducible_rng(); UNMOCK(crypto_rand); } diff --git a/src/tools/tor-resolve.c b/src/tools/tor-resolve.c index 98b3a4a74c..5d97696c18 100644 --- a/src/tools/tor-resolve.c +++ b/src/tools/tor-resolve.c @@ -424,6 +424,7 @@ do_resolve(const char *hostname, if (parsed < 2) { log_err(LD_NET, "Failed to parse SOCKS5 method selection " "message"); + socks5_server_method_free(m); goto err; } |