diff options
360 files changed, 5209 insertions, 760 deletions
diff --git a/.gitignore b/.gitignore index d33b79736d..3163c12765 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,8 @@ .dirstamp *.trs *.log +# Calltool stuff +.*.graph # Stuff made by our makefiles *.bak # Python droppings @@ -13,7 +13,7 @@ Tor is distributed under this license: Copyright (c) 2001-2004, Roger Dingledine Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson -Copyright (c) 2007-2016, The Tor Project, Inc. +Copyright (c) 2007-2017, The Tor Project, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/Makefile.am b/Makefile.am index 3d50594bf4..aeac0b85c4 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1,6 +1,6 @@ # Copyright (c) 2001-2004, Roger Dingledine # Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson -# Copyright (c) 2007-2015, The Tor Project, Inc. +# Copyright (c) 2007-2017, The Tor Project, Inc. # See LICENSE for licensing information ACLOCAL_AMFLAGS = -I m4 diff --git a/acinclude.m4 b/acinclude.m4 index 193d3a7a08..49d4f14471 100644 --- a/acinclude.m4 +++ b/acinclude.m4 @@ -2,7 +2,7 @@ dnl Helper macros for Tor configure.ac dnl Copyright (c) 2001-2004, Roger Dingledine dnl Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson dnl Copyright (c) 2007-2008, Roger Dingledine, Nick Mathewson -dnl Copyright (c) 2007-2015, The Tor Project, Inc. +dnl Copyright (c) 2007-2017, The Tor Project, Inc. dnl See LICENSE for licensing information AC_DEFUN([TOR_EXTEND_CODEPATH], diff --git a/changes/bug17170 b/changes/bug17170 new file mode 100644 index 0000000000..24a9b94fcf --- /dev/null +++ b/changes/bug17170 @@ -0,0 +1,3 @@ + o Documentation: + - Note that bandwidth-limiting options don't affect TCP headers or DNS. + Closes ticket 17170. diff --git a/changes/bug20059 b/changes/bug20059 new file mode 100644 index 0000000000..091fab06d1 --- /dev/null +++ b/changes/bug20059 @@ -0,0 +1,3 @@ + o Minor bugfixes (relay): + - Avoid a double-marked-circuit warning that can happen when we receive + DESTROY cells under heavy load. Fixes bug 20059; bugfix on 0.1.0.1-rc. diff --git a/changes/bug21496 b/changes/bug21496 new file mode 100644 index 0000000000..24ac85a769 --- /dev/null +++ b/changes/bug21496 @@ -0,0 +1,4 @@ + o Minor features (safety): + - Add an explict check to extrainfo_parse_entry_from_string() for NULL + inputs. We don't believe this can actually happen, but it may help + silence a warning from the Clang analyzer. Closes ticket 21496. diff --git a/changes/bug21540 b/changes/bug21540 new file mode 100644 index 0000000000..0cf684b7f2 --- /dev/null +++ b/changes/bug21540 @@ -0,0 +1,4 @@ + o Minor bugfixes (windows, relay): + - Resolve "Failure from drain_fd: No error" warnings on Windows + relays. Fixes bug 21540; bugfix on 0.2.6.3-alpha. + diff --git a/changes/bug21641 b/changes/bug21641 new file mode 100644 index 0000000000..96fdf5fe22 --- /dev/null +++ b/changes/bug21641 @@ -0,0 +1,5 @@ + o Minor feature (defaults, directory): + - Onion key rotation and expiry intervals are now defined as a network + consensus parameter as per proposal #274. The default lifetime of an + onion key is bumped from 7 to 28 days. Old onion keys will expire after 7 + days by default. Fixes bug #21641. diff --git a/changes/bug21682 b/changes/bug21682 new file mode 100644 index 0000000000..ab7126e4d6 --- /dev/null +++ b/changes/bug21682 @@ -0,0 +1,3 @@ + o Minor bugfixes (memory leaks): + - Fix a memory leak when using GETCONF on a port option. + Fixes bug 21682; bugfix on 0.3.0.3-alpha. diff --git a/changes/bug21703 b/changes/bug21703 new file mode 100644 index 0000000000..3034fc5e4b --- /dev/null +++ b/changes/bug21703 @@ -0,0 +1,4 @@ + o Minor features (controller): + - Warn the first time that a controller requests data in the + long-deprecated 'GETINFO network-status' format. Closes ticket 21703. + diff --git a/changes/bug21720 b/changes/bug21720 new file mode 100644 index 0000000000..6d2fbcf711 --- /dev/null +++ b/changes/bug21720 @@ -0,0 +1,5 @@ + o Documentation: + - Update the description of the directory server options in the manual + page, to clarify that DirPort is no longer necessary to be a directory + cache. Closes ticket 21720. + diff --git a/changes/bug21771 b/changes/bug21771 new file mode 100644 index 0000000000..f814c75c0b --- /dev/null +++ b/changes/bug21771 @@ -0,0 +1,5 @@ + o Minor bugfixes (guard selection): + - Fix a guard selection bug where Tor would refuse to bootstrap in some + cases if the user swapped a bridge for another bridge in their + configuration file. + Fixes bug 21771; bugfix on 0.3.0.1-alpha. Reported by "torvlnt33r". diff --git a/changes/bug21788 b/changes/bug21788 new file mode 100644 index 0000000000..48deec2bb7 --- /dev/null +++ b/changes/bug21788 @@ -0,0 +1,3 @@ + o Minor bugfixes: + - Fix a small memory leak at exit from the backtrace handler code. + Fixes bug 21788; bugfix on 0.2.5.2-alpha. Patch from Daniel Pinto. diff --git a/changes/bug21799 b/changes/bug21799 new file mode 100644 index 0000000000..ee2e904a35 --- /dev/null +++ b/changes/bug21799 @@ -0,0 +1,6 @@ + o Minor bugfixes (tests): + - Run the entry_guard_parse_from_state_full test with the time set + to a specific date. (The guard state that this test was parsing + contained guards that had expired since the test was first + written.) Fixes bug 21799; bugfix on 0.3.0.1-alpha. + diff --git a/changes/consdiff_21643 b/changes/consdiff_21643 new file mode 100644 index 0000000000..38d465673b --- /dev/null +++ b/changes/consdiff_21643 @@ -0,0 +1,5 @@ + o Major features (internals): + - Add an ed diff/patch backend, optimized for consensus documents. + This backend will be the basis of our consensus diff implementation. + Most of the work here was done + by Daniel MartÃ. Closes ticket 21643. diff --git a/changes/data_dir_default_doc b/changes/data_dir_default_doc new file mode 100644 index 0000000000..6b49bb2a65 --- /dev/null +++ b/changes/data_dir_default_doc @@ -0,0 +1,3 @@ + o Documentation: + - Correct the documentation about the default DataDirectory value. + Closes ticket 21151. diff --git a/changes/faster-keccak b/changes/faster-keccak new file mode 100644 index 0000000000..45fc1526a8 --- /dev/null +++ b/changes/faster-keccak @@ -0,0 +1,4 @@ + o Minor features (performance): + - The minimal keccak implementation we include now accesses memory + more efficiently, especially on little-endian systems. + Closes ticket 21737. diff --git a/changes/feature21622 b/changes/feature21622 new file mode 100644 index 0000000000..163b90b724 --- /dev/null +++ b/changes/feature21622 @@ -0,0 +1,8 @@ + o Minor feature (hidden services): + - Log a message when a hidden service reaches its introduction point + circuit limit, and when that limit is reset. + Follow up to ticket 21594, closes ticket 21622. + - Add more information to the message logged when a hidden service + descriptor has fewer introduction points than specified in + HiddenServiceNumIntroductionPoints. + Follow up to tickets 21598 and 21599, closes ticket 21622. diff --git a/changes/storagedir b/changes/storagedir new file mode 100644 index 0000000000..afaaab397f --- /dev/null +++ b/changes/storagedir @@ -0,0 +1,5 @@ + o Minor features (infrastructure, seccomp2 sandbox): + - We now have a document storage backend compatible with the Linux + seccomp2 sandbox. The long-term plan is to use this backend for + consensus documents and for storing unparseable directory + material. Closes ticket 21645. diff --git a/changes/ticket21729 b/changes/ticket21729 new file mode 100644 index 0000000000..51d117311b --- /dev/null +++ b/changes/ticket21729 @@ -0,0 +1,7 @@ + o Minor features (logging): + - Log files are no longer created world-readable by default. + (Previously, most distributors would store the logs in a + non-world-readable location to prevent inappropriate access. This + change is an extra precaution.) Closes ticket 21729; patch from + toralf. + diff --git a/configure.ac b/configure.ac index 05ac9fe846..0d4fa9838b 100644 --- a/configure.ac +++ b/configure.ac @@ -1,6 +1,6 @@ dnl Copyright (c) 2001-2004, Roger Dingledine dnl Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson -dnl Copyright (c) 2007-2015, The Tor Project, Inc. +dnl Copyright (c) 2007-2017, The Tor Project, Inc. dnl See LICENSE for licensing information AC_PREREQ([2.63]) diff --git a/doc/HACKING/CodingStandards.md b/doc/HACKING/CodingStandards.md index 01212a9919..c7787a72cc 100644 --- a/doc/HACKING/CodingStandards.md +++ b/doc/HACKING/CodingStandards.md @@ -4,9 +4,10 @@ Coding conventions for Tor tl;dr: - Run configure with `--enable-fatal-warnings` - - Run `make check-spaces` to catch whitespace errors - Document your functions - Write unit tests + - Run `make test-full` to test against all unit and integration tests. + - Run `make distcheck` to ensure the distribution works - Add a file in `changes` for your branch. Patch checklist @@ -22,10 +23,12 @@ preference) Did you remember... - To build your code while configured with `--enable-fatal-warnings`? - - To run `make check-spaces` on your code? - To run `make check-docs` to see whether all new options are on the manpage? - To write unit tests, as possible? + - To run `make test-full` to test against all unit and integration tests (or + `make test-full-online` if you have a working connection to the internet)? + - To test that the distribution will actually work via `make distcheck`? - To base your code on the appropriate branch? - To include a file in the `changes` directory as appropriate? diff --git a/doc/tor.1.txt b/doc/tor.1.txt index de2e2b4a04..568771a123 100644 --- a/doc/tor.1.txt +++ b/doc/tor.1.txt @@ -185,6 +185,9 @@ GENERAL OPTIONS course, more is better; we recommend at least 250 KBytes (2 mbits) if possible. (Default: 1 GByte) + + + Note that this option, and other bandwidth-limiting options, apply to TCP + data only: They do not count TCP headers or DNS traffic. + + + With this option, and in other options that take arguments in bytes, KBytes, and so on, other formats are also supported. Notably, "KBytes" can also be written as "kilobytes" or "kb"; "MBytes" can be written as @@ -391,7 +394,9 @@ GENERAL OPTIONS [[DataDirectory]] **DataDirectory** __DIR__:: Store working data in DIR. Can not be changed while tor is running. - (Default: @LOCALSTATEDIR@/lib/tor) + (Default: ~/.tor if your home directory is not /; otherwise, + @LOCALSTATEDIR@/lib/tor. On Windows, the default is + your ApplicationData folder.) [[DataDirectoryGroupReadable]] **DataDirectoryGroupReadable** **0**|**1**:: If this option is set to 0, don't allow the filesystem group to read the @@ -2121,8 +2126,9 @@ is non-zero): DIRECTORY SERVER OPTIONS ------------------------ -The following options are useful only for directory servers (that is, -if DirPort is non-zero): +The following options are useful only for directory servers. (Relays with +enough bandwidth automatically become directory servers; see DirCache for +details.) [[DirPortFrontPage]] **DirPortFrontPage** __FILENAME__:: When this option is set, it takes an HTML file and publishes it as "/" on diff --git a/scripts/codegen/fuzzing_include_am.py b/scripts/codegen/fuzzing_include_am.py new file mode 100755 index 0000000000..91e9e5f3f6 --- /dev/null +++ b/scripts/codegen/fuzzing_include_am.py @@ -0,0 +1,154 @@ + + +FUZZERS = """ + consensus + descriptor + diff + diff-apply + extrainfo + hsdescv2 + http + iptsv2 + microdesc + vrs +""" + + +PREAMBLE = r""" +FUZZING_CPPFLAGS = \ + $(src_test_AM_CPPFLAGS) $(TEST_CPPFLAGS) +FUZZING_CFLAGS = \ + $(AM_CFLAGS) $(TEST_CFLAGS) +FUZZING_LDFLAG = \ + @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ @TOR_LDFLAGS_libevent@ +FUZZING_LIBS = \ + src/or/libtor-testing.a \ + src/common/libor-crypto-testing.a \ + $(LIBKECCAK_TINY) \ + $(LIBDONNA) \ + src/common/libor-testing.a \ + src/common/libor-ctime-testing.a \ + src/common/libor-event-testing.a \ + src/trunnel/libor-trunnel-testing.a \ + @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ \ + @TOR_LIBEVENT_LIBS@ \ + @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ \ + @TOR_SYSTEMD_LIBS@ + +oss-fuzz-prereqs: \ + src/or/libtor-testing.a \ + src/common/libor-crypto-testing.a \ + $(LIBKECCAK_TINY) \ + $(LIBDONNA) \ + src/common/libor-testing.a \ + src/common/libor-ctime-testing.a \ + src/common/libor-event-testing.a \ + src/trunnel/libor-trunnel-testing.a + +noinst_HEADERS += \ + src/test/fuzz/fuzzing.h + +LIBFUZZER = /home/nickm/build/libfuzz/libFuzzer.a +LIBFUZZER_CPPFLAGS = $(FUZZING_CPPFLAGS) -DLLVM_FUZZ +LIBFUZZER_CFLAGS = $(FUZZING_CFLAGS) +LIBFUZZER_LDFLAG = $(FUZZING_LDFLAG) +LIBFUZZER_LIBS = $(FUZZING_LIBS) $(LIBFUZZER) -lstdc++ + +LIBOSS_FUZZ_CPPFLAGS = $(FUZZING_CPPFLAGS) -DLLVM_FUZZ +LIBOSS_FUZZ_CFLAGS = $(FUZZING_CFLAGS) +""" + +POSTAMBLE = r""" +noinst_PROGRAMS += $(FUZZERS) $(LIBFUZZER_FUZZERS) +noinst_LIBRARIES += $(OSS_FUZZ_FUZZERS) +oss-fuzz-fuzzers: oss-fuzz-prereqs $(OSS_FUZZ_FUZZERS) +fuzzers: $(FUZZERS) $(LIBFUZZER_FUZZERS) + +test-fuzz-corpora: $(FUZZERS) + $(top_srcdir)/src/test/fuzz_static_testcases.sh +""" + +########### No user serviceable parts will follow. + +PREAMBLE = PREAMBLE.strip() +POSTAMBLE = POSTAMBLE.strip() # If I use it, it's a word! +FUZZERS = FUZZERS.split() +FUZZERS.sort() + +WARNING = """ +# This file was generated by fuzzing_include_am.py; do not hand-edit unless +# you enjoy having your changes erased. +""".strip() + +print(WARNING) + +print(PREAMBLE) + +print("\n# ===== AFL fuzzers") + +def get_id_name(s): + return s.replace("-", "_") + +for fuzzer in FUZZERS: + idname = get_id_name(fuzzer) + print("""\ +src_test_fuzz_fuzz_{name}_SOURCES = \\ + src/test/fuzz/fuzzing_common.c \\ + src/test/fuzz/fuzz_{name}.c +src_test_fuzz_fuzz_{name}_CPPFLAGS = $(FUZZING_CPPFLAGS) +src_test_fuzz_fuzz_{name}_CFLAGS = $(FUZZING_CFLAGS) +src_test_fuzz_fuzz_{name}_LDFLAGS = $(FUZZING_LDFLAG) +src_test_fuzz_fuzz_{name}_LDADD = $(FUZZING_LIBS) +""".format(name=idname)) + +print("FUZZERS = \\") +print(" \\\n".join("\tsrc/test/fuzz/fuzz-{name}".format(name=fuzzer) + for fuzzer in FUZZERS)) + +print("\n# ===== libfuzzer") +print("\nif LIBFUZZER_ENABLED") + +for fuzzer in FUZZERS: + idname = get_id_name(fuzzer) + print("""\ +src_test_fuzz_lf_fuzz_{name}_SOURCES = \\ + $(src_test_fuzz_fuzz_{name}_SOURCES) +src_test_fuzz_lf_fuzz_{name}_CPPFLAGS = $(LIBFUZZER_CPPFLAGS) +src_test_fuzz_lf_fuzz_{name}_CFLAGS = $(LIBFUZZER_CFLAGS) +src_test_fuzz_lf_fuzz_{name}_LDFLAGS = $(LIBFUZZER_LDFLAG) +src_test_fuzz_lf_fuzz_{name}_LDADD = $(LIBFUZZER_LIBS) +""".format(name=idname)) + +print("LIBFUZZER_FUZZERS = \\") +print(" \\\n".join("\tsrc/test/fuzz/lf-fuzz-{name}".format(name=fuzzer) + for fuzzer in FUZZERS)) + +print(""" +else +LIBFUZZER_FUZZERS = +endif""") + +print("\n# ===== oss-fuzz\n") +print("if OSS_FUZZ_ENABLED") + +for fuzzer in FUZZERS: + idname = get_id_name(fuzzer) + print("""\ +src_test_fuzz_liboss_fuzz_{name}_a_SOURCES = \\ + $(src_test_fuzz_fuzz_{name}_SOURCES) +src_test_fuzz_liboss_fuzz_{name}_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS) +src_test_fuzz_liboss_fuzz_{name}_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS) +""".format(name=idname)) + +print("OSS_FUZZ_FUZZERS = \\") +print(" \\\n".join("\tsrc/test/fuzz/liboss-fuzz-{name}.a".format(name=fuzzer) + for fuzzer in FUZZERS)) + +print(""" +else +OSS_FUZZ_FUZZERS = +endif""") + +print("") + +print(POSTAMBLE) diff --git a/scripts/codegen/gen_server_ciphers.py b/scripts/codegen/gen_server_ciphers.py index 7470f8a025..7ea39c540d 100755 --- a/scripts/codegen/gen_server_ciphers.py +++ b/scripts/codegen/gen_server_ciphers.py @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright 2014-2015, The Tor Project, Inc +# Copyright 2014-2017, The Tor Project, Inc # See LICENSE for licensing information # This script parses openssl headers to find ciphersuite names, determines diff --git a/scripts/codegen/get_mozilla_ciphers.py b/scripts/codegen/get_mozilla_ciphers.py index b07746c2e2..946957ac77 100755 --- a/scripts/codegen/get_mozilla_ciphers.py +++ b/scripts/codegen/get_mozilla_ciphers.py @@ -1,6 +1,6 @@ #!/usr/bin/python # coding=utf-8 -# Copyright 2011-2015, The Tor Project, Inc +# Copyright 2011-2017, The Tor Project, Inc # original version by Arturo Filastò # See LICENSE for licensing information diff --git a/scripts/codegen/makedesc.py b/scripts/codegen/makedesc.py index d4ba21efae..8d9d4edaaf 100644 --- a/scripts/codegen/makedesc.py +++ b/scripts/codegen/makedesc.py @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright 2014-2015, The Tor Project, Inc. +# Copyright 2014-2017, The Tor Project, Inc. # See LICENSE for license information # This is a kludgey python script that uses ctypes and openssl to sign diff --git a/scripts/maint/format_changelog.py b/scripts/maint/format_changelog.py index 2429c24417..c5a0cfc81b 100755 --- a/scripts/maint/format_changelog.py +++ b/scripts/maint/format_changelog.py @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (c) 2014-2015, The Tor Project, Inc. +# Copyright (c) 2014-2017, The Tor Project, Inc. # See LICENSE for licensing information # # This script reformats a section of the changelog to wrap everything to diff --git a/scripts/maint/redox.py b/scripts/maint/redox.py index 43f5b6eb16..12aed6463a 100755 --- a/scripts/maint/redox.py +++ b/scripts/maint/redox.py @@ -1,6 +1,6 @@ #!/usr/bin/python # -# Copyright (c) 2008-2015, The Tor Project, Inc. +# Copyright (c) 2008-2017, The Tor Project, Inc. # See LICENSE for licensing information. # # Hi! diff --git a/scripts/maint/sortChanges.py b/scripts/maint/sortChanges.py index d6ec0e269d..22e40fd369 100755 --- a/scripts/maint/sortChanges.py +++ b/scripts/maint/sortChanges.py @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (c) 2014-2015, The Tor Project, Inc. +# Copyright (c) 2014-2017, The Tor Project, Inc. # See LICENSE for licensing information """This script sorts a bunch of changes files listed on its command diff --git a/scripts/maint/updateCopyright.pl b/scripts/maint/updateCopyright.pl index 8bd6a18210..beb0b8f26e 100755 --- a/scripts/maint/updateCopyright.pl +++ b/scripts/maint/updateCopyright.pl @@ -1,7 +1,7 @@ #!/usr/bin/perl -i -w -p -$NEWYEAR=2016; +$NEWYEAR=2017; -s/Copyright(.*) (201[^6]), The Tor Project/Copyright$1 $2-${NEWYEAR}, The Tor Project/; +s/Copyright(.*) (201[^7]), The Tor Project/Copyright$1 $2-${NEWYEAR}, The Tor Project/; s/Copyright(.*)-(20..), The Tor Project/Copyright$1-${NEWYEAR}, The Tor Project/; diff --git a/src/common/address.c b/src/common/address.c index 2693239146..701daace57 100644 --- a/src/common/address.c +++ b/src/common/address.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/address.h b/src/common/address.h index 0dc6edae37..ce85b3d81d 100644 --- a/src/common/address.h +++ b/src/common/address.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/aes.c b/src/common/aes.c index 35c2d1e3a5..73abef143e 100644 --- a/src/common/aes.c +++ b/src/common/aes.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/aes.h b/src/common/aes.h index 1cda53f2fa..6fd9c3ea16 100644 --- a/src/common/aes.h +++ b/src/common/aes.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /* Implements a minimal interface to counter-mode AES. */ diff --git a/src/common/backtrace.c b/src/common/backtrace.c index 61096952d8..0f0fa857bf 100644 --- a/src/common/backtrace.c +++ b/src/common/backtrace.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2016, The Tor Project, Inc. */ +/* Copyright (c) 2013-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/backtrace.h b/src/common/backtrace.h index b53fd2c668..8cd1dcf06c 100644 --- a/src/common/backtrace.h +++ b/src/common/backtrace.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2016, The Tor Project, Inc. */ +/* Copyright (c) 2013-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_BACKTRACE_H diff --git a/src/common/compat.c b/src/common/compat.c index 0dbede6bed..390cff7dba 100644 --- a/src/common/compat.c +++ b/src/common/compat.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/compat.h b/src/common/compat.h index ee1c9454de..68fad9b598 100644 --- a/src/common/compat.h +++ b/src/common/compat.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_COMPAT_H diff --git a/src/common/compat_libevent.c b/src/common/compat_libevent.c index 4a3b1af922..1146d02168 100644 --- a/src/common/compat_libevent.c +++ b/src/common/compat_libevent.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2009-2016, The Tor Project, Inc. */ +/* Copyright (c) 2009-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/compat_libevent.h b/src/common/compat_libevent.h index c2e34764e4..96c48d53a1 100644 --- a/src/common/compat_libevent.h +++ b/src/common/compat_libevent.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2009-2016, The Tor Project, Inc. */ +/* Copyright (c) 2009-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_COMPAT_LIBEVENT_H diff --git a/src/common/compat_openssl.h b/src/common/compat_openssl.h index 1bfe188075..2b94fe5b4e 100644 --- a/src/common/compat_openssl.h +++ b/src/common/compat_openssl.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_COMPAT_OPENSSL_H diff --git a/src/common/compat_pthreads.c b/src/common/compat_pthreads.c index c1ae66c1d2..8e4b833573 100644 --- a/src/common/compat_pthreads.c +++ b/src/common/compat_pthreads.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/compat_threads.c b/src/common/compat_threads.c index f4809060d6..7b28a9494a 100644 --- a/src/common/compat_threads.c +++ b/src/common/compat_threads.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -94,51 +94,73 @@ in_main_thread(void) } #if defined(HAVE_EVENTFD) || defined(HAVE_PIPE) -/* As write(), but retry on EINTR */ +/* As write(), but retry on EINTR, and return the negative error code on + * error. */ static int write_ni(int fd, const void *buf, size_t n) { int r; again: r = (int) write(fd, buf, n); - if (r < 0 && errno == EINTR) - goto again; + if (r < 0) { + if (errno == EINTR) + goto again; + else + return -errno; + } return r; } -/* As read(), but retry on EINTR */ +/* As read(), but retry on EINTR, and return the negative error code on error. + */ static int read_ni(int fd, void *buf, size_t n) { int r; again: r = (int) read(fd, buf, n); - if (r < 0 && errno == EINTR) - goto again; + if (r < 0) { + if (errno == EINTR) + goto again; + else + return -errno; + } return r; } #endif -/** As send(), but retry on EINTR. */ +/** As send(), but retry on EINTR, and return the negative error code on + * error. */ static int send_ni(int fd, const void *buf, size_t n, int flags) { int r; again: r = (int) send(fd, buf, n, flags); - if (r < 0 && ERRNO_IS_EINTR(tor_socket_errno(fd))) - goto again; + if (r < 0) { + int error = tor_socket_errno(fd); + if (ERRNO_IS_EINTR(error)) + goto again; + else + return -error; + } return r; } -/** As recv(), but retry on EINTR. */ +/** As recv(), but retry on EINTR, and return the negative error code on + * error. */ static int recv_ni(int fd, void *buf, size_t n, int flags) { int r; again: r = (int) recv(fd, buf, n, flags); - if (r < 0 && ERRNO_IS_EINTR(tor_socket_errno(fd))) - goto again; + if (r < 0) { + int error = tor_socket_errno(fd); + if (ERRNO_IS_EINTR(error)) + goto again; + else + return -error; + } return r; } @@ -149,7 +171,7 @@ eventfd_alert(int fd) { uint64_t u = 1; int r = write_ni(fd, (void*)&u, sizeof(u)); - if (r < 0 && errno != EAGAIN) + if (r < 0 && -r != EAGAIN) return -1; return 0; } @@ -160,8 +182,8 @@ eventfd_drain(int fd) { uint64_t u = 0; int r = read_ni(fd, (void*)&u, sizeof(u)); - if (r < 0 && errno != EAGAIN) - return -1; + if (r < 0 && -r != EAGAIN) + return r; return 0; } #endif @@ -172,8 +194,8 @@ static int pipe_alert(int fd) { ssize_t r = write_ni(fd, "x", 1); - if (r < 0 && errno != EAGAIN) - return -1; + if (r < 0 && -r != EAGAIN) + return (int)r; return 0; } @@ -188,7 +210,7 @@ pipe_drain(int fd) r = read_ni(fd, buf, sizeof(buf)); } while (r > 0); if (r < 0 && errno != EAGAIN) - return -1; + return -errno; /* A value of r = 0 means EOF on the fd so successfully drained. */ return 0; } @@ -200,13 +222,13 @@ static int sock_alert(tor_socket_t fd) { ssize_t r = send_ni(fd, "x", 1, 0); - if (r < 0 && !ERRNO_IS_EAGAIN(tor_socket_errno(fd))) - return -1; + if (r < 0 && !ERRNO_IS_EAGAIN(-r)) + return (int)r; return 0; } /** Drain all the input from a socket <b>fd</b>, and ignore it. Return 0 on - * success, -1 on error. */ + * success, -errno on error. */ static int sock_drain(tor_socket_t fd) { @@ -215,8 +237,8 @@ sock_drain(tor_socket_t fd) do { r = recv_ni(fd, buf, sizeof(buf), 0); } while (r > 0); - if (r < 0 && !ERRNO_IS_EAGAIN(tor_socket_errno(fd))) - return -1; + if (r < 0 && !ERRNO_IS_EAGAIN(-r)) + return (int)r; /* A value of r = 0 means EOF on the fd so successfully drained. */ return 0; } diff --git a/src/common/compat_threads.h b/src/common/compat_threads.h index 171a9f93ff..2943573b23 100644 --- a/src/common/compat_threads.h +++ b/src/common/compat_threads.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_COMPAT_THREADS_H diff --git a/src/common/compat_time.c b/src/common/compat_time.c index d044bbe1d7..2ccaa36e49 100644 --- a/src/common/compat_time.c +++ b/src/common/compat_time.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/compat_time.h b/src/common/compat_time.h index 2262446e57..90194c5ebc 100644 --- a/src/common/compat_time.h +++ b/src/common/compat_time.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/compat_winthreads.c b/src/common/compat_winthreads.c index 735be4ad17..915368b94f 100644 --- a/src/common/compat_winthreads.c +++ b/src/common/compat_winthreads.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/container.c b/src/common/container.c index 1448ab403c..689e7e55e9 100644 --- a/src/common/container.c +++ b/src/common/container.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/container.h b/src/common/container.h index 00c3ca81ad..db68d892f5 100644 --- a/src/common/container.h +++ b/src/common/container.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_CONTAINER_H @@ -224,6 +224,7 @@ char *smartlist_join_strings2(smartlist_t *sl, const char *join, #define SMARTLIST_FOREACH_END(var) \ var = NULL; \ + (void) var ## _sl_idx; \ } STMT_END /** diff --git a/src/common/crypto.c b/src/common/crypto.c index 7cb3330bde..a5eb7b5c9a 100644 --- a/src/common/crypto.c +++ b/src/common/crypto.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/crypto.h b/src/common/crypto.h index 42345f80e8..62c78b5d77 100644 --- a/src/common/crypto.h +++ b/src/common/crypto.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/crypto_curve25519.c b/src/common/crypto_curve25519.c index 5f328e124c..b99f13a93b 100644 --- a/src/common/crypto_curve25519.c +++ b/src/common/crypto_curve25519.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2012-2016, The Tor Project, Inc. */ +/* Copyright (c) 2012-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/crypto_curve25519.h b/src/common/crypto_curve25519.h index 4011820949..e7790edac0 100644 --- a/src/common/crypto_curve25519.h +++ b/src/common/crypto_curve25519.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2012-2016, The Tor Project, Inc. */ +/* Copyright (c) 2012-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_CRYPTO_CURVE25519_H diff --git a/src/common/crypto_ed25519.c b/src/common/crypto_ed25519.c index 525d25a3e0..f97b289169 100644 --- a/src/common/crypto_ed25519.c +++ b/src/common/crypto_ed25519.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2016, The Tor Project, Inc. */ +/* Copyright (c) 2013-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/crypto_ed25519.h b/src/common/crypto_ed25519.h index f4a4adad68..77a3313adc 100644 --- a/src/common/crypto_ed25519.h +++ b/src/common/crypto_ed25519.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2012-2016, The Tor Project, Inc. */ +/* Copyright (c) 2012-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_CRYPTO_ED25519_H diff --git a/src/common/crypto_format.c b/src/common/crypto_format.c index aa2a9d1fb0..1d090a8770 100644 --- a/src/common/crypto_format.c +++ b/src/common/crypto_format.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/crypto_format.h b/src/common/crypto_format.h index 86c29d319c..390916cf04 100644 --- a/src/common/crypto_format.h +++ b/src/common/crypto_format.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_CRYPTO_FORMAT_H diff --git a/src/common/crypto_pwbox.c b/src/common/crypto_pwbox.c index 31e37c007d..db8892e376 100644 --- a/src/common/crypto_pwbox.c +++ b/src/common/crypto_pwbox.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2014-2016, The Tor Project, Inc. */ +/* Copyright (c) 2014-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/crypto_s2k.c b/src/common/crypto_s2k.c index 5dbd2ad91f..076df815a9 100644 --- a/src/common/crypto_s2k.c +++ b/src/common/crypto_s2k.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/crypto_s2k.h b/src/common/crypto_s2k.h index 9b186450b1..04212b868a 100644 --- a/src/common/crypto_s2k.h +++ b/src/common/crypto_s2k.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_CRYPTO_S2K_H_INCLUDED diff --git a/src/common/di_ops.c b/src/common/di_ops.c index 4ed49e1164..e47998107d 100644 --- a/src/common/di_ops.c +++ b/src/common/di_ops.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2011-2016, The Tor Project, Inc. */ +/* Copyright (c) 2011-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/di_ops.h b/src/common/di_ops.h index 0a154302bf..e174fcc4e4 100644 --- a/src/common/di_ops.h +++ b/src/common/di_ops.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/handles.h b/src/common/handles.h index 1ee2322579..6d7262ab80 100644 --- a/src/common/handles.h +++ b/src/common/handles.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/include.am b/src/common/include.am index 40c463c9d9..b5d6ac4ac5 100644 --- a/src/common/include.am +++ b/src/common/include.am @@ -93,6 +93,7 @@ LIBOR_A_SRC = \ src/common/util_format.c \ src/common/util_process.c \ src/common/sandbox.c \ + src/common/storagedir.c \ src/common/workqueue.c \ $(libor_extra_source) \ $(threads_impl_source) \ @@ -157,6 +158,7 @@ COMMONHEADERS = \ src/common/procmon.h \ src/common/pubsub.h \ src/common/sandbox.h \ + src/common/storagedir.h \ src/common/testsupport.h \ src/common/timers.h \ src/common/torgzip.h \ diff --git a/src/common/log.c b/src/common/log.c index 5f7151bf0c..6a5819064a 100644 --- a/src/common/log.c +++ b/src/common/log.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -1086,7 +1086,7 @@ add_file_log(const log_severity_list_t *severity, const char *filename, int open_flags = O_WRONLY|O_CREAT; open_flags |= truncate_log ? O_TRUNC : O_APPEND; - fd = tor_open_cloexec(filename, open_flags, 0644); + fd = tor_open_cloexec(filename, open_flags, 0640); if (fd<0) return -1; if (tor_fd_seekend(fd)<0) { @@ -1177,7 +1177,7 @@ static const char *domain_list[] = { "GENERAL", "CRYPTO", "NET", "CONFIG", "FS", "PROTOCOL", "MM", "HTTP", "APP", "CONTROL", "CIRC", "REND", "BUG", "DIR", "DIRSERV", "OR", "EDGE", "ACCT", "HIST", "HANDSHAKE", "HEARTBEAT", "CHANNEL", - "SCHED", "GUARD", NULL + "SCHED", "GUARD", "CONSDIFF", NULL }; /** Return a bitmask for the log domain for which <b>domain</b> is the name, diff --git a/src/common/memarea.c b/src/common/memarea.c index 12781e76ea..659d1edf54 100644 --- a/src/common/memarea.c +++ b/src/common/memarea.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2008-2016, The Tor Project, Inc. */ +/* Copyright (c) 2008-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** \file memarea.c diff --git a/src/common/memarea.h b/src/common/memarea.h index 85bca51ad3..85012c1c34 100644 --- a/src/common/memarea.h +++ b/src/common/memarea.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2008-2016, The Tor Project, Inc. */ +/* Copyright (c) 2008-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /* Tor dependencies */ diff --git a/src/common/procmon.c b/src/common/procmon.c index c485c760c7..d49e7f18f5 100644 --- a/src/common/procmon.c +++ b/src/common/procmon.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2011-2016, The Tor Project, Inc. */ +/* Copyright (c) 2011-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/procmon.h b/src/common/procmon.h index 49ead24092..b07cff2c4a 100644 --- a/src/common/procmon.h +++ b/src/common/procmon.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2011-2016, The Tor Project, Inc. */ +/* Copyright (c) 2011-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/pubsub.c b/src/common/pubsub.c index b3faf40e00..336e8a6e7f 100644 --- a/src/common/pubsub.c +++ b/src/common/pubsub.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/pubsub.h b/src/common/pubsub.h index bbb4f02a42..6f4ce08754 100644 --- a/src/common/pubsub.h +++ b/src/common/pubsub.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/sandbox.c b/src/common/sandbox.c index 8e72ebf137..ddb1f02b05 100644 --- a/src/common/sandbox.c +++ b/src/common/sandbox.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/sandbox.h b/src/common/sandbox.h index c5963e3119..a6b83153af 100644 --- a/src/common/sandbox.h +++ b/src/common/sandbox.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/storagedir.c b/src/common/storagedir.c new file mode 100644 index 0000000000..e28a66f693 --- /dev/null +++ b/src/common/storagedir.c @@ -0,0 +1,409 @@ +/* Copyright (c) 2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "container.h" +#include "compat.h" +#include "sandbox.h" +#include "storagedir.h" +#include "torlog.h" +#include "util.h" + +#ifdef HAVE_SYS_TYPES_H +#include <sys/types.h> +#endif +#ifdef HAVE_SYS_STAT_H +#include <sys/stat.h> +#endif +#ifdef HAVE_UNISTD_H +#include <unistd.h> +#endif + +#define FNAME_MIN_NUM 1000 + +/** A storage_dir_t represents a directory full of similar cached + * files. Filenames are decimal integers. Files can be cleaned as needed + * to limit total disk usage. */ +struct storage_dir_t { + /** Directory holding the files for this storagedir. */ + char *directory; + /** Either NULL, or a directory listing of the directory (as a smartlist + * of strings */ + smartlist_t *contents; + /** The largest number of non-temporary files we'll place in the + * directory. */ + int max_files; + /** If true, then 'usage' has been computed. */ + int usage_known; + /** The total number of bytes used in this directory */ + uint64_t usage; +}; + +/** Create or open a new storage directory at <b>dirname</b>, with + * capacity for up to <b>max_files</b> files. + */ +storage_dir_t * +storage_dir_new(const char *dirname, int max_files) +{ + if (check_private_dir(dirname, CPD_CREATE, NULL) < 0) + return NULL; + + storage_dir_t *d = tor_malloc_zero(sizeof(storage_dir_t)); + d->directory = tor_strdup(dirname); + d->max_files = max_files; + return d; +} + +/** + * Drop all in-RAM storage for <b>d</b>. Does not delete any files. + */ +void +storage_dir_free(storage_dir_t *d) +{ + if (d == NULL) + return; + tor_free(d->directory); + if (d->contents) { + SMARTLIST_FOREACH(d->contents, char *, cp, tor_free(cp)); + smartlist_free(d->contents); + } + tor_free(d); +} + +/** + * Tell the sandbox (if any) configured by <b>cfg</b> to allow the + * operations that <b>d</b> will need. + * + * The presence of this function is why we need an upper limit on the + * number of filers in a storage_dir_t: we need to approve file + * operaitons one by one. + */ +int +storage_dir_register_with_sandbox(storage_dir_t *d, sandbox_cfg_t **cfg) +{ + int problems = 0; + int idx; + for (idx = FNAME_MIN_NUM; idx < FNAME_MIN_NUM + d->max_files; ++idx) { + char *path = NULL, *tmppath = NULL; + tor_asprintf(&path, "%s/%d", d->directory, idx); + tor_asprintf(&tmppath, "%s/%d.tmp", d->directory, idx); + + problems += sandbox_cfg_allow_open_filename(cfg, path); + problems += sandbox_cfg_allow_open_filename(cfg, tmppath); + problems += sandbox_cfg_allow_stat_filename(cfg, path); + problems += sandbox_cfg_allow_stat_filename(cfg, tmppath); + problems += sandbox_cfg_allow_rename(cfg, tmppath, path); + + tor_free(path); + tor_free(tmppath); + } + + return problems ? -1 : 0; +} + +/** + * Remove all files in <b>d</b> whose names end with ".tmp". + * + * Requires that the contents field of <b>d</b> is set. + */ +static void +storage_dir_clean_tmpfiles(storage_dir_t *d) +{ + if (!d->contents) + return; + SMARTLIST_FOREACH_BEGIN(d->contents, char *, fname) { + if (strcmpend(fname, ".tmp")) + continue; + char *path = NULL; + tor_asprintf(&path, "%s/%s", d->directory, fname); + if (unlink(sandbox_intern_string(path))) { + log_warn(LD_FS, "Unable to unlink %s", escaped(path)); + tor_free(path); + continue; + } + tor_free(path); + SMARTLIST_DEL_CURRENT(d->contents, fname); + tor_free(fname); + } SMARTLIST_FOREACH_END(fname); + + d->usage_known = 0; +} + +/** + * Re-scan the directory <b>d</b> to learn its contents. + */ +static int +storage_dir_rescan(storage_dir_t *d) +{ + if (d->contents) { + SMARTLIST_FOREACH(d->contents, char *, cp, tor_free(cp)); + smartlist_free(d->contents); + } + d->usage = 0; + d->usage_known = 0; + if (NULL == (d->contents = tor_listdir(d->directory))) { + return -1; + } + storage_dir_clean_tmpfiles(d); + return 0; +} + +/** + * Return a smartlist containing the filenames within <b>d</b>. + */ +const smartlist_t * +storage_dir_list(storage_dir_t *d) +{ + if (! d->contents) + storage_dir_rescan(d); + return d->contents; +} + +/** + * Return the total number of bytes used for storage in <b>d</b>. + */ +uint64_t +storage_dir_get_usage(storage_dir_t *d) +{ + if (d->usage_known) + return d->usage; + + uint64_t total = 0; + SMARTLIST_FOREACH_BEGIN(storage_dir_list(d), const char *, cp) { + char *path = NULL; + struct stat st; + tor_asprintf(&path, "%s/%s", d->directory, cp); + if (stat(sandbox_intern_string(path), &st) == 0) { + total += st.st_size; + } + tor_free(path); + } SMARTLIST_FOREACH_END(cp); + + d->usage = total; + d->usage_known = 1; + return d->usage; +} + +/** Mmap a specified file within <b>d</b>. */ +tor_mmap_t * +storage_dir_map(storage_dir_t *d, const char *fname) +{ + char *path = NULL; + tor_asprintf(&path, "%s/%s", d->directory, fname); + tor_mmap_t *result = tor_mmap_file(path); + tor_free(path); + return result; +} + +/** Read a file within <b>d</b> into a newly allocated buffer. Set + * *<b>sz_out</b> to its size. */ +uint8_t * +storage_dir_read(storage_dir_t *d, const char *fname, int bin, size_t *sz_out) +{ + const int flags = bin ? RFTS_BIN : 0; + + char *path = NULL; + tor_asprintf(&path, "%s/%s", d->directory, fname); + struct stat st; + char *contents = read_file_to_str(path, flags, &st); + if (contents && sz_out) { + // it fits in RAM, so we know its size is less than SIZE_MAX + tor_assert((uint64_t)st.st_size <= SIZE_MAX); + *sz_out = (size_t) st.st_size; + } + + tor_free(path); + return (uint8_t *) contents; +} + +/** Helper: Find an unused filename within the directory */ +static char * +find_unused_fname(storage_dir_t *d) +{ + if (!d->contents) { + if (storage_dir_rescan(d) < 0) + return NULL; + } + + char buf[16]; + int i; + /* Yuck; this is quadratic. Fortunately, that shouldn't matter much, + * since disk writes are more expensive by a lot. */ + for (i = FNAME_MIN_NUM; i < FNAME_MIN_NUM + d->max_files; ++i) { + tor_snprintf(buf, sizeof(buf), "%d", i); + if (!smartlist_contains_string(d->contents, buf)) { + return tor_strdup(buf); + } + } + return NULL; +} + +/** Try to write the <b>length</b> bytes at <b>data</b> into a new file + * in <b>d</b>. On success, return 0 and set *<b>fname_out</b> to a + * newly allocated string containing the filename. On failure, return + * -1. */ +int +storage_dir_save_bytes_to_file(storage_dir_t *d, + const uint8_t *data, + size_t length, + int binary, + char **fname_out) +{ + char *fname = find_unused_fname(d); + if (!fname) + return -1; + + char *path = NULL; + tor_asprintf(&path, "%s/%s", d->directory, fname); + + int r = write_bytes_to_file(path, (const char *)data, length, binary); + if (r == 0) { + if (d->usage_known) + d->usage += length; + if (fname_out) { + *fname_out = tor_strdup(fname); + } + if (d->contents) + smartlist_add(d->contents, tor_strdup(fname)); + } + tor_free(fname); + tor_free(path); + return r; +} + +/** + * As storage_dir_save_bytes_to_file, but saves a NUL-terminated string + * <b>str</b>. + */ +int +storage_dir_save_string_to_file(storage_dir_t *d, + const char *str, + int binary, + char **fname_out) +{ + return storage_dir_save_bytes_to_file(d, + (const uint8_t*)str, strlen(str), binary, fname_out); +} + +/** + * Remove the file called <b>fname</b> from <b>d</b>. + */ +void +storage_dir_remove_file(storage_dir_t *d, + const char *fname) +{ + char *path = NULL; + tor_asprintf(&path, "%s/%s", d->directory, fname); + const char *ipath = sandbox_intern_string(path); + + uint64_t size = 0; + if (d->usage_known) { + struct stat st; + if (stat(ipath, &st) == 0) { + size = st.st_size; + } + } + if (unlink(ipath) == 0) { + d->usage -= size; + } else { + log_warn(LD_FS, "Unable to unlink %s", escaped(path)); + tor_free(path); + return; + } + if (d->contents) { + smartlist_string_remove(d->contents, fname); + } + + tor_free(path); +} + +/** Helper type: used to sort the members of storage directory by mtime. */ +typedef struct shrinking_dir_entry_t { + time_t mtime; + uint64_t size; + char *path; +} shrinking_dir_entry_t; + +/** Helper: use with qsort to sort shrinking_dir_entry_t structs. */ +static int +shrinking_dir_entry_compare(const void *a_, const void *b_) +{ + const shrinking_dir_entry_t *a = a_; + const shrinking_dir_entry_t *b = b_; + + if (a->mtime < b->mtime) + return -1; + else if (a->mtime > b->mtime) + return 1; + else + return 0; +} + +/** + * Try to free space by removing the oldest files in <b>d</b>. Delete + * until no more than <b>target_size</b> bytes are left, and at least + * <b>min_to_remove</b> files have been removed... or until there is + * nothing left to remove. + * + * Return 0 on success; -1 on failure. + */ +int +storage_dir_shrink(storage_dir_t *d, + uint64_t target_size, + int min_to_remove) +{ + if (d->usage_known && d->usage <= target_size && !min_to_remove) { + /* Already small enough. */ + return 0; + } + + if (storage_dir_rescan(d) < 0) + return -1; + + const uint64_t orig_usage = storage_dir_get_usage(d); + if (orig_usage <= target_size && !min_to_remove) { + /* Okay, small enough after rescan! */ + return 0; + } + + const int n = smartlist_len(d->contents); + shrinking_dir_entry_t *ents = tor_calloc(n, sizeof(shrinking_dir_entry_t)); + SMARTLIST_FOREACH_BEGIN(d->contents, const char *, fname) { + shrinking_dir_entry_t *ent = &ents[fname_sl_idx]; + struct stat st; + tor_asprintf(&ent->path, "%s/%s", d->directory, fname); + if (stat(sandbox_intern_string(ent->path), &st) == 0) { + ent->mtime = st.st_mtime; + ent->size = st.st_size; + } + } SMARTLIST_FOREACH_END(fname); + + qsort(ents, n, sizeof(shrinking_dir_entry_t), shrinking_dir_entry_compare); + + int idx = 0; + while ((d->usage > target_size || min_to_remove > 0) && idx < n) { + if (unlink(sandbox_intern_string(ents[idx].path)) == 0) { + if (! BUG(d->usage < ents[idx].size)) { + d->usage -= ents[idx].size; + } + --min_to_remove; + } + ++idx; + } + + for (idx = 0; idx < n; ++idx) { + tor_free(ents[idx].path); + } + tor_free(ents); + + storage_dir_rescan(d); + + return 0; +} + +/** Remove all files in <b>d</b>. */ +int +storage_dir_remove_all(storage_dir_t *d) +{ + return storage_dir_shrink(d, 0, d->max_files); +} + diff --git a/src/common/storagedir.h b/src/common/storagedir.h new file mode 100644 index 0000000000..29be941b14 --- /dev/null +++ b/src/common/storagedir.h @@ -0,0 +1,37 @@ +/* Copyright (c) 2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_STORAGEDIR_H +#define TOR_STORAGEDIR_H + +typedef struct storage_dir_t storage_dir_t; + +struct sandbox_cfg_elem; + +storage_dir_t * storage_dir_new(const char *dirname, int n_files); +void storage_dir_free(storage_dir_t *d); +int storage_dir_register_with_sandbox(storage_dir_t *d, + struct sandbox_cfg_elem **cfg); +const smartlist_t *storage_dir_list(storage_dir_t *d); +uint64_t storage_dir_get_usage(storage_dir_t *d); +tor_mmap_t *storage_dir_map(storage_dir_t *d, const char *fname); +uint8_t *storage_dir_read(storage_dir_t *d, const char *fname, int bin, + size_t *sz_out); +int storage_dir_save_bytes_to_file(storage_dir_t *d, + const uint8_t *data, + size_t length, + int binary, + char **fname_out); +int storage_dir_save_string_to_file(storage_dir_t *d, + const char *data, + int binary, + char **fname_out); +void storage_dir_remove_file(storage_dir_t *d, + const char *fname); +int storage_dir_shrink(storage_dir_t *d, + uint64_t target_size, + int min_to_remove); +int storage_dir_remove_all(storage_dir_t *d); + +#endif + diff --git a/src/common/testsupport.h b/src/common/testsupport.h index 9ad2ba77e0..9fc96199b4 100644 --- a/src/common/testsupport.h +++ b/src/common/testsupport.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2016, The Tor Project, Inc. */ +/* Copyright (c) 2013-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_TESTSUPPORT_H diff --git a/src/common/timers.c b/src/common/timers.c index e1ad47b15b..6f4a6c30f0 100644 --- a/src/common/timers.c +++ b/src/common/timers.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/timers.h b/src/common/timers.h index c5246a3335..e816630e6d 100644 --- a/src/common/timers.h +++ b/src/common/timers.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_TIMERS_H diff --git a/src/common/torgzip.c b/src/common/torgzip.c index 04ae353cf4..223cd307a7 100644 --- a/src/common/torgzip.c +++ b/src/common/torgzip.c @@ -1,6 +1,6 @@ /* Copyright (c) 2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/torgzip.h b/src/common/torgzip.h index 00f62dcb45..430b152fe4 100644 --- a/src/common/torgzip.h +++ b/src/common/torgzip.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/torint.h b/src/common/torint.h index 58c30f41a8..ee31459e94 100644 --- a/src/common/torint.h +++ b/src/common/torint.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/torlog.h b/src/common/torlog.h index bc957858d9..6e374b1c11 100644 --- a/src/common/torlog.h +++ b/src/common/torlog.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -101,8 +101,10 @@ #define LD_SCHED (1u<<22) /** Guard nodes */ #define LD_GUARD (1u<<23) +/** Generation and application of consensus diffs. */ +#define LD_CONSDIFF (1u<<24) /** Number of logging domains in the code. */ -#define N_LOGGING_DOMAINS 24 +#define N_LOGGING_DOMAINS 25 /** This log message is not safe to send to a callback-based logger * immediately. Used as a flag, not a log domain. */ diff --git a/src/common/tortls.c b/src/common/tortls.c index 1594f3be00..3034fbcd57 100644 --- a/src/common/tortls.c +++ b/src/common/tortls.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/tortls.h b/src/common/tortls.h index 6510fdbe64..dcb82fcb23 100644 --- a/src/common/tortls.h +++ b/src/common/tortls.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_TORTLS_H diff --git a/src/common/util.c b/src/common/util.c index a8b44cbda7..93befe9d77 100644 --- a/src/common/util.c +++ b/src/common/util.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -5270,7 +5270,7 @@ tor_get_lines_from_handle, (int fd, enum stream_status *stream_status_out)) goto done; if (!lines) lines = smartlist_new(); - smartlist_add_strdup(lines, stdout_buf); + smartlist_split_string(lines, stdout_buf, "\n", 0, 0); } done: diff --git a/src/common/util.h b/src/common/util.h index a8d7978c24..181acb623f 100644 --- a/src/common/util.h +++ b/src/common/util.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/util_bug.c b/src/common/util_bug.c index c7bfdefe80..3d990e3700 100644 --- a/src/common/util_bug.c +++ b/src/common/util_bug.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/util_bug.h b/src/common/util_bug.h index e02778110b..7879f880ec 100644 --- a/src/common/util_bug.h +++ b/src/common/util_bug.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/util_format.c b/src/common/util_format.c index 6e0a04586a..51b299b1ba 100644 --- a/src/common/util_format.c +++ b/src/common/util_format.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/util_format.h b/src/common/util_format.h index 20ac711d10..5fd82d6d99 100644 --- a/src/common/util_format.h +++ b/src/common/util_format.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_UTIL_FORMAT_H diff --git a/src/common/util_process.c b/src/common/util_process.c index abda63720c..9e9679b099 100644 --- a/src/common/util_process.c +++ b/src/common/util_process.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/util_process.h b/src/common/util_process.h index d38301a354..c3a63498b5 100644 --- a/src/common/util_process.h +++ b/src/common/util_process.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2011-2016, The Tor Project, Inc. */ +/* Copyright (c) 2011-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/workqueue.c b/src/common/workqueue.c index e1fb663a2a..ec6e257b4c 100644 --- a/src/common/workqueue.c +++ b/src/common/workqueue.c @@ -510,12 +510,13 @@ replyqueue_get_socket(replyqueue_t *rq) void replyqueue_process(replyqueue_t *queue) { - if (queue->alert.drain_fn(queue->alert.read_fd) < 0) { + int r = queue->alert.drain_fn(queue->alert.read_fd); + if (r < 0) { //LCOV_EXCL_START static ratelim_t warn_limit = RATELIM_INIT(7200); log_fn_ratelim(&warn_limit, LOG_WARN, LD_GENERAL, "Failure from drain_fd: %s", - tor_socket_strerror(tor_socket_errno(queue->alert.read_fd))); + tor_socket_strerror(-r)); //LCOV_EXCL_STOP } diff --git a/src/common/workqueue.h b/src/common/workqueue.h index 54276767b0..5d14e7f6a7 100644 --- a/src/common/workqueue.h +++ b/src/common/workqueue.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2016, The Tor Project, Inc. */ +/* Copyright (c) 2013-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_WORKQUEUE_H diff --git a/src/ext/byteorder.h b/src/ext/byteorder.h new file mode 100644 index 0000000000..c8ba52184b --- /dev/null +++ b/src/ext/byteorder.h @@ -0,0 +1,67 @@ +/* <MIT License> + Copyright (c) 2013-2014 Marek Majkowski <marek@popcount.org> + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. + </MIT License> + + Original location: + https://github.com/majek/csiphash/ + + Solution inspired by code from: + Samuel Neves (supercop/crypto_auth/siphash24/little) + djb (supercop/crypto_auth/siphash24/little2) + Jean-Philippe Aumasson (https://131002.net/siphash/siphash24.c) +*/ + +/* This code is extracted from csiphash.h */ + +#if defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && \ + __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ +# define _le64toh(x) ((uint64_t)(x)) +#elif defined(_WIN32) +/* Windows is always little endian, unless you're on xbox360 + http://msdn.microsoft.com/en-us/library/b0084kay(v=vs.80).aspx */ +# define _le64toh(x) ((uint64_t)(x)) +#elif defined(__APPLE__) +# include <libkern/OSByteOrder.h> +# define _le64toh(x) OSSwapLittleToHostInt64(x) +#elif defined(sun) || defined(__sun) +# include <sys/byteorder.h> +# define _le64toh(x) LE_64(x) + +#else + +/* See: http://sourceforge.net/p/predef/wiki/Endianness/ */ +# if defined(__FreeBSD__) || defined(__NetBSD__) || defined(OpenBSD) +# include <sys/endian.h> +# else +# include <endian.h> +# endif +# if defined(__BYTE_ORDER) && defined(__LITTLE_ENDIAN) && \ + __BYTE_ORDER == __LITTLE_ENDIAN +# define _le64toh(x) ((uint64_t)(x)) +# else +# if defined(OpenBSD) +# define _le64toh(x) letoh64(x) +# else +# define _le64toh(x) le64toh(x) +# endif +# endif + +#endif diff --git a/src/ext/csiphash.c b/src/ext/csiphash.c index 8348c66048..1029bbbad3 100644 --- a/src/ext/csiphash.c +++ b/src/ext/csiphash.c @@ -35,41 +35,7 @@ #include "util.h" /* for memcpy */ #include <string.h> - -#if defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && \ - __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ -# define _le64toh(x) ((uint64_t)(x)) -#elif defined(_WIN32) -/* Windows is always little endian, unless you're on xbox360 - http://msdn.microsoft.com/en-us/library/b0084kay(v=vs.80).aspx */ -# define _le64toh(x) ((uint64_t)(x)) -#elif defined(__APPLE__) -# include <libkern/OSByteOrder.h> -# define _le64toh(x) OSSwapLittleToHostInt64(x) -#elif defined(sun) || defined(__sun) -# include <sys/byteorder.h> -# define _le64toh(x) LE_64(x) - -#else - -/* See: http://sourceforge.net/p/predef/wiki/Endianness/ */ -# if defined(__FreeBSD__) || defined(__NetBSD__) || defined(OpenBSD) -# include <sys/endian.h> -# else -# include <endian.h> -# endif -# if defined(__BYTE_ORDER) && defined(__LITTLE_ENDIAN) && \ - __BYTE_ORDER == __LITTLE_ENDIAN -# define _le64toh(x) ((uint64_t)(x)) -# else -# if defined(OpenBSD) -# define _le64toh(x) letoh64(x) -# else -# define _le64toh(x) le64toh(x) -# endif -# endif - -#endif +#include "byteorder.h" #define ROTATE(x, b) (uint64_t)( ((x) << (b)) | ( (x) >> (64 - (b))) ) diff --git a/src/ext/ht.h b/src/ext/ht.h index a441d0b685..caa420e9b5 100644 --- a/src/ext/ht.h +++ b/src/ext/ht.h @@ -1,6 +1,6 @@ /* Copyright (c) 2002, Christopher Clark. * Copyright (c) 2005-2006, Nick Mathewson. - * Copyright (c) 2007-2015, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See license at end. */ /* Based on ideas by Christopher Clark and interfaces from Niels Provos. */ diff --git a/src/ext/include.am b/src/ext/include.am index f00f3e031e..c18e58857e 100644 --- a/src/ext/include.am +++ b/src/ext/include.am @@ -5,6 +5,7 @@ EXTRA_DIST += src/ext/README EXTHEADERS = \ src/ext/ht.h \ + src/ext/byteorder.h \ src/ext/tinytest.h \ src/ext/tor_readpassphrase.h \ src/ext/strlcat.c \ diff --git a/src/ext/keccak-tiny/keccak-tiny-unrolled.c b/src/ext/keccak-tiny/keccak-tiny-unrolled.c index d1342c3601..d8d7fe335a 100644 --- a/src/ext/keccak-tiny/keccak-tiny-unrolled.c +++ b/src/ext/keccak-tiny/keccak-tiny-unrolled.c @@ -10,28 +10,21 @@ #include <string.h> #include "crypto.h" +#include "byteorder.h" /******** Endianness conversion helpers ********/ static inline uint64_t loadu64le(const unsigned char *x) { - uint64_t r = 0; - size_t i; - - for (i = 0; i < 8; ++i) { - r |= (uint64_t)x[i] << 8 * i; - } - return r; + uint64_t r; + memcpy(&r, x, sizeof(r)); + return _le64toh(r); } static inline void storeu64le(uint8_t *x, uint64_t u) { - size_t i; - - for(i=0; i<8; ++i) { - x[i] = u; - u >>= 8; - } + uint64_t val = _le64toh(u); + memcpy(x, &val, sizeof(u)); } /******** The Keccak-f[1600] permutation ********/ diff --git a/src/ext/trunnel/trunnel-impl.h b/src/ext/trunnel/trunnel-impl.h index bc805851b1..85c847b3fd 100644 --- a/src/ext/trunnel/trunnel-impl.h +++ b/src/ext/trunnel/trunnel-impl.h @@ -5,7 +5,7 @@ /* trunnel-impl.h -- Implementation helpers for trunnel, included by * generated trunnel files * - * Copyright 2014-2015, The Tor Project, Inc. + * Copyright 2014-2017, The Tor Project, Inc. * See license at the end of this file for copying information. */ diff --git a/src/ext/trunnel/trunnel.c b/src/ext/trunnel/trunnel.c index 8f72351277..e98a2e0b66 100644 --- a/src/ext/trunnel/trunnel.c +++ b/src/ext/trunnel/trunnel.c @@ -4,7 +4,7 @@ */ /* trunnel.c -- Helper functions to implement trunnel. * - * Copyright 2014-2015, The Tor Project, Inc. + * Copyright 2014-2017, The Tor Project, Inc. * See license at the end of this file for copying information. * * See trunnel-impl.h for documentation of these functions. diff --git a/src/ext/trunnel/trunnel.h b/src/ext/trunnel/trunnel.h index 85bbcc5451..dd78553c77 100644 --- a/src/ext/trunnel/trunnel.h +++ b/src/ext/trunnel/trunnel.h @@ -5,7 +5,7 @@ /* trunnel.h -- Public declarations for trunnel, to be included * in trunnel header files. - * Copyright 2014-2015, The Tor Project, Inc. + * Copyright 2014-2017, The Tor Project, Inc. * See license at the end of this file for copying information. */ diff --git a/src/or/addressmap.c b/src/or/addressmap.c index 85a6434f4a..c92af38254 100644 --- a/src/or/addressmap.c +++ b/src/or/addressmap.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/addressmap.h b/src/or/addressmap.h index 67648d0518..80f453b4c1 100644 --- a/src/or/addressmap.h +++ b/src/or/addressmap.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_ADDRESSMAP_H diff --git a/src/or/bridges.c b/src/or/bridges.c index 88154c6c8a..255e0a4f7c 100644 --- a/src/or/bridges.c +++ b/src/or/bridges.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/bridges.h b/src/or/bridges.h index 27ea5e197c..3bfc782f9a 100644 --- a/src/or/bridges.h +++ b/src/or/bridges.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/buffers.c b/src/or/buffers.c index 155b1935ed..e559f80a1e 100644 --- a/src/or/buffers.c +++ b/src/or/buffers.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/buffers.h b/src/or/buffers.h index bb53b3bbff..c6a5ffaad5 100644 --- a/src/or/buffers.h +++ b/src/or/buffers.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/channel.c b/src/or/channel.c index 45f1602ab2..e79fc0760b 100644 --- a/src/or/channel.c +++ b/src/or/channel.c @@ -1,4 +1,4 @@ -/* * Copyright (c) 2012-2016, The Tor Project, Inc. */ +/* * Copyright (c) 2012-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/channel.h b/src/or/channel.h index 26aa93b5e2..33dceb1be0 100644 --- a/src/or/channel.h +++ b/src/or/channel.h @@ -1,4 +1,4 @@ -/* * Copyright (c) 2012-2016, The Tor Project, Inc. */ +/* * Copyright (c) 2012-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/channeltls.c b/src/or/channeltls.c index dbed95fb43..9d9e7446ab 100644 --- a/src/or/channeltls.c +++ b/src/or/channeltls.c @@ -1,4 +1,4 @@ -/* * Copyright (c) 2012-2016, The Tor Project, Inc. */ +/* * Copyright (c) 2012-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/channeltls.h b/src/or/channeltls.h index 729e595615..1f9a39d8a4 100644 --- a/src/or/channeltls.h +++ b/src/or/channeltls.h @@ -1,4 +1,4 @@ -/* * Copyright (c) 2012-2016, The Tor Project, Inc. */ +/* * Copyright (c) 2012-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/circpathbias.c b/src/or/circpathbias.c index cdcb6deae4..4c0bd9e455 100644 --- a/src/or/circpathbias.c +++ b/src/or/circpathbias.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/circpathbias.h b/src/or/circpathbias.h index ce76689d5f..2a4c316807 100644 --- a/src/or/circpathbias.h +++ b/src/or/circpathbias.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/circuitbuild.c b/src/or/circuitbuild.c index 14e829b03e..f08e2ee6af 100644 --- a/src/or/circuitbuild.c +++ b/src/or/circuitbuild.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/circuitbuild.h b/src/or/circuitbuild.h index ddb070b427..62a80b2b89 100644 --- a/src/or/circuitbuild.h +++ b/src/or/circuitbuild.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/circuitlist.c b/src/or/circuitlist.c index 54a7db9dbf..738119fa18 100644 --- a/src/or/circuitlist.c +++ b/src/or/circuitlist.c @@ -1,7 +1,7 @@ /* Copyright 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/circuitlist.h b/src/or/circuitlist.h index 6abee37dc4..d647062f46 100644 --- a/src/or/circuitlist.h +++ b/src/or/circuitlist.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/circuitmux.c b/src/or/circuitmux.c index 96a3647aab..ee0d5c718b 100644 --- a/src/or/circuitmux.c +++ b/src/or/circuitmux.c @@ -1,4 +1,4 @@ -/* * Copyright (c) 2012-2016, The Tor Project, Inc. */ +/* * Copyright (c) 2012-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/circuitmux.h b/src/or/circuitmux.h index 00745ac4a1..42a46aaa47 100644 --- a/src/or/circuitmux.h +++ b/src/or/circuitmux.h @@ -1,4 +1,4 @@ -/* * Copyright (c) 2012-2016, The Tor Project, Inc. */ +/* * Copyright (c) 2012-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/circuitmux_ewma.c b/src/or/circuitmux_ewma.c index 0219459cdb..c2440b13f0 100644 --- a/src/or/circuitmux_ewma.c +++ b/src/or/circuitmux_ewma.c @@ -1,4 +1,4 @@ -/* * Copyright (c) 2012-2016, The Tor Project, Inc. */ +/* * Copyright (c) 2012-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/circuitmux_ewma.h b/src/or/circuitmux_ewma.h index a7b8961ac6..1f04408789 100644 --- a/src/or/circuitmux_ewma.h +++ b/src/or/circuitmux_ewma.h @@ -1,4 +1,4 @@ -/* * Copyright (c) 2012-2016, The Tor Project, Inc. */ +/* * Copyright (c) 2012-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/circuitstats.c b/src/or/circuitstats.c index 48cb89dc37..fe6f723bd5 100644 --- a/src/or/circuitstats.c +++ b/src/or/circuitstats.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/circuitstats.h b/src/or/circuitstats.h index 23910276ae..4977b26c0d 100644 --- a/src/or/circuitstats.h +++ b/src/or/circuitstats.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/circuituse.c b/src/or/circuituse.c index 113643a4c9..8d233e0cb6 100644 --- a/src/or/circuituse.c +++ b/src/or/circuituse.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/circuituse.h b/src/or/circuituse.h index e5f8700ea2..ad4c214a3b 100644 --- a/src/or/circuituse.h +++ b/src/or/circuituse.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/command.c b/src/or/command.c index 5866c386e4..0992e97b8b 100644 --- a/src/or/command.c +++ b/src/or/command.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/command.h b/src/or/command.h index 12cda6a463..5079d42e75 100644 --- a/src/or/command.h +++ b/src/or/command.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/config.c b/src/or/config.c index a527571cb0..7ae40053ee 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -2347,7 +2347,7 @@ print_usage(void) printf( "Copyright (c) 2001-2004, Roger Dingledine\n" "Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson\n" -"Copyright (c) 2007-2016, The Tor Project, Inc.\n\n" +"Copyright (c) 2007-2017, The Tor Project, Inc.\n\n" "tor -f <torrc> [args]\n" "See man page for options, or https://www.torproject.org/ for " "documentation.\n"); @@ -2815,7 +2815,7 @@ compute_publishserverdescriptor(or_options_t *options) #define MAX_PREDICTED_CIRCS_RELEVANCE (60*60) /** Highest allowable value for RendPostPeriod. */ -#define MAX_DIR_PERIOD (MIN_ONION_KEY_LIFETIME/2) +#define MAX_DIR_PERIOD ((7*24*60*60)/2) /** Lowest allowable value for MaxCircuitDirtiness; if this is too low, Tor * will generate too many circuits and potentially overload the network. */ diff --git a/src/or/config.h b/src/or/config.h index 6645532514..bb7802c990 100644 --- a/src/or/config.h +++ b/src/or/config.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/confparse.c b/src/or/confparse.c index 9e352aca27..ae0b3d5582 100644 --- a/src/or/confparse.c +++ b/src/or/confparse.c @@ -2,7 +2,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -779,6 +779,7 @@ config_get_assigned_option(const config_format_t *fmt, const void *options, tor_free(result); return NULL; case CONFIG_TYPE_LINELIST_S: + tor_free(result->key); tor_free(result); result = config_lines_dup_and_filter(*(const config_line_t **)value, key); diff --git a/src/or/confparse.h b/src/or/confparse.h index 8d915d266b..5db6c8e4b6 100644 --- a/src/or/confparse.h +++ b/src/or/confparse.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_CONFPARSE_H diff --git a/src/or/connection.c b/src/or/connection.c index 3c51df3df8..09e316d214 100644 --- a/src/or/connection.c +++ b/src/or/connection.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/connection.h b/src/or/connection.h index d25e002fa4..df6fc64709 100644 --- a/src/or/connection.h +++ b/src/or/connection.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/connection_edge.c b/src/or/connection_edge.c index f892dd7186..2c60d8dddb 100644 --- a/src/or/connection_edge.c +++ b/src/or/connection_edge.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/connection_edge.h b/src/or/connection_edge.h index 61b5752aed..e4780b3c7d 100644 --- a/src/or/connection_edge.h +++ b/src/or/connection_edge.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/connection_or.c b/src/or/connection_or.c index cefe42c4db..61da43e119 100644 --- a/src/or/connection_or.c +++ b/src/or/connection_or.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/connection_or.h b/src/or/connection_or.h index 80a5bddb14..40008426e9 100644 --- a/src/or/connection_or.h +++ b/src/or/connection_or.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/consdiff.c b/src/or/consdiff.c new file mode 100644 index 0000000000..d2a2af1b5f --- /dev/null +++ b/src/or/consdiff.c @@ -0,0 +1,1302 @@ +/* Copyright (c) 2014, Daniel Martà + * Copyright (c) 2014, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file consdiff.c + * \brief Consensus diff implementation, including both the generation and the + * application of diffs in a minimal ed format. + * + * The consensus diff application is done in consdiff_apply_diff, which relies + * on apply_ed_diff for the main ed diff part and on some digest helper + * functions to check the digest hashes found in the consensus diff header. + * + * The consensus diff generation is more complex. consdiff_gen_diff generates + * it, relying on gen_ed_diff to generate the ed diff and some digest helper + * functions to generate the digest hashes. + * + * gen_ed_diff is the tricky bit. In it simplest form, it will take quadratic + * time and linear space to generate an ed diff given two smartlists. As shown + * in its comment section, calling calc_changes on the entire two consensuses + * will calculate what is to be added and what is to be deleted in the diff. + * Its comment section briefly explains how it works. + * + * In our case specific to consensuses, we take advantage of the fact that + * consensuses list routers sorted by their identities. We use that + * information to avoid running calc_changes on the whole smartlists. + * gen_ed_diff will navigate through the two consensuses identity by identity + * and will send small couples of slices to calc_changes, keeping the running + * time near-linear. This is explained in more detail in the gen_ed_diff + * comments. + * + * The allocation strategy tries to save time and memory by avoiding needless + * copies. Instead of actually splitting the inputs into separate strings, we + * allocate cdline_t objects, each of which represents a line in the original + * object or in the output. We use memarea_t allocators to manage the + * temporary memory we use when generating or applying diffs. + **/ + +#define CONSDIFF_PRIVATE + +#include "or.h" +#include "consdiff.h" +#include "memarea.h" +#include "routerparse.h" + +static const char* ns_diff_version = "network-status-diff-version 1"; +static const char* hash_token = "hash"; + +static char *consensus_join_lines(const smartlist_t *inp); + +/** Return true iff a and b have the same contents. */ +STATIC int +lines_eq(const cdline_t *a, const cdline_t *b) +{ + return a->len == b->len && fast_memeq(a->s, b->s, a->len); +} + +/** Return true iff a has the same contents as the nul-terminated string b. */ +STATIC int +line_str_eq(const cdline_t *a, const char *b) +{ + const size_t len = strlen(b); + tor_assert(len <= UINT32_MAX); + cdline_t bline = { b, (uint32_t)len }; + return lines_eq(a, &bline); +} + +/** Add a cdline_t to <b>lst</b> holding as its contents the nul-terminated + * string s. Use the provided memory area for storage. */ +STATIC void +smartlist_add_linecpy(smartlist_t *lst, memarea_t *area, const char *s) +{ + size_t len = strlen(s); + const char *ss = memarea_memdup(area, s, len); + cdline_t *line = memarea_alloc(area, sizeof(cdline_t)); + line->s = ss; + line->len = (uint32_t)len; + smartlist_add(lst, line); +} + +/** Compute the digest of <b>cons</b>, and store the result in + * <b>digest_out</b>. Return 0 on success, -1 on failure. */ +/* This is a separate, mockable function so that we can override it when + * fuzzing. */ +MOCK_IMPL(STATIC int, +consensus_compute_digest,(const char *cons, + consensus_digest_t *digest_out)) +{ + int r = crypto_digest256((char*)digest_out->sha3_256, + cons, strlen(cons), DIGEST_SHA3_256); + return r; +} + +/** Return true iff <b>d1</b> and <b>d2</b> contain the same digest */ +/* This is a separate, mockable function so that we can override it when + * fuzzing. */ +MOCK_IMPL(STATIC int, +consensus_digest_eq,(const uint8_t *d1, + const uint8_t *d2)) +{ + return fast_memeq(d1, d2, DIGEST256_LEN); +} + +/** Create (allocate) a new slice from a smartlist. Assumes that the start + * and the end indexes are within the bounds of the initial smartlist. The end + * element is not part of the resulting slice. If end is -1, the slice is to + * reach the end of the smartlist. + */ +STATIC smartlist_slice_t * +smartlist_slice(const smartlist_t *list, int start, int end) +{ + int list_len = smartlist_len(list); + tor_assert(start >= 0); + tor_assert(start <= list_len); + if (end == -1) { + end = list_len; + } + tor_assert(start <= end); + + smartlist_slice_t *slice = tor_malloc(sizeof(smartlist_slice_t)); + slice->list = list; + slice->offset = start; + slice->len = end - start; + return slice; +} + +/** Helper: Compute the longest common subsequence lengths for the two slices. + * Used as part of the diff generation to find the column at which to split + * slice2 while still having the optimal solution. + * If direction is -1, the navigation is reversed. Otherwise it must be 1. + * The length of the resulting integer array is that of the second slice plus + * one. + */ +STATIC int * +lcs_lengths(const smartlist_slice_t *slice1, const smartlist_slice_t *slice2, + int direction) +{ + size_t a_size = sizeof(int) * (slice2->len+1); + + /* Resulting lcs lengths. */ + int *result = tor_malloc_zero(a_size); + /* Copy of the lcs lengths from the last iteration. */ + int *prev = tor_malloc(a_size); + + tor_assert(direction == 1 || direction == -1); + + int si = slice1->offset; + if (direction == -1) { + si += (slice1->len-1); + } + + for (int i = 0; i < slice1->len; ++i, si+=direction) { + + const cdline_t *line1 = smartlist_get(slice1->list, si); + /* Store the last results. */ + memcpy(prev, result, a_size); + + int sj = slice2->offset; + if (direction == -1) { + sj += (slice2->len-1); + } + + for (int j = 0; j < slice2->len; ++j, sj+=direction) { + + const cdline_t *line2 = smartlist_get(slice2->list, sj); + if (lines_eq(line1, line2)) { + /* If the lines are equal, the lcs is one line longer. */ + result[j + 1] = prev[j] + 1; + } else { + /* If not, see what lcs parent path is longer. */ + result[j + 1] = MAX(result[j], prev[j + 1]); + } + } + } + tor_free(prev); + return result; +} + +/** Helper: Trim any number of lines that are equally at the start or the end + * of both slices. + */ +STATIC void +trim_slices(smartlist_slice_t *slice1, smartlist_slice_t *slice2) +{ + while (slice1->len>0 && slice2->len>0) { + const cdline_t *line1 = smartlist_get(slice1->list, slice1->offset); + const cdline_t *line2 = smartlist_get(slice2->list, slice2->offset); + if (!lines_eq(line1, line2)) { + break; + } + slice1->offset++; slice1->len--; + slice2->offset++; slice2->len--; + } + + int i1 = (slice1->offset+slice1->len)-1; + int i2 = (slice2->offset+slice2->len)-1; + + while (slice1->len>0 && slice2->len>0) { + const cdline_t *line1 = smartlist_get(slice1->list, i1); + const cdline_t *line2 = smartlist_get(slice2->list, i2); + if (!lines_eq(line1, line2)) { + break; + } + i1--; + slice1->len--; + i2--; + slice2->len--; + } +} + +/** Like smartlist_string_pos, but uses a cdline_t, and is restricted to the + * bounds of the slice. + */ +STATIC int +smartlist_slice_string_pos(const smartlist_slice_t *slice, + const cdline_t *string) +{ + int end = slice->offset + slice->len; + for (int i = slice->offset; i < end; ++i) { + const cdline_t *el = smartlist_get(slice->list, i); + if (lines_eq(el, string)) { + return i; + } + } + return -1; +} + +/** Helper: Set all the appropriate changed booleans to true. The first slice + * must be of length 0 or 1. All the lines of slice1 and slice2 which are not + * present in the other slice will be set to changed in their bool array. + * The two changed bool arrays are passed in the same order as the slices. + */ +STATIC void +set_changed(bitarray_t *changed1, bitarray_t *changed2, + const smartlist_slice_t *slice1, const smartlist_slice_t *slice2) +{ + int toskip = -1; + tor_assert(slice1->len == 0 || slice1->len == 1); + + if (slice1->len == 1) { + const cdline_t *line_common = smartlist_get(slice1->list, slice1->offset); + toskip = smartlist_slice_string_pos(slice2, line_common); + if (toskip == -1) { + bitarray_set(changed1, slice1->offset); + } + } + int end = slice2->offset + slice2->len; + for (int i = slice2->offset; i < end; ++i) { + if (i != toskip) { + bitarray_set(changed2, i); + } + } +} + +/* + * Helper: Given that slice1 has been split by half into top and bot, we want + * to fetch the column at which to split slice2 so that we are still on track + * to the optimal diff solution, i.e. the shortest one. We use lcs_lengths + * since the shortest diff is just another way to say the longest common + * subsequence. + */ +static int +optimal_column_to_split(const smartlist_slice_t *top, + const smartlist_slice_t *bot, + const smartlist_slice_t *slice2) +{ + int *lens_top = lcs_lengths(top, slice2, 1); + int *lens_bot = lcs_lengths(bot, slice2, -1); + int column=0, max_sum=-1; + + for (int i = 0; i < slice2->len+1; ++i) { + int sum = lens_top[i] + lens_bot[slice2->len-i]; + if (sum > max_sum) { + column = i; + max_sum = sum; + } + } + tor_free(lens_top); + tor_free(lens_bot); + + return column; +} + +/** + * Helper: Figure out what elements are new or gone on the second smartlist + * relative to the first smartlist, and store the booleans in the bitarrays. + * True on the first bitarray means the element is gone, true on the second + * bitarray means it's new. + * + * In its base case, either of the smartlists is of length <= 1 and we can + * quickly see what elements are new or are gone. In the other case, we will + * split one smartlist by half and we'll use optimal_column_to_split to find + * the optimal column at which to split the second smartlist so that we are + * finding the smallest diff possible. + */ +STATIC void +calc_changes(smartlist_slice_t *slice1, + smartlist_slice_t *slice2, + bitarray_t *changed1, bitarray_t *changed2) +{ + trim_slices(slice1, slice2); + + if (slice1->len <= 1) { + set_changed(changed1, changed2, slice1, slice2); + + } else if (slice2->len <= 1) { + set_changed(changed2, changed1, slice2, slice1); + + /* Keep on splitting the slices in two. */ + } else { + smartlist_slice_t *top, *bot, *left, *right; + + /* Split the first slice in half. */ + int mid = slice1->len/2; + top = smartlist_slice(slice1->list, slice1->offset, slice1->offset+mid); + bot = smartlist_slice(slice1->list, slice1->offset+mid, + slice1->offset+slice1->len); + + /* Split the second slice by the optimal column. */ + int mid2 = optimal_column_to_split(top, bot, slice2); + left = smartlist_slice(slice2->list, slice2->offset, slice2->offset+mid2); + right = smartlist_slice(slice2->list, slice2->offset+mid2, + slice2->offset+slice2->len); + + calc_changes(top, left, changed1, changed2); + calc_changes(bot, right, changed1, changed2); + tor_free(top); + tor_free(bot); + tor_free(left); + tor_free(right); + } +} + +/* This table is from crypto.c. The SP and PAD defines are different. */ +#define NOT_VALID_BASE64 255 +#define X NOT_VALID_BASE64 +#define SP NOT_VALID_BASE64 +#define PAD NOT_VALID_BASE64 +static const uint8_t base64_compare_table[256] = { + X, X, X, X, X, X, X, X, X, SP, SP, SP, X, SP, X, X, + X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, + SP, X, X, X, X, X, X, X, X, X, X, 62, X, X, X, 63, + 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, X, X, X, PAD, X, X, + X, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, X, X, X, X, X, + X, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, + 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, X, X, X, X, X, + X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, + X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, + X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, + X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, + X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, + X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, + X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, + X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, +}; + +/** Helper: Get the identity hash from a router line, assuming that the line + * at least appears to be a router line and thus starts with "r ". + * + * If an identity hash is found, store it (without decoding it) in + * <b>hash_out</b>, and return 0. On failure, return -1. + */ +STATIC int +get_id_hash(const cdline_t *line, cdline_t *hash_out) +{ + if (line->len < 2) + return -1; + + /* Skip the router name. */ + const char *hash = memchr(line->s + 2, ' ', line->len - 2); + if (!hash) { + return -1; + } + + hash++; + const char *hash_end = hash; + /* Stop when the first non-base64 character is found. Use unsigned chars to + * avoid negative indexes causing crashes. + */ + while (base64_compare_table[*((unsigned char*)hash_end)] + != NOT_VALID_BASE64 && + hash_end < line->s + line->len) { + hash_end++; + } + + /* Empty hash. */ + if (hash_end == hash) { + return -1; + } + + hash_out->s = hash; + /* Always true because lines are limited to this length */ + tor_assert(hash_end >= hash); + tor_assert((size_t)(hash_end - hash) <= UINT32_MAX); + hash_out->len = (uint32_t)(hash_end - hash); + + return 0; +} + +/** Helper: Check that a line is a valid router entry. We must at least be + * able to fetch a proper identity hash from it for it to be valid. + */ +STATIC int +is_valid_router_entry(const cdline_t *line) +{ + if (line->len < 2 || fast_memneq(line->s, "r ", 2)) + return 0; + cdline_t tmp; + return (get_id_hash(line, &tmp) == 0); +} + +/** Helper: Find the next router line starting at the current position. + * Assumes that cur is lower than the length of the smartlist, i.e. it is a + * line within the bounds of the consensus. The only exception is when we + * don't want to skip the first line, in which case cur will be -1. + */ +STATIC int +next_router(const smartlist_t *cons, int cur) +{ + int len = smartlist_len(cons); + tor_assert(cur >= -1 && cur < len); + + if (++cur >= len) { + return len; + } + + const cdline_t *line = smartlist_get(cons, cur); + while (!is_valid_router_entry(line)) { + if (++cur >= len) { + return len; + } + line = smartlist_get(cons, cur); + } + return cur; +} + +/** Helper: compare two base64-encoded identity hashes, which may be of + * different lengths. Comparison ends when the first non-base64 char is found. + */ +STATIC int +base64cmp(const cdline_t *hash1, const cdline_t *hash2) +{ + /* NULL is always lower, useful for last_hash which starts at NULL. */ + if (!hash1->s && !hash2->s) { + return 0; + } + if (!hash1->s) { + return -1; + } + if (!hash2->s) { + return 1; + } + + /* Don't index with a char; char may be signed. */ + const unsigned char *a = (unsigned char*)hash1->s; + const unsigned char *b = (unsigned char*)hash2->s; + const unsigned char *a_end = a + hash1->len; + const unsigned char *b_end = b + hash2->len; + while (1) { + uint8_t av = base64_compare_table[*a]; + uint8_t bv = base64_compare_table[*b]; + if (av == NOT_VALID_BASE64) { + if (bv == NOT_VALID_BASE64) { + /* Both ended with exactly the same characters. */ + return 0; + } else { + /* hash2 goes on longer than hash1 and thus hash1 is lower. */ + return -1; + } + } else if (bv == NOT_VALID_BASE64) { + /* hash1 goes on longer than hash2 and thus hash1 is greater. */ + return 1; + } else if (av < bv) { + /* The first difference shows that hash1 is lower. */ + return -1; + } else if (av > bv) { + /* The first difference shows that hash1 is greater. */ + return 1; + } else { + a++; + b++; + if (a == a_end) { + if (b == b_end) { + return 0; + } else { + return -1; + } + } else if (b == b_end) { + return 1; + } + } + } +} + +/** Structure used to remember the previous and current identity hash of + * the "r " lines in a consensus, to enforce well-ordering. */ +typedef struct router_id_iterator_t { + cdline_t last_hash; + cdline_t hash; +} router_id_iterator_t; + +/** + * Initializer for a router_id_iterator_t. + */ +#define ROUTER_ID_ITERATOR_INIT { { NULL, 0 }, { NULL, 0 } } + +/** Given an index *<b>idxp</b> into the consensus at <b>cons</b>, advance + * the index to the next router line ("r ...") in the consensus, or to + * an index one after the end of the list if there is no such line. + * + * Use <b>iter</b> to record the hash of the found router line, if any, + * and to enforce ordering on the hashes. If the hashes are mis-ordered, + * return -1. Else, return 0. + **/ +static int +find_next_router_line(const smartlist_t *cons, + const char *consname, + int *idxp, + router_id_iterator_t *iter) +{ + *idxp = next_router(cons, *idxp); + if (*idxp < smartlist_len(cons)) { + memcpy(&iter->last_hash, &iter->hash, sizeof(cdline_t)); + if (get_id_hash(smartlist_get(cons, *idxp), &iter->hash) < 0 || + base64cmp(&iter->hash, &iter->last_hash) <= 0) { + log_warn(LD_CONSDIFF, "Refusing to generate consensus diff because " + "the %s consensus doesn't have its router entries sorted " + "properly.", consname); + return -1; + } + } + return 0; +} + +/** Generate an ed diff as a smartlist from two consensuses, also given as + * smartlists. Will return NULL if the diff could not be generated, which can + * happen if any lines the script had to add matched "." or if the routers + * were not properly ordered. + * + * All cdline_t objects in the resulting object are either references to lines + * in one of the inputs, or are newly allocated lines in the provided memarea. + * + * This implementation is consensus-specific. To generate an ed diff for any + * given input in quadratic time, you can replace all the code until the + * navigation in reverse order with the following: + * + * int len1 = smartlist_len(cons1); + * int len2 = smartlist_len(cons2); + * bitarray_t *changed1 = bitarray_init_zero(len1); + * bitarray_t *changed2 = bitarray_init_zero(len2); + * cons1_sl = smartlist_slice(cons1, 0, -1); + * cons2_sl = smartlist_slice(cons2, 0, -1); + * calc_changes(cons1_sl, cons2_sl, changed1, changed2); + */ +STATIC smartlist_t * +gen_ed_diff(const smartlist_t *cons1, const smartlist_t *cons2, + memarea_t *area) +{ + int len1 = smartlist_len(cons1); + int len2 = smartlist_len(cons2); + smartlist_t *result = smartlist_new(); + + /* Initialize the changed bitarrays to zero, so that calc_changes only needs + * to set the ones that matter and leave the rest untouched. + */ + bitarray_t *changed1 = bitarray_init_zero(len1); + bitarray_t *changed2 = bitarray_init_zero(len2); + int i1=-1, i2=-1; + int start1=0, start2=0; + + /* To check that hashes are ordered properly */ + router_id_iterator_t iter1 = ROUTER_ID_ITERATOR_INIT; + router_id_iterator_t iter2 = ROUTER_ID_ITERATOR_INIT; + + /* i1 and i2 are initialized at the first line of each consensus. They never + * reach past len1 and len2 respectively, since next_router doesn't let that + * happen. i1 and i2 are advanced by at least one line at each iteration as + * long as they have not yet reached len1 and len2, so the loop is + * guaranteed to end, and each pair of (i1,i2) will be inspected at most + * once. + */ + while (i1 < len1 || i2 < len2) { + + /* Advance each of the two navigation positions by one router entry if not + * yet at the end. + */ + if (i1 < len1) { + if (find_next_router_line(cons1, "base", &i1, &iter1) < 0) { + goto error_cleanup; + } + } + + if (i2 < len2) { + if (find_next_router_line(cons2, "target", &i2, &iter2) < 0) { + goto error_cleanup; + } + } + + /* If we have reached the end of both consensuses, there is no need to + * compare hashes anymore, since this is the last iteration. + */ + if (i1 < len1 || i2 < len2) { + + /* Keep on advancing the lower (by identity hash sorting) position until + * we have two matching positions. The only other possible outcome is + * that a lower position reaches the end of the consensus before it can + * reach a hash that is no longer the lower one. Since there will always + * be a lower hash for as long as the loop runs, one of the two indexes + * will always be incremented, thus assuring that the loop must end + * after a finite number of iterations. If that cannot be because said + * consensus has already reached the end, both are extended to their + * respecting ends since we are done. + */ + int cmp = base64cmp(&iter1.hash, &iter2.hash); + while (cmp != 0) { + if (i1 < len1 && cmp < 0) { + if (find_next_router_line(cons1, "base", &i1, &iter1) < 0) { + goto error_cleanup; + } + if (i1 == len1) { + /* We finished the first consensus, so grab all the remaining + * lines of the second consensus and finish up. + */ + i2 = len2; + break; + } + } else if (i2 < len2 && cmp > 0) { + if (find_next_router_line(cons2, "target", &i2, &iter2) < 0) { + goto error_cleanup; + } + if (i2 == len2) { + /* We finished the second consensus, so grab all the remaining + * lines of the first consensus and finish up. + */ + i1 = len1; + break; + } + } else { + i1 = len1; + i2 = len2; + break; + } + cmp = base64cmp(&iter1.hash, &iter2.hash); + } + } + + /* Make slices out of these chunks (up to the common router entry) and + * calculate the changes for them. + * Error if any of the two slices are longer than 10K lines. That should + * never happen with any pair of real consensuses. Feeding more than 10K + * lines to calc_changes would be very slow anyway. + */ +#define MAX_LINE_COUNT (10000) + if (i1-start1 > MAX_LINE_COUNT || i2-start2 > MAX_LINE_COUNT) { + log_warn(LD_CONSDIFF, "Refusing to generate consensus diff because " + "we found too few common router ids."); + goto error_cleanup; + } + + smartlist_slice_t *cons1_sl = smartlist_slice(cons1, start1, i1); + smartlist_slice_t *cons2_sl = smartlist_slice(cons2, start2, i2); + calc_changes(cons1_sl, cons2_sl, changed1, changed2); + tor_free(cons1_sl); + tor_free(cons2_sl); + start1 = i1, start2 = i2; + } + + /* Navigate the changes in reverse order and generate one ed command for + * each chunk of changes. + */ + i1=len1-1, i2=len2-1; + char buf[128]; + while (i1 > 0 || i2 > 0) { + + int start1x, start2x, end1, end2, added, deleted; + + /* We are at a point were no changed bools are true, so just keep going. */ + if (!(i1 >= 0 && bitarray_is_set(changed1, i1)) && + !(i2 >= 0 && bitarray_is_set(changed2, i2))) { + if (i1 >= 0) { + i1--; + } + if (i2 >= 0) { + i2--; + } + continue; + } + + end1 = i1, end2 = i2; + + /* Grab all contiguous changed lines */ + while (i1 >= 0 && bitarray_is_set(changed1, i1)) { + i1--; + } + while (i2 >= 0 && bitarray_is_set(changed2, i2)) { + i2--; + } + + start1x = i1+1, start2x = i2+1; + added = end2-i2, deleted = end1-i1; + + if (added == 0) { + if (deleted == 1) { + tor_snprintf(buf, sizeof(buf), "%id", start1x+1); + smartlist_add_linecpy(result, area, buf); + } else { + tor_snprintf(buf, sizeof(buf), "%i,%id", start1x+1, start1x+deleted); + smartlist_add_linecpy(result, area, buf); + } + } else { + int i; + if (deleted == 0) { + tor_snprintf(buf, sizeof(buf), "%ia", start1x); + smartlist_add_linecpy(result, area, buf); + } else if (deleted == 1) { + tor_snprintf(buf, sizeof(buf), "%ic", start1x+1); + smartlist_add_linecpy(result, area, buf); + } else { + tor_snprintf(buf, sizeof(buf), "%i,%ic", start1x+1, start1x+deleted); + smartlist_add_linecpy(result, area, buf); + } + + for (i = start2x; i <= end2; ++i) { + cdline_t *line = smartlist_get(cons2, i); + if (line_str_eq(line, ".")) { + log_warn(LD_CONSDIFF, "Cannot generate consensus diff because " + "one of the lines to be added is \".\"."); + goto error_cleanup; + } + smartlist_add(result, line); + } + smartlist_add_linecpy(result, area, "."); + } + } + + bitarray_free(changed1); + bitarray_free(changed2); + + return result; + + error_cleanup: + + bitarray_free(changed1); + bitarray_free(changed2); + + smartlist_free(result); + + return NULL; +} + +/* Helper: Read a base-10 number between 0 and INT32_MAX from <b>s</b> and + * store it in <b>num_out</b>. Advance <b>s</b> to the characer immediately + * after the number. Return 0 on success, -1 on failure. */ +static int +get_linenum(const char **s, int *num_out) +{ + int ok; + char *next; + *num_out = (int) tor_parse_long(*s, 10, 0, INT32_MAX, &ok, &next); + if (ok && next) { + *s = next; + return 0; + } else { + return -1; + } +} + +/** Apply the ed diff, starting at <b>diff_starting_line</b>, to the consensus + * and return a new consensus, also as a line-based smartlist. Will return + * NULL if the ed diff is not properly formatted. + * + * All cdline_t objects in the resulting object are references to lines + * in one of the inputs; nothing is copied. + */ +STATIC smartlist_t * +apply_ed_diff(const smartlist_t *cons1, const smartlist_t *diff, + int diff_starting_line) +{ + int diff_len = smartlist_len(diff); + int j = smartlist_len(cons1); + smartlist_t *cons2 = smartlist_new(); + + for (int i=diff_starting_line; i<diff_len; ++i) { + const cdline_t *diff_cdline = smartlist_get(diff, i); + char diff_line[128]; + + if (diff_cdline->len > sizeof(diff_line) - 1) { + log_warn(LD_CONSDIFF, "Could not apply consensus diff because " + "an ed command was far too long"); + goto error_cleanup; + } + /* Copy the line to make it nul-terminated. */ + memcpy(diff_line, diff_cdline->s, diff_cdline->len); + diff_line[diff_cdline->len] = 0; + const char *ptr = diff_line; + int start = 0, end = 0; + if (get_linenum(&ptr, &start) < 0) { + log_warn(LD_CONSDIFF, "Could not apply consensus diff because " + "an ed command was missing a line number."); + goto error_cleanup; + } + if (*ptr == ',') { + /* Two-item range */ + ++ptr; + if (get_linenum(&ptr, &end) < 0) { + log_warn(LD_CONSDIFF, "Could not apply consensus diff because " + "an ed command was missing a range end line number."); + goto error_cleanup; + } + /* Incoherent range. */ + if (end <= start) { + log_warn(LD_CONSDIFF, "Could not apply consensus diff because " + "an invalid range was found in an ed command."); + goto error_cleanup; + } + } else { + /* We'll take <n1> as <n1>,<n1> for simplicity. */ + end = start; + } + + if (end > j) { + log_warn(LD_CONSDIFF, "Could not apply consensus diff because " + "its commands are not properly sorted in reverse order."); + goto error_cleanup; + } + + if (*ptr == '\0') { + log_warn(LD_CONSDIFF, "Could not apply consensus diff because " + "a line with no ed command was found"); + goto error_cleanup; + } + + if (*(ptr+1) != '\0') { + log_warn(LD_CONSDIFF, "Could not apply consensus diff because " + "an ed command longer than one char was found."); + goto error_cleanup; + } + + char action = *ptr; + + switch (action) { + case 'a': + case 'c': + case 'd': + break; + default: + log_warn(LD_CONSDIFF, "Could not apply consensus diff because " + "an unrecognised ed command was found."); + goto error_cleanup; + } + + /* Add unchanged lines. */ + for (; j && j > end; --j) { + cdline_t *cons_line = smartlist_get(cons1, j-1); + smartlist_add(cons2, cons_line); + } + + /* Ignore removed lines. */ + if (action == 'c' || action == 'd') { + while (--j >= start) { + /* Skip line */ + } + } + + /* Add new lines in reverse order, since it will all be reversed at the + * end. + */ + if (action == 'a' || action == 'c') { + int added_end = i; + + i++; /* Skip the line with the range and command. */ + while (i < diff_len) { + if (line_str_eq(smartlist_get(diff, i), ".")) { + break; + } + if (++i == diff_len) { + log_warn(LD_CONSDIFF, "Could not apply consensus diff because " + "it has lines to be inserted that don't end with a \".\"."); + goto error_cleanup; + } + } + + int added_i = i-1; + + /* It would make no sense to add zero new lines. */ + if (added_i == added_end) { + log_warn(LD_CONSDIFF, "Could not apply consensus diff because " + "it has an ed command that tries to insert zero lines."); + goto error_cleanup; + } + + while (added_i > added_end) { + cdline_t *added_line = smartlist_get(diff, added_i--); + smartlist_add(cons2, added_line); + } + } + } + + /* Add remaining unchanged lines. */ + for (; j > 0; --j) { + cdline_t *cons_line = smartlist_get(cons1, j-1); + smartlist_add(cons2, cons_line); + } + + /* Reverse the whole thing since we did it from the end. */ + smartlist_reverse(cons2); + return cons2; + + error_cleanup: + + smartlist_free(cons2); + + return NULL; +} + +/** Generate a consensus diff as a smartlist from two given consensuses, also + * as smartlists. Will return NULL if the consensus diff could not be + * generated. Neither of the two consensuses are modified in any way, so it's + * up to the caller to free their resources. + */ +smartlist_t * +consdiff_gen_diff(const smartlist_t *cons1, + const smartlist_t *cons2, + const consensus_digest_t *digests1, + const consensus_digest_t *digests2, + memarea_t *area) +{ + smartlist_t *ed_diff = gen_ed_diff(cons1, cons2, area); + /* ed diff could not be generated - reason already logged by gen_ed_diff. */ + if (!ed_diff) { + goto error_cleanup; + } + + /* See that the script actually produces what we want. */ + smartlist_t *ed_cons2 = apply_ed_diff(cons1, ed_diff, 0); + if (!ed_cons2) { + /* LCOV_EXCL_START -- impossible if diff generation is correct */ + log_warn(LD_BUG|LD_CONSDIFF, "Refusing to generate consensus diff because " + "the generated ed diff could not be tested to successfully generate " + "the target consensus."); + goto error_cleanup; + /* LCOV_EXCL_STOP */ + } + + int cons2_eq = 1; + if (smartlist_len(cons2) == smartlist_len(ed_cons2)) { + SMARTLIST_FOREACH_BEGIN(cons2, const cdline_t *, line1) { + const cdline_t *line2 = smartlist_get(ed_cons2, line1_sl_idx); + if (! lines_eq(line1, line2) ) { + cons2_eq = 0; + break; + } + } SMARTLIST_FOREACH_END(line1); + } else { + cons2_eq = 0; + } + smartlist_free(ed_cons2); + if (!cons2_eq) { + /* LCOV_EXCL_START -- impossible if diff generation is correct. */ + log_warn(LD_BUG|LD_CONSDIFF, "Refusing to generate consensus diff because " + "the generated ed diff did not generate the target consensus " + "successfully when tested."); + goto error_cleanup; + /* LCOV_EXCL_STOP */ + } + + char cons1_hash_hex[HEX_DIGEST256_LEN+1]; + char cons2_hash_hex[HEX_DIGEST256_LEN+1]; + base16_encode(cons1_hash_hex, HEX_DIGEST256_LEN+1, + (const char*)digests1->sha3_256, DIGEST256_LEN); + base16_encode(cons2_hash_hex, HEX_DIGEST256_LEN+1, + (const char*)digests2->sha3_256, DIGEST256_LEN); + + /* Create the resulting consensus diff. */ + char buf[160]; + smartlist_t *result = smartlist_new(); + tor_snprintf(buf, sizeof(buf), "%s", ns_diff_version); + smartlist_add_linecpy(result, area, buf); + tor_snprintf(buf, sizeof(buf), "%s %s %s", hash_token, + cons1_hash_hex, cons2_hash_hex); + smartlist_add_linecpy(result, area, buf); + smartlist_add_all(result, ed_diff); + smartlist_free(ed_diff); + return result; + + error_cleanup: + + if (ed_diff) { + /* LCOV_EXCL_START -- ed_diff is NULL except in unreachable cases above */ + smartlist_free(ed_diff); + /* LCOV_EXCL_STOP */ + } + + return NULL; +} + +/** Fetch the digest of the base consensus in the consensus diff, encoded in + * base16 as found in the diff itself. digest1_out and digest2_out must be of + * length DIGEST256_LEN or larger if not NULL. + */ +int +consdiff_get_digests(const smartlist_t *diff, + char *digest1_out, + char *digest2_out) +{ + smartlist_t *hash_words = NULL; + const cdline_t *format; + char cons1_hash[DIGEST256_LEN], cons2_hash[DIGEST256_LEN]; + char *cons1_hash_hex, *cons2_hash_hex; + if (smartlist_len(diff) < 2) { + log_info(LD_CONSDIFF, "The provided consensus diff is too short."); + goto error_cleanup; + } + + /* Check that it's the format and version we know. */ + format = smartlist_get(diff, 0); + if (!line_str_eq(format, ns_diff_version)) { + log_warn(LD_CONSDIFF, "The provided consensus diff format is not known."); + goto error_cleanup; + } + + /* Grab the base16 digests. */ + hash_words = smartlist_new(); + { + const cdline_t *line2 = smartlist_get(diff, 1); + char *h = tor_memdup_nulterm(line2->s, line2->len); + smartlist_split_string(hash_words, h, " ", 0, 0); + tor_free(h); + } + + /* There have to be three words, the first of which must be hash_token. */ + if (smartlist_len(hash_words) != 3 || + strcmp(smartlist_get(hash_words, 0), hash_token)) { + log_info(LD_CONSDIFF, "The provided consensus diff does not include " + "the necessary digests."); + goto error_cleanup; + } + + /* Expected hashes as found in the consensus diff header. They must be of + * length HEX_DIGEST256_LEN, normally 64 hexadecimal characters. + * If any of the decodings fail, error to make sure that the hashes are + * proper base16-encoded digests. + */ + cons1_hash_hex = smartlist_get(hash_words, 1); + cons2_hash_hex = smartlist_get(hash_words, 2); + if (strlen(cons1_hash_hex) != HEX_DIGEST256_LEN || + strlen(cons2_hash_hex) != HEX_DIGEST256_LEN) { + log_info(LD_CONSDIFF, "The provided consensus diff includes " + "base16-encoded digests of incorrect size."); + goto error_cleanup; + } + + if (base16_decode(cons1_hash, DIGEST256_LEN, + cons1_hash_hex, HEX_DIGEST256_LEN) != DIGEST256_LEN || + base16_decode(cons2_hash, DIGEST256_LEN, + cons2_hash_hex, HEX_DIGEST256_LEN) != DIGEST256_LEN) { + log_info(LD_CONSDIFF, "The provided consensus diff includes " + "malformed digests."); + goto error_cleanup; + } + + if (digest1_out) { + memcpy(digest1_out, cons1_hash, DIGEST256_LEN); + } + if (digest2_out) { + memcpy(digest2_out, cons2_hash, DIGEST256_LEN); + } + + SMARTLIST_FOREACH(hash_words, char *, cp, tor_free(cp)); + smartlist_free(hash_words); + return 0; + + error_cleanup: + + if (hash_words) { + SMARTLIST_FOREACH(hash_words, char *, cp, tor_free(cp)); + smartlist_free(hash_words); + } + return 1; +} + +/** Apply the consensus diff to the given consensus and return a new + * consensus, also as a line-based smartlist. Will return NULL if the diff + * could not be applied. Neither the consensus nor the diff are modified in + * any way, so it's up to the caller to free their resources. + */ +char * +consdiff_apply_diff(const smartlist_t *cons1, + const smartlist_t *diff, + const consensus_digest_t *digests1) +{ + smartlist_t *cons2 = NULL; + char *cons2_str = NULL; + char e_cons1_hash[DIGEST256_LEN]; + char e_cons2_hash[DIGEST256_LEN]; + + if (consdiff_get_digests(diff, e_cons1_hash, e_cons2_hash) != 0) { + goto error_cleanup; + } + + /* See that the consensus that was given to us matches its hash. */ + if (!consensus_digest_eq(digests1->sha3_256, + (const uint8_t*)e_cons1_hash)) { + char hex_digest1[HEX_DIGEST256_LEN+1]; + char e_hex_digest1[HEX_DIGEST256_LEN+1]; + log_warn(LD_CONSDIFF, "Refusing to apply consensus diff because " + "the base consensus doesn't match the digest as found in " + "the consensus diff header."); + base16_encode(hex_digest1, HEX_DIGEST256_LEN+1, + (const char *)digests1->sha3_256, DIGEST256_LEN); + base16_encode(e_hex_digest1, HEX_DIGEST256_LEN+1, + e_cons1_hash, DIGEST256_LEN); + log_warn(LD_CONSDIFF, "Expected: %s; found: %s", + hex_digest1, e_hex_digest1); + goto error_cleanup; + } + + /* Grab the ed diff and calculate the resulting consensus. */ + /* Skip the first two lines. */ + cons2 = apply_ed_diff(cons1, diff, 2); + + /* ed diff could not be applied - reason already logged by apply_ed_diff. */ + if (!cons2) { + goto error_cleanup; + } + + cons2_str = consensus_join_lines(cons2); + + consensus_digest_t cons2_digests; + if (consensus_compute_digest(cons2_str, &cons2_digests) < 0) { + /* LCOV_EXCL_START -- digest can't fail */ + log_warn(LD_CONSDIFF, "Could not compute digests of the consensus " + "resulting from applying a consensus diff."); + goto error_cleanup; + /* LCOV_EXCL_STOP */ + } + + /* See that the resulting consensus matches its hash. */ + if (!consensus_digest_eq(cons2_digests.sha3_256, + (const uint8_t*)e_cons2_hash)) { + log_warn(LD_CONSDIFF, "Refusing to apply consensus diff because " + "the resulting consensus doesn't match the digest as found in " + "the consensus diff header."); + char hex_digest2[HEX_DIGEST256_LEN+1]; + char e_hex_digest2[HEX_DIGEST256_LEN+1]; + base16_encode(hex_digest2, HEX_DIGEST256_LEN+1, + (const char *)cons2_digests.sha3_256, DIGEST256_LEN); + base16_encode(e_hex_digest2, HEX_DIGEST256_LEN+1, + e_cons2_hash, DIGEST256_LEN); + log_warn(LD_CONSDIFF, "Expected: %s; found: %s", + hex_digest2, e_hex_digest2); + goto error_cleanup; + } + + goto done; + + error_cleanup: + tor_free(cons2_str); /* Sets it to NULL */ + + done: + if (cons2) { + smartlist_free(cons2); + } + + return cons2_str; +} + +/** Any consensus line longer than this means that the input is invalid. */ +#define CONSENSUS_LINE_MAX_LEN (1<<20) + +/** + * Helper: For every NL-terminated line in <b>s</b>, add a cdline referring to + * that line (without trailing newline) to <b>out</b>. Return -1 if there are + * any non-NL terminated lines; 0 otherwise. + * + * Unlike tor_split_lines, this function avoids ambiguity on its + * handling of a final line that isn't NL-terminated. + * + * All cdline_t objects are allocated in the provided memarea. Strings + * are not copied: if <b>s</b> changes or becomes invalid, then all + * generated cdlines will become invalid. + */ +STATIC int +consensus_split_lines(smartlist_t *out, const char *s, memarea_t *area) +{ + while (*s) { + const char *eol = strchr(s, '\n'); + if (!eol) { + /* File doesn't end with newline. */ + return -1; + } + if (eol - s > CONSENSUS_LINE_MAX_LEN) { + /* Line is far too long. */ + return -1; + } + cdline_t *line = memarea_alloc(area, sizeof(cdline_t)); + line->s = s; + line->len = (uint32_t)(eol - s); + smartlist_add(out, line); + s = eol+1; + } + return 0; +} + +/** Given a list of cdline_t, return a newly allocated string containing + * all of the lines, terminated with NL, concatenated. + * + * Unlike smartlist_join_strings(), avoids lossy operations on empty + * lists. */ +static char * +consensus_join_lines(const smartlist_t *inp) +{ + size_t n = 0; + SMARTLIST_FOREACH(inp, const cdline_t *, cdline, n += cdline->len + 1); + n += 1; + char *result = tor_malloc(n); + char *out = result; + SMARTLIST_FOREACH_BEGIN(inp, const cdline_t *, cdline) { + memcpy(out, cdline->s, cdline->len); + out += cdline->len; + *out++ = '\n'; + } SMARTLIST_FOREACH_END(cdline); + *out++ = '\0'; + tor_assert(out == result+n); + return result; +} + +/** Given two consensus documents, try to compute a diff between them. On + * success, retun a newly allocated string containing that diff. On failure, + * return NULL. */ +char * +consensus_diff_generate(const char *cons1, + const char *cons2) +{ + consensus_digest_t d1, d2; + smartlist_t *lines1 = NULL, *lines2 = NULL, *result_lines = NULL; + int r1, r2; + char *result = NULL; + + r1 = consensus_compute_digest(cons1, &d1); + r2 = consensus_compute_digest(cons2, &d2); + if (BUG(r1 < 0 || r2 < 0)) + return NULL; // LCOV_EXCL_LINE + + memarea_t *area = memarea_new(); + lines1 = smartlist_new(); + lines2 = smartlist_new(); + if (consensus_split_lines(lines1, cons1, area) < 0) + goto done; + if (consensus_split_lines(lines2, cons2, area) < 0) + goto done; + + result_lines = consdiff_gen_diff(lines1, lines2, &d1, &d2, area); + + done: + if (result_lines) { + result = consensus_join_lines(result_lines); + smartlist_free(result_lines); + } + + memarea_drop_all(area); + smartlist_free(lines1); + smartlist_free(lines2); + + return result; +} + +/** Given a consensus document and a diff, try to apply the diff to the + * consensus. On success return a newly allocated string containing the new + * consensus. On failure, return NULL. */ +char * +consensus_diff_apply(const char *consensus, + const char *diff) +{ + consensus_digest_t d1; + smartlist_t *lines1 = NULL, *lines2 = NULL; + int r1; + char *result = NULL; + memarea_t *area = memarea_new(); + + r1 = consensus_compute_digest(consensus, &d1); + if (BUG(r1 < 0)) + return NULL; // LCOV_EXCL_LINE + + lines1 = smartlist_new(); + lines2 = smartlist_new(); + if (consensus_split_lines(lines1, consensus, area) < 0) + goto done; + if (consensus_split_lines(lines2, diff, area) < 0) + goto done; + + result = consdiff_apply_diff(lines1, lines2, &d1); + + done: + smartlist_free(lines1); + smartlist_free(lines2); + memarea_drop_all(area); + + return result; +} + diff --git a/src/or/consdiff.h b/src/or/consdiff.h new file mode 100644 index 0000000000..e9d175136e --- /dev/null +++ b/src/or/consdiff.h @@ -0,0 +1,93 @@ +/* Copyright (c) 2014, Daniel Martà + * Copyright (c) 2014, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_CONSDIFF_H +#define TOR_CONSDIFF_H + +#include "or.h" + +char *consensus_diff_generate(const char *cons1, + const char *cons2); +char *consensus_diff_apply(const char *consensus, + const char *diff); + +#ifdef CONSDIFF_PRIVATE +struct memarea_t; + +/** Line type used for constructing consensus diffs. Each of these lines + * refers to a chunk of memory allocated elsewhere, and is not necessarily + * NUL-terminated: this helps us avoid copies and save memory. */ +typedef struct cdline_t { + const char *s; + uint32_t len; +} cdline_t; + +typedef struct consensus_digest_t { + uint8_t sha3_256[DIGEST256_LEN]; +} consensus_digest_t; + +STATIC smartlist_t *consdiff_gen_diff(const smartlist_t *cons1, + const smartlist_t *cons2, + const consensus_digest_t *digests1, + const consensus_digest_t *digests2, + struct memarea_t *area); +STATIC char *consdiff_apply_diff(const smartlist_t *cons1, + const smartlist_t *diff, + const consensus_digest_t *digests1); +STATIC int consdiff_get_digests(const smartlist_t *diff, + char *digest1_out, + char *digest2_out); + +/** Data structure to define a slice of a smarltist. */ +typedef struct smartlist_slice_t { + /** + * Smartlist that this slice is made from. + * References the whole original smartlist that the slice was made out of. + * */ + const smartlist_t *list; + /** Starting position of the slice in the smartlist. */ + int offset; + /** Length of the slice, i.e. the number of elements it holds. */ + int len; +} smartlist_slice_t; +STATIC smartlist_t *gen_ed_diff(const smartlist_t *cons1, + const smartlist_t *cons2, + struct memarea_t *area); +STATIC smartlist_t *apply_ed_diff(const smartlist_t *cons1, + const smartlist_t *diff, + int start_line); +STATIC void calc_changes(smartlist_slice_t *slice1, smartlist_slice_t *slice2, + bitarray_t *changed1, bitarray_t *changed2); +STATIC smartlist_slice_t *smartlist_slice(const smartlist_t *list, + int start, int end); +STATIC int next_router(const smartlist_t *cons, int cur); +STATIC int *lcs_lengths(const smartlist_slice_t *slice1, + const smartlist_slice_t *slice2, + int direction); +STATIC void trim_slices(smartlist_slice_t *slice1, smartlist_slice_t *slice2); +STATIC int base64cmp(const cdline_t *hash1, const cdline_t *hash2); +STATIC int get_id_hash(const cdline_t *line, cdline_t *hash_out); +STATIC int is_valid_router_entry(const cdline_t *line); +STATIC int smartlist_slice_string_pos(const smartlist_slice_t *slice, + const cdline_t *string); +STATIC void set_changed(bitarray_t *changed1, bitarray_t *changed2, + const smartlist_slice_t *slice1, + const smartlist_slice_t *slice2); +STATIC int consensus_split_lines(smartlist_t *out, const char *s, + struct memarea_t *area); +STATIC void smartlist_add_linecpy(smartlist_t *lst, struct memarea_t *area, + const char *s); +STATIC int lines_eq(const cdline_t *a, const cdline_t *b); +STATIC int line_str_eq(const cdline_t *a, const char *b); + +MOCK_DECL(STATIC int, + consensus_compute_digest,(const char *cons, + consensus_digest_t *digest_out)); +MOCK_DECL(STATIC int, + consensus_digest_eq,(const uint8_t *d1, + const uint8_t *d2)); +#endif + +#endif + diff --git a/src/or/control.c b/src/or/control.c index 8ab31f18f7..30067293dc 100644 --- a/src/or/control.c +++ b/src/or/control.c @@ -1,5 +1,5 @@ /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -2047,6 +2047,12 @@ getinfo_helper_dir(control_connection_t *control_conn, } } } else if (!strcmp(question, "network-status")) { /* v1 */ + static int network_status_warned = 0; + if (!network_status_warned) { + log_warn(LD_CONTROL, "GETINFO network-status is deprecated; it will " + "go away in a future version of Tor."); + network_status_warned = 1; + } routerlist_t *routerlist = router_get_routerlist(); if (!routerlist || !routerlist->routers || list_server_status_v1(routerlist->routers, answer, 1) < 0) { diff --git a/src/or/control.h b/src/or/control.h index 16ba1ed8f0..a786dfe1af 100644 --- a/src/or/control.h +++ b/src/or/control.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/cpuworker.c b/src/or/cpuworker.c index fd6de6ea7c..790c147d7b 100644 --- a/src/or/cpuworker.c +++ b/src/or/cpuworker.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/cpuworker.h b/src/or/cpuworker.h index 62cf0eb164..fd426e21f6 100644 --- a/src/or/cpuworker.h +++ b/src/or/cpuworker.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/dircollate.c b/src/or/dircollate.c index 033a7afe0f..172364c5f5 100644 --- a/src/or/dircollate.c +++ b/src/or/dircollate.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/dircollate.h b/src/or/dircollate.h index 358c730cbb..52214282b9 100644 --- a/src/or/dircollate.h +++ b/src/or/dircollate.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/directory.c b/src/or/directory.c index fa76afac66..1b999ee7c3 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define DIRECTORY_PRIVATE diff --git a/src/or/directory.h b/src/or/directory.h index a45a0aa3f7..0c5db3e070 100644 --- a/src/or/directory.h +++ b/src/or/directory.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/dirserv.c b/src/or/dirserv.c index 33d25618fa..b78af307ce 100644 --- a/src/or/dirserv.c +++ b/src/or/dirserv.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define DIRSERV_PRIVATE @@ -823,6 +823,9 @@ directory_remove_invalid(void) * Allocate and return a description of the status of the server <b>desc</b>, * for use in a v1-style router-status line. The server is listed * as running iff <b>is_live</b> is true. + * + * This is deprecated: it's only used for controllers that want outputs in + * the old format. */ static char * list_single_server_status(const routerinfo_t *desc, int is_live) @@ -935,6 +938,9 @@ dirserv_set_router_is_running(routerinfo_t *router, time_t now) * *<b>router_status_out</b>. Return 0 on success, -1 on failure. * * If for_controller is true, include the routers with very old descriptors. + * + * This is deprecated: it's only used for controllers that want outputs in + * the old format. */ int list_server_status_v1(smartlist_t *routers, char **router_status_out, diff --git a/src/or/dirserv.h b/src/or/dirserv.h index beebe47d3d..4bc31d8f3b 100644 --- a/src/or/dirserv.h +++ b/src/or/dirserv.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/dirvote.c b/src/or/dirvote.c index e92d3b49dc..f5e29eb786 100644 --- a/src/or/dirvote.c +++ b/src/or/dirvote.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define DIRVOTE_PRIVATE diff --git a/src/or/dirvote.h b/src/or/dirvote.h index ac7db69db2..e342dc78ea 100644 --- a/src/or/dirvote.h +++ b/src/or/dirvote.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/dns.c b/src/or/dns.c index 7651501ec3..cd84a64758 100644 --- a/src/or/dns.c +++ b/src/or/dns.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/dns.h b/src/or/dns.h index 951a2a3467..a81cbd20da 100644 --- a/src/or/dns.h +++ b/src/or/dns.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/dns_structs.h b/src/or/dns_structs.h index bc6067213d..dc00e9f7b9 100644 --- a/src/or/dns_structs.h +++ b/src/or/dns_structs.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/dnsserv.c b/src/or/dnsserv.c index 8768b2a1d1..54a22a5150 100644 --- a/src/or/dnsserv.c +++ b/src/or/dnsserv.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2007-2016, The Tor Project, Inc. */ +/* Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/dnsserv.h b/src/or/dnsserv.h index ad0e248c83..6c0643b8dc 100644 --- a/src/or/dnsserv.h +++ b/src/or/dnsserv.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/entrynodes.c b/src/or/entrynodes.c index 729e4b0391..7a27cd0f44 100644 --- a/src/or/entrynodes.c +++ b/src/or/entrynodes.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -989,9 +989,11 @@ get_max_sample_size(guard_selection_t *gs, const int using_bridges = (gs->type == GS_TYPE_BRIDGE); const int min_sample = get_min_filtered_sample_size(); - /* With bridges, max_sample is "all of them" */ + /* If we are in bridge mode, expand our sample set as needed without worrying + * about max size. We should respect the user's wishes to use many bridges if + * that's what they have specified in their configuration file. */ if (using_bridges) - return n_guards; + return INT_MAX; const int max_sample_by_pct = (int)(n_guards * get_max_sample_threshold()); const int max_sample_absolute = get_max_sample_size_absolute(); diff --git a/src/or/entrynodes.h b/src/or/entrynodes.h index f02901f5d7..400a84269d 100644 --- a/src/or/entrynodes.h +++ b/src/or/entrynodes.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/ext_orport.c b/src/or/ext_orport.c index 676adfd8bf..b60d2e55c8 100644 --- a/src/or/ext_orport.c +++ b/src/or/ext_orport.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2012-2016, The Tor Project, Inc. */ +/* Copyright (c) 2012-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/ext_orport.h b/src/or/ext_orport.h index 33d954e8d0..b2cd05db8f 100644 --- a/src/or/ext_orport.h +++ b/src/or/ext_orport.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef EXT_ORPORT_H diff --git a/src/or/fp_pair.c b/src/or/fp_pair.c index eeeb0f1de3..f730106d06 100644 --- a/src/or/fp_pair.c +++ b/src/or/fp_pair.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2016, The Tor Project, Inc. */ +/* Copyright (c) 2013-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/fp_pair.h b/src/or/fp_pair.h index b1466581d2..4cea3eda6d 100644 --- a/src/or/fp_pair.h +++ b/src/or/fp_pair.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2016, The Tor Project, Inc. */ +/* Copyright (c) 2013-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/geoip.c b/src/or/geoip.c index 74811ea643..c76071a152 100644 --- a/src/or/geoip.c +++ b/src/or/geoip.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2007-2016, The Tor Project, Inc. */ +/* Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/geoip.h b/src/or/geoip.h index 070296dd07..55ca8ca28c 100644 --- a/src/or/geoip.h +++ b/src/or/geoip.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/hibernate.c b/src/or/hibernate.c index c2b3bbb839..f2a4c71764 100644 --- a/src/or/hibernate.c +++ b/src/or/hibernate.c @@ -1,5 +1,5 @@ /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/hibernate.h b/src/or/hibernate.h index fa9da6de39..8bdb65a927 100644 --- a/src/or/hibernate.h +++ b/src/or/hibernate.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/hs_cache.c b/src/or/hs_cache.c index 43cd8c3258..29681b42b5 100644 --- a/src/or/hs_cache.c +++ b/src/or/hs_cache.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/hs_cache.h b/src/or/hs_cache.h index ba95e73338..ed00424234 100644 --- a/src/or/hs_cache.h +++ b/src/or/hs_cache.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/hs_circuitmap.c b/src/or/hs_circuitmap.c index 5003b4b593..32017f1234 100644 --- a/src/or/hs_circuitmap.c +++ b/src/or/hs_circuitmap.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/hs_circuitmap.h b/src/or/hs_circuitmap.h index b587039310..6e0c2d44f2 100644 --- a/src/or/hs_circuitmap.h +++ b/src/or/hs_circuitmap.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/hs_common.c b/src/or/hs_common.c index de96946ab5..7e0b6ca1bc 100644 --- a/src/or/hs_common.c +++ b/src/or/hs_common.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/hs_common.h b/src/or/hs_common.h index e0ab510ea4..7ac2a15ea1 100644 --- a/src/or/hs_common.h +++ b/src/or/hs_common.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/hs_descriptor.c b/src/or/hs_descriptor.c index f16a2fdc14..0480f63308 100644 --- a/src/or/hs_descriptor.c +++ b/src/or/hs_descriptor.c @@ -1,9 +1,55 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** * \file hs_descriptor.c * \brief Handle hidden service descriptor encoding/decoding. + * + * \details + * Here is a graphical depiction of an HS descriptor and its layers: + * + * +------------------------------------------------------+ + * |DESCRIPTOR HEADER: | + * | hs-descriptor 3 | + * | descriptor-lifetime 180 | + * | ... | + * | superencrypted | + * |+---------------------------------------------------+ | + * ||SUPERENCRYPTED LAYER (aka OUTER ENCRYPTED LAYER): | | + * || desc-auth-type x25519 | | + * || desc-auth-ephemeral-key | | + * || auth-client | | + * || auth-client | | + * || ... | | + * || encrypted | | + * ||+-------------------------------------------------+| | + * |||ENCRYPTED LAYER (aka INNER ENCRYPTED LAYER): || | + * ||| create2-formats || | + * ||| intro-auth-required || | + * ||| introduction-point || | + * ||| introduction-point || | + * ||| ... || | + * ||+-------------------------------------------------+| | + * |+---------------------------------------------------+ | + * +------------------------------------------------------+ + * + * The DESCRIPTOR HEADER section is completely unencrypted and contains generic + * descriptor metadata. + * + * The SUPERENCRYPTED LAYER section is the first layer of encryption, and it's + * encrypted using the blinded public key of the hidden service to protect + * against entities who don't know its onion address. The clients of the hidden + * service know its onion address and blinded public key, whereas third-parties + * (like HSDirs) don't know it (except if it's a public hidden service). + * + * The ENCRYPTED LAYER section is the second layer of encryption, and it's + * encrypted using the client authorization key material (if those exist). When + * client authorization is enabled, this second layer of encryption protects + * the descriptor content from unauthorized entities. If client authorization + * is disabled, this second layer of encryption does not provide any extra + * security but is still present. The plaintext of this layer contains all the + * information required to connect to the hidden service like its list of + * introduction points. **/ /* For unit tests.*/ @@ -23,11 +69,12 @@ #define str_desc_cert "descriptor-signing-key-cert" #define str_rev_counter "revision-counter" #define str_superencrypted "superencrypted" +#define str_encrypted "encrypted" #define str_signature "signature" #define str_lifetime "descriptor-lifetime" /* Constant string value for the encrypted part of the descriptor. */ #define str_create2_formats "create2-formats" -#define str_auth_required "authentication-required" +#define str_intro_auth_required "intro-auth-required" #define str_single_onion "single-onion-service" #define str_intro_point "introduction-point" #define str_ip_auth_key "auth-key" @@ -36,16 +83,20 @@ #define str_intro_point_start "\n" str_intro_point " " /* Constant string value for the construction to encrypt the encrypted data * section. */ -#define str_enc_hsdir_data "hsdir-superencrypted-data" +#define str_enc_const_superencryption "hsdir-superencrypted-data" +#define str_enc_const_encryption "hsdir-encrypted-data" /* Prefix required to compute/verify HS desc signatures */ #define str_desc_sig_prefix "Tor onion service descriptor sig v3" +#define str_desc_auth_type "desc-auth-type" +#define str_desc_auth_key "desc-auth-ephemeral-key" +#define str_desc_auth_client "auth-client" +#define str_encrypted "encrypted" /* Authentication supported types. */ static const struct { hs_desc_auth_type_t type; const char *identifier; -} auth_types[] = { - { HS_DESC_AUTH_PASSWORD, "password" }, +} intro_auth_types[] = { { HS_DESC_AUTH_ED25519, "ed25519" }, /* Indicate end of array. */ { 0, NULL } @@ -62,10 +113,19 @@ static token_rule_t hs_desc_v3_token_table[] = { END_OF_TABLE }; +/* Descriptor ruleset for the superencrypted section. */ +static token_rule_t hs_desc_superencrypted_v3_token_table[] = { + T1_START(str_desc_auth_type, R3_DESC_AUTH_TYPE, GE(1), NO_OBJ), + T1(str_desc_auth_key, R3_DESC_AUTH_KEY, GE(1), NO_OBJ), + T1N(str_desc_auth_client, R3_DESC_AUTH_CLIENT, GE(3), NO_OBJ), + T1(str_encrypted, R3_ENCRYPTED, NO_ARGS, NEED_OBJ), + END_OF_TABLE +}; + /* Descriptor ruleset for the encrypted section. */ static token_rule_t hs_desc_encrypted_v3_token_table[] = { T1_START(str_create2_formats, R3_CREATE2_FORMATS, CONCAT_ARGS, NO_OBJ), - T01(str_auth_required, R3_AUTHENTICATION_REQUIRED, ARGS, NO_OBJ), + T01(str_intro_auth_required, R3_INTRO_AUTH_REQUIRED, ARGS, NO_OBJ), T01(str_single_onion, R3_SINGLE_ONION_SERVICE, ARGS, NO_OBJ), END_OF_TABLE }; @@ -107,8 +167,8 @@ desc_plaintext_data_free_contents(hs_desc_plaintext_data_t *desc) return; } - if (desc->encrypted_blob) { - tor_free(desc->encrypted_blob); + if (desc->superencrypted_blob) { + tor_free(desc->superencrypted_blob); } tor_cert_free(desc->signing_key_cert); @@ -123,9 +183,9 @@ desc_encrypted_data_free_contents(hs_desc_encrypted_data_t *desc) return; } - if (desc->auth_types) { - SMARTLIST_FOREACH(desc->auth_types, char *, a, tor_free(a)); - smartlist_free(desc->auth_types); + if (desc->intro_auth_types) { + SMARTLIST_FOREACH(desc->intro_auth_types, char *, a, tor_free(a)); + smartlist_free(desc->intro_auth_types); } if (desc->intro_points) { SMARTLIST_FOREACH(desc->intro_points, hs_desc_intro_point_t *, ip, @@ -135,6 +195,135 @@ desc_encrypted_data_free_contents(hs_desc_encrypted_data_t *desc) memwipe(desc, 0, sizeof(*desc)); } +/* Using a key, salt and encrypted payload, build a MAC and put it in mac_out. + * We use SHA3-256 for the MAC computation. + * This function can't fail. */ +static void +build_mac(const uint8_t *mac_key, size_t mac_key_len, + const uint8_t *salt, size_t salt_len, + const uint8_t *encrypted, size_t encrypted_len, + uint8_t *mac_out, size_t mac_len) +{ + crypto_digest_t *digest; + + const uint64_t mac_len_netorder = tor_htonll(mac_key_len); + const uint64_t salt_len_netorder = tor_htonll(salt_len); + + tor_assert(mac_key); + tor_assert(salt); + tor_assert(encrypted); + tor_assert(mac_out); + + digest = crypto_digest256_new(DIGEST_SHA3_256); + /* As specified in section 2.5 of proposal 224, first add the mac key + * then add the salt first and then the encrypted section. */ + + crypto_digest_add_bytes(digest, (const char *) &mac_len_netorder, 8); + crypto_digest_add_bytes(digest, (const char *) mac_key, mac_key_len); + crypto_digest_add_bytes(digest, (const char *) &salt_len_netorder, 8); + crypto_digest_add_bytes(digest, (const char *) salt, salt_len); + crypto_digest_add_bytes(digest, (const char *) encrypted, encrypted_len); + crypto_digest_get_digest(digest, (char *) mac_out, mac_len); + crypto_digest_free(digest); +} + +/* Using a given decriptor object, build the secret input needed for the + * KDF and put it in the dst pointer which is an already allocated buffer + * of size dstlen. */ +static void +build_secret_input(const hs_descriptor_t *desc, uint8_t *dst, size_t dstlen) +{ + size_t offset = 0; + + tor_assert(desc); + tor_assert(dst); + tor_assert(HS_DESC_ENCRYPTED_SECRET_INPUT_LEN <= dstlen); + + /* XXX use the destination length as the memcpy length */ + /* Copy blinded public key. */ + memcpy(dst, desc->plaintext_data.blinded_pubkey.pubkey, + sizeof(desc->plaintext_data.blinded_pubkey.pubkey)); + offset += sizeof(desc->plaintext_data.blinded_pubkey.pubkey); + /* Copy subcredential. */ + memcpy(dst + offset, desc->subcredential, sizeof(desc->subcredential)); + offset += sizeof(desc->subcredential); + /* Copy revision counter value. */ + set_uint64(dst + offset, tor_ntohll(desc->plaintext_data.revision_counter)); + offset += sizeof(uint64_t); + tor_assert(HS_DESC_ENCRYPTED_SECRET_INPUT_LEN == offset); +} + +/* Do the KDF construction and put the resulting data in key_out which is of + * key_out_len length. It uses SHAKE-256 as specified in the spec. */ +static void +build_kdf_key(const hs_descriptor_t *desc, + const uint8_t *salt, size_t salt_len, + uint8_t *key_out, size_t key_out_len, + int is_superencrypted_layer) +{ + uint8_t secret_input[HS_DESC_ENCRYPTED_SECRET_INPUT_LEN]; + crypto_xof_t *xof; + + tor_assert(desc); + tor_assert(salt); + tor_assert(key_out); + + /* Build the secret input for the KDF computation. */ + build_secret_input(desc, secret_input, sizeof(secret_input)); + + xof = crypto_xof_new(); + /* Feed our KDF. [SHAKE it like a polaroid picture --Yawning]. */ + crypto_xof_add_bytes(xof, secret_input, sizeof(secret_input)); + crypto_xof_add_bytes(xof, salt, salt_len); + + /* Feed in the right string constant based on the desc layer */ + if (is_superencrypted_layer) { + crypto_xof_add_bytes(xof, (const uint8_t *) str_enc_const_superencryption, + strlen(str_enc_const_superencryption)); + } else { + crypto_xof_add_bytes(xof, (const uint8_t *) str_enc_const_encryption, + strlen(str_enc_const_encryption)); + } + + /* Eat from our KDF. */ + crypto_xof_squeeze_bytes(xof, key_out, key_out_len); + crypto_xof_free(xof); + memwipe(secret_input, 0, sizeof(secret_input)); +} + +/* Using the given descriptor and salt, run it through our KDF function and + * then extract a secret key in key_out, the IV in iv_out and MAC in mac_out. + * This function can't fail. */ +static void +build_secret_key_iv_mac(const hs_descriptor_t *desc, + const uint8_t *salt, size_t salt_len, + uint8_t *key_out, size_t key_len, + uint8_t *iv_out, size_t iv_len, + uint8_t *mac_out, size_t mac_len, + int is_superencrypted_layer) +{ + size_t offset = 0; + uint8_t kdf_key[HS_DESC_ENCRYPTED_KDF_OUTPUT_LEN]; + + tor_assert(desc); + tor_assert(salt); + tor_assert(key_out); + tor_assert(iv_out); + tor_assert(mac_out); + + build_kdf_key(desc, salt, salt_len, kdf_key, sizeof(kdf_key), + is_superencrypted_layer); + /* Copy the bytes we need for both the secret key and IV. */ + memcpy(key_out, kdf_key, key_len); + offset += key_len; + memcpy(iv_out, kdf_key + offset, iv_len); + offset += iv_len; + memcpy(mac_out, kdf_key + offset, mac_len); + /* Extra precaution to make sure we are not out of bound. */ + tor_assert((offset + mac_len) == sizeof(kdf_key)); + memwipe(kdf_key, 0, sizeof(kdf_key)); +} + /* === ENCODING === */ /* Encode the given link specifier objects into a newly allocated string. @@ -363,142 +552,23 @@ encode_intro_point(const ed25519_public_key_t *sig_key, return encoded_ip; } -/* Using a given decriptor object, build the secret input needed for the - * KDF and put it in the dst pointer which is an already allocated buffer - * of size dstlen. */ -static void -build_secret_input(const hs_descriptor_t *desc, uint8_t *dst, size_t dstlen) -{ - size_t offset = 0; - - tor_assert(desc); - tor_assert(dst); - tor_assert(HS_DESC_ENCRYPTED_SECRET_INPUT_LEN <= dstlen); - - /* XXX use the destination length as the memcpy length */ - /* Copy blinded public key. */ - memcpy(dst, desc->plaintext_data.blinded_pubkey.pubkey, - sizeof(desc->plaintext_data.blinded_pubkey.pubkey)); - offset += sizeof(desc->plaintext_data.blinded_pubkey.pubkey); - /* Copy subcredential. */ - memcpy(dst + offset, desc->subcredential, sizeof(desc->subcredential)); - offset += sizeof(desc->subcredential); - /* Copy revision counter value. */ - set_uint64(dst + offset, tor_ntohll(desc->plaintext_data.revision_counter)); - offset += sizeof(uint64_t); - tor_assert(HS_DESC_ENCRYPTED_SECRET_INPUT_LEN == offset); -} - -/* Do the KDF construction and put the resulting data in key_out which is of - * key_out_len length. It uses SHAKE-256 as specified in the spec. */ -static void -build_kdf_key(const hs_descriptor_t *desc, - const uint8_t *salt, size_t salt_len, - uint8_t *key_out, size_t key_out_len) -{ - uint8_t secret_input[HS_DESC_ENCRYPTED_SECRET_INPUT_LEN]; - crypto_xof_t *xof; - - tor_assert(desc); - tor_assert(salt); - tor_assert(key_out); - - /* Build the secret input for the KDF computation. */ - build_secret_input(desc, secret_input, sizeof(secret_input)); - - xof = crypto_xof_new(); - /* Feed our KDF. [SHAKE it like a polaroid picture --Yawning]. */ - crypto_xof_add_bytes(xof, secret_input, sizeof(secret_input)); - crypto_xof_add_bytes(xof, salt, salt_len); - crypto_xof_add_bytes(xof, (const uint8_t *) str_enc_hsdir_data, - strlen(str_enc_hsdir_data)); - /* Eat from our KDF. */ - crypto_xof_squeeze_bytes(xof, key_out, key_out_len); - crypto_xof_free(xof); - memwipe(secret_input, 0, sizeof(secret_input)); -} - -/* Using the given descriptor and salt, run it through our KDF function and - * then extract a secret key in key_out, the IV in iv_out and MAC in mac_out. - * This function can't fail. */ -static void -build_secret_key_iv_mac(const hs_descriptor_t *desc, - const uint8_t *salt, size_t salt_len, - uint8_t *key_out, size_t key_len, - uint8_t *iv_out, size_t iv_len, - uint8_t *mac_out, size_t mac_len) -{ - size_t offset = 0; - uint8_t kdf_key[HS_DESC_ENCRYPTED_KDF_OUTPUT_LEN]; - - tor_assert(desc); - tor_assert(salt); - tor_assert(key_out); - tor_assert(iv_out); - tor_assert(mac_out); - - build_kdf_key(desc, salt, salt_len, kdf_key, sizeof(kdf_key)); - /* Copy the bytes we need for both the secret key and IV. */ - memcpy(key_out, kdf_key, key_len); - offset += key_len; - memcpy(iv_out, kdf_key + offset, iv_len); - offset += iv_len; - memcpy(mac_out, kdf_key + offset, mac_len); - /* Extra precaution to make sure we are not out of bound. */ - tor_assert((offset + mac_len) == sizeof(kdf_key)); - memwipe(kdf_key, 0, sizeof(kdf_key)); -} - -/* Using a key, salt and encrypted payload, build a MAC and put it in mac_out. - * We use SHA3-256 for the MAC computation. - * This function can't fail. */ -static void -build_mac(const uint8_t *mac_key, size_t mac_key_len, - const uint8_t *salt, size_t salt_len, - const uint8_t *encrypted, size_t encrypted_len, - uint8_t *mac_out, size_t mac_len) -{ - crypto_digest_t *digest; - - const uint64_t mac_len_netorder = tor_htonll(mac_key_len); - const uint64_t salt_len_netorder = tor_htonll(salt_len); - - tor_assert(mac_key); - tor_assert(salt); - tor_assert(encrypted); - tor_assert(mac_out); - - digest = crypto_digest256_new(DIGEST_SHA3_256); - /* As specified in section 2.5 of proposal 224, first add the mac key - * then add the salt first and then the encrypted section. */ - - crypto_digest_add_bytes(digest, (const char *) &mac_len_netorder, 8); - crypto_digest_add_bytes(digest, (const char *) mac_key, mac_key_len); - crypto_digest_add_bytes(digest, (const char *) &salt_len_netorder, 8); - crypto_digest_add_bytes(digest, (const char *) salt, salt_len); - crypto_digest_add_bytes(digest, (const char *) encrypted, encrypted_len); - crypto_digest_get_digest(digest, (char *) mac_out, mac_len); - crypto_digest_free(digest); -} - /* Given a source length, return the new size including padding for the * plaintext encryption. */ static size_t compute_padded_plaintext_length(size_t plaintext_len) { size_t plaintext_padded_len; + const int padding_block_length = HS_DESC_SUPERENC_PLAINTEXT_PAD_MULTIPLE; /* Make sure we won't overflow. */ - tor_assert(plaintext_len <= - (SIZE_T_CEILING - HS_DESC_PLAINTEXT_PADDING_MULTIPLE)); - - /* Get the extra length we need to add. For example, if srclen is 234 bytes, - * this will expand to (2 * 128) == 256 thus an extra 22 bytes. */ - plaintext_padded_len = CEIL_DIV(plaintext_len, - HS_DESC_PLAINTEXT_PADDING_MULTIPLE) * - HS_DESC_PLAINTEXT_PADDING_MULTIPLE; + tor_assert(plaintext_len <= (SIZE_T_CEILING - padding_block_length)); + + /* Get the extra length we need to add. For example, if srclen is 10200 + * bytes, this will expand to (2 * 10k) == 20k thus an extra 9800 bytes. */ + plaintext_padded_len = CEIL_DIV(plaintext_len, padding_block_length) * + padding_block_length; /* Can never be extra careful. Make sure we are _really_ padded. */ - tor_assert(!(plaintext_padded_len % HS_DESC_PLAINTEXT_PADDING_MULTIPLE)); + tor_assert(!(plaintext_padded_len % padding_block_length)); return plaintext_padded_len; } @@ -530,7 +600,8 @@ build_plaintext_padding(const char *plaintext, size_t plaintext_len, * data. Return size of the encrypted data buffer. */ static size_t build_encrypted(const uint8_t *key, const uint8_t *iv, const char *plaintext, - size_t plaintext_len, uint8_t **encrypted_out) + size_t plaintext_len, uint8_t **encrypted_out, + int is_superencrypted_layer) { size_t encrypted_len; uint8_t *padded_plaintext, *encrypted; @@ -541,15 +612,21 @@ build_encrypted(const uint8_t *key, const uint8_t *iv, const char *plaintext, tor_assert(plaintext); tor_assert(encrypted_out); + /* If we are encrypting the middle layer of the descriptor, we need to first + pad the plaintext */ + if (is_superencrypted_layer) { + encrypted_len = build_plaintext_padding(plaintext, plaintext_len, + &padded_plaintext); + /* Extra precautions that we have a valid padding length. */ + tor_assert(!(encrypted_len % HS_DESC_SUPERENC_PLAINTEXT_PAD_MULTIPLE)); + } else { /* No padding required for inner layers */ + padded_plaintext = tor_memdup(plaintext, plaintext_len); + encrypted_len = plaintext_len; + } + /* This creates a cipher for AES. It can't fail. */ cipher = crypto_cipher_new_with_iv_and_bits(key, iv, HS_DESC_ENCRYPTED_BIT_SIZE); - /* This can't fail. */ - encrypted_len = build_plaintext_padding(plaintext, plaintext_len, - &padded_plaintext); - /* Extra precautions that we have a valie padding length. */ - tor_assert(encrypted_len <= HS_DESC_PADDED_PLAINTEXT_MAX_LEN); - tor_assert(!(encrypted_len % HS_DESC_PLAINTEXT_PADDING_MULTIPLE)); /* We use a stream cipher so the encrypted length will be the same as the * plaintext padded length. */ encrypted = tor_malloc_zero(encrypted_len); @@ -563,12 +640,13 @@ build_encrypted(const uint8_t *key, const uint8_t *iv, const char *plaintext, return encrypted_len; } -/* Encrypt the given plaintext buffer and using the descriptor to get the +/* Encrypt the given <b>plaintext</b> buffer using <b>desc</b> to get the * keys. Set encrypted_out with the encrypted data and return the length of - * it. */ + * it. <b>is_superencrypted_layer</b> is set if this is the outer encrypted + * layer of the descriptor. */ static size_t encrypt_descriptor_data(const hs_descriptor_t *desc, const char *plaintext, - char **encrypted_out) + char **encrypted_out, int is_superencrypted_layer) { char *final_blob; size_t encrypted_len, final_blob_len, offset = 0; @@ -589,11 +667,13 @@ encrypt_descriptor_data(const hs_descriptor_t *desc, const char *plaintext, build_secret_key_iv_mac(desc, salt, sizeof(salt), secret_key, sizeof(secret_key), secret_iv, sizeof(secret_iv), - mac_key, sizeof(mac_key)); + mac_key, sizeof(mac_key), + is_superencrypted_layer); /* Build the encrypted part that is do the actual encryption. */ encrypted_len = build_encrypted(secret_key, secret_iv, plaintext, - strlen(plaintext), &encrypted); + strlen(plaintext), &encrypted, + is_superencrypted_layer); memwipe(secret_key, 0, sizeof(secret_key)); memwipe(secret_iv, 0, sizeof(secret_iv)); /* This construction is specified in section 2.5 of proposal 224. */ @@ -625,20 +705,89 @@ encrypt_descriptor_data(const hs_descriptor_t *desc, const char *plaintext, return final_blob_len; } -/* Take care of encoding the encrypted data section and then encrypting it - * with the descriptor's key. A newly allocated NUL terminated string pointer - * containing the encrypted encoded blob is put in encrypted_blob_out. Return - * 0 on success else a negative value. */ -static int -encode_encrypted_data(const hs_descriptor_t *desc, - char **encrypted_blob_out) +/* Create and return a string containing a fake client-auth entry. It's the + * responsibility of the caller to free the returned string. This function will + * never fail. */ +static char * +get_fake_auth_client_str(void) { - int ret = -1; - char *encoded_str, *encrypted_blob; - smartlist_t *lines = smartlist_new(); + char *auth_client_str = NULL; + /* We are gonna fill these arrays with fake base64 data. They are all double + * the size of their binary representation to fit the base64 overhead. */ + char client_id_b64[8*2]; + char iv_b64[16*2]; + char encrypted_cookie_b64[16*2]; + int retval; + + /* This is a macro to fill a field with random data and then base64 it. */ +#define FILL_WITH_FAKE_DATA_AND_BASE64(field) STMT_BEGIN \ + crypto_rand((char *)field, sizeof(field)); \ + retval = base64_encode_nopad(field##_b64, sizeof(field##_b64), \ + field, sizeof(field)); \ + tor_assert(retval > 0); \ + STMT_END + + { /* Get those fakes! */ + uint8_t client_id[8]; /* fake client-id */ + uint8_t iv[16]; /* fake IV (initialization vector) */ + uint8_t encrypted_cookie[16]; /* fake encrypted cookie */ + + FILL_WITH_FAKE_DATA_AND_BASE64(client_id); + FILL_WITH_FAKE_DATA_AND_BASE64(iv); + FILL_WITH_FAKE_DATA_AND_BASE64(encrypted_cookie); + } + + /* Build the final string */ + tor_asprintf(&auth_client_str, "%s %s %s %s", str_desc_auth_client, + client_id_b64, iv_b64, encrypted_cookie_b64); + +#undef FILL_WITH_FAKE_DATA_AND_BASE64 + + return auth_client_str; +} - tor_assert(desc); - tor_assert(encrypted_blob_out); +/** How many lines of "client-auth" we want in our descriptors; fake or not. */ +#define CLIENT_AUTH_ENTRIES_BLOCK_SIZE 16 + +/** Create the "client-auth" part of the descriptor and return a + * newly-allocated string with it. It's the responsibility of the caller to + * free the returned string. */ +static char * +get_fake_auth_client_lines(void) +{ + /* XXX: Client authorization is still not implemented, so all this function + does is make fake clients */ + int i = 0; + smartlist_t *auth_client_lines = smartlist_new(); + char *auth_client_lines_str = NULL; + + /* Make a line for each fake client */ + const int num_fake_clients = CLIENT_AUTH_ENTRIES_BLOCK_SIZE; + for (i = 0; i < num_fake_clients; i++) { + char *auth_client_str = get_fake_auth_client_str(); + tor_assert(auth_client_str); + smartlist_add(auth_client_lines, auth_client_str); + } + + /* Join all lines together to form final string */ + auth_client_lines_str = smartlist_join_strings(auth_client_lines, + "\n", 1, NULL); + /* Cleanup the mess */ + SMARTLIST_FOREACH(auth_client_lines, char *, a, tor_free(a)); + smartlist_free(auth_client_lines); + + return auth_client_lines_str; +} + +/* Create the inner layer of the descriptor (which includes the intro points, + * etc.). Return a newly-allocated string with the layer plaintext, or NULL if + * an error occured. It's the responsibility of the caller to free the returned + * string. */ +static char * +get_inner_encrypted_layer_plaintext(const hs_descriptor_t *desc) +{ + char *encoded_str = NULL; + smartlist_t *lines = smartlist_new(); /* Build the start of the section prior to the introduction points. */ { @@ -649,12 +798,12 @@ encode_encrypted_data(const hs_descriptor_t *desc, smartlist_add_asprintf(lines, "%s %d\n", str_create2_formats, ONION_HANDSHAKE_TYPE_NTOR); - if (desc->encrypted_data.auth_types && - smartlist_len(desc->encrypted_data.auth_types)) { + if (desc->encrypted_data.intro_auth_types && + smartlist_len(desc->encrypted_data.intro_auth_types)) { /* Put the authentication-required line. */ - char *buf = smartlist_join_strings(desc->encrypted_data.auth_types, " ", - 0, NULL); - smartlist_add_asprintf(lines, "%s %s\n", str_auth_required, buf); + char *buf = smartlist_join_strings(desc->encrypted_data.intro_auth_types, + " ", 0, NULL); + smartlist_add_asprintf(lines, "%s %s\n", str_intro_auth_required, buf); tor_free(buf); } @@ -679,31 +828,159 @@ encode_encrypted_data(const hs_descriptor_t *desc, * then encrypt it. */ encoded_str = smartlist_join_strings(lines, "", 0, NULL); - /* Encrypt the section into an encrypted blob that we'll base64 encode - * before returning it. */ + err: + SMARTLIST_FOREACH(lines, char *, l, tor_free(l)); + smartlist_free(lines); + + return encoded_str; +} + +/* Create the middle layer of the descriptor, which includes the client auth + * data and the encrypted inner layer (provided as a base64 string at + * <b>layer2_b64_ciphertext</b>). Return a newly-allocated string with the + * layer plaintext, or NULL if an error occured. It's the responsibility of the + * caller to free the returned string. */ +static char * +get_outer_encrypted_layer_plaintext(const hs_descriptor_t *desc, + const char *layer2_b64_ciphertext) +{ + char *layer1_str = NULL; + smartlist_t *lines = smartlist_new(); + + /* XXX: Disclaimer: This function generates only _fake_ client auth + * data. Real client auth is not yet implemented, but client auth data MUST + * always be present in descriptors. In the future this function will be + * refactored to use real client auth data if they exist (#20700). */ + (void) *desc; + + /* Specify auth type */ + smartlist_add_asprintf(lines, "%s %s\n", str_desc_auth_type, "x25519"); + + { /* Create fake ephemeral x25519 key */ + char fake_key_base64[CURVE25519_BASE64_PADDED_LEN + 1]; + curve25519_keypair_t fake_x25519_keypair; + if (curve25519_keypair_generate(&fake_x25519_keypair, 0) < 0) { + goto done; + } + if (curve25519_public_to_base64(fake_key_base64, + &fake_x25519_keypair.pubkey) < 0) { + goto done; + } + smartlist_add_asprintf(lines, "%s %s\n", + str_desc_auth_key, fake_key_base64); + /* No need to memwipe any of these fake keys. They will go unused. */ + } + + { /* Create fake auth-client lines. */ + char *auth_client_lines = get_fake_auth_client_lines(); + tor_assert(auth_client_lines); + smartlist_add(lines, auth_client_lines); + } + + /* create encrypted section */ { - char *enc_b64; - ssize_t enc_b64_len, ret_len, enc_len; + smartlist_add_asprintf(lines, + "%s\n" + "-----BEGIN MESSAGE-----\n" + "%s" + "-----END MESSAGE-----", + str_encrypted, layer2_b64_ciphertext); + } - enc_len = encrypt_descriptor_data(desc, encoded_str, &encrypted_blob); - tor_free(encoded_str); - /* Get the encoded size plus a NUL terminating byte. */ - enc_b64_len = base64_encode_size(enc_len, BASE64_ENCODE_MULTILINE) + 1; - enc_b64 = tor_malloc_zero(enc_b64_len); - /* Base64 the encrypted blob before returning it. */ - ret_len = base64_encode(enc_b64, enc_b64_len, encrypted_blob, enc_len, - BASE64_ENCODE_MULTILINE); - /* Return length doesn't count the NUL byte. */ - tor_assert(ret_len == (enc_b64_len - 1)); - tor_free(encrypted_blob); - *encrypted_blob_out = enc_b64; + layer1_str = smartlist_join_strings(lines, "", 0, NULL); + + done: + SMARTLIST_FOREACH(lines, char *, a, tor_free(a)); + smartlist_free(lines); + + return layer1_str; +} + +/* Encrypt <b>encoded_str</b> into an encrypted blob and then base64 it before + * returning it. <b>desc</b> is provided to derive the encryption + * keys. <b>is_superencrypted_layer</b> is set if <b>encoded_str</b> is the + * middle (superencrypted) layer of the descriptor. It's the responsibility of + * the caller to free the returned string. */ +static char * +encrypt_desc_data_and_base64(const hs_descriptor_t *desc, + const char *encoded_str, + int is_superencrypted_layer) +{ + char *enc_b64; + ssize_t enc_b64_len, ret_len, enc_len; + char *encrypted_blob = NULL; + + enc_len = encrypt_descriptor_data(desc, encoded_str, &encrypted_blob, + is_superencrypted_layer); + /* Get the encoded size plus a NUL terminating byte. */ + enc_b64_len = base64_encode_size(enc_len, BASE64_ENCODE_MULTILINE) + 1; + enc_b64 = tor_malloc_zero(enc_b64_len); + /* Base64 the encrypted blob before returning it. */ + ret_len = base64_encode(enc_b64, enc_b64_len, encrypted_blob, enc_len, + BASE64_ENCODE_MULTILINE); + /* Return length doesn't count the NUL byte. */ + tor_assert(ret_len == (enc_b64_len - 1)); + tor_free(encrypted_blob); + + return enc_b64; +} + +/* Generate and encode the superencrypted portion of <b>desc</b>. This also + * involves generating the encrypted portion of the descriptor, and performing + * the superencryption. A newly allocated NUL-terminated string pointer + * containing the encrypted encoded blob is put in encrypted_blob_out. Return 0 + * on success else a negative value. */ +static int +encode_superencrypted_data(const hs_descriptor_t *desc, + char **encrypted_blob_out) +{ + int ret = -1; + char *layer2_str = NULL; + char *layer2_b64_ciphertext = NULL; + char *layer1_str = NULL; + char *layer1_b64_ciphertext = NULL; + + tor_assert(desc); + tor_assert(encrypted_blob_out); + + /* Func logic: We first create the inner layer of the descriptor (layer2). + * We then encrypt it and use it to create the middle layer of the descriptor + * (layer1). Finally we superencrypt the middle layer and return it to our + * caller. */ + + /* Create inner descriptor layer */ + layer2_str = get_inner_encrypted_layer_plaintext(desc); + if (!layer2_str) { + goto err; + } + + /* Encrypt and b64 the inner layer */ + layer2_b64_ciphertext = encrypt_desc_data_and_base64(desc, layer2_str, 0); + if (!layer2_b64_ciphertext) { + goto err; + } + + /* Now create middle descriptor layer given the inner layer */ + layer1_str = get_outer_encrypted_layer_plaintext(desc,layer2_b64_ciphertext); + if (!layer1_str) { + goto err; } + + /* Encrypt and base64 the middle layer */ + layer1_b64_ciphertext = encrypt_desc_data_and_base64(desc, layer1_str, 1); + if (!layer1_b64_ciphertext) { + goto err; + } + /* Success! */ ret = 0; err: - SMARTLIST_FOREACH(lines, char *, l, tor_free(l)); - smartlist_free(lines); + tor_free(layer1_str); + tor_free(layer2_str); + tor_free(layer2_b64_ciphertext); + + *encrypted_blob_out = layer1_b64_ciphertext; return ret; } @@ -756,7 +1033,7 @@ desc_encode_v3(const hs_descriptor_t *desc, /* Build the superencrypted data section. */ { char *enc_b64_blob=NULL; - if (encode_encrypted_data(desc, &enc_b64_blob) < 0) { + if (encode_superencrypted_data(desc, &enc_b64_blob) < 0) { goto err; } smartlist_add_asprintf(lines, @@ -796,6 +1073,13 @@ desc_encode_v3(const hs_descriptor_t *desc, encoded_str = smartlist_join_strings(lines, "\n", 1, NULL); *encoded_out = encoded_str; + if (strlen(encoded_str) >= hs_cache_get_max_descriptor_size()) { + log_warn(LD_GENERAL, "We just made an HS descriptor that's too big (%d)." + "Failing.", (int)strlen(encoded_str)); + tor_free(encoded_str); + goto err; + } + /* XXX: Trigger a control port event. */ /* Success! */ @@ -894,14 +1178,14 @@ decode_auth_type(hs_desc_encrypted_data_t *desc, const char *list) tor_assert(desc); tor_assert(list); - desc->auth_types = smartlist_new(); - smartlist_split_string(desc->auth_types, list, " ", 0, 0); + desc->intro_auth_types = smartlist_new(); + smartlist_split_string(desc->intro_auth_types, list, " ", 0, 0); /* Validate the types that we at least know about one. */ - SMARTLIST_FOREACH_BEGIN(desc->auth_types, const char *, auth) { - for (int idx = 0; auth_types[idx].identifier; idx++) { - if (!strncmp(auth, auth_types[idx].identifier, - strlen(auth_types[idx].identifier))) { + SMARTLIST_FOREACH_BEGIN(desc->intro_auth_types, const char *, auth) { + for (int idx = 0; intro_auth_types[idx].identifier; idx++) { + if (!strncmp(auth, intro_auth_types[idx].identifier, + strlen(intro_auth_types[idx].identifier))) { match = 1; break; } @@ -1023,41 +1307,31 @@ cert_parse_and_validate(tor_cert_t **cert_out, const char *data, STATIC int encrypted_data_length_is_valid(size_t len) { - /* Check for the minimum length possible. */ - if (len < HS_DESC_ENCRYPTED_MIN_LEN) { + /* Make sure there is enough data for the salt and the mac. The equality is + there to ensure that there is at least one byte of encrypted data. */ + if (len <= HS_DESC_ENCRYPTED_SALT_LEN + DIGEST256_LEN) { log_warn(LD_REND, "Length of descriptor's encrypted data is too small. " "Got %lu but minimum value is %d", - (unsigned long)len, HS_DESC_ENCRYPTED_MIN_LEN); + (unsigned long)len, HS_DESC_ENCRYPTED_SALT_LEN + DIGEST256_LEN); goto err; } - /* Encrypted data has the salt and MAC concatenated to it so remove those - * from the validation calculation. */ - len -= HS_DESC_ENCRYPTED_SALT_LEN + DIGEST256_LEN; - - /* Check that it's aligned on the block size of the crypto algorithm. */ - if (len % HS_DESC_PLAINTEXT_PADDING_MULTIPLE) { - log_warn(LD_REND, "Length of descriptor's encrypted data is invalid. " - "Got %lu which is not a multiple of %d.", - (unsigned long) len, HS_DESC_PLAINTEXT_PADDING_MULTIPLE); - goto err; - } - - /* XXX: Check maximum size. Will strongly depends on the maximum intro point - * allowed we decide on and probably if they will all have to use the legacy - * key which is bigger than the ed25519 key. */ - return 1; err: return 0; } -/* Decrypt the encrypted section of the descriptor using the given descriptor - * object desc. A newly allocated NUL terminated string is put in - * decrypted_out. Return the length of decrypted_out on success else 0 is - * returned and decrypted_out is set to NULL. */ +/** Decrypt an encrypted descriptor layer at <b>encrypted_blob</b> of size + * <b>encrypted_blob_size</b>. Use the descriptor object <b>desc</b> to + * generate the right decryption keys; set <b>decrypted_out</b> to the + * plaintext. If <b>is_superencrypted_layer</b> is set, this is the outter + * encrypted layer of the descriptor. */ static size_t -desc_decrypt_data_v3(const hs_descriptor_t *desc, char **decrypted_out) +decrypt_desc_layer(const hs_descriptor_t *desc, + const uint8_t *encrypted_blob, + size_t encrypted_blob_size, + int is_superencrypted_layer, + char **decrypted_out) { uint8_t *decrypted = NULL; uint8_t secret_key[HS_DESC_ENCRYPTED_KEY_LEN], secret_iv[CIPHER_IV_LEN]; @@ -1067,41 +1341,33 @@ desc_decrypt_data_v3(const hs_descriptor_t *desc, char **decrypted_out) tor_assert(decrypted_out); tor_assert(desc); - tor_assert(desc->plaintext_data.encrypted_blob); + tor_assert(encrypted_blob); - /* Construction is as follow: SALT | ENCRYPTED_DATA | MAC */ - if (!encrypted_data_length_is_valid( - desc->plaintext_data.encrypted_blob_size)) { + /* Construction is as follow: SALT | ENCRYPTED_DATA | MAC . + * Make sure we have enough space for all these things. */ + if (!encrypted_data_length_is_valid(encrypted_blob_size)) { goto err; } /* Start of the blob thus the salt. */ - salt = desc->plaintext_data.encrypted_blob; + salt = encrypted_blob; + /* Next is the encrypted data. */ - encrypted = desc->plaintext_data.encrypted_blob + - HS_DESC_ENCRYPTED_SALT_LEN; - encrypted_len = desc->plaintext_data.encrypted_blob_size - + encrypted = encrypted_blob + HS_DESC_ENCRYPTED_SALT_LEN; + encrypted_len = encrypted_blob_size - (HS_DESC_ENCRYPTED_SALT_LEN + DIGEST256_LEN); + tor_assert(encrypted_len > 0); /* guaranteed by the check above */ - /* At the very end is the MAC. Make sure it's of the right size. */ - { - desc_mac = encrypted + encrypted_len; - size_t desc_mac_size = desc->plaintext_data.encrypted_blob_size - - (desc_mac - desc->plaintext_data.encrypted_blob); - if (desc_mac_size != DIGEST256_LEN) { - log_warn(LD_REND, "Service descriptor MAC length of encrypted data " - "is invalid (%lu, expected %u)", - (unsigned long) desc_mac_size, DIGEST256_LEN); - goto err; - } - } + /* And last comes the MAC. */ + desc_mac = encrypted_blob + encrypted_blob_size - DIGEST256_LEN; /* KDF construction resulting in a key from which the secret key, IV and MAC * key are extracted which is what we need for the decryption. */ build_secret_key_iv_mac(desc, salt, HS_DESC_ENCRYPTED_SALT_LEN, secret_key, sizeof(secret_key), secret_iv, sizeof(secret_iv), - mac_key, sizeof(mac_key)); + mac_key, sizeof(mac_key), + is_superencrypted_layer); /* Build MAC. */ build_mac(mac_key, sizeof(mac_key), salt, HS_DESC_ENCRYPTED_SALT_LEN, @@ -1131,7 +1397,7 @@ desc_decrypt_data_v3(const hs_descriptor_t *desc, char **decrypted_out) } { - /* Adjust length to remove NULL padding bytes */ + /* Adjust length to remove NUL padding bytes */ uint8_t *end = memchr(decrypted, 0, encrypted_len); result_len = encrypted_len; if (end) { @@ -1157,6 +1423,164 @@ desc_decrypt_data_v3(const hs_descriptor_t *desc, char **decrypted_out) return result_len; } +/* Basic validation that the superencrypted client auth portion of the + * descriptor is well-formed and recognized. Return True if so, otherwise + * return False. */ +static int +superencrypted_auth_data_is_valid(smartlist_t *tokens) +{ + /* XXX: This is just basic validation for now. When we implement client auth, + we can refactor this function so that it actually parses and saves the + data. */ + + { /* verify desc auth type */ + const directory_token_t *tok; + tok = find_by_keyword(tokens, R3_DESC_AUTH_TYPE); + tor_assert(tok->n_args >= 1); + if (strcmp(tok->args[0], "x25519")) { + log_warn(LD_DIR, "Unrecognized desc auth type"); + return 0; + } + } + + { /* verify desc auth key */ + const directory_token_t *tok; + curve25519_public_key_t k; + tok = find_by_keyword(tokens, R3_DESC_AUTH_KEY); + tor_assert(tok->n_args >= 1); + if (curve25519_public_from_base64(&k, tok->args[0]) < 0) { + log_warn(LD_DIR, "Bogus desc auth key in HS desc"); + return 0; + } + } + + /* verify desc auth client items */ + SMARTLIST_FOREACH_BEGIN(tokens, const directory_token_t *, tok) { + if (tok->tp == R3_DESC_AUTH_CLIENT) { + tor_assert(tok->n_args >= 3); + } + } SMARTLIST_FOREACH_END(tok); + + return 1; +} + +/* Parse <b>message</b>, the plaintext of the superencrypted portion of an HS + * descriptor. Set <b>encrypted_out</b> to the encrypted blob, and return its + * size */ +STATIC size_t +decode_superencrypted(const char *message, size_t message_len, + uint8_t **encrypted_out) +{ + int retval = 0; + memarea_t *area = NULL; + smartlist_t *tokens = NULL; + + area = memarea_new(); + tokens = smartlist_new(); + if (tokenize_string(area, message, message + message_len, tokens, + hs_desc_superencrypted_v3_token_table, 0) < 0) { + log_warn(LD_REND, "Superencrypted portion is not parseable"); + goto err; + } + + /* Do some rudimentary validation of the authentication data */ + if (!superencrypted_auth_data_is_valid(tokens)) { + log_warn(LD_REND, "Invalid auth data"); + goto err; + } + + /* Extract the encrypted data section. */ + { + const directory_token_t *tok; + tok = find_by_keyword(tokens, R3_ENCRYPTED); + tor_assert(tok->object_body); + if (strcmp(tok->object_type, "MESSAGE") != 0) { + log_warn(LD_REND, "Desc superencrypted data section is invalid"); + goto err; + } + /* Make sure the length of the encrypted blob is valid. */ + if (!encrypted_data_length_is_valid(tok->object_size)) { + goto err; + } + + /* Copy the encrypted blob to the descriptor object so we can handle it + * latter if needed. */ + tor_assert(tok->object_size <= INT_MAX); + *encrypted_out = tor_memdup(tok->object_body, tok->object_size); + retval = (int) tok->object_size; + } + + err: + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); + smartlist_free(tokens); + if (area) { + memarea_drop_all(area); + } + + return retval; +} + +/* Decrypt both the superencrypted and the encrypted section of the descriptor + * using the given descriptor object <b>desc</b>. A newly allocated NUL + * terminated string is put in decrypted_out which contains the inner encrypted + * layer of the descriptor. Return the length of decrypted_out on success else + * 0 is returned and decrypted_out is set to NULL. */ +static size_t +desc_decrypt_all(const hs_descriptor_t *desc, char **decrypted_out) +{ + size_t decrypted_len = 0; + size_t encrypted_len = 0; + size_t superencrypted_len = 0; + char *superencrypted_plaintext = NULL; + uint8_t *encrypted_blob = NULL; + + /** Function logic: This function takes us from the descriptor header to the + * inner encrypted layer, by decrypting and decoding the middle descriptor + * layer. In the end we return the contents of the inner encrypted layer to + * our caller. */ + + /* 1. Decrypt middle layer of descriptor */ + superencrypted_len = decrypt_desc_layer(desc, + desc->plaintext_data.superencrypted_blob, + desc->plaintext_data.superencrypted_blob_size, + 1, + &superencrypted_plaintext); + if (!superencrypted_len) { + log_warn(LD_REND, "Decrypting superencrypted desc failed."); + goto err; + } + tor_assert(superencrypted_plaintext); + + /* 2. Parse "superencrypted" */ + encrypted_len = decode_superencrypted(superencrypted_plaintext, + superencrypted_len, + &encrypted_blob); + if (!encrypted_len) { + log_warn(LD_REND, "Decrypting encrypted desc failed."); + goto err; + } + tor_assert(encrypted_blob); + + /* 3. Decrypt "encrypted" and set decrypted_out */ + char *decrypted_desc; + decrypted_len = decrypt_desc_layer(desc, + encrypted_blob, encrypted_len, + 0, &decrypted_desc); + if (!decrypted_len) { + log_warn(LD_REND, "Decrypting encrypted desc failed."); + goto err; + } + tor_assert(decrypted_desc); + + *decrypted_out = decrypted_desc; + + err: + tor_free(superencrypted_plaintext); + tor_free(encrypted_blob); + + return decrypted_len; +} + /* Given the start of a section and the end of it, decode a single * introduction point from that section. Return a newly allocated introduction * point object containing the decoded data. Return NULL if the section can't @@ -1289,7 +1713,9 @@ decode_introduction_point(const hs_descriptor_t *desc, const char *start) tor_cert_free(cross_cert); SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_free(tokens); - memarea_drop_all(area); + if (area) { + memarea_drop_all(area); + } return ip; } @@ -1508,8 +1934,8 @@ desc_decode_plaintext_v3(smartlist_t *tokens, /* Copy the encrypted blob to the descriptor object so we can handle it * latter if needed. */ - desc->encrypted_blob = tor_memdup(tok->object_body, tok->object_size); - desc->encrypted_blob_size = tok->object_size; + desc->superencrypted_blob = tor_memdup(tok->object_body, tok->object_size); + desc->superencrypted_blob_size = tok->object_size; /* Extract signature and verify it. */ tok = find_by_keyword(tokens, R3_SIGNATURE); @@ -1543,10 +1969,9 @@ desc_decode_encrypted_v3(const hs_descriptor_t *desc, tor_assert(desc); tor_assert(desc_encrypted_out); - /* Decrypt the encrypted data that is located in the plaintext section in - * the descriptor as a blob of bytes. The following functions will use the - * keys found in the same section. */ - message_len = desc_decrypt_data_v3(desc, &message); + /* Decrypt the superencrypted data that is located in the plaintext section + * in the descriptor as a blob of bytes. */ + message_len = desc_decrypt_all(desc, &message); if (!message_len) { log_warn(LD_REND, "Service descriptor decryption failed."); goto err; @@ -1572,7 +1997,7 @@ desc_decode_encrypted_v3(const hs_descriptor_t *desc, } /* Authentication type. It's optional but only once. */ - tok = find_opt_by_keyword(tokens, R3_AUTHENTICATION_REQUIRED); + tok = find_opt_by_keyword(tokens, R3_INTRO_AUTH_REQUIRED); if (tok) { if (!decode_auth_type(desc_encrypted_out, tok->args[0])) { log_warn(LD_REND, "Service descriptor authentication type has " @@ -1654,7 +2079,7 @@ hs_desc_decode_encrypted(const hs_descriptor_t *desc, /* Calling this function without an encrypted blob to parse is a code flow * error. The plaintext parsing should never succeed in the first place * without an encrypted section. */ - tor_assert(desc->plaintext_data.encrypted_blob); + tor_assert(desc->plaintext_data.superencrypted_blob); /* Let's make sure we have a supported version as well. By correctly parsing * the plaintext, this should not fail. */ if (BUG(!hs_desc_is_supported_version(version))) { @@ -1906,6 +2331,6 @@ hs_desc_plaintext_obj_size(const hs_desc_plaintext_data_t *data) { tor_assert(data); return (sizeof(*data) + sizeof(*data->signing_key_cert) + - data->encrypted_blob_size); + data->superencrypted_blob_size); } diff --git a/src/or/hs_descriptor.h b/src/or/hs_descriptor.h index b520d24471..b7d512c06b 100644 --- a/src/or/hs_descriptor.h +++ b/src/or/hs_descriptor.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -41,24 +41,11 @@ * the secret IV and MAC key length which is the length of H() output. */ #define HS_DESC_ENCRYPTED_KDF_OUTPUT_LEN \ CIPHER256_KEY_LEN + CIPHER_IV_LEN + DIGEST256_LEN -/* We need to pad the plaintext version of the encrypted data section before - * encryption and it has to be a multiple of this value. */ -#define HS_DESC_PLAINTEXT_PADDING_MULTIPLE 128 -/* XXX: Let's make sure this makes sense as an upper limit for the padded - * plaintext section. Then we should enforce it as now only an assert will be - * triggered if we are above it. */ -/* Once padded, this is the maximum length in bytes for the plaintext. */ -#define HS_DESC_PADDED_PLAINTEXT_MAX_LEN 8192 -/* Minimum length in bytes of the encrypted portion of the descriptor. */ -#define HS_DESC_ENCRYPTED_MIN_LEN \ - HS_DESC_ENCRYPTED_SALT_LEN + \ - HS_DESC_PLAINTEXT_PADDING_MULTIPLE + DIGEST256_LEN +/* Pad plaintext of superencrypted data section before encryption so that its + * length is a multiple of this value. */ +#define HS_DESC_SUPERENC_PLAINTEXT_PAD_MULTIPLE 10000 /* Maximum length in bytes of a full hidden service descriptor. */ #define HS_DESC_MAX_LEN 50000 /* 50kb max size */ -/* The minimum amount of fields a descriptor should contain. The parsing of - * the fields are version specific so the only required field, as a generic - * view of a descriptor, is 1 that is the version field. */ -#define HS_DESC_PLAINTEXT_MIN_FIELDS 1 /* Key length for the descriptor symmetric encryption. As specified in the * protocol, we use AES-256 for the encrypted section of the descriptor. The @@ -68,8 +55,7 @@ /* Type of authentication in the descriptor. */ typedef enum { - HS_DESC_AUTH_PASSWORD = 1, - HS_DESC_AUTH_ED25519 = 2, + HS_DESC_AUTH_ED25519 = 1 } hs_desc_auth_type_t; /* Type of encryption key in the descriptor. */ @@ -132,7 +118,7 @@ typedef struct hs_desc_encrypted_data_t { /* A list of authentication types that a client must at least support one * in order to contact the service. Contains NULL terminated strings. */ - smartlist_t *auth_types; + smartlist_t *intro_auth_types; /* Is this descriptor a single onion service? */ unsigned int single_onion_service : 1; @@ -167,11 +153,11 @@ typedef struct hs_desc_plaintext_data_t { * has changed. Spec specifies this as a 8 bytes positive integer. */ uint64_t revision_counter; - /* Decoding only: The base64-decoded encrypted blob from the descriptor */ - uint8_t *encrypted_blob; + /* Decoding only: The b64-decoded superencrypted blob from the descriptor */ + uint8_t *superencrypted_blob; - /* Decoding only: Size of the encrypted_blob */ - size_t encrypted_blob_size; + /* Decoding only: Size of the superencrypted_blob */ + size_t superencrypted_blob_size; } hs_desc_plaintext_data_t; /* Service descriptor in its decoded form. */ @@ -242,6 +228,8 @@ STATIC int desc_sig_is_valid(const char *b64_sig, const ed25519_public_key_t *signing_pubkey, const char *encoded_desc, size_t encoded_len); STATIC void desc_intro_point_free(hs_desc_intro_point_t *ip); +STATIC size_t decode_superencrypted(const char *message, size_t message_len, + uint8_t **encrypted_out); #endif /* HS_DESCRIPTOR_PRIVATE */ #endif /* TOR_HS_DESCRIPTOR_H */ diff --git a/src/or/hs_intropoint.c b/src/or/hs_intropoint.c index db4ba7982a..a7282aba8f 100644 --- a/src/or/hs_intropoint.c +++ b/src/or/hs_intropoint.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/hs_intropoint.h b/src/or/hs_intropoint.h index e6024a858f..3a84a48dd8 100644 --- a/src/or/hs_intropoint.h +++ b/src/or/hs_intropoint.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/hs_service.c b/src/or/hs_service.c index 7b12b63065..f3073ce6c0 100644 --- a/src/or/hs_service.c +++ b/src/or/hs_service.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/hs_service.h b/src/or/hs_service.h index 994521fc50..a4e87891da 100644 --- a/src/or/hs_service.h +++ b/src/or/hs_service.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/include.am b/src/or/include.am index 4e54deca55..d65d0d3657 100644 --- a/src/or/include.am +++ b/src/or/include.am @@ -36,6 +36,7 @@ LIBTOR_A_SOURCES = \ src/or/connection.c \ src/or/connection_edge.c \ src/or/connection_or.c \ + src/or/consdiff.c \ src/or/control.c \ src/or/cpuworker.c \ src/or/dircollate.c \ @@ -151,6 +152,7 @@ ORHEADERS = \ src/or/connection.h \ src/or/connection_edge.h \ src/or/connection_or.h \ + src/or/consdiff.h \ src/or/control.h \ src/or/cpuworker.h \ src/or/dircollate.h \ diff --git a/src/or/keypin.c b/src/or/keypin.c index 2d4c4e92d2..1698dc184f 100644 --- a/src/or/keypin.c +++ b/src/or/keypin.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2014-2016, The Tor Project, Inc. */ +/* Copyright (c) 2014-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/keypin.h b/src/or/keypin.h index 673f24d9e3..2564f5befb 100644 --- a/src/or/keypin.h +++ b/src/or/keypin.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2014-2016, The Tor Project, Inc. */ +/* Copyright (c) 2014-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_KEYPIN_H diff --git a/src/or/main.c b/src/or/main.c index 475587eacd..5b73aea70c 100644 --- a/src/or/main.c +++ b/src/or/main.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -1161,6 +1161,7 @@ static int periodic_events_initialized = 0; #define CALLBACK(name) \ static int name ## _callback(time_t, const or_options_t *) CALLBACK(rotate_onion_key); +CALLBACK(check_onion_keys_expiry_time); CALLBACK(check_ed_keys); CALLBACK(launch_descriptor_fetches); CALLBACK(rotate_x509_certificate); @@ -1192,6 +1193,7 @@ CALLBACK(heartbeat); static periodic_event_item_t periodic_events[] = { CALLBACK(rotate_onion_key), + CALLBACK(check_onion_keys_expiry_time), CALLBACK(check_ed_keys), CALLBACK(launch_descriptor_fetches), CALLBACK(rotate_x509_certificate), @@ -1472,17 +1474,18 @@ run_scheduled_events(time_t now) pt_configure_remaining_proxies(); } -/* Periodic callback: Every MIN_ONION_KEY_LIFETIME seconds, rotate the onion - * keys, shut down and restart all cpuworkers, and update our descriptor if - * necessary. +/* Periodic callback: rotate the onion keys after the period defined by the + * "onion-key-rotation-days" consensus parameter, shut down and restart all + * cpuworkers, and update our descriptor if necessary. */ static int rotate_onion_key_callback(time_t now, const or_options_t *options) { if (server_mode(options)) { - time_t rotation_time = get_onion_key_set_at()+MIN_ONION_KEY_LIFETIME; + int onion_key_lifetime = get_onion_key_lifetime(); + time_t rotation_time = get_onion_key_set_at()+onion_key_lifetime; if (rotation_time > now) { - return safe_timer_diff(now, rotation_time); + return ONION_KEY_CONSENSUS_CHECK_INTERVAL; } log_info(LD_GENERAL,"Rotating onion key."); @@ -1493,11 +1496,35 @@ rotate_onion_key_callback(time_t now, const or_options_t *options) } if (advertised_server_mode() && !options->DisableNetwork) router_upload_dir_desc_to_dirservers(0); - return MIN_ONION_KEY_LIFETIME; + return ONION_KEY_CONSENSUS_CHECK_INTERVAL; } return PERIODIC_EVENT_NO_UPDATE; } +/* Period callback: Check if our old onion keys are still valid after the + * period of time defined by the consensus parameter + * "onion-key-grace-period-days", otherwise expire them by setting them to + * NULL. + */ +static int +check_onion_keys_expiry_time_callback(time_t now, const or_options_t *options) +{ + if (server_mode(options)) { + int onion_key_grace_period = get_onion_key_grace_period(); + time_t expiry_time = get_onion_key_set_at()+onion_key_grace_period; + if (expiry_time > now) { + return ONION_KEY_CONSENSUS_CHECK_INTERVAL; + } + + log_info(LD_GENERAL, "Expiring old onion keys."); + expire_old_onion_keys(); + cpuworkers_rotate_keyinfo(); + return ONION_KEY_CONSENSUS_CHECK_INTERVAL; + } + + return PERIODIC_EVENT_NO_UPDATE; +} + /* Periodic callback: Every 30 seconds, check whether it's time to make new * Ed25519 subkeys. */ @@ -3159,6 +3186,7 @@ tor_free_all(int postfork) if (!postfork) { escaped(NULL); esc_router_info(NULL); + clean_up_backtrace_handler(); logs_free_all(); /* free log strings. do this last so logs keep working. */ } } diff --git a/src/or/main.h b/src/or/main.h index 07b22598b1..915d82b7ba 100644 --- a/src/or/main.h +++ b/src/or/main.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/microdesc.c b/src/or/microdesc.c index b9b0e83ce9..a4e6b409c4 100644 --- a/src/or/microdesc.c +++ b/src/or/microdesc.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2009-2016, The Tor Project, Inc. */ +/* Copyright (c) 2009-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/microdesc.h b/src/or/microdesc.h index 5b1365d48c..943873066e 100644 --- a/src/or/microdesc.h +++ b/src/or/microdesc.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/networkstatus.c b/src/or/networkstatus.c index 508cf6c5b6..9d27e576e6 100644 --- a/src/or/networkstatus.c +++ b/src/or/networkstatus.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/networkstatus.h b/src/or/networkstatus.h index 66cd84c88e..8a9f5f0b09 100644 --- a/src/or/networkstatus.h +++ b/src/or/networkstatus.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/nodelist.c b/src/or/nodelist.c index 96e95baf5a..dc4d9a63e3 100644 --- a/src/or/nodelist.c +++ b/src/or/nodelist.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/nodelist.h b/src/or/nodelist.h index 8456d21c6c..b94a066913 100644 --- a/src/or/nodelist.h +++ b/src/or/nodelist.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/ntmain.c b/src/or/ntmain.c index 4c65805b32..e087bd6937 100644 --- a/src/or/ntmain.c +++ b/src/or/ntmain.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/ntmain.h b/src/or/ntmain.h index 31bf38c62c..4b771b1828 100644 --- a/src/or/ntmain.h +++ b/src/or/ntmain.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/onion.c b/src/or/onion.c index 42b9ca4b18..a98b97cb1d 100644 --- a/src/or/onion.c +++ b/src/or/onion.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -238,7 +238,9 @@ onion_pending_add(or_circuit_t *circ, create_cell_t *onionskin) onion_queue_entry_remove(head); log_info(LD_CIRC, "Circuit create request is too old; canceling due to overload."); - circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_RESOURCELIMIT); + if (! TO_CIRCUIT(circ)->marked_for_close) { + circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_RESOURCELIMIT); + } } return 0; } diff --git a/src/or/onion.h b/src/or/onion.h index 19e4a7c381..37a7b08cb6 100644 --- a/src/or/onion.h +++ b/src/or/onion.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/onion_fast.c b/src/or/onion_fast.c index 8dcbfe22d8..146943a273 100644 --- a/src/or/onion_fast.c +++ b/src/or/onion_fast.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/onion_fast.h b/src/or/onion_fast.h index b9626002c3..b31f8e9492 100644 --- a/src/or/onion_fast.h +++ b/src/or/onion_fast.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/onion_ntor.c b/src/or/onion_ntor.c index ded97ee73d..902260b54b 100644 --- a/src/or/onion_ntor.c +++ b/src/or/onion_ntor.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2012-2016, The Tor Project, Inc. */ +/* Copyright (c) 2012-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/onion_ntor.h b/src/or/onion_ntor.h index f637b437fd..158c499de4 100644 --- a/src/or/onion_ntor.h +++ b/src/or/onion_ntor.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2012-2016, The Tor Project, Inc. */ +/* Copyright (c) 2012-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_ONION_NTOR_H diff --git a/src/or/onion_tap.c b/src/or/onion_tap.c index 2769300945..9a65fcf014 100644 --- a/src/or/onion_tap.c +++ b/src/or/onion_tap.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/onion_tap.h b/src/or/onion_tap.h index a2880f6e98..bd625231f4 100644 --- a/src/or/onion_tap.h +++ b/src/or/onion_tap.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/or.h b/src/or/or.h index 83a10299be..d592e4b91c 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -147,8 +147,27 @@ /** Maximum size of a single extrainfo document, as above. */ #define MAX_EXTRAINFO_UPLOAD_SIZE 50000 -/** How often do we rotate onion keys? */ -#define MIN_ONION_KEY_LIFETIME (7*24*60*60) +/** Minimum lifetime for an onion key in days. */ +#define MIN_ONION_KEY_LIFETIME_DAYS (1) + +/** Maximum lifetime for an onion key in days. */ +#define MAX_ONION_KEY_LIFETIME_DAYS (90) + +/** Default lifetime for an onion key in days. */ +#define DEFAULT_ONION_KEY_LIFETIME_DAYS (28) + +/** Minimum grace period for acceptance of an onion key in days. + * The maximum value is defined in proposal #274 as being the current network + * consensus parameter for "onion-key-rotation-days". */ +#define MIN_ONION_KEY_GRACE_PERIOD_DAYS (1) + +/** Default grace period for acceptance of an onion key in days. */ +#define DEFAULT_ONION_KEY_GRACE_PERIOD_DAYS (7) + +/** How often we should check the network consensus if it is time to rotate or + * expire onion keys. */ +#define ONION_KEY_CONSENSUS_CHECK_INTERVAL (60*60) + /** How often do we rotate TLS contexts? */ #define MAX_SSL_KEY_LIFETIME_INTERNAL (2*60*60) diff --git a/src/or/parsecommon.c b/src/or/parsecommon.c index ec2cec69f7..7959867875 100644 --- a/src/or/parsecommon.c +++ b/src/or/parsecommon.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/parsecommon.h b/src/or/parsecommon.h index 15e9f7ae85..f4974a9683 100644 --- a/src/or/parsecommon.h +++ b/src/or/parsecommon.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -157,12 +157,16 @@ typedef enum { R3_SUPERENCRYPTED, R3_SIGNATURE, R3_CREATE2_FORMATS, - R3_AUTHENTICATION_REQUIRED, + R3_INTRO_AUTH_REQUIRED, R3_SINGLE_ONION_SERVICE, R3_INTRODUCTION_POINT, R3_INTRO_AUTH_KEY, R3_INTRO_ENC_KEY, R3_INTRO_ENC_KEY_CERTIFICATION, + R3_DESC_AUTH_TYPE, + R3_DESC_AUTH_KEY, + R3_DESC_AUTH_CLIENT, + R3_ENCRYPTED, R_IPO_IDENTIFIER, R_IPO_IP_ADDRESS, diff --git a/src/or/periodic.c b/src/or/periodic.c index d02d4a7bbb..6896b41c86 100644 --- a/src/or/periodic.c +++ b/src/or/periodic.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2015-2016, The Tor Project, Inc. */ +/* Copyright (c) 2015-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/periodic.h b/src/or/periodic.h index 021bb4ef5c..88d00cc7e9 100644 --- a/src/or/periodic.h +++ b/src/or/periodic.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2015-2016, The Tor Project, Inc. */ +/* Copyright (c) 2015-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_PERIODIC_H diff --git a/src/or/policies.c b/src/or/policies.c index 2aa6373f3e..3d49a6110c 100644 --- a/src/or/policies.c +++ b/src/or/policies.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/policies.h b/src/or/policies.h index f73f850c21..ce08d497e9 100644 --- a/src/or/policies.h +++ b/src/or/policies.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/protover.c b/src/or/protover.c index 88d549ab35..ee58927fc5 100644 --- a/src/or/protover.c +++ b/src/or/protover.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/protover.h b/src/or/protover.h index 5c658931ea..22667bed79 100644 --- a/src/or/protover.h +++ b/src/or/protover.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/reasons.c b/src/or/reasons.c index a1566e2299..e6c325f1b3 100644 --- a/src/or/reasons.c +++ b/src/or/reasons.c @@ -1,5 +1,5 @@ /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/reasons.h b/src/or/reasons.h index 2e12c93728..1cadf4e89e 100644 --- a/src/or/reasons.h +++ b/src/or/reasons.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/relay.c b/src/or/relay.c index 6b3f34f3e5..a7a5071198 100644 --- a/src/or/relay.c +++ b/src/or/relay.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/relay.h b/src/or/relay.h index 3acf3ee0e3..a160cd5551 100644 --- a/src/or/relay.h +++ b/src/or/relay.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/rendcache.c b/src/or/rendcache.c index 12c23ea87c..11b60b36a1 100644 --- a/src/or/rendcache.c +++ b/src/or/rendcache.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2015-2016, The Tor Project, Inc. */ +/* Copyright (c) 2015-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/rendcache.h b/src/or/rendcache.h index 746f142fcc..1bd3be2243 100644 --- a/src/or/rendcache.h +++ b/src/or/rendcache.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2015-2016, The Tor Project, Inc. */ +/* Copyright (c) 2015-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/rendclient.c b/src/or/rendclient.c index 06744ad795..8d2ae03c9e 100644 --- a/src/or/rendclient.c +++ b/src/or/rendclient.c @@ -1,5 +1,5 @@ /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/rendclient.h b/src/or/rendclient.h index 164305a773..ff0f4084fd 100644 --- a/src/or/rendclient.h +++ b/src/or/rendclient.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/rendcommon.c b/src/or/rendcommon.c index bc53762fb6..e1236bdd0f 100644 --- a/src/or/rendcommon.c +++ b/src/or/rendcommon.c @@ -1,5 +1,5 @@ /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/rendcommon.h b/src/or/rendcommon.h index 942ace5761..94c2480d86 100644 --- a/src/or/rendcommon.h +++ b/src/or/rendcommon.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/rendmid.c b/src/or/rendmid.c index 57c8cfac92..a7b37c7cc9 100644 --- a/src/or/rendmid.c +++ b/src/or/rendmid.c @@ -1,5 +1,5 @@ /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/rendmid.h b/src/or/rendmid.h index 347d745853..daf9e2885e 100644 --- a/src/or/rendmid.h +++ b/src/or/rendmid.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/rendservice.c b/src/or/rendservice.c index eeccd4347b..ac231c2d1f 100644 --- a/src/or/rendservice.c +++ b/src/or/rendservice.c @@ -1,5 +1,5 @@ /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -83,6 +83,7 @@ static const smartlist_t* rend_get_service_list( const smartlist_t* substitute_service_list); static smartlist_t* rend_get_service_list_mutable( smartlist_t* substitute_service_list); +static int rend_max_intro_circs_per_period(unsigned int n_intro_points_wanted); /** Represents the mapping from a virtual port of a rendezvous service to * a real port on some IP. @@ -1025,6 +1026,45 @@ rend_service_del_ephemeral(const char *service_id) return 0; } +/* There can be 1 second's delay due to second_elapsed_callback, and perhaps + * another few seconds due to blocking calls. */ +#define INTRO_CIRC_RETRY_PERIOD_SLOP 10 + +/** Log information about the intro point creation rate and current intro + * points for service, upgrading the log level from min_severity to warn if + * we have stopped launching new intro point circuits. */ +static void +rend_log_intro_limit(const rend_service_t *service, int min_severity) +{ + int exceeded_limit = (service->n_intro_circuits_launched >= + rend_max_intro_circs_per_period( + service->n_intro_points_wanted)); + int severity = min_severity; + /* We stopped creating circuits */ + if (exceeded_limit) { + severity = LOG_WARN; + } + time_t intro_period_elapsed = time(NULL) - service->intro_period_started; + tor_assert_nonfatal(intro_period_elapsed >= 0); + /* We delayed resuming circuits longer than expected */ + int exceeded_elapsed = (intro_period_elapsed > INTRO_CIRC_RETRY_PERIOD + + INTRO_CIRC_RETRY_PERIOD_SLOP); + if (exceeded_elapsed) { + severity = LOG_WARN; + } + log_fn(severity, LD_REND, "Hidden service %s %s %d intro points in the last " + "%d seconds%s. Intro circuit launches are limited to %d per %d " + "seconds.", + service->service_id, + exceeded_limit ? "exceeded launch limit with" : "launched", + service->n_intro_circuits_launched, + (int)intro_period_elapsed, + exceeded_elapsed ? " (delayed)" : "", + rend_max_intro_circs_per_period(service->n_intro_points_wanted), + INTRO_CIRC_RETRY_PERIOD); + rend_service_dump_stats(severity); +} + /** Replace the old value of <b>service</b>-\>desc with one that reflects * the other fields in service. */ @@ -1094,7 +1134,8 @@ rend_service_update_descriptor(rend_service_t *service) "descriptor was updated with %d instead.", service->service_id, service->n_intro_points_wanted, have_intro); - rend_service_dump_stats(severity); + /* Now log an informative message about how we might have got here. */ + rend_log_intro_limit(service, severity); } } @@ -4095,8 +4136,12 @@ rend_consider_services_intro_points(void) /* This retry period is important here so we don't stress circuit * creation. */ + if (now > service->intro_period_started + INTRO_CIRC_RETRY_PERIOD) { - /* One period has elapsed; we can try building circuits again. */ + /* One period has elapsed: + * - if we stopped, we can try building circuits again, + * - if we haven't, we reset the circuit creation counts. */ + rend_log_intro_limit(service, LOG_INFO); service->intro_period_started = now; service->n_intro_circuits_launched = 0; } else if (service->n_intro_circuits_launched >= @@ -4104,6 +4149,7 @@ rend_consider_services_intro_points(void) service->n_intro_points_wanted)) { /* We have failed too many times in this period; wait for the next * one before we try to initiate any more connections. */ + rend_log_intro_limit(service, LOG_WARN); continue; } diff --git a/src/or/rendservice.h b/src/or/rendservice.h index 85daaae4e2..6f45f473a5 100644 --- a/src/or/rendservice.h +++ b/src/or/rendservice.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/rephist.c b/src/or/rephist.c index 8bcd7396aa..231130f13c 100644 --- a/src/or/rephist.c +++ b/src/or/rephist.c @@ -1,5 +1,5 @@ /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/rephist.h b/src/or/rephist.h index ff4810a56d..6dd88a3544 100644 --- a/src/or/rephist.h +++ b/src/or/rephist.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/replaycache.c b/src/or/replaycache.c index 8290fa6964..3d42deb90a 100644 --- a/src/or/replaycache.c +++ b/src/or/replaycache.c @@ -1,4 +1,4 @@ - /* Copyright (c) 2012-2016, The Tor Project, Inc. */ + /* Copyright (c) 2012-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/replaycache.h b/src/or/replaycache.h index 64a6caf5f5..0d637939a4 100644 --- a/src/or/replaycache.h +++ b/src/or/replaycache.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2012-2016, The Tor Project, Inc. */ +/* Copyright (c) 2012-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/router.c b/src/or/router.c index e4fa72a283..7fb49e8eba 100644 --- a/src/or/router.c +++ b/src/or/router.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define ROUTER_PRIVATE @@ -148,6 +148,51 @@ dup_onion_keys(crypto_pk_t **key, crypto_pk_t **last) tor_mutex_release(key_lock); } +/** Expire our old set of onion keys. This is done by setting + * last_curve25519_onion_key and lastonionkey to all zero's and NULL + * respectively. + * + * This function does not perform any grace period checks for the old onion + * keys. + */ +void +expire_old_onion_keys(void) +{ + char *fname = NULL; + + tor_mutex_acquire(key_lock); + + /* Free lastonionkey and set it to NULL. */ + if (lastonionkey) { + crypto_pk_free(lastonionkey); + lastonionkey = NULL; + } + + /* We zero out the keypair. See the tor_mem_is_zero() check made in + * construct_ntor_key_map() below. */ + memset(&last_curve25519_onion_key, 0, sizeof(last_curve25519_onion_key)); + + tor_mutex_release(key_lock); + + fname = get_datadir_fname2("keys", "secret_onion_key.old"); + if (file_status(fname) == FN_FILE) { + if (tor_unlink(fname) != 0) { + log_warn(LD_FS, "Couldn't unlink old onion key file %s: %s", + fname, strerror(errno)); + } + } + tor_free(fname); + + fname = get_datadir_fname2("keys", "secret_onion_key_ntor.old"); + if (file_status(fname) == FN_FILE) { + if (tor_unlink(fname) != 0) { + log_warn(LD_FS, "Couldn't unlink old ntor onion key file %s: %s", + fname, strerror(errno)); + } + } + tor_free(fname); +} + /** Return the current secret onion key for the ntor handshake. Must only * be called from the main thread. */ static const curve25519_keypair_t * @@ -683,6 +728,47 @@ v3_authority_check_key_expiry(void) last_warned = now; } +/** Get the lifetime of an onion key in days. This value is defined by the + * network consesus parameter "onion-key-rotation-days". Always returns a value + * between <b>MIN_ONION_KEY_LIFETIME_DAYS</b> and + * <b>MAX_ONION_KEY_LIFETIME_DAYS</b>. + */ +static int +get_onion_key_rotation_days_(void) +{ + return networkstatus_get_param(NULL, + "onion-key-rotation-days", + DEFAULT_ONION_KEY_LIFETIME_DAYS, + MIN_ONION_KEY_LIFETIME_DAYS, + MAX_ONION_KEY_LIFETIME_DAYS); +} + +/** Get the current lifetime of an onion key in seconds. This value is defined + * by the network consesus parameter "onion-key-rotation-days", but the value + * is converted to seconds. + */ +int +get_onion_key_lifetime(void) +{ + return get_onion_key_rotation_days_()*24*60*60; +} + +/** Get the grace period of an onion key in seconds. This value is defined by + * the network consesus parameter "onion-key-grace-period-days", but the value + * is converted to seconds. + */ +int +get_onion_key_grace_period(void) +{ + int grace_period; + grace_period = networkstatus_get_param(NULL, + "onion-key-grace-period-days", + DEFAULT_ONION_KEY_GRACE_PERIOD_DAYS, + MIN_ONION_KEY_GRACE_PERIOD_DAYS, + get_onion_key_rotation_days_()); + return grace_period*24*60*60; +} + /** Set up Tor's TLS contexts, based on our configuration and keys. Return 0 * on success, and -1 on failure. */ int @@ -928,7 +1014,7 @@ init_keys(void) /* We have no LastRotatedOnionKey set; either we just created the key * or it's a holdover from 0.1.2.4-alpha-dev or earlier. In either case, * start the clock ticking now so that we will eventually rotate it even - * if we don't stay up for a full MIN_ONION_KEY_LIFETIME. */ + * if we don't stay up for the full lifetime of an onion key. */ state->LastRotatedOnionKey = onionkey_set_at = now; or_state_mark_dirty(state, options->AvoidDiskWrites ? time(NULL)+3600 : 0); @@ -2760,7 +2846,7 @@ router_dump_router_to_string(routerinfo_t *router, make_ntor_onion_key_crosscert(ntor_keypair, &router->cache_info.signing_key_cert->signing_key, router->cache_info.published_on, - MIN_ONION_KEY_LIFETIME, &sign); + get_onion_key_lifetime(), &sign); if (!cert) { log_warn(LD_BUG,"make_ntor_onion_key_crosscert failed!"); goto err; diff --git a/src/or/router.h b/src/or/router.h index c30a0301b7..9c5def5218 100644 --- a/src/or/router.h +++ b/src/or/router.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -27,10 +27,13 @@ crypto_pk_t *get_my_v3_authority_signing_key(void); authority_cert_t *get_my_v3_legacy_cert(void); crypto_pk_t *get_my_v3_legacy_signing_key(void); void dup_onion_keys(crypto_pk_t **key, crypto_pk_t **last); +void expire_old_onion_keys(void); void rotate_onion_key(void); crypto_pk_t *init_key_from_file(const char *fname, int generate, int severity, int log_greeting); void v3_authority_check_key_expiry(void); +int get_onion_key_lifetime(void); +int get_onion_key_grace_period(void); di_digest256_map_t *construct_ntor_key_map(void); void ntor_key_map_free(di_digest256_map_t *map); diff --git a/src/or/routerkeys.c b/src/or/routerkeys.c index 6259e3f5a0..01f5e98bd1 100644 --- a/src/or/routerkeys.c +++ b/src/or/routerkeys.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2014-2016, The Tor Project, Inc. */ +/* Copyright (c) 2014-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/routerkeys.h b/src/or/routerkeys.h index d2027f4bbe..6453c785b5 100644 --- a/src/or/routerkeys.h +++ b/src/or/routerkeys.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2014-2016, The Tor Project, Inc. */ +/* Copyright (c) 2014-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_ROUTERKEYS_H diff --git a/src/or/routerlist.c b/src/or/routerlist.c index 45b3838792..0b0bb4b1d2 100644 --- a/src/or/routerlist.c +++ b/src/or/routerlist.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/routerlist.h b/src/or/routerlist.h index 8b68d69f28..5376369e8d 100644 --- a/src/or/routerlist.h +++ b/src/or/routerlist.h @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/routerparse.c b/src/or/routerparse.c index 0336c035b4..7220b992c4 100644 --- a/src/or/routerparse.c +++ b/src/or/routerparse.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -2030,6 +2030,9 @@ extrainfo_parse_entry_from_string(const char *s, const char *end, * parse that's covered by the hash. */ int can_dl_again = 0; + if (BUG(s == NULL)) + return NULL; + if (!end) { end = s + strlen(s); } diff --git a/src/or/routerparse.h b/src/or/routerparse.h index 648f29b0d3..e8d71b6dc9 100644 --- a/src/or/routerparse.h +++ b/src/or/routerparse.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/routerset.c b/src/or/routerset.c index d0df0a74e6..4906c6a51d 100644 --- a/src/or/routerset.c +++ b/src/or/routerset.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/routerset.h b/src/or/routerset.h index 2e3b4b0fe0..a63677b471 100644 --- a/src/or/routerset.h +++ b/src/or/routerset.h @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/scheduler.c b/src/or/scheduler.c index 033e6d119c..fac545fba7 100644 --- a/src/or/scheduler.c +++ b/src/or/scheduler.c @@ -1,4 +1,4 @@ -/* * Copyright (c) 2013-2016, The Tor Project, Inc. */ +/* * Copyright (c) 2013-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "or.h" diff --git a/src/or/scheduler.h b/src/or/scheduler.h index 3dcfd2faca..e29c13de7e 100644 --- a/src/or/scheduler.h +++ b/src/or/scheduler.h @@ -1,4 +1,4 @@ -/* * Copyright (c) 2013-2016, The Tor Project, Inc. */ +/* * Copyright (c) 2013-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/shared_random.c b/src/or/shared_random.c index f798a51a9f..73b4c1d0ea 100644 --- a/src/or/shared_random.c +++ b/src/or/shared_random.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/shared_random.h b/src/or/shared_random.h index dbb8effeaa..ae390c469e 100644 --- a/src/or/shared_random.h +++ b/src/or/shared_random.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_SHARED_RANDOM_H diff --git a/src/or/shared_random_state.c b/src/or/shared_random_state.c index 87db9031ee..89d2e8d7f6 100644 --- a/src/or/shared_random_state.c +++ b/src/or/shared_random_state.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/shared_random_state.h b/src/or/shared_random_state.h index 43a7f1d284..3526ad47d3 100644 --- a/src/or/shared_random_state.h +++ b/src/or/shared_random_state.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_SHARED_RANDOM_STATE_H diff --git a/src/or/statefile.c b/src/or/statefile.c index a95ba8533c..d0606b3012 100644 --- a/src/or/statefile.c +++ b/src/or/statefile.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/statefile.h b/src/or/statefile.h index b13743481d..10c09324bc 100644 --- a/src/or/statefile.h +++ b/src/or/statefile.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_STATEFILE_H diff --git a/src/or/status.c b/src/or/status.c index fce6a10157..f7be41e412 100644 --- a/src/or/status.c +++ b/src/or/status.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2010-2016, The Tor Project, Inc. */ +/* Copyright (c) 2010-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/status.h b/src/or/status.h index b97e835037..c1a0033ce0 100644 --- a/src/or/status.h +++ b/src/or/status.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2010-2016, The Tor Project, Inc. */ +/* Copyright (c) 2010-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_STATUS_H diff --git a/src/or/tor_main.c b/src/or/tor_main.c index d67eda2ac9..a3a8838602 100644 --- a/src/or/tor_main.c +++ b/src/or/tor_main.c @@ -1,6 +1,6 @@ /* Copyright 2001-2004 Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ extern const char tor_git_revision[]; diff --git a/src/or/torcert.c b/src/or/torcert.c index c58f3da2d3..ef9e78bfe7 100644 --- a/src/or/torcert.c +++ b/src/or/torcert.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2014-2016, The Tor Project, Inc. */ +/* Copyright (c) 2014-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/torcert.h b/src/or/torcert.h index 090f6b5811..51f7665f1e 100644 --- a/src/or/torcert.h +++ b/src/or/torcert.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2014-2016, The Tor Project, Inc. */ +/* Copyright (c) 2014-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TORCERT_H_INCLUDED diff --git a/src/or/transports.c b/src/or/transports.c index 535393b1a1..31849a8d15 100644 --- a/src/or/transports.c +++ b/src/or/transports.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2011-2016, The Tor Project, Inc. */ +/* Copyright (c) 2011-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/transports.h b/src/or/transports.h index 7de90dcbec..44a9626e50 100644 --- a/src/or/transports.h +++ b/src/or/transports.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/test/Makefile.nmake b/src/test/Makefile.nmake index 0ba56d7036..575198388b 100644 --- a/src/test/Makefile.nmake +++ b/src/test/Makefile.nmake @@ -12,7 +12,7 @@ LIBS = ..\..\..\build-alpha\lib\libevent.lib \ crypt32.lib gdi32.lib user32.lib TEST_OBJECTS = test.obj test_addr.obj test_channel.obj test_channeltls.obj \ - test_containers.obj \ + test_consdiff.obj test_containers.obj \ test_controller_events.obj test_crypto.obj test_data.obj test_dir.obj \ test_checkdir.obj test_microdesc.obj test_pt.obj test_util.obj \ test_config.obj test_connection.obj \ diff --git a/src/test/bench.c b/src/test/bench.c index 99bc686f30..5da9168cdc 100644 --- a/src/test/bench.c +++ b/src/test/bench.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ extern const char tor_git_revision[]; @@ -28,6 +28,7 @@ const char tor_git_revision[] = ""; #include "crypto_curve25519.h" #include "onion_ntor.h" #include "crypto_ed25519.h" +#include "consdiff.h" #if defined(HAVE_CLOCK_GETTIME) && defined(CLOCK_PROCESS_CPUTIME_ID) static uint64_t nanostart; @@ -674,6 +675,27 @@ main(int argc, const char **argv) tor_threads_init(); + if (argc == 4 && !strcmp(argv[1], "diff")) { + init_logging(1); + const int N = 200; + char *f1 = read_file_to_str(argv[2], RFTS_BIN, NULL); + char *f2 = read_file_to_str(argv[3], RFTS_BIN, NULL); + if (! f1 || ! f2) { + perror("X"); + return 1; + } + for (i = 0; i < N; ++i) { + char *diff = consensus_diff_generate(f1, f2); + tor_free(diff); + } + char *diff = consensus_diff_generate(f1, f2); + printf("%s", diff); + tor_free(f1); + tor_free(f2); + tor_free(diff); + return 0; + } + for (i = 1; i < argc; ++i) { if (!strcmp(argv[i], "--list")) { list = 1; diff --git a/src/test/bt_test.py b/src/test/bt_test.py index 30591453b9..4cb3326042 100755 --- a/src/test/bt_test.py +++ b/src/test/bt_test.py @@ -1,4 +1,4 @@ -# Copyright 2013-2015, The Tor Project, Inc +# Copyright 2013-2017, The Tor Project, Inc # See LICENSE for licensing information """ diff --git a/src/test/ed25519_exts_ref.py b/src/test/ed25519_exts_ref.py index d5a3a79910..af5010415e 100644 --- a/src/test/ed25519_exts_ref.py +++ b/src/test/ed25519_exts_ref.py @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright 2014-2015, The Tor Project, Inc +# Copyright 2014-2017, The Tor Project, Inc # See LICENSE for licensing information """ diff --git a/src/test/fakechans.h b/src/test/fakechans.h index fa0e37dbe6..c0de430e3d 100644 --- a/src/test/fakechans.h +++ b/src/test/fakechans.h @@ -1,4 +1,4 @@ - /* Copyright (c) 2014-2016, The Tor Project, Inc. */ + /* Copyright (c) 2014-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_FAKECHANS_H diff --git a/src/test/fuzz/dict/http b/src/test/fuzz/dict/http index 1a7b61e8d4..3b0531579d 100644 --- a/src/test/fuzz/dict/http +++ b/src/test/fuzz/dict/http @@ -4,7 +4,7 @@ # # Extracted from directory_handle_command() in the tor source code # -# Copyright (c) 2016, The Tor Project, Inc. +# Copyright (c) 2016-2017, The Tor Project, Inc. # See LICENSE for licensing information # # Usage: diff --git a/src/test/fuzz/fuzz_consensus.c b/src/test/fuzz/fuzz_consensus.c index f5d22f69ae..6610ade7ad 100644 --- a/src/test/fuzz/fuzz_consensus.c +++ b/src/test/fuzz/fuzz_consensus.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define ROUTERPARSE_PRIVATE #include "or.h" diff --git a/src/test/fuzz/fuzz_descriptor.c b/src/test/fuzz/fuzz_descriptor.c index d19386d77f..1a50beae17 100644 --- a/src/test/fuzz/fuzz_descriptor.c +++ b/src/test/fuzz/fuzz_descriptor.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define ROUTERPARSE_PRIVATE #include "or.h" diff --git a/src/test/fuzz/fuzz_diff.c b/src/test/fuzz/fuzz_diff.c new file mode 100644 index 0000000000..c241f63dc4 --- /dev/null +++ b/src/test/fuzz/fuzz_diff.c @@ -0,0 +1,67 @@ +/* Copyright (c) 2016, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#define CONSDIFF_PRIVATE + +#include "orconfig.h" +#include "or.h" +#include "consdiff.h" + +#include "fuzzing.h" + +static int +mock_consensus_compute_digest_(const char *c, consensus_digest_t *d) +{ + (void)c; + memset(d->sha3_256, 3, sizeof(d->sha3_256)); + return 0; +} + +int +fuzz_init(void) +{ + MOCK(consensus_compute_digest, mock_consensus_compute_digest_); + return 0; +} + +int +fuzz_cleanup(void) +{ + UNMOCK(consensus_compute_digest); + return 0; +} + +int +fuzz_main(const uint8_t *stdin_buf, size_t data_size) +{ +#define SEP "=====\n" +#define SEPLEN strlen(SEP) + const uint8_t *separator = tor_memmem(stdin_buf, data_size, SEP, SEPLEN); + if (! separator) + return 0; + size_t c1_len = separator - stdin_buf; + char *c1 = tor_memdup_nulterm(stdin_buf, c1_len); + size_t c2_len = data_size - c1_len - SEPLEN; + char *c2 = tor_memdup_nulterm(separator + SEPLEN, c2_len); + + char *c3 = consensus_diff_generate(c1, c2); + + if (c3) { + char *c4 = consensus_diff_apply(c1, c3); + tor_assert(c4); + if (strcmp(c2, c4)) { + printf("%s\n", escaped(c1)); + printf("%s\n", escaped(c2)); + printf("%s\n", escaped(c3)); + printf("%s\n", escaped(c4)); + } + tor_assert(! strcmp(c2, c4)); + tor_free(c3); + tor_free(c4); + } + tor_free(c1); + tor_free(c2); + + return 0; +} + diff --git a/src/test/fuzz/fuzz_diff_apply.c b/src/test/fuzz/fuzz_diff_apply.c new file mode 100644 index 0000000000..8d7bf751bf --- /dev/null +++ b/src/test/fuzz/fuzz_diff_apply.c @@ -0,0 +1,65 @@ +/* Copyright (c) 2016, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#define CONSDIFF_PRIVATE + +#include "orconfig.h" +#include "or.h" +#include "consdiff.h" + +#include "fuzzing.h" + +static int +mock_consensus_compute_digest_(const char *c, consensus_digest_t *d) +{ + (void)c; + memset(d->sha3_256, 3, sizeof(d->sha3_256)); + return 0; +} + +static int +mock_consensus_digest_eq_(const uint8_t *a, const uint8_t *b) +{ + (void)a; + (void)b; + return 1; +} + +int +fuzz_init(void) +{ + MOCK(consensus_compute_digest, mock_consensus_compute_digest_); + MOCK(consensus_digest_eq, mock_consensus_digest_eq_); + return 0; +} + +int +fuzz_cleanup(void) +{ + UNMOCK(consensus_compute_digest); + UNMOCK(consensus_digest_eq); + return 0; +} + +int +fuzz_main(const uint8_t *stdin_buf, size_t data_size) +{ +#define SEP "=====\n" +#define SEPLEN strlen(SEP) + const uint8_t *separator = tor_memmem(stdin_buf, data_size, SEP, SEPLEN); + if (! separator) + return 0; + size_t c1_len = separator - stdin_buf; + char *c1 = tor_memdup_nulterm(stdin_buf, c1_len); + size_t c2_len = data_size - c1_len - SEPLEN; + char *c2 = tor_memdup_nulterm(separator + SEPLEN, c2_len); + + char *c3 = consensus_diff_apply(c1, c2); + + tor_free(c1); + tor_free(c2); + tor_free(c3); + + return 0; +} + diff --git a/src/test/fuzz/fuzz_extrainfo.c b/src/test/fuzz/fuzz_extrainfo.c index 6251e606d0..2a3de7ecf7 100644 --- a/src/test/fuzz/fuzz_extrainfo.c +++ b/src/test/fuzz/fuzz_extrainfo.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define ROUTERPARSE_PRIVATE #include "or.h" diff --git a/src/test/fuzz/fuzz_hsdescv2.c b/src/test/fuzz/fuzz_hsdescv2.c index 53b7cbe2f7..19db265716 100644 --- a/src/test/fuzz/fuzz_hsdescv2.c +++ b/src/test/fuzz/fuzz_hsdescv2.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define ROUTERPARSE_PRIVATE #include "or.h" diff --git a/src/test/fuzz/fuzz_http.c b/src/test/fuzz/fuzz_http.c index 01c3815f18..ce52acfb5b 100644 --- a/src/test/fuzz/fuzz_http.c +++ b/src/test/fuzz/fuzz_http.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/test/fuzz/fuzz_iptsv2.c b/src/test/fuzz/fuzz_iptsv2.c index 341d4880bd..4abde0c16d 100644 --- a/src/test/fuzz/fuzz_iptsv2.c +++ b/src/test/fuzz/fuzz_iptsv2.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define ROUTERPARSE_PRIVATE #include "or.h" diff --git a/src/test/fuzz/fuzz_microdesc.c b/src/test/fuzz/fuzz_microdesc.c index bb89546191..396115026e 100644 --- a/src/test/fuzz/fuzz_microdesc.c +++ b/src/test/fuzz/fuzz_microdesc.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define ROUTERPARSE_PRIVATE #include "or.h" diff --git a/src/test/fuzz/fuzz_vrs.c b/src/test/fuzz/fuzz_vrs.c index 9301a9bcc8..baf0610a0b 100644 --- a/src/test/fuzz/fuzz_vrs.c +++ b/src/test/fuzz/fuzz_vrs.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define ROUTERPARSE_PRIVATE #define NETWORKSTATUS_PRIVATE diff --git a/src/test/fuzz/fuzzing.h b/src/test/fuzz/fuzzing.h index 4295743458..aecdbb4e52 100644 --- a/src/test/fuzz/fuzzing.h +++ b/src/test/fuzz/fuzzing.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef FUZZING_H #define FUZZING_H diff --git a/src/test/fuzz/fuzzing_common.c b/src/test/fuzz/fuzzing_common.c index e4920d3ee7..f540176204 100644 --- a/src/test/fuzz/fuzzing_common.c +++ b/src/test/fuzz/fuzzing_common.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define CRYPTO_ED25519_PRIVATE #include "orconfig.h" diff --git a/src/test/fuzz/include.am b/src/test/fuzz/include.am index 806710879b..1b608c6885 100644 --- a/src/test/fuzz/include.am +++ b/src/test/fuzz/include.am @@ -1,4 +1,5 @@ - +# This file was generated by fuzzing_include_am.py; do not hand-edit unless +# you enjoy having your changes erased. FUZZING_CPPFLAGS = \ $(src_test_AM_CPPFLAGS) $(TEST_CPPFLAGS) FUZZING_CFLAGS = \ @@ -32,6 +33,16 @@ oss-fuzz-prereqs: \ noinst_HEADERS += \ src/test/fuzz/fuzzing.h +LIBFUZZER = /home/nickm/build/libfuzz/libFuzzer.a +LIBFUZZER_CPPFLAGS = $(FUZZING_CPPFLAGS) -DLLVM_FUZZ +LIBFUZZER_CFLAGS = $(FUZZING_CFLAGS) +LIBFUZZER_LDFLAG = $(FUZZING_LDFLAG) +LIBFUZZER_LIBS = $(FUZZING_LIBS) $(LIBFUZZER) -lstdc++ + +LIBOSS_FUZZ_CPPFLAGS = $(FUZZING_CPPFLAGS) -DLLVM_FUZZ +LIBOSS_FUZZ_CFLAGS = $(FUZZING_CFLAGS) + +# ===== AFL fuzzers src_test_fuzz_fuzz_consensus_SOURCES = \ src/test/fuzz/fuzzing_common.c \ src/test/fuzz/fuzz_consensus.c @@ -48,13 +59,29 @@ src_test_fuzz_fuzz_descriptor_CFLAGS = $(FUZZING_CFLAGS) src_test_fuzz_fuzz_descriptor_LDFLAGS = $(FUZZING_LDFLAG) src_test_fuzz_fuzz_descriptor_LDADD = $(FUZZING_LIBS) -src_test_fuzz_fuzz_http_SOURCES = \ +src_test_fuzz_fuzz_diff_SOURCES = \ src/test/fuzz/fuzzing_common.c \ - src/test/fuzz/fuzz_http.c -src_test_fuzz_fuzz_http_CPPFLAGS = $(FUZZING_CPPFLAGS) -src_test_fuzz_fuzz_http_CFLAGS = $(FUZZING_CFLAGS) -src_test_fuzz_fuzz_http_LDFLAGS = $(FUZZING_LDFLAG) -src_test_fuzz_fuzz_http_LDADD = $(FUZZING_LIBS) + src/test/fuzz/fuzz_diff.c +src_test_fuzz_fuzz_diff_CPPFLAGS = $(FUZZING_CPPFLAGS) +src_test_fuzz_fuzz_diff_CFLAGS = $(FUZZING_CFLAGS) +src_test_fuzz_fuzz_diff_LDFLAGS = $(FUZZING_LDFLAG) +src_test_fuzz_fuzz_diff_LDADD = $(FUZZING_LIBS) + +src_test_fuzz_fuzz_diff_apply_SOURCES = \ + src/test/fuzz/fuzzing_common.c \ + src/test/fuzz/fuzz_diff_apply.c +src_test_fuzz_fuzz_diff_apply_CPPFLAGS = $(FUZZING_CPPFLAGS) +src_test_fuzz_fuzz_diff_apply_CFLAGS = $(FUZZING_CFLAGS) +src_test_fuzz_fuzz_diff_apply_LDFLAGS = $(FUZZING_LDFLAG) +src_test_fuzz_fuzz_diff_apply_LDADD = $(FUZZING_LIBS) + +src_test_fuzz_fuzz_extrainfo_SOURCES = \ + src/test/fuzz/fuzzing_common.c \ + src/test/fuzz/fuzz_extrainfo.c +src_test_fuzz_fuzz_extrainfo_CPPFLAGS = $(FUZZING_CPPFLAGS) +src_test_fuzz_fuzz_extrainfo_CFLAGS = $(FUZZING_CFLAGS) +src_test_fuzz_fuzz_extrainfo_LDFLAGS = $(FUZZING_LDFLAG) +src_test_fuzz_fuzz_extrainfo_LDADD = $(FUZZING_LIBS) src_test_fuzz_fuzz_hsdescv2_SOURCES = \ src/test/fuzz/fuzzing_common.c \ @@ -64,6 +91,14 @@ src_test_fuzz_fuzz_hsdescv2_CFLAGS = $(FUZZING_CFLAGS) src_test_fuzz_fuzz_hsdescv2_LDFLAGS = $(FUZZING_LDFLAG) src_test_fuzz_fuzz_hsdescv2_LDADD = $(FUZZING_LIBS) +src_test_fuzz_fuzz_http_SOURCES = \ + src/test/fuzz/fuzzing_common.c \ + src/test/fuzz/fuzz_http.c +src_test_fuzz_fuzz_http_CPPFLAGS = $(FUZZING_CPPFLAGS) +src_test_fuzz_fuzz_http_CFLAGS = $(FUZZING_CFLAGS) +src_test_fuzz_fuzz_http_LDFLAGS = $(FUZZING_LDFLAG) +src_test_fuzz_fuzz_http_LDADD = $(FUZZING_LIBS) + src_test_fuzz_fuzz_iptsv2_SOURCES = \ src/test/fuzz/fuzzing_common.c \ src/test/fuzz/fuzz_iptsv2.c @@ -72,14 +107,6 @@ src_test_fuzz_fuzz_iptsv2_CFLAGS = $(FUZZING_CFLAGS) src_test_fuzz_fuzz_iptsv2_LDFLAGS = $(FUZZING_LDFLAG) src_test_fuzz_fuzz_iptsv2_LDADD = $(FUZZING_LIBS) -src_test_fuzz_fuzz_extrainfo_SOURCES = \ - src/test/fuzz/fuzzing_common.c \ - src/test/fuzz/fuzz_extrainfo.c -src_test_fuzz_fuzz_extrainfo_CPPFLAGS = $(FUZZING_CPPFLAGS) -src_test_fuzz_fuzz_extrainfo_CFLAGS = $(FUZZING_CFLAGS) -src_test_fuzz_fuzz_extrainfo_LDFLAGS = $(FUZZING_LDFLAG) -src_test_fuzz_fuzz_extrainfo_LDADD = $(FUZZING_LIBS) - src_test_fuzz_fuzz_microdesc_SOURCES = \ src/test/fuzz/fuzzing_common.c \ src/test/fuzz/fuzz_microdesc.c @@ -99,19 +126,16 @@ src_test_fuzz_fuzz_vrs_LDADD = $(FUZZING_LIBS) FUZZERS = \ src/test/fuzz/fuzz-consensus \ src/test/fuzz/fuzz-descriptor \ + src/test/fuzz/fuzz-diff \ + src/test/fuzz/fuzz-diff-apply \ src/test/fuzz/fuzz-extrainfo \ - src/test/fuzz/fuzz-http \ src/test/fuzz/fuzz-hsdescv2 \ + src/test/fuzz/fuzz-http \ src/test/fuzz/fuzz-iptsv2 \ src/test/fuzz/fuzz-microdesc \ src/test/fuzz/fuzz-vrs - -LIBFUZZER = /home/nickm/build/libfuzz/libFuzzer.a -LIBFUZZER_CPPFLAGS = $(FUZZING_CPPFLAGS) -DLLVM_FUZZ -LIBFUZZER_CFLAGS = $(FUZZING_CFLAGS) -LIBFUZZER_LDFLAG = $(FUZZING_LDFLAG) -LIBFUZZER_LIBS = $(FUZZING_LIBS) $(LIBFUZZER) -lstdc++ +# ===== libfuzzer if LIBFUZZER_ENABLED src_test_fuzz_lf_fuzz_consensus_SOURCES = \ @@ -128,6 +152,20 @@ src_test_fuzz_lf_fuzz_descriptor_CFLAGS = $(LIBFUZZER_CFLAGS) src_test_fuzz_lf_fuzz_descriptor_LDFLAGS = $(LIBFUZZER_LDFLAG) src_test_fuzz_lf_fuzz_descriptor_LDADD = $(LIBFUZZER_LIBS) +src_test_fuzz_lf_fuzz_diff_SOURCES = \ + $(src_test_fuzz_fuzz_diff_SOURCES) +src_test_fuzz_lf_fuzz_diff_CPPFLAGS = $(LIBFUZZER_CPPFLAGS) +src_test_fuzz_lf_fuzz_diff_CFLAGS = $(LIBFUZZER_CFLAGS) +src_test_fuzz_lf_fuzz_diff_LDFLAGS = $(LIBFUZZER_LDFLAG) +src_test_fuzz_lf_fuzz_diff_LDADD = $(LIBFUZZER_LIBS) + +src_test_fuzz_lf_fuzz_diff_apply_SOURCES = \ + $(src_test_fuzz_fuzz_diff_apply_SOURCES) +src_test_fuzz_lf_fuzz_diff_apply_CPPFLAGS = $(LIBFUZZER_CPPFLAGS) +src_test_fuzz_lf_fuzz_diff_apply_CFLAGS = $(LIBFUZZER_CFLAGS) +src_test_fuzz_lf_fuzz_diff_apply_LDFLAGS = $(LIBFUZZER_LDFLAG) +src_test_fuzz_lf_fuzz_diff_apply_LDADD = $(LIBFUZZER_LIBS) + src_test_fuzz_lf_fuzz_extrainfo_SOURCES = \ $(src_test_fuzz_fuzz_extrainfo_SOURCES) src_test_fuzz_lf_fuzz_extrainfo_CPPFLAGS = $(LIBFUZZER_CPPFLAGS) @@ -135,13 +173,6 @@ src_test_fuzz_lf_fuzz_extrainfo_CFLAGS = $(LIBFUZZER_CFLAGS) src_test_fuzz_lf_fuzz_extrainfo_LDFLAGS = $(LIBFUZZER_LDFLAG) src_test_fuzz_lf_fuzz_extrainfo_LDADD = $(LIBFUZZER_LIBS) -src_test_fuzz_lf_fuzz_http_SOURCES = \ - $(src_test_fuzz_fuzz_http_SOURCES) -src_test_fuzz_lf_fuzz_http_CPPFLAGS = $(LIBFUZZER_CPPFLAGS) -src_test_fuzz_lf_fuzz_http_CFLAGS = $(LIBFUZZER_CFLAGS) -src_test_fuzz_lf_fuzz_http_LDFLAGS = $(LIBFUZZER_LDFLAG) -src_test_fuzz_lf_fuzz_http_LDADD = $(LIBFUZZER_LIBS) - src_test_fuzz_lf_fuzz_hsdescv2_SOURCES = \ $(src_test_fuzz_fuzz_hsdescv2_SOURCES) src_test_fuzz_lf_fuzz_hsdescv2_CPPFLAGS = $(LIBFUZZER_CPPFLAGS) @@ -149,6 +180,13 @@ src_test_fuzz_lf_fuzz_hsdescv2_CFLAGS = $(LIBFUZZER_CFLAGS) src_test_fuzz_lf_fuzz_hsdescv2_LDFLAGS = $(LIBFUZZER_LDFLAG) src_test_fuzz_lf_fuzz_hsdescv2_LDADD = $(LIBFUZZER_LIBS) +src_test_fuzz_lf_fuzz_http_SOURCES = \ + $(src_test_fuzz_fuzz_http_SOURCES) +src_test_fuzz_lf_fuzz_http_CPPFLAGS = $(LIBFUZZER_CPPFLAGS) +src_test_fuzz_lf_fuzz_http_CFLAGS = $(LIBFUZZER_CFLAGS) +src_test_fuzz_lf_fuzz_http_LDFLAGS = $(LIBFUZZER_LDFLAG) +src_test_fuzz_lf_fuzz_http_LDADD = $(LIBFUZZER_LIBS) + src_test_fuzz_lf_fuzz_iptsv2_SOURCES = \ $(src_test_fuzz_fuzz_iptsv2_SOURCES) src_test_fuzz_lf_fuzz_iptsv2_CPPFLAGS = $(LIBFUZZER_CPPFLAGS) @@ -173,9 +211,11 @@ src_test_fuzz_lf_fuzz_vrs_LDADD = $(LIBFUZZER_LIBS) LIBFUZZER_FUZZERS = \ src/test/fuzz/lf-fuzz-consensus \ src/test/fuzz/lf-fuzz-descriptor \ + src/test/fuzz/lf-fuzz-diff \ + src/test/fuzz/lf-fuzz-diff-apply \ src/test/fuzz/lf-fuzz-extrainfo \ - src/test/fuzz/lf-fuzz-http \ src/test/fuzz/lf-fuzz-hsdescv2 \ + src/test/fuzz/lf-fuzz-http \ src/test/fuzz/lf-fuzz-iptsv2 \ src/test/fuzz/lf-fuzz-microdesc \ src/test/fuzz/lf-fuzz-vrs @@ -184,10 +224,9 @@ else LIBFUZZER_FUZZERS = endif -if OSS_FUZZ_ENABLED -LIBOSS_FUZZ_CPPFLAGS = $(FUZZING_CPPFLAGS) -DLLVM_FUZZ -LIBOSS_FUZZ_CFLAGS = $(FUZZING_CFLAGS) +# ===== oss-fuzz +if OSS_FUZZ_ENABLED src_test_fuzz_liboss_fuzz_consensus_a_SOURCES = \ $(src_test_fuzz_fuzz_consensus_SOURCES) src_test_fuzz_liboss_fuzz_consensus_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS) @@ -198,21 +237,31 @@ src_test_fuzz_liboss_fuzz_descriptor_a_SOURCES = \ src_test_fuzz_liboss_fuzz_descriptor_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS) src_test_fuzz_liboss_fuzz_descriptor_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS) +src_test_fuzz_liboss_fuzz_diff_a_SOURCES = \ + $(src_test_fuzz_fuzz_diff_SOURCES) +src_test_fuzz_liboss_fuzz_diff_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS) +src_test_fuzz_liboss_fuzz_diff_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS) + +src_test_fuzz_liboss_fuzz_diff_apply_a_SOURCES = \ + $(src_test_fuzz_fuzz_diff_apply_SOURCES) +src_test_fuzz_liboss_fuzz_diff_apply_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS) +src_test_fuzz_liboss_fuzz_diff_apply_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS) + src_test_fuzz_liboss_fuzz_extrainfo_a_SOURCES = \ $(src_test_fuzz_fuzz_extrainfo_SOURCES) src_test_fuzz_liboss_fuzz_extrainfo_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS) src_test_fuzz_liboss_fuzz_extrainfo_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS) -src_test_fuzz_liboss_fuzz_http_a_SOURCES = \ - $(src_test_fuzz_fuzz_http_SOURCES) -src_test_fuzz_liboss_fuzz_http_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS) -src_test_fuzz_liboss_fuzz_http_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS) - src_test_fuzz_liboss_fuzz_hsdescv2_a_SOURCES = \ $(src_test_fuzz_fuzz_hsdescv2_SOURCES) src_test_fuzz_liboss_fuzz_hsdescv2_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS) src_test_fuzz_liboss_fuzz_hsdescv2_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS) +src_test_fuzz_liboss_fuzz_http_a_SOURCES = \ + $(src_test_fuzz_fuzz_http_SOURCES) +src_test_fuzz_liboss_fuzz_http_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS) +src_test_fuzz_liboss_fuzz_http_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS) + src_test_fuzz_liboss_fuzz_iptsv2_a_SOURCES = \ $(src_test_fuzz_fuzz_iptsv2_SOURCES) src_test_fuzz_liboss_fuzz_iptsv2_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS) @@ -231,12 +280,15 @@ src_test_fuzz_liboss_fuzz_vrs_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS) OSS_FUZZ_FUZZERS = \ src/test/fuzz/liboss-fuzz-consensus.a \ src/test/fuzz/liboss-fuzz-descriptor.a \ + src/test/fuzz/liboss-fuzz-diff.a \ + src/test/fuzz/liboss-fuzz-diff-apply.a \ src/test/fuzz/liboss-fuzz-extrainfo.a \ - src/test/fuzz/liboss-fuzz-http.a \ src/test/fuzz/liboss-fuzz-hsdescv2.a \ + src/test/fuzz/liboss-fuzz-http.a \ src/test/fuzz/liboss-fuzz-iptsv2.a \ src/test/fuzz/liboss-fuzz-microdesc.a \ src/test/fuzz/liboss-fuzz-vrs.a + else OSS_FUZZ_FUZZERS = endif diff --git a/src/test/fuzz_static_testcases.sh b/src/test/fuzz_static_testcases.sh index bfe1677573..3cb45ad5e6 100755 --- a/src/test/fuzz_static_testcases.sh +++ b/src/test/fuzz_static_testcases.sh @@ -1,6 +1,6 @@ #!/bin/sh -# Copyright (c) 2016, The Tor Project, Inc. +# Copyright (c) 2016-2017, The Tor Project, Inc. # See LICENSE for licensing information set -e diff --git a/src/test/include.am b/src/test/include.am index 1c0726fd3a..7ced7baa77 100644 --- a/src/test/include.am +++ b/src/test/include.am @@ -86,6 +86,7 @@ src_test_test_SOURCES = \ src/test/test_compat_libevent.c \ src/test/test_config.c \ src/test/test_connection.c \ + src/test/test_consdiff.c \ src/test/test_containers.c \ src/test/test_controller.c \ src/test/test_controller_events.c \ @@ -129,6 +130,7 @@ src_test_test_SOURCES = \ src/test/test_shared_random.c \ src/test/test_socks.c \ src/test/test_status.c \ + src/test/test_storagedir.c \ src/test/test_threads.c \ src/test/test_tortls.c \ src/test/test_util.c \ diff --git a/src/test/log_test_helpers.c b/src/test/log_test_helpers.c index c788a33c17..d5a39cfeee 100644 --- a/src/test/log_test_helpers.c +++ b/src/test/log_test_helpers.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2015-2016, The Tor Project, Inc. */ +/* Copyright (c) 2015-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define LOG_PRIVATE #include "torlog.h" diff --git a/src/test/log_test_helpers.h b/src/test/log_test_helpers.h index 19c5a881a4..f7798c0249 100644 --- a/src/test/log_test_helpers.h +++ b/src/test/log_test_helpers.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2014-2016, The Tor Project, Inc. */ +/* Copyright (c) 2014-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "or.h" diff --git a/src/test/ntor_ref.py b/src/test/ntor_ref.py index df065853f3..c753588f97 100755 --- a/src/test/ntor_ref.py +++ b/src/test/ntor_ref.py @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright 2012-2015, The Tor Project, Inc +# Copyright 2012-2017, The Tor Project, Inc # See LICENSE for licensing information """ diff --git a/src/test/rend_test_helpers.c b/src/test/rend_test_helpers.c index 377337bcb9..f7880046fb 100644 --- a/src/test/rend_test_helpers.c +++ b/src/test/rend_test_helpers.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2014-2016, The Tor Project, Inc. */ +/* Copyright (c) 2014-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "or.h" diff --git a/src/test/rend_test_helpers.h b/src/test/rend_test_helpers.h index 180a4e8fde..486adba436 100644 --- a/src/test/rend_test_helpers.h +++ b/src/test/rend_test_helpers.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2014-2016, The Tor Project, Inc. */ +/* Copyright (c) 2014-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "or.h" diff --git a/src/test/test-child.c b/src/test/test-child.c index fdf3ccec0a..f0bdb3ea26 100644 --- a/src/test/test-child.c +++ b/src/test/test-child.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2011-2016, The Tor Project, Inc. */ +/* Copyright (c) 2011-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/test/test-timers.c b/src/test/test-timers.c index b5fcade7f8..99715f4333 100644 --- a/src/test/test-timers.c +++ b/src/test/test-timers.c @@ -1,4 +1,4 @@ -/* Copyright 2016, The Tor Project, Inc. */ +/* Copyright 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/test/test.c b/src/test/test.c index 866408e856..d9723d55d3 100644 --- a/src/test/test.c +++ b/src/test/test.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -1194,6 +1194,7 @@ struct testgroup_t testgroups[] = { { "compat/libevent/", compat_libevent_tests }, { "config/", config_tests }, { "connection/", connection_tests }, + { "consdiff/", consdiff_tests }, { "container/", container_tests }, { "control/", controller_tests }, { "control/event/", controller_event_tests }, @@ -1232,6 +1233,7 @@ struct testgroup_t testgroups[] = { { "socks/", socks_tests }, { "shared-random/", sr_tests }, { "status/" , status_tests }, + { "storagedir/", storagedir_tests }, { "tortls/", tortls_tests }, { "util/", util_tests }, { "util/format/", util_format_tests }, diff --git a/src/test/test.h b/src/test/test.h index 2bd58f51c8..0692040656 100644 --- a/src/test/test.h +++ b/src/test/test.h @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2003, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_TEST_H @@ -189,6 +189,7 @@ extern struct testcase_t circuituse_tests[]; extern struct testcase_t compat_libevent_tests[]; extern struct testcase_t config_tests[]; extern struct testcase_t connection_tests[]; +extern struct testcase_t consdiff_tests[]; extern struct testcase_t container_tests[]; extern struct testcase_t controller_tests[]; extern struct testcase_t controller_event_tests[]; @@ -227,6 +228,7 @@ extern struct testcase_t routerkeys_tests[]; extern struct testcase_t routerlist_tests[]; extern struct testcase_t routerset_tests[]; extern struct testcase_t scheduler_tests[]; +extern struct testcase_t storagedir_tests[]; extern struct testcase_t socks_tests[]; extern struct testcase_t status_tests[]; extern struct testcase_t thread_tests[]; diff --git a/src/test/test_addr.c b/src/test/test_addr.c index 49e248014f..b6d2445bad 100644 --- a/src/test/test_addr.c +++ b/src/test/test_addr.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define ADDRESSMAP_PRIVATE diff --git a/src/test/test_address.c b/src/test/test_address.c index 0d142ad483..50a0574522 100644 --- a/src/test/test_address.c +++ b/src/test/test_address.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2014-2016, The Tor Project, Inc. */ +/* Copyright (c) 2014-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define ADDRESS_PRIVATE diff --git a/src/test/test_bt_cl.c b/src/test/test_bt_cl.c index 709d599f52..ed588ecc5b 100644 --- a/src/test/test_bt_cl.c +++ b/src/test/test_bt_cl.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2012-2016, The Tor Project, Inc. */ +/* Copyright (c) 2012-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/test/test_buffers.c b/src/test/test_buffers.c index 9e7bdb8911..d14165c45c 100644 --- a/src/test/test_buffers.c +++ b/src/test/test_buffers.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define BUFFERS_PRIVATE diff --git a/src/test/test_cell_formats.c b/src/test/test_cell_formats.c index 22c34b6d6c..007f7e3d3e 100644 --- a/src/test/test_cell_formats.c +++ b/src/test/test_cell_formats.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/test/test_cell_queue.c b/src/test/test_cell_queue.c index 93ac9854d8..69e89b69b0 100644 --- a/src/test/test_cell_queue.c +++ b/src/test/test_cell_queue.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2016, The Tor Project, Inc. */ +/* Copyright (c) 2013-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define CIRCUITLIST_PRIVATE diff --git a/src/test/test_channel.c b/src/test/test_channel.c index 862bd6dfa6..f5999b8e67 100644 --- a/src/test/test_channel.c +++ b/src/test/test_channel.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2016, The Tor Project, Inc. */ +/* Copyright (c) 2013-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define TOR_CHANNEL_INTERNAL_ diff --git a/src/test/test_channeltls.c b/src/test/test_channeltls.c index fd98ee40fb..96c5eba9a5 100644 --- a/src/test/test_channeltls.c +++ b/src/test/test_channeltls.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2014-2016, The Tor Project, Inc. */ +/* Copyright (c) 2014-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/test/test_checkdir.c b/src/test/test_checkdir.c index fbb33f87f6..38f3360b61 100644 --- a/src/test/test_checkdir.c +++ b/src/test/test_checkdir.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2014-2016, The Tor Project, Inc. */ +/* Copyright (c) 2014-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/test/test_circuitlist.c b/src/test/test_circuitlist.c index 7eed5fe225..61d3b43059 100644 --- a/src/test/test_circuitlist.c +++ b/src/test/test_circuitlist.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2016, The Tor Project, Inc. */ +/* Copyright (c) 2013-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define TOR_CHANNEL_INTERNAL_ diff --git a/src/test/test_circuitmux.c b/src/test/test_circuitmux.c index 9e8fb54964..779783299d 100644 --- a/src/test/test_circuitmux.c +++ b/src/test/test_circuitmux.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2016, The Tor Project, Inc. */ +/* Copyright (c) 2013-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define TOR_CHANNEL_INTERNAL_ diff --git a/src/test/test_circuituse.c b/src/test/test_circuituse.c index 27a87660ff..5cc9fe571e 100644 --- a/src/test/test_circuituse.c +++ b/src/test/test_circuituse.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define CIRCUITLIST_PRIVATE diff --git a/src/test/test_compat_libevent.c b/src/test/test_compat_libevent.c index 0443cc0b1c..7dd8e65194 100644 --- a/src/test/test_compat_libevent.c +++ b/src/test/test_compat_libevent.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2010-2016, The Tor Project, Inc. */ +/* Copyright (c) 2010-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define COMPAT_LIBEVENT_PRIVATE diff --git a/src/test/test_config.c b/src/test/test_config.c index 3b19cad036..f0874e0e0b 100644 --- a/src/test/test_config.c +++ b/src/test/test_config.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/test/test_connection.c b/src/test/test_connection.c index 5cda4f3175..05dd6f4ec9 100644 --- a/src/test/test_connection.c +++ b/src/test/test_connection.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2015-2016, The Tor Project, Inc. */ +/* Copyright (c) 2015-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/test/test_consdiff.c b/src/test/test_consdiff.c new file mode 100644 index 0000000000..1b4e2ad3c1 --- /dev/null +++ b/src/test/test_consdiff.c @@ -0,0 +1,1097 @@ +/* Copyright (c) 2014, Daniel Martà + * Copyright (c) 2014, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#define CONSDIFF_PRIVATE + +#include "or.h" +#include "test.h" + +#include "consdiff.h" +#include "memarea.h" +#include "log_test_helpers.h" + +#define tt_str_eq_line(a,b) \ + tt_assert(line_str_eq((b),(a))) + +static void +test_consdiff_smartlist_slice(void *arg) +{ + smartlist_t *sl = smartlist_new(); + smartlist_slice_t *sls; + + /* Create a regular smartlist. */ + (void)arg; + smartlist_add(sl, (void*)1); + smartlist_add(sl, (void*)2); + smartlist_add(sl, (void*)3); + smartlist_add(sl, (void*)4); + smartlist_add(sl, (void*)5); + + /* See if the slice was done correctly. */ + sls = smartlist_slice(sl, 2, 5); + tt_ptr_op(sl, OP_EQ, sls->list); + tt_ptr_op((void*)3, OP_EQ, smartlist_get(sls->list, sls->offset)); + tt_ptr_op((void*)5, OP_EQ, + smartlist_get(sls->list, sls->offset + (sls->len-1))); + tor_free(sls); + + /* See that using -1 as the end does get to the last element. */ + sls = smartlist_slice(sl, 2, -1); + tt_ptr_op(sl, OP_EQ, sls->list); + tt_ptr_op((void*)3, OP_EQ, smartlist_get(sls->list, sls->offset)); + tt_ptr_op((void*)5, OP_EQ, + smartlist_get(sls->list, sls->offset + (sls->len-1))); + + done: + tor_free(sls); + smartlist_free(sl); +} + +static void +test_consdiff_smartlist_slice_string_pos(void *arg) +{ + smartlist_t *sl = smartlist_new(); + smartlist_slice_t *sls; + memarea_t *area = memarea_new(); + + /* Create a regular smartlist. */ + (void)arg; + consensus_split_lines(sl, "a\nd\nc\na\nb\n", area); + + /* See that smartlist_slice_string_pos respects the bounds of the slice. */ + sls = smartlist_slice(sl, 2, 5); + cdline_t a_line = { "a", 1 }; + tt_int_op(3, OP_EQ, smartlist_slice_string_pos(sls, &a_line)); + cdline_t d_line = { "d", 1 }; + tt_int_op(-1, OP_EQ, smartlist_slice_string_pos(sls, &d_line)); + + done: + tor_free(sls); + smartlist_free(sl); + memarea_drop_all(area); +} + +static void +test_consdiff_lcs_lengths(void *arg) +{ + smartlist_t *sl1 = smartlist_new(); + smartlist_t *sl2 = smartlist_new(); + smartlist_slice_t *sls1, *sls2; + int *lengths1, *lengths2; + memarea_t *area = memarea_new(); + + /* Expected lcs lengths in regular and reverse order. */ + int e_lengths1[] = { 0, 1, 2, 3, 3, 4 }; + int e_lengths2[] = { 0, 1, 1, 2, 3, 4 }; + + (void)arg; + consensus_split_lines(sl1, "a\nb\nc\nd\ne\n", area); + consensus_split_lines(sl2, "a\nc\nd\ni\ne\n", area); + + sls1 = smartlist_slice(sl1, 0, -1); + sls2 = smartlist_slice(sl2, 0, -1); + + lengths1 = lcs_lengths(sls1, sls2, 1); + lengths2 = lcs_lengths(sls1, sls2, -1); + tt_mem_op(e_lengths1, OP_EQ, lengths1, sizeof(int) * 6); + tt_mem_op(e_lengths2, OP_EQ, lengths2, sizeof(int) * 6); + + done: + tor_free(lengths1); + tor_free(lengths2); + tor_free(sls1); + tor_free(sls2); + smartlist_free(sl1); + smartlist_free(sl2); + memarea_drop_all(area); +} + +static void +test_consdiff_trim_slices(void *arg) +{ + smartlist_t *sl1 = smartlist_new(); + smartlist_t *sl2 = smartlist_new(); + smartlist_t *sl3 = smartlist_new(); + smartlist_t *sl4 = smartlist_new(); + smartlist_slice_t *sls1, *sls2, *sls3, *sls4; + memarea_t *area = memarea_new(); + + (void)arg; + consensus_split_lines(sl1, "a\nb\nb\nb\nd\n", area); + consensus_split_lines(sl2, "a\nc\nc\nc\nd\n", area); + consensus_split_lines(sl3, "a\nb\nb\nb\na\n", area); + consensus_split_lines(sl4, "c\nb\nb\nb\nc\n", area); + sls1 = smartlist_slice(sl1, 0, -1); + sls2 = smartlist_slice(sl2, 0, -1); + sls3 = smartlist_slice(sl3, 0, -1); + sls4 = smartlist_slice(sl4, 0, -1); + + /* They should be trimmed by one line at each end. */ + tt_int_op(5, OP_EQ, sls1->len); + tt_int_op(5, OP_EQ, sls2->len); + trim_slices(sls1, sls2); + tt_int_op(3, OP_EQ, sls1->len); + tt_int_op(3, OP_EQ, sls2->len); + + /* They should not be trimmed at all. */ + tt_int_op(5, OP_EQ, sls3->len); + tt_int_op(5, OP_EQ, sls4->len); + trim_slices(sls3, sls4); + tt_int_op(5, OP_EQ, sls3->len); + tt_int_op(5, OP_EQ, sls4->len); + + done: + tor_free(sls1); + tor_free(sls2); + tor_free(sls3); + tor_free(sls4); + smartlist_free(sl1); + smartlist_free(sl2); + smartlist_free(sl3); + smartlist_free(sl4); + memarea_drop_all(area); +} + +static void +test_consdiff_set_changed(void *arg) +{ + smartlist_t *sl1 = smartlist_new(); + smartlist_t *sl2 = smartlist_new(); + bitarray_t *changed1 = bitarray_init_zero(4); + bitarray_t *changed2 = bitarray_init_zero(4); + smartlist_slice_t *sls1, *sls2; + memarea_t *area = memarea_new(); + + (void)arg; + consensus_split_lines(sl1, "a\nb\na\na\n", area); + consensus_split_lines(sl2, "a\na\na\na\n", area); + + /* Length of sls1 is 0. */ + sls1 = smartlist_slice(sl1, 0, 0); + sls2 = smartlist_slice(sl2, 1, 3); + set_changed(changed1, changed2, sls1, sls2); + + /* The former is not changed, the latter changes all of its elements. */ + tt_assert(!bitarray_is_set(changed1, 0)); + tt_assert(!bitarray_is_set(changed1, 1)); + tt_assert(!bitarray_is_set(changed1, 2)); + tt_assert(!bitarray_is_set(changed1, 3)); + + tt_assert(!bitarray_is_set(changed2, 0)); + tt_assert(bitarray_is_set(changed2, 1)); + tt_assert(bitarray_is_set(changed2, 2)); + tt_assert(!bitarray_is_set(changed2, 3)); + bitarray_clear(changed2, 1); + bitarray_clear(changed2, 2); + + /* Length of sls1 is 1 and its element is in sls2. */ + tor_free(sls1); + sls1 = smartlist_slice(sl1, 0, 1); + set_changed(changed1, changed2, sls1, sls2); + + /* The latter changes all elements but the (first) common one. */ + tt_assert(!bitarray_is_set(changed1, 0)); + tt_assert(!bitarray_is_set(changed1, 1)); + tt_assert(!bitarray_is_set(changed1, 2)); + tt_assert(!bitarray_is_set(changed1, 3)); + + tt_assert(!bitarray_is_set(changed2, 0)); + tt_assert(!bitarray_is_set(changed2, 1)); + tt_assert(bitarray_is_set(changed2, 2)); + tt_assert(!bitarray_is_set(changed2, 3)); + bitarray_clear(changed2, 2); + + /* Length of sls1 is 1 and its element is not in sls2. */ + tor_free(sls1); + sls1 = smartlist_slice(sl1, 1, 2); + set_changed(changed1, changed2, sls1, sls2); + + /* The former changes its element, the latter changes all elements. */ + tt_assert(!bitarray_is_set(changed1, 0)); + tt_assert(bitarray_is_set(changed1, 1)); + tt_assert(!bitarray_is_set(changed1, 2)); + tt_assert(!bitarray_is_set(changed1, 3)); + + tt_assert(!bitarray_is_set(changed2, 0)); + tt_assert(bitarray_is_set(changed2, 1)); + tt_assert(bitarray_is_set(changed2, 2)); + tt_assert(!bitarray_is_set(changed2, 3)); + + done: + bitarray_free(changed1); + bitarray_free(changed2); + smartlist_free(sl1); + smartlist_free(sl2); + tor_free(sls1); + tor_free(sls2); + memarea_drop_all(area); +} + +static void +test_consdiff_calc_changes(void *arg) +{ + smartlist_t *sl1 = smartlist_new(); + smartlist_t *sl2 = smartlist_new(); + smartlist_slice_t *sls1, *sls2; + bitarray_t *changed1 = bitarray_init_zero(4); + bitarray_t *changed2 = bitarray_init_zero(4); + memarea_t *area = memarea_new(); + + (void)arg; + consensus_split_lines(sl1, "a\na\na\na\n", area); + consensus_split_lines(sl2, "a\na\na\na\n", area); + + sls1 = smartlist_slice(sl1, 0, -1); + sls2 = smartlist_slice(sl2, 0, -1); + calc_changes(sls1, sls2, changed1, changed2); + + /* Nothing should be set to changed. */ + tt_assert(!bitarray_is_set(changed1, 0)); + tt_assert(!bitarray_is_set(changed1, 1)); + tt_assert(!bitarray_is_set(changed1, 2)); + tt_assert(!bitarray_is_set(changed1, 3)); + + tt_assert(!bitarray_is_set(changed2, 0)); + tt_assert(!bitarray_is_set(changed2, 1)); + tt_assert(!bitarray_is_set(changed2, 2)); + tt_assert(!bitarray_is_set(changed2, 3)); + + smartlist_clear(sl2); + consensus_split_lines(sl2, "a\nb\na\nb\n", area); + tor_free(sls1); + tor_free(sls2); + sls1 = smartlist_slice(sl1, 0, -1); + sls2 = smartlist_slice(sl2, 0, -1); + calc_changes(sls1, sls2, changed1, changed2); + + /* Two elements are changed. */ + tt_assert(!bitarray_is_set(changed1, 0)); + tt_assert(bitarray_is_set(changed1, 1)); + tt_assert(bitarray_is_set(changed1, 2)); + tt_assert(!bitarray_is_set(changed1, 3)); + bitarray_clear(changed1, 1); + bitarray_clear(changed1, 2); + + tt_assert(!bitarray_is_set(changed2, 0)); + tt_assert(bitarray_is_set(changed2, 1)); + tt_assert(!bitarray_is_set(changed2, 2)); + tt_assert(bitarray_is_set(changed2, 3)); + bitarray_clear(changed1, 1); + bitarray_clear(changed1, 3); + + smartlist_clear(sl2); + consensus_split_lines(sl2, "b\nb\nb\nb\n", area); + tor_free(sls1); + tor_free(sls2); + sls1 = smartlist_slice(sl1, 0, -1); + sls2 = smartlist_slice(sl2, 0, -1); + calc_changes(sls1, sls2, changed1, changed2); + + /* All elements are changed. */ + tt_assert(bitarray_is_set(changed1, 0)); + tt_assert(bitarray_is_set(changed1, 1)); + tt_assert(bitarray_is_set(changed1, 2)); + tt_assert(bitarray_is_set(changed1, 3)); + + tt_assert(bitarray_is_set(changed2, 0)); + tt_assert(bitarray_is_set(changed2, 1)); + tt_assert(bitarray_is_set(changed2, 2)); + tt_assert(bitarray_is_set(changed2, 3)); + + done: + bitarray_free(changed1); + bitarray_free(changed2); + smartlist_free(sl1); + smartlist_free(sl2); + tor_free(sls1); + tor_free(sls2); + memarea_drop_all(area); +} + +static void +test_consdiff_get_id_hash(void *arg) +{ + (void)arg; + + cdline_t line1 = { "r name", 6 }; + cdline_t line2 = { "r name _hash_isnt_base64 etc", 28 }; + cdline_t line3 = { "r name hash+valid+base64 etc", 28 }; + cdline_t tmp; + + /* No hash. */ + tt_int_op(-1, OP_EQ, get_id_hash(&line1, &tmp)); + /* The hash contains characters that are not base64. */ + tt_int_op(-1, OP_EQ, get_id_hash(&line2, &tmp)); + + /* valid hash. */ + tt_int_op(0, OP_EQ, get_id_hash(&line3, &tmp)); + tt_ptr_op(tmp.s, OP_EQ, line3.s + 7); + tt_uint_op(tmp.len, OP_EQ, line3.len - 11); + + done: + ; +} + +static void +test_consdiff_is_valid_router_entry(void *arg) +{ + /* Doesn't start with "r ". */ + (void)arg; + cdline_t line0 = { "foo", 3 }; + tt_int_op(0, OP_EQ, is_valid_router_entry(&line0)); + + /* These are already tested with get_id_hash, but make sure it's run + * properly. */ + + cdline_t line1 = { "r name", 6 }; + cdline_t line2 = { "r name _hash_isnt_base64 etc", 28 }; + cdline_t line3 = { "r name hash+valid+base64 etc", 28 }; + tt_int_op(0, OP_EQ, is_valid_router_entry(&line1)); + tt_int_op(0, OP_EQ, is_valid_router_entry(&line2)); + tt_int_op(1, OP_EQ, is_valid_router_entry(&line3)); + + done: + ; +} + +static void +test_consdiff_next_router(void *arg) +{ + smartlist_t *sl = smartlist_new(); + memarea_t *area = memarea_new(); + (void)arg; + smartlist_add_linecpy(sl, area, "foo"); + smartlist_add_linecpy(sl, area, + "r name hash+longer+than+27+chars+and+valid+base64 etc"); + smartlist_add_linecpy(sl, area, "foo"); + smartlist_add_linecpy(sl, area, "foo"); + smartlist_add_linecpy(sl, area, + "r name hash+longer+than+27+chars+and+valid+base64 etc"); + smartlist_add_linecpy(sl, area, "foo"); + + /* Not currently on a router entry line, finding the next one. */ + tt_int_op(1, OP_EQ, next_router(sl, 0)); + tt_int_op(4, OP_EQ, next_router(sl, 2)); + + /* Already at the beginning of a router entry line, ignore it. */ + tt_int_op(4, OP_EQ, next_router(sl, 1)); + + /* There are no more router entries, so return the line after the last. */ + tt_int_op(6, OP_EQ, next_router(sl, 4)); + tt_int_op(6, OP_EQ, next_router(sl, 5)); + + done: + smartlist_free(sl); + memarea_drop_all(area); +} + +static int +base64cmp_wrapper(const char *a, const char *b) +{ + cdline_t aa = { a, a ? (uint32_t) strlen(a) : 0 }; + cdline_t bb = { b, b ? (uint32_t) strlen(b) : 0 }; + return base64cmp(&aa, &bb); +} + +static void +test_consdiff_base64cmp(void *arg) +{ + /* NULL arguments. */ + (void)arg; + tt_int_op(0, OP_EQ, base64cmp_wrapper(NULL, NULL)); + tt_int_op(-1, OP_EQ, base64cmp_wrapper(NULL, "foo")); + tt_int_op(1, OP_EQ, base64cmp_wrapper("bar", NULL)); + + /* Nil base64 values. */ + tt_int_op(0, OP_EQ, base64cmp_wrapper("", "")); + tt_int_op(0, OP_EQ, base64cmp_wrapper("_", "&")); + + /* Exact same valid strings. */ + tt_int_op(0, OP_EQ, base64cmp_wrapper("abcABC/+", "abcABC/+")); + /* Both end with an invalid base64 char other than '\0'. */ + tt_int_op(0, OP_EQ, base64cmp_wrapper("abcABC/+ ", "abcABC/+ ")); + /* Only one ends with an invalid base64 char other than '\0'. */ + tt_int_op(-1, OP_EQ, base64cmp_wrapper("abcABC/+ ", "abcABC/+a")); + + /* Comparisons that would return differently with strcmp(). */ + tt_int_op(-1, OP_EQ, strcmp("/foo", "Afoo")); + tt_int_op(1, OP_EQ, base64cmp_wrapper("/foo", "Afoo")); + tt_int_op(1, OP_EQ, strcmp("Afoo", "0foo")); + tt_int_op(-1, OP_EQ, base64cmp_wrapper("Afoo", "0foo")); + + /* Comparisons that would return the same as with strcmp(). */ + tt_int_op(1, OP_EQ, strcmp("afoo", "Afoo")); + tt_int_op(1, OP_EQ, base64cmp_wrapper("afoo", "Afoo")); + + /* Different lengths */ + tt_int_op(-1, OP_EQ, base64cmp_wrapper("afoo", "afooo")); + tt_int_op(1, OP_EQ, base64cmp_wrapper("afooo", "afoo")); + + done: + ; +} + +static void +test_consdiff_gen_ed_diff(void *arg) +{ + smartlist_t *cons1=NULL, *cons2=NULL, *diff=NULL; + int i; + memarea_t *area = memarea_new(); + setup_capture_of_logs(LOG_WARN); + + (void)arg; + cons1 = smartlist_new(); + cons2 = smartlist_new(); + + /* Identity hashes are not sorted properly, return NULL. */ + smartlist_add_linecpy(cons1, area, "r name bbbbbbbbbbbbbbbbbbbbbbbbbbb etc"); + smartlist_add_linecpy(cons1, area, "foo"); + smartlist_add_linecpy(cons1, area, "r name aaaaaaaaaaaaaaaaaaaaaaaaaaa etc"); + smartlist_add_linecpy(cons1, area, "bar"); + + smartlist_add_linecpy(cons2, area, "r name aaaaaaaaaaaaaaaaaaaaaaaaaaa etc"); + smartlist_add_linecpy(cons2, area, "foo"); + smartlist_add_linecpy(cons2, area, "r name ccccccccccccccccccccccccccc etc"); + smartlist_add_linecpy(cons2, area, "bar"); + + diff = gen_ed_diff(cons1, cons2, area); + tt_ptr_op(NULL, OP_EQ, diff); + expect_single_log_msg_containing("Refusing to generate consensus diff " + "because the base consensus doesn't have its router entries sorted " + "properly."); + + /* Same, but now with the second consensus. */ + mock_clean_saved_logs(); + diff = gen_ed_diff(cons2, cons1, area); + tt_ptr_op(NULL, OP_EQ, diff); + expect_single_log_msg_containing("Refusing to generate consensus diff " + "because the target consensus doesn't have its router entries sorted " + "properly."); + + /* Same as the two above, but with the reversed thing immediately after a + match. (The code handles this differently) */ + smartlist_del(cons1, 0); + smartlist_add_linecpy(cons1, area, "r name aaaaaaaaaaaaaaaaaaaaaaaaaaa etc"); + + mock_clean_saved_logs(); + diff = gen_ed_diff(cons1, cons2, area); + tt_ptr_op(NULL, OP_EQ, diff); + expect_single_log_msg_containing("Refusing to generate consensus diff " + "because the base consensus doesn't have its router entries sorted " + "properly."); + + mock_clean_saved_logs(); + diff = gen_ed_diff(cons2, cons1, area); + tt_ptr_op(NULL, OP_EQ, diff); + expect_single_log_msg_containing("Refusing to generate consensus diff " + "because the target consensus doesn't have its router entries sorted " + "properly."); + + /* Identity hashes are repeated, return NULL. */ + smartlist_clear(cons1); + + smartlist_add_linecpy(cons1, area, "r name bbbbbbbbbbbbbbbbbbbbbbbbbbb etc"); + smartlist_add_linecpy(cons1, area, "foo"); + smartlist_add_linecpy(cons1, area, "r name bbbbbbbbbbbbbbbbbbbbbbbbbbb etc"); + smartlist_add_linecpy(cons1, area, "bar"); + + mock_clean_saved_logs(); + diff = gen_ed_diff(cons1, cons2, area); + tt_ptr_op(NULL, OP_EQ, diff); + expect_single_log_msg_containing("Refusing to generate consensus diff " + "because the base consensus doesn't have its router entries sorted " + "properly."); + + /* We have to add a line that is just a dot, return NULL. */ + smartlist_clear(cons1); + smartlist_clear(cons2); + + smartlist_add_linecpy(cons1, area, "foo1"); + smartlist_add_linecpy(cons1, area, "foo2"); + + smartlist_add_linecpy(cons2, area, "foo1"); + smartlist_add_linecpy(cons2, area, "."); + smartlist_add_linecpy(cons2, area, "foo2"); + + mock_clean_saved_logs(); + diff = gen_ed_diff(cons1, cons2, area); + tt_ptr_op(NULL, OP_EQ, diff); + expect_single_log_msg_containing("Cannot generate consensus diff " + "because one of the lines to be added is \".\"."); + +#define MAX_LINE_COUNT (10000) + /* Too many lines to be fed to the quadratic-time function. */ + smartlist_clear(cons1); + smartlist_clear(cons2); + + for (i=0; i < MAX_LINE_COUNT; ++i) smartlist_add_linecpy(cons1, area, "a"); + for (i=0; i < MAX_LINE_COUNT; ++i) smartlist_add_linecpy(cons1, area, "b"); + + mock_clean_saved_logs(); + diff = gen_ed_diff(cons1, cons2, area); + + tt_ptr_op(NULL, OP_EQ, diff); + expect_single_log_msg_containing("Refusing to generate consensus diff " + "because we found too few common router ids."); + + /* We have dot lines, but they don't interfere with the script format. */ + smartlist_clear(cons1); + smartlist_clear(cons2); + + smartlist_add_linecpy(cons1, area, "foo1"); + smartlist_add_linecpy(cons1, area, "."); + smartlist_add_linecpy(cons1, area, "."); + smartlist_add_linecpy(cons1, area, "foo2"); + + smartlist_add_linecpy(cons2, area, "foo1"); + smartlist_add_linecpy(cons2, area, "."); + smartlist_add_linecpy(cons2, area, "foo2"); + + diff = gen_ed_diff(cons1, cons2, area); + tt_ptr_op(NULL, OP_NE, diff); + smartlist_free(diff); + + /* Empty diff tests. */ + smartlist_clear(cons1); + smartlist_clear(cons2); + + diff = gen_ed_diff(cons1, cons2, area); + tt_ptr_op(NULL, OP_NE, diff); + tt_int_op(0, OP_EQ, smartlist_len(diff)); + smartlist_free(diff); + + smartlist_add_linecpy(cons1, area, "foo"); + smartlist_add_linecpy(cons1, area, "bar"); + + smartlist_add_linecpy(cons2, area, "foo"); + smartlist_add_linecpy(cons2, area, "bar"); + + diff = gen_ed_diff(cons1, cons2, area); + tt_ptr_op(NULL, OP_NE, diff); + tt_int_op(0, OP_EQ, smartlist_len(diff)); + smartlist_free(diff); + + /* Everything is deleted. */ + smartlist_clear(cons2); + + diff = gen_ed_diff(cons1, cons2, area); + tt_ptr_op(NULL, OP_NE, diff); + tt_int_op(1, OP_EQ, smartlist_len(diff)); + tt_str_eq_line("1,2d", smartlist_get(diff, 0)); + + smartlist_free(diff); + + /* Everything is added. */ + diff = gen_ed_diff(cons2, cons1, area); + tt_ptr_op(NULL, OP_NE, diff); + tt_int_op(4, OP_EQ, smartlist_len(diff)); + tt_str_eq_line("0a", smartlist_get(diff, 0)); + tt_str_eq_line("foo", smartlist_get(diff, 1)); + tt_str_eq_line("bar", smartlist_get(diff, 2)); + tt_str_eq_line(".", smartlist_get(diff, 3)); + + smartlist_free(diff); + + /* Everything is changed. */ + smartlist_add_linecpy(cons2, area, "foo2"); + smartlist_add_linecpy(cons2, area, "bar2"); + diff = gen_ed_diff(cons1, cons2, area); + tt_ptr_op(NULL, OP_NE, diff); + tt_int_op(4, OP_EQ, smartlist_len(diff)); + tt_str_eq_line("1,2c", smartlist_get(diff, 0)); + tt_str_eq_line("foo2", smartlist_get(diff, 1)); + tt_str_eq_line("bar2", smartlist_get(diff, 2)); + tt_str_eq_line(".", smartlist_get(diff, 3)); + + smartlist_free(diff); + + /* Test 'a', 'c' and 'd' together. See that it is done in reverse order. */ + smartlist_clear(cons1); + smartlist_clear(cons2); + consensus_split_lines(cons1, "A\nB\nC\nD\nE\n", area); + consensus_split_lines(cons2, "A\nC\nO\nE\nU\n", area); + diff = gen_ed_diff(cons1, cons2, area); + tt_ptr_op(NULL, OP_NE, diff); + tt_int_op(7, OP_EQ, smartlist_len(diff)); + tt_str_eq_line("5a", smartlist_get(diff, 0)); + tt_str_eq_line("U", smartlist_get(diff, 1)); + tt_str_eq_line(".", smartlist_get(diff, 2)); + tt_str_eq_line("4c", smartlist_get(diff, 3)); + tt_str_eq_line("O", smartlist_get(diff, 4)); + tt_str_eq_line(".", smartlist_get(diff, 5)); + tt_str_eq_line("2d", smartlist_get(diff, 6)); + + /* TODO: small real use-cases, i.e. consensuses. */ + + done: + teardown_capture_of_logs(); + smartlist_free(cons1); + smartlist_free(cons2); + smartlist_free(diff); + memarea_drop_all(area); +} + +static void +test_consdiff_apply_ed_diff(void *arg) +{ + smartlist_t *cons1=NULL, *cons2=NULL, *diff=NULL; + memarea_t *area = memarea_new(); + (void)arg; + cons1 = smartlist_new(); + diff = smartlist_new(); + setup_capture_of_logs(LOG_WARN); + + consensus_split_lines(cons1, "A\nB\nC\nD\nE\n", area); + + /* Command without range. */ + smartlist_add_linecpy(diff, area, "a"); + cons2 = apply_ed_diff(cons1, diff, 0); + tt_ptr_op(NULL, OP_EQ, cons2); + smartlist_clear(diff); + expect_single_log_msg_containing("an ed command was missing a line number"); + + /* Range without command. */ + smartlist_add_linecpy(diff, area, "1"); + mock_clean_saved_logs(); + cons2 = apply_ed_diff(cons1, diff, 0); + tt_ptr_op(NULL, OP_EQ, cons2); + expect_single_log_msg_containing("a line with no ed command was found"); + + smartlist_clear(diff); + + /* Range without end. */ + smartlist_add_linecpy(diff, area, "1,"); + mock_clean_saved_logs(); + cons2 = apply_ed_diff(cons1, diff, 0); + tt_ptr_op(NULL, OP_EQ, cons2); + expect_single_log_msg_containing("an ed command was missing a range " + "end line number."); + + smartlist_clear(diff); + + /* Incoherent ranges. */ + smartlist_add_linecpy(diff, area, "1,1"); + mock_clean_saved_logs(); + cons2 = apply_ed_diff(cons1, diff, 0); + tt_ptr_op(NULL, OP_EQ, cons2); + expect_single_log_msg_containing("an invalid range was found"); + + smartlist_clear(diff); + + smartlist_add_linecpy(diff, area, "3,2"); + mock_clean_saved_logs(); + cons2 = apply_ed_diff(cons1, diff, 0); + tt_ptr_op(NULL, OP_EQ, cons2); + expect_single_log_msg_containing("an invalid range was found"); + + smartlist_clear(diff); + + /* Script is not in reverse order. */ + smartlist_add_linecpy(diff, area, "1d"); + smartlist_add_linecpy(diff, area, "3d"); + mock_clean_saved_logs(); + cons2 = apply_ed_diff(cons1, diff, 0); + tt_ptr_op(NULL, OP_EQ, cons2); + expect_single_log_msg_containing("its commands are not properly sorted"); + + smartlist_clear(diff); + + /* Script contains unrecognised commands longer than one char. */ + smartlist_add_linecpy(diff, area, "1foo"); + mock_clean_saved_logs(); + cons2 = apply_ed_diff(cons1, diff, 0); + tt_ptr_op(NULL, OP_EQ, cons2); + expect_single_log_msg_containing("an ed command longer than one char was " + "found"); + + smartlist_clear(diff); + + /* Script contains unrecognised commands. */ + smartlist_add_linecpy(diff, area, "1e"); + mock_clean_saved_logs(); + cons2 = apply_ed_diff(cons1, diff, 0); + tt_ptr_op(NULL, OP_EQ, cons2); + expect_single_log_msg_containing("an unrecognised ed command was found"); + + smartlist_clear(diff); + + /* Command that should be followed by at least one line and a ".", but + * isn't. */ + smartlist_add_linecpy(diff, area, "0a"); + mock_clean_saved_logs(); + cons2 = apply_ed_diff(cons1, diff, 0); + tt_ptr_op(NULL, OP_EQ, cons2); + expect_single_log_msg_containing("it has an ed command that tries to " + "insert zero lines."); + + /* Now it is followed by a ".", but it inserts zero lines. */ + smartlist_add_linecpy(diff, area, "."); + mock_clean_saved_logs(); + cons2 = apply_ed_diff(cons1, diff, 0); + tt_ptr_op(NULL, OP_EQ, cons2); + expect_single_log_msg_containing("it has an ed command that tries to " + "insert zero lines."); + + smartlist_clear(diff); + + /* Now it it inserts something, but has no terminator. */ + smartlist_add_linecpy(diff, area, "0a"); + smartlist_add_linecpy(diff, area, "hello"); + mock_clean_saved_logs(); + cons2 = apply_ed_diff(cons1, diff, 0); + tt_ptr_op(NULL, OP_EQ, cons2); + expect_single_log_msg_containing("lines to be inserted that don't end with " + "a \".\"."); + + smartlist_clear(diff); + + /* Test appending text, 'a'. */ + consensus_split_lines(diff, "3a\nU\nO\n.\n0a\nV\n.\n", area); + cons2 = apply_ed_diff(cons1, diff, 0); + tt_ptr_op(NULL, OP_NE, cons2); + tt_int_op(8, OP_EQ, smartlist_len(cons2)); + tt_str_eq_line("V", smartlist_get(cons2, 0)); + tt_str_eq_line("A", smartlist_get(cons2, 1)); + tt_str_eq_line("B", smartlist_get(cons2, 2)); + tt_str_eq_line("C", smartlist_get(cons2, 3)); + tt_str_eq_line("U", smartlist_get(cons2, 4)); + tt_str_eq_line("O", smartlist_get(cons2, 5)); + tt_str_eq_line("D", smartlist_get(cons2, 6)); + tt_str_eq_line("E", smartlist_get(cons2, 7)); + + smartlist_clear(diff); + smartlist_free(cons2); + + /* Test deleting text, 'd'. */ + consensus_split_lines(diff, "4d\n1,2d\n", area); + cons2 = apply_ed_diff(cons1, diff, 0); + tt_ptr_op(NULL, OP_NE, cons2); + tt_int_op(2, OP_EQ, smartlist_len(cons2)); + tt_str_eq_line("C", smartlist_get(cons2, 0)); + tt_str_eq_line("E", smartlist_get(cons2, 1)); + + smartlist_clear(diff); + smartlist_free(cons2); + + /* Test changing text, 'c'. */ + consensus_split_lines(diff, "4c\nT\nX\n.\n1, 2c\nM\n.\n", area); + cons2 = apply_ed_diff(cons1, diff, 0); + tt_ptr_op(NULL, OP_NE, cons2); + tt_int_op(5, OP_EQ, smartlist_len(cons2)); + tt_str_eq_line("M", smartlist_get(cons2, 0)); + tt_str_eq_line("C", smartlist_get(cons2, 1)); + tt_str_eq_line("T", smartlist_get(cons2, 2)); + tt_str_eq_line("X", smartlist_get(cons2, 3)); + tt_str_eq_line("E", smartlist_get(cons2, 4)); + + smartlist_clear(diff); + smartlist_free(cons2); + + /* Test 'a', 'd' and 'c' together. */ + consensus_split_lines(diff, "4c\nT\nX\n.\n2d\n0a\nM\n.\n", area); + cons2 = apply_ed_diff(cons1, diff, 0); + tt_ptr_op(NULL, OP_NE, cons2); + tt_int_op(6, OP_EQ, smartlist_len(cons2)); + tt_str_eq_line("M", smartlist_get(cons2, 0)); + tt_str_eq_line("A", smartlist_get(cons2, 1)); + tt_str_eq_line("C", smartlist_get(cons2, 2)); + tt_str_eq_line("T", smartlist_get(cons2, 3)); + tt_str_eq_line("X", smartlist_get(cons2, 4)); + tt_str_eq_line("E", smartlist_get(cons2, 5)); + + done: + teardown_capture_of_logs(); + smartlist_free(cons1); + smartlist_free(cons2); + smartlist_free(diff); + memarea_drop_all(area); +} + +static void +test_consdiff_gen_diff(void *arg) +{ + char *cons1_str=NULL, *cons2_str=NULL; + smartlist_t *cons1=NULL, *cons2=NULL, *diff=NULL; + consensus_digest_t digests1, digests2; + memarea_t *area = memarea_new(); + (void)arg; + cons1 = smartlist_new(); + cons2 = smartlist_new(); + + /* Identity hashes are not sorted properly, return NULL. + * Already tested in gen_ed_diff, but see that a NULL ed diff also makes + * gen_diff return NULL. */ + cons1_str = tor_strdup( + "network-status-version foo\n" + "r name bbbbbbbbbbbbbbbbb etc\nfoo\n" + "r name aaaaaaaaaaaaaaaaa etc\nbar\n" + "directory-signature foo bar\nbar\n" + ); + cons2_str = tor_strdup( + "network-status-version foo\n" + "r name aaaaaaaaaaaaaaaaa etc\nfoo\n" + "r name ccccccccccccccccc etc\nbar\n" + "directory-signature foo bar\nbar\n" + ); + + tt_int_op(0, OP_EQ, + consensus_compute_digest(cons1_str, &digests1)); + tt_int_op(0, OP_EQ, + consensus_compute_digest(cons2_str, &digests2)); + + consensus_split_lines(cons1, cons1_str, area); + consensus_split_lines(cons2, cons2_str, area); + + diff = consdiff_gen_diff(cons1, cons2, &digests1, &digests2, area); + tt_ptr_op(NULL, OP_EQ, diff); + + /* Check that the headers are done properly. */ + tor_free(cons1_str); + cons1_str = tor_strdup( + "network-status-version foo\n" + "r name ccccccccccccccccc etc\nfoo\n" + "r name eeeeeeeeeeeeeeeee etc\nbar\n" + "directory-signature foo bar\nbar\n" + ); + tt_int_op(0, OP_EQ, + consensus_compute_digest(cons1_str, &digests1)); + smartlist_clear(cons1); + consensus_split_lines(cons1, cons1_str, area); + diff = consdiff_gen_diff(cons1, cons2, &digests1, &digests2, area); + tt_ptr_op(NULL, OP_NE, diff); + tt_int_op(7, OP_EQ, smartlist_len(diff)); + tt_assert(line_str_eq(smartlist_get(diff, 0), + "network-status-diff-version 1")); + tt_assert(line_str_eq(smartlist_get(diff, 1), "hash " + "06646D6CF563A41869D3B02E73254372AE3140046C5E7D83C9F71E54976AF9B4 " + "7AFECEFA4599BA33D603653E3D2368F648DF4AC4723929B0F7CF39281596B0C1")); + tt_assert(line_str_eq(smartlist_get(diff, 2), "3,4d")); + tt_assert(line_str_eq(smartlist_get(diff, 3), "1a")); + tt_assert(line_str_eq(smartlist_get(diff, 4), + "r name aaaaaaaaaaaaaaaaa etc")); + tt_assert(line_str_eq(smartlist_get(diff, 5), "foo")); + tt_assert(line_str_eq(smartlist_get(diff, 6), ".")); + + /* TODO: small real use-cases, i.e. consensuses. */ + + done: + tor_free(cons1_str); + tor_free(cons2_str); + smartlist_free(cons1); + smartlist_free(cons2); + smartlist_free(diff); + memarea_drop_all(area); +} + +static void +test_consdiff_apply_diff(void *arg) +{ + smartlist_t *cons1=NULL, *diff=NULL; + char *cons1_str=NULL, *cons2 = NULL; + consensus_digest_t digests1; + (void)arg; + memarea_t *area = memarea_new(); + cons1 = smartlist_new(); + diff = smartlist_new(); + setup_capture_of_logs(LOG_INFO); + + cons1_str = tor_strdup( + "network-status-version foo\n" + "r name ccccccccccccccccc etc\nfoo\n" + "r name eeeeeeeeeeeeeeeee etc\nbar\n" + "directory-signature foo bar\nbar\n" + ); + tt_int_op(0, OP_EQ, + consensus_compute_digest(cons1_str, &digests1)); + consensus_split_lines(cons1, cons1_str, area); + + /* diff doesn't have enough lines. */ + cons2 = consdiff_apply_diff(cons1, diff, &digests1); + tt_ptr_op(NULL, OP_EQ, cons2); + expect_single_log_msg_containing("too short") + + /* first line doesn't match format-version string. */ + smartlist_add_linecpy(diff, area, "foo-bar"); + smartlist_add_linecpy(diff, area, "header-line"); + mock_clean_saved_logs(); + cons2 = consdiff_apply_diff(cons1, diff, &digests1); + tt_ptr_op(NULL, OP_EQ, cons2); + expect_single_log_msg_containing("format is not known") + + /* The first word of the second header line is not "hash". */ + smartlist_clear(diff); + smartlist_add_linecpy(diff, area, "network-status-diff-version 1"); + smartlist_add_linecpy(diff, area, "word a b"); + smartlist_add_linecpy(diff, area, "x"); + mock_clean_saved_logs(); + cons2 = consdiff_apply_diff(cons1, diff, &digests1); + tt_ptr_op(NULL, OP_EQ, cons2); + expect_single_log_msg_containing("does not include the necessary digests") + + /* Wrong number of words after "hash". */ + smartlist_clear(diff); + smartlist_add_linecpy(diff, area, "network-status-diff-version 1"); + smartlist_add_linecpy(diff, area, "hash a b c"); + mock_clean_saved_logs(); + cons2 = consdiff_apply_diff(cons1, diff, &digests1); + tt_ptr_op(NULL, OP_EQ, cons2); + expect_single_log_msg_containing("does not include the necessary digests") + + /* base16 digests do not have the expected length. */ + smartlist_clear(diff); + smartlist_add_linecpy(diff, area, "network-status-diff-version 1"); + smartlist_add_linecpy(diff, area, "hash aaa bbb"); + mock_clean_saved_logs(); + cons2 = consdiff_apply_diff(cons1, diff, &digests1); + tt_ptr_op(NULL, OP_EQ, cons2); + expect_single_log_msg_containing("includes base16-encoded digests of " + "incorrect size") + + /* base16 digests contain non-base16 characters. */ + smartlist_clear(diff); + smartlist_add_linecpy(diff, area, "network-status-diff-version 1"); + smartlist_add_linecpy(diff, area, "hash" + " ????????????????????????????????????????????????????????????????" + " ----------------------------------------------------------------"); + mock_clean_saved_logs(); + cons2 = consdiff_apply_diff(cons1, diff, &digests1); + tt_ptr_op(NULL, OP_EQ, cons2); + expect_single_log_msg_containing("includes malformed digests") + + /* Invalid ed diff. + * As tested in apply_ed_diff, but check that apply_diff does return NULL if + * the ed diff can't be applied. */ + smartlist_clear(diff); + smartlist_add_linecpy(diff, area, "network-status-diff-version 1"); + smartlist_add_linecpy(diff, area, "hash" + /* sha3 of cons1. */ + " 06646D6CF563A41869D3B02E73254372AE3140046C5E7D83C9F71E54976AF9B4" + /* sha256 of cons2. */ + " 635D34593020C08E5ECD865F9986E29D50028EFA62843766A8197AD228A7F6AA"); + smartlist_add_linecpy(diff, area, "foobar"); + mock_clean_saved_logs(); + cons2 = consdiff_apply_diff(cons1, diff, &digests1); + tt_ptr_op(NULL, OP_EQ, cons2); + expect_single_log_msg_containing("because an ed command was missing a line " + "number") + + /* Base consensus doesn't match its digest as found in the diff. */ + smartlist_clear(diff); + smartlist_add_linecpy(diff, area, "network-status-diff-version 1"); + smartlist_add_linecpy(diff, area, "hash" + /* bogus sha256. */ + " 3333333333333333333333333333333333333333333333333333333333333333" + /* sha256 of cons2. */ + " 635D34593020C08E5ECD865F9986E29D50028EFA62843766A8197AD228A7F6AA"); + mock_clean_saved_logs(); + cons2 = consdiff_apply_diff(cons1, diff, &digests1); + tt_ptr_op(NULL, OP_EQ, cons2); + expect_log_msg_containing("base consensus doesn't match the digest " + "as found"); + + /* Resulting consensus doesn't match its digest as found in the diff. */ + smartlist_clear(diff); + smartlist_add_linecpy(diff, area, "network-status-diff-version 1"); + smartlist_add_linecpy(diff, area, "hash" + /* sha3 of cons1. */ + " 06646D6CF563A41869D3B02E73254372AE3140046C5E7D83C9F71E54976AF9B4" + /* bogus sha3. */ + " 3333333333333333333333333333333333333333333333333333333333333333"); + mock_clean_saved_logs(); + cons2 = consdiff_apply_diff(cons1, diff, &digests1); + tt_ptr_op(NULL, OP_EQ, cons2); + expect_log_msg_containing("resulting consensus doesn't match the " + "digest as found"); + +#if 0 + /* XXXX No longer possible, since we aren't using the other algorithm. */ + /* Resulting consensus digest cannot be computed */ + smartlist_clear(diff); + smartlist_add_linecpy(diff, area, "network-status-diff-version 1"); + smartlist_add_linecpy(diff, area, "hash" + /* sha3 of cons1. */ + " 06646D6CF563A41869D3B02E73254372AE3140046C5E7D83C9F71E54976AF9B4" + /* bogus sha3. */ + " 3333333333333333333333333333333333333333333333333333333333333333"); + smartlist_add_linecpy(diff, area, "1,2d"); // remove starting line + mock_clean_saved_logs(); + cons2 = consdiff_apply_diff(cons1, diff, &digests1); + tt_ptr_op(NULL, OP_EQ, cons2); + expect_log_msg_containing("Could not compute digests of the consensus " + "resulting from applying a consensus diff."); +#endif + + /* Very simple test, only to see that nothing errors. */ + smartlist_clear(diff); + smartlist_add_linecpy(diff, area, "network-status-diff-version 1"); + smartlist_add_linecpy(diff, area, "hash" + /* sha3 of cons1. */ + " 06646D6CF563A41869D3B02E73254372AE3140046C5E7D83C9F71E54976AF9B4" + /* sha3 of cons2. */ + " 90A418881B2FCAB3D9E60EE02E4D666D56CFA38F8A3B7AA3E0ADBA530DDA9353"); + smartlist_add_linecpy(diff, area, "3c"); + smartlist_add_linecpy(diff, area, "sample"); + smartlist_add_linecpy(diff, area, "."); + cons2 = consdiff_apply_diff(cons1, diff, &digests1); + tt_ptr_op(NULL, OP_NE, cons2); + tt_str_op( + "network-status-version foo\n" + "r name ccccccccccccccccc etc\nsample\n" + "r name eeeeeeeeeeeeeeeee etc\nbar\n" + "directory-signature foo bar\nbar\n", OP_EQ, + cons2); + tor_free(cons2); + + /* Check that lowercase letters in base16-encoded digests work too. */ + smartlist_clear(diff); + smartlist_add_linecpy(diff, area, "network-status-diff-version 1"); + smartlist_add_linecpy(diff, area, "hash" + /* sha3 of cons1. */ + " 06646d6cf563a41869d3b02e73254372ae3140046c5e7d83c9f71e54976af9b4" + /* sha3 of cons2. */ + " 90a418881b2fcab3d9e60ee02e4d666d56cfa38f8a3b7aa3e0adba530dda9353"); + smartlist_add_linecpy(diff, area, "3c"); + smartlist_add_linecpy(diff, area, "sample"); + smartlist_add_linecpy(diff, area, "."); + cons2 = consdiff_apply_diff(cons1, diff, &digests1); + tt_ptr_op(NULL, OP_NE, cons2); + tt_str_op( + "network-status-version foo\n" + "r name ccccccccccccccccc etc\nsample\n" + "r name eeeeeeeeeeeeeeeee etc\nbar\n" + "directory-signature foo bar\nbar\n", OP_EQ, + cons2); + tor_free(cons2); + + smartlist_clear(diff); + + done: + teardown_capture_of_logs(); + tor_free(cons1_str); + smartlist_free(cons1); + smartlist_free(diff); + memarea_drop_all(area); +} + +#define CONSDIFF_LEGACY(name) \ + { #name, test_consdiff_ ## name , 0, NULL, NULL } + +struct testcase_t consdiff_tests[] = { + CONSDIFF_LEGACY(smartlist_slice), + CONSDIFF_LEGACY(smartlist_slice_string_pos), + CONSDIFF_LEGACY(lcs_lengths), + CONSDIFF_LEGACY(trim_slices), + CONSDIFF_LEGACY(set_changed), + CONSDIFF_LEGACY(calc_changes), + CONSDIFF_LEGACY(get_id_hash), + CONSDIFF_LEGACY(is_valid_router_entry), + CONSDIFF_LEGACY(next_router), + CONSDIFF_LEGACY(base64cmp), + CONSDIFF_LEGACY(gen_ed_diff), + CONSDIFF_LEGACY(apply_ed_diff), + CONSDIFF_LEGACY(gen_diff), + CONSDIFF_LEGACY(apply_diff), + END_OF_TESTCASES +}; + diff --git a/src/test/test_containers.c b/src/test/test_containers.c index 41f3f873de..54484a2a91 100644 --- a/src/test/test_containers.c +++ b/src/test/test_containers.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/test/test_controller.c b/src/test/test_controller.c index 206315e289..592f91a988 100644 --- a/src/test/test_controller.c +++ b/src/test/test_controller.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2015-2016, The Tor Project, Inc. */ +/* Copyright (c) 2015-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define CONTROL_PRIVATE diff --git a/src/test/test_controller_events.c b/src/test/test_controller_events.c index 11e1e3dc8f..901ad7ab3d 100644 --- a/src/test/test_controller_events.c +++ b/src/test/test_controller_events.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2016, The Tor Project, Inc. */ +/* Copyright (c) 2013-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define CONNECTION_PRIVATE diff --git a/src/test/test_crypto.c b/src/test/test_crypto.c index d66ddccd4f..3a6c222bc4 100644 --- a/src/test/test_crypto.c +++ b/src/test/test_crypto.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/test/test_crypto_slow.c b/src/test/test_crypto_slow.c index 0d7d65ac73..6d676ff9b9 100644 --- a/src/test/test_crypto_slow.c +++ b/src/test/test_crypto_slow.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/test/test_data.c b/src/test/test_data.c index 788489a097..ce6c3394f6 100644 --- a/src/test/test_data.c +++ b/src/test/test_data.c @@ -1,6 +1,6 @@ /* Copyright 2001-2004 Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "test.h" diff --git a/src/test/test_dir.c b/src/test/test_dir.c index 3906206696..e0c92a74c5 100644 --- a/src/test/test_dir.c +++ b/src/test/test_dir.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" @@ -329,7 +329,7 @@ test_dir_formats(void *arg) ntor_cc = make_ntor_onion_key_crosscert(&r2_onion_keypair, &kp1.pubkey, r2->cache_info.published_on, - MIN_ONION_KEY_LIFETIME, + get_onion_key_lifetime(), &ntor_cc_sign); tt_assert(ntor_cc); base64_encode(cert_buf, sizeof(cert_buf), diff --git a/src/test/test_dir_common.c b/src/test/test_dir_common.c index ca43dd4c04..fca70249bd 100644 --- a/src/test/test_dir_common.c +++ b/src/test/test_dir_common.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/test/test_dir_common.h b/src/test/test_dir_common.h index 9682b0db49..65b9cf6436 100644 --- a/src/test/test_dir_common.h +++ b/src/test/test_dir_common.h @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "or.h" diff --git a/src/test/test_dir_handle_get.c b/src/test/test_dir_handle_get.c index 02ee6ad6f2..cfda314693 100644 --- a/src/test/test_dir_handle_get.c +++ b/src/test/test_dir_handle_get.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define RENDCOMMON_PRIVATE diff --git a/src/test/test_entryconn.c b/src/test/test_entryconn.c index 50848cfec2..12a631630b 100644 --- a/src/test/test_entryconn.c +++ b/src/test/test_entryconn.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2014-2016, The Tor Project, Inc. */ +/* Copyright (c) 2014-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/test/test_entrynodes.c b/src/test/test_entrynodes.c index 249fd1f973..3db7e63ee3 100644 --- a/src/test/test_entrynodes.c +++ b/src/test/test_entrynodes.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2014-2016, The Tor Project, Inc. */ +/* Copyright (c) 2014-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" @@ -584,6 +584,9 @@ test_entry_guard_parse_from_state_full(void *arg) smartlist_t *text = smartlist_new(); char *joined = NULL; + // So nodes aren't expired. This is Tue, 13 Dec 2016 09:37:14 GMT + update_approx_time(1481621834); + MOCK(entry_guard_is_listed, mock_entry_guard_is_listed); dummy_state = state; diff --git a/src/test/test_extorport.c b/src/test/test_extorport.c index 1f92780177..965e1824f2 100644 --- a/src/test/test_extorport.c +++ b/src/test/test_extorport.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2016, The Tor Project, Inc. */ +/* Copyright (c) 2013-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define CONNECTION_PRIVATE diff --git a/src/test/test_guardfraction.c b/src/test/test_guardfraction.c index 8173e44d47..56006f3cc3 100644 --- a/src/test/test_guardfraction.c +++ b/src/test/test_guardfraction.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2014-2016, The Tor Project, Inc. */ +/* Copyright (c) 2014-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define DIRSERV_PRIVATE diff --git a/src/test/test_handles.c b/src/test/test_handles.c index 536a478689..7ddee6e376 100644 --- a/src/test/test_handles.c +++ b/src/test/test_handles.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/test/test_helpers.c b/src/test/test_helpers.c index 5b84366e6d..a894336f07 100644 --- a/src/test/test_helpers.c +++ b/src/test/test_helpers.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2014-2016, The Tor Project, Inc. */ +/* Copyright (c) 2014-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/test/test_helpers.h b/src/test/test_helpers.h index c6d4d9c41f..293f80d173 100644 --- a/src/test/test_helpers.h +++ b/src/test/test_helpers.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2014-2016, The Tor Project, Inc. */ +/* Copyright (c) 2014-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_TEST_HELPERS_H diff --git a/src/test/test_hs.c b/src/test/test_hs.c index fbaabe91d8..3aa2eea7cb 100644 --- a/src/test/test_hs.c +++ b/src/test/test_hs.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2007-2016, The Tor Project, Inc. */ +/* Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/test/test_hs_cache.c b/src/test/test_hs_cache.c index 1943d0ffac..bc4bfabde9 100644 --- a/src/test/test_hs_cache.c +++ b/src/test/test_hs_cache.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -93,8 +93,8 @@ helper_build_hs_desc(uint64_t revision_counter, uint32_t lifetime, /* Setup encrypted data section. */ desc->encrypted_data.create2_ntor = 1; - desc->encrypted_data.auth_types = smartlist_new(); - smartlist_add(desc->encrypted_data.auth_types, tor_strdup("ed25519")); + desc->encrypted_data.intro_auth_types = smartlist_new(); + smartlist_add(desc->encrypted_data.intro_auth_types, tor_strdup("ed25519")); desc->encrypted_data.intro_points = smartlist_new(); /* Add an intro point. */ smartlist_add(desc->encrypted_data.intro_points, @@ -333,7 +333,7 @@ helper_fetch_desc_from_hsdir(const ed25519_public_key_t *blinded_key) size_t body_used = 0; fetch_from_buf_http(TO_CONN(conn)->outbuf, &headers, MAX_HEADERS_SIZE, - &received_desc, &body_used, 10000, 0); + &received_desc, &body_used, HS_DESC_MAX_LEN, 0); tor_free(headers); } diff --git a/src/test/test_hs_descriptor.c b/src/test/test_hs_descriptor.c index 02a71aa473..a1a1f14fb8 100644 --- a/src/test/test_hs_descriptor.c +++ b/src/test/test_hs_descriptor.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -15,6 +15,9 @@ #include "test.h" #include "torcert.h" +#include "test_helpers.h" +#include "log_test_helpers.h" + static hs_desc_intro_point_t * helper_build_intro_point(const ed25519_keypair_t *blinded_kp, time_t now, const char *addr, int legacy) @@ -105,9 +108,9 @@ helper_build_hs_desc(unsigned int no_ip, ed25519_public_key_t *signing_pubkey) /* Setup encrypted data section. */ desc->encrypted_data.create2_ntor = 1; - desc->encrypted_data.auth_types = smartlist_new(); + desc->encrypted_data.intro_auth_types = smartlist_new(); desc->encrypted_data.single_onion_service = 1; - smartlist_add(desc->encrypted_data.auth_types, tor_strdup("ed25519")); + smartlist_add(desc->encrypted_data.intro_auth_types, tor_strdup("ed25519")); desc->encrypted_data.intro_points = smartlist_new(); if (!no_ip) { /* Add four intro points. */ @@ -157,14 +160,17 @@ helper_compare_hs_desc(const hs_descriptor_t *desc1, desc2->encrypted_data.create2_ntor); /* Authentication type. */ - tt_int_op(!!desc1->encrypted_data.auth_types, ==, - !!desc2->encrypted_data.auth_types); - if (desc1->encrypted_data.auth_types && desc2->encrypted_data.auth_types) { - tt_int_op(smartlist_len(desc1->encrypted_data.auth_types), ==, - smartlist_len(desc2->encrypted_data.auth_types)); - for (int i = 0; i < smartlist_len(desc1->encrypted_data.auth_types); i++) { - tt_str_op(smartlist_get(desc1->encrypted_data.auth_types, i), OP_EQ, - smartlist_get(desc2->encrypted_data.auth_types, i)); + tt_int_op(!!desc1->encrypted_data.intro_auth_types, ==, + !!desc2->encrypted_data.intro_auth_types); + if (desc1->encrypted_data.intro_auth_types && + desc2->encrypted_data.intro_auth_types) { + tt_int_op(smartlist_len(desc1->encrypted_data.intro_auth_types), ==, + smartlist_len(desc2->encrypted_data.intro_auth_types)); + for (int i = 0; + i < smartlist_len(desc1->encrypted_data.intro_auth_types); + i++) { + tt_str_op(smartlist_get(desc1->encrypted_data.intro_auth_types, i),OP_EQ, + smartlist_get(desc2->encrypted_data.intro_auth_types, i)); } } @@ -311,13 +317,13 @@ test_descriptor_padding(void *arg) /* Example: if l = 129, the ceiled division gives 2 and then multiplied by 128 * to give 256. With l = 127, ceiled division gives 1 then times 128. */ #define PADDING_EXPECTED_LEN(l) \ - CEIL_DIV(l, HS_DESC_PLAINTEXT_PADDING_MULTIPLE) * \ - HS_DESC_PLAINTEXT_PADDING_MULTIPLE + CEIL_DIV(l, HS_DESC_SUPERENC_PLAINTEXT_PAD_MULTIPLE) * \ + HS_DESC_SUPERENC_PLAINTEXT_PAD_MULTIPLE (void) arg; { /* test #1: no padding */ - plaintext_len = HS_DESC_PLAINTEXT_PADDING_MULTIPLE; + plaintext_len = HS_DESC_SUPERENC_PLAINTEXT_PAD_MULTIPLE; plaintext = tor_malloc(plaintext_len); padded_len = build_plaintext_padding(plaintext, plaintext_len, &padded_plaintext); @@ -333,7 +339,7 @@ test_descriptor_padding(void *arg) } { /* test #2: one byte padding? */ - plaintext_len = HS_DESC_PLAINTEXT_PADDING_MULTIPLE - 1; + plaintext_len = HS_DESC_SUPERENC_PLAINTEXT_PAD_MULTIPLE - 1; plaintext = tor_malloc(plaintext_len); padded_plaintext = NULL; padded_len = build_plaintext_padding(plaintext, plaintext_len, @@ -350,7 +356,7 @@ test_descriptor_padding(void *arg) } { /* test #3: Lots more bytes of padding? */ - plaintext_len = HS_DESC_PLAINTEXT_PADDING_MULTIPLE + 1; + plaintext_len = HS_DESC_SUPERENC_PLAINTEXT_PAD_MULTIPLE + 1; plaintext = tor_malloc(plaintext_len); padded_plaintext = NULL; padded_len = build_plaintext_padding(plaintext, plaintext_len, @@ -587,19 +593,11 @@ test_encrypted_data_len(void *arg) /* No length, error. */ ret = encrypted_data_length_is_valid(0); tt_int_op(ret, OP_EQ, 0); - /* Not a multiple of our encryption algorithm (thus no padding). It's - * suppose to be aligned on HS_DESC_PLAINTEXT_PADDING_MULTIPLE. */ - value = HS_DESC_PLAINTEXT_PADDING_MULTIPLE * 10 - 1; - ret = encrypted_data_length_is_valid(value); - tt_int_op(ret, OP_EQ, 0); /* Valid value. */ - value = HS_DESC_PADDED_PLAINTEXT_MAX_LEN + HS_DESC_ENCRYPTED_SALT_LEN + - DIGEST256_LEN; + value = HS_DESC_ENCRYPTED_SALT_LEN + DIGEST256_LEN + 1; ret = encrypted_data_length_is_valid(value); tt_int_op(ret, OP_EQ, 1); - /* XXX: Test maximum possible size. */ - done: ; } @@ -1006,6 +1004,103 @@ test_desc_signature(void *arg) tor_free(data); } +/* bad desc auth type */ +static const char bad_superencrypted_text1[] = "desc-auth-type scoobysnack\n" + "desc-auth-ephemeral-key A/O8DVtnUheb3r1JqoB8uJB7wxXL1XJX3eny4yB+eFA=\n" + "auth-client oiNrQB8WwKo S5D02W7vKgiWIMygrBl8RQ FB//SfOBmLEx1kViEWWL1g\n" + "encrypted\n" + "-----BEGIN MESSAGE-----\n" + "YmVpbmcgb24gbW91bnRhaW5zLCB0aGlua2luZyBhYm91dCBjb21wdXRlcnMsIGlzIG5vdC" + "BiYWQgYXQgYWxs\n" + "-----END MESSAGE-----\n"; + +/* bad ephemeral key */ +static const char bad_superencrypted_text2[] = "desc-auth-type x25519\n" + "desc-auth-ephemeral-key differentalphabet\n" + "auth-client oiNrQB8WwKo S5D02W7vKgiWIMygrBl8RQ FB//SfOBmLEx1kViEWWL1g\n" + "encrypted\n" + "-----BEGIN MESSAGE-----\n" + "YmVpbmcgb24gbW91bnRhaW5zLCB0aGlua2luZyBhYm91dCBjb21wdXRlcnMsIGlzIG5vdC" + "BiYWQgYXQgYWxs\n" + "-----END MESSAGE-----\n"; + +/* bad encrypted msg */ +static const char bad_superencrypted_text3[] = "desc-auth-type x25519\n" + "desc-auth-ephemeral-key A/O8DVtnUheb3r1JqoB8uJB7wxXL1XJX3eny4yB+eFA=\n" + "auth-client oiNrQB8WwKo S5D02W7vKgiWIMygrBl8RQ FB//SfOBmLEx1kViEWWL1g\n" + "encrypted\n" + "-----BEGIN MESSAGE-----\n" + "SO SMALL NOT GOOD\n" + "-----END MESSAGE-----\n"; + +static const char correct_superencrypted_text[] = "desc-auth-type x25519\n" + "desc-auth-ephemeral-key A/O8DVtnUheb3r1JqoB8uJB7wxXL1XJX3eny4yB+eFA=\n" + "auth-client oiNrQB8WwKo S5D02W7vKgiWIMygrBl8RQ FB//SfOBmLEx1kViEWWL1g\n" + "auth-client Od09Qu636Qo /PKLzqewAdS/+0+vZC+MvQ dpw4NFo13zDnuPz45rxrOg\n" + "auth-client JRr840iGYN0 8s8cxYqF7Lx23+NducC4Qg zAafl4wPLURkuEjJreZq1g\n" + "encrypted\n" + "-----BEGIN MESSAGE-----\n" + "YmVpbmcgb24gbW91bnRhaW5zLCB0aGlua2luZyBhYm91dCBjb21wdXRlcnMsIGlzIG5vdC" + "BiYWQgYXQgYWxs\n" + "-----END MESSAGE-----\n"; + +static const char correct_encrypted_plaintext[] = "being on mountains, " + "thinking about computers, is not bad at all"; + +static void +test_parse_hs_desc_superencrypted(void *arg) +{ + (void) arg; + size_t retval; + uint8_t *encrypted_out = NULL; + + { + setup_full_capture_of_logs(LOG_WARN); + retval = decode_superencrypted(bad_superencrypted_text1, + strlen(bad_superencrypted_text1), + &encrypted_out); + tt_u64_op(retval, ==, 0); + tt_assert(!encrypted_out); + expect_log_msg_containing("Unrecognized desc auth type"); + teardown_capture_of_logs(); + } + + { + setup_full_capture_of_logs(LOG_WARN); + retval = decode_superencrypted(bad_superencrypted_text2, + strlen(bad_superencrypted_text2), + &encrypted_out); + tt_u64_op(retval, ==, 0); + tt_assert(!encrypted_out); + expect_log_msg_containing("Bogus desc auth key in HS desc"); + teardown_capture_of_logs(); + } + + { + setup_full_capture_of_logs(LOG_WARN); + retval = decode_superencrypted(bad_superencrypted_text3, + strlen(bad_superencrypted_text3), + &encrypted_out); + tt_u64_op(retval, ==, 0); + tt_assert(!encrypted_out); + expect_log_msg_containing("Length of descriptor\'s encrypted data " + "is too small."); + teardown_capture_of_logs(); + } + + /* Now finally the good one */ + retval = decode_superencrypted(correct_superencrypted_text, + strlen(correct_superencrypted_text), + &encrypted_out); + + tt_u64_op(retval, ==, strlen(correct_encrypted_plaintext)); + tt_mem_op(encrypted_out, OP_EQ, correct_encrypted_plaintext, + strlen(correct_encrypted_plaintext)); + + done: + tor_free(encrypted_out); +} + struct testcase_t hs_descriptor[] = { /* Encoding tests. */ { "cert_encoding", test_cert_encoding, TT_FORK, @@ -1035,6 +1130,9 @@ struct testcase_t hs_descriptor[] = { { "desc_signature", test_desc_signature, TT_FORK, NULL, NULL }, + { "parse_hs_desc_superencrypted", test_parse_hs_desc_superencrypted, + TT_FORK, NULL, NULL }, + END_OF_TESTCASES }; diff --git a/src/test/test_hs_intropoint.c b/src/test/test_hs_intropoint.c index b207cd4ce3..2de2d957a2 100644 --- a/src/test/test_hs_intropoint.c +++ b/src/test/test_hs_intropoint.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/test/test_hs_service.c b/src/test/test_hs_service.c index 039d727cea..ae9c0a0300 100644 --- a/src/test/test_hs_service.c +++ b/src/test/test_hs_service.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/test/test_introduce.c b/src/test/test_introduce.c index 810b03c93d..cfb8d83b1d 100644 --- a/src/test/test_introduce.c +++ b/src/test/test_introduce.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2012-2016, The Tor Project, Inc. */ +/* Copyright (c) 2012-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/test/test_keypin.c b/src/test/test_keypin.c index 95657349c6..d2ec8e9ca7 100644 --- a/src/test/test_keypin.c +++ b/src/test/test_keypin.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2014-2016, The Tor Project, Inc. */ +/* Copyright (c) 2014-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/test/test_link_handshake.c b/src/test/test_link_handshake.c index 421f3aaedf..4e9fe040ff 100644 --- a/src/test/test_link_handshake.c +++ b/src/test/test_link_handshake.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2014-2016, The Tor Project, Inc. */ +/* Copyright (c) 2014-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/test/test_logging.c b/src/test/test_logging.c index 15471e46d0..94b3e4ea68 100644 --- a/src/test/test_logging.c +++ b/src/test/test_logging.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2016, The Tor Project, Inc. */ +/* Copyright (c) 2013-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/test/test_microdesc.c b/src/test/test_microdesc.c index 2ae605b8db..b8d96491d4 100644 --- a/src/test/test_microdesc.c +++ b/src/test/test_microdesc.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2010-2016, The Tor Project, Inc. */ +/* Copyright (c) 2010-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/test/test_nodelist.c b/src/test/test_nodelist.c index d58f8a7fca..256354415c 100644 --- a/src/test/test_nodelist.c +++ b/src/test/test_nodelist.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2007-2016, The Tor Project, Inc. */ +/* Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/test/test_ntor_cl.c b/src/test/test_ntor_cl.c index a560e5fc5e..d0eea85d6f 100644 --- a/src/test/test_ntor_cl.c +++ b/src/test/test_ntor_cl.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2012-2016, The Tor Project, Inc. */ +/* Copyright (c) 2012-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/test/test_oom.c b/src/test/test_oom.c index 0f97972032..f03a504d1d 100644 --- a/src/test/test_oom.c +++ b/src/test/test_oom.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2014-2016, The Tor Project, Inc. */ +/* Copyright (c) 2014-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /* Unit tests for OOM handling logic */ diff --git a/src/test/test_oos.c b/src/test/test_oos.c index db06625116..9fd6bce5ae 100644 --- a/src/test/test_oos.c +++ b/src/test/test_oos.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /* Unit tests for OOS handler */ diff --git a/src/test/test_options.c b/src/test/test_options.c index d5782e9ec0..1ba6a05e4a 100644 --- a/src/test/test_options.c +++ b/src/test/test_options.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define CONFIG_PRIVATE diff --git a/src/test/test_policy.c b/src/test/test_policy.c index e86d0f0274..1b2fac4325 100644 --- a/src/test/test_policy.c +++ b/src/test/test_policy.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2016, The Tor Project, Inc. */ +/* Copyright (c) 2013-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "or.h" diff --git a/src/test/test_procmon.c b/src/test/test_procmon.c index 9e63fc006d..5c52af8693 100644 --- a/src/test/test_procmon.c +++ b/src/test/test_procmon.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2010-2016, The Tor Project, Inc. */ +/* Copyright (c) 2010-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define PROCMON_PRIVATE diff --git a/src/test/test_protover.c b/src/test/test_protover.c index f00955d1b4..5626816024 100644 --- a/src/test/test_protover.c +++ b/src/test/test_protover.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define PROTOVER_PRIVATE diff --git a/src/test/test_pt.c b/src/test/test_pt.c index f50c75ab48..79b03171bc 100644 --- a/src/test/test_pt.c +++ b/src/test/test_pt.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/test/test_pubsub.c b/src/test/test_pubsub.c index 547d6c6b32..2f047d9f2c 100644 --- a/src/test/test_pubsub.c +++ b/src/test/test_pubsub.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/test/test_relay.c b/src/test/test_relay.c index 4713c79ea5..238d4c5baf 100644 --- a/src/test/test_relay.c +++ b/src/test/test_relay.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2014-2016, The Tor Project, Inc. */ +/* Copyright (c) 2014-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "or.h" diff --git a/src/test/test_relaycell.c b/src/test/test_relaycell.c index fb6748965a..eea1f5dc80 100644 --- a/src/test/test_relaycell.c +++ b/src/test/test_relaycell.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2014-2016, The Tor Project, Inc. */ +/* Copyright (c) 2014-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /* Unit tests for handling different kinds of relay cell */ diff --git a/src/test/test_rendcache.c b/src/test/test_rendcache.c index 0d53c78817..59d3be9003 100644 --- a/src/test/test_rendcache.c +++ b/src/test/test_rendcache.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2010-2016, The Tor Project, Inc. */ +/* Copyright (c) 2010-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/test/test_replay.c b/src/test/test_replay.c index e882bc6164..80e7203716 100644 --- a/src/test/test_replay.c +++ b/src/test/test_replay.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2012-2016, The Tor Project, Inc. */ +/* Copyright (c) 2012-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define REPLAYCACHE_PRIVATE diff --git a/src/test/test_routerkeys.c b/src/test/test_routerkeys.c index 64692d28a0..263923f50f 100644 --- a/src/test/test_routerkeys.c +++ b/src/test/test_routerkeys.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/test/test_routerlist.c b/src/test/test_routerlist.c index 78f1cf16b7..0b4b6c5c44 100644 --- a/src/test/test_routerlist.c +++ b/src/test/test_routerlist.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2014-2016, The Tor Project, Inc. */ +/* Copyright (c) 2014-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/test/test_scheduler.c b/src/test/test_scheduler.c index 05ea8e86e8..4c536b0905 100644 --- a/src/test/test_scheduler.c +++ b/src/test/test_scheduler.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2014-2016, The Tor Project, Inc. */ +/* Copyright (c) 2014-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/test/test_slow.c b/src/test/test_slow.c index 7c9f0b1cc2..e640702499 100644 --- a/src/test/test_slow.c +++ b/src/test/test_slow.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/test/test_socks.c b/src/test/test_socks.c index 62ff12fe15..bb1be11f2b 100644 --- a/src/test/test_socks.c +++ b/src/test/test_socks.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "or.h" diff --git a/src/test/test_storagedir.c b/src/test/test_storagedir.c new file mode 100644 index 0000000000..6678629df6 --- /dev/null +++ b/src/test/test_storagedir.c @@ -0,0 +1,270 @@ +/* Copyright (c) 2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "or.h" +#include "storagedir.h" +#include "test.h" + +#ifdef HAVE_UTIME_H +#include <utime.h> +#endif + +static void +test_storagedir_empty(void *arg) +{ + char *dirname = tor_strdup(get_fname_rnd("store_dir")); + storage_dir_t *d = NULL; + (void)arg; + + tt_int_op(FN_NOENT, OP_EQ, file_status(dirname)); + + d = storage_dir_new(dirname, 10); + tt_assert(d); + + tt_int_op(FN_DIR, OP_EQ, file_status(dirname)); + + tt_int_op(0, OP_EQ, smartlist_len(storage_dir_list(d))); + tt_u64_op(0, OP_EQ, storage_dir_get_usage(d)); + + storage_dir_free(d); + d = storage_dir_new(dirname, 10); + tt_assert(d); + + tt_int_op(FN_DIR, OP_EQ, file_status(dirname)); + + tt_int_op(0, OP_EQ, smartlist_len(storage_dir_list(d))); + tt_u64_op(0, OP_EQ, storage_dir_get_usage(d)); + + done: + storage_dir_free(d); + tor_free(dirname); +} + +static void +test_storagedir_basic(void *arg) +{ + char *dirname = tor_strdup(get_fname_rnd("store_dir")); + storage_dir_t *d = NULL; + uint8_t *junk = NULL, *bytes = NULL; + const size_t junklen = 1024; + char *fname1 = NULL, *fname2 = NULL; + const char hello_str[] = "then what are we but cold, alone ... ?"; + tor_mmap_t *mapping = NULL; + (void)arg; + + junk = tor_malloc(junklen); + crypto_rand((void*)junk, junklen); + + d = storage_dir_new(dirname, 10); + tt_assert(d); + tt_u64_op(0, OP_EQ, storage_dir_get_usage(d)); + + int r; + r = storage_dir_save_string_to_file(d, hello_str, 1, &fname1); + tt_int_op(r, OP_EQ, 0); + tt_ptr_op(fname1, OP_NE, NULL); + tt_u64_op(strlen(hello_str), OP_EQ, storage_dir_get_usage(d)); + + r = storage_dir_save_bytes_to_file(d, junk, junklen, 1, &fname2); + tt_int_op(r, OP_EQ, 0); + tt_ptr_op(fname2, OP_NE, NULL); + + tt_str_op(fname1, OP_NE, fname2); + + tt_int_op(2, OP_EQ, smartlist_len(storage_dir_list(d))); + tt_u64_op(junklen + strlen(hello_str), OP_EQ, storage_dir_get_usage(d)); + tt_assert(smartlist_contains_string(storage_dir_list(d), fname1)); + tt_assert(smartlist_contains_string(storage_dir_list(d), fname2)); + + storage_dir_free(d); + d = storage_dir_new(dirname, 10); + tt_assert(d); + tt_int_op(2, OP_EQ, smartlist_len(storage_dir_list(d))); + tt_u64_op(junklen + strlen(hello_str), OP_EQ, storage_dir_get_usage(d)); + tt_assert(smartlist_contains_string(storage_dir_list(d), fname1)); + tt_assert(smartlist_contains_string(storage_dir_list(d), fname2)); + + size_t n; + bytes = storage_dir_read(d, fname2, 1, &n); + tt_assert(bytes); + tt_u64_op(n, OP_EQ, junklen); + tt_mem_op(bytes, OP_EQ, junk, junklen); + + mapping = storage_dir_map(d, fname1); + tt_assert(mapping); + tt_u64_op(mapping->size, OP_EQ, strlen(hello_str)); + tt_mem_op(mapping->data, OP_EQ, hello_str, strlen(hello_str)); + + done: + tor_free(dirname); + tor_free(junk); + tor_free(bytes); + tor_munmap_file(mapping); + storage_dir_free(d); + tor_free(fname1); + tor_free(fname2); +} + +static void +test_storagedir_deletion(void *arg) +{ + (void)arg; + char *dirname = tor_strdup(get_fname_rnd("store_dir")); + storage_dir_t *d = NULL; + char *fn1 = NULL, *fn2 = NULL; + char *bytes = NULL; + int r; + const char str1[] = "There are nine and sixty ways to disguise communiques"; + const char str2[] = "And rather more than one of them is right"; + + // Make sure the directory is there. */ + d = storage_dir_new(dirname, 10); + storage_dir_free(d); + d = NULL; + + tor_asprintf(&fn1, "%s/1007", dirname); + r = write_str_to_file(fn1, str1, 0); + tt_int_op(r, OP_EQ, 0); + + tor_asprintf(&fn2, "%s/1003.tmp", dirname); + r = write_str_to_file(fn2, str2, 0); + tt_int_op(r, OP_EQ, 0); + + // The tempfile should be deleted the next time we list the directory. + d = storage_dir_new(dirname, 10); + tt_int_op(1, OP_EQ, smartlist_len(storage_dir_list(d))); + tt_u64_op(strlen(str1), OP_EQ, storage_dir_get_usage(d)); + tt_int_op(FN_FILE, OP_EQ, file_status(fn1)); + tt_int_op(FN_NOENT, OP_EQ, file_status(fn2)); + + bytes = (char*) storage_dir_read(d, "1007", 1, NULL); + tt_str_op(bytes, OP_EQ, str1); + + // Should have no effect; file already gone. + storage_dir_remove_file(d, "1003.tmp"); + tt_int_op(1, OP_EQ, smartlist_len(storage_dir_list(d))); + tt_u64_op(strlen(str1), OP_EQ, storage_dir_get_usage(d)); + + // Actually remove a file. + storage_dir_remove_file(d, "1007"); + tt_int_op(FN_NOENT, OP_EQ, file_status(fn1)); + tt_int_op(0, OP_EQ, smartlist_len(storage_dir_list(d))); + tt_u64_op(0, OP_EQ, storage_dir_get_usage(d)); + + done: + tor_free(dirname); + tor_free(fn1); + tor_free(fn2); + storage_dir_free(d); + tor_free(bytes); +} + +static void +test_storagedir_full(void *arg) +{ + (void)arg; + + char *dirname = tor_strdup(get_fname_rnd("store_dir")); + storage_dir_t *d = NULL; + const char str[] = "enemies of the peephole"; + int r; + + d = storage_dir_new(dirname, 3); + tt_assert(d); + + r = storage_dir_save_string_to_file(d, str, 1, NULL); + tt_int_op(r, OP_EQ, 0); + r = storage_dir_save_string_to_file(d, str, 1, NULL); + tt_int_op(r, OP_EQ, 0); + r = storage_dir_save_string_to_file(d, str, 1, NULL); + tt_int_op(r, OP_EQ, 0); + + // These should fail! + r = storage_dir_save_string_to_file(d, str, 1, NULL); + tt_int_op(r, OP_EQ, -1); + r = storage_dir_save_string_to_file(d, str, 1, NULL); + tt_int_op(r, OP_EQ, -1); + + tt_u64_op(strlen(str) * 3, OP_EQ, storage_dir_get_usage(d)); + + done: + tor_free(dirname); + storage_dir_free(d); +} + +static void +test_storagedir_cleaning(void *arg) +{ + (void)arg; + + char *dirname = tor_strdup(get_fname_rnd("store_dir")); + storage_dir_t *d = NULL; + const char str[] = + "On a mountain halfway between Reno and Rome / " + "We have a machine in a plexiglass dome / " + "Which listens and looks into everyone's home." + " -- Dr. Seuss"; + char *fns[8]; + int r, i; + + memset(fns, 0, sizeof(fns)); + d = storage_dir_new(dirname, 10); + tt_assert(d); + + for (i = 0; i < 8; ++i) { + r = storage_dir_save_string_to_file(d, str+i*2, 1, &fns[i]); + tt_int_op(r, OP_EQ, 0); + } + + /* Now we're going to make sure all the files have distinct mtimes. */ + time_t now = time(NULL); + struct utimbuf ub; + ub.actime = now; + ub.modtime -= 1000; + for (i = 0; i < 8; ++i) { + char *f = NULL; + tor_asprintf(&f, "%s/%s", dirname, fns[i]); + r = utime(f, &ub); + tor_free(f); + tt_int_op(r, OP_EQ, 0); + ub.modtime += 5; + } + + const uint64_t usage_orig = storage_dir_get_usage(d); + /* No changes needed if we are already under target. */ + storage_dir_shrink(d, 1024*1024, 0); + tt_u64_op(usage_orig, OP_EQ, storage_dir_get_usage(d)); + + /* Get rid of at least one byte. This will delete fns[0]. */ + storage_dir_shrink(d, usage_orig - 1, 0); + tt_u64_op(usage_orig, OP_GT, storage_dir_get_usage(d)); + tt_u64_op(usage_orig - strlen(str), OP_EQ, storage_dir_get_usage(d)); + + /* Get rid of at least two files. This will delete fns[1] and fns[2]. */ + storage_dir_shrink(d, 1024*1024, 2); + tt_u64_op(usage_orig - strlen(str)*3 + 6, OP_EQ, storage_dir_get_usage(d)); + + /* Get rid of everything. */ + storage_dir_remove_all(d); + tt_u64_op(0, OP_EQ, storage_dir_get_usage(d)); + + done: + tor_free(dirname); + storage_dir_free(d); + for (i = 0; i < 8; ++i) { + tor_free(fns[i]); + } +} + +#define ENT(name) \ + { #name, test_storagedir_ ## name, TT_FORK, NULL, NULL } + +struct testcase_t storagedir_tests[] = { + ENT(empty), + ENT(basic), + ENT(deletion), + ENT(full), + ENT(cleaning), + END_OF_TESTCASES +}; + diff --git a/src/test/test_switch_id.c b/src/test/test_switch_id.c index 322f5bdc7a..44e02e87b3 100644 --- a/src/test/test_switch_id.c +++ b/src/test/test_switch_id.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2015-2016, The Tor Project, Inc. */ +/* Copyright (c) 2015-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "or.h" diff --git a/src/test/test_threads.c b/src/test/test_threads.c index ebbc95c7ca..18a9407ff7 100644 --- a/src/test/test_threads.c +++ b/src/test/test_threads.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/test/test_tortls.c b/src/test/test_tortls.c index 4bfcea211d..18ffd4a8c1 100644 --- a/src/test/test_tortls.c +++ b/src/test/test_tortls.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2010-2016, The Tor Project, Inc. */ +/* Copyright (c) 2010-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define TORTLS_PRIVATE diff --git a/src/test/test_util.c b/src/test/test_util.c index 130a019d5c..203f9dd1c4 100644 --- a/src/test/test_util.c +++ b/src/test/test_util.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" @@ -4029,6 +4029,16 @@ test_util_string_from_pipe(void *ptr) tt_mem_op(buf, OP_EQ, "B\0\xff\xff", sizeof(buf)); errno = 0; + /* Send in multiple lines. */ + retlen = write(test_pipe[1], "A\nB", 3); + tt_int_op(retlen, OP_EQ, 3); + + status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1); + tt_int_op(errno, OP_EQ, 0); + tt_int_op(status, OP_EQ, IO_STREAM_OKAY); + tt_str_op(buf, OP_EQ, "A\nB"); + errno = 0; + /* Send in a line and close */ retlen = write(test_pipe[1], "AB", 2); tt_int_op(retlen, OP_EQ, 2); diff --git a/src/test/test_util_format.c b/src/test/test_util_format.c index 21a6923c6d..a689b2de6c 100644 --- a/src/test/test_util_format.c +++ b/src/test/test_util_format.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2010-2016, The Tor Project, Inc. */ +/* Copyright (c) 2010-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/test/test_util_process.c b/src/test/test_util_process.c index 4e75b97f3d..70292f2287 100644 --- a/src/test/test_util_process.c +++ b/src/test/test_util_process.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2010-2016, The Tor Project, Inc. */ +/* Copyright (c) 2010-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define UTIL_PROCESS_PRIVATE diff --git a/src/test/test_util_slow.c b/src/test/test_util_slow.c index 96a2fa756b..3e5d78948d 100644 --- a/src/test/test_util_slow.c +++ b/src/test/test_util_slow.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/test/test_workqueue.c b/src/test/test_workqueue.c index ccb8d0c8ca..10714457f7 100644 --- a/src/test/test_workqueue.c +++ b/src/test/test_workqueue.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "or.h" diff --git a/src/test/testing_common.c b/src/test/testing_common.c index caeae13a38..bb2bcbf44c 100644 --- a/src/test/testing_common.c +++ b/src/test/testing_common.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ extern const char tor_git_revision[]; diff --git a/src/test/testing_rsakeys.c b/src/test/testing_rsakeys.c index 134770bb0d..5dff233a69 100644 --- a/src/test/testing_rsakeys.c +++ b/src/test/testing_rsakeys.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/tools/tor-checkkey.c b/src/tools/tor-checkkey.c index 3e16fd0336..f2f709dd78 100644 --- a/src/tools/tor-checkkey.c +++ b/src/tools/tor-checkkey.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2008-2015, The Tor Project, Inc. */ +/* Copyright (c) 2008-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/tools/tor-gencert.c b/src/tools/tor-gencert.c index db308485e6..395535697f 100644 --- a/src/tools/tor-gencert.c +++ b/src/tools/tor-gencert.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2007-2015, The Tor Project, Inc. */ +/* Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/tools/tor-resolve.c b/src/tools/tor-resolve.c index 6ac866d3c0..1e2409a131 100644 --- a/src/tools/tor-resolve.c +++ b/src/tools/tor-resolve.c @@ -1,5 +1,5 @@ /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson - * Copyright (c) 2007-2015, The Tor Project, Inc. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ |