diff options
80 files changed, 4104 insertions, 2807 deletions
@@ -1,3 +1,90 @@ +Changes in version 0.3.1.6-rc - 2017-09-05 + Tor 0.3.1.6-rc fixes a few small bugs and annoyances in the 0.3.1 + release series, including a bug that produced weird behavior on + Windows directory caches. + + This is the first release candidate in the Tor 0.3.1 series. If we + find no new bugs or regressions here, the first stable 0.3.1 release + will be nearly identical to it. + + o Major bugfixes (windows, directory cache): + - On Windows, do not try to delete cached consensus documents and + diffs before they are unmapped from memory--Windows won't allow + that. Instead, allow the consensus cache directory to grow larger, + to hold files that might need to stay around longer. Fixes bug + 22752; bugfix on 0.3.1.1-alpha. + + o Minor features (directory authority): + - Improve the message that authorities report to relays that present + RSA/Ed25519 keypairs that conflict with previously pinned keys. + Closes ticket 22348. + + o Minor features (geoip): + - Update geoip and geoip6 to the August 3 2017 Maxmind GeoLite2 + Country database. + + o Minor features (testing): + - Add more tests for compression backend initialization. Closes + ticket 22286. + + o Minor bugfixes (directory cache): + - Fix a memory leak when recovering space in the consensus cache. + Fixes bug 23139; bugfix on 0.3.1.1-alpha. + + o Minor bugfixes (hidden service): + - Increase the number of circuits that a service is allowed to + open over a specific period of time. The value was lower than it + should be (8 vs 12) in the normal case of 3 introduction points. + Fixes bug 22159; bugfix on 0.3.0.5-rc. + - Fix a BUG warning during HSv3 descriptor decoding that could be + cause by a specially crafted descriptor. Fixes bug 23233; bugfix + on 0.3.0.1-alpha. Bug found by "haxxpop". + - Rate-limit the log messages if we exceed the maximum number of + allowed intro circuits. Fixes bug 22159; bugfix on 0.3.1.1-alpha. + + o Minor bugfixes (logging, relay): + - Remove a forgotten debugging message when an introduction point + successfully establishes a hidden service prop224 circuit with + a client. + - Change three other log_warn() for an introduction point to + protocol warnings, because they can be failure from the network + and are not relevant to the operator. Fixes bug 23078; bugfix on + 0.3.0.1-alpha and 0.3.0.2-alpha. + + o Minor bugfixes (relay): + - When a relay is not running as a directory cache, it will no + longer generate compressed consensuses and consensus diff + information. Previously, this was a waste of disk and CPU. Fixes + bug 23275; bugfix on 0.3.1.1-alpha. + + o Minor bugfixes (robustness, error handling): + - Improve our handling of the cases where OpenSSL encounters a + memory error while encoding keys and certificates. We haven't + observed these errors in the wild, but if they do happen, we now + detect and respond better. Fixes bug 19418; bugfix on all versions + of Tor. Reported by Guido Vranken. + + o Minor bugfixes (stability): + - Avoid crashing on a double-free when unable to load or process an + included file. Fixes bug 23155; bugfix on 0.3.1.1-alpha. Found + with the clang static analyzer. + + o Minor bugfixes (testing): + - Fix an undersized buffer in test-memwipe.c. Fixes bug 23291; + bugfix on 0.2.7.2-alpha. Found and patched by Ties Stuij. + - Port the hs_ntor handshake test to work correctly with recent + versions of the pysha3 module. Fixes bug 23071; bugfix + on 0.3.1.1-alpha. + + o Minor bugfixes (Windows service): + - When running as a Windows service, set the ID of the main thread + correctly. Failure to do so made us fail to send log messages to + the controller in 0.2.1.16-rc, slowed down controller event + delivery in 0.2.7.3-rc and later, and crash with an assertion + failure in 0.3.1.1-alpha. Fixes bug 23081; bugfix on 0.2.1.6-alpha. + Patch and diagnosis from "Vort". + + Changes in version 0.3.0.10 - 2017-08-02 Tor 0.3.0.10 backports a collection of small-to-medium bugfixes from the current Tor alpha series. OpenBSD users and TPROXY users @@ -21,7 +108,7 @@ Changes in version 0.3.0.10 - 2017-08-02 o Major bugfixes (openbsd, denial-of-service, backport from 0.3.1.5-alpha): - Avoid an assertion failure bug affecting our implementation of inet_pton(AF_INET6) on certain OpenBSD systems whose strtol() - handling of "0xfoo" differs from what we had expected. Fixes bug + handling of "0xbar" differs from what we had expected. Fixes bug 22789; bugfix on 0.2.3.8-alpha. Also tracked as TROVE-2017-007. o Minor features (backport from 0.3.1.5-alpha): @@ -93,7 +180,7 @@ Changes in version 0.3.1.5-alpha - 2017-08-01 o Major bugfixes (openbsd, denial-of-service): - Avoid an assertion failure bug affecting our implementation of inet_pton(AF_INET6) on certain OpenBSD systems whose strtol() - handling of "0xfoo" differs from what we had expected. Fixes bug + handling of "0xbar" differs from what we had expected. Fixes bug 22789; bugfix on 0.2.3.8-alpha. Also tracked as TROVE-2017-007. o Major bugfixes (relay, performance): diff --git a/ReleaseNotes b/ReleaseNotes index 668ee533cf..0012c060f5 100644 --- a/ReleaseNotes +++ b/ReleaseNotes @@ -25,7 +25,7 @@ Changes in version 0.3.0.10 - 2017-08-02 o Major bugfixes (openbsd, denial-of-service, backport from 0.3.1.5-alpha): - Avoid an assertion failure bug affecting our implementation of inet_pton(AF_INET6) on certain OpenBSD systems whose strtol() - handling of "0xfoo" differs from what we had expected. Fixes bug + handling of "0xbar" differs from what we had expected. Fixes bug 22789; bugfix on 0.2.3.8-alpha. Also tracked as TROVE-2017-007. o Minor features (backport from 0.3.1.5-alpha): diff --git a/changes/bug19418 b/changes/bug19418 deleted file mode 100644 index fb5f6ad5df..0000000000 --- a/changes/bug19418 +++ /dev/null @@ -1,7 +0,0 @@ - o Minor bugfixes (robustness, error handling): - - Improve our handling of the cases where OpenSSL encounters a - memory error while encoding keys and certificates. We haven't - observed these happening in the wild, but if they do happen, - we now detect and respond better. Fixes bug 19418; bugfix - on all versions of Tor. Reported by Guido Vranken. - diff --git a/changes/bug22159 b/changes/bug22159 deleted file mode 100644 index c319c7e322..0000000000 --- a/changes/bug22159 +++ /dev/null @@ -1,7 +0,0 @@ - o Minor bugfixes (hidden service): - - A service is allowed to open a maximum number of circuits for a specific - period of time. That value was lower than it should be (8 vs 12) in the - normal case of 3 introduction points. Fixes bug 22159.; bugfix on - tor-0.3.0.5-rc. - - Rate limit the log if we ever go above the maximum number of allowed - intro circuits. Fixes bug 22159.; bugfix on tor-0.3.1.1-alpha. diff --git a/changes/bug22286 b/changes/bug22286 deleted file mode 100644 index f72e8fe2c7..0000000000 --- a/changes/bug22286 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor features (tests): - - Add a couple more tests for compression backend initialization. - Closes ticket 22286. diff --git a/changes/bug23071 b/changes/bug23071 deleted file mode 100644 index 4756dd6252..0000000000 --- a/changes/bug23071 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor bugfixes (tests): - - Port the hs_ntor handshake test to work correctly with recent - versions of the pysha3 module. Fixes bug 23071; bugfix on - 0.3.1.1-alpha. - diff --git a/changes/bug23078 b/changes/bug23078 deleted file mode 100644 index 67624007cf..0000000000 --- a/changes/bug23078 +++ /dev/null @@ -1,7 +0,0 @@ - o Minor bugfixes (logging, relay): - - Remove a log_warn() that has been forgotten when an introduction point - successfully established a hidden service prop224 circuit with a client. - - Three other log_warn() for an introduction point have been changed to - protocol warning because they can be failure from the network and are - not relevant to the operator. Fixes bug 23078; bugfix on - tor-0.3.0.1-alpha and tor-0.3.0.2-alpha. diff --git a/changes/bug23081 b/changes/bug23081 deleted file mode 100644 index 76c4e30971..0000000000 --- a/changes/bug23081 +++ /dev/null @@ -1,8 +0,0 @@ - o Minor bugfixes (Windows service): - - When running as a Windows service, set the ID of the main thread - correctly. Failure to do so made us fail to send log messages - to the controller in 0.2.1.16-rc, slowed down controller - event delivery in 0.2.7.3-rc and later, and crash with an assertion - failure in 0.3.1.1-alpha. Fixes bug 23081; bugfix on 0.2.1.6-alpha. - Patch and diagnosis from "Vort". - diff --git a/changes/bug23139 b/changes/bug23139 deleted file mode 100644 index ed63ce85e2..0000000000 --- a/changes/bug23139 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (directory cache): - - Fix a memory leak in the code that recovers space in the consensus - directory cache. Fixes bug 23139; bugfix on 0.3.1.1-alpha. diff --git a/changes/bug23155 b/changes/bug23155 deleted file mode 100644 index 4c24ab136c..0000000000 --- a/changes/bug23155 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (stability): - - Avoid crashing on double-free when unable to load or process - an included file. Fixes bug 23155; bugfix on 0.3.1.1-alpha. - Found with the clang static analyzer. diff --git a/changes/bug23233 b/changes/bug23233 deleted file mode 100644 index 689a99a2a8..0000000000 --- a/changes/bug23233 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (hidden service): - - Fix a BUG alert during HSv3 descriptor decoding that could trigger with a - specially crafted descriptor. Fixes bug #23233; bugfix on 0.3.0.1-alpha. - Bug found by "haxxpop". diff --git a/changes/bug23291 b/changes/bug23291 deleted file mode 100644 index a5b0efda0a..0000000000 --- a/changes/bug23291 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (testing): - - Fix an undersized buffer in test-memwipe.c. Fixes bug 23291; bugfix on - 0.2.7.2-alpha. Found and patched by Ties Stuij. diff --git a/changes/bug23366 b/changes/bug23366 new file mode 100644 index 0000000000..85e370f61d --- /dev/null +++ b/changes/bug23366 @@ -0,0 +1,4 @@ + o Minor bugfixes (test): + - Fix a broken OutboundAddress option unit test because the parsing + function was never returning an error on failure. Fixes bug #23366.; + bugfix on tor-0.3.0.3-alpha. diff --git a/changes/feature22407 b/changes/feature22407 new file mode 100644 index 0000000000..aec6c15f46 --- /dev/null +++ b/changes/feature22407 @@ -0,0 +1,5 @@ + o Minor features (client): + - You can now use Tor as a tunneled HTTP proxy: use the HTTPTunnelPort + option to open a port that accepts HTTP CONNECT requests. + Closes ticket 22407. + diff --git a/changes/feature23090 b/changes/feature23090 new file mode 100644 index 0000000000..44cdac5ca1 --- /dev/null +++ b/changes/feature23090 @@ -0,0 +1,3 @@ + o Minor features (linux seccomp2 sandbox): + - If the sandbox filter fails to load, suggest to the user that their + kernel might not support seccomp2. Closes ticket 23090. diff --git a/changes/geoip-august2017 b/changes/geoip-august2017 deleted file mode 100644 index 2dab18a63f..0000000000 --- a/changes/geoip-august2017 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor features: - - Update geoip and geoip6 to the August 3 2017 Maxmind GeoLite2 - Country database. - diff --git a/changes/refactor-buffer b/changes/refactor-buffer new file mode 100644 index 0000000000..36b0296728 --- /dev/null +++ b/changes/refactor-buffer @@ -0,0 +1,3 @@ + o Code simplifications and refactoring: + - Split the portions of the buffer.c module that handle particular + protocols into separate modules. Part of ticket 23149. diff --git a/changes/ticket22348 b/changes/ticket22348 deleted file mode 100644 index 49ae94cdf3..0000000000 --- a/changes/ticket22348 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor features (directory authority): - - Improve the message that authorities report to relays when - the RSA/Ed25519 key pair they present conflicts with a previously - pinned key. Closes ticket 22348. - diff --git a/doc/HACKING/CodingStandardsRust.md b/doc/HACKING/CodingStandardsRust.md new file mode 100644 index 0000000000..d0b17c1604 --- /dev/null +++ b/doc/HACKING/CodingStandardsRust.md @@ -0,0 +1,469 @@ + + Rust Coding Standards +======================= + +You MUST follow the standards laid out in `.../doc/HACKING/CodingStandards.md`, +where applicable. + + Module/Crate Declarations +--------------------------- + +Each Tor C module which is being rewritten MUST be in its own crate. +See the structure of `.../src/rust` for examples. + +In your crate, you MUST use `lib.rs` ONLY for pulling in external +crates (e.g. `extern crate libc;`) and exporting public objects from +other Rust modules (e.g. `pub use mymodule::foo;`). For example, if +you create a crate in `.../src/rust/yourcrate`, your Rust code should +live in `.../src/rust/yourcrate/yourcode.rs` and the public interface +to it should be exported in `.../src/rust/yourcrate/lib.rs`. + +If your code is to be called from Tor C code, you MUST define a safe +`ffi.rs`. See the "Safety" section further down for more details. + +For example, in a hypothetical `tor_addition` Rust module: + +In `.../src/rust/tor_addition/addition.rs`: + + pub fn get_sum(a: i32, b: i32) -> i32 { + a + b + } + +In `.../src/rust/tor_addition/lib.rs`: + + pub use addition::*; + +In `.../src/rust/tor_addition/ffi.rs`: + + #[no_mangle] + pub extern "C" fn tor_get_sum(a: c_int, b: c_int) -> c_int { + get_sum(a, b) + } + +If your Rust code must call out to parts of Tor's C code, you must +declare the functions you are calling in the `external` crate, located +at `.../src/rust/external`. + +<!-- XXX get better examples of how to declare these externs, when/how they --> +<!-- XXX are unsafe, what they are expected to do —isis --> + +Modules should strive to be below 500 lines (tests excluded). Single +responsibility and limited dependencies should be a guiding standard. + +If you have any external modules as dependencies (e.g. `extern crate +libc;`), you MUST declare them in your crate's `lib.rs` and NOT in any +other module. + + Dependencies +-------------- + +In general, we use modules from only the Rust standard library +whenever possible. We will review including external crates on a +case-by-case basis. + + Documentation +--------------- + +You MUST include `#[deny(missing_docs)]` in your crate. + +For function/method comments, you SHOULD include a one-sentence, "first person" +description of function behaviour (see requirements for documentation as +described in `.../src/HACKING/CodingStandards.md`), then an `# Inputs` section +for inputs or initialisation values, a `# Returns` section for return +values/types, a `# Warning` section containing warnings for unsafe behaviours or +panics that could happen. For publicly accessible +types/constants/objects/functions/methods, you SHOULD also include an +`# Examples` section with runnable doctests. + +You MUST document your module with _module docstring_ comments, +i.e. `//!` at the beginning of each line. + + Style +------- + +You SHOULD consider breaking up large literal numbers with `_` when it makes it +more human readable to do so, e.g. `let x: u64 = 100_000_000_000`. + + Testing +--------- + +All code MUST be unittested and integration tested. + +Public functions/objects exported from a crate SHOULD include doctests +describing how the function/object is expected to be used. + +Integration tests SHOULD go into a `tests/` directory inside your +crate. Unittests SHOULD go into their own module inside the module +they are testing, e.g. in `.../src/rust/tor_addition/addition.rs` you +should put: + + #[cfg(test)] + mod test { + use super::*; + + #[test] + fn addition_with_zero() { + let sum: i32 = get_sum(5i32, 0i32); + assert_eq!(sum, 5); + } + } + + Benchmarking +-------------- + +The external `test` crate can be used for most benchmarking. However, using +this crate requires nightly Rust. Since we may want to switch to a more +stable Rust compiler eventually, we shouldn't do things which will automatically +break builds for stable compilers. Therefore, you MUST feature-gate your +benchmarks in the following manner. + +If you wish to benchmark some of your Rust code, you MUST put the +following in the `[features]` section of your crate's `Cargo.toml`: + + [features] + bench = [] + +Next, in your crate's `lib.rs` you MUST put: + + #[cfg(all(test, feature = "bench"))] + extern crate test; + +This ensures that the external crate `test`, which contains utilities +for basic benchmarks, is only used when running benchmarks via `cargo +bench --features bench`. + +Finally, to write your benchmark code, in +`.../src/rust/tor_addition/addition.rs` you SHOULD put: + + #[cfg(all(test, features = "bench"))] + mod bench { + use test::Bencher; + use super::*; + + #[bench] + fn addition_small_integers(b: &mut Bencher) { + b.iter(| | get_sum(5i32, 0i32)); + } + } + + Fuzzing +--------- + +If you wish to fuzz parts of your code, please see the +[`cargo fuzz`](https://github.com/rust-fuzz/cargo-fuzz) crate, which uses +[libfuzzer-sys](https://github.com/rust-fuzz/libfuzzer-sys). + + Whitespace & Formatting +------------------------- + +You MUST run `rustfmt` (https://github.com/rust-lang-nursery/rustfmt) +on your code before your code will be merged. You can install rustfmt +by doing `cargo install rustfmt-nightly` and then run it with `cargo +fmt`. + + Safety +-------- + +You SHOULD read [the nomicon](https://doc.rust-lang.org/nomicon/) before writing +Rust FFI code. It is *highly advised* that you read and write normal Rust code +before attempting to write FFI or any other unsafe code. + +Here are some additional bits of advice and rules: + +0. Any behaviours which Rust considers to be undefined are forbidden + + From https://doc.rust-lang.org/reference/behavior-considered-undefined.html: + + > Behavior considered undefined + > + > The following is a list of behavior which is forbidden in all Rust code, + > including within unsafe blocks and unsafe functions. Type checking provides the + > guarantee that these issues are never caused by safe code. + > + > * Data races + > * Dereferencing a null/dangling raw pointer + > * Reads of [undef](http://llvm.org/docs/LangRef.html#undefined-values) + > (uninitialized) memory + > * Breaking the + > [pointer aliasing rules](http://llvm.org/docs/LangRef.html#pointer-aliasing-rules) + > with raw pointers (a subset of the rules used by C) + > * `&mut T` and `&T` follow LLVM’s scoped noalias model, except if the `&T` + > contains an `UnsafeCell<U>`. Unsafe code must not violate these aliasing + > guarantees. + > * Mutating non-mutable data (that is, data reached through a shared + > reference or data owned by a `let` binding), unless that data is + > contained within an `UnsafeCell<U>`. + > * Invoking undefined behavior via compiler intrinsics: + > - Indexing outside of the bounds of an object with + > `std::ptr::offset` (`offset` intrinsic), with the exception of + > one byte past the end which is permitted. + > - Using `std::ptr::copy_nonoverlapping_memory` (`memcpy32`/`memcpy64` + > intrinsics) on overlapping buffers + > * Invalid values in primitive types, even in private fields/locals: + > - Dangling/null references or boxes + > - A value other than `false` (0) or `true` (1) in a `bool` + > - A discriminant in an `enum` not included in the type definition + > - A value in a `char` which is a surrogate or above `char::MAX` + > - Non-UTF-8 byte sequences in a `str` + > * Unwinding into Rust from foreign code or unwinding from Rust into foreign + > code. Rust's failure system is not compatible with exception handling in other + > languages. Unwinding must be caught and handled at FFI boundaries. + +1. `unwrap()` + + If you call `unwrap()`, anywhere, even in a test, you MUST include + an inline comment stating how the unwrap will either 1) never fail, + or 2) should fail (i.e. in a unittest). + + You SHOULD NOT use `unwrap()` anywhere in which it is possible to handle the + potential error with either `expect()` or the eel operator, `?`. + For example, consider a function which parses a string into an integer: + + fn parse_port_number(config_string: &str) -> u16 { + u16::from_str_radix(config_string, 10).unwrap() + } + + There are numerous ways this can fail, and the `unwrap()` will cause the + whole program to byte the dust! Instead, either you SHOULD use `expect()` + (or another equivalent function which will return an `Option` or a `Result`) + and change the return type to be compatible: + + fn parse_port_number(config_string: &str) -> Option<u16> { + u16::from_str_radix(config_string, 10).expect("Couldn't parse port into a u16") + } + + or you SHOULD use `or()` (or another similar method): + + fn parse_port_number(config_string: &str) -> Option<u16> { + u16::from_str_radix(config_string, 10).or(Err("Couldn't parse port into a u16") + } + + Using methods like `or()` can be particularly handy when you must do + something afterwards with the data, for example, if we wanted to guarantee + that the port is high. Combining these methods with the eel operator (`?`) + makes this even easier: + + fn parse_port_number(config_string: &str) -> Result<u16, Err> { + let port = u16::from_str_radix(config_string, 10).or(Err("Couldn't parse port into a u16"))?; + + if port > 1024 { + return Ok(port); + } else { + return Err("Low ports not allowed"); + } + } + +2. `unsafe` + + If you use `unsafe`, you MUST describe a contract in your + documentation which describes how and when the unsafe code may + fail, and what expectations are made w.r.t. the interfaces to + unsafe code. This is also REQUIRED for major pieces of FFI between + C and Rust. + + When creating an FFI in Rust for C code to call, it is NOT REQUIRED + to declare the entire function `unsafe`. For example, rather than doing: + + #[no_mangle] + pub unsafe extern "C" fn increment_and_combine_numbers(mut numbers: [u8; 4]) -> u32 { + for number in &mut numbers { + *number += 1; + } + std::mem::transmute::<[u8; 4], u32>(numbers) + } + + You SHOULD instead do: + + #[no_mangle] + pub extern "C" fn increment_and_combine_numbers(mut numbers: [u8; 4]) -> u32 { + for index in 0..numbers.len() { + numbers[index] += 1; + } + unsafe { + std::mem::transmute::<[u8; 4], u32>(numbers) + } + } + +3. Pass only integer types and bytes over the boundary + + The only non-integer type which may cross the FFI boundary is + bytes, e.g. `&[u8]`. This SHOULD be done on the Rust side by + passing a pointer (`*mut libc::c_char`) and a length + (`libc::size_t`). + + One might be tempted to do this via doing + `CString::new("blah").unwrap().into_raw()`. This has several problems: + + a) If you do `CString::new("bl\x00ah")` then the unwrap() will fail + due to the additional NULL terminator, causing a dangling + pointer to be returned (as well as a potential use-after-free). + + b) Returning the raw pointer will cause the CString to run its deallocator, + which causes any C code which tries to access the contents to dereference a + NULL pointer. + + c) If we were to do `as_raw()` this would result in a potential double-free + since the Rust deallocator would run and possibly Tor's deallocator. + + d) Calling `into_raw()` without later using the same pointer in Rust to call + `from_raw()` and then deallocate in Rust can result in a + [memory leak](https://doc.rust-lang.org/std/ffi/struct.CString.html#method.into_raw). + + [It was determined](https://github.com/rust-lang/rust/pull/41074) that this + is safe to do if you use the same allocator in C and Rust and also specify + the memory alignment for CString (except that there is no way to specify + the alignment for CString). It is believed that the alignment is always 1, + which would mean it's safe to dealloc the resulting `*mut c_char` in Tor's + C code. However, the Rust developers are not willing to guarantee the + stability of, or a contract for, this behaviour, citing concerns that this + is potentially extremely and subtly unsafe. + +4. Perform an allocation on the other side of the boundary + + After crossing the boundary, the other side MUST perform an + allocation to copy the data and is therefore responsible for + freeing that memory later. + +5. No touching other language's enums + + Rust enums should never be touched from C (nor can they be safely + `#[repr(C)]`) nor vice versa: + + > "The chosen size is the default enum size for the target platform's C + > ABI. Note that enum representation in C is implementation defined, so this is + > really a "best guess". In particular, this may be incorrect when the C code + > of interest is compiled with certain flags." + + (from https://gankro.github.io/nomicon/other-reprs.html) + +6. Type safety + + Wherever possible and sensical, you SHOULD create new types in a + manner which prevents type confusion or misuse. For example, + rather than using an untyped mapping between strings and integers + like so: + + use std::collections::HashMap; + + pub fn get_elements_with_over_9000_points(map: &HashMap<String, usize>) -> Vec<String> { + ... + } + + It would be safer to define a new type, such that some other usage + of `HashMap<String, usize>` cannot be confused for this type: + + pub struct DragonBallZPowers(pub HashMap<String, usize>); + + impl DragonBallZPowers { + pub fn over_nine_thousand<'a>(&'a self) -> Vec<&'a String> { + let mut powerful_enough: Vec<&'a String> = Vec::with_capacity(5); + + for (character, power) in &self.0 { + if *power > 9000 { + powerful_enough.push(character); + } + } + powerful_enough + } + } + + Note the following code, which uses Rust's type aliasing, is valid + but it does NOT meet the desired type safety goals: + + pub type Power = usize; + + pub fn over_nine_thousand(power: &Power) -> bool { + if *power > 9000 { + return true; + } + false + } + + // We can still do the following: + let his_power: usize = 9001; + over_nine_thousand(&his_power); + +7. Unsafe mucking around with lifetimes + + Because lifetimes are technically, in type theory terms, a kind, i.e. a + family of types, individual lifetimes can be treated as types. For example, + one can arbitrarily extend and shorten lifetime using `std::mem::transmute`: + + struct R<'a>(&'a i32); + + unsafe fn extend_lifetime<'b>(r: R<'b>) -> R<'static> { + std::mem::transmute::<R<'b>, R<'static>>(r) + } + + unsafe fn shorten_invariant_lifetime<'b, 'c>(r: &'b mut R<'static>) -> &'b mut R<'c> { + std::mem::transmute::<&'b mut R<'static>, &'b mut R<'c>>(r) + } + + Calling `extend_lifetime()` would cause an `R` passed into it to live forever + for the life of the program (the `'static` lifetime). Similarly, + `shorten_invariant_lifetime()` could be used to take something meant to live + forever, and cause it to disappear! This is incredibly unsafe. If you're + going to be mucking around with lifetimes like this, first, you better have + an extremely good reason, and second, you may as be honest and explicit about + it, and for ferris' sake just use a raw pointer. + + In short, just because lifetimes can be treated like types doesn't mean you + should do it. + +8. Doing excessively unsafe things when there's a safer alternative + + Similarly to #7, often there are excessively unsafe ways to do a task and a + simpler, safer way. You MUST choose the safer option where possible. + + For example, `std::mem::transmute` can be abused in ways where casting with + `as` would be both simpler and safer: + + // Don't do this + let ptr = &0; + let ptr_num_transmute = unsafe { std::mem::transmute::<&i32, usize>(ptr)}; + + // Use an `as` cast instead + let ptr_num_cast = ptr as *const i32 as usize; + + In fact, using `std::mem::transmute` for *any* reason is a code smell and as + such SHOULD be avoided. + +9. Casting integers with `as` + + This is generally fine to do, but it has some behaviours which you should be + aware of. Casting down chops off the high bits, e.g.: + + let x: u32 = 4294967295; + println!("{}", x as u16); // prints 65535 + + Some cases which you MUST NOT do include: + + * Casting an `u128` down to an `f32` or vice versa (e.g. + `u128::MAX as f32` but this isn't only a problem with overflowing + as it is also undefined behaviour for `42.0f32 as u128`), + + * Casting between integers and floats when the thing being cast + cannot fit into the type it is being casted into, e.g.: + + println!("{}", 42949.0f32 as u8); // prints 197 in debug mode and 0 in release + println!("{}", 1.04E+17 as u8); // prints 0 in both modes + println!("{}", (0.0/0.0) as i64); // prints whatever the heck LLVM wants + + Because this behaviour is undefined, it can even produce segfaults in + safe Rust code. For example, the following program built in release + mode segfaults: + + #[inline(never)] + pub fn trigger_ub(sl: &[u8; 666]) -> &[u8] { + // Note that the float is out of the range of `usize`, invoking UB when casting. + let idx = 1e99999f64 as usize; + &sl[idx..] // The bound check is elided due to `idx` being of an undefined value. + } + + fn main() { + println!("{}", trigger_ub(&[1; 666])[999999]); // ~ out of bound + } + + And in debug mode panics with: + + thread 'main' panicked at 'slice index starts at 140721821254240 but ends at 666', /checkout/src/libcore/slice/mod.rs:754:4 diff --git a/doc/HACKING/GettingStartedRust.md b/doc/HACKING/GettingStartedRust.md new file mode 100644 index 0000000000..cfb5196aa4 --- /dev/null +++ b/doc/HACKING/GettingStartedRust.md @@ -0,0 +1,158 @@ + + Hacking on Rust in Tor +======================== + + Getting Started +----------------- + +Please read or review our documentation on Rust coding standards +(`.../doc/HACKING/CodingStandardsRust.md`) before doing anything. + +Please also read +[the Rust Code of Conduct](https://www.rust-lang.org/en-US/conduct.html). We aim +to follow the good example set by the Rust community and be excellent to one +another. Let's be careful with each other, so we can be memory-safe together! + +Next, please contact us before rewriting anything! Rust in Tor is still an +experiment. It is an experiment that we very much want to see succeed, so we're +going slowly and carefully. For the moment, it's also a completely +volunteer-driven effort: while many, if not most, of us are paid to work on Tor, +we are not yet funded to write Rust code for Tor. Please be patient with the +other people who are working on getting more Rust code into Tor, because they +are graciously donating their free time to contribute to this effort. + + Resources for learning Rust +----------------------------- + +**Beginning resources** + +The primary resource for learning Rust is +[The Book](https://doc.rust-lang.org/book/). If you'd like to start writing +Rust immediately, without waiting for anything to install, there is +[an interactive browser-based playground](https://play.rust-lang.org/). + +**Advanced resources** + +If you're interested in playing with various Rust compilers and viewing a very +nicely displayed output of the generated assembly, there is +[the Godbolt compiler explorer](https://rust.godbolt.org/) + +For learning how to write unsafe Rust, read +[The Rustonomicon](https://doc.rust-lang.org/nomicon/). + +For learning everything you ever wanted to know about Rust macros, there is +[The Little Book of Rust Macros](https://danielkeep.github.io/tlborm/book/index.html). + + Compiling Tor with Rust enabled +--------------------------------- + +You will need to run the `configure` script with the `--enable-rust` flag to +explicitly build with Rust. Additionally, you will need to specify where to +fetch Rust dependencies, as we allow for either fetching dependencies from Cargo +or specifying a local directory. + +**Fetch dependencies from Cargo** + + ./configure --enable-rust --enable-cargo-online-mode + +**Using a local dependency cache** + +**NOTE**: local dependency caches which were not *originally* created via + `--enable-cargo-online-mode` are broken. See https://bugs.torproject.org/22907 + +To specify a local directory: + + RUST_DEPENDENCIES='path_to_dependencies_directory' ./configure --enable-rust + +(Note that RUST_DEPENDENCIES must be the full path to the directory; it cannot +be relative.) + +You'll need the following Rust dependencies (as of this writing): + + libc==0.2.22 + +To get them, do: + + mkdir path_to_dependencies_directory + cd path_to_dependencies_directory + git clone https://github.com/rust-lang/libc + cd libc + git checkout 0.2.22 + cargo package + cd .. + ln -s libc/target/package/libc-0.2.22 libc-0.2.22 + + + Identifying which modules to rewrite +====================================== + +The places in the Tor codebase that are good candidates for porting to Rust are: + +1. loosely coupled to other Tor submodules, +2. have high test coverage, and +3. would benefit from being implemented in a memory safe language. + +Help in either identifying places such as this, or working to improve existing +areas of the C codebase by adding regression tests and simplifying dependencies, +would be really helpful. + +Furthermore, as submodules in C are implemented in Rust, this is a good +opportunity to refactor, add more tests, and split modules into smaller areas of +responsibility. + +A good first step is to build a module-level callgraph to understand how +interconnected your target module is. + + git clone https://git.torproject.org/user/nickm/calltool.git + cd tor + CFLAGS=0 ./configure + ../calltool/src/main.py module_callgraph + +The output will tell you each module name, along with a set of every module that +the module calls. Modules which call fewer other modules are better targets. + + Writing your Rust module +========================== + +Strive to change the C API as little as possible. + +We are currently targetting Rust nightly, *for now*. We expect this to change +moving forward, as we understand more about which nightly features we need. It +is on our TODO list to try to cultivate good standing with various distro +maintainers of `rustc` and `cargo`, in order to ensure that whatever version we +solidify on is readily available. + + Adding your Rust module to Tor's build system +----------------------------------------------- + +0. Your translation of the C module should live in its own crate(s) + in the `.../tor/src/rust/` directory. +1. Add your crate to `.../tor/src/rust/Cargo.toml`, in the + `[workspace.members]` section. +2. Append your crate's static library to the `rust_ldadd` definition + (underneath `if USE_RUST`) in `.../tor/Makefile.am`. + + How to test your Rust code +---------------------------- + +Everything should be tested full stop. Even non-public functionality. + +Be sure to edit `.../tor/src/test/test_rust.sh` to add the name of your crate to +the `crates` variable! This will ensure that `cargo test` is run on your crate. + +Configure Tor's build system to build with Rust enabled: + + ./configure --enable-fatal-warnings --enable-rust --enable-cargo-online-mode + +Tor's test should be run by doing: + + make check + +Tor's integration tests should also pass: + + make test-stem + + Submitting a patch +===================== + +Please follow the instructions in `.../doc/HACKING/GettingStarted.md`. diff --git a/doc/HACKING/ReleasingTor.md b/doc/HACKING/ReleasingTor.md index 8fece92c20..2987121611 100644 --- a/doc/HACKING/ReleasingTor.md +++ b/doc/HACKING/ReleasingTor.md @@ -146,7 +146,8 @@ new Tor release: - {weasel,gk,mikeperry} at torproject dot org - {blueness} at gentoo dot org - - {paul,vincent} at invizbox dot io + - {paul} at invizbox dot io + - {vincent} at invizbox dot com - {lfleischer} at archlinux dot org - {Nathan} at freitas dot net - {mike} at tig dot as diff --git a/doc/tor.1.txt b/doc/tor.1.txt index 85af8c3a3d..aec1767059 100644 --- a/doc/tor.1.txt +++ b/doc/tor.1.txt @@ -786,7 +786,8 @@ CLIENT OPTIONS -------------- The following options are useful only for clients (that is, if -**SocksPort**, **TransPort**, **DNSPort**, or **NATDPort** is non-zero): +**SocksPort**, **HTTPTunnelPort**, **TransPort**, **DNSPort**, or +**NATDPort** is non-zero): [[Bridge]] **Bridge** [__transport__] __IP__:__ORPort__ [__fingerprint__]:: When set along with UseBridges, instructs Tor to use the relay at @@ -1110,7 +1111,9 @@ The following options are useful only for clients (that is, if Unsupported and force-disabled when using Unix domain sockets.) **IsolateSOCKSAuth**;; Don't share circuits with streams for which different - SOCKS authentication was provided. (On by default; + SOCKS authentication was provided. (For HTTPTunnelPort + connections, this option looks at the Proxy-Authorization and + X-Tor-Stream-Isolation headers. On by default; you can disable it with **NoIsolateSOCKSAuth**.) **IsolateClientProtocol**;; Don't share circuits with streams using a different protocol. @@ -1331,6 +1334,14 @@ The following options are useful only for clients (that is, if the node "foo". Disabled by default since attacking websites and exit relays can use it to manipulate your path selection. (Default: 0) +[[HTTPTunnelPort]] **HTTPTunnelPort** \['address':]__port__|**auto** [_isolation flags_]:: + Open this port to listen for proxy connections using the "HTTP CONNECT" + protocol instead of SOCKS. Set this to 0 + 0 if you don't want to allow "HTTP CONNECT" connections. Set the port + to "auto" to have Tor pick a port for you. This directive can be + specified multiple times to bind to multiple addresses/ports. See + SOCKSPort for an explanation of isolation flags. (Default: 0) + [[TransPort]] **TransPort** \['address':]__port__|**auto** [_isolation flags_]:: Open this port to listen for transparent proxy connections. Set this to 0 if you don't want to allow transparent proxy connections. Set the port diff --git a/scripts/codegen/fuzzing_include_am.py b/scripts/codegen/fuzzing_include_am.py index 045d0f8922..e01bc8daa9 100755 --- a/scripts/codegen/fuzzing_include_am.py +++ b/scripts/codegen/fuzzing_include_am.py @@ -8,6 +8,7 @@ FUZZERS = """ extrainfo hsdescv2 http + http-connect iptsv2 microdesc vrs diff --git a/src/common/buffers.c b/src/common/buffers.c new file mode 100644 index 0000000000..50673646df --- /dev/null +++ b/src/common/buffers.c @@ -0,0 +1,1069 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file buffers.c + * \brief Implements a generic buffer interface. + * + * A buf_t is a (fairly) opaque byte-oriented FIFO that can read to or flush + * from memory, sockets, file descriptors, TLS connections, or another buf_t. + * Buffers are implemented as linked lists of memory chunks. + * + * All socket-backed and TLS-based connection_t objects have a pair of + * buffers: one for incoming data, and one for outcoming data. These are fed + * and drained from functions in connection.c, trigged by events that are + * monitored in main.c. + **/ + +#define BUFFERS_PRIVATE +#include "orconfig.h" +#include <stddef.h> +#include "buffers.h" +#include "compat.h" +#include "compress.h" +#include "util.h" +#include "torint.h" +#include "torlog.h" +#ifdef HAVE_UNISTD_H +#include <unistd.h> +#endif + +//#define PARANOIA + +#ifdef PARANOIA +/** Helper: If PARANOIA is defined, assert that the buffer in local variable + * <b>buf</b> is well-formed. */ +#define check() STMT_BEGIN buf_assert_ok(buf); STMT_END +#else +#define check() STMT_NIL +#endif + +/* Implementation notes: + * + * After flirting with memmove, and dallying with ring-buffers, we're finally + * getting up to speed with the 1970s and implementing buffers as a linked + * list of small chunks. Each buffer has such a list; data is removed from + * the head of the list, and added at the tail. The list is singly linked, + * and the buffer keeps a pointer to the head and the tail. + * + * Every chunk, except the tail, contains at least one byte of data. Data in + * each chunk is contiguous. + * + * When you need to treat the first N characters on a buffer as a contiguous + * string, use the buf_pullup function to make them so. Don't do this more + * than necessary. + * + * The major free Unix kernels have handled buffers like this since, like, + * forever. + */ + +/* Chunk manipulation functions */ + +#define CHUNK_HEADER_LEN offsetof(chunk_t, mem[0]) + +/* We leave this many NUL bytes at the end of the buffer. */ +#ifdef DISABLE_MEMORY_SENTINELS +#define SENTINEL_LEN 0 +#else +#define SENTINEL_LEN 4 +#endif + +/* Header size plus NUL bytes at the end */ +#define CHUNK_OVERHEAD (CHUNK_HEADER_LEN + SENTINEL_LEN) + +/** Return the number of bytes needed to allocate a chunk to hold + * <b>memlen</b> bytes. */ +#define CHUNK_ALLOC_SIZE(memlen) (CHUNK_OVERHEAD + (memlen)) +/** Return the number of usable bytes in a chunk allocated with + * malloc(<b>memlen</b>). */ +#define CHUNK_SIZE_WITH_ALLOC(memlen) ((memlen) - CHUNK_OVERHEAD) + +#define DEBUG_SENTINEL + +#if defined(DEBUG_SENTINEL) && !defined(DISABLE_MEMORY_SENTINELS) +#define DBG_S(s) s +#else +#define DBG_S(s) (void)0 +#endif + +#ifdef DISABLE_MEMORY_SENTINELS +#define CHUNK_SET_SENTINEL(chunk, alloclen) STMT_NIL +#else +#define CHUNK_SET_SENTINEL(chunk, alloclen) do { \ + uint8_t *a = (uint8_t*) &(chunk)->mem[(chunk)->memlen]; \ + DBG_S(uint8_t *b = &((uint8_t*)(chunk))[(alloclen)-SENTINEL_LEN]); \ + DBG_S(tor_assert(a == b)); \ + memset(a,0,SENTINEL_LEN); \ + } while (0) +#endif + +/** Move all bytes stored in <b>chunk</b> to the front of <b>chunk</b>->mem, + * to free up space at the end. */ +static inline void +chunk_repack(chunk_t *chunk) +{ + if (chunk->datalen && chunk->data != &chunk->mem[0]) { + memmove(chunk->mem, chunk->data, chunk->datalen); + } + chunk->data = &chunk->mem[0]; +} + +/** Keep track of total size of allocated chunks for consistency asserts */ +static size_t total_bytes_allocated_in_chunks = 0; +static void +buf_chunk_free_unchecked(chunk_t *chunk) +{ + if (!chunk) + return; +#ifdef DEBUG_CHUNK_ALLOC + tor_assert(CHUNK_ALLOC_SIZE(chunk->memlen) == chunk->DBG_alloc); +#endif + tor_assert(total_bytes_allocated_in_chunks >= + CHUNK_ALLOC_SIZE(chunk->memlen)); + total_bytes_allocated_in_chunks -= CHUNK_ALLOC_SIZE(chunk->memlen); + tor_free(chunk); +} +static inline chunk_t * +chunk_new_with_alloc_size(size_t alloc) +{ + chunk_t *ch; + ch = tor_malloc(alloc); + ch->next = NULL; + ch->datalen = 0; +#ifdef DEBUG_CHUNK_ALLOC + ch->DBG_alloc = alloc; +#endif + ch->memlen = CHUNK_SIZE_WITH_ALLOC(alloc); + total_bytes_allocated_in_chunks += alloc; + ch->data = &ch->mem[0]; + CHUNK_SET_SENTINEL(ch, alloc); + return ch; +} + +/** Expand <b>chunk</b> until it can hold <b>sz</b> bytes, and return a + * new pointer to <b>chunk</b>. Old pointers are no longer valid. */ +static inline chunk_t * +chunk_grow(chunk_t *chunk, size_t sz) +{ + off_t offset; + const size_t memlen_orig = chunk->memlen; + const size_t orig_alloc = CHUNK_ALLOC_SIZE(memlen_orig); + const size_t new_alloc = CHUNK_ALLOC_SIZE(sz); + tor_assert(sz > chunk->memlen); + offset = chunk->data - chunk->mem; + chunk = tor_realloc(chunk, new_alloc); + chunk->memlen = sz; + chunk->data = chunk->mem + offset; +#ifdef DEBUG_CHUNK_ALLOC + tor_assert(chunk->DBG_alloc == orig_alloc); + chunk->DBG_alloc = new_alloc; +#endif + total_bytes_allocated_in_chunks += new_alloc - orig_alloc; + CHUNK_SET_SENTINEL(chunk, new_alloc); + return chunk; +} + +/** Every chunk should take up at least this many bytes. */ +#define MIN_CHUNK_ALLOC 256 +/** No chunk should take up more than this many bytes. */ +#define MAX_CHUNK_ALLOC 65536 + +/** Return the allocation size we'd like to use to hold <b>target</b> + * bytes. */ +size_t +buf_preferred_chunk_size(size_t target) +{ + tor_assert(target <= SIZE_T_CEILING - CHUNK_OVERHEAD); + if (CHUNK_ALLOC_SIZE(target) >= MAX_CHUNK_ALLOC) + return CHUNK_ALLOC_SIZE(target); + size_t sz = MIN_CHUNK_ALLOC; + while (CHUNK_SIZE_WITH_ALLOC(sz) < target) { + sz <<= 1; + } + return sz; +} + +/** Collapse data from the first N chunks from <b>buf</b> into buf->head, + * growing it as necessary, until buf->head has the first <b>bytes</b> bytes + * of data from the buffer, or until buf->head has all the data in <b>buf</b>. + * + * Set *<b>head_out</b> to point to the first byte of available data, and + * *<b>len_out</b> to the number of bytes of data available at + * *<b>head_out</b>. Note that *<b>len_out</b> may be more or less than + * <b>bytes</b>, depending on the number of bytes available. + */ +void +buf_pullup(buf_t *buf, size_t bytes, const char **head_out, size_t *len_out) +{ + chunk_t *dest, *src; + size_t capacity; + if (!buf->head) { + *head_out = NULL; + *len_out = 0; + return; + } + + check(); + if (buf->datalen < bytes) + bytes = buf->datalen; + + capacity = bytes; + if (buf->head->datalen >= bytes) { + *head_out = buf->head->data; + *len_out = buf->head->datalen; + return; + } + + if (buf->head->memlen >= capacity) { + /* We don't need to grow the first chunk, but we might need to repack it.*/ + size_t needed = capacity - buf->head->datalen; + if (CHUNK_REMAINING_CAPACITY(buf->head) < needed) + chunk_repack(buf->head); + tor_assert(CHUNK_REMAINING_CAPACITY(buf->head) >= needed); + } else { + chunk_t *newhead; + size_t newsize; + /* We need to grow the chunk. */ + chunk_repack(buf->head); + newsize = CHUNK_SIZE_WITH_ALLOC(buf_preferred_chunk_size(capacity)); + newhead = chunk_grow(buf->head, newsize); + tor_assert(newhead->memlen >= capacity); + if (newhead != buf->head) { + if (buf->tail == buf->head) + buf->tail = newhead; + buf->head = newhead; + } + } + + dest = buf->head; + while (dest->datalen < bytes) { + size_t n = bytes - dest->datalen; + src = dest->next; + tor_assert(src); + if (n >= src->datalen) { + memcpy(CHUNK_WRITE_PTR(dest), src->data, src->datalen); + dest->datalen += src->datalen; + dest->next = src->next; + if (buf->tail == src) + buf->tail = dest; + buf_chunk_free_unchecked(src); + } else { + memcpy(CHUNK_WRITE_PTR(dest), src->data, n); + dest->datalen += n; + src->data += n; + src->datalen -= n; + tor_assert(dest->datalen == bytes); + } + } + + check(); + *head_out = buf->head->data; + *len_out = buf->head->datalen; +} + +#ifdef TOR_UNIT_TESTS +/* Write sz bytes from cp into a newly allocated buffer buf. + * Returns NULL when passed a NULL cp or zero sz. + * Asserts on failure: only for use in unit tests. + * buf must be freed using buf_free(). */ +buf_t * +buf_new_with_data(const char *cp, size_t sz) +{ + /* Validate arguments */ + if (!cp || sz <= 0) { + return NULL; + } + + tor_assert(sz < SSIZE_T_CEILING); + + /* Allocate a buffer */ + buf_t *buf = buf_new_with_capacity(sz); + tor_assert(buf); + buf_assert_ok(buf); + tor_assert(!buf->head); + + /* Allocate a chunk that is sz bytes long */ + buf->head = chunk_new_with_alloc_size(CHUNK_ALLOC_SIZE(sz)); + buf->tail = buf->head; + tor_assert(buf->head); + buf_assert_ok(buf); + tor_assert(buf_allocation(buf) >= sz); + + /* Copy the data and size the buffers */ + tor_assert(sz <= buf_slack(buf)); + tor_assert(sz <= CHUNK_REMAINING_CAPACITY(buf->head)); + memcpy(&buf->head->mem[0], cp, sz); + buf->datalen = sz; + buf->head->datalen = sz; + buf->head->data = &buf->head->mem[0]; + buf_assert_ok(buf); + + /* Make sure everything is large enough */ + tor_assert(buf_allocation(buf) >= sz); + tor_assert(buf_allocation(buf) >= buf_datalen(buf) + buf_slack(buf)); + /* Does the buffer implementation allocate more than the requested size? + * (for example, by rounding up). If so, these checks will fail. */ + tor_assert(buf_datalen(buf) == sz); + tor_assert(buf_slack(buf) == 0); + + return buf; +} +#endif + +/** Remove the first <b>n</b> bytes from buf. */ +void +buf_drain(buf_t *buf, size_t n) +{ + tor_assert(buf->datalen >= n); + while (n) { + tor_assert(buf->head); + if (buf->head->datalen > n) { + buf->head->datalen -= n; + buf->head->data += n; + buf->datalen -= n; + return; + } else { + chunk_t *victim = buf->head; + n -= victim->datalen; + buf->datalen -= victim->datalen; + buf->head = victim->next; + if (buf->tail == victim) + buf->tail = NULL; + buf_chunk_free_unchecked(victim); + } + } + check(); +} + +/** Create and return a new buf with default chunk capacity <b>size</b>. + */ +buf_t * +buf_new_with_capacity(size_t size) +{ + buf_t *b = buf_new(); + b->default_chunk_size = buf_preferred_chunk_size(size); + return b; +} + +/** Allocate and return a new buffer with default capacity. */ +buf_t * +buf_new(void) +{ + buf_t *buf = tor_malloc_zero(sizeof(buf_t)); + buf->magic = BUFFER_MAGIC; + buf->default_chunk_size = 4096; + return buf; +} + +size_t +buf_get_default_chunk_size(const buf_t *buf) +{ + return buf->default_chunk_size; +} + +/** Remove all data from <b>buf</b>. */ +void +buf_clear(buf_t *buf) +{ + chunk_t *chunk, *next; + buf->datalen = 0; + for (chunk = buf->head; chunk; chunk = next) { + next = chunk->next; + buf_chunk_free_unchecked(chunk); + } + buf->head = buf->tail = NULL; +} + +/** Return the number of bytes stored in <b>buf</b> */ +MOCK_IMPL(size_t, +buf_datalen, (const buf_t *buf)) +{ + return buf->datalen; +} + +/** Return the total length of all chunks used in <b>buf</b>. */ +size_t +buf_allocation(const buf_t *buf) +{ + size_t total = 0; + const chunk_t *chunk; + for (chunk = buf->head; chunk; chunk = chunk->next) { + total += CHUNK_ALLOC_SIZE(chunk->memlen); + } + return total; +} + +/** Return the number of bytes that can be added to <b>buf</b> without + * performing any additional allocation. */ +size_t +buf_slack(const buf_t *buf) +{ + if (!buf->tail) + return 0; + else + return CHUNK_REMAINING_CAPACITY(buf->tail); +} + +/** Release storage held by <b>buf</b>. */ +void +buf_free(buf_t *buf) +{ + if (!buf) + return; + + buf_clear(buf); + buf->magic = 0xdeadbeef; + tor_free(buf); +} + +/** Return a new copy of <b>in_chunk</b> */ +static chunk_t * +chunk_copy(const chunk_t *in_chunk) +{ + chunk_t *newch = tor_memdup(in_chunk, CHUNK_ALLOC_SIZE(in_chunk->memlen)); + total_bytes_allocated_in_chunks += CHUNK_ALLOC_SIZE(in_chunk->memlen); +#ifdef DEBUG_CHUNK_ALLOC + newch->DBG_alloc = CHUNK_ALLOC_SIZE(in_chunk->memlen); +#endif + newch->next = NULL; + if (in_chunk->data) { + off_t offset = in_chunk->data - in_chunk->mem; + newch->data = newch->mem + offset; + } + return newch; +} + +/** Return a new copy of <b>buf</b> */ +buf_t * +buf_copy(const buf_t *buf) +{ + chunk_t *ch; + buf_t *out = buf_new(); + out->default_chunk_size = buf->default_chunk_size; + for (ch = buf->head; ch; ch = ch->next) { + chunk_t *newch = chunk_copy(ch); + if (out->tail) { + out->tail->next = newch; + out->tail = newch; + } else { + out->head = out->tail = newch; + } + } + out->datalen = buf->datalen; + return out; +} + +/** Append a new chunk with enough capacity to hold <b>capacity</b> bytes to + * the tail of <b>buf</b>. If <b>capped</b>, don't allocate a chunk bigger + * than MAX_CHUNK_ALLOC. */ +chunk_t * +buf_add_chunk_with_capacity(buf_t *buf, size_t capacity, int capped) +{ + chunk_t *chunk; + + if (CHUNK_ALLOC_SIZE(capacity) < buf->default_chunk_size) { + chunk = chunk_new_with_alloc_size(buf->default_chunk_size); + } else if (capped && CHUNK_ALLOC_SIZE(capacity) > MAX_CHUNK_ALLOC) { + chunk = chunk_new_with_alloc_size(MAX_CHUNK_ALLOC); + } else { + chunk = chunk_new_with_alloc_size(buf_preferred_chunk_size(capacity)); + } + + chunk->inserted_time = (uint32_t)monotime_coarse_absolute_msec(); + + if (buf->tail) { + tor_assert(buf->head); + buf->tail->next = chunk; + buf->tail = chunk; + } else { + tor_assert(!buf->head); + buf->head = buf->tail = chunk; + } + check(); + return chunk; +} + +/** Return the age of the oldest chunk in the buffer <b>buf</b>, in + * milliseconds. Requires the current monotonic time, in truncated msec, + * as its input <b>now</b>. + */ +uint32_t +buf_get_oldest_chunk_timestamp(const buf_t *buf, uint32_t now) +{ + if (buf->head) { + return now - buf->head->inserted_time; + } else { + return 0; + } +} + +size_t +buf_get_total_allocation(void) +{ + return total_bytes_allocated_in_chunks; +} + +/** Read up to <b>at_most</b> bytes from the socket <b>fd</b> into + * <b>chunk</b> (which must be on <b>buf</b>). If we get an EOF, set + * *<b>reached_eof</b> to 1. Return -1 on error, 0 on eof or blocking, + * and the number of bytes read otherwise. */ +static inline int +read_to_chunk(buf_t *buf, chunk_t *chunk, tor_socket_t fd, size_t at_most, + int *reached_eof, int *socket_error) +{ + ssize_t read_result; + if (at_most > CHUNK_REMAINING_CAPACITY(chunk)) + at_most = CHUNK_REMAINING_CAPACITY(chunk); + read_result = tor_socket_recv(fd, CHUNK_WRITE_PTR(chunk), at_most, 0); + + if (read_result < 0) { + int e = tor_socket_errno(fd); + if (!ERRNO_IS_EAGAIN(e)) { /* it's a real error */ +#ifdef _WIN32 + if (e == WSAENOBUFS) + log_warn(LD_NET,"recv() failed: WSAENOBUFS. Not enough ram?"); +#endif + *socket_error = e; + return -1; + } + return 0; /* would block. */ + } else if (read_result == 0) { + log_debug(LD_NET,"Encountered eof on fd %d", (int)fd); + *reached_eof = 1; + return 0; + } else { /* actually got bytes. */ + buf->datalen += read_result; + chunk->datalen += read_result; + log_debug(LD_NET,"Read %ld bytes. %d on inbuf.", (long)read_result, + (int)buf->datalen); + tor_assert(read_result < INT_MAX); + return (int)read_result; + } +} + +/** Read from socket <b>s</b>, writing onto end of <b>buf</b>. Read at most + * <b>at_most</b> bytes, growing the buffer as necessary. If recv() returns 0 + * (because of EOF), set *<b>reached_eof</b> to 1 and return 0. Return -1 on + * error; else return the number of bytes read. + */ +/* XXXX indicate "read blocked" somehow? */ +int +buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most, + int *reached_eof, + int *socket_error) +{ + /* XXXX It's stupid to overload the return values for these functions: + * "error status" and "number of bytes read" are not mutually exclusive. + */ + int r = 0; + size_t total_read = 0; + + check(); + tor_assert(reached_eof); + tor_assert(SOCKET_OK(s)); + + if (BUG(buf->datalen >= INT_MAX)) + return -1; + if (BUG(buf->datalen >= INT_MAX - at_most)) + return -1; + + while (at_most > total_read) { + size_t readlen = at_most - total_read; + chunk_t *chunk; + if (!buf->tail || CHUNK_REMAINING_CAPACITY(buf->tail) < MIN_READ_LEN) { + chunk = buf_add_chunk_with_capacity(buf, at_most, 1); + if (readlen > chunk->memlen) + readlen = chunk->memlen; + } else { + size_t cap = CHUNK_REMAINING_CAPACITY(buf->tail); + chunk = buf->tail; + if (cap < readlen) + readlen = cap; + } + + r = read_to_chunk(buf, chunk, s, readlen, reached_eof, socket_error); + check(); + if (r < 0) + return r; /* Error */ + tor_assert(total_read+r < INT_MAX); + total_read += r; + if ((size_t)r < readlen) { /* eof, block, or no more to read. */ + break; + } + } + return (int)total_read; +} + +/** Helper for buf_flush_to_socket(): try to write <b>sz</b> bytes from chunk + * <b>chunk</b> of buffer <b>buf</b> onto socket <b>s</b>. On success, deduct + * the bytes written from *<b>buf_flushlen</b>. Return the number of bytes + * written on success, 0 on blocking, -1 on failure. + */ +static inline int +flush_chunk(tor_socket_t s, buf_t *buf, chunk_t *chunk, size_t sz, + size_t *buf_flushlen) +{ + ssize_t write_result; + + if (sz > chunk->datalen) + sz = chunk->datalen; + write_result = tor_socket_send(s, chunk->data, sz, 0); + + if (write_result < 0) { + int e = tor_socket_errno(s); + if (!ERRNO_IS_EAGAIN(e)) { /* it's a real error */ +#ifdef _WIN32 + if (e == WSAENOBUFS) + log_warn(LD_NET,"write() failed: WSAENOBUFS. Not enough ram?"); +#endif + return -1; + } + log_debug(LD_NET,"write() would block, returning."); + return 0; + } else { + *buf_flushlen -= write_result; + buf_drain(buf, write_result); + tor_assert(write_result < INT_MAX); + return (int)write_result; + } +} + +/** Write data from <b>buf</b> to the socket <b>s</b>. Write at most + * <b>sz</b> bytes, decrement *<b>buf_flushlen</b> by + * the number of bytes actually written, and remove the written bytes + * from the buffer. Return the number of bytes written on success, + * -1 on failure. Return 0 if write() would block. + */ +int +buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz, + size_t *buf_flushlen) +{ + /* XXXX It's stupid to overload the return values for these functions: + * "error status" and "number of bytes flushed" are not mutually exclusive. + */ + int r; + size_t flushed = 0; + tor_assert(buf_flushlen); + tor_assert(SOCKET_OK(s)); + tor_assert(*buf_flushlen <= buf->datalen); + tor_assert(sz <= *buf_flushlen); + + check(); + while (sz) { + size_t flushlen0; + tor_assert(buf->head); + if (buf->head->datalen >= sz) + flushlen0 = sz; + else + flushlen0 = buf->head->datalen; + + r = flush_chunk(s, buf, buf->head, flushlen0, buf_flushlen); + check(); + if (r < 0) + return r; + flushed += r; + sz -= r; + if (r == 0 || (size_t)r < flushlen0) /* can't flush any more now. */ + break; + } + tor_assert(flushed < INT_MAX); + return (int)flushed; +} + +/** Append <b>string_len</b> bytes from <b>string</b> to the end of + * <b>buf</b>. + * + * Return the new length of the buffer on success, -1 on failure. + */ +int +buf_add(buf_t *buf, const char *string, size_t string_len) +{ + if (!string_len) + return (int)buf->datalen; + check(); + + if (BUG(buf->datalen >= INT_MAX)) + return -1; + if (BUG(buf->datalen >= INT_MAX - string_len)) + return -1; + + while (string_len) { + size_t copy; + if (!buf->tail || !CHUNK_REMAINING_CAPACITY(buf->tail)) + buf_add_chunk_with_capacity(buf, string_len, 1); + + copy = CHUNK_REMAINING_CAPACITY(buf->tail); + if (copy > string_len) + copy = string_len; + memcpy(CHUNK_WRITE_PTR(buf->tail), string, copy); + string_len -= copy; + string += copy; + buf->datalen += copy; + buf->tail->datalen += copy; + } + + check(); + tor_assert(buf->datalen < INT_MAX); + return (int)buf->datalen; +} + +/** Helper: copy the first <b>string_len</b> bytes from <b>buf</b> + * onto <b>string</b>. + */ +void +buf_peek(const buf_t *buf, char *string, size_t string_len) +{ + chunk_t *chunk; + + tor_assert(string); + /* make sure we don't ask for too much */ + tor_assert(string_len <= buf->datalen); + /* buf_assert_ok(buf); */ + + chunk = buf->head; + while (string_len) { + size_t copy = string_len; + tor_assert(chunk); + if (chunk->datalen < copy) + copy = chunk->datalen; + memcpy(string, chunk->data, copy); + string_len -= copy; + string += copy; + chunk = chunk->next; + } +} + +/** Remove <b>string_len</b> bytes from the front of <b>buf</b>, and store + * them into <b>string</b>. Return the new buffer size. <b>string_len</b> + * must be \<= the number of bytes on the buffer. + */ +int +buf_get_bytes(buf_t *buf, char *string, size_t string_len) +{ + /* There must be string_len bytes in buf; write them onto string, + * then memmove buf back (that is, remove them from buf). + * + * Return the number of bytes still on the buffer. */ + + check(); + buf_peek(buf, string, string_len); + buf_drain(buf, string_len); + check(); + tor_assert(buf->datalen < INT_MAX); + return (int)buf->datalen; +} + +/** Move up to *<b>buf_flushlen</b> bytes from <b>buf_in</b> to + * <b>buf_out</b>, and modify *<b>buf_flushlen</b> appropriately. + * Return the number of bytes actually copied. + */ +int +buf_move_to_buf(buf_t *buf_out, buf_t *buf_in, size_t *buf_flushlen) +{ + /* We can do way better here, but this doesn't turn up in any profiles. */ + char b[4096]; + size_t cp, len; + + if (BUG(buf_out->datalen >= INT_MAX)) + return -1; + if (BUG(buf_out->datalen >= INT_MAX - *buf_flushlen)) + return -1; + + len = *buf_flushlen; + if (len > buf_in->datalen) + len = buf_in->datalen; + + cp = len; /* Remember the number of bytes we intend to copy. */ + tor_assert(cp < INT_MAX); + while (len) { + /* This isn't the most efficient implementation one could imagine, since + * it does two copies instead of 1, but I kinda doubt that this will be + * critical path. */ + size_t n = len > sizeof(b) ? sizeof(b) : len; + buf_get_bytes(buf_in, b, n); + buf_add(buf_out, b, n); + len -= n; + } + *buf_flushlen -= cp; + return (int)cp; +} + +/** Internal structure: represents a position in a buffer. */ +typedef struct buf_pos_t { + const chunk_t *chunk; /**< Which chunk are we pointing to? */ + int pos;/**< Which character inside the chunk's data are we pointing to? */ + size_t chunk_pos; /**< Total length of all previous chunks. */ +} buf_pos_t; + +/** Initialize <b>out</b> to point to the first character of <b>buf</b>.*/ +static void +buf_pos_init(const buf_t *buf, buf_pos_t *out) +{ + out->chunk = buf->head; + out->pos = 0; + out->chunk_pos = 0; +} + +/** Advance <b>out</b> to the first appearance of <b>ch</b> at the current + * position of <b>out</b>, or later. Return -1 if no instances are found; + * otherwise returns the absolute position of the character. */ +static off_t +buf_find_pos_of_char(char ch, buf_pos_t *out) +{ + const chunk_t *chunk; + int pos; + tor_assert(out); + if (out->chunk) { + if (out->chunk->datalen) { + tor_assert(out->pos < (off_t)out->chunk->datalen); + } else { + tor_assert(out->pos == 0); + } + } + pos = out->pos; + for (chunk = out->chunk; chunk; chunk = chunk->next) { + char *cp = memchr(chunk->data+pos, ch, chunk->datalen - pos); + if (cp) { + out->chunk = chunk; + tor_assert(cp - chunk->data < INT_MAX); + out->pos = (int)(cp - chunk->data); + return out->chunk_pos + out->pos; + } else { + out->chunk_pos += chunk->datalen; + pos = 0; + } + } + return -1; +} + +/** Advance <b>pos</b> by a single character, if there are any more characters + * in the buffer. Returns 0 on success, -1 on failure. */ +static inline int +buf_pos_inc(buf_pos_t *pos) +{ + ++pos->pos; + if (pos->pos == (off_t)pos->chunk->datalen) { + if (!pos->chunk->next) + return -1; + pos->chunk_pos += pos->chunk->datalen; + pos->chunk = pos->chunk->next; + pos->pos = 0; + } + return 0; +} + +/** Return true iff the <b>n</b>-character string in <b>s</b> appears + * (verbatim) at <b>pos</b>. */ +static int +buf_matches_at_pos(const buf_pos_t *pos, const char *s, size_t n) +{ + buf_pos_t p; + if (!n) + return 1; + + memcpy(&p, pos, sizeof(p)); + + while (1) { + char ch = p.chunk->data[p.pos]; + if (ch != *s) + return 0; + ++s; + /* If we're out of characters that don't match, we match. Check this + * _before_ we test incrementing pos, in case we're at the end of the + * string. */ + if (--n == 0) + return 1; + if (buf_pos_inc(&p)<0) + return 0; + } +} + +/** Return the first position in <b>buf</b> at which the <b>n</b>-character + * string <b>s</b> occurs, or -1 if it does not occur. */ +int +buf_find_string_offset(const buf_t *buf, const char *s, size_t n) +{ + buf_pos_t pos; + buf_pos_init(buf, &pos); + while (buf_find_pos_of_char(*s, &pos) >= 0) { + if (buf_matches_at_pos(&pos, s, n)) { + tor_assert(pos.chunk_pos + pos.pos < INT_MAX); + return (int)(pos.chunk_pos + pos.pos); + } else { + if (buf_pos_inc(&pos)<0) + return -1; + } + } + return -1; +} + +/** Return 1 iff <b>buf</b> starts with <b>cmd</b>. <b>cmd</b> must be a null + * terminated string, of no more than PEEK_BUF_STARTSWITH_MAX bytes. */ +int +buf_peek_startswith(const buf_t *buf, const char *cmd) +{ + char tmp[PEEK_BUF_STARTSWITH_MAX]; + size_t clen = strlen(cmd); + if (BUG(clen > sizeof(tmp))) + return 0; + if (buf->datalen < clen) + return 0; + buf_peek(buf, tmp, clen); + return fast_memeq(tmp, cmd, clen); +} + +/** Return the index within <b>buf</b> at which <b>ch</b> first appears, + * or -1 if <b>ch</b> does not appear on buf. */ +static off_t +buf_find_offset_of_char(buf_t *buf, char ch) +{ + chunk_t *chunk; + off_t offset = 0; + for (chunk = buf->head; chunk; chunk = chunk->next) { + char *cp = memchr(chunk->data, ch, chunk->datalen); + if (cp) + return offset + (cp - chunk->data); + else + offset += chunk->datalen; + } + return -1; +} + +/** Try to read a single LF-terminated line from <b>buf</b>, and write it + * (including the LF), NUL-terminated, into the *<b>data_len</b> byte buffer + * at <b>data_out</b>. Set *<b>data_len</b> to the number of bytes in the + * line, not counting the terminating NUL. Return 1 if we read a whole line, + * return 0 if we don't have a whole line yet, and return -1 if the line + * length exceeds *<b>data_len</b>. + */ +int +buf_get_line(buf_t *buf, char *data_out, size_t *data_len) +{ + size_t sz; + off_t offset; + + if (!buf->head) + return 0; + + offset = buf_find_offset_of_char(buf, '\n'); + if (offset < 0) + return 0; + sz = (size_t) offset; + if (sz+2 > *data_len) { + *data_len = sz + 2; + return -1; + } + buf_get_bytes(buf, data_out, sz+1); + data_out[sz+1] = '\0'; + *data_len = sz+1; + return 1; +} + +/** Compress on uncompress the <b>data_len</b> bytes in <b>data</b> using the + * compression state <b>state</b>, appending the result to <b>buf</b>. If + * <b>done</b> is true, flush the data in the state and finish the + * compression/uncompression. Return -1 on failure, 0 on success. */ +int +buf_add_compress(buf_t *buf, tor_compress_state_t *state, + const char *data, size_t data_len, + const int done) +{ + char *next; + size_t old_avail, avail; + int over = 0; + + do { + int need_new_chunk = 0; + if (!buf->tail || ! CHUNK_REMAINING_CAPACITY(buf->tail)) { + size_t cap = data_len / 4; + buf_add_chunk_with_capacity(buf, cap, 1); + } + next = CHUNK_WRITE_PTR(buf->tail); + avail = old_avail = CHUNK_REMAINING_CAPACITY(buf->tail); + switch (tor_compress_process(state, &next, &avail, + &data, &data_len, done)) { + case TOR_COMPRESS_DONE: + over = 1; + break; + case TOR_COMPRESS_ERROR: + return -1; + case TOR_COMPRESS_OK: + if (data_len == 0) { + tor_assert_nonfatal(!done); + over = 1; + } + break; + case TOR_COMPRESS_BUFFER_FULL: + if (avail) { + /* The compression module says we need more room + * (TOR_COMPRESS_BUFFER_FULL). Start a new chunk automatically, + * whether were going to or not. */ + need_new_chunk = 1; + } + if (data_len == 0 && !done) { + /* We've consumed all the input data, though, so there's no + * point in forging ahead right now. */ + over = 1; + } + break; + } + buf->datalen += old_avail - avail; + buf->tail->datalen += old_avail - avail; + if (need_new_chunk) { + buf_add_chunk_with_capacity(buf, data_len/4, 1); + } + + } while (!over); + check(); + return 0; +} + +/** Set *<b>output</b> to contain a copy of the data in *<b>input</b> */ +int +buf_set_to_copy(buf_t **output, + const buf_t *input) +{ + if (*output) + buf_free(*output); + *output = buf_copy(input); + return 0; +} + +/** Log an error and exit if <b>buf</b> is corrupted. + */ +void +buf_assert_ok(buf_t *buf) +{ + tor_assert(buf); + tor_assert(buf->magic == BUFFER_MAGIC); + + if (! buf->head) { + tor_assert(!buf->tail); + tor_assert(buf->datalen == 0); + } else { + chunk_t *ch; + size_t total = 0; + tor_assert(buf->tail); + for (ch = buf->head; ch; ch = ch->next) { + total += ch->datalen; + tor_assert(ch->datalen <= ch->memlen); + tor_assert(ch->data >= &ch->mem[0]); + tor_assert(ch->data <= &ch->mem[0]+ch->memlen); + if (ch->data == &ch->mem[0]+ch->memlen) { + static int warned = 0; + if (! warned) { + log_warn(LD_BUG, "Invariant violation in buf.c related to #15083"); + warned = 1; + } + } + tor_assert(ch->data+ch->datalen <= &ch->mem[0] + ch->memlen); + if (!ch->next) + tor_assert(ch == buf->tail); + } + tor_assert(buf->datalen == total); + } +} + diff --git a/src/or/buffers.h b/src/common/buffers.h index d884084385..5a52b2a81c 100644 --- a/src/or/buffers.h +++ b/src/common/buffers.h @@ -12,8 +12,15 @@ #ifndef TOR_BUFFERS_H #define TOR_BUFFERS_H +#include "compat.h" +#include "compat.h" +#include "torint.h" #include "testsupport.h" +typedef struct buf_t buf_t; + +struct tor_compress_state_t; + buf_t *buf_new(void); buf_t *buf_new_with_capacity(size_t size); size_t buf_get_default_chunk_size(const buf_t *buf); @@ -28,50 +35,39 @@ size_t buf_slack(const buf_t *buf); uint32_t buf_get_oldest_chunk_timestamp(const buf_t *buf, uint32_t now); size_t buf_get_total_allocation(void); -int read_to_buf(tor_socket_t s, size_t at_most, buf_t *buf, int *reached_eof, - int *socket_error); -int read_to_buf_tls(tor_tls_t *tls, size_t at_most, buf_t *buf); +int buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most, + int *reached_eof, + int *socket_error); -int flush_buf(tor_socket_t s, buf_t *buf, size_t sz, size_t *buf_flushlen); -int flush_buf_tls(tor_tls_t *tls, buf_t *buf, size_t sz, size_t *buf_flushlen); +int buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz, + size_t *buf_flushlen); -int write_to_buf(const char *string, size_t string_len, buf_t *buf); -int write_to_buf_compress(buf_t *buf, tor_compress_state_t *state, +int buf_add(buf_t *buf, const char *string, size_t string_len); +int buf_add_compress(buf_t *buf, struct tor_compress_state_t *state, const char *data, size_t data_len, int done); -int move_buf_to_buf(buf_t *buf_out, buf_t *buf_in, size_t *buf_flushlen); -int fetch_from_buf(char *string, size_t string_len, buf_t *buf); -int fetch_var_cell_from_buf(buf_t *buf, var_cell_t **out, int linkproto); -int fetch_from_buf_http(buf_t *buf, - char **headers_out, size_t max_headerlen, - char **body_out, size_t *body_used, size_t max_bodylen, - int force_complete); -socks_request_t *socks_request_new(void); -void socks_request_free(socks_request_t *req); -int fetch_from_buf_socks(buf_t *buf, socks_request_t *req, - int log_sockstype, int safe_socks); -int fetch_from_buf_socks_client(buf_t *buf, int state, char **reason); -int fetch_from_buf_line(buf_t *buf, char *data_out, size_t *data_len); - -int peek_buf_has_control0_command(buf_t *buf); -#define PEEK_BUF_STARTSWITH_MAX 16 -int peek_buf_startswith(const buf_t *buf, const char *cmd); -int peek_buf_has_http_command(const buf_t *buf); +int buf_move_to_buf(buf_t *buf_out, buf_t *buf_in, size_t *buf_flushlen); +void buf_peek(const buf_t *buf, char *string, size_t string_len); +void buf_drain(buf_t *buf, size_t n); +int buf_get_bytes(buf_t *buf, char *string, size_t string_len); +int buf_get_line(buf_t *buf, char *data_out, size_t *data_len); -int fetch_ext_or_command_from_buf(buf_t *buf, ext_or_cmd_t **out); +#define PEEK_BUF_STARTSWITH_MAX 16 +int buf_peek_startswith(const buf_t *buf, const char *cmd); int buf_set_to_copy(buf_t **output, const buf_t *input); -void assert_buf_ok(buf_t *buf); +void buf_assert_ok(buf_t *buf); + +int buf_find_string_offset(const buf_t *buf, const char *s, size_t n); +void buf_pullup(buf_t *buf, size_t bytes, + const char **head_out, size_t *len_out); #ifdef BUFFERS_PRIVATE -STATIC int buf_find_string_offset(const buf_t *buf, const char *s, size_t n); -STATIC void buf_pullup(buf_t *buf, size_t bytes); #ifdef TOR_UNIT_TESTS -void buf_get_first_chunk_data(const buf_t *buf, const char **cp, size_t *sz); buf_t *buf_new_with_data(const char *cp, size_t sz); #endif -STATIC size_t preferred_chunk_size(size_t target); +size_t buf_preferred_chunk_size(size_t target); #define DEBUG_CHUNK_ALLOC /** A single chunk on a buffer. */ @@ -101,11 +97,28 @@ struct buf_t { chunk_t *head; /**< First chunk in the list, or NULL for none. */ chunk_t *tail; /**< Last chunk in the list, or NULL for none. */ }; -#endif -#ifdef BUFFERS_PRIVATE -STATIC int buf_http_find_content_length(const char *headers, size_t headerlen, - size_t *result_out); +chunk_t *buf_add_chunk_with_capacity(buf_t *buf, size_t capacity, int capped); +/** If a read onto the end of a chunk would be smaller than this number, then + * just start a new chunk. */ +#define MIN_READ_LEN 8 + +/** Return the number of bytes that can be written onto <b>chunk</b> without + * running out of space. */ +static inline size_t +CHUNK_REMAINING_CAPACITY(const chunk_t *chunk) +{ + return (chunk->mem + chunk->memlen) - (chunk->data + chunk->datalen); +} + +/** Return the next character in <b>chunk</b> onto which data can be appended. + * If the chunk is full, this might be off the end of chunk->mem. */ +static inline char * +CHUNK_WRITE_PTR(chunk_t *chunk) +{ + return chunk->data + chunk->datalen; +} + #endif #endif diff --git a/src/common/buffers_tls.c b/src/common/buffers_tls.c new file mode 100644 index 0000000000..8dbd4bcc8e --- /dev/null +++ b/src/common/buffers_tls.c @@ -0,0 +1,175 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#define BUFFERS_PRIVATE +#include "orconfig.h" +#include <stddef.h> +#include "buffers.h" +#include "buffers_tls.h" +#include "compat.h" +#include "compress.h" +#include "util.h" +#include "torint.h" +#include "torlog.h" +#include "tortls.h" +#ifdef HAVE_UNISTD_H +#include <unistd.h> +#endif + +/** As read_to_chunk(), but return (negative) error code on error, blocking, + * or TLS, and the number of bytes read otherwise. */ +static inline int +read_to_chunk_tls(buf_t *buf, chunk_t *chunk, tor_tls_t *tls, + size_t at_most) +{ + int read_result; + + tor_assert(CHUNK_REMAINING_CAPACITY(chunk) >= at_most); + read_result = tor_tls_read(tls, CHUNK_WRITE_PTR(chunk), at_most); + if (read_result < 0) + return read_result; + buf->datalen += read_result; + chunk->datalen += read_result; + return read_result; +} + +/** As read_to_buf, but reads from a TLS connection, and returns a TLS + * status value rather than the number of bytes read. + * + * Using TLS on OR connections complicates matters in two ways. + * + * First, a TLS stream has its own read buffer independent of the + * connection's read buffer. (TLS needs to read an entire frame from + * the network before it can decrypt any data. Thus, trying to read 1 + * byte from TLS can require that several KB be read from the network + * and decrypted. The extra data is stored in TLS's decrypt buffer.) + * Because the data hasn't been read by Tor (it's still inside the TLS), + * this means that sometimes a connection "has stuff to read" even when + * poll() didn't return POLLIN. The tor_tls_get_pending_bytes function is + * used in connection.c to detect TLS objects with non-empty internal + * buffers and read from them again. + * + * Second, the TLS stream's events do not correspond directly to network + * events: sometimes, before a TLS stream can read, the network must be + * ready to write -- or vice versa. + */ +int +buf_read_from_tls(buf_t *buf, tor_tls_t *tls, size_t at_most) +{ + int r = 0; + size_t total_read = 0; + + check_no_tls_errors(); + + if (BUG(buf->datalen >= INT_MAX)) + return -1; + if (BUG(buf->datalen >= INT_MAX - at_most)) + return -1; + + while (at_most > total_read) { + size_t readlen = at_most - total_read; + chunk_t *chunk; + if (!buf->tail || CHUNK_REMAINING_CAPACITY(buf->tail) < MIN_READ_LEN) { + chunk = buf_add_chunk_with_capacity(buf, at_most, 1); + if (readlen > chunk->memlen) + readlen = chunk->memlen; + } else { + size_t cap = CHUNK_REMAINING_CAPACITY(buf->tail); + chunk = buf->tail; + if (cap < readlen) + readlen = cap; + } + + r = read_to_chunk_tls(buf, chunk, tls, readlen); + if (r < 0) + return r; /* Error */ + tor_assert(total_read+r < INT_MAX); + total_read += r; + if ((size_t)r < readlen) /* eof, block, or no more to read. */ + break; + } + return (int)total_read; +} + +/** Helper for buf_flush_to_tls(): try to write <b>sz</b> bytes from chunk + * <b>chunk</b> of buffer <b>buf</b> onto socket <b>s</b>. (Tries to write + * more if there is a forced pending write size.) On success, deduct the + * bytes written from *<b>buf_flushlen</b>. Return the number of bytes + * written on success, and a TOR_TLS error code on failure or blocking. + */ +static inline int +flush_chunk_tls(tor_tls_t *tls, buf_t *buf, chunk_t *chunk, + size_t sz, size_t *buf_flushlen) +{ + int r; + size_t forced; + char *data; + + forced = tor_tls_get_forced_write_size(tls); + if (forced > sz) + sz = forced; + if (chunk) { + data = chunk->data; + tor_assert(sz <= chunk->datalen); + } else { + data = NULL; + tor_assert(sz == 0); + } + r = tor_tls_write(tls, data, sz); + if (r < 0) + return r; + if (*buf_flushlen > (size_t)r) + *buf_flushlen -= r; + else + *buf_flushlen = 0; + buf_drain(buf, r); + log_debug(LD_NET,"flushed %d bytes, %d ready to flush, %d remain.", + r,(int)*buf_flushlen,(int)buf->datalen); + return r; +} + +/** As buf_flush_to_socket(), but writes data to a TLS connection. Can write + * more than <b>flushlen</b> bytes. + */ +int +buf_flush_to_tls(buf_t *buf, tor_tls_t *tls, size_t flushlen, + size_t *buf_flushlen) +{ + int r; + size_t flushed = 0; + ssize_t sz; + tor_assert(buf_flushlen); + tor_assert(*buf_flushlen <= buf->datalen); + tor_assert(flushlen <= *buf_flushlen); + sz = (ssize_t) flushlen; + + /* we want to let tls write even if flushlen is zero, because it might + * have a partial record pending */ + check_no_tls_errors(); + + do { + size_t flushlen0; + if (buf->head) { + if ((ssize_t)buf->head->datalen >= sz) + flushlen0 = sz; + else + flushlen0 = buf->head->datalen; + } else { + flushlen0 = 0; + } + + r = flush_chunk_tls(tls, buf, buf->head, flushlen0, buf_flushlen); + if (r < 0) + return r; + flushed += r; + sz -= r; + if (r == 0) /* Can't flush any more now. */ + break; + } while (sz > 0); + tor_assert(flushed < INT_MAX); + return (int)flushed; +} + diff --git a/src/common/buffers_tls.h b/src/common/buffers_tls.h new file mode 100644 index 0000000000..e91dfeba0b --- /dev/null +++ b/src/common/buffers_tls.h @@ -0,0 +1,19 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_BUFFERS_TLS_H +#define TOR_BUFFERS_TLS_H + +struct buf_t; +struct tor_tls_t; + +int buf_read_from_tls(struct buf_t *buf, + struct tor_tls_t *tls, size_t at_most); +int buf_flush_to_tls(struct buf_t *buf, struct tor_tls_t *tls, + size_t sz, size_t *buf_flushlen); + +#endif + diff --git a/src/common/include.am b/src/common/include.am index 1253888815..cd5eea3404 100644 --- a/src/common/include.am +++ b/src/common/include.am @@ -81,6 +81,7 @@ src_common_libor_ctime_testing_a_CFLAGS = @CFLAGS_CONSTTIME@ $(TEST_CFLAGS) LIBOR_A_SRC = \ src/common/address.c \ src/common/backtrace.c \ + src/common/buffers.c \ src/common/compat.c \ src/common/compat_threads.c \ src/common/compat_time.c \ @@ -110,6 +111,7 @@ src/common/src_common_libor_testing_a-log.$(OBJEXT) \ LIBOR_CRYPTO_A_SRC = \ src/common/aes.c \ + src/common/buffers_tls.c \ src/common/compress.c \ src/common/compress_lzma.c \ src/common/compress_none.c \ @@ -147,6 +149,8 @@ src_common_libor_event_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) COMMONHEADERS = \ src/common/address.h \ src/common/backtrace.h \ + src/common/buffers.h \ + src/common/buffers_tls.h \ src/common/aes.h \ src/common/ciphers.inc \ src/common/compat.h \ diff --git a/src/common/sandbox.c b/src/common/sandbox.c index 18beaabe14..a85b1406fa 100644 --- a/src/common/sandbox.c +++ b/src/common/sandbox.c @@ -1638,7 +1638,9 @@ install_syscall_filter(sandbox_cfg_t* cfg) // loading the seccomp2 filter if ((rc = seccomp_load(ctx))) { - log_err(LD_BUG, "(Sandbox) failed to load: %d (%s)!", rc, + log_err(LD_BUG, "(Sandbox) failed to load: %d (%s)! " + "Are you sure that your kernel has seccomp2 support? The " + "sandbox won't work without it.", rc, strerror(-rc)); goto end; } diff --git a/src/or/buffers.c b/src/or/buffers.c deleted file mode 100644 index bd84103c37..0000000000 --- a/src/or/buffers.c +++ /dev/null @@ -1,2214 +0,0 @@ -/* Copyright (c) 2001 Matej Pfajfar. - * Copyright (c) 2001-2004, Roger Dingledine. - * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2017, The Tor Project, Inc. */ -/* See LICENSE for licensing information */ - -/** - * \file buffers.c - * \brief Implements a generic buffer interface. - * - * A buf_t is a (fairly) opaque byte-oriented FIFO that can read to or flush - * from memory, sockets, file descriptors, TLS connections, or another buf_t. - * Buffers are implemented as linked lists of memory chunks. - * - * All socket-backed and TLS-based connection_t objects have a pair of - * buffers: one for incoming data, and one for outcoming data. These are fed - * and drained from functions in connection.c, trigged by events that are - * monitored in main.c. - * - * This module has basic support for reading and writing on buf_t objects. It - * also contains specialized functions for handling particular protocols - * on a buf_t backend, including SOCKS (used in connection_edge.c), Tor cells - * (used in connection_or.c and channeltls.c), HTTP (used in directory.c), and - * line-oriented communication (used in control.c). - **/ -#define BUFFERS_PRIVATE -#include "or.h" -#include "addressmap.h" -#include "buffers.h" -#include "config.h" -#include "connection_edge.h" -#include "connection_or.h" -#include "control.h" -#include "reasons.h" -#include "ext_orport.h" -#include "util.h" -#include "torlog.h" -#ifdef HAVE_UNISTD_H -#include <unistd.h> -#endif - -//#define PARANOIA - -#ifdef PARANOIA -/** Helper: If PARANOIA is defined, assert that the buffer in local variable - * <b>buf</b> is well-formed. */ -#define check() STMT_BEGIN assert_buf_ok(buf); STMT_END -#else -#define check() STMT_NIL -#endif - -/* Implementation notes: - * - * After flirting with memmove, and dallying with ring-buffers, we're finally - * getting up to speed with the 1970s and implementing buffers as a linked - * list of small chunks. Each buffer has such a list; data is removed from - * the head of the list, and added at the tail. The list is singly linked, - * and the buffer keeps a pointer to the head and the tail. - * - * Every chunk, except the tail, contains at least one byte of data. Data in - * each chunk is contiguous. - * - * When you need to treat the first N characters on a buffer as a contiguous - * string, use the buf_pullup function to make them so. Don't do this more - * than necessary. - * - * The major free Unix kernels have handled buffers like this since, like, - * forever. - */ - -static void socks_request_set_socks5_error(socks_request_t *req, - socks5_reply_status_t reason); - -static int parse_socks(const char *data, size_t datalen, socks_request_t *req, - int log_sockstype, int safe_socks, ssize_t *drain_out, - size_t *want_length_out); -static int parse_socks_client(const uint8_t *data, size_t datalen, - int state, char **reason, - ssize_t *drain_out); - -/* Chunk manipulation functions */ - -#define CHUNK_HEADER_LEN offsetof(chunk_t, mem[0]) - -/* We leave this many NUL bytes at the end of the buffer. */ -#ifdef DISABLE_MEMORY_SENTINELS -#define SENTINEL_LEN 0 -#else -#define SENTINEL_LEN 4 -#endif - -/* Header size plus NUL bytes at the end */ -#define CHUNK_OVERHEAD (CHUNK_HEADER_LEN + SENTINEL_LEN) - -/** Return the number of bytes needed to allocate a chunk to hold - * <b>memlen</b> bytes. */ -#define CHUNK_ALLOC_SIZE(memlen) (CHUNK_OVERHEAD + (memlen)) -/** Return the number of usable bytes in a chunk allocated with - * malloc(<b>memlen</b>). */ -#define CHUNK_SIZE_WITH_ALLOC(memlen) ((memlen) - CHUNK_OVERHEAD) - -#define DEBUG_SENTINEL - -#if defined(DEBUG_SENTINEL) && !defined(DISABLE_MEMORY_SENTINELS) -#define DBG_S(s) s -#else -#define DBG_S(s) (void)0 -#endif - -#ifdef DISABLE_MEMORY_SENTINELS -#define CHUNK_SET_SENTINEL(chunk, alloclen) STMT_NIL -#else -#define CHUNK_SET_SENTINEL(chunk, alloclen) do { \ - uint8_t *a = (uint8_t*) &(chunk)->mem[(chunk)->memlen]; \ - DBG_S(uint8_t *b = &((uint8_t*)(chunk))[(alloclen)-SENTINEL_LEN]); \ - DBG_S(tor_assert(a == b)); \ - memset(a,0,SENTINEL_LEN); \ - } while (0) -#endif - -/** Return the next character in <b>chunk</b> onto which data can be appended. - * If the chunk is full, this might be off the end of chunk->mem. */ -static inline char * -CHUNK_WRITE_PTR(chunk_t *chunk) -{ - return chunk->data + chunk->datalen; -} - -/** Return the number of bytes that can be written onto <b>chunk</b> without - * running out of space. */ -static inline size_t -CHUNK_REMAINING_CAPACITY(const chunk_t *chunk) -{ - return (chunk->mem + chunk->memlen) - (chunk->data + chunk->datalen); -} - -/** Move all bytes stored in <b>chunk</b> to the front of <b>chunk</b>->mem, - * to free up space at the end. */ -static inline void -chunk_repack(chunk_t *chunk) -{ - if (chunk->datalen && chunk->data != &chunk->mem[0]) { - memmove(chunk->mem, chunk->data, chunk->datalen); - } - chunk->data = &chunk->mem[0]; -} - -/** Keep track of total size of allocated chunks for consistency asserts */ -static size_t total_bytes_allocated_in_chunks = 0; -static void -buf_chunk_free_unchecked(chunk_t *chunk) -{ - if (!chunk) - return; -#ifdef DEBUG_CHUNK_ALLOC - tor_assert(CHUNK_ALLOC_SIZE(chunk->memlen) == chunk->DBG_alloc); -#endif - tor_assert(total_bytes_allocated_in_chunks >= - CHUNK_ALLOC_SIZE(chunk->memlen)); - total_bytes_allocated_in_chunks -= CHUNK_ALLOC_SIZE(chunk->memlen); - tor_free(chunk); -} -static inline chunk_t * -chunk_new_with_alloc_size(size_t alloc) -{ - chunk_t *ch; - ch = tor_malloc(alloc); - ch->next = NULL; - ch->datalen = 0; -#ifdef DEBUG_CHUNK_ALLOC - ch->DBG_alloc = alloc; -#endif - ch->memlen = CHUNK_SIZE_WITH_ALLOC(alloc); - total_bytes_allocated_in_chunks += alloc; - ch->data = &ch->mem[0]; - CHUNK_SET_SENTINEL(ch, alloc); - return ch; -} - -/** Expand <b>chunk</b> until it can hold <b>sz</b> bytes, and return a - * new pointer to <b>chunk</b>. Old pointers are no longer valid. */ -static inline chunk_t * -chunk_grow(chunk_t *chunk, size_t sz) -{ - off_t offset; - const size_t memlen_orig = chunk->memlen; - const size_t orig_alloc = CHUNK_ALLOC_SIZE(memlen_orig); - const size_t new_alloc = CHUNK_ALLOC_SIZE(sz); - tor_assert(sz > chunk->memlen); - offset = chunk->data - chunk->mem; - chunk = tor_realloc(chunk, new_alloc); - chunk->memlen = sz; - chunk->data = chunk->mem + offset; -#ifdef DEBUG_CHUNK_ALLOC - tor_assert(chunk->DBG_alloc == orig_alloc); - chunk->DBG_alloc = new_alloc; -#endif - total_bytes_allocated_in_chunks += new_alloc - orig_alloc; - CHUNK_SET_SENTINEL(chunk, new_alloc); - return chunk; -} - -/** If a read onto the end of a chunk would be smaller than this number, then - * just start a new chunk. */ -#define MIN_READ_LEN 8 -/** Every chunk should take up at least this many bytes. */ -#define MIN_CHUNK_ALLOC 256 -/** No chunk should take up more than this many bytes. */ -#define MAX_CHUNK_ALLOC 65536 - -/** Return the allocation size we'd like to use to hold <b>target</b> - * bytes. */ -STATIC size_t -preferred_chunk_size(size_t target) -{ - tor_assert(target <= SIZE_T_CEILING - CHUNK_OVERHEAD); - if (CHUNK_ALLOC_SIZE(target) >= MAX_CHUNK_ALLOC) - return CHUNK_ALLOC_SIZE(target); - size_t sz = MIN_CHUNK_ALLOC; - while (CHUNK_SIZE_WITH_ALLOC(sz) < target) { - sz <<= 1; - } - return sz; -} - -/** Collapse data from the first N chunks from <b>buf</b> into buf->head, - * growing it as necessary, until buf->head has the first <b>bytes</b> bytes - * of data from the buffer, or until buf->head has all the data in <b>buf</b>. - */ -STATIC void -buf_pullup(buf_t *buf, size_t bytes) -{ - chunk_t *dest, *src; - size_t capacity; - if (!buf->head) - return; - - check(); - if (buf->datalen < bytes) - bytes = buf->datalen; - - capacity = bytes; - if (buf->head->datalen >= bytes) - return; - - if (buf->head->memlen >= capacity) { - /* We don't need to grow the first chunk, but we might need to repack it.*/ - size_t needed = capacity - buf->head->datalen; - if (CHUNK_REMAINING_CAPACITY(buf->head) < needed) - chunk_repack(buf->head); - tor_assert(CHUNK_REMAINING_CAPACITY(buf->head) >= needed); - } else { - chunk_t *newhead; - size_t newsize; - /* We need to grow the chunk. */ - chunk_repack(buf->head); - newsize = CHUNK_SIZE_WITH_ALLOC(preferred_chunk_size(capacity)); - newhead = chunk_grow(buf->head, newsize); - tor_assert(newhead->memlen >= capacity); - if (newhead != buf->head) { - if (buf->tail == buf->head) - buf->tail = newhead; - buf->head = newhead; - } - } - - dest = buf->head; - while (dest->datalen < bytes) { - size_t n = bytes - dest->datalen; - src = dest->next; - tor_assert(src); - if (n >= src->datalen) { - memcpy(CHUNK_WRITE_PTR(dest), src->data, src->datalen); - dest->datalen += src->datalen; - dest->next = src->next; - if (buf->tail == src) - buf->tail = dest; - buf_chunk_free_unchecked(src); - } else { - memcpy(CHUNK_WRITE_PTR(dest), src->data, n); - dest->datalen += n; - src->data += n; - src->datalen -= n; - tor_assert(dest->datalen == bytes); - } - } - - check(); -} - -#ifdef TOR_UNIT_TESTS -/* Return the data from the first chunk of buf in cp, and its length in sz. */ -void -buf_get_first_chunk_data(const buf_t *buf, const char **cp, size_t *sz) -{ - if (!buf || !buf->head) { - *cp = NULL; - *sz = 0; - } else { - *cp = buf->head->data; - *sz = buf->head->datalen; - } -} - -/* Write sz bytes from cp into a newly allocated buffer buf. - * Returns NULL when passed a NULL cp or zero sz. - * Asserts on failure: only for use in unit tests. - * buf must be freed using buf_free(). */ -buf_t * -buf_new_with_data(const char *cp, size_t sz) -{ - /* Validate arguments */ - if (!cp || sz <= 0) { - return NULL; - } - - tor_assert(sz < SSIZE_T_CEILING); - - /* Allocate a buffer */ - buf_t *buf = buf_new_with_capacity(sz); - tor_assert(buf); - assert_buf_ok(buf); - tor_assert(!buf->head); - - /* Allocate a chunk that is sz bytes long */ - buf->head = chunk_new_with_alloc_size(CHUNK_ALLOC_SIZE(sz)); - buf->tail = buf->head; - tor_assert(buf->head); - assert_buf_ok(buf); - tor_assert(buf_allocation(buf) >= sz); - - /* Copy the data and size the buffers */ - tor_assert(sz <= buf_slack(buf)); - tor_assert(sz <= CHUNK_REMAINING_CAPACITY(buf->head)); - memcpy(&buf->head->mem[0], cp, sz); - buf->datalen = sz; - buf->head->datalen = sz; - buf->head->data = &buf->head->mem[0]; - assert_buf_ok(buf); - - /* Make sure everything is large enough */ - tor_assert(buf_allocation(buf) >= sz); - tor_assert(buf_allocation(buf) >= buf_datalen(buf) + buf_slack(buf)); - /* Does the buffer implementation allocate more than the requested size? - * (for example, by rounding up). If so, these checks will fail. */ - tor_assert(buf_datalen(buf) == sz); - tor_assert(buf_slack(buf) == 0); - - return buf; -} -#endif - -/** Remove the first <b>n</b> bytes from buf. */ -static inline void -buf_remove_from_front(buf_t *buf, size_t n) -{ - tor_assert(buf->datalen >= n); - while (n) { - tor_assert(buf->head); - if (buf->head->datalen > n) { - buf->head->datalen -= n; - buf->head->data += n; - buf->datalen -= n; - return; - } else { - chunk_t *victim = buf->head; - n -= victim->datalen; - buf->datalen -= victim->datalen; - buf->head = victim->next; - if (buf->tail == victim) - buf->tail = NULL; - buf_chunk_free_unchecked(victim); - } - } - check(); -} - -/** Create and return a new buf with default chunk capacity <b>size</b>. - */ -buf_t * -buf_new_with_capacity(size_t size) -{ - buf_t *b = buf_new(); - b->default_chunk_size = preferred_chunk_size(size); - return b; -} - -/** Allocate and return a new buffer with default capacity. */ -buf_t * -buf_new(void) -{ - buf_t *buf = tor_malloc_zero(sizeof(buf_t)); - buf->magic = BUFFER_MAGIC; - buf->default_chunk_size = 4096; - return buf; -} - -size_t -buf_get_default_chunk_size(const buf_t *buf) -{ - return buf->default_chunk_size; -} - -/** Remove all data from <b>buf</b>. */ -void -buf_clear(buf_t *buf) -{ - chunk_t *chunk, *next; - buf->datalen = 0; - for (chunk = buf->head; chunk; chunk = next) { - next = chunk->next; - buf_chunk_free_unchecked(chunk); - } - buf->head = buf->tail = NULL; -} - -/** Return the number of bytes stored in <b>buf</b> */ -MOCK_IMPL(size_t, -buf_datalen, (const buf_t *buf)) -{ - return buf->datalen; -} - -/** Return the total length of all chunks used in <b>buf</b>. */ -size_t -buf_allocation(const buf_t *buf) -{ - size_t total = 0; - const chunk_t *chunk; - for (chunk = buf->head; chunk; chunk = chunk->next) { - total += CHUNK_ALLOC_SIZE(chunk->memlen); - } - return total; -} - -/** Return the number of bytes that can be added to <b>buf</b> without - * performing any additional allocation. */ -size_t -buf_slack(const buf_t *buf) -{ - if (!buf->tail) - return 0; - else - return CHUNK_REMAINING_CAPACITY(buf->tail); -} - -/** Release storage held by <b>buf</b>. */ -void -buf_free(buf_t *buf) -{ - if (!buf) - return; - - buf_clear(buf); - buf->magic = 0xdeadbeef; - tor_free(buf); -} - -/** Return a new copy of <b>in_chunk</b> */ -static chunk_t * -chunk_copy(const chunk_t *in_chunk) -{ - chunk_t *newch = tor_memdup(in_chunk, CHUNK_ALLOC_SIZE(in_chunk->memlen)); - total_bytes_allocated_in_chunks += CHUNK_ALLOC_SIZE(in_chunk->memlen); -#ifdef DEBUG_CHUNK_ALLOC - newch->DBG_alloc = CHUNK_ALLOC_SIZE(in_chunk->memlen); -#endif - newch->next = NULL; - if (in_chunk->data) { - off_t offset = in_chunk->data - in_chunk->mem; - newch->data = newch->mem + offset; - } - return newch; -} - -/** Return a new copy of <b>buf</b> */ -buf_t * -buf_copy(const buf_t *buf) -{ - chunk_t *ch; - buf_t *out = buf_new(); - out->default_chunk_size = buf->default_chunk_size; - for (ch = buf->head; ch; ch = ch->next) { - chunk_t *newch = chunk_copy(ch); - if (out->tail) { - out->tail->next = newch; - out->tail = newch; - } else { - out->head = out->tail = newch; - } - } - out->datalen = buf->datalen; - return out; -} - -/** Append a new chunk with enough capacity to hold <b>capacity</b> bytes to - * the tail of <b>buf</b>. If <b>capped</b>, don't allocate a chunk bigger - * than MAX_CHUNK_ALLOC. */ -static chunk_t * -buf_add_chunk_with_capacity(buf_t *buf, size_t capacity, int capped) -{ - chunk_t *chunk; - - if (CHUNK_ALLOC_SIZE(capacity) < buf->default_chunk_size) { - chunk = chunk_new_with_alloc_size(buf->default_chunk_size); - } else if (capped && CHUNK_ALLOC_SIZE(capacity) > MAX_CHUNK_ALLOC) { - chunk = chunk_new_with_alloc_size(MAX_CHUNK_ALLOC); - } else { - chunk = chunk_new_with_alloc_size(preferred_chunk_size(capacity)); - } - - chunk->inserted_time = (uint32_t)monotime_coarse_absolute_msec(); - - if (buf->tail) { - tor_assert(buf->head); - buf->tail->next = chunk; - buf->tail = chunk; - } else { - tor_assert(!buf->head); - buf->head = buf->tail = chunk; - } - check(); - return chunk; -} - -/** Return the age of the oldest chunk in the buffer <b>buf</b>, in - * milliseconds. Requires the current monotonic time, in truncated msec, - * as its input <b>now</b>. - */ -uint32_t -buf_get_oldest_chunk_timestamp(const buf_t *buf, uint32_t now) -{ - if (buf->head) { - return now - buf->head->inserted_time; - } else { - return 0; - } -} - -size_t -buf_get_total_allocation(void) -{ - return total_bytes_allocated_in_chunks; -} - -/** Read up to <b>at_most</b> bytes from the socket <b>fd</b> into - * <b>chunk</b> (which must be on <b>buf</b>). If we get an EOF, set - * *<b>reached_eof</b> to 1. Return -1 on error, 0 on eof or blocking, - * and the number of bytes read otherwise. */ -static inline int -read_to_chunk(buf_t *buf, chunk_t *chunk, tor_socket_t fd, size_t at_most, - int *reached_eof, int *socket_error) -{ - ssize_t read_result; - if (at_most > CHUNK_REMAINING_CAPACITY(chunk)) - at_most = CHUNK_REMAINING_CAPACITY(chunk); - read_result = tor_socket_recv(fd, CHUNK_WRITE_PTR(chunk), at_most, 0); - - if (read_result < 0) { - int e = tor_socket_errno(fd); - if (!ERRNO_IS_EAGAIN(e)) { /* it's a real error */ -#ifdef _WIN32 - if (e == WSAENOBUFS) - log_warn(LD_NET,"recv() failed: WSAENOBUFS. Not enough ram?"); -#endif - *socket_error = e; - return -1; - } - return 0; /* would block. */ - } else if (read_result == 0) { - log_debug(LD_NET,"Encountered eof on fd %d", (int)fd); - *reached_eof = 1; - return 0; - } else { /* actually got bytes. */ - buf->datalen += read_result; - chunk->datalen += read_result; - log_debug(LD_NET,"Read %ld bytes. %d on inbuf.", (long)read_result, - (int)buf->datalen); - tor_assert(read_result < INT_MAX); - return (int)read_result; - } -} - -/** As read_to_chunk(), but return (negative) error code on error, blocking, - * or TLS, and the number of bytes read otherwise. */ -static inline int -read_to_chunk_tls(buf_t *buf, chunk_t *chunk, tor_tls_t *tls, - size_t at_most) -{ - int read_result; - - tor_assert(CHUNK_REMAINING_CAPACITY(chunk) >= at_most); - read_result = tor_tls_read(tls, CHUNK_WRITE_PTR(chunk), at_most); - if (read_result < 0) - return read_result; - buf->datalen += read_result; - chunk->datalen += read_result; - return read_result; -} - -/** Read from socket <b>s</b>, writing onto end of <b>buf</b>. Read at most - * <b>at_most</b> bytes, growing the buffer as necessary. If recv() returns 0 - * (because of EOF), set *<b>reached_eof</b> to 1 and return 0. Return -1 on - * error; else return the number of bytes read. - */ -/* XXXX indicate "read blocked" somehow? */ -int -read_to_buf(tor_socket_t s, size_t at_most, buf_t *buf, int *reached_eof, - int *socket_error) -{ - /* XXXX It's stupid to overload the return values for these functions: - * "error status" and "number of bytes read" are not mutually exclusive. - */ - int r = 0; - size_t total_read = 0; - - check(); - tor_assert(reached_eof); - tor_assert(SOCKET_OK(s)); - - if (BUG(buf->datalen >= INT_MAX)) - return -1; - if (BUG(buf->datalen >= INT_MAX - at_most)) - return -1; - - while (at_most > total_read) { - size_t readlen = at_most - total_read; - chunk_t *chunk; - if (!buf->tail || CHUNK_REMAINING_CAPACITY(buf->tail) < MIN_READ_LEN) { - chunk = buf_add_chunk_with_capacity(buf, at_most, 1); - if (readlen > chunk->memlen) - readlen = chunk->memlen; - } else { - size_t cap = CHUNK_REMAINING_CAPACITY(buf->tail); - chunk = buf->tail; - if (cap < readlen) - readlen = cap; - } - - r = read_to_chunk(buf, chunk, s, readlen, reached_eof, socket_error); - check(); - if (r < 0) - return r; /* Error */ - tor_assert(total_read+r < INT_MAX); - total_read += r; - if ((size_t)r < readlen) { /* eof, block, or no more to read. */ - break; - } - } - return (int)total_read; -} - -/** As read_to_buf, but reads from a TLS connection, and returns a TLS - * status value rather than the number of bytes read. - * - * Using TLS on OR connections complicates matters in two ways. - * - * First, a TLS stream has its own read buffer independent of the - * connection's read buffer. (TLS needs to read an entire frame from - * the network before it can decrypt any data. Thus, trying to read 1 - * byte from TLS can require that several KB be read from the network - * and decrypted. The extra data is stored in TLS's decrypt buffer.) - * Because the data hasn't been read by Tor (it's still inside the TLS), - * this means that sometimes a connection "has stuff to read" even when - * poll() didn't return POLLIN. The tor_tls_get_pending_bytes function is - * used in connection.c to detect TLS objects with non-empty internal - * buffers and read from them again. - * - * Second, the TLS stream's events do not correspond directly to network - * events: sometimes, before a TLS stream can read, the network must be - * ready to write -- or vice versa. - */ -int -read_to_buf_tls(tor_tls_t *tls, size_t at_most, buf_t *buf) -{ - int r = 0; - size_t total_read = 0; - - check_no_tls_errors(); - - check(); - - if (BUG(buf->datalen >= INT_MAX)) - return -1; - if (BUG(buf->datalen >= INT_MAX - at_most)) - return -1; - - while (at_most > total_read) { - size_t readlen = at_most - total_read; - chunk_t *chunk; - if (!buf->tail || CHUNK_REMAINING_CAPACITY(buf->tail) < MIN_READ_LEN) { - chunk = buf_add_chunk_with_capacity(buf, at_most, 1); - if (readlen > chunk->memlen) - readlen = chunk->memlen; - } else { - size_t cap = CHUNK_REMAINING_CAPACITY(buf->tail); - chunk = buf->tail; - if (cap < readlen) - readlen = cap; - } - - r = read_to_chunk_tls(buf, chunk, tls, readlen); - check(); - if (r < 0) - return r; /* Error */ - tor_assert(total_read+r < INT_MAX); - total_read += r; - if ((size_t)r < readlen) /* eof, block, or no more to read. */ - break; - } - return (int)total_read; -} - -/** Helper for flush_buf(): try to write <b>sz</b> bytes from chunk - * <b>chunk</b> of buffer <b>buf</b> onto socket <b>s</b>. On success, deduct - * the bytes written from *<b>buf_flushlen</b>. Return the number of bytes - * written on success, 0 on blocking, -1 on failure. - */ -static inline int -flush_chunk(tor_socket_t s, buf_t *buf, chunk_t *chunk, size_t sz, - size_t *buf_flushlen) -{ - ssize_t write_result; - - if (sz > chunk->datalen) - sz = chunk->datalen; - write_result = tor_socket_send(s, chunk->data, sz, 0); - - if (write_result < 0) { - int e = tor_socket_errno(s); - if (!ERRNO_IS_EAGAIN(e)) { /* it's a real error */ -#ifdef _WIN32 - if (e == WSAENOBUFS) - log_warn(LD_NET,"write() failed: WSAENOBUFS. Not enough ram?"); -#endif - return -1; - } - log_debug(LD_NET,"write() would block, returning."); - return 0; - } else { - *buf_flushlen -= write_result; - buf_remove_from_front(buf, write_result); - tor_assert(write_result < INT_MAX); - return (int)write_result; - } -} - -/** Helper for flush_buf_tls(): try to write <b>sz</b> bytes from chunk - * <b>chunk</b> of buffer <b>buf</b> onto socket <b>s</b>. (Tries to write - * more if there is a forced pending write size.) On success, deduct the - * bytes written from *<b>buf_flushlen</b>. Return the number of bytes - * written on success, and a TOR_TLS error code on failure or blocking. - */ -static inline int -flush_chunk_tls(tor_tls_t *tls, buf_t *buf, chunk_t *chunk, - size_t sz, size_t *buf_flushlen) -{ - int r; - size_t forced; - char *data; - - forced = tor_tls_get_forced_write_size(tls); - if (forced > sz) - sz = forced; - if (chunk) { - data = chunk->data; - tor_assert(sz <= chunk->datalen); - } else { - data = NULL; - tor_assert(sz == 0); - } - r = tor_tls_write(tls, data, sz); - if (r < 0) - return r; - if (*buf_flushlen > (size_t)r) - *buf_flushlen -= r; - else - *buf_flushlen = 0; - buf_remove_from_front(buf, r); - log_debug(LD_NET,"flushed %d bytes, %d ready to flush, %d remain.", - r,(int)*buf_flushlen,(int)buf->datalen); - return r; -} - -/** Write data from <b>buf</b> to the socket <b>s</b>. Write at most - * <b>sz</b> bytes, decrement *<b>buf_flushlen</b> by - * the number of bytes actually written, and remove the written bytes - * from the buffer. Return the number of bytes written on success, - * -1 on failure. Return 0 if write() would block. - */ -int -flush_buf(tor_socket_t s, buf_t *buf, size_t sz, size_t *buf_flushlen) -{ - /* XXXX It's stupid to overload the return values for these functions: - * "error status" and "number of bytes flushed" are not mutually exclusive. - */ - int r; - size_t flushed = 0; - tor_assert(buf_flushlen); - tor_assert(SOCKET_OK(s)); - tor_assert(*buf_flushlen <= buf->datalen); - tor_assert(sz <= *buf_flushlen); - - check(); - while (sz) { - size_t flushlen0; - tor_assert(buf->head); - if (buf->head->datalen >= sz) - flushlen0 = sz; - else - flushlen0 = buf->head->datalen; - - r = flush_chunk(s, buf, buf->head, flushlen0, buf_flushlen); - check(); - if (r < 0) - return r; - flushed += r; - sz -= r; - if (r == 0 || (size_t)r < flushlen0) /* can't flush any more now. */ - break; - } - tor_assert(flushed < INT_MAX); - return (int)flushed; -} - -/** As flush_buf(), but writes data to a TLS connection. Can write more than - * <b>flushlen</b> bytes. - */ -int -flush_buf_tls(tor_tls_t *tls, buf_t *buf, size_t flushlen, - size_t *buf_flushlen) -{ - int r; - size_t flushed = 0; - ssize_t sz; - tor_assert(buf_flushlen); - tor_assert(*buf_flushlen <= buf->datalen); - tor_assert(flushlen <= *buf_flushlen); - sz = (ssize_t) flushlen; - - /* we want to let tls write even if flushlen is zero, because it might - * have a partial record pending */ - check_no_tls_errors(); - - check(); - do { - size_t flushlen0; - if (buf->head) { - if ((ssize_t)buf->head->datalen >= sz) - flushlen0 = sz; - else - flushlen0 = buf->head->datalen; - } else { - flushlen0 = 0; - } - - r = flush_chunk_tls(tls, buf, buf->head, flushlen0, buf_flushlen); - check(); - if (r < 0) - return r; - flushed += r; - sz -= r; - if (r == 0) /* Can't flush any more now. */ - break; - } while (sz > 0); - tor_assert(flushed < INT_MAX); - return (int)flushed; -} - -/** Append <b>string_len</b> bytes from <b>string</b> to the end of - * <b>buf</b>. - * - * Return the new length of the buffer on success, -1 on failure. - */ -int -write_to_buf(const char *string, size_t string_len, buf_t *buf) -{ - if (!string_len) - return (int)buf->datalen; - check(); - - if (BUG(buf->datalen >= INT_MAX)) - return -1; - if (BUG(buf->datalen >= INT_MAX - string_len)) - return -1; - - while (string_len) { - size_t copy; - if (!buf->tail || !CHUNK_REMAINING_CAPACITY(buf->tail)) - buf_add_chunk_with_capacity(buf, string_len, 1); - - copy = CHUNK_REMAINING_CAPACITY(buf->tail); - if (copy > string_len) - copy = string_len; - memcpy(CHUNK_WRITE_PTR(buf->tail), string, copy); - string_len -= copy; - string += copy; - buf->datalen += copy; - buf->tail->datalen += copy; - } - - check(); - tor_assert(buf->datalen < INT_MAX); - return (int)buf->datalen; -} - -/** Helper: copy the first <b>string_len</b> bytes from <b>buf</b> - * onto <b>string</b>. - */ -static inline void -peek_from_buf(char *string, size_t string_len, const buf_t *buf) -{ - chunk_t *chunk; - - tor_assert(string); - /* make sure we don't ask for too much */ - tor_assert(string_len <= buf->datalen); - /* assert_buf_ok(buf); */ - - chunk = buf->head; - while (string_len) { - size_t copy = string_len; - tor_assert(chunk); - if (chunk->datalen < copy) - copy = chunk->datalen; - memcpy(string, chunk->data, copy); - string_len -= copy; - string += copy; - chunk = chunk->next; - } -} - -/** Remove <b>string_len</b> bytes from the front of <b>buf</b>, and store - * them into <b>string</b>. Return the new buffer size. <b>string_len</b> - * must be \<= the number of bytes on the buffer. - */ -int -fetch_from_buf(char *string, size_t string_len, buf_t *buf) -{ - /* There must be string_len bytes in buf; write them onto string, - * then memmove buf back (that is, remove them from buf). - * - * Return the number of bytes still on the buffer. */ - - check(); - peek_from_buf(string, string_len, buf); - buf_remove_from_front(buf, string_len); - check(); - tor_assert(buf->datalen < INT_MAX); - return (int)buf->datalen; -} - -/** True iff the cell command <b>command</b> is one that implies a - * variable-length cell in Tor link protocol <b>linkproto</b>. */ -static inline int -cell_command_is_var_length(uint8_t command, int linkproto) -{ - /* If linkproto is v2 (2), CELL_VERSIONS is the only variable-length cells - * work as implemented here. If it's 1, there are no variable-length cells. - * Tor does not support other versions right now, and so can't negotiate - * them. - */ - switch (linkproto) { - case 1: - /* Link protocol version 1 has no variable-length cells. */ - return 0; - case 2: - /* In link protocol version 2, VERSIONS is the only variable-length cell */ - return command == CELL_VERSIONS; - case 0: - case 3: - default: - /* In link protocol version 3 and later, and in version "unknown", - * commands 128 and higher indicate variable-length. VERSIONS is - * grandfathered in. */ - return command == CELL_VERSIONS || command >= 128; - } -} - -/** Check <b>buf</b> for a variable-length cell according to the rules of link - * protocol version <b>linkproto</b>. If one is found, pull it off the buffer - * and assign a newly allocated var_cell_t to *<b>out</b>, and return 1. - * Return 0 if whatever is on the start of buf_t is not a variable-length - * cell. Return 1 and set *<b>out</b> to NULL if there seems to be the start - * of a variable-length cell on <b>buf</b>, but the whole thing isn't there - * yet. */ -int -fetch_var_cell_from_buf(buf_t *buf, var_cell_t **out, int linkproto) -{ - char hdr[VAR_CELL_MAX_HEADER_SIZE]; - var_cell_t *result; - uint8_t command; - uint16_t length; - const int wide_circ_ids = linkproto >= MIN_LINK_PROTO_FOR_WIDE_CIRC_IDS; - const int circ_id_len = get_circ_id_size(wide_circ_ids); - const unsigned header_len = get_var_cell_header_size(wide_circ_ids); - check(); - *out = NULL; - if (buf->datalen < header_len) - return 0; - peek_from_buf(hdr, header_len, buf); - - command = get_uint8(hdr + circ_id_len); - if (!(cell_command_is_var_length(command, linkproto))) - return 0; - - length = ntohs(get_uint16(hdr + circ_id_len + 1)); - if (buf->datalen < (size_t)(header_len+length)) - return 1; - result = var_cell_new(length); - result->command = command; - if (wide_circ_ids) - result->circ_id = ntohl(get_uint32(hdr)); - else - result->circ_id = ntohs(get_uint16(hdr)); - - buf_remove_from_front(buf, header_len); - peek_from_buf((char*) result->payload, length, buf); - buf_remove_from_front(buf, length); - check(); - - *out = result; - return 1; -} - -/** Move up to *<b>buf_flushlen</b> bytes from <b>buf_in</b> to - * <b>buf_out</b>, and modify *<b>buf_flushlen</b> appropriately. - * Return the number of bytes actually copied. - */ -int -move_buf_to_buf(buf_t *buf_out, buf_t *buf_in, size_t *buf_flushlen) -{ - /* We can do way better here, but this doesn't turn up in any profiles. */ - char b[4096]; - size_t cp, len; - - if (BUG(buf_out->datalen >= INT_MAX)) - return -1; - if (BUG(buf_out->datalen >= INT_MAX - *buf_flushlen)) - return -1; - - len = *buf_flushlen; - if (len > buf_in->datalen) - len = buf_in->datalen; - - cp = len; /* Remember the number of bytes we intend to copy. */ - tor_assert(cp < INT_MAX); - while (len) { - /* This isn't the most efficient implementation one could imagine, since - * it does two copies instead of 1, but I kinda doubt that this will be - * critical path. */ - size_t n = len > sizeof(b) ? sizeof(b) : len; - fetch_from_buf(b, n, buf_in); - write_to_buf(b, n, buf_out); - len -= n; - } - *buf_flushlen -= cp; - return (int)cp; -} - -/** Internal structure: represents a position in a buffer. */ -typedef struct buf_pos_t { - const chunk_t *chunk; /**< Which chunk are we pointing to? */ - int pos;/**< Which character inside the chunk's data are we pointing to? */ - size_t chunk_pos; /**< Total length of all previous chunks. */ -} buf_pos_t; - -/** Initialize <b>out</b> to point to the first character of <b>buf</b>.*/ -static void -buf_pos_init(const buf_t *buf, buf_pos_t *out) -{ - out->chunk = buf->head; - out->pos = 0; - out->chunk_pos = 0; -} - -/** Advance <b>out</b> to the first appearance of <b>ch</b> at the current - * position of <b>out</b>, or later. Return -1 if no instances are found; - * otherwise returns the absolute position of the character. */ -static off_t -buf_find_pos_of_char(char ch, buf_pos_t *out) -{ - const chunk_t *chunk; - int pos; - tor_assert(out); - if (out->chunk) { - if (out->chunk->datalen) { - tor_assert(out->pos < (off_t)out->chunk->datalen); - } else { - tor_assert(out->pos == 0); - } - } - pos = out->pos; - for (chunk = out->chunk; chunk; chunk = chunk->next) { - char *cp = memchr(chunk->data+pos, ch, chunk->datalen - pos); - if (cp) { - out->chunk = chunk; - tor_assert(cp - chunk->data < INT_MAX); - out->pos = (int)(cp - chunk->data); - return out->chunk_pos + out->pos; - } else { - out->chunk_pos += chunk->datalen; - pos = 0; - } - } - return -1; -} - -/** Advance <b>pos</b> by a single character, if there are any more characters - * in the buffer. Returns 0 on success, -1 on failure. */ -static inline int -buf_pos_inc(buf_pos_t *pos) -{ - ++pos->pos; - if (pos->pos == (off_t)pos->chunk->datalen) { - if (!pos->chunk->next) - return -1; - pos->chunk_pos += pos->chunk->datalen; - pos->chunk = pos->chunk->next; - pos->pos = 0; - } - return 0; -} - -/** Return true iff the <b>n</b>-character string in <b>s</b> appears - * (verbatim) at <b>pos</b>. */ -static int -buf_matches_at_pos(const buf_pos_t *pos, const char *s, size_t n) -{ - buf_pos_t p; - if (!n) - return 1; - - memcpy(&p, pos, sizeof(p)); - - while (1) { - char ch = p.chunk->data[p.pos]; - if (ch != *s) - return 0; - ++s; - /* If we're out of characters that don't match, we match. Check this - * _before_ we test incrementing pos, in case we're at the end of the - * string. */ - if (--n == 0) - return 1; - if (buf_pos_inc(&p)<0) - return 0; - } -} - -/** Return the first position in <b>buf</b> at which the <b>n</b>-character - * string <b>s</b> occurs, or -1 if it does not occur. */ -STATIC int -buf_find_string_offset(const buf_t *buf, const char *s, size_t n) -{ - buf_pos_t pos; - buf_pos_init(buf, &pos); - while (buf_find_pos_of_char(*s, &pos) >= 0) { - if (buf_matches_at_pos(&pos, s, n)) { - tor_assert(pos.chunk_pos + pos.pos < INT_MAX); - return (int)(pos.chunk_pos + pos.pos); - } else { - if (buf_pos_inc(&pos)<0) - return -1; - } - } - return -1; -} - -/** - * Scan the HTTP headers in the <b>headerlen</b>-byte memory range at - * <b>headers</b>, looking for a "Content-Length" header. Try to set - * *<b>result_out</b> to the numeric value of that header if possible. - * Return -1 if the header was malformed, 0 if it was missing, and 1 if - * it was present and well-formed. - */ -STATIC int -buf_http_find_content_length(const char *headers, size_t headerlen, - size_t *result_out) -{ - const char *p, *newline; - char *len_str, *eos=NULL; - size_t remaining, result; - int ok; - *result_out = 0; /* The caller shouldn't look at this unless the - * return value is 1, but let's prevent confusion */ - -#define CONTENT_LENGTH "\r\nContent-Length: " - p = (char*) tor_memstr(headers, headerlen, CONTENT_LENGTH); - if (p == NULL) - return 0; - - tor_assert(p >= headers && p < headers+headerlen); - remaining = (headers+headerlen)-p; - p += strlen(CONTENT_LENGTH); - remaining -= strlen(CONTENT_LENGTH); - - newline = memchr(p, '\n', remaining); - if (newline == NULL) - return -1; - - len_str = tor_memdup_nulterm(p, newline-p); - /* We limit the size to INT_MAX because other parts of the buffer.c - * code don't like buffers to be any bigger than that. */ - result = (size_t) tor_parse_uint64(len_str, 10, 0, INT_MAX, &ok, &eos); - if (eos && !tor_strisspace(eos)) { - ok = 0; - } else { - *result_out = result; - } - tor_free(len_str); - - return ok ? 1 : -1; -} - -/** There is a (possibly incomplete) http statement on <b>buf</b>, of the - * form "\%s\\r\\n\\r\\n\%s", headers, body. (body may contain NULs.) - * If a) the headers include a Content-Length field and all bytes in - * the body are present, or b) there's no Content-Length field and - * all headers are present, then: - * - * - strdup headers into <b>*headers_out</b>, and NUL-terminate it. - * - memdup body into <b>*body_out</b>, and NUL-terminate it. - * - Then remove them from <b>buf</b>, and return 1. - * - * - If headers or body is NULL, discard that part of the buf. - * - If a headers or body doesn't fit in the arg, return -1. - * (We ensure that the headers or body don't exceed max len, - * _even if_ we're planning to discard them.) - * - If force_complete is true, then succeed even if not all of the - * content has arrived. - * - * Else, change nothing and return 0. - */ -int -fetch_from_buf_http(buf_t *buf, - char **headers_out, size_t max_headerlen, - char **body_out, size_t *body_used, size_t max_bodylen, - int force_complete) -{ - char *headers; - size_t headerlen, bodylen, contentlen=0; - int crlf_offset; - int r; - - check(); - if (!buf->head) - return 0; - - crlf_offset = buf_find_string_offset(buf, "\r\n\r\n", 4); - if (crlf_offset > (int)max_headerlen || - (crlf_offset < 0 && buf->datalen > max_headerlen)) { - log_debug(LD_HTTP,"headers too long."); - return -1; - } else if (crlf_offset < 0) { - log_debug(LD_HTTP,"headers not all here yet."); - return 0; - } - /* Okay, we have a full header. Make sure it all appears in the first - * chunk. */ - if ((int)buf->head->datalen < crlf_offset + 4) - buf_pullup(buf, crlf_offset+4); - headerlen = crlf_offset + 4; - - headers = buf->head->data; - bodylen = buf->datalen - headerlen; - log_debug(LD_HTTP,"headerlen %d, bodylen %d.", (int)headerlen, (int)bodylen); - - if (max_headerlen <= headerlen) { - log_warn(LD_HTTP,"headerlen %d larger than %d. Failing.", - (int)headerlen, (int)max_headerlen-1); - return -1; - } - if (max_bodylen <= bodylen) { - log_warn(LD_HTTP,"bodylen %d larger than %d. Failing.", - (int)bodylen, (int)max_bodylen-1); - return -1; - } - - r = buf_http_find_content_length(headers, headerlen, &contentlen); - if (r == -1) { - log_warn(LD_PROTOCOL, "Content-Length is bogus; maybe " - "someone is trying to crash us."); - return -1; - } else if (r == 1) { - /* if content-length is malformed, then our body length is 0. fine. */ - log_debug(LD_HTTP,"Got a contentlen of %d.",(int)contentlen); - if (bodylen < contentlen) { - if (!force_complete) { - log_debug(LD_HTTP,"body not all here yet."); - return 0; /* not all there yet */ - } - } - if (bodylen > contentlen) { - bodylen = contentlen; - log_debug(LD_HTTP,"bodylen reduced to %d.",(int)bodylen); - } - } else { - tor_assert(r == 0); - /* Leave bodylen alone */ - } - - /* all happy. copy into the appropriate places, and return 1 */ - if (headers_out) { - *headers_out = tor_malloc(headerlen+1); - fetch_from_buf(*headers_out, headerlen, buf); - (*headers_out)[headerlen] = 0; /* NUL terminate it */ - } - if (body_out) { - tor_assert(body_used); - *body_used = bodylen; - *body_out = tor_malloc(bodylen+1); - fetch_from_buf(*body_out, bodylen, buf); - (*body_out)[bodylen] = 0; /* NUL terminate it */ - } - check(); - return 1; -} - -/** - * Wait this many seconds before warning the user about using SOCKS unsafely - * again. */ -#define SOCKS_WARN_INTERVAL 5 - -/** Warn that the user application has made an unsafe socks request using - * protocol <b>socks_protocol</b> on port <b>port</b>. Don't warn more than - * once per SOCKS_WARN_INTERVAL, unless <b>safe_socks</b> is set. */ -static void -log_unsafe_socks_warning(int socks_protocol, const char *address, - uint16_t port, int safe_socks) -{ - static ratelim_t socks_ratelim = RATELIM_INIT(SOCKS_WARN_INTERVAL); - - if (safe_socks) { - log_fn_ratelim(&socks_ratelim, LOG_WARN, LD_APP, - "Your application (using socks%d to port %d) is giving " - "Tor only an IP address. Applications that do DNS resolves " - "themselves may leak information. Consider using Socks4A " - "(e.g. via privoxy or socat) instead. For more information, " - "please see https://wiki.torproject.org/TheOnionRouter/" - "TorFAQ#SOCKSAndDNS.%s", - socks_protocol, - (int)port, - safe_socks ? " Rejecting." : ""); - } - control_event_client_status(LOG_WARN, - "DANGEROUS_SOCKS PROTOCOL=SOCKS%d ADDRESS=%s:%d", - socks_protocol, address, (int)port); -} - -/** Do not attempt to parse socks messages longer than this. This value is - * actually significantly higher than the longest possible socks message. */ -#define MAX_SOCKS_MESSAGE_LEN 512 - -/** Return a new socks_request_t. */ -socks_request_t * -socks_request_new(void) -{ - return tor_malloc_zero(sizeof(socks_request_t)); -} - -/** Free all storage held in the socks_request_t <b>req</b>. */ -void -socks_request_free(socks_request_t *req) -{ - if (!req) - return; - if (req->username) { - memwipe(req->username, 0x10, req->usernamelen); - tor_free(req->username); - } - if (req->password) { - memwipe(req->password, 0x04, req->passwordlen); - tor_free(req->password); - } - memwipe(req, 0xCC, sizeof(socks_request_t)); - tor_free(req); -} - -/** There is a (possibly incomplete) socks handshake on <b>buf</b>, of one - * of the forms - * - socks4: "socksheader username\\0" - * - socks4a: "socksheader username\\0 destaddr\\0" - * - socks5 phase one: "version #methods methods" - * - socks5 phase two: "version command 0 addresstype..." - * If it's a complete and valid handshake, and destaddr fits in - * MAX_SOCKS_ADDR_LEN bytes, then pull the handshake off the buf, - * assign to <b>req</b>, and return 1. - * - * If it's invalid or too big, return -1. - * - * Else it's not all there yet, leave buf alone and return 0. - * - * If you want to specify the socks reply, write it into <b>req->reply</b> - * and set <b>req->replylen</b>, else leave <b>req->replylen</b> alone. - * - * If <b>log_sockstype</b> is non-zero, then do a notice-level log of whether - * the connection is possibly leaking DNS requests locally or not. - * - * If <b>safe_socks</b> is true, then reject unsafe socks protocols. - * - * If returning 0 or -1, <b>req->address</b> and <b>req->port</b> are - * undefined. - */ -int -fetch_from_buf_socks(buf_t *buf, socks_request_t *req, - int log_sockstype, int safe_socks) -{ - int res; - ssize_t n_drain; - size_t want_length = 128; - - if (buf->datalen < 2) /* version and another byte */ - return 0; - - do { - n_drain = 0; - buf_pullup(buf, want_length); - tor_assert(buf->head && buf->head->datalen >= 2); - want_length = 0; - - res = parse_socks(buf->head->data, buf->head->datalen, req, log_sockstype, - safe_socks, &n_drain, &want_length); - - if (n_drain < 0) - buf_clear(buf); - else if (n_drain > 0) - buf_remove_from_front(buf, n_drain); - - } while (res == 0 && buf->head && want_length < buf->datalen && - buf->datalen >= 2); - - return res; -} - -/** The size of the header of an Extended ORPort message: 2 bytes for - * COMMAND, 2 bytes for BODYLEN */ -#define EXT_OR_CMD_HEADER_SIZE 4 - -/** Read <b>buf</b>, which should contain an Extended ORPort message - * from a transport proxy. If well-formed, create and populate - * <b>out</b> with the Extended ORport message. Return 0 if the - * buffer was incomplete, 1 if it was well-formed and -1 if we - * encountered an error while parsing it. */ -int -fetch_ext_or_command_from_buf(buf_t *buf, ext_or_cmd_t **out) -{ - char hdr[EXT_OR_CMD_HEADER_SIZE]; - uint16_t len; - - check(); - if (buf->datalen < EXT_OR_CMD_HEADER_SIZE) - return 0; - peek_from_buf(hdr, sizeof(hdr), buf); - len = ntohs(get_uint16(hdr+2)); - if (buf->datalen < (unsigned)len + EXT_OR_CMD_HEADER_SIZE) - return 0; - *out = ext_or_cmd_new(len); - (*out)->cmd = ntohs(get_uint16(hdr)); - (*out)->len = len; - buf_remove_from_front(buf, EXT_OR_CMD_HEADER_SIZE); - fetch_from_buf((*out)->body, len, buf); - return 1; -} - -/** Create a SOCKS5 reply message with <b>reason</b> in its REP field and - * have Tor send it as error response to <b>req</b>. - */ -static void -socks_request_set_socks5_error(socks_request_t *req, - socks5_reply_status_t reason) -{ - req->replylen = 10; - memset(req->reply,0,10); - - req->reply[0] = 0x05; // VER field. - req->reply[1] = reason; // REP field. - req->reply[3] = 0x01; // ATYP field. -} - -static const char SOCKS_PROXY_IS_NOT_AN_HTTP_PROXY_MSG[] = - "HTTP/1.0 501 Tor is not an HTTP Proxy\r\n" - "Content-Type: text/html; charset=iso-8859-1\r\n\r\n" - "<html>\n" - "<head>\n" - "<title>Tor is not an HTTP Proxy</title>\n" - "</head>\n" - "<body>\n" - "<h1>Tor is not an HTTP Proxy</h1>\n" - "<p>\n" - "It appears you have configured your web browser to use Tor as " - "an HTTP proxy.\n\n" - "This is not correct: Tor is a SOCKS proxy, not an HTTP proxy.\n" - "Please configure your client accordingly.\n" - "</p>\n" - "<p>\n" - "See <a href=\"https://www.torproject.org/documentation.html\">" - "https://www.torproject.org/documentation.html</a> for more " - "information.\n" - "<!-- Plus this comment, to make the body response more than 512 bytes, so " - " IE will be willing to display it. Comment comment comment comment " - " comment comment comment comment comment comment comment comment.-->\n" - "</p>\n" - "</body>\n" - "</html>\n"; - -/** Implementation helper to implement fetch_from_*_socks. Instead of looking - * at a buffer's contents, we look at the <b>datalen</b> bytes of data in - * <b>data</b>. Instead of removing data from the buffer, we set - * <b>drain_out</b> to the amount of data that should be removed (or -1 if the - * buffer should be cleared). Instead of pulling more data into the first - * chunk of the buffer, we set *<b>want_length_out</b> to the number of bytes - * we'd like to see in the input buffer, if they're available. */ -static int -parse_socks(const char *data, size_t datalen, socks_request_t *req, - int log_sockstype, int safe_socks, ssize_t *drain_out, - size_t *want_length_out) -{ - unsigned int len; - char tmpbuf[TOR_ADDR_BUF_LEN+1]; - tor_addr_t destaddr; - uint32_t destip; - uint8_t socksver; - char *next, *startaddr; - unsigned char usernamelen, passlen; - struct in_addr in; - - if (datalen < 2) { - /* We always need at least 2 bytes. */ - *want_length_out = 2; - return 0; - } - - if (req->socks_version == 5 && !req->got_auth) { - /* See if we have received authentication. Strictly speaking, we should - also check whether we actually negotiated username/password - authentication. But some broken clients will send us authentication - even if we negotiated SOCKS_NO_AUTH. */ - if (*data == 1) { /* username/pass version 1 */ - /* Format is: authversion [1 byte] == 1 - usernamelen [1 byte] - username [usernamelen bytes] - passlen [1 byte] - password [passlen bytes] */ - usernamelen = (unsigned char)*(data + 1); - if (datalen < 2u + usernamelen + 1u) { - *want_length_out = 2u + usernamelen + 1u; - return 0; - } - passlen = (unsigned char)*(data + 2u + usernamelen); - if (datalen < 2u + usernamelen + 1u + passlen) { - *want_length_out = 2u + usernamelen + 1u + passlen; - return 0; - } - req->replylen = 2; /* 2 bytes of response */ - req->reply[0] = 1; /* authversion == 1 */ - req->reply[1] = 0; /* authentication successful */ - log_debug(LD_APP, - "socks5: Accepted username/password without checking."); - if (usernamelen) { - req->username = tor_memdup(data+2u, usernamelen); - req->usernamelen = usernamelen; - } - if (passlen) { - req->password = tor_memdup(data+3u+usernamelen, passlen); - req->passwordlen = passlen; - } - *drain_out = 2u + usernamelen + 1u + passlen; - req->got_auth = 1; - *want_length_out = 7; /* Minimal socks5 command. */ - return 0; - } else if (req->auth_type == SOCKS_USER_PASS) { - /* unknown version byte */ - log_warn(LD_APP, "Socks5 username/password version %d not recognized; " - "rejecting.", (int)*data); - return -1; - } - } - - socksver = *data; - - switch (socksver) { /* which version of socks? */ - case 5: /* socks5 */ - - if (req->socks_version != 5) { /* we need to negotiate a method */ - unsigned char nummethods = (unsigned char)*(data+1); - int have_user_pass, have_no_auth; - int r=0; - tor_assert(!req->socks_version); - if (datalen < 2u+nummethods) { - *want_length_out = 2u+nummethods; - return 0; - } - if (!nummethods) - return -1; - req->replylen = 2; /* 2 bytes of response */ - req->reply[0] = 5; /* socks5 reply */ - have_user_pass = (memchr(data+2, SOCKS_USER_PASS, nummethods) !=NULL); - have_no_auth = (memchr(data+2, SOCKS_NO_AUTH, nummethods) !=NULL); - if (have_user_pass && !(have_no_auth && req->socks_prefer_no_auth)) { - req->auth_type = SOCKS_USER_PASS; - req->reply[1] = SOCKS_USER_PASS; /* tell client to use "user/pass" - auth method */ - req->socks_version = 5; /* remember we've already negotiated auth */ - log_debug(LD_APP,"socks5: accepted method 2 (username/password)"); - r=0; - } else if (have_no_auth) { - req->reply[1] = SOCKS_NO_AUTH; /* tell client to use "none" auth - method */ - req->socks_version = 5; /* remember we've already negotiated auth */ - log_debug(LD_APP,"socks5: accepted method 0 (no authentication)"); - r=0; - } else { - log_warn(LD_APP, - "socks5: offered methods don't include 'no auth' or " - "username/password. Rejecting."); - req->reply[1] = '\xFF'; /* reject all methods */ - r=-1; - } - /* Remove packet from buf. Some SOCKS clients will have sent extra - * junk at this point; let's hope it's an authentication message. */ - *drain_out = 2u + nummethods; - - return r; - } - if (req->auth_type != SOCKS_NO_AUTH && !req->got_auth) { - log_warn(LD_APP, - "socks5: negotiated authentication, but none provided"); - return -1; - } - /* we know the method; read in the request */ - log_debug(LD_APP,"socks5: checking request"); - if (datalen < 7) {/* basic info plus >=1 for addr plus 2 for port */ - *want_length_out = 7; - return 0; /* not yet */ - } - req->command = (unsigned char) *(data+1); - if (req->command != SOCKS_COMMAND_CONNECT && - req->command != SOCKS_COMMAND_RESOLVE && - req->command != SOCKS_COMMAND_RESOLVE_PTR) { - /* not a connect or resolve or a resolve_ptr? we don't support it. */ - socks_request_set_socks5_error(req,SOCKS5_COMMAND_NOT_SUPPORTED); - - log_warn(LD_APP,"socks5: command %d not recognized. Rejecting.", - req->command); - return -1; - } - switch (*(data+3)) { /* address type */ - case 1: /* IPv4 address */ - case 4: /* IPv6 address */ { - const int is_v6 = *(data+3) == 4; - const unsigned addrlen = is_v6 ? 16 : 4; - log_debug(LD_APP,"socks5: ipv4 address type"); - if (datalen < 6+addrlen) {/* ip/port there? */ - *want_length_out = 6+addrlen; - return 0; /* not yet */ - } - - if (is_v6) - tor_addr_from_ipv6_bytes(&destaddr, data+4); - else - tor_addr_from_ipv4n(&destaddr, get_uint32(data+4)); - - tor_addr_to_str(tmpbuf, &destaddr, sizeof(tmpbuf), 1); - - if (strlen(tmpbuf)+1 > MAX_SOCKS_ADDR_LEN) { - socks_request_set_socks5_error(req, SOCKS5_GENERAL_ERROR); - log_warn(LD_APP, - "socks5 IP takes %d bytes, which doesn't fit in %d. " - "Rejecting.", - (int)strlen(tmpbuf)+1,(int)MAX_SOCKS_ADDR_LEN); - return -1; - } - strlcpy(req->address,tmpbuf,sizeof(req->address)); - req->port = ntohs(get_uint16(data+4+addrlen)); - *drain_out = 6+addrlen; - if (req->command != SOCKS_COMMAND_RESOLVE_PTR && - !addressmap_have_mapping(req->address,0)) { - log_unsafe_socks_warning(5, req->address, req->port, safe_socks); - if (safe_socks) { - socks_request_set_socks5_error(req, SOCKS5_NOT_ALLOWED); - return -1; - } - } - return 1; - } - case 3: /* fqdn */ - log_debug(LD_APP,"socks5: fqdn address type"); - if (req->command == SOCKS_COMMAND_RESOLVE_PTR) { - socks_request_set_socks5_error(req, - SOCKS5_ADDRESS_TYPE_NOT_SUPPORTED); - log_warn(LD_APP, "socks5 received RESOLVE_PTR command with " - "hostname type. Rejecting."); - return -1; - } - len = (unsigned char)*(data+4); - if (datalen < 7+len) { /* addr/port there? */ - *want_length_out = 7+len; - return 0; /* not yet */ - } - if (len+1 > MAX_SOCKS_ADDR_LEN) { - socks_request_set_socks5_error(req, SOCKS5_GENERAL_ERROR); - log_warn(LD_APP, - "socks5 hostname is %d bytes, which doesn't fit in " - "%d. Rejecting.", len+1,MAX_SOCKS_ADDR_LEN); - return -1; - } - memcpy(req->address,data+5,len); - req->address[len] = 0; - req->port = ntohs(get_uint16(data+5+len)); - *drain_out = 5+len+2; - - if (!string_is_valid_hostname(req->address)) { - socks_request_set_socks5_error(req, SOCKS5_GENERAL_ERROR); - - log_warn(LD_PROTOCOL, - "Your application (using socks5 to port %d) gave Tor " - "a malformed hostname: %s. Rejecting the connection.", - req->port, escaped_safe_str_client(req->address)); - return -1; - } - if (log_sockstype) - log_notice(LD_APP, - "Your application (using socks5 to port %d) instructed " - "Tor to take care of the DNS resolution itself if " - "necessary. This is good.", req->port); - return 1; - default: /* unsupported */ - socks_request_set_socks5_error(req, - SOCKS5_ADDRESS_TYPE_NOT_SUPPORTED); - log_warn(LD_APP,"socks5: unsupported address type %d. Rejecting.", - (int) *(data+3)); - return -1; - } - tor_assert(0); - break; - case 4: { /* socks4 */ - enum {socks4, socks4a} socks4_prot = socks4a; - const char *authstart, *authend; - /* http://ss5.sourceforge.net/socks4.protocol.txt */ - /* http://ss5.sourceforge.net/socks4A.protocol.txt */ - - req->socks_version = 4; - if (datalen < SOCKS4_NETWORK_LEN) {/* basic info available? */ - *want_length_out = SOCKS4_NETWORK_LEN; - return 0; /* not yet */ - } - // buf_pullup(buf, 1280); - req->command = (unsigned char) *(data+1); - if (req->command != SOCKS_COMMAND_CONNECT && - req->command != SOCKS_COMMAND_RESOLVE) { - /* not a connect or resolve? we don't support it. (No resolve_ptr with - * socks4.) */ - log_warn(LD_APP,"socks4: command %d not recognized. Rejecting.", - req->command); - return -1; - } - - req->port = ntohs(get_uint16(data+2)); - destip = ntohl(get_uint32(data+4)); - if ((!req->port && req->command!=SOCKS_COMMAND_RESOLVE) || !destip) { - log_warn(LD_APP,"socks4: Port or DestIP is zero. Rejecting."); - return -1; - } - if (destip >> 8) { - log_debug(LD_APP,"socks4: destip not in form 0.0.0.x."); - in.s_addr = htonl(destip); - tor_inet_ntoa(&in,tmpbuf,sizeof(tmpbuf)); - if (strlen(tmpbuf)+1 > MAX_SOCKS_ADDR_LEN) { - log_debug(LD_APP,"socks4 addr (%d bytes) too long. Rejecting.", - (int)strlen(tmpbuf)); - return -1; - } - log_debug(LD_APP, - "socks4: successfully read destip (%s)", - safe_str_client(tmpbuf)); - socks4_prot = socks4; - } - - authstart = data + SOCKS4_NETWORK_LEN; - next = memchr(authstart, 0, - datalen-SOCKS4_NETWORK_LEN); - if (!next) { - if (datalen >= 1024) { - log_debug(LD_APP, "Socks4 user name too long; rejecting."); - return -1; - } - log_debug(LD_APP,"socks4: Username not here yet."); - *want_length_out = datalen+1024; /* More than we need, but safe */ - return 0; - } - authend = next; - tor_assert(next < data+datalen); - - startaddr = NULL; - if (socks4_prot != socks4a && - !addressmap_have_mapping(tmpbuf,0)) { - log_unsafe_socks_warning(4, tmpbuf, req->port, safe_socks); - - if (safe_socks) - return -1; - } - if (socks4_prot == socks4a) { - if (next+1 == data+datalen) { - log_debug(LD_APP,"socks4: No part of destaddr here yet."); - *want_length_out = datalen + 1024; /* More than we need, but safe */ - return 0; - } - startaddr = next+1; - next = memchr(startaddr, 0, data + datalen - startaddr); - if (!next) { - if (datalen >= 1024) { - log_debug(LD_APP,"socks4: Destaddr too long."); - return -1; - } - log_debug(LD_APP,"socks4: Destaddr not all here yet."); - *want_length_out = datalen + 1024; /* More than we need, but safe */ - return 0; - } - if (MAX_SOCKS_ADDR_LEN <= next-startaddr) { - log_warn(LD_APP,"socks4: Destaddr too long. Rejecting."); - return -1; - } - // tor_assert(next < buf->cur+buf->datalen); - - if (log_sockstype) - log_notice(LD_APP, - "Your application (using socks4a to port %d) instructed " - "Tor to take care of the DNS resolution itself if " - "necessary. This is good.", req->port); - } - log_debug(LD_APP,"socks4: Everything is here. Success."); - strlcpy(req->address, startaddr ? startaddr : tmpbuf, - sizeof(req->address)); - if (!string_is_valid_hostname(req->address)) { - log_warn(LD_PROTOCOL, - "Your application (using socks4 to port %d) gave Tor " - "a malformed hostname: %s. Rejecting the connection.", - req->port, escaped_safe_str_client(req->address)); - return -1; - } - if (authend != authstart) { - req->got_auth = 1; - req->usernamelen = authend - authstart; - req->username = tor_memdup(authstart, authend - authstart); - } - /* next points to the final \0 on inbuf */ - *drain_out = next - data + 1; - return 1; - } - case 'G': /* get */ - case 'H': /* head */ - case 'P': /* put/post */ - case 'C': /* connect */ - strlcpy((char*)req->reply, SOCKS_PROXY_IS_NOT_AN_HTTP_PROXY_MSG, - MAX_SOCKS_REPLY_LEN); - req->replylen = strlen((char*)req->reply)+1; - /* fall through */ - default: /* version is not socks4 or socks5 */ - log_warn(LD_APP, - "Socks version %d not recognized. (Tor is not an http proxy.)", - *(data)); - { - /* Tell the controller the first 8 bytes. */ - char *tmp = tor_strndup(data, datalen < 8 ? datalen : 8); - control_event_client_status(LOG_WARN, - "SOCKS_UNKNOWN_PROTOCOL DATA=\"%s\"", - escaped(tmp)); - tor_free(tmp); - } - return -1; - } -} - -/** Inspect a reply from SOCKS server stored in <b>buf</b> according - * to <b>state</b>, removing the protocol data upon success. Return 0 on - * incomplete response, 1 on success and -1 on error, in which case - * <b>reason</b> is set to a descriptive message (free() when finished - * with it). - * - * As a special case, 2 is returned when user/pass is required - * during SOCKS5 handshake and user/pass is configured. - */ -int -fetch_from_buf_socks_client(buf_t *buf, int state, char **reason) -{ - ssize_t drain = 0; - int r; - if (buf->datalen < 2) - return 0; - - buf_pullup(buf, MAX_SOCKS_MESSAGE_LEN); - tor_assert(buf->head && buf->head->datalen >= 2); - - r = parse_socks_client((uint8_t*)buf->head->data, buf->head->datalen, - state, reason, &drain); - if (drain > 0) - buf_remove_from_front(buf, drain); - else if (drain < 0) - buf_clear(buf); - - return r; -} - -/** Implementation logic for fetch_from_*_socks_client. */ -static int -parse_socks_client(const uint8_t *data, size_t datalen, - int state, char **reason, - ssize_t *drain_out) -{ - unsigned int addrlen; - *drain_out = 0; - if (datalen < 2) - return 0; - - switch (state) { - case PROXY_SOCKS4_WANT_CONNECT_OK: - /* Wait for the complete response */ - if (datalen < 8) - return 0; - - if (data[1] != 0x5a) { - *reason = tor_strdup(socks4_response_code_to_string(data[1])); - return -1; - } - - /* Success */ - *drain_out = 8; - return 1; - - case PROXY_SOCKS5_WANT_AUTH_METHOD_NONE: - /* we don't have any credentials */ - if (data[1] != 0x00) { - *reason = tor_strdup("server doesn't support any of our " - "available authentication methods"); - return -1; - } - - log_info(LD_NET, "SOCKS 5 client: continuing without authentication"); - *drain_out = -1; - return 1; - - case PROXY_SOCKS5_WANT_AUTH_METHOD_RFC1929: - /* we have a username and password. return 1 if we can proceed without - * providing authentication, or 2 otherwise. */ - switch (data[1]) { - case 0x00: - log_info(LD_NET, "SOCKS 5 client: we have auth details but server " - "doesn't require authentication."); - *drain_out = -1; - return 1; - case 0x02: - log_info(LD_NET, "SOCKS 5 client: need authentication."); - *drain_out = -1; - return 2; - /* fall through */ - } - - *reason = tor_strdup("server doesn't support any of our available " - "authentication methods"); - return -1; - - case PROXY_SOCKS5_WANT_AUTH_RFC1929_OK: - /* handle server reply to rfc1929 authentication */ - if (data[1] != 0x00) { - *reason = tor_strdup("authentication failed"); - return -1; - } - - log_info(LD_NET, "SOCKS 5 client: authentication successful."); - *drain_out = -1; - return 1; - - case PROXY_SOCKS5_WANT_CONNECT_OK: - /* response is variable length. BND.ADDR, etc, isn't needed - * (don't bother with buf_pullup()), but make sure to eat all - * the data used */ - - /* wait for address type field to arrive */ - if (datalen < 4) - return 0; - - switch (data[3]) { - case 0x01: /* ip4 */ - addrlen = 4; - break; - case 0x04: /* ip6 */ - addrlen = 16; - break; - case 0x03: /* fqdn (can this happen here?) */ - if (datalen < 5) - return 0; - addrlen = 1 + data[4]; - break; - default: - *reason = tor_strdup("invalid response to connect request"); - return -1; - } - - /* wait for address and port */ - if (datalen < 6 + addrlen) - return 0; - - if (data[1] != 0x00) { - *reason = tor_strdup(socks5_response_code_to_string(data[1])); - return -1; - } - - *drain_out = 6 + addrlen; - return 1; - } - - /* shouldn't get here... */ - tor_assert(0); - - return -1; -} - -/** Return true if <b>cmd</b> looks like a HTTP (proxy) request. */ -int -peek_buf_has_http_command(const buf_t *buf) -{ - if (peek_buf_startswith(buf, "CONNECT ") || - peek_buf_startswith(buf, "DELETE ") || - peek_buf_startswith(buf, "GET ") || - peek_buf_startswith(buf, "POST ") || - peek_buf_startswith(buf, "PUT " )) - return 1; - return 0; -} - -/** Return 1 iff <b>buf</b> starts with <b>cmd</b>. <b>cmd</b> must be a null - * terminated string, of no more than PEEK_BUF_STARTSWITH_MAX bytes. */ -int -peek_buf_startswith(const buf_t *buf, const char *cmd) -{ - char tmp[PEEK_BUF_STARTSWITH_MAX]; - size_t clen = strlen(cmd); - if (BUG(clen > sizeof(tmp))) - return 0; - if (buf->datalen < clen) - return 0; - peek_from_buf(tmp, clen, buf); - return fast_memeq(tmp, cmd, clen); -} - -/** Return 1 iff buf looks more like it has an (obsolete) v0 controller - * command on it than any valid v1 controller command. */ -int -peek_buf_has_control0_command(buf_t *buf) -{ - if (buf->datalen >= 4) { - char header[4]; - uint16_t cmd; - peek_from_buf(header, sizeof(header), buf); - cmd = ntohs(get_uint16(header+2)); - if (cmd <= 0x14) - return 1; /* This is definitely not a v1 control command. */ - } - return 0; -} - -/** Return the index within <b>buf</b> at which <b>ch</b> first appears, - * or -1 if <b>ch</b> does not appear on buf. */ -static off_t -buf_find_offset_of_char(buf_t *buf, char ch) -{ - chunk_t *chunk; - off_t offset = 0; - for (chunk = buf->head; chunk; chunk = chunk->next) { - char *cp = memchr(chunk->data, ch, chunk->datalen); - if (cp) - return offset + (cp - chunk->data); - else - offset += chunk->datalen; - } - return -1; -} - -/** Try to read a single LF-terminated line from <b>buf</b>, and write it - * (including the LF), NUL-terminated, into the *<b>data_len</b> byte buffer - * at <b>data_out</b>. Set *<b>data_len</b> to the number of bytes in the - * line, not counting the terminating NUL. Return 1 if we read a whole line, - * return 0 if we don't have a whole line yet, and return -1 if the line - * length exceeds *<b>data_len</b>. - */ -int -fetch_from_buf_line(buf_t *buf, char *data_out, size_t *data_len) -{ - size_t sz; - off_t offset; - - if (!buf->head) - return 0; - - offset = buf_find_offset_of_char(buf, '\n'); - if (offset < 0) - return 0; - sz = (size_t) offset; - if (sz+2 > *data_len) { - *data_len = sz + 2; - return -1; - } - fetch_from_buf(data_out, sz+1, buf); - data_out[sz+1] = '\0'; - *data_len = sz+1; - return 1; -} - -/** Compress on uncompress the <b>data_len</b> bytes in <b>data</b> using the - * compression state <b>state</b>, appending the result to <b>buf</b>. If - * <b>done</b> is true, flush the data in the state and finish the - * compression/uncompression. Return -1 on failure, 0 on success. */ -int -write_to_buf_compress(buf_t *buf, tor_compress_state_t *state, - const char *data, size_t data_len, - const int done) -{ - char *next; - size_t old_avail, avail; - int over = 0; - - do { - int need_new_chunk = 0; - if (!buf->tail || ! CHUNK_REMAINING_CAPACITY(buf->tail)) { - size_t cap = data_len / 4; - buf_add_chunk_with_capacity(buf, cap, 1); - } - next = CHUNK_WRITE_PTR(buf->tail); - avail = old_avail = CHUNK_REMAINING_CAPACITY(buf->tail); - switch (tor_compress_process(state, &next, &avail, - &data, &data_len, done)) { - case TOR_COMPRESS_DONE: - over = 1; - break; - case TOR_COMPRESS_ERROR: - return -1; - case TOR_COMPRESS_OK: - if (data_len == 0) { - tor_assert_nonfatal(!done); - over = 1; - } - break; - case TOR_COMPRESS_BUFFER_FULL: - if (avail) { - /* The compression module says we need more room - * (TOR_COMPRESS_BUFFER_FULL). Start a new chunk automatically, - * whether were going to or not. */ - need_new_chunk = 1; - } - if (data_len == 0 && !done) { - /* We've consumed all the input data, though, so there's no - * point in forging ahead right now. */ - over = 1; - } - break; - } - buf->datalen += old_avail - avail; - buf->tail->datalen += old_avail - avail; - if (need_new_chunk) { - buf_add_chunk_with_capacity(buf, data_len/4, 1); - } - - } while (!over); - check(); - return 0; -} - -/** Set *<b>output</b> to contain a copy of the data in *<b>input</b> */ -int -buf_set_to_copy(buf_t **output, - const buf_t *input) -{ - if (*output) - buf_free(*output); - *output = buf_copy(input); - return 0; -} - -/** Log an error and exit if <b>buf</b> is corrupted. - */ -void -assert_buf_ok(buf_t *buf) -{ - tor_assert(buf); - tor_assert(buf->magic == BUFFER_MAGIC); - - if (! buf->head) { - tor_assert(!buf->tail); - tor_assert(buf->datalen == 0); - } else { - chunk_t *ch; - size_t total = 0; - tor_assert(buf->tail); - for (ch = buf->head; ch; ch = ch->next) { - total += ch->datalen; - tor_assert(ch->datalen <= ch->memlen); - tor_assert(ch->data >= &ch->mem[0]); - tor_assert(ch->data <= &ch->mem[0]+ch->memlen); - if (ch->data == &ch->mem[0]+ch->memlen) { - static int warned = 0; - if (! warned) { - log_warn(LD_BUG, "Invariant violation in buf.c related to #15083"); - warned = 1; - } - } - tor_assert(ch->data+ch->datalen <= &ch->mem[0] + ch->memlen); - if (!ch->next) - tor_assert(ch == buf->tail); - } - tor_assert(buf->datalen == total); - } -} - diff --git a/src/or/channeltls.c b/src/or/channeltls.c index 6547451181..4ccd3b5fbf 100644 --- a/src/or/channeltls.c +++ b/src/or/channeltls.c @@ -847,7 +847,7 @@ channel_tls_write_packed_cell_method(channel_t *chan, tor_assert(packed_cell); if (tlschan->conn) { - connection_write_to_buf(packed_cell->body, cell_network_size, + connection_buf_add(packed_cell->body, cell_network_size, TO_CONN(tlschan->conn)); /* This is where the cell is finished; used to be done from relay.c */ diff --git a/src/or/config.c b/src/or/config.c index 30853724e4..eb89d6f5ee 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -372,6 +372,7 @@ static config_var_t option_vars_[] = { V(HTTPProxyAuthenticator, STRING, NULL), V(HTTPSProxy, STRING, NULL), V(HTTPSProxyAuthenticator, STRING, NULL), + VPORT(HTTPTunnelPort), V(IPv6Exit, BOOL, "0"), VAR("ServerTransportPlugin", LINELIST, ServerTransportPlugin, NULL), V(ServerTransportListenAddr, LINELIST, NULL), @@ -2915,7 +2916,8 @@ options_validate_single_onion(or_options_t *options, char **msg) const int client_port_set = (options->SocksPort_set || options->TransPort_set || options->NATDPort_set || - options->DNSPort_set); + options->DNSPort_set || + options->HTTPTunnelPort_set); if (rend_service_non_anonymous_mode_enabled(options) && client_port_set && !options->Tor2webMode) { REJECT("HiddenServiceNonAnonymousMode is incompatible with using Tor as " @@ -7000,6 +7002,15 @@ parse_ports(or_options_t *options, int validate_only, *msg = tor_strdup("Invalid NatdPort configuration"); goto err; } + if (parse_port_config(ports, + options->HTTPTunnelPort_lines, + "HTTP Tunnel", CONN_TYPE_AP_HTTP_CONNECT_LISTENER, + "127.0.0.1", 0, + ((validate_only ? 0 : CL_PORT_WARN_NONLOCAL) + | CL_PORT_TAKES_HOSTNAMES | gw_flag)) < 0) { + *msg = tor_strdup("Invalid HTTPTunnelPort configuration"); + goto err; + } { unsigned control_port_flags = CL_PORT_NO_STREAM_OPTIONS | CL_PORT_WARN_NONLOCAL; @@ -7077,6 +7088,8 @@ parse_ports(or_options_t *options, int validate_only, !! count_real_listeners(ports, CONN_TYPE_AP_TRANS_LISTENER, 1); options->NATDPort_set = !! count_real_listeners(ports, CONN_TYPE_AP_NATD_LISTENER, 1); + options->HTTPTunnelPort_set = + !! count_real_listeners(ports, CONN_TYPE_AP_HTTP_CONNECT_LISTENER, 1); /* Use options->ControlSocket to test if a control socket is set */ options->ControlPort_set = !! count_real_listeners(ports, CONN_TYPE_CONTROL_LISTENER, 0); @@ -7921,13 +7934,28 @@ parse_outbound_addresses(or_options_t *options, int validate_only, char **msg) memset(&options->OutboundBindAddresses, 0, sizeof(options->OutboundBindAddresses)); } - parse_outbound_address_lines(options->OutboundBindAddress, - OUTBOUND_ADDR_EXIT_AND_OR, options, validate_only, msg); - parse_outbound_address_lines(options->OutboundBindAddressOR, - OUTBOUND_ADDR_OR, options, validate_only, msg); - parse_outbound_address_lines(options->OutboundBindAddressExit, - OUTBOUND_ADDR_EXIT, options, validate_only, msg); + + if (parse_outbound_address_lines(options->OutboundBindAddress, + OUTBOUND_ADDR_EXIT_AND_OR, options, + validate_only, msg) < 0) { + goto err; + } + + if (parse_outbound_address_lines(options->OutboundBindAddressOR, + OUTBOUND_ADDR_OR, options, validate_only, + msg) < 0) { + goto err; + } + + if (parse_outbound_address_lines(options->OutboundBindAddressExit, + OUTBOUND_ADDR_EXIT, options, validate_only, + msg) < 0) { + goto err; + } + return 0; + err: + return -1; } /** Load one of the geoip files, <a>family</a> determining which diff --git a/src/or/connection.c b/src/or/connection.c index 31a682387d..e2392b52e3 100644 --- a/src/or/connection.c +++ b/src/or/connection.c @@ -37,7 +37,7 @@ * they call connection_stop_reading() or connection_stop_writing(). * * To queue data to be written on a connection, call - * connection_write_to_buf(). When data arrives, the + * connection_buf_add(). When data arrives, the * connection_process_inbuf() callback is invoked, which dispatches to a * type-specific function (such as connection_edge_process_inbuf() for * example). Connection types that need notice of when data has been written @@ -58,6 +58,7 @@ #include "or.h" #include "bridges.h" #include "buffers.h" +#include "buffers_tls.h" /* * Define this so we get channel internal functions, since we're implementing * part of a subclass (channel_tls_t). @@ -86,6 +87,8 @@ #include "hs_common.h" #include "hs_ident.h" #include "nodelist.h" +#include "proto_http.h" +#include "proto_socks.h" #include "policies.h" #include "reasons.h" #include "relay.h" @@ -124,8 +127,9 @@ static int connection_finished_flushing(connection_t *conn); static int connection_flushed_some(connection_t *conn); static int connection_finished_connecting(connection_t *conn); static int connection_reached_eof(connection_t *conn); -static int connection_read_to_buf(connection_t *conn, ssize_t *max_to_read, - int *socket_error); +static int connection_buf_read_from_socket(connection_t *conn, + ssize_t *max_to_read, + int *socket_error); static int connection_process_inbuf(connection_t *conn, int package_partial); static void client_check_address_changed(tor_socket_t sock); static void set_constrained_socket_buffers(tor_socket_t sock, int size); @@ -158,7 +162,8 @@ static smartlist_t *outgoing_addrs = NULL; case CONN_TYPE_CONTROL_LISTENER: \ case CONN_TYPE_AP_TRANS_LISTENER: \ case CONN_TYPE_AP_NATD_LISTENER: \ - case CONN_TYPE_AP_DNS_LISTENER + case CONN_TYPE_AP_DNS_LISTENER: \ + case CONN_TYPE_AP_HTTP_CONNECT_LISTENER /**************************************************************/ @@ -185,6 +190,7 @@ conn_type_to_string(int type) case CONN_TYPE_CONTROL: return "Control"; case CONN_TYPE_EXT_OR: return "Extended OR"; case CONN_TYPE_EXT_OR_LISTENER: return "Extended OR listener"; + case CONN_TYPE_AP_HTTP_CONNECT_LISTENER: return "HTTP tunnel listener"; default: log_warn(LD_BUG, "unknown connection type %d", type); tor_snprintf(buf, sizeof(buf), "unknown [%d]", type); @@ -1702,6 +1708,8 @@ connection_init_accepted_conn(connection_t *conn, TO_ENTRY_CONN(conn)->is_transparent_ap = 1; conn->state = AP_CONN_STATE_NATD_WAIT; break; + case CONN_TYPE_AP_HTTP_CONNECT_LISTENER: + conn->state = AP_CONN_STATE_HTTP_CONNECT_WAIT; } break; case CONN_TYPE_DIR: @@ -2140,7 +2148,7 @@ connection_proxy_connect(connection_t *conn, int type) fmt_addrport(&conn->addr, conn->port)); } - connection_write_to_buf(buf, strlen(buf), conn); + connection_buf_add(buf, strlen(buf), conn); conn->proxy_state = PROXY_HTTPS_WANT_CONNECT_OK; break; } @@ -2206,7 +2214,7 @@ connection_proxy_connect(connection_t *conn, int type) buf[8] = 0; /* no userid */ } - connection_write_to_buf((char *)buf, buf_size, conn); + connection_buf_add((char *)buf, buf_size, conn); tor_free(buf); conn->proxy_state = PROXY_SOCKS4_WANT_CONNECT_OK; @@ -2237,7 +2245,7 @@ connection_proxy_connect(connection_t *conn, int type) conn->proxy_state = PROXY_SOCKS5_WANT_AUTH_METHOD_NONE; } - connection_write_to_buf((char *)buf, 2 + buf[1], conn); + connection_buf_add((char *)buf, 2 + buf[1], conn); break; } @@ -2343,7 +2351,7 @@ connection_send_socks5_connect(connection_t *conn) memcpy(buf + 20, &port, 2); } - connection_write_to_buf((char *)buf, reqsize, conn); + connection_buf_add((char *)buf, reqsize, conn); conn->proxy_state = PROXY_SOCKS5_WANT_CONNECT_OK; } @@ -2469,7 +2477,7 @@ connection_read_proxy_handshake(connection_t *conn) if (socks_args_string) tor_free(socks_args_string); - connection_write_to_buf((char *)buf, reqsize, conn); + connection_buf_add((char *)buf, reqsize, conn); conn->proxy_state = PROXY_SOCKS5_WANT_AUTH_RFC1929_OK; ret = 0; @@ -3368,7 +3376,7 @@ connection_bucket_should_increase(int bucket, or_connection_t *conn) /** Read bytes from conn-\>s and process them. * - * It calls connection_read_to_buf() to bring in any new bytes, + * It calls connection_buf_read_from_socket() to bring in any new bytes, * and then calls connection_process_inbuf() to process them. * * Mark the connection and return -1 if you want to close it, else @@ -3394,6 +3402,7 @@ connection_handle_read_impl(connection_t *conn) case CONN_TYPE_AP_LISTENER: case CONN_TYPE_AP_TRANS_LISTENER: case CONN_TYPE_AP_NATD_LISTENER: + case CONN_TYPE_AP_HTTP_CONNECT_LISTENER: return connection_handle_listener_read(conn, CONN_TYPE_AP); case CONN_TYPE_DIR_LISTENER: return connection_handle_listener_read(conn, CONN_TYPE_DIR); @@ -3410,7 +3419,7 @@ connection_handle_read_impl(connection_t *conn) tor_assert(!conn->marked_for_close); before = buf_datalen(conn->inbuf); - if (connection_read_to_buf(conn, &max_to_read, &socket_error) < 0) { + if (connection_buf_read_from_socket(conn, &max_to_read, &socket_error) < 0) { /* There's a read error; kill the connection.*/ if (conn->type == CONN_TYPE_OR) { connection_or_notify_error(TO_OR_CONN(conn), @@ -3507,7 +3516,7 @@ connection_handle_read(connection_t *conn) * Return -1 if we want to break conn, else return 0. */ static int -connection_read_to_buf(connection_t *conn, ssize_t *max_to_read, +connection_buf_read_from_socket(connection_t *conn, ssize_t *max_to_read, int *socket_error) { int result; @@ -3548,7 +3557,7 @@ connection_read_to_buf(connection_t *conn, ssize_t *max_to_read, initial_size = buf_datalen(conn->inbuf); /* else open, or closing */ - result = read_to_buf_tls(or_conn->tls, at_most, conn->inbuf); + result = buf_read_from_tls(conn->inbuf, or_conn->tls, at_most); if (TOR_TLS_IS_ERROR(result) || result == TOR_TLS_CLOSE) or_conn->tls_error = result; else @@ -3597,7 +3606,7 @@ connection_read_to_buf(connection_t *conn, ssize_t *max_to_read, /* If we have any pending bytes, we read them now. This *can* * take us over our read allotment, but really we shouldn't be * believing that SSL bytes are the same as TCP bytes anyway. */ - int r2 = read_to_buf_tls(or_conn->tls, pending, conn->inbuf); + int r2 = buf_read_from_tls(conn->inbuf, or_conn->tls, pending); if (BUG(r2<0)) { log_warn(LD_BUG, "apparently, reading pending bytes can fail."); return -1; @@ -3609,7 +3618,7 @@ connection_read_to_buf(connection_t *conn, ssize_t *max_to_read, result, (long)n_read, (long)n_written); } else if (conn->linked) { if (conn->linked_conn) { - result = move_buf_to_buf(conn->inbuf, conn->linked_conn->outbuf, + result = buf_move_to_buf(conn->inbuf, conn->linked_conn->outbuf, &conn->linked_conn->outbuf_flushlen); } else { result = 0; @@ -3627,8 +3636,10 @@ connection_read_to_buf(connection_t *conn, ssize_t *max_to_read, /* !connection_speaks_cells, !conn->linked_conn. */ int reached_eof = 0; CONN_LOG_PROTECT(conn, - result = read_to_buf(conn->s, at_most, conn->inbuf, &reached_eof, - socket_error)); + result = buf_read_from_socket(conn->inbuf, conn->s, + at_most, + &reached_eof, + socket_error)); if (reached_eof) conn->inbuf_reached_eof = 1; @@ -3697,17 +3708,17 @@ connection_read_to_buf(connection_t *conn, ssize_t *max_to_read, /** A pass-through to fetch_from_buf. */ int -connection_fetch_from_buf(char *string, size_t len, connection_t *conn) +connection_buf_get_bytes(char *string, size_t len, connection_t *conn) { - return fetch_from_buf(string, len, conn->inbuf); + return buf_get_bytes(conn->inbuf, string, len); } -/** As fetch_from_buf_line(), but read from a connection's input buffer. */ +/** As buf_get_line(), but read from a connection's input buffer. */ int -connection_fetch_from_buf_line(connection_t *conn, char *data, +connection_buf_get_line(connection_t *conn, char *data, size_t *data_len) { - return fetch_from_buf_line(conn->inbuf, data, data_len); + return buf_get_line(conn->inbuf, data, data_len); } /** As fetch_from_buf_http, but fetches from a connection's input buffer_t as @@ -3744,7 +3755,7 @@ connection_outbuf_too_full(connection_t *conn) * * This function gets called either from conn_write_callback() in main.c * when libevent tells us that conn wants to write, or below - * from connection_write_to_buf() when an entire TLS record is ready. + * from connection_buf_add() when an entire TLS record is ready. * * Update <b>conn</b>-\>timestamp_lastwritten to now, and call flush_buf * or flush_buf_tls appropriately. If it succeeds and there are no more @@ -3855,7 +3866,7 @@ connection_handle_write_impl(connection_t *conn, int force) /* else open, or closing */ initial_size = buf_datalen(conn->outbuf); - result = flush_buf_tls(or_conn->tls, conn->outbuf, + result = buf_flush_to_tls(conn->outbuf, or_conn->tls, max_to_write, &conn->outbuf_flushlen); /* If we just flushed the last bytes, tell the channel on the @@ -3918,8 +3929,8 @@ connection_handle_write_impl(connection_t *conn, int force) result = (int)(initial_size-buf_datalen(conn->outbuf)); } else { CONN_LOG_PROTECT(conn, - result = flush_buf(conn->s, conn->outbuf, - max_to_write, &conn->outbuf_flushlen)); + result = buf_flush_to_socket(conn->outbuf, conn->s, + max_to_write, &conn->outbuf_flushlen)); if (result < 0) { if (CONN_IS_EDGE(conn)) connection_edge_end_errno(TO_EDGE_CONN(conn)); @@ -4059,11 +4070,11 @@ connection_write_to_buf_impl_,(const char *string, size_t len, if (zlib) { dir_connection_t *dir_conn = TO_DIR_CONN(conn); int done = zlib < 0; - CONN_LOG_PROTECT(conn, r = write_to_buf_compress(conn->outbuf, + CONN_LOG_PROTECT(conn, r = buf_add_compress(conn->outbuf, dir_conn->compress_state, string, len, done)); } else { - CONN_LOG_PROTECT(conn, r = write_to_buf(string, len, conn->outbuf)); + CONN_LOG_PROTECT(conn, r = buf_add(conn->outbuf, string, len)); } if (r < 0) { if (CONN_IS_EDGE(conn)) { @@ -4307,6 +4318,7 @@ connection_is_listener(connection_t *conn) conn->type == CONN_TYPE_AP_TRANS_LISTENER || conn->type == CONN_TYPE_AP_DNS_LISTENER || conn->type == CONN_TYPE_AP_NATD_LISTENER || + conn->type == CONN_TYPE_AP_HTTP_CONNECT_LISTENER || conn->type == CONN_TYPE_DIR_LISTENER || conn->type == CONN_TYPE_CONTROL_LISTENER) return 1; @@ -4974,9 +4986,9 @@ assert_connection_ok(connection_t *conn, time_t now) /* buffers */ if (conn->inbuf) - assert_buf_ok(conn->inbuf); + buf_assert_ok(conn->inbuf); if (conn->outbuf) - assert_buf_ok(conn->outbuf); + buf_assert_ok(conn->outbuf); if (conn->type == CONN_TYPE_OR) { or_connection_t *or_conn = TO_OR_CONN(conn); diff --git a/src/or/connection.h b/src/or/connection.h index 0bcf0ccdce..57b9a4a1df 100644 --- a/src/or/connection.h +++ b/src/or/connection.h @@ -123,8 +123,8 @@ void connection_bucket_refill(int seconds_elapsed, time_t now); int connection_handle_read(connection_t *conn); -int connection_fetch_from_buf(char *string, size_t len, connection_t *conn); -int connection_fetch_from_buf_line(connection_t *conn, char *data, +int connection_buf_get_bytes(char *string, size_t len, connection_t *conn); +int connection_buf_get_line(connection_t *conn, char *data, size_t *data_len); int connection_fetch_from_buf_http(connection_t *conn, char **headers_out, size_t max_headerlen, @@ -139,18 +139,18 @@ int connection_flush(connection_t *conn); MOCK_DECL(void, connection_write_to_buf_impl_, (const char *string, size_t len, connection_t *conn, int zlib)); /* DOCDOC connection_write_to_buf */ -static void connection_write_to_buf(const char *string, size_t len, +static void connection_buf_add(const char *string, size_t len, connection_t *conn); /* DOCDOC connection_write_to_buf_compress */ -static void connection_write_to_buf_compress(const char *string, size_t len, +static void connection_buf_add_compress(const char *string, size_t len, dir_connection_t *conn, int done); static inline void -connection_write_to_buf(const char *string, size_t len, connection_t *conn) +connection_buf_add(const char *string, size_t len, connection_t *conn) { connection_write_to_buf_impl_(string, len, conn, 0); } static inline void -connection_write_to_buf_compress(const char *string, size_t len, +connection_buf_add_compress(const char *string, size_t len, dir_connection_t *conn, int done) { connection_write_to_buf_impl_(string, len, TO_CONN(conn), done ? -1 : 1); diff --git a/src/or/connection_edge.c b/src/or/connection_edge.c index a98b32450b..a9fdeee0ee 100644 --- a/src/or/connection_edge.c +++ b/src/or/connection_edge.c @@ -82,6 +82,8 @@ #include "main.h" #include "nodelist.h" #include "policies.h" +#include "proto_http.h" +#include "proto_socks.h" #include "reasons.h" #include "relay.h" #include "rendclient.h" @@ -241,6 +243,11 @@ connection_edge_process_inbuf(edge_connection_t *conn, int package_partial) return -1; } return 0; + case AP_CONN_STATE_HTTP_CONNECT_WAIT: + if (connection_ap_process_http_connect(EDGE_TO_ENTRY_CONN(conn)) < 0) { + return -1; + } + return 0; case AP_CONN_STATE_OPEN: case EXIT_CONN_STATE_OPEN: if (connection_edge_package_raw_inbuf(conn, package_partial, NULL) < 0) { @@ -490,6 +497,7 @@ connection_edge_finished_flushing(edge_connection_t *conn) case AP_CONN_STATE_CONNECT_WAIT: case AP_CONN_STATE_CONTROLLER_WAIT: case AP_CONN_STATE_RESOLVE_WAIT: + case AP_CONN_STATE_HTTP_CONNECT_WAIT: return 0; default: log_warn(LD_BUG, "Called in unexpected state %d.",conn->base_.state); @@ -1181,10 +1189,10 @@ consider_plaintext_ports(entry_connection_t *conn, uint16_t port) * See connection_ap_handshake_rewrite_and_attach()'s * documentation for arguments and return value. */ -int -connection_ap_rewrite_and_attach_if_allowed(entry_connection_t *conn, - origin_circuit_t *circ, - crypt_path_t *cpath) +MOCK_IMPL(int, +connection_ap_rewrite_and_attach_if_allowed,(entry_connection_t *conn, + origin_circuit_t *circ, + crypt_path_t *cpath)) { const or_options_t *options = get_options(); @@ -2275,7 +2283,7 @@ connection_ap_handshake_process_socks(entry_connection_t *conn) if (socks->replylen) { had_reply = 1; - connection_write_to_buf((const char*)socks->reply, socks->replylen, + connection_buf_add((const char*)socks->reply, socks->replylen, base_conn); socks->replylen = 0; if (sockshere == -1) { @@ -2372,7 +2380,7 @@ connection_ap_process_natd(entry_connection_t *conn) /* look for LF-terminated "[DEST ip_addr port]" * where ip_addr is a dotted-quad and port is in string form */ - err = connection_fetch_from_buf_line(ENTRY_TO_CONN(conn), tmp_buf, &tlen); + err = connection_buf_get_line(ENTRY_TO_CONN(conn), tmp_buf, &tlen); if (err == 0) return 0; if (err < 0) { @@ -2421,6 +2429,108 @@ connection_ap_process_natd(entry_connection_t *conn) return connection_ap_rewrite_and_attach_if_allowed(conn, NULL, NULL); } +/** Called on an HTTP CONNECT entry connection when some bytes have arrived, + * but we have not yet received a full HTTP CONNECT request. Try to parse an + * HTTP CONNECT request from the connection's inbuf. On success, set up the + * connection's socks_request field and try to attach the connection. On + * failure, send an HTTP reply, and mark the connection. + */ +STATIC int +connection_ap_process_http_connect(entry_connection_t *conn) +{ + if (BUG(ENTRY_TO_CONN(conn)->state != AP_CONN_STATE_HTTP_CONNECT_WAIT)) + return -1; + + char *headers = NULL, *body = NULL; + char *command = NULL, *addrport = NULL; + char *addr = NULL; + size_t bodylen = 0; + + const char *errmsg = NULL; + int rv = 0; + + const int http_status = + fetch_from_buf_http(ENTRY_TO_CONN(conn)->inbuf, &headers, 8192, + &body, &bodylen, 1024, 0); + if (http_status < 0) { + /* Bad http status */ + errmsg = "HTTP/1.0 400 Bad Request\r\n\r\n"; + goto err; + } else if (http_status == 0) { + /* no HTTP request yet. */ + goto done; + } + + const int cmd_status = parse_http_command(headers, &command, &addrport); + if (cmd_status < 0) { + errmsg = "HTTP/1.0 400 Bad Request\r\n\r\n"; + goto err; + } + tor_assert(command); + tor_assert(addrport); + if (strcasecmp(command, "connect")) { + errmsg = "HTTP/1.0 405 Method Not Allowed\r\n\r\n"; + goto err; + } + + tor_assert(conn->socks_request); + socks_request_t *socks = conn->socks_request; + uint16_t port; + if (tor_addr_port_split(LOG_WARN, addrport, &addr, &port) < 0) { + errmsg = "HTTP/1.0 400 Bad Request\r\n\r\n"; + goto err; + } + if (strlen(addr) >= MAX_SOCKS_ADDR_LEN) { + errmsg = "HTTP/1.0 414 Request-URI Too Long\r\n\r\n"; + goto err; + } + + /* Abuse the 'username' and 'password' fields here. They are already an + * abuse. */ + { + char *authorization = http_get_header(headers, "Proxy-Authorization: "); + if (authorization) { + socks->username = authorization; // steal reference + socks->usernamelen = strlen(authorization); + } + char *isolation = http_get_header(headers, "X-Tor-Stream-Isolation: "); + if (isolation) { + socks->password = isolation; // steal reference + socks->passwordlen = strlen(isolation); + } + } + + socks->command = SOCKS_COMMAND_CONNECT; + socks->listener_type = CONN_TYPE_AP_HTTP_CONNECT_LISTENER; + strlcpy(socks->address, addr, sizeof(socks->address)); + socks->port = port; + + control_event_stream_status(conn, STREAM_EVENT_NEW, 0); + + rv = connection_ap_rewrite_and_attach_if_allowed(conn, NULL, NULL); + + // XXXX send a "100 Continue" message? + + goto done; + + err: + if (BUG(errmsg == NULL)) + errmsg = "HTTP/1.0 400 Bad Request\r\n\r\n"; + log_warn(LD_EDGE, "Saying %s", escaped(errmsg)); + connection_buf_add(errmsg, strlen(errmsg), ENTRY_TO_CONN(conn)); + connection_mark_unattached_ap(conn, + END_STREAM_REASON_HTTPPROTOCOL| + END_STREAM_REASON_FLAG_ALREADY_SOCKS_REPLIED); + + done: + tor_free(headers); + tor_free(body); + tor_free(command); + tor_free(addrport); + tor_free(addr); + return rv; +} + /** Iterate over the two bytes of stream_id until we get one that is not * already in use; return it. Return 0 if can't get a unique stream_id. */ @@ -3040,15 +3150,22 @@ connection_ap_handshake_socks_reply(entry_connection_t *conn, char *reply, return; } if (replylen) { /* we already have a reply in mind */ - connection_write_to_buf(reply, replylen, ENTRY_TO_CONN(conn)); + connection_buf_add(reply, replylen, ENTRY_TO_CONN(conn)); conn->socks_request->has_finished = 1; return; } - if (conn->socks_request->socks_version == 4) { + if (conn->socks_request->listener_type == + CONN_TYPE_AP_HTTP_CONNECT_LISTENER) { + const char *response = end_reason_to_http_connect_response_line(endreason); + if (!response) { + response = "HTTP/1.0 400 Bad Request\r\n\r\n"; + } + connection_buf_add(response, strlen(response), ENTRY_TO_CONN(conn)); + } else if (conn->socks_request->socks_version == 4) { memset(buf,0,SOCKS4_NETWORK_LEN); buf[1] = (status==SOCKS5_SUCCEEDED ? SOCKS4_GRANTED : SOCKS4_REJECT); /* leave version, destport, destip zero */ - connection_write_to_buf(buf, SOCKS4_NETWORK_LEN, ENTRY_TO_CONN(conn)); + connection_buf_add(buf, SOCKS4_NETWORK_LEN, ENTRY_TO_CONN(conn)); } else if (conn->socks_request->socks_version == 5) { size_t buf_len; memset(buf,0,sizeof(buf)); @@ -3067,7 +3184,7 @@ connection_ap_handshake_socks_reply(entry_connection_t *conn, char *reply, /* 4 bytes for the header, 2 bytes for the port, 16 for the address. */ buf_len = 22; } - connection_write_to_buf(buf,buf_len,ENTRY_TO_CONN(conn)); + connection_buf_add(buf,buf_len,ENTRY_TO_CONN(conn)); } /* If socks_version isn't 4 or 5, don't send anything. * This can happen in the case of AP bridges. */ diff --git a/src/or/connection_edge.h b/src/or/connection_edge.h index 914238fc1f..e47043f7fe 100644 --- a/src/or/connection_edge.h +++ b/src/or/connection_edge.h @@ -89,9 +89,10 @@ int connection_ap_process_transparent(entry_connection_t *conn); int address_is_invalid_destination(const char *address, int client); -int connection_ap_rewrite_and_attach_if_allowed(entry_connection_t *conn, - origin_circuit_t *circ, - crypt_path_t *cpath); +MOCK_DECL(int, connection_ap_rewrite_and_attach_if_allowed, + (entry_connection_t *conn, + origin_circuit_t *circ, + crypt_path_t *cpath)); int connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn, origin_circuit_t *circ, crypt_path_t *cpath); @@ -188,6 +189,8 @@ typedef struct { STATIC void connection_ap_handshake_rewrite(entry_connection_t *conn, rewrite_result_t *out); + +STATIC int connection_ap_process_http_connect(entry_connection_t *conn); #endif #endif diff --git a/src/or/connection_or.c b/src/or/connection_or.c index fc304e6f18..d890b58da6 100644 --- a/src/or/connection_or.c +++ b/src/or/connection_or.c @@ -46,6 +46,7 @@ #include "microdesc.h" #include "networkstatus.h" #include "nodelist.h" +#include "proto_cell.h" #include "reasons.h" #include "relay.h" #include "rephist.h" @@ -1978,7 +1979,7 @@ connection_or_write_cell_to_buf(const cell_t *cell, or_connection_t *conn) if (cell->command == CELL_PADDING) rep_hist_padding_count_write(PADDING_TYPE_CELL); - connection_write_to_buf(networkcell.body, cell_network_size, TO_CONN(conn)); + connection_buf_add(networkcell.body, cell_network_size, TO_CONN(conn)); /* Touch the channel's active timestamp if there is one */ if (conn->chan) { @@ -2008,8 +2009,8 @@ connection_or_write_var_cell_to_buf,(const var_cell_t *cell, tor_assert(cell); tor_assert(conn); n = var_cell_pack_header(cell, hdr, conn->wide_circ_ids); - connection_write_to_buf(hdr, n, TO_CONN(conn)); - connection_write_to_buf((char*)cell->payload, + connection_buf_add(hdr, n, TO_CONN(conn)); + connection_buf_add((char*)cell->payload, cell->payload_len, TO_CONN(conn)); if (conn->base_.state == OR_CONN_STATE_OR_HANDSHAKING_V3) or_handshake_state_record_var_cell(conn, conn->handshake_state, cell, 0); @@ -2084,7 +2085,7 @@ connection_or_process_cells_from_inbuf(or_connection_t *conn) channel_timestamp_active(TLS_CHAN_TO_BASE(conn->chan)); circuit_build_times_network_is_live(get_circuit_build_times_mutable()); - connection_fetch_from_buf(buf, cell_network_size, TO_CONN(conn)); + connection_buf_get_bytes(buf, cell_network_size, TO_CONN(conn)); /* retrieve cell info from buf (create the host-order struct from the * network-order string) */ diff --git a/src/or/conscache.c b/src/or/conscache.c index 5ffa129bbe..9e13ce8e43 100644 --- a/src/or/conscache.c +++ b/src/or/conscache.c @@ -9,6 +9,14 @@ #define CCE_MAGIC 0x17162253 +#ifdef _WIN32 +/* On Windows, unlink won't work on a file if the file is actively mmap()ed. + * That forces us to be less aggressive about unlinking files, and causes other + * changes throughout our logic. + */ +#define MUST_UNMAP_TO_UNLINK +#endif + /** * A consensus_cache_entry_t is a reference-counted handle to an * item in a consensus_cache_t. It can be mmapped into RAM, or not, @@ -49,6 +57,11 @@ struct consensus_cache_t { storage_dir_t *dir; /** List of all the entries in the directory. */ smartlist_t *entries; + + /** The maximum number of entries that we'd like to allow in this cache. + * This is the same as the storagedir limit when MUST_UNMAP_TO_UNLINK is + * not defined. */ + unsigned max_entries; }; static void consensus_cache_clear(consensus_cache_t *cache); @@ -64,9 +77,26 @@ static void consensus_cache_entry_unmap(consensus_cache_entry_t *ent); consensus_cache_t * consensus_cache_open(const char *subdir, int max_entries) { + int storagedir_max_entries; consensus_cache_t *cache = tor_malloc_zero(sizeof(consensus_cache_t)); char *directory = get_datadir_fname(subdir); - cache->dir = storage_dir_new(directory, max_entries); + cache->max_entries = max_entries; + +#ifdef MUST_UNMAP_TO_UNLINK + /* If we can't unlink the files that we're still using, then we need to + * tell the storagedir backend to allow far more files than this consensus + * cache actually wants, so that it can hold files which, from this cache's + * perspective, have become useless. + */ +#define VERY_LARGE_STORAGEDIR_LIMIT (1000*1000) + storagedir_max_entries = VERY_LARGE_STORAGEDIR_LIMIT; +#else + /* Otherwise, we can just tell the storagedir to use the same limits + * as this cache. */ + storagedir_max_entries = max_entries; +#endif + + cache->dir = storage_dir_new(directory, storagedir_max_entries); tor_free(directory); if (!cache->dir) { tor_free(cache); @@ -77,6 +107,24 @@ consensus_cache_open(const char *subdir, int max_entries) return cache; } +/** Return true if it's okay to put more entries in this cache than + * its official file limit. + * + * (We need this method on Windows, where we can't unlink files that are still + * in use, and therefore might need to temporarily exceed the file limit until + * the no-longer-wanted files are deletable.) + */ +int +consensus_cache_may_overallocate(consensus_cache_t *cache) +{ + (void) cache; +#ifdef MUST_UNMAP_TO_UNLINK + return 1; +#else + return 0; +#endif +} + /** * Tell the sandbox (if any) configured by <b>cfg</b> to allow the * operations that <b>cache</b> will need. @@ -85,6 +133,19 @@ int consensus_cache_register_with_sandbox(consensus_cache_t *cache, struct sandbox_cfg_elem **cfg) { +#ifdef MUST_UNMAP_TO_UNLINK + /* Our Linux sandbox doesn't support huge file lists like the one that would + * be generated by using VERY_LARGE_STORAGEDIR_LIMIT above in + * consensus_cache_open(). Since the Linux sandbox is the only one we have + * right now, we just assert that we never reach this point when we've had + * to use VERY_LARGE_STORAGEDIR_LIMIT. + * + * If at some point in the future we have a different sandbox mechanism that + * can handle huge file lists, we can remove this assertion or make it + * conditional. + */ + tor_assert_nonfatal_unreached(); +#endif return storage_dir_register_with_sandbox(cache->dir, cfg); } @@ -406,23 +467,36 @@ int consensus_cache_get_n_filenames_available(consensus_cache_t *cache) { tor_assert(cache); - int max = storage_dir_get_max_files(cache->dir); + int max = cache->max_entries; int used = smartlist_len(storage_dir_list(cache->dir)); +#ifdef MUST_UNMAP_TO_UNLINK + if (used > max) + return 0; +#else tor_assert_nonfatal(max >= used); +#endif return max - used; } /** * Delete every element of <b>cache</b> has been marked with - * consensus_cache_entry_mark_for_removal. If <b>force</b> is false, - * retain those entries which are not in use except by the cache. + * consensus_cache_entry_mark_for_removal. If <b>force</b> is false, + * retain those entries which are in use by something other than the cache. */ void consensus_cache_delete_pending(consensus_cache_t *cache, int force) { SMARTLIST_FOREACH_BEGIN(cache->entries, consensus_cache_entry_t *, ent) { tor_assert_nonfatal(ent->in_cache == cache); - if (! force) { + int force_ent = force; +#ifdef MUST_UNMAP_TO_UNLINK + /* We cannot delete anything with an active mmap on win32, so no + * force-deletion. */ + if (ent->map) { + force_ent = 0; + } +#endif + if (! force_ent) { if (ent->refcnt > 1 || BUG(ent->in_cache == NULL)) { /* Somebody is using this entry right now */ continue; diff --git a/src/or/conscache.h b/src/or/conscache.h index aef54201f0..a0d74c4e08 100644 --- a/src/or/conscache.h +++ b/src/or/conscache.h @@ -14,6 +14,7 @@ HANDLE_DECL(consensus_cache_entry, consensus_cache_entry_t, ) consensus_cache_t *consensus_cache_open(const char *subdir, int max_entries); void consensus_cache_free(consensus_cache_t *cache); struct sandbox_cfg_elem; +int consensus_cache_may_overallocate(consensus_cache_t *cache); int consensus_cache_register_with_sandbox(consensus_cache_t *cache, struct sandbox_cfg_elem **cfg); void consensus_cache_unmap_lazy(consensus_cache_t *cache, time_t cutoff); diff --git a/src/or/consdiffmgr.c b/src/or/consdiffmgr.c index 928fc26f54..831d5d45c3 100644 --- a/src/or/consdiffmgr.c +++ b/src/or/consdiffmgr.c @@ -1136,7 +1136,7 @@ consdiffmgr_ensure_space_for_files(int n) return 0; } // Let's get more assertive: clean out unused stuff, and force-remove - // the files. + // the files that we can. consdiffmgr_cleanup(); consensus_cache_delete_pending(cache, 1); const int n_to_remove = n - consensus_cache_get_n_filenames_available(cache); @@ -1159,6 +1159,14 @@ consdiffmgr_ensure_space_for_files(int n) smartlist_free(objects); consensus_cache_delete_pending(cache, 1); + + if (consensus_cache_may_overallocate(cache)) { + /* If we're allowed to throw extra files into the cache, let's do so + * rather getting upset. + */ + return 0; + } + if (BUG(n_marked < n_to_remove)) return -1; else diff --git a/src/or/control.c b/src/or/control.c index bc173a6e1c..eb6f213520 100644 --- a/src/or/control.c +++ b/src/or/control.c @@ -64,6 +64,8 @@ #include "networkstatus.h" #include "nodelist.h" #include "policies.h" +#include "proto_control0.h" +#include "proto_http.h" #include "reasons.h" #include "rendclient.h" #include "rendcommon.h" @@ -355,7 +357,7 @@ static inline void connection_write_str_to_buf(const char *s, control_connection_t *conn) { size_t len = strlen(s); - connection_write_to_buf(s, len, TO_CONN(conn)); + connection_buf_add(s, len, TO_CONN(conn)); } /** Given a <b>len</b>-character string in <b>data</b>, made of lines @@ -565,7 +567,7 @@ connection_printf_to_buf(control_connection_t *conn, const char *format, ...) tor_assert(0); } - connection_write_to_buf(buf, (size_t)len, TO_CONN(conn)); + connection_buf_add(buf, (size_t)len, TO_CONN(conn)); tor_free(buf); } @@ -790,7 +792,7 @@ queued_events_flush_all(int force) SMARTLIST_FOREACH_BEGIN(controllers, control_connection_t *, control_conn) { if (control_conn->event_mask & bit) { - connection_write_to_buf(ev->msg, msg_len, TO_CONN(control_conn)); + connection_buf_add(ev->msg, msg_len, TO_CONN(control_conn)); } } SMARTLIST_FOREACH_END(control_conn); @@ -1072,7 +1074,7 @@ handle_control_getconf(control_connection_t *conn, uint32_t body_len, tor_assert(strlen(tmp)>4); tmp[3] = ' '; msg = smartlist_join_strings(answers, "", 0, &msg_len); - connection_write_to_buf(msg, msg_len, TO_CONN(conn)); + connection_buf_add(msg, msg_len, TO_CONN(conn)); } else { connection_write_str_to_buf("250 OK\r\n", conn); } @@ -1654,12 +1656,12 @@ handle_control_mapaddress(control_connection_t *conn, uint32_t len, if (smartlist_len(reply)) { ((char*)smartlist_get(reply,smartlist_len(reply)-1))[3] = ' '; r = smartlist_join_strings(reply, "\r\n", 1, &sz); - connection_write_to_buf(r, sz, TO_CONN(conn)); + connection_buf_add(r, sz, TO_CONN(conn)); tor_free(r); } else { const char *response = "512 syntax error: not enough arguments to mapaddress.\r\n"; - connection_write_to_buf(response, strlen(response), TO_CONN(conn)); + connection_buf_add(response, strlen(response), TO_CONN(conn)); } SMARTLIST_FOREACH(reply, char *, cp, tor_free(cp)); @@ -3244,7 +3246,7 @@ handle_control_getinfo(control_connection_t *conn, uint32_t len, size_t esc_len; esc_len = write_escaped_data(v, strlen(v), &esc); connection_printf_to_buf(conn, "250+%s=\r\n", k); - connection_write_to_buf(esc, esc_len, TO_CONN(conn)); + connection_buf_add(esc, esc_len, TO_CONN(conn)); tor_free(esc); } } @@ -4985,7 +4987,7 @@ connection_control_process_inbuf(control_connection_t *conn) sizeof(buf)-6); body_len = 2+strlen(buf+6)+2; /* code, msg, nul. */ set_uint16(buf+0, htons(body_len)); - connection_write_to_buf(buf, 4+body_len, TO_CONN(conn)); + connection_buf_add(buf, 4+body_len, TO_CONN(conn)); connection_mark_and_flush(TO_CONN(conn)); return 0; @@ -5007,7 +5009,7 @@ connection_control_process_inbuf(control_connection_t *conn) /* First, fetch a line. */ do { data_len = conn->incoming_cmd_len - conn->incoming_cmd_cur_len; - r = connection_fetch_from_buf_line(TO_CONN(conn), + r = connection_buf_get_line(TO_CONN(conn), conn->incoming_cmd+conn->incoming_cmd_cur_len, &data_len); if (r == 0) diff --git a/src/or/directory.c b/src/or/directory.c index 57dfdd9cac..007235d108 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -104,7 +104,6 @@ static void directory_send_command(dir_connection_t *conn, int direct, const directory_request_t *request); static int body_is_plausible(const char *body, size_t body_len, int purpose); -static char *http_get_header(const char *headers, const char *which); static void http_set_address_origin(const char *headers, connection_t *conn); static void connection_dir_download_routerdesc_failed(dir_connection_t *conn); static void connection_dir_bridge_routerdesc_failed(dir_connection_t *conn); @@ -1912,11 +1911,11 @@ directory_send_command(dir_connection_t *conn, request_len = strlen(request); total_request_len += request_len; - connection_write_to_buf(request, request_len, TO_CONN(conn)); + connection_buf_add(request, request_len, TO_CONN(conn)); url_len = strlen(url); total_request_len += url_len; - connection_write_to_buf(url, url_len, TO_CONN(conn)); + connection_buf_add(url, url_len, TO_CONN(conn)); tor_free(url); if (!strcmp(httpcommand, "POST") || payload) { @@ -1933,11 +1932,11 @@ directory_send_command(dir_connection_t *conn, request_len = strlen(request); total_request_len += request_len; - connection_write_to_buf(request, request_len, TO_CONN(conn)); + connection_buf_add(request, request_len, TO_CONN(conn)); if (payload) { /* then send the payload afterwards too */ - connection_write_to_buf(payload, payload_len, TO_CONN(conn)); + connection_buf_add(payload, payload_len, TO_CONN(conn)); total_request_len += payload_len; } @@ -1966,15 +1965,41 @@ directory_send_command(dir_connection_t *conn, STATIC int parse_http_url(const char *headers, char **url) { + char *command = NULL; + if (parse_http_command(headers, &command, url) < 0) { + return -1; + } + if (strcmpstart(*url, "/tor/")) { + char *new_url = NULL; + tor_asprintf(&new_url, "/tor%s%s", + *url[0] == '/' ? "" : "/", + *url); + tor_free(*url); + *url = new_url; + } + tor_free(command); + return 0; +} + +/** Parse an HTTP request line at the start of a headers string. On failure, + * return -1. On success, set *<b>command_out</b> to a copy of the HTTP + * command ("get", "post", etc), set *<b>url_out</b> to a copy of the URL, and + * return 0. */ +int +parse_http_command(const char *headers, char **command_out, char **url_out) +{ + const char *command, *end_of_command; char *s, *start, *tmp; s = (char *)eat_whitespace_no_nl(headers); if (!*s) return -1; + command = s; s = (char *)find_whitespace(s); /* get past GET/POST */ if (!*s) return -1; + end_of_command = s; s = (char *)eat_whitespace_no_nl(s); if (!*s) return -1; - start = s; /* this is it, assuming it's valid */ + start = s; /* this is the URL, assuming it's valid */ s = (char *)find_whitespace(start); if (!*s) return -1; @@ -2005,13 +2030,8 @@ parse_http_url(const char *headers, char **url) return -1; } - if (s-start < 5 || strcmpstart(start,"/tor/")) { /* need to rewrite it */ - *url = tor_malloc(s - start + 5); - strlcpy(*url,"/tor", s-start+5); - strlcat((*url)+4, start, s-start+1); - } else { - *url = tor_strndup(start, s-start); - } + *url_out = tor_memdup_nulterm(start, s-start); + *command_out = tor_memdup_nulterm(command, end_of_command - command); return 0; } @@ -2019,7 +2039,7 @@ parse_http_url(const char *headers, char **url) * <b>which</b>. The key should be given with a terminating colon and space; * this function copies everything after, up to but not including the * following \\r\\n. */ -static char * +char * http_get_header(const char *headers, const char *which) { const char *cp = headers; @@ -3466,7 +3486,7 @@ write_http_status_line(dir_connection_t *conn, int status, return; } log_debug(LD_DIRSERV,"Wrote status 'HTTP/1.0 %d %s'", status, reason_phrase); - connection_write_to_buf(buf, strlen(buf), TO_CONN(conn)); + connection_buf_add(buf, strlen(buf), TO_CONN(conn)); } /** Write the header for an HTTP/1.0 response onto <b>conn</b>-\>outbuf, @@ -3539,7 +3559,7 @@ write_http_response_header_impl(dir_connection_t *conn, ssize_t length, memcpy(cp, "\r\n", 3); else tor_assert(0); - connection_write_to_buf(tmp, strlen(tmp), TO_CONN(conn)); + connection_buf_add(tmp, strlen(tmp), TO_CONN(conn)); } /** As write_http_response_header_impl, but sets encoding and content-typed @@ -3902,7 +3922,7 @@ handle_get_frontpage(dir_connection_t *conn, const get_handler_args_t *args) * this page no matter what.] */ write_http_response_header_impl(conn, dlen, "text/html", "identity", NULL, DIRPORTFRONTPAGE_CACHE_LIFETIME); - connection_write_to_buf(frontpage, dlen, TO_CONN(conn)); + connection_buf_add(frontpage, dlen, TO_CONN(conn)); } else { write_http_status_line(conn, 404, "Not found"); } @@ -4539,15 +4559,15 @@ handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args) conn->compress_state = tor_compress_new(1, compress_method, choose_compression_level(estimated_len)); SMARTLIST_FOREACH(items, const char *, c, - connection_write_to_buf_compress(c, strlen(c), conn, 0)); - connection_write_to_buf_compress("", 0, conn, 1); + connection_buf_add_compress(c, strlen(c), conn, 0)); + connection_buf_add_compress("", 0, conn, 1); } else { SMARTLIST_FOREACH(items, const char *, c, - connection_write_to_buf(c, strlen(c), TO_CONN(conn))); + connection_buf_add(c, strlen(c), TO_CONN(conn))); } } else { SMARTLIST_FOREACH(dir_items, cached_dir_t *, d, - connection_write_to_buf(compress_method != NO_METHOD ? + connection_buf_add(compress_method != NO_METHOD ? d->dir_compressed : d->dir, compress_method != NO_METHOD ? d->dir_compressed_len : d->dir_len, @@ -4795,14 +4815,14 @@ handle_get_keys(dir_connection_t *conn, const get_handler_args_t *args) conn->compress_state = tor_compress_new(1, compress_method, choose_compression_level(len)); SMARTLIST_FOREACH(certs, authority_cert_t *, c, - connection_write_to_buf_compress( + connection_buf_add_compress( c->cache_info.signed_descriptor_body, c->cache_info.signed_descriptor_len, conn, 0)); - connection_write_to_buf_compress("", 0, conn, 1); + connection_buf_add_compress("", 0, conn, 1); } else { SMARTLIST_FOREACH(certs, authority_cert_t *, c, - connection_write_to_buf(c->cache_info.signed_descriptor_body, + connection_buf_add(c->cache_info.signed_descriptor_body, c->cache_info.signed_descriptor_len, TO_CONN(conn))); } @@ -4831,7 +4851,7 @@ handle_get_hs_descriptor_v2(dir_connection_t *conn, switch (rend_cache_lookup_v2_desc_as_dir(query, &descp)) { case 1: /* valid */ write_http_response_header(conn, strlen(descp), NO_METHOD, 0); - connection_write_to_buf(descp, strlen(descp), TO_CONN(conn)); + connection_buf_add(descp, strlen(descp), TO_CONN(conn)); break; case 0: /* well-formed but not present */ write_http_status_line(conn, 404, "Not found"); @@ -4883,7 +4903,7 @@ handle_get_hs_descriptor_v3(dir_connection_t *conn, /* Found requested descriptor! Pass it to this nice client. */ write_http_response_header(conn, strlen(desc_str), NO_METHOD, 0); - connection_write_to_buf(desc_str, strlen(desc_str), TO_CONN(conn)); + connection_buf_add(desc_str, strlen(desc_str), TO_CONN(conn)); done: return 0; @@ -4922,7 +4942,7 @@ handle_get_networkstatus_bridges(dir_connection_t *conn, status = networkstatus_getinfo_by_purpose("bridge", time(NULL)); size_t dlen = strlen(status); write_http_response_header(conn, dlen, NO_METHOD, 0); - connection_write_to_buf(status, dlen, TO_CONN(conn)); + connection_buf_add(status, dlen, TO_CONN(conn)); tor_free(status); goto done; } @@ -4939,7 +4959,7 @@ handle_get_robots(dir_connection_t *conn, const get_handler_args_t *args) const char robots[] = "User-agent: *\r\nDisallow: /\r\n"; size_t len = strlen(robots); write_http_response_header(conn, len, NO_METHOD, ROBOTS_CACHE_LIFETIME); - connection_write_to_buf(robots, len, TO_CONN(conn)); + connection_buf_add(robots, len, TO_CONN(conn)); } return 0; } diff --git a/src/or/directory.h b/src/or/directory.h index 42bcb55eb2..1200728411 100644 --- a/src/or/directory.h +++ b/src/or/directory.h @@ -87,6 +87,9 @@ MOCK_DECL(void, directory_initiate_request, (directory_request_t *request)); int parse_http_response(const char *headers, int *code, time_t *date, compress_method_t *compression, char **response); +int parse_http_command(const char *headers, + char **command_out, char **url_out); +char *http_get_header(const char *headers, const char *which); int connection_dir_is_encrypted(const dir_connection_t *conn); int connection_dir_reached_eof(dir_connection_t *conn); diff --git a/src/or/dirserv.c b/src/or/dirserv.c index 4faca9454b..b3a0b0d23f 100644 --- a/src/or/dirserv.c +++ b/src/or/dirserv.c @@ -3617,9 +3617,9 @@ spooled_resource_flush_some(spooled_resource_t *spooled, return SRFS_DONE; } if (conn->compress_state) { - connection_write_to_buf_compress((const char*)body, bodylen, conn, 0); + connection_buf_add_compress((const char*)body, bodylen, conn, 0); } else { - connection_write_to_buf((const char*)body, bodylen, TO_CONN(conn)); + connection_buf_add((const char*)body, bodylen, TO_CONN(conn)); } return SRFS_DONE; } else { @@ -3656,11 +3656,11 @@ spooled_resource_flush_some(spooled_resource_t *spooled, return SRFS_ERR; ssize_t bytes = (ssize_t) MIN(DIRSERV_CACHED_DIR_CHUNK_SIZE, remaining); if (conn->compress_state) { - connection_write_to_buf_compress( + connection_buf_add_compress( ptr + spooled->cached_dir_offset, bytes, conn, 0); } else { - connection_write_to_buf(ptr + spooled->cached_dir_offset, + connection_buf_add(ptr + spooled->cached_dir_offset, bytes, TO_CONN(conn)); } spooled->cached_dir_offset += bytes; @@ -3925,7 +3925,7 @@ connection_dirserv_flushed_some(dir_connection_t *conn) if (conn->compress_state) { /* Flush the compression state: there could be more bytes pending in there, * and we don't want to omit bytes. */ - connection_write_to_buf_compress("", 0, conn, 1); + connection_buf_add_compress("", 0, conn, 1); tor_compress_free(conn->compress_state); conn->compress_state = NULL; } diff --git a/src/or/ext_orport.c b/src/or/ext_orport.c index 01dc06ce13..c22a2f13d8 100644 --- a/src/or/ext_orport.c +++ b/src/or/ext_orport.c @@ -23,8 +23,9 @@ #include "ext_orport.h" #include "control.h" #include "config.h" -#include "util.h" #include "main.h" +#include "proto_ext_or.h" +#include "util.h" /** Allocate and return a structure capable of holding an Extended * ORPort message of body length <b>len</b>. */ @@ -69,10 +70,10 @@ connection_write_ext_or_command(connection_t *conn, return -1; set_uint16(header, htons(command)); set_uint16(header+2, htons(bodylen)); - connection_write_to_buf(header, 4, conn); + connection_buf_add(header, 4, conn); if (bodylen) { tor_assert(body); - connection_write_to_buf(body, bodylen, conn); + connection_buf_add(body, bodylen, conn); } return 0; } @@ -170,7 +171,7 @@ connection_ext_or_auth_neg_auth_type(connection_t *conn) if (connection_get_inbuf_len(conn) < 1) return 0; - if (connection_fetch_from_buf(authtype, 1, conn) < 0) + if (connection_buf_get_bytes(authtype, 1, conn) < 0) return -1; log_debug(LD_GENERAL, "Client wants us to use %d auth type", authtype[0]); @@ -310,7 +311,7 @@ connection_ext_or_auth_handle_client_nonce(connection_t *conn) if (connection_get_inbuf_len(conn) < EXT_OR_PORT_AUTH_NONCE_LEN) return 0; - if (connection_fetch_from_buf(client_nonce, + if (connection_buf_get_bytes(client_nonce, EXT_OR_PORT_AUTH_NONCE_LEN, conn) < 0) return -1; @@ -325,7 +326,7 @@ connection_ext_or_auth_handle_client_nonce(connection_t *conn) &reply, &reply_len) < 0) return -1; - connection_write_to_buf(reply, reply_len, conn); + connection_buf_add(reply, reply_len, conn); memwipe(reply, 0, reply_len); tor_free(reply); @@ -347,9 +348,9 @@ static void connection_ext_or_auth_send_result(connection_t *conn, int success) { if (success) - connection_write_to_buf("\x01", 1, conn); + connection_buf_add("\x01", 1, conn); else - connection_write_to_buf("\x00", 1, conn); + connection_buf_add("\x00", 1, conn); } /** Receive the client's hash from <b>conn</b>, validate that it's @@ -367,7 +368,7 @@ connection_ext_or_auth_handle_client_hash(connection_t *conn) if (connection_get_inbuf_len(conn) < EXT_OR_PORT_AUTH_HASH_LEN) return 0; - if (connection_fetch_from_buf(provided_client_hash, + if (connection_buf_get_bytes(provided_client_hash, EXT_OR_PORT_AUTH_HASH_LEN, conn) < 0) return -1; @@ -637,7 +638,7 @@ connection_ext_or_start_auth(or_connection_t *or_conn) log_debug(LD_GENERAL, "ExtORPort authentication: Sending supported authentication types"); - connection_write_to_buf((const char *)authtypes, sizeof(authtypes), conn); + connection_buf_add((const char *)authtypes, sizeof(authtypes), conn); conn->state = EXT_OR_CONN_STATE_AUTH_WAIT_AUTH_TYPE; return 0; diff --git a/src/or/hs_client.c b/src/or/hs_client.c index 2acc639e49..0e8fc6c24b 100644 --- a/src/or/hs_client.c +++ b/src/or/hs_client.c @@ -1264,3 +1264,11 @@ hs_client_reextend_intro_circuit(origin_circuit_t *circ) return ret; } +/* Release all the storage held by the client subsystem. */ +void +hs_client_free_all(void) +{ + /* Purge the hidden service request cache. */ + hs_purge_last_hid_serv_requests(); +} + diff --git a/src/or/hs_client.h b/src/or/hs_client.h index 8ed0501c91..86784f52c3 100644 --- a/src/or/hs_client.h +++ b/src/or/hs_client.h @@ -46,5 +46,7 @@ extend_info_t *hs_client_get_random_intro_from_edge( int hs_client_reextend_intro_circuit(origin_circuit_t *circ); +void hs_client_free_all(void); + #endif /* TOR_HS_CLIENT_H */ diff --git a/src/or/hs_common.c b/src/or/hs_common.c index d866ab6a8f..5ea44b97e7 100644 --- a/src/or/hs_common.c +++ b/src/or/hs_common.c @@ -19,6 +19,7 @@ #include "nodelist.h" #include "hs_cache.h" #include "hs_common.h" +#include "hs_client.h" #include "hs_ident.h" #include "hs_service.h" #include "policies.h" @@ -1206,12 +1207,16 @@ node_has_hsdir_index(const node_t *node, int is_for_next_period) if (BUG(node->hsdir_index == NULL) || BUG(tor_mem_is_zero((const char*)node->hsdir_index->current, DIGEST256_LEN))) { + log_warn(LD_BUG, "Zero current index (ri: %p, rs: %p, md: %p)", + node->ri, node->rs, node->md); return 0; } if (is_for_next_period && BUG(tor_mem_is_zero((const char*)node->hsdir_index->next, DIGEST256_LEN))) { + log_warn(LD_BUG, "Zero next index (ri: %p, rs: %p, md: %p)", + node->ri, node->rs, node->md); return 0; } @@ -1700,6 +1705,7 @@ hs_free_all(void) hs_circuitmap_free_all(); hs_service_free_all(); hs_cache_free_all(); + hs_client_free_all(); } /* For the given origin circuit circ, decrement the number of rendezvous diff --git a/src/or/hs_service.c b/src/or/hs_service.c index 5ff118222d..706890e34f 100644 --- a/src/or/hs_service.c +++ b/src/or/hs_service.c @@ -78,6 +78,7 @@ static smartlist_t *hs_service_staging_list; static int consider_republishing_hs_descriptors = 0; static void set_descriptor_revision_counter(hs_descriptor_t *hs_desc); +static void move_descriptors(hs_service_t *src, hs_service_t *dst); /* Helper: Function to compare two objects in the service map. Return 1 if the * two service have the same master public identity key. */ @@ -714,37 +715,6 @@ close_service_circuits(hs_service_t *service) close_service_rp_circuits(service); } -/* Move introduction points from the src descriptor to the dst descriptor. The - * destination service intropoints are wiped out if any before moving. */ -static void -move_descriptor_intro_points(hs_service_descriptor_t *src, - hs_service_descriptor_t *dst) -{ - tor_assert(src); - tor_assert(dst); - - digest256map_free(dst->intro_points.map, service_intro_point_free_); - dst->intro_points.map = src->intro_points.map; - /* Nullify the source. */ - src->intro_points.map = NULL; -} - -/* Move introduction points from the src service to the dst service. The - * destination service intropoints are wiped out if any before moving. */ -static void -move_intro_points(hs_service_t *src, hs_service_t *dst) -{ - tor_assert(src); - tor_assert(dst); - - if (src->desc_current && dst->desc_current) { - move_descriptor_intro_points(src->desc_current, dst->desc_current); - } - if (src->desc_next && dst->desc_next) { - move_descriptor_intro_points(src->desc_next, dst->desc_next); - } -} - /* Move every ephemeral services from the src service map to the dst service * map. It is possible that a service can't be register to the dst map which * won't stop the process of moving them all but will trigger a log warn. */ @@ -785,6 +755,26 @@ service_escaped_dir(const hs_service_t *s) escaped(s->config.directory_path); } +/** Move the hidden service state from <b>src</b> to <b>dst</b>. We do this + * when we receive a SIGHUP: <b>dst</b> is the post-HUP service */ +static void +move_hs_state(hs_service_t *src_service, hs_service_t *dst_service) +{ + tor_assert(src_service); + tor_assert(dst_service); + + hs_service_state_t *src = &src_service->state; + hs_service_state_t *dst = &dst_service->state; + + /* Let's do a shallow copy */ + dst->intro_circ_retry_started_time = src->intro_circ_retry_started_time; + dst->num_intro_circ_launched = src->num_intro_circ_launched; + dst->in_overlap_period = src->in_overlap_period; + dst->replay_cache_rend_cookie = src->replay_cache_rend_cookie; + + src->replay_cache_rend_cookie = NULL; /* steal pointer reference */ +} + /* Register services that are in the staging list. Once this function returns, * the global service map will be set with the right content and all non * surviving services will be cleaned up. */ @@ -817,13 +807,15 @@ register_all_services(void) * transfer the intro points to it. */ s = find_service(hs_service_map, &snew->keys.identity_pk); if (s) { - /* Pass ownership of intro points from s (the current service) to snew - * (the newly configured one). */ - move_intro_points(s, snew); + /* Pass ownership of the descriptors from s (the current service) to + * snew (the newly configured one). */ + move_descriptors(s, snew); + move_hs_state(s, snew); /* Remove the service from the global map because after this, we need to * go over the remaining service in that map that aren't surviving the * reload to close their circuits. */ remove_service(hs_service_map, s); + hs_service_free(s); } /* Great, this service is now ready to be added to our new map. */ if (BUG(register_service(new_service_map, snew) < 0)) { @@ -972,8 +964,6 @@ service_descriptor_free(hs_service_descriptor_t *desc) hs_descriptor_free(desc->desc); memwipe(&desc->signing_kp, 0, sizeof(desc->signing_kp)); memwipe(&desc->blinded_kp, 0, sizeof(desc->blinded_kp)); - SMARTLIST_FOREACH(desc->hsdir_missing_info, char *, id, tor_free(id)); - smartlist_free(desc->hsdir_missing_info); /* Cleanup all intro points. */ digest256map_free(desc->intro_points.map, service_intro_point_free_); digestmap_free(desc->intro_points.failed_id, tor_free_); @@ -993,11 +983,37 @@ service_descriptor_new(void) /* Initialize the intro points map. */ sdesc->intro_points.map = digest256map_new(); sdesc->intro_points.failed_id = digestmap_new(); - sdesc->hsdir_missing_info = smartlist_new(); sdesc->previous_hsdirs = smartlist_new(); return sdesc; } +/* Move descriptor(s) from the src service to the dst service. We do this + * during SIGHUP when we re-create our hidden services. */ +static void +move_descriptors(hs_service_t *src, hs_service_t *dst) +{ + tor_assert(src); + tor_assert(dst); + + if (src->desc_current) { + /* Nothing should be there, but clean it up just in case */ + if (BUG(dst->desc_current)) { + service_descriptor_free(dst->desc_current); + } + dst->desc_current = src->desc_current; + src->desc_current = NULL; + } + + if (src->desc_next) { + /* Nothing should be there, but clean it up just in case */ + if (BUG(dst->desc_next)) { + service_descriptor_free(dst->desc_next); + } + dst->desc_next = src->desc_next; + src->desc_next = NULL; + } +} + /* From the given service, remove all expired failing intro points for each * descriptor. */ static void @@ -1546,7 +1562,6 @@ service_desc_note_upload(hs_service_descriptor_t *desc, const node_t *hsdir) if (!smartlist_contains_string(desc->previous_hsdirs, b64_digest)) { smartlist_add_strdup(desc->previous_hsdirs, b64_digest); - smartlist_sort_strings(desc->previous_hsdirs); } } @@ -2317,6 +2332,10 @@ upload_descriptor_to_all(const hs_service_t *service, hs_get_responsible_hsdirs(&desc->blinded_kp.pubkey, desc->time_period_num, for_next_period, 0, responsible_dirs); + /** Clear list of previous hsdirs since we are about to upload to a new + * list. Let's keep it up to date. */ + service_desc_clear_previous_hsdirs(desc); + /* For each responsible HSDir we have, initiate an upload command. */ SMARTLIST_FOREACH_BEGIN(responsible_dirs, const routerstatus_t *, hsdir_rs) { @@ -2324,18 +2343,6 @@ upload_descriptor_to_all(const hs_service_t *service, /* Getting responsible hsdir implies that the node_t object exists for the * routerstatus_t found in the consensus else we have a problem. */ tor_assert(hsdir_node); - /* Do not upload to an HSDir we don't have a descriptor for. */ - if (!node_has_descriptor(hsdir_node)) { - log_info(LD_REND, "Missing descriptor for HSDir %s. Not uploading " - "descriptor. We'll try later once we have it.", - safe_str_client(node_describe(hsdir_node))); - /* Once we get new directory information, this HSDir will be retried if - * we ever get the descriptor. */ - smartlist_add(desc->hsdir_missing_info, - tor_memdup(hsdir_rs->identity_digest, DIGEST_LEN)); - continue; - } - /* Upload this descriptor to the chosen directory. */ upload_descriptor_to_hsdir(service, desc, hsdir_node); } SMARTLIST_FOREACH_END(hsdir_rs); @@ -2366,9 +2373,8 @@ STATIC int service_desc_hsdirs_changed(const hs_service_t *service, const hs_service_descriptor_t *desc) { - int retval = 0; + int should_reupload = 0; smartlist_t *responsible_dirs = smartlist_new(); - smartlist_t *b64_responsible_dirs = smartlist_new(); /* No desc upload has happened yet: it will happen eventually */ if (!desc->previous_hsdirs || !smartlist_len(desc->previous_hsdirs)) { @@ -2379,32 +2385,24 @@ service_desc_hsdirs_changed(const hs_service_t *service, hs_get_responsible_hsdirs(&desc->blinded_kp.pubkey, desc->time_period_num, service->desc_next == desc, 0, responsible_dirs); - /* Make a second list with their b64ed identity digests, so that we can - * compare it with out previous list of hsdirs */ + /* Check if any new hsdirs have been added to the responsible hsdirs set: + * Iterate over the list of new hsdirs, and reupload if any of them is not + * present in the list of previous hsdirs. + */ SMARTLIST_FOREACH_BEGIN(responsible_dirs, const routerstatus_t *, hsdir_rs) { char b64_digest[BASE64_DIGEST_LEN+1] = {0}; digest_to_base64(b64_digest, hsdir_rs->identity_digest); - smartlist_add_strdup(b64_responsible_dirs, b64_digest); - } SMARTLIST_FOREACH_END(hsdir_rs); - - /* Sort this new smartlist so that we can compare it with the other one */ - smartlist_sort_strings(b64_responsible_dirs); - /* Check whether the set of HSDirs changed */ - if (!smartlist_strings_eq(b64_responsible_dirs, desc->previous_hsdirs)) { - log_info(LD_GENERAL, "Received new dirinfo and set of hsdirs changed!"); - retval = 1; - } else { - log_debug(LD_GENERAL, "No change in hsdir set!"); - } + if (!smartlist_contains_string(desc->previous_hsdirs, b64_digest)) { + should_reupload = 1; + break; + } + } SMARTLIST_FOREACH_END(hsdir_rs); done: smartlist_free(responsible_dirs); - SMARTLIST_FOREACH(b64_responsible_dirs, char*, s, tor_free(s)); - smartlist_free(b64_responsible_dirs); - - return retval; + return should_reupload; } /* Return 1 if the given descriptor from the given service can be uploaded @@ -2590,9 +2588,10 @@ service_rendezvous_circ_has_opened(origin_circuit_t *circ) tor_assert(circ->hs_ident); tor_assert(TO_CIRCUIT(circ)->purpose == CIRCUIT_PURPOSE_S_CONNECT_REND); - /* Declare the circuit dirty to avoid reuse, and for path-bias */ - if (!TO_CIRCUIT(circ)->timestamp_dirty) - TO_CIRCUIT(circ)->timestamp_dirty = time(NULL); + /* Declare the circuit dirty to avoid reuse, and for path-bias. We set the + * timestamp regardless of its content because that circuit could have been + * cannibalized so in any cases, we are about to use that circuit more. */ + TO_CIRCUIT(circ)->timestamp_dirty = time(NULL); pathbias_count_use_attempt(circ); /* Get the corresponding service and intro point. */ @@ -2724,58 +2723,6 @@ service_handle_introduce2(origin_circuit_t *circ, const uint8_t *payload, return -1; } -/* For a given service and a descriptor of that service, consider retrying to - * upload the descriptor to any directories from which we had missing - * information when originally tried to be uploaded. This is called when our - * directory information has changed. */ -static void -consider_hsdir_upload_retry(const hs_service_t *service, - hs_service_descriptor_t *desc) -{ - smartlist_t *responsible_dirs = NULL; - smartlist_t *still_missing_dirs = NULL; - - tor_assert(service); - tor_assert(desc); - - responsible_dirs = smartlist_new(); - still_missing_dirs = smartlist_new(); - - /* We first need to get responsible directories from the latest consensus so - * we can then make sure that the node that we were missing information for - * is still responsible for this descriptor. */ - hs_get_responsible_hsdirs(&desc->blinded_kp.pubkey, desc->time_period_num, - service->desc_next == desc, 0, responsible_dirs); - - SMARTLIST_FOREACH_BEGIN(responsible_dirs, const routerstatus_t *, rs) { - const node_t *node; - const char *id = rs->identity_digest; - if (!smartlist_contains_digest(desc->hsdir_missing_info, id)) { - continue; - } - /* We do need a node_t object and descriptor to perform an upload. If - * found, we remove the id from the missing dir list else we add it to the - * still missing dir list to keep track of id that are still missing. */ - node = node_get_by_id(id); - if (node && node_has_descriptor(node)) { - upload_descriptor_to_hsdir(service, desc, node); - smartlist_remove(desc->hsdir_missing_info, id); - } else { - smartlist_add(still_missing_dirs, tor_memdup(id, DIGEST_LEN)); - } - } SMARTLIST_FOREACH_END(rs); - - /* Switch the still missing dir list with the current missing dir list in - * the descriptor. It is possible that the list ends up empty which is what - * we want if we have no more missing dir. */ - SMARTLIST_FOREACH(desc->hsdir_missing_info, char *, id, tor_free(id)); - smartlist_free(desc->hsdir_missing_info); - desc->hsdir_missing_info = still_missing_dirs; - - /* No ownership of the routerstatus_t object in this list. */ - smartlist_free(responsible_dirs); -} - /* Add to list every filename used by service. This is used by the sandbox * subsystem. */ static void @@ -2802,16 +2749,6 @@ service_add_fnames_to_list(const hs_service_t *service, smartlist_t *list) /* Public API */ /* ========== */ -/* We just received a new batch of descriptors which might affect the shape of - * the HSDir hash ring. Signal that we should reexamine the hash ring and - * re-upload our HS descriptors if needed. */ -void -hs_hsdir_set_changed_consider_reupload(void) -{ - log_info(LD_REND, "New dirinfo arrived: consider reuploading descriptor"); - consider_republishing_hs_descriptors = 1; -} - /* Return the number of service we have configured and usable. */ unsigned int hs_service_get_num_services(void) @@ -2981,22 +2918,14 @@ hs_service_lists_fnames_for_sandbox(smartlist_t *file_list, } /* Called when our internal view of the directory has changed. We might have - * new descriptors for hidden service directories that we didn't have before - * so try them if it's the case. */ + * received a new batch of descriptors which might affect the shape of the + * HSDir hash ring. Signal that we should reexamine the hash ring and + * re-upload our HS descriptors if needed. */ void hs_service_dir_info_changed(void) { - /* For each service we have, check every descriptor and consider retrying to - * upload it to directories that we might have had missing information - * previously that is missing a router descriptor. */ - FOR_EACH_SERVICE_BEGIN(service) { - FOR_EACH_DESCRIPTOR_BEGIN(service, desc) { - /* This cleans up the descriptor missing hsdir information list if a - * successful upload is made or if any of the directory aren't - * responsible anymore for the service descriptor. */ - consider_hsdir_upload_retry(service, desc); - } FOR_EACH_DESCRIPTOR_END; - } FOR_EACH_SERVICE_END; + log_info(LD_REND, "New dirinfo arrived: consider reuploading descriptor"); + consider_republishing_hs_descriptors = 1; } /* Called when we get an INTRODUCE2 cell on the circ. Respond to the cell and diff --git a/src/or/hs_service.h b/src/or/hs_service.h index 57717fc927..317b9d795d 100644 --- a/src/or/hs_service.h +++ b/src/or/hs_service.h @@ -123,17 +123,10 @@ typedef struct hs_service_descriptor_t { * couldn't pick any nodes. */ unsigned int missing_intro_points : 1; - /* List of identity digests for hidden service directories to which we - * couldn't upload this descriptor because we didn't have its router - * descriptor at the time. If this list is non-empty, only the relays in this - * list are re-tried to upload this descriptor when our directory information - * have been updated. */ - smartlist_t *hsdir_missing_info; - /** List of the responsible HSDirs (their b64ed identity digest) last time we * uploaded this descriptor. If the set of responsible HSDirs is different * from this list, this means we received new dirinfo and we need to - * reupload our descriptor. This list is always sorted lexicographically. */ + * reupload our descriptor. */ smartlist_t *previous_hsdirs; } hs_service_descriptor_t; @@ -266,7 +259,6 @@ void hs_service_lists_fnames_for_sandbox(smartlist_t *file_list, smartlist_t *dir_list); int hs_service_set_conn_addr_port(const origin_circuit_t *circ, edge_connection_t *conn); -void hs_hsdir_set_changed_consider_reupload(void); void hs_service_dir_info_changed(void); void hs_service_run_scheduled_events(time_t now); diff --git a/src/or/include.am b/src/or/include.am index 69b505fcd4..021f5f9d5d 100644 --- a/src/or/include.am +++ b/src/or/include.am @@ -20,7 +20,6 @@ EXTRA_DIST+= src/or/ntmain.c src/or/Makefile.nmake LIBTOR_A_SOURCES = \ src/or/addressmap.c \ src/or/bridges.c \ - src/or/buffers.c \ src/or/channel.c \ src/or/channelpadding.c \ src/or/channeltls.c \ @@ -79,6 +78,11 @@ LIBTOR_A_SOURCES = \ src/or/parsecommon.c \ src/or/periodic.c \ src/or/protover.c \ + src/or/proto_cell.c \ + src/or/proto_control0.c \ + src/or/proto_ext_or.c \ + src/or/proto_http.c \ + src/or/proto_socks.c \ src/or/policies.c \ src/or/reasons.c \ src/or/relay.c \ @@ -150,7 +154,6 @@ endif ORHEADERS = \ src/or/addressmap.h \ src/or/bridges.h \ - src/or/buffers.h \ src/or/channel.h \ src/or/channelpadding.h \ src/or/channeltls.h \ @@ -215,6 +218,11 @@ ORHEADERS = \ src/or/periodic.h \ src/or/policies.h \ src/or/protover.h \ + src/or/proto_cell.h \ + src/or/proto_control0.h \ + src/or/proto_ext_or.h \ + src/or/proto_http.h \ + src/or/proto_socks.h \ src/or/reasons.h \ src/or/relay.h \ src/or/rendcache.h \ diff --git a/src/or/main.c b/src/or/main.c index 5d51d1dead..cd726776c4 100644 --- a/src/or/main.c +++ b/src/or/main.c @@ -52,6 +52,7 @@ #include "backtrace.h" #include "bridges.h" #include "buffers.h" +#include "buffers_tls.h" #include "channel.h" #include "channeltls.h" #include "channelpadding.h" @@ -835,7 +836,7 @@ conn_close_if_marked(int i) (int)conn->outbuf_flushlen, conn->marked_for_close_file, conn->marked_for_close); if (conn->linked_conn) { - retval = move_buf_to_buf(conn->linked_conn->inbuf, conn->outbuf, + retval = buf_move_to_buf(conn->linked_conn->inbuf, conn->outbuf, &conn->outbuf_flushlen); if (retval >= 0) { /* The linked conn will notice that it has data when it notices that @@ -849,12 +850,13 @@ conn_close_if_marked(int i) connection_wants_to_flush(conn)); } else if (connection_speaks_cells(conn)) { if (conn->state == OR_CONN_STATE_OPEN) { - retval = flush_buf_tls(TO_OR_CONN(conn)->tls, conn->outbuf, sz, + retval = buf_flush_to_tls(conn->outbuf, TO_OR_CONN(conn)->tls, sz, &conn->outbuf_flushlen); } else retval = -1; /* never flush non-open broken tls connections */ } else { - retval = flush_buf(conn->s, conn->outbuf, sz, &conn->outbuf_flushlen); + retval = buf_flush_to_socket(conn->outbuf, conn->s, sz, + &conn->outbuf_flushlen); } if (retval >= 0 && /* Technically, we could survive things like TLS_WANT_WRITE here. But don't bother for now. */ @@ -1483,7 +1485,7 @@ run_scheduled_events(time_t now) /* 12. launch diff computations. (This is free if there are none to * launch.) */ - if (server_mode(options)) { + if (dir_server_mode(options)) { consdiffmgr_rescan(); } } @@ -2110,7 +2112,8 @@ hs_service_callback(time_t now, const or_options_t *options) /* We need to at least be able to build circuits and that we actually have * a working network. */ - if (!have_completed_a_circuit() || net_is_disabled()) { + if (!have_completed_a_circuit() || net_is_disabled() || + networkstatus_get_live_consensus(now) == NULL) { goto end; } diff --git a/src/or/networkstatus.c b/src/or/networkstatus.c index 69bff55cff..e25a3d316a 100644 --- a/src/or/networkstatus.c +++ b/src/or/networkstatus.c @@ -1683,7 +1683,8 @@ any_client_port_set(const or_options_t *options) options->TransPort_set || options->NATDPort_set || options->ControlPort_set || - options->DNSPort_set); + options->DNSPort_set || + options->HTTPTunnelPort_set); } /** @@ -2021,7 +2022,7 @@ networkstatus_set_current_consensus(const char *consensus, &c->digests, c->digest_sha3_as_signed, c->valid_after); - if (server_mode(get_options())) { + if (dir_server_mode(get_options())) { consdiffmgr_add_consensus(consensus, c); } } diff --git a/src/or/nodelist.c b/src/or/nodelist.c index f670870f61..03b315e682 100644 --- a/src/or/nodelist.c +++ b/src/or/nodelist.c @@ -1832,7 +1832,7 @@ router_dir_info_changed(void) { need_to_update_have_min_dir_info = 1; rend_hsdir_routers_changed(); - hs_hsdir_set_changed_consider_reupload(); + hs_service_dir_info_changed(); } /** Return a string describing what we're missing before we have enough diff --git a/src/or/or.h b/src/or/or.h index a2707b4d40..5a543b498c 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -226,8 +226,10 @@ typedef enum { #define CONN_TYPE_EXT_OR 16 /** Type for sockets listening for Extended ORPort connections. */ #define CONN_TYPE_EXT_OR_LISTENER 17 +/** Type for sockets listening for HTTP CONNECT tunnel connections. */ +#define CONN_TYPE_AP_HTTP_CONNECT_LISTENER 18 -#define CONN_TYPE_MAX_ 17 +#define CONN_TYPE_MAX_ 19 /* !!!! If _CONN_TYPE_MAX is ever over 31, we must grow the type field in * connection_t. */ @@ -348,7 +350,9 @@ typedef enum { /** State for a transparent natd connection: waiting for original * destination. */ #define AP_CONN_STATE_NATD_WAIT 12 -#define AP_CONN_STATE_MAX_ 12 +/** State for an HTTP tunnel: waiting for an HTTP CONNECT command. */ +#define AP_CONN_STATE_HTTP_CONNECT_WAIT 13 +#define AP_CONN_STATE_MAX_ 13 /** True iff the AP_CONN_STATE_* value <b>s</b> means that the corresponding * edge connection is not attached to any circuit. */ @@ -648,6 +652,10 @@ typedef enum { /** The target address is in a private network (like 127.0.0.1 or 10.0.0.1); * you don't want to do that over a randomly chosen exit */ #define END_STREAM_REASON_PRIVATE_ADDR 262 +/** This is an HTTP tunnel connection and the client used or misused HTTP in a + * way we can't handle. + */ +#define END_STREAM_REASON_HTTPPROTOCOL 263 /** Bitwise-and this value with endreason to mask out all flags. */ #define END_STREAM_REASON_MASK 511 @@ -1179,11 +1187,8 @@ typedef struct { uint16_t length; /**< How long is the payload body? */ } relay_header_t; -typedef struct buf_t buf_t; typedef struct socks_request_t socks_request_t; -#define buf_t buf_t - typedef struct entry_port_cfg_t { /* Client port types (socks, dns, trans, natd) only: */ uint8_t isolation_flags; /**< Zero or more isolation flags */ @@ -1243,6 +1248,8 @@ typedef struct server_port_cfg_t { #define CONTROL_CONNECTION_MAGIC 0x8abc765du #define LISTENER_CONNECTION_MAGIC 0x1a1ac741u +struct buf_t; + /** Description of a connection to another host or process, and associated * data. * @@ -1314,8 +1321,9 @@ typedef struct connection_t { struct event *read_event; /**< Libevent event structure. */ struct event *write_event; /**< Libevent event structure. */ - buf_t *inbuf; /**< Buffer holding data read over this connection. */ - buf_t *outbuf; /**< Buffer holding data to write over this connection. */ + struct buf_t *inbuf; /**< Buffer holding data read over this connection. */ + struct buf_t *outbuf; /**< Buffer holding data to write over this + * connection. */ size_t outbuf_flushlen; /**< How much data should we try to flush from the * outbuf? */ time_t timestamp_lastread; /**< When was the last time libevent said we could @@ -1722,11 +1730,11 @@ typedef struct entry_connection_t { /** For AP connections only: buffer for data that we have sent * optimistically, which we might need to re-send if we have to * retry this connection. */ - buf_t *pending_optimistic_data; + struct buf_t *pending_optimistic_data; /* For AP connections only: buffer for data that we previously sent * optimistically which we are currently re-sending as we retry this * connection. */ - buf_t *sending_optimistic_data; + struct buf_t *sending_optimistic_data; /** If this is a DNSPort connection, this field holds the pending DNS * request that we're going to try to answer. */ @@ -3701,6 +3709,8 @@ typedef struct { } TransProxyType_parsed; config_line_t *NATDPort_lines; /**< Ports to listen on for transparent natd * connections. */ + /** Ports to listen on for HTTP Tunnel connections. */ + config_line_t *HTTPTunnelPort_lines; config_line_t *ControlPort_lines; /**< Ports to listen on for control * connections. */ config_line_t *ControlSocket; /**< List of Unix Domain Sockets to listen on @@ -3727,7 +3737,8 @@ typedef struct { * configured in one of the _lines options above. * For client ports, also true if there is a unix socket configured. * If you are checking for client ports, you may want to use: - * SocksPort_set || TransPort_set || NATDPort_set || DNSPort_set + * SocksPort_set || TransPort_set || NATDPort_set || DNSPort_set || + * HTTPTunnelPort_set * rather than SocksPort_set. * * @{ @@ -3740,6 +3751,7 @@ typedef struct { unsigned int DirPort_set : 1; unsigned int DNSPort_set : 1; unsigned int ExtORPort_set : 1; + unsigned int HTTPTunnelPort_set : 1; /**@}*/ int AssumeReachable; /**< Whether to publish our descriptor regardless. */ diff --git a/src/or/proto_cell.c b/src/or/proto_cell.c new file mode 100644 index 0000000000..4485ab4e9e --- /dev/null +++ b/src/or/proto_cell.c @@ -0,0 +1,83 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "or.h" +#include "buffers.h" +#include "proto_cell.h" + +#include "connection_or.h" + +/** True iff the cell command <b>command</b> is one that implies a + * variable-length cell in Tor link protocol <b>linkproto</b>. */ +static inline int +cell_command_is_var_length(uint8_t command, int linkproto) +{ + /* If linkproto is v2 (2), CELL_VERSIONS is the only variable-length cells + * work as implemented here. If it's 1, there are no variable-length cells. + * Tor does not support other versions right now, and so can't negotiate + * them. + */ + switch (linkproto) { + case 1: + /* Link protocol version 1 has no variable-length cells. */ + return 0; + case 2: + /* In link protocol version 2, VERSIONS is the only variable-length cell */ + return command == CELL_VERSIONS; + case 0: + case 3: + default: + /* In link protocol version 3 and later, and in version "unknown", + * commands 128 and higher indicate variable-length. VERSIONS is + * grandfathered in. */ + return command == CELL_VERSIONS || command >= 128; + } +} + +/** Check <b>buf</b> for a variable-length cell according to the rules of link + * protocol version <b>linkproto</b>. If one is found, pull it off the buffer + * and assign a newly allocated var_cell_t to *<b>out</b>, and return 1. + * Return 0 if whatever is on the start of buf_t is not a variable-length + * cell. Return 1 and set *<b>out</b> to NULL if there seems to be the start + * of a variable-length cell on <b>buf</b>, but the whole thing isn't there + * yet. */ +int +fetch_var_cell_from_buf(buf_t *buf, var_cell_t **out, int linkproto) +{ + char hdr[VAR_CELL_MAX_HEADER_SIZE]; + var_cell_t *result; + uint8_t command; + uint16_t length; + const int wide_circ_ids = linkproto >= MIN_LINK_PROTO_FOR_WIDE_CIRC_IDS; + const int circ_id_len = get_circ_id_size(wide_circ_ids); + const unsigned header_len = get_var_cell_header_size(wide_circ_ids); + *out = NULL; + if (buf_datalen(buf) < header_len) + return 0; + buf_peek(buf, hdr, header_len); + + command = get_uint8(hdr + circ_id_len); + if (!(cell_command_is_var_length(command, linkproto))) + return 0; + + length = ntohs(get_uint16(hdr + circ_id_len + 1)); + if (buf_datalen(buf) < (size_t)(header_len+length)) + return 1; + result = var_cell_new(length); + result->command = command; + if (wide_circ_ids) + result->circ_id = ntohl(get_uint32(hdr)); + else + result->circ_id = ntohs(get_uint16(hdr)); + + buf_drain(buf, header_len); + buf_peek(buf, (char*) result->payload, length); + buf_drain(buf, length); + + *out = result; + return 1; +} + diff --git a/src/or/proto_cell.h b/src/or/proto_cell.h new file mode 100644 index 0000000000..91729a391e --- /dev/null +++ b/src/or/proto_cell.h @@ -0,0 +1,17 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_PROTO_CELL_H +#define TOR_PROTO_CELL_H + +struct buf_t; +struct var_cell_t; + +int fetch_var_cell_from_buf(struct buf_t *buf, struct var_cell_t **out, + int linkproto); + +#endif + diff --git a/src/or/proto_control0.c b/src/or/proto_control0.c new file mode 100644 index 0000000000..c17ba34948 --- /dev/null +++ b/src/or/proto_control0.c @@ -0,0 +1,26 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "or.h" +#include "buffers.h" +#include "proto_control0.h" + +/** Return 1 iff buf looks more like it has an (obsolete) v0 controller + * command on it than any valid v1 controller command. */ +int +peek_buf_has_control0_command(buf_t *buf) +{ + if (buf_datalen(buf) >= 4) { + char header[4]; + uint16_t cmd; + buf_peek(buf, header, sizeof(header)); + cmd = ntohs(get_uint16(header+2)); + if (cmd <= 0x14) + return 1; /* This is definitely not a v1 control command. */ + } + return 0; +} + diff --git a/src/or/proto_control0.h b/src/or/proto_control0.h new file mode 100644 index 0000000000..6df6bebaa5 --- /dev/null +++ b/src/or/proto_control0.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_PROTO_CONTROL0_H +#define TOR_PROTO_CONTROL0_H + +struct buf_t; +int peek_buf_has_control0_command(struct buf_t *buf); + +#endif + diff --git a/src/or/proto_ext_or.c b/src/or/proto_ext_or.c new file mode 100644 index 0000000000..057cf109ec --- /dev/null +++ b/src/or/proto_ext_or.c @@ -0,0 +1,40 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "or.h" +#include "buffers.h" +#include "ext_orport.h" +#include "proto_ext_or.h" + +/** The size of the header of an Extended ORPort message: 2 bytes for + * COMMAND, 2 bytes for BODYLEN */ +#define EXT_OR_CMD_HEADER_SIZE 4 + +/** Read <b>buf</b>, which should contain an Extended ORPort message + * from a transport proxy. If well-formed, create and populate + * <b>out</b> with the Extended ORport message. Return 0 if the + * buffer was incomplete, 1 if it was well-formed and -1 if we + * encountered an error while parsing it. */ +int +fetch_ext_or_command_from_buf(buf_t *buf, ext_or_cmd_t **out) +{ + char hdr[EXT_OR_CMD_HEADER_SIZE]; + uint16_t len; + + if (buf_datalen(buf) < EXT_OR_CMD_HEADER_SIZE) + return 0; + buf_peek(buf, hdr, sizeof(hdr)); + len = ntohs(get_uint16(hdr+2)); + if (buf_datalen(buf) < (unsigned)len + EXT_OR_CMD_HEADER_SIZE) + return 0; + *out = ext_or_cmd_new(len); + (*out)->cmd = ntohs(get_uint16(hdr)); + (*out)->len = len; + buf_drain(buf, EXT_OR_CMD_HEADER_SIZE); + buf_get_bytes(buf, (*out)->body, len); + return 1; +} + diff --git a/src/or/proto_ext_or.h b/src/or/proto_ext_or.h new file mode 100644 index 0000000000..2ef185356c --- /dev/null +++ b/src/or/proto_ext_or.h @@ -0,0 +1,17 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_PROTO_EXT_OR_H +#define TOR_PROTO_EXT_OR_H + +struct buf_t; +struct ext_or_cmt_t; + +int fetch_ext_or_command_from_buf(struct buf_t *buf, + struct ext_or_cmd_t **out); + +#endif + diff --git a/src/or/proto_http.c b/src/or/proto_http.c new file mode 100644 index 0000000000..3762429e1e --- /dev/null +++ b/src/or/proto_http.c @@ -0,0 +1,171 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#define PROTO_HTTP_PRIVATE +#include "or.h" +#include "buffers.h" +#include "proto_http.h" + +/** Return true if <b>cmd</b> looks like a HTTP (proxy) request. */ +int +peek_buf_has_http_command(const buf_t *buf) +{ + if (buf_peek_startswith(buf, "CONNECT ") || + buf_peek_startswith(buf, "DELETE ") || + buf_peek_startswith(buf, "GET ") || + buf_peek_startswith(buf, "POST ") || + buf_peek_startswith(buf, "PUT " )) + return 1; + return 0; +} + +/** There is a (possibly incomplete) http statement on <b>buf</b>, of the + * form "\%s\\r\\n\\r\\n\%s", headers, body. (body may contain NULs.) + * If a) the headers include a Content-Length field and all bytes in + * the body are present, or b) there's no Content-Length field and + * all headers are present, then: + * + * - strdup headers into <b>*headers_out</b>, and NUL-terminate it. + * - memdup body into <b>*body_out</b>, and NUL-terminate it. + * - Then remove them from <b>buf</b>, and return 1. + * + * - If headers or body is NULL, discard that part of the buf. + * - If a headers or body doesn't fit in the arg, return -1. + * (We ensure that the headers or body don't exceed max len, + * _even if_ we're planning to discard them.) + * - If force_complete is true, then succeed even if not all of the + * content has arrived. + * + * Else, change nothing and return 0. + */ +int +fetch_from_buf_http(buf_t *buf, + char **headers_out, size_t max_headerlen, + char **body_out, size_t *body_used, size_t max_bodylen, + int force_complete) +{ + const char *headers; + size_t headerlen, bodylen, contentlen=0; + int crlf_offset; + int r; + + if (buf_datalen(buf) == 0) + return 0; + + crlf_offset = buf_find_string_offset(buf, "\r\n\r\n", 4); + if (crlf_offset > (int)max_headerlen || + (crlf_offset < 0 && buf_datalen(buf) > max_headerlen)) { + log_debug(LD_HTTP,"headers too long."); + return -1; + } else if (crlf_offset < 0) { + log_debug(LD_HTTP,"headers not all here yet."); + return 0; + } + /* Okay, we have a full header. Make sure it all appears in the first + * chunk. */ + headerlen = crlf_offset + 4; + size_t headers_in_chunk = 0; + buf_pullup(buf, headerlen, &headers, &headers_in_chunk); + + bodylen = buf_datalen(buf) - headerlen; + log_debug(LD_HTTP,"headerlen %d, bodylen %d.", (int)headerlen, (int)bodylen); + + if (max_headerlen <= headerlen) { + log_warn(LD_HTTP,"headerlen %d larger than %d. Failing.", + (int)headerlen, (int)max_headerlen-1); + return -1; + } + if (max_bodylen <= bodylen) { + log_warn(LD_HTTP,"bodylen %d larger than %d. Failing.", + (int)bodylen, (int)max_bodylen-1); + return -1; + } + + r = buf_http_find_content_length(headers, headerlen, &contentlen); + if (r == -1) { + log_warn(LD_PROTOCOL, "Content-Length is bogus; maybe " + "someone is trying to crash us."); + return -1; + } else if (r == 1) { + /* if content-length is malformed, then our body length is 0. fine. */ + log_debug(LD_HTTP,"Got a contentlen of %d.",(int)contentlen); + if (bodylen < contentlen) { + if (!force_complete) { + log_debug(LD_HTTP,"body not all here yet."); + return 0; /* not all there yet */ + } + } + if (bodylen > contentlen) { + bodylen = contentlen; + log_debug(LD_HTTP,"bodylen reduced to %d.",(int)bodylen); + } + } else { + tor_assert(r == 0); + /* Leave bodylen alone */ + } + + /* all happy. copy into the appropriate places, and return 1 */ + if (headers_out) { + *headers_out = tor_malloc(headerlen+1); + buf_get_bytes(buf, *headers_out, headerlen); + (*headers_out)[headerlen] = 0; /* NUL terminate it */ + } + if (body_out) { + tor_assert(body_used); + *body_used = bodylen; + *body_out = tor_malloc(bodylen+1); + buf_get_bytes(buf, *body_out, bodylen); + (*body_out)[bodylen] = 0; /* NUL terminate it */ + } + return 1; +} + +/** + * Scan the HTTP headers in the <b>headerlen</b>-byte memory range at + * <b>headers</b>, looking for a "Content-Length" header. Try to set + * *<b>result_out</b> to the numeric value of that header if possible. + * Return -1 if the header was malformed, 0 if it was missing, and 1 if + * it was present and well-formed. + */ +STATIC int +buf_http_find_content_length(const char *headers, size_t headerlen, + size_t *result_out) +{ + const char *p, *newline; + char *len_str, *eos=NULL; + size_t remaining, result; + int ok; + *result_out = 0; /* The caller shouldn't look at this unless the + * return value is 1, but let's prevent confusion */ + +#define CONTENT_LENGTH "\r\nContent-Length: " + p = (char*) tor_memstr(headers, headerlen, CONTENT_LENGTH); + if (p == NULL) + return 0; + + tor_assert(p >= headers && p < headers+headerlen); + remaining = (headers+headerlen)-p; + p += strlen(CONTENT_LENGTH); + remaining -= strlen(CONTENT_LENGTH); + + newline = memchr(p, '\n', remaining); + if (newline == NULL) + return -1; + + len_str = tor_memdup_nulterm(p, newline-p); + /* We limit the size to INT_MAX because other parts of the buffer.c + * code don't like buffers to be any bigger than that. */ + result = (size_t) tor_parse_uint64(len_str, 10, 0, INT_MAX, &ok, &eos); + if (eos && !tor_strisspace(eos)) { + ok = 0; + } else { + *result_out = result; + } + tor_free(len_str); + + return ok ? 1 : -1; +} + diff --git a/src/or/proto_http.h b/src/or/proto_http.h new file mode 100644 index 0000000000..dbff823cce --- /dev/null +++ b/src/or/proto_http.h @@ -0,0 +1,24 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_PROTO_HTTP_H +#define TOR_PROTO_HTTP_H + +struct buf_t; + +int fetch_from_buf_http(struct buf_t *buf, + char **headers_out, size_t max_headerlen, + char **body_out, size_t *body_used, size_t max_bodylen, + int force_complete); +int peek_buf_has_http_command(const struct buf_t *buf); + +#ifdef PROTO_HTTP_PRIVATE +STATIC int buf_http_find_content_length(const char *headers, size_t headerlen, + size_t *result_out); +#endif + +#endif + diff --git a/src/or/proto_socks.c b/src/or/proto_socks.c new file mode 100644 index 0000000000..f92d614918 --- /dev/null +++ b/src/or/proto_socks.c @@ -0,0 +1,698 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "or.h" +#include "addressmap.h" +#include "buffers.h" +#include "control.h" +#include "config.h" +#include "ext_orport.h" +#include "proto_socks.h" +#include "reasons.h" + +static void socks_request_set_socks5_error(socks_request_t *req, + socks5_reply_status_t reason); + +static int parse_socks(const char *data, size_t datalen, socks_request_t *req, + int log_sockstype, int safe_socks, ssize_t *drain_out, + size_t *want_length_out); +static int parse_socks_client(const uint8_t *data, size_t datalen, + int state, char **reason, + ssize_t *drain_out); +/** + * Wait this many seconds before warning the user about using SOCKS unsafely + * again. */ +#define SOCKS_WARN_INTERVAL 5 + +/** Warn that the user application has made an unsafe socks request using + * protocol <b>socks_protocol</b> on port <b>port</b>. Don't warn more than + * once per SOCKS_WARN_INTERVAL, unless <b>safe_socks</b> is set. */ +static void +log_unsafe_socks_warning(int socks_protocol, const char *address, + uint16_t port, int safe_socks) +{ + static ratelim_t socks_ratelim = RATELIM_INIT(SOCKS_WARN_INTERVAL); + + if (safe_socks) { + log_fn_ratelim(&socks_ratelim, LOG_WARN, LD_APP, + "Your application (using socks%d to port %d) is giving " + "Tor only an IP address. Applications that do DNS resolves " + "themselves may leak information. Consider using Socks4A " + "(e.g. via privoxy or socat) instead. For more information, " + "please see https://wiki.torproject.org/TheOnionRouter/" + "TorFAQ#SOCKSAndDNS.%s", + socks_protocol, + (int)port, + safe_socks ? " Rejecting." : ""); + } + control_event_client_status(LOG_WARN, + "DANGEROUS_SOCKS PROTOCOL=SOCKS%d ADDRESS=%s:%d", + socks_protocol, address, (int)port); +} + +/** Do not attempt to parse socks messages longer than this. This value is + * actually significantly higher than the longest possible socks message. */ +#define MAX_SOCKS_MESSAGE_LEN 512 + +/** Return a new socks_request_t. */ +socks_request_t * +socks_request_new(void) +{ + return tor_malloc_zero(sizeof(socks_request_t)); +} + +/** Free all storage held in the socks_request_t <b>req</b>. */ +void +socks_request_free(socks_request_t *req) +{ + if (!req) + return; + if (req->username) { + memwipe(req->username, 0x10, req->usernamelen); + tor_free(req->username); + } + if (req->password) { + memwipe(req->password, 0x04, req->passwordlen); + tor_free(req->password); + } + memwipe(req, 0xCC, sizeof(socks_request_t)); + tor_free(req); +} + +/** There is a (possibly incomplete) socks handshake on <b>buf</b>, of one + * of the forms + * - socks4: "socksheader username\\0" + * - socks4a: "socksheader username\\0 destaddr\\0" + * - socks5 phase one: "version #methods methods" + * - socks5 phase two: "version command 0 addresstype..." + * If it's a complete and valid handshake, and destaddr fits in + * MAX_SOCKS_ADDR_LEN bytes, then pull the handshake off the buf, + * assign to <b>req</b>, and return 1. + * + * If it's invalid or too big, return -1. + * + * Else it's not all there yet, leave buf alone and return 0. + * + * If you want to specify the socks reply, write it into <b>req->reply</b> + * and set <b>req->replylen</b>, else leave <b>req->replylen</b> alone. + * + * If <b>log_sockstype</b> is non-zero, then do a notice-level log of whether + * the connection is possibly leaking DNS requests locally or not. + * + * If <b>safe_socks</b> is true, then reject unsafe socks protocols. + * + * If returning 0 or -1, <b>req->address</b> and <b>req->port</b> are + * undefined. + */ +int +fetch_from_buf_socks(buf_t *buf, socks_request_t *req, + int log_sockstype, int safe_socks) +{ + int res; + ssize_t n_drain; + size_t want_length = 128; + const char *head = NULL; + size_t datalen = 0; + + if (buf_datalen(buf) < 2) /* version and another byte */ + return 0; + + do { + n_drain = 0; + buf_pullup(buf, want_length, &head, &datalen); + tor_assert(head && datalen >= 2); + want_length = 0; + + res = parse_socks(head, datalen, req, log_sockstype, + safe_socks, &n_drain, &want_length); + + if (n_drain < 0) + buf_clear(buf); + else if (n_drain > 0) + buf_drain(buf, n_drain); + + } while (res == 0 && head && want_length < buf_datalen(buf) && + buf_datalen(buf) >= 2); + + return res; +} + +/** Create a SOCKS5 reply message with <b>reason</b> in its REP field and + * have Tor send it as error response to <b>req</b>. + */ +static void +socks_request_set_socks5_error(socks_request_t *req, + socks5_reply_status_t reason) +{ + req->replylen = 10; + memset(req->reply,0,10); + + req->reply[0] = 0x05; // VER field. + req->reply[1] = reason; // REP field. + req->reply[3] = 0x01; // ATYP field. +} + +static const char SOCKS_PROXY_IS_NOT_AN_HTTP_PROXY_MSG[] = + "HTTP/1.0 501 Tor is not an HTTP Proxy\r\n" + "Content-Type: text/html; charset=iso-8859-1\r\n\r\n" + "<html>\n" + "<head>\n" + "<title>Tor is not an HTTP Proxy</title>\n" + "</head>\n" + "<body>\n" + "<h1>Tor is not an HTTP Proxy</h1>\n" + "<p>\n" + "It appears you have configured your web browser to use Tor as " + "an HTTP proxy.\n\n" + "This is not correct: Tor is a SOCKS proxy, not an HTTP proxy.\n" + "Please configure your client accordingly.\n" + "</p>\n" + "<p>\n" + "See <a href=\"https://www.torproject.org/documentation.html\">" + "https://www.torproject.org/documentation.html</a> for more " + "information.\n" + "<!-- Plus this comment, to make the body response more than 512 bytes, so " + " IE will be willing to display it. Comment comment comment comment " + " comment comment comment comment comment comment comment comment.-->\n" + "</p>\n" + "</body>\n" + "</html>\n"; + +/** Implementation helper to implement fetch_from_*_socks. Instead of looking + * at a buffer's contents, we look at the <b>datalen</b> bytes of data in + * <b>data</b>. Instead of removing data from the buffer, we set + * <b>drain_out</b> to the amount of data that should be removed (or -1 if the + * buffer should be cleared). Instead of pulling more data into the first + * chunk of the buffer, we set *<b>want_length_out</b> to the number of bytes + * we'd like to see in the input buffer, if they're available. */ +static int +parse_socks(const char *data, size_t datalen, socks_request_t *req, + int log_sockstype, int safe_socks, ssize_t *drain_out, + size_t *want_length_out) +{ + unsigned int len; + char tmpbuf[TOR_ADDR_BUF_LEN+1]; + tor_addr_t destaddr; + uint32_t destip; + uint8_t socksver; + char *next, *startaddr; + unsigned char usernamelen, passlen; + struct in_addr in; + + if (datalen < 2) { + /* We always need at least 2 bytes. */ + *want_length_out = 2; + return 0; + } + + if (req->socks_version == 5 && !req->got_auth) { + /* See if we have received authentication. Strictly speaking, we should + also check whether we actually negotiated username/password + authentication. But some broken clients will send us authentication + even if we negotiated SOCKS_NO_AUTH. */ + if (*data == 1) { /* username/pass version 1 */ + /* Format is: authversion [1 byte] == 1 + usernamelen [1 byte] + username [usernamelen bytes] + passlen [1 byte] + password [passlen bytes] */ + usernamelen = (unsigned char)*(data + 1); + if (datalen < 2u + usernamelen + 1u) { + *want_length_out = 2u + usernamelen + 1u; + return 0; + } + passlen = (unsigned char)*(data + 2u + usernamelen); + if (datalen < 2u + usernamelen + 1u + passlen) { + *want_length_out = 2u + usernamelen + 1u + passlen; + return 0; + } + req->replylen = 2; /* 2 bytes of response */ + req->reply[0] = 1; /* authversion == 1 */ + req->reply[1] = 0; /* authentication successful */ + log_debug(LD_APP, + "socks5: Accepted username/password without checking."); + if (usernamelen) { + req->username = tor_memdup(data+2u, usernamelen); + req->usernamelen = usernamelen; + } + if (passlen) { + req->password = tor_memdup(data+3u+usernamelen, passlen); + req->passwordlen = passlen; + } + *drain_out = 2u + usernamelen + 1u + passlen; + req->got_auth = 1; + *want_length_out = 7; /* Minimal socks5 command. */ + return 0; + } else if (req->auth_type == SOCKS_USER_PASS) { + /* unknown version byte */ + log_warn(LD_APP, "Socks5 username/password version %d not recognized; " + "rejecting.", (int)*data); + return -1; + } + } + + socksver = *data; + + switch (socksver) { /* which version of socks? */ + case 5: /* socks5 */ + + if (req->socks_version != 5) { /* we need to negotiate a method */ + unsigned char nummethods = (unsigned char)*(data+1); + int have_user_pass, have_no_auth; + int r=0; + tor_assert(!req->socks_version); + if (datalen < 2u+nummethods) { + *want_length_out = 2u+nummethods; + return 0; + } + if (!nummethods) + return -1; + req->replylen = 2; /* 2 bytes of response */ + req->reply[0] = 5; /* socks5 reply */ + have_user_pass = (memchr(data+2, SOCKS_USER_PASS, nummethods) !=NULL); + have_no_auth = (memchr(data+2, SOCKS_NO_AUTH, nummethods) !=NULL); + if (have_user_pass && !(have_no_auth && req->socks_prefer_no_auth)) { + req->auth_type = SOCKS_USER_PASS; + req->reply[1] = SOCKS_USER_PASS; /* tell client to use "user/pass" + auth method */ + req->socks_version = 5; /* remember we've already negotiated auth */ + log_debug(LD_APP,"socks5: accepted method 2 (username/password)"); + r=0; + } else if (have_no_auth) { + req->reply[1] = SOCKS_NO_AUTH; /* tell client to use "none" auth + method */ + req->socks_version = 5; /* remember we've already negotiated auth */ + log_debug(LD_APP,"socks5: accepted method 0 (no authentication)"); + r=0; + } else { + log_warn(LD_APP, + "socks5: offered methods don't include 'no auth' or " + "username/password. Rejecting."); + req->reply[1] = '\xFF'; /* reject all methods */ + r=-1; + } + /* Remove packet from buf. Some SOCKS clients will have sent extra + * junk at this point; let's hope it's an authentication message. */ + *drain_out = 2u + nummethods; + + return r; + } + if (req->auth_type != SOCKS_NO_AUTH && !req->got_auth) { + log_warn(LD_APP, + "socks5: negotiated authentication, but none provided"); + return -1; + } + /* we know the method; read in the request */ + log_debug(LD_APP,"socks5: checking request"); + if (datalen < 7) {/* basic info plus >=1 for addr plus 2 for port */ + *want_length_out = 7; + return 0; /* not yet */ + } + req->command = (unsigned char) *(data+1); + if (req->command != SOCKS_COMMAND_CONNECT && + req->command != SOCKS_COMMAND_RESOLVE && + req->command != SOCKS_COMMAND_RESOLVE_PTR) { + /* not a connect or resolve or a resolve_ptr? we don't support it. */ + socks_request_set_socks5_error(req,SOCKS5_COMMAND_NOT_SUPPORTED); + + log_warn(LD_APP,"socks5: command %d not recognized. Rejecting.", + req->command); + return -1; + } + switch (*(data+3)) { /* address type */ + case 1: /* IPv4 address */ + case 4: /* IPv6 address */ { + const int is_v6 = *(data+3) == 4; + const unsigned addrlen = is_v6 ? 16 : 4; + log_debug(LD_APP,"socks5: ipv4 address type"); + if (datalen < 6+addrlen) {/* ip/port there? */ + *want_length_out = 6+addrlen; + return 0; /* not yet */ + } + + if (is_v6) + tor_addr_from_ipv6_bytes(&destaddr, data+4); + else + tor_addr_from_ipv4n(&destaddr, get_uint32(data+4)); + + tor_addr_to_str(tmpbuf, &destaddr, sizeof(tmpbuf), 1); + + if (strlen(tmpbuf)+1 > MAX_SOCKS_ADDR_LEN) { + socks_request_set_socks5_error(req, SOCKS5_GENERAL_ERROR); + log_warn(LD_APP, + "socks5 IP takes %d bytes, which doesn't fit in %d. " + "Rejecting.", + (int)strlen(tmpbuf)+1,(int)MAX_SOCKS_ADDR_LEN); + return -1; + } + strlcpy(req->address,tmpbuf,sizeof(req->address)); + req->port = ntohs(get_uint16(data+4+addrlen)); + *drain_out = 6+addrlen; + if (req->command != SOCKS_COMMAND_RESOLVE_PTR && + !addressmap_have_mapping(req->address,0)) { + log_unsafe_socks_warning(5, req->address, req->port, safe_socks); + if (safe_socks) { + socks_request_set_socks5_error(req, SOCKS5_NOT_ALLOWED); + return -1; + } + } + return 1; + } + case 3: /* fqdn */ + log_debug(LD_APP,"socks5: fqdn address type"); + if (req->command == SOCKS_COMMAND_RESOLVE_PTR) { + socks_request_set_socks5_error(req, + SOCKS5_ADDRESS_TYPE_NOT_SUPPORTED); + log_warn(LD_APP, "socks5 received RESOLVE_PTR command with " + "hostname type. Rejecting."); + return -1; + } + len = (unsigned char)*(data+4); + if (datalen < 7+len) { /* addr/port there? */ + *want_length_out = 7+len; + return 0; /* not yet */ + } + if (len+1 > MAX_SOCKS_ADDR_LEN) { + socks_request_set_socks5_error(req, SOCKS5_GENERAL_ERROR); + log_warn(LD_APP, + "socks5 hostname is %d bytes, which doesn't fit in " + "%d. Rejecting.", len+1,MAX_SOCKS_ADDR_LEN); + return -1; + } + memcpy(req->address,data+5,len); + req->address[len] = 0; + req->port = ntohs(get_uint16(data+5+len)); + *drain_out = 5+len+2; + + if (!string_is_valid_hostname(req->address)) { + socks_request_set_socks5_error(req, SOCKS5_GENERAL_ERROR); + + log_warn(LD_PROTOCOL, + "Your application (using socks5 to port %d) gave Tor " + "a malformed hostname: %s. Rejecting the connection.", + req->port, escaped_safe_str_client(req->address)); + return -1; + } + if (log_sockstype) + log_notice(LD_APP, + "Your application (using socks5 to port %d) instructed " + "Tor to take care of the DNS resolution itself if " + "necessary. This is good.", req->port); + return 1; + default: /* unsupported */ + socks_request_set_socks5_error(req, + SOCKS5_ADDRESS_TYPE_NOT_SUPPORTED); + log_warn(LD_APP,"socks5: unsupported address type %d. Rejecting.", + (int) *(data+3)); + return -1; + } + tor_assert(0); + break; + case 4: { /* socks4 */ + enum {socks4, socks4a} socks4_prot = socks4a; + const char *authstart, *authend; + /* http://ss5.sourceforge.net/socks4.protocol.txt */ + /* http://ss5.sourceforge.net/socks4A.protocol.txt */ + + req->socks_version = 4; + if (datalen < SOCKS4_NETWORK_LEN) {/* basic info available? */ + *want_length_out = SOCKS4_NETWORK_LEN; + return 0; /* not yet */ + } + // buf_pullup(buf, 1280); + req->command = (unsigned char) *(data+1); + if (req->command != SOCKS_COMMAND_CONNECT && + req->command != SOCKS_COMMAND_RESOLVE) { + /* not a connect or resolve? we don't support it. (No resolve_ptr with + * socks4.) */ + log_warn(LD_APP,"socks4: command %d not recognized. Rejecting.", + req->command); + return -1; + } + + req->port = ntohs(get_uint16(data+2)); + destip = ntohl(get_uint32(data+4)); + if ((!req->port && req->command!=SOCKS_COMMAND_RESOLVE) || !destip) { + log_warn(LD_APP,"socks4: Port or DestIP is zero. Rejecting."); + return -1; + } + if (destip >> 8) { + log_debug(LD_APP,"socks4: destip not in form 0.0.0.x."); + in.s_addr = htonl(destip); + tor_inet_ntoa(&in,tmpbuf,sizeof(tmpbuf)); + if (strlen(tmpbuf)+1 > MAX_SOCKS_ADDR_LEN) { + log_debug(LD_APP,"socks4 addr (%d bytes) too long. Rejecting.", + (int)strlen(tmpbuf)); + return -1; + } + log_debug(LD_APP, + "socks4: successfully read destip (%s)", + safe_str_client(tmpbuf)); + socks4_prot = socks4; + } + + authstart = data + SOCKS4_NETWORK_LEN; + next = memchr(authstart, 0, + datalen-SOCKS4_NETWORK_LEN); + if (!next) { + if (datalen >= 1024) { + log_debug(LD_APP, "Socks4 user name too long; rejecting."); + return -1; + } + log_debug(LD_APP,"socks4: Username not here yet."); + *want_length_out = datalen+1024; /* More than we need, but safe */ + return 0; + } + authend = next; + tor_assert(next < data+datalen); + + startaddr = NULL; + if (socks4_prot != socks4a && + !addressmap_have_mapping(tmpbuf,0)) { + log_unsafe_socks_warning(4, tmpbuf, req->port, safe_socks); + + if (safe_socks) + return -1; + } + if (socks4_prot == socks4a) { + if (next+1 == data+datalen) { + log_debug(LD_APP,"socks4: No part of destaddr here yet."); + *want_length_out = datalen + 1024; /* More than we need, but safe */ + return 0; + } + startaddr = next+1; + next = memchr(startaddr, 0, data + datalen - startaddr); + if (!next) { + if (datalen >= 1024) { + log_debug(LD_APP,"socks4: Destaddr too long."); + return -1; + } + log_debug(LD_APP,"socks4: Destaddr not all here yet."); + *want_length_out = datalen + 1024; /* More than we need, but safe */ + return 0; + } + if (MAX_SOCKS_ADDR_LEN <= next-startaddr) { + log_warn(LD_APP,"socks4: Destaddr too long. Rejecting."); + return -1; + } + // tor_assert(next < buf->cur+buf_datalen(buf)); + + if (log_sockstype) + log_notice(LD_APP, + "Your application (using socks4a to port %d) instructed " + "Tor to take care of the DNS resolution itself if " + "necessary. This is good.", req->port); + } + log_debug(LD_APP,"socks4: Everything is here. Success."); + strlcpy(req->address, startaddr ? startaddr : tmpbuf, + sizeof(req->address)); + if (!string_is_valid_hostname(req->address)) { + log_warn(LD_PROTOCOL, + "Your application (using socks4 to port %d) gave Tor " + "a malformed hostname: %s. Rejecting the connection.", + req->port, escaped_safe_str_client(req->address)); + return -1; + } + if (authend != authstart) { + req->got_auth = 1; + req->usernamelen = authend - authstart; + req->username = tor_memdup(authstart, authend - authstart); + } + /* next points to the final \0 on inbuf */ + *drain_out = next - data + 1; + return 1; + } + case 'G': /* get */ + case 'H': /* head */ + case 'P': /* put/post */ + case 'C': /* connect */ + strlcpy((char*)req->reply, SOCKS_PROXY_IS_NOT_AN_HTTP_PROXY_MSG, + MAX_SOCKS_REPLY_LEN); + req->replylen = strlen((char*)req->reply)+1; + /* fall through */ + default: /* version is not socks4 or socks5 */ + log_warn(LD_APP, + "Socks version %d not recognized. (Tor is not an http proxy.)", + *(data)); + { + /* Tell the controller the first 8 bytes. */ + char *tmp = tor_strndup(data, datalen < 8 ? datalen : 8); + control_event_client_status(LOG_WARN, + "SOCKS_UNKNOWN_PROTOCOL DATA=\"%s\"", + escaped(tmp)); + tor_free(tmp); + } + return -1; + } +} + +/** Inspect a reply from SOCKS server stored in <b>buf</b> according + * to <b>state</b>, removing the protocol data upon success. Return 0 on + * incomplete response, 1 on success and -1 on error, in which case + * <b>reason</b> is set to a descriptive message (free() when finished + * with it). + * + * As a special case, 2 is returned when user/pass is required + * during SOCKS5 handshake and user/pass is configured. + */ +int +fetch_from_buf_socks_client(buf_t *buf, int state, char **reason) +{ + ssize_t drain = 0; + int r; + const char *head = NULL; + size_t datalen = 0; + + if (buf_datalen(buf) < 2) + return 0; + + buf_pullup(buf, MAX_SOCKS_MESSAGE_LEN, &head, &datalen); + tor_assert(head && datalen >= 2); + + r = parse_socks_client((uint8_t*)head, datalen, + state, reason, &drain); + if (drain > 0) + buf_drain(buf, drain); + else if (drain < 0) + buf_clear(buf); + + return r; +} + +/** Implementation logic for fetch_from_*_socks_client. */ +static int +parse_socks_client(const uint8_t *data, size_t datalen, + int state, char **reason, + ssize_t *drain_out) +{ + unsigned int addrlen; + *drain_out = 0; + if (datalen < 2) + return 0; + + switch (state) { + case PROXY_SOCKS4_WANT_CONNECT_OK: + /* Wait for the complete response */ + if (datalen < 8) + return 0; + + if (data[1] != 0x5a) { + *reason = tor_strdup(socks4_response_code_to_string(data[1])); + return -1; + } + + /* Success */ + *drain_out = 8; + return 1; + + case PROXY_SOCKS5_WANT_AUTH_METHOD_NONE: + /* we don't have any credentials */ + if (data[1] != 0x00) { + *reason = tor_strdup("server doesn't support any of our " + "available authentication methods"); + return -1; + } + + log_info(LD_NET, "SOCKS 5 client: continuing without authentication"); + *drain_out = -1; + return 1; + + case PROXY_SOCKS5_WANT_AUTH_METHOD_RFC1929: + /* we have a username and password. return 1 if we can proceed without + * providing authentication, or 2 otherwise. */ + switch (data[1]) { + case 0x00: + log_info(LD_NET, "SOCKS 5 client: we have auth details but server " + "doesn't require authentication."); + *drain_out = -1; + return 1; + case 0x02: + log_info(LD_NET, "SOCKS 5 client: need authentication."); + *drain_out = -1; + return 2; + /* fall through */ + } + + *reason = tor_strdup("server doesn't support any of our available " + "authentication methods"); + return -1; + + case PROXY_SOCKS5_WANT_AUTH_RFC1929_OK: + /* handle server reply to rfc1929 authentication */ + if (data[1] != 0x00) { + *reason = tor_strdup("authentication failed"); + return -1; + } + + log_info(LD_NET, "SOCKS 5 client: authentication successful."); + *drain_out = -1; + return 1; + + case PROXY_SOCKS5_WANT_CONNECT_OK: + /* response is variable length. BND.ADDR, etc, isn't needed + * (don't bother with buf_pullup()), but make sure to eat all + * the data used */ + + /* wait for address type field to arrive */ + if (datalen < 4) + return 0; + + switch (data[3]) { + case 0x01: /* ip4 */ + addrlen = 4; + break; + case 0x04: /* ip6 */ + addrlen = 16; + break; + case 0x03: /* fqdn (can this happen here?) */ + if (datalen < 5) + return 0; + addrlen = 1 + data[4]; + break; + default: + *reason = tor_strdup("invalid response to connect request"); + return -1; + } + + /* wait for address and port */ + if (datalen < 6 + addrlen) + return 0; + + if (data[1] != 0x00) { + *reason = tor_strdup(socks5_response_code_to_string(data[1])); + return -1; + } + + *drain_out = 6 + addrlen; + return 1; + } + + /* shouldn't get here... */ + tor_assert(0); + + return -1; +} + diff --git a/src/or/proto_socks.h b/src/or/proto_socks.h new file mode 100644 index 0000000000..4a24773199 --- /dev/null +++ b/src/or/proto_socks.h @@ -0,0 +1,20 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_PROTO_SOCKS_H +#define TOR_PROTO_SOCKS_H + +struct socks_request_t; +struct buf_t; + +struct socks_request_t *socks_request_new(void); +void socks_request_free(struct socks_request_t *req); +int fetch_from_buf_socks(struct buf_t *buf, socks_request_t *req, + int log_sockstype, int safe_socks); +int fetch_from_buf_socks_client(buf_t *buf, int state, char **reason); + +#endif + diff --git a/src/or/reasons.c b/src/or/reasons.c index e6c325f1b3..717cf57c24 100644 --- a/src/or/reasons.c +++ b/src/or/reasons.c @@ -45,6 +45,8 @@ stream_end_reason_to_control_string(int reason) case END_STREAM_REASON_CANT_ATTACH: return "CANT_ATTACH"; case END_STREAM_REASON_NET_UNREACHABLE: return "NET_UNREACHABLE"; case END_STREAM_REASON_SOCKSPROTOCOL: return "SOCKS_PROTOCOL"; + // XXXX Controlspec + case END_STREAM_REASON_HTTPPROTOCOL: return "HTTP_PROTOCOL"; case END_STREAM_REASON_PRIVATE_ADDR: return "PRIVATE_ADDR"; @@ -138,6 +140,11 @@ stream_end_reason_to_socks5_response(int reason) return SOCKS5_NET_UNREACHABLE; case END_STREAM_REASON_SOCKSPROTOCOL: return SOCKS5_GENERAL_ERROR; + case END_STREAM_REASON_HTTPPROTOCOL: + // LCOV_EXCL_START + tor_assert_nonfatal_unreached(); + return SOCKS5_GENERAL_ERROR; + // LCOV_EXCL_STOP case END_STREAM_REASON_PRIVATE_ADDR: return SOCKS5_GENERAL_ERROR; @@ -442,3 +449,48 @@ bandwidth_weight_rule_to_string(bandwidth_weight_rule_t rule) } } +/** Given a RELAY_END reason value, convert it to an HTTP response to be + * send over an HTTP tunnel connection. */ +const char * +end_reason_to_http_connect_response_line(int endreason) +{ + endreason &= END_STREAM_REASON_MASK; + /* XXXX these are probably all wrong. Should they all be 502? */ + switch (endreason) { + case 0: + return "HTTP/1.0 200 OK\r\n\r\n"; + case END_STREAM_REASON_MISC: + return "HTTP/1.0 500 Internal Server Error\r\n\r\n"; + case END_STREAM_REASON_RESOLVEFAILED: + return "HTTP/1.0 404 Not Found (resolve failed)\r\n\r\n"; + case END_STREAM_REASON_NOROUTE: + return "HTTP/1.0 404 Not Found (no route)\r\n\r\n"; + case END_STREAM_REASON_CONNECTREFUSED: + return "HTTP/1.0 403 Forbidden (connection refused)\r\n\r\n"; + case END_STREAM_REASON_EXITPOLICY: + return "HTTP/1.0 403 Forbidden (exit policy)\r\n\r\n"; + case END_STREAM_REASON_DESTROY: + return "HTTP/1.0 502 Bad Gateway (destroy cell received)\r\n\r\n"; + case END_STREAM_REASON_DONE: + return "HTTP/1.0 502 Bad Gateway (unexpected close)\r\n\r\n"; + case END_STREAM_REASON_TIMEOUT: + return "HTTP/1.0 504 Gateway Timeout\r\n\r\n"; + case END_STREAM_REASON_HIBERNATING: + return "HTTP/1.0 502 Bad Gateway (hibernating server)\r\n\r\n"; + case END_STREAM_REASON_INTERNAL: + return "HTTP/1.0 502 Bad Gateway (internal error)\r\n\r\n"; + case END_STREAM_REASON_RESOURCELIMIT: + return "HTTP/1.0 502 Bad Gateway (resource limit)\r\n\r\n"; + case END_STREAM_REASON_CONNRESET: + return "HTTP/1.0 403 Forbidden (connection reset)\r\n\r\n"; + case END_STREAM_REASON_TORPROTOCOL: + return "HTTP/1.0 502 Bad Gateway (tor protocol violation)\r\n\r\n"; + case END_STREAM_REASON_ENTRYPOLICY: + return "HTTP/1.0 403 Forbidden (entry policy violation)\r\n\r\n"; + case END_STREAM_REASON_NOTDIRECTORY: /* Fall Through */ + default: + tor_assert_nonfatal_unreached(); + return "HTTP/1.0 500 Internal Server Error (weird end reason)\r\n\r\n"; + } +} + diff --git a/src/or/reasons.h b/src/or/reasons.h index 1cadf4e89e..f98db55bdd 100644 --- a/src/or/reasons.h +++ b/src/or/reasons.h @@ -26,6 +26,7 @@ const char *socks4_response_code_to_string(uint8_t code); const char *socks5_response_code_to_string(uint8_t code); const char *bandwidth_weight_rule_to_string(enum bandwidth_weight_rule_t rule); +const char *end_reason_to_http_connect_response_line(int endreason); #endif diff --git a/src/or/relay.c b/src/or/relay.c index 18ccc65b80..d8f09d320d 100644 --- a/src/or/relay.c +++ b/src/or/relay.c @@ -1467,8 +1467,9 @@ connection_edge_process_relay_cell_not_open( circuit_log_path(LOG_INFO,LD_APP,TO_ORIGIN_CIRCUIT(circ)); /* don't send a socks reply to transparent conns */ tor_assert(entry_conn->socks_request != NULL); - if (!entry_conn->socks_request->has_finished) + if (!entry_conn->socks_request->has_finished) { connection_ap_handshake_socks_reply(entry_conn, NULL, 0, 0); + } /* Was it a linked dir conn? If so, a dir request just started to * fetch something; this could be a bootstrap status milestone. */ @@ -1670,7 +1671,7 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, } stats_n_data_bytes_received += rh.length; - connection_write_to_buf((char*)(cell->payload + RELAY_HEADER_SIZE), + connection_buf_add((char*)(cell->payload + RELAY_HEADER_SIZE), rh.length, TO_CONN(conn)); #ifdef MEASUREMENTS_21206 @@ -2038,13 +2039,13 @@ connection_edge_package_raw_inbuf(edge_connection_t *conn, int package_partial, /* XXXX We could be more efficient here by sometimes packing * previously-sent optimistic data in the same cell with data * from the inbuf. */ - fetch_from_buf(payload, length, entry_conn->sending_optimistic_data); + buf_get_bytes(entry_conn->sending_optimistic_data, payload, length); if (!buf_datalen(entry_conn->sending_optimistic_data)) { buf_free(entry_conn->sending_optimistic_data); entry_conn->sending_optimistic_data = NULL; } } else { - connection_fetch_from_buf(payload, length, TO_CONN(conn)); + connection_buf_get_bytes(payload, length, TO_CONN(conn)); } log_debug(domain,TOR_SOCKET_T_FORMAT": Packaging %d bytes (%d waiting).", @@ -2056,7 +2057,7 @@ connection_edge_package_raw_inbuf(edge_connection_t *conn, int package_partial, retry */ if (!entry_conn->pending_optimistic_data) entry_conn->pending_optimistic_data = buf_new(); - write_to_buf(payload, length, entry_conn->pending_optimistic_data); + buf_add(entry_conn->pending_optimistic_data, payload, length); } if (connection_edge_send_command(conn, RELAY_COMMAND_DATA, diff --git a/src/or/rendservice.c b/src/or/rendservice.c index c17d77f970..9edb7cc4b7 100644 --- a/src/or/rendservice.c +++ b/src/or/rendservice.c @@ -3332,9 +3332,10 @@ rend_service_rendezvous_has_opened(origin_circuit_t *circuit) NULL); rend_cookie = circuit->rend_data->rend_cookie; - /* Declare the circuit dirty to avoid reuse, and for path-bias */ - if (!circuit->base_.timestamp_dirty) - circuit->base_.timestamp_dirty = time(NULL); + /* Declare the circuit dirty to avoid reuse, and for path-bias. We set the + * timestamp regardless of its content because that circuit could have been + * cannibalized so in any cases, we are about to use that circuit more. */ + circuit->base_.timestamp_dirty = time(NULL); /* This may be redundant */ pathbias_count_use_attempt(circuit); diff --git a/src/test/fuzz/fuzz_http_connect.c b/src/test/fuzz/fuzz_http_connect.c new file mode 100644 index 0000000000..dc674070b2 --- /dev/null +++ b/src/test/fuzz/fuzz_http_connect.c @@ -0,0 +1,106 @@ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" + +#define BUFFERS_PRIVATE +#define CONNECTION_EDGE_PRIVATE + +#include "or.h" +#include "backtrace.h" +#include "buffers.h" +#include "config.h" +#include "connection.h" +#include "connection_edge.h" +#include "proto_socks.h" +#include "torlog.h" + +#include "fuzzing.h" + +static void +mock_connection_write_to_buf_impl_(const char *string, size_t len, + connection_t *conn, int compressed) +{ + log_debug(LD_GENERAL, "%sResponse:\n%u\nConnection: %p\n%s\n", + compressed ? "Compressed " : "", (unsigned)len, conn, string); +} + +static void +mock_connection_mark_unattached_ap_(entry_connection_t *conn, int endreason, + int line, const char *file) +{ + (void)conn; + (void)endreason; + (void)line; + (void)file; +} + +static int +mock_connection_ap_rewrite_and_attach_if_allowed(entry_connection_t *conn, + origin_circuit_t *circ, + crypt_path_t *cpath) +{ + (void)conn; + (void)circ; + (void)cpath; + return 0; +} + +int +fuzz_init(void) +{ + /* Set up fake response handler */ + MOCK(connection_write_to_buf_impl_, mock_connection_write_to_buf_impl_); + /* Set up the fake handler functions */ + MOCK(connection_mark_unattached_ap_, mock_connection_mark_unattached_ap_); + MOCK(connection_ap_rewrite_and_attach_if_allowed, + mock_connection_ap_rewrite_and_attach_if_allowed); + + return 0; +} + +int +fuzz_cleanup(void) +{ + UNMOCK(connection_write_to_buf_impl_); + UNMOCK(connection_mark_unattached_ap_); + UNMOCK(connection_ap_rewrite_and_attach_if_allowed); + return 0; +} + +int +fuzz_main(const uint8_t *stdin_buf, size_t data_size) +{ + entry_connection_t conn; + + /* Set up the fake connection */ + memset(&conn, 0, sizeof(conn)); + conn.edge_.base_.type = CONN_TYPE_AP; + conn.edge_.base_.state = AP_CONN_STATE_HTTP_CONNECT_WAIT; + conn.socks_request = tor_malloc_zero(sizeof(socks_request_t)); + conn.socks_request->listener_type = CONN_TYPE_AP_HTTP_CONNECT_LISTENER; + + conn.edge_.base_.inbuf = buf_new_with_data((char*)stdin_buf, data_size); + if (!conn.edge_.base_.inbuf) { + log_debug(LD_GENERAL, "Zero-Length-Input\n"); + goto done; + } + + /* Parse the headers */ + int rv = connection_ap_process_http_connect(&conn); + + /* TODO: check the output is correctly parsed based on the input */ + + log_debug(LD_GENERAL, "Result:\n%d\n", rv); + + goto done; + + done: + /* Reset. */ + socks_request_free(conn.socks_request); + buf_free(conn.edge_.base_.inbuf); + conn.edge_.base_.inbuf = NULL; + + return 0; +} + diff --git a/src/test/fuzz/include.am b/src/test/fuzz/include.am index 8a043cd76e..472cbfc8a5 100644 --- a/src/test/fuzz/include.am +++ b/src/test/fuzz/include.am @@ -103,6 +103,14 @@ src_test_fuzz_fuzz_http_CFLAGS = $(FUZZING_CFLAGS) src_test_fuzz_fuzz_http_LDFLAGS = $(FUZZING_LDFLAG) src_test_fuzz_fuzz_http_LDADD = $(FUZZING_LIBS) +src_test_fuzz_fuzz_http_connect_SOURCES = \ + src/test/fuzz/fuzzing_common.c \ + src/test/fuzz/fuzz_http_connect.c +src_test_fuzz_fuzz_http_connect_CPPFLAGS = $(FUZZING_CPPFLAGS) +src_test_fuzz_fuzz_http_connect_CFLAGS = $(FUZZING_CFLAGS) +src_test_fuzz_fuzz_http_connect_LDFLAGS = $(FUZZING_LDFLAG) +src_test_fuzz_fuzz_http_connect_LDADD = $(FUZZING_LIBS) + src_test_fuzz_fuzz_iptsv2_SOURCES = \ src/test/fuzz/fuzzing_common.c \ src/test/fuzz/fuzz_iptsv2.c @@ -135,6 +143,7 @@ FUZZERS = \ src/test/fuzz/fuzz-extrainfo \ src/test/fuzz/fuzz-hsdescv2 \ src/test/fuzz/fuzz-http \ + src/test/fuzz/fuzz-http-connect \ src/test/fuzz/fuzz-iptsv2 \ src/test/fuzz/fuzz-microdesc \ src/test/fuzz/fuzz-vrs @@ -191,6 +200,13 @@ src_test_fuzz_lf_fuzz_http_CFLAGS = $(LIBFUZZER_CFLAGS) src_test_fuzz_lf_fuzz_http_LDFLAGS = $(LIBFUZZER_LDFLAG) src_test_fuzz_lf_fuzz_http_LDADD = $(LIBFUZZER_LIBS) +src_test_fuzz_lf_fuzz_http_connect_SOURCES = \ + $(src_test_fuzz_fuzz_http_connect_SOURCES) +src_test_fuzz_lf_fuzz_http_connect_CPPFLAGS = $(LIBFUZZER_CPPFLAGS) +src_test_fuzz_lf_fuzz_http_connect_CFLAGS = $(LIBFUZZER_CFLAGS) +src_test_fuzz_lf_fuzz_http_connect_LDFLAGS = $(LIBFUZZER_LDFLAG) +src_test_fuzz_lf_fuzz_http_connect_LDADD = $(LIBFUZZER_LIBS) + src_test_fuzz_lf_fuzz_iptsv2_SOURCES = \ $(src_test_fuzz_fuzz_iptsv2_SOURCES) src_test_fuzz_lf_fuzz_iptsv2_CPPFLAGS = $(LIBFUZZER_CPPFLAGS) @@ -220,6 +236,7 @@ LIBFUZZER_FUZZERS = \ src/test/fuzz/lf-fuzz-extrainfo \ src/test/fuzz/lf-fuzz-hsdescv2 \ src/test/fuzz/lf-fuzz-http \ + src/test/fuzz/lf-fuzz-http-connect \ src/test/fuzz/lf-fuzz-iptsv2 \ src/test/fuzz/lf-fuzz-microdesc \ src/test/fuzz/lf-fuzz-vrs @@ -266,6 +283,11 @@ src_test_fuzz_liboss_fuzz_http_a_SOURCES = \ src_test_fuzz_liboss_fuzz_http_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS) src_test_fuzz_liboss_fuzz_http_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS) +src_test_fuzz_liboss_fuzz_http_connect_a_SOURCES = \ + $(src_test_fuzz_fuzz_http_connect_SOURCES) +src_test_fuzz_liboss_fuzz_http_connect_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS) +src_test_fuzz_liboss_fuzz_http_connect_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS) + src_test_fuzz_liboss_fuzz_iptsv2_a_SOURCES = \ $(src_test_fuzz_fuzz_iptsv2_SOURCES) src_test_fuzz_liboss_fuzz_iptsv2_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS) @@ -289,6 +311,7 @@ OSS_FUZZ_FUZZERS = \ src/test/fuzz/liboss-fuzz-extrainfo.a \ src/test/fuzz/liboss-fuzz-hsdescv2.a \ src/test/fuzz/liboss-fuzz-http.a \ + src/test/fuzz/liboss-fuzz-http-connect.a \ src/test/fuzz/liboss-fuzz-iptsv2.a \ src/test/fuzz/liboss-fuzz-microdesc.a \ src/test/fuzz/liboss-fuzz-vrs.a diff --git a/src/test/test_buffers.c b/src/test/test_buffers.c index a85fb95f0a..dfc1fcb24f 100644 --- a/src/test/test_buffers.c +++ b/src/test/test_buffers.c @@ -4,9 +4,16 @@ /* See LICENSE for licensing information */ #define BUFFERS_PRIVATE +#define PROTO_HTTP_PRIVATE #include "or.h" #include "buffers.h" +#include "buffers_tls.h" #include "ext_orport.h" +#include "proto_cell.h" +#include "proto_ext_or.h" +#include "proto_http.h" +#include "proto_control0.h" +#include "proto_socks.h" #include "test.h" /** Run unit tests for buffers.c */ @@ -38,15 +45,15 @@ test_buffers_basic(void *arg) for (j=0;j<256;++j) { str[j] = (char)j; } - write_to_buf(str, 256, buf); - write_to_buf(str, 256, buf); + buf_add(buf, str, 256); + buf_add(buf, str, 256); tt_int_op(buf_datalen(buf),OP_EQ, 512); - fetch_from_buf(str2, 200, buf); + buf_get_bytes(buf, str2, 200); tt_mem_op(str,OP_EQ, str2, 200); tt_int_op(buf_datalen(buf),OP_EQ, 312); memset(str2, 0, sizeof(str2)); - fetch_from_buf(str2, 256, buf); + buf_get_bytes(buf, str2, 256); tt_mem_op(str+200,OP_EQ, str2, 56); tt_mem_op(str,OP_EQ, str2+56, 200); tt_int_op(buf_datalen(buf),OP_EQ, 56); @@ -54,16 +61,16 @@ test_buffers_basic(void *arg) /* Okay, now we should be 512 bytes into the 4096-byte buffer. If we add * another 3584 bytes, we hit the end. */ for (j=0;j<15;++j) { - write_to_buf(str, 256, buf); + buf_add(buf, str, 256); } - assert_buf_ok(buf); + buf_assert_ok(buf); tt_int_op(buf_datalen(buf),OP_EQ, 3896); - fetch_from_buf(str2, 56, buf); + buf_get_bytes(buf, str2, 56); tt_int_op(buf_datalen(buf),OP_EQ, 3840); tt_mem_op(str+200,OP_EQ, str2, 56); for (j=0;j<15;++j) { memset(str2, 0, sizeof(str2)); - fetch_from_buf(str2, 256, buf); + buf_get_bytes(buf, str2, 256); tt_mem_op(str,OP_EQ, str2, 256); } tt_int_op(buf_datalen(buf),OP_EQ, 0); @@ -73,38 +80,38 @@ test_buffers_basic(void *arg) /* Okay, now make sure growing can work. */ buf = buf_new_with_capacity(16); //test_eq(buf_capacity(buf), 16); - write_to_buf(str+1, 255, buf); + buf_add(buf, str+1, 255); //test_eq(buf_capacity(buf), 256); - fetch_from_buf(str2, 254, buf); + buf_get_bytes(buf, str2, 254); tt_mem_op(str+1,OP_EQ, str2, 254); //test_eq(buf_capacity(buf), 256); - assert_buf_ok(buf); - write_to_buf(str, 32, buf); + buf_assert_ok(buf); + buf_add(buf, str, 32); //test_eq(buf_capacity(buf), 256); - assert_buf_ok(buf); - write_to_buf(str, 256, buf); - assert_buf_ok(buf); + buf_assert_ok(buf); + buf_add(buf, str, 256); + buf_assert_ok(buf); //test_eq(buf_capacity(buf), 512); tt_int_op(buf_datalen(buf),OP_EQ, 33+256); - fetch_from_buf(str2, 33, buf); + buf_get_bytes(buf, str2, 33); tt_int_op(*str2,OP_EQ, str[255]); tt_mem_op(str2+1,OP_EQ, str, 32); //test_eq(buf_capacity(buf), 512); tt_int_op(buf_datalen(buf),OP_EQ, 256); - fetch_from_buf(str2, 256, buf); + buf_get_bytes(buf, str2, 256); tt_mem_op(str,OP_EQ, str2, 256); /* now try shrinking: case 1. */ buf_free(buf); buf = buf_new_with_capacity(33668); for (j=0;j<67;++j) { - write_to_buf(str,255, buf); + buf_add(buf, str,255); } //test_eq(buf_capacity(buf), 33668); tt_int_op(buf_datalen(buf),OP_EQ, 17085); for (j=0; j < 40; ++j) { - fetch_from_buf(str2, 255,buf); + buf_get_bytes(buf, str2, 255); tt_mem_op(str2,OP_EQ, str, 255); } @@ -112,18 +119,18 @@ test_buffers_basic(void *arg) buf_free(buf); buf = buf_new_with_capacity(33668); for (j=0;j<67;++j) { - write_to_buf(str,255, buf); + buf_add(buf, str, 255); } for (j=0; j < 20; ++j) { - fetch_from_buf(str2, 255,buf); + buf_get_bytes(buf, str2, 255); tt_mem_op(str2,OP_EQ, str, 255); } for (j=0;j<80;++j) { - write_to_buf(str,255, buf); + buf_add(buf, str, 255); } //test_eq(buf_capacity(buf),33668); for (j=0; j < 120; ++j) { - fetch_from_buf(str2, 255,buf); + buf_get_bytes(buf, str2, 255); tt_mem_op(str2,OP_EQ, str, 255); } @@ -132,27 +139,27 @@ test_buffers_basic(void *arg) buf = buf_new_with_capacity(4096); buf2 = buf_new_with_capacity(4096); for (j=0;j<100;++j) - write_to_buf(str, 255, buf); + buf_add(buf, str, 255); tt_int_op(buf_datalen(buf),OP_EQ, 25500); for (j=0;j<100;++j) { r = 10; - move_buf_to_buf(buf2, buf, &r); + buf_move_to_buf(buf2, buf, &r); tt_int_op(r,OP_EQ, 0); } tt_int_op(buf_datalen(buf),OP_EQ, 24500); tt_int_op(buf_datalen(buf2),OP_EQ, 1000); for (j=0;j<3;++j) { - fetch_from_buf(str2, 255, buf2); + buf_get_bytes(buf2, str2, 255); tt_mem_op(str2,OP_EQ, str, 255); } r = 8192; /*big move*/ - move_buf_to_buf(buf2, buf, &r); + buf_move_to_buf(buf2, buf, &r); tt_int_op(r,OP_EQ, 0); r = 30000; /* incomplete move */ - move_buf_to_buf(buf2, buf, &r); + buf_move_to_buf(buf2, buf, &r); tt_int_op(r,OP_EQ, 13692); for (j=0;j<97;++j) { - fetch_from_buf(str2, 255, buf2); + buf_get_bytes(buf2, str2, 255); tt_mem_op(str2,OP_EQ, str, 255); } buf_free(buf); @@ -162,7 +169,7 @@ test_buffers_basic(void *arg) buf = buf_new_with_capacity(5); cp = "Testing. This is a moderately long Testing string."; for (j = 0; cp[j]; j++) - write_to_buf(cp+j, 1, buf); + buf_add(buf, cp+j, 1); tt_int_op(0,OP_EQ, buf_find_string_offset(buf, "Testing", 7)); tt_int_op(1,OP_EQ, buf_find_string_offset(buf, "esting", 6)); tt_int_op(1,OP_EQ, buf_find_string_offset(buf, "est", 3)); @@ -180,7 +187,7 @@ test_buffers_basic(void *arg) { char *mem = tor_malloc_zero(65536); buf = buf_new(); - write_to_buf(mem, 65536, buf); + buf_add(buf, mem, 65536); tor_free(mem); tt_int_op(buf_datalen(buf), OP_EQ, 65536); @@ -215,8 +222,7 @@ test_buffer_pullup(void *arg) /* There are a bunch of cases for pullup. One is the trivial case. Let's mess around with an empty buffer. */ - buf_pullup(buf, 16); - buf_get_first_chunk_data(buf, &cp, &sz); + buf_pullup(buf, 16, &cp, &sz); tt_ptr_op(cp, OP_EQ, NULL); tt_uint_op(sz, OP_EQ, 0); @@ -227,65 +233,62 @@ test_buffer_pullup(void *arg) /* Let's add some data. */ crypto_rand(stuff, 16384); - write_to_buf(stuff, 3000, buf); - write_to_buf(stuff+3000, 3000, buf); - buf_get_first_chunk_data(buf, &cp, &sz); + buf_add(buf, stuff, 3000); + buf_add(buf, stuff+3000, 3000); + buf_pullup(buf, 0, &cp, &sz); tt_ptr_op(cp, OP_NE, NULL); tt_int_op(sz, OP_LE, 4096); /* Make room for 3000 bytes in the first chunk, so that the pullup-move code * can get tested. */ - tt_int_op(fetch_from_buf(tmp, 3000, buf), OP_EQ, 3000); + tt_int_op(buf_get_bytes(buf, tmp, 3000), OP_EQ, 3000); tt_mem_op(tmp,OP_EQ, stuff, 3000); - buf_pullup(buf, 2048); - assert_buf_ok(buf); - buf_get_first_chunk_data(buf, &cp, &sz); + buf_pullup(buf, 2048, &cp, &sz); + buf_assert_ok(buf); tt_ptr_op(cp, OP_NE, NULL); tt_int_op(sz, OP_GE, 2048); tt_mem_op(cp,OP_EQ, stuff+3000, 2048); tt_int_op(3000, OP_EQ, buf_datalen(buf)); - tt_int_op(fetch_from_buf(tmp, 3000, buf), OP_EQ, 0); + tt_int_op(buf_get_bytes(buf, tmp, 3000), OP_EQ, 0); tt_mem_op(tmp,OP_EQ, stuff+3000, 2048); buf_free(buf); /* Now try the large-chunk case. */ buf = buf_new_with_capacity(3000); /* rounds up to next power of 2. */ - write_to_buf(stuff, 4000, buf); - write_to_buf(stuff+4000, 4000, buf); - write_to_buf(stuff+8000, 4000, buf); - write_to_buf(stuff+12000, 4000, buf); + buf_add(buf, stuff, 4000); + buf_add(buf, stuff+4000, 4000); + buf_add(buf, stuff+8000, 4000); + buf_add(buf, stuff+12000, 4000); tt_int_op(buf_datalen(buf), OP_EQ, 16000); - buf_get_first_chunk_data(buf, &cp, &sz); + buf_pullup(buf, 0, &cp, &sz); tt_ptr_op(cp, OP_NE, NULL); tt_int_op(sz, OP_LE, 4096); - buf_pullup(buf, 12500); - assert_buf_ok(buf); - buf_get_first_chunk_data(buf, &cp, &sz); + buf_pullup(buf, 12500, &cp, &sz); + buf_assert_ok(buf); tt_ptr_op(cp, OP_NE, NULL); tt_int_op(sz, OP_GE, 12500); tt_mem_op(cp,OP_EQ, stuff, 12500); tt_int_op(buf_datalen(buf), OP_EQ, 16000); - fetch_from_buf(tmp, 12400, buf); + buf_get_bytes(buf, tmp, 12400); tt_mem_op(tmp,OP_EQ, stuff, 12400); tt_int_op(buf_datalen(buf), OP_EQ, 3600); - fetch_from_buf(tmp, 3500, buf); + buf_get_bytes(buf, tmp, 3500); tt_mem_op(tmp,OP_EQ, stuff+12400, 3500); - fetch_from_buf(tmp, 100, buf); + buf_get_bytes(buf, tmp, 100); tt_mem_op(tmp,OP_EQ, stuff+15900, 10); buf_free(buf); /* Make sure that the pull-up-whole-buffer case works */ buf = buf_new_with_capacity(3000); /* rounds up to next power of 2. */ - write_to_buf(stuff, 4000, buf); - write_to_buf(stuff+4000, 4000, buf); - fetch_from_buf(tmp, 100, buf); /* dump 100 bytes from first chunk */ - buf_pullup(buf, 16000); /* Way too much. */ - assert_buf_ok(buf); - buf_get_first_chunk_data(buf, &cp, &sz); + buf_add(buf, stuff, 4000); + buf_add(buf, stuff+4000, 4000); + buf_get_bytes(buf, tmp, 100); /* dump 100 bytes from first chunk */ + buf_pullup(buf, 16000, &cp, &sz); + buf_assert_ok(buf); tt_ptr_op(cp, OP_NE, NULL); tt_int_op(sz, OP_EQ, 7900); tt_mem_op(cp,OP_EQ, stuff+100, 7900); @@ -321,23 +324,23 @@ test_buffer_copy(void *arg) /* Now try with a short buffer. */ s = "And now comes an act of enormous enormance!"; len = strlen(s); - write_to_buf(s, len, buf); + buf_add(buf, s, len); tt_int_op(len, OP_EQ, buf_datalen(buf)); /* Add junk to buf2 so we can test replacing.*/ - write_to_buf("BLARG", 5, buf2); + buf_add(buf2, "BLARG", 5); tt_int_op(0, OP_EQ, buf_set_to_copy(&buf2, buf)); tt_int_op(len, OP_EQ, buf_datalen(buf2)); - fetch_from_buf(b, len, buf2); + buf_get_bytes(buf2, b, len); tt_mem_op(b, OP_EQ, s, len); /* Now free buf2 and retry so we can test allocating */ buf_free(buf2); buf2 = NULL; tt_int_op(0, OP_EQ, buf_set_to_copy(&buf2, buf)); tt_int_op(len, OP_EQ, buf_datalen(buf2)); - fetch_from_buf(b, len, buf2); + buf_get_bytes(buf2, b, len); tt_mem_op(b, OP_EQ, s, len); /* Clear buf for next test */ - fetch_from_buf(b, len, buf); + buf_get_bytes(buf, b, len); tt_int_op(buf_datalen(buf),OP_EQ,0); /* Okay, now let's try a bigger buffer. */ @@ -347,13 +350,13 @@ test_buffer_copy(void *arg) len = strlen(s); for (i = 0; i < 256; ++i) { b[0]=i; - write_to_buf(b, 1, buf); - write_to_buf(s, len, buf); + buf_add(buf, b, 1); + buf_add(buf, s, len); } tt_int_op(0, OP_EQ, buf_set_to_copy(&buf2, buf)); tt_int_op(buf_datalen(buf2), OP_EQ, buf_datalen(buf)); for (i = 0; i < 256; ++i) { - fetch_from_buf(b, len+1, buf2); + buf_get_bytes(buf2, b, len+1); tt_int_op((unsigned char)b[0],OP_EQ,i); tt_mem_op(b+1, OP_EQ, s, len); } @@ -378,13 +381,13 @@ test_buffer_ext_or_cmd(void *arg) tt_ptr_op(NULL, OP_EQ, cmd); /* Three bytes: shouldn't work. */ - write_to_buf("\x00\x20\x00", 3, buf); + buf_add(buf, "\x00\x20\x00", 3); tt_int_op(0, OP_EQ, fetch_ext_or_command_from_buf(buf, &cmd)); tt_ptr_op(NULL, OP_EQ, cmd); tt_int_op(3, OP_EQ, buf_datalen(buf)); /* 0020 0000: That's a nil command. It should work. */ - write_to_buf("\x00", 1, buf); + buf_add(buf, "\x00", 1); tt_int_op(1, OP_EQ, fetch_ext_or_command_from_buf(buf, &cmd)); tt_ptr_op(NULL, OP_NE, cmd); tt_int_op(0x20, OP_EQ, cmd->cmd); @@ -394,10 +397,10 @@ test_buffer_ext_or_cmd(void *arg) cmd = NULL; /* Now try a length-6 command with one byte missing. */ - write_to_buf("\x10\x21\x00\x06""abcde", 9, buf); + buf_add(buf, "\x10\x21\x00\x06""abcde", 9); tt_int_op(0, OP_EQ, fetch_ext_or_command_from_buf(buf, &cmd)); tt_ptr_op(NULL, OP_EQ, cmd); - write_to_buf("f", 1, buf); + buf_add(buf, "f", 1); tt_int_op(1, OP_EQ, fetch_ext_or_command_from_buf(buf, &cmd)); tt_ptr_op(NULL, OP_NE, cmd); tt_int_op(0x1021, OP_EQ, cmd->cmd); @@ -408,7 +411,7 @@ test_buffer_ext_or_cmd(void *arg) cmd = NULL; /* Now try a length-10 command with 4 extra bytes. */ - write_to_buf("\xff\xff\x00\x0aloremipsum\x10\x00\xff\xff", 18, buf); + buf_add(buf, "\xff\xff\x00\x0aloremipsum\x10\x00\xff\xff", 18); tt_int_op(1, OP_EQ, fetch_ext_or_command_from_buf(buf, &cmd)); tt_ptr_op(NULL, OP_NE, cmd); tt_int_op(0xffff, OP_EQ, cmd->cmd); @@ -422,7 +425,7 @@ test_buffer_ext_or_cmd(void *arg) * waiting. */ tt_int_op(0, OP_EQ, fetch_ext_or_command_from_buf(buf, &cmd)); tmp = tor_malloc_zero(65535); - write_to_buf(tmp, 65535, buf); + buf_add(buf, tmp, 65535); tt_int_op(1, OP_EQ, fetch_ext_or_command_from_buf(buf, &cmd)); tt_ptr_op(NULL, OP_NE, cmd); tt_int_op(0x1000, OP_EQ, cmd->cmd); @@ -458,36 +461,36 @@ test_buffer_allocation_tracking(void *arg) tt_int_op(buf_allocation(buf1), OP_EQ, 0); tt_int_op(buf_get_total_allocation(), OP_EQ, 0); - write_to_buf(junk, 4000, buf1); - write_to_buf(junk, 4000, buf1); - write_to_buf(junk, 4000, buf1); - write_to_buf(junk, 4000, buf1); + buf_add(buf1, junk, 4000); + buf_add(buf1, junk, 4000); + buf_add(buf1, junk, 4000); + buf_add(buf1, junk, 4000); tt_int_op(buf_allocation(buf1), OP_EQ, 16384); - fetch_from_buf(junk, 100, buf1); + buf_get_bytes(buf1, junk, 100); tt_int_op(buf_allocation(buf1), OP_EQ, 16384); /* still 4 4k chunks */ tt_int_op(buf_get_total_allocation(), OP_EQ, 16384); - fetch_from_buf(junk, 4096, buf1); /* drop a 1k chunk... */ + buf_get_bytes(buf1, junk, 4096); /* drop a 1k chunk... */ tt_int_op(buf_allocation(buf1), OP_EQ, 3*4096); /* now 3 4k chunks */ tt_int_op(buf_get_total_allocation(), OP_EQ, 12288); /* that chunk was really freed. */ - write_to_buf(junk, 4000, buf2); + buf_add(buf2, junk, 4000); tt_int_op(buf_allocation(buf2), OP_EQ, 4096); /* another 4k chunk. */ /* * We bounce back up to 16384 by allocating a new chunk. */ tt_int_op(buf_get_total_allocation(), OP_EQ, 16384); - write_to_buf(junk, 4000, buf2); + buf_add(buf2, junk, 4000); tt_int_op(buf_allocation(buf2), OP_EQ, 8192); /* another 4k chunk. */ tt_int_op(buf_get_total_allocation(), OP_EQ, 5*4096); /* that chunk was new. */ /* Make a really huge buffer */ for (i = 0; i < 1000; ++i) { - write_to_buf(junk, 4000, buf2); + buf_add(buf2, junk, 4000); } tt_int_op(buf_allocation(buf2), OP_GE, 4008000); tt_int_op(buf_get_total_allocation(), OP_GE, 4008000); @@ -530,7 +533,7 @@ test_buffer_time_tracking(void *arg) tt_int_op(0, OP_EQ, buf_get_oldest_chunk_timestamp(buf, START_MSEC)); tt_int_op(0, OP_EQ, buf_get_oldest_chunk_timestamp(buf, START_MSEC+1000)); - write_to_buf("ABCDEFG", 7, buf); + buf_add(buf, "ABCDEFG", 7); tt_int_op(1000, OP_EQ, buf_get_oldest_chunk_timestamp(buf, START_MSEC+1000)); buf2 = buf_copy(buf); @@ -541,7 +544,7 @@ test_buffer_time_tracking(void *arg) /* Now add more bytes; enough to overflow the first chunk. */ monotime_coarse_set_mock_time_nsec(START_NSEC + 123 * (uint64_t)1000000); for (i = 0; i < 600; ++i) - write_to_buf("ABCDEFG", 7, buf); + buf_add(buf, "ABCDEFG", 7); tt_int_op(4207, OP_EQ, buf_datalen(buf)); /* The oldest bytes are still in the front. */ @@ -549,12 +552,12 @@ test_buffer_time_tracking(void *arg) /* Once those bytes are dropped, the chunk is still on the first * timestamp. */ - fetch_from_buf(tmp, 100, buf); + buf_get_bytes(buf, tmp, 100); tt_int_op(2000, OP_EQ, buf_get_oldest_chunk_timestamp(buf, START_MSEC+2000)); /* But once we discard the whole first chunk, we get the data in the second * chunk. */ - fetch_from_buf(tmp, 4000, buf); + buf_get_bytes(buf, tmp, 4000); tt_int_op(107, OP_EQ, buf_datalen(buf)); tt_int_op(2000, OP_EQ, buf_get_oldest_chunk_timestamp(buf, START_MSEC+2123)); @@ -562,12 +565,12 @@ test_buffer_time_tracking(void *arg) its time gets updated */ monotime_coarse_set_mock_time_nsec(START_NSEC + 5617 * (uint64_t)1000000); for (i = 0; i < 600; ++i) - write_to_buf("ABCDEFG", 7, buf); + buf_add(buf, "ABCDEFG", 7); tt_int_op(4307, OP_EQ, buf_datalen(buf)); tt_int_op(2000, OP_EQ, buf_get_oldest_chunk_timestamp(buf, START_MSEC+2123)); - fetch_from_buf(tmp, 4000, buf); - fetch_from_buf(tmp, 306, buf); + buf_get_bytes(buf, tmp, 4000); + buf_get_bytes(buf, tmp, 306); tt_int_op(0, OP_EQ, buf_get_oldest_chunk_timestamp(buf, START_MSEC+5617)); tt_int_op(383, OP_EQ, buf_get_oldest_chunk_timestamp(buf, START_MSEC+6000)); @@ -593,23 +596,23 @@ test_buffers_compress_fin_at_chunk_end_impl(compress_method_t method, sz = buf_get_default_chunk_size(buf); msg = tor_malloc_zero(sz); - write_to_buf(msg, 1, buf); + buf_add(buf, msg, 1); tt_assert(buf->head); /* Fill up the chunk so the compression stuff won't fit in one chunk. */ tt_uint_op(buf->head->memlen, OP_LT, sz); headerjunk = buf->head->memlen - 7; - write_to_buf(msg, headerjunk-1, buf); + buf_add(buf, msg, headerjunk-1); tt_uint_op(buf->head->datalen, OP_EQ, headerjunk); tt_uint_op(buf_datalen(buf), OP_EQ, headerjunk); /* Write an empty string, with finalization on. */ compress_state = tor_compress_new(1, method, level); - tt_int_op(write_to_buf_compress(buf, compress_state, "", 0, 1), OP_EQ, 0); + tt_int_op(buf_add_compress(buf, compress_state, "", 0, 1), OP_EQ, 0); in_len = buf_datalen(buf); contents = tor_malloc(in_len); - tt_int_op(fetch_from_buf(contents, in_len, buf), OP_EQ, 0); + tt_int_op(buf_get_bytes(buf, contents, in_len), OP_EQ, 0); if (method == NO_METHOD) { tt_uint_op(in_len, OP_EQ, headerjunk); @@ -652,23 +655,23 @@ test_buffers_compress_impl(compress_method_t method, msg = tor_malloc(512); crypto_rand(msg, 512); - tt_int_op(write_to_buf_compress(buf, compress_state, + tt_int_op(buf_add_compress(buf, compress_state, msg, 128, 0), OP_EQ, 0); - tt_int_op(write_to_buf_compress(buf, compress_state, + tt_int_op(buf_add_compress(buf, compress_state, msg+128, 128, 0), OP_EQ, 0); - tt_int_op(write_to_buf_compress(buf, compress_state, + tt_int_op(buf_add_compress(buf, compress_state, msg+256, 256, 0), OP_EQ, 0); done = !finalize_with_nil; - tt_int_op(write_to_buf_compress(buf, compress_state, + tt_int_op(buf_add_compress(buf, compress_state, "all done", 9, done), OP_EQ, 0); if (finalize_with_nil) { - tt_int_op(write_to_buf_compress(buf, compress_state, "", 0, 1), OP_EQ, 0); + tt_int_op(buf_add_compress(buf, compress_state, "", 0, 1), OP_EQ, 0); } in_len = buf_datalen(buf); contents = tor_malloc(in_len); - tt_int_op(fetch_from_buf(contents, in_len, buf), OP_EQ, 0); + tt_int_op(buf_get_bytes(buf, contents, in_len), OP_EQ, 0); tt_int_op(0, OP_EQ, tor_uncompress(&expanded, &out_len, contents, in_len, @@ -762,11 +765,11 @@ test_buffers_tls_read_mocked(void *arg) buf = buf_new(); next_reply_val[0] = 1024; - tt_int_op(128, OP_EQ, read_to_buf_tls(NULL, 128, buf)); + tt_int_op(128, OP_EQ, buf_read_from_tls(buf, NULL, 128)); next_reply_val[0] = 5000; next_reply_val[1] = 5000; - tt_int_op(6000, OP_EQ, read_to_buf_tls(NULL, 6000, buf)); + tt_int_op(6000, OP_EQ, buf_read_from_tls(buf, NULL, 6000)); done: UNMOCK(tor_tls_read); @@ -780,17 +783,17 @@ test_buffers_chunk_size(void *arg) (void)arg; const int min = 256; const int max = 65536; - tt_uint_op(preferred_chunk_size(3), OP_EQ, min); - tt_uint_op(preferred_chunk_size(25), OP_EQ, min); - tt_uint_op(preferred_chunk_size(0), OP_EQ, min); - tt_uint_op(preferred_chunk_size(256), OP_EQ, 512); - tt_uint_op(preferred_chunk_size(65400), OP_EQ, max); + tt_uint_op(buf_preferred_chunk_size(3), OP_EQ, min); + tt_uint_op(buf_preferred_chunk_size(25), OP_EQ, min); + tt_uint_op(buf_preferred_chunk_size(0), OP_EQ, min); + tt_uint_op(buf_preferred_chunk_size(256), OP_EQ, 512); + tt_uint_op(buf_preferred_chunk_size(65400), OP_EQ, max); /* Here, we're implicitly saying that the chunk header overhead is * between 1 and 100 bytes. 24..48 would probably be more accurate. */ - tt_uint_op(preferred_chunk_size(65536), OP_GT, 65536); - tt_uint_op(preferred_chunk_size(65536), OP_LT, 65536+100); - tt_uint_op(preferred_chunk_size(165536), OP_GT, 165536); - tt_uint_op(preferred_chunk_size(165536), OP_LT, 165536+100); + tt_uint_op(buf_preferred_chunk_size(65536), OP_GT, 65536); + tt_uint_op(buf_preferred_chunk_size(65536), OP_LT, 65536+100); + tt_uint_op(buf_preferred_chunk_size(165536), OP_GT, 165536); + tt_uint_op(buf_preferred_chunk_size(165536), OP_LT, 165536+100); done: ; } @@ -846,19 +849,19 @@ test_buffer_peek_startswith(void *arg) buf = buf_new(); tt_ptr_op(buf, OP_NE, NULL); - tt_assert(peek_buf_startswith(buf, "")); - tt_assert(! peek_buf_startswith(buf, "X")); + tt_assert(buf_peek_startswith(buf, "")); + tt_assert(! buf_peek_startswith(buf, "X")); - write_to_buf("Tor", 3, buf); + buf_add(buf, "Tor", 3); - tt_assert(peek_buf_startswith(buf, "")); - tt_assert(peek_buf_startswith(buf, "T")); - tt_assert(peek_buf_startswith(buf, "To")); - tt_assert(peek_buf_startswith(buf, "Tor")); - tt_assert(! peek_buf_startswith(buf, "Top")); - tt_assert(! peek_buf_startswith(buf, "For")); - tt_assert(! peek_buf_startswith(buf, "Tork")); - tt_assert(! peek_buf_startswith(buf, "Torpor")); + tt_assert(buf_peek_startswith(buf, "")); + tt_assert(buf_peek_startswith(buf, "T")); + tt_assert(buf_peek_startswith(buf, "To")); + tt_assert(buf_peek_startswith(buf, "Tor")); + tt_assert(! buf_peek_startswith(buf, "Top")); + tt_assert(! buf_peek_startswith(buf, "For")); + tt_assert(! buf_peek_startswith(buf, "Tork")); + tt_assert(! buf_peek_startswith(buf, "Torpor")); done: buf_free(buf); diff --git a/src/test/test_dir_handle_get.c b/src/test/test_dir_handle_get.c index 75fe6249ad..c4c74df662 100644 --- a/src/test/test_dir_handle_get.c +++ b/src/test/test_dir_handle_get.c @@ -28,6 +28,7 @@ #include "entrynodes.h" #include "routerparse.h" #include "networkstatus.h" +#include "proto_http.h" #include "geoip.h" #include "dirserv.h" #include "dirvote.h" diff --git a/src/test/test_extorport.c b/src/test/test_extorport.c index fc9f27a5ac..9e8721ee36 100644 --- a/src/test/test_extorport.c +++ b/src/test/test_extorport.c @@ -78,7 +78,7 @@ connection_write_to_buf_impl_replacement(const char *string, size_t len, tor_assert(string); tor_assert(conn); - write_to_buf(string, len, conn->outbuf); + buf_add(conn->outbuf, string, len); } static char * @@ -89,7 +89,7 @@ buf_get_contents(buf_t *buf, size_t *sz_out) if (*sz_out >= ULONG_MAX) return NULL; /* C'mon, really? */ out = tor_malloc(*sz_out + 1); - if (fetch_from_buf(out, (unsigned long)*sz_out, buf) != 0) { + if (buf_get_bytes(buf, out, (unsigned long)*sz_out) != 0) { tor_free(out); return NULL; } @@ -399,14 +399,14 @@ handshake_start(or_connection_t *conn, int receiving) #define WRITE(s,n) \ do { \ - write_to_buf((s), (n), TO_CONN(conn)->inbuf); \ + buf_add(TO_CONN(conn)->inbuf, (s), (n)); \ } while (0) #define CONTAINS(s,n) \ do { \ tt_int_op((n), OP_LE, sizeof(b)); \ tt_int_op(buf_datalen(TO_CONN(conn)->outbuf), OP_EQ, (n)); \ if ((n)) { \ - fetch_from_buf(b, (n), TO_CONN(conn)->outbuf); \ + buf_get_bytes(TO_CONN(conn)->outbuf, b, (n)); \ tt_mem_op(b, OP_EQ, (s), (n)); \ } \ } while (0) diff --git a/src/test/test_helpers.c b/src/test/test_helpers.c index 4c0580cff3..0da9cf64d0 100644 --- a/src/test/test_helpers.c +++ b/src/test/test_helpers.c @@ -114,7 +114,7 @@ connection_write_to_buf_mock(const char *string, size_t len, tor_assert(string); tor_assert(conn); - write_to_buf(string, len, conn->outbuf); + buf_add(conn->outbuf, string, len); } /* Set up a fake origin circuit with the specified number of cells, diff --git a/src/test/test_hs_cache.c b/src/test/test_hs_cache.c index cbd88acff1..950c0483d1 100644 --- a/src/test/test_hs_cache.c +++ b/src/test/test_hs_cache.c @@ -15,6 +15,7 @@ #include "rendcache.h" #include "directory.h" #include "connection.h" +#include "proto_http.h" #include "hs_test_helpers.h" #include "test_helpers.h" diff --git a/src/test/test_hs_common.c b/src/test/test_hs_common.c index b9215ea187..9980892951 100644 --- a/src/test/test_hs_common.c +++ b/src/test/test_hs_common.c @@ -21,6 +21,7 @@ #include "networkstatus.h" #include "directory.h" #include "nodelist.h" +#include "routerlist.h" #include "statefile.h" /** Test the validation of HS v3 addresses */ @@ -362,20 +363,26 @@ test_desc_overlap_period_testnet(void *arg) static void helper_add_hsdir_to_networkstatus(networkstatus_t *ns, - const uint8_t *identity, - const uint8_t *curr_hsdir_index, + int identity_idx, const char *nickname, int is_hsdir) { routerstatus_t *rs = tor_malloc_zero(sizeof(routerstatus_t)); routerinfo_t *ri = tor_malloc_zero(sizeof(routerinfo_t)); - + uint8_t identity[DIGEST_LEN]; + uint8_t curr_hsdir_index[DIGEST256_LEN]; tor_addr_t ipv4_addr; + + memset(identity, identity_idx, sizeof(identity)); + memset(curr_hsdir_index, identity_idx, sizeof(curr_hsdir_index)); + memcpy(rs->identity_digest, identity, DIGEST_LEN); rs->is_hs_dir = is_hsdir; rs->supports_v3_hsdir = 1; + strlcpy(rs->nickname, nickname, sizeof(rs->nickname)); tor_addr_parse(&ipv4_addr, "1.2.3.4"); ri->addr = tor_addr_to_ipv4h(&ipv4_addr); + rs->addr = tor_addr_to_ipv4h(&ipv4_addr); ri->nickname = tor_strdup(nickname); ri->protocol_list = tor_strdup("HSDir=1-2 LinkAuth=3"); memcpy(ri->cache_info.identity_digest, identity, DIGEST_LEN); @@ -388,7 +395,7 @@ helper_add_hsdir_to_networkstatus(networkstatus_t *ns, smartlist_add(ns->routerstatus_list, rs); done: - ; + routerinfo_free(ri); } static networkstatus_t *mock_ns = NULL; @@ -412,6 +419,7 @@ mock_networkstatus_get_latest_consensus(void) mock_ns->valid_until = now+2; /* Create routerstatus list */ mock_ns->routerstatus_list = smartlist_new(); + mock_ns->type = NS_TYPE_CONSENSUS; return mock_ns; } @@ -435,36 +443,15 @@ test_responsible_hsdirs(void *arg) ns = networkstatus_get_latest_consensus(); { /* First router: HSdir */ - uint8_t identity[DIGEST_LEN]; - uint8_t curr_hsdir_index[DIGEST256_LEN]; - char nickname[] = "let_me"; - memset(identity, 1, sizeof(identity)); - memset(curr_hsdir_index, 1, sizeof(curr_hsdir_index)); - - helper_add_hsdir_to_networkstatus(ns, identity, - curr_hsdir_index, nickname, 1); + helper_add_hsdir_to_networkstatus(ns, 1, "igor", 1); } { /* Second HSDir */ - uint8_t identity[DIGEST_LEN]; - uint8_t curr_hsdir_index[DIGEST256_LEN]; - char nickname[] = "show_you"; - memset(identity, 2, sizeof(identity)); - memset(curr_hsdir_index, 2, sizeof(curr_hsdir_index)); - - helper_add_hsdir_to_networkstatus(ns, identity, - curr_hsdir_index, nickname, 1); + helper_add_hsdir_to_networkstatus(ns, 2, "victor", 1); } { /* Third relay but not HSDir */ - uint8_t identity[DIGEST_LEN]; - uint8_t curr_hsdir_index[DIGEST256_LEN]; - char nickname[] = "how_to_dance"; - memset(identity, 3, sizeof(identity)); - memset(curr_hsdir_index, 3, sizeof(curr_hsdir_index)); - - helper_add_hsdir_to_networkstatus(ns, identity, - curr_hsdir_index, nickname, 0); + helper_add_hsdir_to_networkstatus(ns, 3, "spyro", 0); } ed25519_keypair_t kp; @@ -579,32 +566,19 @@ test_desc_reupload_logic(void *arg) tt_int_op(hs_service_get_num_services(), OP_EQ, 1); /* Now let's create our hash ring: */ - { /* First HSDir */ - uint8_t identity[DIGEST_LEN]; - uint8_t curr_hsdir_index[DIGEST256_LEN]; - char nickname[] = "let_me"; - memset(identity, 1, sizeof(identity)); - memset(curr_hsdir_index, 1, sizeof(curr_hsdir_index)); - - helper_add_hsdir_to_networkstatus(ns, identity, - curr_hsdir_index, nickname, 1); - } - - { /* Second HSDir */ - uint8_t identity[DIGEST_LEN]; - uint8_t curr_hsdir_index[DIGEST256_LEN]; - char nickname[] = "show_you"; - memset(identity, 2, sizeof(identity)); - memset(curr_hsdir_index, 2, sizeof(curr_hsdir_index)); - - helper_add_hsdir_to_networkstatus(ns, identity, - curr_hsdir_index, nickname, 1); + { + helper_add_hsdir_to_networkstatus(ns, 1, "dingus", 1); + helper_add_hsdir_to_networkstatus(ns, 2, "clive", 1); + helper_add_hsdir_to_networkstatus(ns, 3, "aaron", 1); + helper_add_hsdir_to_networkstatus(ns, 4, "lizzie", 1); + helper_add_hsdir_to_networkstatus(ns, 5, "daewon", 1); + helper_add_hsdir_to_networkstatus(ns, 6, "clarke", 1); } /* Now let's upload our desc to all hsdirs */ upload_descriptor_to_all(service, desc, 0); /* Check that previous hsdirs were populated */ - tt_int_op(smartlist_len(desc->previous_hsdirs), OP_EQ, 2); + tt_int_op(smartlist_len(desc->previous_hsdirs), OP_EQ, 6); /* Poison next upload time so that we can see if it was changed by * router_dir_info_changed(). No changes in hash ring so far, so the upload @@ -613,17 +587,23 @@ test_desc_reupload_logic(void *arg) router_dir_info_changed(); tt_int_op(desc->next_upload_time, OP_EQ, 42); - /* Now change the HSDir hash ring by adding another node */ - - { /* Third HSDir */ - uint8_t identity[DIGEST_LEN]; - uint8_t curr_hsdir_index[DIGEST256_LEN]; - char nickname[] = "how_to_dance"; - memset(identity, 3, sizeof(identity)); - memset(curr_hsdir_index, 3, sizeof(curr_hsdir_index)); + /* Now change the HSDir hash ring by swapping nora for aaron. + * Start by clearing the hash ring */ + { + SMARTLIST_FOREACH(ns->routerstatus_list, + routerstatus_t *, rs, routerstatus_free(rs)); + smartlist_clear(ns->routerstatus_list); + nodelist_free_all(); + routerlist_free_all(); + } - helper_add_hsdir_to_networkstatus(ns, identity, - curr_hsdir_index, nickname, 1); + { /* Now add back all the nodes */ + helper_add_hsdir_to_networkstatus(ns, 1, "dingus", 1); + helper_add_hsdir_to_networkstatus(ns, 2, "clive", 1); + helper_add_hsdir_to_networkstatus(ns, 4, "lizzie", 1); + helper_add_hsdir_to_networkstatus(ns, 5, "daewon", 1); + helper_add_hsdir_to_networkstatus(ns, 6, "clarke", 1); + helper_add_hsdir_to_networkstatus(ns, 7, "nora", 1); } /* Now call service_desc_hsdirs_changed() and see that it detected the hash @@ -631,6 +611,35 @@ test_desc_reupload_logic(void *arg) time_t now = approx_time(); tt_assert(now); tt_int_op(service_desc_hsdirs_changed(service, desc), OP_EQ, 1); + tt_int_op(smartlist_len(desc->previous_hsdirs), OP_EQ, 6); + + /* Now order another upload and see that we keep having 6 prev hsdirs */ + upload_descriptor_to_all(service, desc, 0); + /* Check that previous hsdirs were populated */ + tt_int_op(smartlist_len(desc->previous_hsdirs), OP_EQ, 6); + + /* Now restore the HSDir hash ring to its original state by swapping back + aaron for nora */ + /* First clear up the hash ring */ + { + SMARTLIST_FOREACH(ns->routerstatus_list, + routerstatus_t *, rs, routerstatus_free(rs)); + smartlist_clear(ns->routerstatus_list); + nodelist_free_all(); + routerlist_free_all(); + } + + { /* Now populate the hash ring again */ + helper_add_hsdir_to_networkstatus(ns, 1, "dingus", 1); + helper_add_hsdir_to_networkstatus(ns, 2, "clive", 1); + helper_add_hsdir_to_networkstatus(ns, 3, "aaron", 1); + helper_add_hsdir_to_networkstatus(ns, 4, "lizzie", 1); + helper_add_hsdir_to_networkstatus(ns, 5, "daewon", 1); + helper_add_hsdir_to_networkstatus(ns, 6, "clarke", 1); + } + + /* Check that our algorithm catches this change of hsdirs */ + tt_int_op(service_desc_hsdirs_changed(service, desc), OP_EQ, 1); /* Now pretend that the descriptor changed, and order a reupload to all HSDirs. Make sure that the set of previous HSDirs was cleared. */ @@ -639,9 +648,14 @@ test_desc_reupload_logic(void *arg) /* Now reupload again: see that the prev hsdir set got populated again. */ upload_descriptor_to_all(service, desc, 0); - tt_int_op(smartlist_len(desc->previous_hsdirs), OP_EQ, 3); + tt_int_op(smartlist_len(desc->previous_hsdirs), OP_EQ, 6); done: + SMARTLIST_FOREACH(ns->routerstatus_list, + routerstatus_t *, rs, routerstatus_free(rs)); + smartlist_clear(ns->routerstatus_list); + networkstatus_vote_free(ns); + nodelist_free_all(); hs_free_all(); } diff --git a/src/test/test_oom.c b/src/test/test_oom.c index f03a504d1d..cf28690a28 100644 --- a/src/test/test_oom.c +++ b/src/test/test_oom.c @@ -67,7 +67,7 @@ add_bytes_to_buf(buf_t *buf, size_t n_bytes) while (n_bytes) { size_t this_add = n_bytes > sizeof(b) ? sizeof(b) : n_bytes; crypto_rand(b, this_add); - write_to_buf(b, this_add, buf); + buf_add(buf, b, this_add); n_bytes -= this_add; } } diff --git a/src/test/test_options.c b/src/test/test_options.c index 5f35c47f79..92e6982823 100644 --- a/src/test/test_options.c +++ b/src/test/test_options.c @@ -528,6 +528,8 @@ test_options_validate__outbound_addresses(void *ignored) ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg); tt_int_op(ret, OP_EQ, -1); + tt_str_op(msg, OP_EQ, "Multiple outbound bind addresses configured: " + "xxyy!!!sdfaf"); done: free_options_test_data(tdata); diff --git a/src/test/test_socks.c b/src/test/test_socks.c index 7c0960f0f9..6dc8215a95 100644 --- a/src/test/test_socks.c +++ b/src/test/test_socks.c @@ -6,6 +6,7 @@ #include "or.h" #include "buffers.h" #include "config.h" +#include "proto_socks.h" #include "test.h" typedef struct socks_test_data_t { @@ -43,7 +44,7 @@ static const struct testcase_setup_t socks_setup = { buf_t *buf = testdata->buf; \ socks_request_t *socks = testdata->req; #define ADD_DATA(buf, s) \ - write_to_buf(s, sizeof(s)-1, buf) + buf_add(buf, s, sizeof(s)-1) static void socks_request_clear(socks_request_t *socks) |