From b06b95d81d362b08f98d98cb45b3c7a651273004 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 29 Apr 2009 14:00:14 -0400 Subject: [PATCH 001/187] Add foo.o and foo~ files to gitignore. --- .gitignore | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index b15ee9282c..f5d2d4ee96 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,10 @@ -# Global ignores +# Editor droppings \#*\# .#* +*~ +# C stuff +*.o +# Diff droppings *.orig *.rej # gcov stuff From e3ec061bc0ba9aa1bad1615f78fa7fe602e194e0 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 29 Apr 2009 14:18:32 -0400 Subject: [PATCH 002/187] Bump master version to 0.2.2.0-alpha-dev. --- configure.in | 2 +- contrib/tor-mingw.nsi.in | 2 +- src/win32/orconfig.h | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/configure.in b/configure.in index d802d1fba3..a7ea5e1bbc 100644 --- a/configure.in +++ b/configure.in @@ -5,7 +5,7 @@ dnl Copyright (c) 2007-2008, The Tor Project, Inc. dnl See LICENSE for licensing information AC_INIT -AM_INIT_AUTOMAKE(tor, 0.2.1.14-rc) +AM_INIT_AUTOMAKE(tor, 0.2.2.0-alpha-dev) AM_CONFIG_HEADER(orconfig.h) AC_CANONICAL_HOST diff --git a/contrib/tor-mingw.nsi.in b/contrib/tor-mingw.nsi.in index bb1ee74da7..b8c4bec5d1 100644 --- a/contrib/tor-mingw.nsi.in +++ b/contrib/tor-mingw.nsi.in @@ -9,7 +9,7 @@ !include "FileFunc.nsh" !insertmacro GetParameters -!define VERSION "0.2.1.14-rc" +!define VERSION "0.2.2.0-alpha-dev" !define INSTALLER "tor-${VERSION}-win32.exe" !define WEBSITE "https://www.torproject.org/" !define LICENSE "LICENSE" diff --git a/src/win32/orconfig.h b/src/win32/orconfig.h index dca86631aa..08e601456b 100644 --- a/src/win32/orconfig.h +++ b/src/win32/orconfig.h @@ -226,6 +226,6 @@ #define USING_TWOS_COMPLEMENT /* Version number of package */ -#define VERSION "0.2.1.14-rc" +#define VERSION "0.2.2.0-alpha-dev" From 6ac3a8b0cd1946e90a4c997e8fad1bfc63890abd Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Sat, 3 Jan 2009 21:43:05 -0500 Subject: [PATCH 003/187] Command-line option to dump SHA1 digests of all source files. Now, when you call tor --digests, it dumps the SHA1 digest of each source file that Tor was built with. We support both 'sha1sum' and 'openssl sha1'. If the user is building from a tarball and they haven't edited anything, they don't need any program that calculates SHA1. If they _have_ modified a file but they don't have a program to calculate SHA1, we try to build so we do not output digests. --- .gitignore | 2 ++ ChangeLog | 6 ++++++ configure.in | 4 ++++ src/common/Makefile.am | 14 ++++++++++++++ src/common/util.c | 7 +++++++ src/common/util.h | 2 ++ src/or/Makefile.am | 16 +++++++++++++++- src/or/config.c | 8 ++++++++ 8 files changed, 58 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index f5d2d4ee96..8c80e09c1a 100644 --- a/.gitignore +++ b/.gitignore @@ -107,6 +107,7 @@ # /src/common/ /src/common/Makefile /src/common/Makefile.in +/src/common/common_sha1.i /src/common/libor.a /src/common/libor-crypto.a @@ -120,6 +121,7 @@ # /src/or/ /src/or/Makefile /src/or/Makefile.in +/src/or/or_sha1.i /src/or/micro-revision.* /src/or/tor /src/or/test diff --git a/ChangeLog b/ChangeLog index 3ca41e13b5..5103456d37 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,9 @@ +Changes in version 0.2.2.1-alpha - 2009-??-?? + o Minor features + - New --digests command-line switch to output the digests of the source + files Tor was built with. + + Changes in version 0.2.1.15??? - ????-??-?? o Minor bugfixes: - Actually return -1 in the error case for read_bandwidth_usage. Bug diff --git a/configure.in b/configure.in index a7ea5e1bbc..2b8a915355 100644 --- a/configure.in +++ b/configure.in @@ -107,6 +107,10 @@ AC_PROG_CC AC_PROG_CPP AC_PROG_MAKE_SET AC_PROG_RANLIB +AC_PROG_SED + +AC_PATH_PROG([SHA1SUM], [sha1sum], none) +AC_PATH_PROG([OPENSSL], [openssl], none) TORUSER=_tor AC_ARG_WITH(tor-user, diff --git a/src/common/Makefile.am b/src/common/Makefile.am index 105c413343..78c8ab5516 100644 --- a/src/common/Makefile.am +++ b/src/common/Makefile.am @@ -1,6 +1,8 @@ noinst_LIBRARIES = libor.a libor-crypto.a +EXTRA_DIST = common_sha1.i + #CFLAGS = -Wall -Wpointer-arith -O2 if USE_OPENBSD_MALLOC @@ -14,3 +16,15 @@ libor_a_SOURCES = address.c log.c util.c compat.c container.c mempool.c \ libor_crypto_a_SOURCES = crypto.c aes.c tortls.c torgzip.c noinst_HEADERS = address.h log.h crypto.h test.h util.h compat.h aes.h torint.h tortls.h strlcpy.c strlcat.c torgzip.h container.h ht.h mempool.h memarea.h ciphers.inc + +common_sha1.i: $(libor_SOURCES) $(libor_crypto_a_SOURCES) $(noinst_HEADERS) + if test "@SHA1SUM@" != none; then \ + @SHA1SUM@ $(libor_SOURCES) $(libor_crypto_a_SOURCES) $(noinst_HEADERS) | @SED@ -n 's/^\(.*\)$$/"\1\\n"/p' > common_sha1.i; \ + elif test "@OPENSSL@" != none; then \ + @OPENSSL@ sha1 $(libor_SOURCES) $(libor_crypto_a_SOURCES) $(noinst_HEADERS) | @SED@ -n 's/SHA1(\(.*\))= \(.*\)/"\2 \1\\n"/p' > common_sha1.i; \ + else \ + rm common_sha1.i; \ + touch common_sha1.i; \ + fi + +util.o: common_sha1.i diff --git a/src/common/util.c b/src/common/util.c index 6310811961..bb227e4b0c 100644 --- a/src/common/util.c +++ b/src/common/util.c @@ -2562,3 +2562,10 @@ write_pidfile(char *filename) } } +const char * +libor_get_digests(void) +{ + return "" +#include "common_sha1.i" + ; +} diff --git a/src/common/util.h b/src/common/util.h index cbf5334c15..363a23b661 100644 --- a/src/common/util.h +++ b/src/common/util.h @@ -294,5 +294,7 @@ void start_daemon(void); void finish_daemon(const char *desired_cwd); void write_pidfile(char *filename); +const char *libor_get_digests(void); + #endif diff --git a/src/or/Makefile.am b/src/or/Makefile.am index 28d7e736dc..03e7900944 100644 --- a/src/or/Makefile.am +++ b/src/or/Makefile.am @@ -10,7 +10,7 @@ else tor_platform_source= endif -EXTRA_DIST=ntmain.c +EXTRA_DIST=ntmain.c or_sha1.i tor_SOURCES = buffers.c circuitbuild.c circuitlist.c \ circuituse.c command.c config.c \ @@ -52,6 +52,8 @@ test_LDADD = ../common/libor.a ../common/libor-crypto.a \ noinst_HEADERS = or.h eventdns.h eventdns_tor.h micro-revision.i +config.o: or_sha1.i + tor_main.o: micro-revision.i micro-revision.i: FORCE @@ -103,5 +105,17 @@ micro-revision.i: FORCE mv micro-revision.tmp micro-revision.i; \ fi; true +or_sha1.i: $(tor_SOURCES) test_data.c test.c + if test "@SHA1SUM@" != none; then \ + @SHA1SUM@ $(tor_SOURCES) test_data.c test.c | @SED@ -n 's/^\(.*\)$$/"\1\\n"/p' > or_sha1.i; \ + elif test "@OPENSSL@" != none; then \ + @OPENSSL@ sha1 $(tor_SOURCES) test_data.c test.c | @SED@ -n 's/SHA1(\(.*\))= \(.*\)/"\2 \1\\n"/p' > or_sha1.i; \ + else \ + rm or_sha1.i; \ + touch or_sha1.i; \ + fi + + + #Dummy target to ensure that micro-revision.i _always_ gets built. FORCE: diff --git a/src/or/config.c b/src/or/config.c index aa61ecf9a5..14418d2beb 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -3927,6 +3927,14 @@ options_init_from_torrc(int argc, char **argv) printf("Tor version %s.\n",get_version()); exit(0); } + if (argc > 1 && (!strcmp(argv[1],"--digests"))) { + printf("Tor version %s.\n",get_version()); + printf("%s", libor_get_digests()); + printf("%s", "" +#include "or_sha1.i" + ); + exit(0); + } /* Go through command-line variables */ if (!global_cmdline_options) { From 9043932789c516a263c16729f53773d8ef0121a4 Mon Sep 17 00:00:00 2001 From: Jacob Appelbaum Date: Wed, 29 Apr 2009 14:35:06 -0700 Subject: [PATCH 004/187] Make the torify script use torify when available. Update Torify to use torsocks by default and to warn when the user is using the older tsocks program. Update torify and the torify man page to refelect changes to torify. Add warnings in both. --- contrib/torify.1 | 18 ++++++++++--- contrib/torify.in | 67 ++++++++++++++++++++++++++++++++++------------- 2 files changed, 63 insertions(+), 22 deletions(-) diff --git a/contrib/torify.1 b/contrib/torify.1 index b08d468451..9ae4e40d9d 100644 --- a/contrib/torify.1 +++ b/contrib/torify.1 @@ -1,22 +1,27 @@ .TH torify 1 "" Jan-2009 "" -.\" manual page by Peter Palfrader +.\" manual page by Peter Palfrader and Jacob Appelbaum .SH NAME .LP -torify \- wrapper for tsocks and tor +torify \- wrapper for torsocks or tsocks and tor .SH SYNOPSIS \fBtorify\fP\ \fIapplication\fP\ [\fIapplication's\ arguments\fP] .SH DESCRIPTION -\fBtorify\fR is a simple wrapper that calls tsocks with a tor specific +\fBtorify\fR is a simple wrapper that attempts to find the best underlying Tor +wrapper available on a system. It calls torsocks or tsocks with a tor specific configuration file. +torsocks is an improved wrapper that explictly rejects UDP, safely resolves DNS +lookups and properly socksifies your TCP connections. + tsocks itself is a wrapper between the tsocks library and the application that you would like to run socksified. -Please note that since tsocks uses LD_PRELOAD, torify cannot be applied +Please note that since both method use LD_PRELOAD, torify cannot be applied to suid binaries. +.SH WARNING You should also be aware that the way tsocks currently works only TCP connections are socksified. Be aware that this will in most circumstances not include hostname lookups which would still be routed through your @@ -25,8 +30,13 @@ normal system resolver to your usual resolving nameservers. The The Tor FAQ at https://wiki.torproject.org/noreply/TheOnionRouter/TorFAQ might have further information on this subject. +When used with torsocks, torify should not leak DNS requests or UDP data. + +Both will leak ICMP data. + .SH SEE ALSO .BR tor (1), .BR tor-resolve (1), +.BR torsocks (1), .BR tsocks (1), .BR tsocks.conf (5). diff --git a/contrib/torify.in b/contrib/torify.in index 05645fd07c..6b1e440dc1 100755 --- a/contrib/torify.in +++ b/contrib/torify.in @@ -10,36 +10,67 @@ # Define and ensure we have tsocks # XXX: what if we don't have which? +TORSOCKS="`which torsocks`" TSOCKS="`which tsocks`" +PROG="" if [ ! -x "$TSOCKS" ] then - echo "$0: Can't find tsocks in PATH. Perhaps you haven't installed it?" >&2 - exit 1 + echo "$0: Can't find tsocks in PATH. Perhaps you haven't installed it?" >&2 +else + PROG=$TSOCKS +fi +if [ ! -x "$TORSOCKS" ] +then + echo "$0: Can't find torsocks in PATH. Perhaps you haven't installed it?" >&2 +else + PROG=$TORSOCKS +fi + +if [ ! -x "$PROG" ] +then + echo "$0: Can't find the required tor helpers in our PATH. Perhaps you haven't installed them?" >&2 + exit 1; fi # Check for any argument list if [ "$#" = 0 ] then - echo "Usage: $0 [...]" >&2 - exit 1 + echo "Usage: $0 [-hv] [...]" >&2 + exit 1 fi if [ "$#" = 1 ] && ( [ "$1" = "-h" ] || [ "$1" = "--help" ] ) then - echo "Usage: $0 [...]" - exit 0 + echo "Usage: $0 [-hv] [...]" + exit 0 fi -# Define our tsocks config file -TSOCKS_CONF_FILE="@CONFDIR@/tor-tsocks.conf" -export TSOCKS_CONF_FILE - -# Check that we've got a tsocks config file -if [ -r "$TSOCKS_CONF_FILE" ] +if [ "$1" = "-v" ] || [ "$1" = "--verbose" ] then - exec tsocks "$@" - echo "$0: Failed to exec tsocks $@" >&2 - exit 1 -else - echo "$0: Missing tsocks configuration file \"$TSOCKS_CONF_FILE\"." >&2 - exit 1 + echo "We're armed with the following tsocks: $TSOCKS" + echo "We're armed with the following torsocks: $TORSOCKS" + echo "We're attempting to use $PROG for all tor action." + shift 1 +fi + +if [ "$PROG" == "$TSOCKS" ] +then + # Define our tsocks config file + TSOCKS_CONF_FILE="/etc/tor/tor-tsocks.conf" + export TSOCKS_CONF_FILE + + # Check that we've got a tsocks config file + if [ -r "$TSOCKS_CONF_FILE" ] + then + echo "WARNING: tsocks is known to leak DNS and UDP data." >&2 + exec tsocks "$@" + echo "$0: Failed to exec tsocks $@" >&2 + exit 1 + else + echo "$0: Missing tsocks configuration file \"$TSOCKS_CONF_FILE\"." >&2 + exit 1 + fi +fi +if [ "$PROG" == "$TORSOCKS" ] +then + exec torsocks "$@" fi From 4e4be180ec2a286adb35e0b180bb2af2c7b50f82 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 29 Apr 2009 19:05:56 -0400 Subject: [PATCH 005/187] Changelog entry for Jake's torify patch. --- ChangeLog | 1 + 1 file changed, 1 insertion(+) diff --git a/ChangeLog b/ChangeLog index 5103456d37..9d3cf2bcc6 100644 --- a/ChangeLog +++ b/ChangeLog @@ -2,6 +2,7 @@ Changes in version 0.2.2.1-alpha - 2009-??-?? o Minor features - New --digests command-line switch to output the digests of the source files Tor was built with. + - The "torify" script now uses torsocks where available. Changes in version 0.2.1.15??? - ????-??-?? From 596e852c313c1bb9d422c086f5702b80b5bbb4e1 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 1 May 2009 06:25:18 -0400 Subject: [PATCH 006/187] Remove the long-deprecated GETINFO addr-mappings/ --- ChangeLog | 4 ++++ src/or/control.c | 23 +++-------------------- 2 files changed, 7 insertions(+), 20 deletions(-) diff --git a/ChangeLog b/ChangeLog index 9d3cf2bcc6..236567099b 100644 --- a/ChangeLog +++ b/ChangeLog @@ -4,6 +4,10 @@ Changes in version 0.2.2.1-alpha - 2009-??-?? files Tor was built with. - The "torify" script now uses torsocks where available. + o Deprecated and removed features: + - The controller no longer accepts the old obsolete "addr-mappings/" + GETINFO value. + Changes in version 0.2.1.15??? - ????-??-?? o Minor bugfixes: diff --git a/src/or/control.c b/src/or/control.c index 5571fd30a6..deb725ec2e 100644 --- a/src/or/control.c +++ b/src/or/control.c @@ -1753,19 +1753,10 @@ getinfo_helper_events(control_connection_t *control_conn, *answer = smartlist_join_strings(status, "\r\n", 0, NULL); SMARTLIST_FOREACH(status, char *, cp, tor_free(cp)); smartlist_free(status); - } else if (!strcmpstart(question, "addr-mappings/") || - !strcmpstart(question, "address-mappings/")) { + } else if (!strcmpstart(question, "address-mappings/")) { time_t min_e, max_e; smartlist_t *mappings; - int want_expiry = !strcmpstart(question, "address-mappings/"); - if (!strcmpstart(question, "addr-mappings/")) { - /* XXXX022 This has been deprecated since 0.2.0.3-alpha, and has - generated a warning since 0.2.1.10-alpha; remove late in 0.2.2.x. */ - log_warn(LD_CONTROL, "Controller used obsolete addr-mappings/ GETINFO " - "key; use address-mappings/ instead."); - } - question += strlen(want_expiry ? "address-mappings/" - : "addr-mappings/"); + question += strlen("address-mappings/"); if (!strcmp(question, "all")) { min_e = 0; max_e = TIME_MAX; } else if (!strcmp(question, "cache")) { @@ -1778,7 +1769,7 @@ getinfo_helper_events(control_connection_t *control_conn, return 0; } mappings = smartlist_create(); - addressmap_get_mappings(mappings, min_e, max_e, want_expiry); + addressmap_get_mappings(mappings, min_e, max_e, 1); *answer = smartlist_join_strings(mappings, "\r\n", 0, NULL); SMARTLIST_FOREACH(mappings, char *, cp, tor_free(cp)); smartlist_free(mappings); @@ -1940,14 +1931,6 @@ static const getinfo_item_t getinfo_items[] = { DOC("address-mappings/config", "Current address mappings from configuration."), DOC("address-mappings/control", "Current address mappings from controller."), - PREFIX("addr-mappings/", events, NULL), - DOC("addr-mappings/all", "Current address mappings without expiry times."), - DOC("addr-mappings/cache", - "Current cached DNS replies without expiry times."), - DOC("addr-mappings/config", - "Current address mappings from configuration without expiry times."), - DOC("addr-mappings/control", - "Current address mappings from controller without expiry times."), PREFIX("status/", events, NULL), DOC("status/circuit-established", "Whether we think client functionality is working."), From e188ab9e75db5a9767c98e6cd3914e22b591bbdc Mon Sep 17 00:00:00 2001 From: phobos Date: Fri, 1 May 2009 23:05:54 -0400 Subject: [PATCH 007/187] clean up distro detection and remove dead comment blocks --- tor.spec.in | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/tor.spec.in b/tor.spec.in index ec00415f48..6133ce6049 100644 --- a/tor.spec.in +++ b/tor.spec.in @@ -15,13 +15,6 @@ %define toruser @TORUSER@ %define torgroup @TORGROUP@ -## Target a specific arch and OS -# -# default is i386 linux - -## Override any system rpm macros -# - ## Version song and dance # # This should be the Tor version number, as it appears on the tarball, @@ -54,14 +47,11 @@ %if %{is_fc} %define ostag %(sed -e 's/^.*release /fc/' -e 's/ .*$//' -e 's/\\./_/g' < /etc/fedora-release) -%else +%endif + %if %{is_rh} %define ostag %(sed -e 's/^.*release /rh/' -e 's/ .*$//' -e 's/\\./_/g' < /etc/redhat-release) %endif -%endif - -# These are probably wrong... just placeholders should we actually -# end up supporting these distributions %if %{is_mdk} %define ostag mdk @@ -290,6 +280,9 @@ exit 0 %changelog +* Fri May 01 2009 Andrew Lewman +- clean up distro detection and remove dead comment blocks + * Sun Feb 22 2009 Andrew Lewman - update the description, vendor, and packager From 4f9b12e2480d74427e80d6ecd68383b11340ea30 Mon Sep 17 00:00:00 2001 From: Marian Date: Sat, 2 May 2009 12:39:56 -0400 Subject: [PATCH 008/187] More notes and an extra element in xxx-what-uses-sha1.txt --- .../proposals/ideas/xxx-what-uses-sha1.txt | 51 ++++++++++++++----- 1 file changed, 39 insertions(+), 12 deletions(-) diff --git a/doc/spec/proposals/ideas/xxx-what-uses-sha1.txt b/doc/spec/proposals/ideas/xxx-what-uses-sha1.txt index 9b6e20c586..1e621129be 100644 --- a/doc/spec/proposals/ideas/xxx-what-uses-sha1.txt +++ b/doc/spec/proposals/ideas/xxx-what-uses-sha1.txt @@ -1,8 +1,8 @@ Filename: xxx-what-uses-sha1.txt Title: Where does Tor use SHA-1 today? Version: $Revision$ -Last-Modified: $Date$ -Author: Nick Mathewson +Last-Modified: 1-May-2009 +Authors: Nick Mathewson, Marian Created: 30-Dec-2008 Status: Meta @@ -15,9 +15,15 @@ Introduction: too long. According to smart crypto people, the SHA-2 functions (SHA-256, etc) - share too much of SHA-1's structure to be very good. Some people - like other hash functions; most of these have not seen enough - analysis to be widely regarded as an extra-good idea. + share too much of SHA-1's structure to be very good. RIPEMD-160 is + also based on flawed past hashes. Some people think other hash + functions (e.g. Whirlpool and Tiger) are not as bad; most of these + have not seen enough analysis to be used yet. + + Here is a 2006 paper about hash algorithms. + http://www.sane.nl/sane2006/program/final-papers/R10.pdf + + (Todo: Ask smart crypto people.) By 2012, the NIST SHA-3 competition will be done, and with luck we'll have something good to switch too. But it's probably a bad idea to @@ -85,19 +91,41 @@ What Tor uses hashes for today: A. All signatures are generated on the SHA-1 of their corresponding documents, using PKCS1 padding. + * In dir-spec.txt, section 1.3, it states, + "SIGNATURE" Object contains a signature (using the signing key) + of the PKCS1-padded digest of the entire document, taken from + the beginning of the Initial item, through the newline after + the Signature Item's keyword and its arguments." + So our attacker, Malcom, could generate a collision for the hash + that is signed. Thus, a second pre-image attack is possible. + Vulnerable to regular collision attack only if key is stolen. + If the key is stolen, Malcom could distribute two different + copies of the document which have the same hash. Maybe useful + for a partitioning attack? B. Router descriptors identify their corresponding extra-info documents by their SHA-1 digest. + * A third party might use a second pre-image attack to generate a + false extra-info document that has the same hash. The router + itself might use a regular collision attack to generate multiple + extra-info documents with the same hash, which might be useful + for a partitioning attack. C. Fingerprints in router descriptors are taken using SHA-1. - D. Fingerprints in authority certs are taken using SHA-1. - E. Fingerprints in dir-source lines of votes and consensuses are taken + * The fingerprint must match the public key. Not sure what would + happen if two routers had different public keys but the same + fingerprint. There could perhaps be unpredictable behaviour. + D. In router descriptors, routers in the same "Family" may be listed + by server nicknames or hexdigests. + * Does not seem critical. + E. Fingerprints in authority certs are taken using SHA-1. + F. Fingerprints in dir-source lines of votes and consensuses are taken using SHA-1. - F. Networkstatuses refer to routers identity keys and descriptors by their + G. Networkstatuses refer to routers identity keys and descriptors by their SHA-1 digests. - G. Directory-signature lines identify which key is doing the signing by + H. Directory-signature lines identify which key is doing the signing by the SHA-1 digests of the authority's signing key and its identity key. - H. The following items are downloaded by the SHA-1 of their contents: + I. The following items are downloaded by the SHA-1 of their contents: XXXX list them - I. The following items are downloaded by the SHA-1 of an identity key: + J. The following items are downloaded by the SHA-1 of an identity key: XXXX list them too. 4. The rendezvous protocol @@ -137,4 +165,3 @@ What Tor uses hashes for today: hashes of their identity keys. E. The deprecated .exit notation uses SHA-1 hashes of identity keys - From 08e4a077207a5def143cf62ad34ce6c76e6f8300 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Sat, 2 May 2009 11:49:45 -0500 Subject: [PATCH 009/187] delete trailing whitespace in xxx-what-uses-sha1.txt --- .../proposals/ideas/xxx-what-uses-sha1.txt | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/doc/spec/proposals/ideas/xxx-what-uses-sha1.txt b/doc/spec/proposals/ideas/xxx-what-uses-sha1.txt index 1e621129be..3112aee1f0 100644 --- a/doc/spec/proposals/ideas/xxx-what-uses-sha1.txt +++ b/doc/spec/proposals/ideas/xxx-what-uses-sha1.txt @@ -15,10 +15,10 @@ Introduction: too long. According to smart crypto people, the SHA-2 functions (SHA-256, etc) - share too much of SHA-1's structure to be very good. RIPEMD-160 is - also based on flawed past hashes. Some people think other hash - functions (e.g. Whirlpool and Tiger) are not as bad; most of these - have not seen enough analysis to be used yet. + share too much of SHA-1's structure to be very good. RIPEMD-160 is + also based on flawed past hashes. Some people think other hash + functions (e.g. Whirlpool and Tiger) are not as bad; most of these + have not seen enough analysis to be used yet. Here is a 2006 paper about hash algorithms. http://www.sane.nl/sane2006/program/final-papers/R10.pdf @@ -91,29 +91,29 @@ What Tor uses hashes for today: A. All signatures are generated on the SHA-1 of their corresponding documents, using PKCS1 padding. - * In dir-spec.txt, section 1.3, it states, - "SIGNATURE" Object contains a signature (using the signing key) - of the PKCS1-padded digest of the entire document, taken from - the beginning of the Initial item, through the newline after + * In dir-spec.txt, section 1.3, it states, + "SIGNATURE" Object contains a signature (using the signing key) + of the PKCS1-padded digest of the entire document, taken from + the beginning of the Initial item, through the newline after the Signature Item's keyword and its arguments." - So our attacker, Malcom, could generate a collision for the hash - that is signed. Thus, a second pre-image attack is possible. + So our attacker, Malcom, could generate a collision for the hash + that is signed. Thus, a second pre-image attack is possible. Vulnerable to regular collision attack only if key is stolen. - If the key is stolen, Malcom could distribute two different + If the key is stolen, Malcom could distribute two different copies of the document which have the same hash. Maybe useful for a partitioning attack? B. Router descriptors identify their corresponding extra-info documents by their SHA-1 digest. - * A third party might use a second pre-image attack to generate a - false extra-info document that has the same hash. The router - itself might use a regular collision attack to generate multiple - extra-info documents with the same hash, which might be useful + * A third party might use a second pre-image attack to generate a + false extra-info document that has the same hash. The router + itself might use a regular collision attack to generate multiple + extra-info documents with the same hash, which might be useful for a partitioning attack. C. Fingerprints in router descriptors are taken using SHA-1. - * The fingerprint must match the public key. Not sure what would - happen if two routers had different public keys but the same + * The fingerprint must match the public key. Not sure what would + happen if two routers had different public keys but the same fingerprint. There could perhaps be unpredictable behaviour. - D. In router descriptors, routers in the same "Family" may be listed + D. In router descriptors, routers in the same "Family" may be listed by server nicknames or hexdigests. * Does not seem critical. E. Fingerprints in authority certs are taken using SHA-1. From 7e411f4b302e01f36626ca3476bdbac225383410 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Sat, 2 May 2009 11:52:38 -0500 Subject: [PATCH 010/187] Initial division of SHA1 uses by collision/preimage needs. More thought is needed on each collision opportunity to figure out how exploitable it is. --- .../proposals/ideas/xxx-what-uses-sha1.txt | 53 ++++++++++++++++++- 1 file changed, 51 insertions(+), 2 deletions(-) diff --git a/doc/spec/proposals/ideas/xxx-what-uses-sha1.txt b/doc/spec/proposals/ideas/xxx-what-uses-sha1.txt index 3112aee1f0..b0be567a4a 100644 --- a/doc/spec/proposals/ideas/xxx-what-uses-sha1.txt +++ b/doc/spec/proposals/ideas/xxx-what-uses-sha1.txt @@ -60,35 +60,81 @@ Why now? one look silly. +Triage + + How severe are these problems? Let's divide them into these + categories: + PREIMAGE -- A SHA-1 usage that only depends on preimage + resistance + COLLISION -- A SHA-1 usage that depends on collision + resistance, but the only party who could mount a + collision-based attack is already in a trusted role + (like a distribution signer or a directory authority). + COLLISION -- A SHA-1 usage that depends on collision resistance + and doesn't need the attacker to have any special keys. + + There is no need to put much effort into fixing PREIMAGE usages in + the near-term: even _MD5_ is still preimage resistant. To fix + COLLISION usages is not too important either, since + anyone who has the key to sign the code can mount far worse + attacks. It would be good to fix COLLISION usages, + since we try to resist. The COLLISION usages are the most + important to fix. + + Additionally, we need to consider the impact of a successful attack + in each of these cases. SHA-1 collisions are still expensive even + if recent results are verified, and anybody with the resources to + compute one also has the resources to mount a decent Sybil attack. + + Let be pessimistic, and not assume that producing collisions of a given + format is actually any harder than producing collisions at all. + What Tor uses hashes for today: 1. Infrastructure. A. Our X.509 certificates are signed with SHA-1. + COLLSION B. TLS uses SHA-1 (and MD5) internally to generate keys. + PREIMAGE? C. Some of the TLS ciphersuites we allow use SHA-1. + PREIMAGE? D. When we sign our code with GPG, it might be using SHA-1. + COLLISION E. Our GPG keys might be authenticated with SHA-1. + COLLISION F. OpenSSL's random number generator uses SHA-1, I believe. + PREIMAGE 2. The Tor protocol A. Everything we sign, we sign using SHA-1-based OAEP-MGF1. + PREIMAGE? B. Our CREATE cell format uses SHA-1 for: OAEP padding. + PREIMAGE? C. Our EXTEND cells use SHA-1 to hash the identity key of the target server. + COLLISION D. Our CREATED cells use SHA-1 to hash the derived key data. + ?? E. The data we use in CREATE_FAST cells to generate a key is the length of a SHA-1. + NONE F. The data we send back in a CREATED/CREATED_FAST cell is the length of a SHA-1. - G. We use SHA-1 to derive our circuit keys from the negotiated g^xy value. + NONE + G. We use SHA-1 to derive our circuit keys from the negotiated g^xy + value. + NONE H. We use SHA-1 to derive the digest field of each RELAY cell, but that's used more as a checksum than as a strong digest. + NONE 3. Directory services + [All are COLLISION or COLLISION ] + A. All signatures are generated on the SHA-1 of their corresponding documents, using PKCS1 padding. * In dir-spec.txt, section 1.3, it states, @@ -159,9 +205,12 @@ What Tor uses hashes for today: A. We log information about servers based on SHA-1 hashes of their identity keys. + COLLISION B. The controller identifies servers based on SHA-1 hashes of their identity keys. + COLLISION C. Nearly all of our configuration options that list servers allow SHA-1 hashes of their identity keys. + COLLISION E. The deprecated .exit notation uses SHA-1 hashes of identity keys - + COLLISION From e25fdce900b305b3d0aa2bd54594d0189386d4db Mon Sep 17 00:00:00 2001 From: Karsten Loesing Date: Sat, 2 May 2009 21:47:03 +0200 Subject: [PATCH 011/187] Documentation fix. --- src/or/relay.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/or/relay.c b/src/or/relay.c index b4592e85f8..2d751e31c9 100644 --- a/src/or/relay.c +++ b/src/or/relay.c @@ -1660,7 +1660,7 @@ prev_circ_on_conn_p(circuit_t *circ, or_connection_t *conn) } /** Add circ to the list of circuits with pending cells on - * conn. No effect if circ is already unlinked. */ + * conn. No effect if circ is already linked. */ void make_circuit_active_on_conn(circuit_t *circ, or_connection_t *conn) { @@ -1686,7 +1686,7 @@ make_circuit_active_on_conn(circuit_t *circ, or_connection_t *conn) assert_active_circuits_ok_paranoid(conn); } -/** Remove circ to the list of circuits with pending cells on +/** Remove circ from the list of circuits with pending cells on * conn. No effect if circ is already unlinked. */ void make_circuit_inactive_on_conn(circuit_t *circ, or_connection_t *conn) From 4ebcc4da344540ecacf10301448550cc77af45a4 Mon Sep 17 00:00:00 2001 From: Karsten Loesing Date: Sat, 2 May 2009 22:00:54 +0200 Subject: [PATCH 012/187] Update copyright to 2009. --- src/common/address.c | 2 +- src/common/address.h | 2 +- src/common/aes.c | 2 +- src/common/aes.h | 2 +- src/common/compat.c | 2 +- src/common/compat.h | 2 +- src/common/container.c | 2 +- src/common/container.h | 2 +- src/common/crypto.c | 2 +- src/common/crypto.h | 2 +- src/common/ht.h | 5 +++-- src/common/log.c | 2 +- src/common/log.h | 2 +- src/common/memarea.c | 2 +- src/common/memarea.h | 2 +- src/common/mempool.c | 2 +- src/common/mempool.h | 2 +- src/common/test.h | 2 +- src/common/torgzip.c | 2 +- src/common/torgzip.h | 2 +- src/common/torint.h | 2 +- src/common/tortls.c | 2 +- src/common/tortls.h | 2 +- src/common/util.c | 2 +- src/common/util.h | 2 +- src/or/buffers.c | 2 +- src/or/circuitbuild.c | 2 +- src/or/circuitlist.c | 2 +- src/or/circuituse.c | 2 +- src/or/command.c | 2 +- src/or/config.c | 4 ++-- src/or/connection.c | 2 +- src/or/connection_edge.c | 2 +- src/or/connection_or.c | 2 +- src/or/control.c | 2 +- src/or/cpuworker.c | 2 +- src/or/directory.c | 2 +- src/or/dirserv.c | 2 +- src/or/dirvote.c | 2 +- src/or/dns.c | 2 +- src/or/dnsserv.c | 2 +- src/or/eventdns_tor.h | 2 +- src/or/geoip.c | 2 +- src/or/hibernate.c | 2 +- src/or/main.c | 2 +- src/or/networkstatus.c | 2 +- src/or/ntmain.c | 2 +- src/or/onion.c | 2 +- src/or/or.h | 2 +- src/or/policies.c | 2 +- src/or/reasons.c | 2 +- src/or/relay.c | 2 +- src/or/rendclient.c | 2 +- src/or/rendcommon.c | 2 +- src/or/rendmid.c | 2 +- src/or/rendservice.c | 2 +- src/or/rephist.c | 2 +- src/or/router.c | 2 +- src/or/routerlist.c | 2 +- src/or/routerparse.c | 2 +- src/or/test.c | 2 +- src/or/test_data.c | 2 +- src/or/tor_main.c | 2 +- src/tools/tor-gencert.c | 2 +- src/tools/tor-resolve.c | 2 +- 65 files changed, 68 insertions(+), 67 deletions(-) diff --git a/src/common/address.c b/src/common/address.c index 59412363ab..ced1d0cd8f 100644 --- a/src/common/address.c +++ b/src/common/address.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2008, The Tor Project, Inc. */ + * Copyright (c) 2007-2009, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/address.h b/src/common/address.h index 521a360fa1..9480d64274 100644 --- a/src/common/address.h +++ b/src/common/address.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2008, The Tor Project, Inc. */ + * Copyright (c) 2007-2009, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/aes.c b/src/common/aes.c index 60df36ecdd..5a979bfa58 100644 --- a/src/common/aes.c +++ b/src/common/aes.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2008, The Tor Project, Inc. */ + * Copyright (c) 2007-2009, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/aes.h b/src/common/aes.h index a566fb4b70..f3e5796c78 100644 --- a/src/common/aes.h +++ b/src/common/aes.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2008, The Tor Project, Inc. */ + * Copyright (c) 2007-2009, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /* Implements a minimal interface to counter-mode AES. */ diff --git a/src/common/compat.c b/src/common/compat.c index cfd428133e..82957722c9 100644 --- a/src/common/compat.c +++ b/src/common/compat.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2008, The Tor Project, Inc. */ + * Copyright (c) 2007-2009, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/compat.h b/src/common/compat.h index b80bdd1222..0e2875dabc 100644 --- a/src/common/compat.h +++ b/src/common/compat.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledinex * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2008, The Tor Project, Inc. */ + * Copyright (c) 2007-2009, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef _TOR_COMPAT_H diff --git a/src/common/container.c b/src/common/container.c index 0a2cdd82e7..1ef0044f1b 100644 --- a/src/common/container.c +++ b/src/common/container.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2008, The Tor Project, Inc. */ + * Copyright (c) 2007-2009, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/container.h b/src/common/container.h index b4ec4ee246..f7f835874b 100644 --- a/src/common/container.h +++ b/src/common/container.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2008, The Tor Project, Inc. */ + * Copyright (c) 2007-2009, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef _TOR_CONTAINER_H diff --git a/src/common/crypto.c b/src/common/crypto.c index 0abd30be1a..bc7eb0c6ca 100644 --- a/src/common/crypto.c +++ b/src/common/crypto.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2008, The Tor Project, Inc. */ + * Copyright (c) 2007-2009, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/crypto.h b/src/common/crypto.h index 906098e06f..dd353ef030 100644 --- a/src/common/crypto.h +++ b/src/common/crypto.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2008, The Tor Project, Inc. */ + * Copyright (c) 2007-2009, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/ht.h b/src/common/ht.h index 54b030b047..ff1e5551cc 100644 --- a/src/common/ht.h +++ b/src/common/ht.h @@ -1,5 +1,6 @@ -/* Copyright 2002 Christopher Clark */ -/* Copyright 2005 Nick Mathewson */ +/* Copyright (c) 2002, Christopher Clark. + * Copyright (c) 2005-2006, Nick Mathewson. + * Copyright (c) 2007-2009, The Tor Project, Inc. */ /* See license at end. */ /* Based on ideas by Christopher Clark and interfaces from Niels Provos. */ diff --git a/src/common/log.c b/src/common/log.c index 421be67a23..a7b0c12c4a 100644 --- a/src/common/log.c +++ b/src/common/log.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2008, The Tor Project, Inc. */ + * Copyright (c) 2007-2009, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/log.h b/src/common/log.h index 66f5ee39bf..c72fb6fb16 100644 --- a/src/common/log.h +++ b/src/common/log.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2008, The Tor Project, Inc. */ + * Copyright (c) 2007-2009, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/memarea.c b/src/common/memarea.c index 6e0bda0c8a..7771f2c2ac 100644 --- a/src/common/memarea.c +++ b/src/common/memarea.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2008, The Tor Project, Inc. */ +/* Copyright (c) 2008-2009, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** \file memarea.c diff --git a/src/common/memarea.h b/src/common/memarea.h index 2e3c884802..04ffae7cae 100644 --- a/src/common/memarea.h +++ b/src/common/memarea.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2008, The Tor Project, Inc. */ +/* Copyright (c) 2008-2009, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /* Tor dependencies */ diff --git a/src/common/mempool.c b/src/common/mempool.c index 3825957e7d..bbfa3c256a 100644 --- a/src/common/mempool.c +++ b/src/common/mempool.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2007-2008, The Tor Project, Inc. */ +/* Copyright (c) 2007-2009, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #if 1 /* Tor dependencies */ diff --git a/src/common/mempool.h b/src/common/mempool.h index ca8cc1432f..d6813f1baa 100644 --- a/src/common/mempool.h +++ b/src/common/mempool.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2007-2008, The Tor Project, Inc. */ +/* Copyright (c) 2007-2009, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/test.h b/src/common/test.h index e3b5744283..52a249ccb0 100644 --- a/src/common/test.h +++ b/src/common/test.h @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2003, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2008, The Tor Project, Inc. */ + * Copyright (c) 2007-2009, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef _TOR_TEST_H diff --git a/src/common/torgzip.c b/src/common/torgzip.c index f75ef83503..196f5521f7 100644 --- a/src/common/torgzip.c +++ b/src/common/torgzip.c @@ -1,6 +1,6 @@ /* Copyright (c) 2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2008, The Tor Project, Inc. */ + * Copyright (c) 2007-2009, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/torgzip.h b/src/common/torgzip.h index 6b994c2fc4..d6432283c3 100644 --- a/src/common/torgzip.h +++ b/src/common/torgzip.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2008, The Tor Project, Inc. */ + * Copyright (c) 2007-2009, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/torint.h b/src/common/torint.h index 657974d579..1f7421174a 100644 --- a/src/common/torint.h +++ b/src/common/torint.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2008, The Tor Project, Inc. */ + * Copyright (c) 2007-2009, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/tortls.c b/src/common/tortls.c index 9847bc960a..581b35848a 100644 --- a/src/common/tortls.c +++ b/src/common/tortls.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2008, The Tor Project, Inc. */ + * Copyright (c) 2007-2009, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/tortls.h b/src/common/tortls.h index 37e762ed3a..44e3b499ef 100644 --- a/src/common/tortls.h +++ b/src/common/tortls.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2008, The Tor Project, Inc. */ + * Copyright (c) 2007-2009, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef _TOR_TORTLS_H diff --git a/src/common/util.c b/src/common/util.c index bb227e4b0c..69cc0c4951 100644 --- a/src/common/util.c +++ b/src/common/util.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2008, The Tor Project, Inc. */ + * Copyright (c) 2007-2009, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/util.h b/src/common/util.h index 363a23b661..7fb540b3d2 100644 --- a/src/common/util.h +++ b/src/common/util.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2008, The Tor Project, Inc. */ + * Copyright (c) 2007-2009, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/buffers.c b/src/or/buffers.c index 6b8ac09918..00db5d0916 100644 --- a/src/or/buffers.c +++ b/src/or/buffers.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2008, The Tor Project, Inc. */ + * Copyright (c) 2007-2009, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/circuitbuild.c b/src/or/circuitbuild.c index 3ed291a3ff..e56290fb50 100644 --- a/src/or/circuitbuild.c +++ b/src/or/circuitbuild.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2008, The Tor Project, Inc. */ + * Copyright (c) 2007-2009, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/circuitlist.c b/src/or/circuitlist.c index 224443e55d..252eaf9f8e 100644 --- a/src/or/circuitlist.c +++ b/src/or/circuitlist.c @@ -1,7 +1,7 @@ /* Copyright 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2008, The Tor Project, Inc. */ + * Copyright (c) 2007-2009, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/circuituse.c b/src/or/circuituse.c index a1c8d1c148..4ee99146e6 100644 --- a/src/or/circuituse.c +++ b/src/or/circuituse.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2008, The Tor Project, Inc. */ + * Copyright (c) 2007-2009, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/command.c b/src/or/command.c index c6e2c12979..9481e5bcc6 100644 --- a/src/or/command.c +++ b/src/or/command.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2008, The Tor Project, Inc. */ + * Copyright (c) 2007-2009, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/config.c b/src/or/config.c index 14418d2beb..76de52748c 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2008, The Tor Project, Inc. */ + * Copyright (c) 2007-2009, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -2246,7 +2246,7 @@ print_usage(void) printf( "Copyright (c) 2001-2004, Roger Dingledine\n" "Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson\n" -"Copyright (c) 2007-2008, The Tor Project, Inc.\n\n" +"Copyright (c) 2007-2009, The Tor Project, Inc.\n\n" "tor -f [args]\n" "See man page for options, or https://www.torproject.org/ for " "documentation.\n"); diff --git a/src/or/connection.c b/src/or/connection.c index 3812dd8e3e..3a01c28652 100644 --- a/src/or/connection.c +++ b/src/or/connection.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2008, The Tor Project, Inc. */ + * Copyright (c) 2007-2009, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/connection_edge.c b/src/or/connection_edge.c index c2f4f68440..b350c08d29 100644 --- a/src/or/connection_edge.c +++ b/src/or/connection_edge.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2008, The Tor Project, Inc. */ + * Copyright (c) 2007-2009, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/connection_or.c b/src/or/connection_or.c index 4e729ad6f2..b4e80926be 100644 --- a/src/or/connection_or.c +++ b/src/or/connection_or.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2008, The Tor Project, Inc. */ + * Copyright (c) 2007-2009, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/control.c b/src/or/control.c index deb725ec2e..53b9952481 100644 --- a/src/or/control.c +++ b/src/or/control.c @@ -1,5 +1,5 @@ /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2008, The Tor Project, Inc. */ + * Copyright (c) 2007-2009, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/cpuworker.c b/src/or/cpuworker.c index 99e06df35e..99829e89b5 100644 --- a/src/or/cpuworker.c +++ b/src/or/cpuworker.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2008, The Tor Project, Inc. */ + * Copyright (c) 2007-2009, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/directory.c b/src/or/directory.c index c72061a9f9..082eca2006 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2008, The Tor Project, Inc. */ + * Copyright (c) 2007-2009, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "or.h" diff --git a/src/or/dirserv.c b/src/or/dirserv.c index 9fc30154e3..dde8959f88 100644 --- a/src/or/dirserv.c +++ b/src/or/dirserv.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2008, The Tor Project, Inc. */ + * Copyright (c) 2007-2009, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define DIRSERV_PRIVATE diff --git a/src/or/dirvote.c b/src/or/dirvote.c index 3db1954a88..f0feca95a8 100644 --- a/src/or/dirvote.c +++ b/src/or/dirvote.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2008, The Tor Project, Inc. */ + * Copyright (c) 2007-2009, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define DIRVOTE_PRIVATE diff --git a/src/or/dns.c b/src/or/dns.c index add4b47403..1742e3dd4b 100644 --- a/src/or/dns.c +++ b/src/or/dns.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2008, The Tor Project, Inc. */ + * Copyright (c) 2007-2009, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/dnsserv.c b/src/or/dnsserv.c index 1f964780e9..6020f9a781 100644 --- a/src/or/dnsserv.c +++ b/src/or/dnsserv.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2007-2008, The Tor Project, Inc. */ +/* Copyright (c) 2007-2009, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/eventdns_tor.h b/src/or/eventdns_tor.h index f4361209d5..3d84fc7794 100644 --- a/src/or/eventdns_tor.h +++ b/src/or/eventdns_tor.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2007-2008, The Tor Project, Inc. */ +/* Copyright (c) 2007-2009, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/or/geoip.c b/src/or/geoip.c index 61f1207e59..eb0d7e43f4 100644 --- a/src/or/geoip.c +++ b/src/or/geoip.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2007-2008, The Tor Project, Inc. */ +/* Copyright (c) 2007-2009, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/hibernate.c b/src/or/hibernate.c index d62036afb9..8f347a48cb 100644 --- a/src/or/hibernate.c +++ b/src/or/hibernate.c @@ -1,5 +1,5 @@ /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2008, The Tor Project, Inc. */ + * Copyright (c) 2007-2009, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/main.c b/src/or/main.c index dd15024b7b..69ed12f12a 100644 --- a/src/or/main.c +++ b/src/or/main.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2008, The Tor Project, Inc. */ + * Copyright (c) 2007-2009, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/networkstatus.c b/src/or/networkstatus.c index 8e340a4a57..54ccb113ce 100644 --- a/src/or/networkstatus.c +++ b/src/or/networkstatus.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2008, The Tor Project, Inc. */ + * Copyright (c) 2007-2009, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/ntmain.c b/src/or/ntmain.c index 385e1066a9..2304be6526 100644 --- a/src/or/ntmain.c +++ b/src/or/ntmain.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2008, The Tor Project, Inc. */ + * Copyright (c) 2007-2009, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define MAIN_PRIVATE diff --git a/src/or/onion.c b/src/or/onion.c index 92dc12b988..9e8b43c23f 100644 --- a/src/or/onion.c +++ b/src/or/onion.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2008, The Tor Project, Inc. */ + * Copyright (c) 2007-2009, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/or.h b/src/or/or.h index aa3824a3dc..731cf5e9a8 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2008, The Tor Project, Inc. */ + * Copyright (c) 2007-2009, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/policies.c b/src/or/policies.c index a2ab13a90a..6bff4e4dad 100644 --- a/src/or/policies.c +++ b/src/or/policies.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2008, The Tor Project, Inc. */ + * Copyright (c) 2007-2009, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/reasons.c b/src/or/reasons.c index c427a8dc74..a252f83198 100644 --- a/src/or/reasons.c +++ b/src/or/reasons.c @@ -1,5 +1,5 @@ /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2008, The Tor Project, Inc. */ + * Copyright (c) 2007-2009, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/relay.c b/src/or/relay.c index 2d751e31c9..472d638c81 100644 --- a/src/or/relay.c +++ b/src/or/relay.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2008, The Tor Project, Inc. */ + * Copyright (c) 2007-2009, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/rendclient.c b/src/or/rendclient.c index c68a297c02..fb50e8dc00 100644 --- a/src/or/rendclient.c +++ b/src/or/rendclient.c @@ -1,5 +1,5 @@ /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2008, The Tor Project, Inc. */ + * Copyright (c) 2007-2009, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/rendcommon.c b/src/or/rendcommon.c index 1de0dd67ba..d21eb42efe 100644 --- a/src/or/rendcommon.c +++ b/src/or/rendcommon.c @@ -1,5 +1,5 @@ /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2008, The Tor Project, Inc. */ + * Copyright (c) 2007-2009, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/rendmid.c b/src/or/rendmid.c index 7f64fecbc0..57f5b6daed 100644 --- a/src/or/rendmid.c +++ b/src/or/rendmid.c @@ -1,5 +1,5 @@ /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2008, The Tor Project, Inc. */ + * Copyright (c) 2007-2009, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/rendservice.c b/src/or/rendservice.c index 9d677067cc..94c619ced8 100644 --- a/src/or/rendservice.c +++ b/src/or/rendservice.c @@ -1,5 +1,5 @@ /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2008, The Tor Project, Inc. */ + * Copyright (c) 2007-2009, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/rephist.c b/src/or/rephist.c index 5a84d31c62..7bacd3abb6 100644 --- a/src/or/rephist.c +++ b/src/or/rephist.c @@ -1,5 +1,5 @@ /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2008, The Tor Project, Inc. */ + * Copyright (c) 2007-2009, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/router.c b/src/or/router.c index 13fe268da6..45ea18b14f 100644 --- a/src/or/router.c +++ b/src/or/router.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2008, The Tor Project, Inc. */ + * Copyright (c) 2007-2009, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define ROUTER_PRIVATE diff --git a/src/or/routerlist.c b/src/or/routerlist.c index 3e78ed5479..c6bfca4546 100644 --- a/src/or/routerlist.c +++ b/src/or/routerlist.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2008, The Tor Project, Inc. */ + * Copyright (c) 2007-2009, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/routerparse.c b/src/or/routerparse.c index 3e33067d18..c08a058a09 100644 --- a/src/or/routerparse.c +++ b/src/or/routerparse.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2008, The Tor Project, Inc. */ + * Copyright (c) 2007-2009, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/test.c b/src/or/test.c index 03c6013e13..bd0f485dd8 100644 --- a/src/or/test.c +++ b/src/or/test.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2008, The Tor Project, Inc. */ + * Copyright (c) 2007-2009, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /* Ordinarily defined in tor_main.c; this bit is just here to provide one diff --git a/src/or/test_data.c b/src/or/test_data.c index 61fa37af44..bd7d989856 100644 --- a/src/or/test_data.c +++ b/src/or/test_data.c @@ -1,6 +1,6 @@ /* Copyright 2001-2004 Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2008, The Tor Project, Inc. */ + * Copyright (c) 2007-2009, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** First of 3 example authority certificates for unit testing. */ diff --git a/src/or/tor_main.c b/src/or/tor_main.c index 5890c9aa9c..137da02c41 100644 --- a/src/or/tor_main.c +++ b/src/or/tor_main.c @@ -1,6 +1,6 @@ /* Copyright 2001-2004 Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2008, The Tor Project, Inc. */ + * Copyright (c) 2007-2009, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** String describing which Tor subversion repository version the source was diff --git a/src/tools/tor-gencert.c b/src/tools/tor-gencert.c index cbb8826e48..4971668c9f 100644 --- a/src/tools/tor-gencert.c +++ b/src/tools/tor-gencert.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2007-2008 The Tor Project, Inc. */ +/* Copyright (c) 2007-2009 The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/tools/tor-resolve.c b/src/tools/tor-resolve.c index 6ea2dc2352..ca75a5a56c 100644 --- a/src/tools/tor-resolve.c +++ b/src/tools/tor-resolve.c @@ -1,5 +1,5 @@ /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson - * Copyright (c) 2007-2008, The Tor Project, Inc. + * Copyright (c) 2007-2009, The Tor Project, Inc. */ /* See LICENSE for licensing information */ From 0d68da2381780e2f8010b52a9e062947554699d5 Mon Sep 17 00:00:00 2001 From: Karsten Loesing Date: Sat, 2 May 2009 23:31:58 +0200 Subject: [PATCH 013/187] Drop version 0 hidserv support on service side. --- src/or/rendcommon.c | 55 --------------- src/or/rendservice.c | 155 ++++++++----------------------------------- src/or/test.c | 87 ++++-------------------- 3 files changed, 39 insertions(+), 258 deletions(-) diff --git a/src/or/rendcommon.c b/src/or/rendcommon.c index d21eb42efe..c813c4959d 100644 --- a/src/or/rendcommon.c +++ b/src/or/rendcommon.c @@ -655,61 +655,6 @@ rend_encode_v2_descriptors(smartlist_t *descs_out, return seconds_valid; } -/** Encode a service descriptor for desc, and sign it with - * key. Store the descriptor in *str_out, and set - * *len_out to its length. - */ -int -rend_encode_service_descriptor(rend_service_descriptor_t *desc, - crypto_pk_env_t *key, - char **str_out, size_t *len_out) -{ - char *cp; - char *end; - int i, r; - size_t asn1len; - size_t buflen = - PK_BYTES*2*(smartlist_len(desc->intro_nodes)+2);/*Too long, but ok*/ - cp = *str_out = tor_malloc(buflen); - end = cp + PK_BYTES*2*(smartlist_len(desc->intro_nodes)+1); - r = crypto_pk_asn1_encode(desc->pk, cp+2, end-(cp+2)); - if (r < 0) { - tor_free(*str_out); - return -1; - } - asn1len = r; - set_uint16(cp, htons((uint16_t)asn1len)); - cp += 2+asn1len; - set_uint32(cp, htonl((uint32_t)desc->timestamp)); - cp += 4; - set_uint16(cp, htons((uint16_t)smartlist_len(desc->intro_nodes))); - cp += 2; - for (i=0; i < smartlist_len(desc->intro_nodes); ++i) { - rend_intro_point_t *intro = smartlist_get(desc->intro_nodes, i); - char ipoint[HEX_DIGEST_LEN+2]; - const size_t ipoint_len = HEX_DIGEST_LEN+1; - ipoint[0] = '$'; - base16_encode(ipoint+1, HEX_DIGEST_LEN+1, - intro->extend_info->identity_digest, - DIGEST_LEN); - tor_assert(strlen(ipoint) == ipoint_len); - /* Assert that appending ipoint and its NUL won't over overrun the - * buffer. */ - tor_assert(cp + ipoint_len+1 < *str_out + buflen); - memcpy(cp, ipoint, ipoint_len+1); - cp += ipoint_len+1; - } - note_crypto_pk_op(REND_SERVER); - r = crypto_pk_private_sign_digest(key, cp, *str_out, cp-*str_out); - if (r<0) { - tor_free(*str_out); - return -1; - } - cp += r; - *len_out = (size_t)(cp-*str_out); - return 0; -} - /** Parse a service descriptor at str (len bytes). On * success, return a newly alloced service_descriptor_t. On failure, * return NULL. diff --git a/src/or/rendservice.c b/src/or/rendservice.c index 94c619ced8..b30e708f08 100644 --- a/src/or/rendservice.c +++ b/src/or/rendservice.c @@ -42,8 +42,6 @@ typedef struct rend_service_t { /* Fields specified in config file */ char *directory; /**< where in the filesystem it stores it */ smartlist_t *ports; /**< List of rend_service_port_config_t */ - int descriptor_version; /**< Rendezvous descriptor version that will be - * published. */ rend_auth_type_t auth_type; /**< Client authorization type or 0 if no client * authorization is performed. */ smartlist_t *clients; /**< List of rend_authorized_client_t's of @@ -156,36 +154,6 @@ rend_add_service(rend_service_t *service) service->intro_nodes = smartlist_create(); - /* If the service is configured to publish unversioned (v0) and versioned - * descriptors (v2 or higher), split it up into two separate services - * (unless it is configured to perform client authorization). */ - if (service->descriptor_version == -1) { - if (service->auth_type == REND_NO_AUTH) { - rend_service_t *v0_service = tor_malloc_zero(sizeof(rend_service_t)); - v0_service->directory = tor_strdup(service->directory); - v0_service->ports = smartlist_create(); - SMARTLIST_FOREACH(service->ports, rend_service_port_config_t *, p, { - rend_service_port_config_t *copy = - tor_malloc_zero(sizeof(rend_service_port_config_t)); - memcpy(copy, p, sizeof(rend_service_port_config_t)); - smartlist_add(v0_service->ports, copy); - }); - v0_service->intro_period_started = service->intro_period_started; - v0_service->descriptor_version = 0; /* Unversioned descriptor. */ - v0_service->auth_type = REND_NO_AUTH; - rend_add_service(v0_service); - } - - service->descriptor_version = 2; /* Versioned descriptor. */ - } - - if (service->auth_type != REND_NO_AUTH && !service->descriptor_version) { - log_warn(LD_CONFIG, "Hidden service with client authorization and " - "version 0 descriptors configured; ignoring."); - rend_service_free(service); - return; - } - if (service->auth_type != REND_NO_AUTH && smartlist_len(service->clients) == 0) { log_warn(LD_CONFIG, "Hidden service with client authorization but no " @@ -307,7 +275,6 @@ rend_config_services(or_options_t *options, int validate_only) service->directory = tor_strdup(line->value); service->ports = smartlist_create(); service->intro_period_started = time(NULL); - service->descriptor_version = -1; /**< All descriptor versions. */ continue; } if (!service) { @@ -433,35 +400,13 @@ rend_config_services(or_options_t *options, int validate_only) return -1; } } else { - smartlist_t *versions; - char *version_str; - int i, version, ver_ok=1, versions_bitmask = 0; tor_assert(!strcasecmp(line->key, "HiddenServiceVersion")); - versions = smartlist_create(); - smartlist_split_string(versions, line->value, ",", - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - for (i = 0; i < smartlist_len(versions); i++) { - version_str = smartlist_get(versions, i); - if (strlen(version_str) != 1 || strspn(version_str, "02") != 1) { - log_warn(LD_CONFIG, - "HiddenServiceVersion can only be 0 and/or 2."); - SMARTLIST_FOREACH(versions, char *, cp, tor_free(cp)); - smartlist_free(versions); - rend_service_free(service); - return -1; - } - version = (int)tor_parse_long(version_str, 10, 0, INT_MAX, &ver_ok, - NULL); - if (!ver_ok) - continue; - versions_bitmask |= 1 << version; + if (strcmp(line->value, "2")) { + log_warn(LD_CONFIG, + "The only supported HiddenServiceVersion is 2."); + rend_service_free(service); + return -1; } - /* If exactly one version is set, change descriptor_version to that - * value; otherwise leave it at -1. */ - if (versions_bitmask == 1 << 0) service->descriptor_version = 0; - if (versions_bitmask == 1 << 2) service->descriptor_version = 2; - SMARTLIST_FOREACH(versions, char *, cp, tor_free(cp)); - smartlist_free(versions); } } if (service) { @@ -483,8 +428,7 @@ rend_config_services(or_options_t *options, int validate_only) * probably ok? */ SMARTLIST_FOREACH(rend_service_list, rend_service_t *, new, { SMARTLIST_FOREACH(old_service_list, rend_service_t *, old, { - if (!strcmp(old->directory, new->directory) && - old->descriptor_version == new->descriptor_version) { + if (!strcmp(old->directory, new->directory)) { smartlist_add_all(new->intro_nodes, old->intro_nodes); smartlist_clear(old->intro_nodes); smartlist_add(surviving_services, old); @@ -507,8 +451,7 @@ rend_config_services(or_options_t *options, int validate_only) tor_assert(oc->rend_data); SMARTLIST_FOREACH(surviving_services, rend_service_t *, ptr, { if (!memcmp(ptr->pk_digest, oc->rend_data->rend_pk_digest, - DIGEST_LEN) && - ptr->descriptor_version == oc->rend_data->rend_desc_version) { + DIGEST_LEN)) { keep_it = 1; break; } @@ -548,7 +491,6 @@ rend_service_update_descriptor(rend_service_t *service) d = service->desc = tor_malloc_zero(sizeof(rend_service_descriptor_t)); d->pk = crypto_pk_dup_key(service->private_key); d->timestamp = time(NULL); - d->version = service->descriptor_version; d->intro_nodes = smartlist_create(); /* Support intro protocols 2 and 3. */ d->protocols = (1 << 2) + (1 << 3); @@ -798,16 +740,18 @@ rend_service_load_keys(void) } /** Return the service whose public key has a digest of digest and - * which publishes the given descriptor version. Return NULL if no - * such service exists. + * which publishes the given descriptor version. (There is no other + * version than 2, so version is ignored.) Return NULL if no such service + * exists. */ static rend_service_t * rend_service_get_by_pk_digest_and_version(const char* digest, uint8_t version) { + (void) version; SMARTLIST_FOREACH(rend_service_list, rend_service_t*, s, - if (!memcmp(s->pk_digest,digest,DIGEST_LEN) && - s->descriptor_version == version) return s); + if (!memcmp(s->pk_digest,digest,DIGEST_LEN)) + return s); return NULL; } @@ -953,12 +897,8 @@ rend_service_introduce(origin_circuit_t *circuit, const char *request, return -1; } - /* if descriptor version is 2, use intro key instead of service key. */ - if (circuit->rend_data->rend_desc_version == 0) { - intro_key = service->private_key; - } else { - intro_key = circuit->intro_key; - } + /* use intro key instead of service key. */ + intro_key = circuit->intro_key; /* first DIGEST_LEN bytes of request is intro or service pk digest */ crypto_pk_get_digest(intro_key, intro_key_digest); @@ -1201,7 +1141,6 @@ rend_service_introduce(origin_circuit_t *circuit, const char *request, memcpy(launched->rend_data->rend_cookie, r_cookie, REND_COOKIE_LEN); strlcpy(launched->rend_data->onion_address, service->service_id, sizeof(launched->rend_data->onion_address)); - launched->rend_data->rend_desc_version = service->descriptor_version; launched->build_state->pending_final_cpath = cpath = tor_malloc_zero(sizeof(crypt_path_t)); cpath->magic = CRYPT_PATH_MAGIC; @@ -1323,9 +1262,7 @@ rend_service_launch_establish_intro(rend_service_t *service, strlcpy(launched->rend_data->onion_address, service->service_id, sizeof(launched->rend_data->onion_address)); memcpy(launched->rend_data->rend_pk_digest, service->pk_digest, DIGEST_LEN); - launched->rend_data->rend_desc_version = service->descriptor_version; - if (service->descriptor_version == 2) - launched->intro_key = crypto_pk_dup_key(intro->intro_key); + launched->intro_key = crypto_pk_dup_key(intro->intro_key); if (launched->_base.state == CIRCUIT_STATE_OPEN) rend_service_intro_has_opened(launched); return 0; @@ -1401,13 +1338,8 @@ rend_service_intro_has_opened(origin_circuit_t *circuit) "Established circuit %d as introduction point for service %s", circuit->_base.n_circ_id, serviceid); - /* If the introduction point will not be used in an unversioned - * descriptor, use the intro key instead of the service key in - * ESTABLISH_INTRO. */ - if (service->descriptor_version == 0) - intro_key = service->private_key; - else - intro_key = circuit->intro_key; + /* Use the intro key instead of the service key in ESTABLISH_INTRO. */ + intro_key = circuit->intro_key; /* Build the payload for a RELAY_ESTABLISH_INTRO cell. */ r = crypto_pk_asn1_encode(intro_key, buf+2, RELAY_PAYLOAD_SIZE-2); @@ -1695,9 +1627,8 @@ directory_post_to_hs_dir(rend_service_descriptor_t *renddesc, smartlist_free(successful_uploads); } -/** Encode and sign up-to-date v0 and/or v2 service descriptors for - * service, and upload it/them to all the dirservers/to the - * responsible hidden service directories. +/** Encode and sign an up-to-date service descriptor for service, + * and upload it/them to the responsible hidden service directories. */ static void upload_service_descriptor(rend_service_t *service) @@ -1709,35 +1640,8 @@ upload_service_descriptor(rend_service_t *service) rendpostperiod = get_options()->RendPostPeriod; - /* Upload unversioned (v0) descriptor? */ - if (service->descriptor_version == 0 && - get_options()->PublishHidServDescriptors) { - char *desc; - size_t desc_len; - /* Encode the descriptor. */ - if (rend_encode_service_descriptor(service->desc, - service->private_key, - &desc, &desc_len)<0) { - log_warn(LD_BUG, "Internal error: couldn't encode service descriptor; " - "not uploading."); - return; - } - - /* Post it to the dirservers */ - rend_get_service_id(service->desc->pk, serviceid); - log_info(LD_REND, "Sending publish request for hidden service %s", - serviceid); - directory_post_to_dirservers(DIR_PURPOSE_UPLOAD_RENDDESC, - ROUTER_PURPOSE_GENERAL, - HIDSERV_AUTHORITY, desc, desc_len, 0); - tor_free(desc); - service->next_upload_time = now + rendpostperiod; - uploaded = 1; - } - - /* Upload v2 descriptor? */ - if (service->descriptor_version == 2 && - get_options()->PublishHidServDescriptors) { + /* Upload descriptor? */ + if (get_options()->PublishHidServDescriptors) { networkstatus_t *c = networkstatus_get_latest_consensus(); if (c && smartlist_len(c->routerstatus_list) > 0) { int seconds_valid, i, j, num_descs; @@ -1876,8 +1780,7 @@ rend_services_introduce(void) for (j=0; j < smartlist_len(service->intro_nodes); ++j) { intro = smartlist_get(service->intro_nodes, j); router = router_get_by_digest(intro->extend_info->identity_digest); - if (!router || !find_intro_circuit(intro, service->pk_digest, - service->descriptor_version)) { + if (!router || !find_intro_circuit(intro, service->pk_digest, -1)) { log_info(LD_REND,"Giving up on %s as intro point for %s.", intro->extend_info->nickname, service->service_id); if (service->desc) { @@ -1941,10 +1844,8 @@ rend_services_introduce(void) smartlist_add(intro_routers, router); intro = tor_malloc_zero(sizeof(rend_intro_point_t)); intro->extend_info = extend_info_from_router(router); - if (service->descriptor_version == 2) { - intro->intro_key = crypto_new_pk_env(); - tor_assert(!crypto_pk_generate_key(intro->intro_key)); - } + intro->intro_key = crypto_new_pk_env(); + tor_assert(!crypto_pk_generate_key(intro->intro_key)); smartlist_add(service->intro_nodes, intro); log_info(LD_REND, "Picked router %s as an intro point for %s.", router->nickname, service->service_id); @@ -2037,8 +1938,7 @@ rend_consider_descriptor_republication(void) for (i=0; i < smartlist_len(rend_service_list); ++i) { service = smartlist_get(rend_service_list, i); - if (service->descriptor_version && service->desc && - !service->desc->all_uploads_performed) { + if (service->desc && !service->desc->all_uploads_performed) { /* If we failed in uploading a descriptor last time, try again *without* * updating the descriptor's contents. */ upload_service_descriptor(service); @@ -2066,8 +1966,7 @@ rend_service_dump_stats(int severity) intro = smartlist_get(service->intro_nodes, j); safe_name = safe_str(intro->extend_info->nickname); - circ = find_intro_circuit(intro, service->pk_digest, - service->descriptor_version); + circ = find_intro_circuit(intro, service->pk_digest, -1); if (!circ) { log(severity, LD_GENERAL, " Intro point %d at %s: no circuit", j, safe_name); diff --git a/src/or/test.c b/src/or/test.c index bd0f485dd8..6ce0aaeb47 100644 --- a/src/or/test.c +++ b/src/or/test.c @@ -4000,78 +4000,6 @@ test_policies(void) } } -/** Run unit tests for basic rendezvous functions. */ -static void -test_rend_fns(void) -{ - char address1[] = "fooaddress.onion"; - char address2[] = "aaaaaaaaaaaaaaaa.onion"; - char address3[] = "fooaddress.exit"; - char address4[] = "www.torproject.org"; - rend_service_descriptor_t *d1 = - tor_malloc_zero(sizeof(rend_service_descriptor_t)); - rend_service_descriptor_t *d2 = NULL; - char *encoded = NULL; - size_t len; - time_t now; - int i; - crypto_pk_env_t *pk1 = pk_generate(0), *pk2 = pk_generate(1); - - /* Test unversioned (v0) descriptor */ - d1->pk = crypto_pk_dup_key(pk1); - now = time(NULL); - d1->timestamp = now; - d1->version = 0; - d1->intro_nodes = smartlist_create(); - for (i = 0; i < 3; i++) { - rend_intro_point_t *intro = tor_malloc_zero(sizeof(rend_intro_point_t)); - intro->extend_info = tor_malloc_zero(sizeof(extend_info_t)); - crypto_rand(intro->extend_info->identity_digest, DIGEST_LEN); - intro->extend_info->nickname[0] = '$'; - base16_encode(intro->extend_info->nickname+1, HEX_DIGEST_LEN+1, - intro->extend_info->identity_digest, DIGEST_LEN); - smartlist_add(d1->intro_nodes, intro); - } - test_assert(! rend_encode_service_descriptor(d1, pk1, &encoded, &len)); - d2 = rend_parse_service_descriptor(encoded, len); - test_assert(d2); - - test_assert(!crypto_pk_cmp_keys(d1->pk, d2->pk)); - test_eq(d2->timestamp, now); - test_eq(d2->version, 0); - test_eq(d2->protocols, 1<<2); - test_eq(smartlist_len(d2->intro_nodes), 3); - for (i = 0; i < 3; i++) { - rend_intro_point_t *intro1 = smartlist_get(d1->intro_nodes, i); - rend_intro_point_t *intro2 = smartlist_get(d2->intro_nodes, i); - test_streq(intro1->extend_info->nickname, - intro2->extend_info->nickname); - } - - test_assert(BAD_HOSTNAME == parse_extended_hostname(address1)); - test_assert(ONION_HOSTNAME == parse_extended_hostname(address2)); - test_assert(EXIT_HOSTNAME == parse_extended_hostname(address3)); - test_assert(NORMAL_HOSTNAME == parse_extended_hostname(address4)); - - crypto_free_pk_env(pk1); - crypto_free_pk_env(pk2); - pk1 = pk2 = NULL; - rend_service_descriptor_free(d1); - rend_service_descriptor_free(d2); - d1 = d2 = NULL; - - done: - if (pk1) - crypto_free_pk_env(pk1); - if (pk2) - crypto_free_pk_env(pk2); - if (d1) - rend_service_descriptor_free(d1); - if (d2) - rend_service_descriptor_free(d2); - tor_free(encoded); -} - /** Run AES performance benchmarks. */ static void bench_aes(void) @@ -4539,9 +4467,9 @@ test_crypto_base32_decode(void) ; } -/** Test encoding and parsing of v2 rendezvous service descriptors. */ +/** Test encoding and parsing of rendezvous service descriptors. */ static void -test_rend_fns_v2(void) +test_rend_fns(void) { rend_service_descriptor_t *generated = NULL, *parsed = NULL; char service_id[DIGEST_LEN]; @@ -4556,6 +4484,16 @@ test_rend_fns_v2(void) size_t intro_points_size; size_t encoded_size; int i; + char address1[] = "fooaddress.onion"; + char address2[] = "aaaaaaaaaaaaaaaa.onion"; + char address3[] = "fooaddress.exit"; + char address4[] = "www.torproject.org"; + + test_assert(BAD_HOSTNAME == parse_extended_hostname(address1)); + test_assert(ONION_HOSTNAME == parse_extended_hostname(address2)); + test_assert(EXIT_HOSTNAME == parse_extended_hostname(address3)); + test_assert(NORMAL_HOSTNAME == parse_extended_hostname(address4)); + pk1 = pk_generate(0); pk2 = pk_generate(1); generated = tor_malloc_zero(sizeof(rend_service_descriptor_t)); @@ -4760,7 +4698,6 @@ static struct { ENT(v3_networkstatus), ENT(policies), ENT(rend_fns), - SUBENT(rend_fns, v2), ENT(geoip), DISABLED(bench_aes), From f79688ddef0e4d4abd5e92afbb11c5ee9864056d Mon Sep 17 00:00:00 2001 From: Karsten Loesing Date: Sun, 3 May 2009 00:09:18 +0200 Subject: [PATCH 014/187] Clients do not request version 0 hidserv descs anymore. --- src/or/circuituse.c | 5 --- src/or/connection.c | 14 ++----- src/or/connection_edge.c | 10 ----- src/or/directory.c | 21 ---------- src/or/or.h | 11 ----- src/or/rendclient.c | 87 +++++++++++----------------------------- src/or/rendservice.c | 41 ++++++++----------- 7 files changed, 43 insertions(+), 146 deletions(-) diff --git a/src/or/circuituse.c b/src/or/circuituse.c index 4ee99146e6..1dd15d6af9 100644 --- a/src/or/circuituse.c +++ b/src/or/circuituse.c @@ -1110,12 +1110,7 @@ circuit_get_open_circ_or_launch(edge_connection_t *conn, log_info(LD_REND, "No intro points for '%s': refetching service descriptor.", safe_str(conn->rend_data->onion_address)); - /* Fetch both, v0 and v2 rend descriptors in parallel. Use whichever - * arrives first. Exception: When using client authorization, only - * fetch v2 descriptors.*/ rend_client_refetch_v2_renddesc(conn->rend_data); - if (conn->rend_data->auth_type == REND_NO_AUTH) - rend_client_refetch_renddesc(conn->rend_data->onion_address); conn->_base.state = AP_CONN_STATE_RENDDESC_WAIT; return 0; } diff --git a/src/or/connection.c b/src/or/connection.c index 3a01c28652..812d021531 100644 --- a/src/or/connection.c +++ b/src/or/connection.c @@ -544,13 +544,6 @@ connection_about_to_close_connection(connection_t *conn) * failed: forget about this router, and maybe try again. */ connection_dir_request_failed(dir_conn); } - if (conn->purpose == DIR_PURPOSE_FETCH_RENDDESC && dir_conn->rend_data) { - /* Give it a try. However, there is no re-fetching for v0 rend - * descriptors; if the response is empty or the descriptor is - * unusable, close pending connections (unless a v2 request is - * still in progress). */ - rend_client_desc_trynow(dir_conn->rend_data->onion_address, 0); - } /* If we were trying to fetch a v2 rend desc and did not succeed, * retry as needed. (If a fetch is successful, the connection state * is changed to DIR_PURPOSE_HAS_FETCHED_RENDDESC to mark that @@ -2576,8 +2569,8 @@ connection_get_by_type_state(int type, int state) /** Return a connection of type type that has rendquery equal * to rendquery, and that is not marked for close. If state - * is non-zero, conn must be of that state too. If rendversion is - * nonnegative, conn must be fetching that rendversion, too. + * is non-zero, conn must be of that state too. (rendversion is + * ignored.) */ connection_t * connection_get_by_type_state_rendquery(int type, int state, @@ -2585,6 +2578,7 @@ connection_get_by_type_state_rendquery(int type, int state, int rendversion) { smartlist_t *conns = get_connection_array(); + (void) rendversion; tor_assert(type == CONN_TYPE_DIR || type == CONN_TYPE_AP || type == CONN_TYPE_EXIT); @@ -2597,8 +2591,6 @@ connection_get_by_type_state_rendquery(int type, int state, (!state || state == conn->state)) { if (type == CONN_TYPE_DIR && TO_DIR_CONN(conn)->rend_data && - (rendversion < 0 || - rendversion == TO_DIR_CONN(conn)->rend_data->rend_desc_version) && !rend_cmp_service_ids(rendquery, TO_DIR_CONN(conn)->rend_data->onion_address)) return conn; diff --git a/src/or/connection_edge.c b/src/or/connection_edge.c index b350c08d29..f4585d89a9 100644 --- a/src/or/connection_edge.c +++ b/src/or/connection_edge.c @@ -1675,12 +1675,7 @@ connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, conn->_base.state = AP_CONN_STATE_RENDDESC_WAIT; log_info(LD_REND, "Unknown descriptor %s. Fetching.", safe_str(conn->rend_data->onion_address)); - /* Fetch both, v0 and v2 rend descriptors in parallel. Use whichever - * arrives first. Exception: When using client authorization, only - * fetch v2 descriptors.*/ rend_client_refetch_v2_renddesc(conn->rend_data); - if (conn->rend_data->auth_type == REND_NO_AUTH) - rend_client_refetch_renddesc(conn->rend_data->onion_address); } else { /* r > 0 */ /** How long after we receive a hidden service descriptor do we consider * it valid? */ @@ -1697,12 +1692,7 @@ connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, conn->_base.state = AP_CONN_STATE_RENDDESC_WAIT; log_info(LD_REND, "Stale descriptor %s. Refetching.", safe_str(conn->rend_data->onion_address)); - /* Fetch both, v0 and v2 rend descriptors in parallel. Use whichever - * arrives first. Exception: When using client authorization, only - * fetch v2 descriptors.*/ rend_client_refetch_v2_renddesc(conn->rend_data); - if (conn->rend_data->auth_type == REND_NO_AUTH) - rend_client_refetch_renddesc(conn->rend_data->onion_address); } } return 0; diff --git a/src/or/directory.c b/src/or/directory.c index 082eca2006..f20efc1d31 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -1050,31 +1050,10 @@ directory_send_command(dir_connection_t *conn, httpcommand = "POST"; url = tor_strdup("/tor/post/consensus-signature"); break; - case DIR_PURPOSE_FETCH_RENDDESC: - tor_assert(resource); - tor_assert(!payload); - - /* this must be true or we wouldn't be doing the lookup */ - tor_assert(strlen(resource) <= REND_SERVICE_ID_LEN_BASE32); - /* This breaks the function abstraction. */ - conn->rend_data = tor_malloc_zero(sizeof(rend_data_t)); - strlcpy(conn->rend_data->onion_address, resource, - sizeof(conn->rend_data->onion_address)); - conn->rend_data->rend_desc_version = 0; - - httpcommand = "GET"; - /* Request the most recent versioned descriptor. */ - // (XXXX We were going to switch this to fetch rendezvous1 descriptors, - // but that never got testing, and it wasn't a good design.) - len = strlen(resource)+32; - url = tor_malloc(len); - tor_snprintf(url, len, "/tor/rendezvous/%s", resource); - break; case DIR_PURPOSE_FETCH_RENDDESC_V2: tor_assert(resource); tor_assert(strlen(resource) <= REND_DESC_ID_V2_LEN_BASE32); tor_assert(!payload); - conn->rend_data->rend_desc_version = 2; httpcommand = "GET"; len = strlen(resource) + 32; url = tor_malloc(len); diff --git a/src/or/or.h b/src/or/or.h index 731cf5e9a8..496165f4bd 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -730,12 +730,6 @@ typedef struct rend_data_t { /** Rendezvous cookie used by both, client and service. */ char rend_cookie[REND_COOKIE_LEN]; - - /** Rendezvous descriptor version that is used by a service. Used to - * distinguish introduction and rendezvous points belonging to the same - * rendezvous service ID, but different descriptor versions. - */ - uint8_t rend_desc_version; } rend_data_t; /** Time interval for tracking possible replays of INTRODUCE2 cells. @@ -4014,7 +4008,6 @@ void rend_client_introcirc_has_opened(origin_circuit_t *circ); void rend_client_rendcirc_has_opened(origin_circuit_t *circ); int rend_client_introduction_acked(origin_circuit_t *circ, const char *request, size_t request_len); -void rend_client_refetch_renddesc(const char *query); void rend_client_refetch_v2_renddesc(const rend_data_t *rend_query); int rend_client_remove_intro_point(extend_info_t *failed_intro, const rend_data_t *rend_query); @@ -4089,10 +4082,6 @@ void rend_process_relay_cell(circuit_t *circ, const crypt_path_t *layer_hint, int command, size_t length, const char *payload); void rend_service_descriptor_free(rend_service_descriptor_t *desc); -int rend_encode_service_descriptor(rend_service_descriptor_t *desc, - crypto_pk_env_t *key, - char **str_out, - size_t *len_out); rend_service_descriptor_t *rend_parse_service_descriptor(const char *str, size_t len); int rend_get_service_id(crypto_pk_env_t *pk, char *out); diff --git a/src/or/rendclient.c b/src/or/rendclient.c index fb50e8dc00..2f41beb4d4 100644 --- a/src/or/rendclient.c +++ b/src/or/rendclient.c @@ -63,7 +63,7 @@ rend_client_send_introduction(origin_circuit_t *introcirc, rend_cache_entry_t *entry; crypt_path_t *cpath; off_t dh_offset; - crypto_pk_env_t *intro_key; /* either Bob's public key or an intro key. */ + crypto_pk_env_t *intro_key = NULL; tor_assert(introcirc->_base.purpose == CIRCUIT_PURPOSE_C_INTRODUCING); tor_assert(rendcirc->_base.purpose == CIRCUIT_PURPOSE_C_REND_READY); @@ -80,23 +80,19 @@ rend_client_send_introduction(origin_circuit_t *introcirc, goto err; } - /* first 20 bytes of payload are the hash of bob's pk */ - if (entry->parsed->version == 0) { /* unversioned descriptor */ - intro_key = entry->parsed->pk; - } else { /* versioned descriptor */ - intro_key = NULL; - SMARTLIST_FOREACH(entry->parsed->intro_nodes, rend_intro_point_t *, - intro, { - if (!memcmp(introcirc->build_state->chosen_exit->identity_digest, - intro->extend_info->identity_digest, DIGEST_LEN)) { - intro_key = intro->intro_key; - break; - } - }); - if (!intro_key) { - log_warn(LD_BUG, "Internal error: could not find intro key."); - goto err; + /* first 20 bytes of payload are the hash of the intro key */ + intro_key = NULL; + SMARTLIST_FOREACH(entry->parsed->intro_nodes, rend_intro_point_t *, + intro, { + if (!memcmp(introcirc->build_state->chosen_exit->identity_digest, + intro->extend_info->identity_digest, DIGEST_LEN)) { + intro_key = intro->intro_key; + break; } + }); + if (!intro_key) { + log_warn(LD_BUG, "Internal error: could not find intro key."); + goto err; } if (crypto_pk_get_digest(intro_key, payload)<0) { log_warn(LD_BUG, "Internal error: couldn't hash public key."); @@ -451,28 +447,6 @@ directory_get_from_hs_dir(const char *desc_id, const rend_data_t *rend_query) return 1; } -/** If we are not currently fetching a rendezvous service descriptor - * for the service ID query, start a directory connection to fetch a - * new one. - */ -void -rend_client_refetch_renddesc(const char *query) -{ - if (!get_options()->FetchHidServDescriptors) - return; - log_info(LD_REND, "Fetching rendezvous descriptor for service %s", - escaped_safe_str(query)); - if (connection_get_by_type_state_rendquery(CONN_TYPE_DIR, 0, query, 0)) { - log_info(LD_REND,"Would fetch a new renddesc here (for %s), but one is " - "already in progress.", escaped_safe_str(query)); - } else { - /* not one already; initiate a dir rend desc lookup */ - directory_get_from_dirserver(DIR_PURPOSE_FETCH_RENDDESC, - ROUTER_PURPOSE_GENERAL, query, - PDS_RETRY_IF_NO_SERVERS); - } -} - /** Start a connection to a hidden service directory to fetch a v2 * rendezvous service descriptor for the base32-encoded service ID * query. @@ -552,12 +526,7 @@ rend_client_remove_intro_point(extend_info_t *failed_intro, if (r==0) { log_info(LD_REND, "Unknown service %s. Re-fetching descriptor.", escaped_safe_str(rend_query->onion_address)); - /* Fetch both, v0 and v2 rend descriptors in parallel. Use whichever - * arrives first. Exception: When using client authorization, only - * fetch v2 descriptors.*/ rend_client_refetch_v2_renddesc(rend_query); - if (rend_query->auth_type == REND_NO_AUTH) - rend_client_refetch_renddesc(rend_query->onion_address); return 0; } @@ -575,12 +544,7 @@ rend_client_remove_intro_point(extend_info_t *failed_intro, log_info(LD_REND, "No more intro points remain for %s. Re-fetching descriptor.", escaped_safe_str(rend_query->onion_address)); - /* Fetch both, v0 and v2 rend descriptors in parallel. Use whichever - * arrives first. Exception: When using client authorization, only - * fetch v2 descriptors.*/ rend_client_refetch_v2_renddesc(rend_query); - if (rend_query->auth_type == REND_NO_AUTH) - rend_client_refetch_renddesc(rend_query->onion_address); /* move all pending streams back to renddesc_wait */ while ((conn = connection_get_by_type_state_rendquery(CONN_TYPE_AP, @@ -696,11 +660,11 @@ rend_client_receive_rendezvous(origin_circuit_t *circ, const char *request, /** Find all the apconns in state AP_CONN_STATE_RENDDESC_WAIT that * are waiting on query. If there's a working cache entry here - * with at least one intro point, move them to the next state. If - * rend_version is non-negative, fail connections that have - * requested query unless there are still descriptor fetch - * requests in progress for other descriptor versions than - * rend_version. + * with at least one intro point, move them to the next state. + * (rend_version was used to keep the connection open when + * there were still descriptor fetch requests in progress for other + * descriptor versions than rend_version, but this is obsolete + * now that we support only version 2.) */ void rend_client_desc_trynow(const char *query, int rend_version) @@ -708,8 +672,9 @@ rend_client_desc_trynow(const char *query, int rend_version) edge_connection_t *conn; rend_cache_entry_t *entry; time_t now = time(NULL); - smartlist_t *conns = get_connection_array(); + (void) rend_version; + SMARTLIST_FOREACH(conns, connection_t *, _conn, { if (_conn->type != CONN_TYPE_AP || @@ -743,15 +708,9 @@ rend_client_desc_trynow(const char *query, int rend_version) connection_mark_unattached_ap(conn, END_STREAM_REASON_CANT_ATTACH); } } else { /* 404, or fetch didn't get that far */ - /* Unless there are requests for another descriptor version pending, - * close the connection. */ - if (rend_version >= 0 && - !connection_get_by_type_state_rendquery(CONN_TYPE_DIR, 0, query, - rend_version == 0 ? 2 : 0)) { - log_notice(LD_REND,"Closing stream for '%s.onion': hidden service is " - "unavailable (try again later).", safe_str(query)); - connection_mark_unattached_ap(conn, END_STREAM_REASON_RESOLVEFAILED); - } + log_notice(LD_REND,"Closing stream for '%s.onion': hidden service is " + "unavailable (try again later).", safe_str(query)); + connection_mark_unattached_ap(conn, END_STREAM_REASON_RESOLVEFAILED); } }); } diff --git a/src/or/rendservice.c b/src/or/rendservice.c index b30e708f08..d0bed2353b 100644 --- a/src/or/rendservice.c +++ b/src/or/rendservice.c @@ -458,10 +458,9 @@ rend_config_services(or_options_t *options, int validate_only) }); if (keep_it) continue; - log_info(LD_REND, "Closing intro point %s for service %s version %d.", + log_info(LD_REND, "Closing intro point %s for service %s.", safe_str(oc->build_state->chosen_exit->nickname), - oc->rend_data->onion_address, - oc->rend_data->rend_desc_version); + oc->rend_data->onion_address); circuit_mark_for_close(circ, END_CIRC_REASON_FINISHED); /* XXXX Is there another reason we should use here? */ } @@ -889,8 +888,7 @@ rend_service_introduce(origin_circuit_t *circuit, const char *request, /* look up service depending on circuit. */ service = rend_service_get_by_pk_digest_and_version( - circuit->rend_data->rend_pk_digest, - circuit->rend_data->rend_desc_version); + circuit->rend_data->rend_pk_digest, -1); if (!service) { log_warn(LD_REND, "Got an INTRODUCE2 cell for an unrecognized service %s.", escaped(serviceid)); @@ -1269,12 +1267,14 @@ rend_service_launch_establish_intro(rend_service_t *service, } /** Return the number of introduction points that are or have been - * established for the given service address and rendezvous version. */ + * established for the given service address in query. + * (rend_version is ignored.) */ static int count_established_intro_points(const char *query, int rend_version) { int num_ipos = 0; circuit_t *circ; + (void) rend_version; for (circ = _circuit_get_global_list(); circ; circ = circ->next) { if (!circ->marked_for_close && circ->state == CIRCUIT_STATE_OPEN && @@ -1282,7 +1282,6 @@ count_established_intro_points(const char *query, int rend_version) circ->purpose == CIRCUIT_PURPOSE_S_INTRO)) { origin_circuit_t *oc = TO_ORIGIN_CIRCUIT(circ); if (oc->rend_data && - oc->rend_data->rend_desc_version == rend_version && !rend_cmp_service_ids(query, oc->rend_data->onion_address)) num_ipos++; } @@ -1313,8 +1312,7 @@ rend_service_intro_has_opened(origin_circuit_t *circuit) circuit->rend_data->rend_pk_digest, REND_SERVICE_ID_LEN); service = rend_service_get_by_pk_digest_and_version( - circuit->rend_data->rend_pk_digest, - circuit->rend_data->rend_desc_version); + circuit->rend_data->rend_pk_digest, -1); if (!service) { log_warn(LD_REND, "Unrecognized service ID %s on introduction circuit %d.", serviceid, circuit->_base.n_circ_id); @@ -1324,8 +1322,7 @@ rend_service_intro_has_opened(origin_circuit_t *circuit) /* If we already have enough introduction circuits for this service, * redefine this one as a general circuit. */ - if (count_established_intro_points(serviceid, - circuit->rend_data->rend_desc_version) > NUM_INTRO_POINTS) { + if (count_established_intro_points(serviceid, -1) > NUM_INTRO_POINTS) { log_info(LD_CIRC|LD_REND, "We have just finished an introduction " "circuit, but we already have enough. Redefining purpose to " "general."); @@ -1399,8 +1396,7 @@ rend_service_intro_established(origin_circuit_t *circuit, const char *request, } tor_assert(circuit->rend_data); service = rend_service_get_by_pk_digest_and_version( - circuit->rend_data->rend_pk_digest, - circuit->rend_data->rend_desc_version); + circuit->rend_data->rend_pk_digest, -1); if (!service) { log_warn(LD_REND, "Unknown service on introduction circuit %d.", circuit->_base.n_circ_id); @@ -1451,8 +1447,7 @@ rend_service_rendezvous_has_opened(origin_circuit_t *circuit) circuit->_base.n_circ_id, hexcookie, serviceid); service = rend_service_get_by_pk_digest_and_version( - circuit->rend_data->rend_pk_digest, - circuit->rend_data->rend_desc_version); + circuit->rend_data->rend_pk_digest, -1); if (!service) { log_warn(LD_GENERAL, "Internal error: unrecognized service ID on " "introduction circuit."); @@ -1508,23 +1503,23 @@ rend_service_rendezvous_has_opened(origin_circuit_t *circuit) */ /** Return the (possibly non-open) introduction circuit ending at - * intro for the service whose public key is pk_digest and - * which publishes descriptor of version desc_version. Return - * NULL if no such service is found. + * intro for the service whose public key is pk_digest. + * (desc_version is ignored). Return NULL if no such service is + * found. */ static origin_circuit_t * find_intro_circuit(rend_intro_point_t *intro, const char *pk_digest, int desc_version) { origin_circuit_t *circ = NULL; + (void) desc_version; tor_assert(intro); while ((circ = circuit_get_next_by_pk_and_purpose(circ,pk_digest, CIRCUIT_PURPOSE_S_INTRO))) { if (!memcmp(circ->build_state->chosen_exit->identity_digest, intro->extend_info->identity_digest, DIGEST_LEN) && - circ->rend_data && - circ->rend_data->rend_desc_version == desc_version) { + circ->rend_data) { return circ; } } @@ -1534,8 +1529,7 @@ find_intro_circuit(rend_intro_point_t *intro, const char *pk_digest, CIRCUIT_PURPOSE_S_ESTABLISH_INTRO))) { if (!memcmp(circ->build_state->chosen_exit->identity_digest, intro->extend_info->identity_digest, DIGEST_LEN) && - circ->rend_data && - circ->rend_data->rend_desc_version == desc_version) { + circ->rend_data) { return circ; } } @@ -1998,8 +1992,7 @@ rend_service_set_connection_addr_port(edge_connection_t *conn, base32_encode(serviceid, REND_SERVICE_ID_LEN_BASE32+1, circ->rend_data->rend_pk_digest, REND_SERVICE_ID_LEN); service = rend_service_get_by_pk_digest_and_version( - circ->rend_data->rend_pk_digest, - circ->rend_data->rend_desc_version); + circ->rend_data->rend_pk_digest, -1); if (!service) { log_warn(LD_REND, "Couldn't find any service associated with pk %s on " "rendezvous circuit %d; closing.", From 592f9571690514ebff0df9732b927bbfdba136a7 Mon Sep 17 00:00:00 2001 From: Karsten Loesing Date: Sun, 3 May 2009 00:34:36 +0200 Subject: [PATCH 015/187] Update rend-spec.txt. --- ChangeLog | 4 ++ doc/spec/rend-spec.txt | 110 ++++++++++++++++++----------------------- 2 files changed, 51 insertions(+), 63 deletions(-) diff --git a/ChangeLog b/ChangeLog index 236567099b..4ca2b1e950 100644 --- a/ChangeLog +++ b/ChangeLog @@ -7,6 +7,10 @@ Changes in version 0.2.2.1-alpha - 2009-??-?? o Deprecated and removed features: - The controller no longer accepts the old obsolete "addr-mappings/" GETINFO value. + - Hidden services no longer publish version 0 descriptors, and clients + do not request or use version 0 descriptors. However, the authorities + still accept and serve version 0 descriptors when contacted by older + hidden services/clients. Changes in version 0.2.1.15??? - ????-??-?? diff --git a/doc/spec/rend-spec.txt b/doc/spec/rend-spec.txt index e3fbe2253b..5c2f69871b 100644 --- a/doc/spec/rend-spec.txt +++ b/doc/spec/rend-spec.txt @@ -145,33 +145,10 @@ $Id$ 1.2. Bob's OP generates service descriptors. The first time the OP provides an advertised service, it generates - a public/private keypair (stored locally). Periodically, the OP - generates and publishes a descriptor of type "V0". + a public/private keypair (stored locally). - The "V0" descriptor contains: - - KL Key length [2 octets] - PK Bob's public key [KL octets] - TS A timestamp [4 octets] - NI Number of introduction points [2 octets] - Ipt A list of NUL-terminated ORs [variable] - SIG Signature of above fields [variable] - - KL is the length of PK, in octets. - TS is the number of seconds elapsed since Jan 1, 1970. - - The members of Ipt may be either (a) nicknames, or (b) identity key - digests, encoded in hex, and prefixed with a '$'. Clients must - accept both forms. Services must only generate the second form. - Once 0.0.9.x is obsoleted, we can drop the first form. - - [It's ok for Bob to advertise 0 introduction points. He might want - to do that if he previously advertised some introduction points, - and now he doesn't have any. -RD] - - Beginning with 0.2.0.10-alpha, Bob's OP encodes "V2" descriptors in - addition to "V0" descriptors. The format of a "V2" descriptor is as - follows: + Beginning with 0.2.0.10-alpha, Bob's OP encodes "V2" descriptors. The + format of a "V2" descriptor is as follows: "rendezvous-service-descriptor" descriptor-id NL @@ -340,6 +317,10 @@ $Id$ (This ends the fields in the encrypted portion of the descriptor.) + [It's ok for Bob to advertise 0 introduction points. He might want + to do that if he previously advertised some introduction points, + and now he doesn't have any. -RD] + "signature" NL signature-string [At end, exactly once] @@ -349,6 +330,21 @@ $Id$ 1.2.1. Other descriptor formats we don't use. + Support for the V0 descriptor format was dropped in 0.2.2.0-alpha-dev: + + KL Key length [2 octets] + PK Bob's public key [KL octets] + TS A timestamp [4 octets] + NI Number of introduction points [2 octets] + Ipt A list of NUL-terminated ORs [variable] + SIG Signature of above fields [variable] + + KL is the length of PK, in octets. + TS is the number of seconds elapsed since Jan 1, 1970. + + The members of Ipt may be either (a) nicknames, or (b) identity key + digests, encoded in hex, and prefixed with a '$'. + The V1 descriptor format was understood and accepted from 0.1.1.5-alpha-cvs to 0.2.0.6-alpha-dev, but no Tors generated it and it was removed: @@ -409,7 +405,7 @@ $Id$ RELAY_ESTABLISH_INTRO cell, containing: KL Key length [2 octets] - PK Bob's public key [KL octets] + PK Introduction public key [KL octets] HS Hash of session info [20 octets] SIG Signature of above information [variable] @@ -431,16 +427,13 @@ $Id$ currently associated with PK. On success, the OR sends Bob a RELAY_INTRO_ESTABLISHED cell with an empty payload. - If a hidden service is configured to publish only v2 hidden service - descriptors, Bob's OP does not include its own public key in the - RELAY_ESTABLISH_INTRO cell, but the public key of a freshly generated - key pair. The OP also includes these fresh public keys in the v2 hidden - service descriptor together with the other introduction point - information. The reason is that the introduction point does not need to - and therefore should not know for which hidden service it works, so as - to prevent it from tracking the hidden service's activity. If the hidden - service is configured to publish both, v0 and v2 descriptors, two - separate sets of introduction points are established. + Bob's OP does not include its own public key in the RELAY_ESTABLISH_INTRO + cell, but the public key of a freshly generated introduction key pair. + The OP also includes these fresh public keys in the v2 hidden service + descriptor together with the other introduction point information. The + reason is that the introduction point does not need to and therefore + should not know for which hidden service it works, so as to prevent it + from tracking the hidden service's activity. 1.4. Bob's OP advertises his service descriptor(s). @@ -464,10 +457,8 @@ $Id$ after its timestamp. At least every 18 hours, Bob's OP uploads a fresh descriptor. - If Bob's OP is configured to publish v2 descriptors instead of or in - addition to v0 descriptors, it does so to a changing subset of all v2 - hidden service directories instead of the authoritative directory - servers. Therefore, Bob's OP opens a stream via Tor to each + Bob's OP publishes v2 descriptors to a changing subset of all v2 hidden + service directories. Therefore, Bob's OP opens a stream via Tor to each responsible hidden service directory. (He may re-use old circuits for this.) Over this stream, Bob's OP makes an HTTP 'POST' request to a URL "/tor/rendezvous2/publish" relative to the hidden service @@ -520,12 +511,21 @@ $Id$ 1.6. Alice's OP retrieves a service descriptor. - Alice opens a stream to a directory server via Tor, and makes an HTTP GET - request for the document '/tor/rendezvous/', where '' is replaced - with the encoding of Bob's public key as described above. (She may re-use - old circuits for this.) The directory replies with a 404 HTTP response if - it does not recognize , and otherwise returns Bob's most recently - uploaded service descriptor. + Similarly to the description in section 1.4, Alice's OP fetches a v2 + descriptor from a randomly chosen hidden service directory out of the + changing subset of 6 nodes. If the request is unsuccessful, Alice retries + the other remaining responsible hidden service directories in a random + order. Alice relies on Bob to care about a potential clock skew between + the two by possibly storing two sets of descriptors (see end of section + 1.4). + + Alice's OP opens a stream via Tor to the chosen v2 hidden service + directory. (She may re-use old circuits for this.) Over this stream, + Alice's OP makes an HTTP 'GET' request for the document + "/tor/rendezvous2/", where z is replaced with the encoding of the + descriptor ID. The directory replies with a 404 HTTP response if it does + not recognize , and otherwise returns Bob's most recently uploaded + service descriptor. If Alice's OP receives a 404 response, it tries the other directory servers, and only fails the lookup if none recognize the public key hash. @@ -541,22 +541,6 @@ $Id$ [Caching may make her partitionable, but she fetched it anonymously, and we can't very well *not* cache it. -RD] - Alice's OP fetches v2 descriptors in parallel to v0 descriptors. Similarly - to the description in section 1.4, the OP fetches a v2 descriptor from a - randomly chosen hidden service directory out of the changing subset of - 6 nodes. If the request is unsuccessful, Alice retries the other - remaining responsible hidden service directories in a random order. - Alice relies on Bob to care about a potential clock skew between the two - by possibly storing two sets of descriptors (see end of section 1.4). - - Alice's OP opens a stream via Tor to the chosen v2 hidden service - directory. (She may re-use old circuits for this.) Over this stream, - Alice's OP makes an HTTP 'GET' request for the document - "/tor/rendezvous2/", where z is replaced with the encoding of the - descriptor ID. The directory replies with a 404 HTTP response if it does - not recognize , and otherwise returns Bob's most recently uploaded - service descriptor. - 1.7. Alice's OP establishes a rendezvous point. When Alice requests a connection to a given location-hidden service, From 948835c6809932a3013846302a74f1ccf9d4c99c Mon Sep 17 00:00:00 2001 From: Karsten Loesing Date: Sun, 3 May 2009 01:04:25 +0200 Subject: [PATCH 016/187] Remove unused rendversion parameters. YAGNI. --- src/or/connection.c | 7 ++---- src/or/directory.c | 4 ++-- src/or/or.h | 5 ++--- src/or/rendclient.c | 20 ++++++----------- src/or/rendservice.c | 53 ++++++++++++++++++-------------------------- 5 files changed, 35 insertions(+), 54 deletions(-) diff --git a/src/or/connection.c b/src/or/connection.c index 812d021531..7b14f7550a 100644 --- a/src/or/connection.c +++ b/src/or/connection.c @@ -2569,16 +2569,13 @@ connection_get_by_type_state(int type, int state) /** Return a connection of type type that has rendquery equal * to rendquery, and that is not marked for close. If state - * is non-zero, conn must be of that state too. (rendversion is - * ignored.) + * is non-zero, conn must be of that state too. */ connection_t * connection_get_by_type_state_rendquery(int type, int state, - const char *rendquery, - int rendversion) + const char *rendquery) { smartlist_t *conns = get_connection_array(); - (void) rendversion; tor_assert(type == CONN_TYPE_DIR || type == CONN_TYPE_AP || type == CONN_TYPE_EXIT); diff --git a/src/or/directory.c b/src/or/directory.c index f20efc1d31..ec31d77eb2 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -1899,7 +1899,7 @@ connection_dir_client_reached_eof(dir_connection_t *conn) /* Success, or at least there's a v2 descriptor already * present. Notify pending connections about this. */ conn->_base.purpose = DIR_PURPOSE_HAS_FETCHED_RENDDESC; - rend_client_desc_trynow(conn->rend_data->onion_address, -1); + rend_client_desc_trynow(conn->rend_data->onion_address); } break; case 404: @@ -1946,7 +1946,7 @@ connection_dir_client_reached_eof(dir_connection_t *conn) log_info(LD_REND, "Successfully fetched v2 rendezvous " "descriptor."); conn->_base.purpose = DIR_PURPOSE_HAS_FETCHED_RENDDESC; - rend_client_desc_trynow(conn->rend_data->onion_address, -1); + rend_client_desc_trynow(conn->rend_data->onion_address); break; } break; diff --git a/src/or/or.h b/src/or/or.h index 496165f4bd..854b541d3d 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -2985,8 +2985,7 @@ connection_t *connection_get_by_type_addr_port_purpose(int type, uint16_t port, int purpose); connection_t *connection_get_by_type_state(int type, int state); connection_t *connection_get_by_type_state_rendquery(int type, int state, - const char *rendquery, - int rendversion); + const char *rendquery); #define connection_speaks_cells(conn) ((conn)->type == CONN_TYPE_OR) int connection_is_listener(connection_t *conn); @@ -4015,7 +4014,7 @@ int rend_client_rendezvous_acked(origin_circuit_t *circ, const char *request, size_t request_len); int rend_client_receive_rendezvous(origin_circuit_t *circ, const char *request, size_t request_len); -void rend_client_desc_trynow(const char *query, int rend_version); +void rend_client_desc_trynow(const char *query); extend_info_t *rend_client_get_random_intro(const rend_data_t *rend_query); diff --git a/src/or/rendclient.c b/src/or/rendclient.c index 2f41beb4d4..3be67c9f83 100644 --- a/src/or/rendclient.c +++ b/src/or/rendclient.c @@ -498,8 +498,8 @@ rend_client_refetch_v2_renddesc(const rend_data_t *rend_query) log_info(LD_REND, "Could not pick one of the responsible hidden " "service directories to fetch descriptors, because " "we already tried them all unsuccessfully."); - /* Close pending connections (unless a v0 request is still going on). */ - rend_client_desc_trynow(rend_query->onion_address, 2); + /* Close pending connections. */ + rend_client_desc_trynow(rend_query->onion_address); return; } @@ -549,7 +549,7 @@ rend_client_remove_intro_point(extend_info_t *failed_intro, /* move all pending streams back to renddesc_wait */ while ((conn = connection_get_by_type_state_rendquery(CONN_TYPE_AP, AP_CONN_STATE_CIRCUIT_WAIT, - rend_query->onion_address, -1))) { + rend_query->onion_address))) { conn->state = AP_CONN_STATE_RENDDESC_WAIT; } @@ -658,22 +658,16 @@ rend_client_receive_rendezvous(origin_circuit_t *circ, const char *request, return -1; } -/** Find all the apconns in state AP_CONN_STATE_RENDDESC_WAIT that - * are waiting on query. If there's a working cache entry here - * with at least one intro point, move them to the next state. - * (rend_version was used to keep the connection open when - * there were still descriptor fetch requests in progress for other - * descriptor versions than rend_version, but this is obsolete - * now that we support only version 2.) - */ +/** Find all the apconns in state AP_CONN_STATE_RENDDESC_WAIT that are + * waiting on query. If there's a working cache entry here with at + * least one intro point, move them to the next state. */ void -rend_client_desc_trynow(const char *query, int rend_version) +rend_client_desc_trynow(const char *query) { edge_connection_t *conn; rend_cache_entry_t *entry; time_t now = time(NULL); smartlist_t *conns = get_connection_array(); - (void) rend_version; SMARTLIST_FOREACH(conns, connection_t *, _conn, { diff --git a/src/or/rendservice.c b/src/or/rendservice.c index d0bed2353b..8dc1290499 100644 --- a/src/or/rendservice.c +++ b/src/or/rendservice.c @@ -10,8 +10,7 @@ #include "or.h" static origin_circuit_t *find_intro_circuit(rend_intro_point_t *intro, - const char *pk_digest, - int desc_version); + const char *pk_digest); /** Represents the mapping from a virtual port of a rendezvous service to * a real port on some IP. @@ -56,7 +55,7 @@ typedef struct rend_service_t { * or are trying to establish. */ time_t intro_period_started; /**< Start of the current period to build * introduction points. */ - int n_intro_circuits_launched; /**< count of intro circuits we have + int n_intro_circuits_launched; /**< Count of intro circuits we have * established in this period. */ rend_service_descriptor_t *desc; /**< Current hidden service descriptor. */ time_t desc_is_dirty; /**< Time at which changes to the hidden service @@ -497,7 +496,7 @@ rend_service_update_descriptor(rend_service_t *service) for (i = 0; i < smartlist_len(service->intro_nodes); ++i) { rend_intro_point_t *intro_svc = smartlist_get(service->intro_nodes, i); rend_intro_point_t *intro_desc; - circ = find_intro_circuit(intro_svc, service->pk_digest, d->version); + circ = find_intro_circuit(intro_svc, service->pk_digest); if (!circ || circ->_base.purpose != CIRCUIT_PURPOSE_S_INTRO) continue; @@ -738,16 +737,12 @@ rend_service_load_keys(void) return r; } -/** Return the service whose public key has a digest of digest and - * which publishes the given descriptor version. (There is no other - * version than 2, so version is ignored.) Return NULL if no such service - * exists. +/** Return the service whose public key has a digest of digest, or + * NULL if no such service exists. */ static rend_service_t * -rend_service_get_by_pk_digest_and_version(const char* digest, - uint8_t version) +rend_service_get_by_pk_digest(const char* digest) { - (void) version; SMARTLIST_FOREACH(rend_service_list, rend_service_t*, s, if (!memcmp(s->pk_digest,digest,DIGEST_LEN)) return s); @@ -887,8 +882,8 @@ rend_service_introduce(origin_circuit_t *circuit, const char *request, } /* look up service depending on circuit. */ - service = rend_service_get_by_pk_digest_and_version( - circuit->rend_data->rend_pk_digest, -1); + service = rend_service_get_by_pk_digest( + circuit->rend_data->rend_pk_digest); if (!service) { log_warn(LD_REND, "Got an INTRODUCE2 cell for an unrecognized service %s.", escaped(serviceid)); @@ -1267,14 +1262,12 @@ rend_service_launch_establish_intro(rend_service_t *service, } /** Return the number of introduction points that are or have been - * established for the given service address in query. - * (rend_version is ignored.) */ + * established for the given service address in query. */ static int -count_established_intro_points(const char *query, int rend_version) +count_established_intro_points(const char *query) { int num_ipos = 0; circuit_t *circ; - (void) rend_version; for (circ = _circuit_get_global_list(); circ; circ = circ->next) { if (!circ->marked_for_close && circ->state == CIRCUIT_STATE_OPEN && @@ -1311,8 +1304,8 @@ rend_service_intro_has_opened(origin_circuit_t *circuit) base32_encode(serviceid, REND_SERVICE_ID_LEN_BASE32+1, circuit->rend_data->rend_pk_digest, REND_SERVICE_ID_LEN); - service = rend_service_get_by_pk_digest_and_version( - circuit->rend_data->rend_pk_digest, -1); + service = rend_service_get_by_pk_digest( + circuit->rend_data->rend_pk_digest); if (!service) { log_warn(LD_REND, "Unrecognized service ID %s on introduction circuit %d.", serviceid, circuit->_base.n_circ_id); @@ -1322,7 +1315,7 @@ rend_service_intro_has_opened(origin_circuit_t *circuit) /* If we already have enough introduction circuits for this service, * redefine this one as a general circuit. */ - if (count_established_intro_points(serviceid, -1) > NUM_INTRO_POINTS) { + if (count_established_intro_points(serviceid) > NUM_INTRO_POINTS) { log_info(LD_CIRC|LD_REND, "We have just finished an introduction " "circuit, but we already have enough. Redefining purpose to " "general."); @@ -1395,8 +1388,8 @@ rend_service_intro_established(origin_circuit_t *circuit, const char *request, goto err; } tor_assert(circuit->rend_data); - service = rend_service_get_by_pk_digest_and_version( - circuit->rend_data->rend_pk_digest, -1); + service = rend_service_get_by_pk_digest( + circuit->rend_data->rend_pk_digest); if (!service) { log_warn(LD_REND, "Unknown service on introduction circuit %d.", circuit->_base.n_circ_id); @@ -1446,8 +1439,8 @@ rend_service_rendezvous_has_opened(origin_circuit_t *circuit) "cookie %s for service %s", circuit->_base.n_circ_id, hexcookie, serviceid); - service = rend_service_get_by_pk_digest_and_version( - circuit->rend_data->rend_pk_digest, -1); + service = rend_service_get_by_pk_digest( + circuit->rend_data->rend_pk_digest); if (!service) { log_warn(LD_GENERAL, "Internal error: unrecognized service ID on " "introduction circuit."); @@ -1508,11 +1501,9 @@ rend_service_rendezvous_has_opened(origin_circuit_t *circuit) * found. */ static origin_circuit_t * -find_intro_circuit(rend_intro_point_t *intro, const char *pk_digest, - int desc_version) +find_intro_circuit(rend_intro_point_t *intro, const char *pk_digest) { origin_circuit_t *circ = NULL; - (void) desc_version; tor_assert(intro); while ((circ = circuit_get_next_by_pk_and_purpose(circ,pk_digest, @@ -1774,7 +1765,7 @@ rend_services_introduce(void) for (j=0; j < smartlist_len(service->intro_nodes); ++j) { intro = smartlist_get(service->intro_nodes, j); router = router_get_by_digest(intro->extend_info->identity_digest); - if (!router || !find_intro_circuit(intro, service->pk_digest, -1)) { + if (!router || !find_intro_circuit(intro, service->pk_digest)) { log_info(LD_REND,"Giving up on %s as intro point for %s.", intro->extend_info->nickname, service->service_id); if (service->desc) { @@ -1960,7 +1951,7 @@ rend_service_dump_stats(int severity) intro = smartlist_get(service->intro_nodes, j); safe_name = safe_str(intro->extend_info->nickname); - circ = find_intro_circuit(intro, service->pk_digest, -1); + circ = find_intro_circuit(intro, service->pk_digest); if (!circ) { log(severity, LD_GENERAL, " Intro point %d at %s: no circuit", j, safe_name); @@ -1991,8 +1982,8 @@ rend_service_set_connection_addr_port(edge_connection_t *conn, log_debug(LD_REND,"beginning to hunt for addr/port"); base32_encode(serviceid, REND_SERVICE_ID_LEN_BASE32+1, circ->rend_data->rend_pk_digest, REND_SERVICE_ID_LEN); - service = rend_service_get_by_pk_digest_and_version( - circ->rend_data->rend_pk_digest, -1); + service = rend_service_get_by_pk_digest( + circ->rend_data->rend_pk_digest); if (!service) { log_warn(LD_REND, "Couldn't find any service associated with pk %s on " "rendezvous circuit %d; closing.", From ae8e0aeca62e4fd47423a393072d0dc084ed6ca3 Mon Sep 17 00:00:00 2001 From: Roger Dingledine Date: Tue, 5 May 2009 08:52:22 -0400 Subject: [PATCH 017/187] Fix typos in proposal 141. --- doc/spec/proposals/141-jit-sd-downloads.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/spec/proposals/141-jit-sd-downloads.txt b/doc/spec/proposals/141-jit-sd-downloads.txt index b0c2b2cbcd..1546e854bd 100644 --- a/doc/spec/proposals/141-jit-sd-downloads.txt +++ b/doc/spec/proposals/141-jit-sd-downloads.txt @@ -63,8 +63,8 @@ Status: Draft which tries to convey a server's capacity to clients. Currently we weigh servers differently for different purposes. There - is a weigh for when we use a server as a guard node (our entry to the - Tor network), there is one weigh we assign servers for exit duties, + is a weight for when we use a server as a guard node (our entry to the + Tor network), there is one weight we assign servers for exit duties, and a third for when we need intermediate (middle) nodes. 2.2 Exit information @@ -80,7 +80,7 @@ Status: Draft 2.3 Capability information - Server descriptors contain information about the specific version or + Server descriptors contain information about the specific version of the Tor protocol they understand [proposal 105]. Furthermore the server descriptor also contains the exact version of From df8a47464a4e9adf30757a378ddb1f8aa317dbfa Mon Sep 17 00:00:00 2001 From: Roger Dingledine Date: Tue, 5 May 2009 08:53:12 -0400 Subject: [PATCH 018/187] add a new proposal for authorities to vote for bandwidth "offsets" --- doc/spec/proposals/000-index.txt | 2 + doc/spec/proposals/160-bandwidth-offset.txt | 80 +++++++++++++++++++++ 2 files changed, 82 insertions(+) create mode 100644 doc/spec/proposals/160-bandwidth-offset.txt diff --git a/doc/spec/proposals/000-index.txt b/doc/spec/proposals/000-index.txt index d75157650d..dd91efeccd 100644 --- a/doc/spec/proposals/000-index.txt +++ b/doc/spec/proposals/000-index.txt @@ -82,6 +82,7 @@ Proposals by number: 157 Make certificate downloads specific [ACCEPTED] 158 Clients download consensus + microdescriptors [OPEN] 159 Exit Scanning [OPEN] +160 Authorities vote for bandwidth offsets in consensus [OPEN] Proposals by status: @@ -103,6 +104,7 @@ Proposals by status: 156 Tracking blocked ports on the client side [for 0.2.?] 158 Clients download consensus + microdescriptors 159 Exit Scanning + 160 Authorities vote for bandwidth offsets in consensus [for 0.2.2.x] ACCEPTED: 110 Avoiding infinite length circuits [for 0.2.1.x] [in 0.2.1.3-alpha] 117 IPv6 exits [for 0.2.1.x] diff --git a/doc/spec/proposals/160-bandwidth-offset.txt b/doc/spec/proposals/160-bandwidth-offset.txt new file mode 100644 index 0000000000..078754b771 --- /dev/null +++ b/doc/spec/proposals/160-bandwidth-offset.txt @@ -0,0 +1,80 @@ +Filename: 160-bandwidth-offset.txt +Title: Authorities vote for bandwidth offsets in consensus +Version: $Revision$ +Last-Modified: $Date$ +Author: Roger Dingledine +Created: 4-May-2009 +Status: Open +Target: 0.2.2.x + +1. Motivation + + As part of proposal 141, we moved the bandwidth value for each relay + into the consensus. Now clients can know how they should load balance + even before they've fetched the corresponding relay descriptors. + + Putting the bandwidth in the consensus also lets the directory + authorities choose more accurate numbers to advertise, if we come up + with a better algorithm for deciding weightings. + + Our original plan was to teach directory authorities how to measure + bandwidth themselves; then every authority would vote for the bandwidth + it prefers, and we'd take the median of votes as usual. + + The problem comes when we have 7 authorities, and only a few of them + have smarter bandwidth allocation algorithms. So long as the majority + of them are voting for the number in the relay descriptor, the minority + that have better numbers will be ignored. + +2. Options + + One fix would be to demand that every authority also run the + new bandwidth measurement algorithms: in that case, part of the + responsibility of being an authority operator is that you need to run + this code too. But in practice we can't really require all current + authority operators to do that; and if we want to expand the set of + authority operators even further, it will become even more impractical. + Also, bandwidth testing adds load to the network, so we don't really + want to require that the number of concurrent bandwidth tests match + the number of authorities we have. + + The better fix is to allow certain authorities to specify that they are + voting on bandwidth "offsets": how much they think the weight should + be changed for the relay in question. We should put the offset vote in + the stanza for the relay in question, so a given authority can choose + which relays to express preferences for and which not. + +3. Security implications + + If only some authorities choose to vote on an offset, then a majority of + those voting authorities can arbitrarily change the bandwidth weighting + for the relay. At the extreme, if there's only one offset-voting + authority, then that authority can dictate which relays clients will + find attractive. + + This problem isn't entirely new: we already have the worry wrt + the subset of authorities that vote for BadExit. + + To make it not so bad, we should deploy at least three offset-voting + authorities. + + Also, authorities that know how to vote for offsets should vote for + an offset of zero for new nodes, rather than choosing not to vote on + any offset in those cases. + +4. Design + + First, we need a new consensus method to support this new calculation. + + Now v3 votes can have a new weight on the "w" line: + "Bandwidth_Offset=" INT. + Once we're using the new consensus method, the new way to compute the + Bandwidth weight is by taking the old vote (explained in proposal 141: + median, then choose the lower number in the case of ties), and adding + or subtracting the median offset (using the offset closer to 0 in the + case of ties, and with a sum of 0 if the sum is negative). + + Then the actual consensus looks just the same as it did before, + so clients never have to know that this additional calculation is + happening. + From d0cb5e5b0bfa55ac7e3ec4f28cdea309aaeefd51 Mon Sep 17 00:00:00 2001 From: Sebastian Hahn Date: Tue, 5 May 2009 15:48:23 +0200 Subject: [PATCH 019/187] SVN-specific metadata should no longer be part of a proposal Update the proposal creation instructions to remove the Version and Last-Modified fields. --- doc/spec/proposals/001-process.txt | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/doc/spec/proposals/001-process.txt b/doc/spec/proposals/001-process.txt index 3a767b5fa4..5c12d01648 100644 --- a/doc/spec/proposals/001-process.txt +++ b/doc/spec/proposals/001-process.txt @@ -47,7 +47,7 @@ How to change the specs now: Like an RFC, every proposal gets a number. Unlike RFCs, proposals can change over time and keep the same number, until they are finally accepted or rejected. The history for each proposal - will be stored in the Tor Subversion repository. + will be stored in the Tor repository. Once a proposal is in the repository, we should discuss and improve it until we've reached consensus that it's a good idea, and that it's @@ -82,9 +82,7 @@ How new proposals get added: What should go in a proposal: Every proposal should have a header containing these fields: - Filename, Title, Version, Last-Modified, Author, Created, Status. - The Version and Last-Modified fields should use the SVN Revision and Date - tags respectively. + Filename, Title, Author, Created, Status. These fields are optional but recommended: Target, Implemented-In. From b875a7a0f20be4108c1ef9c77b1c50cba34f1e40 Mon Sep 17 00:00:00 2001 From: Sebastian Hahn Date: Tue, 5 May 2009 15:55:32 +0200 Subject: [PATCH 020/187] checkSpace.pl doesn't need to worry about svn tags anymore --- contrib/checkSpace.pl | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/contrib/checkSpace.pl b/contrib/checkSpace.pl index 37f079c52b..db061a0828 100755 --- a/contrib/checkSpace.pl +++ b/contrib/checkSpace.pl @@ -42,9 +42,8 @@ for $fn (@ARGV) { $lastnil = 0; } ## Terminals are still 80 columns wide in my world. I refuse to - ## accept double-line lines. Except, of course, svn Id tags - ## can make us go long. - if (/^.{80}/ && !/\$Id: /) { + ## accept double-line lines. + if (/^.{80}/) { print " Wide:$fn:$.\n"; } ### Juju to skip over comments and strings, since the tests From d2308279120276720281472005b3beea39798634 Mon Sep 17 00:00:00 2001 From: Sebastian Hahn Date: Tue, 5 May 2009 16:16:31 +0200 Subject: [PATCH 021/187] Remove all svn metadata minus what I missed. Tor doesn't use SVN anymore, making $Revision$, $Id$ and $Date$ meaningless. Remove them without replacement. --- Doxyfile.in | 1 - Makefile.am | 1 - acinclude.m4 | 1 - configure.in | 1 - contrib/checkOptionDocs.pl | 1 - contrib/cross.sh | 1 - contrib/id_to_fp.c | 1 - contrib/nagios-check-tor-authority-cert | 2 -- contrib/osx/package.sh | 1 - contrib/polipo/package.sh | 1 - contrib/rc.subr | 1 - contrib/tor-resolve.py | 1 - doc/TODO.021 | 1 - doc/TODO.external | 1 - doc/TODO.future | 1 - doc/design-paper/latex8.bst | 2 -- doc/design-paper/usenix.sty | 2 -- doc/spec/address-spec.txt | 1 - doc/spec/bridges-spec.txt | 1 - doc/spec/control-spec-v0.txt | 1 - doc/spec/control-spec.txt | 1 - doc/spec/dir-spec-v1.txt | 1 - doc/spec/dir-spec-v2.txt | 1 - doc/spec/dir-spec.txt | 1 - doc/spec/path-spec.txt | 1 - doc/spec/proposals/000-index.txt | 2 -- doc/spec/proposals/001-process.txt | 2 -- doc/spec/proposals/098-todo.txt | 2 -- doc/spec/proposals/099-misc.txt | 2 -- doc/spec/proposals/100-tor-spec-udp.txt | 2 -- doc/spec/proposals/101-dir-voting.txt | 2 -- doc/spec/proposals/102-drop-opt.txt | 2 -- doc/spec/proposals/103-multilevel-keys.txt | 2 -- doc/spec/proposals/104-short-descriptors.txt | 2 -- doc/spec/proposals/105-handshake-revision.txt | 2 -- doc/spec/proposals/106-less-tls-constraint.txt | 2 -- doc/spec/proposals/107-uptime-sanity-checking.txt | 2 -- doc/spec/proposals/108-mtbf-based-stability.txt | 2 -- doc/spec/proposals/109-no-sharing-ips.txt | 2 -- doc/spec/proposals/110-avoid-infinite-circuits.txt | 2 -- doc/spec/proposals/111-local-traffic-priority.txt | 2 -- doc/spec/proposals/112-bring-back-pathlencoinweight.txt | 2 -- doc/spec/proposals/113-fast-authority-interface.txt | 2 -- doc/spec/proposals/114-distributed-storage.txt | 2 -- doc/spec/proposals/115-two-hop-paths.txt | 2 -- doc/spec/proposals/116-two-hop-paths-from-guard.txt | 2 -- doc/spec/proposals/117-ipv6-exits.txt | 2 -- doc/spec/proposals/118-multiple-orports.txt | 2 -- doc/spec/proposals/119-controlport-auth.txt | 2 -- doc/spec/proposals/120-shutdown-descriptors.txt | 2 -- doc/spec/proposals/121-hidden-service-authentication.txt | 2 -- doc/spec/proposals/122-unnamed-flag.txt | 2 -- doc/spec/proposals/123-autonaming.txt | 2 -- doc/spec/proposals/124-tls-certificates.txt | 2 -- doc/spec/proposals/125-bridges.txt | 2 -- doc/spec/proposals/126-geoip-reporting.txt | 2 -- doc/spec/proposals/127-dirport-mirrors-downloads.txt | 2 -- doc/spec/proposals/128-bridge-families.txt | 2 -- doc/spec/proposals/129-reject-plaintext-ports.txt | 2 -- doc/spec/proposals/130-v2-conn-protocol.txt | 2 -- doc/spec/proposals/131-verify-tor-usage.txt | 2 -- doc/spec/proposals/132-browser-check-tor-service.txt | 2 -- doc/spec/proposals/135-private-tor-networks.txt | 2 -- doc/spec/proposals/137-bootstrap-phases.txt | 2 -- doc/spec/proposals/138-remove-down-routers-from-consensus.txt | 2 -- doc/spec/proposals/140-consensus-diffs.txt | 2 -- doc/spec/proposals/141-jit-sd-downloads.txt | 2 -- doc/spec/proposals/142-combine-intro-and-rend-points.txt | 2 -- doc/spec/proposals/143-distributed-storage-improvements.txt | 2 -- doc/spec/proposals/145-newguard-flag.txt | 2 -- doc/spec/proposals/146-long-term-stability.txt | 2 -- doc/spec/proposals/147-prevoting-opinions.txt | 2 -- doc/spec/proposals/148-uniform-client-end-reason.txt | 2 -- doc/spec/proposals/149-using-netinfo-data.txt | 2 -- doc/spec/proposals/150-exclude-exit-nodes.txt | 1 - doc/spec/proposals/151-path-selection-improvements.txt | 2 -- doc/spec/proposals/152-single-hop-circuits.txt | 2 -- doc/spec/proposals/153-automatic-software-update-protocol.txt | 2 -- doc/spec/proposals/154-automatic-updates.txt | 2 -- doc/spec/proposals/155-four-hidden-service-improvements.txt | 2 -- doc/spec/proposals/156-tracking-blocked-ports.txt | 2 -- doc/spec/proposals/157-specific-cert-download.txt | 2 -- doc/spec/proposals/158-microdescriptors.txt | 2 -- doc/spec/proposals/159-exit-scanning.txt | 2 -- doc/spec/proposals/160-bandwidth-offset.txt | 2 -- doc/spec/proposals/ideas/xxx-hide-platform.txt | 2 -- doc/spec/proposals/ideas/xxx-port-knocking.txt | 2 -- doc/spec/proposals/ideas/xxx-separate-streams-by-port.txt | 2 -- doc/spec/proposals/ideas/xxx-what-uses-sha1.txt | 2 -- doc/spec/rend-spec.txt | 1 - doc/spec/socks-extensions.txt | 1 - doc/spec/tor-spec.txt | 1 - doc/spec/version-spec.txt | 1 - src/config/torrc.complete.in | 2 -- 94 files changed, 161 deletions(-) diff --git a/Doxyfile.in b/Doxyfile.in index b4d21c334d..24355f5f04 100644 --- a/Doxyfile.in +++ b/Doxyfile.in @@ -1,4 +1,3 @@ -# $Id$ # Doxyfile 1.5.1 # This file describes the settings to be used by the documentation system diff --git a/Makefile.am b/Makefile.am index b6e559ef0c..d4ebbb1fda 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1,4 +1,3 @@ -# $Id$ # Copyright (c) 2001-2004, Roger Dingledine # Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson # Copyright (c) 2007-2008, The Tor Project, Inc. diff --git a/acinclude.m4 b/acinclude.m4 index 47d421c260..766ca1e62e 100644 --- a/acinclude.m4 +++ b/acinclude.m4 @@ -1,4 +1,3 @@ -dnl $Id$ dnl Helper macros for Tor configure.in dnl Copyright (c) 2001-2004, Roger Dingledine dnl Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson diff --git a/configure.in b/configure.in index 2b8a915355..8a4101f41d 100644 --- a/configure.in +++ b/configure.in @@ -1,4 +1,3 @@ -dnl $Id$ dnl Copyright (c) 2001-2004, Roger Dingledine dnl Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson dnl Copyright (c) 2007-2008, The Tor Project, Inc. diff --git a/contrib/checkOptionDocs.pl b/contrib/checkOptionDocs.pl index ca3fba55e3..c2e8757362 100755 --- a/contrib/checkOptionDocs.pl +++ b/contrib/checkOptionDocs.pl @@ -1,5 +1,4 @@ #!/usr/bin/perl -w -# $Id use strict; my %options = (); diff --git a/contrib/cross.sh b/contrib/cross.sh index e660be780d..af68755dbf 100755 --- a/contrib/cross.sh +++ b/contrib/cross.sh @@ -1,5 +1,4 @@ #!/bin/bash -# $Id$ # Copyright 2006 Michael Mohr with modifications by Roger Dingledine # See LICENSE for licensing information. diff --git a/contrib/id_to_fp.c b/contrib/id_to_fp.c index 73395e16c1..55b025dfaf 100644 --- a/contrib/id_to_fp.c +++ b/contrib/id_to_fp.c @@ -1,5 +1,4 @@ /* Copyright 2006 Nick Mathewson; see LICENSE for licensing information */ -/* $Id$ */ /* id_to_fp.c : Helper for directory authority ops. When somebody sends us * a private key, this utility converts the private key into a fingerprint diff --git a/contrib/nagios-check-tor-authority-cert b/contrib/nagios-check-tor-authority-cert index 0e2c1d06c4..46dc7284b7 100755 --- a/contrib/nagios-check-tor-authority-cert +++ b/contrib/nagios-check-tor-authority-cert @@ -8,8 +8,6 @@ # Usage: nagios-check-tor-authority-cert # e.g.: nagios-check-tor-authority-cert A9AC67E64B200BBF2FA26DF194AC0469E2A948C6 -# $Id$ - # Copyright (c) 2008 Peter Palfrader # # Permission is hereby granted, free of charge, to any person obtaining diff --git a/contrib/osx/package.sh b/contrib/osx/package.sh index 040c7cd4c9..eeb0f95078 100644 --- a/contrib/osx/package.sh +++ b/contrib/osx/package.sh @@ -1,5 +1,4 @@ #!/bin/sh -# $Id$ # Copyright 2004-2005 Nick Mathewson. # Copyright 2005-2007 Andrew Lewman # Copyright 2008 The Tor Project, Inc. diff --git a/contrib/polipo/package.sh b/contrib/polipo/package.sh index 83f74212be..4ec72c81d8 100644 --- a/contrib/polipo/package.sh +++ b/contrib/polipo/package.sh @@ -1,5 +1,4 @@ #!/bin/sh -# $Id: package.sh 8992 2006-12-23 03:12:09Z phobos $ # Copyright 2004-2005 Nick Mathewson & Andrew Lewman. # Copyright 2005-2008 Andrew Lewman # This is licensed under the Modified BSD License. diff --git a/contrib/rc.subr b/contrib/rc.subr index 117ae71d47..d757e89528 100644 --- a/contrib/rc.subr +++ b/contrib/rc.subr @@ -1,5 +1,4 @@ #!/bin/sh -# $Id$ # $FreeBSD: ports/security/tor-devel/files/tor.in,v 1.1 2006/02/17 22:21:25 mnag Exp $ # # (rc.subr written by Peter Thoenen for Net/FreeBSD) diff --git a/contrib/tor-resolve.py b/contrib/tor-resolve.py index 919bc876cc..47ae1a0c38 100755 --- a/contrib/tor-resolve.py +++ b/contrib/tor-resolve.py @@ -1,5 +1,4 @@ #!/usr/bin/python -#$Id$ import socket import struct diff --git a/doc/TODO.021 b/doc/TODO.021 index 881ba5ee4b..37c5b9845b 100644 --- a/doc/TODO.021 +++ b/doc/TODO.021 @@ -1,4 +1,3 @@ -$Id$ Legend: SPEC!! - Not specified SPEC - Spec not finalized diff --git a/doc/TODO.external b/doc/TODO.external index c02d6aca54..948ad6a975 100644 --- a/doc/TODO.external +++ b/doc/TODO.external @@ -1,4 +1,3 @@ -$Id$ Legend: SPEC!! - Not specified SPEC - Spec not finalized diff --git a/doc/TODO.future b/doc/TODO.future index 64169ecfec..a6cc95150e 100644 --- a/doc/TODO.future +++ b/doc/TODO.future @@ -1,4 +1,3 @@ -$Id$ Legend: SPEC!! - Not specified SPEC - Spec not finalized diff --git a/doc/design-paper/latex8.bst b/doc/design-paper/latex8.bst index 2dd3249633..bae8e209ee 100644 --- a/doc/design-paper/latex8.bst +++ b/doc/design-paper/latex8.bst @@ -1,8 +1,6 @@ % --------------------------------------------------------------- % -% $Id$ -% % by Paolo.Ienne@di.epfl.ch % diff --git a/doc/design-paper/usenix.sty b/doc/design-paper/usenix.sty index 4442f11574..575c854e77 100644 --- a/doc/design-paper/usenix.sty +++ b/doc/design-paper/usenix.sty @@ -5,8 +5,6 @@ % \usepackage{usenix-2e} % and put {\rm ....} around the author names. % -% $Id$ -% % The following definitions are modifications of standard article.sty % definitions, arranged to do a better job of matching the USENIX % guidelines. diff --git a/doc/spec/address-spec.txt b/doc/spec/address-spec.txt index 2a84d857e6..95641b49b8 100644 --- a/doc/spec/address-spec.txt +++ b/doc/spec/address-spec.txt @@ -1,4 +1,3 @@ -$Id$ Special Hostnames in Tor Nick Mathewson diff --git a/doc/spec/bridges-spec.txt b/doc/spec/bridges-spec.txt index 4a9b373c8e..647118815c 100644 --- a/doc/spec/bridges-spec.txt +++ b/doc/spec/bridges-spec.txt @@ -1,4 +1,3 @@ -$Id$ Tor bridges specification diff --git a/doc/spec/control-spec-v0.txt b/doc/spec/control-spec-v0.txt index faf75a64a4..3515d395a6 100644 --- a/doc/spec/control-spec-v0.txt +++ b/doc/spec/control-spec-v0.txt @@ -1,4 +1,3 @@ -$Id$ TC: A Tor control protocol (Version 0) diff --git a/doc/spec/control-spec.txt b/doc/spec/control-spec.txt index 576c5dcd53..6ca45aa27f 100644 --- a/doc/spec/control-spec.txt +++ b/doc/spec/control-spec.txt @@ -1,4 +1,3 @@ -$Id$ TC: A Tor control protocol (Version 1) diff --git a/doc/spec/dir-spec-v1.txt b/doc/spec/dir-spec-v1.txt index 286df664e2..a92fc7999a 100644 --- a/doc/spec/dir-spec-v1.txt +++ b/doc/spec/dir-spec-v1.txt @@ -1,4 +1,3 @@ -$Id$ Tor Protocol Specification diff --git a/doc/spec/dir-spec-v2.txt b/doc/spec/dir-spec-v2.txt index 4873c4a728..d1be27f3db 100644 --- a/doc/spec/dir-spec-v2.txt +++ b/doc/spec/dir-spec-v2.txt @@ -1,4 +1,3 @@ -$Id$ Tor directory protocol, version 2 diff --git a/doc/spec/dir-spec.txt b/doc/spec/dir-spec.txt index 9a2a62bc46..0a3b0c11bc 100644 --- a/doc/spec/dir-spec.txt +++ b/doc/spec/dir-spec.txt @@ -1,4 +1,3 @@ -$Id$ Tor directory protocol, version 3 diff --git a/doc/spec/path-spec.txt b/doc/spec/path-spec.txt index dceb21dad7..b53e4bda66 100644 --- a/doc/spec/path-spec.txt +++ b/doc/spec/path-spec.txt @@ -1,4 +1,3 @@ -$Id$ Tor Path Specification diff --git a/doc/spec/proposals/000-index.txt b/doc/spec/proposals/000-index.txt index dd91efeccd..d54d5a2a47 100644 --- a/doc/spec/proposals/000-index.txt +++ b/doc/spec/proposals/000-index.txt @@ -1,7 +1,5 @@ Filename: 000-index.txt Title: Index of Tor Proposals -Version: $Revision$ -Last-Modified: $Date$ Author: Nick Mathewson Created: 26-Jan-2007 Status: Meta diff --git a/doc/spec/proposals/001-process.txt b/doc/spec/proposals/001-process.txt index 5c12d01648..2ed0a3066b 100644 --- a/doc/spec/proposals/001-process.txt +++ b/doc/spec/proposals/001-process.txt @@ -1,7 +1,5 @@ Filename: 001-process.txt Title: The Tor Proposal Process -Version: $Revision$ -Last-Modified: $Date$ Author: Nick Mathewson Created: 30-Jan-2007 Status: Meta diff --git a/doc/spec/proposals/098-todo.txt b/doc/spec/proposals/098-todo.txt index e891ea890c..a0bbbeb568 100644 --- a/doc/spec/proposals/098-todo.txt +++ b/doc/spec/proposals/098-todo.txt @@ -1,7 +1,5 @@ Filename: 098-todo.txt Title: Proposals that should be written -Version: $Revision$ -Last-Modified: $Date$ Author: Nick Mathewson, Roger Dingledine Created: 26-Jan-2007 Status: Meta diff --git a/doc/spec/proposals/099-misc.txt b/doc/spec/proposals/099-misc.txt index ba13ea2a71..a3621dd25f 100644 --- a/doc/spec/proposals/099-misc.txt +++ b/doc/spec/proposals/099-misc.txt @@ -1,7 +1,5 @@ Filename: 099-misc.txt Title: Miscellaneous proposals -Version: $Revision$ -Last-Modified: $Date$ Author: Various Created: 26-Jan-2007 Status: Meta diff --git a/doc/spec/proposals/100-tor-spec-udp.txt b/doc/spec/proposals/100-tor-spec-udp.txt index 8224682ec8..7f062222c5 100644 --- a/doc/spec/proposals/100-tor-spec-udp.txt +++ b/doc/spec/proposals/100-tor-spec-udp.txt @@ -1,7 +1,5 @@ Filename: 100-tor-spec-udp.txt Title: Tor Unreliable Datagram Extension Proposal -Version: $Revision$ -Last-Modified: $Date$ Author: Marc Liberatore Created: 23 Feb 2006 Status: Dead diff --git a/doc/spec/proposals/101-dir-voting.txt b/doc/spec/proposals/101-dir-voting.txt index be900a641e..634d3f1948 100644 --- a/doc/spec/proposals/101-dir-voting.txt +++ b/doc/spec/proposals/101-dir-voting.txt @@ -1,7 +1,5 @@ Filename: 101-dir-voting.txt Title: Voting on the Tor Directory System -Version: $Revision$ -Last-Modified: $Date$ Author: Nick Mathewson Created: Nov 2006 Status: Closed diff --git a/doc/spec/proposals/102-drop-opt.txt b/doc/spec/proposals/102-drop-opt.txt index 8f6a38ae6c..490376bb53 100644 --- a/doc/spec/proposals/102-drop-opt.txt +++ b/doc/spec/proposals/102-drop-opt.txt @@ -1,7 +1,5 @@ Filename: 102-drop-opt.txt Title: Dropping "opt" from the directory format -Version: $Revision$ -Last-Modified: $Date$ Author: Nick Mathewson Created: Jan 2007 Status: Closed diff --git a/doc/spec/proposals/103-multilevel-keys.txt b/doc/spec/proposals/103-multilevel-keys.txt index ef51e18047..c8a7a6677b 100644 --- a/doc/spec/proposals/103-multilevel-keys.txt +++ b/doc/spec/proposals/103-multilevel-keys.txt @@ -1,7 +1,5 @@ Filename: 103-multilevel-keys.txt Title: Splitting identity key from regularly used signing key. -Version: $Revision$ -Last-Modified: $Date$ Author: Nick Mathewson Created: Jan 2007 Status: Closed diff --git a/doc/spec/proposals/104-short-descriptors.txt b/doc/spec/proposals/104-short-descriptors.txt index a1c42c8ff7..90e0764fe6 100644 --- a/doc/spec/proposals/104-short-descriptors.txt +++ b/doc/spec/proposals/104-short-descriptors.txt @@ -1,7 +1,5 @@ Filename: 104-short-descriptors.txt Title: Long and Short Router Descriptors -Version: $Revision$ -Last-Modified: $Date$ Author: Nick Mathewson Created: Jan 2007 Status: Closed diff --git a/doc/spec/proposals/105-handshake-revision.txt b/doc/spec/proposals/105-handshake-revision.txt index f6c209e71b..791a016c26 100644 --- a/doc/spec/proposals/105-handshake-revision.txt +++ b/doc/spec/proposals/105-handshake-revision.txt @@ -1,7 +1,5 @@ Filename: 105-handshake-revision.txt Title: Version negotiation for the Tor protocol. -Version: $Revision$ -Last-Modified: $Date$ Author: Nick Mathewson, Roger Dingledine Created: Jan 2007 Status: Closed diff --git a/doc/spec/proposals/106-less-tls-constraint.txt b/doc/spec/proposals/106-less-tls-constraint.txt index 35d6bf1066..7e7621df69 100644 --- a/doc/spec/proposals/106-less-tls-constraint.txt +++ b/doc/spec/proposals/106-less-tls-constraint.txt @@ -1,7 +1,5 @@ Filename: 106-less-tls-constraint.txt Title: Checking fewer things during TLS handshakes -Version: $Revision$ -Last-Modified: $Date$ Author: Nick Mathewson Created: 9-Feb-2007 Status: Closed diff --git a/doc/spec/proposals/107-uptime-sanity-checking.txt b/doc/spec/proposals/107-uptime-sanity-checking.txt index b11be89380..922129b21d 100644 --- a/doc/spec/proposals/107-uptime-sanity-checking.txt +++ b/doc/spec/proposals/107-uptime-sanity-checking.txt @@ -1,7 +1,5 @@ Filename: 107-uptime-sanity-checking.txt Title: Uptime Sanity Checking -Version: $Revision$ -Last-Modified: $Date$ Author: Kevin Bauer & Damon McCoy Created: 8-March-2007 Status: Closed diff --git a/doc/spec/proposals/108-mtbf-based-stability.txt b/doc/spec/proposals/108-mtbf-based-stability.txt index 2c66481530..294103760b 100644 --- a/doc/spec/proposals/108-mtbf-based-stability.txt +++ b/doc/spec/proposals/108-mtbf-based-stability.txt @@ -1,7 +1,5 @@ Filename: 108-mtbf-based-stability.txt Title: Base "Stable" Flag on Mean Time Between Failures -Version: $Revision$ -Last-Modified: $Date$ Author: Nick Mathewson Created: 10-Mar-2007 Status: Closed diff --git a/doc/spec/proposals/109-no-sharing-ips.txt b/doc/spec/proposals/109-no-sharing-ips.txt index 1a88b00c0f..5438cf049a 100644 --- a/doc/spec/proposals/109-no-sharing-ips.txt +++ b/doc/spec/proposals/109-no-sharing-ips.txt @@ -1,7 +1,5 @@ Filename: 109-no-sharing-ips.txt Title: No more than one server per IP address. -Version: $Revision$ -Last-Modified: $Date$ Author: Kevin Bauer & Damon McCoy Created: 9-March-2007 Status: Closed diff --git a/doc/spec/proposals/110-avoid-infinite-circuits.txt b/doc/spec/proposals/110-avoid-infinite-circuits.txt index 1834cd34a7..fffc41c25a 100644 --- a/doc/spec/proposals/110-avoid-infinite-circuits.txt +++ b/doc/spec/proposals/110-avoid-infinite-circuits.txt @@ -1,7 +1,5 @@ Filename: 110-avoid-infinite-circuits.txt Title: Avoiding infinite length circuits -Version: $Revision$ -Last-Modified: $Date$ Author: Roger Dingledine Created: 13-Mar-2007 Status: Accepted diff --git a/doc/spec/proposals/111-local-traffic-priority.txt b/doc/spec/proposals/111-local-traffic-priority.txt index f8a37efc94..9411463c21 100644 --- a/doc/spec/proposals/111-local-traffic-priority.txt +++ b/doc/spec/proposals/111-local-traffic-priority.txt @@ -1,7 +1,5 @@ Filename: 111-local-traffic-priority.txt Title: Prioritizing local traffic over relayed traffic -Version: $Revision$ -Last-Modified: $Date$ Author: Roger Dingledine Created: 14-Mar-2007 Status: Closed diff --git a/doc/spec/proposals/112-bring-back-pathlencoinweight.txt b/doc/spec/proposals/112-bring-back-pathlencoinweight.txt index e7cc6b4e36..3f6c3376f0 100644 --- a/doc/spec/proposals/112-bring-back-pathlencoinweight.txt +++ b/doc/spec/proposals/112-bring-back-pathlencoinweight.txt @@ -1,7 +1,5 @@ Filename: 112-bring-back-pathlencoinweight.txt Title: Bring Back Pathlen Coin Weight -Version: $Revision$ -Last-Modified: $Date$ Author: Mike Perry Created: Status: Superseded diff --git a/doc/spec/proposals/113-fast-authority-interface.txt b/doc/spec/proposals/113-fast-authority-interface.txt index 20cf33e429..8912b53220 100644 --- a/doc/spec/proposals/113-fast-authority-interface.txt +++ b/doc/spec/proposals/113-fast-authority-interface.txt @@ -1,7 +1,5 @@ Filename: 113-fast-authority-interface.txt Title: Simplifying directory authority administration -Version: $Revision$ -Last-Modified: $Date$ Author: Nick Mathewson Created: Status: Superseded diff --git a/doc/spec/proposals/114-distributed-storage.txt b/doc/spec/proposals/114-distributed-storage.txt index e9271fb82d..91a787d301 100644 --- a/doc/spec/proposals/114-distributed-storage.txt +++ b/doc/spec/proposals/114-distributed-storage.txt @@ -1,7 +1,5 @@ Filename: 114-distributed-storage.txt Title: Distributed Storage for Tor Hidden Service Descriptors -Version: $Revision$ -Last-Modified: $Date$ Author: Karsten Loesing Created: 13-May-2007 Status: Closed diff --git a/doc/spec/proposals/115-two-hop-paths.txt b/doc/spec/proposals/115-two-hop-paths.txt index ee10d949c4..9854c9ad55 100644 --- a/doc/spec/proposals/115-two-hop-paths.txt +++ b/doc/spec/proposals/115-two-hop-paths.txt @@ -1,7 +1,5 @@ Filename: 115-two-hop-paths.txt Title: Two Hop Paths -Version: $Revision$ -Last-Modified: $Date$ Author: Mike Perry Created: Status: Dead diff --git a/doc/spec/proposals/116-two-hop-paths-from-guard.txt b/doc/spec/proposals/116-two-hop-paths-from-guard.txt index 454b344abf..f45625350b 100644 --- a/doc/spec/proposals/116-two-hop-paths-from-guard.txt +++ b/doc/spec/proposals/116-two-hop-paths-from-guard.txt @@ -1,7 +1,5 @@ Filename: 116-two-hop-paths-from-guard.txt Title: Two hop paths from entry guards -Version: $Revision$ -Last-Modified: $Date$ Author: Michael Lieberman Created: 26-Jun-2007 Status: Dead diff --git a/doc/spec/proposals/117-ipv6-exits.txt b/doc/spec/proposals/117-ipv6-exits.txt index c8402821ed..00cd7cef10 100644 --- a/doc/spec/proposals/117-ipv6-exits.txt +++ b/doc/spec/proposals/117-ipv6-exits.txt @@ -1,7 +1,5 @@ Filename: 117-ipv6-exits.txt Title: IPv6 exits -Version: $Revision$ -Last-Modified: $Date$ Author: coderman Created: 10-Jul-2007 Status: Accepted diff --git a/doc/spec/proposals/118-multiple-orports.txt b/doc/spec/proposals/118-multiple-orports.txt index 1bef2504d9..2381ec7ca3 100644 --- a/doc/spec/proposals/118-multiple-orports.txt +++ b/doc/spec/proposals/118-multiple-orports.txt @@ -1,7 +1,5 @@ Filename: 118-multiple-orports.txt Title: Advertising multiple ORPorts at once -Version: $Revision$ -Last-Modified: $Date$ Author: Nick Mathewson Created: 09-Jul-2007 Status: Accepted diff --git a/doc/spec/proposals/119-controlport-auth.txt b/doc/spec/proposals/119-controlport-auth.txt index dc57a27368..9ed1cc1cbe 100644 --- a/doc/spec/proposals/119-controlport-auth.txt +++ b/doc/spec/proposals/119-controlport-auth.txt @@ -1,7 +1,5 @@ Filename: 119-controlport-auth.txt Title: New PROTOCOLINFO command for controllers -Version: $Revision$ -Last-Modified: $Date$ Author: Roger Dingledine Created: 14-Aug-2007 Status: Closed diff --git a/doc/spec/proposals/120-shutdown-descriptors.txt b/doc/spec/proposals/120-shutdown-descriptors.txt index dc1265b03b..5cfe2b5bc6 100644 --- a/doc/spec/proposals/120-shutdown-descriptors.txt +++ b/doc/spec/proposals/120-shutdown-descriptors.txt @@ -1,7 +1,5 @@ Filename: 120-shutdown-descriptors.txt Title: Shutdown descriptors when Tor servers stop -Version: $Revision$ -Last-Modified: $Date$ Author: Roger Dingledine Created: 15-Aug-2007 Status: Dead diff --git a/doc/spec/proposals/121-hidden-service-authentication.txt b/doc/spec/proposals/121-hidden-service-authentication.txt index 828bf3c92d..0d92b53a8c 100644 --- a/doc/spec/proposals/121-hidden-service-authentication.txt +++ b/doc/spec/proposals/121-hidden-service-authentication.txt @@ -1,7 +1,5 @@ Filename: 121-hidden-service-authentication.txt Title: Hidden Service Authentication -Version: $Revision$ -Last-Modified: $Date$ Author: Tobias Kamm, Thomas Lauterbach, Karsten Loesing, Ferdinand Rieger, Christoph Weingarten Created: 10-Sep-2007 diff --git a/doc/spec/proposals/122-unnamed-flag.txt b/doc/spec/proposals/122-unnamed-flag.txt index 6502b9c560..2ce7bb22b9 100644 --- a/doc/spec/proposals/122-unnamed-flag.txt +++ b/doc/spec/proposals/122-unnamed-flag.txt @@ -1,7 +1,5 @@ Filename: 122-unnamed-flag.txt Title: Network status entries need a new Unnamed flag -Version: $Revision$ -Last-Modified: $Date$ Author: Roger Dingledine Created: 04-Oct-2007 Status: Closed diff --git a/doc/spec/proposals/123-autonaming.txt b/doc/spec/proposals/123-autonaming.txt index 6cd25329f8..74c486985d 100644 --- a/doc/spec/proposals/123-autonaming.txt +++ b/doc/spec/proposals/123-autonaming.txt @@ -1,7 +1,5 @@ Filename: 123-autonaming.txt Title: Naming authorities automatically create bindings -Version: $Revision$ -Last-Modified: $Date$ Author: Peter Palfrader Created: 2007-10-11 Status: Closed diff --git a/doc/spec/proposals/124-tls-certificates.txt b/doc/spec/proposals/124-tls-certificates.txt index 0a47772732..9472d14af8 100644 --- a/doc/spec/proposals/124-tls-certificates.txt +++ b/doc/spec/proposals/124-tls-certificates.txt @@ -1,7 +1,5 @@ Filename: 124-tls-certificates.txt Title: Blocking resistant TLS certificate usage -Version: $Revision$ -Last-Modified: $Date$ Author: Steven J. Murdoch Created: 2007-10-25 Status: Superseded diff --git a/doc/spec/proposals/125-bridges.txt b/doc/spec/proposals/125-bridges.txt index 8bb3169780..9d95729d42 100644 --- a/doc/spec/proposals/125-bridges.txt +++ b/doc/spec/proposals/125-bridges.txt @@ -1,7 +1,5 @@ Filename: 125-bridges.txt Title: Behavior for bridge users, bridge relays, and bridge authorities -Version: $Revision$ -Last-Modified: $Date$ Author: Roger Dingledine Created: 11-Nov-2007 Status: Closed diff --git a/doc/spec/proposals/126-geoip-reporting.txt b/doc/spec/proposals/126-geoip-reporting.txt index d48a08ba38..9f3b21c670 100644 --- a/doc/spec/proposals/126-geoip-reporting.txt +++ b/doc/spec/proposals/126-geoip-reporting.txt @@ -1,7 +1,5 @@ Filename: 126-geoip-reporting.txt Title: Getting GeoIP data and publishing usage summaries -Version: $Revision$ -Last-Modified: $Date$ Author: Roger Dingledine Created: 2007-11-24 Status: Closed diff --git a/doc/spec/proposals/127-dirport-mirrors-downloads.txt b/doc/spec/proposals/127-dirport-mirrors-downloads.txt index 1b55a02d61..72d6c0cb9f 100644 --- a/doc/spec/proposals/127-dirport-mirrors-downloads.txt +++ b/doc/spec/proposals/127-dirport-mirrors-downloads.txt @@ -1,7 +1,5 @@ Filename: 127-dirport-mirrors-downloads.txt Title: Relaying dirport requests to Tor download site / website -Version: $Revision$ -Last-Modified: $Date$ Author: Roger Dingledine Created: 2007-12-02 Status: Draft diff --git a/doc/spec/proposals/128-bridge-families.txt b/doc/spec/proposals/128-bridge-families.txt index e8a0050c3c..e5bdcf95cb 100644 --- a/doc/spec/proposals/128-bridge-families.txt +++ b/doc/spec/proposals/128-bridge-families.txt @@ -1,7 +1,5 @@ Filename: 128-bridge-families.txt Title: Families of private bridges -Version: $Revision$ -Last-Modified: $Date$ Author: Roger Dingledine Created: 2007-12-xx Status: Dead diff --git a/doc/spec/proposals/129-reject-plaintext-ports.txt b/doc/spec/proposals/129-reject-plaintext-ports.txt index d4767d03d8..8080ff5b75 100644 --- a/doc/spec/proposals/129-reject-plaintext-ports.txt +++ b/doc/spec/proposals/129-reject-plaintext-ports.txt @@ -1,7 +1,5 @@ Filename: 129-reject-plaintext-ports.txt Title: Block Insecure Protocols by Default -Version: $Revision$ -Last-Modified: $Date$ Author: Kevin Bauer & Damon McCoy Created: 2008-01-15 Status: Closed diff --git a/doc/spec/proposals/130-v2-conn-protocol.txt b/doc/spec/proposals/130-v2-conn-protocol.txt index 16f5bf2844..60e742a622 100644 --- a/doc/spec/proposals/130-v2-conn-protocol.txt +++ b/doc/spec/proposals/130-v2-conn-protocol.txt @@ -1,7 +1,5 @@ Filename: 130-v2-conn-protocol.txt Title: Version 2 Tor connection protocol -Version: $Revision$ -Last-Modified: $Date$ Author: Nick Mathewson Created: 2007-10-25 Status: Closed diff --git a/doc/spec/proposals/131-verify-tor-usage.txt b/doc/spec/proposals/131-verify-tor-usage.txt index 2687139189..d3c6efe75a 100644 --- a/doc/spec/proposals/131-verify-tor-usage.txt +++ b/doc/spec/proposals/131-verify-tor-usage.txt @@ -1,7 +1,5 @@ Filename: 131-verify-tor-usage.txt Title: Help users to verify they are using Tor -Version: $Revision$ -Last-Modified: $Date$ Author: Steven J. Murdoch Created: 2008-01-25 Status: Needs-Revision diff --git a/doc/spec/proposals/132-browser-check-tor-service.txt b/doc/spec/proposals/132-browser-check-tor-service.txt index d07a10dcde..6132e5d060 100644 --- a/doc/spec/proposals/132-browser-check-tor-service.txt +++ b/doc/spec/proposals/132-browser-check-tor-service.txt @@ -1,7 +1,5 @@ Filename: 132-browser-check-tor-service.txt Title: A Tor Web Service For Verifying Correct Browser Configuration -Version: $Revision$ -Last-Modified: $Date$ Author: Robert Hogan Created: 2008-03-08 Status: Draft diff --git a/doc/spec/proposals/135-private-tor-networks.txt b/doc/spec/proposals/135-private-tor-networks.txt index 131bbb9068..19ef68b7b1 100644 --- a/doc/spec/proposals/135-private-tor-networks.txt +++ b/doc/spec/proposals/135-private-tor-networks.txt @@ -1,7 +1,5 @@ Filename: 135-private-tor-networks.txt Title: Simplify Configuration of Private Tor Networks -Version: $Revision$ -Last-Modified: $Date$ Author: Karsten Loesing Created: 29-Apr-2008 Status: Closed diff --git a/doc/spec/proposals/137-bootstrap-phases.txt b/doc/spec/proposals/137-bootstrap-phases.txt index 18d3dfae12..ebe044c707 100644 --- a/doc/spec/proposals/137-bootstrap-phases.txt +++ b/doc/spec/proposals/137-bootstrap-phases.txt @@ -1,7 +1,5 @@ Filename: 137-bootstrap-phases.txt Title: Keep controllers informed as Tor bootstraps -Version: $Revision$ -Last-Modified: $Date$ Author: Roger Dingledine Created: 07-Jun-2008 Status: Closed diff --git a/doc/spec/proposals/138-remove-down-routers-from-consensus.txt b/doc/spec/proposals/138-remove-down-routers-from-consensus.txt index a07764d536..776911b5c9 100644 --- a/doc/spec/proposals/138-remove-down-routers-from-consensus.txt +++ b/doc/spec/proposals/138-remove-down-routers-from-consensus.txt @@ -1,7 +1,5 @@ Filename: 138-remove-down-routers-from-consensus.txt Title: Remove routers that are not Running from consensus documents -Version: $Revision$ -Last-Modified: $Date$ Author: Peter Palfrader Created: 11-Jun-2008 Status: Closed diff --git a/doc/spec/proposals/140-consensus-diffs.txt b/doc/spec/proposals/140-consensus-diffs.txt index da63bfe23c..b4a8ab4f85 100644 --- a/doc/spec/proposals/140-consensus-diffs.txt +++ b/doc/spec/proposals/140-consensus-diffs.txt @@ -1,7 +1,5 @@ Filename: 140-consensus-diffs.txt Title: Provide diffs between consensuses -Version: $Revision$ -Last-Modified: $Date$ Author: Peter Palfrader Created: 13-Jun-2008 Status: Accepted diff --git a/doc/spec/proposals/141-jit-sd-downloads.txt b/doc/spec/proposals/141-jit-sd-downloads.txt index 1546e854bd..2ac7a086b7 100644 --- a/doc/spec/proposals/141-jit-sd-downloads.txt +++ b/doc/spec/proposals/141-jit-sd-downloads.txt @@ -1,7 +1,5 @@ Filename: 141-jit-sd-downloads.txt Title: Download server descriptors on demand -Version: $Revision$ -Last-Modified: $Date$ Author: Peter Palfrader Created: 15-Jun-2008 Status: Draft diff --git a/doc/spec/proposals/142-combine-intro-and-rend-points.txt b/doc/spec/proposals/142-combine-intro-and-rend-points.txt index 3456b285a9..3abd5c863d 100644 --- a/doc/spec/proposals/142-combine-intro-and-rend-points.txt +++ b/doc/spec/proposals/142-combine-intro-and-rend-points.txt @@ -1,7 +1,5 @@ Filename: 142-combine-intro-and-rend-points.txt Title: Combine Introduction and Rendezvous Points -Version: $Revision$ -Last-Modified: $Date$ Author: Karsten Loesing, Christian Wilms Created: 27-Jun-2008 Status: Dead diff --git a/doc/spec/proposals/143-distributed-storage-improvements.txt b/doc/spec/proposals/143-distributed-storage-improvements.txt index 8789d84663..0f7468f1dc 100644 --- a/doc/spec/proposals/143-distributed-storage-improvements.txt +++ b/doc/spec/proposals/143-distributed-storage-improvements.txt @@ -1,7 +1,5 @@ Filename: 143-distributed-storage-improvements.txt Title: Improvements of Distributed Storage for Tor Hidden Service Descriptors -Version: $Revision$ -Last-Modified: $Date$ Author: Karsten Loesing Created: 28-Jun-2008 Status: Open diff --git a/doc/spec/proposals/145-newguard-flag.txt b/doc/spec/proposals/145-newguard-flag.txt index 31d707d725..9e61e30be9 100644 --- a/doc/spec/proposals/145-newguard-flag.txt +++ b/doc/spec/proposals/145-newguard-flag.txt @@ -1,7 +1,5 @@ Filename: 145-newguard-flag.txt Title: Separate "suitable as a guard" from "suitable as a new guard" -Version: $Revision$ -Last-Modified: $Date$ Author: Nick Mathewson Created: 1-Jul-2008 Status: Open diff --git a/doc/spec/proposals/146-long-term-stability.txt b/doc/spec/proposals/146-long-term-stability.txt index 7cfd58f564..9af0017441 100644 --- a/doc/spec/proposals/146-long-term-stability.txt +++ b/doc/spec/proposals/146-long-term-stability.txt @@ -1,7 +1,5 @@ Filename: 146-long-term-stability.txt Title: Add new flag to reflect long-term stability -Version: $Revision$ -Last-Modified: $Date$ Author: Nick Mathewson Created: 19-Jun-2008 Status: Open diff --git a/doc/spec/proposals/147-prevoting-opinions.txt b/doc/spec/proposals/147-prevoting-opinions.txt index 2b8cf30e46..3d9659c984 100644 --- a/doc/spec/proposals/147-prevoting-opinions.txt +++ b/doc/spec/proposals/147-prevoting-opinions.txt @@ -1,7 +1,5 @@ Filename: 147-prevoting-opinions.txt Title: Eliminate the need for v2 directories in generating v3 directories -Version: $Revision$ -Last-Modified: $Date$ Author: Nick Mathewson Created: 2-Jul-2008 Status: Accepted diff --git a/doc/spec/proposals/148-uniform-client-end-reason.txt b/doc/spec/proposals/148-uniform-client-end-reason.txt index cec81253ea..1db3b3e596 100644 --- a/doc/spec/proposals/148-uniform-client-end-reason.txt +++ b/doc/spec/proposals/148-uniform-client-end-reason.txt @@ -1,7 +1,5 @@ Filename: 148-uniform-client-end-reason.txt Title: Stream end reasons from the client side should be uniform -Version: $Revision$ -Last-Modified: $Date$ Author: Roger Dingledine Created: 2-Jul-2008 Status: Closed diff --git a/doc/spec/proposals/149-using-netinfo-data.txt b/doc/spec/proposals/149-using-netinfo-data.txt index 4919514b4c..cbfc759f22 100644 --- a/doc/spec/proposals/149-using-netinfo-data.txt +++ b/doc/spec/proposals/149-using-netinfo-data.txt @@ -1,7 +1,5 @@ Filename: 149-using-netinfo-data.txt Title: Using data from NETINFO cells -Version: $Revision$ -Last-Modified: $Date$ Author: Nick Mathewson Created: 2-Jul-2008 Status: Open diff --git a/doc/spec/proposals/150-exclude-exit-nodes.txt b/doc/spec/proposals/150-exclude-exit-nodes.txt index b73a9cc4d1..b497ae62c1 100644 --- a/doc/spec/proposals/150-exclude-exit-nodes.txt +++ b/doc/spec/proposals/150-exclude-exit-nodes.txt @@ -1,6 +1,5 @@ Filename: 150-exclude-exit-nodes.txt Title: Exclude Exit Nodes from a circuit -Version: $Revision$ Author: Mfr Created: 2008-06-15 Status: Closed diff --git a/doc/spec/proposals/151-path-selection-improvements.txt b/doc/spec/proposals/151-path-selection-improvements.txt index e3c8f35451..3d5f07d3ab 100644 --- a/doc/spec/proposals/151-path-selection-improvements.txt +++ b/doc/spec/proposals/151-path-selection-improvements.txt @@ -1,7 +1,5 @@ Filename: 151-path-selection-improvements.txt Title: Improving Tor Path Selection -Version: -Last-Modified: Author: Fallon Chen, Mike Perry Created: 5-Jul-2008 Status: Draft diff --git a/doc/spec/proposals/152-single-hop-circuits.txt b/doc/spec/proposals/152-single-hop-circuits.txt index e49a4250e0..d0b28b1c72 100644 --- a/doc/spec/proposals/152-single-hop-circuits.txt +++ b/doc/spec/proposals/152-single-hop-circuits.txt @@ -1,7 +1,5 @@ Filename: 152-single-hop-circuits.txt Title: Optionally allow exit from single-hop circuits -Version: -Last-Modified: Author: Geoff Goodell Created: 13-Jul-2008 Status: Closed diff --git a/doc/spec/proposals/153-automatic-software-update-protocol.txt b/doc/spec/proposals/153-automatic-software-update-protocol.txt index 7bc809d440..c2979bb695 100644 --- a/doc/spec/proposals/153-automatic-software-update-protocol.txt +++ b/doc/spec/proposals/153-automatic-software-update-protocol.txt @@ -1,7 +1,5 @@ Filename: 153-automatic-software-update-protocol.txt Title: Automatic software update protocol -Version: $Revision$ -Last-Modified: $Date$ Author: Jacob Appelbaum Created: 14-July-2008 Status: Superseded diff --git a/doc/spec/proposals/154-automatic-updates.txt b/doc/spec/proposals/154-automatic-updates.txt index 00a820de08..4c2c6d3899 100644 --- a/doc/spec/proposals/154-automatic-updates.txt +++ b/doc/spec/proposals/154-automatic-updates.txt @@ -1,7 +1,5 @@ Filename: 154-automatic-updates.txt Title: Automatic Software Update Protocol -Version: $Revision$ -Last-Modified: $Date$ Author: Matt Edman Created: 30-July-2008 Status: Superseded diff --git a/doc/spec/proposals/155-four-hidden-service-improvements.txt b/doc/spec/proposals/155-four-hidden-service-improvements.txt index f528f8baf2..e342bf1c39 100644 --- a/doc/spec/proposals/155-four-hidden-service-improvements.txt +++ b/doc/spec/proposals/155-four-hidden-service-improvements.txt @@ -1,7 +1,5 @@ Filename: 155-four-hidden-service-improvements.txt Title: Four Improvements of Hidden Service Performance -Version: $Revision$ -Last-Modified: $Date$ Author: Karsten Loesing, Christian Wilms Created: 25-Sep-2008 Status: Finished diff --git a/doc/spec/proposals/156-tracking-blocked-ports.txt b/doc/spec/proposals/156-tracking-blocked-ports.txt index 1e7b0d963f..419de7e74c 100644 --- a/doc/spec/proposals/156-tracking-blocked-ports.txt +++ b/doc/spec/proposals/156-tracking-blocked-ports.txt @@ -1,7 +1,5 @@ Filename: 156-tracking-blocked-ports.txt Title: Tracking blocked ports on the client side -Version: $Revision$ -Last-Modified: $Date$ Author: Robert Hogan Created: 14-Oct-2008 Status: Open diff --git a/doc/spec/proposals/157-specific-cert-download.txt b/doc/spec/proposals/157-specific-cert-download.txt index e54a987277..204b20973a 100644 --- a/doc/spec/proposals/157-specific-cert-download.txt +++ b/doc/spec/proposals/157-specific-cert-download.txt @@ -1,7 +1,5 @@ Filename: 157-specific-cert-download.txt Title: Make certificate downloads specific -Version: $Revision$ -Last-Modified: $Date$ Author: Nick Mathewson Created: 2-Dec-2008 Status: Accepted diff --git a/doc/spec/proposals/158-microdescriptors.txt b/doc/spec/proposals/158-microdescriptors.txt index f478a3c834..3b9a690a2b 100644 --- a/doc/spec/proposals/158-microdescriptors.txt +++ b/doc/spec/proposals/158-microdescriptors.txt @@ -1,7 +1,5 @@ Filename: 158-microdescriptors.txt Title: Clients download consensus + microdescriptors -Version: $Revision$ -Last-Modified: $Date$ Author: Roger Dingledine Created: 17-Jan-2009 Status: Open diff --git a/doc/spec/proposals/159-exit-scanning.txt b/doc/spec/proposals/159-exit-scanning.txt index fbc69aa9e6..7090f2ed08 100644 --- a/doc/spec/proposals/159-exit-scanning.txt +++ b/doc/spec/proposals/159-exit-scanning.txt @@ -1,7 +1,5 @@ Filename: 159-exit-scanning.txt Title: Exit Scanning -Version: $Revision$ -Last-Modified: $Date$ Author: Mike Perry Created: 13-Feb-2009 Status: Open diff --git a/doc/spec/proposals/160-bandwidth-offset.txt b/doc/spec/proposals/160-bandwidth-offset.txt index 078754b771..f618d0eeb2 100644 --- a/doc/spec/proposals/160-bandwidth-offset.txt +++ b/doc/spec/proposals/160-bandwidth-offset.txt @@ -1,7 +1,5 @@ Filename: 160-bandwidth-offset.txt Title: Authorities vote for bandwidth offsets in consensus -Version: $Revision$ -Last-Modified: $Date$ Author: Roger Dingledine Created: 4-May-2009 Status: Open diff --git a/doc/spec/proposals/ideas/xxx-hide-platform.txt b/doc/spec/proposals/ideas/xxx-hide-platform.txt index 3fed5cfbd4..ad19fb1fd4 100644 --- a/doc/spec/proposals/ideas/xxx-hide-platform.txt +++ b/doc/spec/proposals/ideas/xxx-hide-platform.txt @@ -1,7 +1,5 @@ Filename: xxx-hide-platform.txt Title: Hide Tor Platform Information -Version: $Revision$ -Last-Modified: $Date$ Author: Jacob Appelbaum Created: 24-July-2008 Status: Draft diff --git a/doc/spec/proposals/ideas/xxx-port-knocking.txt b/doc/spec/proposals/ideas/xxx-port-knocking.txt index 9fbcdf3545..85c27ec52d 100644 --- a/doc/spec/proposals/ideas/xxx-port-knocking.txt +++ b/doc/spec/proposals/ideas/xxx-port-knocking.txt @@ -1,7 +1,5 @@ Filename: xxx-port-knocking.txt Title: Port knocking for bridge scanning resistance -Version: $Revision$ -Last-Modified: $Date$ Author: Jacob Appelbaum Created: 19-April-2009 Status: Draft diff --git a/doc/spec/proposals/ideas/xxx-separate-streams-by-port.txt b/doc/spec/proposals/ideas/xxx-separate-streams-by-port.txt index cebde65a9b..f26c1e580f 100644 --- a/doc/spec/proposals/ideas/xxx-separate-streams-by-port.txt +++ b/doc/spec/proposals/ideas/xxx-separate-streams-by-port.txt @@ -1,7 +1,5 @@ Filename: xxx-separate-streams-by-port.txt Title: Separate streams across circuits by destination port -Version: $Revision$ -Last-Modified: $Date$ Author: Robert Hogan Created: 21-Oct-2008 Status: Draft diff --git a/doc/spec/proposals/ideas/xxx-what-uses-sha1.txt b/doc/spec/proposals/ideas/xxx-what-uses-sha1.txt index b0be567a4a..e176bb8f9c 100644 --- a/doc/spec/proposals/ideas/xxx-what-uses-sha1.txt +++ b/doc/spec/proposals/ideas/xxx-what-uses-sha1.txt @@ -1,7 +1,5 @@ Filename: xxx-what-uses-sha1.txt Title: Where does Tor use SHA-1 today? -Version: $Revision$ -Last-Modified: 1-May-2009 Authors: Nick Mathewson, Marian Created: 30-Dec-2008 Status: Meta diff --git a/doc/spec/rend-spec.txt b/doc/spec/rend-spec.txt index 5c2f69871b..f030092679 100644 --- a/doc/spec/rend-spec.txt +++ b/doc/spec/rend-spec.txt @@ -1,4 +1,3 @@ -$Id$ Tor Rendezvous Specification diff --git a/doc/spec/socks-extensions.txt b/doc/spec/socks-extensions.txt index 8d58987f35..62d86acd9f 100644 --- a/doc/spec/socks-extensions.txt +++ b/doc/spec/socks-extensions.txt @@ -1,4 +1,3 @@ -$Id$ Tor's extensions to the SOCKS protocol 1. Overview diff --git a/doc/spec/tor-spec.txt b/doc/spec/tor-spec.txt index 77a91cad8e..9ae22a92d7 100644 --- a/doc/spec/tor-spec.txt +++ b/doc/spec/tor-spec.txt @@ -1,4 +1,3 @@ -$Id$ Tor Protocol Specification diff --git a/doc/spec/version-spec.txt b/doc/spec/version-spec.txt index 842271ae19..265717f409 100644 --- a/doc/spec/version-spec.txt +++ b/doc/spec/version-spec.txt @@ -1,4 +1,3 @@ -$Id$ HOW TOR VERSION NUMBERS WORK diff --git a/src/config/torrc.complete.in b/src/config/torrc.complete.in index 310458a5c0..2fbf494e56 100644 --- a/src/config/torrc.complete.in +++ b/src/config/torrc.complete.in @@ -1,5 +1,3 @@ -# $Id$ -# Last updated on $Date$ #################################################################### ## This config file is divided into four sections. They are: ## 1. Global Options (clients and servers) From deea710cdcbbcdafa9fd67f8f9a7e85f7fdbb652 Mon Sep 17 00:00:00 2001 From: Sebastian Hahn Date: Tue, 5 May 2009 15:47:36 +0200 Subject: [PATCH 022/187] Fix typo in 001-process.txt --- doc/spec/proposals/001-process.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/spec/proposals/001-process.txt b/doc/spec/proposals/001-process.txt index 2ed0a3066b..636ba2c2fa 100644 --- a/doc/spec/proposals/001-process.txt +++ b/doc/spec/proposals/001-process.txt @@ -93,7 +93,7 @@ What should go in a proposal: what the proposal's about, what it does, and about what state it's in. After the Overview, the proposal becomes more free-form. Depending on its - the length and complexity, the proposal can break into sections as + length and complexity, the proposal can break into sections as appropriate, or follow a short discursive format. Every proposal should contain at least the following information before it is "ACCEPTED", though the information does not need to be in sections with these names. From b9b16ef9a5c7c84f41f95e0e2c8266f4c1e2dce2 Mon Sep 17 00:00:00 2001 From: Sebastian Hahn Date: Tue, 5 May 2009 16:34:15 +0200 Subject: [PATCH 023/187] Add a missing newline --- src/common/util.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/common/util.c b/src/common/util.c index 69cc0c4951..c1b3ea5507 100644 --- a/src/common/util.c +++ b/src/common/util.c @@ -2569,3 +2569,4 @@ libor_get_digests(void) #include "common_sha1.i" ; } + From fdbdb4dc15004fdb74c57b9bc67cbfca24c6e7ea Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 8 May 2009 12:35:36 -0400 Subject: [PATCH 024/187] Include the *_sha1.i files in their own *_codedigest.c files. This way we do not need to rebuild util.c and/or config.c whenever any unrelated source file in src/common or src/or has changed. --- src/common/Makefile.am | 4 ++-- src/common/util.c | 8 -------- src/common/util_codedigest.c | 11 +++++++++++ src/or/Makefile.am | 6 +++--- src/or/config.c | 4 +--- src/or/config_codedigest.c | 10 ++++++++++ src/or/or.h | 2 ++ 7 files changed, 29 insertions(+), 16 deletions(-) create mode 100644 src/common/util_codedigest.c create mode 100644 src/or/config_codedigest.c diff --git a/src/common/Makefile.am b/src/common/Makefile.am index 78c8ab5516..ad54e1aaaa 100644 --- a/src/common/Makefile.am +++ b/src/common/Makefile.am @@ -12,7 +12,7 @@ libor_extra_source= endif libor_a_SOURCES = address.c log.c util.c compat.c container.c mempool.c \ - memarea.c $(libor_extra_source) + memarea.c util_codedigest.c $(libor_extra_source) libor_crypto_a_SOURCES = crypto.c aes.c tortls.c torgzip.c noinst_HEADERS = address.h log.h crypto.h test.h util.h compat.h aes.h torint.h tortls.h strlcpy.c strlcat.c torgzip.h container.h ht.h mempool.h memarea.h ciphers.inc @@ -27,4 +27,4 @@ common_sha1.i: $(libor_SOURCES) $(libor_crypto_a_SOURCES) $(noinst_HEADERS) touch common_sha1.i; \ fi -util.o: common_sha1.i +util_codedigest.o: common_sha1.i diff --git a/src/common/util.c b/src/common/util.c index c1b3ea5507..d153df8cf1 100644 --- a/src/common/util.c +++ b/src/common/util.c @@ -2562,11 +2562,3 @@ write_pidfile(char *filename) } } -const char * -libor_get_digests(void) -{ - return "" -#include "common_sha1.i" - ; -} - diff --git a/src/common/util_codedigest.c b/src/common/util_codedigest.c new file mode 100644 index 0000000000..88fe508b92 --- /dev/null +++ b/src/common/util_codedigest.c @@ -0,0 +1,11 @@ + +#include "util.h" + +const char * +libor_get_digests(void) +{ + return "" +#include "common_sha1.i" + ; +} + diff --git a/src/or/Makefile.am b/src/or/Makefile.am index 03e7900944..11923fbd35 100644 --- a/src/or/Makefile.am +++ b/src/or/Makefile.am @@ -20,7 +20,7 @@ tor_SOURCES = buffers.c circuitbuild.c circuitlist.c \ networkstatus.c onion.c policies.c \ reasons.c relay.c rendcommon.c rendclient.c rendmid.c \ rendservice.c rephist.c router.c routerlist.c routerparse.c \ - eventdns.c \ + eventdns.c config_codedigest.c \ tor_main.c AM_CPPFLAGS = -DSHARE_DATADIR="\"$(datadir)\"" \ @@ -42,7 +42,7 @@ test_SOURCES = buffers.c circuitbuild.c circuitlist.c \ networkstatus.c onion.c policies.c \ reasons.c relay.c rendcommon.c rendclient.c rendmid.c \ rendservice.c rephist.c router.c routerlist.c routerparse.c \ - eventdns.c \ + eventdns.c config_codedigest.c \ test_data.c test.c test_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ \ @@ -52,7 +52,7 @@ test_LDADD = ../common/libor.a ../common/libor-crypto.a \ noinst_HEADERS = or.h eventdns.h eventdns_tor.h micro-revision.i -config.o: or_sha1.i +config_codedigest.o: or_sha1.i tor_main.o: micro-revision.i diff --git a/src/or/config.c b/src/or/config.c index 76de52748c..93177a166c 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -3930,9 +3930,7 @@ options_init_from_torrc(int argc, char **argv) if (argc > 1 && (!strcmp(argv[1],"--digests"))) { printf("Tor version %s.\n",get_version()); printf("%s", libor_get_digests()); - printf("%s", "" -#include "or_sha1.i" - ); + printf("%s", tor_get_digests()); exit(0); } diff --git a/src/or/config_codedigest.c b/src/or/config_codedigest.c new file mode 100644 index 0000000000..67fd9847c2 --- /dev/null +++ b/src/or/config_codedigest.c @@ -0,0 +1,10 @@ + +const char *tor_get_digests(void); + +const char * +tor_get_digests(void) +{ + return "" +#include "or_sha1.i" + ; +} diff --git a/src/or/or.h b/src/or/or.h index 854b541d3d..fd75229649 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -2911,6 +2911,8 @@ int options_need_geoip_info(or_options_t *options, const char **reason_out); int getinfo_helper_config(control_connection_t *conn, const char *question, char **answer); +const char *tor_get_digests(void); + #ifdef CONFIG_PRIVATE /* Used only by config.c and test.c */ or_options_t *options_new(void); From 7a2c8e55afc3a83969215da60e3652097b2ba44a Mon Sep 17 00:00:00 2001 From: Marian Date: Fri, 8 May 2009 12:46:43 -0400 Subject: [PATCH 025/187] Clarify some points in what-uses-sha1. --- .../proposals/ideas/xxx-what-uses-sha1.txt | 65 +++++++++++++++---- 1 file changed, 51 insertions(+), 14 deletions(-) diff --git a/doc/spec/proposals/ideas/xxx-what-uses-sha1.txt b/doc/spec/proposals/ideas/xxx-what-uses-sha1.txt index e176bb8f9c..10ada5f33a 100644 --- a/doc/spec/proposals/ideas/xxx-what-uses-sha1.txt +++ b/doc/spec/proposals/ideas/xxx-what-uses-sha1.txt @@ -61,31 +61,49 @@ Why now? Triage How severe are these problems? Let's divide them into these - categories: - PREIMAGE -- A SHA-1 usage that only depends on preimage - resistance + categories, where H(x) is the SHA-1 hash of x: + PREIMAGE -- find any x such that a H(x) has a chosen value + -- A SHA-1 usage that only depends on preimage + resistance + * Also SECOND PREIMAGE. Given x, find a y not equal to + x such that H(x) = H(y) COLLISION -- A SHA-1 usage that depends on collision resistance, but the only party who could mount a collision-based attack is already in a trusted role (like a distribution signer or a directory authority). - COLLISION -- A SHA-1 usage that depends on collision resistance + COLLISION -- find any x and y such that H(x) = H(y) -- A + SHA-1 usage that depends on collision resistance and doesn't need the attacker to have any special keys. - There is no need to put much effort into fixing PREIMAGE usages in - the near-term: even _MD5_ is still preimage resistant. To fix - COLLISION usages is not too important either, since - anyone who has the key to sign the code can mount far worse - attacks. It would be good to fix COLLISION usages, - since we try to resist. The COLLISION usages are the most - important to fix. + There is no need to put much effort into fixing PREIMAGE and + SECOND PREIMAGE usages in the near-term: while SHA-1 is + theoretically broken with regards to those attacks, no practical + attack has been published as far as we know. The difference + between finding any collisions and finding a second preimage is + like the difference between finding any two people with the same + birthday and finding someone with the same birthday as you + personally. To fix COLLISION usages is not too + important either, since anyone who has the key to sign the code + can mount far worse attacks. It would be good to fix + COLLISION usages, since we try to resist bad authorities + to a limited extent. The COLLISION usages are the most important + to fix. + + Kelsey and Schneier published a theoretical second preimage attack + against SHA-1 in 2005, so it would be a good idea to fix PREIMAGE + and SECOND PREIMAGE usages after fixing COLLISION usages or where fixes + require minimal effort. + + http://www.schneier.com/paper-preimages.html Additionally, we need to consider the impact of a successful attack in each of these cases. SHA-1 collisions are still expensive even if recent results are verified, and anybody with the resources to compute one also has the resources to mount a decent Sybil attack. - Let be pessimistic, and not assume that producing collisions of a given - format is actually any harder than producing collisions at all. + Let's be pessimistic, and not assume that producing collisions of + a given format is actually any harder than producing collisions at + all. What Tor uses hashes for today: @@ -96,10 +114,16 @@ What Tor uses hashes for today: COLLSION B. TLS uses SHA-1 (and MD5) internally to generate keys. PREIMAGE? + * At least breaking SHA-1 and MD5 simultaneously is + much more difficult than breaking either + independently. C. Some of the TLS ciphersuites we allow use SHA-1. PREIMAGE? D. When we sign our code with GPG, it might be using SHA-1. COLLISION + * GPG 1.4 and up have writing support for SHA-2 hashes. + This blog has help for converting: + http://www.schwer.us/journal/2005/02/19/sha-1-broken-and-gnupg-gpg/ E. Our GPG keys might be authenticated with SHA-1. COLLISION F. OpenSSL's random number generator uses SHA-1, I believe. @@ -179,6 +203,12 @@ What Tor uses hashes for today: establishment requests. B. Hidden servers use SHA-1 in multiple places when generating hidden service descriptors. + * The permanent-id is the first 80 bits of the SHA-1 hash of the + public key + ** time-period performs caclulations using the permanent-id + * The secret-id-part is the SHA-1 has of the time period, the + descriptor-cookie, and replica. + * Hash of introduction point's identity key. C. Hidden servers performing basic-type client authorization for their services use SHA-1 when encrypting introduction points contained in hidden service descriptors. @@ -187,17 +217,24 @@ What Tor uses hashes for today: identifier or not. E. Hidden servers use SHA-1 to derive .onion addresses of their services. + * What's worse, it only uses the first 80 bits of the SHA-1 hash. + However, the rend-spec.txt says we aren't worried about arbitrary + collisons? F. Clients use SHA-1 to generate the current hidden service descriptor identifiers for a given .onion address. G. Hidden servers use SHA-1 to remember digests of the first parts of Diffie-Hellman handshakes contained in introduction requests in order - to detect replays. + to detect replays. See the RELAY_ESTABLISH_INTRO cell. We seem to be + taking a hash of a hash here. H. Hidden servers use SHA-1 during the Diffie-Hellman key exchange with a connecting client. 5. The bridge protocol XXXX write me + + A. Client may attempt to query for bridges where he knows a digest + (probably SHA-1) before a direct query. 6. The Tor user interface From 183b5905bb58c8ce21cc25d8c97193e699cb767a Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 8 May 2009 12:49:15 -0400 Subject: [PATCH 026/187] Remove some stuff from the SHA-1 paragraph. We don't need to explain the difference between 2nd preimage and collision: anybody who doesn't know can use wikipedia. --- .../proposals/ideas/xxx-what-uses-sha1.txt | 20 ++++++++----------- 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/doc/spec/proposals/ideas/xxx-what-uses-sha1.txt b/doc/spec/proposals/ideas/xxx-what-uses-sha1.txt index 10ada5f33a..b3ca3eea5a 100644 --- a/doc/spec/proposals/ideas/xxx-what-uses-sha1.txt +++ b/doc/spec/proposals/ideas/xxx-what-uses-sha1.txt @@ -75,18 +75,14 @@ Triage SHA-1 usage that depends on collision resistance and doesn't need the attacker to have any special keys. - There is no need to put much effort into fixing PREIMAGE and - SECOND PREIMAGE usages in the near-term: while SHA-1 is - theoretically broken with regards to those attacks, no practical - attack has been published as far as we know. The difference - between finding any collisions and finding a second preimage is - like the difference between finding any two people with the same - birthday and finding someone with the same birthday as you - personally. To fix COLLISION usages is not too - important either, since anyone who has the key to sign the code - can mount far worse attacks. It would be good to fix - COLLISION usages, since we try to resist bad authorities - to a limited extent. The COLLISION usages are the most important + There is no need to put much effort into fixing PREIMAGE and SECOND + PREIMAGE usages in the near-term: while there have been some + theoretical results doing these attacks against SHA-1, they don't + seem to be close to practical yet. To fix COLLISION + usages is not too important either, since anyone who has the key to + sign the code can mount far worse attacks. It would be good to fix + COLLISION usages, since we try to resist bad authorities + to a limited extent. The COLLISION usages are the most important to fix. Kelsey and Schneier published a theoretical second preimage attack From 36c2db2b2e065ea7eab50a748dd8ad023dd46b49 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 8 May 2009 19:09:51 -0400 Subject: [PATCH 027/187] Add TODO.022 items based on discussion with arma --- doc/TODO.022 | 38 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/doc/TODO.022 b/doc/TODO.022 index 3eeae006cb..652fdec101 100644 --- a/doc/TODO.022 +++ b/doc/TODO.022 @@ -8,7 +8,7 @@ NOTE 2: It's easy to list stuff like this with no time estimates and 0.2.2, figure out how long the stuff we want will take, and triage accordingly, or vice versa. -- Design +- Design only - Begin design work for UDP transition; identify areas where we need to make changes or instrument stuff early. @@ -30,14 +30,31 @@ NOTE 2: It's easy to list stuff like this with no time estimates and - Figure out good ways to instrument Tor internals so we can tell how well our bandwidth and flow-control stuff is actually working. + - What ports eat the bandwidth? + - How full do queues get? + - How much latency do queues get? -- Features + - Rate limit at clients: + - Give clients an upper bound on how much they're willing to use + the network if they're not relaying? + - ... or group client circuits by IP at the server and rate-limit + like that. + + + +- Other features - Proposals to implement: - 146: reflect long-term stability - 147: Stop using v2 directories to generate v3 votes. + - Start pinging as soon as we learn about a relay, not on a + 22-minute cycle. Prioritize new and volatile relays for + testing. - Proposals to improve and implement - 158: microdescriptors + - 160: list bandwidth in consensus + - and actually set it reasonably + - and actually use it. - Proposals to improve and implement if not broken - IPv6 support. (Parts of 117, but figure out how to handle DNS @@ -46,6 +63,23 @@ NOTE 2: It's easy to list stuff like this with no time estimates and - 149: learn info from netinfo cells. - 134: handle authority fragmentation (Needs more analysis) + - Needs a proposal, or at least some design + - Detect client-status better + - Have authorities report relay and voting status better: make it + easy to answer, "Why is my server not listed/not Guard/not + Running/etc" + - Have consensuses come in multiple "flavours". + - Weaken the requirements for being a Guard, based on K's + measurements. + - Adaptive timeouts for giving up on circuits and streams. + - Have weight for picking new nodes as guards be less stupid + - Lagged weight updates in consensuses: don't just move abruptly. + d Don't kill a circuit on the first failed extend. + +- Installers + - Switch to MSI on win32 + - Use Thandy, perhaps? + - Deprecations - Make .exit safe, or make it off-by-default. From 6feb149db9715498c6cf4803bfb02b5bcdcf2706 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 12 May 2009 12:50:06 -0400 Subject: [PATCH 028/187] Claim/assign some proposal-writing tasks for 0.2.2. (Proposals assigned to others are purely in the realm of speculation.) --- doc/TODO.022 | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/doc/TODO.022 b/doc/TODO.022 index 652fdec101..a0644f1af8 100644 --- a/doc/TODO.022 +++ b/doc/TODO.022 @@ -12,6 +12,8 @@ NOTE 2: It's easy to list stuff like this with no time estimates and - Begin design work for UDP transition; identify areas where we need to make changes or instrument stuff early. + [multiple weeks, ongoing. Need to do a draft early.] + - Performance, mostly protocol-neutral. - Work with Libevent 2.0's bufferevent interface - Identify any performance stuff we need to push back into @@ -44,7 +46,7 @@ NOTE 2: It's easy to list stuff like this with no time estimates and - Other features - Proposals to implement: - - 146: reflect long-term stability + - 146: reflect long-term stability in consensuses - 147: Stop using v2 directories to generate v3 votes. - Start pinging as soon as we learn about a relay, not on a 22-minute cycle. Prioritize new and volatile relays for @@ -52,7 +54,9 @@ NOTE 2: It's easy to list stuff like this with no time estimates and - Proposals to improve and implement - 158: microdescriptors +N - Revise proposal - 160: list bandwidth in consensus +RNM? - Finish proposal - and actually set it reasonably - and actually use it. @@ -65,15 +69,24 @@ NOTE 2: It's easy to list stuff like this with no time estimates and - Needs a proposal, or at least some design - Detect client-status better +N - Write proposal - Have authorities report relay and voting status better: make it easy to answer, "Why is my server not listed/not Guard/not Running/etc" +N - Write proposal - Have consensuses come in multiple "flavours". +N - Write proposal - Weaken the requirements for being a Guard, based on K's measurements. +K - Finish measurements +K? - Write proposal - Adaptive timeouts for giving up on circuits and streams. - - Have weight for picking new nodes as guards be less stupid +M/N - Revise proposal 151 + - Downweight guards more sensibly: be more forgiving about using + Guard nodes as non-first-hop. + - Write proposal. - Lagged weight updates in consensuses: don't just move abruptly. +M? - Write proposal d Don't kill a circuit on the first failed extend. - Installers From d9650cfa50772d52b7f9ad7a6deeb5b3fdc751f3 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 12 May 2009 15:10:23 -0400 Subject: [PATCH 029/187] Add sentinel values to the end of memarea chunks. This might detect some possible causes of bug 930, and will at least make sure we aren't doing some dumb memory-corruption stuff with the heap and router-parsing. --- ChangeLog | 3 +++ src/common/memarea.c | 37 ++++++++++++++++++++++++++++++++++--- 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/ChangeLog b/ChangeLog index 799e3311fc..eff171db2f 100644 --- a/ChangeLog +++ b/ChangeLog @@ -3,6 +3,9 @@ Changes in version 0.2.2.1-alpha - 2009-??-?? - New --digests command-line switch to output the digests of the source files Tor was built with. - The "torify" script now uses torsocks where available. + - The memarea code now uses a sentinel value at the end of each area + to make sure nothing writes beyond the end of an area. This might + help debug some conceivable causes of bug 930. o Deprecated and removed features: - The controller no longer accepts the old obsolete "addr-mappings/" diff --git a/src/common/memarea.c b/src/common/memarea.c index 7771f2c2ac..8d363fd5ac 100644 --- a/src/common/memarea.c +++ b/src/common/memarea.c @@ -13,6 +13,10 @@ #include "compat.h" #include "log.h" +/** If true, we try to detect any attempts to write beyond the length of a + * memarea. */ +#define USE_SENTINELS + /** All returned pointers should be aligned to the nearest multiple of this * value. */ #define MEMAREA_ALIGN SIZEOF_VOID_P @@ -25,6 +29,24 @@ #error "void* is neither 4 nor 8 bytes long. I don't know how to align stuff." #endif +#ifdef USE_SENTINELS +#define SENTINEL_VAL 0x90806622u +#define SENTINEL_LEN sizeof(uint32_t) +#define SET_SENTINEL(chunk) \ + STMT_BEGIN \ + set_uint32( &(chunk)->u.mem[chunk->mem_size], SENTINEL_VAL ); \ + STMT_END +#define CHECK_SENTINEL(chunk) \ + STMT_BEGIN \ + uint32_t sent_val = get_uint32(&(chunk)->u.mem[chunk->mem_size]); \ + tor_assert(sent_val == SENTINEL_VAL); \ + STMT_END +#else +#define SENTINEL_LEN 0 +#define SET_SENTINEL(chunk) STMT_NIL +#define CHECK_SENTINEL(chunk) STMT_NIL +#endif + /** Increment ptr until it is aligned to MEMAREA_ALIGN. */ static INLINE void * realign_pointer(void *ptr) @@ -78,15 +100,20 @@ alloc_chunk(size_t sz, int freelist_ok) freelist = res->next_chunk; res->next_chunk = NULL; --freelist_len; + CHECK_SENTINEL(res); return res; } else { size_t chunk_size = freelist_ok ? CHUNK_SIZE : sz; - memarea_chunk_t *res = tor_malloc_roundup(&chunk_size); + memarea_chunk_t *res; + chunk_size += SENTINEL_LEN; + res = tor_malloc_roundup(&chunk_size); res->next_chunk = NULL; - res->mem_size = chunk_size - CHUNK_HEADER_SIZE; + res->mem_size = chunk_size - CHUNK_HEADER_SIZE - SENTINEL_LEN; res->next_mem = res->u.mem; - tor_assert(res->next_mem+res->mem_size == ((char*)res)+chunk_size); + tor_assert(res->next_mem+res->mem_size+SENTINEL_LEN == + ((char*)res)+chunk_size); tor_assert(realign_pointer(res->next_mem) == res->next_mem); + SET_SENTINEL(res); return res; } } @@ -96,6 +123,7 @@ alloc_chunk(size_t sz, int freelist_ok) static void chunk_free(memarea_chunk_t *chunk) { + CHECK_SENTINEL(chunk); if (freelist_len < MAX_FREELIST_LEN) { ++freelist_len; chunk->next_chunk = freelist; @@ -182,6 +210,7 @@ memarea_alloc(memarea_t *area, size_t sz) memarea_chunk_t *chunk = area->first; char *result; tor_assert(chunk); + CHECK_SENTINEL(chunk); if (chunk->next_mem+sz > chunk->u.mem+chunk->mem_size) { if (sz+CHUNK_HEADER_SIZE >= CHUNK_SIZE) { /* This allocation is too big. Stick it in a special chunk, and put @@ -255,6 +284,7 @@ memarea_get_stats(memarea_t *area, size_t *allocated_out, size_t *used_out) size_t a = 0, u = 0; memarea_chunk_t *chunk; for (chunk = area->first; chunk; chunk = chunk->next_chunk) { + CHECK_SENTINEL(chunk); a += CHUNK_HEADER_SIZE + chunk->mem_size; tor_assert(chunk->next_mem >= chunk->u.mem); u += CHUNK_HEADER_SIZE + (chunk->next_mem - chunk->u.mem); @@ -271,6 +301,7 @@ memarea_assert_ok(memarea_t *area) tor_assert(area->first); for (chunk = area->first; chunk; chunk = chunk->next_chunk) { + CHECK_SENTINEL(chunk); tor_assert(chunk->next_mem >= chunk->u.mem); tor_assert(chunk->next_mem <= chunk->u.mem+chunk->mem_size+MEMAREA_ALIGN); } From ce768fc06e093da114ddfebfc2400c47115c2c38 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 12 May 2009 23:00:05 -0400 Subject: [PATCH 030/187] Add proposal 161: computing bandwidth adjustments --- doc/spec/proposals/000-index.txt | 2 + .../161-computing-bandwidth-adjustments.txt | 115 ++++++++++++++++++ 2 files changed, 117 insertions(+) create mode 100644 doc/spec/proposals/161-computing-bandwidth-adjustments.txt diff --git a/doc/spec/proposals/000-index.txt b/doc/spec/proposals/000-index.txt index d54d5a2a47..efc418c9a8 100644 --- a/doc/spec/proposals/000-index.txt +++ b/doc/spec/proposals/000-index.txt @@ -81,6 +81,7 @@ Proposals by number: 158 Clients download consensus + microdescriptors [OPEN] 159 Exit Scanning [OPEN] 160 Authorities vote for bandwidth offsets in consensus [OPEN] +161 Computing Bandwidth Adjustments [OPEN] Proposals by status: @@ -103,6 +104,7 @@ Proposals by status: 158 Clients download consensus + microdescriptors 159 Exit Scanning 160 Authorities vote for bandwidth offsets in consensus [for 0.2.2.x] + 161 Computing Bandwidth Adjustments [for 0.2.2.x] ACCEPTED: 110 Avoiding infinite length circuits [for 0.2.1.x] [in 0.2.1.3-alpha] 117 IPv6 exits [for 0.2.1.x] diff --git a/doc/spec/proposals/161-computing-bandwidth-adjustments.txt b/doc/spec/proposals/161-computing-bandwidth-adjustments.txt new file mode 100644 index 0000000000..e64a35bcd9 --- /dev/null +++ b/doc/spec/proposals/161-computing-bandwidth-adjustments.txt @@ -0,0 +1,115 @@ +Title: Computing Bandwidth Adjustments +Filename: 161-computing-bandwidth-adjustments.txt +Author: Mike Perry +Created: 12-May-2009 +Target: 0.2.2.x +Status: Open + + +1. Motivation + + There is high variance in the performance of the Tor network. Despite + our efforts to balance load evenly across the Tor nodes, some nodes are + significantly slower and more overloaded than others. + + Proposal 160 describes how we can augment the directory authorities to + vote on measured bandwidths for routers. This proposal describes what + goes into the measuring process. + + +2. Measurement Selection + + The general idea is to determine a load factor representing the ratio + of the capacity of measured nodes to the rest of the network. This load + factor could be computed from three potentially relevant statistics: + circuit failure rates, circuit extend times, or stream capacity. + + Circuit failure rates and circuit extend times appear to be + non-linearly proportional to node load. We've observed that the same + nodes when scanned at US nighttime hours (when load is presumably + lower) exhibit almost no circuit failure, and significantly faster + extend times than when scanned during the day. + + Stream capacity, however, is much more uniform, even during US + nighttime hours. Moreover, it is a more intuitive representation of + node capacity, and also less dependent upon distance and latency + if amortized over large stream fetches. + + +2. Average Stream Bandwidth Calculation + + The average stream bandwidths are obtained by dividing the network + into 3% slices according to advertised node bandwidth, yielding + about 45 nodes per slice in the current network. + + Two hop circuits are built using nodes from the same slice, and a large + file is downloaded via these circuits. This process is repeated + several hundred times, and average stream capacities are assigned to + each node from these results. + + +3. Ratio Calculation Options + + There are two options for deriving the ratios themselves. They can + be obtained by dividing each nodes' average stream capacity by + either the average for the slice, or the average for the network as a + whole. + + Dividing by the network-wide average has the advantage that it will + account for issues related to unbalancing between higher vs lower + capacity, such as Steven Murdoch's queuing theory weighting result. + + Dividing by the slice average has the advantage that many scans can + be run in parallel from a single authority, and that results are + typically available sooner after a given scan takes place. + + +3. Ratio Filtering + + After the base ratios are calculated, a second pass is performed + to remove any streams with nodes of ratios less than X=0.5 from + the results of other nodes. In addition, all outlying streams + with capacity of one standard deviation below a node's average + are also removed. + + The final ratio result will be calculated as the maximum of + these two resulting ratios if both are less than 1.0, the minimum + if both are greater than 1.0, and the mean if one is greater + and one is less than 1.0. + + +4. Security implications + + The ratio filtering will deal with cases of sabotage by dropping + both very slow outliers in stream average calculations, as well + as dropping streams that used very slow nodes from the calculation + of other nodes. + + This scheme will not address nodes that try to game the system by + providing better service to scanners. The scanners can be detected + at the entry by IP address, and at the exit by the destination fetch. + + Measures can be taken to obfuscate and separate the scanners' source + IP address from the directory authority IP address. For instance, + scans can happen offsite and the results can be rsynced into the + authorities. The destination fetch can also be obscured by using SSL + and periodically changing the large document that is fetched. + + Neither of these methods are foolproof, but such nodes can already + lie about their bandwidth to attract more traffic, so this solution + does not set us back any in that regard. + + +4. Integration with Proposal 160 + + The final results will be produced for the voting mechanism + described in Proposal 160 by multiplying the derived ratio by + the average observed advertised bandwidth during the course of the + scan. This will produce a new bandwidth value that will be + output into a file consisting of lines of the form: + + SP new_bandwidth NL + + This file can be either copied or rsynced into a directory readable + by the directory authority. + From 1b61c8bf377759345e761b0e7bac35c9fbf52f61 Mon Sep 17 00:00:00 2001 From: potentiate Date: Thu, 14 May 2009 05:20:27 -0700 Subject: [PATCH 031/187] Fix misreporting of stream bandwidths. --- src/or/connection.c | 1 + src/or/connection_edge.c | 1 + src/or/control.c | 22 ++++++++++++++++++++++ src/or/or.h | 1 + 4 files changed, 25 insertions(+) diff --git a/src/or/connection.c b/src/or/connection.c index 7b14f7550a..17cfdcba82 100644 --- a/src/or/connection.c +++ b/src/or/connection.c @@ -611,6 +611,7 @@ connection_about_to_close_connection(connection_t *conn) conn->marked_for_close_file, conn->marked_for_close); dnsserv_reject_request(edge_conn); } + control_event_stream_bandwidth(edge_conn); control_event_stream_status(edge_conn, STREAM_EVENT_CLOSED, edge_conn->end_reason); circ = circuit_get_by_edge_conn(edge_conn); diff --git a/src/or/connection_edge.c b/src/or/connection_edge.c index f4585d89a9..eff20bd139 100644 --- a/src/or/connection_edge.c +++ b/src/or/connection_edge.c @@ -160,6 +160,7 @@ connection_edge_destroy(circid_t circ_id, edge_connection_t *conn) "CircID %d: At an edge. Marking connection for close.", circ_id); if (conn->_base.type == CONN_TYPE_AP) { connection_mark_unattached_ap(conn, END_STREAM_REASON_DESTROY); + control_event_stream_bandwidth(conn); control_event_stream_status(conn, STREAM_EVENT_CLOSED, END_STREAM_REASON_DESTROY); conn->end_reason |= END_STREAM_REASON_FLAG_ALREADY_SENT_CLOSED; diff --git a/src/or/control.c b/src/or/control.c index 53b9952481..c5fd688e3d 100644 --- a/src/or/control.c +++ b/src/or/control.c @@ -3258,6 +3258,28 @@ control_event_or_conn_status(or_connection_t *conn, or_conn_status_event_t tp, return 0; } +/** + * Print out STREAM_BW event for a single conn + */ +int +control_event_stream_bandwidth(edge_connection_t *edge_conn) +{ + if (EVENT_IS_INTERESTING(EVENT_STREAM_BANDWIDTH_USED)) { + if (!edge_conn->n_read && !edge_conn->n_written) + return 0; + + send_control_event(EVENT_STREAM_BANDWIDTH_USED, ALL_NAMES, + "650 STREAM_BW "U64_FORMAT" %lu %lu\r\n", + U64_PRINTF_ARG(edge_conn->_base.global_identifier), + (unsigned long)edge_conn->n_read, + (unsigned long)edge_conn->n_written); + + edge_conn->n_written = edge_conn->n_read = 0; + } + + return 0; +} + /** A second or more has elapsed: tell any interested control * connections how much bandwidth streams have used. */ int diff --git a/src/or/or.h b/src/or/or.h index fd75229649..7ac9e80004 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -3220,6 +3220,7 @@ int control_event_stream_status(edge_connection_t *conn, int control_event_or_conn_status(or_connection_t *conn, or_conn_status_event_t e, int reason); int control_event_bandwidth_used(uint32_t n_read, uint32_t n_written); +int control_event_stream_bandwidth(edge_connection_t *edge_conn); int control_event_stream_bandwidth_used(void); void control_event_logmsg(int severity, unsigned int domain, const char *msg); int control_event_descriptors_changed(smartlist_t *routers); From 2a4c6207ad3c9e19187946e39f71926ca3c7929d Mon Sep 17 00:00:00 2001 From: Mike Perry Date: Thu, 14 May 2009 04:57:41 -0700 Subject: [PATCH 032/187] Update proposal 161 to reflect mailinglist discussion. --- .../161-computing-bandwidth-adjustments.txt | 80 +++++++++++++++---- 1 file changed, 65 insertions(+), 15 deletions(-) diff --git a/doc/spec/proposals/161-computing-bandwidth-adjustments.txt b/doc/spec/proposals/161-computing-bandwidth-adjustments.txt index e64a35bcd9..4383a7a121 100644 --- a/doc/spec/proposals/161-computing-bandwidth-adjustments.txt +++ b/doc/spec/proposals/161-computing-bandwidth-adjustments.txt @@ -36,19 +36,27 @@ Status: Open if amortized over large stream fetches. -2. Average Stream Bandwidth Calculation +3. Average Stream Bandwidth Calculation - The average stream bandwidths are obtained by dividing the network - into 3% slices according to advertised node bandwidth, yielding - about 45 nodes per slice in the current network. + The average stream bandwidths are obtained by dividing the network into + slices of 50 nodes each, grouped according to advertised node bandwidth. Two hop circuits are built using nodes from the same slice, and a large - file is downloaded via these circuits. This process is repeated - several hundred times, and average stream capacities are assigned to - each node from these results. + file is downloaded via these circuits. For nodes in the first 15% of the + network, a 500K file will be used. For nodes in the next 15%, a 250K file + will be used. For nodes in next 15%, a 100K file will be used. The + remainder of the nodes will fetch a 75K file.[1] + This process is repeated 250 times, and average stream capacities are + assigned to each node from these results. + + In the future, a node generator type can be created to ensure that + each node is chosen to participate in an equal number of circuits, + and the selection will continue until every live node is chosen + to participate in at least 7 circuits. + -3. Ratio Calculation Options +4. Ratio Calculation Options There are two options for deriving the ratios themselves. They can be obtained by dividing each nodes' average stream capacity by @@ -64,7 +72,7 @@ Status: Open typically available sooner after a given scan takes place. -3. Ratio Filtering +5. Ratio Filtering After the base ratios are calculated, a second pass is performed to remove any streams with nodes of ratios less than X=0.5 from @@ -78,7 +86,41 @@ Status: Open and one is less than 1.0. -4. Security implications +6. Pseudocode for Ratio Calculation Algorithm + + Here is the complete pseudocode for the ratio algorithm: + + Slices = {S | S is 50 nodes of similar consensus capacity} + for S in Slices: + while exists node N in S with circ_chosen(N) < 7: + fetch_slice_file(build_2hop_circuit(N, (exit in S))) + for N in S: + BW_measured(N) = MEAN(b | b is bandwidth of a stream through N) + Bw_stddev(N) = STDDEV(b | b is bandwidth of a stream through N) + Bw_avg(S) = MEAN(b | b = BW_measured(N) for all N in S) + Normal_Routers(S) = {N | Bw_measured(N)/Bw_avg(S) > 0.5 } + for N in S: + Normal_Streams(N) = + {stream via N | all nodes in stream not in {Normal_Routers(S)-N} + and bandwidth > BW_measured(N)-Bw_stddev(N)} + BW_Norm_measured(N) = MEAN(b | b is a bandwidth of Normal_Streams(N)) + + Bw_net_avg(Slices) = MEAN(BW_measured(N) for all N in Slices) + Bw_Norm_net_avg(Slices) = MEAN(BW_Norm_measured(N) for all N in Slices) + + for N in all Slices: + Bw_net_ratio(N) = Bw_measured(N)/Bw_net_avg(Slices) + Bw_Norm_net_ratio(N) = Bw_measured2(N)/Bw_Norm_net_avg(Slices) + + if Bw_net_ratio(N) < 1.0 and Bw_Norm_net_ratio(N) < 1.0: + ResultRatio(N) = MAX(Bw_net_ratio(N), Bw_Norm_net_ratio(N)) + else if Bw_net_ratio(N) > 1.0 and Bw_Norm_net_ratio(N) > 1.0: + ResultRatio(N) = MIN(Bw_net_ratio(N), Bw_Norm_net_ratio(N)) + else: + ResultRatio(N) = MEAN(Bw_net_ratio(N), Bw_Norm_net_ratio(N)) + + +7. Security implications The ratio filtering will deal with cases of sabotage by dropping both very slow outliers in stream average calculations, as well @@ -100,16 +142,24 @@ Status: Open does not set us back any in that regard. -4. Integration with Proposal 160 +8. Integration with Proposal 160 The final results will be produced for the voting mechanism described in Proposal 160 by multiplying the derived ratio by - the average observed advertised bandwidth during the course of the - scan. This will produce a new bandwidth value that will be - output into a file consisting of lines of the form: + the average published consensus bandwidth during the course of the + scan, and taking the weighted average with the previous consensus + bandwidth: - SP new_bandwidth NL + Bw_new = (Bw_current * Alpha + Bw_scan_avg*Bw_ratio)/(Alpha + 1) + The Alpha parameter is a smoothing parameter intended to prevent + rapid oscillation between loaded and unloaded conditions. + + This will produce a new bandwidth value that will be output into a + file consisting of lines of the form: + + node_id= SP bw= NL + This file can be either copied or rsynced into a directory readable by the directory authority. From dc1253af7bcf8ddce19933cc3d6a284015fcffbe Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 15 May 2009 13:03:42 -0400 Subject: [PATCH 033/187] Add new proposal 162: "Publish the consensus in multiple flavors" --- doc/spec/proposals/000-index.txt | 2 + doc/spec/proposals/162-consensus-flavors.txt | 175 +++++++++++++++++++ 2 files changed, 177 insertions(+) create mode 100644 doc/spec/proposals/162-consensus-flavors.txt diff --git a/doc/spec/proposals/000-index.txt b/doc/spec/proposals/000-index.txt index efc418c9a8..ab681acc46 100644 --- a/doc/spec/proposals/000-index.txt +++ b/doc/spec/proposals/000-index.txt @@ -82,6 +82,7 @@ Proposals by number: 159 Exit Scanning [OPEN] 160 Authorities vote for bandwidth offsets in consensus [OPEN] 161 Computing Bandwidth Adjustments [OPEN] +162 Publish the consensus in multiple flavors [OPEN] Proposals by status: @@ -105,6 +106,7 @@ Proposals by status: 159 Exit Scanning 160 Authorities vote for bandwidth offsets in consensus [for 0.2.2.x] 161 Computing Bandwidth Adjustments [for 0.2.2.x] + 162 Publish the consensus in multiple flavors [for 0.2.2] ACCEPTED: 110 Avoiding infinite length circuits [for 0.2.1.x] [in 0.2.1.3-alpha] 117 IPv6 exits [for 0.2.1.x] diff --git a/doc/spec/proposals/162-consensus-flavors.txt b/doc/spec/proposals/162-consensus-flavors.txt new file mode 100644 index 0000000000..2765b05b68 --- /dev/null +++ b/doc/spec/proposals/162-consensus-flavors.txt @@ -0,0 +1,175 @@ +Filename: 162-consensus-flavors.txt +Title: Publish the consensus in multiple flavors +Author: Nick Mathewson +Created: 14-May-2009 +Target: 0.2.2 +Status: Open + + +Overview: + + This proposal describes a way to publish each consensus in + multiple simultaneous formats, or "flavors". This will reduce the + amount of time needed to deploy new consensus-like documents, and + reduce the size of consensus documents in the long term. + +Motivation: + + In the future, we will almost surely want different fields and + data in the network-status document. Examples include: + - Publishing hashes of microdescriptors instead of hashes of + full descriptors (Proposal 158). + - Including different digests of descriptors, instead of the + perhaps-soon-to-be-totally-broken SHA1. + + Note that in both cases, from the client's point of view, this + information _replaces_ older information. If we're using a + SHA256 hash, we don't need to see the SHA1. If clients only want + microdescriptors, they don't (necessarily) need to see hashes of + other things. + + Our past approach to cases like this has been to shovel all of + the data into the consensus document. But this is rather poor + for bandwidth. Adding a single SHA256 hash to a consensus for + each router increases the compressed consensus size by 47%. In + comparison, replacing a single SHA1 hash with a SHA256 hash for + each listed router increases the consensus size by only 18%. + +Design in brief: + + Let the voting process will remain as it is, until a consensus is + generated. With future versions of the voting algorithm, instead + of just a single consensus being generated, multiple consensus + "flavors" are produced. + + Consensuses (all of them) include a list of which flavors are + being generated. Caches fetch and serve all flavors of consensus + that are listed, regardless of whether they can parse or validate + them, and serve them to clients. Thus, once this design is in + place, we won't need to deploy more cache changes in order to get + new flavors of consensus to be cached. + + Clients download only the consensus flavor they want. + +A note on hashes: + + Everything in this document is specified to use SHA256, and to be + upgradeable to use better hashes in the future. + +Spec modifications: + + 1. URLs and changes to the current consensus format. + + Every consensus flavor has a name consisting of a sequence of one + or more alphanumeric characters and dashes. For compatibility + current descriptor flavor is called "ns". + + The supported consensus flavors are defined as part of the + authorities' consensus method. + + For each supported flavors, every authority calculates another + consensus document of as-yet-unspecified format, and exchange + detached signatures for these documents as in the current consensus + design. + + In addition to the consensus currently served at + /tor/status-vote/(current|next)/consensus.z , authorities serve + another consensus of each flavor "F" from the location + /tor/status-vote/(current|next)/F/consensus.z. + + When caches serve these documents, they do so from the same + locations. + + 2. Document format: generic consensus. + + The format of a flavored consensus is as-yet-unspecified, except + that the first line is: + "network-status-version" SP version SP flavor NL + + where version is 3 or higher, and the flavor is a string + consisting of alphanumeric characters and dashes, matching the + corresponding flavor listed in the unflavored consensus. + + 3. Document format: detached signatures. + + In addition to the current detached signature format, we allow + the first line to take the form, + "consensus-digest" SP flavor SP 1*(Algname "=" Digest) NL + + The consensus-signatures URL should contain the signatures + for _all_ flavors of consensus. + + 4. The consensus index: + + Authorities additionally generate and serve a consensus-index + document. Its format is: + + Header ValidAfter ValidUntil Documents Signatures + + Header = "consensus-index" SP version NL + ValidAfter = as in a consensus + ValidUntil = as in a consensus + Documents = Document* + Document = "document" SP flavor SP SignedLength + 1*(SP AlgorithmName "=" Digest) NL + Signatures = Signature * + Signature = "directory-signature" SP algname SP identity + SP signing-key-digest NL signature + + There must be one Document line for each generated consensus flavor + Each Document line describes the length of the signed portion of + a consensus (the signatures themselves are not included), along + with one or more digests of that signed portion. Digests are + given in hex. The algorithm "sha256" MUST be included; others + are allowed. + + The algname part of a signature describes what algorithm was + used to hash the identity and signing keys, and to compute the + signature. The algorithm "sha256" MUST be recognized; + signatures with unrecognized algorithms MUST be ignored. + (See below). + + The consensus index is made available at + /tor/status-vote/(current|next)/consensus-index.z. + + Caches should fetch this document so they can check the + correctness of the different consensus documents they fetch. + They do not need to check anything about an unrecognized + consensus document beyond its digest. + + 4.1. The "sha256" signature format. + + The 'SHA256' signature format for directory objects is defined as + the RSA signature of the OAEP+-padded SHA256 digest of the SHA256 + digest of the the item to be signed. When checking signatures, + the signature MUST be treated as valid if the signed material + begins with SHA256(SHA256(document)); this allows us to add other + data later. + +Considerations: + + - We should not create a new flavor of consensus when adding a + field wouldn't be too onerous. + + - We should not proliferate flavors lightly: clients will be + distinguishable based on which flavor they download. + +Migration: + + - Stage one: authorities begin generating and serving + consensus-index files. + + - Stage two: Caches begin downloading consenusus-index files, + validating them, and using them to decide what flavors of + consensus documents to cache. They download all listed + documents, and compare them to the digests given in the + consensus. + + - Stage three: Once we want to make a significant change to the + consensus format, we deploy another flavor of consensus at the + authorities. This will immediately start getting cached by the + caches, and clients can start fetching the new flavor without + waiting a version or two for enough caches to begin supporting + it. + + From 143e6677ff04e374822277b7dc4f6e23302ad9cd Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 15 May 2009 16:32:18 -0400 Subject: [PATCH 034/187] Add acks section to prop 162. --- doc/spec/proposals/162-consensus-flavors.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/spec/proposals/162-consensus-flavors.txt b/doc/spec/proposals/162-consensus-flavors.txt index 2765b05b68..80fee1e688 100644 --- a/doc/spec/proposals/162-consensus-flavors.txt +++ b/doc/spec/proposals/162-consensus-flavors.txt @@ -172,4 +172,7 @@ Migration: waiting a version or two for enough caches to begin supporting it. +Acknowledgements: + Aspects of this design and its applications to hash migration were + heavily influenced by IRC conversations with Marian. From 573aeb769ed9e624d451ffbb98b10d20c7d6f6f2 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Sat, 16 May 2009 00:42:27 -0400 Subject: [PATCH 035/187] Major revision of proposal 158. The big changes are to go from a "caches compute the micro-descriptor" format to an "authorities generate microdescriptors" format. See or-dev discussions of January 2009 for full rationales. --- doc/spec/proposals/158-microdescriptors.txt | 155 +++++++++----------- 1 file changed, 68 insertions(+), 87 deletions(-) diff --git a/doc/spec/proposals/158-microdescriptors.txt b/doc/spec/proposals/158-microdescriptors.txt index 3b9a690a2b..59c14c3bcc 100644 --- a/doc/spec/proposals/158-microdescriptors.txt +++ b/doc/spec/proposals/158-microdescriptors.txt @@ -4,6 +4,15 @@ Author: Roger Dingledine Created: 17-Jan-2009 Status: Open +0. History + + 15 May 2009: Substantially revised based on discussions on or-dev + from late January. Removed the notion of voting on how to choose + microdescriptors; made it just a function of the consesus method. + (This lets us avoid the possibility of "desynchronization.") + Added suggestion to use a new consensus flavor. Specified use of + SHA256 for new hashes. -nickm + 1. Overview This proposal replaces section 3.2 of proposal 141, which was @@ -22,6 +31,10 @@ Status: Open them, we'll need to resume considering some design like the one in proposal 141. + Note also that any descriptor element which clients need to use to + decide which servers to fetch info about, or which servers to fetch + info from, needs to stay in the consensus. + 2. Motivation See @@ -34,89 +47,69 @@ Status: Open 3. Design There are three pieces to the proposal. First, authorities will list in - their votes (and thus in the consensus) what relay descriptor elements - are included in the microdescriptor, and also list the expected hash + their votes (and thus in the consensus) the expected hash of microdescriptor for each relay. Second, directory mirrors will serve microdescriptors. Third, clients will ask for them and cache them. 3.1. Consensus changes - V3 votes should include a new line: - microdescriptor-elements bar baz foo - listing each descriptor element (sorted alphabetically) that authority - included when it calculated its expected microdescriptor hashes. + If the authorities choose a consensus method of a given version or + later, a microdescriptor format is implicit in that version. + A microdescriptor should in every case be a pure function of the + router descriptor and the conensus method. - We also need to include the hash of each expected microdescriptor in + In votes, need to include the hash of each expected microdescriptor in the routerstatus section. I suggest a new "m" line for each stanza, - with the base64 of the hash of the elements that the authority voted - for above. + with the base64 of the SHA256 hash of the router's microdescriptor. + + For every consensus method that an authority supports, it includes a + separate "m" line in each router section of its vote, containing: + "m" SP methods SP digest NL + where methods is a comma-separated list of the consensus methods + that the authority believes will produce "digest". + + (As with base64 encoding of SHA1 hashes in consensuses, let's + omit the trailing =s) The consensus microdescriptor-elements and "m" lines are then computed as described in Section 3.1.2 below. - I believe that means we need a new consensus-method "6" that knows - how to compute the microdescriptor-elements and add "m" lines. + (This means we need a new consensus-method that knows + how to compute the microdescriptor-elements and add "m" lines.) 3.1.1. Descriptor elements to include for now - To start, the element list that authorities suggest should be - family onion-key - - (Note that the or-dev posts above only mention onion-key, but if - we don't also include family then clients will never learn it. It - seemed like it should be relatively static, so putting it in the - microdescriptor is smarter than trying to fit it into the consensus.) - - We could imagine a config option "family,onion-key" so authorities - could change their voted preferences without needing to upgrade. + In the first version, the microdescriptor should contain the + onion-key element and the family element from the router descriptor. 3.1.2. Computing consensus for microdescriptor-elements and "m" lines - One approach is for the consensus microdescriptor-elements line to - include every element listed by a majority of authorities, sorted. The - problem here is that it will no longer be deterministic what the correct - hash for the "m" line should be. We could imagine telling the authority - to go look in its descriptor and produce the right hash itself, but - we don't want consensus calculation to be based on external data like - that. (Plus, the authority may not have the descriptor that everybody - else voted to use.) + When we generating a consensus, we use whichever m line + unambiguously corresponds to the descriptor digest that will be + included in the consensus. (If there are multiple m lines for that + descriptor digest, we use whichever is most common. If they are + equally common, we break ties in the favor of the lexically + earliest. Either way, we should log a warning: That's likely a + bug.) - The better approach is to take the exact set that has the most votes - (breaking ties by the set that has the most elements, and breaking - ties after that by whichever is alphabetically first). That will - increase the odds that we actually get a microdescriptor hash that - is both a) for the descriptor we're putting in the consensus, and b) - over the elements that we're declaring it should be for. + The "m" lines in a consensus contain only the digest, not a list of + consensus methods. - Then the "m" line for a given relay is the one that gets the most votes - from authorities that both a) voted for the microdescriptor-elements - line we're using, and b) voted for the descriptor we're using. +3.1.3. A new flavor of consensus - (If there's a tie, use the smaller hash. But really, if there are - multiple such votes and they differ about a microdescriptor, we caught - one of them lying or being buggy. We should log it to track down why.) + Rather than inserting "m" lines in the current consensus format, + they should be included in a new consensus flavor (see proposal + 162). - If there are no such votes, then we leave out the "m" line for that - relay. That means clients should avoid it for this time period. (As - an extension it could instead mean that clients should fetch the - descriptor and figure out its microdescriptor themselves. But let's - not get ahead of ourselves.) + This flavor can safely omit descriptor digests. - It would be nice to have a more foolproof way to agree on what - microdescriptor hash each authority should vote for, so we can avoid - missing "m" lines. Just switching to a new consensus-method each time - we change the set of microdescriptor-elements won't help though, since - each authority will still have to decide what hash to vote for before - knowing what consensus-method will be used. + We still need to descide whether to move ports into microdescriptors + or not. In either case, they can be removed from the current "ns" + flavor of consensus, since no current clients use them, and they + take up about 5% of the compressed consensus. - Here's one way we could do it. Each vote / consensus includes - the microdescriptor-elements that were used to compute the hashes, - and also a preferred-microdescriptor-elements set. If an authority - has a consensus from the previous period, then it should use the - consensus preferred-microdescriptor-elements when computing its votes - for microdescriptor-elements and the appropriate hashes in the upcoming - period. (If it has no previous consensus, then it just writes its - own preferences in both lines.) + This new consensus flavor should be signed with the sha256 signature + format as documented in proposal 162. 3.2. Directory mirrors serve microdescriptors @@ -125,8 +118,10 @@ Status: Open continue to serve normal relay descriptors too, a) to serve old clients and b) to be able to construct microdescriptors on the fly.) - The microdescriptors with hashes ,, should be available at: - http:///tor/micro/d/++.z + The microdescriptors with base64 hashes ,, should be available at: + http:///tor/micro/d/--.z + (We use base64 for size and for consistency with the consensus + format. We use -s instead of +s to separate these items, since All the microdescriptors from the current consensus should also be available at: @@ -134,24 +129,9 @@ Status: Open so a client that's bootstrapping doesn't need to send a 70KB URL just to name every microdescriptor it's looking for. - The format of a microdescriptor is the header line - "microdescriptor-header" - followed by each element (keyword and body), alphabetically. There's - no need to mention what hash it's for, since it's self-identifying: - you can hash the elements to learn this. - - (Do we need a footer line to show that it's over, or is the next - microdescriptor line or EOF enough of a hint? A footer line wouldn't - hurt much. Also, no fair voting for the microdescriptor-element - "microdescriptor-header".) - + Microdescriptors have no header or footer. The hash of the microdescriptor is simply the hash of the concatenated - elements -- not counting the header line or hypothetical footer line. - Unless you prefer that? - - Is there a reasonable way to version these things? We could say that - the microdescriptor-header line can contain arguments which clients - must ignore if they don't understand them. Any better ways? + elements. Directory mirrors should check to make sure that the microdescriptors they're about to serve match the right hashes (either the hashes from @@ -168,10 +148,14 @@ Status: Open When a client gets a new consensus, it looks to see if there are any microdescriptors it needs to learn. If it needs to learn more than some threshold of the microdescriptors (half?), it requests 'all', - else it requests only the missing ones. + else it requests only the missing ones. Clients MAY try to + determine whether the upload bandwidth for listing the + microdescriptors they want is more or less than the download + bandwidth for the microdescriptors they do not want. Clients maintain a cache of microdescriptors along with metadata like - when it was last referenced by a consensus. They keep a microdescriptor + when it was last referenced by a consensus, and which identity key + it corresponds to. They keep a microdescriptor until it hasn't been mentioned in any consensus for a week. Future clients might cache them for longer or shorter times. @@ -192,14 +176,11 @@ Status: Open Phase one, the directory authorities should start voting on microdescriptors and microdescriptor elements, and putting them in the - consensus. This should happen during the 0.2.1.x series, and should - be relatively easy to do. + consensus. Phase two, directory mirrors should learn how to serve them, and learn - how to read the consensus to find out what they should be serving. This - phase could be done either in 0.2.1.x or early in 0.2.2.x, depending - on how messy it turns out to be and how quickly we get around to it. + how to read the consensus to find out what they should be serving. Phase three, clients should start fetching and caching them instead - of normal descriptors. This should happen post 0.2.1.x. + of normal descriptors. From 2619e3594258054f039617c517ad87a29358db4e Mon Sep 17 00:00:00 2001 From: Marian Date: Sat, 16 May 2009 11:57:26 -0400 Subject: [PATCH 036/187] New proposal draft about migrating ciphers and hashes in the Tor protocol. --- .../xxx-choosing-crypto-in-tor-protocol.txt | 138 ++++++++++++++++++ 1 file changed, 138 insertions(+) create mode 100644 doc/spec/proposals/ideas/xxx-choosing-crypto-in-tor-protocol.txt diff --git a/doc/spec/proposals/ideas/xxx-choosing-crypto-in-tor-protocol.txt b/doc/spec/proposals/ideas/xxx-choosing-crypto-in-tor-protocol.txt new file mode 100644 index 0000000000..e8489570f7 --- /dev/null +++ b/doc/spec/proposals/ideas/xxx-choosing-crypto-in-tor-protocol.txt @@ -0,0 +1,138 @@ +Filename: xxx-choosing-crypto-in-tor-protocol.txt +Title: Picking cryptographic standards in the Tor wire protocol +Author: Marian +Created: 2009-05-16 +Status: Draft + +Motivation: + + SHA-1 is horribly outdated and not suited for security critical + purposes. SHA-2, RIPEMD-160, Whirlpool and Tigerare good options + for a short-term replacement, but in the long run, we will + probably want to upgrade to the winner or a semi-finalist of the + SHA-3 competition. + + For a 2006 comparison of different hash algorithms, read: + http://www.sane.nl/sane2006/program/final-papers/R10.pdf + + Other reading about SHA-1: + http://www.schneier.com/blog/archives/2005/02/sha1_broken.html + http://www.schneier.com/blog/archives/2005/08/new_cryptanalyt.html + http://www.schneier.com/paper-preimages.html + + Additionally, AES has been theoretically broken for years. While + the attack is still not efficient enough that the public sector + has been able to prove that it works, we should probably consider + the time between a theoretical attack and a practical attack as an + opportunity to figure out how to upgrade to a better algorithm, + such as Twofish. + + See: + http://schneier.com/crypto-gram-0209.html#1 + +Design: + + I suggest that nodes should publish in directories which + cryptographic standards, such as hash algorithms and ciphers, + they support. Clients communicating with nodes will then + pick whichever of those cryptographic standards they prefer + the most. In the case that the node does not publish which + cryptographic standards it supports, the client should assume + that the server supports the older standards, such as SHA-1 + and AES, until such time as we choose to desupport those + standards. + + Node to node communications could work similarly. However, in + case they both support a set of algorithms but have different + preferences, the disagreement would have to be resolved + somehow. Two possibilities include: + * the node requesting communications presents which + cryptographic standards it supports in the request. The + other node picks. + * both nodes send each other lists of what they support and + what version of Tor they are using. The newer node picks, + based on the assumption that the newer node has the most up + to date information about which hash algorithm is the best. + Of course, the node could lie about its version, but then + again, it could also maliciously choose only to support older + algorithms. + + Using this method, we could potentially add server side support + to hash algorithms and ciphers before we instruct clients to + begin preferring those hash algorithms and ciphers. In this way, + the clients could upgrade and the servers would already support + the newly preferred hash algorithms and ciphers, even if the + servers were still using older versions of Tor, so long as the + older versions of Tor were at least new enough to have server + side support. + + This would make quickly upgrading to new hash algorithms and + ciphers easier. This could be very useful when new attacks + are published. + + One concern is that client preferences could expose the client + to segmentation attacks. To mitigate this, we suggest hardcoding + preferences in the client, to prevent the client from choosing + to use a new hash algorithm or cipher that no one else is using + yet. While offering a preference might be useful in case a client + with an older version of Tor wants to start using the newer hash + algorithm or cipher that everyone else is using, if the client + cares enough, he or she can just upgrade Tor. + + We may also have to worry about nodes which, through laziness or + maliciousness, refuse to start supporting new hash algorithms or + ciphers. This must be balanced with the need to maintain + backward compatibility so the client will have a large selection + of nodes to pick from. Adding new hash algorithms and ciphers + long before we suggest nodes start using them can help mitigate + this. However, eventually, once sufficient nodes support new + standards, client side support for older standards should be + disabled, particularly if there are practical rather than merely + theoretical attacks. + + Server side support for older standards can be kept much longer + than client side support, since clients using older hashes and + ciphers are really only hurting theirselvse. + + If server side support for a hash algorithm or cipher is added + but never preferred before we decide we don't really want it, + support can be removed without having to worry about backward + compatibility. + +Security implications: + Improving cryptography will improve Tor's security. However, if + clients pick different cryptographic standards, they could be + partitioned based on their cryptographic preferences. We also + need to worry about nodes refusing to support new standards. + These issues are detailed above. + +Specification: + + Todo. Need better understanding of how Tor currently works or + help from someone who does. + +Compatibility: + + This idea is intended to allow easier upgrading of cryptographic + hash algorithms and ciphers while maintaining backwards + compatibility. However, at some point, backwards compatibility + with very old hashes and ciphers should be dropped for security + reasons. + +Implementation: + + Todo. + +Performance and scalability nodes: + + Better hashes and cipher are someimes a little more CPU intensive + than weaker ones. For instance, on most computers AES is a little + faster than Twofish. However, in that example, I consider Twofish's + additional security worth the tradeoff. + +Acknowledgements: + + Discussed this on IRC with a few people, mostly Nick Mathewson. + Nick was particularly helpful in explaining how Tor works, + explaining goals, and providing various links to Tor + specifications. From 095c26f63e47624e80a45a9ae7790f7045188f22 Mon Sep 17 00:00:00 2001 From: Roger Dingledine Date: Sun, 17 May 2009 00:51:23 -0400 Subject: [PATCH 037/187] minor tweak on changelog entries --- ChangeLog | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index 4dbe2193cf..b84a3d947a 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,7 +1,7 @@ Changes in version 0.2.2.1-alpha - 2009-??-?? o Minor features - - New --digests command-line switch to output the digests of the source - files Tor was built with. + - New --digests command-line switch to output the digests of the + source files Tor was built with. - The "torify" script now uses torsocks where available. - The memarea code now uses a sentinel value at the end of each area to make sure nothing writes beyond the end of an area. This might From 7d84b84608f7425208e60020dcede1561e53a170 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 19 May 2009 10:40:20 -0400 Subject: [PATCH 038/187] Clearer valgrind instructions for non-Debian openssls --- doc/HACKING | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/doc/HACKING b/doc/HACKING index 50b5d80d18..3d3f2c1dfc 100644 --- a/doc/HACKING +++ b/doc/HACKING @@ -11,12 +11,20 @@ 0.1. Useful command-lines that are non-trivial to reproduce but can help with tracking bugs or leaks. +0.1.1. Dmalloc + dmalloc -l ~/dmalloc.log (run the commands it tells you) ./configure --with-dmalloc +0.2.2. Valgrind + valgrind --leak-check=yes --error-limit=no --show-reachable=yes src/or/tor +(Note that if you get a zillion openssl warnings, you will also need to + pass --undef-value-errors=no to valgrind, or rebuild your openssl + with -DPURIFY.) + 0.2. Running gcov for unit test coverage make clean From f55a70b6405f133ade42738ed0357f602b3207f8 Mon Sep 17 00:00:00 2001 From: Mike Perry Date: Tue, 19 May 2009 21:24:18 -0700 Subject: [PATCH 039/187] Update proposal 160 with comments from mailinglist. Also add implementation details and a timestampt to the output of 161. --- doc/spec/proposals/160-bandwidth-offset.txt | 44 ++++++++++++++----- .../161-computing-bandwidth-adjustments.txt | 37 +++++++++++++--- 2 files changed, 64 insertions(+), 17 deletions(-) diff --git a/doc/spec/proposals/160-bandwidth-offset.txt b/doc/spec/proposals/160-bandwidth-offset.txt index f618d0eeb2..6d04f916e7 100644 --- a/doc/spec/proposals/160-bandwidth-offset.txt +++ b/doc/spec/proposals/160-bandwidth-offset.txt @@ -37,10 +37,8 @@ Target: 0.2.2.x the number of authorities we have. The better fix is to allow certain authorities to specify that they are - voting on bandwidth "offsets": how much they think the weight should - be changed for the relay in question. We should put the offset vote in - the stanza for the relay in question, so a given authority can choose - which relays to express preferences for and which not. + voting on bandwidth measurements: more accurate bandwidth values that + have actually been evaluated. In this way, authorities 3. Security implications @@ -64,15 +62,41 @@ Target: 0.2.2.x First, we need a new consensus method to support this new calculation. - Now v3 votes can have a new weight on the "w" line: - "Bandwidth_Offset=" INT. + Now v3 votes can have an additional value on the "w" line: + "w Bandwidth=X Measured=" INT. + Once we're using the new consensus method, the new way to compute the - Bandwidth weight is by taking the old vote (explained in proposal 141: - median, then choose the lower number in the case of ties), and adding - or subtracting the median offset (using the offset closer to 0 in the - case of ties, and with a sum of 0 if the sum is negative). + Bandwidth weight is by checking if there are at least 3 "Measured" + votes. If so, the median of these is taken. Otherwise, the median + of the "Bandwidth=" values are taken, as described in Proposal 141. Then the actual consensus looks just the same as it did before, so clients never have to know that this additional calculation is happening. +5. Implementation + + The Measured values will be read from a file provided by the scanners + described in proposal 161. Files with a timestamp older than 3 days + will be ignored. + + The file will be read in from dirserv_generate_networkstatus_vote_obj() + in a location specified by a new config option "V3MeasuredBandwidths". + A helper function will be called to populate new 'measured' and + 'has_measured' fields of the routerstatus_t 'routerstatuses' list with + values read from this file. + + An additional for_vote flag will be passed to + routerstatus_format_entry() from format_networkstatus_vote(), which will + indicate that the "Measured=" string should be appended to the "w Bandwith=" + line with the measured value in the struct. + + routerstatus_parse_entry_from_string() will be modified to parse the + "Measured=" lines into routerstatus_t struct fields. + + Finally, networkstatus_compute_consensus() will set rs_out.bandwidth + to the median of the measured values if there are more than 3, otherwise + it will use the bandwidth value median as normal. + + + diff --git a/doc/spec/proposals/161-computing-bandwidth-adjustments.txt b/doc/spec/proposals/161-computing-bandwidth-adjustments.txt index 4383a7a121..b02dc64eb2 100644 --- a/doc/spec/proposals/161-computing-bandwidth-adjustments.txt +++ b/doc/spec/proposals/161-computing-bandwidth-adjustments.txt @@ -66,10 +66,7 @@ Status: Open Dividing by the network-wide average has the advantage that it will account for issues related to unbalancing between higher vs lower capacity, such as Steven Murdoch's queuing theory weighting result. - - Dividing by the slice average has the advantage that many scans can - be run in parallel from a single authority, and that results are - typically available sooner after a given scan takes place. + For this reason, we will opt for network-wide averages. 5. Ratio Filtering @@ -142,7 +139,26 @@ Status: Open does not set us back any in that regard. -8. Integration with Proposal 160 +8. Parallelization + + Because each slice takes as long as 6 hours to complete, we will want + to parallelize as much as possible. This will be done by concurrently + running multiple scanners from each authority to deal with different + segments of the network. Each scanner piece will continually loop + over a portion of the network, outputting files of the form: + + node_id= SP strm_bw= SP + filt_bw= NL + + The most recent file from each scanner will be periodically gathered + by another script that uses them to produce network-wide averages + and calculate ratios as per the algorithm in section 6. Because nodes + may shift in capacity, they may appear in more than one slice and/or + appear more than once in the file set. The line that yields a ratio + closest to 1.0 will be chosen in this case. + + +9. Integration with Proposal 160 The final results will be produced for the voting mechanism described in Proposal 160 by multiplying the derived ratio by @@ -158,8 +174,15 @@ Status: Open This will produce a new bandwidth value that will be output into a file consisting of lines of the form: - node_id= SP bw= NL - + node_id= SP bw= NL + + The first line of the file will contain a timestamp in UNIX time() + seconds. This will be used by the authority to decide if the + measured values are too old to use. + This file can be either copied or rsynced into a directory readable by the directory authority. + +1. Exact values for each segment are still being determined via +test scans. From 4e1b20b1d3761d3d9199287ea5697b6ea6f7d088 Mon Sep 17 00:00:00 2001 From: Mike Perry Date: Thu, 21 May 2009 21:38:50 -0700 Subject: [PATCH 040/187] Finish incomplete sentence. --- doc/spec/proposals/160-bandwidth-offset.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/doc/spec/proposals/160-bandwidth-offset.txt b/doc/spec/proposals/160-bandwidth-offset.txt index 6d04f916e7..7ca74dfae3 100644 --- a/doc/spec/proposals/160-bandwidth-offset.txt +++ b/doc/spec/proposals/160-bandwidth-offset.txt @@ -38,7 +38,10 @@ Target: 0.2.2.x The better fix is to allow certain authorities to specify that they are voting on bandwidth measurements: more accurate bandwidth values that - have actually been evaluated. In this way, authorities + have actually been evaluated. In this way, authorities can vote on + the median measured value if sufficient measured votes exist for a router, + and otherwise fall back to the median value taken from the published router + descriptors. 3. Security implications From 047bc09565eff6a699b4ce0a608b6ecec1de2c8f Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 22 May 2009 02:57:54 -0400 Subject: [PATCH 041/187] New short proposal about detecting if a node is a client. --- doc/spec/proposals/000-index.txt | 2 + doc/spec/proposals/163-detecting-clients.txt | 115 +++++++++++++++++++ 2 files changed, 117 insertions(+) create mode 100644 doc/spec/proposals/163-detecting-clients.txt diff --git a/doc/spec/proposals/000-index.txt b/doc/spec/proposals/000-index.txt index ab681acc46..8c8b0ab5d4 100644 --- a/doc/spec/proposals/000-index.txt +++ b/doc/spec/proposals/000-index.txt @@ -83,6 +83,7 @@ Proposals by number: 160 Authorities vote for bandwidth offsets in consensus [OPEN] 161 Computing Bandwidth Adjustments [OPEN] 162 Publish the consensus in multiple flavors [OPEN] +163 Detecting whether a connection comes from a client [OPEN] Proposals by status: @@ -107,6 +108,7 @@ Proposals by status: 160 Authorities vote for bandwidth offsets in consensus [for 0.2.2.x] 161 Computing Bandwidth Adjustments [for 0.2.2.x] 162 Publish the consensus in multiple flavors [for 0.2.2] + 163 Detecting whether a connection comes from a client [for 0.2.2] ACCEPTED: 110 Avoiding infinite length circuits [for 0.2.1.x] [in 0.2.1.3-alpha] 117 IPv6 exits [for 0.2.1.x] diff --git a/doc/spec/proposals/163-detecting-clients.txt b/doc/spec/proposals/163-detecting-clients.txt new file mode 100644 index 0000000000..d838b17063 --- /dev/null +++ b/doc/spec/proposals/163-detecting-clients.txt @@ -0,0 +1,115 @@ +Filename: 163-detecting-clients.txt +Title: Detecting whether a connection comes from a client +Author: Nick Mathewson +Created: 22-May-2009 +Target: 0.2.2 +Status: Open + + +Overview: + + Some aspects of Tor's design require relays to distinguish + connections from clients from connections that come from relays. + The existing means for doing this is easy to spoof. We propose + a better approach. + +Motivation: + + There are at least two reasons for which Tor servers want to tell + which connections come from clients and which come from other + servers: + + 1) Some exits, proposal 152 notwithstanding, want to disallow + their use as single-hop proxies. + 2) Some performance-related proposals involve prioritizing + traffic from relays, or limiting traffic per client (but not + per relay). + + Right now, we detect client vs server status based on how the + client opens circuits. (Check out the code that implements the + AllowSingleHopExits option if you want all the details.) This + method is depressingly easy to fake, though. This document + proposes better means. + +Goals: + + To make grabbing relay privileges at least as difficult as just + running a relay. + + In the analysis below, "using server privileges" means taking any + action that only servers are supposed to do, like delivering a + BEGIN cell to an exit node that doesn't allow single hop exits, + or claiming server-like amounts of bandwidth. + +Passive detection: + + A connection is definitely a client connection if it takes one of + the TLS methods during setup that does not establish an identity + key. + + A circuit is definitely a client circuit if it is initiated with + a CREATE_FAST cell, though the node could be a client or a server. + + A node that's listed in a recent consensus is probably a server. + + A node to which we have successfully extended circuits from + multiple origins is probably a server. + +Active detection: + + If a node doesn't try to use server privileges at all, we never + need to care whether it's a server. + + When a node or circuit tries to use server privileges, if it is + "definitely a client" as per above, we can refuse it immediately. + + If it's "probably a server" as per above, we can accept it. + + Otherwise, we have either a client, or a server that is neither + listed in any consensus or used by any other clients -- in other + words, a new or private server. + + For these servers, we should attempt to build one or more test + circuits through them. If enough of the circuits succeed, the + node is a real relay. If not, it is probably a client. + + While we are waiting for the test circuits to succeed, we should + allow a short grace period in which server privileges are + permitted. When a test is done, we should remember its outcome + for a while, so we don't need to do it again. + +Why it's hard to do good testing: + + Doing a test circuit starting with an unlisted router requires + only that we have an open connection for it. Doing a test + circuit starting elsewhere _through_ an unlisted router--though + more reliable-- would require that we have a known address, port, + identity key, and onion key for the router. Only the address and + identity key are easily available via the current Tor protocol in + all cases. + + We could fix this part by requiring that all servers support + BEGIN_DIR and support downloading at least a current descriptor + for themselves. + +Open questions: + + What are the thresholds for the needed numbers of circuits + for us to decide that a node is a relay? + + [Suggested answer: two circuits from two distinct hosts.] + + How do we pick grace periods? How long do we remember the + outcome of a test? + + [Suggested answer: 10 minute grace period; 48 hour memory of + test outcomes.] + + If we can build circuits starting at a suspect node, but we don't + have enough information to try extending circuits elsewhere + through the node, should we conclude that the node is + "server-like" or not? + + [Suggested answer: for now, just try making circuits through + the node. Extend this to extending circuits as needed.] + From 0adb8c83860699fee4ccd19a3cfa0c5abd2c4070 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 22 May 2009 02:58:42 -0400 Subject: [PATCH 042/187] Short proposal on reporting why authorities voted as they did --- doc/spec/proposals/000-index.txt | 2 + .../proposals/164-reporting-server-status.txt | 91 +++++++++++++++++++ 2 files changed, 93 insertions(+) create mode 100644 doc/spec/proposals/164-reporting-server-status.txt diff --git a/doc/spec/proposals/000-index.txt b/doc/spec/proposals/000-index.txt index 8c8b0ab5d4..26622e83a5 100644 --- a/doc/spec/proposals/000-index.txt +++ b/doc/spec/proposals/000-index.txt @@ -84,6 +84,7 @@ Proposals by number: 161 Computing Bandwidth Adjustments [OPEN] 162 Publish the consensus in multiple flavors [OPEN] 163 Detecting whether a connection comes from a client [OPEN] +164 Reporting the status of server votes [OPEN] Proposals by status: @@ -109,6 +110,7 @@ Proposals by status: 161 Computing Bandwidth Adjustments [for 0.2.2.x] 162 Publish the consensus in multiple flavors [for 0.2.2] 163 Detecting whether a connection comes from a client [for 0.2.2] + 164 Reporting the status of server votes [for 0.2.2] ACCEPTED: 110 Avoiding infinite length circuits [for 0.2.1.x] [in 0.2.1.3-alpha] 117 IPv6 exits [for 0.2.1.x] diff --git a/doc/spec/proposals/164-reporting-server-status.txt b/doc/spec/proposals/164-reporting-server-status.txt new file mode 100644 index 0000000000..705f5f1a84 --- /dev/null +++ b/doc/spec/proposals/164-reporting-server-status.txt @@ -0,0 +1,91 @@ +Filename: 164-reporting-server-status.txt +Title: Reporting the status of server votes +Author: Nick Mathewson +Created: 22-May-2009 +Target: 0.2.2 +Status: Open + + +Overview: + + When a given node isn't listed in the directory, it isn't always easy + to tell why. This proposal suggest a quick-and-dirty way for + authorities to export not only how they voted, but why, and a way to + collate the information. + +Motivation: + + Right now, if you want to know the reason why your server was listed + a certain way in the Tor directory, the following steps are + recommended: + + - Look through your log for reports of what the authority said + when you tried to upload. + + - Look at the consensus; see if you're listed. + + - Wait a while, see if things get better. + + - Download the votes from all the authorities, and see how they + voted. Try to figure out why. + + - If you think they'll listen to you, ask some authority + operators to look you up in their mtbf files and logs to see + why they voted as they did. + + This is far too hard. + +Solution: + + We should add a new vote-like information-only document that + authorities serve on request. Call it a "vote info". It is + generated at the same time as a vote, but used only for + determining why a server voted as it did. It is served from + /tor/status-vote-info/current/authority[.z] + + It differs from a vote in that: + + * Its vote-status field is 'vote-info'. + + * It includes routers that the authority would not include + in its vote. + + For these, it includes an "omitted" line with an English + message explaining why they were omitted. + + * For each router, it includes a line describing its WFU and + MTBF. The format is: + + "stability up-since='date'" + "uptime down-since='date'" + + * It describes the WFU and MTBF thresholds it requires to + vote for a given router in various roles in the header. + The format is: + + "flag-requirement " + + e.g. + + "flag-requirement Guard uptime > 80" + + * It includes info on routers all of whose descriptors that + were uploaded but rejected over the past few hours. The + "r" lines for these are the same as for regular routers. + The other lines are omitted for these routers, and are + replaced with a single "rejected" line, explaining (in + English) why the router was rejected. + + + A status site (like Torweather or Torstatus or another + tool) can poll these files when they are generated, collate + the data, and make it available to server operators. + +Risks: + + This document makes no provisions for caching these "vote + info" documents. If many people wind up fetching them + aggressively from the authorities, that would be bad. + + + From a3fadddd4a08d1eeb1f0cf100d3fd6c60f445fcc Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 22 May 2009 14:06:39 -0400 Subject: [PATCH 043/187] Improved bug-957 fix for 0.2.2. Really, our idiocy was that we were calling event_set() on the same event more than once, which sometimes led to us calling event_set() on an event that was already inserted, thus making it look uninserted. With this patch, we just initialize the timeout events when we create the requests and nameservers, and we don't need to worry about double-add and double-del cases at all. --- src/or/eventdns.c | 76 +++++++---------------------------------------- 1 file changed, 11 insertions(+), 65 deletions(-) diff --git a/src/or/eventdns.c b/src/or/eventdns.c index cd1330314b..079e353e97 100644 --- a/src/or/eventdns.c +++ b/src/or/eventdns.c @@ -173,8 +173,6 @@ struct evdns_request { /* these objects are kept in a circular list */ struct evdns_request *next, *prev; - u16 timeout_event_deleted; /**< Debugging: where was timeout_event - * deleted? 0 for "it's added." */ struct event timeout_event; u16 trans_id; /* the transaction id */ @@ -214,8 +212,6 @@ struct nameserver { struct event event; /* these objects are kept in a circular list */ struct nameserver *next, *prev; - u16 timeout_event_deleted; /**< Debugging: where was timeout_event - * deleted? 0 for "it's added." */ struct event timeout_event; /* used to keep the timeout for */ /* when we next probe this server. */ /* Valid if state == 0 */ @@ -474,51 +470,10 @@ sockaddr_eq(const struct sockaddr *sa1, const struct sockaddr *sa2, return 1; } -/* for debugging bug 929. XXXX022 */ -static int -_add_timeout_event(u16 *lineno, struct event *ev, struct timeval *to) -{ - *lineno = 0; - return evtimer_add(ev, to); -} -#define add_timeout_event(s, to) \ - (_add_timeout_event(&(s)->timeout_event_deleted, &(s)->timeout_event, (to))) - -/* for debugging bug 929. XXXX022 */ -static int -_del_timeout_event(u16 *lineno, struct event *ev, int line) -{ - if (*lineno) { - log(EVDNS_LOG_DEBUG, - "Duplicate timeout event_del from line %d: first call " - "was at %d.", line, (int)*lineno); - return 0; - } else { - *lineno = (u16)line; - return event_del(ev); - } -} -#define del_timeout_event(s) \ - (_del_timeout_event(&(s)->timeout_event_deleted, &(s)->timeout_event, \ - __LINE__)) -/* For debugging bug 929/957. XXXX022 */ -static int -_del_timeout_event_if_set(u16 *lineno, struct event *ev, int line) -{ - if (*lineno == 0) { - log(EVDNS_LOG_DEBUG, - "Event that I thought was non-added as of line %d " - "was actually added on line %d", - line, (int)*lineno); - *lineno = line; - return event_del(ev); - } - return 0; -} -#define del_timeout_event_if_set(s) \ - _del_timeout_event_if_set(&(s)->timeout_event_deleted, \ - &(s)->timeout_event, \ - __LINE__) +#define add_timeout_event(s, to) \ + (event_add(&(s)->timeout_event, (to))) +#define del_timeout_event(s) \ + (event_del(&(s)->timeout_event)) /* This walks the list of inflight requests to find the */ /* one with a matching transaction id. Returns NULL on */ @@ -555,7 +510,7 @@ static void nameserver_probe_failed(struct nameserver *const ns) { const struct timeval * timeout; del_timeout_event(ns); - CLEAR(&ns->timeout_event); + if (ns->state == 1) { /* This can happen if the nameserver acts in a way which makes us mark */ /* it as bad and then starts sending good replies. */ @@ -567,8 +522,6 @@ nameserver_probe_failed(struct nameserver *const ns) { global_nameserver_timeouts_length - 1)]; ns->failed_times++; - del_timeout_event_if_set(ns); - evtimer_set(&ns->timeout_event, nameserver_prod_callback, ns); if (add_timeout_event(ns, (struct timeval *) timeout) < 0) { log(EVDNS_LOG_WARN, "Error from libevent when adding timer event for %s", @@ -597,8 +550,6 @@ nameserver_failed(struct nameserver *const ns, const char *msg) { ns->state = 0; ns->failed_times = 1; - del_timeout_event_if_set(ns); - evtimer_set(&ns->timeout_event, nameserver_prod_callback, ns); if (add_timeout_event(ns, (struct timeval *) &global_nameserver_timeouts[0]) < 0) { log(EVDNS_LOG_WARN, "Error from libevent when adding timer event for %s", @@ -634,7 +585,6 @@ nameserver_up(struct nameserver *const ns) { log(EVDNS_LOG_WARN, "Nameserver %s is back up", debug_ntop((struct sockaddr *)&ns->address)); del_timeout_event(ns); - CLEAR(&ns->timeout_event); ns->state = 1; ns->failed_times = 0; ns->timedout = 0; @@ -666,7 +616,6 @@ request_finished(struct evdns_request *const req, struct evdns_request **head) { log(EVDNS_LOG_DEBUG, "Removing timeout for request %lx", (unsigned long) req); del_timeout_event(req); - CLEAR(&req->timeout_event); search_request_finished(req); global_requests_inflight--; @@ -2046,7 +1995,6 @@ evdns_request_timeout_callback(int fd, short events, void *arg) { * request_finished; that one already deletes the timeout event. * XXXX021 port this change to libevent. */ del_timeout_event(req); - CLEAR(&req->timeout_event); evdns_request_transmit(req); } } @@ -2109,8 +2057,7 @@ evdns_request_transmit(struct evdns_request *req) { /* transmitted; we need to check for timeout. */ log(EVDNS_LOG_DEBUG, "Setting timeout for request %lx", (unsigned long) req); - del_timeout_event_if_set(req); - evtimer_set(&req->timeout_event, evdns_request_timeout_callback, req); + if (add_timeout_event(req, &global_timeout) < 0) { log(EVDNS_LOG_WARN, "Error from libevent when adding timer for request %lx", @@ -2225,7 +2172,6 @@ evdns_clear_nameservers_and_suspend(void) (void) event_del(&server->event); CLEAR(&server->event); del_timeout_event(server); - CLEAR(&server->timeout_event); if (server->socket >= 0) CLOSE_SOCKET(server->socket); CLEAR(server); @@ -2243,7 +2189,6 @@ evdns_clear_nameservers_and_suspend(void) req->ns = NULL; /* ???? What to do about searches? */ del_timeout_event(req); - CLEAR(&req->timeout_event); req->trans_id = 0; req->transmit_me = 0; @@ -2318,7 +2263,8 @@ _evdns_nameserver_add_impl(const struct sockaddr *address, if (!ns) return -1; memset(ns, 0, sizeof(struct nameserver)); - ns->timeout_event_deleted = __LINE__; + + evtimer_set(&ns->timeout_event, nameserver_prod_callback, ns); ns->socket = socket(PF_INET, SOCK_DGRAM, 0); if (ns->socket < 0) { err = 1; goto out1; } @@ -2553,7 +2499,8 @@ request_new(int type, const char *name, int flags, } memset(req, 0, sizeof(struct evdns_request)); - req->timeout_event_deleted = __LINE__; + + evtimer_set(&req->timeout_event, evdns_request_timeout_callback, req); if (global_randomize_case) { unsigned i; @@ -3384,8 +3331,7 @@ evdns_shutdown(int fail_requests) if (server->socket >= 0) CLOSE_SOCKET(server->socket); (void) event_del(&server->event); - if (server->state == 0) - del_timeout_event(server); + del_timeout_event(server); CLEAR(server); mm_free(server); if (server_next == server_head) From 9f69b3d94c86aa470d7927fe7f6a96693e5baf0c Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 22 May 2009 15:55:40 -0400 Subject: [PATCH 044/187] [prop140] Note another useful restriction of ed format. --- doc/spec/proposals/140-consensus-diffs.txt | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/doc/spec/proposals/140-consensus-diffs.txt b/doc/spec/proposals/140-consensus-diffs.txt index b4a8ab4f85..8bc4070bfe 100644 --- a/doc/spec/proposals/140-consensus-diffs.txt +++ b/doc/spec/proposals/140-consensus-diffs.txt @@ -5,6 +5,11 @@ Created: 13-Jun-2008 Status: Accepted Target: 0.2.2.x +0. History + + 22-May-2009: Restricted the ed format even more strictly for ease of + implementation. -nickm + 1. Overview. Tor clients and servers need a list of which relays are on the @@ -133,6 +138,10 @@ Target: 0.2.2.x Note that line numbers always apply to the file after all previous commands have already been applied. + The commands MUST apply to the file from back to front, such that + lines are only ever referred to by their position in the original + file. + The "current line" is either the first line of the file, if this is the first command, the last line of a block we added in an append or change command, or the line immediate following a set of lines we just From 793e97bb2a610dbd4cf8cfb2795f8b680d55a116 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 22 May 2009 23:30:52 -0400 Subject: [PATCH 045/187] Add a quick macro to calculate hashtable memory usage --- src/common/ht.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/common/ht.h b/src/common/ht.h index ff1e5551cc..be747529ab 100644 --- a/src/common/ht.h +++ b/src/common/ht.h @@ -42,6 +42,10 @@ #define HT_SIZE(head) \ ((head)->hth_n_entries) +/* Return memory usage for a hashtable (not counting the entries themselves) */ +#define HT_MEM_USAGE(head) \ + (sizeof(*head) + (head)->hth_table_length * sizeof(void*)) + #define HT_FIND(name, head, elm) name##_HT_FIND((head), (elm)) #define HT_INSERT(name, head, elm) name##_HT_INSERT((head), (elm)) #define HT_REPLACE(name, head, elm) name##_HT_REPLACE((head), (elm)) From 75f963e9517ba8702fe1ed1d470e28b0462fb3d2 Mon Sep 17 00:00:00 2001 From: Jacob Appelbaum Date: Fri, 22 May 2009 23:33:44 -0400 Subject: [PATCH 046/187] Log the number and size of DNS cache entries on SIGUSR1. Specifically if you send SIGUSR1, it will add two lines to the log file: May 22 07:41:59.576 [notice] Our DNS cache has 3364 entries. May 22 07:41:59.576 [notice] Our DNS cache size is approximately 1022656 bytes. [tweaked a bit by nickm] --- src/or/dns.c | 25 +++++++++++++++++++++++++ src/or/main.c | 1 + src/or/or.h | 3 ++- 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/src/or/dns.c b/src/or/dns.c index 1742e3dd4b..88d08a8ffe 100644 --- a/src/or/dns.c +++ b/src/or/dns.c @@ -1612,6 +1612,31 @@ assert_resolve_ok(cached_resolve_t *resolve) } } +/** Return the number of DNS cache entries as an int */ +static int +dns_cache_entry_count(void) +{ + return HT_SIZE(&cache_root); +} + +/** Log memory information about our internal DNS cache at level 'severity'. */ +void +dump_dns_mem_usage(int severity) +{ + /* This should never be larger than INT_MAX. */ + int hash_count = dns_cache_entry_count(); + size_t hash_mem = sizeof(struct cached_resolve_t) * hash_count; + hash_mem += HT_MEM_USAGE(&cache_root); + + /* Print out the count and estimated size of our &cache_root. It undercounts + hostnames in cached reverse resolves. + */ + log(severity, LD_MM, "Our DNS cache has %d entries.", hash_count); + log(severity, LD_MM, "Our DNS cache size is approximately %u bytes.", + (unsigned)hash_mem); +} + + #ifdef DEBUG_DNS_CACHE /** Exit with an assertion if the DNS cache is corrupt. */ static void diff --git a/src/or/main.c b/src/or/main.c index 69ed12f12a..b151b1f666 100644 --- a/src/or/main.c +++ b/src/or/main.c @@ -1588,6 +1588,7 @@ dumpmemusage(int severity) U64_PRINTF_ARG(rephist_total_alloc), rephist_total_num); dump_routerlist_mem_usage(severity); dump_cell_pool_usage(severity); + dump_dns_mem_usage(severity); buf_dump_freelist_sizes(severity); tor_log_mallinfo(severity); } diff --git a/src/or/or.h b/src/or/or.h index 7ac9e80004..d5b36c85a6 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -483,7 +483,7 @@ typedef enum { #define CIRCUIT_PURPOSE_IS_ORIGIN(p) ((p)>_CIRCUIT_PURPOSE_OR_MAX) /** True iff the circuit purpose p is for a circuit that originated * here to serve as a client. (Hidden services don't count here.) */ -#define CIRCUIT_PURPOSE_IS_CLIENT(p) \ +#define CIRCUIT_PURPOSE_IS_CLIENT(p) \ ((p)> _CIRCUIT_PURPOSE_OR_MAX && \ (p)<=_CIRCUIT_PURPOSE_C_MAX) /** True iff the circuit_t c is actually an origin_circuit_t. */ @@ -3573,6 +3573,7 @@ int dns_resolve(edge_connection_t *exitconn); void dns_launch_correctness_checks(void); int dns_seems_to_be_broken(void); void dns_reset_correctness_checks(void); +void dump_dns_mem_usage(int severity); /********************************* dnsserv.c ************************/ From 7703b887f5db0d8d62e9eb87305bb8e2113276f1 Mon Sep 17 00:00:00 2001 From: Martin Peck Date: Sat, 23 May 2009 16:42:44 -0700 Subject: [PATCH 047/187] Add support for dynamic OpenSSL hardware crypto acceleration engines. --- src/common/crypto.c | 59 ++++++++++++++++++++++++++++++++-------- src/common/crypto.h | 4 ++- src/common/tortls.c | 1 - src/or/config.c | 19 +++++++++++-- src/or/main.c | 4 ++- src/or/or.h | 2 ++ src/or/router.c | 4 ++- src/or/test.c | 2 +- src/tools/tor-checkkey.c | 2 +- src/tools/tor-gencert.c | 2 +- 10 files changed, 77 insertions(+), 22 deletions(-) diff --git a/src/common/crypto.c b/src/common/crypto.c index bc7eb0c6ca..7b5743f64a 100644 --- a/src/common/crypto.c +++ b/src/common/crypto.c @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -166,36 +167,70 @@ log_engine(const char *fn, ENGINE *e) } } +/** Try to load an engine in a shared library via fully qualified path. + */ +static ENGINE * +try_load_engine(const char *path, const char *engine) +{ + ENGINE *e = ENGINE_by_id("dynamic"); + if (e) { + if (!ENGINE_ctrl_cmd_string(e, "ID", engine, 0) || + !ENGINE_ctrl_cmd_string(e, "DIR_LOAD", "2", 0) || + !ENGINE_ctrl_cmd_string(e, "DIR_ADD", path, 0) || + !ENGINE_ctrl_cmd_string(e, "LOAD", NULL, 0)) { + ENGINE_free(e); + e = NULL; + } + } + return e; +} + /** Initialize the crypto library. Return 0 on success, -1 on failure. */ int -crypto_global_init(int useAccel) +crypto_global_init(int useAccel, const char *accelName, const char *accelDir) { if (!_crypto_global_initialized) { ERR_load_crypto_strings(); OpenSSL_add_all_algorithms(); _crypto_global_initialized = 1; setup_openssl_threading(); - /* XXX the below is a bug, since we can't know if we're supposed - * to be using hardware acceleration or not. we should arrange - * for this function to be called before init_keys. But make it - * not complain loudly, at least until we make acceleration work. */ - if (useAccel < 0) { - log_info(LD_CRYPTO, "Initializing OpenSSL via tor_tls_init()."); - } if (useAccel > 0) { + ENGINE *e = NULL; log_info(LD_CRYPTO, "Initializing OpenSSL engine support."); ENGINE_load_builtin_engines(); - if (!ENGINE_register_all_complete()) - return -1; - - /* XXXX make sure this isn't leaking. */ + ENGINE_register_all_complete(); + if (accelName) { + if (accelDir) { + log_info(LD_CRYPTO, "Trying to load dynamic OpenSSL engine \"%s\"" + " via path \"%s\".", accelName, accelDir); + e = try_load_engine(accelName, accelDir); + } else { + log_info(LD_CRYPTO, "Initializing dynamic OpenSSL engine \"%s\"" + " acceleration support.", accelName); + e = ENGINE_by_id(accelName); + } + if (!e) { + log_warn(LD_CRYPTO, "Unable to load dynamic OpenSSL engine \"%s\".", + accelName); + } else { + log_info(LD_CRYPTO, "Loaded dynamic OpenSSL engine \"%s\".", + accelName); + } + } + if (e) { + log_info(LD_CRYPTO, "Loaded OpenSSL hardware acceleration engine," + " setting default ciphers."); + ENGINE_set_default(e, ENGINE_METHOD_ALL); + } log_engine("RSA", ENGINE_get_default_RSA()); log_engine("DH", ENGINE_get_default_DH()); log_engine("RAND", ENGINE_get_default_RAND()); log_engine("SHA1", ENGINE_get_digest_engine(NID_sha1)); log_engine("3DES", ENGINE_get_cipher_engine(NID_des_ede3_ecb)); log_engine("AES", ENGINE_get_cipher_engine(NID_aes_128_ecb)); + } else { + log_info(LD_CRYPTO, "NOT using OpenSSL engine support."); } return crypto_seed_rng(1); } diff --git a/src/common/crypto.h b/src/common/crypto.h index dd353ef030..fa6735d788 100644 --- a/src/common/crypto.h +++ b/src/common/crypto.h @@ -55,7 +55,9 @@ typedef struct crypto_digest_env_t crypto_digest_env_t; typedef struct crypto_dh_env_t crypto_dh_env_t; /* global state */ -int crypto_global_init(int hardwareAccel); +int crypto_global_init(int hardwareAccel, + const char *accelName, + const char *accelPath); void crypto_thread_cleanup(void); int crypto_global_cleanup(void); diff --git a/src/common/tortls.c b/src/common/tortls.c index 581b35848a..106a9d40ee 100644 --- a/src/common/tortls.c +++ b/src/common/tortls.c @@ -308,7 +308,6 @@ tor_tls_init(void) if (!tls_library_is_initialized) { SSL_library_init(); SSL_load_error_strings(); - crypto_global_init(-1); tls_library_is_initialized = 1; } } diff --git a/src/or/config.c b/src/or/config.c index a4461a6fe7..0d176670e4 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -222,6 +222,8 @@ static config_var_t _option_vars[] = { #endif OBSOLETE("Group"), V(HardwareAccel, BOOL, "0"), + V(AccelName, STRING, NULL), + V(AccelDir, FILENAME, NULL), V(HashedControlPassword, LINELIST, NULL), V(HidServDirectoryV2, BOOL, "1"), VAR("HiddenServiceDir", LINELIST_S, RendConfigLines, NULL), @@ -444,6 +446,10 @@ static config_var_description_t options_description[] = { * FetchUselessDescriptors */ { "HardwareAccel", "If set, Tor tries to use hardware crypto accelerators " "when it can." }, + { "AccelName", "If set, try to use hardware crypto accelerator with this " + "specific ID." }, + { "AccelDir", "If set, look in this directory for the dynamic hardware " + "engine in addition to OpenSSL default path." }, /* HashedControlPassword */ { "HTTPProxy", "Force Tor to make all HTTP directory requests through this " "host:port (or host:80 if port is not set)." }, @@ -3602,6 +3608,11 @@ options_validate(or_options_t *old_options, or_options_t *options, "testing Tor network!"); } + if (options->AccelName && !options->HardwareAccel) + options->HardwareAccel = 1; + if (options->AccelDir && !options->AccelName) + REJECT("Can't use hardware crypto accelerator dir without engine name."); + return 0; #undef REJECT #undef COMPLAIN @@ -3659,9 +3670,11 @@ options_transition_allowed(or_options_t *old, or_options_t *new_val, return -1; } - if (old->HardwareAccel != new_val->HardwareAccel) { - *msg = tor_strdup("While Tor is running, changing HardwareAccel is " - "not allowed."); + if ((old->HardwareAccel != new_val->HardwareAccel) + || (old->AccelName != new_val->AccelName) + || (old->AccelDir != new_val->AccelDir)) { + *msg = tor_strdup("While Tor is running, changing OpenSSL hardware " + "acceleration engine is not allowed."); return -1; } diff --git a/src/or/main.c b/src/or/main.c index b151b1f666..456d9fab16 100644 --- a/src/or/main.c +++ b/src/or/main.c @@ -1818,7 +1818,9 @@ tor_init(int argc, char *argv[]) "and you probably shouldn't."); #endif - if (crypto_global_init(get_options()->HardwareAccel)) { + if (crypto_global_init(get_options()->HardwareAccel, + get_options()->AccelName, + get_options()->AccelDir)) { log_err(LD_BUG, "Unable to initialize OpenSSL. Exiting."); return -1; } diff --git a/src/or/or.h b/src/or/or.h index d5b36c85a6..77582309e9 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -2422,6 +2422,8 @@ typedef struct { * log whether it was DNS-leaking or not? */ int HardwareAccel; /**< Boolean: Should we enable OpenSSL hardware * acceleration where available? */ + char *AccelName; /**< Optional hardware acceleration engine name. */ + char *AccelDir; /**< Optional hardware acceleration engine search dir. */ int UseEntryGuards; /**< Boolean: Do we try to enter from a smallish number * of fixed nodes? */ int NumEntryGuards; /**< How many entry guards do we try to establish? */ diff --git a/src/or/router.c b/src/or/router.c index da922b7508..7a567be30d 100644 --- a/src/or/router.c +++ b/src/or/router.c @@ -442,7 +442,9 @@ init_keys(void) key_lock = tor_mutex_new(); /* There are a couple of paths that put us here before */ - if (crypto_global_init(get_options()->HardwareAccel)) { + if (crypto_global_init(get_options()->HardwareAccel, + get_options()->AccelName, + get_options()->AccelDir)) { log_err(LD_BUG, "Unable to initialize OpenSSL. Exiting."); return -1; } diff --git a/src/or/test.c b/src/or/test.c index 6ce0aaeb47..ef61e62ee1 100644 --- a/src/or/test.c +++ b/src/or/test.c @@ -4793,7 +4793,7 @@ main(int c, char**v) } options->command = CMD_RUN_UNITTESTS; - crypto_global_init(0); + crypto_global_init(0, NULL, NULL); rep_hist_init(); network_init(); setup_directory(); diff --git a/src/tools/tor-checkkey.c b/src/tools/tor-checkkey.c index b29b52d8db..6416dbfbb3 100644 --- a/src/tools/tor-checkkey.c +++ b/src/tools/tor-checkkey.c @@ -29,7 +29,7 @@ int main(int c, char **v) return 1; } - if (crypto_global_init(0)) { + if (crypto_global_init(0, NULL, NULL)) { fprintf(stderr, "Couldn't initialize crypto library.\n"); return 1; } diff --git a/src/tools/tor-gencert.c b/src/tools/tor-gencert.c index 4971668c9f..d2ea4eb109 100644 --- a/src/tools/tor-gencert.c +++ b/src/tools/tor-gencert.c @@ -496,7 +496,7 @@ main(int argc, char **argv) init_logging(); /* Don't bother using acceleration. */ - if (crypto_global_init(0)) { + if (crypto_global_init(0, NULL, NULL)) { fprintf(stderr, "Couldn't initialize crypto library.\n"); return 1; } From f26abf5f540e446db18d146cffaeb23c84ea50ce Mon Sep 17 00:00:00 2001 From: Sebastian Hahn Date: Sun, 24 May 2009 02:42:37 +0200 Subject: [PATCH 048/187] fix codestyle issues --- src/or/config_codedigest.c | 1 + src/or/dns.c | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/src/or/config_codedigest.c b/src/or/config_codedigest.c index 67fd9847c2..be9eaa331d 100644 --- a/src/or/config_codedigest.c +++ b/src/or/config_codedigest.c @@ -8,3 +8,4 @@ tor_get_digests(void) #include "or_sha1.i" ; } + diff --git a/src/or/dns.c b/src/or/dns.c index 88d08a8ffe..af056e8de3 100644 --- a/src/or/dns.c +++ b/src/or/dns.c @@ -1636,7 +1636,6 @@ dump_dns_mem_usage(int severity) (unsigned)hash_mem); } - #ifdef DEBUG_DNS_CACHE /** Exit with an assertion if the DNS cache is corrupt. */ static void From 7df5c031e8deea647a23ff102de0b2c29d5a59ed Mon Sep 17 00:00:00 2001 From: Roger Dingledine Date: Sun, 24 May 2009 17:04:31 -0400 Subject: [PATCH 049/187] add mikeperry's notes about fairer round-robin for rate limiting --- doc/spec/proposals/ideas/xxx-bwrate-algs.txt | 106 +++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100644 doc/spec/proposals/ideas/xxx-bwrate-algs.txt diff --git a/doc/spec/proposals/ideas/xxx-bwrate-algs.txt b/doc/spec/proposals/ideas/xxx-bwrate-algs.txt new file mode 100644 index 0000000000..757f5bc55e --- /dev/null +++ b/doc/spec/proposals/ideas/xxx-bwrate-algs.txt @@ -0,0 +1,106 @@ +# The following two algorithms + + +# Algorithm 1 +# TODO: Burst and Relay/Regular differentiation + +BwRate = Bandwidth Rate in Bytes Per Second +GlobalWriteBucket = 0 +GlobalReadBucket = 0 +Epoch = Token Fill Rate in seconds: suggest 50ms=.050 +SecondCounter = 0 +MinWriteBytes = Minimum amount bytes per write + +Every Epoch Seconds: + UseMinWriteBytes = MinWriteBytes + WriteCnt = 0 + ReadCnt = 0 + BytesRead = 0 + + For Each Open OR Conn with pending write data: + WriteCnt++ + For Each Open OR Conn: + ReadCnt++ + + BytesToRead = (BwRate*Epoch + GlobalReadBucket)/ReadCnt + BytesToWrite = (BwRate*Epoch + GlobalWriteBucket)/WriteCnt + + if BwRate/WriteCnt < MinWriteBytes: + # If we aren't likely to accumulate enough bytes in a second to + # send a whole cell for our connections, send partials + Log(NOTICE, "Too many ORCons to write full blocks. Sending short packets.") + UseMinWriteBytes = 1 + # Other option: We could switch to plan 2 here + + # Service each writable ORConn. If there are any partial writes, + # return remaining bytes from this epoch to the global pool + For Each Open OR Conn with pending write data: + ORConn->write_bucket += BytesToWrite + if ORConn->write_bucket > UseMinWriteBytes: + w = write(ORConn, MIN(len(ORConn->write_data), ORConn->write_bucket)) + # possible that w < ORConn->write_data here due to TCP pushback. + # We should restore the rest of the write_bucket to the global + # buffer + GlobalWriteBucket += (ORConn->write_bucket - w) + ORConn->write_bucket = 0 + + For Each Open OR Conn: + r = read_nonblock(ORConn, BytesToRead) + BytesRead += r + + SecondCounter += Epoch + if SecondCounter < 1: + # Save unused bytes from this epoch to be used later in the second + GlobalReadBucket += (BwRate*Epoch - BytesRead) + else: + SecondCounter = 0 + GlobalReadBucket = 0 + GlobalWriteBucket = 0 + For Each ORConn: + ORConn->write_bucket = 0 + + + +# Alternate plan for Writing fairly. Reads would still be covered +# by plan 1 as there is no additional network overhead for short reads, +# so we don't need to try to avoid them. +# +# I think this is actually pretty similar to what we do now, but +# with the addition that the bytes accumulate up to the second mark +# and we try to keep track of our position in the write list here +# (unless libevent is doing that for us already and I just don't see it) +# +# TODO: Burst and Relay/Regular differentiation + +# XXX: The inability to send single cells will cause us to block +# on EXTEND cells for low-bandwidth node pairs.. +BwRate = Bandwidth Rate in Bytes Per Second +WriteBytes = Bytes per write +Epoch = MAX(MIN(WriteBytes/BwRate, .333s), .050s) + +SecondCounter = 0 +GlobalWriteBucket = 0 + +# New connections are inserted at Head-1 (the 'tail' of this circular list) +# This is not 100% fifo for all node data, but it is the best we can do +# without insane amounts of additional queueing complexity. +WriteConnList = List of Open OR Conns with pending write data > WriteBytes +WriteConnHead = 0 + +Every Epoch Seconds: + GlobalWriteBucket += BwRate*Epoch + WriteListEnd = WriteConnHead + + do + ORCONN = WriteConnList[WriteConnHead] + w = write(ORConn, WriteBytes) + GlobalWriteBucket -= w + WriteConnHead += 1 + while GlobalWriteBucket > 0 and WriteConnHead != WriteListEnd + + SecondCounter += Epoch + if SecondCounter >= 1: + SecondCounter = 0 + GlobalWriteBucket = 0 + + From 4c548cd51b087801b02c96b56978bd03ffdf8938 Mon Sep 17 00:00:00 2001 From: Roger Dingledine Date: Sun, 24 May 2009 17:05:13 -0400 Subject: [PATCH 050/187] early thoughts on a safer way to reach a service via a particular exit. --- .../proposals/ideas/xxx-encrypted-services.txt | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 doc/spec/proposals/ideas/xxx-encrypted-services.txt diff --git a/doc/spec/proposals/ideas/xxx-encrypted-services.txt b/doc/spec/proposals/ideas/xxx-encrypted-services.txt new file mode 100644 index 0000000000..3414f3c4fb --- /dev/null +++ b/doc/spec/proposals/ideas/xxx-encrypted-services.txt @@ -0,0 +1,18 @@ + +the basic idea might be to generate a keypair, and sign little statements +like "this key corresponds to this relay id", and publish them on karsten's +hs dht. + +so if you want to talk to it, you look it up, then go to that exit. +and by 'go to' i mean 'build a tor circuit like normal except you're sure +where to exit' + +connecting to it is slower than usual, but once you're connected, it's no +slower than normal tor. +and you get what wikileaks wants from its hidden service, which is really +just the UI piece. +indymedia also wants this. + +might be interesting to let an encrypted service list more than one relay, +too. + From 9a4b0446b094382d3f0833e0e1217671c9946697 Mon Sep 17 00:00:00 2001 From: Roger Dingledine Date: Sun, 24 May 2009 17:01:02 -0400 Subject: [PATCH 051/187] fix a comment --- src/or/main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/or/main.c b/src/or/main.c index b151b1f666..61e9d9f6a7 100644 --- a/src/or/main.c +++ b/src/or/main.c @@ -774,7 +774,7 @@ run_connection_housekeeping(int i, time_t now) } } -/** Honor a NEWNYM request: make future requests unlinkability to past +/** Honor a NEWNYM request: make future requests unlinkable to past * requests. */ static void signewnym_impl(time_t now) From 05e55d82b6a10e20be2d447c35f42a1bef9d7da7 Mon Sep 17 00:00:00 2001 From: Roger Dingledine Date: Sun, 24 May 2009 17:01:30 -0400 Subject: [PATCH 052/187] simplify options_act() --- src/or/config.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/or/config.c b/src/or/config.c index a4461a6fe7..0dd0931ece 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -1280,14 +1280,14 @@ options_act(or_options_t *old_options) return 0; /* Finish backgrounding the process */ - if (running_tor && options->RunAsDaemon) { + if (options->RunAsDaemon) { /* We may be calling this for the n'th time (on SIGHUP), but it's safe. */ finish_daemon(options->DataDirectory); } /* Write our pid to the pid file. If we do not have write permissions we * will log a warning */ - if (running_tor && options->PidFile) + if (options->PidFile) write_pidfile(options->PidFile); /* Register addressmap directives */ From b1e61353ecab03a3714bd52d5a45a933ce4f76d1 Mon Sep 17 00:00:00 2001 From: Roger Dingledine Date: Sun, 24 May 2009 17:03:13 -0400 Subject: [PATCH 053/187] note that proposal 160 is written. note that section 4.5 is done. --- doc/TODO.external | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/doc/TODO.external b/doc/TODO.external index 948ad6a975..12048fa344 100644 --- a/doc/TODO.external +++ b/doc/TODO.external @@ -84,16 +84,9 @@ SM - 4.1, balance traffic better (rejigger the bandwidth numbers at the authorities based on Steven's algorithm), or Mike's plan (relay scanning to identify the unbalanced relays and fix them on the fly), or both. - - Figure out how to actually modify bandwidths in the consensus. We - may need to change the consensus voting algorithm to decide what - bandwidth to advertise based on something other than median: - if 7 authorities provide bandwidths, and 2 are doing scanning, - then the 5 that aren't scanning will outvote any changes. Should - all 7 scan? Should only some vote? Extra points if it doesn't - change all the numbers every new consensus, so consensus diffing - is still practical. -? - 4.5, Older entry guards are overloaded - - Pick a conservative timeout like a month, and implement. + - Implement Proposal 160 + o 4.5, Older entry guards are overloaded + o Pick a conservative timeout like a month, and implement. M - 5.2, better timeouts for giving up on circuits/streams - clients gather data about circuit timeouts, and then abandon circuits that take more than a std dev above that. From b998fed9b967924aa880f73c685284a052c8acfe Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 8 May 2009 12:12:20 -0400 Subject: [PATCH 054/187] Remove support for events without the extended format or long names. Supporting the old formats made our code complex; running without them has been discouraged since 0.2.1.x. --- ChangeLog | 3 + doc/spec/control-spec.txt | 10 +- src/or/circuitbuild.c | 30 ++-- src/or/control.c | 295 ++++++++++---------------------------- src/or/dirserv.c | 12 +- src/or/or.h | 6 - 6 files changed, 101 insertions(+), 255 deletions(-) diff --git a/ChangeLog b/ChangeLog index ee9f814f5a..9bbb3b7d7c 100644 --- a/ChangeLog +++ b/ChangeLog @@ -14,6 +14,9 @@ Changes in version 0.2.2.1-alpha - 2009-??-?? do not request or use version 0 descriptors. However, the authorities still accept and serve version 0 descriptors when contacted by older hidden services/clients. + - The EXTENDED_EVENTS and VERBOSE_NAMES controller features are now + always on; using them is necessary for correct forward-compatible + controllers. Changes in version 0.2.1.15??? - ????-??-?? diff --git a/doc/spec/control-spec.txt b/doc/spec/control-spec.txt index 6ca45aa27f..0c739eca03 100644 --- a/doc/spec/control-spec.txt +++ b/doc/spec/control-spec.txt @@ -773,9 +773,8 @@ Same as passing 'EXTENDED' to SETEVENTS; this is the preferred way to request the extended event syntax. - This will not be always-enabled until at least two stable releases - after 0.1.2.3-alpha, the release where it was first used for - anything. + This feaure was first used in 0.1.2.3-alpha. It is always-on in + Tor 0.2.2.1-alpha and later. VERBOSE_NAMES @@ -786,8 +785,9 @@ LongName format includes a Fingerprint, an indication of Named status, and a Nickname (if one is known). - This will not be always-enabled until at least two stable releases - after 0.1.2.2-alpha, the release where it was first available. + This will not be always-enabled until at least two stable + releases after 0.1.2.2-alpha, the release where it was first + available. It is always-on in Tor 0.2.2.1-alpha and later. 3.20. RESOLVE diff --git a/src/or/circuitbuild.c b/src/or/circuitbuild.c index e56290fb50..d62fae842a 100644 --- a/src/or/circuitbuild.c +++ b/src/or/circuitbuild.c @@ -2849,8 +2849,7 @@ int getinfo_helper_entry_guards(control_connection_t *conn, const char *question, char **answer) { - int use_long_names = conn->use_long_names; - + (void) conn; if (!strcmp(question,"entry-guards") || !strcmp(question,"helper-nodes")) { smartlist_t *sl = smartlist_create(); @@ -2858,12 +2857,13 @@ getinfo_helper_entry_guards(control_connection_t *conn, char nbuf[MAX_VERBOSE_NICKNAME_LEN+1]; if (!entry_guards) entry_guards = smartlist_create(); - SMARTLIST_FOREACH(entry_guards, entry_guard_t *, e, - { + SMARTLIST_FOREACH_BEGIN(entry_guards, entry_guard_t *, e) { size_t len = MAX_VERBOSE_NICKNAME_LEN+ISO_TIME_LEN+32; char *c = tor_malloc(len); const char *status = NULL; time_t when = 0; + routerinfo_t *ri; + if (!e->made_contact) { status = "never-connected"; } else if (e->bad_since) { @@ -2872,19 +2872,17 @@ getinfo_helper_entry_guards(control_connection_t *conn, } else { status = "up"; } - if (use_long_names) { - routerinfo_t *ri = router_get_by_digest(e->identity); - if (ri) { - router_get_verbose_nickname(nbuf, ri); - } else { - nbuf[0] = '$'; - base16_encode(nbuf+1, sizeof(nbuf)-1, e->identity, DIGEST_LEN); - /* e->nickname field is not very reliable if we don't know about - * this router any longer; don't include it. */ - } + + ri = router_get_by_digest(e->identity); + if (ri) { + router_get_verbose_nickname(nbuf, ri); } else { - base16_encode(nbuf, sizeof(nbuf), e->identity, DIGEST_LEN); + nbuf[0] = '$'; + base16_encode(nbuf+1, sizeof(nbuf)-1, e->identity, DIGEST_LEN); + /* e->nickname field is not very reliable if we don't know about + * this router any longer; don't include it. */ } + if (when) { format_iso_time(tbuf, when); tor_snprintf(c, len, "%s %s %s\n", nbuf, status, tbuf); @@ -2892,7 +2890,7 @@ getinfo_helper_entry_guards(control_connection_t *conn, tor_snprintf(c, len, "%s %s\n", nbuf, status); } smartlist_add(sl, c); - }); + } SMARTLIST_FOREACH_END(e); *answer = smartlist_join_strings(sl, "", 0, NULL); SMARTLIST_FOREACH(sl, char *, c, tor_free(c)); smartlist_free(sl); diff --git a/src/or/control.c b/src/or/control.c index c5fd688e3d..2726412d93 100644 --- a/src/or/control.c +++ b/src/or/control.c @@ -54,13 +54,9 @@ **/ typedef uint32_t event_mask_t; -/** An event mask of all the events that controller with the LONG_NAMES option - * set is interested in receiving. */ -static event_mask_t global_event_mask1long = 0; - -/** An event mask of all the events that controller with the SHORT_NAMES option - * set is interested in receiving. */ -static event_mask_t global_event_mask1short = 0; +/** An event mask of all the events that any controller is interested in + * receiving. */ +static event_mask_t global_event_mask = 0; /** True iff we have disabled log messages from being sent to the controller */ static int disable_log_messages = 0; @@ -68,13 +64,7 @@ static int disable_log_messages = 0; /** Macro: true if any control connection is interested in events of type * e. */ #define EVENT_IS_INTERESTING(e) \ - ((global_event_mask1long|global_event_mask1short) & (1<<(e))) -/** Macro: true if any control connection with the LONG_NAMES option is - * interested in events of type e. */ -#define EVENT_IS_INTERESTING1L(e) (global_event_mask1long & (1<<(e))) -/** Macro: true if any control connection with the SHORT_NAMES option is - * interested in events of type e. */ -#define EVENT_IS_INTERESTING1S(e) (global_event_mask1short & (1<<(e))) + (global_event_mask & (1<<(e))) /** If we're using cookie-type authentication, how long should our cookies be? */ @@ -95,25 +85,13 @@ static char authentication_cookie[AUTHENTICATION_COOKIE_LEN]; * of this so we can respond to getinfo status/bootstrap-phase queries. */ static char last_sent_bootstrap_message[BOOTSTRAP_MSG_LEN]; -/** Flag for event_format_t. Indicates that we should use the old - * name format of nickname|hexdigest +/** Flag for event_format_t. Indicates that we should use the one standard + format. */ -#define SHORT_NAMES 1 -/** Flag for event_format_t. Indicates that we should use the new - * name format of $hexdigest[=~]nickname - */ -#define LONG_NAMES 2 -#define ALL_NAMES (SHORT_NAMES|LONG_NAMES) -/** Flag for event_format_t. Indicates that we should use the new event - * format where extra event fields are allowed using a NAME=VAL format. */ -#define EXTENDED_FORMAT 4 -/** Flag for event_format_t. Indicates that we are using the old event format - * where extra fields aren't allowed. */ -#define NONEXTENDED_FORMAT 8 -#define ALL_FORMATS (EXTENDED_FORMAT|NONEXTENDED_FORMAT) +#define ALL_FORMATS 1 /** Bit field of flags to select how to format a controller event. Recognized - * flags are SHORT_NAMES, LONG_NAMES, EXTENDED_FORMAT, NONEXTENDED_FORMAT. */ + * flag is ALL_FORMATS. */ typedef int event_format_t; static void connection_printf_to_buf(control_connection_t *conn, @@ -123,9 +101,6 @@ static void send_control_done(control_connection_t *conn); static void send_control_event(uint16_t event, event_format_t which, const char *format, ...) CHECK_PRINTF(3,4); -static void send_control_event_extended(uint16_t event, event_format_t which, - const char *format, ...) - CHECK_PRINTF(3,4); static int handle_control_setconf(control_connection_t *conn, uint32_t len, char *body); static int handle_control_resetconf(control_connection_t *conn, uint32_t len, @@ -174,7 +149,7 @@ static int handle_control_usefeature(control_connection_t *conn, const char *body); static int write_stream_target_to_buf(edge_connection_t *conn, char *buf, size_t len); -static void orconn_target_get_name(int long_names, char *buf, size_t len, +static void orconn_target_get_name(char *buf, size_t len, or_connection_t *conn); static char *get_cookie_file(void); @@ -214,25 +189,19 @@ control_update_global_event_mask(void) { smartlist_t *conns = get_connection_array(); event_mask_t old_mask, new_mask; - old_mask = global_event_mask1short; - old_mask |= global_event_mask1long; + old_mask = global_event_mask; - global_event_mask1short = 0; - global_event_mask1long = 0; + global_event_mask = 0; SMARTLIST_FOREACH(conns, connection_t *, _conn, { if (_conn->type == CONN_TYPE_CONTROL && STATE_IS_OPEN(_conn->state)) { control_connection_t *conn = TO_CONTROL_CONN(_conn); - if (conn->use_long_names) - global_event_mask1long |= conn->event_mask; - else - global_event_mask1short |= conn->event_mask; + global_event_mask |= conn->event_mask; } }); - new_mask = global_event_mask1short; - new_mask |= global_event_mask1long; + new_mask = global_event_mask; /* Handle the aftermath. Set up the log callback to tell us only what * we want to hear...*/ @@ -542,28 +511,15 @@ send_control_event_string(uint16_t event, event_format_t which, const char *msg) { smartlist_t *conns = get_connection_array(); + (void)which; tor_assert(event >= _EVENT_MIN && event <= _EVENT_MAX); - SMARTLIST_FOREACH(conns, connection_t *, conn, - { + SMARTLIST_FOREACH_BEGIN(conns, connection_t *, conn) { if (conn->type == CONN_TYPE_CONTROL && !conn->marked_for_close && conn->state == CONTROL_CONN_STATE_OPEN) { control_connection_t *control_conn = TO_CONTROL_CONN(conn); - if (control_conn->use_long_names) { - if (!(which & LONG_NAMES)) - continue; - } else { - if (!(which & SHORT_NAMES)) - continue; - } - if (control_conn->use_extended_events) { - if (!(which & EXTENDED_FORMAT)) - continue; - } else { - if (!(which & NONEXTENDED_FORMAT)) - continue; - } + if (control_conn->event_mask & (1<event. The event's body is created by the printf-style format in * format, and other arguments as provided. * - * If extended is true, and the format contains a single '@' character, - * it will be replaced with a space and all text after that character will be - * sent only to controllers that have enabled extended events. - * * Currently the length of the message is limited to 1024 (including the * ending \\r\\n\\0). */ static void -send_control_event_impl(uint16_t event, event_format_t which, int extended, - const char *format, va_list ap) +send_control_event_impl(uint16_t event, event_format_t which, + const char *format, va_list ap) { /* This is just a little longer than the longest allowed log message */ #define SEND_CONTROL1_EVENT_BUFFERSIZE 10064 int r; char buf[SEND_CONTROL1_EVENT_BUFFERSIZE]; size_t len; - char *cp; r = tor_vsnprintf(buf, sizeof(buf), format, ap); if (r<0) { @@ -618,15 +569,7 @@ send_control_event_impl(uint16_t event, event_format_t which, int extended, buf[SEND_CONTROL1_EVENT_BUFFERSIZE-3] = '\r'; } - if (extended && (cp = strchr(buf, '@'))) { - which &= ~ALL_FORMATS; - *cp = ' '; - send_control_event_string(event, which|EXTENDED_FORMAT, buf); - memcpy(cp, "\r\n\0", 3); - send_control_event_string(event, which|NONEXTENDED_FORMAT, buf); - } else { - send_control_event_string(event, which|ALL_FORMATS, buf); - } + send_control_event_string(event, which|ALL_FORMATS, buf); } /** Send an event to all v1 controllers that are listening for code @@ -641,27 +584,7 @@ send_control_event(uint16_t event, event_format_t which, { va_list ap; va_start(ap, format); - send_control_event_impl(event, which, 0, format, ap); - va_end(ap); -} - -/** Send an event to all v1 controllers that are listening for code - * event. The event's body is created by the printf-style format in - * format, and other arguments as provided. - * - * If the format contains a single '@' character, it will be replaced with a - * space and all text after that character will be sent only to controllers - * that have enabled extended events. - * - * Currently the length of the message is limited to 1024 (including the - * ending \\n\\r\\0. */ -static void -send_control_event_extended(uint16_t event, event_format_t which, - const char *format, ...) -{ - va_list ap; - va_start(ap, format); - send_control_event_impl(event, which, 1, format, ap); + send_control_event_impl(event, which, format, ap); va_end(ap); } @@ -948,7 +871,6 @@ handle_control_setevents(control_connection_t *conn, uint32_t len, { uint16_t event_code; uint32_t event_mask = 0; - unsigned int extended = 0; smartlist_t *events = smartlist_create(); (void) len; @@ -958,7 +880,6 @@ handle_control_setevents(control_connection_t *conn, uint32_t len, SMARTLIST_FOREACH_BEGIN(events, const char *, ev) { if (!strcasecmp(ev, "EXTENDED")) { - extended = 1; continue; } else if (!strcasecmp(ev, "CIRC")) event_code = EVENT_CIRCUIT_STATUS; @@ -1016,8 +937,6 @@ handle_control_setevents(control_connection_t *conn, uint32_t len, smartlist_free(events); conn->event_mask = event_mask; - if (extended) - conn->use_extended_events = 1; control_update_global_event_mask(); send_control_done(conn); @@ -1463,6 +1382,7 @@ static int getinfo_helper_dir(control_connection_t *control_conn, const char *question, char **answer) { + (void) control_conn; if (!strcmpstart(question, "desc/id/")) { routerinfo_t *ri = router_get_by_hexdigest(question+strlen("desc/id/")); if (ri) { @@ -1597,10 +1517,8 @@ getinfo_helper_dir(control_connection_t *control_conn, } } else if (!strcmp(question, "network-status")) { /* v1 */ routerlist_t *routerlist = router_get_routerlist(); - int verbose = control_conn->use_long_names; if (!routerlist || !routerlist->routers || - list_server_status_v1(routerlist->routers, answer, - verbose ? 2 : 1) < 0) { + list_server_status_v1(routerlist->routers, answer, 1) < 0) { return -1; } } else if (!strcmpstart(question, "extra-info/digest/")) { @@ -1636,6 +1554,7 @@ static int getinfo_helper_events(control_connection_t *control_conn, const char *question, char **answer) { + (void) control_conn; if (!strcmp(question, "circuit-status")) { circuit_t *circ; smartlist_t *status = smartlist_create(); @@ -1646,10 +1565,9 @@ getinfo_helper_events(control_connection_t *control_conn, const char *purpose; if (! CIRCUIT_IS_ORIGIN(circ) || circ->marked_for_close) continue; - if (control_conn->use_long_names) - path = circuit_list_path_for_controller(TO_ORIGIN_CIRCUIT(circ)); - else - path = circuit_list_path(TO_ORIGIN_CIRCUIT(circ),0); + + path = circuit_list_path_for_controller(TO_ORIGIN_CIRCUIT(circ)); + if (circ->state == CIRCUIT_STATE_OPEN) state = "BUILT"; else if (strlen(path)) @@ -1727,8 +1645,7 @@ getinfo_helper_events(control_connection_t *control_conn, } else if (!strcmp(question, "orconn-status")) { smartlist_t *conns = get_connection_array(); smartlist_t *status = smartlist_create(); - SMARTLIST_FOREACH(conns, connection_t *, base_conn, - { + SMARTLIST_FOREACH_BEGIN(conns, connection_t *, base_conn) { const char *state; char *s; char name[128]; @@ -1743,13 +1660,12 @@ getinfo_helper_events(control_connection_t *control_conn, state = "LAUNCHED"; else state = "NEW"; - orconn_target_get_name(control_conn->use_long_names, name, sizeof(name), - conn); + orconn_target_get_name(name, sizeof(name), conn); slen = strlen(name)+strlen(state)+2; s = tor_malloc(slen+1); tor_snprintf(s, slen, "%s %s", name, state); smartlist_add(status, s); - }); + } SMARTLIST_FOREACH_END(base_conn); *answer = smartlist_join_strings(status, "\r\n", 0, NULL); SMARTLIST_FOREACH(status, char *, cp, tor_free(cp)); smartlist_free(status); @@ -2688,7 +2604,6 @@ handle_control_usefeature(control_connection_t *conn, const char *body) { smartlist_t *args; - int verbose_names = 0, extended_events = 0; int bad = 0; (void) len; /* body is nul-terminated; it's safe to ignore the length */ args = smartlist_create(); @@ -2696,9 +2611,9 @@ handle_control_usefeature(control_connection_t *conn, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); SMARTLIST_FOREACH(args, const char *, arg, { if (!strcasecmp(arg, "VERBOSE_NAMES")) - verbose_names = 1; + ; else if (!strcasecmp(arg, "EXTENDED_EVENTS")) - extended_events = 1; + ; else { connection_printf_to_buf(conn, "552 Unrecognized feature \"%s\"\r\n", arg); @@ -2708,12 +2623,6 @@ handle_control_usefeature(control_connection_t *conn, }); if (!bad) { - if (verbose_names) { - conn->use_long_names = 1; - control_update_global_event_mask(); - } - if (extended_events) - conn->use_extended_events = 1; send_control_done(conn); } @@ -3011,20 +2920,11 @@ control_event_circuit_status(origin_circuit_t *circ, circuit_status_event_t tp, tor_free(reason); } - if (EVENT_IS_INTERESTING1S(EVENT_CIRCUIT_STATUS)) { - char *path = circuit_list_path(circ,0); - const char *sp = strlen(path) ? " " : ""; - send_control_event_extended(EVENT_CIRCUIT_STATUS, SHORT_NAMES, - "650 CIRC %lu %s%s%s@%s\r\n", - (unsigned long)circ->global_identifier, - status, sp, path, extended_buf); - tor_free(path); - } - if (EVENT_IS_INTERESTING1L(EVENT_CIRCUIT_STATUS)) { + { char *vpath = circuit_list_path_for_controller(circ); const char *sp = strlen(vpath) ? " " : ""; - send_control_event_extended(EVENT_CIRCUIT_STATUS, LONG_NAMES, - "650 CIRC %lu %s%s%s@%s\r\n", + send_control_event(EVENT_CIRCUIT_STATUS, ALL_FORMATS, + "650 CIRC %lu %s%s%s %s\r\n", (unsigned long)circ->global_identifier, status, sp, vpath, extended_buf); tor_free(vpath); @@ -3160,8 +3060,8 @@ control_event_stream_status(edge_connection_t *conn, stream_status_event_t tp, circ = circuit_get_by_edge_conn(conn); if (circ && CIRCUIT_IS_ORIGIN(circ)) origin_circ = TO_ORIGIN_CIRCUIT(circ); - send_control_event_extended(EVENT_STREAM_STATUS, ALL_NAMES, - "650 STREAM "U64_FORMAT" %s %lu %s@%s%s%s\r\n", + send_control_event(EVENT_STREAM_STATUS, ALL_FORMATS, + "650 STREAM "U64_FORMAT" %s %lu %s %s%s%s\r\n", U64_PRINTF_ARG(conn->_base.global_identifier), status, origin_circ? (unsigned long)origin_circ->global_identifier : 0ul, @@ -3174,30 +3074,21 @@ control_event_stream_status(edge_connection_t *conn, stream_status_event_t tp, /** Figure out the best name for the target router of an OR connection * conn, and write it into the len-character buffer - * name. Use verbose names if long_names is set. */ + * name. */ static void -orconn_target_get_name(int long_names, - char *name, size_t len, or_connection_t *conn) +orconn_target_get_name(char *name, size_t len, or_connection_t *conn) { - if (! long_names) { - if (conn->nickname) - strlcpy(name, conn->nickname, len); - else - tor_snprintf(name, len, "%s:%d", - conn->_base.address, conn->_base.port); + routerinfo_t *ri = router_get_by_digest(conn->identity_digest); + if (ri) { + tor_assert(len > MAX_VERBOSE_NICKNAME_LEN); + router_get_verbose_nickname(name, ri); + } else if (! tor_digest_is_zero(conn->identity_digest)) { + name[0] = '$'; + base16_encode(name+1, len-1, conn->identity_digest, + DIGEST_LEN); } else { - routerinfo_t *ri = router_get_by_digest(conn->identity_digest); - if (ri) { - tor_assert(len > MAX_VERBOSE_NICKNAME_LEN); - router_get_verbose_nickname(name, ri); - } else if (! tor_digest_is_zero(conn->identity_digest)) { - name[0] = '$'; - base16_encode(name+1, len-1, conn->identity_digest, - DIGEST_LEN); - } else { - tor_snprintf(name, len, "%s:%d", - conn->_base.address, conn->_base.port); - } + tor_snprintf(name, len, "%s:%d", + conn->_base.address, conn->_base.port); } } @@ -3236,24 +3127,13 @@ control_event_or_conn_status(or_connection_t *conn, or_conn_status_event_t tp, reason ? " " : "", ncircs); } - if (EVENT_IS_INTERESTING1S(EVENT_OR_CONN_STATUS)) { - orconn_target_get_name(0, name, sizeof(name), conn); - send_control_event_extended(EVENT_OR_CONN_STATUS, SHORT_NAMES, - "650 ORCONN %s %s@%s%s%s\r\n", - name, status, - reason ? "REASON=" : "", - orconn_end_reason_to_control_string(reason), - ncircs_buf); - } - if (EVENT_IS_INTERESTING1L(EVENT_OR_CONN_STATUS)) { - orconn_target_get_name(1, name, sizeof(name), conn); - send_control_event_extended(EVENT_OR_CONN_STATUS, LONG_NAMES, - "650 ORCONN %s %s@%s%s%s\r\n", - name, status, - reason ? "REASON=" : "", - orconn_end_reason_to_control_string(reason), - ncircs_buf); - } + orconn_target_get_name(name, sizeof(name), conn); + send_control_event(EVENT_OR_CONN_STATUS, ALL_FORMATS, + "650 ORCONN %s %s %s%s%s\r\n", + name, status, + reason ? "REASON=" : "", + orconn_end_reason_to_control_string(reason), + ncircs_buf); return 0; } @@ -3297,7 +3177,7 @@ control_event_stream_bandwidth_used(void) if (!edge_conn->n_read && !edge_conn->n_written) continue; - send_control_event(EVENT_STREAM_BANDWIDTH_USED, ALL_NAMES, + send_control_event(EVENT_STREAM_BANDWIDTH_USED, ALL_FORMATS, "650 STREAM_BW "U64_FORMAT" %lu %lu\r\n", U64_PRINTF_ARG(edge_conn->_base.global_identifier), (unsigned long)edge_conn->n_read, @@ -3317,7 +3197,7 @@ int control_event_bandwidth_used(uint32_t n_read, uint32_t n_written) { if (EVENT_IS_INTERESTING(EVENT_BANDWIDTH_USED)) { - send_control_event(EVENT_BANDWIDTH_USED, ALL_NAMES, + send_control_event(EVENT_BANDWIDTH_USED, ALL_FORMATS, "650 BW %lu %lu\r\n", (unsigned long)n_read, (unsigned long)n_written); @@ -3382,7 +3262,7 @@ control_event_logmsg(int severity, uint32_t domain, const char *msg) default: s = "UnknownLogSeverity"; break; } ++disable_log_messages; - send_control_event(event, ALL_NAMES, "650 %s %s\r\n", s, b?b:msg); + send_control_event(event, ALL_FORMATS, "650 %s %s\r\n", s, b?b:msg); --disable_log_messages; tor_free(b); } @@ -3395,31 +3275,12 @@ control_event_logmsg(int severity, uint32_t domain, const char *msg) int control_event_descriptors_changed(smartlist_t *routers) { - size_t len; char *msg; - smartlist_t *identities = NULL; - char buf[HEX_DIGEST_LEN+1]; if (!EVENT_IS_INTERESTING(EVENT_NEW_DESC)) return 0; - if (EVENT_IS_INTERESTING1S(EVENT_NEW_DESC)) { - identities = smartlist_create(); - SMARTLIST_FOREACH(routers, routerinfo_t *, r, - { - base16_encode(buf,sizeof(buf),r->cache_info.identity_digest,DIGEST_LEN); - smartlist_add(identities, tor_strdup(buf)); - }); - } - if (EVENT_IS_INTERESTING1S(EVENT_NEW_DESC)) { - char *ids = smartlist_join_strings(identities, " ", 0, &len); - size_t ids_len = strlen(ids)+32; - msg = tor_malloc(ids_len); - tor_snprintf(msg, ids_len, "650 NEWDESC %s\r\n", ids); - send_control_event_string(EVENT_NEW_DESC, SHORT_NAMES|ALL_FORMATS, msg); - tor_free(ids); - tor_free(msg); - } - if (EVENT_IS_INTERESTING1L(EVENT_NEW_DESC)) { + + { smartlist_t *names = smartlist_create(); char *ids; size_t names_len; @@ -3432,16 +3293,12 @@ control_event_descriptors_changed(smartlist_t *routers) names_len = strlen(ids)+32; msg = tor_malloc(names_len); tor_snprintf(msg, names_len, "650 NEWDESC %s\r\n", ids); - send_control_event_string(EVENT_NEW_DESC, LONG_NAMES|ALL_FORMATS, msg); + send_control_event_string(EVENT_NEW_DESC, ALL_FORMATS, msg); tor_free(ids); tor_free(msg); SMARTLIST_FOREACH(names, char *, cp, tor_free(cp)); smartlist_free(names); } - if (identities) { - SMARTLIST_FOREACH(identities, char *, cp, tor_free(cp)); - smartlist_free(identities); - } return 0; } @@ -3458,17 +3315,17 @@ control_event_address_mapped(const char *from, const char *to, time_t expires, return 0; if (expires < 3 || expires == TIME_MAX) - send_control_event_extended(EVENT_ADDRMAP, ALL_NAMES, - "650 ADDRMAP %s %s NEVER@%s\r\n", from, to, + send_control_event(EVENT_ADDRMAP, ALL_FORMATS, + "650 ADDRMAP %s %s NEVER %s\r\n", from, to, error?error:""); else { char buf[ISO_TIME_LEN+1]; char buf2[ISO_TIME_LEN+1]; format_local_iso_time(buf,expires); format_iso_time(buf2,expires); - send_control_event_extended(EVENT_ADDRMAP, ALL_NAMES, + send_control_event(EVENT_ADDRMAP, ALL_FORMATS, "650 ADDRMAP %s %s \"%s\"" - "@%s%sEXPIRES=\"%s\"\r\n", + " %s%sEXPIRES=\"%s\"\r\n", from, to, buf, error?error:"", error?" ":"", buf2); @@ -3508,9 +3365,9 @@ control_event_or_authdir_new_descriptor(const char *action, buf = tor_malloc(totallen); strlcpy(buf, firstline, totallen); strlcpy(buf+strlen(firstline), esc, totallen); - send_control_event_string(EVENT_AUTHDIR_NEWDESCS, ALL_NAMES|ALL_FORMATS, + send_control_event_string(EVENT_AUTHDIR_NEWDESCS, ALL_FORMATS, buf); - send_control_event_string(EVENT_AUTHDIR_NEWDESCS, ALL_NAMES|ALL_FORMATS, + send_control_event_string(EVENT_AUTHDIR_NEWDESCS, ALL_FORMATS, "650 OK\r\n"); tor_free(esc); tor_free(buf); @@ -3548,8 +3405,8 @@ control_event_networkstatus_changed_helper(smartlist_t *statuses, SMARTLIST_FOREACH(strs, char *, cp, tor_free(cp)); smartlist_free(strs); tor_free(s); - send_control_event_string(event, ALL_NAMES|ALL_FORMATS, esc); - send_control_event_string(event, ALL_NAMES|ALL_FORMATS, + send_control_event_string(event, ALL_FORMATS, esc); + send_control_event_string(event, ALL_FORMATS, "650 OK\r\n"); tor_free(esc); @@ -3598,7 +3455,7 @@ control_event_networkstatus_changed_single(routerstatus_t *rs) int control_event_my_descriptor_changed(void) { - send_control_event(EVENT_DESCCHANGED, ALL_NAMES, "650 DESCCHANGED\r\n"); + send_control_event(EVENT_DESCCHANGED, ALL_FORMATS, "650 DESCCHANGED\r\n"); return 0; } @@ -3646,7 +3503,7 @@ control_event_status(int type, int severity, const char *format, va_list args) return -1; } - send_control_event_impl(type, ALL_NAMES|ALL_FORMATS, 0, format_buf, args); + send_control_event_impl(type, ALL_FORMATS, format_buf, args); return 0; } @@ -3710,7 +3567,7 @@ control_event_guard(const char *nickname, const char *digest, if (!EVENT_IS_INTERESTING(EVENT_GUARD)) return 0; - if (EVENT_IS_INTERESTING1L(EVENT_GUARD)) { + { char buf[MAX_VERBOSE_NICKNAME_LEN+1]; routerinfo_t *ri = router_get_by_digest(digest); if (ri) { @@ -3718,13 +3575,9 @@ control_event_guard(const char *nickname, const char *digest, } else { tor_snprintf(buf, sizeof(buf), "$%s~%s", hbuf, nickname); } - send_control_event(EVENT_GUARD, LONG_NAMES, + send_control_event(EVENT_GUARD, ALL_FORMATS, "650 GUARD ENTRY %s %s\r\n", buf, status); } - if (EVENT_IS_INTERESTING1S(EVENT_GUARD)) { - send_control_event(EVENT_GUARD, SHORT_NAMES, - "650 GUARD ENTRY $%s %s\r\n", hbuf, status); - } return 0; } diff --git a/src/or/dirserv.c b/src/or/dirserv.c index dde8959f88..f586076448 100644 --- a/src/or/dirserv.c +++ b/src/or/dirserv.c @@ -964,7 +964,6 @@ dirserv_set_router_is_running(routerinfo_t *router, time_t now) * *router_status_out. Return 0 on success, -1 on failure. * * If for_controller is true, include the routers with very old descriptors. - * If for_controller is >1, use the verbose nickname format. */ int list_server_status_v1(smartlist_t *routers, char **router_status_out, @@ -984,23 +983,22 @@ list_server_status_v1(smartlist_t *routers, char **router_status_out, rs_entries = smartlist_create(); - SMARTLIST_FOREACH(routers, routerinfo_t *, ri, - { + SMARTLIST_FOREACH_BEGIN(routers, routerinfo_t *, ri) { if (authdir) { /* Update router status in routerinfo_t. */ dirserv_set_router_is_running(ri, now); } - if (for_controller == 1 || ri->cache_info.published_on >= cutoff) - smartlist_add(rs_entries, list_single_server_status(ri, ri->is_running)); - else if (for_controller > 2) { + if (for_controller) { char name_buf[MAX_VERBOSE_NICKNAME_LEN+2]; char *cp = name_buf; if (!ri->is_running) *cp++ = '!'; router_get_verbose_nickname(cp, ri); smartlist_add(rs_entries, tor_strdup(name_buf)); + } else if (ri->cache_info.published_on >= cutoff) { + smartlist_add(rs_entries, list_single_server_status(ri, ri->is_running)); } - }); + } SMARTLIST_FOREACH_END(ri); *router_status_out = smartlist_join_strings(rs_entries, " ", 0, NULL); diff --git a/src/or/or.h b/src/or/or.h index d5b36c85a6..deb388ab26 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -1163,12 +1163,6 @@ typedef struct control_connection_t { uint32_t event_mask; /**< Bitfield: which events does this controller * care about? */ - unsigned int use_long_names:1; /**< True if we should use long nicknames - * on this (v1) connection. Only settable - * via v1 controllers. */ - /** For control connections only. If set, we send extended info with control - * events as appropriate. */ - unsigned int use_extended_events:1; /** True if we have sent a protocolinfo reply on this connection. */ unsigned int have_sent_protocolinfo:1; From 3b47be20ec377cd249904d24953d821b2dca5604 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 25 May 2009 12:55:59 -0400 Subject: [PATCH 055/187] Update newer control_event_stream_bandwidth code to revised control impl --- src/or/control.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/or/control.c b/src/or/control.c index 2726412d93..3e69bbfd41 100644 --- a/src/or/control.c +++ b/src/or/control.c @@ -3148,7 +3148,7 @@ control_event_stream_bandwidth(edge_connection_t *edge_conn) if (!edge_conn->n_read && !edge_conn->n_written) return 0; - send_control_event(EVENT_STREAM_BANDWIDTH_USED, ALL_NAMES, + send_control_event(EVENT_STREAM_BANDWIDTH_USED, ALL_FORMATS, "650 STREAM_BW "U64_FORMAT" %lu %lu\r\n", U64_PRINTF_ARG(edge_conn->_base.global_identifier), (unsigned long)edge_conn->n_read, From bf7e5d6553acd8d8ce37e80586083620d66d0160 Mon Sep 17 00:00:00 2001 From: Karsten Loesing Date: Tue, 26 May 2009 21:32:56 +0200 Subject: [PATCH 056/187] Some cleanups on geoip.c. Use two bitfields for last_seen and action in clientmap_entry_t rather than having both share a 32-bit. Also, documentation fixes. --- src/or/geoip.c | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/src/or/geoip.c b/src/or/geoip.c index eb0d7e43f4..fe847ed8d8 100644 --- a/src/or/geoip.c +++ b/src/or/geoip.c @@ -42,7 +42,7 @@ static strmap_t *country_idxplus1_by_lc_code = NULL; static smartlist_t *geoip_entries = NULL; /** Return the index of the country's entry in the GeoIP DB - * if it is a valid 2-letter country code, otherwise return zero. + * if it is a valid 2-letter country code, otherwise return -1. */ country_t geoip_get_country(const char *country) @@ -261,8 +261,8 @@ geoip_is_loaded(void) typedef struct clientmap_entry_t { HT_ENTRY(clientmap_entry_t) node; uint32_t ipaddr; - time_t last_seen; /* The last 2 bits of this value hold the client - * operation. */ + unsigned int last_seen_in_minutes:30; + unsigned int action:2; } clientmap_entry_t; #define ACTION_MASK 3 @@ -289,7 +289,7 @@ clientmap_entry_hash(const clientmap_entry_t *a) static INLINE int clientmap_entries_eq(const clientmap_entry_t *a, const clientmap_entry_t *b) { - return a->ipaddr == b->ipaddr; + return a->ipaddr == b->ipaddr && a->action == b->action; } HT_PROTOTYPE(clientmap, clientmap_entry_t, node, clientmap_entry_hash, @@ -298,7 +298,8 @@ HT_GENERATE(clientmap, clientmap_entry_t, node, clientmap_entry_hash, clientmap_entries_eq, 0.6, malloc, realloc, free); /** Note that we've seen a client connect from the IP addr (host order) - * at time now. Ignored by all but bridges. */ + * at time now. Ignored by all but bridges and directories if + * configured accordingly. */ void geoip_note_client_seen(geoip_client_action_t action, uint32_t addr, time_t now) @@ -337,20 +338,18 @@ geoip_note_client_seen(geoip_client_action_t action, current_request_period_starts += REQUEST_HIST_PERIOD; if (n_old_request_periods < REQUEST_HIST_LEN-1) ++n_old_request_periods; - } + } - /* We use the low 3 bits of the time to encode the action. Since we're - * potentially remembering tons of clients, we don't want to make - * clientmap_entry_t larger than it has to be. */ - now = (now & ~ACTION_MASK) | (((int)action) & ACTION_MASK); lookup.ipaddr = addr; + lookup.action = (int)action; ent = HT_FIND(clientmap, &client_history, &lookup); if (ent) { - ent->last_seen = now; + ent->last_seen_in_minutes = now / 60; } else { ent = tor_malloc_zero(sizeof(clientmap_entry_t)); ent->ipaddr = addr; - ent->last_seen = now; + ent->last_seen_in_minutes = now / 60; + ent->action = (int)action; HT_INSERT(clientmap, &client_history, ent); } @@ -377,8 +376,8 @@ geoip_note_client_seen(geoip_client_action_t action, static int _remove_old_client_helper(struct clientmap_entry_t *ent, void *_cutoff) { - time_t cutoff = *(time_t*)_cutoff; - if (ent->last_seen < cutoff) { + time_t cutoff = *(time_t*)_cutoff / 60; + if (ent->last_seen_in_minutes < cutoff) { tor_free(ent); return 1; } else { @@ -477,7 +476,7 @@ geoip_get_client_history(time_t now, geoip_client_action_t action) #endif HT_FOREACH(ent, clientmap, &client_history) { int country; - if (((*ent)->last_seen & ACTION_MASK) != (int)action) + if ((*ent)->action != (int)action) continue; country = geoip_get_country_by_ip((*ent)->ipaddr); if (country < 0) From 54c97c9133ebbeea6aed9b4526b19fbc0d88377b Mon Sep 17 00:00:00 2001 From: Karsten Loesing Date: Wed, 27 May 2009 17:52:46 +0200 Subject: [PATCH 057/187] Change the way how directories that are configured with --enable-geoip-stats write geoip stats to disk. - Write geoip stats to disk every 24 hours, not every hour. - Remove configuration options and define reasonable defaults. - Clear history of client requests every 24 hours (which wasn't done at all before). --- ChangeLog | 2 ++ src/or/config.c | 8 ++++---- src/or/geoip.c | 36 ++++++++++++++++++++++++------------ src/or/main.c | 8 -------- src/or/or.h | 12 ++++++++++-- src/or/router.c | 7 +++++-- 6 files changed, 45 insertions(+), 28 deletions(-) diff --git a/ChangeLog b/ChangeLog index 8c6d7a652e..036d0e10d8 100644 --- a/ChangeLog +++ b/ChangeLog @@ -6,6 +6,8 @@ Changes in version 0.2.2.1-alpha - 2009-??-?? - The memarea code now uses a sentinel value at the end of each area to make sure nothing writes beyond the end of an area. This might help debug some conceivable causes of bug 930. + - Directories that are configured with the --enable-geoip-stats flag + now write their GeoIP stats to disk exactly every 24 hours. o Deprecated and removed features: - The controller no longer accepts the old obsolete "addr-mappings/" diff --git a/src/or/config.c b/src/or/config.c index 0dd0931ece..e462b3e76a 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -187,10 +187,10 @@ static config_var_t _option_vars[] = { V(DirPortFrontPage, FILENAME, NULL), OBSOLETE("DirPostPeriod"), #ifdef ENABLE_GEOIP_STATS - V(DirRecordUsageByCountry, BOOL, "0"), - V(DirRecordUsageGranularity, UINT, "4"), - V(DirRecordUsageRetainIPs, INTERVAL, "14 days"), - V(DirRecordUsageSaveInterval, INTERVAL, "6 hours"), + OBSOLETE("DirRecordUsageByCountry"), + OBSOLETE("DirRecordUsageGranularity"), + OBSOLETE("DirRecordUsageRetainIPs"), + OBSOLETE("DirRecordUsageSaveInterval"), #endif VAR("DirServer", LINELIST, DirServers, NULL), V(DNSPort, UINT, "0"), diff --git a/src/or/geoip.c b/src/or/geoip.c index fe847ed8d8..6aff1f6233 100644 --- a/src/or/geoip.c +++ b/src/or/geoip.c @@ -12,6 +12,7 @@ #include "ht.h" static void clear_geoip_db(void); +static void dump_geoip_stats(void); /** An entry from the GeoIP file: maps an IP range to a country. */ typedef struct geoip_entry_t { @@ -21,9 +22,9 @@ typedef struct geoip_entry_t { } geoip_entry_t; /** For how many periods should we remember per-country request history? */ -#define REQUEST_HIST_LEN 3 +#define REQUEST_HIST_LEN 1 /** How long are the periods for which we should remember request history? */ -#define REQUEST_HIST_PERIOD (8*60*60) +#define REQUEST_HIST_PERIOD (24*60*60) /** A per-country record for GeoIP request history. */ typedef struct geoip_country_t { @@ -313,8 +314,7 @@ geoip_note_client_seen(geoip_client_action_t action, #ifndef ENABLE_GEOIP_STATS return; #else - if (options->BridgeRelay || options->BridgeAuthoritativeDir || - !options->DirRecordUsageByCountry) + if (options->BridgeRelay || options->BridgeAuthoritativeDir) return; #endif } @@ -327,6 +327,14 @@ geoip_note_client_seen(geoip_client_action_t action, current_request_period_starts = now; break; } + /* Also discard all items in the client history that are too old. + * (This only works here because bridge and directory stats are + * independent. Otherwise, we'd only want to discard those items + * with action GEOIP_CLIENT_NETWORKSTATUS{_V2}.) */ + geoip_remove_old_clients(current_request_period_starts); + /* Before rotating, write the current stats to disk. */ + dump_geoip_stats(); + /* Now rotate request period */ SMARTLIST_FOREACH(geoip_countries, geoip_country_t *, c, { memmove(&c->n_v2_ns_requests[0], &c->n_v2_ns_requests[1], sizeof(uint32_t)*(REQUEST_HIST_LEN-1)); @@ -458,9 +466,13 @@ char * geoip_get_client_history(time_t now, geoip_client_action_t action) { char *result = NULL; + unsigned min_observation_time = GEOIP_MIN_OBSERVATION_TIME; +#ifdef ENABLE_GEOIP_STATS + min_observation_time = DIR_RECORD_USAGE_MIN_OBSERVATION_TIME; +#endif if (!geoip_is_loaded()) return NULL; - if (client_history_starts < (now - GEOIP_MIN_OBSERVATION_TIME)) { + if (client_history_starts < (now - min_observation_time)) { char buf[32]; smartlist_t *chunks = NULL; smartlist_t *entries = NULL; @@ -471,8 +483,7 @@ geoip_get_client_history(time_t now, geoip_client_action_t action) unsigned total = 0; unsigned granularity = IP_GRANULARITY; #ifdef ENABLE_GEOIP_STATS - if (get_options()->DirRecordUsageByCountry) - granularity = get_options()->DirRecordUsageGranularity; + granularity = DIR_RECORD_USAGE_GRANULARITY; #endif HT_FOREACH(ent, clientmap, &client_history) { int country; @@ -538,12 +549,13 @@ geoip_get_request_history(time_t now, geoip_client_action_t action) smartlist_t *entries, *strings; char *result; unsigned granularity = IP_GRANULARITY; + unsigned min_observation_time = GEOIP_MIN_OBSERVATION_TIME; #ifdef ENABLE_GEOIP_STATS - if (get_options()->DirRecordUsageByCountry) - granularity = get_options()->DirRecordUsageGranularity; + granularity = DIR_RECORD_USAGE_GRANULARITY; + min_observation_time = DIR_RECORD_USAGE_MIN_OBSERVATION_TIME; #endif - if (client_history_starts >= (now - GEOIP_MIN_OBSERVATION_TIME)) + if (client_history_starts >= (now - min_observation_time)) return NULL; if (action != GEOIP_CLIENT_NETWORKSTATUS && action != GEOIP_CLIENT_NETWORKSTATUS_V2) @@ -584,7 +596,7 @@ geoip_get_request_history(time_t now, geoip_client_action_t action) } /** Store all our geoip statistics into $DATADIR/geoip-stats. */ -void +static void dump_geoip_stats(void) { #ifdef ENABLE_GEOIP_STATS @@ -601,7 +613,7 @@ dump_geoip_stats(void) data_v3 = geoip_get_client_history(now, GEOIP_CLIENT_NETWORKSTATUS); format_iso_time(since, geoip_get_history_start()); format_iso_time(written, now); - out = start_writing_to_stdio_file(filename, OPEN_FLAGS_REPLACE, + out = start_writing_to_stdio_file(filename, OPEN_FLAGS_APPEND, 0600, &open_file); if (!out) goto done; diff --git a/src/or/main.c b/src/or/main.c index 61e9d9f6a7..9163b8c26b 100644 --- a/src/or/main.c +++ b/src/or/main.c @@ -807,7 +807,6 @@ run_scheduled_events(time_t now) static time_t time_to_clean_caches = 0; static time_t time_to_recheck_bandwidth = 0; static time_t time_to_check_for_expired_networkstatus = 0; - static time_t time_to_dump_geoip_stats = 0; static time_t time_to_retry_dns_init = 0; or_options_t *options = get_options(); int i; @@ -935,13 +934,6 @@ run_scheduled_events(time_t now) time_to_check_for_expired_networkstatus = now + CHECK_EXPIRED_NS_INTERVAL; } - if (time_to_dump_geoip_stats < now) { -#define DUMP_GEOIP_STATS_INTERVAL (60*60); - if (time_to_dump_geoip_stats) - dump_geoip_stats(); - time_to_dump_geoip_stats = now + DUMP_GEOIP_STATS_INTERVAL; - } - /* Remove old information from rephist and the rend cache. */ if (time_to_clean_caches < now) { rep_history_clean(now - options->RephistTrackTime); diff --git a/src/or/or.h b/src/or/or.h index deb388ab26..c13ca0b50f 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -2528,7 +2528,7 @@ typedef struct { * the bridge authority guess which countries have blocked access to us. */ int BridgeRecordUsageByCountry; -#ifdef ENABLE_GEOIP_STATS +#if 0 /** If true, and Tor is built with GEOIP_STATS support, and we're a * directory, record how many directory requests we get from each country. */ int DirRecordUsageByCountry; @@ -3583,6 +3583,15 @@ int dnsserv_launch_request(const char *name, int is_reverse); /********************************* geoip.c **************************/ +/** Round all GeoIP results to the next multiple of this value, to avoid + * leaking information. */ +#define DIR_RECORD_USAGE_GRANULARITY 8 +/** Time interval: Flush geoip data to disk this often. */ +#define DIR_RECORD_USAGE_RETAIN_IPS (24*60*60) +/** How long do we have to have observed per-country request history before + * we are willing to talk about it? */ +#define DIR_RECORD_USAGE_MIN_OBSERVATION_TIME (24*60*60) + #ifdef GEOIP_PRIVATE int geoip_parse_entry(const char *line); #endif @@ -3614,7 +3623,6 @@ char *geoip_get_request_history(time_t now, geoip_client_action_t action); int getinfo_helper_geoip(control_connection_t *control_conn, const char *question, char **answer); void geoip_free_all(void); -void dump_geoip_stats(void); /********************************* hibernate.c **********************/ diff --git a/src/or/router.c b/src/or/router.c index da922b7508..6d6f8dad39 100644 --- a/src/or/router.c +++ b/src/or/router.c @@ -1915,10 +1915,13 @@ extrainfo_get_client_geoip_summary(time_t now) static time_t last_purged_at = 0; int geoip_purge_interval = 48*60*60; #ifdef ENABLE_GEOIP_STATS - if (get_options()->DirRecordUsageByCountry) - geoip_purge_interval = get_options()->DirRecordUsageRetainIPs; + geoip_purge_interval = DIR_RECORD_USAGE_RETAIN_IPS; #endif if (now > last_purged_at+geoip_purge_interval) { + /* (Note that this also discards items in the client history with + * action GEOIP_CLIENT_NETWORKSTATUS{_V2}, which doesn't matter + * because bridge and directory stats are independent. Keep in mind + * for future extensions, though.) */ geoip_remove_old_clients(now-geoip_purge_interval); last_purged_at = now; } From 181674e9bba9f4a0155f6172e437931d9560219c Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 27 May 2009 14:19:24 -0400 Subject: [PATCH 058/187] Fix a signed/unsigned comparison warning --- src/or/geoip.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/or/geoip.c b/src/or/geoip.c index 6aff1f6233..64ced6ea90 100644 --- a/src/or/geoip.c +++ b/src/or/geoip.c @@ -466,7 +466,7 @@ char * geoip_get_client_history(time_t now, geoip_client_action_t action) { char *result = NULL; - unsigned min_observation_time = GEOIP_MIN_OBSERVATION_TIME; + int min_observation_time = GEOIP_MIN_OBSERVATION_TIME; #ifdef ENABLE_GEOIP_STATS min_observation_time = DIR_RECORD_USAGE_MIN_OBSERVATION_TIME; #endif @@ -549,7 +549,7 @@ geoip_get_request_history(time_t now, geoip_client_action_t action) smartlist_t *entries, *strings; char *result; unsigned granularity = IP_GRANULARITY; - unsigned min_observation_time = GEOIP_MIN_OBSERVATION_TIME; + int min_observation_time = GEOIP_MIN_OBSERVATION_TIME; #ifdef ENABLE_GEOIP_STATS granularity = DIR_RECORD_USAGE_GRANULARITY; min_observation_time = DIR_RECORD_USAGE_MIN_OBSERVATION_TIME; From e86ad6b7fb49a497fa75b79bc15d004b02ebd371 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 27 May 2009 14:33:44 -0400 Subject: [PATCH 059/187] Reject proposal 134 --- doc/spec/proposals/000-index.txt | 5 +++-- doc/spec/proposals/134-robust-voting.txt | 22 ++++++++++++++++++++-- doc/spec/proposals/reindex.py | 2 +- 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/doc/spec/proposals/000-index.txt b/doc/spec/proposals/000-index.txt index 26622e83a5..b2907e0607 100644 --- a/doc/spec/proposals/000-index.txt +++ b/doc/spec/proposals/000-index.txt @@ -54,7 +54,7 @@ Proposals by number: 131 Help users to verify they are using Tor [NEEDS-REVISION] 132 A Tor Web Service For Verifying Correct Browser Configuration [DRAFT] 133 Incorporate Unreachable ORs into the Tor Network [DRAFT] -134 More robust consensus voting with diverse authority sets [ACCEPTED] +134 More robust consensus voting with diverse authority sets [REJECTED] 135 Simplify Configuration of Private Tor Networks [CLOSED] 136 Mass authority migration with legacy keys [CLOSED] 137 Keep controllers informed as Tor bootstraps [CLOSED] @@ -115,7 +115,6 @@ Proposals by status: 110 Avoiding infinite length circuits [for 0.2.1.x] [in 0.2.1.3-alpha] 117 IPv6 exits [for 0.2.1.x] 118 Advertising multiple ORPorts at once [for 0.2.1.x] - 134 More robust consensus voting with diverse authority sets [for 0.2.2.x] 140 Provide diffs between consensuses [for 0.2.2.x] 147 Eliminate the need for v2 directories in generating v3 directories [for 0.2.1.x] 157 Make certificate downloads specific [for 0.2.1.x] @@ -167,3 +166,5 @@ Proposals by status: 120 Shutdown descriptors when Tor servers stop 128 Families of private bridges 142 Combine Introduction and Rendezvous Points + REJECTED: + 134 More robust consensus voting with diverse authority sets diff --git a/doc/spec/proposals/134-robust-voting.txt b/doc/spec/proposals/134-robust-voting.txt index 5d5e77fa3b..c5dfb3b47f 100644 --- a/doc/spec/proposals/134-robust-voting.txt +++ b/doc/spec/proposals/134-robust-voting.txt @@ -2,8 +2,10 @@ Filename: 134-robust-voting.txt Title: More robust consensus voting with diverse authority sets Author: Peter Palfrader Created: 2008-04-01 -Status: Accepted -Target: 0.2.2.x +Status: Rejected + +History: + 2009 May 27: Added note on rejecting this proposal -- Nick Overview: @@ -103,3 +105,19 @@ Possible Attacks/Open Issues/Some thinking required: Q: Can this ever force us to build a consensus with authorities we do not recognize? A: No, we can never build a fully connected set with them in step 3. + +------------------------------ + +I'm rejecting this proposal as insecure. + +Suppose that we have a clique of size N, and M hostile members in the +clique. If these hostile members stop declaring trust for up to M-1 +good members of the clique, the clique with the hostile members will +in it will be larger than the one without them. + +The M hostile members will constitute a majority of this new clique +when M > (N-(M-1)) / 2, or when M > (N + 1) / 3. This breaks our +requirement that an adversary must compromise a majority of authorities +in order to control the consensus. + +-- Nick diff --git a/doc/spec/proposals/reindex.py b/doc/spec/proposals/reindex.py index 2b4c02516b..980bc0659f 100755 --- a/doc/spec/proposals/reindex.py +++ b/doc/spec/proposals/reindex.py @@ -4,7 +4,7 @@ import re, os class Error(Exception): pass STATUSES = """DRAFT NEEDS-REVISION NEEDS-RESEARCH OPEN ACCEPTED META FINISHED - CLOSED SUPERSEDED DEAD""".split() + CLOSED SUPERSEDED DEAD REJECTED""".split() REQUIRED_FIELDS = [ "Filename", "Status", "Title" ] CONDITIONAL_FIELDS = { "OPEN" : [ "Target" ], "ACCEPTED" : [ "Target "], From 2c8b56c029988adf65c38fc34fdba6d91cd0608c Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 28 May 2009 12:48:38 -0400 Subject: [PATCH 060/187] Proposal 165: Easy migration for voting authority sets. This proposal tries to do what 134 tried to do, but more securely and with less whiz-bang NP-hard computation. Peter had the basic idea. --- doc/spec/proposals/000-index.txt | 2 + .../proposals/165-simple-robust-voting.txt | 119 ++++++++++++++++++ 2 files changed, 121 insertions(+) create mode 100644 doc/spec/proposals/165-simple-robust-voting.txt diff --git a/doc/spec/proposals/000-index.txt b/doc/spec/proposals/000-index.txt index b2907e0607..b26f382f6a 100644 --- a/doc/spec/proposals/000-index.txt +++ b/doc/spec/proposals/000-index.txt @@ -85,6 +85,7 @@ Proposals by number: 162 Publish the consensus in multiple flavors [OPEN] 163 Detecting whether a connection comes from a client [OPEN] 164 Reporting the status of server votes [OPEN] +165 Easy migration for voting authority sets [OPEN] Proposals by status: @@ -111,6 +112,7 @@ Proposals by status: 162 Publish the consensus in multiple flavors [for 0.2.2] 163 Detecting whether a connection comes from a client [for 0.2.2] 164 Reporting the status of server votes [for 0.2.2] + 165 Easy migration for voting authority sets ACCEPTED: 110 Avoiding infinite length circuits [for 0.2.1.x] [in 0.2.1.3-alpha] 117 IPv6 exits [for 0.2.1.x] diff --git a/doc/spec/proposals/165-simple-robust-voting.txt b/doc/spec/proposals/165-simple-robust-voting.txt new file mode 100644 index 0000000000..d8993c9e95 --- /dev/null +++ b/doc/spec/proposals/165-simple-robust-voting.txt @@ -0,0 +1,119 @@ +Filename: 165-simple-robust-voting.txt +Title: Easy migration for voting authority sets +Author: Nick Mathewson +Created: 2009-05-28 +Status: Open + +Overview: + + This proposal describes any easy-to-implement, easy-to-verify way to + change the set of authorities without creating a "flag day" situation. + +Motivation: + + From proposal 134 ("More robust consensus voting with diverse + authority sets") by Peter Palfrader: + + Right now there are about five authoritative directory servers + in the Tor network, tho this number is expected to rise to about + 15 eventually. + + Adding a new authority requires synchronized action from all + operators of directory authorities so that at any time during the + update at least half of all authorities are running and agree on + who is an authority. The latter requirement is there so that the + authorities can arrive at a common consensus: Each authority + builds the consensus based on the votes from all authorities it + recognizes, and so a different set of recognized authorities will + lead to a different consensus document. + + In response to this problem, proposal 134 suggested that every + candidate authority list in its vote whom it believes to be an + authority. These A-says-B-is-an-authority relationships form a + directed graph. Each authority then iteratively finds the largest + clique in the graph and remove it, until they find one containing + them. They vote with this clique. + + Proposal 134 had some problems: + + - It had a security problem in that M hostile authorities in a + clique could effectively kick out M-1 honest authorities. This + could enable a minority of the original authorities to take over. + + - It was too complex in its implications to analyze well: it took us + over a year to realize that it was insecure. + + - It tried to solve a bigger problem: general fragmentation of + authority trust. Really, all we wanted to have was the ability to + add and remove authorities without forcing a flag day. + +Proposed protocol design: + + A "Voting Set" is a set of authorities. Each authority has a list of + the voting sets it considers acceptable. These sets must always + contain the authority itself. Each authority lists all of these + voting sets in its votes. + + Authorities exchange votes with every other authority in any of their + voting sets. + + When it comes time to calculate a consensus, an authority votes with + whichever voting set it lists that is listed by the most members of + that set. + + For example, suppose authority A recognizes two sets, "A B C D" and + "A E F G H". Suppose that the first set is recognized by all of A, + B, C, and D, whereas the second set is recognized only by A, E, and + F. Because the first set is recognize by more of the authorities in + it than the other one, A will vote with the first set. + + Ties are broken in favor of some arbitrary function of the identity + keys of the authorities in the set. + +How to migrate authority sets: + + In steady state, each authority should list only the current actual + voting set as accepted. + + When we want to add an authority, we list two voting sets: one + containing all the old authorities, and one containing the old + authorities and the new authority too. Once all authorities are + listing the new set of authorities, they will start preferring that + set because of its size. + + When we want to remove an authority, we list two voting sets: one + containing all the authorities, and one omitting the authority we + want to remove. Once enough authorities list the new set as + acceptable, we start having authorities stop listing the old set. + Once there are more listing the new set than the old set, the new set + will win. + +Data format changes: + + Add a new 'voting-set' line to the vote document format. Allow it to + occur any number of times. Its format is: + + voting-set SP 'fingerprint' SP 'fingerprint' ... NL + + where each fingerprint is the hex fingerprint of an identity key of + an authority. Sort fingerprints in ascending order. + + When the consensus method is at least 'X' (decide this when we + implement the proposal), add this line to the consensus format as + well, before the first dir-source line. [This information is not + redundant with the dir-source sections in the consensus: If an + authority is recognized didn't vote, that authority will appear in + the voting-set line but not in the dir-source sections.] + + We don't need to list other information about authorities in our + vote. + +Migration issues: + + We should keep track somewhere of which Tor client versions + recognized which authorities. + +Acknowledgments: + + The design came out of an IRC conversation with Peter Palfrader. He + had the basic idea first. From 3599e9051e48f9a4eaa469e30df021aedc3429fc Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 28 May 2009 16:19:05 -0400 Subject: [PATCH 061/187] Restore changes from Karsten's "Remove unused rendversion parameters". These were made undone by a merge. --- src/or/rendclient.c | 40 ++++++++++++++-------------------------- 1 file changed, 14 insertions(+), 26 deletions(-) diff --git a/src/or/rendclient.c b/src/or/rendclient.c index fb50e8dc00..6fe3a6938e 100644 --- a/src/or/rendclient.c +++ b/src/or/rendclient.c @@ -455,14 +455,14 @@ directory_get_from_hs_dir(const char *desc_id, const rend_data_t *rend_query) * for the service ID query, start a directory connection to fetch a * new one. */ -void +static void rend_client_refetch_renddesc(const char *query) { if (!get_options()->FetchHidServDescriptors) return; log_info(LD_REND, "Fetching rendezvous descriptor for service %s", escaped_safe_str(query)); - if (connection_get_by_type_state_rendquery(CONN_TYPE_DIR, 0, query, 0)) { + if (connection_get_by_type_state_rendquery(CONN_TYPE_DIR, 0, query)) { log_info(LD_REND,"Would fetch a new renddesc here (for %s), but one is " "already in progress.", escaped_safe_str(query)); } else { @@ -524,8 +524,8 @@ rend_client_refetch_v2_renddesc(const rend_data_t *rend_query) log_info(LD_REND, "Could not pick one of the responsible hidden " "service directories to fetch descriptors, because " "we already tried them all unsuccessfully."); - /* Close pending connections (unless a v0 request is still going on). */ - rend_client_desc_trynow(rend_query->onion_address, 2); + /* Close pending connections. */ + rend_client_desc_trynow(rend_query->onion_address); return; } @@ -585,7 +585,7 @@ rend_client_remove_intro_point(extend_info_t *failed_intro, /* move all pending streams back to renddesc_wait */ while ((conn = connection_get_by_type_state_rendquery(CONN_TYPE_AP, AP_CONN_STATE_CIRCUIT_WAIT, - rend_query->onion_address, -1))) { + rend_query->onion_address))) { conn->state = AP_CONN_STATE_RENDDESC_WAIT; } @@ -694,24 +694,18 @@ rend_client_receive_rendezvous(origin_circuit_t *circ, const char *request, return -1; } -/** Find all the apconns in state AP_CONN_STATE_RENDDESC_WAIT that - * are waiting on query. If there's a working cache entry here - * with at least one intro point, move them to the next state. If - * rend_version is non-negative, fail connections that have - * requested query unless there are still descriptor fetch - * requests in progress for other descriptor versions than - * rend_version. - */ +/** Find all the apconns in state AP_CONN_STATE_RENDDESC_WAIT that are + * waiting on query. If there's a working cache entry here with at + * least one intro point, move them to the next state. */ void -rend_client_desc_trynow(const char *query, int rend_version) +rend_client_desc_trynow(const char *query) { edge_connection_t *conn; rend_cache_entry_t *entry; time_t now = time(NULL); smartlist_t *conns = get_connection_array(); - SMARTLIST_FOREACH(conns, connection_t *, _conn, - { + SMARTLIST_FOREACH_BEGIN(conns, connection_t *, _conn) { if (_conn->type != CONN_TYPE_AP || _conn->state != AP_CONN_STATE_RENDDESC_WAIT || _conn->marked_for_close) @@ -743,17 +737,11 @@ rend_client_desc_trynow(const char *query, int rend_version) connection_mark_unattached_ap(conn, END_STREAM_REASON_CANT_ATTACH); } } else { /* 404, or fetch didn't get that far */ - /* Unless there are requests for another descriptor version pending, - * close the connection. */ - if (rend_version >= 0 && - !connection_get_by_type_state_rendquery(CONN_TYPE_DIR, 0, query, - rend_version == 0 ? 2 : 0)) { - log_notice(LD_REND,"Closing stream for '%s.onion': hidden service is " - "unavailable (try again later).", safe_str(query)); - connection_mark_unattached_ap(conn, END_STREAM_REASON_RESOLVEFAILED); - } + log_notice(LD_REND,"Closing stream for '%s.onion': hidden service is " + "unavailable (try again later).", safe_str(query)); + connection_mark_unattached_ap(conn, END_STREAM_REASON_RESOLVEFAILED); } - }); + } SMARTLIST_FOREACH_END(_conn); } /** Return a newly allocated extend_info_t* for a randomly chosen introduction From 594854364460e9fe62998c191ee62c994a1cb336 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 29 May 2009 14:51:32 -0400 Subject: [PATCH 062/187] Clarify some of proposal 165 based on discussion --- .../proposals/165-simple-robust-voting.txt | 48 ++++++++++++------- 1 file changed, 31 insertions(+), 17 deletions(-) diff --git a/doc/spec/proposals/165-simple-robust-voting.txt b/doc/spec/proposals/165-simple-robust-voting.txt index d8993c9e95..e33d1772c9 100644 --- a/doc/spec/proposals/165-simple-robust-voting.txt +++ b/doc/spec/proposals/165-simple-robust-voting.txt @@ -50,16 +50,21 @@ Motivation: Proposed protocol design: A "Voting Set" is a set of authorities. Each authority has a list of - the voting sets it considers acceptable. These sets must always - contain the authority itself. Each authority lists all of these - voting sets in its votes. + the voting sets it considers acceptable. These sets are chosen + manually by the authority operators. they must always contain the + authority itself. Each authority lists all of these voting sets in + its votes. Authorities exchange votes with every other authority in any of their voting sets. When it comes time to calculate a consensus, an authority votes with whichever voting set it lists that is listed by the most members of - that set. + that set. In other words, given two sets S1 and S2 that an authority + lists, that authority will prefer to vote with S1 over S2 whenever + the number of other authorities in S1 that themselves list S1 is + higher than the number of other authorities in S2 that themselves + list S2. For example, suppose authority A recognizes two sets, "A B C D" and "A E F G H". Suppose that the first set is recognized by all of A, @@ -72,21 +77,30 @@ Proposed protocol design: How to migrate authority sets: - In steady state, each authority should list only the current actual - voting set as accepted. + In steady state, each authority operator should list only the current + actual voting set as accepted. - When we want to add an authority, we list two voting sets: one - containing all the old authorities, and one containing the old - authorities and the new authority too. Once all authorities are - listing the new set of authorities, they will start preferring that - set because of its size. + When we want to add an authority, each authority operator configures + his or her server to list two voting sets: one containing all the old + authorities, and one containing the old authorities and the new + authority too. Once all authorities are listing the new set of + authorities, they will start voting with that set because of its + size. - When we want to remove an authority, we list two voting sets: one - containing all the authorities, and one omitting the authority we - want to remove. Once enough authorities list the new set as - acceptable, we start having authorities stop listing the old set. - Once there are more listing the new set than the old set, the new set - will win. + What if one or two authority operators are slow to list the new set? + Then the other operators can stop listing the old set once there are + enough authorities listing the new set to make its voting successful. + (Note that these authorities not listing the new set will still have + their votes counted, since they themselves will be members of the new + set. They will only fail to sign the consensus generated by the + other authorities who are using the new set.) + + When we want to remove an authority, the operators list two voting + sets: one containing all the authorities, and one omitting the + authority we want to remove. Once enough authorities list the new + set as acceptable, we start having authority operators stop listing + the old set. Once there are more listing the new set than the old + set, the new set will win. Data format changes: From db576626e1d09c183d0c5153a78ebff4b95fc53b Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 29 May 2009 15:12:38 -0400 Subject: [PATCH 063/187] Update 0.2.2 TODO item status --- doc/TODO.022 | 39 +++++++++++++++++++++++++-------------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/doc/TODO.022 b/doc/TODO.022 index a0644f1af8..8bf50659fb 100644 --- a/doc/TODO.022 +++ b/doc/TODO.022 @@ -11,13 +11,14 @@ NOTE 2: It's easy to list stuff like this with no time estimates and - Design only - Begin design work for UDP transition; identify areas where we need to make changes or instrument stuff early. - [multiple weeks, ongoing. Need to do a draft early.] - Performance, mostly protocol-neutral. - Work with Libevent 2.0's bufferevent interface - Identify any performance stuff we need to push back into libevent to make it as fast as we want. + - Get a decent rate-limiting feature into Libevent + - Get openssl support into Libevent. - Revise how we do bandwidth limiting and round-robining between circuits on a connection. @@ -42,6 +43,7 @@ NOTE 2: It's easy to list stuff like this with no time estimates and - ... or group client circuits by IP at the server and rate-limit like that. + - Use if-modified-since to download consensuses - Other features @@ -54,34 +56,43 @@ NOTE 2: It's easy to list stuff like this with no time estimates and - Proposals to improve and implement - 158: microdescriptors -N - Revise proposal + o Revise proposal + - Implement - 160: list bandwidth in consensus -RNM? - Finish proposal +RNM? . Finish proposal - and actually set it reasonably - and actually use it. - Proposals to improve and implement if not broken - - IPv6 support. (Parts of 117, but figure out how to handle DNS + D IPv6 support. (Parts of 117, but figure out how to handle DNS requests.) - 140: Directory diffs + - Need a decent simple C diff implementation. + - Need a decent simple C ed patch implementation. - 149: learn info from netinfo cells. - - 134: handle authority fragmentation (Needs more analysis) - - - Needs a proposal, or at least some design - - Detect client-status better -N - Write proposal - - Have authorities report relay and voting status better: make it + o Start discussion + - Revise proposal based on discussion. + X 134: handle authority fragmentation (Needs more analysis) + - 165: Easy migration for voting authority sets + - 163: Detect client-status better + o Write proposal + - Possibly implement, depending on discussion. + - 164: Have authorities report relay and voting status better: make it easy to answer, "Why is my server not listed/not Guard/not Running/etc" -N - Write proposal - - Have consensuses come in multiple "flavours". -N - Write proposal + o Write proposal + - Possibly implement, depending on discussion + - 162: Have consensuses come in multiple "flavours". + o Write proposal + - Possibly implement, depending on discussion. + + - Needs a proposal, or at least some design - Weaken the requirements for being a Guard, based on K's measurements. K - Finish measurements K? - Write proposal - Adaptive timeouts for giving up on circuits and streams. -M/N - Revise proposal 151 +M - Revise proposal 151 - Downweight guards more sensibly: be more forgiving about using Guard nodes as non-first-hop. - Write proposal. From 58168322477a9f847c14ac6282218e9f56a37c28 Mon Sep 17 00:00:00 2001 From: Martin Peck Date: Sat, 30 May 2009 13:53:35 -0700 Subject: [PATCH 064/187] Add documentation for new AccelName and AccelDir dynamic hardware acceleration engine options. --- doc/tor.1.in | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/doc/tor.1.in b/doc/tor.1.in index 55f9a8e043..c9b344257d 100644 --- a/doc/tor.1.in +++ b/doc/tor.1.in @@ -354,6 +354,16 @@ If non-zero, try to use crypto hardware acceleration when available. This is untested and probably buggy. (Default: 0) .LP .TP +\fBAccelName \fR\fIEngineName\fP +When using OpenSSL hardware crypto acceleration attempt to load the dynamic +engine of this name. This must be used for any dynamic hardware engine. +.LP +.TP +\fBAccelDir \fR\fIPath\fP +Specify this option if using dynamic hardware acceleration and the engine +implementation library resides somewhere other than the OpenSSL default. +.LP +.TP \fBAvoidDiskWrites \fR\fB0\fR|\fB1\fP If non-zero, try to write to disk less frequently than we would otherwise. This is useful when running on flash memory or other media that support only From dfebc88d56e2c5b470fce20cd7b25914cf2507d4 Mon Sep 17 00:00:00 2001 From: Karsten Loesing Date: Sat, 30 May 2009 23:35:21 +0200 Subject: [PATCH 065/187] Warn and exit when we are configured to measure GeoIP statistics, but have no GeoIP database. --- src/or/config.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/or/config.c b/src/or/config.c index 0cf0db7578..c0f473bf2d 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -1378,6 +1378,14 @@ options_act(or_options_t *old_options) geoip_load_file(actual_fname, options); tor_free(actual_fname); } +#ifdef ENABLE_GEOIP_STATS + /* Check if GeoIP database could be loaded. */ + if (!geoip_is_loaded()) { + log_warn(LD_CONFIG, "Configured to measure GeoIP statistics, but no " + "GeoIP database found!"); + return -1; + } +#endif /* Check if we need to parse and add the EntryNodes config option. */ if (options->EntryNodes && (!old_options || From 65836d246dce0b5928bf6d9d580fce5786a23eb1 Mon Sep 17 00:00:00 2001 From: Martin Peck Date: Sat, 30 May 2009 15:15:43 -0700 Subject: [PATCH 066/187] Remove warning about untested HardwareAccel option and clarify usage. --- doc/tor.1.in | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/doc/tor.1.in b/doc/tor.1.in index c9b344257d..db0637a75f 100644 --- a/doc/tor.1.in +++ b/doc/tor.1.in @@ -350,16 +350,17 @@ On startup, setuid to this user and setgid to their primary group. .LP .TP \fBHardwareAccel \fR\fB0\fR|\fB1\fP -If non-zero, try to use crypto hardware acceleration when -available. This is untested and probably buggy. (Default: 0) +If non-zero, try to use built-in (static) crypto hardware acceleration when +available. (Default: 0) .LP .TP -\fBAccelName \fR\fIEngineName\fP +\fBAccelName \fR\fINAME\fP When using OpenSSL hardware crypto acceleration attempt to load the dynamic -engine of this name. This must be used for any dynamic hardware engine. +engine of this name. This must be used for any dynamic hardware engine. Names +can be verified with the openssl engine command. .LP .TP -\fBAccelDir \fR\fIPath\fP +\fBAccelDir \fR\fIDIR\fP Specify this option if using dynamic hardware acceleration and the engine implementation library resides somewhere other than the OpenSSL default. .LP From 3ca10bb62faaed6a5242ff760ed6c76330a750ef Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Sun, 31 May 2009 13:36:18 -0400 Subject: [PATCH 067/187] Fix-ups for dynamic OpenSSL engine patch. Include a changelog, and don't try to compare strings with !=. --- ChangeLog | 6 +++++- src/or/config.c | 4 ++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/ChangeLog b/ChangeLog index ee9f814f5a..ff01403b78 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,9 @@ Changes in version 0.2.2.1-alpha - 2009-??-?? - o Minor features + o Major features: + - Add support for dynamic OpenSSL hardware crypto acceleration engines + via new AccelName and AccelDir options. + + o Minor features: - New --digests command-line switch to output the digests of the source files Tor was built with. - The "torify" script now uses torsocks where available. diff --git a/src/or/config.c b/src/or/config.c index 0d176670e4..dea76391d1 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -3671,8 +3671,8 @@ options_transition_allowed(or_options_t *old, or_options_t *new_val, } if ((old->HardwareAccel != new_val->HardwareAccel) - || (old->AccelName != new_val->AccelName) - || (old->AccelDir != new_val->AccelDir)) { + || !opt_streq(old->AccelName, new_val->AccelName) + || !opt_streq(old->AccelDir, new_val->AccelDir)) { *msg = tor_strdup("While Tor is running, changing OpenSSL hardware " "acceleration engine is not allowed."); return -1; From 169c019a609890ed0dda826db6e6d354fb2edb8a Mon Sep 17 00:00:00 2001 From: Sebastian Hahn Date: Sat, 30 May 2009 03:15:54 +0200 Subject: [PATCH 068/187] spelling fixes for proposals --- doc/spec/proposals/149-using-netinfo-data.txt | 4 ++-- doc/spec/proposals/158-microdescriptors.txt | 8 ++++---- doc/spec/proposals/162-consensus-flavors.txt | 4 ++-- doc/spec/proposals/165-simple-robust-voting.txt | 6 +++--- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/doc/spec/proposals/149-using-netinfo-data.txt b/doc/spec/proposals/149-using-netinfo-data.txt index cbfc759f22..8bf8375d5d 100644 --- a/doc/spec/proposals/149-using-netinfo-data.txt +++ b/doc/spec/proposals/149-using-netinfo-data.txt @@ -22,14 +22,14 @@ Motivation idea of their own IP addresses, so they can publish correct descriptors. This is also in NETINFO cells. -Learning the time and IP +Learning the time and IP address We need to think about attackers here. Just because a router tells us that we have a given IP or a given clock skew doesn't mean that it's true. We believe this information only if we've heard it from a majority of the routers we've connected to recently, including at least 3 routers. Routers only believe this information if the - majority inclues at least one authority. + majority includes at least one authority. Avoiding MITM attacks diff --git a/doc/spec/proposals/158-microdescriptors.txt b/doc/spec/proposals/158-microdescriptors.txt index 59c14c3bcc..c8a35422f8 100644 --- a/doc/spec/proposals/158-microdescriptors.txt +++ b/doc/spec/proposals/158-microdescriptors.txt @@ -58,8 +58,8 @@ Status: Open A microdescriptor should in every case be a pure function of the router descriptor and the conensus method. - In votes, need to include the hash of each expected microdescriptor in - the routerstatus section. I suggest a new "m" line for each stanza, + In votes, we need to include the hash of each expected microdescriptor + in the routerstatus section. I suggest a new "m" line for each stanza, with the base64 of the SHA256 hash of the router's microdescriptor. For every consensus method that an authority supports, it includes a @@ -84,7 +84,7 @@ Status: Open 3.1.2. Computing consensus for microdescriptor-elements and "m" lines - When we generating a consensus, we use whichever m line + When we are generating a consensus, we use whichever m line unambiguously corresponds to the descriptor digest that will be included in the consensus. (If there are multiple m lines for that descriptor digest, we use whichever is most common. If they are @@ -103,7 +103,7 @@ Status: Open This flavor can safely omit descriptor digests. - We still need to descide whether to move ports into microdescriptors + We still need to decide whether to move ports into microdescriptors or not. In either case, they can be removed from the current "ns" flavor of consensus, since no current clients use them, and they take up about 5% of the compressed consensus. diff --git a/doc/spec/proposals/162-consensus-flavors.txt b/doc/spec/proposals/162-consensus-flavors.txt index 80fee1e688..da1405749d 100644 --- a/doc/spec/proposals/162-consensus-flavors.txt +++ b/doc/spec/proposals/162-consensus-flavors.txt @@ -37,7 +37,7 @@ Motivation: Design in brief: - Let the voting process will remain as it is, until a consensus is + Let the voting process remain as it is, until a consensus is generated. With future versions of the voting algorithm, instead of just a single consensus being generated, multiple consensus "flavors" are produced. @@ -116,7 +116,7 @@ Spec modifications: Signature = "directory-signature" SP algname SP identity SP signing-key-digest NL signature - There must be one Document line for each generated consensus flavor + There must be one Document line for each generated consensus flavor. Each Document line describes the length of the signed portion of a consensus (the signatures themselves are not included), along with one or more digests of that signed portion. Digests are diff --git a/doc/spec/proposals/165-simple-robust-voting.txt b/doc/spec/proposals/165-simple-robust-voting.txt index e33d1772c9..f813285a83 100644 --- a/doc/spec/proposals/165-simple-robust-voting.txt +++ b/doc/spec/proposals/165-simple-robust-voting.txt @@ -51,14 +51,14 @@ Proposed protocol design: A "Voting Set" is a set of authorities. Each authority has a list of the voting sets it considers acceptable. These sets are chosen - manually by the authority operators. they must always contain the + manually by the authority operators. They must always contain the authority itself. Each authority lists all of these voting sets in its votes. Authorities exchange votes with every other authority in any of their voting sets. - When it comes time to calculate a consensus, an authority votes with + When it is time to calculate a consensus, an authority votes with whichever voting set it lists that is listed by the most members of that set. In other words, given two sets S1 and S2 that an authority lists, that authority will prefer to vote with S1 over S2 whenever @@ -116,7 +116,7 @@ Data format changes: implement the proposal), add this line to the consensus format as well, before the first dir-source line. [This information is not redundant with the dir-source sections in the consensus: If an - authority is recognized didn't vote, that authority will appear in + authority is recognized but didn't vote, that authority will appear in the voting-set line but not in the dir-source sections.] We don't need to list other information about authorities in our From 08fd7e61c718d2016705a52365e219f9d42181c6 Mon Sep 17 00:00:00 2001 From: Roger Dingledine Date: Sun, 7 Jun 2009 15:07:23 -0400 Subject: [PATCH 069/187] proposals tweaks patch is attached --roger >From 674f087ab98e1711bb533acf23ee88c7c2a1dfdb Mon Sep 17 00:00:00 2001 From: Roger Dingledine Date: Sun, 7 Jun 2009 14:37:32 -0400 Subject: [PATCH] minor edits on proposals --- doc/spec/proposals/158-microdescriptors.txt | 7 ++++--- doc/spec/proposals/162-consensus-flavors.txt | 16 ++++++++-------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/doc/spec/proposals/158-microdescriptors.txt b/doc/spec/proposals/158-microdescriptors.txt index c8a35422f8..6f53b9dd58 100644 --- a/doc/spec/proposals/158-microdescriptors.txt +++ b/doc/spec/proposals/158-microdescriptors.txt @@ -8,7 +8,7 @@ Status: Open 15 May 2009: Substantially revised based on discussions on or-dev from late January. Removed the notion of voting on how to choose - microdescriptors; made it just a function of the consesus method. + microdescriptors; made it just a function of the consensus method. (This lets us avoid the possibility of "desynchronization.") Added suggestion to use a new consensus flavor. Specified use of SHA256 for new hashes. -nickm @@ -47,7 +47,7 @@ Status: Open 3. Design There are three pieces to the proposal. First, authorities will list in - their votes (and thus in the consensus) the expected hash + their votes (and thus in the consensus) the expected hash of microdescriptor for each relay. Second, directory mirrors will serve microdescriptors. Third, clients will ask for them and cache them. @@ -111,7 +111,7 @@ Status: Open This new consensus flavor should be signed with the sha256 signature format as documented in proposal 162. -3.2. Directory mirrors serve microdescriptors +3.2. Directory mirrors fetch, cache, and serve microdescriptors Directory mirrors should then read the microdescriptor-elements line from the consensus, and learn how to answer requests. (Directory mirrors @@ -122,6 +122,7 @@ Status: Open http:///tor/micro/d/--.z (We use base64 for size and for consistency with the consensus format. We use -s instead of +s to separate these items, since + ... since...? All the microdescriptors from the current consensus should also be available at: diff --git a/doc/spec/proposals/162-consensus-flavors.txt b/doc/spec/proposals/162-consensus-flavors.txt index da1405749d..caff96c0d7 100644 --- a/doc/spec/proposals/162-consensus-flavors.txt +++ b/doc/spec/proposals/162-consensus-flavors.txt @@ -5,7 +5,6 @@ Created: 14-May-2009 Target: 0.2.2 Status: Open - Overview: This proposal describes a way to publish each consensus in @@ -67,8 +66,8 @@ Spec modifications: The supported consensus flavors are defined as part of the authorities' consensus method. - For each supported flavors, every authority calculates another - consensus document of as-yet-unspecified format, and exchange + For each supported flavor, every authority calculates another + consensus document of as-yet-unspecified format, and exchanges detached signatures for these documents as in the current consensus design. @@ -112,7 +111,7 @@ Spec modifications: Documents = Document* Document = "document" SP flavor SP SignedLength 1*(SP AlgorithmName "=" Digest) NL - Signatures = Signature * + Signatures = Signature* Signature = "directory-signature" SP algname SP identity SP signing-key-digest NL signature @@ -141,15 +140,15 @@ Spec modifications: The 'SHA256' signature format for directory objects is defined as the RSA signature of the OAEP+-padded SHA256 digest of the SHA256 - digest of the the item to be signed. When checking signatures, - the signature MUST be treated as valid if the signed material + digest of the item to be signed. When checking signatures, + the signature MUST be treated as valid if the signature material begins with SHA256(SHA256(document)); this allows us to add other data later. Considerations: - We should not create a new flavor of consensus when adding a - field wouldn't be too onerous. + field instead wouldn't be too onerous. - We should not proliferate flavors lightly: clients will be distinguishable based on which flavor they download. @@ -159,7 +158,7 @@ Migration: - Stage one: authorities begin generating and serving consensus-index files. - - Stage two: Caches begin downloading consenusus-index files, + - Stage two: Caches begin downloading consensus-index files, validating them, and using them to decide what flavors of consensus documents to cache. They download all listed documents, and compare them to the digests given in the @@ -176,3 +175,4 @@ Acknowledgements: Aspects of this design and its applications to hash migration were heavily influenced by IRC conversations with Marian. + From 7404c936d011b9891fbe36fcb7ae4c59eace1fc7 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Sun, 7 Jun 2009 23:53:05 -0400 Subject: [PATCH 070/187] Finish an incomplete sentence in a proposal. --- doc/spec/proposals/158-microdescriptors.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/spec/proposals/158-microdescriptors.txt b/doc/spec/proposals/158-microdescriptors.txt index 6f53b9dd58..3dff9f85c7 100644 --- a/doc/spec/proposals/158-microdescriptors.txt +++ b/doc/spec/proposals/158-microdescriptors.txt @@ -122,7 +122,7 @@ Status: Open http:///tor/micro/d/--.z (We use base64 for size and for consistency with the consensus format. We use -s instead of +s to separate these items, since - ... since...? + the + character use used in base64 encoding.) All the microdescriptors from the current consensus should also be available at: From 77f5ad6b07dcdd3271d56f5e9edde3dc56ffe411 Mon Sep 17 00:00:00 2001 From: Karsten Loesing Date: Fri, 12 Jun 2009 02:05:21 +0200 Subject: [PATCH 071/187] Restore changes from f79688d that got lost somehow during a merge. --- src/or/rendclient.c | 62 ++++++++++----------------------------------- 1 file changed, 13 insertions(+), 49 deletions(-) diff --git a/src/or/rendclient.c b/src/or/rendclient.c index 6fe3a6938e..d9c0a9e713 100644 --- a/src/or/rendclient.c +++ b/src/or/rendclient.c @@ -63,7 +63,7 @@ rend_client_send_introduction(origin_circuit_t *introcirc, rend_cache_entry_t *entry; crypt_path_t *cpath; off_t dh_offset; - crypto_pk_env_t *intro_key; /* either Bob's public key or an intro key. */ + crypto_pk_env_t *intro_key = NULL; tor_assert(introcirc->_base.purpose == CIRCUIT_PURPOSE_C_INTRODUCING); tor_assert(rendcirc->_base.purpose == CIRCUIT_PURPOSE_C_REND_READY); @@ -80,23 +80,19 @@ rend_client_send_introduction(origin_circuit_t *introcirc, goto err; } - /* first 20 bytes of payload are the hash of bob's pk */ - if (entry->parsed->version == 0) { /* unversioned descriptor */ - intro_key = entry->parsed->pk; - } else { /* versioned descriptor */ - intro_key = NULL; - SMARTLIST_FOREACH(entry->parsed->intro_nodes, rend_intro_point_t *, - intro, { - if (!memcmp(introcirc->build_state->chosen_exit->identity_digest, - intro->extend_info->identity_digest, DIGEST_LEN)) { - intro_key = intro->intro_key; - break; - } - }); - if (!intro_key) { - log_warn(LD_BUG, "Internal error: could not find intro key."); - goto err; + /* first 20 bytes of payload are the hash of the intro key */ + intro_key = NULL; + SMARTLIST_FOREACH(entry->parsed->intro_nodes, rend_intro_point_t *, + intro, { + if (!memcmp(introcirc->build_state->chosen_exit->identity_digest, + intro->extend_info->identity_digest, DIGEST_LEN)) { + intro_key = intro->intro_key; + break; } + }); + if (!intro_key) { + log_warn(LD_BUG, "Internal error: could not find intro key."); + goto err; } if (crypto_pk_get_digest(intro_key, payload)<0) { log_warn(LD_BUG, "Internal error: couldn't hash public key."); @@ -451,28 +447,6 @@ directory_get_from_hs_dir(const char *desc_id, const rend_data_t *rend_query) return 1; } -/** If we are not currently fetching a rendezvous service descriptor - * for the service ID query, start a directory connection to fetch a - * new one. - */ -static void -rend_client_refetch_renddesc(const char *query) -{ - if (!get_options()->FetchHidServDescriptors) - return; - log_info(LD_REND, "Fetching rendezvous descriptor for service %s", - escaped_safe_str(query)); - if (connection_get_by_type_state_rendquery(CONN_TYPE_DIR, 0, query)) { - log_info(LD_REND,"Would fetch a new renddesc here (for %s), but one is " - "already in progress.", escaped_safe_str(query)); - } else { - /* not one already; initiate a dir rend desc lookup */ - directory_get_from_dirserver(DIR_PURPOSE_FETCH_RENDDESC, - ROUTER_PURPOSE_GENERAL, query, - PDS_RETRY_IF_NO_SERVERS); - } -} - /** Start a connection to a hidden service directory to fetch a v2 * rendezvous service descriptor for the base32-encoded service ID * query. @@ -552,12 +526,7 @@ rend_client_remove_intro_point(extend_info_t *failed_intro, if (r==0) { log_info(LD_REND, "Unknown service %s. Re-fetching descriptor.", escaped_safe_str(rend_query->onion_address)); - /* Fetch both, v0 and v2 rend descriptors in parallel. Use whichever - * arrives first. Exception: When using client authorization, only - * fetch v2 descriptors.*/ rend_client_refetch_v2_renddesc(rend_query); - if (rend_query->auth_type == REND_NO_AUTH) - rend_client_refetch_renddesc(rend_query->onion_address); return 0; } @@ -575,12 +544,7 @@ rend_client_remove_intro_point(extend_info_t *failed_intro, log_info(LD_REND, "No more intro points remain for %s. Re-fetching descriptor.", escaped_safe_str(rend_query->onion_address)); - /* Fetch both, v0 and v2 rend descriptors in parallel. Use whichever - * arrives first. Exception: When using client authorization, only - * fetch v2 descriptors.*/ rend_client_refetch_v2_renddesc(rend_query); - if (rend_query->auth_type == REND_NO_AUTH) - rend_client_refetch_renddesc(rend_query->onion_address); /* move all pending streams back to renddesc_wait */ while ((conn = connection_get_by_type_state_rendquery(CONN_TYPE_AP, From 6a32beb8771d94dd35460ce391eeb4455f2ada3f Mon Sep 17 00:00:00 2001 From: Karsten Loesing Date: Fri, 12 Jun 2009 11:39:35 +0200 Subject: [PATCH 072/187] Apply fix for bug 997. Hidden service clients didn't use a cached service descriptor that was older than 15 minutes, but wouldn't fetch a new one either. Now, use a cached descriptor no matter how old it is and only fetch a new one when all introduction points fail. Fix for bug 997. Patch from Marcus Griep. --- ChangeLog | 7 +++++++ src/or/connection_edge.c | 22 ++++++---------------- 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/ChangeLog b/ChangeLog index 3afc590258..d2a2261844 100644 --- a/ChangeLog +++ b/ChangeLog @@ -13,6 +13,13 @@ Changes in version 0.2.2.1-alpha - 2009-??-?? - Directories that are configured with the --enable-geoip-stats flag now write their GeoIP stats to disk exactly every 24 hours. + o Minor bugfixes + - Hidden service clients didn't use a cached service descriptor that + was older than 15 minutes, but wouldn't fetch a new one either. Now, + use a cached descriptor no matter how old it is and only fetch a new + one when all introduction points fail. Fix for bug 997. Patch from + Marcus Griep. + o Deprecated and removed features: - The controller no longer accepts the old obsolete "addr-mappings/" GETINFO value. diff --git a/src/or/connection_edge.c b/src/or/connection_edge.c index 6207ad3178..fe25c25c58 100644 --- a/src/or/connection_edge.c +++ b/src/or/connection_edge.c @@ -1678,22 +1678,12 @@ connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, safe_str(conn->rend_data->onion_address)); rend_client_refetch_v2_renddesc(conn->rend_data); } else { /* r > 0 */ -/** How long after we receive a hidden service descriptor do we consider - * it valid? */ -#define NUM_SECONDS_BEFORE_HS_REFETCH (60*15) - if (now - entry->received < NUM_SECONDS_BEFORE_HS_REFETCH) { - conn->_base.state = AP_CONN_STATE_CIRCUIT_WAIT; - log_info(LD_REND, "Descriptor is here and fresh enough. Great."); - if (connection_ap_handshake_attach_circuit(conn) < 0) { - if (!conn->_base.marked_for_close) - connection_mark_unattached_ap(conn, END_STREAM_REASON_CANT_ATTACH); - return -1; - } - } else { - conn->_base.state = AP_CONN_STATE_RENDDESC_WAIT; - log_info(LD_REND, "Stale descriptor %s. Re-fetching.", - safe_str(conn->rend_data->onion_address)); - rend_client_refetch_v2_renddesc(conn->rend_data); + conn->_base.state = AP_CONN_STATE_CIRCUIT_WAIT; + log_info(LD_REND, "Descriptor is here. Great."); + if (connection_ap_handshake_attach_circuit(conn) < 0) { + if (!conn->_base.marked_for_close) + connection_mark_unattached_ap(conn, END_STREAM_REASON_CANT_ATTACH); + return -1; } } return 0; From cb1617f18e94b244dc0847658e006057040dcc37 Mon Sep 17 00:00:00 2001 From: Roger Dingledine Date: Fri, 12 Jun 2009 11:18:02 -0400 Subject: [PATCH 073/187] Check answer_len in the remap_addr case of process_relay_cell_not_open. Fix an edge case where a malicious exit relay could convince a controller that the client's DNS question resolves to an internal IP address. Bug found and fixed by "optimist"; bugfix on 0.1.2.8-beta. --- ChangeLog | 5 +++++ src/or/relay.c | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 3afc590258..4d43e7a6f3 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,9 @@ Changes in version 0.2.2.1-alpha - 2009-??-?? + o Security fixes: + - Fix an edge case where a malicious exit relay could convince a + controller that the client's DNS question resolves to an internal IP + address. Bug found and fixed by "optimist"; bugfix on 0.1.2.8-beta. + o Major features: - Add support for dynamic OpenSSL hardware crypto acceleration engines via new AccelName and AccelDir options. diff --git a/src/or/relay.c b/src/or/relay.c index 4e09c0040b..e9baac6a45 100644 --- a/src/or/relay.c +++ b/src/or/relay.c @@ -947,7 +947,7 @@ connection_edge_process_relay_cell_not_open( cell->payload+RELAY_HEADER_SIZE+2, /*answer*/ ttl, -1); - if (answer_type == RESOLVED_TYPE_IPV4) { + if (answer_type == RESOLVED_TYPE_IPV4 && answer_len >= 4) { uint32_t addr = ntohl(get_uint32(cell->payload+RELAY_HEADER_SIZE+2)); remap_event_helper(conn, addr); } From e5b88dc83fb44622f2b0f9c0c242907d1c02311f Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 4 Jun 2009 01:05:23 -0400 Subject: [PATCH 074/187] Update Tor to use Libevent 2.0 APIs when available. This patch adds a new compat_libevent.[ch] set of files, and moves our Libevent compatibility and utilitity functions there. We build them into a separate .a so that nothing else in src/commmon depends on Libevent (partially fixing bug 507). Also, do not use our own built-in evdns copy when we have Libevent 2.0, whose evdns is finally good enough (thus fixing Bug 920). --- .gitignore | 1 + configure.in | 6 +- src/common/Makefile.am | 5 +- src/common/compat_libevent.c | 153 +++++++++++++++++++++++++++++++++++ src/common/compat_libevent.h | 39 +++++++++ src/common/log.c | 61 -------------- src/common/log.h | 2 - src/or/Makefile.am | 30 +++---- src/or/config.c | 8 +- src/or/dns.c | 17 +++- src/or/dnsserv.c | 7 ++ src/or/eventdns.c | 1 + src/or/main.c | 45 +++++------ src/or/or.h | 7 +- src/tools/Makefile.am | 6 +- 15 files changed, 269 insertions(+), 119 deletions(-) create mode 100644 src/common/compat_libevent.c create mode 100644 src/common/compat_libevent.h diff --git a/.gitignore b/.gitignore index 6eacae932d..9dcf3a4fe6 100644 --- a/.gitignore +++ b/.gitignore @@ -111,6 +111,7 @@ /src/common/common_sha1.i /src/common/libor.a /src/common/libor-crypto.a +/src/common/libor-event.a # /src/config/ /src/config/Makefile diff --git a/configure.in b/configure.in index 8a4101f41d..5720244ba0 100644 --- a/configure.in +++ b/configure.in @@ -273,15 +273,19 @@ save_CPPFLAGS="$CPPFLAGS" LIBS="-levent $TOR_LIB_WS32 $LIBS" LDFLAGS="$TOR_LDFLAGS_libevent $LDFLAGS" CPPFLAGS="$TOR_CPPFLAGS_libevent $CPPFLAGS" -AC_CHECK_FUNCS(event_get_version event_get_method event_set_log_callback) +AC_CHECK_FUNCS(event_get_version event_get_method event_set_log_callback evdns_set_outgoing_bind_address) AC_CHECK_MEMBERS([struct event.min_heap_idx], , , [#include ]) +AC_CHECK_HEADERS(event2/event.h event2/dns.h) + LIBS="$save_LIBS" LDFLAGS="$save_LDFLAGS" CPPFLAGS="$save_CPPFLAGS" +AM_CONDITIONAL(USE_EXTERNAL_EVDNS, test x$ac_cv_header_event2_dns_h = xyes) + dnl ------------------------------------------------------ dnl Where do you live, openssl? And how do we call you? diff --git a/src/common/Makefile.am b/src/common/Makefile.am index ad54e1aaaa..78f73db25f 100644 --- a/src/common/Makefile.am +++ b/src/common/Makefile.am @@ -1,5 +1,5 @@ -noinst_LIBRARIES = libor.a libor-crypto.a +noinst_LIBRARIES = libor.a libor-crypto.a libor-event.a EXTRA_DIST = common_sha1.i @@ -14,8 +14,9 @@ endif libor_a_SOURCES = address.c log.c util.c compat.c container.c mempool.c \ memarea.c util_codedigest.c $(libor_extra_source) libor_crypto_a_SOURCES = crypto.c aes.c tortls.c torgzip.c +libor_event_a_SOURCES = compat_libevent.c -noinst_HEADERS = address.h log.h crypto.h test.h util.h compat.h aes.h torint.h tortls.h strlcpy.c strlcat.c torgzip.h container.h ht.h mempool.h memarea.h ciphers.inc +noinst_HEADERS = address.h log.h crypto.h test.h util.h compat.h aes.h torint.h tortls.h strlcpy.c strlcat.c torgzip.h container.h ht.h mempool.h memarea.h ciphers.inc compat_libevent.h common_sha1.i: $(libor_SOURCES) $(libor_crypto_a_SOURCES) $(noinst_HEADERS) if test "@SHA1SUM@" != none; then \ diff --git a/src/common/compat_libevent.c b/src/common/compat_libevent.c new file mode 100644 index 0000000000..d2ef112079 --- /dev/null +++ b/src/common/compat_libevent.c @@ -0,0 +1,153 @@ +/* Copyright (c) 2009, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file compat_libevent.c + * \brief Wrappers to handle porting between different versions of libevent. + * + * In an ideal world, we'd just use Libevent 2.0 from now on. But as of June + * 2009, Libevent 2.0 is still in alpha, and we will have old versions of + * Libevent for the forseeable future. + **/ + +#include "orconfig.h" +#include "compat_libevent.h" + +#include "compat.h" +#include "util.h" +#include "log.h" + +#ifdef HAVE_EVENT2_EVENT_H +#include +#else +#include +#endif + +#ifdef HAVE_EVENT_SET_LOG_CALLBACK +/** A string which, if it appears in a libevent log, should be ignored. */ +static const char *suppress_msg = NULL; +/** Callback function passed to event_set_log() so we can intercept + * log messages from libevent. */ +static void +libevent_logging_callback(int severity, const char *msg) +{ + char buf[1024]; + size_t n; + if (suppress_msg && strstr(msg, suppress_msg)) + return; + n = strlcpy(buf, msg, sizeof(buf)); + if (n && n < sizeof(buf) && buf[n-1] == '\n') { + buf[n-1] = '\0'; + } + switch (severity) { + case _EVENT_LOG_DEBUG: + log(LOG_DEBUG, LD_NET, "Message from libevent: %s", buf); + break; + case _EVENT_LOG_MSG: + log(LOG_INFO, LD_NET, "Message from libevent: %s", buf); + break; + case _EVENT_LOG_WARN: + log(LOG_WARN, LD_GENERAL, "Warning from libevent: %s", buf); + break; + case _EVENT_LOG_ERR: + log(LOG_ERR, LD_GENERAL, "Error from libevent: %s", buf); + break; + default: + log(LOG_WARN, LD_GENERAL, "Message [%d] from libevent: %s", + severity, buf); + break; + } +} +/** Set hook to intercept log messages from libevent. */ +void +configure_libevent_logging(void) +{ + event_set_log_callback(libevent_logging_callback); +} +/** Ignore any libevent log message that contains msg. */ +void +suppress_libevent_log_msg(const char *msg) +{ + suppress_msg = msg; +} +#else +void +configure_libevent_logging(void) +{ +} +void +suppress_libevent_log_msg(const char *msg) +{ + (void)msg; +} +#endif + +#ifndef HAVE_EVENT2_EVENT_H +/** Work-alike replacement for event_new() on pre-Libevent-2.0 systems. */ +struct event * +tor_event_new(struct event_base *base, int sock, short what, + void (*cb)(int, short, void *), void *arg) +{ + struct event *e = tor_malloc_zero(sizeof(struct event)); + event_set(e, sock, what, cb, arg); + event_base_set(base, e); + return e; +} +/** Work-alike replacement for evtimer_new() on pre-Libevent-2.0 systems. */ +struct event * +tor_evtimer_new(struct event_base *base, + void (*cb)(int, short, void *), void *arg) +{ + return tor_event_new(base, -1, 0, cb, arg); +} +/** Work-alike replacement for evsignal_new() on pre-Libevent-2.0 systems. */ +struct event * +tor_evsignal_new(struct event_base * base, int sig, + void (*cb)(int, short, void *), void *arg) +{ + return tor_event_new(base, sig, EV_SIGNAL, cb, arg); +} +/** Work-alike replacement for event_free() on pre-Libevent-2.0 systems. */ +void +tor_event_free(struct event *ev) +{ + event_del(ev); + tor_free(ev); +} +#endif + +/** Global event base for use by the main thread. */ +struct event_base *the_event_base = NULL; + +/** Initialize the Libevent library and set up the event base. */ +void +tor_libevent_initialize(void) +{ + tor_assert(the_event_base == NULL); +#ifdef HAVE_EVENT2_EVENT_H + the_event_base = event_base_new(); +#else + the_event_base = event_init(); +#endif +} + +/** Return the current Libevent event base that we're set up to use. */ +struct event_base * +tor_libevent_get_base(void) +{ + return the_event_base; +} + +/** Return the name of the Libevent backend we're using. */ +const char * +tor_libevent_get_method(void) +{ +#ifdef HAVE_EVENT2_EVENT_H + return event_base_get_method(the_event_base); +#elif defined(HAVE_EVENT_GET_METHOD) + return event_get_method(); +#else + return ""; +#endif +} + diff --git a/src/common/compat_libevent.h b/src/common/compat_libevent.h new file mode 100644 index 0000000000..a95285f460 --- /dev/null +++ b/src/common/compat_libevent.h @@ -0,0 +1,39 @@ +/* Copyright (c) 2009, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef _TOR_COMPAT_LIBEVENT_H +#define _TOR_COMPAT_LIBEVENT_H + +#include "orconfig.h" + +#ifdef HAVE_EVENT2_EVENT_H +#include +#else +#include +#define evutil_socket_t int +#endif + +void configure_libevent_logging(void); +void suppress_libevent_log_msg(const char *msg); + +#ifdef HAVE_EVENT2_EVENT_H +#define tor_event_new event_new +#define tor_evtimer_new evtimer_new +#define tor_evsignal_new evsignal_new +#define tor_event_free event_free +#else +struct event *tor_event_new(struct event_base * base, evutil_socket_t sock, + short what, void (*cb)(evutil_socket_t, short, void *), void *arg); +struct event *tor_evtimer_new(struct event_base * base, + void (*cb)(evutil_socket_t, short, void *), void *arg); +struct event *tor_evsignal_new(struct event_base * base, int sig, + void (*cb)(evutil_socket_t, short, void *), void *arg); +void tor_event_free(struct event *ev); +#endif + +void tor_libevent_initialize(void); +struct event_base *tor_libevent_get_base(void); +const char *tor_libevent_get_method(void); + +#endif + diff --git a/src/common/log.c b/src/common/log.c index a7b0c12c4a..b12462a42f 100644 --- a/src/common/log.c +++ b/src/common/log.c @@ -36,8 +36,6 @@ #include "log.h" #include "container.h" -#include - #define TRUNCATED_STR "[...truncated]" #define TRUNCATED_STR_LEN 14 @@ -921,65 +919,6 @@ switch_logs_debug(void) UNLOCK_LOGS(); } -#ifdef HAVE_EVENT_SET_LOG_CALLBACK -/** A string which, if it appears in a libevent log, should be ignored. */ -static const char *suppress_msg = NULL; -/** Callback function passed to event_set_log() so we can intercept - * log messages from libevent. */ -static void -libevent_logging_callback(int severity, const char *msg) -{ - char buf[1024]; - size_t n; - if (suppress_msg && strstr(msg, suppress_msg)) - return; - n = strlcpy(buf, msg, sizeof(buf)); - if (n && n < sizeof(buf) && buf[n-1] == '\n') { - buf[n-1] = '\0'; - } - switch (severity) { - case _EVENT_LOG_DEBUG: - log(LOG_DEBUG, LD_NET, "Message from libevent: %s", buf); - break; - case _EVENT_LOG_MSG: - log(LOG_INFO, LD_NET, "Message from libevent: %s", buf); - break; - case _EVENT_LOG_WARN: - log(LOG_WARN, LD_GENERAL, "Warning from libevent: %s", buf); - break; - case _EVENT_LOG_ERR: - log(LOG_ERR, LD_GENERAL, "Error from libevent: %s", buf); - break; - default: - log(LOG_WARN, LD_GENERAL, "Message [%d] from libevent: %s", - severity, buf); - break; - } -} -/** Set hook to intercept log messages from libevent. */ -void -configure_libevent_logging(void) -{ - event_set_log_callback(libevent_logging_callback); -} -/** Ignore any libevent log message that contains msg. */ -void -suppress_libevent_log_msg(const char *msg) -{ - suppress_msg = msg; -} -#else -void -configure_libevent_logging(void) -{ -} -void -suppress_libevent_log_msg(const char *msg) -{ - (void)msg; -} -#endif - #if 0 static void dump_log_info(logfile_t *lf) diff --git a/src/common/log.h b/src/common/log.h index 834b1724b3..6745baabc6 100644 --- a/src/common/log.h +++ b/src/common/log.h @@ -134,8 +134,6 @@ void add_temp_log(int min_severity); void close_temp_logs(void); void rollback_log_changes(void); void mark_logs_temp(void); -void configure_libevent_logging(void); -void suppress_libevent_log_msg(const char *msg); void change_callback_log_severity(int loglevelMin, int loglevelMax, log_callback cb); void log_set_application_name(const char *name); diff --git a/src/or/Makefile.am b/src/or/Makefile.am index 11923fbd35..c967a8846f 100644 --- a/src/or/Makefile.am +++ b/src/or/Makefile.am @@ -12,7 +12,13 @@ endif EXTRA_DIST=ntmain.c or_sha1.i -tor_SOURCES = buffers.c circuitbuild.c circuitlist.c \ +if USE_EXTERNAL_EVDNS +evdns_source= +else +evdns_source=eventdns.c +endif + +COMMON_SRC = buffers.c circuitbuild.c circuitlist.c \ circuituse.c command.c config.c \ connection.c connection_edge.c connection_or.c control.c \ cpuworker.c directory.c dirserv.c dirvote.c \ @@ -20,8 +26,9 @@ tor_SOURCES = buffers.c circuitbuild.c circuitlist.c \ networkstatus.c onion.c policies.c \ reasons.c relay.c rendcommon.c rendclient.c rendmid.c \ rendservice.c rephist.c router.c routerlist.c routerparse.c \ - eventdns.c config_codedigest.c \ - tor_main.c + $(evdns_source) config_codedigest.c + +tor_SOURCES = $(COMMON_SRC) tor_main.c AM_CPPFLAGS = -DSHARE_DATADIR="\"$(datadir)\"" \ -DLOCALSTATEDIR="\"$(localstatedir)\"" \ @@ -33,22 +40,15 @@ AM_CPPFLAGS = -DSHARE_DATADIR="\"$(datadir)\"" \ tor_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ @TOR_LDFLAGS_libevent@ tor_LDADD = ../common/libor.a ../common/libor-crypto.a \ - -lz -levent -lssl -lcrypto @TOR_LIB_WS32@ @TOR_LIB_GDI@ -test_SOURCES = buffers.c circuitbuild.c circuitlist.c \ - circuituse.c command.c config.c \ - connection.c connection_edge.c connection_or.c control.c \ - cpuworker.c directory.c dirserv.c dirvote.c \ - dns.c dnsserv.c geoip.c hibernate.c main.c $(tor_platform_source) \ - networkstatus.c onion.c policies.c \ - reasons.c relay.c rendcommon.c rendclient.c rendmid.c \ - rendservice.c rephist.c router.c routerlist.c routerparse.c \ - eventdns.c config_codedigest.c \ - test_data.c test.c + ../common/libor-event.a \ + -lz -levent -lssl -lcrypto @TOR_LIB_WS32@ @TOR_LIB_GDI@ +test_SOURCES = $(COMMON_SRC) test_data.c test.c test_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ \ @TOR_LDFLAGS_libevent@ test_LDADD = ../common/libor.a ../common/libor-crypto.a \ - -lz -levent -lssl -lcrypto @TOR_LIB_WS32@ @TOR_LIB_GDI@ + ../common/libor-event.a \ + -lz -levent -lssl -lcrypto @TOR_LIB_WS32@ @TOR_LIB_GDI@ noinst_HEADERS = or.h eventdns.h eventdns_tor.h micro-revision.i diff --git a/src/or/config.c b/src/or/config.c index 9d5fb3b120..e5f27f9acd 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -4903,15 +4903,17 @@ init_libevent(void) /* Your libevent is ancient. */ #endif - event_init(); + tor_libevent_initialize(); + suppress_libevent_log_msg(NULL); #if defined(HAVE_EVENT_GET_VERSION) && defined(HAVE_EVENT_GET_METHOD) /* Making this a NOTICE for now so we can link bugs to a libevent versions * or methods better. */ log(LOG_NOTICE, LD_GENERAL, "Initialized libevent version %s using method %s. Good.", - event_get_version(), event_get_method()); - check_libevent_version(event_get_method(), get_options()->ORPort != 0); + event_get_version(), tor_libevent_get_method()); + check_libevent_version(tor_libevent_get_method(), + get_options()->ORPort != 0); #else log(LOG_NOTICE, LD_GENERAL, "Initialized old libevent (version 1.0b or earlier)."); diff --git a/src/or/dns.c b/src/or/dns.c index b8dd460889..ba34b406b9 100644 --- a/src/or/dns.c +++ b/src/or/dns.c @@ -13,7 +13,15 @@ #include "or.h" #include "ht.h" +#ifdef HAVE_EVENT2_DNS_H +#include +#include +#else #include "eventdns.h" +#ifndef HAVE_EVDNS_SET_DEFAULT_OUTGOING_BIND_ADDRESS +#define HAVE_EVDNS_SET_DEFAULT_OUTGOING_BIND_ADDRESS +#endif +#endif /** Longest hostname we're willing to resolve. */ #define MAX_ADDRESSLEN 256 @@ -1108,6 +1116,7 @@ configure_nameservers(int force) conf_fname = "/etc/resolv.conf"; #endif +#ifdef HAVE_EVDNS_SET_DEFAULT_OUTGOING_BIND_ADDRESS if (options->OutboundBindAddress) { tor_addr_t addr; if (tor_addr_from_str(&addr, options->OutboundBindAddress) < 0) { @@ -1127,6 +1136,7 @@ configure_nameservers(int force) } } } +#endif if (options->ServerDNSRandomizeCase) evdns_set_option("randomize-case:", "1", DNS_OPTIONS_ALL); @@ -1547,7 +1557,7 @@ dns_launch_wildcard_checks(void) void dns_launch_correctness_checks(void) { - static struct event launch_event; + static struct event *launch_event = NULL; struct timeval timeout; if (!get_options()->ServerDNSDetectHijacking) return; @@ -1555,10 +1565,11 @@ dns_launch_correctness_checks(void) /* Wait a while before launching requests for test addresses, so we can * get the results from checking for wildcarding. */ - evtimer_set(&launch_event, launch_test_addresses, NULL); + if (! launch_event) + launch_event = tor_evtimer_new(NULL, launch_test_addresses, NULL); timeout.tv_sec = 30; timeout.tv_usec = 0; - if (evtimer_add(&launch_event, &timeout)<0) { + if (evtimer_add(launch_event, &timeout)<0) { log_warn(LD_BUG, "Couldn't add timer for checking for dns hijacking"); } } diff --git a/src/or/dnsserv.c b/src/or/dnsserv.c index 6020f9a781..08cf653093 100644 --- a/src/or/dnsserv.c +++ b/src/or/dnsserv.c @@ -9,7 +9,14 @@ **/ #include "or.h" +#ifdef HAVE_EVENT2_DNS_H +#include +#include +/* XXXX022 this implies we want an improved evdns */ +#include +#else #include "eventdns.h" +#endif /** Helper function: called by evdns whenever the client sends a request to our * DNSPort. We need to eventually answer the request req. diff --git a/src/or/eventdns.c b/src/or/eventdns.c index 34c16248dd..00e63b5212 100644 --- a/src/or/eventdns.c +++ b/src/or/eventdns.c @@ -89,6 +89,7 @@ #include #include "eventdns.h" + #ifdef WIN32 #include #include diff --git a/src/or/main.c b/src/or/main.c index ee1ff3437b..1b716f93a5 100644 --- a/src/or/main.c +++ b/src/or/main.c @@ -127,12 +127,10 @@ connection_add(connection_t *conn) smartlist_add(connection_array, conn); if (conn->s >= 0 || conn->linked) { - conn->read_event = tor_malloc_zero(sizeof(struct event)); - conn->write_event = tor_malloc_zero(sizeof(struct event)); - event_set(conn->read_event, conn->s, EV_READ|EV_PERSIST, - conn_read_callback, conn); - event_set(conn->write_event, conn->s, EV_WRITE|EV_PERSIST, - conn_write_callback, conn); + conn->read_event = tor_event_new(tor_libevent_get_base(), + conn->s, EV_READ|EV_PERSIST, conn_read_callback, conn); + conn->write_event = tor_event_new(tor_libevent_get_base(), + conn->s, EV_WRITE|EV_PERSIST, conn_write_callback, conn); } log_debug(LD_NET,"new conn type %s, socket %d, address %s, n_conns %d.", @@ -393,11 +391,11 @@ connection_start_reading_from_linked_conn(connection_t *conn) smartlist_add(active_linked_connection_lst, conn); if (!called_loop_once) { /* This is the first event on the list; we won't be in LOOP_ONCE mode, - * so we need to make sure that the event_loop() actually exits at the - * end of its run through the current connections and - * lets us activate read events for linked connections. */ + * so we need to make sure that the event_base_loop() actually exits at + * the end of its run through the current connections and lets us + * activate read events for linked connections. */ struct timeval tv = { 0, 0 }; - event_loopexit(&tv); + event_base_loopexit(tor_libevent_get_base(), &tv); } } else { tor_assert(smartlist_isin(active_linked_connection_lst, conn)); @@ -1140,8 +1138,8 @@ second_elapsed_callback(int fd, short event, void *args) (void)event; (void)args; if (!timeout_event) { - timeout_event = tor_malloc_zero(sizeof(struct event)); - evtimer_set(timeout_event, second_elapsed_callback, NULL); + timeout_event = tor_evtimer_new(tor_libevent_get_base(), + second_elapsed_callback, NULL); one_second.tv_sec = 1; one_second.tv_usec = 0; } @@ -1213,7 +1211,7 @@ second_elapsed_callback(int fd, short event, void *args) } #endif - if (evtimer_add(timeout_event, &one_second)) + if (event_add(timeout_event, &one_second)) log_err(LD_NET, "Error from libevent when setting one-second timeout event"); } @@ -1424,20 +1422,16 @@ do_main_loop(void) /* poll until we have an event, or the second ends, or until we have * some active linked connections to trigger events for. */ - loop_result = event_loop(called_loop_once ? EVLOOP_ONCE : 0); + loop_result = event_base_loop(tor_libevent_get_base(), + called_loop_once ? EVLOOP_ONCE : 0); /* let catch() handle things like ^c, and otherwise don't worry about it */ if (loop_result < 0) { int e = tor_socket_errno(-1); /* let the program survive things like ^z */ if (e != EINTR && !ERRNO_IS_EINPROGRESS(e)) { -#ifdef HAVE_EVENT_GET_METHOD log_err(LD_NET,"libevent call with %s failed: %s [%d]", - event_get_method(), tor_socket_strerror(e), e); -#else - log_err(LD_NET,"libevent call failed: %s [%d]", - tor_socket_strerror(e), e); -#endif + tor_libevent_get_method(), tor_socket_strerror(e), e); return -1; #ifndef MS_WINDOWS } else if (e == EINVAL) { @@ -1704,7 +1698,7 @@ handle_signals(int is_parent) { #ifndef MS_WINDOWS /* do signal stuff only on Unix */ int i; - static int signals[] = { + static const int signals[] = { SIGINT, /* do a controlled slow shutdown */ SIGTERM, /* to terminate now */ SIGPIPE, /* otherwise SIGPIPE kills us */ @@ -1716,12 +1710,13 @@ handle_signals(int is_parent) #endif SIGCHLD, /* handle dns/cpu workers that exit */ -1 }; - static struct event signal_events[16]; /* bigger than it has to be. */ + static struct event *signal_events[16]; /* bigger than it has to be. */ if (is_parent) { for (i = 0; signals[i] >= 0; ++i) { - signal_set(&signal_events[i], signals[i], signal_callback, - (void*)(uintptr_t)signals[i]); - if (signal_add(&signal_events[i], NULL)) + signal_events[i] = tor_evsignal_new( + tor_libevent_get_base(), signals[i], signal_callback, + (void*)(uintptr_t)signals[i]); + if (event_add(signal_events[i], NULL)) log_warn(LD_BUG, "Error from libevent when adding event for signal %d", signals[i]); } diff --git a/src/or/or.h b/src/or/or.h index 2fa60264f5..59425fa7cb 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -91,8 +91,7 @@ #include "util.h" #include "torgzip.h" #include "address.h" - -#include +#include "compat_libevent.h" /* These signals are defined to help control_signal_act work. */ @@ -922,8 +921,8 @@ typedef struct connection_t { * connection. */ unsigned int linked_conn_is_closed:1; - int s; /**< Our socket; -1 if this connection is closed, or has no - * socket. */ + /** Our socket; -1 if this connection is closed, or has no socket. */ + evutil_socket_t s; int conn_array_index; /**< Index into the global connection array. */ struct event *read_event; /**< Libevent event structure. */ struct event *write_event; /**< Libevent event structure. */ diff --git a/src/tools/Makefile.am b/src/tools/Makefile.am index 79393d6854..b1e8bafb26 100644 --- a/src/tools/Makefile.am +++ b/src/tools/Makefile.am @@ -3,16 +3,16 @@ noinst_PROGRAMS = tor-checkkey tor_resolve_SOURCES = tor-resolve.c tor_resolve_LDFLAGS = @TOR_LDFLAGS_libevent@ -tor_resolve_LDADD = ../common/libor.a -levent @TOR_LIB_WS32@ +tor_resolve_LDADD = ../common/libor.a @TOR_LIB_WS32@ tor_gencert_SOURCES = tor-gencert.c tor_gencert_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ \ @TOR_LDFLAGS_libevent@ tor_gencert_LDADD = ../common/libor.a ../common/libor-crypto.a \ - -lz -lcrypto -levent @TOR_LIB_WS32@ @TOR_LIB_GDI@ + -lz -lcrypto @TOR_LIB_WS32@ @TOR_LIB_GDI@ tor_checkkey_SOURCES = tor-checkkey.c tor_checkkey_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ \ @TOR_LDFLAGS_libevent@ tor_checkkey_LDADD = ../common/libor.a ../common/libor-crypto.a \ - -lz -lcrypto -levent @TOR_LIB_WS32@ @TOR_LIB_GDI@ + -lz -lcrypto @TOR_LIB_WS32@ @TOR_LIB_GDI@ From c0af3cdfb6816659031fed5a1b8feec5d457b608 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 4 Jun 2009 03:16:26 -0400 Subject: [PATCH 075/187] Move the Libvent setup logic into compat_libevent from config. This has been some pretty ugly and voodoo-laden code. I've tried to clean it up a bit, but more work probably remains. --- configure.in | 2 +- src/common/compat_libevent.c | 305 +++++++++++++++++++++++++++++++++++ src/common/compat_libevent.h | 11 +- src/or/config.c | 251 +--------------------------- src/or/or.h | 6 + 5 files changed, 328 insertions(+), 247 deletions(-) diff --git a/configure.in b/configure.in index 5720244ba0..f9a7b033e8 100644 --- a/configure.in +++ b/configure.in @@ -273,7 +273,7 @@ save_CPPFLAGS="$CPPFLAGS" LIBS="-levent $TOR_LIB_WS32 $LIBS" LDFLAGS="$TOR_LDFLAGS_libevent $LDFLAGS" CPPFLAGS="$TOR_CPPFLAGS_libevent $CPPFLAGS" -AC_CHECK_FUNCS(event_get_version event_get_method event_set_log_callback evdns_set_outgoing_bind_address) +AC_CHECK_FUNCS(event_get_version event_get_version_number event_get_method event_set_log_callback evdns_set_outgoing_bind_address) AC_CHECK_MEMBERS([struct event.min_heap_idx], , , [#include ]) diff --git a/src/common/compat_libevent.c b/src/common/compat_libevent.c index d2ef112079..ce9f6a0fd3 100644 --- a/src/common/compat_libevent.c +++ b/src/common/compat_libevent.c @@ -23,6 +23,21 @@ #include #endif +/** A number representing a version of Libevent. + + This is a 4-byte number, with the first three bytes representing the + major, minor, and patchlevel respectively of the the library. The fourth + byte is unused. + + This is equivalent to the format of LIBEVENT_VERSION_NUMBER on Libevent + 2.0.1 or later. For versions of Libevent before 1.4.0, which followed the + format of "1.0, 1.0a, 1.0b", we define 1.0 to be equivalent to 1.0.0, 1.0a + to be equivalent to 1.0.1, and so on. +*/ +typedef uint32_t le_version_t; + +static le_version_t tor_get_libevent_version(const char **v_out); + #ifdef HAVE_EVENT_SET_LOG_CALLBACK /** A string which, if it appears in a libevent log, should be ignored. */ static const char *suppress_msg = NULL; @@ -119,16 +134,51 @@ tor_event_free(struct event *ev) /** Global event base for use by the main thread. */ struct event_base *the_event_base = NULL; + +/* This is what passes for version detection on OSX. We set + * MACOSX_KQUEUE_IS_BROKEN to true iff we're on a version of OSX before + * 10.4.0 (aka 1040). */ +#ifdef __APPLE__ +#ifdef __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ +#define MACOSX_KQUEUE_IS_BROKEN \ + (__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 1040) +#else +#define MACOSX_KQUEUE_IS_BROKEN 0 +#endif +#endif + /** Initialize the Libevent library and set up the event base. */ void tor_libevent_initialize(void) { tor_assert(the_event_base == NULL); + +#ifdef __APPLE__ + if (MACOSX_KQUEUE_IS_BROKEN || + tor_get_libevent_version(NULL) < V_OLD(1,1,'b')) { + setenv("EVENT_NOKQUEUE","1",1); + } +#endif + #ifdef HAVE_EVENT2_EVENT_H the_event_base = event_base_new(); #else the_event_base = event_init(); #endif + +#if defined(HAVE_EVENT_GET_VERSION) && defined(HAVE_EVENT_GET_METHOD) + /* Making this a NOTICE for now so we can link bugs to a libevent versions + * or methods better. */ + log(LOG_NOTICE, LD_GENERAL, + "Initialized libevent version %s using method %s. Good.", + event_get_version(), tor_libevent_get_method()); +#else + log(LOG_NOTICE, LD_GENERAL, + "Initialized old libevent (version 1.0b or earlier)."); + log(LOG_WARN, LD_GENERAL, + "You have a *VERY* old version of libevent. It is likely to be buggy; " + "please build Tor with a more recent version."); +#endif } /** Return the current Libevent event base that we're set up to use. */ @@ -151,3 +201,258 @@ tor_libevent_get_method(void) #endif } +/* Macros: returns the number of a libevent version. */ +#define V(major, minor, patch) \ + (((major) << 24) | ((minor) << 16) | ((patch) << 8)) +#define V_OLD(major, minor, patch) \ + V((major), (minor), (patch)-'a'+1) + +#define LE_OLD V(0,0,0) +#define LE_OTHER V(0,0,99) + +/** Return the le_version_t for the current version of libevent. If the + * version is very new, return LE_OTHER. If the version is so old that it + * doesn't support event_get_version(), return LE_OLD. DOCDOC */ +static le_version_t +tor_decode_libevent_version(const char *v) +{ + unsigned major, minor, patchlevel; + char c, extra; + int fields; + + /* Try the new preferred "1.4.11-stable" format. */ + fields = sscanf(v, "%u.%u.%u%c", &major, &minor, &patchlevel, &c); + if (fields == 3 || + (fields == 4 && (c == '-' || c == '_'))) { + return V(major,minor,patchlevel); + } + + /* Try the old "1.3e" format. */ + fields = sscanf(v, "%u.%u%c%c", &major, &minor, &c, &extra); + if (fields == 3 && TOR_ISALPHA(c)) { + return V_OLD(major, minor, c); + } else if (fields == 2) { + return V(major, minor, 0); + } + + return LE_OTHER; +} + +/** Return an integer representing the binary interface of a Libevent library. + * Two different versions with different numbers are sure not to be binary + * compatible. Two different versions with the same numbers have a decent + * chance of binary compatibility.*/ +static int +le_versions_compatibility(le_version_t v) +{ + if (v == LE_OTHER) + return 0; + if (v < V_OLD(1,0,'c')) + return 1; + else if (v < V(1,4,0)) + return 2; + else if (v < V(1,4,99)) + return 3; + else if (v < V(2,0,1)) + return 4; + else /* Everything 2.0 and later should be compatible. */ + return 5; +} + +/** Return the version number of the currently running version of Libevent. + See le_version_t for info on the format. + */ +static le_version_t +tor_get_libevent_version(const char **v_out) +{ + const char *v; + le_version_t r; +#if defined(HAVE_EVENT_GET_VERSION_NUMBER) + v = event_get_version(); + r = event_get_version_number(); +#elif defined (HAVE_EVENT_GET_VERSION) + v = event_get_version(); + r = tor_decode_libevent_version(v); +#else + v = "pre-1.0c"; + r = LE_OLD; +#endif + if (v_out) + *v_out = v; + return r; +} + +/** Return a string representation of the version of the currently running + * version of Libevent. */ +const char * +tor_libevent_get_version_str(void) +{ +#ifdef HAVE_EVENT_GET_VERSION + return event_get_version(); +#else + return "pre-1.0c"; +#endif +} + +/** + * Compare the current Libevent method and version to a list of versions + * which are known not to work. Warn the user as appropriate. + */ +void +tor_check_libevent_version(const char *m, int server, + const char **badness_out) +{ + int buggy = 0, iffy = 0, slow = 0, thread_unsafe = 0; + le_version_t version; + const char *v = NULL; + const char *badness = NULL; + const char *sad_os = ""; + + version = tor_get_libevent_version(&v); + + /* XXX Would it be worthwhile disabling the methods that we know + * are buggy, rather than just warning about them and then proceeding + * to use them? If so, we should probably not wrap this whole thing + * in HAVE_EVENT_GET_VERSION and HAVE_EVENT_GET_METHOD. -RD */ + /* XXXX The problem is that it's not trivial to get libevent to change it's + * method once it's initialized, and it's not trivial to tell what method it + * will use without initializing it. I guess we could preemptively disable + * buggy libevent modes based on the version _before_ initializing it, + * though, but then there's no good way (afaict) to warn "I would have used + * kqueue, but instead I'm using select." -NM */ + /* XXXX022 revist the above; it is fixable now. */ + if (!strcmp(m, "kqueue")) { + if (version < V_OLD(1,1,'b')) + buggy = 1; + } else if (!strcmp(m, "epoll")) { + if (version < V(1,1,0)) + iffy = 1; + } else if (!strcmp(m, "poll")) { + if (version < V_OLD(1,0,'e')) + buggy = 1; + if (version < V(1,1,0)) + slow = 1; + } else if (!strcmp(m, "select")) { + if (version < V(1,1,0)) + slow = 1; + } else if (!strcmp(m, "win32")) { + if (version < V_OLD(1,1,'b')) + buggy = 1; + } + + /* Libevent versions before 1.3b do very badly on operating systems with + * user-space threading implementations. */ +#if defined(__OpenBSD__) || defined(__FreeBSD__) || defined(__NetBSD__) + if (server && version < V_OLD(1,3,'b')) + thread_unsafe = 1; + sad_os = "BSD variants"; + } +#elif defined(__APPLE__) || defined(__darwin__) + if (server && version < V_OLD(1,3,'b')) + thread_unsafe = 1; + sad_os = "Mac OS X"; + } +#endif + + if (thread_unsafe) { + log(LOG_WARN, LD_GENERAL, + "Libevent version %s often crashes when running a Tor server with %s. " + "Please use the latest version of libevent (1.3b or later)",v,sad_os); + badness = "BROKEN"; + } else if (buggy) { + log(LOG_WARN, LD_GENERAL, + "There are serious bugs in using %s with libevent %s. " + "Please use the latest version of libevent.", m, v); + badness = "BROKEN"; + } else if (iffy) { + log(LOG_WARN, LD_GENERAL, + "There are minor bugs in using %s with libevent %s. " + "You may want to use the latest version of libevent.", m, v); + badness = "BUGGY"; + } else if (slow && server) { + log(LOG_WARN, LD_GENERAL, + "libevent %s can be very slow with %s. " + "When running a server, please use the latest version of libevent.", + v,m); + badness = "SLOW"; + } + + *badness_out = badness; +} + +#if defined(LIBEVENT_VERSION) +#define HEADER_VERSION LIBEVENT_VERSION +#elif defined(_EVENT_VERSION) +#define HEADER_VERSION _EVENT_VERSION +#endif + +/** See whether the headers we were built against differ from the library we + * linked against so much that we're likely to crash. If so, warn the + * user. */ +void +tor_check_libevent_header_compatibility(void) +{ + (void) le_versions_compatibility; + (void) tor_decode_libevent_version; + + /* In libevent versions before 2.0, it's hard to keep binary compatibility + * between upgrades, and unpleasant to detect when the version we compiled + * against is unlike the version we have linked against. Here's how. */ +#if defined(HEADER_VERSION) && defined(HAVE_EVENT_GET_VERSION) + /* We have a header-file version and a function-call version. Easy. */ + if (strcmp(HEADER_VERSION, event_get_version())) { + le_version_t v1, v2; + int compat1 = -1, compat2 = -1; + int verybad; + v1 = tor_decode_libevent_version(HEADER_VERSION); + v2 = tor_decode_libevent_version(event_get_version()); + compat1 = le_versions_compatibility(v1); + compat2 = le_versions_compatibility(v2); + + verybad = compat1 != compat2; + + log(verybad ? LOG_WARN : LOG_NOTICE, + LD_GENERAL, "We were compiled with headers from version %s " + "of Libevent, but we're using a Libevent library that says it's " + "version %s.", HEADER_VERSION, event_get_version()); + if (verybad) + log_warn(LD_GENERAL, "This will almost certainly make Tor crash."); + else + log_info(LD_GENERAL, "I think these versions are binary-compatible."); + } +#elif defined(HAVE_EVENT_GET_VERSION) + /* event_get_version but no _EVENT_VERSION. We might be in 1.4.0-beta or + earlier, where that's normal. To see whether we were compiled with an + earlier version, let's see whether the struct event defines MIN_HEAP_IDX. + */ +#ifdef HAVE_STRUCT_EVENT_MIN_HEAP_IDX + /* The header files are 1.4.0-beta or later. If the version is not + * 1.4.0-beta, we are incompatible. */ + { + if (strcmp(event_get_version(), "1.4.0-beta")) { + log_warn(LD_GENERAL, "It's a little hard to tell, but you seem to have " + "Libevent 1.4.0-beta header files, whereas you have linked " + "against Libevent %s. This will probably make Tor crash.", + event_get_version()); + } + } +#else + /* Our headers are 1.3e or earlier. If the library version is not 1.4.x or + later, we're probably fine. */ + { + const char *v = event_get_version(); + if ((v[0] == '1' && v[2] == '.' && v[3] > '3') || v[0] > '1') { + log_warn(LD_GENERAL, "It's a little hard to tell, but you seem to have " + "Libevent header file from 1.3e or earlier, whereas you have " + "linked against Libevent %s. This will probably make Tor " + "crash.", event_get_version()); + } + } +#endif + +#elif defined(HEADER_VERSION) +#warn "_EVENT_VERSION is defined but not get_event_version(): Libevent is odd." +#else + /* Your libevent is ancient. */ +#endif +} diff --git a/src/common/compat_libevent.h b/src/common/compat_libevent.h index a95285f460..5482c479fe 100644 --- a/src/common/compat_libevent.h +++ b/src/common/compat_libevent.h @@ -6,10 +6,12 @@ #include "orconfig.h" +struct event; +struct event_base; + #ifdef HAVE_EVENT2_EVENT_H -#include +#include #else -#include #define evutil_socket_t int #endif @@ -34,6 +36,9 @@ void tor_event_free(struct event *ev); void tor_libevent_initialize(void); struct event_base *tor_libevent_get_base(void); const char *tor_libevent_get_method(void); +void tor_check_libevent_version(const char *m, int server, + const char **badness_out); +void tor_check_libevent_header_compatibility(void); +const char *tor_libevent_get_version_str(void); #endif - diff --git a/src/or/config.c b/src/or/config.c index e5f27f9acd..bcabb67a99 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -706,20 +706,6 @@ static uint64_t config_parse_memunit(const char *s, int *ok); static int config_parse_interval(const char *s, int *ok); static void init_libevent(void); static int opt_streq(const char *s1, const char *s2); -/** Versions of libevent. */ -typedef enum { - /* Note: we compare these, so it's important that "old" precede everything, - * and that "other" come last. */ - LE_OLD=0, LE_10C, LE_10D, LE_10E, LE_11, LE_11A, LE_11B, LE_12, LE_12A, - LE_13, LE_13A, LE_13B, LE_13C, LE_13D, LE_13E, - LE_140, LE_141, LE_142, LE_143, LE_144, LE_145, LE_146, LE_147, LE_148, - LE_1499, - LE_OTHER -} le_version_t; -static le_version_t decode_libevent_version(const char *v, int *bincompat_out); -#if defined(HAVE_EVENT_GET_VERSION) && defined(HAVE_EVENT_GET_METHOD) -static void check_libevent_version(const char *m, int server); -#endif /** Magic value for or_options_t. */ #define OR_OPTIONS_MAGIC 9090909 @@ -4813,258 +4799,37 @@ config_parse_interval(const char *s, int *ok) return (int)r; } -/* This is what passes for version detection on OSX. We set - * MACOSX_KQUEUE_IS_BROKEN to true iff we're on a version of OSX before - * 10.4.0 (aka 1040). */ -#ifdef __APPLE__ -#ifdef __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ -#define MACOSX_KQUEUE_IS_BROKEN \ - (__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 1040) -#else -#define MACOSX_KQUEUE_IS_BROKEN 0 -#endif -#endif - /** * Initialize the libevent library. */ static void init_libevent(void) { + const char *badness=NULL; + configure_libevent_logging(); /* If the kernel complains that some method (say, epoll) doesn't * exist, we don't care about it, since libevent will cope. */ suppress_libevent_log_msg("Function not implemented"); -#ifdef __APPLE__ - if (MACOSX_KQUEUE_IS_BROKEN || - decode_libevent_version(event_get_version(), NULL) < LE_11B) { - setenv("EVENT_NOKQUEUE","1",1); - } -#endif - /* In libevent versions before 2.0, it's hard to keep binary compatibility - * between upgrades, and unpleasant to detect when the version we compiled - * against is unlike the version we have linked against. Here's how. */ -#if defined(_EVENT_VERSION) && defined(HAVE_EVENT_GET_VERSION) - /* We have a header-file version and a function-call version. Easy. */ - if (strcmp(_EVENT_VERSION, event_get_version())) { - int compat1 = -1, compat2 = -1; - int verybad, prettybad ; - decode_libevent_version(_EVENT_VERSION, &compat1); - decode_libevent_version(event_get_version(), &compat2); - verybad = compat1 != compat2; - prettybad = (compat1 == -1 || compat2 == -1) && compat1 != compat2; - - log(verybad ? LOG_WARN : (prettybad ? LOG_NOTICE : LOG_INFO), - LD_GENERAL, "We were compiled with headers from version %s " - "of Libevent, but we're using a Libevent library that says it's " - "version %s.", _EVENT_VERSION, event_get_version()); - if (verybad) - log_warn(LD_GENERAL, "This will almost certainly make Tor crash."); - else if (prettybad) - log_notice(LD_GENERAL, "If Tor crashes, this might be why."); - else - log_info(LD_GENERAL, "I think these versions are binary-compatible."); - } -#elif defined(HAVE_EVENT_GET_VERSION) - /* event_get_version but no _EVENT_VERSION. We might be in 1.4.0-beta or - earlier, where that's normal. To see whether we were compiled with an - earlier version, let's see whether the struct event defines MIN_HEAP_IDX. - */ -#ifdef HAVE_STRUCT_EVENT_MIN_HEAP_IDX - /* The header files are 1.4.0-beta or later. If the version is not - * 1.4.0-beta, we are incompatible. */ - { - if (strcmp(event_get_version(), "1.4.0-beta")) { - log_warn(LD_GENERAL, "It's a little hard to tell, but you seem to have " - "Libevent 1.4.0-beta header files, whereas you have linked " - "against Libevent %s. This will probably make Tor crash.", - event_get_version()); - } - } -#else - /* Our headers are 1.3e or earlier. If the library version is not 1.4.x or - later, we're probably fine. */ - { - const char *v = event_get_version(); - if ((v[0] == '1' && v[2] == '.' && v[3] > '3') || v[0] > '1') { - log_warn(LD_GENERAL, "It's a little hard to tell, but you seem to have " - "Libevent header file from 1.3e or earlier, whereas you have " - "linked against Libevent %s. This will probably make Tor " - "crash.", event_get_version()); - } - } -#endif - -#elif defined(_EVENT_VERSION) -#warn "_EVENT_VERSION is defined but not get_event_version(): Libevent is odd." -#else - /* Your libevent is ancient. */ -#endif + tor_check_libevent_header_compatibility(); tor_libevent_initialize(); suppress_libevent_log_msg(NULL); -#if defined(HAVE_EVENT_GET_VERSION) && defined(HAVE_EVENT_GET_METHOD) - /* Making this a NOTICE for now so we can link bugs to a libevent versions - * or methods better. */ - log(LOG_NOTICE, LD_GENERAL, - "Initialized libevent version %s using method %s. Good.", - event_get_version(), tor_libevent_get_method()); - check_libevent_version(tor_libevent_get_method(), - get_options()->ORPort != 0); -#else - log(LOG_NOTICE, LD_GENERAL, - "Initialized old libevent (version 1.0b or earlier)."); - log(LOG_WARN, LD_GENERAL, - "You have a *VERY* old version of libevent. It is likely to be buggy; " - "please build Tor with a more recent version."); -#endif -} -/** Table mapping return value of event_get_version() to le_version_t. */ -static const struct { - const char *name; le_version_t version; int bincompat; -} le_version_table[] = { - /* earlier versions don't have get_version. */ - { "1.0c", LE_10C, 1}, - { "1.0d", LE_10D, 1}, - { "1.0e", LE_10E, 1}, - { "1.1", LE_11, 1 }, - { "1.1a", LE_11A, 1 }, - { "1.1b", LE_11B, 1 }, - { "1.2", LE_12, 1 }, - { "1.2a", LE_12A, 1 }, - { "1.3", LE_13, 1 }, - { "1.3a", LE_13A, 1 }, - { "1.3b", LE_13B, 1 }, - { "1.3c", LE_13C, 1 }, - { "1.3d", LE_13D, 1 }, - { "1.3e", LE_13E, 1 }, - { "1.4.0-beta", LE_140, 2 }, - { "1.4.1-beta", LE_141, 2 }, - { "1.4.2-rc", LE_142, 2 }, - { "1.4.3-stable", LE_143, 2 }, - { "1.4.4-stable", LE_144, 2 }, - { "1.4.5-stable", LE_145, 2 }, - { "1.4.6-stable", LE_146, 2 }, - { "1.4.7-stable", LE_147, 2 }, - { "1.4.8-stable", LE_148, 2 }, - { "1.4.99-trunk", LE_1499, 3 }, - { NULL, LE_OTHER, 0 } -}; - -/** Return the le_version_t for the current version of libevent. If the - * version is very new, return LE_OTHER. If the version is so old that it - * doesn't support event_get_version(), return LE_OLD. */ -static le_version_t -decode_libevent_version(const char *v, int *bincompat_out) -{ - int i; - for (i=0; le_version_table[i].name; ++i) { - if (!strcmp(le_version_table[i].name, v)) { - if (bincompat_out) - *bincompat_out = le_version_table[i].bincompat; - return le_version_table[i].version; - } - } - if (v[0] != '1' && bincompat_out) - *bincompat_out = 100; - else if (!strcmpstart(v, "1.4") && bincompat_out) - *bincompat_out = 2; - return LE_OTHER; -} - -#if defined(HAVE_EVENT_GET_VERSION) && defined(HAVE_EVENT_GET_METHOD) -/** - * Compare the given libevent method and version to a list of versions - * which are known not to work. Warn the user as appropriate. - */ -static void -check_libevent_version(const char *m, int server) -{ - int buggy = 0, iffy = 0, slow = 0, thread_unsafe = 0; - le_version_t version; - const char *v = event_get_version(); - const char *badness = NULL; - const char *sad_os = ""; - - version = decode_libevent_version(v, NULL); - - /* XXX Would it be worthwhile disabling the methods that we know - * are buggy, rather than just warning about them and then proceeding - * to use them? If so, we should probably not wrap this whole thing - * in HAVE_EVENT_GET_VERSION and HAVE_EVENT_GET_METHOD. -RD */ - /* XXXX The problem is that it's not trivial to get libevent to change it's - * method once it's initialized, and it's not trivial to tell what method it - * will use without initializing it. I guess we could preemptively disable - * buggy libevent modes based on the version _before_ initializing it, - * though, but then there's no good way (afaict) to warn "I would have used - * kqueue, but instead I'm using select." -NM */ - if (!strcmp(m, "kqueue")) { - if (version < LE_11B) - buggy = 1; - } else if (!strcmp(m, "epoll")) { - if (version < LE_11) - iffy = 1; - } else if (!strcmp(m, "poll")) { - if (version < LE_10E) - buggy = 1; - else if (version < LE_11) - slow = 1; - } else if (!strcmp(m, "select")) { - if (version < LE_11) - slow = 1; - } else if (!strcmp(m, "win32")) { - if (version < LE_11B) - buggy = 1; - } - - /* Libevent versions before 1.3b do very badly on operating systems with - * user-space threading implementations. */ -#if defined(__OpenBSD__) || defined(__FreeBSD__) || defined(__NetBSD__) - if (server && version < LE_13B) { - thread_unsafe = 1; - sad_os = "BSD variants"; - } -#elif defined(__APPLE__) || defined(__darwin__) - if (server && version < LE_13B) { - thread_unsafe = 1; - sad_os = "Mac OS X"; - } -#endif - - if (thread_unsafe) { - log(LOG_WARN, LD_GENERAL, - "Libevent version %s often crashes when running a Tor server with %s. " - "Please use the latest version of libevent (1.3b or later)",v,sad_os); - badness = "BROKEN"; - } else if (buggy) { - log(LOG_WARN, LD_GENERAL, - "There are serious bugs in using %s with libevent %s. " - "Please use the latest version of libevent.", m, v); - badness = "BROKEN"; - } else if (iffy) { - log(LOG_WARN, LD_GENERAL, - "There are minor bugs in using %s with libevent %s. " - "You may want to use the latest version of libevent.", m, v); - badness = "BUGGY"; - } else if (slow && server) { - log(LOG_WARN, LD_GENERAL, - "libevent %s can be very slow with %s. " - "When running a server, please use the latest version of libevent.", - v,m); - badness = "SLOW"; - } + tor_check_libevent_version(tor_libevent_get_method(), + get_options()->ORPort != 0, + &badness); if (badness) { + const char *v = tor_libevent_get_version_str(); + const char *m = tor_libevent_get_method(); control_event_general_status(LOG_WARN, "BAD_LIBEVENT VERSION=%s METHOD=%s BADNESS=%s RECOVERED=NO", v, m, badness); } - } -#endif /** Return the persistent state struct for this Tor. */ or_state_t * diff --git a/src/or/or.h b/src/or/or.h index 59425fa7cb..e4989fc432 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -93,6 +93,12 @@ #include "address.h" #include "compat_libevent.h" +#ifdef HAVE_EVENT2_EVENT_H +#include +#else +#include +#endif + /* These signals are defined to help control_signal_act work. */ #ifndef SIGHUP From 33b1d714e779a714c2d55e5da09d0e3c682f273a Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 4 Jun 2009 03:24:53 -0400 Subject: [PATCH 076/187] Make Tor compile with Libevent 1.0 again. --- configure.in | 2 +- src/common/compat_libevent.c | 13 ++++++++++++- src/common/compat_libevent.h | 9 +++++++++ src/or/main.c | 2 +- 4 files changed, 23 insertions(+), 3 deletions(-) diff --git a/configure.in b/configure.in index f9a7b033e8..f1ac83687c 100644 --- a/configure.in +++ b/configure.in @@ -273,7 +273,7 @@ save_CPPFLAGS="$CPPFLAGS" LIBS="-levent $TOR_LIB_WS32 $LIBS" LDFLAGS="$TOR_LDFLAGS_libevent $LDFLAGS" CPPFLAGS="$TOR_CPPFLAGS_libevent $CPPFLAGS" -AC_CHECK_FUNCS(event_get_version event_get_version_number event_get_method event_set_log_callback evdns_set_outgoing_bind_address) +AC_CHECK_FUNCS(event_get_version event_get_version_number event_get_method event_set_log_callback evdns_set_outgoing_bind_address event_base_loopexit) AC_CHECK_MEMBERS([struct event.min_heap_idx], , , [#include ]) diff --git a/src/common/compat_libevent.c b/src/common/compat_libevent.c index ce9f6a0fd3..d9843c9c3c 100644 --- a/src/common/compat_libevent.c +++ b/src/common/compat_libevent.c @@ -134,7 +134,6 @@ tor_event_free(struct event *ev) /** Global event base for use by the main thread. */ struct event_base *the_event_base = NULL; - /* This is what passes for version detection on OSX. We set * MACOSX_KQUEUE_IS_BROKEN to true iff we're on a version of OSX before * 10.4.0 (aka 1040). */ @@ -188,6 +187,18 @@ tor_libevent_get_base(void) return the_event_base; } + +#ifndef HAVE_EVENT_BASE_LOOPEXIT +/* Replacement for event_base_loopexit on some very old versions of Libevent + that we are not yet brave enough to deprecate. */ +int +tor_event_base_loopexit(struct event_base *base, struct timeval *tv) +{ + tor_assert(base == the_event_base); + return event_loopexit(tv); +} +#endif + /** Return the name of the Libevent backend we're using. */ const char * tor_libevent_get_method(void) diff --git a/src/common/compat_libevent.h b/src/common/compat_libevent.h index 5482c479fe..b70f8d3083 100644 --- a/src/common/compat_libevent.h +++ b/src/common/compat_libevent.h @@ -33,6 +33,15 @@ struct event *tor_evsignal_new(struct event_base * base, int sig, void tor_event_free(struct event *ev); #endif +/* XXXX022 If we can drop support for Libevent before 1.1, we can + * do without this wrapper. */ +#ifdef HAVE_EVENT_BASE_LOOPEXIT +#define tor_event_base_loopexit event_base_loopexit +#else +struct timeval; +int tor_event_base_loopexit(struct event_base *base, struct timeval *tv); +#endif + void tor_libevent_initialize(void); struct event_base *tor_libevent_get_base(void); const char *tor_libevent_get_method(void); diff --git a/src/or/main.c b/src/or/main.c index 1b716f93a5..c808845192 100644 --- a/src/or/main.c +++ b/src/or/main.c @@ -395,7 +395,7 @@ connection_start_reading_from_linked_conn(connection_t *conn) * the end of its run through the current connections and lets us * activate read events for linked connections. */ struct timeval tv = { 0, 0 }; - event_base_loopexit(tor_libevent_get_base(), &tv); + tor_event_base_loopexit(tor_libevent_get_base(), &tv); } } else { tor_assert(smartlist_isin(active_linked_connection_lst, conn)); From 1e709c79d12b2366d4e1c1ea517e313cc29ee3cd Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 4 Jun 2009 14:49:16 -0400 Subject: [PATCH 077/187] Isolate Libevent API dependency to just main.c and dns.c in src/or. The rest of the code was only including event.h so that it could see EV_READ and EV_WRITE, which we were using as part of the connection_watch_events interface for no very good reason. --- src/or/connection.c | 19 ------------------- src/or/connection_edge.c | 6 +++--- src/or/connection_or.c | 2 +- src/or/directory.c | 4 ++-- src/or/dns.c | 2 ++ src/or/main.c | 33 +++++++++++++++++++++++++++++---- src/or/or.h | 14 ++++++-------- 7 files changed, 43 insertions(+), 37 deletions(-) diff --git a/src/or/connection.c b/src/or/connection.c index 66a62c549a..c8406fee3f 100644 --- a/src/or/connection.c +++ b/src/or/connection.c @@ -299,25 +299,6 @@ connection_link_connections(connection_t *conn_a, connection_t *conn_b) conn_b->linked_conn = conn_a; } -/** Tell libevent that we don't care about conn any more. */ -void -connection_unregister_events(connection_t *conn) -{ - if (conn->read_event) { - if (event_del(conn->read_event)) - log_warn(LD_BUG, "Error removing read event for %d", conn->s); - tor_free(conn->read_event); - } - if (conn->write_event) { - if (event_del(conn->write_event)) - log_warn(LD_BUG, "Error removing write event for %d", conn->s); - tor_free(conn->write_event); - } - if (conn->dns_server_port) { - dnsserv_close_listener(conn); - } -} - /** Deallocate memory used by conn. Deallocate its buffers if * necessary, close its socket if necessary, and mark the directory as dirty * if conn is an OR or OP connection. diff --git a/src/or/connection_edge.c b/src/or/connection_edge.c index 6207ad3178..944b68e367 100644 --- a/src/or/connection_edge.c +++ b/src/or/connection_edge.c @@ -334,7 +334,7 @@ connection_edge_finished_connecting(edge_connection_t *edge_conn) safe_str(fmt_addr(&conn->addr))); conn->state = EXIT_CONN_STATE_OPEN; - connection_watch_events(conn, EV_READ); /* stop writing, continue reading */ + connection_watch_events(conn, READ_EVENT);/* stop writing, continue reading */ if (connection_wants_to_flush(conn)) /* in case there are any queued relay * cells */ connection_start_writing(conn); @@ -2727,7 +2727,7 @@ connection_exit_connect(edge_connection_t *edge_conn) case 0: conn->state = EXIT_CONN_STATE_CONNECTING; - connection_watch_events(conn, EV_WRITE | EV_READ); + connection_watch_events(conn, READ_EVENT | WRITE_EVENT); /* writable indicates finish; * readable/error indicates broken link in windows-land. */ return; @@ -2740,7 +2740,7 @@ connection_exit_connect(edge_connection_t *edge_conn) log_warn(LD_BUG,"newly connected conn had data waiting!"); // connection_start_writing(conn); } - connection_watch_events(conn, EV_READ); + connection_watch_events(conn, READ_EVENT); /* also, deliver a 'connected' cell back through the circuit. */ if (connection_edge_is_rendezvous_stream(edge_conn)) { diff --git a/src/or/connection_or.c b/src/or/connection_or.c index b4e80926be..54dc1ab2f1 100644 --- a/src/or/connection_or.c +++ b/src/or/connection_or.c @@ -792,7 +792,7 @@ connection_or_connect(const tor_addr_t *_addr, uint16_t port, connection_free(TO_CONN(conn)); return NULL; case 0: - connection_watch_events(TO_CONN(conn), EV_READ | EV_WRITE); + connection_watch_events(TO_CONN(conn), READ_EVENT | WRITE_EVENT); /* writable indicates finish, readable indicates broken link, error indicates broken link on windows */ return conn; diff --git a/src/or/directory.c b/src/or/directory.c index d86495dca5..1b972a345b 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -794,7 +794,7 @@ directory_initiate_command_rend(const char *address, const tor_addr_t *_addr, payload, payload_len, supports_conditional_consensus, if_modified_since); - connection_watch_events(TO_CONN(conn), EV_READ | EV_WRITE); + connection_watch_events(TO_CONN(conn), READ_EVENT | WRITE_EVENT); /* writable indicates finish, readable indicates broken link, error indicates broken link in windowsland. */ } @@ -833,7 +833,7 @@ directory_initiate_command_rend(const char *address, const tor_addr_t *_addr, payload, payload_len, supports_conditional_consensus, if_modified_since); - connection_watch_events(TO_CONN(conn), EV_READ | EV_WRITE); + connection_watch_events(TO_CONN(conn), READ_EVENT|WRITE_EVENT); connection_start_reading(TO_CONN(linked_conn)); } } diff --git a/src/or/dns.c b/src/or/dns.c index ba34b406b9..8d00d23eee 100644 --- a/src/or/dns.c +++ b/src/or/dns.c @@ -14,9 +14,11 @@ #include "or.h" #include "ht.h" #ifdef HAVE_EVENT2_DNS_H +#include #include #include #else +#include #include "eventdns.h" #ifndef HAVE_EVDNS_SET_DEFAULT_OUTGOING_BIND_ADDRESS #define HAVE_EVDNS_SET_DEFAULT_OUTGOING_BIND_ADDRESS diff --git a/src/or/main.c b/src/or/main.c index c808845192..c3dae2ad54 100644 --- a/src/or/main.c +++ b/src/or/main.c @@ -18,6 +18,12 @@ #endif #include "memarea.h" +#ifdef HAVE_EVENT2_EVENT_H +#include +#else +#include +#endif + void evdns_shutdown(int); /********* PROTOTYPES **********/ @@ -140,6 +146,25 @@ connection_add(connection_t *conn) return 0; } +/** Tell libevent that we don't care about conn any more. */ +void +connection_unregister_events(connection_t *conn) +{ + if (conn->read_event) { + if (event_del(conn->read_event)) + log_warn(LD_BUG, "Error removing read event for %d", conn->s); + tor_free(conn->read_event); + } + if (conn->write_event) { + if (event_del(conn->write_event)) + log_warn(LD_BUG, "Error removing write event for %d", conn->s); + tor_free(conn->write_event); + } + if (conn->dns_server_port) { + dnsserv_close_listener(conn); + } +} + /** Remove the connection from the global list, and remove the * corresponding poll entry. Calling this function will shift the last * connection (if any) into the position occupied by conn. @@ -244,17 +269,17 @@ get_connection_array(void) } /** Set the event mask on conn to events. (The event - * mask is a bitmask whose bits are EV_READ and EV_WRITE.) + * mask is a bitmask whose bits are READ_EVENT and WRITE_EVENT) */ void -connection_watch_events(connection_t *conn, short events) +connection_watch_events(connection_t *conn, watchable_events_t events) { - if (events & EV_READ) + if (events & READ_EVENT) connection_start_reading(conn); else connection_stop_reading(conn); - if (events & EV_WRITE) + if (events & WRITE_EVENT) connection_start_writing(conn); else connection_stop_writing(conn); diff --git a/src/or/or.h b/src/or/or.h index e4989fc432..b6ee72d9b5 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -93,12 +93,6 @@ #include "address.h" #include "compat_libevent.h" -#ifdef HAVE_EVENT2_EVENT_H -#include -#else -#include -#endif - /* These signals are defined to help control_signal_act work. */ #ifndef SIGHUP @@ -2931,7 +2925,6 @@ control_connection_t *control_connection_new(int socket_family); connection_t *connection_new(int type, int socket_family); void connection_link_connections(connection_t *conn_a, connection_t *conn_b); -void connection_unregister_events(connection_t *conn); void connection_free(connection_t *conn); void connection_free_all(void); void connection_about_to_close_connection(connection_t *conn); @@ -3652,13 +3645,18 @@ extern int has_completed_circuit; int connection_add(connection_t *conn); int connection_remove(connection_t *conn); +void connection_unregister_events(connection_t *conn); int connection_in_array(connection_t *conn); void add_connection_to_closeable_list(connection_t *conn); int connection_is_on_closeable_list(connection_t *conn); smartlist_t *get_connection_array(void); -void connection_watch_events(connection_t *conn, short events); +typedef enum watchable_events { + READ_EVENT=0x02, + WRITE_EVENT=0x04 +} watchable_events_t; +void connection_watch_events(connection_t *conn, watchable_events_t events); int connection_is_reading(connection_t *conn); void connection_stop_reading(connection_t *conn); void connection_start_reading(connection_t *conn); From 74bf885b2d280c3006018cf9256c2ca3781bfcf9 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 12 Jun 2009 15:09:09 -0400 Subject: [PATCH 078/187] Whitespace and osx fixes on libevent2 patch. --- src/common/compat_libevent.c | 24 ++++++++++++------------ src/common/compat_libevent.h | 1 + src/or/connection_edge.c | 2 +- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/src/common/compat_libevent.c b/src/common/compat_libevent.c index d9843c9c3c..73308b4eab 100644 --- a/src/common/compat_libevent.c +++ b/src/common/compat_libevent.c @@ -36,6 +36,15 @@ */ typedef uint32_t le_version_t; +/* Macros: returns the number of a libevent version. */ +#define V(major, minor, patch) \ + (((major) << 24) | ((minor) << 16) | ((patch) << 8)) +#define V_OLD(major, minor, patch) \ + V((major), (minor), (patch)-'a'+1) + +#define LE_OLD V(0,0,0) +#define LE_OTHER V(0,0,99) + static le_version_t tor_get_libevent_version(const char **v_out); #ifdef HAVE_EVENT_SET_LOG_CALLBACK @@ -187,7 +196,6 @@ tor_libevent_get_base(void) return the_event_base; } - #ifndef HAVE_EVENT_BASE_LOOPEXIT /* Replacement for event_base_loopexit on some very old versions of Libevent that we are not yet brave enough to deprecate. */ @@ -212,15 +220,6 @@ tor_libevent_get_method(void) #endif } -/* Macros: returns the number of a libevent version. */ -#define V(major, minor, patch) \ - (((major) << 24) | ((minor) << 16) | ((patch) << 8)) -#define V_OLD(major, minor, patch) \ - V((major), (minor), (patch)-'a'+1) - -#define LE_OLD V(0,0,0) -#define LE_OTHER V(0,0,99) - /** Return the le_version_t for the current version of libevent. If the * version is very new, return LE_OTHER. If the version is so old that it * doesn't support event_get_version(), return LE_OLD. DOCDOC */ @@ -354,12 +353,12 @@ tor_check_libevent_version(const char *m, int server, /* Libevent versions before 1.3b do very badly on operating systems with * user-space threading implementations. */ #if defined(__OpenBSD__) || defined(__FreeBSD__) || defined(__NetBSD__) - if (server && version < V_OLD(1,3,'b')) + if (server && version < V_OLD(1,3,'b')) { thread_unsafe = 1; sad_os = "BSD variants"; } #elif defined(__APPLE__) || defined(__darwin__) - if (server && version < V_OLD(1,3,'b')) + if (server && version < V_OLD(1,3,'b')) { thread_unsafe = 1; sad_os = "Mac OS X"; } @@ -467,3 +466,4 @@ tor_check_libevent_header_compatibility(void) /* Your libevent is ancient. */ #endif } + diff --git a/src/common/compat_libevent.h b/src/common/compat_libevent.h index b70f8d3083..d2e76ce4e2 100644 --- a/src/common/compat_libevent.h +++ b/src/common/compat_libevent.h @@ -51,3 +51,4 @@ void tor_check_libevent_header_compatibility(void); const char *tor_libevent_get_version_str(void); #endif + diff --git a/src/or/connection_edge.c b/src/or/connection_edge.c index 944b68e367..5600d52aaf 100644 --- a/src/or/connection_edge.c +++ b/src/or/connection_edge.c @@ -334,7 +334,7 @@ connection_edge_finished_connecting(edge_connection_t *edge_conn) safe_str(fmt_addr(&conn->addr))); conn->state = EXIT_CONN_STATE_OPEN; - connection_watch_events(conn, READ_EVENT);/* stop writing, continue reading */ + connection_watch_events(conn, READ_EVENT); /* stop writing, keep reading */ if (connection_wants_to_flush(conn)) /* in case there are any queued relay * cells */ connection_start_writing(conn); From ca8708a9ce3f8e6a4b3aadf2e35edfd2b49c1a2d Mon Sep 17 00:00:00 2001 From: Karsten Loesing Date: Sat, 13 Jun 2009 12:21:58 +0200 Subject: [PATCH 079/187] Fix more of bug 997. Fix refetching of hidden service descriptors when all introduction points have turned out to not work. --- ChangeLog | 2 ++ src/or/rendclient.c | 7 +++---- src/or/rendcommon.c | 5 +++++ 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/ChangeLog b/ChangeLog index d2a2261844..2e4eadc688 100644 --- a/ChangeLog +++ b/ChangeLog @@ -19,6 +19,8 @@ Changes in version 0.2.2.1-alpha - 2009-??-?? use a cached descriptor no matter how old it is and only fetch a new one when all introduction points fail. Fix for bug 997. Patch from Marcus Griep. + - Fix refetching of hidden service descriptors when all introduction + points have turned out to not work. Fixes more of bug 997. o Deprecated and removed features: - The controller no longer accepts the old obsolete "addr-mappings/" diff --git a/src/or/rendclient.c b/src/or/rendclient.c index d9c0a9e713..00b77d8370 100644 --- a/src/or/rendclient.c +++ b/src/or/rendclient.c @@ -447,10 +447,9 @@ directory_get_from_hs_dir(const char *desc_id, const rend_data_t *rend_query) return 1; } -/** Start a connection to a hidden service directory to fetch a v2 - * rendezvous service descriptor for the base32-encoded service ID - * query. - */ +/** Unless we already have a descriptor for rend_query with at least + * one (possibly) working introduction point in it, start a connection to a + * hidden service directory to fetch a v2 rendezvous service descriptor. */ void rend_client_refetch_v2_renddesc(const rend_data_t *rend_query) { diff --git a/src/or/rendcommon.c b/src/or/rendcommon.c index c813c4959d..df7195e3ea 100644 --- a/src/or/rendcommon.c +++ b/src/or/rendcommon.c @@ -911,6 +911,11 @@ rend_cache_lookup_entry(const char *query, int version, rend_cache_entry_t **e) } if (!*e) return 0; + tor_assert((*e)->parsed && (*e)->parsed->intro_nodes); + /* XXX022 hack for now, to return "not found" if there are no intro + * points remaining. See bug 997. */ + if (smartlist_len((*e)->parsed->intro_nodes) == 0) + return 0; return 1; } From bdd5785f075d89123f6ac8110ea86f31981ae3b1 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 15 Jun 2009 14:18:32 -0400 Subject: [PATCH 080/187] Small changes to proposals 158 and 162 based on comments from arma --- doc/spec/proposals/158-microdescriptors.txt | 63 ++++++++++++-------- doc/spec/proposals/162-consensus-flavors.txt | 2 +- 2 files changed, 38 insertions(+), 27 deletions(-) diff --git a/doc/spec/proposals/158-microdescriptors.txt b/doc/spec/proposals/158-microdescriptors.txt index 3dff9f85c7..e6966c0cef 100644 --- a/doc/spec/proposals/158-microdescriptors.txt +++ b/doc/spec/proposals/158-microdescriptors.txt @@ -13,6 +13,8 @@ Status: Open Added suggestion to use a new consensus flavor. Specified use of SHA256 for new hashes. -nickm + 15 June 2009: Cleaned up based on comments from Roger. -nickm + 1. Overview This proposal replaces section 3.2 of proposal 141, which was @@ -20,9 +22,7 @@ Status: Open circuit-building protocol to fetch a server descriptor inline at each circuit extend, we instead put all of the information that clients need either into the consensus itself, or into a new set of data about each - relay called a microdescriptor. The microdescriptor is a direct - transform from the relay descriptor, so relays don't even need to know - this is happening. + relay called a microdescriptor. Descriptor elements that are small and frequently changing should go in the consensus itself, and descriptor elements that are small and @@ -47,16 +47,17 @@ Status: Open 3. Design There are three pieces to the proposal. First, authorities will list in - their votes (and thus in the consensus) the expected hash - of microdescriptor for each relay. Second, directory mirrors will serve - microdescriptors. Third, clients will ask for them and cache them. + their votes (and thus in the consensus) the expected hash of + microdescriptor for each relay. Second, authorities will serve + microdescriptors, directory mirrors will cache and serve + them. Third, clients will ask for them and cache them. 3.1. Consensus changes If the authorities choose a consensus method of a given version or later, a microdescriptor format is implicit in that version. A microdescriptor should in every case be a pure function of the - router descriptor and the conensus method. + router descriptor and the consensus method. In votes, we need to include the hash of each expected microdescriptor in the routerstatus section. I suggest a new "m" line for each stanza, @@ -64,7 +65,7 @@ Status: Open For every consensus method that an authority supports, it includes a separate "m" line in each router section of its vote, containing: - "m" SP methods SP digest NL + "m" SP methods 1*(SP AlgorithmName "=" digest) NL where methods is a comma-separated list of the consensus methods that the authority believes will produce "digest". @@ -77,20 +78,28 @@ Status: Open (This means we need a new consensus-method that knows how to compute the microdescriptor-elements and add "m" lines.) + The microdescriptor consensus uses the directory-signature format from + proposal 162, with the "sha256" algorithm. + + 3.1.1. Descriptor elements to include for now In the first version, the microdescriptor should contain the - onion-key element and the family element from the router descriptor. + onion-key element, and the family element from the router descriptor, + and the exit policy summary as currently specified in dir-spec.txt. 3.1.2. Computing consensus for microdescriptor-elements and "m" lines When we are generating a consensus, we use whichever m line unambiguously corresponds to the descriptor digest that will be - included in the consensus. (If there are multiple m lines for that - descriptor digest, we use whichever is most common. If they are - equally common, we break ties in the favor of the lexically - earliest. Either way, we should log a warning: That's likely a - bug.) + included in the consensus. + + (If different votes have different microdescriptor digests for a + single pair, then at least one + of the authorities is broken. If this happens, the consensus should + contain whichever microdescriptor digest is most common. If there is + no winner, we break ties in the favor of the lexically earliest. + Either way, we should log a warning: there is definitely a bug.) The "m" lines in a consensus contain only the digest, not a list of consensus methods. @@ -103,26 +112,26 @@ Status: Open This flavor can safely omit descriptor digests. - We still need to decide whether to move ports into microdescriptors - or not. In either case, they can be removed from the current "ns" - flavor of consensus, since no current clients use them, and they - take up about 5% of the compressed consensus. + When we implement this voting method, we can remove the exit policy + summary from the current "ns" flavor of consensus, since no current + clients use them, and they take up about 5% of the compressed + consensus. This new consensus flavor should be signed with the sha256 signature format as documented in proposal 162. 3.2. Directory mirrors fetch, cache, and serve microdescriptors - Directory mirrors should then read the microdescriptor-elements line - from the consensus, and learn how to answer requests. (Directory mirrors - continue to serve normal relay descriptors too, a) to serve old clients - and b) to be able to construct microdescriptors on the fly.) + Directory mirrors should fetch, catch, and serve each microdescriptor + from the authorities. (They need to continue to serve normal relay + descriptors too, to handle old clients.) - The microdescriptors with base64 hashes ,, should be available at: + The microdescriptors with base64 hashes ,, should be + available at: http:///tor/micro/d/--.z (We use base64 for size and for consistency with the consensus format. We use -s instead of +s to separate these items, since - the + character use used in base64 encoding.) + the + character is used in base64 encoding.) All the microdescriptors from the current consensus should also be available at: @@ -173,11 +182,13 @@ Status: Open Another future option would be to fetch some of the microdescriptors anonymously (via a Tor circuit). + Another crazy option (Roger's phrasing) is to do decoy fetches as + well. + 4. Transition and deployment Phase one, the directory authorities should start voting on - microdescriptors and microdescriptor elements, and putting them in the - consensus. + microdescriptors, and putting them in the consensus. Phase two, directory mirrors should learn how to serve them, and learn how to read the consensus to find out what they should be serving. diff --git a/doc/spec/proposals/162-consensus-flavors.txt b/doc/spec/proposals/162-consensus-flavors.txt index caff96c0d7..8fdf9d07bf 100644 --- a/doc/spec/proposals/162-consensus-flavors.txt +++ b/doc/spec/proposals/162-consensus-flavors.txt @@ -134,7 +134,7 @@ Spec modifications: Caches should fetch this document so they can check the correctness of the different consensus documents they fetch. They do not need to check anything about an unrecognized - consensus document beyond its digest. + consensus document beyond its digest and length. 4.1. The "sha256" signature format. From 0caf8dd0b6b047e0cc050837b2eea1746c508a5d Mon Sep 17 00:00:00 2001 From: Sebastian Hahn Date: Tue, 16 Jun 2009 17:30:08 +0200 Subject: [PATCH 081/187] Fix bug 1001 For compatibility with Libevent2, tor_event_new should accept a NULL base without crashing. --- src/common/compat_libevent.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/common/compat_libevent.c b/src/common/compat_libevent.c index 73308b4eab..793e174477 100644 --- a/src/common/compat_libevent.c +++ b/src/common/compat_libevent.c @@ -114,6 +114,8 @@ tor_event_new(struct event_base *base, int sock, short what, { struct event *e = tor_malloc_zero(sizeof(struct event)); event_set(e, sock, what, cb, arg); + if (! base) + base = tor_libevent_get_base(); event_base_set(base, e); return e; } From b871cdf64bc6ee3956fbec8b611588f417eae52f Mon Sep 17 00:00:00 2001 From: Marcus Griep Date: Fri, 12 Jun 2009 12:16:53 -0400 Subject: [PATCH 082/187] Remove bashism from torify script Signed-off-by: Marcus Griep --- contrib/torify.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/torify.in b/contrib/torify.in index 6b1e440dc1..5bf7d4dbcf 100755 --- a/contrib/torify.in +++ b/contrib/torify.in @@ -52,7 +52,7 @@ then shift 1 fi -if [ "$PROG" == "$TSOCKS" ] +if [ "$PROG" = "$TSOCKS" ] then # Define our tsocks config file TSOCKS_CONF_FILE="/etc/tor/tor-tsocks.conf" @@ -70,7 +70,7 @@ then exit 1 fi fi -if [ "$PROG" == "$TORSOCKS" ] +if [ "$PROG" = "$TORSOCKS" ] then exec torsocks "$@" fi From 94e8c34cb770e2206bd0dbf96c3a36137355171d Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 18 Jun 2009 10:07:26 -0400 Subject: [PATCH 083/187] Set EV_PERSIST flag on signal events with Libevent < 2.0. Fix for bug 1007. --- src/common/compat_libevent.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/compat_libevent.c b/src/common/compat_libevent.c index 793e174477..32c6d4c8bc 100644 --- a/src/common/compat_libevent.c +++ b/src/common/compat_libevent.c @@ -131,7 +131,7 @@ struct event * tor_evsignal_new(struct event_base * base, int sig, void (*cb)(int, short, void *), void *arg) { - return tor_event_new(base, sig, EV_SIGNAL, cb, arg); + return tor_event_new(base, sig, EV_SIGNAL|EV_PERSIST, cb, arg); } /** Work-alike replacement for event_free() on pre-Libevent-2.0 systems. */ void From 75472c19c3fdcda913eb8117c917ddfd445b2b77 Mon Sep 17 00:00:00 2001 From: Christopher Davis Date: Thu, 18 Jun 2009 16:59:18 -0700 Subject: [PATCH 084/187] Enable Tor to connect through SOCKS 4/5 proxies Added a sanity check in config.c and a check in directory.c directory_initiate_command_rend() to catch any direct connection attempts when a socks proxy is configured. --- doc/tor.1.in | 19 +++ src/or/buffers.c | 171 ++++++++++++++++++++ src/or/config.c | 46 ++++++ src/or/connection.c | 360 ++++++++++++++++++++++++++++++++++++++++- src/or/connection_or.c | 137 ++++++---------- src/or/directory.c | 9 ++ src/or/or.h | 51 ++++-- 7 files changed, 687 insertions(+), 106 deletions(-) diff --git a/doc/tor.1.in b/doc/tor.1.in index 3ac0f92fe2..45d3ce7bdb 100644 --- a/doc/tor.1.in +++ b/doc/tor.1.in @@ -292,6 +292,25 @@ HTTPS proxy authentication that Tor supports; feel free to submit a patch if you want it to support others. .LP .TP +\fBSocks4Proxy\fR \fIhost\fR[:\fIport\fR]\fP +Tor will make all OR connections through the SOCKS 4 proxy at host:port +(or host:1080 if port is not specified). +.LP +.TP +\fBSocks5Proxy\fR \fIhost\fR[:\fIport\fR]\fP +Tor will make all OR connections through the SOCKS 5 proxy at host:port +(or host:1080 if port is not specified). +.LP +.TP +\fBSocks5ProxyUsername\fR \fIusername\fP +.LP +.TP +\fBSocks5ProxyPassword\fR \fIpassword\fP +If defined, authenticate to the SOCKS 5 server using username and password +in accordance to RFC 1929. Both username and password must be between 1 and 255 +characters. +.LP +.TP \fBKeepalivePeriod \fR\fINUM\fP To keep firewalls from expiring connections, send a padding keepalive cell every NUM seconds on open connections that are in use. If the diff --git a/src/or/buffers.c b/src/or/buffers.c index ed39bc0f82..31725c5f27 100644 --- a/src/or/buffers.c +++ b/src/or/buffers.c @@ -1611,6 +1611,177 @@ fetch_from_buf_socks(buf_t *buf, socks_request_t *req, } } +/** Inspect a reply from SOCKS server stored in buf according + * to state, removing the protocol data upon success. Return 0 on + * incomplete response, 1 on success and -1 on error, in which case + * reason is set to a descriptive message (free() when finished + * with it). + * + * As a special case, 2 is returned when user/pass is required + * during SOCKS5 handshake and user/pass is configured. + */ +int +fetch_from_buf_socks_client(buf_t *buf, int state, char **reason) +{ + unsigned char *data; + size_t addrlen; + + if (buf->datalen < 2) + return 0; + + buf_pullup(buf, 128, 0); + tor_assert(buf->head && buf->head->datalen >= 2); + + data = (unsigned char *) buf->head->data; + + switch (state) { + case PROXY_SOCKS4_WANT_CONNECT_OK: + /* Wait for the complete response */ + if (buf->head->datalen < 8) + return 0; + + if (data[1] != 0x5a) { + switch (data[1]) { + case 0x5b: + *reason = tor_strdup("server rejected connection"); + break; + case 0x5c: + *reason = tor_strdup("server cannot connect to identd " + "on this client"); + break; + case 0x5d: + *reason = tor_strdup("user id does not match identd"); + break; + default: + *reason = tor_strdup("invalid SOCKS 4 response code"); + break; + } + + return -1; + } + + /* Success */ + buf_remove_from_front(buf, 8); + return 1; + + case PROXY_SOCKS5_WANT_AUTH_METHOD_NONE: + /* we don't have any credentials */ + if (data[1] != 0x00) { + *reason = tor_strdup("server doesn't support any of our " + "available authentication methods"); + return -1; + } + + log_info(LD_NET, "SOCKS 5 client: continuing without authentication"); + buf_clear(buf); + return 1; + + case PROXY_SOCKS5_WANT_AUTH_METHOD_RFC1929: + /* we have a username and password. return 1 if we can proceed without + * providing authentication, or 2 otherwise. */ + switch (data[1]) { + case 0x00: + log_info(LD_NET, "SOCKS 5 client: we have auth details but server " + "doesn't require authentication."); + buf_clear(buf); + return 1; + case 0x02: + log_info(LD_NET, "SOCKS 5 client: need authentication."); + buf_clear(buf); + return 2; + /* fall through */ + } + + *reason = tor_strdup("server doesn't support any of our available " + "authentication methods"); + return -1; + + case PROXY_SOCKS5_WANT_AUTH_RFC1929_OK: + /* handle server reply to rfc1929 authentication */ + if (data[1] != 0x00) { + *reason = tor_strdup("authentication failed"); + return -1; + } + + log_info(LD_NET, "SOCKS 5 client: authentication successful."); + buf_clear(buf); + return 1; + + case PROXY_SOCKS5_WANT_CONNECT_OK: + /* response is variable length. BND.ADDR, etc, isn't needed + * (don't bother with buf_pullup()), but make sure to eat all + * the data used */ + + /* wait for address type field to arrive */ + if (buf->datalen < 4) + return 0; + + switch (data[3]) { + case 0x01: /* ip4 */ + addrlen = 4; + break; + case 0x04: /* ip6 */ + addrlen = 16; + break; + case 0x03: /* fqdn (can this happen here?) */ + if (buf->datalen < 5) + return 0; + addrlen = 1 + data[4]; + break; + default: + *reason = tor_strdup("invalid response to connect request"); + return -1; + } + + /* wait for address and port */ + if (buf->datalen < 6 + addrlen) + return 0; + + if (data[1] != 0x00) { + + switch (data[1]) { + case 0x01: + *reason = tor_strdup("general SOCKS server failure"); + break; + case 0x02: + *reason = tor_strdup("connection not allowed by ruleset"); + break; + case 0x03: + *reason = tor_strdup("Network unreachable"); + break; + case 0x04: + *reason = tor_strdup("Host unreachable"); + break; + case 0x05: + *reason = tor_strdup("Connection refused"); + break; + case 0x06: + *reason = tor_strdup("TTL expired"); + break; + case 0x07: + *reason = tor_strdup("Command not supported"); + break; + case 0x08: + *reason = tor_strdup("Address type not supported"); + break; + default: + *reason = tor_strdup("unknown reason"); + break; + } + + return -1; + } + + buf_remove_from_front(buf, 6 + addrlen); + return 1; + } + + /* shouldn't get here... */ + tor_assert(0); + + return -1; +} + /** Return 1 iff buf looks more like it has an (obsolete) v0 controller * command on it than any valid v1 controller command. */ int diff --git a/src/or/config.c b/src/or/config.c index bcabb67a99..e1830fa7c2 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -240,6 +240,10 @@ static config_var_t _option_vars[] = { V(HttpProxyAuthenticator, STRING, NULL), V(HttpsProxy, STRING, NULL), V(HttpsProxyAuthenticator, STRING, NULL), + V(Socks4Proxy, STRING, NULL), + V(Socks5Proxy, STRING, NULL), + V(Socks5ProxyUsername, STRING, NULL), + V(Socks5ProxyPassword, STRING, NULL), OBSOLETE("IgnoreVersion"), V(KeepalivePeriod, INTERVAL, "5 minutes"), VAR("Log", LINELIST, Logs, NULL), @@ -3409,6 +3413,42 @@ options_validate(or_options_t *old_options, or_options_t *options, REJECT("HttpsProxyAuthenticator is too long (>= 48 chars)."); } + if (options->Socks4Proxy) { /* parse it now */ + if (parse_addr_port(LOG_WARN, options->Socks4Proxy, NULL, + &options->Socks4ProxyAddr, + &options->Socks4ProxyPort) <0) + REJECT("Socks4Proxy failed to parse or resolve. Please fix."); + if (options->Socks4ProxyPort == 0) { /* give it a default */ + options->Socks4ProxyPort = 1080; + } + } + + if (options->Socks5Proxy) { /* parse it now */ + if (parse_addr_port(LOG_WARN, options->Socks5Proxy, NULL, + &options->Socks5ProxyAddr, + &options->Socks5ProxyPort) <0) + REJECT("Socks5Proxy failed to parse or resolve. Please fix."); + if (options->Socks5ProxyPort == 0) { /* give it a default */ + options->Socks5ProxyPort = 1080; + } + } + + if (options->Socks5ProxyUsername) { + size_t len; + + len = strlen(options->Socks5ProxyUsername); + if (len < 1 || len > 255) + REJECT("Socks5ProxyUsername must be between 1 and 255 characters."); + + if (!options->Socks5ProxyPassword) + REJECT("Socks5ProxyPassword must be included with Socks5ProxyUsername."); + + len = strlen(options->Socks5ProxyPassword); + if (len < 1 || len > 255) + REJECT("Socks5ProxyPassword must be between 1 and 255 characters."); + } else if (options->Socks5ProxyPassword) + REJECT("Socks5ProxyPassword must be included with Socks5ProxyUsername."); + if (options->HashedControlPassword) { smartlist_t *sl = decode_hashed_passwords(options->HashedControlPassword); if (!sl) { @@ -3557,6 +3597,12 @@ options_validate(or_options_t *old_options, or_options_t *options, if (options->PreferTunneledDirConns && !options->TunnelDirConns) REJECT("Must set TunnelDirConns if PreferTunneledDirConns is set."); + if ((options->Socks4Proxy || options->Socks5Proxy) && + !options->HttpProxy && !options->PreferTunneledDirConns) + REJECT("When Socks4Proxy or Socks5Proxy is configured, " + "PreferTunneledDirConns and TunnelDirConns must both be " + "set to 1, or HttpProxy must be configured."); + if (options->AutomapHostsSuffixes) { SMARTLIST_FOREACH(options->AutomapHostsSuffixes, char *, suf, { diff --git a/src/or/connection.c b/src/or/connection.c index c8406fee3f..ccfbe19ed5 100644 --- a/src/or/connection.c +++ b/src/or/connection.c @@ -32,6 +32,10 @@ static int connection_process_inbuf(connection_t *conn, int package_partial); static void client_check_address_changed(int sock); static void set_constrained_socket_buffers(int sock, int size); +static const char *connection_proxy_state_to_string(int state); +static int connection_read_https_proxy_response(connection_t *conn); +static void connection_send_socks5_connect(connection_t *conn); + /** The last IPv4 address that our network interface seemed to have been * binding to, in host order. We use this to detect when our IP changes. */ static uint32_t last_interface_ip = 0; @@ -92,8 +96,7 @@ conn_state_to_string(int type, int state) case CONN_TYPE_OR: switch (state) { case OR_CONN_STATE_CONNECTING: return "connect()ing"; - case OR_CONN_STATE_PROXY_FLUSHING: return "proxy flushing"; - case OR_CONN_STATE_PROXY_READING: return "proxy reading"; + case OR_CONN_STATE_PROXY_HANDSHAKING: return "handshaking (proxy)"; case OR_CONN_STATE_TLS_HANDSHAKING: return "handshaking (TLS)"; case OR_CONN_STATE_TLS_CLIENT_RENEGOTIATING: return "renegotiating (TLS)"; @@ -1289,6 +1292,353 @@ connection_connect(connection_t *conn, const char *address, return inprogress ? 0 : 1; } +/** Convert state number to string representation for logging purposes. + */ +static const char * +connection_proxy_state_to_string(int state) +{ + static const char *unknown = "???"; + static const char *states[] = { + "PROXY_NONE", + "PROXY_HTTPS_WANT_CONNECT_OK", + "PROXY_SOCKS4_WANT_CONNECT_OK", + "PROXY_SOCKS5_WANT_AUTH_METHOD_NONE", + "PROXY_SOCKS5_WANT_AUTH_METHOD_RFC1929", + "PROXY_SOCKS5_WANT_AUTH_RFC1929_OK", + "PROXY_SOCKS5_WANT_CONNECT_OK", + "PROXY_CONNECTED", + }; + + if (state < PROXY_NONE || state > PROXY_CONNECTED) + return unknown; + + return states[state]; +} + +/** Write a proxy request of type (socks4, socks5, https) to conn + * for conn->addr:conn->port, authenticating with the auth details given + * in the configuration (if available). SOCKS 5 and HTTP CONNECT proxies + * support authentication. + * + * Returns -1 if conn->addr is incompatible with the proxy protocol, and + * 0 otherwise. + * + * Use connection_read_proxy_handshake() to complete the handshake. + */ +int +connection_proxy_connect(connection_t *conn, int type) +{ + or_options_t *options; + + tor_assert(conn); + + options = get_options(); + + switch (type) { + case PROXY_CONNECT: { + char buf[1024]; + char *base64_authenticator=NULL; + const char *authenticator = options->HttpsProxyAuthenticator; + + /* Send HTTP CONNECT and authentication (if available) in + * one request */ + + if (authenticator) { + base64_authenticator = alloc_http_authenticator(authenticator); + if (!base64_authenticator) + log_warn(LD_OR, "Encoding https authenticator failed"); + } + + if (base64_authenticator) { + tor_snprintf(buf, sizeof(buf), "CONNECT %s:%d HTTP/1.1\r\n" + "Proxy-Authorization: Basic %s\r\n\r\n", + fmt_addr(&conn->addr), + conn->port, base64_authenticator); + tor_free(base64_authenticator); + } else { + tor_snprintf(buf, sizeof(buf), "CONNECT %s:%d HTTP/1.0\r\n\r\n", + fmt_addr(&conn->addr), conn->port); + } + + connection_write_to_buf(buf, strlen(buf), conn); + conn->proxy_state = PROXY_HTTPS_WANT_CONNECT_OK; + break; + } + + case PROXY_SOCKS4: { + unsigned char buf[9]; + uint16_t portn; + uint32_t ip4addr; + + /* Send a SOCKS4 connect request with empty user id */ + + if (tor_addr_family(&conn->addr) != AF_INET) { + log_warn(LD_NET, "SOCKS4 client is incompatible with with IPv6"); + return -1; + } + + ip4addr = tor_addr_to_ipv4n(&conn->addr); + portn = htons(conn->port); + + buf[0] = 4; /* version */ + buf[1] = SOCKS_COMMAND_CONNECT; /* command */ + memcpy(buf + 2, &portn, 2); /* port */ + memcpy(buf + 4, &ip4addr, 4); /* addr */ + buf[8] = 0; /* userid (empty) */ + + connection_write_to_buf((char *)buf, sizeof buf, conn); + conn->proxy_state = PROXY_SOCKS4_WANT_CONNECT_OK; + break; + } + + case PROXY_SOCKS5: { + unsigned char buf[4]; /* fields: vers, num methods, method list */ + + /* Send a SOCKS5 greeting (connect request must wait) */ + + buf[0] = 5; /* version */ + + /* number of auth methods */ + if (options->Socks5ProxyUsername) { + buf[1] = 2; + buf[2] = 0x00; /* no authentication */ + buf[3] = 0x02; /* rfc1929 Username/Passwd auth */ + conn->proxy_state = PROXY_SOCKS5_WANT_AUTH_METHOD_RFC1929; + } else { + buf[1] = 1; + buf[2] = 0x00; /* no authentication */ + conn->proxy_state = PROXY_SOCKS5_WANT_AUTH_METHOD_NONE; + } + + connection_write_to_buf((char *)buf, 2 + buf[1], conn); + break; + } + + default: + log_err(LD_BUG, "Invalid proxy protocol, %d", type); + tor_fragile_assert(); + return -1; + } + + log_debug(LD_NET, "set state %s", + connection_proxy_state_to_string(conn->proxy_state)); + + return 0; +} + +/** Read conn's inbuf. If the http response from the proxy is all + * here, make sure it's good news, then return 1. If it's bad news, + * return -1. Else return 0 and hope for better luck next time. + */ +static int +connection_read_https_proxy_response(connection_t *conn) +{ + char *headers; + char *reason=NULL; + int status_code; + time_t date_header; + + switch (fetch_from_buf_http(conn->inbuf, + &headers, MAX_HEADERS_SIZE, + NULL, NULL, 10000, 0)) { + case -1: /* overflow */ + log_warn(LD_PROTOCOL, + "Your https proxy sent back an oversized response. Closing."); + return -1; + case 0: + log_info(LD_NET,"https proxy response not all here yet. Waiting."); + return 0; + /* case 1, fall through */ + } + + if (parse_http_response(headers, &status_code, &date_header, + NULL, &reason) < 0) { + log_warn(LD_NET, + "Unparseable headers from proxy (connecting to '%s'). Closing.", + conn->address); + tor_free(headers); + return -1; + } + if (!reason) reason = tor_strdup("[no reason given]"); + + if (status_code == 200) { + log_info(LD_NET, + "HTTPS connect to '%s' successful! (200 %s) Starting TLS.", + conn->address, escaped(reason)); + tor_free(reason); + return 1; + } + /* else, bad news on the status code */ + log_warn(LD_NET, + "The https proxy sent back an unexpected status code %d (%s). " + "Closing.", + status_code, escaped(reason)); + tor_free(reason); + return -1; +} + +/** Send SOCKS5 CONNECT command to conn, copying conn->addr + * and conn->port into the request. + */ +static void +connection_send_socks5_connect(connection_t *conn) +{ + unsigned char buf[1024]; + size_t reqsize = 6; + uint16_t port = htons(conn->port); + + buf[0] = 5; /* version */ + buf[1] = SOCKS_COMMAND_CONNECT; /* command */ + buf[2] = 0; /* reserved */ + + if (tor_addr_family(&conn->addr) == AF_INET) { + uint32_t addr = tor_addr_to_ipv4n(&conn->addr); + + buf[3] = 1; + reqsize += 4; + memcpy(buf + 4, &addr, 4); + memcpy(buf + 8, &port, 2); + } else { /* AF_INET6 */ + buf[3] = 4; + reqsize += 16; + memcpy(buf + 4, tor_addr_to_in6(&conn->addr), 16); + memcpy(buf + 20, &port, 2); + } + + connection_write_to_buf((char *)buf, reqsize, conn); + + conn->proxy_state = PROXY_SOCKS5_WANT_CONNECT_OK; +} + +/** Call this from connection_*_process_inbuf() to advance the proxy + * handshake. + * + * No matter what proxy protocol is used, if this function returns 1, the + * handshake is complete, and the data remaining on inbuf may contain the + * start of the communication with the requested server. + * + * Returns 0 if the current buffer contains an incomplete response, and -1 + * on error. + */ +int +connection_read_proxy_handshake(connection_t *conn) +{ + int ret = 0; + char *reason = NULL; + + log_debug(LD_NET, "enter state %s", + connection_proxy_state_to_string(conn->proxy_state)); + + switch (conn->proxy_state) { + case PROXY_HTTPS_WANT_CONNECT_OK: + ret = connection_read_https_proxy_response(conn); + if (ret == 1) + conn->proxy_state = PROXY_CONNECTED; + break; + + case PROXY_SOCKS4_WANT_CONNECT_OK: + ret = fetch_from_buf_socks_client(conn->inbuf, + conn->proxy_state, + &reason); + if (ret == 1) + conn->proxy_state = PROXY_CONNECTED; + break; + + case PROXY_SOCKS5_WANT_AUTH_METHOD_NONE: + ret = fetch_from_buf_socks_client(conn->inbuf, + conn->proxy_state, + &reason); + /* no auth needed, do connect */ + if (ret == 1) { + connection_send_socks5_connect(conn); + ret = 0; + } + break; + + case PROXY_SOCKS5_WANT_AUTH_METHOD_RFC1929: + ret = fetch_from_buf_socks_client(conn->inbuf, + conn->proxy_state, + &reason); + + /* send auth if needed, otherwise do connect */ + if (ret == 1) { + connection_send_socks5_connect(conn); + ret = 0; + } else if (ret == 2) { + unsigned char buf[1024]; + size_t reqsize, usize, psize; + const char *user, *pass; + + user = get_options()->Socks5ProxyUsername; + pass = get_options()->Socks5ProxyPassword; + tor_assert(user && pass); + + /* XXX len of user and pass must be <= 255 !!! */ + usize = strlen(user); + psize = strlen(pass); + tor_assert(usize <= 255 && psize <= 255); + reqsize = 3 + usize + psize; + + buf[0] = 1; /* negotiation version */ + buf[1] = usize; + memcpy(buf + 2, user, usize); + buf[2 + usize] = psize; + memcpy(buf + 3 + usize, pass, psize); + + connection_write_to_buf((char *)buf, reqsize, conn); + + conn->proxy_state = PROXY_SOCKS5_WANT_AUTH_RFC1929_OK; + ret = 0; + } + break; + + case PROXY_SOCKS5_WANT_AUTH_RFC1929_OK: + ret = fetch_from_buf_socks_client(conn->inbuf, + conn->proxy_state, + &reason); + /* send the connect request */ + if (ret == 1) { + connection_send_socks5_connect(conn); + ret = 0; + } + break; + + case PROXY_SOCKS5_WANT_CONNECT_OK: + ret = fetch_from_buf_socks_client(conn->inbuf, + conn->proxy_state, + &reason); + if (ret == 1) + conn->proxy_state = PROXY_CONNECTED; + break; + + default: + log_err(LD_BUG, "Invalid proxy_state for reading, %d", + conn->proxy_state); + tor_fragile_assert(); + ret = -1; + break; + } + + log_debug(LD_NET, "leaving state %s", + connection_proxy_state_to_string(conn->proxy_state)); + + if (ret < 0) { + if (reason) { + log_warn(LD_NET, "Proxy Client: unable to connect to %s:%d (%s)", + conn->address, conn->port, escaped(reason)); + tor_free(reason); + } else { + log_warn(LD_NET, "Proxy Client: unable to connect to %s:%d", + conn->address, conn->port); + } + } else if (ret == 1) { + log_info(LD_NET, "Proxy Client: connection to %s:%d successful", + conn->address, conn->port); + } + + return ret; +} + /** * Launch any configured listener connections of type type. (A * listener is configured if port_option is non-zero. If any @@ -2049,7 +2399,7 @@ connection_read_to_buf(connection_t *conn, int *max_to_read, int *socket_error) } if (connection_speaks_cells(conn) && - conn->state > OR_CONN_STATE_PROXY_READING) { + conn->state > OR_CONN_STATE_PROXY_HANDSHAKING) { int pending; or_connection_t *or_conn = TO_OR_CONN(conn); size_t initial_size; @@ -2277,7 +2627,7 @@ connection_handle_write(connection_t *conn, int force) : connection_bucket_write_limit(conn, now); if (connection_speaks_cells(conn) && - conn->state > OR_CONN_STATE_PROXY_READING) { + conn->state > OR_CONN_STATE_PROXY_HANDSHAKING) { or_connection_t *or_conn = TO_OR_CONN(conn); if (conn->state == OR_CONN_STATE_TLS_HANDSHAKING || conn->state == OR_CONN_STATE_TLS_CLIENT_RENEGOTIATING) { @@ -3003,7 +3353,7 @@ assert_connection_ok(connection_t *conn, time_t now) } // tor_assert(conn->addr && conn->port); tor_assert(conn->address); - if (conn->state > OR_CONN_STATE_PROXY_READING) + if (conn->state > OR_CONN_STATE_PROXY_HANDSHAKING) tor_assert(or_conn->tls); } diff --git a/src/or/connection_or.c b/src/or/connection_or.c index 54dc1ab2f1..bbfa611c0d 100644 --- a/src/or/connection_or.c +++ b/src/or/connection_or.c @@ -187,66 +187,6 @@ connection_or_reached_eof(or_connection_t *conn) return 0; } -/** Read conn's inbuf. If the http response from the proxy is all - * here, make sure it's good news, and begin the tls handshake. If - * it's bad news, close the connection and return -1. Else return 0 - * and hope for better luck next time. - */ -static int -connection_or_read_proxy_response(or_connection_t *or_conn) -{ - char *headers; - char *reason=NULL; - int status_code; - time_t date_header; - connection_t *conn = TO_CONN(or_conn); - - switch (fetch_from_buf_http(conn->inbuf, - &headers, MAX_HEADERS_SIZE, - NULL, NULL, 10000, 0)) { - case -1: /* overflow */ - log_warn(LD_PROTOCOL, - "Your https proxy sent back an oversized response. Closing."); - return -1; - case 0: - log_info(LD_OR,"https proxy response not all here yet. Waiting."); - return 0; - /* case 1, fall through */ - } - - if (parse_http_response(headers, &status_code, &date_header, - NULL, &reason) < 0) { - log_warn(LD_OR, - "Unparseable headers from proxy (connecting to '%s'). Closing.", - conn->address); - tor_free(headers); - return -1; - } - if (!reason) reason = tor_strdup("[no reason given]"); - - if (status_code == 200) { - log_info(LD_OR, - "HTTPS connect to '%s' successful! (200 %s) Starting TLS.", - conn->address, escaped(reason)); - tor_free(reason); - if (connection_tls_start_handshake(or_conn, 0) < 0) { - /* TLS handshaking error of some kind. */ - connection_mark_for_close(conn); - - return -1; - } - return 0; - } - /* else, bad news on the status code */ - log_warn(LD_OR, - "The https proxy sent back an unexpected status code %d (%s). " - "Closing.", - status_code, escaped(reason)); - tor_free(reason); - connection_mark_for_close(conn); - return -1; -} - /** Handle any new bytes that have come in on connection conn. * If conn is in 'open' state, hand it to * connection_or_process_cells_from_inbuf() @@ -255,11 +195,24 @@ connection_or_read_proxy_response(or_connection_t *or_conn) int connection_or_process_inbuf(or_connection_t *conn) { + int ret; tor_assert(conn); switch (conn->_base.state) { - case OR_CONN_STATE_PROXY_READING: - return connection_or_read_proxy_response(conn); + case OR_CONN_STATE_PROXY_HANDSHAKING: + ret = connection_read_proxy_handshake(TO_CONN(conn)); + + /* start TLS after handshake completion, or deal with error */ + if (ret == 1) { + tor_assert(TO_CONN(conn)->proxy_state == PROXY_CONNECTED); + if (connection_tls_start_handshake(conn, 0) < 0) + ret = -1; + } + if (ret < 0) { + connection_mark_for_close(TO_CONN(conn)); + } + + return ret; case OR_CONN_STATE_OPEN: case OR_CONN_STATE_OR_HANDSHAKING: return connection_or_process_cells_from_inbuf(conn); @@ -312,11 +265,7 @@ connection_or_finished_flushing(or_connection_t *conn) assert_connection_ok(TO_CONN(conn),0); switch (conn->_base.state) { - case OR_CONN_STATE_PROXY_FLUSHING: - log_debug(LD_OR,"finished sending CONNECT to proxy."); - conn->_base.state = OR_CONN_STATE_PROXY_READING; - connection_stop_writing(TO_CONN(conn)); - break; + case OR_CONN_STATE_PROXY_HANDSHAKING: case OR_CONN_STATE_OPEN: case OR_CONN_STATE_OR_HANDSHAKING: connection_stop_writing(TO_CONN(conn)); @@ -334,6 +283,7 @@ connection_or_finished_flushing(or_connection_t *conn) int connection_or_finished_connecting(or_connection_t *or_conn) { + int proxy_type; connection_t *conn; tor_assert(or_conn); conn = TO_CONN(or_conn); @@ -343,28 +293,24 @@ connection_or_finished_connecting(or_connection_t *or_conn) conn->address,conn->port); control_event_bootstrap(BOOTSTRAP_STATUS_HANDSHAKE, 0); - if (get_options()->HttpsProxy) { - char buf[1024]; - char *base64_authenticator=NULL; - const char *authenticator = get_options()->HttpsProxyAuthenticator; + proxy_type = PROXY_NONE; - if (authenticator) { - base64_authenticator = alloc_http_authenticator(authenticator); - if (!base64_authenticator) - log_warn(LD_OR, "Encoding https authenticator failed"); + if (get_options()->HttpsProxy) + proxy_type = PROXY_CONNECT; + else if (get_options()->Socks4Proxy) + proxy_type = PROXY_SOCKS4; + else if (get_options()->Socks5Proxy) + proxy_type = PROXY_SOCKS5; + + if (proxy_type != PROXY_NONE) { + /* start proxy handshake */ + if (connection_proxy_connect(conn, proxy_type) < 0) { + connection_mark_for_close(conn); + return -1; } - if (base64_authenticator) { - tor_snprintf(buf, sizeof(buf), "CONNECT %s:%d HTTP/1.1\r\n" - "Proxy-Authorization: Basic %s\r\n\r\n", - fmt_addr(&conn->addr), - conn->port, base64_authenticator); - tor_free(base64_authenticator); - } else { - tor_snprintf(buf, sizeof(buf), "CONNECT %s:%d HTTP/1.0\r\n\r\n", - fmt_addr(&conn->addr), conn->port); - } - connection_write_to_buf(buf, strlen(buf), conn); - conn->state = OR_CONN_STATE_PROXY_FLUSHING; + + connection_start_reading(conn); + conn->state = OR_CONN_STATE_PROXY_HANDSHAKING; return 0; } @@ -753,6 +699,7 @@ connection_or_connect(const tor_addr_t *_addr, uint16_t port, or_connection_t *conn; or_options_t *options = get_options(); int socket_error = 0; + int using_proxy = 0; tor_addr_t addr; tor_assert(_addr); @@ -771,19 +718,27 @@ connection_or_connect(const tor_addr_t *_addr, uint16_t port, conn->_base.state = OR_CONN_STATE_CONNECTING; control_event_or_conn_status(conn, OR_CONN_EVENT_LAUNCHED, 0); + /* use a proxy server if available */ if (options->HttpsProxy) { - /* we shouldn't connect directly. use the https proxy instead. */ + using_proxy = 1; tor_addr_from_ipv4h(&addr, options->HttpsProxyAddr); port = options->HttpsProxyPort; + } else if (options->Socks4Proxy) { + using_proxy = 1; + tor_addr_from_ipv4h(&addr, options->Socks4ProxyAddr); + port = options->Socks4ProxyPort; + } else if (options->Socks5Proxy) { + using_proxy = 1; + tor_addr_from_ipv4h(&addr, options->Socks5ProxyAddr); + port = options->Socks5ProxyPort; } switch (connection_connect(TO_CONN(conn), conn->_base.address, &addr, port, &socket_error)) { case -1: /* If the connection failed immediately, and we're using - * an https proxy, our https proxy is down. Don't blame the - * Tor server. */ - if (!options->HttpsProxy) + * a proxy, our proxy is down. Don't blame the Tor server. */ + if (!using_proxy) entry_guard_register_connect_status(conn->identity_digest, 0, 1, time(NULL)); connection_or_connect_failed(conn, diff --git a/src/or/directory.c b/src/or/directory.c index 1b972a345b..d6457e0555 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -747,6 +747,15 @@ directory_initiate_command_rend(const char *address, const tor_addr_t *_addr, log_debug(LD_DIR, "Initiating %s", dir_conn_purpose_to_string(dir_purpose)); + /* ensure that we don't make direct connections when a SOCKS server is + * configured. */ + if (!anonymized_connection && !use_begindir && !options->HttpProxy && + (options->Socks4Proxy || options->Socks5Proxy)) { + log_warn(LD_DIR, "Cannot connect to a directory server through a " + "SOCKS proxy!"); + return; + } + conn = dir_connection_new(AF_INET); /* set up conn so it's got all the data we need to remember */ diff --git a/src/or/or.h b/src/or/or.h index b6ee72d9b5..930599267f 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -220,6 +220,21 @@ typedef enum { /* !!!! If _CONN_TYPE_MAX is ever over 15, we must grow the type field in * connection_t. */ +/* Proxy client types */ +#define PROXY_NONE 0 +#define PROXY_CONNECT 1 +#define PROXY_SOCKS4 2 +#define PROXY_SOCKS5 3 + +/* Proxy client handshake states */ +#define PROXY_HTTPS_WANT_CONNECT_OK 1 +#define PROXY_SOCKS4_WANT_CONNECT_OK 2 +#define PROXY_SOCKS5_WANT_AUTH_METHOD_NONE 3 +#define PROXY_SOCKS5_WANT_AUTH_METHOD_RFC1929 4 +#define PROXY_SOCKS5_WANT_AUTH_RFC1929_OK 5 +#define PROXY_SOCKS5_WANT_CONNECT_OK 6 +#define PROXY_CONNECTED 7 + /** True iff x is an edge connection. */ #define CONN_IS_EDGE(x) \ ((x)->type == CONN_TYPE_EXIT || (x)->type == CONN_TYPE_AP) @@ -240,26 +255,24 @@ typedef enum { #define _OR_CONN_STATE_MIN 1 /** State for a connection to an OR: waiting for connect() to finish. */ #define OR_CONN_STATE_CONNECTING 1 -/** State for a connection to an OR: waiting for proxy command to flush. */ -#define OR_CONN_STATE_PROXY_FLUSHING 2 -/** State for a connection to an OR: waiting for proxy response. */ -#define OR_CONN_STATE_PROXY_READING 3 +/** State for a connection to an OR: waiting for proxy handshake to complete */ +#define OR_CONN_STATE_PROXY_HANDSHAKING 2 /** State for a connection to an OR or client: SSL is handshaking, not done * yet. */ -#define OR_CONN_STATE_TLS_HANDSHAKING 4 +#define OR_CONN_STATE_TLS_HANDSHAKING 3 /** State for a connection to an OR: We're doing a second SSL handshake for * renegotiation purposes. */ -#define OR_CONN_STATE_TLS_CLIENT_RENEGOTIATING 5 +#define OR_CONN_STATE_TLS_CLIENT_RENEGOTIATING 4 /** State for a connection at an OR: We're waiting for the client to * renegotiate. */ -#define OR_CONN_STATE_TLS_SERVER_RENEGOTIATING 6 +#define OR_CONN_STATE_TLS_SERVER_RENEGOTIATING 5 /** State for a connection to an OR: We're done with our SSL handshake, but we * haven't yet negotiated link protocol versions and sent a netinfo cell. */ -#define OR_CONN_STATE_OR_HANDSHAKING 7 +#define OR_CONN_STATE_OR_HANDSHAKING 6 /** State for a connection to an OR: Ready to send/receive cells. */ -#define OR_CONN_STATE_OPEN 8 -#define _OR_CONN_STATE_MAX 8 +#define OR_CONN_STATE_OPEN 7 +#define _OR_CONN_STATE_MAX 7 #define _EXIT_CONN_STATE_MIN 1 /** State for an exit connection: waiting for response from DNS farm. */ @@ -964,6 +977,9 @@ typedef struct connection_t { * to the evdns_server_port is uses to listen to and answer connections. */ struct evdns_server_port *dns_server_port; + /** CONNECT/SOCKS proxy client handshake state (for outgoing connections). */ + unsigned int proxy_state:4; + } connection_t; /** Stores flags and information related to the portion of a v2 Tor OR @@ -2342,6 +2358,16 @@ typedef struct { uint16_t HttpsProxyPort; /**< Parsed port for https proxy, if any. */ char *HttpsProxyAuthenticator; /**< username:password string, if any. */ + char *Socks4Proxy; + uint32_t Socks4ProxyAddr; + uint16_t Socks4ProxyPort; + + char *Socks5Proxy; + uint32_t Socks5ProxyAddr; + uint16_t Socks5ProxyPort; + char *Socks5ProxyUsername; + char *Socks5ProxyPassword; + /** List of configuration lines for replacement directory authorities. * If you just want to replace one class of authority at a time, * use the "Alternate*Authority" options below instead. */ @@ -2679,6 +2705,7 @@ int fetch_from_buf_http(buf_t *buf, int force_complete); int fetch_from_buf_socks(buf_t *buf, socks_request_t *req, int log_sockstype, int safe_socks); +int fetch_from_buf_socks_client(buf_t *buf, int state, char **reason); int fetch_from_buf_line(buf_t *buf, char *data_out, size_t *data_len); int peek_buf_has_control0_command(buf_t *buf); @@ -2939,6 +2966,10 @@ void connection_expire_held_open(void); int connection_connect(connection_t *conn, const char *address, const tor_addr_t *addr, uint16_t port, int *socket_error); + +int connection_proxy_connect(connection_t *conn, int type); +int connection_read_proxy_handshake(connection_t *conn); + int retry_all_listeners(smartlist_t *replaced_conns, smartlist_t *new_conns); From 015189b5df17d3572d27e850336e9d1c9dc83c6d Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 19 Jun 2009 12:40:23 -0400 Subject: [PATCH 085/187] Move SOCKS reason-decoding switches into reasons.c --- src/or/buffers.c | 49 ++---------------------------------------------- src/or/or.h | 2 ++ src/or/reasons.c | 43 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 47 insertions(+), 47 deletions(-) diff --git a/src/or/buffers.c b/src/or/buffers.c index 31725c5f27..ada8bddcb9 100644 --- a/src/or/buffers.c +++ b/src/or/buffers.c @@ -1641,22 +1641,7 @@ fetch_from_buf_socks_client(buf_t *buf, int state, char **reason) return 0; if (data[1] != 0x5a) { - switch (data[1]) { - case 0x5b: - *reason = tor_strdup("server rejected connection"); - break; - case 0x5c: - *reason = tor_strdup("server cannot connect to identd " - "on this client"); - break; - case 0x5d: - *reason = tor_strdup("user id does not match identd"); - break; - default: - *reason = tor_strdup("invalid SOCKS 4 response code"); - break; - } - + *reason = tor_strdup(socks4_response_code_to_string(data[1])); return -1; } @@ -1738,37 +1723,7 @@ fetch_from_buf_socks_client(buf_t *buf, int state, char **reason) return 0; if (data[1] != 0x00) { - - switch (data[1]) { - case 0x01: - *reason = tor_strdup("general SOCKS server failure"); - break; - case 0x02: - *reason = tor_strdup("connection not allowed by ruleset"); - break; - case 0x03: - *reason = tor_strdup("Network unreachable"); - break; - case 0x04: - *reason = tor_strdup("Host unreachable"); - break; - case 0x05: - *reason = tor_strdup("Connection refused"); - break; - case 0x06: - *reason = tor_strdup("TTL expired"); - break; - case 0x07: - *reason = tor_strdup("Command not supported"); - break; - case 0x08: - *reason = tor_strdup("Address type not supported"); - break; - default: - *reason = tor_strdup("unknown reason"); - break; - } - + *reason = tor_strdup(socks5_response_code_to_string(data[1])); return -1; } diff --git a/src/or/or.h b/src/or/or.h index 930599267f..f901121d53 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -3930,6 +3930,8 @@ int tls_error_to_orconn_end_reason(int e); int errno_to_orconn_end_reason(int e); const char *circuit_end_reason_to_control_string(int reason); +const char *socks4_response_code_to_string(uint8_t code); +const char *socks5_response_code_to_string(uint8_t code); /********************************* relay.c ***************************/ diff --git a/src/or/reasons.c b/src/or/reasons.c index a252f83198..5efb08bb41 100644 --- a/src/or/reasons.c +++ b/src/or/reasons.c @@ -326,3 +326,46 @@ circuit_end_reason_to_control_string(int reason) } } +const char * +socks4_response_code_to_string(uint8_t code) +{ + switch (code) { + case 0x5a: + return "connection accepted"; + case 0x5b: + return "server rejected connection"; + case 0x5c: + return "server cannot connect to identd on this client"; + case 0x5d: + return "user id does not match identd"; + default: + return "invalid SOCKS 4 response code"; + } +} + +const char * +socks5_response_code_to_string(uint8_t code) +{ + switch (code) { + case 0x00: + return "connection accepted"; + case 0x01: + return "general SOCKS server failure"; + case 0x02: + return "connection not allowed by ruleset"; + case 0x03: + return "Network unreachable"; + case 0x04: + return "Host unreachable"; + case 0x05: + return "Connection refused"; + case 0x06: + return "TTL expired"; + case 0x07: + return "Command not supported"; + case 0x08: + return "Address type not supported"; + default: + return "unknown reason"; + } +} From 659fc13da5f8cf7c3d7d3be238df709e8de06945 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 19 Jun 2009 12:48:00 -0400 Subject: [PATCH 086/187] Change proxy-address type to tor_addr_t to allow ipv6 proxies. --- src/or/config.c | 12 ++++++------ src/or/connection_or.c | 6 +++--- src/or/directory.c | 2 +- src/or/or.h | 8 ++++---- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/or/config.c b/src/or/config.c index e1830fa7c2..f7696008a6 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -3386,7 +3386,7 @@ options_validate(or_options_t *old_options, or_options_t *options, REJECT("Failed to parse accounting options. See logs for details."); if (options->HttpProxy) { /* parse it now */ - if (parse_addr_port(LOG_WARN, options->HttpProxy, NULL, + if (tor_addr_port_parse(options->HttpProxy, &options->HttpProxyAddr, &options->HttpProxyPort) < 0) REJECT("HttpProxy failed to parse or resolve. Please fix."); if (options->HttpProxyPort == 0) { /* give it a default */ @@ -3400,7 +3400,7 @@ options_validate(or_options_t *old_options, or_options_t *options, } if (options->HttpsProxy) { /* parse it now */ - if (parse_addr_port(LOG_WARN, options->HttpsProxy, NULL, + if (tor_addr_port_parse(options->HttpsProxy, &options->HttpsProxyAddr, &options->HttpsProxyPort) <0) REJECT("HttpsProxy failed to parse or resolve. Please fix."); if (options->HttpsProxyPort == 0) { /* give it a default */ @@ -3414,7 +3414,7 @@ options_validate(or_options_t *old_options, or_options_t *options, } if (options->Socks4Proxy) { /* parse it now */ - if (parse_addr_port(LOG_WARN, options->Socks4Proxy, NULL, + if (tor_addr_port_parse(options->Socks4Proxy, &options->Socks4ProxyAddr, &options->Socks4ProxyPort) <0) REJECT("Socks4Proxy failed to parse or resolve. Please fix."); @@ -3424,9 +3424,9 @@ options_validate(or_options_t *old_options, or_options_t *options, } if (options->Socks5Proxy) { /* parse it now */ - if (parse_addr_port(LOG_WARN, options->Socks5Proxy, NULL, - &options->Socks5ProxyAddr, - &options->Socks5ProxyPort) <0) + if (tor_addr_port_parse(options->Socks5Proxy, + &options->Socks5ProxyAddr, + &options->Socks5ProxyPort) <0) REJECT("Socks5Proxy failed to parse or resolve. Please fix."); if (options->Socks5ProxyPort == 0) { /* give it a default */ options->Socks5ProxyPort = 1080; diff --git a/src/or/connection_or.c b/src/or/connection_or.c index bbfa611c0d..8c8b5496a7 100644 --- a/src/or/connection_or.c +++ b/src/or/connection_or.c @@ -721,15 +721,15 @@ connection_or_connect(const tor_addr_t *_addr, uint16_t port, /* use a proxy server if available */ if (options->HttpsProxy) { using_proxy = 1; - tor_addr_from_ipv4h(&addr, options->HttpsProxyAddr); + tor_addr_copy(&addr, &options->HttpsProxyAddr); port = options->HttpsProxyPort; } else if (options->Socks4Proxy) { using_proxy = 1; - tor_addr_from_ipv4h(&addr, options->Socks4ProxyAddr); + tor_addr_copy(&addr, &options->Socks4ProxyAddr); port = options->Socks4ProxyPort; } else if (options->Socks5Proxy) { using_proxy = 1; - tor_addr_from_ipv4h(&addr, options->Socks5ProxyAddr); + tor_addr_copy(&addr, &options->Socks5ProxyAddr); port = options->Socks5ProxyPort; } diff --git a/src/or/directory.c b/src/or/directory.c index d6457e0555..b015b9e30a 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -781,7 +781,7 @@ directory_initiate_command_rend(const char *address, const tor_addr_t *_addr, /* then we want to connect to dirport directly */ if (options->HttpProxy) { - tor_addr_from_ipv4h(&addr, options->HttpProxyAddr); + tor_addr_copy(&addr, &options->HttpProxyAddr); dir_port = options->HttpProxyPort; } diff --git a/src/or/or.h b/src/or/or.h index f901121d53..bb6cee1ba5 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -2349,21 +2349,21 @@ typedef struct { char *ContactInfo; /**< Contact info to be published in the directory. */ char *HttpProxy; /**< hostname[:port] to use as http proxy, if any. */ - uint32_t HttpProxyAddr; /**< Parsed IPv4 addr for http proxy, if any. */ + tor_addr_t HttpProxyAddr; /**< Parsed IPv4 addr for http proxy, if any. */ uint16_t HttpProxyPort; /**< Parsed port for http proxy, if any. */ char *HttpProxyAuthenticator; /**< username:password string, if any. */ char *HttpsProxy; /**< hostname[:port] to use as https proxy, if any. */ - uint32_t HttpsProxyAddr; /**< Parsed IPv4 addr for https proxy, if any. */ + tor_addr_t HttpsProxyAddr; /**< Parsed addr for https proxy, if any. */ uint16_t HttpsProxyPort; /**< Parsed port for https proxy, if any. */ char *HttpsProxyAuthenticator; /**< username:password string, if any. */ char *Socks4Proxy; - uint32_t Socks4ProxyAddr; + tor_addr_t Socks4ProxyAddr; uint16_t Socks4ProxyPort; char *Socks5Proxy; - uint32_t Socks5ProxyAddr; + tor_addr_t Socks5ProxyAddr; uint16_t Socks5ProxyPort; char *Socks5ProxyUsername; char *Socks5ProxyPassword; From 955a763c742574d08ece34a88982b2f2e848aace Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 22 Jun 2009 14:23:39 -0400 Subject: [PATCH 087/187] path-spec: Clarify what we mean by "a server's bandwidth." This just got a little complicated, since old clients use "clipped advertised bandwith" and new clients now use "consensus bandwidth" but fall back to "clipped advertised bandwidth". --- doc/spec/path-spec.txt | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/doc/spec/path-spec.txt b/doc/spec/path-spec.txt index b53e4bda66..78f3b63bcb 100644 --- a/doc/spec/path-spec.txt +++ b/doc/spec/path-spec.txt @@ -71,6 +71,24 @@ of their choices. is unknown (usually its target IP), but we believe the path probably supports the request according to the rules given below. +1.1. A server's bandwidth + + Old versions of Tor did not report bandwidths in network status + documents, so clients had to learn them from the routers' advertised + server descriptors. + + For versions of Tor prior to 0.2.1.17-rc, everywhere below where we + refer to a server's "bandwidth", we mean its clipped advertised + bandwidth, computed by taking the smaller of the 'rate' and + 'observed' arguments to the "bandwidth" element in the server's + descriptor. If a router's advertised bandwidth is greater than + MAX_BELIEVABLE_BANDWIDTH (currently 10 MB/s), we clipped to that + value. + + For more recent versions of Tor, we take the bandwidth value declared + in the consensus, and fall back to the clipped advertised bandwidth + only if the consensus does not have bandwidths listed. + 2. Building circuits 2.1. When we build @@ -178,16 +196,13 @@ of their choices. multiple candidates for a path element, we choose randomly. For "fast" circuits, we pick a given router as an exit with probability - proportional to its advertised bandwidth [the smaller of the 'rate' and - 'observed' arguments to the "bandwidth" element in its descriptor]. If a - router's advertised bandwidth is greater than MAX_BELIEVABLE_BANDWIDTH - (currently 10 MB/s), we clip to that value. + proportional to its bandwidth. For non-exit positions on "fast" circuits, we pick routers as above, but - we weight the clipped advertised bandwidth of Exit-flagged nodes depending + we weight the bandwidth of Exit-flagged nodes depending on the fraction of bandwidth available from non-Exit nodes. Call the - total clipped advertised bandwidth for Exit nodes under consideration E, - and the total clipped advertised bandwidth for all nodes under + total bandwidth for Exit nodes under consideration E, + and the total bandwidth for all nodes under consideration T. If E Date: Sun, 21 Jun 2009 06:31:34 +0200 Subject: [PATCH 088/187] Log a notice about collecting geoip stats --- src/or/config.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/or/config.c b/src/or/config.c index bcabb67a99..6ad1c3a9ba 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -1380,6 +1380,9 @@ options_act(or_options_t *old_options) "GeoIP database found!"); return -1; } + log_notice(LD_CONFIG, "Configured to measure usage by country and " + "write aggregate statistics to disk. Check the geoip-stats file " + "in your data directory once I've been running for 24 hours."); #endif /* Check if we need to parse and add the EntryNodes config option. */ if (options->EntryNodes && From 4d6af73db88e409764f43fc6cdaa432d667becf3 Mon Sep 17 00:00:00 2001 From: Karsten Loesing Date: Wed, 24 Jun 2009 19:51:45 +0200 Subject: [PATCH 089/187] If configured, write per-port exit statistics to disk periodically. [Original patch series from Karsten, revised and squashed by Nick] --- ChangeLog | 4 + configure.in | 7 ++ src/or/config.c | 11 +++ src/or/connection.c | 10 ++- src/or/connection_edge.c | 2 + src/or/or.h | 14 ++++ src/or/rephist.c | 170 +++++++++++++++++++++++++++++++++++++++ 7 files changed, 216 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index b06d0993c5..0d5af889da 100644 --- a/ChangeLog +++ b/ChangeLog @@ -17,6 +17,10 @@ Changes in version 0.2.2.1-alpha - 2009-??-?? help debug some conceivable causes of bug 930. - Directories that are configured with the --enable-geoip-stats flag now write their GeoIP stats to disk exactly every 24 hours. + - Exit nodes can write statistics on the number of exit streams and + transferred bytes per port to disk every 24 hours. To enable this, + run configure with the --enable-exit-stats option, and set + "ExitPortStatistics 1" in your torrc. o Minor bugfixes - Hidden service clients didn't use a cached service descriptor that diff --git a/configure.in b/configure.in index f1ac83687c..844211588d 100644 --- a/configure.in +++ b/configure.in @@ -85,6 +85,13 @@ case $host in ;; esac +AC_ARG_ENABLE(exit-stats, + AS_HELP_STRING(--enable-exit-stats, enable code for exits to collect per-port statistics)) + +if test "$enable_exit_stats" = "yes"; then + AC_DEFINE(ENABLE_EXIT_STATS, 1, [Defined if we try to collect per-port statistics on exits]) +fi + AC_ARG_ENABLE(geoip-stats, AS_HELP_STRING(--enable-geoip-stats, enable code for directories to collect per-country statistics)) diff --git a/src/or/config.c b/src/or/config.c index 6ad1c3a9ba..f1fea13e58 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -205,6 +205,7 @@ static config_var_t _option_vars[] = { V(ExitNodes, ROUTERSET, NULL), V(ExitPolicy, LINELIST, NULL), V(ExitPolicyRejectPrivate, BOOL, "1"), + V(ExitPortStatistics, BOOL, "0"), V(FallbackNetworkstatusFile, FILENAME, SHARE_DATADIR PATH_SEPARATOR "tor" PATH_SEPARATOR "fallback-consensus"), V(FascistFirewall, BOOL, "0"), @@ -1383,6 +1384,16 @@ options_act(or_options_t *old_options) log_notice(LD_CONFIG, "Configured to measure usage by country and " "write aggregate statistics to disk. Check the geoip-stats file " "in your data directory once I've been running for 24 hours."); +#endif +#ifdef ENABLE_EXIT_STATS + if (options->ExitPortStatistics) + log_notice(LD_CONFIG, "Configured to measure exit port statistics. " + "Look for the exit-stats file that will first be written to " + "the data directory in 24 hours from now."); +#else + if (options->ExitPortStatistics) + log_warn(LD_CONFIG, "ExitPortStatistics enabled, but Tor was built " + "without port statistics support."); #endif /* Check if we need to parse and add the EntryNodes config option. */ if (options->EntryNodes && diff --git a/src/or/connection.c b/src/or/connection.c index c8406fee3f..0897eb4a5e 100644 --- a/src/or/connection.c +++ b/src/or/connection.c @@ -1702,10 +1702,16 @@ connection_buckets_decrement(connection_t *conn, time_t now, tor_fragile_assert(); } - if (num_read > 0) + if (num_read > 0) { + if (conn->type == CONN_TYPE_EXIT) + rep_hist_note_exit_bytes_read(conn->port, num_read, now); rep_hist_note_bytes_read(num_read, now); - if (num_written > 0) + } + if (num_written > 0) { + if (conn->type == CONN_TYPE_EXIT) + rep_hist_note_exit_bytes_written(conn->port, num_written, now); rep_hist_note_bytes_written(num_written, now); + } if (connection_counts_as_relayed_traffic(conn, now)) { global_relayed_read_bucket -= (int)num_read; diff --git a/src/or/connection_edge.c b/src/or/connection_edge.c index 3cd00e2828..dd44190c71 100644 --- a/src/or/connection_edge.c +++ b/src/or/connection_edge.c @@ -333,6 +333,8 @@ connection_edge_finished_connecting(edge_connection_t *edge_conn) escaped_safe_str(conn->address),conn->port, safe_str(fmt_addr(&conn->addr))); + rep_hist_note_exit_stream_opened(conn->port, approx_time()); + conn->state = EXIT_CONN_STATE_OPEN; connection_watch_events(conn, READ_EVENT); /* stop writing, keep reading */ if (connection_wants_to_flush(conn)) /* in case there are any queued relay diff --git a/src/or/or.h b/src/or/or.h index 091264a4ef..935ea523df 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -2475,6 +2475,9 @@ typedef struct { * exit allows it, we use it. */ int AllowSingleHopCircuits; + /** If true, the user wants us to collect statistics on port usage. */ + int ExitPortStatistics; + /** If true, do not believe anybody who tells us that a domain resolves * to an internal address, or that an internal address has a PTR mapping. * Helps avoid some cross-site attacks. */ @@ -3961,6 +3964,17 @@ void rep_hist_note_extend_failed(const char *from_name, const char *to_name); void rep_hist_dump_stats(time_t now, int severity); void rep_hist_note_bytes_read(size_t num_bytes, time_t when); void rep_hist_note_bytes_written(size_t num_bytes, time_t when); +#ifdef ENABLE_EXIT_STATS +void rep_hist_note_exit_bytes_read(uint16_t port, size_t num_bytes, + time_t now); +void rep_hist_note_exit_bytes_written(uint16_t port, size_t num_bytes, + time_t now); +void rep_hist_note_exit_stream_opened(uint16_t port, time_t now); +#else +#define rep_hist_note_exit_bytes_read(p,n,t) STMT_NIL +#define rep_hist_note_exit_bytes_written(p,n,t) STMT_NIL +#define rep_hist_note_exit_stream_opened(p,t) STMT_NIL +#endif int rep_hist_bandwidth_assess(void); char *rep_hist_get_bandwidth_lines(int for_extrainfo); void rep_hist_update_state(or_state_t *state); diff --git a/src/or/rephist.c b/src/or/rephist.c index 13fdb58b5e..f1f502cad2 100644 --- a/src/or/rephist.c +++ b/src/or/rephist.c @@ -1320,6 +1320,176 @@ rep_hist_note_bytes_read(size_t num_bytes, time_t when) add_obs(read_array, when, num_bytes); } +#ifdef ENABLE_EXIT_STATS +/* Some constants */ +/** How long are the intervals for measuring exit stats? */ +#define EXIT_STATS_INTERVAL_SEC (24 * 60 * 60) +/** To what multiple should byte numbers be rounded up? */ +#define EXIT_STATS_ROUND_UP_BYTES 1024 +/** To what multiple should stream counts be rounded up? */ +#define EXIT_STATS_ROUND_UP_STREAMS 4 +/** Number of TCP ports */ +#define EXIT_STATS_NUM_PORTS 65536 + +/* The following data structures are arrays and no fancy smartlists or maps, + * so that all write operations can be done in constant time. This comes at + * the price of some memory (1.25 MB) and linear complexity when writing + * stats. */ +/** Number of bytes read in current period by exit port */ +static uint64_t exit_bytes_read[EXIT_STATS_NUM_PORTS]; +/** Number of bytes written in current period by exit port */ +static uint64_t exit_bytes_written[EXIT_STATS_NUM_PORTS]; +/** Number of streams opened in current period by exit port */ +static uint32_t exit_streams[EXIT_STATS_NUM_PORTS]; + +/** When does the current exit stats period end? */ +static time_t end_of_current_exit_stats_period = 0; + +/** Write exit stats for the current period to disk and reset counters. */ +static void +write_exit_stats(time_t when) +{ + char t[ISO_TIME_LEN+1]; + int r, i, comma; + uint64_t *b; + + char *filename = get_datadir_fname("exit-stats"); + open_file_t *open_file = NULL; + FILE *out = NULL; + + log_debug(LD_HIST, "Considering writing exit port statistics to disk.."); + while (when > end_of_current_exit_stats_period) { + format_iso_time(t, end_of_current_exit_stats_period); + log_info(LD_HIST, "Writing exit port statistics to disk for period " + "ending at %s.", t); + + if (!open_file) { + out = start_writing_to_stdio_file(filename, OPEN_FLAGS_APPEND, + 0600, &open_file); + if (!out) { + log_warn(LD_HIST, "Couldn't open '%s'.", filename); + goto done; + } + } + + /* written yyyy-mm-dd HH:MM:SS (n s) */ + if (fprintf(out, "written %s (%d s)\n", t, EXIT_STATS_INTERVAL_SEC) < 0) + goto done; + + /* kibibytes-(read|written) port=kibibytes,.. */ + for (r = 0; r < 2; r++) { + b = r ? exit_bytes_read : exit_bytes_written; + tor_assert(b); + if (fprintf(out, "%s ", + r == 0 ? "kibibytes-read" : "kibibytes-written")<0) + goto done; + + comma = 0; + for (i = 1; i < EXIT_STATS_NUM_PORTS; i++) { + if (b[i] > 0) { + uint64_t num = b[i]; + num += EXIT_STATS_ROUND_UP_BYTES - 1; + num /= EXIT_STATS_ROUND_UP_BYTES; + num *= EXIT_STATS_ROUND_UP_BYTES; + num /= 1024; + if (fprintf(out, "%s%d="U64_FORMAT, + comma++ ? "," : "", i, + U64_PRINTF_ARG(num)) < 0) + goto done; + } + } + if (fprintf(out, "\n")<0) + goto done; + /* Reset counters */ + memset(b, 0, EXIT_STATS_NUM_PORTS*sizeof(uint64_t)); + } + /* streams-opened port=num,.. */ + if (fprintf(out, "streams-opened ")<0) + goto done; + comma = 0; + for (i = 1; i < EXIT_STATS_NUM_PORTS; i++) { + if (exit_streams[i] > 0) { + uint32_t num = exit_streams[i]; + num += EXIT_STATS_ROUND_UP_STREAMS - 1; + num /= EXIT_STATS_ROUND_UP_STREAMS; + num *= EXIT_STATS_ROUND_UP_STREAMS; + if (fprintf(out, "%s%d=%d", + comma++ ? "," : "", i, num)<0) + goto done; + } + } + if (fprintf(out, "\n")<0) + goto done; + /* Reset counters */ + memset(exit_streams, 0, sizeof(exit_streams)); + end_of_current_exit_stats_period += EXIT_STATS_INTERVAL_SEC; + } + + if (open_file) + finish_writing_to_file(open_file); + open_file = NULL; + done: + if (open_file) + abort_writing_to_file(open_file); + tor_free(filename); +} + +/** Prepare to add an exit stats observation at second when by + * checking whether this observation lies in the current observation + * period; if not, shift the current period forward by one until the + * reported event fits it and write all results in between to disk. */ +static void +add_exit_obs(time_t when) +{ + if (when > end_of_current_exit_stats_period) { + if (end_of_current_exit_stats_period) + write_exit_stats(when); + else + end_of_current_exit_stats_period = when + EXIT_STATS_INTERVAL_SEC; + } +} + +/** Note that we wrote num_bytes to an exit connection to + * port in second when. */ +void +rep_hist_note_exit_bytes_written(uint16_t port, size_t num_bytes, + time_t when) +{ + if (!get_options()->ExitPortStatistics) + return; + add_exit_obs(when); + exit_bytes_written[port] += num_bytes; + log_debug(LD_HIST, "Written %lu bytes to exit connection to port %d.", + (unsigned long)num_bytes, port); +} + +/** Note that we read num_bytes from an exit connection to + * port in second when. */ +void +rep_hist_note_exit_bytes_read(uint16_t port, size_t num_bytes, + time_t when) +{ + if (!get_options()->ExitPortStatistics) + return; + add_exit_obs(when); + exit_bytes_read[port] += num_bytes; + log_debug(LD_HIST, "Read %lu bytes from exit connection to port %d.", + (unsigned long)num_bytes, port); +} + +/** Note that we opened an exit stream to port in second + * when. */ +void +rep_hist_note_exit_stream_opened(uint16_t port, time_t when) +{ + if (!get_options()->ExitPortStatistics) + return; + add_exit_obs(when); + exit_streams[port]++; + log_debug(LD_HIST, "Opened exit stream to port %d", port); +} +#endif + /** Helper: Return the largest value in b->maxima. (This is equal to the * most bandwidth used in any NUM_SECS_ROLLING_MEASURE period for the last * NUM_SECS_BW_SUM_IS_VALID seconds.) From c1f6c2ba30a0ed7d54b03f411fc8b3cb0d660ed1 Mon Sep 17 00:00:00 2001 From: Karsten Loesing Date: Sat, 4 Jul 2009 22:17:08 +0200 Subject: [PATCH 090/187] Estimate v2 and v3 shares as mean values over measurement interval. --- ChangeLog | 2 ++ src/or/geoip.c | 67 +++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 68 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 0d5af889da..c4cc07b311 100644 --- a/ChangeLog +++ b/ChangeLog @@ -17,6 +17,8 @@ Changes in version 0.2.2.1-alpha - 2009-??-?? help debug some conceivable causes of bug 930. - Directories that are configured with the --enable-geoip-stats flag now write their GeoIP stats to disk exactly every 24 hours. + Estimated shares of v2 and v3 requests are determined as averages, + not at the end of a measurement period. - Exit nodes can write statistics on the number of exit streams and transferred bytes per port to disk every 24 hours. To enable this, run configure with the --enable-exit-stats option, and set diff --git a/src/or/geoip.c b/src/or/geoip.c index 41c8f21cdb..e18230d8c3 100644 --- a/src/or/geoip.c +++ b/src/or/geoip.c @@ -298,6 +298,67 @@ HT_PROTOTYPE(clientmap, clientmap_entry_t, node, clientmap_entry_hash, HT_GENERATE(clientmap, clientmap_entry_t, node, clientmap_entry_hash, clientmap_entries_eq, 0.6, malloc, realloc, free); +/** How often do we update our estimate which share of v2 and v3 directory + * requests is sent to us? We could as well trigger updates of shares from + * network status updates, but that means adding a lot of calls into code + * that is independent from geoip stats (and keeping them up-to-date). We + * are perfectly fine with an approximation of 15-minute granularity. */ +#define REQUEST_SHARE_INTERVAL (15 * 60) + +/** When did we last determine which share of v2 and v3 directory requests + * is sent to us? */ +static time_t last_time_determined_shares = 0; + +/** Sum of products of v2 shares times the number of seconds for which we + * consider these shares as valid. */ +static double v2_share_times_seconds; + +/** Sum of products of v3 shares times the number of seconds for which we + * consider these shares as valid. */ +static double v3_share_times_seconds; + +/** Number of seconds we are determining v2 and v3 shares. */ +static int share_seconds; + +/** Try to determine which fraction of v2 and v3 directory requests aimed at + * caches will be sent to us at time now and store that value in + * order to take a mean value later on. */ +static void +geoip_determine_shares(time_t now) +{ + double v2_share = 0.0, v3_share = 0.0; + if (router_get_my_share_of_directory_requests(&v2_share, &v3_share) < 0) + return; + if (last_time_determined_shares) { + v2_share_times_seconds += v2_share * + ((double) (now - last_time_determined_shares)); + v3_share_times_seconds += v3_share * + ((double) (now - last_time_determined_shares)); + share_seconds += now - last_time_determined_shares; + } + last_time_determined_shares = now; +} + +/** Calculate which fraction of v2 and v3 directory requests aimed at caches + * have been sent to us since the last call of this function up to time + * now. Set *v2_share_out and *v3_share_out to the + * fractions of v2 and v3 protocol shares we expect to have seen. Reset + * counters afterwards. Return 0 on success, -1 on failure (e.g. when zero + * seconds have passed since the last call).*/ +static int +geoip_get_mean_shares(time_t now, double *v2_share_out, + double *v3_share_out) +{ + geoip_determine_shares(now); + if (!share_seconds) + return -1; + *v2_share_out = v2_share_times_seconds / ((double) share_seconds); + *v3_share_out = v3_share_times_seconds / ((double) share_seconds); + v2_share_times_seconds = v3_share_times_seconds = 0.0; + share_seconds = 0; + return 0; +} + /** Note that we've seen a client connect from the IP addr (host order) * at time now. Ignored by all but bridges and directories if * configured accordingly. */ @@ -374,6 +435,10 @@ geoip_note_client_seen(geoip_client_action_t action, else ++country->n_v2_ns_requests[REQUEST_HIST_LEN-1]; } + + /* Periodically determine share of requests that we should see */ + if (last_time_determined_shares + REQUEST_SHARE_INTERVAL < now) + geoip_determine_shares(now); } if (!client_history_starts) { @@ -639,7 +704,7 @@ dump_geoip_stats(void) since, data_v3 ? data_v3 : "", data_v2 ? data_v2 : "") < 0) goto done; - if (!router_get_my_share_of_directory_requests(&v2_share, &v3_share)) { + if (!geoip_get_mean_shares(now, &v2_share, &v3_share)) { if (fprintf(out, "v2-ns-share %0.2lf%%\n", v2_share*100) < 0) goto done; if (fprintf(out, "v3-ns-share %0.2lf%%\n", v3_share*100) < 0) From b493a2ccb97e00f4fe3acb5c59c941c2babaeebb Mon Sep 17 00:00:00 2001 From: Karsten Loesing Date: Sun, 5 Jul 2009 19:53:25 +0200 Subject: [PATCH 091/187] If configured, write cell statistics to disk periodically. --- ChangeLog | 4 + configure.in | 7 ++ src/or/circuitlist.c | 5 ++ src/or/config.c | 11 +++ src/or/main.c | 11 +++ src/or/or.h | 26 +++++++ src/or/relay.c | 21 +++++- src/or/rephist.c | 171 +++++++++++++++++++++++++++++++++++++++++++ 8 files changed, 255 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 0d5af889da..b356dee57f 100644 --- a/ChangeLog +++ b/ChangeLog @@ -21,6 +21,10 @@ Changes in version 0.2.2.1-alpha - 2009-??-?? transferred bytes per port to disk every 24 hours. To enable this, run configure with the --enable-exit-stats option, and set "ExitPortStatistics 1" in your torrc. + - Relays write statistics on how long cells spend in their circuit + queues to disk every 24 hours. To enable this, run configure with + the --enable-buffer-stats option, and set "CellStatistics 1" in your + torrc. o Minor bugfixes - Hidden service clients didn't use a cached service descriptor that diff --git a/configure.in b/configure.in index 844211588d..10b05aeffb 100644 --- a/configure.in +++ b/configure.in @@ -99,6 +99,13 @@ if test "$enable_geoip_stats" = "yes"; then AC_DEFINE(ENABLE_GEOIP_STATS, 1, [Defined if we try to collect per-country statistics]) fi +AC_ARG_ENABLE(buffer-stats, + AS_HELP_STRING(--enable-buffer-stats, enable code for relays to collect buffer statistics)) + +if test "$enable_buffer_stats" = "yes"; then + AC_DEFINE(ENABLE_BUFFER_STATS, 1, [Defined if we try to collect buffer statistics]) +fi + AC_ARG_ENABLE(gcc-warnings, AS_HELP_STRING(--enable-gcc-warnings, enable verbose warnings)) diff --git a/src/or/circuitlist.c b/src/or/circuitlist.c index 252eaf9f8e..5a20e7ebde 100644 --- a/src/or/circuitlist.c +++ b/src/or/circuitlist.c @@ -447,6 +447,11 @@ circuit_free(circuit_t *circ) rend_data_free(ocirc->rend_data); } else { or_circuit_t *ocirc = TO_OR_CIRCUIT(circ); +#ifdef ENABLE_BUFFER_STATS + /* Remember cell statistics for this circuit before deallocating. */ + if (get_options()->CellStatistics) + add_circ_to_buffer_stats(circ, time(NULL)); +#endif mem = ocirc; memlen = sizeof(or_circuit_t); tor_assert(circ->magic == OR_CIRCUIT_MAGIC); diff --git a/src/or/config.c b/src/or/config.c index f1fea13e58..f7a5745262 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -162,6 +162,7 @@ static config_var_t _option_vars[] = { V(BridgePassword, STRING, NULL), V(BridgeRecordUsageByCountry, BOOL, "1"), V(BridgeRelay, BOOL, "0"), + V(CellStatistics, BOOL, "0"), V(CircuitBuildTimeout, INTERVAL, "1 minute"), V(CircuitIdleTimeout, INTERVAL, "1 hour"), V(ClientDNSRejectInternalAddresses, BOOL,"1"), @@ -1394,6 +1395,16 @@ options_act(or_options_t *old_options) if (options->ExitPortStatistics) log_warn(LD_CONFIG, "ExitPortStatistics enabled, but Tor was built " "without port statistics support."); +#endif +#ifdef ENABLE_BUFFER_STATS + if (options->CellStatistics) + log_notice(LD_CONFIG, "Configured to measure cell statistics. Look " + "for the buffer-stats file that will first be written to " + "the data directory in 24 hours from now."); +#else + if (options->CellStatistics) + log_warn(LD_CONFIG, "CellStatistics enabled, but Tor was built " + "without cell statistics support."); #endif /* Check if we need to parse and add the EntryNodes config option. */ if (options->EntryNodes && diff --git a/src/or/main.c b/src/or/main.c index 97957a5791..4969212798 100644 --- a/src/or/main.c +++ b/src/or/main.c @@ -830,6 +830,9 @@ run_scheduled_events(time_t now) static time_t time_to_clean_caches = 0; static time_t time_to_recheck_bandwidth = 0; static time_t time_to_check_for_expired_networkstatus = 0; +#ifdef ENABLE_BUFFER_STATS + static time_t time_to_dump_buffer_stats = 0; +#endif static time_t time_to_retry_dns_init = 0; or_options_t *options = get_options(); int i; @@ -957,6 +960,14 @@ run_scheduled_events(time_t now) time_to_check_for_expired_networkstatus = now + CHECK_EXPIRED_NS_INTERVAL; } +#ifdef ENABLE_BUFFER_STATS + if (time_to_dump_buffer_stats < now) { + if (get_options()->CellStatistics && time_to_dump_buffer_stats) + dump_buffer_stats(); + time_to_dump_buffer_stats = now + DUMP_BUFFER_STATS_INTERVAL; + } +#endif + /* Remove old information from rephist and the rend cache. */ if (time_to_clean_caches < now) { rep_history_clean(now - options->RephistTrackTime); diff --git a/src/or/or.h b/src/or/or.h index 935ea523df..2954b125bd 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -23,6 +23,9 @@ #ifndef ENABLE_GEOIP_STATS #define ENABLE_GEOIP_STATS 1 #endif +#ifndef ENABLE_BUFFER_STATS +#define ENABLE_BUFFER_STATS 1 +#endif #endif #ifdef MS_WINDOWS @@ -833,6 +836,9 @@ typedef struct var_cell_t { typedef struct packed_cell_t { struct packed_cell_t *next; /**< Next cell queued on this circuit. */ char body[CELL_NETWORK_SIZE]; /**< Cell as packed for network. */ +#ifdef ENABLE_BUFFER_STATS + struct timeval packed_timeval; /**< When was this cell packed? */ +#endif } packed_cell_t; /** A queue of cells on a circuit, waiting to be added to the @@ -2072,6 +2078,17 @@ typedef struct or_circuit_t { /** True iff this circuit was made with a CREATE_FAST cell. */ unsigned int is_first_hop : 1; + +#ifdef ENABLE_BUFFER_STATS + /** Number of cells that were removed from circuit queue; reset every + * time when writing buffer stats to disk. */ + uint32_t processed_cells; + + /** Total time in milliseconds that cells spent in both app-ward and + * exit-ward queues of this circuit; reset every time when writing + * buffer stats to disk. */ + uint64_t total_cell_waiting_time; +#endif } or_circuit_t; /** Convert a circuit subtype to a circuit_t.*/ @@ -2478,6 +2495,9 @@ typedef struct { /** If true, the user wants us to collect statistics on port usage. */ int ExitPortStatistics; + /** If true, the user wants us to collect cell statistics. */ + int CellStatistics; + /** If true, do not believe anybody who tells us that a domain resolves * to an internal address, or that an internal address has a PTR mapping. * Helps avoid some cross-site attacks. */ @@ -4026,6 +4046,12 @@ void hs_usage_note_fetch_successful(const char *service_id, time_t now); void hs_usage_write_statistics_to_file(time_t now); void hs_usage_free_all(void); +#ifdef ENABLE_BUFFER_STATS +#define DUMP_BUFFER_STATS_INTERVAL (24*60*60) +void add_circ_to_buffer_stats(circuit_t *circ, time_t end_of_interval); +void dump_buffer_stats(void); +#endif + /********************************* rendclient.c ***************************/ void rend_client_introcirc_has_opened(origin_circuit_t *circ); diff --git a/src/or/relay.c b/src/or/relay.c index 3ce05c8858..e5ba6f4035 100644 --- a/src/or/relay.c +++ b/src/or/relay.c @@ -1592,7 +1592,13 @@ cell_queue_append(cell_queue_t *queue, packed_cell_t *cell) void cell_queue_append_packed_copy(cell_queue_t *queue, const cell_t *cell) { - cell_queue_append(queue, packed_cell_copy(cell)); + packed_cell_t *copy = packed_cell_copy(cell); +#ifdef ENABLE_BUFFER_STATS + /* Remember the exact time when this cell was put in the queue. */ + if (get_options()->CellStatistics) + tor_gettimeofday(©->packed_timeval); +#endif + cell_queue_append(queue, copy); } /** Remove and free every cell in queue. */ @@ -1801,6 +1807,19 @@ connection_or_flush_from_first_active_circuit(or_connection_t *conn, int max, packed_cell_t *cell = cell_queue_pop(queue); tor_assert(*next_circ_on_conn_p(circ,conn)); +#ifdef ENABLE_BUFFER_STATS + /* Calculate the exact time that this cell has spent in the queue. */ + if (get_options()->CellStatistics && !CIRCUIT_IS_ORIGIN(circ)) { + struct timeval flushed_from_queue; + uint32_t cell_waiting_time; + or_circuit_t *orcirc = TO_OR_CIRCUIT(circ); + tor_gettimeofday(&flushed_from_queue); + cell_waiting_time = (uint32_t) + (tv_udiff(&cell->packed_timeval, &flushed_from_queue) / 1000); + orcirc->total_cell_waiting_time += cell_waiting_time; + orcirc->processed_cells++; + } +#endif connection_write_to_buf(cell->body, CELL_NETWORK_SIZE, TO_CONN(conn)); packed_cell_free(cell); diff --git a/src/or/rephist.c b/src/or/rephist.c index f1f502cad2..a7f525e542 100644 --- a/src/or/rephist.c +++ b/src/or/rephist.c @@ -2576,3 +2576,174 @@ hs_usage_write_statistics_to_file(time_t now) tor_free(fname); } +/*** cell statistics ***/ + +#ifdef ENABLE_BUFFER_STATS +/** Start of the current buffer stats interval. */ +time_t start_of_buffer_stats_interval; + +typedef struct circ_buffer_stats_t { + uint32_t processed_cells; + double mean_num_cells_in_queue; + double mean_time_cells_in_queue; + uint32_t local_circ_id; +} circ_buffer_stats_t; + +/** Holds stats. */ +smartlist_t *circuits_for_buffer_stats = NULL; + +/** Remember cell statistics for circuit circ at time + * end_of_interval and reset cell counters in case the circuit + * remains open in the next measurement interval. */ +void +add_circ_to_buffer_stats(circuit_t *circ, time_t end_of_interval) +{ + circ_buffer_stats_t *stat; + time_t start_of_interval; + int interval_length; + or_circuit_t *orcirc; + if (CIRCUIT_IS_ORIGIN(circ)) + return; + orcirc = TO_OR_CIRCUIT(circ); + if (!orcirc->processed_cells) + return; + if (!circuits_for_buffer_stats) + circuits_for_buffer_stats = smartlist_create(); + start_of_interval = circ->timestamp_created > + start_of_buffer_stats_interval ? + circ->timestamp_created : + start_of_buffer_stats_interval; + interval_length = (int) (end_of_interval - start_of_interval); + stat = tor_malloc_zero(sizeof(circ_buffer_stats_t)); + stat->processed_cells = orcirc->processed_cells; + /* 1000.0 for s -> ms; 2.0 because of app-ward and exit-ward queues */ + stat->mean_num_cells_in_queue = interval_length == 0 ? 0.0 : + (double) orcirc->total_cell_waiting_time / + (double) interval_length / 1000.0 / 2.0; + stat->mean_time_cells_in_queue = + (double) orcirc->total_cell_waiting_time / + (double) orcirc->processed_cells; + smartlist_add(circuits_for_buffer_stats, stat); + orcirc->total_cell_waiting_time = 0; + orcirc->processed_cells = 0; +} + +/** Sorting helper: return -1, 1, or 0 based on comparison of two + * circ_buffer_stats_t */ +static int +_buffer_stats_compare_entries(const void **_a, const void **_b) +{ + const circ_buffer_stats_t *a = *_a, *b = *_b; + if (a->processed_cells < b->processed_cells) + return 1; + else if (a->processed_cells > b->processed_cells) + return -1; + else + return 0; +} + +/** Append buffer statistics to local file. */ +void +dump_buffer_stats(void) +{ + time_t now = time(NULL); + char *filename; + char written[ISO_TIME_LEN+1]; + open_file_t *open_file = NULL; + FILE *out; +#define SHARES 10 + int processed_cells[SHARES], circs_in_share[SHARES], + number_of_circuits, i; + double queued_cells[SHARES], time_in_queue[SHARES]; + smartlist_t *str_build = smartlist_create(); + char *str = NULL; + char buf[32]; + circuit_t *circ; + /* add current circuits to stats */ + for (circ = _circuit_get_global_list(); circ; circ = circ->next) + add_circ_to_buffer_stats(circ, now); + /* calculate deciles */ + memset(processed_cells, 0, SHARES * sizeof(int)); + memset(circs_in_share, 0, SHARES * sizeof(int)); + memset(queued_cells, 0, SHARES * sizeof(double)); + memset(time_in_queue, 0, SHARES * sizeof(double)); + smartlist_sort(circuits_for_buffer_stats, + _buffer_stats_compare_entries); + number_of_circuits = smartlist_len(circuits_for_buffer_stats); + i = 0; + SMARTLIST_FOREACH_BEGIN(circuits_for_buffer_stats, + circ_buffer_stats_t *, stat) + { + int share = i++ * SHARES / number_of_circuits; + processed_cells[share] += stat->processed_cells; + queued_cells[share] += stat->mean_num_cells_in_queue; + time_in_queue[share] += stat->mean_time_cells_in_queue; + circs_in_share[share]++; + } + SMARTLIST_FOREACH_END(stat); + /* clear buffer stats history */ + SMARTLIST_FOREACH(circuits_for_buffer_stats, circ_buffer_stats_t *, + stat, tor_free(stat)); + smartlist_clear(circuits_for_buffer_stats); + /* write to file */ + filename = get_datadir_fname("buffer-stats"); + out = start_writing_to_stdio_file(filename, OPEN_FLAGS_APPEND, + 0600, &open_file); + if (!out) + goto done; + format_iso_time(written, now); + if (fprintf(out, "written %s (%d s)\n", written, + DUMP_BUFFER_STATS_INTERVAL) < 0) + goto done; + for (i = 0; i < SHARES; i++) { + tor_snprintf(buf, sizeof(buf), "%d", !circs_in_share[i] ? 0 : + processed_cells[i] / circs_in_share[i]); + smartlist_add(str_build, tor_strdup(buf)); + } + str = smartlist_join_strings(str_build, ",", 0, NULL); + if (fprintf(out, "processed-cells %s\n", str) < 0) + goto done; + tor_free(str); + SMARTLIST_FOREACH(str_build, char *, c, tor_free(c)); + smartlist_clear(str_build); + for (i = 0; i < SHARES; i++) { + tor_snprintf(buf, sizeof(buf), "%.2f", circs_in_share[i] == 0 ? 0.0 : + queued_cells[i] / (double) circs_in_share[i]); + smartlist_add(str_build, tor_strdup(buf)); + } + str = smartlist_join_strings(str_build, ",", 0, NULL); + if (fprintf(out, "queued-cells %s\n", str) < 0) + goto done; + tor_free(str); + SMARTLIST_FOREACH(str_build, char *, c, tor_free(c)); + smartlist_clear(str_build); + for (i = 0; i < SHARES; i++) { + tor_snprintf(buf, sizeof(buf), "%.0f", circs_in_share[i] == 0 ? 0.0 : + time_in_queue[i] / (double) circs_in_share[i]); + smartlist_add(str_build, tor_strdup(buf)); + } + str = smartlist_join_strings(str_build, ",", 0, NULL); + if (fprintf(out, "time-in-queue %s\n", str) < 0) + goto done; + tor_free(str); + SMARTLIST_FOREACH(str_build, char *, c, tor_free(c)); + smartlist_free(str_build); + str_build = NULL; + if (fprintf(out, "number-of-circuits-per-share %d\n", + (number_of_circuits + SHARES - 1) / SHARES) < 0) + goto done; + finish_writing_to_file(open_file); + open_file = NULL; + done: + if (open_file) + abort_writing_to_file(open_file); + tor_free(filename); + if (str_build) { + SMARTLIST_FOREACH(str_build, char *, c, tor_free(c)); + smartlist_free(str_build); + } + tor_free(str); +#undef SHARES +} +#endif + From c0b6cb132baaf9cf259bfb09e14c1128d7abc9d6 Mon Sep 17 00:00:00 2001 From: Karsten Loesing Date: Sun, 5 Jul 2009 20:48:16 +0200 Subject: [PATCH 092/187] If configured, write entry-node statistics to disk periodically. --- ChangeLog | 4 ++++ configure.in | 7 +++++++ src/or/config.c | 20 ++++++++++++++++++++ src/or/geoip.c | 42 ++++++++++++++++++++++++++++++++++++++++++ src/or/or.h | 9 ++++++++- src/or/router.c | 3 +++ 6 files changed, 84 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 0d5af889da..a5f35fbeef 100644 --- a/ChangeLog +++ b/ChangeLog @@ -21,6 +21,10 @@ Changes in version 0.2.2.1-alpha - 2009-??-?? transferred bytes per port to disk every 24 hours. To enable this, run configure with the --enable-exit-stats option, and set "ExitPortStatistics 1" in your torrc. + - Entry nodes can write statistics on the rough number and origins of + connecting clients to disk every 24 hours. To enable this, run + configure with the --enable-entry-stats option, and set + "EntryStatistics 1" in your torrc. o Minor bugfixes - Hidden service clients didn't use a cached service descriptor that diff --git a/configure.in b/configure.in index 844211588d..0f83207991 100644 --- a/configure.in +++ b/configure.in @@ -99,6 +99,13 @@ if test "$enable_geoip_stats" = "yes"; then AC_DEFINE(ENABLE_GEOIP_STATS, 1, [Defined if we try to collect per-country statistics]) fi +AC_ARG_ENABLE(entry-stats, + AS_HELP_STRING(--enable-entry-stats, enable code for entry guards to collect per-country statistics)) + +if test "$enable_entry_stats" = "yes"; then + AC_DEFINE(ENABLE_ENTRY_STATS, 1, [Defined if we try to collect per-country statistics]) +fi + AC_ARG_ENABLE(gcc-warnings, AS_HELP_STRING(--enable-gcc-warnings, enable verbose warnings)) diff --git a/src/or/config.c b/src/or/config.c index f1fea13e58..c9ef21bb8c 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -198,6 +198,7 @@ static config_var_t _option_vars[] = { V(DownloadExtraInfo, BOOL, "0"), V(EnforceDistinctSubnets, BOOL, "1"), V(EntryNodes, ROUTERSET, NULL), + V(EntryStatistics, BOOL, "0"), V(TestingEstimatedDescriptorPropagationTime, INTERVAL, "10 minutes"), V(ExcludeNodes, ROUTERSET, NULL), V(ExcludeExitNodes, ROUTERSET, NULL), @@ -1394,6 +1395,25 @@ options_act(or_options_t *old_options) if (options->ExitPortStatistics) log_warn(LD_CONFIG, "ExitPortStatistics enabled, but Tor was built " "without port statistics support."); +#endif +#ifdef ENABLE_ENTRY_STATS + if (options->EntryStatistics) { + if (should_record_bridge_info(options)) { + /* Don't allow measuring statistics on entry guards when configured + * as bridge. */ + log_warn(LD_CONFIG, "Bridges cannot be configured to measure " + "additional GeoIP statistics as entry guards."); + return -1; + } else + log_notice(LD_CONFIG, "Configured to measure entry node " + "statistics. Look for the entry-stats file that will " + "first be written to the data directory in 24 hours " + "from now."); + } +#else + if (options->EntryStatistics) + log_warn(LD_CONFIG, "EntryStatistics enabled, but Tor was built " + "without entry node statistics support."); #endif /* Check if we need to parse and add the EntryNodes config option. */ if (options->EntryNodes && diff --git a/src/or/geoip.c b/src/or/geoip.c index 41c8f21cdb..13a6a28500 100644 --- a/src/or/geoip.c +++ b/src/or/geoip.c @@ -13,6 +13,7 @@ static void clear_geoip_db(void); static void dump_geoip_stats(void); +static void dump_entry_stats(void); /** An entry from the GeoIP file: maps an IP range to a country. */ typedef struct geoip_entry_t { @@ -308,8 +309,13 @@ geoip_note_client_seen(geoip_client_action_t action, or_options_t *options = get_options(); clientmap_entry_t lookup, *ent; if (action == GEOIP_CLIENT_CONNECT) { +#ifdef ENABLE_ENTRY_STATS + if (!options->EntryStatistics) + return; +#else if (!(options->BridgeRelay && options->BridgeRecordUsageByCountry)) return; +#endif /* Did we recently switch from bridge to relay or back? */ if (client_history_starts > now) return; @@ -337,6 +343,8 @@ geoip_note_client_seen(geoip_client_action_t action, geoip_remove_old_clients(current_request_period_starts); /* Before rotating, write the current stats to disk. */ dump_geoip_stats(); + if (get_options()->EntryStatistics) + dump_entry_stats(); /* Now rotate request period */ SMARTLIST_FOREACH(geoip_countries, geoip_country_t *, c, { memmove(&c->n_v2_ns_requests[0], &c->n_v2_ns_requests[1], @@ -657,6 +665,40 @@ dump_geoip_stats(void) #endif } +/** Store all our geoip statistics as entry guards into + * $DATADIR/entry-stats. */ +static void +dump_entry_stats(void) +{ +#ifdef ENABLE_ENTRY_STATS + time_t now = time(NULL); + char *filename = get_datadir_fname("entry-stats"); + char *data = NULL; + char since[ISO_TIME_LEN+1], written[ISO_TIME_LEN+1]; + open_file_t *open_file = NULL; + FILE *out; + + data = geoip_get_client_history(now, GEOIP_CLIENT_CONNECT); + format_iso_time(since, geoip_get_history_start()); + format_iso_time(written, now); + out = start_writing_to_stdio_file(filename, OPEN_FLAGS_APPEND, + 0600, &open_file); + if (!out) + goto done; + if (fprintf(out, "written %s\nstarted-at %s\nips %s\n", + written, since, data ? data : "") < 0) + goto done; + + finish_writing_to_file(open_file); + open_file = NULL; + done: + if (open_file) + abort_writing_to_file(open_file); + tor_free(filename); + tor_free(data); +#endif +} + /** Helper used to implement GETINFO ip-to-country/... controller command. */ int getinfo_helper_geoip(control_connection_t *control_conn, diff --git a/src/or/or.h b/src/or/or.h index 935ea523df..65eac0e9e9 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -2478,6 +2478,9 @@ typedef struct { /** If true, the user wants us to collect statistics on port usage. */ int ExitPortStatistics; + /** If true, the user wants us to collect statistics as entry node. */ + int EntryStatistics; + /** If true, do not believe anybody who tells us that a domain resolves * to an internal address, or that an internal address has a PTR mapping. * Helps avoid some cross-site attacks. */ @@ -3595,6 +3598,10 @@ int dnsserv_launch_request(const char *name, int is_reverse); * we are willing to talk about it? */ #define DIR_RECORD_USAGE_MIN_OBSERVATION_TIME (24*60*60) +/** Time interval: Flush geoip data to disk this often when measuring on an + * entry guard. */ +#define ENTRY_RECORD_USAGE_RETAIN_IPS (24*60*60) + #ifdef GEOIP_PRIVATE int geoip_parse_entry(const char *line); #endif @@ -3610,7 +3617,7 @@ country_t geoip_get_country(const char *countrycode); * the others, we're not. */ typedef enum { - /** We've noticed a connection as a bridge relay. */ + /** We've noticed a connection as a bridge relay or entry guard. */ GEOIP_CLIENT_CONNECT = 0, /** We've served a networkstatus consensus as a directory server. */ GEOIP_CLIENT_NETWORKSTATUS = 1, diff --git a/src/or/router.c b/src/or/router.c index cb73e378c0..bdea4fa764 100644 --- a/src/or/router.c +++ b/src/or/router.c @@ -1918,6 +1918,9 @@ extrainfo_get_client_geoip_summary(time_t now) int geoip_purge_interval = 48*60*60; #ifdef ENABLE_GEOIP_STATS geoip_purge_interval = DIR_RECORD_USAGE_RETAIN_IPS; +#endif +#ifdef ENABLE_ENTRY_STATS + geoip_purge_interval = ENTRY_RECORD_USAGE_RETAIN_IPS; #endif if (now > last_purged_at+geoip_purge_interval) { /* (Note that this also discards items in the client history with From fa2374a16360cb3d9bfb18827c574c8f44345925 Mon Sep 17 00:00:00 2001 From: Karsten Loesing Date: Thu, 9 Jul 2009 15:34:53 +0200 Subject: [PATCH 093/187] List unresolved requests in geoip stats as country '??'. --- ChangeLog | 3 ++- src/or/geoip.c | 11 ++++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index c0a1a9e0cc..3c778c8643 100644 --- a/ChangeLog +++ b/ChangeLog @@ -16,7 +16,8 @@ Changes in version 0.2.2.1-alpha - 2009-??-?? to make sure nothing writes beyond the end of an area. This might help debug some conceivable causes of bug 930. - Directories that are configured with the --enable-geoip-stats flag - now write their GeoIP stats to disk exactly every 24 hours. + now write their GeoIP stats to disk exactly every 24 hours. Also, + unresolved requests are listed with country code '??'. - Exit nodes can write statistics on the number of exit streams and transferred bytes per port to disk every 24 hours. To enable this, run configure with the --enable-exit-stats option, and set diff --git a/src/or/geoip.c b/src/or/geoip.c index 13a6a28500..13b9b7d0f0 100644 --- a/src/or/geoip.c +++ b/src/or/geoip.c @@ -188,7 +188,14 @@ geoip_load_file(const char *filename, or_options_t *options) return -1; } if (!geoip_countries) { + geoip_country_t *geoip_unresolved; geoip_countries = smartlist_create(); + /* Add a geoip_country_t for requests that could not be resolved to a + * country as first element (index 0) to geoip_countries. */ + geoip_unresolved = tor_malloc_zero(sizeof(geoip_country_t)); + strlcpy(geoip_unresolved->countrycode, "??", + sizeof(geoip_unresolved->countrycode)); + smartlist_add(geoip_countries, geoip_unresolved); country_idxplus1_by_lc_code = strmap_new(); } if (geoip_entries) { @@ -375,6 +382,8 @@ geoip_note_client_seen(geoip_client_action_t action, if (action == GEOIP_CLIENT_NETWORKSTATUS || action == GEOIP_CLIENT_NETWORKSTATUS_V2) { int country_idx = geoip_get_country_by_ip(addr); + if (country_idx < 0) + country_idx = 0; /** unresolved requests are stored at index 0. */ if (country_idx >= 0 && country_idx < smartlist_len(geoip_countries)) { geoip_country_t *country = smartlist_get(geoip_countries, country_idx); if (action == GEOIP_CLIENT_NETWORKSTATUS) @@ -505,7 +514,7 @@ geoip_get_client_history(time_t now, geoip_client_action_t action) continue; country = geoip_get_country_by_ip((*ent)->ipaddr); if (country < 0) - continue; + country = 0; /** unresolved requests are stored at index 0. */ tor_assert(0 <= country && country < n_countries); ++counts[country]; ++total; From 7fb206e554db6290ed355cd7fda7d2bcfb23c89f Mon Sep 17 00:00:00 2001 From: Karsten Loesing Date: Thu, 9 Jul 2009 23:55:13 +0200 Subject: [PATCH 094/187] Tweak exit port statistics a bit. Introduce a threshold of 0.01% of bytes that must be read and written per port in order to be included in the statistics. Otherwise we cannot include these statistics in extra-info documents, because they are too big. Change the labels "-written" and "-read" so that the meanings are as intended. --- src/or/rephist.c | 78 ++++++++++++++++++++++++++++++++++-------------- 1 file changed, 56 insertions(+), 22 deletions(-) diff --git a/src/or/rephist.c b/src/or/rephist.c index a7f525e542..ba363ba742 100644 --- a/src/or/rephist.c +++ b/src/or/rephist.c @@ -1330,6 +1330,9 @@ rep_hist_note_bytes_read(size_t num_bytes, time_t when) #define EXIT_STATS_ROUND_UP_STREAMS 4 /** Number of TCP ports */ #define EXIT_STATS_NUM_PORTS 65536 +/** Reciprocal of threshold (= 0.01%) of total bytes that a port needs to + * see in order to be included in exit stats. */ +#define EXIT_STATS_THRESHOLD 10000 /* The following data structures are arrays and no fancy smartlists or maps, * so that all write operations can be done in constant time. This comes at @@ -1351,7 +1354,8 @@ write_exit_stats(time_t when) { char t[ISO_TIME_LEN+1]; int r, i, comma; - uint64_t *b; + uint64_t *b, total_bytes, other_bytes; + uint32_t other_streams; char *filename = get_datadir_fname("exit-stats"); open_file_t *open_file = NULL; @@ -1376,51 +1380,81 @@ write_exit_stats(time_t when) if (fprintf(out, "written %s (%d s)\n", t, EXIT_STATS_INTERVAL_SEC) < 0) goto done; + /* Count the total number of bytes, so that we can attribute all + * observations below a threshold of 1 / EXIT_STATS_THRESHOLD of all + * bytes to a special port 'other'. */ + total_bytes = 0; + for (i = 1; i < EXIT_STATS_NUM_PORTS; i++) { + total_bytes += exit_bytes_read[i]; + total_bytes += exit_bytes_written[i]; + } + /* kibibytes-(read|written) port=kibibytes,.. */ for (r = 0; r < 2; r++) { b = r ? exit_bytes_read : exit_bytes_written; tor_assert(b); if (fprintf(out, "%s ", - r == 0 ? "kibibytes-read" : "kibibytes-written")<0) + r ? "kibibytes-read" : "kibibytes-written")<0) goto done; comma = 0; + other_bytes = 0; for (i = 1; i < EXIT_STATS_NUM_PORTS; i++) { if (b[i] > 0) { - uint64_t num = b[i]; - num += EXIT_STATS_ROUND_UP_BYTES - 1; - num /= EXIT_STATS_ROUND_UP_BYTES; - num *= EXIT_STATS_ROUND_UP_BYTES; - num /= 1024; - if (fprintf(out, "%s%d="U64_FORMAT, - comma++ ? "," : "", i, - U64_PRINTF_ARG(num)) < 0) - goto done; + if (exit_bytes_read[i] + exit_bytes_written[i] > 0 && + (total_bytes / (exit_bytes_read[i] + exit_bytes_written[i]) + < EXIT_STATS_THRESHOLD)) { + uint64_t num = b[i]; + num += EXIT_STATS_ROUND_UP_BYTES - 1; + num /= EXIT_STATS_ROUND_UP_BYTES; + num *= EXIT_STATS_ROUND_UP_BYTES; + num /= 1024; + if (fprintf(out, "%s%d="U64_FORMAT, + comma++ ? "," : "", i, + U64_PRINTF_ARG(num)) < 0) + goto done; + } else + other_bytes += b[i]; } } - if (fprintf(out, "\n")<0) + other_bytes += EXIT_STATS_ROUND_UP_BYTES - 1; + other_bytes /= EXIT_STATS_ROUND_UP_BYTES; + other_bytes *= EXIT_STATS_ROUND_UP_BYTES; + other_bytes /= 1024; + if (fprintf(out, "%sother="U64_FORMAT"\n", + comma ? "," : "", other_bytes)<0) goto done; - /* Reset counters */ - memset(b, 0, EXIT_STATS_NUM_PORTS*sizeof(uint64_t)); } /* streams-opened port=num,.. */ if (fprintf(out, "streams-opened ")<0) goto done; comma = 0; + other_streams = 0; for (i = 1; i < EXIT_STATS_NUM_PORTS; i++) { if (exit_streams[i] > 0) { - uint32_t num = exit_streams[i]; - num += EXIT_STATS_ROUND_UP_STREAMS - 1; - num /= EXIT_STATS_ROUND_UP_STREAMS; - num *= EXIT_STATS_ROUND_UP_STREAMS; - if (fprintf(out, "%s%d=%d", - comma++ ? "," : "", i, num)<0) - goto done; + if (exit_bytes_read[i] + exit_bytes_written[i] > 0 && + (total_bytes / (exit_bytes_read[i] + exit_bytes_written[i]) + < EXIT_STATS_THRESHOLD)) { + uint32_t num = exit_streams[i]; + num += EXIT_STATS_ROUND_UP_STREAMS - 1; + num /= EXIT_STATS_ROUND_UP_STREAMS; + num *= EXIT_STATS_ROUND_UP_STREAMS; + if (fprintf(out, "%s%d=%d", + comma++ ? "," : "", i, num)<0) + goto done; + } else + other_streams += exit_streams[i]; } } - if (fprintf(out, "\n")<0) + other_streams += EXIT_STATS_ROUND_UP_STREAMS - 1; + other_streams /= EXIT_STATS_ROUND_UP_STREAMS; + other_streams *= EXIT_STATS_ROUND_UP_STREAMS; + if (fprintf(out, "%sother=%d\n", + comma ? "," : "", other_streams)<0) goto done; /* Reset counters */ + memset(exit_bytes_read, 0, sizeof(exit_bytes_read)); + memset(exit_bytes_written, 0, sizeof(exit_bytes_written)); memset(exit_streams, 0, sizeof(exit_streams)); end_of_current_exit_stats_period += EXIT_STATS_INTERVAL_SEC; } From d1437245c77b930382f9b36355a3fa67d48ecb38 Mon Sep 17 00:00:00 2001 From: Karsten Loesing Date: Sat, 11 Jul 2009 00:44:27 +0200 Subject: [PATCH 095/187] Simplify the math to round up to the next multiple of some value. --- src/or/geoip.c | 10 ---------- src/or/or.h | 10 ++++++++++ src/or/rephist.c | 22 ++++++++-------------- 3 files changed, 18 insertions(+), 24 deletions(-) diff --git a/src/or/geoip.c b/src/or/geoip.c index 13a6a28500..aac4893117 100644 --- a/src/or/geoip.c +++ b/src/or/geoip.c @@ -461,16 +461,6 @@ _c_hist_compare(const void **_a, const void **_b) * are willing to talk about it? */ #define GEOIP_MIN_OBSERVATION_TIME (12*60*60) -/** Return the lowest x such that x is at least number, and x modulo - * divisor == 0. */ -static INLINE unsigned -round_to_next_multiple_of(unsigned number, unsigned divisor) -{ - number += divisor - 1; - number -= number % divisor; - return number; -} - /** Return a newly allocated comma-separated string containing entries for all * the countries from which we've seen enough clients connect. The entry * format is cc=num where num is the number of IPs we've seen connecting from diff --git a/src/or/or.h b/src/or/or.h index f298d53c27..be38035af7 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -2669,6 +2669,16 @@ struct socks_request_t { * every connection. */ }; +/** Return the lowest x such that x is at least number, and x modulo + * divisor == 0. */ +static INLINE unsigned +round_to_next_multiple_of(unsigned number, unsigned divisor) +{ + number += divisor - 1; + number -= number % divisor; + return number; +} + /* all the function prototypes go here */ /********************************* buffers.c ***************************/ diff --git a/src/or/rephist.c b/src/or/rephist.c index ba363ba742..73dd1315ed 100644 --- a/src/or/rephist.c +++ b/src/or/rephist.c @@ -1404,10 +1404,8 @@ write_exit_stats(time_t when) if (exit_bytes_read[i] + exit_bytes_written[i] > 0 && (total_bytes / (exit_bytes_read[i] + exit_bytes_written[i]) < EXIT_STATS_THRESHOLD)) { - uint64_t num = b[i]; - num += EXIT_STATS_ROUND_UP_BYTES - 1; - num /= EXIT_STATS_ROUND_UP_BYTES; - num *= EXIT_STATS_ROUND_UP_BYTES; + uint64_t num = round_to_next_multiple_of(b[i], + EXIT_STATS_ROUND_UP_BYTES); num /= 1024; if (fprintf(out, "%s%d="U64_FORMAT, comma++ ? "," : "", i, @@ -1417,9 +1415,8 @@ write_exit_stats(time_t when) other_bytes += b[i]; } } - other_bytes += EXIT_STATS_ROUND_UP_BYTES - 1; - other_bytes /= EXIT_STATS_ROUND_UP_BYTES; - other_bytes *= EXIT_STATS_ROUND_UP_BYTES; + other_bytes = round_to_next_multiple_of(other_bytes, + EXIT_STATS_ROUND_UP_BYTES); other_bytes /= 1024; if (fprintf(out, "%sother="U64_FORMAT"\n", comma ? "," : "", other_bytes)<0) @@ -1435,10 +1432,8 @@ write_exit_stats(time_t when) if (exit_bytes_read[i] + exit_bytes_written[i] > 0 && (total_bytes / (exit_bytes_read[i] + exit_bytes_written[i]) < EXIT_STATS_THRESHOLD)) { - uint32_t num = exit_streams[i]; - num += EXIT_STATS_ROUND_UP_STREAMS - 1; - num /= EXIT_STATS_ROUND_UP_STREAMS; - num *= EXIT_STATS_ROUND_UP_STREAMS; + uint32_t num = round_to_next_multiple_of(exit_streams[i], + EXIT_STATS_ROUND_UP_STREAMS); if (fprintf(out, "%s%d=%d", comma++ ? "," : "", i, num)<0) goto done; @@ -1446,9 +1441,8 @@ write_exit_stats(time_t when) other_streams += exit_streams[i]; } } - other_streams += EXIT_STATS_ROUND_UP_STREAMS - 1; - other_streams /= EXIT_STATS_ROUND_UP_STREAMS; - other_streams *= EXIT_STATS_ROUND_UP_STREAMS; + other_streams = round_to_next_multiple_of(other_streams, + EXIT_STATS_ROUND_UP_STREAMS); if (fprintf(out, "%sother=%d\n", comma ? "," : "", other_streams)<0) goto done; From 707a44a7b0e12f9aef541dd69b356e2f123bcda8 Mon Sep 17 00:00:00 2001 From: Karsten Loesing Date: Mon, 13 Jul 2009 22:11:21 +0200 Subject: [PATCH 096/187] Replace two %d with %u's. --- src/or/rephist.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/or/rephist.c b/src/or/rephist.c index 73dd1315ed..f987664449 100644 --- a/src/or/rephist.c +++ b/src/or/rephist.c @@ -1434,7 +1434,7 @@ write_exit_stats(time_t when) < EXIT_STATS_THRESHOLD)) { uint32_t num = round_to_next_multiple_of(exit_streams[i], EXIT_STATS_ROUND_UP_STREAMS); - if (fprintf(out, "%s%d=%d", + if (fprintf(out, "%s%d=%u", comma++ ? "," : "", i, num)<0) goto done; } else @@ -1443,7 +1443,7 @@ write_exit_stats(time_t when) } other_streams = round_to_next_multiple_of(other_streams, EXIT_STATS_ROUND_UP_STREAMS); - if (fprintf(out, "%sother=%d\n", + if (fprintf(out, "%sother=%u\n", comma ? "," : "", other_streams)<0) goto done; /* Reset counters */ From 8f1a973669284e5662bd6f23bc96ecfafa57c554 Mon Sep 17 00:00:00 2001 From: Karsten Loesing Date: Mon, 13 Jul 2009 22:43:06 +0200 Subject: [PATCH 097/187] Two tweaks to exit-port statistics. Add two functions for round_to_next_multiple_of() for uint32_t and uint64_t. Avoid division in every step of the loop over all ports. --- src/common/util.c | 30 ++++++++++++++++++++++++++++++ src/common/util.h | 3 +++ src/or/or.h | 10 ---------- src/or/rephist.c | 33 +++++++++++++++------------------ 4 files changed, 48 insertions(+), 28 deletions(-) diff --git a/src/common/util.c b/src/common/util.c index 7b9e5eb562..0e7f59e620 100644 --- a/src/common/util.c +++ b/src/common/util.c @@ -345,6 +345,36 @@ round_to_power_of_2(uint64_t u64) return low; } +/** Return the lowest x such that x is at least number, and x modulo + * divisor == 0. */ +unsigned +round_to_next_multiple_of(unsigned number, unsigned divisor) +{ + number += divisor - 1; + number -= number % divisor; + return number; +} + +/** Return the lowest x such that x is at least number, and x modulo + * divisor == 0. */ +uint32_t +round_uint32_to_next_multiple_of(uint32_t number, uint32_t divisor) +{ + number += divisor - 1; + number -= number % divisor; + return number; +} + +/** Return the lowest x such that x is at least number, and x modulo + * divisor == 0. */ +uint64_t +round_uint64_to_next_multiple_of(uint64_t number, uint64_t divisor) +{ + number += divisor - 1; + number -= number % divisor; + return number; +} + /* ===== * String manipulation * ===== */ diff --git a/src/common/util.h b/src/common/util.h index 18033f39df..1c5643be39 100644 --- a/src/common/util.h +++ b/src/common/util.h @@ -154,6 +154,9 @@ void tor_log_mallinfo(int severity); /* Math functions */ int tor_log2(uint64_t u64) ATTR_CONST; uint64_t round_to_power_of_2(uint64_t u64); +unsigned round_to_next_multiple_of(unsigned number, unsigned divisor); +uint32_t round_uint32_to_next_multiple_of(uint32_t number, uint32_t divisor); +uint64_t round_uint64_to_next_multiple_of(uint64_t number, uint64_t divisor); /* String manipulation */ diff --git a/src/or/or.h b/src/or/or.h index be38035af7..f298d53c27 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -2669,16 +2669,6 @@ struct socks_request_t { * every connection. */ }; -/** Return the lowest x such that x is at least number, and x modulo - * divisor == 0. */ -static INLINE unsigned -round_to_next_multiple_of(unsigned number, unsigned divisor) -{ - number += divisor - 1; - number -= number % divisor; - return number; -} - /* all the function prototypes go here */ /********************************* buffers.c ***************************/ diff --git a/src/or/rephist.c b/src/or/rephist.c index f987664449..6a716d376b 100644 --- a/src/or/rephist.c +++ b/src/or/rephist.c @@ -1332,7 +1332,7 @@ rep_hist_note_bytes_read(size_t num_bytes, time_t when) #define EXIT_STATS_NUM_PORTS 65536 /** Reciprocal of threshold (= 0.01%) of total bytes that a port needs to * see in order to be included in exit stats. */ -#define EXIT_STATS_THRESHOLD 10000 +#define EXIT_STATS_THRESHOLD_RECIPROCAL 10000 /* The following data structures are arrays and no fancy smartlists or maps, * so that all write operations can be done in constant time. This comes at @@ -1354,7 +1354,7 @@ write_exit_stats(time_t when) { char t[ISO_TIME_LEN+1]; int r, i, comma; - uint64_t *b, total_bytes, other_bytes; + uint64_t *b, total_bytes, threshold_bytes, other_bytes; uint32_t other_streams; char *filename = get_datadir_fname("exit-stats"); @@ -1381,13 +1381,14 @@ write_exit_stats(time_t when) goto done; /* Count the total number of bytes, so that we can attribute all - * observations below a threshold of 1 / EXIT_STATS_THRESHOLD of all - * bytes to a special port 'other'. */ + * observations below a threshold of 1 / EXIT_STATS_THRESHOLD_RECIPROCAL + * of all bytes to a special port 'other'. */ total_bytes = 0; for (i = 1; i < EXIT_STATS_NUM_PORTS; i++) { total_bytes += exit_bytes_read[i]; total_bytes += exit_bytes_written[i]; } + threshold_bytes = total_bytes / EXIT_STATS_THRESHOLD_RECIPROCAL; /* kibibytes-(read|written) port=kibibytes,.. */ for (r = 0; r < 2; r++) { @@ -1401,11 +1402,9 @@ write_exit_stats(time_t when) other_bytes = 0; for (i = 1; i < EXIT_STATS_NUM_PORTS; i++) { if (b[i] > 0) { - if (exit_bytes_read[i] + exit_bytes_written[i] > 0 && - (total_bytes / (exit_bytes_read[i] + exit_bytes_written[i]) - < EXIT_STATS_THRESHOLD)) { - uint64_t num = round_to_next_multiple_of(b[i], - EXIT_STATS_ROUND_UP_BYTES); + if (exit_bytes_read[i] + exit_bytes_written[i] > threshold_bytes) { + uint64_t num = round_uint64_to_next_multiple_of(b[i], + EXIT_STATS_ROUND_UP_BYTES); num /= 1024; if (fprintf(out, "%s%d="U64_FORMAT, comma++ ? "," : "", i, @@ -1415,8 +1414,8 @@ write_exit_stats(time_t when) other_bytes += b[i]; } } - other_bytes = round_to_next_multiple_of(other_bytes, - EXIT_STATS_ROUND_UP_BYTES); + other_bytes = round_uint64_to_next_multiple_of(other_bytes, + EXIT_STATS_ROUND_UP_BYTES); other_bytes /= 1024; if (fprintf(out, "%sother="U64_FORMAT"\n", comma ? "," : "", other_bytes)<0) @@ -1429,11 +1428,9 @@ write_exit_stats(time_t when) other_streams = 0; for (i = 1; i < EXIT_STATS_NUM_PORTS; i++) { if (exit_streams[i] > 0) { - if (exit_bytes_read[i] + exit_bytes_written[i] > 0 && - (total_bytes / (exit_bytes_read[i] + exit_bytes_written[i]) - < EXIT_STATS_THRESHOLD)) { - uint32_t num = round_to_next_multiple_of(exit_streams[i], - EXIT_STATS_ROUND_UP_STREAMS); + if (exit_bytes_read[i] + exit_bytes_written[i] > threshold_bytes) { + uint32_t num = round_uint32_to_next_multiple_of(exit_streams[i], + EXIT_STATS_ROUND_UP_STREAMS); if (fprintf(out, "%s%d=%u", comma++ ? "," : "", i, num)<0) goto done; @@ -1441,8 +1438,8 @@ write_exit_stats(time_t when) other_streams += exit_streams[i]; } } - other_streams = round_to_next_multiple_of(other_streams, - EXIT_STATS_ROUND_UP_STREAMS); + other_streams = round_uint32_to_next_multiple_of(other_streams, + EXIT_STATS_ROUND_UP_STREAMS); if (fprintf(out, "%sother=%u\n", comma ? "," : "", other_streams)<0) goto done; From 9ece0955f75d5bb4e9bd93b2fbbc1064def66774 Mon Sep 17 00:00:00 2001 From: Roger Dingledine Date: Sat, 20 Jun 2009 03:21:52 -0400 Subject: [PATCH 098/187] fix comments and other typos --- src/or/buffers.c | 4 ++-- src/or/config.c | 2 +- src/or/directory.c | 12 ++++++------ src/or/dirvote.c | 2 +- src/or/dns.c | 4 ++-- src/or/networkstatus.c | 2 +- src/or/relay.c | 3 ++- 7 files changed, 15 insertions(+), 14 deletions(-) diff --git a/src/or/buffers.c b/src/or/buffers.c index ed39bc0f82..17d3399635 100644 --- a/src/or/buffers.c +++ b/src/or/buffers.c @@ -1400,7 +1400,7 @@ fetch_from_buf_socks(buf_t *buf, socks_request_t *req, "Tor only an IP address. Applications that do DNS resolves " "themselves may leak information. Consider using Socks4A " "(e.g. via privoxy or socat) instead. For more information, " - "please see http://wiki.noreply.org/noreply/TheOnionRouter/" + "please see https://wiki.torproject.org/TheOnionRouter/" "TorFAQ#SOCKSAndDNS.%s", req->port, safe_socks ? " Rejecting." : ""); /*have_warned_about_unsafe_socks = 1;*/ @@ -1513,7 +1513,7 @@ fetch_from_buf_socks(buf_t *buf, socks_request_t *req, "only an IP address. Applications that do DNS resolves " "themselves may leak information. Consider using Socks4A " "(e.g. via privoxy or socat) instead. For more information, " - "please see http://wiki.noreply.org/noreply/TheOnionRouter/" + "please see https://wiki.torproject.org/TheOnionRouter/" "TorFAQ#SOCKSAndDNS.%s", req->port, safe_socks ? " Rejecting." : ""); /*have_warned_about_unsafe_socks = 1;*/ /*(for now, warn every time)*/ diff --git a/src/or/config.c b/src/or/config.c index 5137f74792..087a907e48 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -2948,7 +2948,7 @@ options_validate(or_options_t *old_options, or_options_t *options, !strcmpstart(uname, "Windows Me"))) { log(LOG_WARN, LD_CONFIG, "Tor is running as a server, but you are " "running %s; this probably won't work. See " - "http://wiki.noreply.org/noreply/TheOnionRouter/TorFAQ#ServerOS " + "https://wiki.torproject.org/TheOnionRouter/TorFAQ#ServerOS " "for details.", uname); } diff --git a/src/or/directory.c b/src/or/directory.c index daf2a15762..0f9cef53df 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -881,7 +881,7 @@ directory_get_consensus_url(int supports_conditional_consensus) if (supports_conditional_consensus) { char *authority_id_list; - smartlist_t *authority_digets = smartlist_create(); + smartlist_t *authority_digests = smartlist_create(); SMARTLIST_FOREACH(router_get_trusted_dir_servers(), trusted_dir_server_t *, ds, @@ -893,10 +893,10 @@ directory_get_consensus_url(int supports_conditional_consensus) hex = tor_malloc(2*CONDITIONAL_CONSENSUS_FPR_LEN+1); base16_encode(hex, 2*CONDITIONAL_CONSENSUS_FPR_LEN+1, ds->v3_identity_digest, CONDITIONAL_CONSENSUS_FPR_LEN); - smartlist_add(authority_digets, hex); + smartlist_add(authority_digests, hex); }); - smartlist_sort(authority_digets, _compare_strs); - authority_id_list = smartlist_join_strings(authority_digets, + smartlist_sort(authority_digests, _compare_strs); + authority_id_list = smartlist_join_strings(authority_digests, "+", 0, NULL); len = strlen(authority_id_list)+64; @@ -904,8 +904,8 @@ directory_get_consensus_url(int supports_conditional_consensus) tor_snprintf(url, len, "/tor/status-vote/current/consensus/%s.z", authority_id_list); - SMARTLIST_FOREACH(authority_digets, char *, cp, tor_free(cp)); - smartlist_free(authority_digets); + SMARTLIST_FOREACH(authority_digests, char *, cp, tor_free(cp)); + smartlist_free(authority_digests); tor_free(authority_id_list); } else { url = tor_strdup("/tor/status-vote/current/consensus.z"); diff --git a/src/or/dirvote.c b/src/or/dirvote.c index 0156e273be..409c2fed69 100644 --- a/src/or/dirvote.c +++ b/src/or/dirvote.c @@ -2183,7 +2183,7 @@ dirvote_get_pending_consensus(void) } /** Return the signatures that we know for the consensus that we're currently - * trying to build */ + * trying to build. */ const char * dirvote_get_pending_detached_signatures(void) { diff --git a/src/or/dns.c b/src/or/dns.c index 8d00d23eee..c6f9e8b833 100644 --- a/src/or/dns.c +++ b/src/or/dns.c @@ -1461,8 +1461,8 @@ evdns_wildcard_check_callback(int result, char type, int count, int ttl, } log(dns_wildcard_one_notice_given ? LOG_INFO : LOG_NOTICE, LD_EXIT, "Your DNS provider gave an answer for \"%s\", which " - "is not supposed to exist. Apparently they are hijacking " - "DNS failures. Trying to correct for this. We've noticed %d " + "is not supposed to exist. Apparently they are hijacking " + "DNS failures. Trying to correct for this. We've noticed %d " "possibly bad address%s so far.", string_address, strmap_size(dns_wildcard_response_count), (strmap_size(dns_wildcard_response_count) == 1) ? "" : "es"); diff --git a/src/or/networkstatus.c b/src/or/networkstatus.c index 573197a53f..e78aefc5fa 100644 --- a/src/or/networkstatus.c +++ b/src/or/networkstatus.c @@ -1154,7 +1154,7 @@ update_consensus_networkstatus_fetch_time(time_t now) } if (dl_interval < 1) dl_interval = 1; - /* We must not try to replace c while it's still the most valid: */ + /* We must not try to replace c while it's still fresh: */ tor_assert(c->fresh_until < start); /* We must download the next one before c is invalid: */ tor_assert(start+dl_interval < c->valid_until); diff --git a/src/or/relay.c b/src/or/relay.c index e5ba6f4035..eb18bbaade 100644 --- a/src/or/relay.c +++ b/src/or/relay.c @@ -992,7 +992,8 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, relay_header_unpack(&rh, cell->payload); // log_fn(LOG_DEBUG,"command %d stream %d", rh.command, rh.stream_id); num_seen++; - log_debug(domain, "Now seen %d relay cells here.", num_seen); + log_debug(domain, "Now seen %d relay cells here (command %d, stream %d).", + num_seen, rh.command, rh.stream_id); if (rh.length > RELAY_PAYLOAD_SIZE) { log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, From 862fbbed7f381cf52a2672be2f12d3c2941a4ce9 Mon Sep 17 00:00:00 2001 From: Roger Dingledine Date: Sat, 20 Jun 2009 01:35:25 -0400 Subject: [PATCH 099/187] resolve atagar's confusion with getinfo accounting --- doc/spec/control-spec.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/spec/control-spec.txt b/doc/spec/control-spec.txt index 0c739eca03..4fdf409742 100644 --- a/doc/spec/control-spec.txt +++ b/doc/spec/control-spec.txt @@ -502,7 +502,7 @@ start and the rest of the interval respectively. The 'interval-start' and 'interval-end' fields are the borders of the current interval; the 'interval-wake' field is the time within the current interval (if any) - where we plan[ned] to start being active. + where we plan[ned] to start being active. The times are GMT. "config/names" A series of lines listing the available configuration options. Each is From b48e3371d748d3b1927c594ee38110c3c68688f2 Mon Sep 17 00:00:00 2001 From: Roger Dingledine Date: Sat, 11 Jul 2009 16:11:37 -0400 Subject: [PATCH 100/187] remove experimental 'getinfo unregistered-servers-' it never really worked, and hasn't been used for years. --- src/or/control.c | 1 - src/or/dirserv.c | 40 ---------------------------------------- src/or/or.h | 2 -- 3 files changed, 43 deletions(-) diff --git a/src/or/control.c b/src/or/control.c index 662da98ec1..068628afd8 100644 --- a/src/or/control.c +++ b/src/or/control.c @@ -1835,7 +1835,6 @@ static const getinfo_item_t getinfo_items[] = { PREFIX("ns/purpose/", networkstatus, "Brief summary of router status by purpose (v2 directory format)."), - PREFIX("unregistered-servers-", dirserv_unregistered, NULL), ITEM("network-status", dir, "Brief summary of router status (v1 directory format)"), ITEM("circuit-status", events, "List of current circuits originating here."), diff --git a/src/or/dirserv.c b/src/or/dirserv.c index f355fdf03e..acce309641 100644 --- a/src/or/dirserv.c +++ b/src/or/dirserv.c @@ -834,46 +834,6 @@ directory_remove_invalid(void) routerlist_assert_ok(rl); } -/** Write a list of unregistered descriptors into a newly allocated - * string and return it. Used by dirserv operators to keep track of - * fast nodes that haven't registered. - */ -int -getinfo_helper_dirserv_unregistered(control_connection_t *control_conn, - const char *question, char **answer_out) -{ - smartlist_t *answerlist; - char buf[1024]; - char *answer; - int min_bw = atoi(question); - routerlist_t *rl = router_get_routerlist(); - - (void) control_conn; - - if (strcmpstart(question, "unregistered-servers-")) - return 0; - question += strlen("unregistered-servers-"); - - answerlist = smartlist_create(); - SMARTLIST_FOREACH(rl->routers, routerinfo_t *, ent, { - uint32_t r = dirserv_router_get_status(ent, NULL); - if (router_get_advertised_bandwidth(ent) >= (size_t)min_bw && - !(r & FP_NAMED)) { - /* then log this one */ - tor_snprintf(buf, sizeof(buf), - "%s: BW %d on '%s'.", - ent->nickname, router_get_advertised_bandwidth(ent), - ent->platform ? ent->platform : ""); - smartlist_add(answerlist, tor_strdup(buf)); - } - }); - answer = smartlist_join_strings(answerlist, "\r\n", 0, NULL); - SMARTLIST_FOREACH(answerlist, char *, cp, tor_free(cp)); - smartlist_free(answerlist); - *answer_out = answer; - return 0; -} - /** Mark the directory as dirty -- when we're next asked for a * directory, we will rebuild it instead of reusing the most recently * generated one. diff --git a/src/or/or.h b/src/or/or.h index f298d53c27..46daa2056a 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -3448,8 +3448,6 @@ enum was_router_added_t dirserv_add_multiple_descriptors( enum was_router_added_t dirserv_add_descriptor(routerinfo_t *ri, const char **msg, const char *source); -int getinfo_helper_dirserv_unregistered(control_connection_t *conn, - const char *question, char **answer); void dirserv_free_descriptors(void); void dirserv_set_router_is_running(routerinfo_t *router, time_t now); int list_server_status_v1(smartlist_t *routers, char **router_status_out, From e7576f92dec47ffb48f481b7a92b3fc0c76105eb Mon Sep 17 00:00:00 2001 From: Jacob Appelbaum Date: Sat, 11 Jul 2009 18:43:33 -0700 Subject: [PATCH 101/187] Add support for a new option: FetchDirInfoExtraEarly This new option will allow clients to download the newest fresh consensus much sooner than they normally would do so, even if they previously set FetchDirInfoEarly. This includes a proper ChangeLog entry and an updated man page. --- ChangeLog | 6 ++++++ doc/tor.1.in | 7 +++++++ src/or/config.c | 5 +++++ src/or/networkstatus.c | 9 +++++++-- src/or/or.h | 3 +++ 5 files changed, 28 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index c0a1a9e0cc..94d8e8fe1a 100644 --- a/ChangeLog +++ b/ChangeLog @@ -29,6 +29,12 @@ Changes in version 0.2.2.1-alpha - 2009-??-?? connecting clients to disk every 24 hours. To enable this, run configure with the --enable-entry-stats option, and set "EntryStatistics 1" in your torrc. + - Certain Tor clients (such as those behind check.torproject.org) may + want to fetch the consensus in an extra early manner. To enable this + a user may now set FetchDirInfoExtraEarly to 1. This also depends on + setting FetchDirInfoEarly to 1. Previous behavior will stay the same + as only certain clients who must have this information sooner should + set this option. o Minor bugfixes - Hidden service clients didn't use a cached service descriptor that diff --git a/doc/tor.1.in b/doc/tor.1.in index 3ac0f92fe2..b6e22310f2 100644 --- a/doc/tor.1.in +++ b/doc/tor.1.in @@ -241,6 +241,13 @@ fetching early. Normal users should leave it off. (Default: 0) .LP .TP +\fBFetchDirInfoExtraEarly \fR\fB0\fR|\fB1\fR\fP +If set to 1, Tor will fetch directory information before other +directory caches. It will attempt to download directory information closer to +the start of the consensus period. Normal users should leave it off. +(Default: 0) +.LP +.TP \fBFetchHidServDescriptors \fR\fB0\fR|\fB1\fR\fP If set to 0, Tor will never fetch any hidden service descriptors from the rendezvous directories. This option is only useful if you're using diff --git a/src/or/config.c b/src/or/config.c index 5137f74792..5495b252cb 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -214,6 +214,7 @@ static config_var_t _option_vars[] = { V(FirewallPorts, CSV, ""), V(FastFirstHopPK, BOOL, "1"), V(FetchDirInfoEarly, BOOL, "0"), + V(FetchDirInfoExtraEarly, BOOL, "0"), V(FetchServerDescriptors, BOOL, "1"), V(FetchHidServDescriptors, BOOL, "1"), V(FetchUselessDescriptors, BOOL, "0"), @@ -3181,6 +3182,10 @@ options_validate(or_options_t *old_options, or_options_t *options, REJECT("HSAuthorityRecordStats is set but we're not running as " "a hidden service authority."); + if (options->FetchDirInfoExtraEarly && !options->FetchDirInfoEarly) + REJECT("FetchDirInfoExtraEarly requires that you also set " + "FetchDirInfoEarly"); + if (options->ConnLimit <= 0) { r = tor_snprintf(buf, sizeof(buf), "ConnLimit must be greater than 0, but was set to %d", diff --git a/src/or/networkstatus.c b/src/or/networkstatus.c index 573197a53f..df17f49548 100644 --- a/src/or/networkstatus.c +++ b/src/or/networkstatus.c @@ -1133,8 +1133,13 @@ update_consensus_networkstatus_fetch_time(time_t now) /* We want to cache the next one at some point after this one * is no longer fresh... */ start = c->fresh_until + CONSENSUS_MIN_SECONDS_BEFORE_CACHING; - /* But only in the first half-interval after that. */ - dl_interval = interval/2; + /* Some clients may need the consensus sooner than others. */ + if (options->FetchDirInfoExtraEarly) { + dl_interval = 60; + } else { + /* But only in the first half-interval after that. */ + dl_interval = interval/2; + } } else { /* We're an ordinary client or a bridge. Give all the caches enough * time to download the consensus. */ diff --git a/src/or/or.h b/src/or/or.h index f298d53c27..e38cf81cfd 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -2444,6 +2444,9 @@ typedef struct { * means directly from the authorities) no matter our other config? */ int FetchDirInfoEarly; + /** Should we fetch our dir info at the start of the consensus period? */ + int FetchDirInfoExtraEarly; + char *VirtualAddrNetwork; /**< Address and mask to hand out for virtual * MAPADDRESS requests. */ int ServerDNSSearchDomains; /**< Boolean: If set, we don't force exit From 59dd9de8581c0cf0a0bd572f8d67b1be2b2a4274 Mon Sep 17 00:00:00 2001 From: Karsten Loesing Date: Fri, 10 Jul 2009 13:37:25 +0200 Subject: [PATCH 102/187] Write number of rejected requests to geoip-stats file. --- src/or/directory.c | 10 +++++-- src/or/geoip.c | 65 ++++++++++++++++++++++++++++++++++++++++++++++ src/or/or.h | 20 ++++++++++++++ 3 files changed, 93 insertions(+), 2 deletions(-) diff --git a/src/or/directory.c b/src/or/directory.c index daf2a15762..aa225d0c43 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -2493,6 +2493,8 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, /* v2 or v3 network status fetch. */ smartlist_t *dir_fps = smartlist_create(); int is_v3 = !strcmpstart(url, "/tor/status-vote"); + geoip_client_action_t act = + is_v3 ? GEOIP_CLIENT_NETWORKSTATUS : GEOIP_CLIENT_NETWORKSTATUS_V2; const char *request_type = NULL; const char *key = url + strlen("/tor/status/"); long lifetime = NETWORKSTATUS_CACHE_LIFETIME; @@ -2517,6 +2519,7 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, write_http_status_line(conn, 404, "Consensus not signed by sufficient " "number of requested authorities"); smartlist_free(dir_fps); + geoip_note_ns_response(act, GEOIP_REJECT_NOT_ENOUGH_SIGS); goto done; } @@ -2529,6 +2532,7 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, if (!smartlist_len(dir_fps)) { /* we failed to create/cache cp */ write_http_status_line(conn, 503, "Network status object unavailable"); smartlist_free(dir_fps); + geoip_note_ns_response(act, GEOIP_REJECT_UNAVAILABLE); goto done; } @@ -2536,11 +2540,13 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, write_http_status_line(conn, 404, "Not found"); SMARTLIST_FOREACH(dir_fps, char *, cp, tor_free(cp)); smartlist_free(dir_fps); + geoip_note_ns_response(act, GEOIP_REJECT_NOT_FOUND); goto done; } else if (!smartlist_len(dir_fps)) { write_http_status_line(conn, 304, "Not modified"); SMARTLIST_FOREACH(dir_fps, char *, cp, tor_free(cp)); smartlist_free(dir_fps); + geoip_note_ns_response(act, GEOIP_REJECT_NOT_MODIFIED); goto done; } @@ -2552,16 +2558,16 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, write_http_status_line(conn, 503, "Directory busy, try again later"); SMARTLIST_FOREACH(dir_fps, char *, fp, tor_free(fp)); smartlist_free(dir_fps); + geoip_note_ns_response(act, GEOIP_REJECT_BUSY); goto done; } #ifdef ENABLE_GEOIP_STATS { - geoip_client_action_t act = - is_v3 ? GEOIP_CLIENT_NETWORKSTATUS : GEOIP_CLIENT_NETWORKSTATUS_V2; struct in_addr in; if (tor_inet_aton((TO_CONN(conn))->address, &in)) geoip_note_client_seen(act, ntohl(in.s_addr), time(NULL)); + geoip_note_ns_response(act, GEOIP_SUCCESS); } #endif diff --git a/src/or/geoip.c b/src/or/geoip.c index aac4893117..0a0fc08f84 100644 --- a/src/or/geoip.c +++ b/src/or/geoip.c @@ -418,6 +418,43 @@ geoip_remove_old_clients(time_t cutoff) client_history_starts = cutoff; } +#ifdef ENABLE_GEOIP_STATS +/** How many responses are we giving to clients requesting v2 network + * statuses? */ +static uint32_t ns_v2_responses[GEOIP_NS_RESPONSE_NUM]; + +/** How many responses are we giving to clients requesting v3 network + * statuses? */ +static uint32_t ns_v3_responses[GEOIP_NS_RESPONSE_NUM]; +#endif + +/** Note that we've rejected a client's request for a v2 or v3 network + * status, encoded in action for reason reason at time + * now. */ +void +geoip_note_ns_response(geoip_client_action_t action, + geoip_ns_response_t response) +{ +#ifdef ENABLE_GEOIP_STATS + static int arrays_initialized = 0; + if (!arrays_initialized) { + memset(ns_v2_responses, 0, sizeof(ns_v2_responses)); + memset(ns_v3_responses, 0, sizeof(ns_v3_responses)); + arrays_initialized = 1; + } + tor_assert(action == GEOIP_CLIENT_NETWORKSTATUS || + action == GEOIP_CLIENT_NETWORKSTATUS_V2); + tor_assert(response < GEOIP_NS_RESPONSE_NUM); + if (action == GEOIP_CLIENT_NETWORKSTATUS) + ns_v3_responses[response]++; + else + ns_v2_responses[response]++; +#else + (void) action; + (void) response; +#endif +} + /** Do not mention any country from which fewer than this number of IPs have * connected. This conceivably avoids reporting information that could * deanonymize users, though analysis is lacking. */ @@ -612,6 +649,7 @@ dump_geoip_stats(void) open_file_t *open_file = NULL; double v2_share = 0.0, v3_share = 0.0; FILE *out; + int i; data_v2 = geoip_get_client_history(now, GEOIP_CLIENT_NETWORKSTATUS_V2); data_v3 = geoip_get_client_history(now, GEOIP_CLIENT_NETWORKSTATUS); @@ -637,6 +675,33 @@ dump_geoip_stats(void) since, data_v3 ? data_v3 : "", data_v2 ? data_v2 : "") < 0) goto done; +#define RESPONSE_GRANULARITY 8 + for (i = 0; i < GEOIP_NS_RESPONSE_NUM; i++) { + ns_v2_responses[i] = round_to_next_multiple_of(ns_v2_responses[i], + RESPONSE_GRANULARITY); + ns_v3_responses[i] = round_to_next_multiple_of(ns_v3_responses[i], + RESPONSE_GRANULARITY); + } +#undef RESPONSE_GRANULARITY + if (fprintf(out, "n-ns-resp ok=%d,not-enough-sigs=%d,unavailable=%d," + "not-found=%d,not-modified=%d,busy=%d\n", + ns_v3_responses[GEOIP_SUCCESS], + ns_v3_responses[GEOIP_REJECT_NOT_ENOUGH_SIGS], + ns_v3_responses[GEOIP_REJECT_UNAVAILABLE], + ns_v3_responses[GEOIP_REJECT_NOT_FOUND], + ns_v3_responses[GEOIP_REJECT_NOT_MODIFIED], + ns_v3_responses[GEOIP_REJECT_BUSY]) < 0) + goto done; + if (fprintf(out, "n-v2-ns-resp ok=%d,unavailable=%d," + "not-found=%d,not-modified=%d,busy=%d\n", + ns_v2_responses[GEOIP_SUCCESS], + ns_v2_responses[GEOIP_REJECT_UNAVAILABLE], + ns_v2_responses[GEOIP_REJECT_NOT_FOUND], + ns_v2_responses[GEOIP_REJECT_NOT_MODIFIED], + ns_v2_responses[GEOIP_REJECT_BUSY]) < 0) + goto done; + memset(ns_v2_responses, 0, sizeof(ns_v2_responses)); + memset(ns_v3_responses, 0, sizeof(ns_v3_responses)); if (!router_get_my_share_of_directory_requests(&v2_share, &v3_share)) { if (fprintf(out, "v2-ns-share %0.2lf%%\n", v2_share*100) < 0) goto done; diff --git a/src/or/or.h b/src/or/or.h index f298d53c27..a6aca94f82 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -3647,6 +3647,26 @@ typedef enum { void geoip_note_client_seen(geoip_client_action_t action, uint32_t addr, time_t now); void geoip_remove_old_clients(time_t cutoff); +/** Indicates either a positive reply or a reason for rejectng a network + * status request that will be included in geoip statistics. */ +typedef enum { + /** Request is answered successfully. */ + GEOIP_SUCCESS = 0, + /** V3 network status is not signed by a sufficient number of requested + * authorities. */ + GEOIP_REJECT_NOT_ENOUGH_SIGS = 1, + /** Requested network status object is unavailable. */ + GEOIP_REJECT_UNAVAILABLE = 2, + /** Requested network status not found. */ + GEOIP_REJECT_NOT_FOUND = 3, + /** Network status has not been modified since If-Modified-Since time. */ + GEOIP_REJECT_NOT_MODIFIED = 4, + /** Directory is busy. */ + GEOIP_REJECT_BUSY = 5, +} geoip_ns_response_t; +#define GEOIP_NS_RESPONSE_NUM 6 +void geoip_note_ns_response(geoip_client_action_t action, + geoip_ns_response_t response); time_t geoip_get_history_start(void); char *geoip_get_client_history(time_t now, geoip_client_action_t action); char *geoip_get_request_history(time_t now, geoip_client_action_t action); From 409386cededa50f17908557a2501580958ac75df Mon Sep 17 00:00:00 2001 From: Karsten Loesing Date: Tue, 14 Jul 2009 01:50:55 +0200 Subject: [PATCH 103/187] Handle unsigned ints correctly. --- src/or/geoip.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/or/geoip.c b/src/or/geoip.c index 0a0fc08f84..a182e213a5 100644 --- a/src/or/geoip.c +++ b/src/or/geoip.c @@ -677,14 +677,14 @@ dump_geoip_stats(void) goto done; #define RESPONSE_GRANULARITY 8 for (i = 0; i < GEOIP_NS_RESPONSE_NUM; i++) { - ns_v2_responses[i] = round_to_next_multiple_of(ns_v2_responses[i], - RESPONSE_GRANULARITY); - ns_v3_responses[i] = round_to_next_multiple_of(ns_v3_responses[i], - RESPONSE_GRANULARITY); + ns_v2_responses[i] = round_uint32_to_next_multiple_of( + ns_v2_responses[i], RESPONSE_GRANULARITY); + ns_v3_responses[i] = round_uint32_to_next_multiple_of( + ns_v3_responses[i], RESPONSE_GRANULARITY); } #undef RESPONSE_GRANULARITY - if (fprintf(out, "n-ns-resp ok=%d,not-enough-sigs=%d,unavailable=%d," - "not-found=%d,not-modified=%d,busy=%d\n", + if (fprintf(out, "n-ns-resp ok=%u,not-enough-sigs=%u,unavailable=%u," + "not-found=%u,not-modified=%u,busy=%u\n", ns_v3_responses[GEOIP_SUCCESS], ns_v3_responses[GEOIP_REJECT_NOT_ENOUGH_SIGS], ns_v3_responses[GEOIP_REJECT_UNAVAILABLE], @@ -692,8 +692,8 @@ dump_geoip_stats(void) ns_v3_responses[GEOIP_REJECT_NOT_MODIFIED], ns_v3_responses[GEOIP_REJECT_BUSY]) < 0) goto done; - if (fprintf(out, "n-v2-ns-resp ok=%d,unavailable=%d," - "not-found=%d,not-modified=%d,busy=%d\n", + if (fprintf(out, "n-v2-ns-resp ok=%u,unavailable=%u," + "not-found=%u,not-modified=%u,busy=%u\n", ns_v2_responses[GEOIP_SUCCESS], ns_v2_responses[GEOIP_REJECT_UNAVAILABLE], ns_v2_responses[GEOIP_REJECT_NOT_FOUND], From 05975e2a89de4d2be9b4c011c0f78f53095077fe Mon Sep 17 00:00:00 2001 From: Roger Dingledine Date: Tue, 14 Jul 2009 14:13:04 -0400 Subject: [PATCH 104/187] make it compile without warnings --- src/or/geoip.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/or/geoip.c b/src/or/geoip.c index 0aa5fd873b..14b5f66263 100644 --- a/src/or/geoip.c +++ b/src/or/geoip.c @@ -347,6 +347,7 @@ geoip_determine_shares(time_t now) last_time_determined_shares = now; } +#ifdef ENABLE_GEOIP_STATS /** Calculate which fraction of v2 and v3 directory requests aimed at caches * have been sent to us since the last call of this function up to time * now. Set *v2_share_out and *v3_share_out to the @@ -366,6 +367,7 @@ geoip_get_mean_shares(time_t now, double *v2_share_out, share_seconds = 0; return 0; } +#endif /** Note that we've seen a client connect from the IP addr (host order) * at time now. Ignored by all but bridges and directories if From 72c5a46b43c6d89c773a99240a95301b91b1f269 Mon Sep 17 00:00:00 2001 From: Roger Dingledine Date: Tue, 14 Jul 2009 23:35:34 -0400 Subject: [PATCH 105/187] move the todo.external file into svn projects --- doc/TODO | 4 +- doc/TODO.external | 188 +--------------------------------------------- 2 files changed, 4 insertions(+), 188 deletions(-) diff --git a/doc/TODO b/doc/TODO index fd023c8bf5..194d6507bc 100644 --- a/doc/TODO +++ b/doc/TODO @@ -4,8 +4,8 @@ We've split out our TODO into three files: TODO.02x is the list of items we're planning to get done in the next stable release. -TODO.external is the list of external constraints and deliverables that -we all need to keep in mind. +TODO.external lives in svn under /projects/todo/. It's the list of +external constraints and deliverables that we all need to keep in mind. TODO.future is the list of other items we plan to get to in later releases. diff --git a/doc/TODO.external b/doc/TODO.external index 12048fa344..2e7e536efc 100644 --- a/doc/TODO.external +++ b/doc/TODO.external @@ -1,188 +1,4 @@ -Legend: -SPEC!! - Not specified -SPEC - Spec not finalized -N - nick claims -R - arma claims -P - phobos claims -S - Steven claims -E - Matt claims -M - Mike claims -J - Jeff claims -I - ioerror claims -W - weasel claims -K - Karsten claims -C - coderman claims - - Not done - * Top priority - . Partially done - o Done - d Deferrable - D Deferred - X Abandoned -======================================================================= - -External constraints: - -For June/July: -NR - Work more on Paul's NRL research problem. - -For March 22: -I * Email auto-responder - * teach gettor how to ask for (and attach) split files. - -K . Metrics. - . With Mike's help, use Torflow to start doing monthly rudimentary - performance evaluations: - . Circuit throughput and latency - - Measure via Broadband and dialup - . Publish a report addressing key long-term metrics questions: - . What metrics should we present? - . What data are available for these metrics? - . What data are missing, and can collect them safely? Can we - publish them safely? - . What systems are available to present this data? - -E . Vidalia improvements - o Vidalia displays by-country user summary for bridge operators -? - write a help page for vidalia, "what is this" - -For mid August: - -Section 0, items that didn't make it into the original roadmap: - -0.1, installers and packaging -C . i18n for the msi bundle files -P . more consistent TBB builds -IC- get a buildbot up again. Have Linux and BSD build machines. - (Windows would be nice but realistically will come later.) -E - Get Tor to work properly on the iPhone. - -3.1, performance work. [Section numbers in here are from performance.pdf] - - High-priority items from performance.pdf -RS - 1.2, new circuit window sizes. make the default package window lower. -R+ - 2.1, squeeze loud circuits - - Evaluate the code to see what stats we can keep about circuit use. - - Write proposals for various meddling. Look at the research papers - that Juliusz pointed us to. Ask our systems friends. Plan to put - a lot of the parameters in the consensus, so we can tune it with - short turnaround times. -E+ - 2.5, Change Vidalia's default exit policy to not click "other - protocols". Or choose not to. Think this through first. -R+ - 2.6, Tell users not to file-share. - - Put statement on the Tor front page - - Put statement on the download pages too - - And the FAQ - - 3.1.2, Tor weather -I - Implement time-to-notification (immediate, a day, a week) -I - Get a relay operator mailing list going, with a plan and supporting - scripts and so on. -R - Link to them from the Tor relay page -R - and the torrc.sample? -SM - 4.1, balance traffic better - - Steven and Mike should decide if we should do Steven's plan - (rejigger the bandwidth numbers at the authorities based on - Steven's algorithm), or Mike's plan (relay scanning to identify - the unbalanced relays and fix them on the fly), or both. - - Implement Proposal 160 - o 4.5, Older entry guards are overloaded - o Pick a conservative timeout like a month, and implement. -M - 5.2, better timeouts for giving up on circuits/streams - - clients gather data about circuit timeouts, and then abandon - circuits that take more than a std dev above that. - -4.1, IOCP / libevent / windows / tor -N - get it working for nick -N - put out a release so other people can start testing it. -N - both the libevent buffer abstraction, and the - tor-uses-libevent-buffer-abstraction. Unless we think that's - unreachable for this milestone? - -4.2.1, risks from becoming a relay -S - Have a clear plan for how users who become relays will be safe, - and be confident that we can build this plan. - - evaluate all the various attacks that are made possible by relaying. - specifically, see "relaying-traffic attacks" in 6.6. - - identify and evaluate ways to make them not a big deal - - setting a low RelayBandwidth - - Nick Hopper's FC08 paper suggesting that we should do a modified - round-robin so we leak less about other circuits - - instructing clients to disable pings in their firewall, etc - - pick the promising ones, improve them so they're even better, and - spec them out so we know how to build them and how much effort is - involved in building them. - -4.5, clients download less directory info -N * deploy proposal 158. -N - decide whether to do proposal 140. if so, construct an implementation - plan for how we'll do it. if not, explain why not. - -5.1, Normalize TLS fingerprint -N o write a draft list of possible attacks for this section, with - estimates about difficulty of attack, difficulty of solution, etc -N - revisit the list and revise our plans as needed -NR- put up a blog post about the two contradictory conclusions: we can - discuss the theory of arms races, and our quandry, without revealing - any specific vulnerabilities. (or decide not to put up a blog post, - and explain why not.) - -5.5, email autoresponder -I . maintenance and keeping it running - -5.7.2, metrics - -XXX. - -6.2, Vidalia work -E - add breakpad support or similar for windows debugging -E o let vidalia change languages without needing a restart -E - Implement the status warning event interface started for the - phase one deliverables. -E - Work with Steve Tyree on building a Vidalia plugin API to enable - building Herdict and TBB plugins. - -6.3, Node scanning -M - Steps toward automation - - Set up email list for results - - Map failure types to potential BadExit lines -M - Improve the ability of SoaT to mimic various real web browsers - - randomizing user agents and locale strings - - caching, XMLHTTPRequest, form posting, content sniffing - - Investigate ideas like running Chrome/xulrunner in parallel -M - Other protocols - - SSH, IMAPS, POPS, SMTPS -M - Add ability to geolocalize exit selection based on scanner location - - Use this to rescan dynamic urls filtered by the URL filter - -6.4, Torbutton development -M - Resolve extension conflicts and other high priority bugs -M - Fix or hack around ugly firefox bugs, especially Timezone issue. - Definitely leaning towards "hack around" unless we see some - level of love from Mozilla. -M - Vidalia New Nym Integration - - Implement for Torbutton to pick up on Vidalia's NEWNYM and clear - cookies based on FoeBud's source - - Do this in such a way that we could adapt polipo to purge cache - if we were so inclined -M - Write up a summary of our options for dealing with the google - you-must-solve-a-captcha-to-search problem, and pick one as our - favorite option. - -6.6, Evaluate new anonymity attacks -S - relaying-traffic attacks - - original murdoch-danezis attack - - nick hopper's latency measurement attack - - columbia bandwidth measurement attack - - christian grothoff's long-circuit attack -S - client attacks - - website fingerprinting - -7.1, Tor VM Research, analysis, and prototyping -C . Get a working package out, meaning other people are testing it. - -7.2, Tor Browser Bundle -I - Port to one of OS X or Linux, and start the port to the other. -I . Make it the recommended Tor download on Windows -I - Make sure it's easy to un-brand TBB in case Firefox asks us to -I - Evaluate CCC's Freedom Stick +[This file moved to svn in /projects/todo/. More people can edit +it more easily there. -RD] From d4b31cf98f85550155156726ad9f3424def07fb7 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 15 Jul 2009 10:02:49 -0400 Subject: [PATCH 106/187] Allow interval and memunit cfg variables to be set to fractions. --- ChangeLog | 3 +++ src/or/config.c | 55 +++++++++++++++++++++++++++++++++++-------------- 2 files changed, 43 insertions(+), 15 deletions(-) diff --git a/ChangeLog b/ChangeLog index 30a776c29e..0ecf3d74b3 100644 --- a/ChangeLog +++ b/ChangeLog @@ -32,6 +32,9 @@ Changes in version 0.2.2.1-alpha - 2009-??-?? connecting clients to disk every 24 hours. To enable this, run configure with the --enable-entry-stats option, and set "EntryStatistics 1" in your torrc. + - Time and memory units in the configuration file can now be set to + fractional units. For example, "2.5 MB" is now a valid value for + AccountingMax. o Minor bugfixes - Hidden service clients didn't use a cached service descriptor that diff --git a/src/or/config.c b/src/or/config.c index 087a907e48..19212fe899 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -4791,30 +4791,55 @@ static struct unit_table_t time_units[] = { static uint64_t config_parse_units(const char *val, struct unit_table_t *u, int *ok) { - uint64_t v; - char *cp; + uint64_t v = 0; + double d = 0; + int use_float = 0; + + smartlist_t *sl; tor_assert(ok); + sl = smartlist_create(); + smartlist_split_string(sl, val, NULL, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 3); - v = tor_parse_uint64(val, 10, 0, UINT64_MAX, ok, &cp); - if (!*ok) - return 0; - if (!cp) { - *ok = 1; - return v; + if (smartlist_len(sl) < 1 || smartlist_len(sl) > 2) { + *ok = 0; + goto done; } - while (TOR_ISSPACE(*cp)) - ++cp; + + v = tor_parse_uint64(smartlist_get(sl,0), 10, 0, UINT64_MAX, ok, NULL); + if (!*ok) { + int r = sscanf(smartlist_get(sl,0), "%lf", &d); + if (r == 0 || d < 0) + goto done; + use_float = 1; + } + + if (smartlist_len(sl) == 1) { + *ok = 1; + v = use_float ? DBL_TO_U64(d) : v; + goto done; + } + for ( ;u->unit;++u) { - if (!strcasecmp(u->unit, cp)) { - v *= u->multiplier; + if (!strcasecmp(u->unit, smartlist_get(sl,1))) { + if (use_float) + v = u->multiplier * d; + else + v *= u->multiplier; *ok = 1; - return v; + goto done; } } - log_warn(LD_CONFIG, "Unknown unit '%s'.", cp); + log_warn(LD_CONFIG, "Unknown unit '%s'.", (char*)smartlist_get(sl,1)); *ok = 0; - return 0; + done: + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_free(sl); + + if (*ok) + return v; + else + return 0; } /** Parse a string in the format "number unit", where unit is a unit of From 3c051325758c0aeeeb44054715e16d8b8717948c Mon Sep 17 00:00:00 2001 From: Karsten Loesing Date: Sun, 12 Jul 2009 16:33:31 +0200 Subject: [PATCH 107/187] Directories now also measure download times of network statuses. --- ChangeLog | 1 + src/or/connection.c | 7 ++ src/or/connection_edge.c | 10 ++ src/or/directory.c | 22 +++- src/or/geoip.c | 215 +++++++++++++++++++++++++++++++++++++++ src/or/or.h | 44 ++++++++ src/or/relay.c | 28 +++++ 7 files changed, 326 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 30a776c29e..6b4f651828 100644 --- a/ChangeLog +++ b/ChangeLog @@ -20,6 +20,7 @@ Changes in version 0.2.2.1-alpha - 2009-??-?? Estimated shares of v2 and v3 requests are determined as averages, not at the end of a measurement period. Also, unresolved requests are listed with country code '??'. + Directories now also measure download times of network statuses. - Exit nodes can write statistics on the number of exit streams and transferred bytes per port to disk every 24 hours. To enable this, run configure with the --enable-exit-stats option, and set diff --git a/src/or/connection.c b/src/or/connection.c index 0897eb4a5e..309421051a 100644 --- a/src/or/connection.c +++ b/src/or/connection.c @@ -2302,6 +2302,13 @@ connection_handle_write(connection_t *conn, int force) /* else open, or closing */ result = flush_buf_tls(or_conn->tls, conn->outbuf, max_to_write, &conn->outbuf_flushlen); +#ifdef ENABLE_GEOIP_STATS + /* If we just flushed the last bytes, check if this tunneled dir + * request is done. */ + if (buf_datalen(conn->outbuf) == 0 && conn->request_id) + geoip_change_dirreq_state(conn->request_id, REQUEST_TUNNELED, + OR_CONN_BUFFER_FLUSHED); +#endif switch (result) { CASE_TOR_TLS_ERROR_ANY: case TOR_TLS_CLOSE: diff --git a/src/or/connection_edge.c b/src/or/connection_edge.c index 560a2433d7..f32563bcdb 100644 --- a/src/or/connection_edge.c +++ b/src/or/connection_edge.c @@ -2551,6 +2551,11 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ) log_debug(LD_EXIT,"Creating new exit connection."); n_stream = edge_connection_new(CONN_TYPE_EXIT, AF_INET); +#ifdef ENABLE_GEOIP_STATS + /* Remember the tunneled request ID in the new edge connection, so that + * we can measure download times. */ + TO_CONN(n_stream)->request_id = circ->request_id; +#endif n_stream->_base.purpose = EXIT_PURPOSE_CONNECT; n_stream->stream_id = rh.stream_id; @@ -2787,6 +2792,11 @@ connection_exit_connect_dir(edge_connection_t *exitconn) dirconn->_base.purpose = DIR_PURPOSE_SERVER; dirconn->_base.state = DIR_CONN_STATE_SERVER_COMMAND_WAIT; +#ifdef ENABLE_GEOIP_STATS + /* Note that the new dir conn belongs to the same tunneled request as + * the edge conn, so that we can measure download times. */ + TO_CONN(dirconn)->request_id = TO_CONN(exitconn)->request_id; +#endif connection_link_connections(TO_CONN(dirconn), TO_CONN(exitconn)); if (connection_add(TO_CONN(exitconn))<0) { diff --git a/src/or/directory.c b/src/or/directory.c index cf9f5543d7..e5230c2cbd 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -2565,9 +2565,18 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, #ifdef ENABLE_GEOIP_STATS { struct in_addr in; - if (tor_inet_aton((TO_CONN(conn))->address, &in)) + if (tor_inet_aton((TO_CONN(conn))->address, &in)) { geoip_note_client_seen(act, ntohl(in.s_addr), time(NULL)); geoip_note_ns_response(act, GEOIP_SUCCESS); + /* Note that a request for a network status has started, so that we + * can measure the download time later on. */ + if (TO_CONN(conn)->request_id) + geoip_start_dirreq(TO_CONN(conn)->request_id, dlen, act, + REQUEST_TUNNELED); + else + geoip_start_dirreq(TO_CONN(conn)->global_identifier, dlen, act, + REQUEST_DIRECT); + } } #endif @@ -3201,6 +3210,17 @@ connection_dir_finished_flushing(dir_connection_t *conn) tor_assert(conn); tor_assert(conn->_base.type == CONN_TYPE_DIR); +#ifdef ENABLE_GEOIP_STATS + /* Note that we have finished writing the directory response. For direct + * connections this means we're done, for tunneled connections its only + * an intermediate step. */ + if (TO_CONN(conn)->request_id) + geoip_change_dirreq_state(TO_CONN(conn)->request_id, REQUEST_TUNNELED, + FLUSHING_DIR_CONN_FINISHED); + else + geoip_change_dirreq_state(TO_CONN(conn)->global_identifier, + REQUEST_DIRECT, FLUSHING_DIR_CONN_FINISHED); +#endif switch (conn->_base.state) { case DIR_CONN_STATE_CLIENT_SENDING: log_debug(LD_DIR,"client finished sending command."); diff --git a/src/or/geoip.c b/src/or/geoip.c index 14b5f66263..a7e46c0b8f 100644 --- a/src/or/geoip.c +++ b/src/or/geoip.c @@ -570,6 +570,204 @@ _c_hist_compare(const void **_a, const void **_b) return strcmp(a->country, b->country); } +/** When there are incomplete directory requests at the end of a 24-hour + * period, consider those requests running for longer than this timeout as + * failed, the others as still running. */ +#define DIRREQ_TIMEOUT (10*60) + +/** Entry in a map from either conn->global_identifier for direct requests + * or a unique circuit identifier for tunneled requests to request time, + * response size, and completion time of a network status request. Used to + * measure download times of requests to derive average client + * bandwidths. */ +typedef struct dirreqdlmap_entry_t { + directory_request_state_t state; /**< State of this directory request. */ + /** Unique identifier for this network status request; this is either the + * conn->global_identifier of the dir conn (direct request) or a new + * locally unique identifier of a circuit (tunneled request). This ID is + * only unique among other direct or tunneled requests, respectively. */ + uint64_t request_id; + /** Is this a direct or a tunneled request? */ + directory_request_type_t type; + int completed:1; /**< Is this request complete? */ + geoip_client_action_t action; /**< Is this a v2 or v3 request? */ + /** When did we receive the request and started sending the response? */ + struct timeval request_time; + size_t response_size; /**< What is the size of the response in bytes? */ + struct timeval completion_time; /**< When did the request succeed? */ +} dirreqdlmap_entry_t; + +/** Map of all directory requests asking for v2 or v3 network statuses in + * the current geoip-stats interval. Keys are strings starting with either + * "dir" for direct requests or "tun" for tunneled requests, followed by + * a unique uint64_t identifier represented as decimal string. Values are + * of type *dirreqdlmap_entry_t. */ +static strmap_t *dirreqdlmap = NULL; + +/** Helper: Put entry into map of directory requests using + * tunneled and request_id as key parts. If there is + * already an entry for that key, print out a BUG warning and return. */ +static void +_dirreqdlmap_put(dirreqdlmap_entry_t *entry, + directory_request_type_t type, uint64_t request_id) +{ + char key[3+20+1]; /* dir|tun + -9223372036854775808 + \0 */ + dirreqdlmap_entry_t *ent; + if (!dirreqdlmap) + dirreqdlmap = strmap_new(); + tor_snprintf(key, sizeof(key), "%s"U64_FORMAT, + type == REQUEST_TUNNELED ? "tun" : "dir", + U64_PRINTF_ARG(request_id)); + ent = strmap_get(dirreqdlmap, key); + if (ent) { + log_warn(LD_BUG, "Error when putting directory request into local " + "map. There is already an entry for the same identifier."); + return; + } + strmap_set(dirreqdlmap, key, entry); +} + +/** Helper: Look up and return an entry in the map of directory requests + * using tunneled and request_id as key parts. If there + * is no such entry, return NULL. */ +static dirreqdlmap_entry_t * +_dirreqdlmap_get(directory_request_type_t type, uint64_t request_id) +{ + char key[3+20+1]; /* dir|tun + -9223372036854775808 + \0 */ + if (!dirreqdlmap) + dirreqdlmap = strmap_new(); + tor_snprintf(key, sizeof(key), "%s"U64_FORMAT, + type == REQUEST_TUNNELED ? "tun" : "dir", + U64_PRINTF_ARG(request_id)); + return strmap_get(dirreqdlmap, key); +} + +/** Note that an either direct or tunneled (see type) directory + * request for a network status with unique ID request_id of size + * response_size and action action (either v2 or v3) has + * started. */ +void +geoip_start_dirreq(uint64_t request_id, size_t response_size, + geoip_client_action_t action, + directory_request_type_t type) +{ + dirreqdlmap_entry_t *ent = tor_malloc_zero(sizeof(dirreqdlmap_entry_t)); + ent->request_id = request_id; + tor_gettimeofday(&ent->request_time); + ent->response_size = response_size; + ent->action = action; + ent->type = type; + _dirreqdlmap_put(ent, type, request_id); +} + +/** Change the state of the either direct or tunneled (see type) + * directory request with request_id to new_state and + * possibly mark it as completed. If no entry can be found for the given + * key parts (e.g., if this is a directory request that we are not + * measuring, or one that was started in the previous measurement period), + * or if the state cannot be advanced to new_state, do nothing. */ +void +geoip_change_dirreq_state(uint64_t request_id, + directory_request_type_t type, + directory_request_state_t new_state) +{ + dirreqdlmap_entry_t *ent = _dirreqdlmap_get(type, request_id); + if (!ent) + return; + if (new_state == REQUEST_IS_FOR_NETWORK_STATUS) + return; + if (new_state - 1 != ent->state) + return; + ent->state = new_state; + if ((type == REQUEST_DIRECT && new_state == FLUSHING_DIR_CONN_FINISHED) || + (type == REQUEST_TUNNELED && new_state == OR_CONN_BUFFER_FLUSHED)) { + tor_gettimeofday(&ent->completion_time); + ent->completed = 1; + } +} + +#ifdef ENABLE_GEOIP_STATS +/** Return a newly allocated comma-separated string containing statistics + * on network status downloads. The string contains the number of completed + * requests, timeouts, and still running requests as well as the download + * times by deciles and quartiles. Return NULL if we have not observed + * requests for long enough. */ +static char * +geoip_get_dirreqdl_history(geoip_client_action_t action, + directory_request_type_t type) +{ + char *result = NULL; + smartlist_t *dirreqdl_times = NULL; + uint32_t complete = 0, timeouts = 0, running = 0; + int i = 0, bufsize = 1024, written; + struct timeval now; + tor_gettimeofday(&now); + if (!dirreqdlmap) + return NULL; + if (action != GEOIP_CLIENT_NETWORKSTATUS && + action != GEOIP_CLIENT_NETWORKSTATUS_V2) + return NULL; + dirreqdl_times = smartlist_create(); + STRMAP_FOREACH_MODIFY(dirreqdlmap, key, dirreqdlmap_entry_t *, ent) { + if (ent->action == action && type == ent->type) { + if (ent->completed) { + uint32_t *bytes_per_second = tor_malloc_zero(sizeof(uint32_t)); + uint32_t time_diff = (uint32_t) tv_udiff(&ent->request_time, + &ent->completion_time); + if (time_diff == 0) + time_diff = 1; /* Avoid DIV/0; "instant" answers are impossible + * anyway by law of nature or something.. */ + *bytes_per_second = 1000000 * ent->response_size / time_diff; + smartlist_add(dirreqdl_times, bytes_per_second); + complete++; + } else { + if (tv_udiff(&ent->request_time, &now) / 1000000 > DIRREQ_TIMEOUT) + timeouts++; + else + running++; + } + tor_free(ent); + MAP_DEL_CURRENT(key); + } + } STRMAP_FOREACH_END; + result = tor_malloc_zero(bufsize); + written = tor_snprintf(result, bufsize, "complete=%d,timeout=%d," + "running=%d", complete, timeouts, running); + if (written < 0) + return NULL; +#define MIN_DIR_REQ_RESPONSES 16 + if (complete >= MIN_DIR_REQ_RESPONSES) { + uint32_t *dltimes = tor_malloc(sizeof(uint32_t) * complete); + SMARTLIST_FOREACH(dirreqdl_times, uint32_t *, dlt, { + dltimes[i++] = *dlt; + tor_free(dlt); + }); + median_uint32(dltimes, complete); /* sort */ + written = tor_snprintf(result + written, bufsize - written, + ",min=%d,d1=%d,d2=%d,q1=%d,d3=%d,d4=%d,md=%d," + "d6=%d,d7=%d,q3=%d,d8=%d,d9=%d,max=%d", + dltimes[0], + dltimes[1*complete/10-1], + dltimes[2*complete/10-1], + dltimes[1*complete/4-1], + dltimes[3*complete/10-1], + dltimes[4*complete/10-1], + dltimes[5*complete/10-1], + dltimes[6*complete/10-1], + dltimes[7*complete/10-1], + dltimes[3*complete/4-1], + dltimes[8*complete/10-1], + dltimes[9*complete/10-1], + dltimes[complete-1]); + tor_free(dltimes); + } + if (written < 0) + result = NULL; + smartlist_free(dirreqdl_times); + return result; +} +#endif + /** How long do we have to have observed per-country request history before we * are willing to talk about it? */ #define GEOIP_MIN_OBSERVATION_TIME (12*60*60) @@ -785,6 +983,23 @@ dump_geoip_stats(void) goto done; } + data_v2 = geoip_get_dirreqdl_history(GEOIP_CLIENT_NETWORKSTATUS_V2, + REQUEST_DIRECT); + data_v3 = geoip_get_dirreqdl_history(GEOIP_CLIENT_NETWORKSTATUS, + REQUEST_DIRECT); + if (fprintf(out, "ns-direct-dl %s\nns-v2-direct-dl %s\n", + data_v3 ? data_v3 : "", data_v2 ? data_v2 : "") < 0) + goto done; + tor_free(data_v2); + tor_free(data_v3); + data_v2 = geoip_get_dirreqdl_history(GEOIP_CLIENT_NETWORKSTATUS_V2, + REQUEST_TUNNELED); + data_v3 = geoip_get_dirreqdl_history(GEOIP_CLIENT_NETWORKSTATUS, + REQUEST_TUNNELED); + if (fprintf(out, "ns-tunneled-dl %s\nns-v2-tunneled-dl %s\n", + data_v3 ? data_v3 : "", data_v2 ? data_v2 : "") < 0) + goto done; + finish_writing_to_file(open_file); open_file = NULL; done: diff --git a/src/or/or.h b/src/or/or.h index c18ef15faf..035d4edc0a 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -970,6 +970,10 @@ typedef struct connection_t { * to the evdns_server_port is uses to listen to and answer connections. */ struct evdns_server_port *dns_server_port; +#ifdef ENABLE_GEOIP_STATS + /** Unique ID for measuring tunneled network status requests. */ + uint64_t request_id; +#endif } connection_t; /** Stores flags and information related to the portion of a v2 Tor OR @@ -1956,6 +1960,10 @@ typedef struct circuit_t { * linked to an OR connection. */ struct circuit_t *prev_active_on_n_conn; struct circuit_t *next; /**< Next circuit in linked list of all circuits. */ +#ifdef ENABLE_GEOIP_STATS + /** Unique ID for measuring tunneled network status requests. */ + uint64_t request_id; +#endif } circuit_t; /** Largest number of relay_early cells that we can send on a given @@ -3672,6 +3680,42 @@ int getinfo_helper_geoip(control_connection_t *control_conn, const char *question, char **answer); void geoip_free_all(void); +/** Directory requests that we are measuring can be either direct or + * tunneled. */ +typedef enum { + REQUEST_DIRECT = 0, + REQUEST_TUNNELED = 1, +} directory_request_type_t; + +/** Possible states for either direct or tunneled directory requests that + * are relevant for determining network status download times. */ +typedef enum { + /** Found that the client requests a network status; applies to both + * direct and tunneled requests; initial state of a request that we are + * measuring. */ + REQUEST_IS_FOR_NETWORK_STATUS = 0, + /** Finished writing a network status to the directory connection; + * applies to both direct and tunneled requests; completes a direct + * request. */ + FLUSHING_DIR_CONN_FINISHED = 1, + /** END cell sent to circuit that initiated a tunneled request. */ + END_CELL_SENT = 2, + /** Flushed last cell from queue of the circuit that initiated a + * tunneled request to the outbuf of the OR connection. */ + CIRC_QUEUE_FLUSHED = 3, + /** Flushed last byte from buffer of the OR connection belonging to the + * circuit that initiated a tunneled request; completes a tunneled + * request. */ + OR_CONN_BUFFER_FLUSHED = 4 +} directory_request_state_t; + +void geoip_start_dirreq(uint64_t request_id, size_t response_size, + geoip_client_action_t action, + directory_request_type_t type); +void geoip_change_dirreq_state(uint64_t request_id, + directory_request_type_t type, + directory_request_state_t new_state); + /********************************* hibernate.c **********************/ int accounting_parse_options(or_options_t *options, int validate_only); diff --git a/src/or/relay.c b/src/or/relay.c index eb18bbaade..580048be7b 100644 --- a/src/or/relay.c +++ b/src/or/relay.c @@ -532,6 +532,14 @@ relay_send_command_from_edge(uint16_t stream_id, circuit_t *circ, log_debug(LD_OR,"delivering %d cell %s.", relay_command, cell_direction == CELL_DIRECTION_OUT ? "forward" : "backward"); +#ifdef ENABLE_GEOIP_STATS + /* If we are sending an END cell and this circuit is used for a tunneled + * directory request, advance its state. */ + if (relay_command == RELAY_COMMAND_END && circ->request_id) + geoip_change_dirreq_state(circ->request_id, REQUEST_TUNNELED, + END_CELL_SENT); +#endif + if (cell_direction == CELL_DIRECTION_OUT && circ->n_conn) { /* if we're using relaybandwidthrate, this conn wants priority */ circ->n_conn->client_used = approx_time(); @@ -1032,6 +1040,18 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, "Begin cell for known stream. Dropping."); return 0; } +#ifdef ENABLE_GEOIP_STATS + if (rh.command == RELAY_COMMAND_BEGIN_DIR) { + /* Assign this circuit and its app-ward OR connection a unique ID, + * so that we can measure download times. The local edge and dir + * connection will be assigned the same ID when they are created + * and linked. */ + static uint64_t next_id = 0; + circ->request_id = ++next_id; + TO_CONN(TO_OR_CIRCUIT(circ)->p_conn)->request_id = circ->request_id; + } +#endif + return connection_exit_begin_conn(cell, circ); case RELAY_COMMAND_DATA: ++stats_n_data_cells_received; @@ -1821,6 +1841,14 @@ connection_or_flush_from_first_active_circuit(or_connection_t *conn, int max, orcirc->processed_cells++; } #endif +#ifdef ENABLE_GEOIP_STATS + /* If we just flushed our queue and this circuit is used for a + * tunneled directory request, possibly advance its state. */ + if (queue->n == 0 && TO_CONN(conn)->request_id) + geoip_change_dirreq_state(TO_CONN(conn)->request_id, + REQUEST_TUNNELED, CIRC_QUEUE_FLUSHED); +#endif + connection_write_to_buf(cell->body, CELL_NETWORK_SIZE, TO_CONN(conn)); packed_cell_free(cell); From 4002980d1cbb95c70528a35aa2491fff4345999d Mon Sep 17 00:00:00 2001 From: Karsten Loesing Date: Mon, 13 Jul 2009 01:30:30 +0200 Subject: [PATCH 108/187] Right, the u in uint stands for unsigned. --- src/or/geoip.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/or/geoip.c b/src/or/geoip.c index a7e46c0b8f..1430f34ae5 100644 --- a/src/or/geoip.c +++ b/src/or/geoip.c @@ -611,7 +611,7 @@ static void _dirreqdlmap_put(dirreqdlmap_entry_t *entry, directory_request_type_t type, uint64_t request_id) { - char key[3+20+1]; /* dir|tun + -9223372036854775808 + \0 */ + char key[3+20+1]; /* dir|tun + 18446744073709551616 + \0 */ dirreqdlmap_entry_t *ent; if (!dirreqdlmap) dirreqdlmap = strmap_new(); @@ -633,7 +633,7 @@ _dirreqdlmap_put(dirreqdlmap_entry_t *entry, static dirreqdlmap_entry_t * _dirreqdlmap_get(directory_request_type_t type, uint64_t request_id) { - char key[3+20+1]; /* dir|tun + -9223372036854775808 + \0 */ + char key[3+20+1]; /* dir|tun + 18446744073709551616 + \0 */ if (!dirreqdlmap) dirreqdlmap = strmap_new(); tor_snprintf(key, sizeof(key), "%s"U64_FORMAT, @@ -731,8 +731,8 @@ geoip_get_dirreqdl_history(geoip_client_action_t action, } } STRMAP_FOREACH_END; result = tor_malloc_zero(bufsize); - written = tor_snprintf(result, bufsize, "complete=%d,timeout=%d," - "running=%d", complete, timeouts, running); + written = tor_snprintf(result, bufsize, "complete=%u,timeout=%u," + "running=%u", complete, timeouts, running); if (written < 0) return NULL; #define MIN_DIR_REQ_RESPONSES 16 @@ -744,8 +744,8 @@ geoip_get_dirreqdl_history(geoip_client_action_t action, }); median_uint32(dltimes, complete); /* sort */ written = tor_snprintf(result + written, bufsize - written, - ",min=%d,d1=%d,d2=%d,q1=%d,d3=%d,d4=%d,md=%d," - "d6=%d,d7=%d,q3=%d,d8=%d,d9=%d,max=%d", + ",min=%u,d1=%u,d2=%u,q1=%u,d3=%u,d4=%u,md=%u," + "d6=%u,d7=%u,q3=%u,d8=%u,d9=%u,max=%u", dltimes[0], dltimes[1*complete/10-1], dltimes[2*complete/10-1], From 85e7f67e1c773e81a0269858169a9992eeb53da6 Mon Sep 17 00:00:00 2001 From: Karsten Loesing Date: Tue, 14 Jul 2009 02:04:20 +0200 Subject: [PATCH 109/187] Round up results to the next multiple of 4. --- src/or/geoip.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/or/geoip.c b/src/or/geoip.c index 1430f34ae5..819c9f0182 100644 --- a/src/or/geoip.c +++ b/src/or/geoip.c @@ -730,6 +730,13 @@ geoip_get_dirreqdl_history(geoip_client_action_t action, MAP_DEL_CURRENT(key); } } STRMAP_FOREACH_END; +#define DIR_REQ_GRANULARITY 4 + complete = round_uint32_to_next_multiple_of(complete, + DIR_REQ_GRANULARITY); + timeouts = round_uint32_to_next_multiple_of(timeouts, + DIR_REQ_GRANULARITY); + running = round_uint32_to_next_multiple_of(running, + DIR_REQ_GRANULARITY); result = tor_malloc_zero(bufsize); written = tor_snprintf(result, bufsize, "complete=%u,timeout=%u," "running=%u", complete, timeouts, running); From 416940d93bac49f78b57a2cf561bd324d75b391f Mon Sep 17 00:00:00 2001 From: Karsten Loesing Date: Tue, 14 Jul 2009 22:24:50 +0200 Subject: [PATCH 110/187] Some tweaks to directory request download times. - Use common prefixes DIRREQ_* and dirreq_*. - Replace enums in structs with bitfields. --- src/or/connection.c | 6 +- src/or/connection_edge.c | 4 +- src/or/directory.c | 17 +++--- src/or/geoip.c | 119 +++++++++++++++++++-------------------- src/or/or.h | 32 +++++------ src/or/relay.c | 17 +++--- 6 files changed, 97 insertions(+), 98 deletions(-) diff --git a/src/or/connection.c b/src/or/connection.c index 309421051a..242a32c2d9 100644 --- a/src/or/connection.c +++ b/src/or/connection.c @@ -2305,9 +2305,9 @@ connection_handle_write(connection_t *conn, int force) #ifdef ENABLE_GEOIP_STATS /* If we just flushed the last bytes, check if this tunneled dir * request is done. */ - if (buf_datalen(conn->outbuf) == 0 && conn->request_id) - geoip_change_dirreq_state(conn->request_id, REQUEST_TUNNELED, - OR_CONN_BUFFER_FLUSHED); + if (buf_datalen(conn->outbuf) == 0 && conn->dirreq_id) + geoip_change_dirreq_state(conn->dirreq_id, DIRREQ_TUNNELED, + DIRREQ_OR_CONN_BUFFER_FLUSHED); #endif switch (result) { CASE_TOR_TLS_ERROR_ANY: diff --git a/src/or/connection_edge.c b/src/or/connection_edge.c index f32563bcdb..f2b499fd6c 100644 --- a/src/or/connection_edge.c +++ b/src/or/connection_edge.c @@ -2554,7 +2554,7 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ) #ifdef ENABLE_GEOIP_STATS /* Remember the tunneled request ID in the new edge connection, so that * we can measure download times. */ - TO_CONN(n_stream)->request_id = circ->request_id; + TO_CONN(n_stream)->dirreq_id = circ->dirreq_id; #endif n_stream->_base.purpose = EXIT_PURPOSE_CONNECT; @@ -2795,7 +2795,7 @@ connection_exit_connect_dir(edge_connection_t *exitconn) #ifdef ENABLE_GEOIP_STATS /* Note that the new dir conn belongs to the same tunneled request as * the edge conn, so that we can measure download times. */ - TO_CONN(dirconn)->request_id = TO_CONN(exitconn)->request_id; + TO_CONN(dirconn)->dirreq_id = TO_CONN(exitconn)->dirreq_id; #endif connection_link_connections(TO_CONN(dirconn), TO_CONN(exitconn)); diff --git a/src/or/directory.c b/src/or/directory.c index e5230c2cbd..c6faeae2e1 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -2570,12 +2570,12 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, geoip_note_ns_response(act, GEOIP_SUCCESS); /* Note that a request for a network status has started, so that we * can measure the download time later on. */ - if (TO_CONN(conn)->request_id) - geoip_start_dirreq(TO_CONN(conn)->request_id, dlen, act, - REQUEST_TUNNELED); + if (TO_CONN(conn)->dirreq_id) + geoip_start_dirreq(TO_CONN(conn)->dirreq_id, dlen, act, + DIRREQ_TUNNELED); else geoip_start_dirreq(TO_CONN(conn)->global_identifier, dlen, act, - REQUEST_DIRECT); + DIRREQ_DIRECT); } } #endif @@ -3214,12 +3214,13 @@ connection_dir_finished_flushing(dir_connection_t *conn) /* Note that we have finished writing the directory response. For direct * connections this means we're done, for tunneled connections its only * an intermediate step. */ - if (TO_CONN(conn)->request_id) - geoip_change_dirreq_state(TO_CONN(conn)->request_id, REQUEST_TUNNELED, - FLUSHING_DIR_CONN_FINISHED); + if (TO_CONN(conn)->dirreq_id) + geoip_change_dirreq_state(TO_CONN(conn)->dirreq_id, DIRREQ_TUNNELED, + DIRREQ_FLUSHING_DIR_CONN_FINISHED); else geoip_change_dirreq_state(TO_CONN(conn)->global_identifier, - REQUEST_DIRECT, FLUSHING_DIR_CONN_FINISHED); + DIRREQ_DIRECT, + DIRREQ_FLUSHING_DIR_CONN_FINISHED); #endif switch (conn->_base.state) { case DIR_CONN_STATE_CLIENT_SENDING: diff --git a/src/or/geoip.c b/src/or/geoip.c index 819c9f0182..0ecc466088 100644 --- a/src/or/geoip.c +++ b/src/or/geoip.c @@ -580,107 +580,106 @@ _c_hist_compare(const void **_a, const void **_b) * response size, and completion time of a network status request. Used to * measure download times of requests to derive average client * bandwidths. */ -typedef struct dirreqdlmap_entry_t { - directory_request_state_t state; /**< State of this directory request. */ +typedef struct dirreq_map_entry_t { /** Unique identifier for this network status request; this is either the * conn->global_identifier of the dir conn (direct request) or a new * locally unique identifier of a circuit (tunneled request). This ID is * only unique among other direct or tunneled requests, respectively. */ - uint64_t request_id; - /** Is this a direct or a tunneled request? */ - directory_request_type_t type; - int completed:1; /**< Is this request complete? */ - geoip_client_action_t action; /**< Is this a v2 or v3 request? */ + uint64_t dirreq_id; + unsigned int state:3; /**< State of this directory request. */ + unsigned int type:1; /**< Is this a direct or a tunneled request? */ + unsigned int completed:1; /**< Is this request complete? */ + unsigned int action:2; /**< Is this a v2 or v3 request? */ /** When did we receive the request and started sending the response? */ struct timeval request_time; size_t response_size; /**< What is the size of the response in bytes? */ struct timeval completion_time; /**< When did the request succeed? */ -} dirreqdlmap_entry_t; +} dirreq_map_entry_t; /** Map of all directory requests asking for v2 or v3 network statuses in * the current geoip-stats interval. Keys are strings starting with either * "dir" for direct requests or "tun" for tunneled requests, followed by * a unique uint64_t identifier represented as decimal string. Values are - * of type *dirreqdlmap_entry_t. */ -static strmap_t *dirreqdlmap = NULL; + * of type *dirreq_map_entry_t. */ +static strmap_t *dirreq_map = NULL; /** Helper: Put entry into map of directory requests using - * tunneled and request_id as key parts. If there is + * tunneled and dirreq_id as key parts. If there is * already an entry for that key, print out a BUG warning and return. */ static void -_dirreqdlmap_put(dirreqdlmap_entry_t *entry, - directory_request_type_t type, uint64_t request_id) +_dirreq_map_put(dirreq_map_entry_t *entry, dirreq_type_t type, + uint64_t dirreq_id) { char key[3+20+1]; /* dir|tun + 18446744073709551616 + \0 */ - dirreqdlmap_entry_t *ent; - if (!dirreqdlmap) - dirreqdlmap = strmap_new(); + dirreq_map_entry_t *ent; + if (!dirreq_map) + dirreq_map = strmap_new(); tor_snprintf(key, sizeof(key), "%s"U64_FORMAT, - type == REQUEST_TUNNELED ? "tun" : "dir", - U64_PRINTF_ARG(request_id)); - ent = strmap_get(dirreqdlmap, key); + type == DIRREQ_TUNNELED ? "tun" : "dir", + U64_PRINTF_ARG(dirreq_id)); + ent = strmap_get(dirreq_map, key); if (ent) { log_warn(LD_BUG, "Error when putting directory request into local " "map. There is already an entry for the same identifier."); return; } - strmap_set(dirreqdlmap, key, entry); + strmap_set(dirreq_map, key, entry); } /** Helper: Look up and return an entry in the map of directory requests - * using tunneled and request_id as key parts. If there + * using tunneled and dirreq_id as key parts. If there * is no such entry, return NULL. */ -static dirreqdlmap_entry_t * -_dirreqdlmap_get(directory_request_type_t type, uint64_t request_id) +static dirreq_map_entry_t * +_dirreq_map_get(dirreq_type_t type, uint64_t dirreq_id) { char key[3+20+1]; /* dir|tun + 18446744073709551616 + \0 */ - if (!dirreqdlmap) - dirreqdlmap = strmap_new(); + if (!dirreq_map) + dirreq_map = strmap_new(); tor_snprintf(key, sizeof(key), "%s"U64_FORMAT, - type == REQUEST_TUNNELED ? "tun" : "dir", - U64_PRINTF_ARG(request_id)); - return strmap_get(dirreqdlmap, key); + type == DIRREQ_TUNNELED ? "tun" : "dir", + U64_PRINTF_ARG(dirreq_id)); + return strmap_get(dirreq_map, key); } /** Note that an either direct or tunneled (see type) directory - * request for a network status with unique ID request_id of size + * request for a network status with unique ID dirreq_id of size * response_size and action action (either v2 or v3) has * started. */ void -geoip_start_dirreq(uint64_t request_id, size_t response_size, - geoip_client_action_t action, - directory_request_type_t type) +geoip_start_dirreq(uint64_t dirreq_id, size_t response_size, + geoip_client_action_t action, dirreq_type_t type) { - dirreqdlmap_entry_t *ent = tor_malloc_zero(sizeof(dirreqdlmap_entry_t)); - ent->request_id = request_id; + dirreq_map_entry_t *ent = tor_malloc_zero(sizeof(dirreq_map_entry_t)); + ent->dirreq_id = dirreq_id; tor_gettimeofday(&ent->request_time); ent->response_size = response_size; ent->action = action; ent->type = type; - _dirreqdlmap_put(ent, type, request_id); + _dirreq_map_put(ent, type, dirreq_id); } /** Change the state of the either direct or tunneled (see type) - * directory request with request_id to new_state and + * directory request with dirreq_id to new_state and * possibly mark it as completed. If no entry can be found for the given * key parts (e.g., if this is a directory request that we are not * measuring, or one that was started in the previous measurement period), * or if the state cannot be advanced to new_state, do nothing. */ void -geoip_change_dirreq_state(uint64_t request_id, - directory_request_type_t type, - directory_request_state_t new_state) +geoip_change_dirreq_state(uint64_t dirreq_id, dirreq_type_t type, + dirreq_state_t new_state) { - dirreqdlmap_entry_t *ent = _dirreqdlmap_get(type, request_id); + dirreq_map_entry_t *ent = _dirreq_map_get(type, dirreq_id); if (!ent) return; - if (new_state == REQUEST_IS_FOR_NETWORK_STATUS) + if (new_state == DIRREQ_IS_FOR_NETWORK_STATUS) return; if (new_state - 1 != ent->state) return; ent->state = new_state; - if ((type == REQUEST_DIRECT && new_state == FLUSHING_DIR_CONN_FINISHED) || - (type == REQUEST_TUNNELED && new_state == OR_CONN_BUFFER_FLUSHED)) { + if ((type == DIRREQ_DIRECT && + new_state == DIRREQ_FLUSHING_DIR_CONN_FINISHED) || + (type == DIRREQ_TUNNELED && + new_state == DIRREQ_OR_CONN_BUFFER_FLUSHED)) { tor_gettimeofday(&ent->completion_time); ent->completed = 1; } @@ -693,22 +692,22 @@ geoip_change_dirreq_state(uint64_t request_id, * times by deciles and quartiles. Return NULL if we have not observed * requests for long enough. */ static char * -geoip_get_dirreqdl_history(geoip_client_action_t action, - directory_request_type_t type) +geoip_get_dirreq_history(geoip_client_action_t action, + dirreq_type_t type) { char *result = NULL; - smartlist_t *dirreqdl_times = NULL; + smartlist_t *dirreq_times = NULL; uint32_t complete = 0, timeouts = 0, running = 0; int i = 0, bufsize = 1024, written; struct timeval now; tor_gettimeofday(&now); - if (!dirreqdlmap) + if (!dirreq_map) return NULL; if (action != GEOIP_CLIENT_NETWORKSTATUS && action != GEOIP_CLIENT_NETWORKSTATUS_V2) return NULL; - dirreqdl_times = smartlist_create(); - STRMAP_FOREACH_MODIFY(dirreqdlmap, key, dirreqdlmap_entry_t *, ent) { + dirreq_times = smartlist_create(); + STRMAP_FOREACH_MODIFY(dirreq_map, key, dirreq_map_entry_t *, ent) { if (ent->action == action && type == ent->type) { if (ent->completed) { uint32_t *bytes_per_second = tor_malloc_zero(sizeof(uint32_t)); @@ -718,7 +717,7 @@ geoip_get_dirreqdl_history(geoip_client_action_t action, time_diff = 1; /* Avoid DIV/0; "instant" answers are impossible * anyway by law of nature or something.. */ *bytes_per_second = 1000000 * ent->response_size / time_diff; - smartlist_add(dirreqdl_times, bytes_per_second); + smartlist_add(dirreq_times, bytes_per_second); complete++; } else { if (tv_udiff(&ent->request_time, &now) / 1000000 > DIRREQ_TIMEOUT) @@ -745,7 +744,7 @@ geoip_get_dirreqdl_history(geoip_client_action_t action, #define MIN_DIR_REQ_RESPONSES 16 if (complete >= MIN_DIR_REQ_RESPONSES) { uint32_t *dltimes = tor_malloc(sizeof(uint32_t) * complete); - SMARTLIST_FOREACH(dirreqdl_times, uint32_t *, dlt, { + SMARTLIST_FOREACH(dirreq_times, uint32_t *, dlt, { dltimes[i++] = *dlt; tor_free(dlt); }); @@ -770,7 +769,7 @@ geoip_get_dirreqdl_history(geoip_client_action_t action, } if (written < 0) result = NULL; - smartlist_free(dirreqdl_times); + smartlist_free(dirreq_times); return result; } #endif @@ -990,19 +989,19 @@ dump_geoip_stats(void) goto done; } - data_v2 = geoip_get_dirreqdl_history(GEOIP_CLIENT_NETWORKSTATUS_V2, - REQUEST_DIRECT); - data_v3 = geoip_get_dirreqdl_history(GEOIP_CLIENT_NETWORKSTATUS, - REQUEST_DIRECT); + data_v2 = geoip_get_dirreq_history(GEOIP_CLIENT_NETWORKSTATUS_V2, + DIRREQ_DIRECT); + data_v3 = geoip_get_dirreq_history(GEOIP_CLIENT_NETWORKSTATUS, + DIRREQ_DIRECT); if (fprintf(out, "ns-direct-dl %s\nns-v2-direct-dl %s\n", data_v3 ? data_v3 : "", data_v2 ? data_v2 : "") < 0) goto done; tor_free(data_v2); tor_free(data_v3); - data_v2 = geoip_get_dirreqdl_history(GEOIP_CLIENT_NETWORKSTATUS_V2, - REQUEST_TUNNELED); - data_v3 = geoip_get_dirreqdl_history(GEOIP_CLIENT_NETWORKSTATUS, - REQUEST_TUNNELED); + data_v2 = geoip_get_dirreq_history(GEOIP_CLIENT_NETWORKSTATUS_V2, + DIRREQ_TUNNELED); + data_v3 = geoip_get_dirreq_history(GEOIP_CLIENT_NETWORKSTATUS, + DIRREQ_TUNNELED); if (fprintf(out, "ns-tunneled-dl %s\nns-v2-tunneled-dl %s\n", data_v3 ? data_v3 : "", data_v2 ? data_v2 : "") < 0) goto done; diff --git a/src/or/or.h b/src/or/or.h index 035d4edc0a..7b91ff7c33 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -972,7 +972,7 @@ typedef struct connection_t { #ifdef ENABLE_GEOIP_STATS /** Unique ID for measuring tunneled network status requests. */ - uint64_t request_id; + uint64_t dirreq_id; #endif } connection_t; @@ -1962,7 +1962,7 @@ typedef struct circuit_t { struct circuit_t *next; /**< Next circuit in linked list of all circuits. */ #ifdef ENABLE_GEOIP_STATS /** Unique ID for measuring tunneled network status requests. */ - uint64_t request_id; + uint64_t dirreq_id; #endif } circuit_t; @@ -3683,9 +3683,9 @@ void geoip_free_all(void); /** Directory requests that we are measuring can be either direct or * tunneled. */ typedef enum { - REQUEST_DIRECT = 0, - REQUEST_TUNNELED = 1, -} directory_request_type_t; + DIRREQ_DIRECT = 0, + DIRREQ_TUNNELED = 1, +} dirreq_type_t; /** Possible states for either direct or tunneled directory requests that * are relevant for determining network status download times. */ @@ -3693,28 +3693,26 @@ typedef enum { /** Found that the client requests a network status; applies to both * direct and tunneled requests; initial state of a request that we are * measuring. */ - REQUEST_IS_FOR_NETWORK_STATUS = 0, + DIRREQ_IS_FOR_NETWORK_STATUS = 0, /** Finished writing a network status to the directory connection; * applies to both direct and tunneled requests; completes a direct * request. */ - FLUSHING_DIR_CONN_FINISHED = 1, + DIRREQ_FLUSHING_DIR_CONN_FINISHED = 1, /** END cell sent to circuit that initiated a tunneled request. */ - END_CELL_SENT = 2, + DIRREQ_END_CELL_SENT = 2, /** Flushed last cell from queue of the circuit that initiated a * tunneled request to the outbuf of the OR connection. */ - CIRC_QUEUE_FLUSHED = 3, + DIRREQ_CIRC_QUEUE_FLUSHED = 3, /** Flushed last byte from buffer of the OR connection belonging to the * circuit that initiated a tunneled request; completes a tunneled * request. */ - OR_CONN_BUFFER_FLUSHED = 4 -} directory_request_state_t; + DIRREQ_OR_CONN_BUFFER_FLUSHED = 4 +} dirreq_state_t; -void geoip_start_dirreq(uint64_t request_id, size_t response_size, - geoip_client_action_t action, - directory_request_type_t type); -void geoip_change_dirreq_state(uint64_t request_id, - directory_request_type_t type, - directory_request_state_t new_state); +void geoip_start_dirreq(uint64_t dirreq_id, size_t response_size, + geoip_client_action_t action, dirreq_type_t type); +void geoip_change_dirreq_state(uint64_t dirreq_id, dirreq_type_t type, + dirreq_state_t new_state); /********************************* hibernate.c **********************/ diff --git a/src/or/relay.c b/src/or/relay.c index 580048be7b..5654736f96 100644 --- a/src/or/relay.c +++ b/src/or/relay.c @@ -535,9 +535,9 @@ relay_send_command_from_edge(uint16_t stream_id, circuit_t *circ, #ifdef ENABLE_GEOIP_STATS /* If we are sending an END cell and this circuit is used for a tunneled * directory request, advance its state. */ - if (relay_command == RELAY_COMMAND_END && circ->request_id) - geoip_change_dirreq_state(circ->request_id, REQUEST_TUNNELED, - END_CELL_SENT); + if (relay_command == RELAY_COMMAND_END && circ->dirreq_id) + geoip_change_dirreq_state(circ->dirreq_id, DIRREQ_TUNNELED, + DIRREQ_END_CELL_SENT); #endif if (cell_direction == CELL_DIRECTION_OUT && circ->n_conn) { @@ -1047,8 +1047,8 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, * connection will be assigned the same ID when they are created * and linked. */ static uint64_t next_id = 0; - circ->request_id = ++next_id; - TO_CONN(TO_OR_CIRCUIT(circ)->p_conn)->request_id = circ->request_id; + circ->dirreq_id = ++next_id; + TO_CONN(TO_OR_CIRCUIT(circ)->p_conn)->dirreq_id = circ->dirreq_id; } #endif @@ -1844,9 +1844,10 @@ connection_or_flush_from_first_active_circuit(or_connection_t *conn, int max, #ifdef ENABLE_GEOIP_STATS /* If we just flushed our queue and this circuit is used for a * tunneled directory request, possibly advance its state. */ - if (queue->n == 0 && TO_CONN(conn)->request_id) - geoip_change_dirreq_state(TO_CONN(conn)->request_id, - REQUEST_TUNNELED, CIRC_QUEUE_FLUSHED); + if (queue->n == 0 && TO_CONN(conn)->dirreq_id) + geoip_change_dirreq_state(TO_CONN(conn)->dirreq_id, + DIRREQ_TUNNELED, + DIRREQ_CIRC_QUEUE_FLUSHED); #endif connection_write_to_buf(cell->body, CELL_NETWORK_SIZE, TO_CONN(conn)); From 8c496d1660c326c0bc2bd5c505255d5ec3b653ec Mon Sep 17 00:00:00 2001 From: Karsten Loesing Date: Wed, 15 Jul 2009 16:32:40 +0200 Subject: [PATCH 111/187] Some tweaks to statistics. Changes to directory request statistics: - Rename GEOIP statistics to DIRREQ statistics, because they now include more than only GeoIP-based statistics, whereas other statistics are GeoIP-dependent, too. - Rename output file from geoip-stats to dirreq-stats. - Add new config option DirReqStatistics that is required to measure directory request statistics. - Clean up ChangeLog. Also ensure that entry guards statistics have access to a local GeoIP database. --- ChangeLog | 14 ++++++++------ configure.in | 8 ++++---- src/or/config.c | 34 ++++++++++++++++++++++++---------- src/or/connection.c | 2 +- src/or/connection_edge.c | 4 ++-- src/or/directory.c | 4 ++-- src/or/geoip.c | 40 ++++++++++++++++++++++++++-------------- src/or/or.h | 14 +++++++++----- src/or/relay.c | 6 +++--- src/or/router.c | 2 +- 10 files changed, 80 insertions(+), 48 deletions(-) diff --git a/ChangeLog b/ChangeLog index 6b4f651828..e8b5000993 100644 --- a/ChangeLog +++ b/ChangeLog @@ -15,12 +15,14 @@ Changes in version 0.2.2.1-alpha - 2009-??-?? - The memarea code now uses a sentinel value at the end of each area to make sure nothing writes beyond the end of an area. This might help debug some conceivable causes of bug 930. - - Directories that are configured with the --enable-geoip-stats flag - now write their GeoIP stats to disk exactly every 24 hours. - Estimated shares of v2 and v3 requests are determined as averages, - not at the end of a measurement period. Also, unresolved requests - are listed with country code '??'. - Directories now also measure download times of network statuses. + - Directories that are configured with the --enable-dirreq-stats flag + and have "DirReqStatistics 1" set write directory request stats to + disk every 24 hours. As compared to the --enable-geoip-stats flag + in 0.2.1.x, there are a few improvements: 1) stats are written to + disk exactly every 24 hours; 2) estimated shares of v2 and v3 + requests are determined as mean values, not at the end of a + measurement period; 3) unresolved requests are listed with country + code '??'; 4) directories also measure download times. - Exit nodes can write statistics on the number of exit streams and transferred bytes per port to disk every 24 hours. To enable this, run configure with the --enable-exit-stats option, and set diff --git a/configure.in b/configure.in index 0f6ed80657..ab7d6480cf 100644 --- a/configure.in +++ b/configure.in @@ -92,11 +92,11 @@ if test "$enable_exit_stats" = "yes"; then AC_DEFINE(ENABLE_EXIT_STATS, 1, [Defined if we try to collect per-port statistics on exits]) fi -AC_ARG_ENABLE(geoip-stats, - AS_HELP_STRING(--enable-geoip-stats, enable code for directories to collect per-country statistics)) +AC_ARG_ENABLE(dirreq-stats, + AS_HELP_STRING(--enable-dirreq-stats, enable code for directories to collect per-country statistics)) -if test "$enable_geoip_stats" = "yes"; then - AC_DEFINE(ENABLE_GEOIP_STATS, 1, [Defined if we try to collect per-country statistics]) +if test "$enable_dirreq_stats" = "yes"; then + AC_DEFINE(ENABLE_DIRREQ_STATS, 1, [Defined if we try to collect per-country statistics]) fi AC_ARG_ENABLE(buffer-stats, diff --git a/src/or/config.c b/src/or/config.c index 087a907e48..1811551556 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -187,12 +187,13 @@ static config_var_t _option_vars[] = { V(DirPort, UINT, "0"), V(DirPortFrontPage, FILENAME, NULL), OBSOLETE("DirPostPeriod"), -#ifdef ENABLE_GEOIP_STATS +#ifdef ENABLE_DIRREQ_STATS OBSOLETE("DirRecordUsageByCountry"), OBSOLETE("DirRecordUsageGranularity"), OBSOLETE("DirRecordUsageRetainIPs"), OBSOLETE("DirRecordUsageSaveInterval"), #endif + V(DirReqStatistics, BOOL, "0"), VAR("DirServer", LINELIST, DirServers, NULL), V(DNSPort, UINT, "0"), V(DNSListenAddress, LINELIST, NULL), @@ -1376,17 +1377,25 @@ options_act(or_options_t *old_options) geoip_load_file(actual_fname, options); tor_free(actual_fname); } -#ifdef ENABLE_GEOIP_STATS - /* Check if GeoIP database could be loaded. */ - if (!geoip_is_loaded()) { - log_warn(LD_CONFIG, "Configured to measure GeoIP statistics, but no " - "GeoIP database found!"); - return -1; + +#ifdef ENABLE_DIRREQ_STATS + if (options->DirReqStatistics) { + /* Check if GeoIP database could be loaded. */ + if (!geoip_is_loaded()) { + log_warn(LD_CONFIG, "Configured to measure directory request " + "statistics, but no GeoIP database found!"); + return -1; + } + log_notice(LD_CONFIG, "Configured to count directory requests by " + "country and write aggregate statistics to disk. Check the " + "dirreq-stats file in your data directory that will first " + "be written in 24 hours from now."); } - log_notice(LD_CONFIG, "Configured to measure usage by country and " - "write aggregate statistics to disk. Check the geoip-stats file " - "in your data directory once I've been running for 24 hours."); +#else + log_warn(LD_CONFIG, "DirReqStatistics enabled, but Tor was built " + "without support for directory request statistics."); #endif + #ifdef ENABLE_EXIT_STATS if (options->ExitPortStatistics) log_notice(LD_CONFIG, "Configured to measure exit port statistics. " @@ -1417,6 +1426,11 @@ options_act(or_options_t *old_options) log_warn(LD_CONFIG, "Bridges cannot be configured to measure " "additional GeoIP statistics as entry guards."); return -1; + } else if (!geoip_is_loaded()) { + /* Check if GeoIP database could be loaded. */ + log_warn(LD_CONFIG, "Configured to measure entry node statistics, " + "but no GeoIP database found!"); + return -1; } else log_notice(LD_CONFIG, "Configured to measure entry node " "statistics. Look for the entry-stats file that will " diff --git a/src/or/connection.c b/src/or/connection.c index 242a32c2d9..dc9c4eace2 100644 --- a/src/or/connection.c +++ b/src/or/connection.c @@ -2302,7 +2302,7 @@ connection_handle_write(connection_t *conn, int force) /* else open, or closing */ result = flush_buf_tls(or_conn->tls, conn->outbuf, max_to_write, &conn->outbuf_flushlen); -#ifdef ENABLE_GEOIP_STATS +#ifdef ENABLE_DIRREQ_STATS /* If we just flushed the last bytes, check if this tunneled dir * request is done. */ if (buf_datalen(conn->outbuf) == 0 && conn->dirreq_id) diff --git a/src/or/connection_edge.c b/src/or/connection_edge.c index f2b499fd6c..7a3d0a55be 100644 --- a/src/or/connection_edge.c +++ b/src/or/connection_edge.c @@ -2551,7 +2551,7 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ) log_debug(LD_EXIT,"Creating new exit connection."); n_stream = edge_connection_new(CONN_TYPE_EXIT, AF_INET); -#ifdef ENABLE_GEOIP_STATS +#ifdef ENABLE_DIRREQ_STATS /* Remember the tunneled request ID in the new edge connection, so that * we can measure download times. */ TO_CONN(n_stream)->dirreq_id = circ->dirreq_id; @@ -2792,7 +2792,7 @@ connection_exit_connect_dir(edge_connection_t *exitconn) dirconn->_base.purpose = DIR_PURPOSE_SERVER; dirconn->_base.state = DIR_CONN_STATE_SERVER_COMMAND_WAIT; -#ifdef ENABLE_GEOIP_STATS +#ifdef ENABLE_DIRREQ_STATS /* Note that the new dir conn belongs to the same tunneled request as * the edge conn, so that we can measure download times. */ TO_CONN(dirconn)->dirreq_id = TO_CONN(exitconn)->dirreq_id; diff --git a/src/or/directory.c b/src/or/directory.c index c6faeae2e1..976c08c19a 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -2562,7 +2562,7 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, goto done; } -#ifdef ENABLE_GEOIP_STATS +#ifdef ENABLE_DIRREQ_STATS { struct in_addr in; if (tor_inet_aton((TO_CONN(conn))->address, &in)) { @@ -3210,7 +3210,7 @@ connection_dir_finished_flushing(dir_connection_t *conn) tor_assert(conn); tor_assert(conn->_base.type == CONN_TYPE_DIR); -#ifdef ENABLE_GEOIP_STATS +#ifdef ENABLE_DIRREQ_STATS /* Note that we have finished writing the directory response. For direct * connections this means we're done, for tunneled connections its only * an intermediate step. */ diff --git a/src/or/geoip.c b/src/or/geoip.c index 0ecc466088..7aeec8f5f1 100644 --- a/src/or/geoip.c +++ b/src/or/geoip.c @@ -347,7 +347,7 @@ geoip_determine_shares(time_t now) last_time_determined_shares = now; } -#ifdef ENABLE_GEOIP_STATS +#ifdef ENABLE_DIRREQ_STATS /** Calculate which fraction of v2 and v3 directory requests aimed at caches * have been sent to us since the last call of this function up to time * now. Set *v2_share_out and *v3_share_out to the @@ -390,10 +390,11 @@ geoip_note_client_seen(geoip_client_action_t action, if (client_history_starts > now) return; } else { -#ifndef ENABLE_GEOIP_STATS +#ifndef ENABLE_DIRREQ_STATS return; #else - if (options->BridgeRelay || options->BridgeAuthoritativeDir) + if (options->BridgeRelay || options->BridgeAuthoritativeDir || + !options->DirReqStatistics) return; #endif } @@ -494,7 +495,7 @@ geoip_remove_old_clients(time_t cutoff) client_history_starts = cutoff; } -#ifdef ENABLE_GEOIP_STATS +#ifdef ENABLE_DIRREQ_STATS /** How many responses are we giving to clients requesting v2 network * statuses? */ static uint32_t ns_v2_responses[GEOIP_NS_RESPONSE_NUM]; @@ -511,8 +512,10 @@ void geoip_note_ns_response(geoip_client_action_t action, geoip_ns_response_t response) { -#ifdef ENABLE_GEOIP_STATS +#ifdef ENABLE_DIRREQ_STATS static int arrays_initialized = 0; + if (!get_options()->DirReqStatistics) + return; if (!arrays_initialized) { memset(ns_v2_responses, 0, sizeof(ns_v2_responses)); memset(ns_v3_responses, 0, sizeof(ns_v3_responses)); @@ -649,7 +652,10 @@ void geoip_start_dirreq(uint64_t dirreq_id, size_t response_size, geoip_client_action_t action, dirreq_type_t type) { - dirreq_map_entry_t *ent = tor_malloc_zero(sizeof(dirreq_map_entry_t)); + dirreq_map_entry_t *ent; + if (!get_options()->DirReqStatistics) + return; + ent = tor_malloc_zero(sizeof(dirreq_map_entry_t)); ent->dirreq_id = dirreq_id; tor_gettimeofday(&ent->request_time); ent->response_size = response_size; @@ -668,7 +674,10 @@ void geoip_change_dirreq_state(uint64_t dirreq_id, dirreq_type_t type, dirreq_state_t new_state) { - dirreq_map_entry_t *ent = _dirreq_map_get(type, dirreq_id); + dirreq_map_entry_t *ent; + if (!get_options()->DirReqStatistics) + return; + ent = _dirreq_map_get(type, dirreq_id); if (!ent) return; if (new_state == DIRREQ_IS_FOR_NETWORK_STATUS) @@ -685,7 +694,7 @@ geoip_change_dirreq_state(uint64_t dirreq_id, dirreq_type_t type, } } -#ifdef ENABLE_GEOIP_STATS +#ifdef ENABLE_DIRREQ_STATS /** Return a newly allocated comma-separated string containing statistics * on network status downloads. The string contains the number of completed * requests, timeouts, and still running requests as well as the download @@ -788,7 +797,7 @@ geoip_get_client_history(time_t now, geoip_client_action_t action) { char *result = NULL; int min_observation_time = GEOIP_MIN_OBSERVATION_TIME; -#ifdef ENABLE_GEOIP_STATS +#ifdef ENABLE_DIRREQ_STATS min_observation_time = DIR_RECORD_USAGE_MIN_OBSERVATION_TIME; #endif if (!geoip_is_loaded()) @@ -803,7 +812,7 @@ geoip_get_client_history(time_t now, geoip_client_action_t action) unsigned *counts = tor_malloc_zero(sizeof(unsigned)*n_countries); unsigned total = 0; unsigned granularity = IP_GRANULARITY; -#ifdef ENABLE_GEOIP_STATS +#ifdef ENABLE_DIRREQ_STATS granularity = DIR_RECORD_USAGE_GRANULARITY; #endif HT_FOREACH(ent, clientmap, &client_history) { @@ -871,7 +880,7 @@ geoip_get_request_history(time_t now, geoip_client_action_t action) char *result; unsigned granularity = IP_GRANULARITY; int min_observation_time = GEOIP_MIN_OBSERVATION_TIME; -#ifdef ENABLE_GEOIP_STATS +#ifdef ENABLE_DIRREQ_STATS granularity = DIR_RECORD_USAGE_GRANULARITY; min_observation_time = DIR_RECORD_USAGE_MIN_OBSERVATION_TIME; #endif @@ -916,14 +925,14 @@ geoip_get_request_history(time_t now, geoip_client_action_t action) return result; } -/** Store all our geoip statistics into $DATADIR/geoip-stats. */ +/** Store all our geoip statistics into $DATADIR/dirreq-stats. */ static void dump_geoip_stats(void) { -#ifdef ENABLE_GEOIP_STATS +#ifdef ENABLE_DIRREQ_STATS time_t now = time(NULL); time_t request_start; - char *filename = get_datadir_fname("geoip-stats"); + char *filename = get_datadir_fname("dirreq-stats"); char *data_v2 = NULL, *data_v3 = NULL; char since[ISO_TIME_LEN+1], written[ISO_TIME_LEN+1]; open_file_t *open_file = NULL; @@ -931,6 +940,9 @@ dump_geoip_stats(void) FILE *out; int i; + if (!get_options()->DirReqStatistics) + goto done; + data_v2 = geoip_get_client_history(now, GEOIP_CLIENT_NETWORKSTATUS_V2); data_v3 = geoip_get_client_history(now, GEOIP_CLIENT_NETWORKSTATUS); format_iso_time(since, geoip_get_history_start()); diff --git a/src/or/or.h b/src/or/or.h index 7b91ff7c33..445df4d78e 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -20,8 +20,8 @@ #ifndef INSTRUMENT_DOWNLOADS #define INSTRUMENT_DOWNLOADS 1 #endif -#ifndef ENABLE_GEOIP_STATS -#define ENABLE_GEOIP_STATS 1 +#ifndef ENABLE_DIRREQ_STATS +#define ENABLE_DIRREQ_STATS 1 #endif #ifndef ENABLE_BUFFER_STATS #define ENABLE_BUFFER_STATS 1 @@ -970,7 +970,7 @@ typedef struct connection_t { * to the evdns_server_port is uses to listen to and answer connections. */ struct evdns_server_port *dns_server_port; -#ifdef ENABLE_GEOIP_STATS +#ifdef ENABLE_DIRREQ_STATS /** Unique ID for measuring tunneled network status requests. */ uint64_t dirreq_id; #endif @@ -1960,7 +1960,7 @@ typedef struct circuit_t { * linked to an OR connection. */ struct circuit_t *prev_active_on_n_conn; struct circuit_t *next; /**< Next circuit in linked list of all circuits. */ -#ifdef ENABLE_GEOIP_STATS +#ifdef ENABLE_DIRREQ_STATS /** Unique ID for measuring tunneled network status requests. */ uint64_t dirreq_id; #endif @@ -2500,6 +2500,10 @@ typedef struct { * exit allows it, we use it. */ int AllowSingleHopCircuits; + /** If true, the user wants us to collect statistics on clients + * requesting network statuses from us as directory. */ + int DirReqStatistics; + /** If true, the user wants us to collect statistics on port usage. */ int ExitPortStatistics; @@ -2564,7 +2568,7 @@ typedef struct { int BridgeRecordUsageByCountry; #if 0 - /** If true, and Tor is built with GEOIP_STATS support, and we're a + /** If true, and Tor is built with DIRREQ_STATS support, and we're a * directory, record how many directory requests we get from each country. */ int DirRecordUsageByCountry; /** Round all GeoIP results to the next multiple of this value, to avoid diff --git a/src/or/relay.c b/src/or/relay.c index 5654736f96..098b95253e 100644 --- a/src/or/relay.c +++ b/src/or/relay.c @@ -532,7 +532,7 @@ relay_send_command_from_edge(uint16_t stream_id, circuit_t *circ, log_debug(LD_OR,"delivering %d cell %s.", relay_command, cell_direction == CELL_DIRECTION_OUT ? "forward" : "backward"); -#ifdef ENABLE_GEOIP_STATS +#ifdef ENABLE_DIRREQ_STATS /* If we are sending an END cell and this circuit is used for a tunneled * directory request, advance its state. */ if (relay_command == RELAY_COMMAND_END && circ->dirreq_id) @@ -1040,7 +1040,7 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, "Begin cell for known stream. Dropping."); return 0; } -#ifdef ENABLE_GEOIP_STATS +#ifdef ENABLE_DIRREQ_STATS if (rh.command == RELAY_COMMAND_BEGIN_DIR) { /* Assign this circuit and its app-ward OR connection a unique ID, * so that we can measure download times. The local edge and dir @@ -1841,7 +1841,7 @@ connection_or_flush_from_first_active_circuit(or_connection_t *conn, int max, orcirc->processed_cells++; } #endif -#ifdef ENABLE_GEOIP_STATS +#ifdef ENABLE_DIRREQ_STATS /* If we just flushed our queue and this circuit is used for a * tunneled directory request, possibly advance its state. */ if (queue->n == 0 && TO_CONN(conn)->dirreq_id) diff --git a/src/or/router.c b/src/or/router.c index bdea4fa764..42a0d56471 100644 --- a/src/or/router.c +++ b/src/or/router.c @@ -1916,7 +1916,7 @@ extrainfo_get_client_geoip_summary(time_t now) { static time_t last_purged_at = 0; int geoip_purge_interval = 48*60*60; -#ifdef ENABLE_GEOIP_STATS +#ifdef ENABLE_DIRREQ_STATS geoip_purge_interval = DIR_RECORD_USAGE_RETAIN_IPS; #endif #ifdef ENABLE_ENTRY_STATS From 989e74e5c0f00b0b423b79ecf34c1e28fe6ab356 Mon Sep 17 00:00:00 2001 From: Sebastian Hahn Date: Wed, 15 Jul 2009 17:26:11 +0200 Subject: [PATCH 112/187] fix long line --- src/or/config.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/or/config.c b/src/or/config.c index 19212fe899..940a205748 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -4799,7 +4799,8 @@ config_parse_units(const char *val, struct unit_table_t *u, int *ok) tor_assert(ok); sl = smartlist_create(); - smartlist_split_string(sl, val, NULL, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 3); + smartlist_split_string(sl, val, NULL, + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 3); if (smartlist_len(sl) < 1 || smartlist_len(sl) > 2) { *ok = 0; From 82795eebb75e2c60b9a294a27100c3f19021dde5 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 16 Jul 2009 14:02:07 -0400 Subject: [PATCH 113/187] Include AccountingMax in torrc.sample "There's room for one more till the end of creation." -- _Sandman_ (Based on patch from David [edeca]. Closes bug 1036.) --- src/config/torrc.sample.in | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/config/torrc.sample.in b/src/config/torrc.sample.in index d0b1ee1591..9250d1eb50 100644 --- a/src/config/torrc.sample.in +++ b/src/config/torrc.sample.in @@ -99,6 +99,19 @@ SocksListenAddress 127.0.0.1 # accept connections only from localhost #RelayBandwidthRate 100 KBytes # Throttle traffic to 100KB/s (800Kbps) #RelayBandwidthBurst 200 KBytes # But allow bursts up to 200KB/s (1600Kbps) +## Use these to restrict the maximum traffic per day, week, or month. +## Note that this threshold applies to sent _and_ to received bytes, +## not to their sum: Setting "1 GB" may allow up to 2 GB total before +## hibernating. +## +## Set a maximum of 1 gigabyte per period. +#AccountingMax 1 GB +## Each period starts daily at midnight (AccountingMax is per day) +#AccountingStart day 00:00 +## Each period starts on the 3rd of the month at 15:00 (AccountingMax +## is per month) +#AccountingStart month 3 15:00 + ## Contact info to be published in the directory, so we can contact you ## if your relay is misconfigured or something else goes wrong. Google ## indexes this, so spammers might also collect it. From cbe432739183ad9cb307f15362489bcc86b405d0 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 15 Jul 2009 12:24:02 -0400 Subject: [PATCH 114/187] Switch dirreq_map to use HT_ functions rather than strmap. --- src/or/geoip.c | 98 ++++++++++++++++++++++++++++++++------------------ 1 file changed, 64 insertions(+), 34 deletions(-) diff --git a/src/or/geoip.c b/src/or/geoip.c index 7aeec8f5f1..3336401220 100644 --- a/src/or/geoip.c +++ b/src/or/geoip.c @@ -584,6 +584,7 @@ _c_hist_compare(const void **_a, const void **_b) * measure download times of requests to derive average client * bandwidths. */ typedef struct dirreq_map_entry_t { + HT_ENTRY(dirreq_map_entry_t) node; /** Unique identifier for this network status request; this is either the * conn->global_identifier of the dir conn (direct request) or a new * locally unique identifier of a circuit (tunneled request). This ID is @@ -600,11 +601,30 @@ typedef struct dirreq_map_entry_t { } dirreq_map_entry_t; /** Map of all directory requests asking for v2 or v3 network statuses in - * the current geoip-stats interval. Keys are strings starting with either - * "dir" for direct requests or "tun" for tunneled requests, followed by - * a unique uint64_t identifier represented as decimal string. Values are + * the current geoip-stats interval. Values are * of type *dirreq_map_entry_t. */ -static strmap_t *dirreq_map = NULL; +static HT_HEAD(dirreqmap, dirreq_map_entry_t) dirreq_map = + HT_INITIALIZER(); + +static int +dirreq_map_ent_eq(const dirreq_map_entry_t *a, + const dirreq_map_entry_t *b) +{ + return a->dirreq_id == b->dirreq_id && a->type == b->type; +} + +static unsigned +dirreq_map_ent_hash(const dirreq_map_entry_t *entry) +{ + unsigned u = (unsigned) entry->dirreq_id; + u += entry->type << 20; + return u; +} + +HT_PROTOTYPE(dirreqmap, dirreq_map_entry_t, node, dirreq_map_ent_hash, + dirreq_map_ent_eq); +HT_GENERATE(dirreqmap, dirreq_map_entry_t, node, dirreq_map_ent_hash, + dirreq_map_ent_eq, 0.6, malloc, realloc, free); /** Helper: Put entry into map of directory requests using * tunneled and dirreq_id as key parts. If there is @@ -613,20 +633,18 @@ static void _dirreq_map_put(dirreq_map_entry_t *entry, dirreq_type_t type, uint64_t dirreq_id) { - char key[3+20+1]; /* dir|tun + 18446744073709551616 + \0 */ - dirreq_map_entry_t *ent; - if (!dirreq_map) - dirreq_map = strmap_new(); - tor_snprintf(key, sizeof(key), "%s"U64_FORMAT, - type == DIRREQ_TUNNELED ? "tun" : "dir", - U64_PRINTF_ARG(dirreq_id)); - ent = strmap_get(dirreq_map, key); - if (ent) { + dirreq_map_entry_t *old_ent; + tor_assert(entry->type == type); + tor_assert(entry->dirreq_id == dirreq_id); + + /* XXXX022 once we're sure the bug case never happens, we can switch + * to HT_INSERT */ + old_ent = HT_REPLACE(dirreqmap, &dirreq_map, entry); + if (old_ent && old_ent != entry) { log_warn(LD_BUG, "Error when putting directory request into local " - "map. There is already an entry for the same identifier."); + "map. There was already an entry for the same identifier."); return; } - strmap_set(dirreq_map, key, entry); } /** Helper: Look up and return an entry in the map of directory requests @@ -635,13 +653,10 @@ _dirreq_map_put(dirreq_map_entry_t *entry, dirreq_type_t type, static dirreq_map_entry_t * _dirreq_map_get(dirreq_type_t type, uint64_t dirreq_id) { - char key[3+20+1]; /* dir|tun + 18446744073709551616 + \0 */ - if (!dirreq_map) - dirreq_map = strmap_new(); - tor_snprintf(key, sizeof(key), "%s"U64_FORMAT, - type == DIRREQ_TUNNELED ? "tun" : "dir", - U64_PRINTF_ARG(dirreq_id)); - return strmap_get(dirreq_map, key); + dirreq_map_entry_t lookup; + lookup.type = type; + lookup.dirreq_id = dirreq_id; + return HT_FIND(dirreqmap, &dirreq_map, &lookup); } /** Note that an either direct or tunneled (see type) directory @@ -708,16 +723,20 @@ geoip_get_dirreq_history(geoip_client_action_t action, smartlist_t *dirreq_times = NULL; uint32_t complete = 0, timeouts = 0, running = 0; int i = 0, bufsize = 1024, written; + dirreq_map_entry_t **ptr, **next, *ent; struct timeval now; + tor_gettimeofday(&now); - if (!dirreq_map) - return NULL; if (action != GEOIP_CLIENT_NETWORKSTATUS && action != GEOIP_CLIENT_NETWORKSTATUS_V2) return NULL; dirreq_times = smartlist_create(); - STRMAP_FOREACH_MODIFY(dirreq_map, key, dirreq_map_entry_t *, ent) { - if (ent->action == action && type == ent->type) { + for (ptr = HT_START(dirreqmap, &dirreq_map); ptr; ptr = next) { + ent = *ptr; + if (ent->action != action || ent->type != type) { + next = HT_NEXT(dirreqmap, &dirreq_map, ptr); + continue; + } else { if (ent->completed) { uint32_t *bytes_per_second = tor_malloc_zero(sizeof(uint32_t)); uint32_t time_diff = (uint32_t) tv_udiff(&ent->request_time, @@ -734,10 +753,10 @@ geoip_get_dirreq_history(geoip_client_action_t action, else running++; } + next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ptr); tor_free(ent); - MAP_DEL_CURRENT(key); } - } STRMAP_FOREACH_END; + } #define DIR_REQ_GRANULARITY 4 complete = round_uint32_to_next_multiple_of(complete, DIR_REQ_GRANULARITY); @@ -1106,13 +1125,24 @@ clear_geoip_db(void) void geoip_free_all(void) { - clientmap_entry_t **ent, **next, *this; - for (ent = HT_START(clientmap, &client_history); ent != NULL; ent = next) { - this = *ent; - next = HT_NEXT_RMV(clientmap, &client_history, ent); - tor_free(this); + { + clientmap_entry_t **ent, **next, *this; + for (ent = HT_START(clientmap, &client_history); ent != NULL; ent = next) { + this = *ent; + next = HT_NEXT_RMV(clientmap, &client_history, ent); + tor_free(this); + } + HT_CLEAR(clientmap, &client_history); + } + { + dirreq_map_entry_t **ent, **next, *this; + for (ent = HT_START(dirreqmap, &dirreq_map); ent != NULL; ent = next) { + this = *ent; + next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ent); + tor_free(this); + } + HT_CLEAR(dirreqmap, &dirreq_map); } - HT_CLEAR(clientmap, &client_history); clear_geoip_db(); } From c0a2d7dc7346384c41435beaa505e3323b2fb9a9 Mon Sep 17 00:00:00 2001 From: Roger Dingledine Date: Thu, 16 Jul 2009 17:15:59 -0400 Subject: [PATCH 115/187] Clean up the accountingmax section in torrc.sample Also, suggest a default of 4GB/day (46KB/s) rather than 1GB/day (11.5KB/s). No use asking low. --- src/config/torrc.sample.in | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/config/torrc.sample.in b/src/config/torrc.sample.in index 9250d1eb50..c3b458543a 100644 --- a/src/config/torrc.sample.in +++ b/src/config/torrc.sample.in @@ -1,5 +1,5 @@ ## Configuration file for a typical Tor user -## Last updated 12 April 2009 for Tor 0.2.1.14-rc. +## Last updated 16 July 2009 for Tor 0.2.2.1-alpha. ## (May or may not work for much older or much newer versions of Tor.) ## ## Lines that begin with "## " try to explain what's going on. Lines @@ -101,11 +101,11 @@ SocksListenAddress 127.0.0.1 # accept connections only from localhost ## Use these to restrict the maximum traffic per day, week, or month. ## Note that this threshold applies to sent _and_ to received bytes, -## not to their sum: Setting "1 GB" may allow up to 2 GB total before -## hibernating. +## not to their sum: Setting "4 GBytes" may allow up to 8 GBytes +## total before hibernating. ## -## Set a maximum of 1 gigabyte per period. -#AccountingMax 1 GB +## Set a maximum of 4 gigabytes each way per period. +#AccountingMax 4 GBytes ## Each period starts daily at midnight (AccountingMax is per day) #AccountingStart day 00:00 ## Each period starts on the 3rd of the month at 15:00 (AccountingMax From b71bbdc69a56b4f1cd88816fa08c4ef8c6445a1b Mon Sep 17 00:00:00 2001 From: Karsten Loesing Date: Thu, 16 Jul 2009 22:29:55 +0200 Subject: [PATCH 116/187] Make exit stats compile on 64-bit machines, too. --- src/or/rephist.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/or/rephist.c b/src/or/rephist.c index 6a716d376b..c0b9ae14a4 100644 --- a/src/or/rephist.c +++ b/src/or/rephist.c @@ -1418,7 +1418,7 @@ write_exit_stats(time_t when) EXIT_STATS_ROUND_UP_BYTES); other_bytes /= 1024; if (fprintf(out, "%sother="U64_FORMAT"\n", - comma ? "," : "", other_bytes)<0) + comma ? "," : "", U64_PRINTF_ARG(other_bytes))<0) goto done; } /* streams-opened port=num,.. */ From 5c0b418a6f7d46b675a6bc91a7fd83cba6889d30 Mon Sep 17 00:00:00 2001 From: Sebastian Hahn Date: Thu, 23 Jul 2009 13:45:04 +0200 Subject: [PATCH 117/187] don't warn about DirReqStatistics when the option is disabled Reported by Zax on #tor --- src/or/config.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/or/config.c b/src/or/config.c index 1ec666ad9a..46d147090b 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -1378,8 +1378,8 @@ options_act(or_options_t *old_options) tor_free(actual_fname); } -#ifdef ENABLE_DIRREQ_STATS if (options->DirReqStatistics) { +#ifdef ENABLE_DIRREQ_STATS /* Check if GeoIP database could be loaded. */ if (!geoip_is_loaded()) { log_warn(LD_CONFIG, "Configured to measure directory request " @@ -1390,11 +1390,11 @@ options_act(or_options_t *old_options) "country and write aggregate statistics to disk. Check the " "dirreq-stats file in your data directory that will first " "be written in 24 hours from now."); - } #else log_warn(LD_CONFIG, "DirReqStatistics enabled, but Tor was built " "without support for directory request statistics."); #endif + } #ifdef ENABLE_EXIT_STATS if (options->ExitPortStatistics) From 884c621aa73edd9727cb97ef249cefdaf8ec4741 Mon Sep 17 00:00:00 2001 From: Karsten Loesing Date: Thu, 23 Jul 2009 10:59:00 -0400 Subject: [PATCH 118/187] Proposal: Including Network Statistics in Extra-Info Documents --- doc/spec/proposals/000-index.txt | 2 + .../166-statistics-extra-info-docs.txt | 391 ++++++++++++++++++ 2 files changed, 393 insertions(+) create mode 100644 doc/spec/proposals/166-statistics-extra-info-docs.txt diff --git a/doc/spec/proposals/000-index.txt b/doc/spec/proposals/000-index.txt index b26f382f6a..af1f40bf6d 100644 --- a/doc/spec/proposals/000-index.txt +++ b/doc/spec/proposals/000-index.txt @@ -86,6 +86,7 @@ Proposals by number: 163 Detecting whether a connection comes from a client [OPEN] 164 Reporting the status of server votes [OPEN] 165 Easy migration for voting authority sets [OPEN] +166 Including Network Statistics in Extra-Info Documents [OPEN] Proposals by status: @@ -113,6 +114,7 @@ Proposals by status: 163 Detecting whether a connection comes from a client [for 0.2.2] 164 Reporting the status of server votes [for 0.2.2] 165 Easy migration for voting authority sets + 166 Including Network Statistics in Extra-Info Documents [for 0.2.2] ACCEPTED: 110 Avoiding infinite length circuits [for 0.2.1.x] [in 0.2.1.3-alpha] 117 IPv6 exits [for 0.2.1.x] diff --git a/doc/spec/proposals/166-statistics-extra-info-docs.txt b/doc/spec/proposals/166-statistics-extra-info-docs.txt new file mode 100644 index 0000000000..3716c049cc --- /dev/null +++ b/doc/spec/proposals/166-statistics-extra-info-docs.txt @@ -0,0 +1,391 @@ +Filename: 166-statistics-extra-info-docs.txt +Title: Including Network Statistics in Extra-Info Documents +Author: Karsten Loesing +Created: 21-Jul-2009 +Target: 0.2.2 +Status: Open + +Change history: + + 21-Jul-2009 Initial proposal for or-dev + + +Overview: + + The Tor network has grown to almost two thousand relays and millions + of casual users over the past few years. With growth has come + increasing performance problems and attempts by some countries to + block access to the Tor network. In order to address these problems, + we need to learn more about the Tor network. This proposal suggests to + measure additional statistics and include them in extra-info documents + to help us understand the Tor network better. + + +Introduction: + + As of May 2009, relays, bridges, and directories gather the following + data for statistical purposes: + + - Relays and bridges count the number of bytes that they have pushed + in 15-minute intervals over the past 24 hours. Relays and bridges + include these data in extra-info documents that they send to the + directory authorities whenever they publish their server descriptor. + + - Bridges further include a rough number of clients per country that + they have seen in the past 48 hours in their extra-info documents. + + - Directories can be configured to count the number of clients they + see per country in the past 24 hours and to write them to a local + file. + + Since then we extended the network statistics in Tor. These statistics + include: + + - Directories now gather more precise statistics about connecting + clients. Fixes include measuring in intervals of exactly 24 hours, + counting unsuccessful requests, measuring download times, etc. The + directories append their statistics to a local file every 24 hours. + + - Entry guards count the number of clients per country per day like + bridges do and write them to a local file every 24 hours. + + - Relays measure statistics of the number of cells in their circuit + queues and how much time these cells spend waiting there. Relays + write these statistics to a local file every 24 hours. + + - Exit nodes count the number of read and written bytes on exit + connections per port as well as the number of opened exit streams + per port in 24-hour intervals. Exit nodes write their statistics to + a local file. + + The following four sections contain descriptions for adding these + statistics to the relays' extra-info documents. + + +Directory request statistics: + + The first type of statistics aims at measuring directory requests sent + by clients to a directory mirror or directory authority. More + precisely, these statistics aim at requests for v2 and v3 network + statuses only. These directory requests are sent non-anonymously, + either via HTTP-like requests to a directory's Dir port or tunneled + over a 1-hop circuit. + + Measuring directory request statistics is useful for several reasons: + First, the number of locally seen directory requests can be used to + estimate the total number of clients in the Tor network. Second, the + country-wise classification of requests using a GeoIP database can + help counting the relative and absolute number of users per country. + Third, the download times can give hints on the available bandwidth + capacity at clients. + + Directory requests do not give any hints on the contents that clients + send or receive over the Tor network. Every client requests network + statuses from the directories, so that there are no anonymity-related + concerns to gather these statistics. It might be, though, that clients + wish to hide the fact that they are connecting to the Tor network. + Therefore, IP addresses are resolved to country codes in memory, + events are accumulated over 24 hours, and numbers are rounded up to + multiples of 4 or 8. + + "dirreq-stats-end" YYYY-MM-DD HH:MM:SS (NSEC s) NL + [At most once.] + + YYYY-MM-DD HH:MM:SS defines the end of the included measurement + interval of length NSEC seconds (86400 seconds by default). + + A "dirreq-stats-end" line, as well as any other "dirreq-*" line, + is only added when the relay has opened its Dir port and after 24 + hours of measuring directory requests. + + "dirreq-v2-ips" CC=N,CC=N,... NL + [At most once.] + "dirreq-v3-ips" CC=N,CC=N,... NL + [At most once.] + + List of mappings from two-letter country codes to the number of + unique IP addresses that have connected from that country to + request a v2/v3 network status, rounded up to the nearest multiple + of 8. Only those IP addresses are counted that the directory can + answer with a 200 OK status code. + + "dirreq-v2-reqs" CC=N,CC=N,... NL + [At most once.] + "dirreq-v3-reqs" CC=N,CC=N,... NL + [At most once.] + + List of mappings from two-letter country codes to the number of + requests for v2/v3 network statuses from that country, rounded up + to the nearest multiple of 8. Only those requests are counted that + the directory can answer with a 200 OK status code. + + "dirreq-v2-share" num% NL + [At most once.] + "dirreq-v3-share" num% NL + [At most once.] + + The share of v2/v3 network status requests that the directory + expects to receive from clients based on its advertised bandwidth + compared to the overall network bandwidth capacity. Shares are + formatted in percent with two decimal places. Shares are + calculated as means over the whole 24-hour interval. + + "dirreq-v2-resp" status=num,... NL + [At most once.] + "dirreq-v3-resp" status=nul,... NL + [At most once.] + + List of mappings from response statuses to the number of requests + for v2/v3 network statuses that were answered with that response + status, rounded up to the nearest multiple of 4. Only response + statuses with at least 1 response are reported. New response + statuses can be added at any time. The current list of response + statuses is as follows: + + "ok": a network status request is answered; this number + corresponds to the sum of all requests as reported in + "dirreq-v2-reqs" or "dirreq-v3-reqs", respectively, before + rounding up. + "not-enough-sigs: a version 3 network status is not signed by a + sufficient number of requested authorities. + "unavailable": a requested network status object is unavailable. + "not-found": a requested network status is not found. + "not-modified": a network status has not been modified since the + If-Modified-Since time that is included in the request. + "busy": the directory is busy. + + "dirreq-v2-direct-dl" key=val,... NL + [At most once.] + "dirreq-v3-direct-dl" key=val,... NL + [At most once.] + "dirreq-v2-tunneled-dl" key=val,... NL + [At most once.] + "dirreq-v3-tunneled-dl" key=val,... NL + [At most once.] + + List of statistics about possible failures in the download process + of v2/v3 network statuses. Requests are either "direct" + HTTP-encoded requests over the relay's directory port, or + "tunneled" requests using a BEGIN_DIR cell over the relay's OR + port. The list of possible statistics can change, and statistics + can be left out from reporting. The current list of statistics is + as follows: + + Successful downloads and failures: + + "complete": a client has finished the download successfully. + "timeout": a download did not finish within 10 minutes after + starting to send the response. + "running": a download is still running at the end of the + measurement period for less than 10 minutes after starting to + send the response. + + Download times: + + "min", "max": smallest and largest measured bandwidth in B/s. + "d[1-4,6-9]": 1st to 4th and 6th to 9th decile of measured + bandwidth in B/s. For a given decile i, i/10 of all downloads + had a smaller bandwidth than di, and (10-i)/10 of all downloads + had a larger bandwidth than di. + "q[1,3]": 1st and 3rd quartile of measured bandwidth in B/s. One + fourth of all downloads had a smaller bandwidth than q1, one + fourth of all downloads had a larger bandwidth than q3, and the + remaining half of all downloads had a bandwidth between q1 and + q3. + "md": median of measured bandwidth in B/s. Half of the downloads + had a smaller bandwidth than md, the other half had a larger + bandwidth than md. + + +Entry guard statistics: + + Entry guard statistics include the number of clients per country and + per day that are connecting directly to an entry guard. + + Entry guard statistics are important to learn more about the + distribution of clients to countries. In the future, this knowledge + can be useful to detect if there are or start to be any restrictions + for clients connecting from specific countries. + + The information which client connects to a given entry guard is very + sensitive. This information must not be combined with the information + what contents are leaving the network at the exit nodes. Therefore, + entry guard statistics need to be aggregated to prevent them from + becoming useful for de-anonymization. Aggregation includes resolving + IP addresses to country codes, counting events over 24-hour intervals, + and rounding up numbers to the next multiple of 8. + + "entry-stats-end" YYYY-MM-DD HH:MM:SS (NSEC s) NL + [At most once.] + + YYYY-MM-DD HH:MM:SS defines the end of the included measurement + interval of length NSEC seconds (86400 seconds by default). + + An "entry-stats-end" line, as well as any other "entry-*" + line, is first added after the relay has been running for at least + 24 hours. + + "entry-ips" CC=N,CC=N,... NL + [At most once.] + + List of mappings from two-letter country codes to the number of + unique IP addresses that have connected from that country to the + relay and which are no known other relays, rounded up to the + nearest multiple of 8. + + +Cell statistics: + + The third type of statistics have to do with the time that cells spend + in circuit queues. In order to gather these statistics, the relay + memorizes when it puts a given cell in a circuit queue and when this + cell is flushed. The relay further notes the life time of the circuit. + These data are sufficient to determine the mean number of cells in a + queue over time and the mean time that cells spend in a queue. + + Cell statistics are necessary to learn more about possible reasons for + the poor network performance of the Tor network, especially high + latencies. The same statistics are also useful to determine the + effects of design changes by comparing today's data with future data. + + There are basically no privacy concerns from measuring cell + statistics, regardless of a node being an entry, middle, or exit node. + + "cell-stats-end" YYYY-MM-DD HH:MM:SS (NSEC s) NL + [At most once.] + + YYYY-MM-DD HH:MM:SS defines the end of the included measurement + interval of length NSEC seconds (86400 seconds by default). + + A "cell-stats-end" line, as well as any other "cell-*" line, + is first added after the relay has been running for at least 24 + hours. + + "cell-processed-cells" num,...,num NL + [At most once.] + + Mean number of processed cells per circuit, subdivided into + deciles of circuits by the number of cells they have processed in + descending order from loudest to quietest circuits. + + "cell-queued-cells" num,...,num NL + [At most once.] + + Mean number of cells contained in queues by circuit decile. These + means are calculated by 1) determining the mean number of cells in + a single circuit between its creation and its termination and 2) + calculating the mean for all circuits in a given decile as + determined in "cell-processed-cells". Numbers have a precision of + two decimal places. + + "cell-time-in-queue" num,...,num NL + [At most once.] + + Mean time cells spend in circuit queues in milliseconds. Times are + calculated by 1) determining the mean time cells spend in the + queue of a single circuit and 2) calculating the mean for all + circuits in a given decile as determined in + "cell-processed-cells". + + "cell-circuits-per-decile" num NL + [At most once.] + + Mean number of circuits that are included in any of the deciles, + rounded up to the next integer. + + +Exit statistics: + + The last type of statistics affects exit nodes counting the number of + bytes written and read and the number of streams opened per port and + per 24 hours. Exit port statistics can be measured from looking of + headers of BEGIN and DATA cells. A BEGIN cell contains the exit port + that is required for the exit node to open a new exit stream. + Subsequent DATA cells coming from the client or being sent back to the + client contain a length field stating how many bytes of application + data are contained in the cell. + + Exit port statistics are important to measure in order to identify + possible load-balancing problems with respect to exit policies. Exit + nodes that permit more ports than others are very likely overloaded + with traffic for those ports plus traffic for other ports. Improving + load balancing in the Tor network improves the overall utilization of + bandwidth capacity. + + Exit traffic is one of the most sensitive parts of network data in the + Tor network. Even though these statistics do not require looking at + traffic contents, statistics are aggregated so that they are not + useful for de-anonymizing users. Only those ports are reported that + have seen at least 0.1% of exiting or incoming bytes, numbers of bytes + are rounded up to full kibibytes (KiB), and stream numbers are rounded + up to the next multiple of 4. + + "exit-stats-end" YYYY-MM-DD HH:MM:SS (NSEC s) NL + [At most once.] + + YYYY-MM-DD HH:MM:SS defines the end of the included measurement + interval of length NSEC seconds (86400 seconds by default). + + An "exit-stats-end" line, as well as any other "exit-*" line, is + first added after the relay has been running for at least 24 hours + and only if the relay permits exiting (where exiting to a single + port and IP address is sufficient). + + "exit-kibibytes-written" port=N,port=N,... NL + [At most once.] + "exit-kibibytes-read" port=N,port=N,... NL + [At most once.] + + List of mappings from ports to the number of kibibytes that the + relay has written to or read from exit connections to that port, + rounded up to the next full kibibyte. + + "exit-streams-opened" port=N,port=N,... NL + [At most once.] + + List of mappings from ports to the number of opened exit streams + to that port, rounded up to the nearest multiple of 4. + + +Implementation notes: + + Right now, relays that are configured accordingly write similar + statistics to those described in this proposal to disk every 24 hours. + With this proposal being implemented, relays include the contents of + these files in extra-info documents. + + The following steps are necessary to implement this proposal: + + 1. The current format of [dirreq|entry|buffer|exit]-stats files needs + to be adapted to the description in this proposal. This step + basically means renaming keywords. + + 2. The timing of writing the four *-stats files should be unified, so + that they are written exactly after 24 hours after starting the + relay. Right now, the measurement intervals for dirreq, entry, and + exit stats starts with the first observed request, and files are + written when observing the first request that occurs more than 24 + hours after the beginning of the measurement interval. With this + proposal, the measurement intervals should all start at the same + time, and files should be written exactly 24 hours later. + + 3. It is advantageous to cache statistics in local files in the data + directory until they are included in extra-info documents. The + reason is that the 24-hour measurement interval can be very + different from the 18-hour publication interval of extra-info + documents. When a relay crashed after finishing a measurement + interval, but before publishing the next extra-info document, + statistics would get lost. Therefore, statistics are written to + disk when finishing a measurement interval and read from disk when + generating an extra-info document. As a result, the *-stats files + need to be overwritten after 24 hours, rather than appending new + statistics to them. Further, the contents of the *-stats files need + to be checked in the process of generating extra-info documents. + + 4. With the statistics patches being tested, the ./configure options + should be removed and the statistics code be compiled by default. + It is still required for relay operators to add configuration + options (DirReqStatistics, ExitPortStatistics, etc.) to enable + gathering statistics. However, in the near future, statistics shall + be enabled gathered by all relays by default, where requiring a + ./configure option would be a barrier for many relay operators. From f80537e3e079ffa18f3b42aff05848fe0407171a Mon Sep 17 00:00:00 2001 From: Karsten Loesing Date: Fri, 24 Jul 2009 01:13:33 +0200 Subject: [PATCH 119/187] Fix unit tests. Now that we require EntryStatistics to be 1 for counting connecting clients, unit tests need to set that config option, too. Reported by Sebastian Hahn. --- src/or/test.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/or/test.c b/src/or/test.c index b2a70eadb3..3103eed828 100644 --- a/src/or/test.c +++ b/src/or/test.c @@ -4799,6 +4799,7 @@ main(int c, char**v) setup_directory(); options_init(options); options->DataDirectory = tor_strdup(temp_dir); + options->EntryStatistics = 1; if (set_options(options, &errmsg) < 0) { printf("Failed to set initial options: %s\n", errmsg); tor_free(errmsg); From 698ec8d7b97b048cdee1ba72fda0e08292cb1a7e Mon Sep 17 00:00:00 2001 From: Roger Dingledine Date: Fri, 24 Jul 2009 18:58:09 -0400 Subject: [PATCH 120/187] clean up the 0.2.2.1-alpha changelog entry --- ChangeLog | 57 ++++++++++++++++++++----------------------------------- 1 file changed, 21 insertions(+), 36 deletions(-) diff --git a/ChangeLog b/ChangeLog index 81bf399a36..acc6e141b0 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,13 +1,27 @@ -Changes in version 0.2.2.1-alpha - 2009-??-?? - o Security fixes: - - Fix an edge case where a malicious exit relay could convince a - controller that the client's DNS question resolves to an internal IP - address. Bug found and fixed by "optimist"; bugfix on 0.1.2.8-beta. - +Changes in version 0.2.2.1-alpha - 2009-07-2? o Major features: - Add support for dynamic OpenSSL hardware crypto acceleration engines via new AccelName and AccelDir options. + o New options for gathering stats safely: + - Directories that configure with --enable-dirreq-stats and set + "DirReqStatistics 1" write directory request stats to disk every + 24 hours. As compared to the --enable-geoip-stats flag in 0.2.1.x, + there are a few improvements: 1) stats are written to disk exactly + every 24 hours; 2) estimated shares of v2 and v3 requests are + determined as mean values, not at the end of a measurement period; + 3) unresolved requests are listed with country code '??'; + 4) directories also measure download times. + - Exit nodes that configure with --enable-exit-stats and set + "ExitPortStatistics 1" write statistics on the number of exit + streams and transferred bytes per port to disk every 24 hours. + - Relays that configure with --enable-buffer-stats and set + "CellStatistics 1" write statistics to disk every 24 hours on how + long cells spend in their circuit queues. + - Entry nodes that configure with --enable-entry-stats and set + "EntryStatistics 1" write statistics to disk every 24 hours on + the rough number and origins of connecting clients. + o Minor features: - New --digests command-line switch to output the digests of the source files Tor was built with. @@ -15,42 +29,13 @@ Changes in version 0.2.2.1-alpha - 2009-??-?? - The memarea code now uses a sentinel value at the end of each area to make sure nothing writes beyond the end of an area. This might help debug some conceivable causes of bug 930. - - Directories that are configured with the --enable-dirreq-stats flag - and have "DirReqStatistics 1" set write directory request stats to - disk every 24 hours. As compared to the --enable-geoip-stats flag - in 0.2.1.x, there are a few improvements: 1) stats are written to - disk exactly every 24 hours; 2) estimated shares of v2 and v3 - requests are determined as mean values, not at the end of a - measurement period; 3) unresolved requests are listed with country - code '??'; 4) directories also measure download times. - - Exit nodes can write statistics on the number of exit streams and - transferred bytes per port to disk every 24 hours. To enable this, - run configure with the --enable-exit-stats option, and set - "ExitPortStatistics 1" in your torrc. - - Relays write statistics on how long cells spend in their circuit - queues to disk every 24 hours. To enable this, run configure with - the --enable-buffer-stats option, and set "CellStatistics 1" in your - torrc. - - Entry nodes can write statistics on the rough number and origins of - connecting clients to disk every 24 hours. To enable this, run - configure with the --enable-entry-stats option, and set - "EntryStatistics 1" in your torrc. - Time and memory units in the configuration file can now be set to fractional units. For example, "2.5 MB" is now a valid value for AccountingMax. - o Minor bugfixes - - Hidden service clients didn't use a cached service descriptor that - was older than 15 minutes, but wouldn't fetch a new one either. Now, - use a cached descriptor no matter how old it is and only fetch a new - one when all introduction points fail. Fix for bug 997. Patch from - Marcus Griep. - - Fix refetching of hidden service descriptors when all introduction - points have turned out to not work. Fixes more of bug 997. - o Deprecated and removed features: - The controller no longer accepts the old obsolete "addr-mappings/" - GETINFO value. + or "unregistered-servers-" GETINFO values. - Hidden services no longer publish version 0 descriptors, and clients do not request or use version 0 descriptors. However, the authorities still accept and serve version 0 descriptors when contacted by older From 7b716878cb6837756bd65ed788e5d8d89d8af56c Mon Sep 17 00:00:00 2001 From: Karsten Loesing Date: Mon, 27 Jul 2009 16:23:53 +0200 Subject: [PATCH 121/187] Fix dirreq and cell stats on 32-bit architectures. When determining how long directory requests take or how long cells spend in queues, we were comparing timestamps on microsecond detail only to convert results to second or millisecond detail later on. But on 32-bit architectures this means that 2^31 microseconds only cover time differences of up to 36 minutes. Instead, compare timestamps on millisecond detail. --- src/common/util.c | 21 ++++++++++++++++++++- src/common/util.h | 1 + src/or/geoip.c | 6 +++--- src/or/relay.c | 3 ++- 4 files changed, 26 insertions(+), 5 deletions(-) diff --git a/src/common/util.c b/src/common/util.c index 0e7f59e620..234180c4d4 100644 --- a/src/common/util.c +++ b/src/common/util.c @@ -1032,7 +1032,8 @@ tv_udiff(const struct timeval *start, const struct timeval *end) long secdiff = end->tv_sec - start->tv_sec; if (labs(secdiff+1) > LONG_MAX/1000000) { - log_warn(LD_GENERAL, "comparing times too far apart."); + log_warn(LD_GENERAL, "comparing times on microsecond detail too far " + "apart: %ld seconds", secdiff); return LONG_MAX; } @@ -1040,6 +1041,24 @@ tv_udiff(const struct timeval *start, const struct timeval *end) return udiff; } +/** Return the number of milliseconds elapsed between *start and *end. + */ +long +tv_mdiff(const struct timeval *start, const struct timeval *end) +{ + long mdiff; + long secdiff = end->tv_sec - start->tv_sec; + + if (labs(secdiff+1) > LONG_MAX/1000) { + log_warn(LD_GENERAL, "comparing times on millisecond detail too far " + "apart: %ld seconds", secdiff); + return LONG_MAX; + } + + mdiff = secdiff*1000L + (end->tv_usec - start->tv_usec) / 1000L; + return mdiff; +} + /** Yield true iff y is a leap-year. */ #define IS_LEAPYEAR(y) (!(y % 4) && ((y % 100) || !(y % 400))) /** Helper: Return the number of leap-days between Jan 1, y1 and Jan 1, y2. */ diff --git a/src/common/util.h b/src/common/util.h index 1c5643be39..c7741a64ac 100644 --- a/src/common/util.h +++ b/src/common/util.h @@ -211,6 +211,7 @@ int base16_decode(char *dest, size_t destlen, const char *src, size_t srclen); /* Time helpers */ long tv_udiff(const struct timeval *start, const struct timeval *end); +long tv_mdiff(const struct timeval *start, const struct timeval *end); time_t tor_timegm(struct tm *tm); #define RFC1123_TIME_LEN 29 void format_rfc1123_time(char *buf, time_t t); diff --git a/src/or/geoip.c b/src/or/geoip.c index 3336401220..5c13154205 100644 --- a/src/or/geoip.c +++ b/src/or/geoip.c @@ -739,16 +739,16 @@ geoip_get_dirreq_history(geoip_client_action_t action, } else { if (ent->completed) { uint32_t *bytes_per_second = tor_malloc_zero(sizeof(uint32_t)); - uint32_t time_diff = (uint32_t) tv_udiff(&ent->request_time, + uint32_t time_diff = (uint32_t) tv_mdiff(&ent->request_time, &ent->completion_time); if (time_diff == 0) time_diff = 1; /* Avoid DIV/0; "instant" answers are impossible * anyway by law of nature or something.. */ - *bytes_per_second = 1000000 * ent->response_size / time_diff; + *bytes_per_second = 1000 * ent->response_size / time_diff; smartlist_add(dirreq_times, bytes_per_second); complete++; } else { - if (tv_udiff(&ent->request_time, &now) / 1000000 > DIRREQ_TIMEOUT) + if (tv_mdiff(&ent->request_time, &now) / 1000 > DIRREQ_TIMEOUT) timeouts++; else running++; diff --git a/src/or/relay.c b/src/or/relay.c index 098b95253e..66b613cca0 100644 --- a/src/or/relay.c +++ b/src/or/relay.c @@ -1836,7 +1836,8 @@ connection_or_flush_from_first_active_circuit(or_connection_t *conn, int max, or_circuit_t *orcirc = TO_OR_CIRCUIT(circ); tor_gettimeofday(&flushed_from_queue); cell_waiting_time = (uint32_t) - (tv_udiff(&cell->packed_timeval, &flushed_from_queue) / 1000); + tv_mdiff(&cell->packed_timeval, &flushed_from_queue); + orcirc->total_cell_waiting_time += cell_waiting_time; orcirc->processed_cells++; } From 6249b0fd7757293876549c58c6cfe351d44a1d11 Mon Sep 17 00:00:00 2001 From: Roger Dingledine Date: Tue, 28 Jul 2009 18:34:35 -0400 Subject: [PATCH 122/187] Fix a signed/unsigned compile warning in 0.2.1.19 --- ChangeLog | 5 +++++ src/or/config.c | 8 ++++---- src/or/or.h | 4 ++-- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/ChangeLog b/ChangeLog index ed76e22f95..4fb38cf22f 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,8 @@ +Changes in version 0.2.1.20 - 2009-??-?? + o Minor bugfixes: + - Fix a signed/unsigned compile warning in 0.2.1.19. + + Changes in version 0.2.1.19 - 2009-07-28 o Major bugfixes: - Make accessing hidden services on 0.2.1.x work right diff --git a/src/or/config.c b/src/or/config.c index 3f45b1e5e2..fa986a6fc2 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -1224,10 +1224,10 @@ options_need_geoip_info(or_options_t *options, const char **reason_out) /** Return the bandwidthrate that we are going to report to the authorities * based on the config options. */ -int +uint32_t get_effective_bwrate(or_options_t *options) { - int bw = (int)options->BandwidthRate; + uint32_t bw = (int)options->BandwidthRate; if (bw > options->MaxAdvertisedBandwidth) bw = (int)options->MaxAdvertisedBandwidth; if (options->RelayBandwidthRate > 0 && bw > options->RelayBandwidthRate) @@ -1237,10 +1237,10 @@ get_effective_bwrate(or_options_t *options) /** Return the bandwidthburst that we are going to report to the authorities * based on the config options. */ -int +uint32_t get_effective_bwburst(or_options_t *options) { - int bw = (int)options->BandwidthBurst; + uint32_t bw = (int)options->BandwidthBurst; if (options->RelayBandwidthBurst > 0 && bw > options->RelayBandwidthBurst) bw = (int)options->RelayBandwidthBurst; return bw; diff --git a/src/or/or.h b/src/or/or.h index 1dcff28d6d..319b3a9d10 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -2926,8 +2926,8 @@ int options_need_geoip_info(or_options_t *options, const char **reason_out); int getinfo_helper_config(control_connection_t *conn, const char *question, char **answer); -int get_effective_bwrate(or_options_t *options); -int get_effective_bwburst(or_options_t *options); +uint32_t get_effective_bwrate(or_options_t *options); +uint32_t get_effective_bwburst(or_options_t *options); #ifdef CONFIG_PRIVATE /* Used only by config.c and test.c */ From 6a960d515f35433344fc65ffdf4d58334bb33c3f Mon Sep 17 00:00:00 2001 From: Roger Dingledine Date: Tue, 28 Jul 2009 20:44:51 -0400 Subject: [PATCH 123/187] credit optimist for the bug 1038 diagnosis also bring the release notes up to date --- ChangeLog | 14 ++++++++++++-- ReleaseNotes | 31 +++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index 4fb38cf22f..132c81597f 100644 --- a/ChangeLog +++ b/ChangeLog @@ -4,9 +4,13 @@ Changes in version 0.2.1.20 - 2009-??-?? Changes in version 0.2.1.19 - 2009-07-28 + Tor 0.2.1.19 fixes a major bug with accessing and providing hidden + services on Tor 0.2.1.3-alpha through 0.2.1.18. + o Major bugfixes: - - Make accessing hidden services on 0.2.1.x work right - again. Bugfix on 0.2.1.3-alpha; workaround for bug 1038. + - Make accessing hidden services on 0.2.1.x work right again. + Bugfix on 0.2.1.3-alpha; workaround for bug 1038. Diagnosis and + part of patch provided by "optimist". o Minor features: - When a relay/bridge is writing out its identity key fingerprint to @@ -25,6 +29,12 @@ Changes in version 0.2.1.19 - 2009-07-28 Changes in version 0.2.1.18 - 2009-07-24 + Tor 0.2.1.18 lays the foundations for performance improvements, + adds status events to help users diagnose bootstrap problems, adds + optional authentication/authorization for hidden services, fixes a + variety of potential anonymity problems, and includes a huge pile of + other features and bug fixes. + o Build fixes: - Add LIBS=-lrt to Makefile.am so the Tor RPMs use a static libevent. diff --git a/ReleaseNotes b/ReleaseNotes index f4e1a00cfb..bd2e3090a3 100644 --- a/ReleaseNotes +++ b/ReleaseNotes @@ -3,7 +3,38 @@ This document summarizes new features and bugfixes in each stable release of Tor. If you want to see more detailed descriptions of the changes in each development snapshot, see the ChangeLog file. +Changes in version 0.2.1.19 - 2009-07-28 + Tor 0.2.1.19 fixes a major bug with accessing and providing hidden + services. + + o Major bugfixes: + - Make accessing hidden services on 0.2.1.x work right again. + Bugfix on 0.2.1.3-alpha; workaround for bug 1038. Diagnosis and + part of patch provided by "optimist". + + o Minor features: + - When a relay/bridge is writing out its identity key fingerprint to + the "fingerprint" file and to its logs, write it without spaces. Now + it will look like the fingerprints in our bridges documentation, + and confuse fewer users. + + o Minor bugfixes: + - Relays no longer publish a new server descriptor if they change + their MaxAdvertisedBandwidth config option but it doesn't end up + changing their advertised bandwidth numbers. Bugfix on 0.2.0.28-rc; + fixes bug 1026. Patch from Sebastian. + - Avoid leaking memory every time we get a create cell but we have + so many already queued that we refuse it. Bugfix on 0.2.0.19-alpha; + fixes bug 1034. Reported by BarkerJr. + + Changes in version 0.2.1.18 - 2009-07-24 + Tor 0.2.1.18 lays the foundations for performance improvements, + adds status events to help users diagnose bootstrap problems, adds + optional authentication/authorization for hidden services, fixes a + variety of potential anonymity problems, and includes a huge pile of + other features and bug fixes. + o Major features (clients): - Start sending "bootstrap phase" status events to the controller, so it can keep the user informed of progress fetching directory From 6bb10a28fb3f0286c61bc39b5fc2ced4aac43a95 Mon Sep 17 00:00:00 2001 From: Roger Dingledine Date: Thu, 30 Jul 2009 03:41:02 -0400 Subject: [PATCH 124/187] Patch from atagar to fix control-spec Specifically, admit that the "newconsensus" event exists, and status/reachability has secretly been status/reachability-succeeded all along, but nobody used it so we didn't notice. --- doc/spec/control-spec.txt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/spec/control-spec.txt b/doc/spec/control-spec.txt index 4fdf409742..20952996b2 100644 --- a/doc/spec/control-spec.txt +++ b/doc/spec/control-spec.txt @@ -219,7 +219,7 @@ "INFO" / "NOTICE" / "WARN" / "ERR" / "NEWDESC" / "ADDRMAP" / "AUTHDIR_NEWDESCS" / "DESCCHANGED" / "STATUS_GENERAL" / "STATUS_CLIENT" / "STATUS_SERVER" / "GUARD" / "NS" / "STREAM_BW" / - "CLIENTS_SEEN" + "CLIENTS_SEEN" / "NEWCONSENSUS" Any events *not* listed in the SETEVENTS line are turned off; thus, sending SETEVENTS with an empty body turns off all event reporting. @@ -562,14 +562,14 @@ states. See Section 4.1.10 for explanations. (Only a few of the status events are available as getinfo's currently. Let us know if you want more exposed.) - "status/reachability/or" + "status/reachability-succeeded/or" 0 or 1, depending on whether we've found our ORPort reachable. - "status/reachability/dir" + "status/reachability-succeeded/dir" 0 or 1, depending on whether we've found our DirPort reachable. - "status/reachability" + "status/reachability-succeeded" "OR=" ("0"/"1") SP "DIR=" ("0"/"1") - Combines status/reachability/*; controllers MUST ignore unrecognized - elements in this entry. + Combines status/reachability-succeeded/*; controllers MUST ignore + unrecognized elements in this entry. "status/bootstrap-phase" Returns the most recent bootstrap phase status event sent. Specifically, it returns a string starting with either From efe966944d6f8fd4374ccc19b10adfe762034f47 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 30 Jul 2009 09:15:07 -0400 Subject: [PATCH 125/187] Fix signed/unsigned comparison warnings in get_effective_bw(rate|burst) --- src/or/config.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/or/config.c b/src/or/config.c index 9f9fef9d9a..cedd7f7529 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -1224,12 +1224,12 @@ options_need_geoip_info(or_options_t *options, const char **reason_out) int get_effective_bwrate(or_options_t *options) { - int bw = (int)options->BandwidthRate; + uint64_t bw = options->BandwidthRate; if (bw > options->MaxAdvertisedBandwidth) - bw = (int)options->MaxAdvertisedBandwidth; + bw = options->MaxAdvertisedBandwidth; if (options->RelayBandwidthRate > 0 && bw > options->RelayBandwidthRate) - bw = (int)options->RelayBandwidthRate; - return bw; + bw = options->RelayBandwidthRate; + return (int)bw; } /** Return the bandwidthburst that we are going to report to the authorities @@ -1237,10 +1237,10 @@ get_effective_bwrate(or_options_t *options) int get_effective_bwburst(or_options_t *options) { - int bw = (int)options->BandwidthBurst; + uint64_t bw = options->BandwidthBurst; if (options->RelayBandwidthBurst > 0 && bw > options->RelayBandwidthBurst) - bw = (int)options->RelayBandwidthBurst; - return bw; + bw = options->RelayBandwidthBurst; + return (int)bw; } /** Fetch the active option list, and take actions based on it. All of the From 4577bda7669885c077624f99657520b7b0f6f96b Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 30 Jul 2009 10:14:12 -0400 Subject: [PATCH 126/187] Cleaner fix for get_effective_bw(rate|burst), with comment on why it is ok. --- src/or/config.c | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/or/config.c b/src/or/config.c index fa986a6fc2..603f1b606d 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -1227,12 +1227,14 @@ options_need_geoip_info(or_options_t *options, const char **reason_out) uint32_t get_effective_bwrate(or_options_t *options) { - uint32_t bw = (int)options->BandwidthRate; + uint64_t bw = options->BandwidthRate; if (bw > options->MaxAdvertisedBandwidth) - bw = (int)options->MaxAdvertisedBandwidth; + bw = options->MaxAdvertisedBandwidth; if (options->RelayBandwidthRate > 0 && bw > options->RelayBandwidthRate) - bw = (int)options->RelayBandwidthRate; - return bw; + bw = options->RelayBandwidthRate; + + /* ensure_bandwidth_cap() makes sure that this cast can't overflow. */ + return (uint32_t)bw; } /** Return the bandwidthburst that we are going to report to the authorities @@ -1240,10 +1242,11 @@ get_effective_bwrate(or_options_t *options) uint32_t get_effective_bwburst(or_options_t *options) { - uint32_t bw = (int)options->BandwidthBurst; + uint64_t bw = options->BandwidthBurst; if (options->RelayBandwidthBurst > 0 && bw > options->RelayBandwidthBurst) - bw = (int)options->RelayBandwidthBurst; - return bw; + bw = options->RelayBandwidthBurst; + /* ensure_bandwidth_cap() makes sure that this cast can't overflow. */ + return (uint32_t)bw; } /** Fetch the active option list, and take actions based on it. All of the From fe36f69ce8e87dba51385e191c2a266578baa810 Mon Sep 17 00:00:00 2001 From: Karsten Loesing Date: Sat, 1 Aug 2009 11:36:51 +0200 Subject: [PATCH 127/187] Make configuration of hidden services with authorization somewhat clearer. --- doc/tor.1.in | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/doc/tor.1.in b/doc/tor.1.in index d85747958b..1a72ebd09f 100644 --- a/doc/tor.1.in +++ b/doc/tor.1.in @@ -476,13 +476,15 @@ used when \fBFascistFirewall\fR is set. This option is deprecated; use ReachableAddresses instead. (Default: 80, 443) .LP .TP -\fBHidServAuth \fR\fIonion-address\fR \fIauth-cookie\fP \fIservice-name\fR +\fBHidServAuth \fR\fIonion-address\fR \fIauth-cookie\fP [\fIservice-name\fR] Client authorization for a hidden service. Valid onion addresses contain 16 characters in a-z2-7 plus ".onion", and valid auth cookies contain 22 characters in A-Za-z0-9+/. The service name is only used for internal purposes, e.g., for Tor controllers. This option may be used multiple times for different hidden services. If a hidden service uses authorization and -this option is not set, the hidden service is not accessible. +this option is not set, the hidden service is not accessible. Hidden +services can be configured to require authorization using the +\fBHiddenServiceAuthorizeClient\fR option. .LP .TP \fBReachableAddresses \fR\fIADDR\fP[\fB/\fP\fIMASK\fP][:\fIPORT\fP]...\fP @@ -1305,7 +1307,8 @@ listed here are authorized to access the hidden service. Valid client names are 1 to 19 characters long and only use characters in A-Za-z0-9+-_ (no spaces). If this option is set, the hidden service is not accessible for clients without authorization any more. Generated authorization data -can be found in the hostname file. +can be found in the hostname file. Clients need to put this authorization +data in their configuration file using \fBHidServAuth\fR. .LP .TP \fBRendPostPeriod \fR\fIN\fR \fBseconds\fR|\fBminutes\fR|\fBhours\fR|\fBdays\fR|\fBweeks\fP From 0d12f12cf59cf8f16b89cd0397e189121c01b2ab Mon Sep 17 00:00:00 2001 From: Andrew Lewman Date: Sat, 1 Aug 2009 22:41:29 -0400 Subject: [PATCH 128/187] add receipts to be wiped. --- contrib/osx/uninstall_tor_bundle.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/osx/uninstall_tor_bundle.sh b/contrib/osx/uninstall_tor_bundle.sh index 9089407edb..b7c439429e 100755 --- a/contrib/osx/uninstall_tor_bundle.sh +++ b/contrib/osx/uninstall_tor_bundle.sh @@ -134,7 +134,7 @@ fi ## clean up echo ". Cleaning up" rm -rf $TEMP_BOM_CONTENTS -rm -rf /Library/Privoxy/ /Library/StartupItems/Privoxy/ /Library/Tor/ /Library/StartupItems/Tor/ /Library/Torbutton/ +rm -rf /Library/Privoxy/ /Library/StartupItems/Privoxy/ /Library/Tor/ /Library/StartupItems/Tor/ /Library/Torbutton/ /Library/Receipts/Privoxy.pkg /Library/Receipts/torbutton.pkg /Library/Receipts/Tor.pkg /Library/Receipts/Vidalia.pkg /Library/Receipts/TorStartup.pkg echo ". Finished" From 431202fb63af6ed872a4a7539bb1073c09cd7061 Mon Sep 17 00:00:00 2001 From: Andrew Lewman Date: Mon, 3 Aug 2009 12:40:43 -0400 Subject: [PATCH 129/187] add geoip file to uninstall --- contrib/tor-mingw.nsi.in | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/tor-mingw.nsi.in b/contrib/tor-mingw.nsi.in index 053c142159..eccf41984a 100644 --- a/contrib/tor-mingw.nsi.in +++ b/contrib/tor-mingw.nsi.in @@ -216,6 +216,7 @@ Function un.InstallFiles Delete "$INSTDIR\tor.ico" Delete "$SMSTARTUP\Tor.lnk" Delete "$INSTDIR\Uninstall.exe" + Delete "$INSTDIR\geoip" FunctionEnd Function un.InstallDirectories From 6fbdf635fa300fb4d9be32cd397dea3bb8cfa4fa Mon Sep 17 00:00:00 2001 From: Mike Perry Date: Fri, 31 Jul 2009 06:33:53 +0200 Subject: [PATCH 130/187] Implement measured bw parsing + unit tests. --- src/or/config.c | 1 + src/or/dirserv.c | 321 ++++++++++++++++++++++++++++++++++------- src/or/dirvote.c | 20 ++- src/or/networkstatus.c | 14 +- src/or/or.h | 36 ++++- src/or/routerparse.c | 18 ++- src/or/test.c | 78 ++++++++++ 7 files changed, 422 insertions(+), 66 deletions(-) diff --git a/src/or/config.c b/src/or/config.c index 1e559070ef..952ac00bef 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -337,6 +337,7 @@ static config_var_t _option_vars[] = { V(V3AuthDistDelay, INTERVAL, "5 minutes"), V(V3AuthNIntervalsValid, UINT, "3"), V(V3AuthUseLegacyKey, BOOL, "0"), + V(V3BandwidthsFile, FILENAME, NULL), VAR("VersioningAuthoritativeDirectory",BOOL,VersioningAuthoritativeDir, "0"), V(VirtualAddrNetwork, STRING, "127.192.0.0/10"), V(WarnPlaintextPorts, CSV, "23,109,110,143"), diff --git a/src/or/dirserv.c b/src/or/dirserv.c index acce309641..d98d8c4ebf 100644 --- a/src/or/dirserv.c +++ b/src/or/dirserv.c @@ -63,6 +63,9 @@ static signed_descriptor_t *get_signed_descriptor_by_fp(const char *fp, time_t publish_cutoff); static int dirserv_add_extrainfo(extrainfo_t *ei, const char **msg); +/************** Measured Bandwidth parsing code ******/ +#define MAX_MEASUREMENT_AGE (3*24*60*60) /* 3 days */ + /************** Fingerprint handling code ************/ #define FP_NAMED 1 /**< Listed in fingerprint file. */ @@ -1854,16 +1857,18 @@ version_from_platform(const char *platform) * which has at least buf_len free characters. Do NUL-termination. * Use the same format as in network-status documents. If version is * non-NULL, add a "v" line for the platform. Return 0 on success, -1 on - * failure. If first_line_only is true, don't include any flags - * or version line. + * failure. + * + * The format argument has three possible values: + * NS_V2 - Output an entry suitable for a V2 NS opinion document + * NS_V3_CONSENSUS - Output the first portion of a V3 NS consensus entry + * NS_V3_VOTE - Output a complete V3 NS vote + * NS_CONTROL_PORT - Output a NS docunent for the control port */ int routerstatus_format_entry(char *buf, size_t buf_len, routerstatus_t *rs, const char *version, - int first_line_only, int v2_format) -/* XXX: first_line_only and v2_format should probably be be both - * replaced by a single purpose parameter. - */ + routerstatus_format_type_t format) { int r; struct in_addr in; @@ -1894,7 +1899,12 @@ routerstatus_format_entry(char *buf, size_t buf_len, log_warn(LD_BUG, "Not enough space in buffer."); return -1; } - if (first_line_only) + + /* TODO: Maybe we want to pass in what we need to build the rest of + * this here, instead of in the caller. Then we could use the + * networkstatus_type_t values, with an additional control port value + * added -MP */ + if (format == NS_V3_CONSENSUS) return 0; cp = buf + strlen(buf); @@ -1931,62 +1941,81 @@ routerstatus_format_entry(char *buf, size_t buf_len, cp += strlen(cp); } - if (!v2_format) { + if (format != NS_V2) { routerinfo_t* desc = router_get_by_digest(rs->identity_digest); - /* Blow up more or less nicely if we didn't get anything or not the - * thing we expected. - */ - if (!desc) { - char id[HEX_DIGEST_LEN+1]; - char dd[HEX_DIGEST_LEN+1]; + if (format != NS_CONTROL_PORT) { + /* Blow up more or less nicely if we didn't get anything or not the + * thing we expected. + */ + if (!desc) { + char id[HEX_DIGEST_LEN+1]; + char dd[HEX_DIGEST_LEN+1]; - base16_encode(id, sizeof(id), rs->identity_digest, DIGEST_LEN); - base16_encode(dd, sizeof(dd), rs->descriptor_digest, DIGEST_LEN); - log_warn(LD_BUG, "Cannot get any descriptor for %s " - "(wanted descriptor %s).", - id, dd); - return -1; - }; - if (memcmp(desc->cache_info.signed_descriptor_digest, - rs->descriptor_digest, - DIGEST_LEN)) { - char rl_d[HEX_DIGEST_LEN+1]; - char rs_d[HEX_DIGEST_LEN+1]; - char id[HEX_DIGEST_LEN+1]; + base16_encode(id, sizeof(id), rs->identity_digest, DIGEST_LEN); + base16_encode(dd, sizeof(dd), rs->descriptor_digest, DIGEST_LEN); + log_warn(LD_BUG, "Cannot get any descriptor for %s " + "(wanted descriptor %s).", + id, dd); + return -1; + }; - base16_encode(rl_d, sizeof(rl_d), - desc->cache_info.signed_descriptor_digest, DIGEST_LEN); - base16_encode(rs_d, sizeof(rs_d), rs->descriptor_digest, DIGEST_LEN); - base16_encode(id, sizeof(id), rs->identity_digest, DIGEST_LEN); - log_err(LD_BUG, "descriptor digest in routerlist does not match " - "the one in routerstatus: %s vs %s " - "(router %s)\n", - rl_d, rs_d, id); + /* This assert can fire for the control port, because + * it can request NS documents before all descriptors + * have been fetched. */ + if (memcmp(desc->cache_info.signed_descriptor_digest, + rs->descriptor_digest, + DIGEST_LEN)) { + char rl_d[HEX_DIGEST_LEN+1]; + char rs_d[HEX_DIGEST_LEN+1]; + char id[HEX_DIGEST_LEN+1]; - tor_assert(!memcmp(desc->cache_info.signed_descriptor_digest, - rs->descriptor_digest, - DIGEST_LEN)); - }; + base16_encode(rl_d, sizeof(rl_d), + desc->cache_info.signed_descriptor_digest, DIGEST_LEN); + base16_encode(rs_d, sizeof(rs_d), rs->descriptor_digest, DIGEST_LEN); + base16_encode(id, sizeof(id), rs->identity_digest, DIGEST_LEN); + log_err(LD_BUG, "descriptor digest in routerlist does not match " + "the one in routerstatus: %s vs %s " + "(router %s)\n", + rl_d, rs_d, id); + + tor_assert(!memcmp(desc->cache_info.signed_descriptor_digest, + rs->descriptor_digest, + DIGEST_LEN)); + }; + } r = tor_snprintf(cp, buf_len - (cp-buf), "w Bandwidth=%d\n", router_get_advertised_bandwidth_capped(desc) / 1024); - if (r<0) { - log_warn(LD_BUG, "Not enough space in buffer."); - return -1; - } - cp += strlen(cp); - summary = policy_summarize(desc->exit_policy); - r = tor_snprintf(cp, buf_len - (cp-buf), "p %s\n", summary); if (r<0) { log_warn(LD_BUG, "Not enough space in buffer."); - tor_free(summary); return -1; } cp += strlen(cp); - tor_free(summary); + if (format == NS_V3_VOTE && rs->has_measured_bw) { + *--cp = '\0'; /* Kill "\n" */ + r = tor_snprintf(cp, buf_len - (cp-buf), + " Measured=%d\n", rs->measured_bw); + if (r<0) { + log_warn(LD_BUG, "Not enough space in buffer for weight line."); + return -1; + } + cp += strlen(cp); + } + + if (desc) { + summary = policy_summarize(desc->exit_policy); + r = tor_snprintf(cp, buf_len - (cp-buf), "p %s\n", summary); + if (r<0) { + log_warn(LD_BUG, "Not enough space in buffer."); + tor_free(summary); + return -1; + } + cp += strlen(cp); + tor_free(summary); + } } return 0; @@ -2189,6 +2218,194 @@ router_clear_status_flags(routerinfo_t *router) router->is_bad_exit = router->is_bad_directory = 0; } +#ifndef HAVE_STRTOK_R +/* + * XXX-MP: If a system lacks strtok_r and we use a non-reentrant strtok, + * we may introduce odd bugs if we call a codepath that also uses strtok + * and resets its internal state. Do we want to abandon use of strtok + * entirely for this reason? Roger mentioned smartlist_split and + * eat_whitespace() as alternatives. + */ +static char * +strtok_r(char *s, const char *delim, char **state) +{ + (void)state; + return strtok(s, delim); +} +#endif + +/** + * Helper function to parse out a line in the measured bandwidth file + * into a measured_bw_line_t output structure. Returns -1 on failure + * or 0 on success. + */ +int +measured_bw_line_parse(measured_bw_line_t *out, const char *orig_line) +{ + char *line = tor_strdup(orig_line); + char *cp = line; + int got_bw = 0; + int got_node_id = 0; + char *strtok_state; /* lame sauce d'jour */ + cp = strtok_r(cp, " \t", &strtok_state); + + if (!cp) { + log_warn(LD_DIRSERV, "Invalid line in bandwidth file: %s", + escaped(orig_line)); + tor_free(line); + return -1; + } + + if (orig_line[strlen(orig_line)-1] != '\n') { + log_warn(LD_DIRSERV, "Incomplete line in bandwidth file: %s", + escaped(orig_line)); + tor_free(line); + return -1; + } + + do { + if (strncasecmp(cp, "bw=", strlen("bw=")) == 0) { + char *endptr; + if (got_bw) { + log_warn(LD_DIRSERV, "Double bw= in bandwidth file line: %s", + escaped(orig_line)); + tor_free(line); + return -1; + } + cp+=strlen("bw="); + errno=0; + out->bw = strtol(cp, &endptr, 0); + if (errno || endptr == cp || (*endptr && !TOR_ISSPACE(*endptr)) || + out->bw < 0) { + log_warn(LD_DIRSERV, "Invalid bandwidth in bandwidth file line: %s", + escaped(orig_line)); + tor_free(line); + return -1; + } + got_bw=1; + } else if (strncasecmp(cp, "node_id=$", strlen("node_id=$")) == 0) { + if (got_node_id) { + log_warn(LD_DIRSERV, "Double node_id= in bandwidth file line: %s", + escaped(orig_line)); + tor_free(line); + return -1; + } + cp+=strlen("node_id=$"); + + if (strlen(cp) != HEX_DIGEST_LEN || + base16_decode(out->node_id, DIGEST_LEN, cp, HEX_DIGEST_LEN)) { + log_warn(LD_DIRSERV, "Invalid node_id in bandwidth file line: %s", + escaped(orig_line)); + tor_free(line); + return -1; + } + strncpy(out->node_hex, cp, sizeof(out->node_hex)); + got_node_id=1; + } + } while ((cp = strtok_r(NULL, " \t", &strtok_state))); + + if (got_bw && got_node_id) { + tor_free(line); + return 0; + } else { + log_warn(LD_DIRSERV, "Incomplete line in bandwidth file: %s", + escaped(orig_line)); + tor_free(line); + return -1; + } +} + +/** + * Helper function to apply a parsed measurement line to a list + * of bandwidth statuses. Returns true if a line is found, + * false otherwise. + */ +int +measured_bw_line_apply(measured_bw_line_t *parsed_line, + smartlist_t *routerstatuses) +{ + routerstatus_t *rs = NULL; + if (!routerstatuses) + return 0; + + rs = smartlist_bsearch(routerstatuses, parsed_line->node_id, + compare_digest_to_routerstatus_entry); + + if (rs) { + rs->has_measured_bw = 1; + rs->measured_bw = parsed_line->bw; + } else { + log_info(LD_DIRSERV, "Node ID %s not found in routerstatus list", + parsed_line->node_hex); + } + + return rs != NULL; +} + +/** + * Read the measured bandwidth file and apply it to the list of + * routerstatuses. Returns -1 on error, 0 otherwise. + */ +int +dirserv_read_measured_bandwidths(const char *from_file, + smartlist_t *routerstatuses) +{ + char line[256]; + FILE *fp = fopen(from_file, "r"); + int applied_lines = 0; + time_t file_time; + int ok; + if (fp == NULL) { + log_warn(LD_CONFIG, "Can't open bandwidth file at configured location: %s", + from_file); + return -1; + } + + fgets(line, sizeof(line), fp); + + if (line[strlen(line)-1] != '\n') { + log_warn(LD_DIRSERV, "Long or truncated time in bandwidth file: %s", + escaped(line)); + fclose(fp); + return -1; + } + + line[strlen(line)-1] = '\0'; + file_time = tor_parse_ulong(line, 10, 0, ULONG_MAX, &ok, NULL); + if (!ok) { + log_warn(LD_DIRSERV, "Non-integer time in bandwidth file: %s", + escaped(line)); + fclose(fp); + return -1; + } + + if ((time(NULL) - file_time) > MAX_MEASUREMENT_AGE) { + log_warn(LD_DIRSERV, "Bandwidth measurement file stale. Age: %u", + (unsigned)(time(NULL) - file_time)); + fclose(fp); + return -1; + } + + if (routerstatuses) + smartlist_sort(routerstatuses, compare_routerstatus_entries); + + while (!feof(fp)) { + measured_bw_line_t parsed_line; + fgets(line, sizeof(line), fp); + + if (measured_bw_line_parse(&parsed_line, line) != -1) { + if (measured_bw_line_apply(&parsed_line, routerstatuses) > 0) + applied_lines++; + } + } + + fclose(fp); + log_notice(LD_DIRSERV, + "Bandwidth measurement file successfully read. " + "Applied %d measurements.", applied_lines); + return 0; +} + /** Return a new networkstatus_t* containing our current opinion. (For v3 * authorities) */ networkstatus_t * @@ -2288,9 +2505,15 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_env_t *private_key, smartlist_add(routerstatuses, vrs); } }); + smartlist_free(routers); digestmap_free(omit_as_sybil, NULL); + if (options->V3BandwidthsFile) { + dirserv_read_measured_bandwidths(options->V3BandwidthsFile, + routerstatuses); + } + v3_out = tor_malloc_zero(sizeof(networkstatus_t)); v3_out->type = NS_TYPE_VOTE; @@ -2494,7 +2717,7 @@ generate_v2_networkstatus_opinion(void) if (digestmap_get(omit_as_sybil, ri->cache_info.identity_digest)) clear_status_flags_on_sybil(&rs); - if (routerstatus_format_entry(outp, endp-outp, &rs, version, 0, 1)) { + if (routerstatus_format_entry(outp, endp-outp, &rs, version, NS_V2)) { log_warn(LD_BUG, "Unable to print router status."); tor_free(version); goto done; diff --git a/src/or/dirvote.c b/src/or/dirvote.c index 409c2fed69..acb658bd23 100644 --- a/src/or/dirvote.c +++ b/src/or/dirvote.c @@ -103,6 +103,7 @@ format_networkstatus_vote(crypto_pk_env_t *private_signing_key, tor_snprintf(status, len, "network-status-version 3\n" "vote-status %s\n" + /* TODO-160: add 6 when ready */ "consensus-methods 1 2 3 4 5\n" "published %s\n" "valid-after %s\n" @@ -142,7 +143,7 @@ format_networkstatus_vote(crypto_pk_env_t *private_signing_key, SMARTLIST_FOREACH(v3_ns->routerstatus_list, vote_routerstatus_t *, vrs, { if (routerstatus_format_entry(outp, endp-outp, &vrs->status, - vrs->version, 0, 0) < 0) { + vrs->version, NS_V3_VOTE) < 0) { log_warn(LD_BUG, "Unable to print router status."); goto err; } @@ -701,7 +702,10 @@ networkstatus_compute_consensus(smartlist_t *votes, smartlist_t *versions = smartlist_create(); smartlist_t *exitsummaries = smartlist_create(); uint32_t *bandwidths = tor_malloc(sizeof(uint32_t) * smartlist_len(votes)); + uint32_t *measured_bws = tor_malloc(sizeof(uint32_t) * + smartlist_len(votes)); int num_bandwidths; + int num_mbws; int *n_voter_flags; /* n_voter_flags[j] is the number of flags that * votes[j] knows about. */ @@ -835,6 +839,7 @@ networkstatus_compute_consensus(smartlist_t *votes, smartlist_clear(chosen_flags); smartlist_clear(versions); num_bandwidths = 0; + num_mbws = 0; /* Okay, go through all the entries for this digest. */ SMARTLIST_FOREACH_BEGIN(votes, networkstatus_t *, v) { @@ -868,6 +873,9 @@ networkstatus_compute_consensus(smartlist_t *votes, } /* count bandwidths */ + if (rs->status.has_measured_bw) + measured_bws[num_mbws++] = rs->status.measured_bw; + if (rs->status.has_bandwidth) bandwidths[num_bandwidths++] = rs->status.bandwidth; } SMARTLIST_FOREACH_END(v); @@ -945,7 +953,10 @@ networkstatus_compute_consensus(smartlist_t *votes, } /* Pick a bandwidth */ - if (consensus_method >= 5 && num_bandwidths > 0) { + if (consensus_method >= 6 && num_mbws > 2) { + rs_out.has_bandwidth = 1; + rs_out.bandwidth = median_uint32(measured_bws, num_mbws); + } else if (consensus_method >= 5 && num_bandwidths > 0) { rs_out.has_bandwidth = 1; rs_out.bandwidth = median_uint32(bandwidths, num_bandwidths); } @@ -1036,7 +1047,8 @@ networkstatus_compute_consensus(smartlist_t *votes, /* Okay!! Now we can write the descriptor... */ /* First line goes into "buf". */ - routerstatus_format_entry(buf, sizeof(buf), &rs_out, NULL, 1, 0); + routerstatus_format_entry(buf, sizeof(buf), &rs_out, NULL, + NS_V3_CONSENSUS); smartlist_add(chunks, tor_strdup(buf)); /* Second line is all flags. The "\n" is missing. */ smartlist_add(chunks, @@ -1055,8 +1067,10 @@ networkstatus_compute_consensus(smartlist_t *votes, log_warn(LD_BUG, "Not enough space in buffer for weight line."); *buf = '\0'; } + smartlist_add(chunks, tor_strdup(buf)); }; + /* Now the exitpolicy summary line. */ if (rs_out.has_exitsummary) { char buf[MAX_POLICY_LINE_LEN+1]; diff --git a/src/or/networkstatus.c b/src/or/networkstatus.c index 97353c01da..b022999607 100644 --- a/src/or/networkstatus.c +++ b/src/or/networkstatus.c @@ -780,8 +780,8 @@ networkstatus_v2_list_clean(time_t now) /** Helper for bsearching a list of routerstatus_t pointers: compare a * digest in the key to the identity digest of a routerstatus_t. */ -static int -_compare_digest_to_routerstatus_entry(const void *_key, const void **_member) +int +compare_digest_to_routerstatus_entry(const void *_key, const void **_member) { const char *key = _key; const routerstatus_t *rs = *_member; @@ -794,7 +794,7 @@ routerstatus_t * networkstatus_v2_find_entry(networkstatus_v2_t *ns, const char *digest) { return smartlist_bsearch(ns->entries, digest, - _compare_digest_to_routerstatus_entry); + compare_digest_to_routerstatus_entry); } /** Return the entry in ns for the identity digest digest, or @@ -803,7 +803,7 @@ routerstatus_t * networkstatus_vote_find_entry(networkstatus_t *ns, const char *digest) { return smartlist_bsearch(ns->routerstatus_list, digest, - _compare_digest_to_routerstatus_entry); + compare_digest_to_routerstatus_entry); } /*XXXX make this static once functions are moved into this file. */ @@ -815,7 +815,7 @@ networkstatus_vote_find_entry_idx(networkstatus_t *ns, const char *digest, int *found_out) { return smartlist_bsearch_idx(ns->routerstatus_list, digest, - _compare_digest_to_routerstatus_entry, + compare_digest_to_routerstatus_entry, found_out); } @@ -868,7 +868,7 @@ router_get_consensus_status_by_id(const char *digest) if (!current_consensus) return NULL; return smartlist_bsearch(current_consensus->routerstatus_list, digest, - _compare_digest_to_routerstatus_entry); + compare_digest_to_routerstatus_entry); } /** Given a nickname (possibly verbose, possibly a hexadecimal digest), return @@ -1827,7 +1827,7 @@ char * networkstatus_getinfo_helper_single(routerstatus_t *rs) { char buf[RS_ENTRY_LEN+1]; - routerstatus_format_entry(buf, sizeof(buf), rs, NULL, 0, 1); + routerstatus_format_entry(buf, sizeof(buf), rs, NULL, NS_CONTROL_PORT); return tor_strdup(buf); } diff --git a/src/or/or.h b/src/or/or.h index 4f215d6448..bdbd3e5007 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -1509,6 +1509,9 @@ typedef struct routerstatus_t { unsigned int has_bandwidth:1; /**< The vote/consensus had bw info */ unsigned int has_exitsummary:1; /**< The vote/consensus had exit summaries */ + unsigned int has_measured_bw:1; /**< The vote/consensus had a measured bw */ + + uint32_t measured_bw; /**< Measured bandwidth (capacity) of the router */ uint32_t bandwidth; /**< Bandwidth (capacity) of the router as reported in * the vote/consensus, in kilobytes/sec. */ @@ -2539,6 +2542,9 @@ typedef struct { * migration purposes? */ int V3AuthUseLegacyKey; + /** Location of bandwidth measurement file */ + char *V3BandwidthsFile; + /** The length of time that we think an initial consensus should be fresh. * Only altered on testing networks. */ int TestingV3AuthInitialVotingInterval; @@ -3437,8 +3443,8 @@ download_status_mark_impossible(download_status_t *dl) * Running Stable Unnamed V2Dir Valid\n". */ #define MAX_FLAG_LINE_LEN 96 /** Length of "w" line for weighting. Currently at most - * "w Bandwidth=\n" */ -#define MAX_WEIGHT_LINE_LEN (13+10) + * "w Bandwidth= Measured=\n" */ +#define MAX_WEIGHT_LINE_LEN (12+10+10+10+1) /** Maximum length of an exit policy summary line. */ #define MAX_POLICY_LINE_LEN (3+MAX_EXITPOLICY_SUMMARY_LEN) /** Amount of space to allocate for each entry: r, s, and v lines. */ @@ -3521,13 +3527,34 @@ int dirserv_remove_old_statuses(smartlist_t *fps, time_t cutoff); int dirserv_have_any_serverdesc(smartlist_t *fps, int spool_src); size_t dirserv_estimate_data_size(smartlist_t *fps, int is_serverdescs, int compressed); +typedef enum { + NS_V2, NS_V3_CONSENSUS, NS_V3_VOTE, NS_CONTROL_PORT +} routerstatus_format_type_t; int routerstatus_format_entry(char *buf, size_t buf_len, routerstatus_t *rs, const char *platform, - int first_line_only, int v2_format); + routerstatus_format_type_t format); void dirserv_free_all(void); void cached_dir_decref(cached_dir_t *d); cached_dir_t *new_cached_dir(char *s, time_t published); +#ifdef DIRSERV_PRIVATE +typedef struct measured_bw_line_t { + char node_id[DIGEST_LEN]; + char node_hex[MAX_HEX_NICKNAME_LEN+1]; + long int bw; +} measured_bw_line_t; + +int +measured_bw_line_parse(measured_bw_line_t *out, const char *line); + +int +measured_bw_line_apply(measured_bw_line_t *parsed_line, + smartlist_t *routerstatuses); +#endif + +int dirserv_read_measured_bandwidths(const char *from_file, + smartlist_t *routerstatuses); + /********************************* dirvote.c ************************/ /** Lowest allowable value for VoteSeconds. */ @@ -3841,6 +3868,8 @@ int router_set_networkstatus_v2(const char *s, time_t arrived_at, v2_networkstatus_source_t source, smartlist_t *requested_fingerprints); void networkstatus_v2_list_clean(time_t now); +int compare_digest_to_routerstatus_entry(const void *_key, + const void **_member); routerstatus_t *networkstatus_v2_find_entry(networkstatus_v2_t *ns, const char *digest); routerstatus_t *networkstatus_vote_find_entry(networkstatus_t *ns, @@ -4705,6 +4734,7 @@ void sort_version_list(smartlist_t *lst, int remove_duplicates); void assert_addr_policy_ok(smartlist_t *t); void dump_distinct_digest_count(int severity); +int compare_routerstatus_entries(const void **_a, const void **_b); networkstatus_v2_t *networkstatus_v2_parse_from_string(const char *s); networkstatus_t *networkstatus_parse_vote_from_string(const char *s, const char **eos_out, diff --git a/src/or/routerparse.c b/src/or/routerparse.c index 8021158e31..c1a7fbcfae 100644 --- a/src/or/routerparse.c +++ b/src/or/routerparse.c @@ -1924,6 +1924,16 @@ routerstatus_parse_entry_from_string(memarea_t *area, goto err; } rs->has_bandwidth = 1; + } else if (!strcmpstart(tok->args[i], "Measured=")) { + int ok; + rs->measured_bw = tor_parse_ulong(strchr(tok->args[i], '=')+1, 10, + 0, UINT32_MAX, &ok, NULL); + if (!ok) { + log_warn(LD_DIR, "Invalid Measured Bandwidth %s", + escaped(tok->args[i])); + goto err; + } + rs->has_measured_bw = 1; } } } @@ -1966,8 +1976,8 @@ routerstatus_parse_entry_from_string(memarea_t *area, } /** Helper to sort a smartlist of pointers to routerstatus_t */ -static int -_compare_routerstatus_entries(const void **_a, const void **_b) +int +compare_routerstatus_entries(const void **_a, const void **_b) { const routerstatus_t *a = *_a, *b = *_b; return memcmp(a->identity_digest, b->identity_digest, DIGEST_LEN); @@ -2114,8 +2124,8 @@ networkstatus_v2_parse_from_string(const char *s) NULL, NULL, 0))) smartlist_add(ns->entries, rs); } - smartlist_sort(ns->entries, _compare_routerstatus_entries); - smartlist_uniq(ns->entries, _compare_routerstatus_entries, + smartlist_sort(ns->entries, compare_routerstatus_entries); + smartlist_uniq(ns->entries, compare_routerstatus_entries, _free_duplicate_routerstatus_entry); if (tokenize_string(area,s, NULL, footer_tokens, dir_footer_token_table,0)) { diff --git a/src/or/test.c b/src/or/test.c index 3103eed828..03351dcb79 100644 --- a/src/or/test.c +++ b/src/or/test.c @@ -3232,6 +3232,72 @@ test_dirutil(void) smartlist_free(sl); } +static void +test_dirutil_measured_bw(void) +{ + measured_bw_line_t mbwl; + int i; + const char *lines_pass[] = { + "node_id=$557365204145532d32353620696e73746561642e bw=1024\n", + "node_id=$557365204145532d32353620696e73746561642e\t bw=1024 \n", + " node_id=$557365204145532d32353620696e73746561642e bw=1024\n", + "\tnoise\tnode_id=$557365204145532d32353620696e73746561642e " + "bw=1024 junk=007\n", + "misc=junk node_id=$557365204145532d32353620696e73746561642e " + "bw=1024 junk=007\n", + "end" + }; + const char *lines_fail[] = { + /* Test possible python stupidity on input */ + "node_id=None bw=1024\n", + "node_id=$None bw=1024\n", + "node_id=$557365204145532d32353620696e73746561642e bw=None\n", + "node_id=$557365204145532d32353620696e73746561642e bw=1024.0\n", + "node_id=$557365204145532d32353620696e73746561642e bw=.1024\n", + "node_id=$557365204145532d32353620696e73746561642e bw=1.024\n", + "node_id=$557365204145532d32353620696e73746561642e bw=1024 bw=0\n", + "node_id=$557365204145532d32353620696e73746561642e bw=1024 bw=None\n", + "node_id=$557365204145532d32353620696e73746561642e bw=-1024\n", + /* Test incomplete writes due to race conditions, partial copies, etc */ + "node_i", + "node_i\n", + "node_id=", + "node_id=\n", + "node_id=$557365204145532d32353620696e73746561642e bw=", + "node_id=$557365204145532d32353620696e73746561642e bw=1024", + "node_id=$557365204145532d32353620696e73746561642e bw=\n", + "node_id=$557365204145532d32353620696e7374", + "node_id=$557365204145532d32353620696e7374\n", + "", + "\n", + " \n ", + " \n\n", + /* Test assorted noise */ + " node_id= ", + "node_id==$557365204145532d32353620696e73746561642e bw==1024\n", + "node_id=$55736520414552d32353620696e73746561642e bw=1024\n", + "node_id=557365204145532d32353620696e73746561642e bw=1024\n", + "node_id= $557365204145532d32353620696e73746561642e bw=0.23\n", + "end" + }; + + for (i = 0; strcmp(lines_fail[i], "end"); i++) { + //fprintf(stderr, "Testing: %s\n", lines_fail[i]); + test_assert(measured_bw_line_parse(&mbwl, lines_fail[i]) == -1); + } + + for (i = 0; strcmp(lines_pass[i], "end"); i++) { + //fprintf(stderr, "Testing: %s %d\n", lines_pass[i], TOR_ISSPACE('\n')); + test_assert(measured_bw_line_parse(&mbwl, lines_pass[i]) == 0); + test_assert(mbwl.bw == 1024); + test_assert(strcmp(mbwl.node_hex, + "557365204145532d32353620696e73746561642e") == 0); + } + +done: + return; +} + extern const char AUTHORITY_CERT_1[]; extern const char AUTHORITY_SIGNKEY_1[]; extern const char AUTHORITY_CERT_2[]; @@ -3512,6 +3578,17 @@ test_v3_networkstatus(void) test_eq(rs->dir_port, 0); test_eq(vrs->flags, U64_LITERAL(254)); // all flags except "authority." + { + measured_bw_line_t mbw; + memset(mbw.node_id, 33, sizeof(mbw.node_id)); + mbw.bw = 1024; + test_assert(measured_bw_line_apply(&mbw, + v1->routerstatus_list) == 1); + vrs = smartlist_get(v1->routerstatus_list, 2); + test_assert(vrs->status.has_measured_bw && + vrs->status.measured_bw == 1024); + } + /* Generate second vote. It disagrees on some of the times, * and doesn't list versions, and knows some crazy flags */ vote->published = now+1; @@ -4695,6 +4772,7 @@ static struct { ENT(onion_handshake), ENT(dir_format), ENT(dirutil), + SUBENT(dirutil, measured_bw), ENT(v3_networkstatus), ENT(policies), ENT(rend_fns), From da88e05edc8c7180b3eb1a3f4c9e953aa1e7ef5b Mon Sep 17 00:00:00 2001 From: Roger Dingledine Date: Sun, 5 Jul 2009 22:47:36 -0400 Subject: [PATCH 131/187] try loading the bandwidth measurement file on startup too, in case it's broken. --- src/or/config.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/or/config.c b/src/or/config.c index 952ac00bef..d0f6c8393b 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -3208,6 +3208,10 @@ options_validate(or_options_t *old_options, or_options_t *options, options->V3AuthoritativeDir)) REJECT("AuthoritativeDir is set, but none of " "(Bridge/HS/V1/V2/V3)AuthoritativeDir is set."); + /* If we have a v3bandwidthsfile and it's broken, complain on startup */ + if (options->V3BandwidthsFile && !old_options) { + dirserv_read_measured_bandwidths(options->V3BandwidthsFile, NULL); + } } if (options->AuthoritativeDir && !options->DirPort) From db297fb94485f7ffd5b244aeec7d200e5ba440b4 Mon Sep 17 00:00:00 2001 From: Mike Perry Date: Mon, 13 Jul 2009 19:21:00 -0700 Subject: [PATCH 132/187] Update dir-spec.txt to describe w line. "Measured=" is present in votes regardless of consensus method. --- doc/spec/dir-spec.txt | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/doc/spec/dir-spec.txt b/doc/spec/dir-spec.txt index 0a3b0c11bc..4bec7b00d0 100644 --- a/doc/spec/dir-spec.txt +++ b/doc/spec/dir-spec.txt @@ -1029,13 +1029,20 @@ descriptors if they would cause "v" lines to be over 128 characters long. - "w" SP "Bandwidth=" INT NL + "w" SP "Bandwidth=" INT [SP "Measured=" INT] NL [At most once.] An estimate of the bandwidth of this server, in an arbitrary unit (currently kilobytes per second). Used to weight router - selection. Other weighting keywords may be added later. + selection. + + Additionally, the Measured= keyword is present in votes by + participating bandwidth measurement authorites to indicate + a measured bandwidth currently produced by measuring stream + capacities. + + Other weighting keywords may be added later. Clients MUST ignore keywords they do not recognize. "p" SP ("accept" / "reject") SP PortList NL @@ -1178,6 +1185,13 @@ rate limit from the router descriptor. It is given in kilobytes per second, and capped at some arbitrary value (currently 10 MB/s). + The Measured= keyword on a "w" line vote is currently computed + by multiplying the previous published consensus bandwidth by the + ratio of the measured average node stream capacity to the network + average. If 3 or more authorities provide a Measured= keyword for + a router, the authorites produce a consensus containing a "w" + Bandwidth= keyword equal to the median of the Measured= votes. + The ports listed in a "p" line should be taken as those ports for which the router's exit policy permits 'most' addresses, ignoring any accept not for all addresses, ignoring all rejects for private @@ -1260,6 +1274,11 @@ one, breaking ties in favor of the lexicographically larger vote.) The port list is encoded as specified in 3.4.2. + * If consensus-method 6 or later is in use and if 3 or more + authorities provide a Measured= keyword in their votes for + a router, the authorities produce a consensus containing a + Bandwidth= keyword equal to the median of the Measured= votes. + The signatures at the end of a consensus document are sorted in ascending order by identity digest. @@ -1280,6 +1299,7 @@ "3" -- Added legacy ID key support to aid in authority ID key rollovers "4" -- No longer list routers that are not running in the consensus "5" -- adds support for "w" and "p" lines. + "6" -- Prefers measured bandwidth values rather than advertised Before generating a consensus, an authority must decide which consensus method to use. To do this, it looks for the highest version number From 011b7324366ec9b4c5546b42cb702b440ab7c672 Mon Sep 17 00:00:00 2001 From: Mike Perry Date: Thu, 25 Jun 2009 17:03:39 -0700 Subject: [PATCH 133/187] Update 161 to reflect current implementation. Also mention rounding step. --- .../161-computing-bandwidth-adjustments.txt | 72 ++++++++----------- 1 file changed, 31 insertions(+), 41 deletions(-) diff --git a/doc/spec/proposals/161-computing-bandwidth-adjustments.txt b/doc/spec/proposals/161-computing-bandwidth-adjustments.txt index b02dc64eb2..a50dba687e 100644 --- a/doc/spec/proposals/161-computing-bandwidth-adjustments.txt +++ b/doc/spec/proposals/161-computing-bandwidth-adjustments.txt @@ -42,31 +42,26 @@ Status: Open slices of 50 nodes each, grouped according to advertised node bandwidth. Two hop circuits are built using nodes from the same slice, and a large - file is downloaded via these circuits. For nodes in the first 15% of the - network, a 500K file will be used. For nodes in the next 15%, a 250K file - will be used. For nodes in next 15%, a 100K file will be used. The - remainder of the nodes will fetch a 75K file.[1] + file is downloaded via these circuits. The file sizes are set based + on node percentile rank as follows: + + 0-10: 4M + 10-20: 2M + 20-30: 1M + 30-50: 512k + 50-75: 256k + 75-100: 128k - This process is repeated 250 times, and average stream capacities are - assigned to each node from these results. - - In the future, a node generator type can be created to ensure that - each node is chosen to participate in an equal number of circuits, - and the selection will continue until every live node is chosen - to participate in at least 7 circuits. - + These sizes are based on measurements performed during test scans. -4. Ratio Calculation Options + This process is repeated until each node has been chosen to participate + in at least 5 circuits. - There are two options for deriving the ratios themselves. They can - be obtained by dividing each nodes' average stream capacity by - either the average for the slice, or the average for the network as a - whole. - Dividing by the network-wide average has the advantage that it will - account for issues related to unbalancing between higher vs lower - capacity, such as Steven Murdoch's queuing theory weighting result. - For this reason, we will opt for network-wide averages. +4. Ratio Calculation + + The ratios are calculated by dividing each measured value by the + network-wide average. 5. Ratio Filtering @@ -77,10 +72,8 @@ Status: Open with capacity of one standard deviation below a node's average are also removed. - The final ratio result will be calculated as the maximum of - these two resulting ratios if both are less than 1.0, the minimum - if both are greater than 1.0, and the mean if one is greater - and one is less than 1.0. + The final ratio result will be the unfiltered ratio if it is + close to 1.0, otherwise it will be the filtered ratio. 6. Pseudocode for Ratio Calculation Algorithm @@ -109,12 +102,7 @@ Status: Open Bw_net_ratio(N) = Bw_measured(N)/Bw_net_avg(Slices) Bw_Norm_net_ratio(N) = Bw_measured2(N)/Bw_Norm_net_avg(Slices) - if Bw_net_ratio(N) < 1.0 and Bw_Norm_net_ratio(N) < 1.0: - ResultRatio(N) = MAX(Bw_net_ratio(N), Bw_Norm_net_ratio(N)) - else if Bw_net_ratio(N) > 1.0 and Bw_Norm_net_ratio(N) > 1.0: - ResultRatio(N) = MIN(Bw_net_ratio(N), Bw_Norm_net_ratio(N)) - else: - ResultRatio(N) = MEAN(Bw_net_ratio(N), Bw_Norm_net_ratio(N)) + ResultRatio(N) = ClosestToOne(Bw_net_ratio(N), Bw_Norm_net_ratio(N)) 7. Security implications @@ -126,14 +114,14 @@ Status: Open This scheme will not address nodes that try to game the system by providing better service to scanners. The scanners can be detected - at the entry by IP address, and at the exit by the destination fetch. + at the entry by IP address, and at the exit by the destination fetch + IP. Measures can be taken to obfuscate and separate the scanners' source IP address from the directory authority IP address. For instance, scans can happen offsite and the results can be rsynced into the - authorities. The destination fetch can also be obscured by using SSL - and periodically changing the large document that is fetched. - + authorities. The destination server IP can also change. + Neither of these methods are foolproof, but such nodes can already lie about their bandwidth to attract more traffic, so this solution does not set us back any in that regard. @@ -148,7 +136,7 @@ Status: Open over a portion of the network, outputting files of the form: node_id= SP strm_bw= SP - filt_bw= NL + filt_bw= ns_bw= NL The most recent file from each scanner will be periodically gathered by another script that uses them to produce network-wide averages @@ -166,10 +154,15 @@ Status: Open scan, and taking the weighted average with the previous consensus bandwidth: - Bw_new = (Bw_current * Alpha + Bw_scan_avg*Bw_ratio)/(Alpha + 1) + Bw_new = Round((Bw_current * Alpha + Bw_scan_avg*Bw_ratio)/(Alpha + 1)) The Alpha parameter is a smoothing parameter intended to prevent - rapid oscillation between loaded and unloaded conditions. + rapid oscillation between loaded and unloaded conditions. It is + currently fixed at 0.333. + + The Round() step consists of rounding to the 3 most significant figures + in base10, and then rounding that result to the nearest 1000, with + a minimum value of 1000. This will produce a new bandwidth value that will be output into a file consisting of lines of the form: @@ -183,6 +176,3 @@ Status: Open This file can be either copied or rsynced into a directory readable by the directory authority. - -1. Exact values for each segment are still being determined via -test scans. From 3a1b9526f89986aedf2cb9bfdf0874804935179f Mon Sep 17 00:00:00 2001 From: Mike Perry Date: Tue, 14 Jul 2009 20:47:51 -0700 Subject: [PATCH 134/187] Update the ratio calculation to prefer faster measurements. Also: simplify complicated filtering steps, always take the most recent measurement, and use slightly smaller file sizes based on measurements. --- .../161-computing-bandwidth-adjustments.txt | 28 ++++++++----------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/doc/spec/proposals/161-computing-bandwidth-adjustments.txt b/doc/spec/proposals/161-computing-bandwidth-adjustments.txt index a50dba687e..786e1afebd 100644 --- a/doc/spec/proposals/161-computing-bandwidth-adjustments.txt +++ b/doc/spec/proposals/161-computing-bandwidth-adjustments.txt @@ -45,12 +45,11 @@ Status: Open file is downloaded via these circuits. The file sizes are set based on node percentile rank as follows: - 0-10: 4M - 10-20: 2M - 20-30: 1M - 30-50: 512k - 50-75: 256k - 75-100: 128k + 0-10: 2M + 10-20: 1M + 20-30: 512k + 30-50: 256k + 50-100: 128k These sizes are based on measurements performed during test scans. @@ -72,8 +71,8 @@ Status: Open with capacity of one standard deviation below a node's average are also removed. - The final ratio result will be the unfiltered ratio if it is - close to 1.0, otherwise it will be the filtered ratio. + The final ratio result will be greater of the unfiltered ratio + and the filtered ratio. 6. Pseudocode for Ratio Calculation Algorithm @@ -88,11 +87,8 @@ Status: Open BW_measured(N) = MEAN(b | b is bandwidth of a stream through N) Bw_stddev(N) = STDDEV(b | b is bandwidth of a stream through N) Bw_avg(S) = MEAN(b | b = BW_measured(N) for all N in S) - Normal_Routers(S) = {N | Bw_measured(N)/Bw_avg(S) > 0.5 } for N in S: - Normal_Streams(N) = - {stream via N | all nodes in stream not in {Normal_Routers(S)-N} - and bandwidth > BW_measured(N)-Bw_stddev(N)} + Normal_Streams(N) = {stream via N | bandwidth >= BW_measured(N)} BW_Norm_measured(N) = MEAN(b | b is a bandwidth of Normal_Streams(N)) Bw_net_avg(Slices) = MEAN(BW_measured(N) for all N in Slices) @@ -100,9 +96,9 @@ Status: Open for N in all Slices: Bw_net_ratio(N) = Bw_measured(N)/Bw_net_avg(Slices) - Bw_Norm_net_ratio(N) = Bw_measured2(N)/Bw_Norm_net_avg(Slices) + Bw_Norm_net_ratio(N) = BW_Norm_measured(N)/Bw_Norm_net_avg(Slices) - ResultRatio(N) = ClosestToOne(Bw_net_ratio(N), Bw_Norm_net_ratio(N)) + ResultRatio(N) = MAX(Bw_net_ratio(N), Bw_Norm_net_ratio(N)) 7. Security implications @@ -142,8 +138,8 @@ Status: Open by another script that uses them to produce network-wide averages and calculate ratios as per the algorithm in section 6. Because nodes may shift in capacity, they may appear in more than one slice and/or - appear more than once in the file set. The line that yields a ratio - closest to 1.0 will be chosen in this case. + appear more than once in the file set. The most recently measured + line will be chosen in this case. 9. Integration with Proposal 160 From b074e61ad35efdf146da14c0b967da3aa24c2cdc Mon Sep 17 00:00:00 2001 From: Mike Perry Date: Mon, 13 Jul 2009 19:21:00 -0700 Subject: [PATCH 135/187] Throw the switch on consensus method 6. --- src/or/dirvote.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/or/dirvote.c b/src/or/dirvote.c index acb658bd23..aadece9096 100644 --- a/src/or/dirvote.c +++ b/src/or/dirvote.c @@ -103,8 +103,10 @@ format_networkstatus_vote(crypto_pk_env_t *private_signing_key, tor_snprintf(status, len, "network-status-version 3\n" "vote-status %s\n" - /* TODO-160: add 6 when ready */ - "consensus-methods 1 2 3 4 5\n" + /* XXX: If you change this value, you also need to + * change consensus_method_is_supported(). + * Perhaps we should unify these somehow? */ + "consensus-methods 1 2 3 4 5 6\n" "published %s\n" "valid-after %s\n" "fresh-until %s\n" @@ -456,7 +458,10 @@ compute_consensus_method(smartlist_t *votes) static int consensus_method_is_supported(int method) { - return (method >= 1) && (method <= 5); + /* XXX: If you change this value, you also need to change + * format_networkstatus_vote(). Perhaps we should unify + * these somehow? */ + return (method >= 1) && (method <= 6); } /** Helper: given lst, a list of version strings such that every From ca676c3924e58f5e07c749678d22315073dd0946 Mon Sep 17 00:00:00 2001 From: Mike Perry Date: Sat, 27 Jun 2009 03:08:18 -0400 Subject: [PATCH 136/187] Display consensus bandwidth to the control port. Also div vote and other bandwidth by 1000, not 1024. --- src/or/dirserv.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/or/dirserv.c b/src/or/dirserv.c index d98d8c4ebf..1a47173686 100644 --- a/src/or/dirserv.c +++ b/src/or/dirserv.c @@ -1943,6 +1943,7 @@ routerstatus_format_entry(char *buf, size_t buf_len, if (format != NS_V2) { routerinfo_t* desc = router_get_by_digest(rs->identity_digest); + u_int32_t bw; if (format != NS_CONTROL_PORT) { /* Blow up more or less nicely if we didn't get anything or not the @@ -1985,9 +1986,14 @@ routerstatus_format_entry(char *buf, size_t buf_len, }; } + if (format == NS_CONTROL_PORT && rs->has_bandwidth) { + bw = rs->bandwidth; + } else { + tor_assert(desc); + bw = router_get_advertised_bandwidth_capped(desc) / 1000; + } r = tor_snprintf(cp, buf_len - (cp-buf), - "w Bandwidth=%d\n", - router_get_advertised_bandwidth_capped(desc) / 1024); + "w Bandwidth=%d\n", bw); if (r<0) { log_warn(LD_BUG, "Not enough space in buffer."); From 8196130f245ea4ccd893ad9b146a63f3fa5e68c1 Mon Sep 17 00:00:00 2001 From: Roger Dingledine Date: Fri, 7 Aug 2009 17:55:17 -0400 Subject: [PATCH 137/187] fix typo in control-spec --- doc/spec/control-spec.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/spec/control-spec.txt b/doc/spec/control-spec.txt index 20952996b2..fc4242ea16 100644 --- a/doc/spec/control-spec.txt +++ b/doc/spec/control-spec.txt @@ -773,7 +773,7 @@ Same as passing 'EXTENDED' to SETEVENTS; this is the preferred way to request the extended event syntax. - This feaure was first used in 0.1.2.3-alpha. It is always-on in + This feature was first used in 0.1.2.3-alpha. It is always-on in Tor 0.2.2.1-alpha and later. VERBOSE_NAMES From 3e4379c2e73bf458cf60c63df44a8d0ec761568c Mon Sep 17 00:00:00 2001 From: Roger Dingledine Date: Fri, 7 Aug 2009 19:26:41 -0400 Subject: [PATCH 138/187] Disable .exit notation unless AllowDotExit is 1. --- doc/tor.1.in | 7 +++++++ src/or/config.c | 1 + src/or/connection_edge.c | 18 ++++++++++++------ src/or/or.h | 9 ++++++++- src/or/test.c | 8 ++++---- 5 files changed, 32 insertions(+), 11 deletions(-) diff --git a/doc/tor.1.in b/doc/tor.1.in index b6e22310f2..fa383ccba6 100644 --- a/doc/tor.1.in +++ b/doc/tor.1.in @@ -690,6 +690,13 @@ resolved. This helps trap accidental attempts to resolve URLs and so on. (Default: 0) .LP .TP +\fBAllowDotOnion \fR\fB0\fR|\fB1\fR\fP +If enabled, we convert "www.google.com.foo.exit" addresses on the +SocksPort/TransPort/NatdPort into "www.google.com" addresses that exit +from the node "foo". Disabled by default since attacking websites and +exit relays can use it to manipulate your path selection. (Default: 0) +.LP +.TP \fBFastFirstHopPK \fR\fB0\fR|\fB1\fR\fP When this option is disabled, Tor uses the public key step for the first hop of creating circuits. Skipping it is generally safe since we have diff --git a/src/or/config.c b/src/or/config.c index 1e559070ef..951f6fab5b 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -134,6 +134,7 @@ static config_var_t _option_vars[] = { V(AccountingMax, MEMUNIT, "0 bytes"), V(AccountingStart, STRING, NULL), V(Address, STRING, NULL), + V(AllowDotExit, BOOL, "0"), V(AllowInvalidNodes, CSV, "middle,rendezvous"), V(AllowNonRFC953Hostnames, BOOL, "0"), V(AllowSingleHopCircuits, BOOL, "0"), diff --git a/src/or/connection_edge.c b/src/or/connection_edge.c index 7a3d0a55be..79496f7a9b 100644 --- a/src/or/connection_edge.c +++ b/src/or/connection_edge.c @@ -1483,7 +1483,8 @@ connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, /* Parse the address provided by SOCKS. Modify it in-place if it * specifies a hidden-service (.onion) or particular exit node (.exit). */ - addresstype = parse_extended_hostname(socks->address); + addresstype = parse_extended_hostname(socks->address, + remapped_to_exit || options->AllowDotExit); if (addresstype == BAD_HOSTNAME) { log_warn(LD_APP, "Invalid onion hostname %s; rejecting", @@ -1496,7 +1497,7 @@ connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, if (addresstype == EXIT_HOSTNAME) { /* foo.exit -- modify conn->chosen_exit_node to specify the exit - * node, and conn->address to hold only the address portion.*/ + * node, and conn->address to hold only the address portion. */ char *s = strrchr(socks->address,'.'); tor_assert(!automap); if (s) { @@ -2902,14 +2903,14 @@ connection_ap_can_use_exit(edge_connection_t *conn, routerinfo_t *exit) /** If address is of the form "y.onion" with a well-formed handle y: * Put a NUL after y, lower-case it, and return ONION_HOSTNAME. * - * If address is of the form "y.exit": + * If address is of the form "y.exit" and allowdotexit is true: * Put a NUL after y and return EXIT_HOSTNAME. * * Otherwise: * Return NORMAL_HOSTNAME and change nothing. */ hostname_type_t -parse_extended_hostname(char *address) +parse_extended_hostname(char *address, int allowdotexit) { char *s; char query[REND_SERVICE_ID_LEN_BASE32+1]; @@ -2918,8 +2919,13 @@ parse_extended_hostname(char *address) if (!s) return NORMAL_HOSTNAME; /* no dot, thus normal */ if (!strcmp(s+1,"exit")) { - *s = 0; /* NUL-terminate it */ - return EXIT_HOSTNAME; /* .exit */ + if (allowdotexit) { + *s = 0; /* NUL-terminate it */ + return EXIT_HOSTNAME; /* .exit */ + } /* else */ + log_warn(LD_APP, "The \".exit\" notation is disabled in Tor due to " + "security risks. Set AllowDotExit in your torrc to enable it."); + /* FFFF send a controller event too to notify Vidalia users */ } if (strcmp(s+1,"onion")) return NORMAL_HOSTNAME; /* neither .exit nor .onion, thus normal */ diff --git a/src/or/or.h b/src/or/or.h index 4f215d6448..253e89af85 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -2508,6 +2508,13 @@ typedef struct { * exit allows it, we use it. */ int AllowSingleHopCircuits; + /** If true, we convert "www.google.com.foo.exit" addresses on the + * socks/trans/natd ports into "www.google.com" addresses that + * exit from the node "foo". Disabled by default since attacking + * websites and exit relays can use it to manipulate your path + * selection. */ + int AllowDotExit; + /** If true, the user wants us to collect statistics on clients * requesting network statuses from us as directory. */ int DirReqStatistics; @@ -3133,7 +3140,7 @@ int hostname_is_noconnect_address(const char *address); typedef enum hostname_type_t { NORMAL_HOSTNAME, ONION_HOSTNAME, EXIT_HOSTNAME, BAD_HOSTNAME } hostname_type_t; -hostname_type_t parse_extended_hostname(char *address); +hostname_type_t parse_extended_hostname(char *address, int allowdotexit); #if defined(HAVE_NET_IF_H) && defined(HAVE_NET_PFVAR_H) int get_pf_socket(void); diff --git a/src/or/test.c b/src/or/test.c index 3103eed828..dffb0a5ade 100644 --- a/src/or/test.c +++ b/src/or/test.c @@ -4489,10 +4489,10 @@ test_rend_fns(void) char address3[] = "fooaddress.exit"; char address4[] = "www.torproject.org"; - test_assert(BAD_HOSTNAME == parse_extended_hostname(address1)); - test_assert(ONION_HOSTNAME == parse_extended_hostname(address2)); - test_assert(EXIT_HOSTNAME == parse_extended_hostname(address3)); - test_assert(NORMAL_HOSTNAME == parse_extended_hostname(address4)); + test_assert(BAD_HOSTNAME == parse_extended_hostname(address1, 1)); + test_assert(ONION_HOSTNAME == parse_extended_hostname(address2, 1)); + test_assert(EXIT_HOSTNAME == parse_extended_hostname(address3, 1)); + test_assert(NORMAL_HOSTNAME == parse_extended_hostname(address4, 1)); pk1 = pk_generate(0); pk2 = pk_generate(1); From 07d95440ef3430f74dfe0fa68f7d84eec9a17f3e Mon Sep 17 00:00:00 2001 From: Roger Dingledine Date: Fri, 7 Aug 2009 19:29:19 -0400 Subject: [PATCH 139/187] oops, fix typo --- doc/tor.1.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/tor.1.in b/doc/tor.1.in index fa383ccba6..19f34f93f9 100644 --- a/doc/tor.1.in +++ b/doc/tor.1.in @@ -690,7 +690,7 @@ resolved. This helps trap accidental attempts to resolve URLs and so on. (Default: 0) .LP .TP -\fBAllowDotOnion \fR\fB0\fR|\fB1\fR\fP +\fBAllowDotExit \fR\fB0\fR|\fB1\fR\fP If enabled, we convert "www.google.com.foo.exit" addresses on the SocksPort/TransPort/NatdPort into "www.google.com" addresses that exit from the node "foo". Disabled by default since attacking websites and From 33762b529694f58d7640ebcbef1bc0f940419c89 Mon Sep 17 00:00:00 2001 From: Jacob Appelbaum Date: Sat, 8 Aug 2009 19:15:22 -0700 Subject: [PATCH 140/187] LetsKillNoConnect removes support for .noconnect This is a patch to remove support for .noconnect. We are removing .noconnect because of a talk at Defcon 17 by Gregory Fleischer. --- ChangeLog | 1 + doc/spec/address-spec.txt | 12 ------------ src/or/connection_edge.c | 16 ---------------- src/or/dnsserv.c | 7 +------ 4 files changed, 2 insertions(+), 34 deletions(-) diff --git a/ChangeLog b/ChangeLog index 1d9b71b766..b1f45f1ffd 100644 --- a/ChangeLog +++ b/ChangeLog @@ -49,6 +49,7 @@ Changes in version 0.2.2.1-alpha - 2009-0?-?? - The EXTENDED_EVENTS and VERBOSE_NAMES controller features are now always on; using them is necessary for correct forward-compatible controllers. + - Removal of support for .noconnect style addresses. Changes in version 0.2.1.20 - 2009-??-?? diff --git a/doc/spec/address-spec.txt b/doc/spec/address-spec.txt index 95641b49b8..fdae9b82a4 100644 --- a/doc/spec/address-spec.txt +++ b/doc/spec/address-spec.txt @@ -53,15 +53,3 @@ When Tor sees an address in this format, it tries to look up and connect to the specified hidden service. See rend-spec.txt for full details. -4. .noconnect - - SYNTAX: [string].noconnect - - When Tor sees an address in this format, it immediately closes the - connection without attaching it to any circuit. This is useful for - controllers that want to test whether a given application is indeed using - the same instance of Tor that they're controlling. - -5. [XXX Is there a ".virtual" address that we expose too, or is that -just intended to be internal? -RD] - diff --git a/src/or/connection_edge.c b/src/or/connection_edge.c index 79496f7a9b..04fffd9f57 100644 --- a/src/or/connection_edge.c +++ b/src/or/connection_edge.c @@ -1875,14 +1875,6 @@ connection_ap_handshake_process_socks(edge_connection_t *conn) return -1; } /* else socks handshake is done, continue processing */ - if (hostname_is_noconnect_address(socks->address)) - { - control_event_stream_status(conn, STREAM_EVENT_NEW, 0); - control_event_stream_status(conn, STREAM_EVENT_CLOSED, 0); - connection_mark_unattached_ap(conn, END_STREAM_REASON_DONE); - return -1; - } - if (SOCKS_COMMAND_IS_CONNECT(socks->command)) control_event_stream_status(conn, STREAM_EVENT_NEW, 0); else @@ -2944,11 +2936,3 @@ failed: return BAD_HOSTNAME; } -/** Check if the address is of the form "y.noconnect" - */ -int -hostname_is_noconnect_address(const char *address) -{ - return ! strcasecmpend(address, ".noconnect"); -} - diff --git a/src/or/dnsserv.c b/src/or/dnsserv.c index 08cf653093..b1e0ff104f 100644 --- a/src/or/dnsserv.c +++ b/src/or/dnsserv.c @@ -92,12 +92,7 @@ evdns_server_callback(struct evdns_server_request *req, void *_data) evdns_server_request_respond(req, DNS_ERR_NONE); return; } - if (q->type == EVDNS_TYPE_A) { - /* Refuse any attempt to resolve a noconnect address, right now. */ - if (hostname_is_noconnect_address(q->name)) { - err = DNS_ERR_REFUSED; - } - } else { + if (q->type != EVDNS_TYPE_A) { tor_assert(q->type == EVDNS_TYPE_PTR); } From 3886467f386732598647a2d3209777ba8d8d7baa Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Sun, 9 Aug 2009 17:27:35 -0700 Subject: [PATCH 141/187] Add a new tor_strtok_r for platforms that don't have one, plus tests. I don't think we actually use (or plan to use) strtok_r in a reentrant way anywhere in our code, but would be nice not to have to think about whether we're doing it. --- src/common/compat.c | 31 +++++++++++++++++++++++++++++++ src/common/compat.h | 7 +++++++ src/or/test.c | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 72 insertions(+) diff --git a/src/common/compat.c b/src/common/compat.c index d62b1ce1f4..29425c2492 100644 --- a/src/common/compat.c +++ b/src/common/compat.c @@ -398,6 +398,37 @@ const char TOR_TOLOWER_TABLE[256] = { 240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255, }; +/** Implementation of strtok_r for platforms whose coders haven't figured out + * how to write one. Hey guys! You can use this code here for free! */ +char * +tor_strtok_r_impl(char *str, const char *sep, char **lasts) +{ + char *cp, *start; + if (str) + start = cp = *lasts = str; + else if (!*lasts) + return NULL; + else + start = cp = *lasts; + + tor_assert(*sep); + if (sep[1]) { + while (*cp && !strchr(sep, *cp)) + ++cp; + } else { + tor_assert(strlen(sep) == 1); + cp = strchr(cp, *sep); + } + + if (!cp || !*cp) { + *lasts = NULL; + } else { + *cp++ = '\0'; + *lasts = cp; + } + return start; +} + #ifdef MS_WINDOWS /** Take a filename and return a pointer to its final element. This * function is called on __FILE__ to fix a MSVC nit where __FILE__ diff --git a/src/common/compat.h b/src/common/compat.h index 4d5a016cf2..3d429486e8 100644 --- a/src/common/compat.h +++ b/src/common/compat.h @@ -267,6 +267,13 @@ extern const char TOR_TOLOWER_TABLE[]; #define TOR_TOLOWER(c) (TOR_TOLOWER_TABLE[(uint8_t)c]) #define TOR_TOUPPER(c) (TOR_TOUPPER_TABLE[(uint8_t)c]) +char *tor_strtok_r_impl(char *str, const char *sep, char **lasts); +#ifdef HAVE_STRTOK_R +#define tor_strok_r(str, sep, lasts) strtok_r(str, sep, lasts) +#else +#define tor_strok_r(str, sep, lasts) tor_strtok_r_impl(str, sep, lasts) +#endif + #ifdef MS_WINDOWS #define _SHORT_FILE_ (tor_fix_source_file(__FILE__)) const char *tor_fix_source_file(const char *fname); diff --git a/src/or/test.c b/src/or/test.c index 3103eed828..67a9c381fe 100644 --- a/src/or/test.c +++ b/src/or/test.c @@ -4284,6 +4284,39 @@ test_util_datadir(void) tor_free(f); } +static void +test_util_strtok(void) +{ + char buf[128]; + char buf2[128]; + char *cp1, *cp2; + strlcpy(buf, "Graved on the dark in gestures of descent", sizeof(buf)); + strlcpy(buf2, "they.seemed;their!own;most.perfect;monument", sizeof(buf2)); + /* -- "Year's End", Richard Wilbur */ + + test_streq("Graved", tor_strtok_r_impl(buf, " ", &cp1)); + test_streq("they", tor_strtok_r_impl(buf2, ".!..;!", &cp2)); +#define S1() tor_strtok_r_impl(NULL, " ", &cp1) +#define S2() tor_strtok_r_impl(NULL, ".!..;!", &cp2) + test_streq("on", S1()); + test_streq("the", S1()); + test_streq("dark", S1()); + test_streq("seemed", S2()); + test_streq("their", S2()); + test_streq("own", S2()); + test_streq("in", S1()); + test_streq("gestures", S1()); + test_streq("of", S1()); + test_streq("most", S2()); + test_streq("perfect", S2()); + test_streq("descent", S1()); + test_streq("monument", S2()); + test_assert(NULL == S1()); + test_assert(NULL == S2()); + done: + ; +} + /** Test AES-CTR encryption and decryption with IV. */ static void test_crypto_aes_iv(void) @@ -4692,6 +4725,7 @@ static struct { SUBENT(util, threads), SUBENT(util, order_functions), SUBENT(util, sscanf), + SUBENT(util, strtok), ENT(onion_handshake), ENT(dir_format), ENT(dirutil), From 1060b4d82429c743b6209f7e0a5a2c6bca0a6450 Mon Sep 17 00:00:00 2001 From: Mike Perry Date: Sun, 9 Aug 2009 18:21:15 -0700 Subject: [PATCH 142/187] Fix issues found by Nick in code review. --- src/or/dirserv.c | 29 ++++++++++++++--------------- src/or/or.h | 8 +++----- 2 files changed, 17 insertions(+), 20 deletions(-) diff --git a/src/or/dirserv.c b/src/or/dirserv.c index 1a47173686..32ddcd0b0a 100644 --- a/src/or/dirserv.c +++ b/src/or/dirserv.c @@ -1943,7 +1943,7 @@ routerstatus_format_entry(char *buf, size_t buf_len, if (format != NS_V2) { routerinfo_t* desc = router_get_by_digest(rs->identity_digest); - u_int32_t bw; + uint32_t bw; if (format != NS_CONTROL_PORT) { /* Blow up more or less nicely if we didn't get anything or not the @@ -2270,7 +2270,8 @@ measured_bw_line_parse(measured_bw_line_t *out, const char *orig_line) } do { - if (strncasecmp(cp, "bw=", strlen("bw=")) == 0) { + if (strcmpstart(cp, "bw=") == 0) { + int parse_ok = 0; char *endptr; if (got_bw) { log_warn(LD_DIRSERV, "Double bw= in bandwidth file line: %s", @@ -2279,17 +2280,16 @@ measured_bw_line_parse(measured_bw_line_t *out, const char *orig_line) return -1; } cp+=strlen("bw="); - errno=0; - out->bw = strtol(cp, &endptr, 0); - if (errno || endptr == cp || (*endptr && !TOR_ISSPACE(*endptr)) || - out->bw < 0) { + + out->bw = tor_parse_long(cp, 0, 0, LONG_MAX, &parse_ok, &endptr); + if (!parse_ok || (*endptr && !TOR_ISSPACE(*endptr))) { log_warn(LD_DIRSERV, "Invalid bandwidth in bandwidth file line: %s", escaped(orig_line)); tor_free(line); return -1; } got_bw=1; - } else if (strncasecmp(cp, "node_id=$", strlen("node_id=$")) == 0) { + } else if (strcmpstart(cp, "node_id=$") == 0) { if (got_node_id) { log_warn(LD_DIRSERV, "Double node_id= in bandwidth file line: %s", escaped(orig_line)); @@ -2367,9 +2367,8 @@ dirserv_read_measured_bandwidths(const char *from_file, return -1; } - fgets(line, sizeof(line), fp); - - if (line[strlen(line)-1] != '\n') { + if (!fgets(line, sizeof(line), fp) + || !strlen(line) || line[strlen(line)-1] != '\n') { log_warn(LD_DIRSERV, "Long or truncated time in bandwidth file: %s", escaped(line)); fclose(fp); @@ -2397,11 +2396,11 @@ dirserv_read_measured_bandwidths(const char *from_file, while (!feof(fp)) { measured_bw_line_t parsed_line; - fgets(line, sizeof(line), fp); - - if (measured_bw_line_parse(&parsed_line, line) != -1) { - if (measured_bw_line_apply(&parsed_line, routerstatuses) > 0) - applied_lines++; + if (fgets(line, sizeof(line), fp) && strlen(line)) { + if (measured_bw_line_parse(&parsed_line, line) != -1) { + if (measured_bw_line_apply(&parsed_line, routerstatuses) > 0) + applied_lines++; + } } } diff --git a/src/or/or.h b/src/or/or.h index bdbd3e5007..f3e639f84e 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -3544,12 +3544,10 @@ typedef struct measured_bw_line_t { long int bw; } measured_bw_line_t; -int -measured_bw_line_parse(measured_bw_line_t *out, const char *line); +int measured_bw_line_parse(measured_bw_line_t *out, const char *line); -int -measured_bw_line_apply(measured_bw_line_t *parsed_line, - smartlist_t *routerstatuses); +int measured_bw_line_apply(measured_bw_line_t *parsed_line, + smartlist_t *routerstatuses); #endif int dirserv_read_measured_bandwidths(const char *from_file, From 9e1fe29bebf57fc38975c552eefaa9cb97dd5c68 Mon Sep 17 00:00:00 2001 From: Mike Perry Date: Sun, 9 Aug 2009 18:42:29 -0700 Subject: [PATCH 143/187] Switch over to tor_strtok_r instead of strtok_r. --- src/common/compat.h | 4 ++-- src/or/dirserv.c | 20 ++------------------ src/or/eventdns.c | 12 ++---------- 3 files changed, 6 insertions(+), 30 deletions(-) diff --git a/src/common/compat.h b/src/common/compat.h index 3d429486e8..edd09d8683 100644 --- a/src/common/compat.h +++ b/src/common/compat.h @@ -269,9 +269,9 @@ extern const char TOR_TOLOWER_TABLE[]; char *tor_strtok_r_impl(char *str, const char *sep, char **lasts); #ifdef HAVE_STRTOK_R -#define tor_strok_r(str, sep, lasts) strtok_r(str, sep, lasts) +#define tor_strtok_r(str, sep, lasts) strtok_r(str, sep, lasts) #else -#define tor_strok_r(str, sep, lasts) tor_strtok_r_impl(str, sep, lasts) +#define tor_strtok_r(str, sep, lasts) tor_strtok_r_impl(str, sep, lasts) #endif #ifdef MS_WINDOWS diff --git a/src/or/dirserv.c b/src/or/dirserv.c index 32ddcd0b0a..3b7b2ff4bc 100644 --- a/src/or/dirserv.c +++ b/src/or/dirserv.c @@ -2224,22 +2224,6 @@ router_clear_status_flags(routerinfo_t *router) router->is_bad_exit = router->is_bad_directory = 0; } -#ifndef HAVE_STRTOK_R -/* - * XXX-MP: If a system lacks strtok_r and we use a non-reentrant strtok, - * we may introduce odd bugs if we call a codepath that also uses strtok - * and resets its internal state. Do we want to abandon use of strtok - * entirely for this reason? Roger mentioned smartlist_split and - * eat_whitespace() as alternatives. - */ -static char * -strtok_r(char *s, const char *delim, char **state) -{ - (void)state; - return strtok(s, delim); -} -#endif - /** * Helper function to parse out a line in the measured bandwidth file * into a measured_bw_line_t output structure. Returns -1 on failure @@ -2253,7 +2237,7 @@ measured_bw_line_parse(measured_bw_line_t *out, const char *orig_line) int got_bw = 0; int got_node_id = 0; char *strtok_state; /* lame sauce d'jour */ - cp = strtok_r(cp, " \t", &strtok_state); + cp = tor_strtok_r(cp, " \t", &strtok_state); if (!cp) { log_warn(LD_DIRSERV, "Invalid line in bandwidth file: %s", @@ -2308,7 +2292,7 @@ measured_bw_line_parse(measured_bw_line_t *out, const char *orig_line) strncpy(out->node_hex, cp, sizeof(out->node_hex)); got_node_id=1; } - } while ((cp = strtok_r(NULL, " \t", &strtok_state))); + } while ((cp = tor_strtok_r(NULL, " \t", &strtok_state))); if (got_bw && got_node_id) { tor_free(line); diff --git a/src/or/eventdns.c b/src/or/eventdns.c index 9578b24cae..b413b6ae97 100644 --- a/src/or/eventdns.c +++ b/src/or/eventdns.c @@ -2889,14 +2889,6 @@ evdns_resolv_set_defaults(int flags) { if (flags & DNS_OPTION_NAMESERVERS) evdns_nameserver_ip_add("127.0.0.1"); } -#ifndef HAVE_STRTOK_R -static char * -strtok_r(char *s, const char *delim, char **state) { - (void)state; - return strtok(s, delim); -} -#endif - /* helper version of atoi which returns -1 on error */ static int strtoint(const char *const str) { @@ -2973,9 +2965,9 @@ static void resolv_conf_parse_line(char *const start, int flags) { char *strtok_state; static const char *const delims = " \t"; -#define NEXT_TOKEN strtok_r(NULL, delims, &strtok_state) +#define NEXT_TOKEN tor_strtok_r(NULL, delims, &strtok_state) - char *const first_token = strtok_r(start, delims, &strtok_state); + char *const first_token = tor_strtok_r(start, delims, &strtok_state); if (!first_token) return; if (!strcmp(first_token, "nameserver") && (flags & DNS_OPTION_NAMESERVERS)) { From 8abe3bac7e66a5ef3f0d3275159e0012ccd7212f Mon Sep 17 00:00:00 2001 From: Roger Dingledine Date: Mon, 10 Aug 2009 01:32:51 -0400 Subject: [PATCH 144/187] Set up urras as the seventh v3 directory authority. --- ChangeLog | 4 ++++ src/or/config.c | 2 ++ 2 files changed, 6 insertions(+) diff --git a/ChangeLog b/ChangeLog index 132c81597f..9e3cd84f50 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,8 @@ Changes in version 0.2.1.20 - 2009-??-?? + o New directory authorities: + - Set up urras (run by Jacob Appelbaum) as the seventh v3 directory + authority. + o Minor bugfixes: - Fix a signed/unsigned compile warning in 0.2.1.19. diff --git a/src/or/config.c b/src/or/config.c index 603f1b606d..84146c1063 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -921,6 +921,8 @@ add_default_trusted_dir_authorities(authority_type_t type) "dannenberg orport=443 no-v2 " "v3ident=585769C78764D58426B8B52B6651A5A71137189A " "213.73.91.31:80 7BE6 83E6 5D48 1413 21C5 ED92 F075 C553 64AC 7123", + "urras orport=80 no-v2 v3ident=80550987E1D626E3EBA5E5E75A458DE0626D088C " + "208.83.223.34:443 0AD3 FA88 4D18 F89E EA2D 89C0 1937 9E0E 7FD9 4417", NULL }; for (i=0; dirservers[i]; i++) { From e53ee805a92e5e8f8afc8aa512d33464be710733 Mon Sep 17 00:00:00 2001 From: Roger Dingledine Date: Mon, 10 Aug 2009 03:06:56 -0400 Subject: [PATCH 145/187] directory-archive scripts now fetch from urras too --- contrib/directory-archive/fetch-all-v3 | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/directory-archive/fetch-all-v3 b/contrib/directory-archive/fetch-all-v3 index 02b3d5d9e8..a4746e02cf 100755 --- a/contrib/directory-archive/fetch-all-v3 +++ b/contrib/directory-archive/fetch-all-v3 @@ -35,6 +35,7 @@ DIRSERVERS="$DIRSERVERS 80.190.246.100:80" # gabelmoo DIRSERVERS="$DIRSERVERS 194.109.206.212:80" # dizum #DIRSERVERS="$DIRSERVERS 128.31.0.34:9032" # moria2 DIRSERVERS="$DIRSERVERS 213.73.91.31:80" # dannenberg +DIRSERVERS="$DIRSERVERS 208.83.223.34:443" # urras TIME=$(date "+%Y%m%d-%H%M%S") . fetch-all-functions From e50b7768b94bbcccef77be255920e3aac80a9555 Mon Sep 17 00:00:00 2001 From: Roger Dingledine Date: Mon, 10 Aug 2009 04:13:18 -0400 Subject: [PATCH 146/187] Send sendmes when we're down 100 cells, not 101. Send circuit or stream sendme cells when our window has decreased by 100 cells, not when it has decreased by 101 cells. Bug uncovered by Karsten when testing the "reduce circuit window" performance patch. Bugfix on the 54th commit on Tor -- from July 2002, before the release of Tor 0.0.0. This is the new winner of the oldest-bug prize. --- ChangeLog | 8 ++++++++ src/or/relay.c | 4 ++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index 9e3cd84f50..994ff4c7b3 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,12 @@ Changes in version 0.2.1.20 - 2009-??-?? + o Major bugfixes: + - Send circuit or stream sendme cells when our window has decreased + by 100 cells, not when it has decreased by 101 cells. Bug uncovered + by Karsten when testing the "reduce circuit window" performance + patch. Bugfix on the 54th commit on Tor -- from July 2002, + before the release of Tor 0.0.0. This is the new winner of the + oldest-bug prize. + o New directory authorities: - Set up urras (run by Jacob Appelbaum) as the seventh v3 directory authority. diff --git a/src/or/relay.c b/src/or/relay.c index 3419e3d190..b26c582b82 100644 --- a/src/or/relay.c +++ b/src/or/relay.c @@ -1358,7 +1358,7 @@ connection_edge_consider_sending_sendme(edge_connection_t *conn) return; } - while (conn->deliver_window < STREAMWINDOW_START - STREAMWINDOW_INCREMENT) { + while (conn->deliver_window <= STREAMWINDOW_START - STREAMWINDOW_INCREMENT) { log_debug(conn->cpath_layer?LD_APP:LD_EXIT, "Outbuf %d, Queuing stream sendme.", (int)conn->_base.outbuf_flushlen); @@ -1472,7 +1472,7 @@ circuit_consider_sending_sendme(circuit_t *circ, crypt_path_t *layer_hint) { // log_fn(LOG_INFO,"Considering: layer_hint is %s", // layer_hint ? "defined" : "null"); - while ((layer_hint ? layer_hint->deliver_window : circ->deliver_window) < + while ((layer_hint ? layer_hint->deliver_window : circ->deliver_window) <= CIRCWINDOW_START - CIRCWINDOW_INCREMENT) { log_debug(LD_CIRC,"Queuing circuit sendme."); if (layer_hint) From a98643c1b5a39764204fcf6835a8b2bee670581d Mon Sep 17 00:00:00 2001 From: Karsten Loesing Date: Tue, 11 Aug 2009 17:33:58 +0200 Subject: [PATCH 147/187] Fix possible segmentation fault on directory authorities. The more verbose logs that were added in ee58153 also include a string that might not have been initialized. This can lead to segfaults, e.g., when setting up private Tor networks. Initialize this string with NULL. --- ChangeLog | 2 ++ src/or/dirserv.c | 7 ++++--- src/or/router.c | 2 +- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/ChangeLog b/ChangeLog index 994ff4c7b3..d64423426e 100644 --- a/ChangeLog +++ b/ChangeLog @@ -13,6 +13,8 @@ Changes in version 0.2.1.20 - 2009-??-?? o Minor bugfixes: - Fix a signed/unsigned compile warning in 0.2.1.19. + - Fix possible segmentation fault on directory authorities. Bugfix on + 0.2.1.14-rc. Changes in version 0.2.1.19 - 2009-07-28 diff --git a/src/or/dirserv.c b/src/or/dirserv.c index a64a01bb80..349e383ab5 100644 --- a/src/or/dirserv.c +++ b/src/or/dirserv.c @@ -652,8 +652,8 @@ dirserv_add_multiple_descriptors(const char *desc, uint8_t purpose, /** Examine the parsed server descriptor in ri and maybe insert it into * the list of server descriptors. Set *msg to a message that should be - * passed back to the origin of this descriptor. Use source to produce - * better log messages. + * passed back to the origin of this descriptor, or NULL if there is no such + * message. Use source to produce better log messages. * * Return the status of the operation * @@ -667,6 +667,7 @@ dirserv_add_descriptor(routerinfo_t *ri, const char **msg, const char *source) routerinfo_t *ri_old; char *desc, *nickname; size_t desclen = 0; + *msg = NULL; /* If it's too big, refuse it now. Otherwise we'll cache it all over the * network and it'll clog everything up. */ @@ -718,7 +719,7 @@ dirserv_add_descriptor(routerinfo_t *ri, const char **msg, const char *source) control_event_or_authdir_new_descriptor("REJECTED", desc, desclen, *msg); log_info(LD_DIRSERV, "Did not add descriptor from '%s' (source: %s): %s.", - nickname, source, *msg); + nickname, source, *msg ? *msg : "(no message)"); } else { smartlist_t *changed; control_event_or_authdir_new_descriptor("ACCEPTED", desc, desclen, *msg); diff --git a/src/or/router.c b/src/or/router.c index 859a1e805a..f0a1e40743 100644 --- a/src/or/router.c +++ b/src/or/router.c @@ -544,7 +544,7 @@ init_keys(void) /* Must be called after keys are initialized. */ mydesc = router_get_my_descriptor(); if (authdir_mode(options)) { - const char *m; + const char *m = NULL; routerinfo_t *ri; /* We need to add our own fingerprint so it gets recognized. */ if (dirserv_add_own_fingerprint(options->Nickname, get_identity_key())) { From 9d16a59fccd997b266b5cacde2acfd691002bf7a Mon Sep 17 00:00:00 2001 From: Karsten Loesing Date: Fri, 14 Aug 2009 14:33:29 +0200 Subject: [PATCH 148/187] Remove ./configure option for exit port statistics. --- configure.in | 7 ------- src/or/config.c | 6 ------ src/or/or.h | 6 ------ src/or/rephist.c | 27 ++++++++++++++++++++++----- 4 files changed, 22 insertions(+), 24 deletions(-) diff --git a/configure.in b/configure.in index ab7d6480cf..7a8764ce91 100644 --- a/configure.in +++ b/configure.in @@ -85,13 +85,6 @@ case $host in ;; esac -AC_ARG_ENABLE(exit-stats, - AS_HELP_STRING(--enable-exit-stats, enable code for exits to collect per-port statistics)) - -if test "$enable_exit_stats" = "yes"; then - AC_DEFINE(ENABLE_EXIT_STATS, 1, [Defined if we try to collect per-port statistics on exits]) -fi - AC_ARG_ENABLE(dirreq-stats, AS_HELP_STRING(--enable-dirreq-stats, enable code for directories to collect per-country statistics)) diff --git a/src/or/config.c b/src/or/config.c index 0b580876fb..6d34008386 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -1426,16 +1426,10 @@ options_act(or_options_t *old_options) #endif } -#ifdef ENABLE_EXIT_STATS if (options->ExitPortStatistics) log_notice(LD_CONFIG, "Configured to measure exit port statistics. " "Look for the exit-stats file that will first be written to " "the data directory in 24 hours from now."); -#else - if (options->ExitPortStatistics) - log_warn(LD_CONFIG, "ExitPortStatistics enabled, but Tor was built " - "without port statistics support."); -#endif #ifdef ENABLE_BUFFER_STATS if (options->CellStatistics) diff --git a/src/or/or.h b/src/or/or.h index 253e89af85..4db824afea 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -4072,17 +4072,11 @@ void rep_hist_note_extend_failed(const char *from_name, const char *to_name); void rep_hist_dump_stats(time_t now, int severity); void rep_hist_note_bytes_read(size_t num_bytes, time_t when); void rep_hist_note_bytes_written(size_t num_bytes, time_t when); -#ifdef ENABLE_EXIT_STATS void rep_hist_note_exit_bytes_read(uint16_t port, size_t num_bytes, time_t now); void rep_hist_note_exit_bytes_written(uint16_t port, size_t num_bytes, time_t now); void rep_hist_note_exit_stream_opened(uint16_t port, time_t now); -#else -#define rep_hist_note_exit_bytes_read(p,n,t) STMT_NIL -#define rep_hist_note_exit_bytes_written(p,n,t) STMT_NIL -#define rep_hist_note_exit_stream_opened(p,t) STMT_NIL -#endif int rep_hist_bandwidth_assess(void); char *rep_hist_get_bandwidth_lines(int for_extrainfo); void rep_hist_update_state(or_state_t *state); diff --git a/src/or/rephist.c b/src/or/rephist.c index c0b9ae14a4..0d370a3b11 100644 --- a/src/or/rephist.c +++ b/src/or/rephist.c @@ -1320,7 +1320,6 @@ rep_hist_note_bytes_read(size_t num_bytes, time_t when) add_obs(read_array, when, num_bytes); } -#ifdef ENABLE_EXIT_STATS /* Some constants */ /** How long are the intervals for measuring exit stats? */ #define EXIT_STATS_INTERVAL_SEC (24 * 60 * 60) @@ -1339,11 +1338,23 @@ rep_hist_note_bytes_read(size_t num_bytes, time_t when) * the price of some memory (1.25 MB) and linear complexity when writing * stats. */ /** Number of bytes read in current period by exit port */ -static uint64_t exit_bytes_read[EXIT_STATS_NUM_PORTS]; +static uint64_t *exit_bytes_read = NULL; /** Number of bytes written in current period by exit port */ -static uint64_t exit_bytes_written[EXIT_STATS_NUM_PORTS]; +static uint64_t *exit_bytes_written = NULL; /** Number of streams opened in current period by exit port */ -static uint32_t exit_streams[EXIT_STATS_NUM_PORTS]; +static uint32_t *exit_streams = NULL; + +/** Set up arrays for exit port statistics. */ +static void +exit_stats_init(void) +{ + exit_bytes_read = tor_malloc_zero(EXIT_STATS_NUM_PORTS * + sizeof(uint64_t)); + exit_bytes_written = tor_malloc_zero(EXIT_STATS_NUM_PORTS * + sizeof(uint64_t)); + exit_streams = tor_malloc_zero(EXIT_STATS_NUM_PORTS * + sizeof(uint32_t)); +} /** When does the current exit stats period end? */ static time_t end_of_current_exit_stats_period = 0; @@ -1362,6 +1373,8 @@ write_exit_stats(time_t when) FILE *out = NULL; log_debug(LD_HIST, "Considering writing exit port statistics to disk.."); + if (!exit_bytes_read) + exit_stats_init(); while (when > end_of_current_exit_stats_period) { format_iso_time(t, end_of_current_exit_stats_period); log_info(LD_HIST, "Writing exit port statistics to disk for period " @@ -1466,6 +1479,8 @@ write_exit_stats(time_t when) static void add_exit_obs(time_t when) { + if (!exit_bytes_read) + exit_stats_init(); if (when > end_of_current_exit_stats_period) { if (end_of_current_exit_stats_period) write_exit_stats(when); @@ -1513,7 +1528,6 @@ rep_hist_note_exit_stream_opened(uint16_t port, time_t when) exit_streams[port]++; log_debug(LD_HIST, "Opened exit stream to port %d", port); } -#endif /** Helper: Return the largest value in b->maxima. (This is equal to the * most bandwidth used in any NUM_SECS_ROLLING_MEASURE period for the last @@ -2049,6 +2063,9 @@ rep_hist_free_all(void) tor_free(read_array); tor_free(write_array); tor_free(last_stability_doc); + tor_free(exit_bytes_read); + tor_free(exit_bytes_written); + tor_free(exit_streams); built_last_stability_doc_at = 0; predicted_ports_free(); } From ab7729f4724efbfac33c069f488e74c8ccc058f1 Mon Sep 17 00:00:00 2001 From: Karsten Loesing Date: Fri, 14 Aug 2009 14:45:53 +0200 Subject: [PATCH 149/187] Remove ./configure option for entry and dir request statistics. --- configure.in | 14 ------- src/or/config.c | 14 +------ src/or/connection.c | 2 - src/or/connection_edge.c | 4 -- src/or/directory.c | 4 -- src/or/geoip.c | 84 +++++++++++++++++++--------------------- src/or/or.h | 19 +++------ src/or/relay.c | 7 +--- src/or/router.c | 12 ++---- src/or/test.c | 6 ++- 10 files changed, 56 insertions(+), 110 deletions(-) diff --git a/configure.in b/configure.in index 7a8764ce91..df228ab18d 100644 --- a/configure.in +++ b/configure.in @@ -92,20 +92,6 @@ if test "$enable_dirreq_stats" = "yes"; then AC_DEFINE(ENABLE_DIRREQ_STATS, 1, [Defined if we try to collect per-country statistics]) fi -AC_ARG_ENABLE(buffer-stats, - AS_HELP_STRING(--enable-buffer-stats, enable code for relays to collect buffer statistics)) - -if test "$enable_buffer_stats" = "yes"; then - AC_DEFINE(ENABLE_BUFFER_STATS, 1, [Defined if we try to collect buffer statistics]) -fi - -AC_ARG_ENABLE(entry-stats, - AS_HELP_STRING(--enable-entry-stats, enable code for entry guards to collect per-country statistics)) - -if test "$enable_entry_stats" = "yes"; then - AC_DEFINE(ENABLE_ENTRY_STATS, 1, [Defined if we try to collect per-country statistics]) -fi - AC_ARG_ENABLE(gcc-warnings, AS_HELP_STRING(--enable-gcc-warnings, enable verbose warnings)) diff --git a/src/or/config.c b/src/or/config.c index 6d34008386..0fc3af47e1 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -188,12 +188,10 @@ static config_var_t _option_vars[] = { V(DirPort, UINT, "0"), V(DirPortFrontPage, FILENAME, NULL), OBSOLETE("DirPostPeriod"), -#ifdef ENABLE_DIRREQ_STATS OBSOLETE("DirRecordUsageByCountry"), OBSOLETE("DirRecordUsageGranularity"), OBSOLETE("DirRecordUsageRetainIPs"), OBSOLETE("DirRecordUsageSaveInterval"), -#endif V(DirReqStatistics, BOOL, "0"), VAR("DirServer", LINELIST, DirServers, NULL), V(DNSPort, UINT, "0"), @@ -1409,7 +1407,6 @@ options_act(or_options_t *old_options) } if (options->DirReqStatistics) { -#ifdef ENABLE_DIRREQ_STATS /* Check if GeoIP database could be loaded. */ if (!geoip_is_loaded()) { log_warn(LD_CONFIG, "Configured to measure directory request " @@ -1420,10 +1417,6 @@ options_act(or_options_t *old_options) "country and write aggregate statistics to disk. Check the " "dirreq-stats file in your data directory that will first " "be written in 24 hours from now."); -#else - log_warn(LD_CONFIG, "DirReqStatistics enabled, but Tor was built " - "without support for directory request statistics."); -#endif } if (options->ExitPortStatistics) @@ -1442,7 +1435,6 @@ options_act(or_options_t *old_options) "without cell statistics support."); #endif -#ifdef ENABLE_ENTRY_STATS if (options->EntryStatistics) { if (should_record_bridge_info(options)) { /* Don't allow measuring statistics on entry guards when configured @@ -1461,11 +1453,7 @@ options_act(or_options_t *old_options) "first be written to the data directory in 24 hours " "from now."); } -#else - if (options->EntryStatistics) - log_warn(LD_CONFIG, "EntryStatistics enabled, but Tor was built " - "without entry node statistics support."); -#endif + /* Check if we need to parse and add the EntryNodes config option. */ if (options->EntryNodes && (!old_options || diff --git a/src/or/connection.c b/src/or/connection.c index dc9c4eace2..af4d1349ba 100644 --- a/src/or/connection.c +++ b/src/or/connection.c @@ -2302,13 +2302,11 @@ connection_handle_write(connection_t *conn, int force) /* else open, or closing */ result = flush_buf_tls(or_conn->tls, conn->outbuf, max_to_write, &conn->outbuf_flushlen); -#ifdef ENABLE_DIRREQ_STATS /* If we just flushed the last bytes, check if this tunneled dir * request is done. */ if (buf_datalen(conn->outbuf) == 0 && conn->dirreq_id) geoip_change_dirreq_state(conn->dirreq_id, DIRREQ_TUNNELED, DIRREQ_OR_CONN_BUFFER_FLUSHED); -#endif switch (result) { CASE_TOR_TLS_ERROR_ANY: case TOR_TLS_CLOSE: diff --git a/src/or/connection_edge.c b/src/or/connection_edge.c index 04fffd9f57..3b399d9bd1 100644 --- a/src/or/connection_edge.c +++ b/src/or/connection_edge.c @@ -2544,11 +2544,9 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ) log_debug(LD_EXIT,"Creating new exit connection."); n_stream = edge_connection_new(CONN_TYPE_EXIT, AF_INET); -#ifdef ENABLE_DIRREQ_STATS /* Remember the tunneled request ID in the new edge connection, so that * we can measure download times. */ TO_CONN(n_stream)->dirreq_id = circ->dirreq_id; -#endif n_stream->_base.purpose = EXIT_PURPOSE_CONNECT; n_stream->stream_id = rh.stream_id; @@ -2785,11 +2783,9 @@ connection_exit_connect_dir(edge_connection_t *exitconn) dirconn->_base.purpose = DIR_PURPOSE_SERVER; dirconn->_base.state = DIR_CONN_STATE_SERVER_COMMAND_WAIT; -#ifdef ENABLE_DIRREQ_STATS /* Note that the new dir conn belongs to the same tunneled request as * the edge conn, so that we can measure download times. */ TO_CONN(dirconn)->dirreq_id = TO_CONN(exitconn)->dirreq_id; -#endif connection_link_connections(TO_CONN(dirconn), TO_CONN(exitconn)); if (connection_add(TO_CONN(exitconn))<0) { diff --git a/src/or/directory.c b/src/or/directory.c index 976c08c19a..900b2f3c5d 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -2562,7 +2562,6 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, goto done; } -#ifdef ENABLE_DIRREQ_STATS { struct in_addr in; if (tor_inet_aton((TO_CONN(conn))->address, &in)) { @@ -2578,7 +2577,6 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, DIRREQ_DIRECT); } } -#endif // note_request(request_type,dlen); (void) request_type; @@ -3210,7 +3208,6 @@ connection_dir_finished_flushing(dir_connection_t *conn) tor_assert(conn); tor_assert(conn->_base.type == CONN_TYPE_DIR); -#ifdef ENABLE_DIRREQ_STATS /* Note that we have finished writing the directory response. For direct * connections this means we're done, for tunneled connections its only * an intermediate step. */ @@ -3221,7 +3218,6 @@ connection_dir_finished_flushing(dir_connection_t *conn) geoip_change_dirreq_state(TO_CONN(conn)->global_identifier, DIRREQ_DIRECT, DIRREQ_FLUSHING_DIR_CONN_FINISHED); -#endif switch (conn->_base.state) { case DIR_CONN_STATE_CLIENT_SENDING: log_debug(LD_DIR,"client finished sending command."); diff --git a/src/or/geoip.c b/src/or/geoip.c index 5c13154205..a07e4b585f 100644 --- a/src/or/geoip.c +++ b/src/or/geoip.c @@ -347,7 +347,6 @@ geoip_determine_shares(time_t now) last_time_determined_shares = now; } -#ifdef ENABLE_DIRREQ_STATS /** Calculate which fraction of v2 and v3 directory requests aimed at caches * have been sent to us since the last call of this function up to time * now. Set *v2_share_out and *v3_share_out to the @@ -367,7 +366,6 @@ geoip_get_mean_shares(time_t now, double *v2_share_out, share_seconds = 0; return 0; } -#endif /** Note that we've seen a client connect from the IP addr (host order) * at time now. Ignored by all but bridges and directories if @@ -379,24 +377,17 @@ geoip_note_client_seen(geoip_client_action_t action, or_options_t *options = get_options(); clientmap_entry_t lookup, *ent; if (action == GEOIP_CLIENT_CONNECT) { -#ifdef ENABLE_ENTRY_STATS - if (!options->EntryStatistics) + /* Only remember statistics as entry guard or as bridge. */ + if (!options->EntryStatistics || + (!(options->BridgeRelay && options->BridgeRecordUsageByCountry))) return; -#else - if (!(options->BridgeRelay && options->BridgeRecordUsageByCountry)) - return; -#endif /* Did we recently switch from bridge to relay or back? */ if (client_history_starts > now) return; } else { -#ifndef ENABLE_DIRREQ_STATS - return; -#else if (options->BridgeRelay || options->BridgeAuthoritativeDir || !options->DirReqStatistics) return; -#endif } /* Rotate the current request period. */ @@ -495,7 +486,6 @@ geoip_remove_old_clients(time_t cutoff) client_history_starts = cutoff; } -#ifdef ENABLE_DIRREQ_STATS /** How many responses are we giving to clients requesting v2 network * statuses? */ static uint32_t ns_v2_responses[GEOIP_NS_RESPONSE_NUM]; @@ -503,7 +493,6 @@ static uint32_t ns_v2_responses[GEOIP_NS_RESPONSE_NUM]; /** How many responses are we giving to clients requesting v3 network * statuses? */ static uint32_t ns_v3_responses[GEOIP_NS_RESPONSE_NUM]; -#endif /** Note that we've rejected a client's request for a v2 or v3 network * status, encoded in action for reason reason at time @@ -512,7 +501,6 @@ void geoip_note_ns_response(geoip_client_action_t action, geoip_ns_response_t response) { -#ifdef ENABLE_DIRREQ_STATS static int arrays_initialized = 0; if (!get_options()->DirReqStatistics) return; @@ -528,10 +516,6 @@ geoip_note_ns_response(geoip_client_action_t action, ns_v3_responses[response]++; else ns_v2_responses[response]++; -#else - (void) action; - (void) response; -#endif } /** Do not mention any country from which fewer than this number of IPs have @@ -709,7 +693,6 @@ geoip_change_dirreq_state(uint64_t dirreq_id, dirreq_type_t type, } } -#ifdef ENABLE_DIRREQ_STATS /** Return a newly allocated comma-separated string containing statistics * on network status downloads. The string contains the number of completed * requests, timeouts, and still running requests as well as the download @@ -800,25 +783,18 @@ geoip_get_dirreq_history(geoip_client_action_t action, smartlist_free(dirreq_times); return result; } -#endif /** How long do we have to have observed per-country request history before we * are willing to talk about it? */ #define GEOIP_MIN_OBSERVATION_TIME (12*60*60) -/** Return a newly allocated comma-separated string containing entries for all - * the countries from which we've seen enough clients connect. The entry - * format is cc=num where num is the number of IPs we've seen connecting from - * that country, and cc is a lowercased country code. Returns NULL if we don't - * want to export geoip data yet. */ -char * -geoip_get_client_history(time_t now, geoip_client_action_t action) +/** Helper for geoip_get_client_history_dirreq() and + * geoip_get_client_history_bridge(). */ +static char * +geoip_get_client_history(time_t now, geoip_client_action_t action, + int min_observation_time, unsigned granularity) { char *result = NULL; - int min_observation_time = GEOIP_MIN_OBSERVATION_TIME; -#ifdef ENABLE_DIRREQ_STATS - min_observation_time = DIR_RECORD_USAGE_MIN_OBSERVATION_TIME; -#endif if (!geoip_is_loaded()) return NULL; if (client_history_starts < (now - min_observation_time)) { @@ -830,10 +806,6 @@ geoip_get_client_history(time_t now, geoip_client_action_t action) clientmap_entry_t **ent; unsigned *counts = tor_malloc_zero(sizeof(unsigned)*n_countries); unsigned total = 0; - unsigned granularity = IP_GRANULARITY; -#ifdef ENABLE_DIRREQ_STATS - granularity = DIR_RECORD_USAGE_GRANULARITY; -#endif HT_FOREACH(ent, clientmap, &client_history) { int country; if ((*ent)->action != (int)action) @@ -889,6 +861,34 @@ geoip_get_client_history(time_t now, geoip_client_action_t action) return result; } +/** Return a newly allocated comma-separated string containing entries for + * all the countries from which we've seen enough clients connect as a + * directory. The entry format is cc=num where num is the number of IPs + * we've seen connecting from that country, and cc is a lowercased country + * code. Returns NULL if we don't want to export geoip data yet. */ +char * +geoip_get_client_history_dirreq(time_t now, + geoip_client_action_t action) +{ + return geoip_get_client_history(now, action, + DIR_RECORD_USAGE_MIN_OBSERVATION_TIME, + DIR_RECORD_USAGE_GRANULARITY); +} + +/** Return a newly allocated comma-separated string containing entries for + * all the countries from which we've seen enough clients connect as a + * bridge. The entry format is cc=num where num is the number of IPs + * we've seen connecting from that country, and cc is a lowercased country + * code. Returns NULL if we don't want to export geoip data yet. */ +char * +geoip_get_client_history_bridge(time_t now, + geoip_client_action_t action) +{ + return geoip_get_client_history(now, action, + GEOIP_MIN_OBSERVATION_TIME, + IP_GRANULARITY); +} + /** Return a newly allocated string holding the per-country request history * for action in a format suitable for an extra-info document, or NULL * on failure. */ @@ -899,10 +899,6 @@ geoip_get_request_history(time_t now, geoip_client_action_t action) char *result; unsigned granularity = IP_GRANULARITY; int min_observation_time = GEOIP_MIN_OBSERVATION_TIME; -#ifdef ENABLE_DIRREQ_STATS - granularity = DIR_RECORD_USAGE_GRANULARITY; - min_observation_time = DIR_RECORD_USAGE_MIN_OBSERVATION_TIME; -#endif if (client_history_starts >= (now - min_observation_time)) return NULL; @@ -948,7 +944,6 @@ geoip_get_request_history(time_t now, geoip_client_action_t action) static void dump_geoip_stats(void) { -#ifdef ENABLE_DIRREQ_STATS time_t now = time(NULL); time_t request_start; char *filename = get_datadir_fname("dirreq-stats"); @@ -962,8 +957,10 @@ dump_geoip_stats(void) if (!get_options()->DirReqStatistics) goto done; - data_v2 = geoip_get_client_history(now, GEOIP_CLIENT_NETWORKSTATUS_V2); - data_v3 = geoip_get_client_history(now, GEOIP_CLIENT_NETWORKSTATUS); + data_v2 = geoip_get_client_history_dirreq(now, + GEOIP_CLIENT_NETWORKSTATUS_V2); + data_v3 = geoip_get_client_history_dirreq(now, + GEOIP_CLIENT_NETWORKSTATUS); format_iso_time(since, geoip_get_history_start()); format_iso_time(written, now); out = start_writing_to_stdio_file(filename, OPEN_FLAGS_APPEND, @@ -1045,7 +1042,6 @@ dump_geoip_stats(void) tor_free(filename); tor_free(data_v2); tor_free(data_v3); -#endif } /** Store all our geoip statistics as entry guards into diff --git a/src/or/or.h b/src/or/or.h index 4db824afea..5fed1ccf4d 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -20,9 +20,6 @@ #ifndef INSTRUMENT_DOWNLOADS #define INSTRUMENT_DOWNLOADS 1 #endif -#ifndef ENABLE_DIRREQ_STATS -#define ENABLE_DIRREQ_STATS 1 -#endif #ifndef ENABLE_BUFFER_STATS #define ENABLE_BUFFER_STATS 1 #endif @@ -975,10 +972,8 @@ typedef struct connection_t { * to the evdns_server_port is uses to listen to and answer connections. */ struct evdns_server_port *dns_server_port; -#ifdef ENABLE_DIRREQ_STATS /** Unique ID for measuring tunneled network status requests. */ uint64_t dirreq_id; -#endif } connection_t; /** Stores flags and information related to the portion of a v2 Tor OR @@ -1965,10 +1960,9 @@ typedef struct circuit_t { * linked to an OR connection. */ struct circuit_t *prev_active_on_n_conn; struct circuit_t *next; /**< Next circuit in linked list of all circuits. */ -#ifdef ENABLE_DIRREQ_STATS + /** Unique ID for measuring tunneled network status requests. */ uint64_t dirreq_id; -#endif } circuit_t; /** Largest number of relay_early cells that we can send on a given @@ -3640,15 +3634,11 @@ int dnsserv_launch_request(const char *name, int is_reverse); * leaking information. */ #define DIR_RECORD_USAGE_GRANULARITY 8 /** Time interval: Flush geoip data to disk this often. */ -#define DIR_RECORD_USAGE_RETAIN_IPS (24*60*60) +#define DIR_ENTRY_RECORD_USAGE_RETAIN_IPS (24*60*60) /** How long do we have to have observed per-country request history before * we are willing to talk about it? */ #define DIR_RECORD_USAGE_MIN_OBSERVATION_TIME (24*60*60) -/** Time interval: Flush geoip data to disk this often when measuring on an - * entry guard. */ -#define ENTRY_RECORD_USAGE_RETAIN_IPS (24*60*60) - #ifdef GEOIP_PRIVATE int geoip_parse_entry(const char *line); #endif @@ -3695,7 +3685,10 @@ typedef enum { void geoip_note_ns_response(geoip_client_action_t action, geoip_ns_response_t response); time_t geoip_get_history_start(void); -char *geoip_get_client_history(time_t now, geoip_client_action_t action); +char *geoip_get_client_history_dirreq(time_t now, + geoip_client_action_t action); +char *geoip_get_client_history_bridge(time_t now, + geoip_client_action_t action); char *geoip_get_request_history(time_t now, geoip_client_action_t action); int getinfo_helper_geoip(control_connection_t *control_conn, const char *question, char **answer); diff --git a/src/or/relay.c b/src/or/relay.c index 76577848f3..63baabc810 100644 --- a/src/or/relay.c +++ b/src/or/relay.c @@ -533,13 +533,11 @@ relay_send_command_from_edge(uint16_t stream_id, circuit_t *circ, log_debug(LD_OR,"delivering %d cell %s.", relay_command, cell_direction == CELL_DIRECTION_OUT ? "forward" : "backward"); -#ifdef ENABLE_DIRREQ_STATS /* If we are sending an END cell and this circuit is used for a tunneled * directory request, advance its state. */ if (relay_command == RELAY_COMMAND_END && circ->dirreq_id) geoip_change_dirreq_state(circ->dirreq_id, DIRREQ_TUNNELED, DIRREQ_END_CELL_SENT); -#endif if (cell_direction == CELL_DIRECTION_OUT && circ->n_conn) { /* if we're using relaybandwidthrate, this conn wants priority */ @@ -1047,7 +1045,6 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, "Begin cell for known stream. Dropping."); return 0; } -#ifdef ENABLE_DIRREQ_STATS if (rh.command == RELAY_COMMAND_BEGIN_DIR) { /* Assign this circuit and its app-ward OR connection a unique ID, * so that we can measure download times. The local edge and dir @@ -1057,7 +1054,6 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, circ->dirreq_id = ++next_id; TO_CONN(TO_OR_CIRCUIT(circ)->p_conn)->dirreq_id = circ->dirreq_id; } -#endif return connection_exit_begin_conn(cell, circ); case RELAY_COMMAND_DATA: @@ -1849,14 +1845,13 @@ connection_or_flush_from_first_active_circuit(or_connection_t *conn, int max, orcirc->processed_cells++; } #endif -#ifdef ENABLE_DIRREQ_STATS + /* If we just flushed our queue and this circuit is used for a * tunneled directory request, possibly advance its state. */ if (queue->n == 0 && TO_CONN(conn)->dirreq_id) geoip_change_dirreq_state(TO_CONN(conn)->dirreq_id, DIRREQ_TUNNELED, DIRREQ_CIRC_QUEUE_FLUSHED); -#endif connection_write_to_buf(cell->body, CELL_NETWORK_SIZE, TO_CONN(conn)); diff --git a/src/or/router.c b/src/or/router.c index d5a2576842..c66b66f0ea 100644 --- a/src/or/router.c +++ b/src/or/router.c @@ -1907,13 +1907,9 @@ char * extrainfo_get_client_geoip_summary(time_t now) { static time_t last_purged_at = 0; - int geoip_purge_interval = 48*60*60; -#ifdef ENABLE_DIRREQ_STATS - geoip_purge_interval = DIR_RECORD_USAGE_RETAIN_IPS; -#endif -#ifdef ENABLE_ENTRY_STATS - geoip_purge_interval = ENTRY_RECORD_USAGE_RETAIN_IPS; -#endif + int geoip_purge_interval = + (get_options()->DirReqStatistics || get_options()->EntryStatistics) ? + DIR_ENTRY_RECORD_USAGE_RETAIN_IPS : 48*60*60; if (now > last_purged_at+geoip_purge_interval) { /* (Note that this also discards items in the client history with * action GEOIP_CLIENT_NETWORKSTATUS{_V2}, which doesn't matter @@ -1922,7 +1918,7 @@ extrainfo_get_client_geoip_summary(time_t now) geoip_remove_old_clients(now-geoip_purge_interval); last_purged_at = now; } - return geoip_get_client_history(now, GEOIP_CLIENT_CONNECT); + return geoip_get_client_history_bridge(now, GEOIP_CLIENT_CONNECT); } /** Return true iff s is a legally valid server nickname. */ diff --git a/src/or/test.c b/src/or/test.c index dffb0a5ade..61109d007a 100644 --- a/src/or/test.c +++ b/src/or/test.c @@ -4625,14 +4625,16 @@ test_geoip(void) /* and 17 observations in ZZ... */ for (i=110; i < 127; ++i) geoip_note_client_seen(GEOIP_CLIENT_CONNECT, i, now); - s = geoip_get_client_history(now+5*24*60*60, GEOIP_CLIENT_CONNECT); + s = geoip_get_client_history_bridge(now+5*24*60*60, + GEOIP_CLIENT_CONNECT); test_assert(s); test_streq("zz=24,ab=16,xy=8", s); tor_free(s); /* Now clear out all the AB observations. */ geoip_remove_old_clients(now-6000); - s = geoip_get_client_history(now+5*24*60*60, GEOIP_CLIENT_CONNECT); + s = geoip_get_client_history_bridge(now+5*24*60*60, + GEOIP_CLIENT_CONNECT); test_assert(s); test_streq("zz=24,xy=8", s); From d97e95cb6242b94002854055a08c3190e731139b Mon Sep 17 00:00:00 2001 From: Karsten Loesing Date: Fri, 14 Aug 2009 14:52:39 +0200 Subject: [PATCH 150/187] Remove ./configure option for cell statistics. --- configure.in | 7 ---- src/or/circuitlist.c | 2 -- src/or/config.c | 6 ---- src/or/main.c | 4 --- src/or/or.h | 10 +----- src/or/relay.c | 80 +++++++++++++++++++++++++++++++++++--------- src/or/rephist.c | 2 -- 7 files changed, 66 insertions(+), 45 deletions(-) diff --git a/configure.in b/configure.in index df228ab18d..735a5240a2 100644 --- a/configure.in +++ b/configure.in @@ -85,13 +85,6 @@ case $host in ;; esac -AC_ARG_ENABLE(dirreq-stats, - AS_HELP_STRING(--enable-dirreq-stats, enable code for directories to collect per-country statistics)) - -if test "$enable_dirreq_stats" = "yes"; then - AC_DEFINE(ENABLE_DIRREQ_STATS, 1, [Defined if we try to collect per-country statistics]) -fi - AC_ARG_ENABLE(gcc-warnings, AS_HELP_STRING(--enable-gcc-warnings, enable verbose warnings)) diff --git a/src/or/circuitlist.c b/src/or/circuitlist.c index 5a20e7ebde..04b1d8edb3 100644 --- a/src/or/circuitlist.c +++ b/src/or/circuitlist.c @@ -447,11 +447,9 @@ circuit_free(circuit_t *circ) rend_data_free(ocirc->rend_data); } else { or_circuit_t *ocirc = TO_OR_CIRCUIT(circ); -#ifdef ENABLE_BUFFER_STATS /* Remember cell statistics for this circuit before deallocating. */ if (get_options()->CellStatistics) add_circ_to_buffer_stats(circ, time(NULL)); -#endif mem = ocirc; memlen = sizeof(or_circuit_t); tor_assert(circ->magic == OR_CIRCUIT_MAGIC); diff --git a/src/or/config.c b/src/or/config.c index 0fc3af47e1..89c94b1997 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -1424,16 +1424,10 @@ options_act(or_options_t *old_options) "Look for the exit-stats file that will first be written to " "the data directory in 24 hours from now."); -#ifdef ENABLE_BUFFER_STATS if (options->CellStatistics) log_notice(LD_CONFIG, "Configured to measure cell statistics. Look " "for the buffer-stats file that will first be written to " "the data directory in 24 hours from now."); -#else - if (options->CellStatistics) - log_warn(LD_CONFIG, "CellStatistics enabled, but Tor was built " - "without cell statistics support."); -#endif if (options->EntryStatistics) { if (should_record_bridge_info(options)) { diff --git a/src/or/main.c b/src/or/main.c index 4969212798..403ae93625 100644 --- a/src/or/main.c +++ b/src/or/main.c @@ -830,9 +830,7 @@ run_scheduled_events(time_t now) static time_t time_to_clean_caches = 0; static time_t time_to_recheck_bandwidth = 0; static time_t time_to_check_for_expired_networkstatus = 0; -#ifdef ENABLE_BUFFER_STATS static time_t time_to_dump_buffer_stats = 0; -#endif static time_t time_to_retry_dns_init = 0; or_options_t *options = get_options(); int i; @@ -960,13 +958,11 @@ run_scheduled_events(time_t now) time_to_check_for_expired_networkstatus = now + CHECK_EXPIRED_NS_INTERVAL; } -#ifdef ENABLE_BUFFER_STATS if (time_to_dump_buffer_stats < now) { if (get_options()->CellStatistics && time_to_dump_buffer_stats) dump_buffer_stats(); time_to_dump_buffer_stats = now + DUMP_BUFFER_STATS_INTERVAL; } -#endif /* Remove old information from rephist and the rend cache. */ if (time_to_clean_caches < now) { diff --git a/src/or/or.h b/src/or/or.h index 5fed1ccf4d..ff2e65c036 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -20,9 +20,6 @@ #ifndef INSTRUMENT_DOWNLOADS #define INSTRUMENT_DOWNLOADS 1 #endif -#ifndef ENABLE_BUFFER_STATS -#define ENABLE_BUFFER_STATS 1 -#endif #endif #ifdef MS_WINDOWS @@ -838,9 +835,7 @@ typedef struct var_cell_t { typedef struct packed_cell_t { struct packed_cell_t *next; /**< Next cell queued on this circuit. */ char body[CELL_NETWORK_SIZE]; /**< Cell as packed for network. */ -#ifdef ENABLE_BUFFER_STATS struct timeval packed_timeval; /**< When was this cell packed? */ -#endif } packed_cell_t; /** A queue of cells on a circuit, waiting to be added to the @@ -849,6 +844,7 @@ typedef struct cell_queue_t { packed_cell_t *head; /**< The first cell, or NULL if the queue is empty. */ packed_cell_t *tail; /**< The last cell, or NULL if the queue is empty. */ int n; /**< The number of cells in the queue. */ + smartlist_t *insertion_times; } cell_queue_t; /** Beginning of a RELAY cell payload. */ @@ -2086,7 +2082,6 @@ typedef struct or_circuit_t { /** True iff this circuit was made with a CREATE_FAST cell. */ unsigned int is_first_hop : 1; -#ifdef ENABLE_BUFFER_STATS /** Number of cells that were removed from circuit queue; reset every * time when writing buffer stats to disk. */ uint32_t processed_cells; @@ -2095,7 +2090,6 @@ typedef struct or_circuit_t { * exit-ward queues of this circuit; reset every time when writing * buffer stats to disk. */ uint64_t total_cell_waiting_time; -#endif } or_circuit_t; /** Convert a circuit subtype to a circuit_t.*/ @@ -4121,11 +4115,9 @@ void hs_usage_note_fetch_successful(const char *service_id, time_t now); void hs_usage_write_statistics_to_file(time_t now); void hs_usage_free_all(void); -#ifdef ENABLE_BUFFER_STATS #define DUMP_BUFFER_STATS_INTERVAL (24*60*60) void add_circ_to_buffer_stats(circuit_t *circ, time_t end_of_interval); void dump_buffer_stats(void); -#endif /********************************* rendclient.c ***************************/ diff --git a/src/or/relay.c b/src/or/relay.c index 63baabc810..add8d4c583 100644 --- a/src/or/relay.c +++ b/src/or/relay.c @@ -1612,16 +1612,48 @@ cell_queue_append(cell_queue_t *queue, packed_cell_t *cell) ++queue->n; } +/** Number of cells added to a circuit queue including their insertion + * time on millisecond detail; used for buffer statistics. */ +typedef struct insertion_time_elem_t { + uint32_t insertion_time; /**< When were cells inserted (in ms starting + at 0:00 of the current day)? */ + unsigned counter; /**< How many cells were inserted? */ +} insertion_time_elem_t; + /** Append a newly allocated copy of cell to the end of queue */ void cell_queue_append_packed_copy(cell_queue_t *queue, const cell_t *cell) { packed_cell_t *copy = packed_cell_copy(cell); -#ifdef ENABLE_BUFFER_STATS - /* Remember the exact time when this cell was put in the queue. */ - if (get_options()->CellStatistics) - tor_gettimeofday(©->packed_timeval); -#endif + /* Remember the time in millis when this cell was put in the queue. */ + if (get_options()->CellStatistics) { + struct timeval now; + uint32_t added; + insertion_time_elem_t *last_elem = NULL; + int add_new_elem = 0; + tor_gettimeofday(&now); + added = now.tv_sec % 86400L * 1000L + now.tv_usec / 1000L; + if (!queue->insertion_times) { + queue->insertion_times = smartlist_create(); + } + if (smartlist_len(queue->insertion_times) < 1) { + add_new_elem = 1; + } else { + last_elem = smartlist_get(queue->insertion_times, + smartlist_len(queue->insertion_times) - 1); + if (last_elem->insertion_time == added) + last_elem->counter++; + else + add_new_elem = 1; + } + if (add_new_elem) { + insertion_time_elem_t *elem = + tor_malloc_zero(sizeof(insertion_time_elem_t)); + elem->insertion_time = added; + elem->counter = 1; + smartlist_add(queue->insertion_times, elem); + } + } cell_queue_append(queue, copy); } @@ -1638,6 +1670,11 @@ cell_queue_clear(cell_queue_t *queue) } queue->head = queue->tail = NULL; queue->n = 0; + if (queue->insertion_times) { + SMARTLIST_FOREACH(queue->insertion_times, void *, e, tor_free(e)); + smartlist_free(queue->insertion_times); + queue->insertion_times = NULL; + } } /** Extract and return the cell at the head of queue; return NULL if @@ -1831,20 +1868,33 @@ connection_or_flush_from_first_active_circuit(or_connection_t *conn, int max, packed_cell_t *cell = cell_queue_pop(queue); tor_assert(*next_circ_on_conn_p(circ,conn)); -#ifdef ENABLE_BUFFER_STATS /* Calculate the exact time that this cell has spent in the queue. */ if (get_options()->CellStatistics && !CIRCUIT_IS_ORIGIN(circ)) { - struct timeval flushed_from_queue; + struct timeval now; + uint32_t flushed; uint32_t cell_waiting_time; - or_circuit_t *orcirc = TO_OR_CIRCUIT(circ); - tor_gettimeofday(&flushed_from_queue); - cell_waiting_time = (uint32_t) - tv_mdiff(&cell->packed_timeval, &flushed_from_queue); - - orcirc->total_cell_waiting_time += cell_waiting_time; - orcirc->processed_cells++; + tor_gettimeofday(&now); + flushed = now.tv_sec % 86400L * 1000L + now.tv_usec / 1000L; + if (!queue->insertion_times || + smartlist_len(queue->insertion_times) < 1) { + log_warn(LD_BUG, "Cannot determine insertion time of cell."); + } else { + or_circuit_t *orcirc = TO_OR_CIRCUIT(circ); + insertion_time_elem_t *elem = smartlist_get( + queue->insertion_times, 0); + cell_waiting_time = (flushed + 86400000L - elem->insertion_time) % + 86400000L; + elem->counter--; + if (elem->counter < 1) { +// TODO this operation is really expensive! write own queue impl? +// smartlist_del(queue->insertion_times, 0); + smartlist_remove(queue->insertion_times, elem); + tor_free(elem); + } + orcirc->total_cell_waiting_time += cell_waiting_time; + orcirc->processed_cells++; + } } -#endif /* If we just flushed our queue and this circuit is used for a * tunneled directory request, possibly advance its state. */ diff --git a/src/or/rephist.c b/src/or/rephist.c index 0d370a3b11..abfd47742c 100644 --- a/src/or/rephist.c +++ b/src/or/rephist.c @@ -2620,7 +2620,6 @@ hs_usage_write_statistics_to_file(time_t now) /*** cell statistics ***/ -#ifdef ENABLE_BUFFER_STATS /** Start of the current buffer stats interval. */ time_t start_of_buffer_stats_interval; @@ -2787,5 +2786,4 @@ dump_buffer_stats(void) tor_free(str); #undef SHARES } -#endif From 9179bcb923f150ed2a3e62ab816709d7b0177f05 Mon Sep 17 00:00:00 2001 From: Karsten Loesing Date: Fri, 14 Aug 2009 15:30:24 +0200 Subject: [PATCH 151/187] Include contents of *-stats files in descriptor. --- src/or/config.c | 1 + src/or/geoip.c | 34 ++++++------- src/or/or.h | 3 ++ src/or/rephist.c | 18 +++---- src/or/router.c | 111 +++++++++++++++++++++++++++++++++++++++++-- src/or/routerparse.c | 50 +++++++++++++++++++ 6 files changed, 185 insertions(+), 32 deletions(-) diff --git a/src/or/config.c b/src/or/config.c index 89c94b1997..7b3f0ad07f 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -208,6 +208,7 @@ static config_var_t _option_vars[] = { V(ExitPolicy, LINELIST, NULL), V(ExitPolicyRejectPrivate, BOOL, "1"), V(ExitPortStatistics, BOOL, "0"), + V(ExtraInfoStatistics, BOOL, "0"), V(FallbackNetworkstatusFile, FILENAME, SHARE_DATADIR PATH_SEPARATOR "tor" PATH_SEPARATOR "fallback-consensus"), V(FascistFirewall, BOOL, "0"), diff --git a/src/or/geoip.c b/src/or/geoip.c index a07e4b585f..824428b7d9 100644 --- a/src/or/geoip.c +++ b/src/or/geoip.c @@ -948,7 +948,7 @@ dump_geoip_stats(void) time_t request_start; char *filename = get_datadir_fname("dirreq-stats"); char *data_v2 = NULL, *data_v3 = NULL; - char since[ISO_TIME_LEN+1], written[ISO_TIME_LEN+1]; + char written[ISO_TIME_LEN+1]; open_file_t *open_file = NULL; double v2_share = 0.0, v3_share = 0.0; FILE *out; @@ -961,14 +961,13 @@ dump_geoip_stats(void) GEOIP_CLIENT_NETWORKSTATUS_V2); data_v3 = geoip_get_client_history_dirreq(now, GEOIP_CLIENT_NETWORKSTATUS); - format_iso_time(since, geoip_get_history_start()); - format_iso_time(written, now); + format_iso_time(written, geoip_get_history_start() + REQUEST_HIST_PERIOD); out = start_writing_to_stdio_file(filename, OPEN_FLAGS_APPEND, 0600, &open_file); if (!out) goto done; - if (fprintf(out, "written %s\nstarted-at %s\nns-ips %s\nns-v2-ips %s\n", - written, since, + if (fprintf(out, "dirreq-stats-end %s (%d s)\ndirreq-v3-ips %s\n" + "dirreq-v2-ips %s\n", written, REQUEST_HIST_PERIOD, data_v3 ? data_v3 : "", data_v2 ? data_v2 : "") < 0) goto done; tor_free(data_v2); @@ -976,11 +975,9 @@ dump_geoip_stats(void) request_start = current_request_period_starts - (n_old_request_periods * REQUEST_HIST_PERIOD); - format_iso_time(since, request_start); data_v2 = geoip_get_request_history(now, GEOIP_CLIENT_NETWORKSTATUS_V2); data_v3 = geoip_get_request_history(now, GEOIP_CLIENT_NETWORKSTATUS); - if (fprintf(out, "requests-start %s\nn-ns-reqs %s\nn-v2-ns-reqs %s\n", - since, + if (fprintf(out, "dirreq-v3-reqs %s\ndirreq-v2-reqs %s\n", data_v3 ? data_v3 : "", data_v2 ? data_v2 : "") < 0) goto done; #define RESPONSE_GRANULARITY 8 @@ -991,7 +988,7 @@ dump_geoip_stats(void) ns_v3_responses[i], RESPONSE_GRANULARITY); } #undef RESPONSE_GRANULARITY - if (fprintf(out, "n-ns-resp ok=%u,not-enough-sigs=%u,unavailable=%u," + if (fprintf(out, "dirreq-v3-resp ok=%u,not-enough-sigs=%u,unavailable=%u," "not-found=%u,not-modified=%u,busy=%u\n", ns_v3_responses[GEOIP_SUCCESS], ns_v3_responses[GEOIP_REJECT_NOT_ENOUGH_SIGS], @@ -1000,7 +997,7 @@ dump_geoip_stats(void) ns_v3_responses[GEOIP_REJECT_NOT_MODIFIED], ns_v3_responses[GEOIP_REJECT_BUSY]) < 0) goto done; - if (fprintf(out, "n-v2-ns-resp ok=%u,unavailable=%u," + if (fprintf(out, "dirreq-v2-resp ok=%u,unavailable=%u," "not-found=%u,not-modified=%u,busy=%u\n", ns_v2_responses[GEOIP_SUCCESS], ns_v2_responses[GEOIP_REJECT_UNAVAILABLE], @@ -1011,9 +1008,9 @@ dump_geoip_stats(void) memset(ns_v2_responses, 0, sizeof(ns_v2_responses)); memset(ns_v3_responses, 0, sizeof(ns_v3_responses)); if (!geoip_get_mean_shares(now, &v2_share, &v3_share)) { - if (fprintf(out, "v2-ns-share %0.2lf%%\n", v2_share*100) < 0) + if (fprintf(out, "dirreq-v2-share %0.2lf%%\n", v2_share*100) < 0) goto done; - if (fprintf(out, "v3-ns-share %0.2lf%%\n", v3_share*100) < 0) + if (fprintf(out, "dirreq-v3-share %0.2lf%%\n", v3_share*100) < 0) goto done; } @@ -1021,7 +1018,7 @@ dump_geoip_stats(void) DIRREQ_DIRECT); data_v3 = geoip_get_dirreq_history(GEOIP_CLIENT_NETWORKSTATUS, DIRREQ_DIRECT); - if (fprintf(out, "ns-direct-dl %s\nns-v2-direct-dl %s\n", + if (fprintf(out, "dirreq-v3-direct-dl %s\ndirreq-v2-direct-dl %s\n", data_v3 ? data_v3 : "", data_v2 ? data_v2 : "") < 0) goto done; tor_free(data_v2); @@ -1030,7 +1027,7 @@ dump_geoip_stats(void) DIRREQ_TUNNELED); data_v3 = geoip_get_dirreq_history(GEOIP_CLIENT_NETWORKSTATUS, DIRREQ_TUNNELED); - if (fprintf(out, "ns-tunneled-dl %s\nns-v2-tunneled-dl %s\n", + if (fprintf(out, "dirreq-v3-tunneled-dl %s\ndirreq-v2-tunneled-dl %s\n", data_v3 ? data_v3 : "", data_v2 ? data_v2 : "") < 0) goto done; @@ -1053,19 +1050,18 @@ dump_entry_stats(void) time_t now = time(NULL); char *filename = get_datadir_fname("entry-stats"); char *data = NULL; - char since[ISO_TIME_LEN+1], written[ISO_TIME_LEN+1]; + char written[ISO_TIME_LEN+1]; open_file_t *open_file = NULL; FILE *out; data = geoip_get_client_history(now, GEOIP_CLIENT_CONNECT); - format_iso_time(since, geoip_get_history_start()); - format_iso_time(written, now); + format_iso_time(written, geoip_get_history_start() + REQUEST_HIST_PERIOD); out = start_writing_to_stdio_file(filename, OPEN_FLAGS_APPEND, 0600, &open_file); if (!out) goto done; - if (fprintf(out, "written %s\nstarted-at %s\nips %s\n", - written, since, data ? data : "") < 0) + if (fprintf(out, "entry-stats-end %s (%d s)\nentry-ips %s\n", + written, REQUEST_HIST_PERIOD, data ? data : "") < 0) goto done; finish_writing_to_file(open_file); diff --git a/src/or/or.h b/src/or/or.h index ff2e65c036..ce4fbd074e 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -2516,6 +2516,9 @@ typedef struct { /** If true, the user wants us to collect statistics as entry node. */ int EntryStatistics; + /** If true, include statistics file contents in extra-info documents. */ + int ExtraInfoStatistics; + /** If true, do not believe anybody who tells us that a domain resolves * to an internal address, or that an internal address has a PTR mapping. * Helps avoid some cross-site attacks. */ diff --git a/src/or/rephist.c b/src/or/rephist.c index abfd47742c..a9bf95da1b 100644 --- a/src/or/rephist.c +++ b/src/or/rephist.c @@ -1390,7 +1390,8 @@ write_exit_stats(time_t when) } /* written yyyy-mm-dd HH:MM:SS (n s) */ - if (fprintf(out, "written %s (%d s)\n", t, EXIT_STATS_INTERVAL_SEC) < 0) + if (fprintf(out, "exit-stats-end %s (%d s)\n", t, + EXIT_STATS_INTERVAL_SEC) < 0) goto done; /* Count the total number of bytes, so that we can attribute all @@ -1408,7 +1409,8 @@ write_exit_stats(time_t when) b = r ? exit_bytes_read : exit_bytes_written; tor_assert(b); if (fprintf(out, "%s ", - r ? "kibibytes-read" : "kibibytes-written")<0) + r ? "exit-kibibytes-read" + : "exit-kibibytes-written") < 0) goto done; comma = 0; @@ -1435,7 +1437,7 @@ write_exit_stats(time_t when) goto done; } /* streams-opened port=num,.. */ - if (fprintf(out, "streams-opened ")<0) + if (fprintf(out, "exit-streams-opened ") < 0) goto done; comma = 0; other_streams = 0; @@ -2733,7 +2735,7 @@ dump_buffer_stats(void) if (!out) goto done; format_iso_time(written, now); - if (fprintf(out, "written %s (%d s)\n", written, + if (fprintf(out, "cell-stats-end %s (%d s)\n", written, DUMP_BUFFER_STATS_INTERVAL) < 0) goto done; for (i = 0; i < SHARES; i++) { @@ -2742,7 +2744,7 @@ dump_buffer_stats(void) smartlist_add(str_build, tor_strdup(buf)); } str = smartlist_join_strings(str_build, ",", 0, NULL); - if (fprintf(out, "processed-cells %s\n", str) < 0) + if (fprintf(out, "cell-processed-cells %s\n", str) < 0) goto done; tor_free(str); SMARTLIST_FOREACH(str_build, char *, c, tor_free(c)); @@ -2753,7 +2755,7 @@ dump_buffer_stats(void) smartlist_add(str_build, tor_strdup(buf)); } str = smartlist_join_strings(str_build, ",", 0, NULL); - if (fprintf(out, "queued-cells %s\n", str) < 0) + if (fprintf(out, "cell-queued-cells %s\n", str) < 0) goto done; tor_free(str); SMARTLIST_FOREACH(str_build, char *, c, tor_free(c)); @@ -2764,13 +2766,13 @@ dump_buffer_stats(void) smartlist_add(str_build, tor_strdup(buf)); } str = smartlist_join_strings(str_build, ",", 0, NULL); - if (fprintf(out, "time-in-queue %s\n", str) < 0) + if (fprintf(out, "cell-time-in-queue %s\n", str) < 0) goto done; tor_free(str); SMARTLIST_FOREACH(str_build, char *, c, tor_free(c)); smartlist_free(str_build); str_build = NULL; - if (fprintf(out, "number-of-circuits-per-share %d\n", + if (fprintf(out, "cell-circuits-per-decile %d\n", (number_of_circuits + SHARES - 1) / SHARES) < 0) goto done; finish_writing_to_file(open_file); diff --git a/src/or/router.c b/src/or/router.c index c66b66f0ea..e1181971c4 100644 --- a/src/or/router.c +++ b/src/or/router.c @@ -1382,8 +1382,10 @@ router_rebuild_descriptor(int force) ei->cache_info.published_on = ri->cache_info.published_on; memcpy(ei->cache_info.identity_digest, ri->cache_info.identity_digest, DIGEST_LEN); - ei->cache_info.signed_descriptor_body = tor_malloc(8192); - if (extrainfo_dump_to_string(ei->cache_info.signed_descriptor_body, 8192, + ei->cache_info.signed_descriptor_body = + tor_malloc(MAX_EXTRAINFO_UPLOAD_SIZE); + if (extrainfo_dump_to_string(ei->cache_info.signed_descriptor_body, + MAX_EXTRAINFO_UPLOAD_SIZE, ei, get_identity_key()) < 0) { log_warn(LD_BUG, "Couldn't generate extra-info descriptor."); extrainfo_free(ei); @@ -1824,6 +1826,31 @@ router_dump_router_to_string(char *s, size_t maxlen, routerinfo_t *router, return (int)written+1; } +/** Load the contents of filename and write its contents to + * **contents. Return 1 for success, 0 if the file does not exit, + * or -1 for failure. */ +static int +load_stats_file(const char *filename, char **contents) +{ + int r = -1; + char *fname = get_datadir_fname(filename); + switch (file_status(fname)) { + case FN_FILE: + if ((*contents = read_file_to_str(fname, 0, NULL))) + r = 1; + break; + case FN_NOENT: + r = 0; + break; + case FN_ERROR: + case FN_DIR: + default: + break; + } + tor_free(fname); + return r; +} + /** Write the contents of extrainfo to the maxlen-byte string * s, signing them with ident_key. Return 0 on success, * negative on failure. */ @@ -1838,6 +1865,7 @@ extrainfo_dump_to_string(char *s, size_t maxlen, extrainfo_t *extrainfo, char *bandwidth_usage; int result; size_t len; + static int write_stats_to_extrainfo = 1; base16_encode(identity, sizeof(identity), extrainfo->cache_info.identity_digest, DIGEST_LEN); @@ -1849,6 +1877,56 @@ extrainfo_dump_to_string(char *s, size_t maxlen, extrainfo_t *extrainfo, "published %s\n%s", extrainfo->nickname, identity, published, bandwidth_usage); + + if (options->ExtraInfoStatistics && write_stats_to_extrainfo) { + char *contents = NULL; + log_info(LD_GENERAL, "Adding stats to extra-info descriptor."); + if (options->DirReqStatistics && + load_stats_file("dirreq-stats", &contents) > 0) { + int pos = strlen(s); + if (strlcpy(s + pos, contents, maxlen - strlen(s)) != + strlen(contents)) { + log_warn(LD_DIR, "Could not write dirreq-stats to extra-info " + "descriptor."); + s[pos] = '\0'; + } + tor_free(contents); + } + if (options->EntryStatistics && + load_stats_file("entry-stats", &contents) > 0) { + int pos = strlen(s); + if (strlcpy(s + pos, contents, maxlen - strlen(s)) != + strlen(contents)) { + log_warn(LD_DIR, "Could not write entry-stats to extra-info " + "descriptor."); + s[pos] = '\0'; + } + tor_free(contents); + } + if (options->CellStatistics && + load_stats_file("buffer-stats", &contents) > 0) { + int pos = strlen(s); + if (strlcpy(s + pos, contents, maxlen - strlen(s)) != + strlen(contents)) { + log_warn(LD_DIR, "Could not write buffer-stats to extra-info " + "descriptor."); + s[pos] = '\0'; + } + tor_free(contents); + } + if (options->ExitPortStatistics && + load_stats_file("exit-stats", &contents) > 0) { + int pos = strlen(s); + if (strlcpy(s + pos, contents, maxlen - strlen(s)) != + strlen(contents)) { + log_warn(LD_DIR, "Could not write exit-stats to extra-info " + "descriptor."); + s[pos] = '\0'; + } + tor_free(contents); + } + } + tor_free(bandwidth_usage); if (result<0) return -1; @@ -1877,22 +1955,45 @@ extrainfo_dump_to_string(char *s, size_t maxlen, extrainfo_t *extrainfo, if (router_append_dirobj_signature(s+len, maxlen-len, digest, ident_key)<0) return -1; -#ifdef DEBUG_ROUTER_DUMP_ROUTER_TO_STRING { char *cp, *s_dup; extrainfo_t *ei_tmp; cp = s_dup = tor_strdup(s); ei_tmp = extrainfo_parse_entry_from_string(cp, NULL, 1, NULL); - if (!ei_tmp) { + if (ei_tmp) { log_err(LD_BUG, "We just generated an extrainfo descriptor we can't parse."); log_err(LD_BUG, "Descriptor was: <<%s>>", s); return -1; + } else { + log_debug(LD_GENERAL, "We generated an extra-info descriptor:\n%s", + s); } tor_free(s_dup); extrainfo_free(ei_tmp); } -#endif + + if (options->ExtraInfoStatistics && write_stats_to_extrainfo) { + char *cp, *s_dup; + extrainfo_t *ei_tmp; + cp = s_dup = tor_strdup(s); + ei_tmp = extrainfo_parse_entry_from_string(cp, NULL, 1, NULL); + if (!ei_tmp) { + log_warn(LD_GENERAL, + "We just generated an extra-info descriptor with " + "statistics that we can't parse. Not adding statistics to " + "this or any future extra-info descriptors. Descriptor " + "was:\n%s", s); + write_stats_to_extrainfo = 0; + tor_free(s); + s = NULL; + extrainfo_dump_to_string(s, maxlen, extrainfo, ident_key); + } + tor_free(s_dup); + extrainfo_free(ei_tmp); + } + + log_info(LD_GENERAL, "Done with dumping our extra-info descriptor."); return (int)strlen(s)+1; } diff --git a/src/or/routerparse.c b/src/or/routerparse.c index 8021158e31..1e505d67c7 100644 --- a/src/or/routerparse.c +++ b/src/or/routerparse.c @@ -62,6 +62,31 @@ typedef enum { K_HIDDEN_SERVICE_DIR, K_ALLOW_SINGLE_HOP_EXITS, + K_DIRREQ_END, + K_DIRREQ_V2_IPS, + K_DIRREQ_V3_IPS, + K_DIRREQ_V2_REQS, + K_DIRREQ_V3_REQS, + K_DIRREQ_V2_SHARE, + K_DIRREQ_V3_SHARE, + K_DIRREQ_V2_RESP, + K_DIRREQ_V3_RESP, + K_DIRREQ_V2_DIR, + K_DIRREQ_V3_DIR, + K_DIRREQ_V2_TUN, + K_DIRREQ_V3_TUN, + K_ENTRY_END, + K_ENTRY_IPS, + K_CELL_END, + K_CELL_PROCESSED, + K_CELL_QUEUED, + K_CELL_TIME, + K_CELL_CIRCS, + K_EXIT_END, + K_EXIT_WRITTEN, + K_EXIT_READ, + K_EXIT_OPENED, + K_DIR_KEY_CERTIFICATE_VERSION, K_DIR_IDENTITY_KEY, K_DIR_KEY_PUBLISHED, @@ -257,6 +282,31 @@ static token_rule_t extrainfo_token_table[] = { T0N("opt", K_OPT, CONCAT_ARGS, OBJ_OK ), T01("read-history", K_READ_HISTORY, ARGS, NO_OBJ ), T01("write-history", K_WRITE_HISTORY, ARGS, NO_OBJ ), + T01("dirreq-stats-end", K_DIRREQ_END, ARGS, NO_OBJ ), + T01("dirreq-v2-ips", K_DIRREQ_V2_IPS, ARGS, NO_OBJ ), + T01("dirreq-v3-ips", K_DIRREQ_V3_IPS, ARGS, NO_OBJ ), + T01("dirreq-v2-reqs", K_DIRREQ_V2_REQS, ARGS, NO_OBJ ), + T01("dirreq-v3-reqs", K_DIRREQ_V3_REQS, ARGS, NO_OBJ ), + T01("dirreq-v2-share", K_DIRREQ_V2_SHARE, ARGS, NO_OBJ ), + T01("dirreq-v3-share", K_DIRREQ_V3_SHARE, ARGS, NO_OBJ ), + T01("dirreq-v2-resp", K_DIRREQ_V2_RESP, ARGS, NO_OBJ ), + T01("dirreq-v3-resp", K_DIRREQ_V3_RESP, ARGS, NO_OBJ ), + T01("dirreq-v2-direct-dl", K_DIRREQ_V2_DIR, ARGS, NO_OBJ ), + T01("dirreq-v3-direct-dl", K_DIRREQ_V3_DIR, ARGS, NO_OBJ ), + T01("dirreq-v2-tunneled-dl", K_DIRREQ_V2_TUN, ARGS, NO_OBJ ), + T01("dirreq-v3-tunneled-dl", K_DIRREQ_V3_TUN, ARGS, NO_OBJ ), + T01("entry-stats-end", K_ENTRY_END, ARGS, NO_OBJ ), + T01("entry-ips", K_ENTRY_IPS, ARGS, NO_OBJ ), + T01("cell-stats-end", K_CELL_END, ARGS, NO_OBJ ), + T01("cell-processed-cells", K_CELL_PROCESSED, ARGS, NO_OBJ ), + T01("cell-queued-cells", K_CELL_QUEUED, ARGS, NO_OBJ ), + T01("cell-time-in-queue", K_CELL_TIME, ARGS, NO_OBJ ), + T01("cell-circuits-per-decile", K_CELL_CIRCS, ARGS, NO_OBJ ), + T01("exit-stats-end", K_EXIT_END, ARGS, NO_OBJ ), + T01("exit-kibibytes-written", K_EXIT_WRITTEN, ARGS, NO_OBJ ), + T01("exit-kibibytes-read", K_EXIT_READ, ARGS, NO_OBJ ), + T01("exit-streams-opened", K_EXIT_OPENED, ARGS, NO_OBJ ), + T1_START( "extra-info", K_EXTRA_INFO, GE(2), NO_OBJ ), END_OF_TABLE From 20c95a3d21d7c635a269d62b5c010a516a4c98c7 Mon Sep 17 00:00:00 2001 From: Karsten Loesing Date: Fri, 14 Aug 2009 16:24:11 +0200 Subject: [PATCH 152/187] Fix trivial bug when uploading extra-info documents. --- src/or/router.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/or/router.c b/src/or/router.c index e1181971c4..e6d6cc37b2 100644 --- a/src/or/router.c +++ b/src/or/router.c @@ -1960,7 +1960,7 @@ extrainfo_dump_to_string(char *s, size_t maxlen, extrainfo_t *extrainfo, extrainfo_t *ei_tmp; cp = s_dup = tor_strdup(s); ei_tmp = extrainfo_parse_entry_from_string(cp, NULL, 1, NULL); - if (ei_tmp) { + if (!ei_tmp) { log_err(LD_BUG, "We just generated an extrainfo descriptor we can't parse."); log_err(LD_BUG, "Descriptor was: <<%s>>", s); From 858a8f809d62e74df8ff71b60ae39631e8d6d5c1 Mon Sep 17 00:00:00 2001 From: Karsten Loesing Date: Sun, 16 Aug 2009 23:42:46 +0200 Subject: [PATCH 153/187] Reduce cell statistics accuracy from 1 ms to 10 ms. --- src/or/relay.c | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/or/relay.c b/src/or/relay.c index add8d4c583..929b2e7707 100644 --- a/src/or/relay.c +++ b/src/or/relay.c @@ -1613,10 +1613,10 @@ cell_queue_append(cell_queue_t *queue, packed_cell_t *cell) } /** Number of cells added to a circuit queue including their insertion - * time on millisecond detail; used for buffer statistics. */ + * time on 10 millisecond detail; used for buffer statistics. */ typedef struct insertion_time_elem_t { - uint32_t insertion_time; /**< When were cells inserted (in ms starting - at 0:00 of the current day)? */ + uint32_t insertion_time; /**< When were cells inserted (in 10 ms steps + * starting at 0:00 of the current day)? */ unsigned counter; /**< How many cells were inserted? */ } insertion_time_elem_t; @@ -1632,7 +1632,8 @@ cell_queue_append_packed_copy(cell_queue_t *queue, const cell_t *cell) insertion_time_elem_t *last_elem = NULL; int add_new_elem = 0; tor_gettimeofday(&now); - added = now.tv_sec % 86400L * 1000L + now.tv_usec / 1000L; +#define SECONDS_IN_A_DAY 86400L + added = now.tv_sec % SECONDS_IN_A_DAY * 10L + now.tv_usec / 100000L; if (!queue->insertion_times) { queue->insertion_times = smartlist_create(); } @@ -1874,7 +1875,7 @@ connection_or_flush_from_first_active_circuit(or_connection_t *conn, int max, uint32_t flushed; uint32_t cell_waiting_time; tor_gettimeofday(&now); - flushed = now.tv_sec % 86400L * 1000L + now.tv_usec / 1000L; + flushed = now.tv_sec % SECONDS_IN_A_DAY * 10L + now.tv_usec / 100000L; if (!queue->insertion_times || smartlist_len(queue->insertion_times) < 1) { log_warn(LD_BUG, "Cannot determine insertion time of cell."); @@ -1882,8 +1883,9 @@ connection_or_flush_from_first_active_circuit(or_connection_t *conn, int max, or_circuit_t *orcirc = TO_OR_CIRCUIT(circ); insertion_time_elem_t *elem = smartlist_get( queue->insertion_times, 0); - cell_waiting_time = (flushed + 86400000L - elem->insertion_time) % - 86400000L; + cell_waiting_time = (flushed + SECONDS_IN_A_DAY * 10L - + elem->insertion_time) % (SECONDS_IN_A_DAY * 10L); +#undef SECONDS_IN_A_DAY elem->counter--; if (elem->counter < 1) { // TODO this operation is really expensive! write own queue impl? From f37af0180d63f59dbb856e897f44a77d0390ab1a Mon Sep 17 00:00:00 2001 From: Karsten Loesing Date: Mon, 17 Aug 2009 13:07:56 +0200 Subject: [PATCH 154/187] Implement queue with O(1) operations, and correct some math. --- src/or/or.h | 18 +++++++++++++-- src/or/relay.c | 59 +++++++++++++++++++++++++------------------------- 2 files changed, 45 insertions(+), 32 deletions(-) diff --git a/src/or/or.h b/src/or/or.h index ce4fbd074e..c9eb4a3f91 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -835,16 +835,30 @@ typedef struct var_cell_t { typedef struct packed_cell_t { struct packed_cell_t *next; /**< Next cell queued on this circuit. */ char body[CELL_NETWORK_SIZE]; /**< Cell as packed for network. */ - struct timeval packed_timeval; /**< When was this cell packed? */ } packed_cell_t; +/** Number of cells added to a circuit queue including their insertion + * time on 10 millisecond detail; used for buffer statistics. */ +typedef struct insertion_time_elem_t { + struct insertion_time_elem_t *next; + uint32_t insertion_time; /**< When were cells inserted (in 10 ms steps + * starting at 0:00 of the current day)? */ + unsigned counter; /**< How many cells were inserted? */ +} insertion_time_elem_t; + +/** Queue of insertion times. */ +typedef struct insertion_time_queue_t { + struct insertion_time_elem_t *first; + struct insertion_time_elem_t *last; +} insertion_time_queue_t; + /** A queue of cells on a circuit, waiting to be added to the * or_connection_t's outbuf. */ typedef struct cell_queue_t { packed_cell_t *head; /**< The first cell, or NULL if the queue is empty. */ packed_cell_t *tail; /**< The last cell, or NULL if the queue is empty. */ int n; /**< The number of cells in the queue. */ - smartlist_t *insertion_times; + insertion_time_queue_t *insertion_times; } cell_queue_t; /** Beginning of a RELAY cell payload. */ diff --git a/src/or/relay.c b/src/or/relay.c index 929b2e7707..66475b4ec1 100644 --- a/src/or/relay.c +++ b/src/or/relay.c @@ -1612,14 +1612,6 @@ cell_queue_append(cell_queue_t *queue, packed_cell_t *cell) ++queue->n; } -/** Number of cells added to a circuit queue including their insertion - * time on 10 millisecond detail; used for buffer statistics. */ -typedef struct insertion_time_elem_t { - uint32_t insertion_time; /**< When were cells inserted (in 10 ms steps - * starting at 0:00 of the current day)? */ - unsigned counter; /**< How many cells were inserted? */ -} insertion_time_elem_t; - /** Append a newly allocated copy of cell to the end of queue */ void cell_queue_append_packed_copy(cell_queue_t *queue, const cell_t *cell) @@ -1629,21 +1621,20 @@ cell_queue_append_packed_copy(cell_queue_t *queue, const cell_t *cell) if (get_options()->CellStatistics) { struct timeval now; uint32_t added; - insertion_time_elem_t *last_elem = NULL; + insertion_time_queue_t *it_queue = queue->insertion_times; int add_new_elem = 0; tor_gettimeofday(&now); #define SECONDS_IN_A_DAY 86400L - added = now.tv_sec % SECONDS_IN_A_DAY * 10L + now.tv_usec / 100000L; - if (!queue->insertion_times) { - queue->insertion_times = smartlist_create(); + added = (now.tv_sec % SECONDS_IN_A_DAY) * 100L + now.tv_usec / 10000L; + if (!it_queue) { + it_queue = tor_malloc_zero(sizeof(insertion_time_queue_t)); + queue->insertion_times = it_queue; } - if (smartlist_len(queue->insertion_times) < 1) { + if (!it_queue->first) { add_new_elem = 1; } else { - last_elem = smartlist_get(queue->insertion_times, - smartlist_len(queue->insertion_times) - 1); - if (last_elem->insertion_time == added) - last_elem->counter++; + if (it_queue->last->insertion_time == added) + it_queue->last->counter++; else add_new_elem = 1; } @@ -1652,7 +1643,12 @@ cell_queue_append_packed_copy(cell_queue_t *queue, const cell_t *cell) tor_malloc_zero(sizeof(insertion_time_elem_t)); elem->insertion_time = added; elem->counter = 1; - smartlist_add(queue->insertion_times, elem); + if (it_queue->last) { + it_queue->last->next = elem; + it_queue->last = elem; + } else { + it_queue->first = it_queue->last = elem; + } } } cell_queue_append(queue, copy); @@ -1672,8 +1668,11 @@ cell_queue_clear(cell_queue_t *queue) queue->head = queue->tail = NULL; queue->n = 0; if (queue->insertion_times) { - SMARTLIST_FOREACH(queue->insertion_times, void *, e, tor_free(e)); - smartlist_free(queue->insertion_times); + while (queue->insertion_times->first) { + insertion_time_elem_t *elem = queue->insertion_times->first; + queue->insertion_times->first = elem->next; + tor_free(elem); + } queue->insertion_times = NULL; } } @@ -1874,23 +1873,23 @@ connection_or_flush_from_first_active_circuit(or_connection_t *conn, int max, struct timeval now; uint32_t flushed; uint32_t cell_waiting_time; + insertion_time_queue_t *it_queue = queue->insertion_times; tor_gettimeofday(&now); - flushed = now.tv_sec % SECONDS_IN_A_DAY * 10L + now.tv_usec / 100000L; - if (!queue->insertion_times || - smartlist_len(queue->insertion_times) < 1) { + flushed = (now.tv_sec % SECONDS_IN_A_DAY) * 100L + + now.tv_usec / 10000L; + if (!it_queue || !it_queue->first) { log_warn(LD_BUG, "Cannot determine insertion time of cell."); } else { or_circuit_t *orcirc = TO_OR_CIRCUIT(circ); - insertion_time_elem_t *elem = smartlist_get( - queue->insertion_times, 0); - cell_waiting_time = (flushed + SECONDS_IN_A_DAY * 10L - - elem->insertion_time) % (SECONDS_IN_A_DAY * 10L); + insertion_time_elem_t *elem = it_queue->first; + cell_waiting_time = (flushed * 10L + SECONDS_IN_A_DAY * 1000L - + elem->insertion_time * 10L) % (SECONDS_IN_A_DAY * 1000L); #undef SECONDS_IN_A_DAY elem->counter--; if (elem->counter < 1) { -// TODO this operation is really expensive! write own queue impl? -// smartlist_del(queue->insertion_times, 0); - smartlist_remove(queue->insertion_times, elem); + it_queue->first = elem->next; + if (elem == it_queue->last) + it_queue->last = NULL; tor_free(elem); } orcirc->total_cell_waiting_time += cell_waiting_time; From 799af41157c74922a8cc7889510e4b1b040ea215 Mon Sep 17 00:00:00 2001 From: Karsten Loesing Date: Mon, 17 Aug 2009 13:27:09 +0200 Subject: [PATCH 155/187] Use memory pool for insertion times. --- src/or/relay.c | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/src/or/relay.c b/src/or/relay.c index 66475b4ec1..b45856d51e 100644 --- a/src/or/relay.c +++ b/src/or/relay.c @@ -1525,6 +1525,10 @@ static int total_cells_allocated = 0; /** A memory pool to allocate packed_cell_t objects. */ static mp_pool_t *cell_pool = NULL; +/** Memory pool to allocate insertion_time_elem_t objects used for cell + * statistics. */ +static mp_pool_t *it_pool = NULL; + /** Allocate structures to hold cells. */ void init_cell_pool(void) @@ -1533,7 +1537,8 @@ init_cell_pool(void) cell_pool = mp_pool_new(sizeof(packed_cell_t), 128*1024); } -/** Free all storage used to hold cells. */ +/** Free all storage used to hold cells (and insertion times if we measure + * cell statistics). */ void free_cell_pool(void) { @@ -1542,6 +1547,10 @@ free_cell_pool(void) mp_pool_destroy(cell_pool); cell_pool = NULL; } + if (it_pool) { + mp_pool_destroy(it_pool); + it_pool = NULL; + } } /** Free excess storage in cell pool. */ @@ -1623,6 +1632,8 @@ cell_queue_append_packed_copy(cell_queue_t *queue, const cell_t *cell) uint32_t added; insertion_time_queue_t *it_queue = queue->insertion_times; int add_new_elem = 0; + if (!it_pool) + it_pool = mp_pool_new(sizeof(insertion_time_elem_t), 1024); tor_gettimeofday(&now); #define SECONDS_IN_A_DAY 86400L added = (now.tv_sec % SECONDS_IN_A_DAY) * 100L + now.tv_usec / 10000L; @@ -1639,8 +1650,8 @@ cell_queue_append_packed_copy(cell_queue_t *queue, const cell_t *cell) add_new_elem = 1; } if (add_new_elem) { - insertion_time_elem_t *elem = - tor_malloc_zero(sizeof(insertion_time_elem_t)); + insertion_time_elem_t *elem = mp_pool_get(it_pool); + elem->next = NULL; elem->insertion_time = added; elem->counter = 1; if (it_queue->last) { @@ -1671,7 +1682,7 @@ cell_queue_clear(cell_queue_t *queue) while (queue->insertion_times->first) { insertion_time_elem_t *elem = queue->insertion_times->first; queue->insertion_times->first = elem->next; - tor_free(elem); + mp_pool_release(elem); } queue->insertion_times = NULL; } @@ -1890,7 +1901,7 @@ connection_or_flush_from_first_active_circuit(or_connection_t *conn, int max, it_queue->first = elem->next; if (elem == it_queue->last) it_queue->last = NULL; - tor_free(elem); + mp_pool_release(elem); } orcirc->total_cell_waiting_time += cell_waiting_time; orcirc->processed_cells++; From dccadb30cd1ceddb9063ba074ba75bf07575e407 Mon Sep 17 00:00:00 2001 From: Karsten Loesing Date: Tue, 18 Aug 2009 15:53:08 +0200 Subject: [PATCH 156/187] Clean up proposal 166 and its implementation. --- .../proposals/166-statistics-extra-info-docs.txt | 14 +++++++------- src/or/connection.c | 2 ++ src/or/connection_edge.c | 3 +++ src/or/or.h | 8 ++++---- src/or/relay.c | 13 +++---------- src/or/rephist.c | 2 +- src/or/router.c | 8 ++++---- 7 files changed, 24 insertions(+), 26 deletions(-) diff --git a/doc/spec/proposals/166-statistics-extra-info-docs.txt b/doc/spec/proposals/166-statistics-extra-info-docs.txt index 3716c049cc..6bc71cafd8 100644 --- a/doc/spec/proposals/166-statistics-extra-info-docs.txt +++ b/doc/spec/proposals/166-statistics-extra-info-docs.txt @@ -298,7 +298,7 @@ Exit statistics: The last type of statistics affects exit nodes counting the number of bytes written and read and the number of streams opened per port and - per 24 hours. Exit port statistics can be measured from looking of + per 24 hours. Exit port statistics can be measured from looking at headers of BEGIN and DATA cells. A BEGIN cell contains the exit port that is required for the exit node to open a new exit stream. Subsequent DATA cells coming from the client or being sent back to the @@ -361,7 +361,7 @@ Implementation notes: basically means renaming keywords. 2. The timing of writing the four *-stats files should be unified, so - that they are written exactly after 24 hours after starting the + that they are written exactly 24 hours after starting the relay. Right now, the measurement intervals for dirreq, entry, and exit stats starts with the first observed request, and files are written when observing the first request that occurs more than 24 @@ -373,14 +373,14 @@ Implementation notes: directory until they are included in extra-info documents. The reason is that the 24-hour measurement interval can be very different from the 18-hour publication interval of extra-info - documents. When a relay crashed after finishing a measurement + documents. When a relay crashes after finishing a measurement interval, but before publishing the next extra-info document, statistics would get lost. Therefore, statistics are written to disk when finishing a measurement interval and read from disk when - generating an extra-info document. As a result, the *-stats files - need to be overwritten after 24 hours, rather than appending new - statistics to them. Further, the contents of the *-stats files need - to be checked in the process of generating extra-info documents. + generating an extra-info document. Only the statistics that were + appended to the *-stats files within the past 24 hours are included + in extra-info documents. Further, the contents of the *-stats files + need to be checked in the process of generating extra-info documents. 4. With the statistics patches being tested, the ./configure options should be removed and the statistics code be compiled by default. diff --git a/src/or/connection.c b/src/or/connection.c index af4d1349ba..48f8278573 100644 --- a/src/or/connection.c +++ b/src/or/connection.c @@ -2302,11 +2302,13 @@ connection_handle_write(connection_t *conn, int force) /* else open, or closing */ result = flush_buf_tls(or_conn->tls, conn->outbuf, max_to_write, &conn->outbuf_flushlen); + /* If we just flushed the last bytes, check if this tunneled dir * request is done. */ if (buf_datalen(conn->outbuf) == 0 && conn->dirreq_id) geoip_change_dirreq_state(conn->dirreq_id, DIRREQ_TUNNELED, DIRREQ_OR_CONN_BUFFER_FLUSHED); + switch (result) { CASE_TOR_TLS_ERROR_ANY: case TOR_TLS_CLOSE: diff --git a/src/or/connection_edge.c b/src/or/connection_edge.c index 3b399d9bd1..1df576d636 100644 --- a/src/or/connection_edge.c +++ b/src/or/connection_edge.c @@ -2544,9 +2544,11 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ) log_debug(LD_EXIT,"Creating new exit connection."); n_stream = edge_connection_new(CONN_TYPE_EXIT, AF_INET); + /* Remember the tunneled request ID in the new edge connection, so that * we can measure download times. */ TO_CONN(n_stream)->dirreq_id = circ->dirreq_id; + n_stream->_base.purpose = EXIT_PURPOSE_CONNECT; n_stream->stream_id = rh.stream_id; @@ -2786,6 +2788,7 @@ connection_exit_connect_dir(edge_connection_t *exitconn) /* Note that the new dir conn belongs to the same tunneled request as * the edge conn, so that we can measure download times. */ TO_CONN(dirconn)->dirreq_id = TO_CONN(exitconn)->dirreq_id; + connection_link_connections(TO_CONN(dirconn), TO_CONN(exitconn)); if (connection_add(TO_CONN(exitconn))<0) { diff --git a/src/or/or.h b/src/or/or.h index c9eb4a3f91..9a0f51f461 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -840,7 +840,7 @@ typedef struct packed_cell_t { /** Number of cells added to a circuit queue including their insertion * time on 10 millisecond detail; used for buffer statistics. */ typedef struct insertion_time_elem_t { - struct insertion_time_elem_t *next; + struct insertion_time_elem_t *next; /**< Next element in queue. */ uint32_t insertion_time; /**< When were cells inserted (in 10 ms steps * starting at 0:00 of the current day)? */ unsigned counter; /**< How many cells were inserted? */ @@ -848,8 +848,8 @@ typedef struct insertion_time_elem_t { /** Queue of insertion times. */ typedef struct insertion_time_queue_t { - struct insertion_time_elem_t *first; - struct insertion_time_elem_t *last; + struct insertion_time_elem_t *first; /**< First element in queue. */ + struct insertion_time_elem_t *last; /**< Last element in queue. */ } insertion_time_queue_t; /** A queue of cells on a circuit, waiting to be added to the @@ -858,7 +858,7 @@ typedef struct cell_queue_t { packed_cell_t *head; /**< The first cell, or NULL if the queue is empty. */ packed_cell_t *tail; /**< The last cell, or NULL if the queue is empty. */ int n; /**< The number of cells in the queue. */ - insertion_time_queue_t *insertion_times; + insertion_time_queue_t *insertion_times; /**< Insertion times of cells. */ } cell_queue_t; /** Beginning of a RELAY cell payload. */ diff --git a/src/or/relay.c b/src/or/relay.c index b45856d51e..a79a4c161a 100644 --- a/src/or/relay.c +++ b/src/or/relay.c @@ -1626,12 +1626,11 @@ void cell_queue_append_packed_copy(cell_queue_t *queue, const cell_t *cell) { packed_cell_t *copy = packed_cell_copy(cell); - /* Remember the time in millis when this cell was put in the queue. */ + /* Remember the time when this cell was put in the queue. */ if (get_options()->CellStatistics) { struct timeval now; uint32_t added; insertion_time_queue_t *it_queue = queue->insertion_times; - int add_new_elem = 0; if (!it_pool) it_pool = mp_pool_new(sizeof(insertion_time_elem_t), 1024); tor_gettimeofday(&now); @@ -1641,15 +1640,9 @@ cell_queue_append_packed_copy(cell_queue_t *queue, const cell_t *cell) it_queue = tor_malloc_zero(sizeof(insertion_time_queue_t)); queue->insertion_times = it_queue; } - if (!it_queue->first) { - add_new_elem = 1; + if (it_queue->last && it_queue->last->insertion_time == added) { + it_queue->last->counter++; } else { - if (it_queue->last->insertion_time == added) - it_queue->last->counter++; - else - add_new_elem = 1; - } - if (add_new_elem) { insertion_time_elem_t *elem = mp_pool_get(it_pool); elem->next = NULL; elem->insertion_time = added; diff --git a/src/or/rephist.c b/src/or/rephist.c index a9bf95da1b..3e4ba672d0 100644 --- a/src/or/rephist.c +++ b/src/or/rephist.c @@ -1336,7 +1336,7 @@ rep_hist_note_bytes_read(size_t num_bytes, time_t when) /* The following data structures are arrays and no fancy smartlists or maps, * so that all write operations can be done in constant time. This comes at * the price of some memory (1.25 MB) and linear complexity when writing - * stats. */ + * stats for measuring relays. */ /** Number of bytes read in current period by exit port */ static uint64_t *exit_bytes_read = NULL; /** Number of bytes written in current period by exit port */ diff --git a/src/or/router.c b/src/or/router.c index e6d6cc37b2..3249f49ce0 100644 --- a/src/or/router.c +++ b/src/or/router.c @@ -1269,6 +1269,7 @@ router_rebuild_descriptor(int force) uint32_t addr; char platform[256]; int hibernating = we_are_hibernating(); + size_t ei_size; or_options_t *options = get_options(); if (desc_clean_since && !force) @@ -1382,11 +1383,10 @@ router_rebuild_descriptor(int force) ei->cache_info.published_on = ri->cache_info.published_on; memcpy(ei->cache_info.identity_digest, ri->cache_info.identity_digest, DIGEST_LEN); - ei->cache_info.signed_descriptor_body = - tor_malloc(MAX_EXTRAINFO_UPLOAD_SIZE); + ei_size = options->ExtraInfoStatistics ? MAX_EXTRAINFO_UPLOAD_SIZE : 8192; + ei->cache_info.signed_descriptor_body = tor_malloc(ei_size); if (extrainfo_dump_to_string(ei->cache_info.signed_descriptor_body, - MAX_EXTRAINFO_UPLOAD_SIZE, - ei, get_identity_key()) < 0) { + ei_size, ei, get_identity_key()) < 0) { log_warn(LD_BUG, "Couldn't generate extra-info descriptor."); extrainfo_free(ei); return -1; From 10fbc998e146b271508066e7680cb0cd00a70c63 Mon Sep 17 00:00:00 2001 From: Karsten Loesing Date: Tue, 18 Aug 2009 16:14:22 +0200 Subject: [PATCH 157/187] Update dir-spec.txt and man page. --- doc/spec/dir-spec.txt | 194 ++++++++++++++++++++++++++++++++++++++++++ doc/tor.1.in | 28 ++++++ 2 files changed, 222 insertions(+) diff --git a/doc/spec/dir-spec.txt b/doc/spec/dir-spec.txt index 0a3b0c11bc..1ab8ac3843 100644 --- a/doc/spec/dir-spec.txt +++ b/doc/spec/dir-spec.txt @@ -641,6 +641,200 @@ "geoip-start" is the time at which we began collecting geoip statistics. + "dirreq-stats-end" YYYY-MM-DD HH:MM:SS (NSEC s) NL + [At most once.] + + YYYY-MM-DD HH:MM:SS defines the end of the included measurement + interval of length NSEC seconds (86400 seconds by default). + + A "dirreq-stats-end" line, as well as any other "dirreq-*" line, + is only added when the relay has opened its Dir port and after 24 + hours of measuring directory requests. + + "dirreq-v2-ips" CC=N,CC=N,... NL + [At most once.] + "dirreq-v3-ips" CC=N,CC=N,... NL + [At most once.] + + List of mappings from two-letter country codes to the number of + unique IP addresses that have connected from that country to + request a v2/v3 network status, rounded up to the nearest multiple + of 8. Only those IP addresses are counted that the directory can + answer with a 200 OK status code. + + "dirreq-v2-reqs" CC=N,CC=N,... NL + [At most once.] + "dirreq-v3-reqs" CC=N,CC=N,... NL + [At most once.] + + List of mappings from two-letter country codes to the number of + requests for v2/v3 network statuses from that country, rounded up + to the nearest multiple of 8. Only those requests are counted that + the directory can answer with a 200 OK status code. + + "dirreq-v2-share" num% NL + [At most once.] + "dirreq-v3-share" num% NL + [At most once.] + + The share of v2/v3 network status requests that the directory + expects to receive from clients based on its advertised bandwidth + compared to the overall network bandwidth capacity. Shares are + formatted in percent with two decimal places. Shares are + calculated as means over the whole 24-hour interval. + + "dirreq-v2-resp" status=num,... NL + [At most once.] + "dirreq-v3-resp" status=nul,... NL + [At most once.] + + List of mappings from response statuses to the number of requests + for v2/v3 network statuses that were answered with that response + status, rounded up to the nearest multiple of 4. Only response + statuses with at least 1 response are reported. New response + statuses can be added at any time. The current list of response + statuses is as follows: + + "ok": a network status request is answered; this number + corresponds to the sum of all requests as reported in + "dirreq-v2-reqs" or "dirreq-v3-reqs", respectively, before + rounding up. + "not-enough-sigs: a version 3 network status is not signed by a + sufficient number of requested authorities. + "unavailable": a requested network status object is unavailable. + "not-found": a requested network status is not found. + "not-modified": a network status has not been modified since the + If-Modified-Since time that is included in the request. + "busy": the directory is busy. + + "dirreq-v2-direct-dl" key=val,... NL + [At most once.] + "dirreq-v3-direct-dl" key=val,... NL + [At most once.] + "dirreq-v2-tunneled-dl" key=val,... NL + [At most once.] + "dirreq-v3-tunneled-dl" key=val,... NL + [At most once.] + + List of statistics about possible failures in the download process + of v2/v3 network statuses. Requests are either "direct" + HTTP-encoded requests over the relay's directory port, or + "tunneled" requests using a BEGIN_DIR cell over the relay's OR + port. The list of possible statistics can change, and statistics + can be left out from reporting. The current list of statistics is + as follows: + + Successful downloads and failures: + + "complete": a client has finished the download successfully. + "timeout": a download did not finish within 10 minutes after + starting to send the response. + "running": a download is still running at the end of the + measurement period for less than 10 minutes after starting to + send the response. + + Download times: + + "min", "max": smallest and largest measured bandwidth in B/s. + "d[1-4,6-9]": 1st to 4th and 6th to 9th decile of measured + bandwidth in B/s. For a given decile i, i/10 of all downloads + had a smaller bandwidth than di, and (10-i)/10 of all downloads + had a larger bandwidth than di. + "q[1,3]": 1st and 3rd quartile of measured bandwidth in B/s. One + fourth of all downloads had a smaller bandwidth than q1, one + fourth of all downloads had a larger bandwidth than q3, and the + remaining half of all downloads had a bandwidth between q1 and + q3. + "md": median of measured bandwidth in B/s. Half of the downloads + had a smaller bandwidth than md, the other half had a larger + bandwidth than md. + + "entry-stats-end" YYYY-MM-DD HH:MM:SS (NSEC s) NL + [At most once.] + + YYYY-MM-DD HH:MM:SS defines the end of the included measurement + interval of length NSEC seconds (86400 seconds by default). + + An "entry-stats-end" line, as well as any other "entry-*" + line, is first added after the relay has been running for at least + 24 hours. + + "entry-ips" CC=N,CC=N,... NL + [At most once.] + + List of mappings from two-letter country codes to the number of + unique IP addresses that have connected from that country to the + relay and which are no known other relays, rounded up to the + nearest multiple of 8. + + "cell-stats-end" YYYY-MM-DD HH:MM:SS (NSEC s) NL + [At most once.] + + YYYY-MM-DD HH:MM:SS defines the end of the included measurement + interval of length NSEC seconds (86400 seconds by default). + + A "cell-stats-end" line, as well as any other "cell-*" line, + is first added after the relay has been running for at least 24 + hours. + + "cell-processed-cells" num,...,num NL + [At most once.] + + Mean number of processed cells per circuit, subdivided into + deciles of circuits by the number of cells they have processed in + descending order from loudest to quietest circuits. + + "cell-queued-cells" num,...,num NL + [At most once.] + + Mean number of cells contained in queues by circuit decile. These + means are calculated by 1) determining the mean number of cells in + a single circuit between its creation and its termination and 2) + calculating the mean for all circuits in a given decile as + determined in "cell-processed-cells". Numbers have a precision of + two decimal places. + + "cell-time-in-queue" num,...,num NL + [At most once.] + + Mean time cells spend in circuit queues in milliseconds. Times are + calculated by 1) determining the mean time cells spend in the + queue of a single circuit and 2) calculating the mean for all + circuits in a given decile as determined in + "cell-processed-cells". + + "cell-circuits-per-decile" num NL + [At most once.] + + Mean number of circuits that are included in any of the deciles, + rounded up to the next integer. + + "exit-stats-end" YYYY-MM-DD HH:MM:SS (NSEC s) NL + [At most once.] + + YYYY-MM-DD HH:MM:SS defines the end of the included measurement + interval of length NSEC seconds (86400 seconds by default). + + An "exit-stats-end" line, as well as any other "exit-*" line, is + first added after the relay has been running for at least 24 hours + and only if the relay permits exiting (where exiting to a single + port and IP address is sufficient). + + "exit-kibibytes-written" port=N,port=N,... NL + [At most once.] + "exit-kibibytes-read" port=N,port=N,... NL + [At most once.] + + List of mappings from ports to the number of kibibytes that the + relay has written to or read from exit connections to that port, + rounded up to the next full kibibyte. + + "exit-streams-opened" port=N,port=N,... NL + [At most once.] + + List of mappings from ports to the number of opened exit streams + to that port, rounded up to the nearest multiple of 4. + "router-signature" NL Signature NL [At end, exactly once.] diff --git a/doc/tor.1.in b/doc/tor.1.in index a49a26d47b..54e35aa6c6 100644 --- a/doc/tor.1.in +++ b/doc/tor.1.in @@ -1056,6 +1056,34 @@ behalf of clients. .TP \fBGeoIPFile \fR\fIfilename\fP A filename containing GeoIP data, for use with BridgeRecordUsageByCountry. +.LP +.TP +\fBCellStatistics \fR\fB0\fR|\fB1\fR\fP +When this option is enabled, Tor writes statistics on the mean time that +cells spend in circuit queues to disk every 24 hours. (Default: 0) +.LP +.TP +\fBDirReqStatistics \fR\fB0\fR|\fB1\fR\fP +When this option is enabled, Tor writes statistics on the number and +response time of network status requests to disk every 24 hours. +(Default: 0) +.LP +.TP +\fBEntryStatistics \fR\fB0\fR|\fB1\fR\fP +When this option is enabled, Tor writes statistics on the number of +directly connecting clients to disk every 24 hours. (Default: 0) +.LP +.TP +\fBExitPortStatistics \fR\fB0\fR|\fB1\fR\fP +When this option is enabled, Tor writes statistics on the number of +relayed bytes and opened stream per exit port to disk every 24 hours. +(Default: 0) +.LP +.TP +\fBExtraInfoStatistics \fR\fB0\fR|\fB1\fR\fP +When this option is enabled, Tor includes previously gathered statistics +in its extra-info documents that it uploads to the directory authorities. +(Default: 0) .SH DIRECTORY SERVER OPTIONS .PP From 4e29f3342754fa8802c364b58f7b2f98e983a291 Mon Sep 17 00:00:00 2001 From: Karsten Loesing Date: Wed, 19 Aug 2009 15:41:12 +0200 Subject: [PATCH 158/187] Write all statistics to disk exactly every 24 hours. --- doc/tor.1.in | 12 ++- src/or/circuitlist.c | 2 +- src/or/config.c | 40 +++---- src/or/connection.c | 4 +- src/or/connection_edge.c | 2 +- src/or/geoip.c | 106 ++++++++++--------- src/or/main.c | 44 +++++++- src/or/or.h | 22 ++-- src/or/rephist.c | 223 ++++++++++++++++++--------------------- 9 files changed, 236 insertions(+), 219 deletions(-) diff --git a/doc/tor.1.in b/doc/tor.1.in index 54e35aa6c6..7606ffb69a 100644 --- a/doc/tor.1.in +++ b/doc/tor.1.in @@ -1060,24 +1060,26 @@ A filename containing GeoIP data, for use with BridgeRecordUsageByCountry. .TP \fBCellStatistics \fR\fB0\fR|\fB1\fR\fP When this option is enabled, Tor writes statistics on the mean time that -cells spend in circuit queues to disk every 24 hours. (Default: 0) +cells spend in circuit queues to disk every 24 hours. Cannot be changed +while Tor is running. (Default: 0) .LP .TP \fBDirReqStatistics \fR\fB0\fR|\fB1\fR\fP When this option is enabled, Tor writes statistics on the number and -response time of network status requests to disk every 24 hours. -(Default: 0) +response time of network status requests to disk every 24 hours. Cannot be +changed while Tor is running. (Default: 0) .LP .TP \fBEntryStatistics \fR\fB0\fR|\fB1\fR\fP When this option is enabled, Tor writes statistics on the number of -directly connecting clients to disk every 24 hours. (Default: 0) +directly connecting clients to disk every 24 hours. Cannot be changed +while Tor is running. (Default: 0) .LP .TP \fBExitPortStatistics \fR\fB0\fR|\fB1\fR\fP When this option is enabled, Tor writes statistics on the number of relayed bytes and opened stream per exit port to disk every 24 hours. -(Default: 0) +Cannot be changed while Tor is running. (Default: 0) .LP .TP \fBExtraInfoStatistics \fR\fB0\fR|\fB1\fR\fP diff --git a/src/or/circuitlist.c b/src/or/circuitlist.c index 04b1d8edb3..065559620c 100644 --- a/src/or/circuitlist.c +++ b/src/or/circuitlist.c @@ -449,7 +449,7 @@ circuit_free(circuit_t *circ) or_circuit_t *ocirc = TO_OR_CIRCUIT(circ); /* Remember cell statistics for this circuit before deallocating. */ if (get_options()->CellStatistics) - add_circ_to_buffer_stats(circ, time(NULL)); + rep_hist_buffer_stats_add_circ(circ, time(NULL)); mem = ocirc; memlen = sizeof(or_circuit_t); tor_assert(circ->magic == OR_CIRCUIT_MAGIC); diff --git a/src/or/config.c b/src/or/config.c index 7b3f0ad07f..acc544b8b8 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -1407,29 +1407,13 @@ options_act(or_options_t *old_options) tor_free(actual_fname); } - if (options->DirReqStatistics) { + if (options->DirReqStatistics && !geoip_is_loaded()) { /* Check if GeoIP database could be loaded. */ - if (!geoip_is_loaded()) { - log_warn(LD_CONFIG, "Configured to measure directory request " - "statistics, but no GeoIP database found!"); - return -1; - } - log_notice(LD_CONFIG, "Configured to count directory requests by " - "country and write aggregate statistics to disk. Check the " - "dirreq-stats file in your data directory that will first " - "be written in 24 hours from now."); + log_warn(LD_CONFIG, "Configured to measure directory request " + "statistics, but no GeoIP database found!"); + return -1; } - if (options->ExitPortStatistics) - log_notice(LD_CONFIG, "Configured to measure exit port statistics. " - "Look for the exit-stats file that will first be written to " - "the data directory in 24 hours from now."); - - if (options->CellStatistics) - log_notice(LD_CONFIG, "Configured to measure cell statistics. Look " - "for the buffer-stats file that will first be written to " - "the data directory in 24 hours from now."); - if (options->EntryStatistics) { if (should_record_bridge_info(options)) { /* Don't allow measuring statistics on entry guards when configured @@ -1442,11 +1426,7 @@ options_act(or_options_t *old_options) log_warn(LD_CONFIG, "Configured to measure entry node statistics, " "but no GeoIP database found!"); return -1; - } else - log_notice(LD_CONFIG, "Configured to measure entry node " - "statistics. Look for the entry-stats file that will " - "first be written to the data directory in 24 hours " - "from now."); + } } /* Check if we need to parse and add the EntryNodes config option. */ @@ -3784,6 +3764,16 @@ options_transition_allowed(or_options_t *old, or_options_t *new_val, return -1; } + if (old->CellStatistics != new_val->CellStatistics || + old->DirReqStatistics != new_val->DirReqStatistics || + old->EntryStatistics != new_val->EntryStatistics || + old->ExitPortStatistics != new_val->ExitPortStatistics) { + *msg = tor_strdup("While Tor is running, changing either " + "CellStatistics, DirReqStatistics, EntryStatistics, " + "or ExitPortStatistics is not allowed."); + return -1; + } + return 0; } diff --git a/src/or/connection.c b/src/or/connection.c index 48f8278573..c961f16c6c 100644 --- a/src/or/connection.c +++ b/src/or/connection.c @@ -1704,12 +1704,12 @@ connection_buckets_decrement(connection_t *conn, time_t now, if (num_read > 0) { if (conn->type == CONN_TYPE_EXIT) - rep_hist_note_exit_bytes_read(conn->port, num_read, now); + rep_hist_note_exit_bytes_read(conn->port, num_read); rep_hist_note_bytes_read(num_read, now); } if (num_written > 0) { if (conn->type == CONN_TYPE_EXIT) - rep_hist_note_exit_bytes_written(conn->port, num_written, now); + rep_hist_note_exit_bytes_written(conn->port, num_written); rep_hist_note_bytes_written(num_written, now); } diff --git a/src/or/connection_edge.c b/src/or/connection_edge.c index 1df576d636..f25202725e 100644 --- a/src/or/connection_edge.c +++ b/src/or/connection_edge.c @@ -333,7 +333,7 @@ connection_edge_finished_connecting(edge_connection_t *edge_conn) escaped_safe_str(conn->address),conn->port, safe_str(fmt_addr(&conn->addr))); - rep_hist_note_exit_stream_opened(conn->port, approx_time()); + rep_hist_note_exit_stream_opened(conn->port); conn->state = EXIT_CONN_STATE_OPEN; connection_watch_events(conn, READ_EVENT); /* stop writing, keep reading */ diff --git a/src/or/geoip.c b/src/or/geoip.c index 824428b7d9..9f37c021c1 100644 --- a/src/or/geoip.c +++ b/src/or/geoip.c @@ -12,8 +12,6 @@ #include "ht.h" static void clear_geoip_db(void); -static void dump_geoip_stats(void); -static void dump_entry_stats(void); /** An entry from the GeoIP file: maps an IP range to a country. */ typedef struct geoip_entry_t { @@ -390,37 +388,6 @@ geoip_note_client_seen(geoip_client_action_t action, return; } - /* Rotate the current request period. */ - while (current_request_period_starts + REQUEST_HIST_PERIOD < now) { - if (!geoip_countries) - geoip_countries = smartlist_create(); - if (!current_request_period_starts) { - current_request_period_starts = now; - break; - } - /* Also discard all items in the client history that are too old. - * (This only works here because bridge and directory stats are - * independent. Otherwise, we'd only want to discard those items - * with action GEOIP_CLIENT_NETWORKSTATUS{_V2}.) */ - geoip_remove_old_clients(current_request_period_starts); - /* Before rotating, write the current stats to disk. */ - dump_geoip_stats(); - if (get_options()->EntryStatistics) - dump_entry_stats(); - /* Now rotate request period */ - SMARTLIST_FOREACH(geoip_countries, geoip_country_t *, c, { - memmove(&c->n_v2_ns_requests[0], &c->n_v2_ns_requests[1], - sizeof(uint32_t)*(REQUEST_HIST_LEN-1)); - memmove(&c->n_v3_ns_requests[0], &c->n_v3_ns_requests[1], - sizeof(uint32_t)*(REQUEST_HIST_LEN-1)); - c->n_v2_ns_requests[REQUEST_HIST_LEN-1] = 0; - c->n_v3_ns_requests[REQUEST_HIST_LEN-1] = 0; - }); - current_request_period_starts += REQUEST_HIST_PERIOD; - if (n_old_request_periods < REQUEST_HIST_LEN-1) - ++n_old_request_periods; - } - lookup.ipaddr = addr; lookup.action = (int)action; ent = HT_FIND(clientmap, &client_history, &lookup); @@ -940,12 +907,20 @@ geoip_get_request_history(time_t now, geoip_client_action_t action) return result; } +/** Start time of directory request stats. */ +static time_t start_of_dirreq_stats_interval; + +/** Initialize directory request stats. */ +void +geoip_dirreq_stats_init(time_t now) +{ + start_of_dirreq_stats_interval = now; +} + /** Store all our geoip statistics into $DATADIR/dirreq-stats. */ -static void -dump_geoip_stats(void) +void +geoip_dirreq_stats_write(time_t now) { - time_t now = time(NULL); - time_t request_start; char *filename = get_datadir_fname("dirreq-stats"); char *data_v2 = NULL, *data_v3 = NULL; char written[ISO_TIME_LEN+1]; @@ -957,11 +932,14 @@ dump_geoip_stats(void) if (!get_options()->DirReqStatistics) goto done; + /* Discard all items in the client history that are too old. */ + geoip_remove_old_clients(start_of_dirreq_stats_interval); + data_v2 = geoip_get_client_history_dirreq(now, GEOIP_CLIENT_NETWORKSTATUS_V2); data_v3 = geoip_get_client_history_dirreq(now, GEOIP_CLIENT_NETWORKSTATUS); - format_iso_time(written, geoip_get_history_start() + REQUEST_HIST_PERIOD); + format_iso_time(written, now); out = start_writing_to_stdio_file(filename, OPEN_FLAGS_APPEND, 0600, &open_file); if (!out) @@ -973,8 +951,6 @@ dump_geoip_stats(void) tor_free(data_v2); tor_free(data_v3); - request_start = current_request_period_starts - - (n_old_request_periods * REQUEST_HIST_PERIOD); data_v2 = geoip_get_request_history(now, GEOIP_CLIENT_NETWORKSTATUS_V2); data_v3 = geoip_get_request_history(now, GEOIP_CLIENT_NETWORKSTATUS); if (fprintf(out, "dirreq-v3-reqs %s\ndirreq-v2-reqs %s\n", @@ -1033,6 +1009,22 @@ dump_geoip_stats(void) finish_writing_to_file(open_file); open_file = NULL; + + /* Rotate request period */ + SMARTLIST_FOREACH(geoip_countries, geoip_country_t *, c, { + memmove(&c->n_v2_ns_requests[0], &c->n_v2_ns_requests[1], + sizeof(uint32_t)*(REQUEST_HIST_LEN-1)); + memmove(&c->n_v3_ns_requests[0], &c->n_v3_ns_requests[1], + sizeof(uint32_t)*(REQUEST_HIST_LEN-1)); + c->n_v2_ns_requests[REQUEST_HIST_LEN-1] = 0; + c->n_v3_ns_requests[REQUEST_HIST_LEN-1] = 0; + }); + current_request_period_starts += REQUEST_HIST_PERIOD; + if (n_old_request_periods < REQUEST_HIST_LEN-1) + ++n_old_request_periods; + + start_of_dirreq_stats_interval = now; + done: if (open_file) abort_writing_to_file(open_file); @@ -1041,29 +1033,46 @@ dump_geoip_stats(void) tor_free(data_v3); } +/** Start time of entry stats. */ +static time_t start_of_entry_stats_interval; + +/** Initialize entry stats. */ +void +geoip_entry_stats_init(time_t now) +{ + start_of_entry_stats_interval = now; +} + /** Store all our geoip statistics as entry guards into * $DATADIR/entry-stats. */ -static void -dump_entry_stats(void) +void +geoip_entry_stats_write(time_t now) { -#ifdef ENABLE_ENTRY_STATS - time_t now = time(NULL); char *filename = get_datadir_fname("entry-stats"); char *data = NULL; char written[ISO_TIME_LEN+1]; open_file_t *open_file = NULL; FILE *out; - data = geoip_get_client_history(now, GEOIP_CLIENT_CONNECT); - format_iso_time(written, geoip_get_history_start() + REQUEST_HIST_PERIOD); + if (!get_options()->EntryStatistics) + goto done; + + /* Discard all items in the client history that are too old. */ + geoip_remove_old_clients(start_of_entry_stats_interval); + + data = geoip_get_client_history_dirreq(now, GEOIP_CLIENT_CONNECT); + format_iso_time(written, now); out = start_writing_to_stdio_file(filename, OPEN_FLAGS_APPEND, 0600, &open_file); if (!out) goto done; - if (fprintf(out, "entry-stats-end %s (%d s)\nentry-ips %s\n", - written, REQUEST_HIST_PERIOD, data ? data : "") < 0) + if (fprintf(out, "entry-stats-end %s (%u s)\nentry-ips %s\n", + written, (unsigned) (now - start_of_entry_stats_interval), + data ? data : "") < 0) goto done; + start_of_entry_stats_interval = now; + finish_writing_to_file(open_file); open_file = NULL; done: @@ -1071,7 +1080,6 @@ dump_entry_stats(void) abort_writing_to_file(open_file); tor_free(filename); tor_free(data); -#endif } /** Helper used to implement GETINFO ip-to-country/... controller command. */ diff --git a/src/or/main.c b/src/or/main.c index 403ae93625..16136cb27f 100644 --- a/src/or/main.c +++ b/src/or/main.c @@ -830,7 +830,7 @@ run_scheduled_events(time_t now) static time_t time_to_clean_caches = 0; static time_t time_to_recheck_bandwidth = 0; static time_t time_to_check_for_expired_networkstatus = 0; - static time_t time_to_dump_buffer_stats = 0; + static time_t time_to_write_stats_files = 0; static time_t time_to_retry_dns_init = 0; or_options_t *options = get_options(); int i; @@ -958,10 +958,44 @@ run_scheduled_events(time_t now) time_to_check_for_expired_networkstatus = now + CHECK_EXPIRED_NS_INTERVAL; } - if (time_to_dump_buffer_stats < now) { - if (get_options()->CellStatistics && time_to_dump_buffer_stats) - dump_buffer_stats(); - time_to_dump_buffer_stats = now + DUMP_BUFFER_STATS_INTERVAL; + /* 1g. Check whether we should write statistics to disk. + */ + if (time_to_write_stats_files >= 0 && time_to_write_stats_files < now) { +#define WRITE_STATS_INTERVAL (24*60*60) + or_options_t *options = get_options(); + if (options->CellStatistics || options->DirReqStatistics || + options->EntryStatistics || options->ExitPortStatistics) { + if (!time_to_write_stats_files) { + /* Initialize stats. */ + if (options->CellStatistics) + rep_hist_buffer_stats_init(now); + if (options->DirReqStatistics) + geoip_dirreq_stats_init(now); + if (options->EntryStatistics) + geoip_entry_stats_init(now); + if (options->ExitPortStatistics) + rep_hist_exit_stats_init(now); + log_notice(LD_CONFIG, "Configured to measure statistics. Look for " + "the *-stats files that will first be written to the " + "data directory in %d hours from now.", + WRITE_STATS_INTERVAL / (60 * 60)); + time_to_write_stats_files = now + WRITE_STATS_INTERVAL; + } else { + /* Write stats to disk. */ + time_to_write_stats_files += WRITE_STATS_INTERVAL; + if (options->CellStatistics) + rep_hist_buffer_stats_write(time_to_write_stats_files); + if (options->DirReqStatistics) + geoip_dirreq_stats_write(time_to_write_stats_files); + if (options->EntryStatistics) + geoip_entry_stats_write(time_to_write_stats_files); + if (options->ExitPortStatistics) + rep_hist_exit_stats_write(time_to_write_stats_files); + } + } else { + /* Never write stats to disk */ + time_to_write_stats_files = -1; + } } /* Remove old information from rephist and the rend cache. */ diff --git a/src/or/or.h b/src/or/or.h index 9a0f51f461..98ab860a0c 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -3739,6 +3739,11 @@ void geoip_start_dirreq(uint64_t dirreq_id, size_t response_size, void geoip_change_dirreq_state(uint64_t dirreq_id, dirreq_type_t type, dirreq_state_t new_state); +void geoip_dirreq_stats_init(time_t now); +void geoip_dirreq_stats_write(time_t now); +void geoip_entry_stats_init(time_t now); +void geoip_entry_stats_write(time_t now); + /********************************* hibernate.c **********************/ int accounting_parse_options(or_options_t *options, int validate_only); @@ -4076,11 +4081,11 @@ void rep_hist_note_extend_failed(const char *from_name, const char *to_name); void rep_hist_dump_stats(time_t now, int severity); void rep_hist_note_bytes_read(size_t num_bytes, time_t when); void rep_hist_note_bytes_written(size_t num_bytes, time_t when); -void rep_hist_note_exit_bytes_read(uint16_t port, size_t num_bytes, - time_t now); -void rep_hist_note_exit_bytes_written(uint16_t port, size_t num_bytes, - time_t now); -void rep_hist_note_exit_stream_opened(uint16_t port, time_t now); +void rep_hist_note_exit_bytes_read(uint16_t port, size_t num_bytes); +void rep_hist_note_exit_bytes_written(uint16_t port, size_t num_bytes); +void rep_hist_note_exit_stream_opened(uint16_t port); +void rep_hist_exit_stats_init(time_t now); +void rep_hist_exit_stats_write(time_t now); int rep_hist_bandwidth_assess(void); char *rep_hist_get_bandwidth_lines(int for_extrainfo); void rep_hist_update_state(or_state_t *state); @@ -4132,9 +4137,10 @@ void hs_usage_note_fetch_successful(const char *service_id, time_t now); void hs_usage_write_statistics_to_file(time_t now); void hs_usage_free_all(void); -#define DUMP_BUFFER_STATS_INTERVAL (24*60*60) -void add_circ_to_buffer_stats(circuit_t *circ, time_t end_of_interval); -void dump_buffer_stats(void); +void rep_hist_buffer_stats_init(time_t now); +void rep_hist_buffer_stats_add_circ(circuit_t *circ, + time_t end_of_interval); +void rep_hist_buffer_stats_write(time_t now); /********************************* rendclient.c ***************************/ diff --git a/src/or/rephist.c b/src/or/rephist.c index 3e4ba672d0..44bf940013 100644 --- a/src/or/rephist.c +++ b/src/or/rephist.c @@ -1321,8 +1321,6 @@ rep_hist_note_bytes_read(size_t num_bytes, time_t when) } /* Some constants */ -/** How long are the intervals for measuring exit stats? */ -#define EXIT_STATS_INTERVAL_SEC (24 * 60 * 60) /** To what multiple should byte numbers be rounded up? */ #define EXIT_STATS_ROUND_UP_BYTES 1024 /** To what multiple should stream counts be rounded up? */ @@ -1344,10 +1342,14 @@ static uint64_t *exit_bytes_written = NULL; /** Number of streams opened in current period by exit port */ static uint32_t *exit_streams = NULL; -/** Set up arrays for exit port statistics. */ -static void -exit_stats_init(void) +/** When does the current exit stats period end? */ +static time_t start_of_exit_stats_interval; + +/** Initialize exit port stats. */ +void +rep_hist_exit_stats_init(time_t now) { + start_of_exit_stats_interval = now; exit_bytes_read = tor_malloc_zero(EXIT_STATS_NUM_PORTS * sizeof(uint64_t)); exit_bytes_written = tor_malloc_zero(EXIT_STATS_NUM_PORTS * @@ -1356,12 +1358,9 @@ exit_stats_init(void) sizeof(uint32_t)); } -/** When does the current exit stats period end? */ -static time_t end_of_current_exit_stats_period = 0; - /** Write exit stats for the current period to disk and reset counters. */ -static void -write_exit_stats(time_t when) +void +rep_hist_exit_stats_write(time_t now) { char t[ISO_TIME_LEN+1]; int r, i, comma; @@ -1372,98 +1371,93 @@ write_exit_stats(time_t when) open_file_t *open_file = NULL; FILE *out = NULL; - log_debug(LD_HIST, "Considering writing exit port statistics to disk.."); - if (!exit_bytes_read) - exit_stats_init(); - while (when > end_of_current_exit_stats_period) { - format_iso_time(t, end_of_current_exit_stats_period); - log_info(LD_HIST, "Writing exit port statistics to disk for period " - "ending at %s.", t); + format_iso_time(t, now); + log_info(LD_HIST, "Writing exit port statistics to disk for period " + "ending at %s.", t); - if (!open_file) { - out = start_writing_to_stdio_file(filename, OPEN_FLAGS_APPEND, - 0600, &open_file); - if (!out) { - log_warn(LD_HIST, "Couldn't open '%s'.", filename); - goto done; - } + if (!open_file) { + out = start_writing_to_stdio_file(filename, OPEN_FLAGS_APPEND, + 0600, &open_file); + if (!out) { + log_warn(LD_HIST, "Couldn't open '%s'.", filename); + goto done; } + } - /* written yyyy-mm-dd HH:MM:SS (n s) */ - if (fprintf(out, "exit-stats-end %s (%d s)\n", t, - EXIT_STATS_INTERVAL_SEC) < 0) + /* written yyyy-mm-dd HH:MM:SS (n s) */ + if (fprintf(out, "exit-stats-end %s (%d s)\n", t, + (unsigned) (now - start_of_exit_stats_interval)) < 0) + goto done; + + /* Count the total number of bytes, so that we can attribute all + * observations below a threshold of 1 / EXIT_STATS_THRESHOLD_RECIPROCAL + * of all bytes to a special port 'other'. */ + total_bytes = 0; + for (i = 1; i < EXIT_STATS_NUM_PORTS; i++) { + total_bytes += exit_bytes_read[i]; + total_bytes += exit_bytes_written[i]; + } + threshold_bytes = total_bytes / EXIT_STATS_THRESHOLD_RECIPROCAL; + + /* exit-kibibytes-(read|written) port=kibibytes,.. */ + for (r = 0; r < 2; r++) { + b = r ? exit_bytes_read : exit_bytes_written; + tor_assert(b); + if (fprintf(out, "%s ", + r ? "exit-kibibytes-read" + : "exit-kibibytes-written") < 0) goto done; - /* Count the total number of bytes, so that we can attribute all - * observations below a threshold of 1 / EXIT_STATS_THRESHOLD_RECIPROCAL - * of all bytes to a special port 'other'. */ - total_bytes = 0; - for (i = 1; i < EXIT_STATS_NUM_PORTS; i++) { - total_bytes += exit_bytes_read[i]; - total_bytes += exit_bytes_written[i]; - } - threshold_bytes = total_bytes / EXIT_STATS_THRESHOLD_RECIPROCAL; - - /* kibibytes-(read|written) port=kibibytes,.. */ - for (r = 0; r < 2; r++) { - b = r ? exit_bytes_read : exit_bytes_written; - tor_assert(b); - if (fprintf(out, "%s ", - r ? "exit-kibibytes-read" - : "exit-kibibytes-written") < 0) - goto done; - - comma = 0; - other_bytes = 0; - for (i = 1; i < EXIT_STATS_NUM_PORTS; i++) { - if (b[i] > 0) { - if (exit_bytes_read[i] + exit_bytes_written[i] > threshold_bytes) { - uint64_t num = round_uint64_to_next_multiple_of(b[i], - EXIT_STATS_ROUND_UP_BYTES); - num /= 1024; - if (fprintf(out, "%s%d="U64_FORMAT, - comma++ ? "," : "", i, - U64_PRINTF_ARG(num)) < 0) - goto done; - } else - other_bytes += b[i]; - } - } - other_bytes = round_uint64_to_next_multiple_of(other_bytes, - EXIT_STATS_ROUND_UP_BYTES); - other_bytes /= 1024; - if (fprintf(out, "%sother="U64_FORMAT"\n", - comma ? "," : "", U64_PRINTF_ARG(other_bytes))<0) - goto done; - } - /* streams-opened port=num,.. */ - if (fprintf(out, "exit-streams-opened ") < 0) - goto done; comma = 0; - other_streams = 0; + other_bytes = 0; for (i = 1; i < EXIT_STATS_NUM_PORTS; i++) { - if (exit_streams[i] > 0) { + if (b[i] > 0) { if (exit_bytes_read[i] + exit_bytes_written[i] > threshold_bytes) { - uint32_t num = round_uint32_to_next_multiple_of(exit_streams[i], - EXIT_STATS_ROUND_UP_STREAMS); - if (fprintf(out, "%s%d=%u", - comma++ ? "," : "", i, num)<0) + uint64_t num = round_uint64_to_next_multiple_of(b[i], + EXIT_STATS_ROUND_UP_BYTES); + num /= 1024; + if (fprintf(out, "%s%d="U64_FORMAT, + comma++ ? "," : "", i, + U64_PRINTF_ARG(num)) < 0) goto done; } else - other_streams += exit_streams[i]; + other_bytes += b[i]; } } - other_streams = round_uint32_to_next_multiple_of(other_streams, - EXIT_STATS_ROUND_UP_STREAMS); - if (fprintf(out, "%sother=%u\n", - comma ? "," : "", other_streams)<0) + other_bytes = round_uint64_to_next_multiple_of(other_bytes, + EXIT_STATS_ROUND_UP_BYTES); + other_bytes /= 1024; + if (fprintf(out, "%sother="U64_FORMAT"\n", + comma ? "," : "", U64_PRINTF_ARG(other_bytes))<0) goto done; - /* Reset counters */ - memset(exit_bytes_read, 0, sizeof(exit_bytes_read)); - memset(exit_bytes_written, 0, sizeof(exit_bytes_written)); - memset(exit_streams, 0, sizeof(exit_streams)); - end_of_current_exit_stats_period += EXIT_STATS_INTERVAL_SEC; } + /* exit-streams-opened port=num,.. */ + if (fprintf(out, "exit-streams-opened ") < 0) + goto done; + comma = 0; + other_streams = 0; + for (i = 1; i < EXIT_STATS_NUM_PORTS; i++) { + if (exit_streams[i] > 0) { + if (exit_bytes_read[i] + exit_bytes_written[i] > threshold_bytes) { + uint32_t num = round_uint32_to_next_multiple_of(exit_streams[i], + EXIT_STATS_ROUND_UP_STREAMS); + if (fprintf(out, "%s%d=%u", + comma++ ? "," : "", i, num)<0) + goto done; + } else + other_streams += exit_streams[i]; + } + } + other_streams = round_uint32_to_next_multiple_of(other_streams, + EXIT_STATS_ROUND_UP_STREAMS); + if (fprintf(out, "%sother=%u\n", + comma ? "," : "", other_streams)<0) + goto done; + /* Reset counters */ + memset(exit_bytes_read, 0, sizeof(exit_bytes_read)); + memset(exit_bytes_written, 0, sizeof(exit_bytes_written)); + memset(exit_streams, 0, sizeof(exit_streams)); + start_of_exit_stats_interval = now; if (open_file) finish_writing_to_file(open_file); @@ -1474,59 +1468,36 @@ write_exit_stats(time_t when) tor_free(filename); } -/** Prepare to add an exit stats observation at second when by - * checking whether this observation lies in the current observation - * period; if not, shift the current period forward by one until the - * reported event fits it and write all results in between to disk. */ -static void -add_exit_obs(time_t when) -{ - if (!exit_bytes_read) - exit_stats_init(); - if (when > end_of_current_exit_stats_period) { - if (end_of_current_exit_stats_period) - write_exit_stats(when); - else - end_of_current_exit_stats_period = when + EXIT_STATS_INTERVAL_SEC; - } -} - /** Note that we wrote num_bytes to an exit connection to - * port in second when. */ + * port. */ void -rep_hist_note_exit_bytes_written(uint16_t port, size_t num_bytes, - time_t when) +rep_hist_note_exit_bytes_written(uint16_t port, size_t num_bytes) { if (!get_options()->ExitPortStatistics) return; - add_exit_obs(when); exit_bytes_written[port] += num_bytes; log_debug(LD_HIST, "Written %lu bytes to exit connection to port %d.", (unsigned long)num_bytes, port); } /** Note that we read num_bytes from an exit connection to - * port in second when. */ + * port. */ void -rep_hist_note_exit_bytes_read(uint16_t port, size_t num_bytes, - time_t when) +rep_hist_note_exit_bytes_read(uint16_t port, size_t num_bytes) { if (!get_options()->ExitPortStatistics) return; - add_exit_obs(when); exit_bytes_read[port] += num_bytes; log_debug(LD_HIST, "Read %lu bytes from exit connection to port %d.", (unsigned long)num_bytes, port); } -/** Note that we opened an exit stream to port in second - * when. */ +/** Note that we opened an exit stream to port. */ void -rep_hist_note_exit_stream_opened(uint16_t port, time_t when) +rep_hist_note_exit_stream_opened(uint16_t port) { if (!get_options()->ExitPortStatistics) return; - add_exit_obs(when); exit_streams[port]++; log_debug(LD_HIST, "Opened exit stream to port %d", port); } @@ -2623,7 +2594,14 @@ hs_usage_write_statistics_to_file(time_t now) /*** cell statistics ***/ /** Start of the current buffer stats interval. */ -time_t start_of_buffer_stats_interval; +static time_t start_of_buffer_stats_interval; + +/** Initialize buffer stats. */ +void +rep_hist_buffer_stats_init(time_t now) +{ + start_of_buffer_stats_interval = now; +} typedef struct circ_buffer_stats_t { uint32_t processed_cells; @@ -2639,7 +2617,7 @@ smartlist_t *circuits_for_buffer_stats = NULL; * end_of_interval and reset cell counters in case the circuit * remains open in the next measurement interval. */ void -add_circ_to_buffer_stats(circuit_t *circ, time_t end_of_interval) +rep_hist_buffer_stats_add_circ(circuit_t *circ, time_t end_of_interval) { circ_buffer_stats_t *stat; time_t start_of_interval; @@ -2687,9 +2665,8 @@ _buffer_stats_compare_entries(const void **_a, const void **_b) /** Append buffer statistics to local file. */ void -dump_buffer_stats(void) +rep_hist_buffer_stats_write(time_t now) { - time_t now = time(NULL); char *filename; char written[ISO_TIME_LEN+1]; open_file_t *open_file = NULL; @@ -2704,7 +2681,7 @@ dump_buffer_stats(void) circuit_t *circ; /* add current circuits to stats */ for (circ = _circuit_get_global_list(); circ; circ = circ->next) - add_circ_to_buffer_stats(circ, now); + rep_hist_buffer_stats_add_circ(circ, now); /* calculate deciles */ memset(processed_cells, 0, SHARES * sizeof(int)); memset(circs_in_share, 0, SHARES * sizeof(int)); @@ -2736,7 +2713,7 @@ dump_buffer_stats(void) goto done; format_iso_time(written, now); if (fprintf(out, "cell-stats-end %s (%d s)\n", written, - DUMP_BUFFER_STATS_INTERVAL) < 0) + (unsigned) (now - start_of_buffer_stats_interval)) < 0) goto done; for (i = 0; i < SHARES; i++) { tor_snprintf(buf, sizeof(buf), "%d", !circs_in_share[i] ? 0 : From a5508583eef66a023d09f3715549f756ea08c268 Mon Sep 17 00:00:00 2001 From: Karsten Loesing Date: Wed, 19 Aug 2009 16:30:00 +0200 Subject: [PATCH 159/187] Include only the last 24 hours of stats in extra-info documents. --- src/or/router.c | 43 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 34 insertions(+), 9 deletions(-) diff --git a/src/or/router.c b/src/or/router.c index 3249f49ce0..f33add8269 100644 --- a/src/or/router.c +++ b/src/or/router.c @@ -1826,18 +1826,38 @@ router_dump_router_to_string(char *s, size_t maxlen, routerinfo_t *router, return (int)written+1; } -/** Load the contents of filename and write its contents to - * **contents. Return 1 for success, 0 if the file does not exit, - * or -1 for failure. */ +/** Load the contents of filename, find the first line starting + * with end_line that has a timestamp after after, and write + * the file contents starting with that line to **contents. Return + * 1 for success, 0 if the file does not exist or does not contain a line + * matching these criteria, or -1 for failure. */ static int -load_stats_file(const char *filename, char **contents) +load_stats_file(const char *filename, const char *end_line, time_t after, + char **out) { int r = -1; char *fname = get_datadir_fname(filename); + char *contents, *start, timestr[ISO_TIME_LEN+1]; + time_t written; switch (file_status(fname)) { case FN_FILE: - if ((*contents = read_file_to_str(fname, 0, NULL))) + if ((contents = read_file_to_str(fname, 0, NULL))) { + do { + if ((start = strstr(contents, end_line))) + goto err; + if (strlen(start) < strlen(end_line) + sizeof(timestr)) + goto err; + if (strlcpy(timestr, start + strlen(end_line), sizeof(timestr))) + goto err; + if (parse_iso_time(timestr, &written) < 0) + goto err; + } while (written < after); + *out = tor_malloc(strlen(start)); + strlcpy(*out, start, strlen(start)); r = 1; + } + err: + tor_free(contents); break; case FN_NOENT: r = 0; @@ -1880,9 +1900,11 @@ extrainfo_dump_to_string(char *s, size_t maxlen, extrainfo_t *extrainfo, if (options->ExtraInfoStatistics && write_stats_to_extrainfo) { char *contents = NULL; + time_t since = time(NULL) - (24*60*60); log_info(LD_GENERAL, "Adding stats to extra-info descriptor."); if (options->DirReqStatistics && - load_stats_file("dirreq-stats", &contents) > 0) { + load_stats_file("dirreq-stats", "dirreq-stats-end", since, + &contents) > 0) { int pos = strlen(s); if (strlcpy(s + pos, contents, maxlen - strlen(s)) != strlen(contents)) { @@ -1893,7 +1915,8 @@ extrainfo_dump_to_string(char *s, size_t maxlen, extrainfo_t *extrainfo, tor_free(contents); } if (options->EntryStatistics && - load_stats_file("entry-stats", &contents) > 0) { + load_stats_file("entry-stats", "entry-stats-end", since, + &contents) > 0) { int pos = strlen(s); if (strlcpy(s + pos, contents, maxlen - strlen(s)) != strlen(contents)) { @@ -1904,7 +1927,8 @@ extrainfo_dump_to_string(char *s, size_t maxlen, extrainfo_t *extrainfo, tor_free(contents); } if (options->CellStatistics && - load_stats_file("buffer-stats", &contents) > 0) { + load_stats_file("buffer-stats", "cell-stats-end", since, + &contents) > 0) { int pos = strlen(s); if (strlcpy(s + pos, contents, maxlen - strlen(s)) != strlen(contents)) { @@ -1915,7 +1939,8 @@ extrainfo_dump_to_string(char *s, size_t maxlen, extrainfo_t *extrainfo, tor_free(contents); } if (options->ExitPortStatistics && - load_stats_file("exit-stats", &contents) > 0) { + load_stats_file("exit-stats", "exit-stats-end", since, + &contents) > 0) { int pos = strlen(s); if (strlcpy(s + pos, contents, maxlen - strlen(s)) != strlen(contents)) { From 42229d1d84095c82331b0bae1002f32f168ee5c9 Mon Sep 17 00:00:00 2001 From: Karsten Loesing Date: Wed, 19 Aug 2009 17:10:40 +0200 Subject: [PATCH 160/187] Move all *-stats file to subdirectory stats/. --- src/or/geoip.c | 19 ++++++++++++++----- src/or/rephist.c | 19 ++++++++++++++----- src/or/router.c | 16 ++++++++-------- 3 files changed, 36 insertions(+), 18 deletions(-) diff --git a/src/or/geoip.c b/src/or/geoip.c index 9f37c021c1..786c40e5b1 100644 --- a/src/or/geoip.c +++ b/src/or/geoip.c @@ -917,11 +917,11 @@ geoip_dirreq_stats_init(time_t now) start_of_dirreq_stats_interval = now; } -/** Store all our geoip statistics into $DATADIR/dirreq-stats. */ +/** Write dirreq statistics to $DATADIR/stats/dirreq-stats. */ void geoip_dirreq_stats_write(time_t now) { - char *filename = get_datadir_fname("dirreq-stats"); + char *statsdir = NULL, *filename = NULL; char *data_v2 = NULL, *data_v3 = NULL; char written[ISO_TIME_LEN+1]; open_file_t *open_file = NULL; @@ -935,6 +935,10 @@ geoip_dirreq_stats_write(time_t now) /* Discard all items in the client history that are too old. */ geoip_remove_old_clients(start_of_dirreq_stats_interval); + statsdir = get_datadir_fname("stats"); + if (check_private_dir(statsdir, CPD_CREATE) < 0) + goto done; + filename = get_datadir_fname("stats"PATH_SEPARATOR"dirreq-stats"); data_v2 = geoip_get_client_history_dirreq(now, GEOIP_CLIENT_NETWORKSTATUS_V2); data_v3 = geoip_get_client_history_dirreq(now, @@ -1029,6 +1033,7 @@ geoip_dirreq_stats_write(time_t now) if (open_file) abort_writing_to_file(open_file); tor_free(filename); + tor_free(statsdir); tor_free(data_v2); tor_free(data_v3); } @@ -1043,12 +1048,11 @@ geoip_entry_stats_init(time_t now) start_of_entry_stats_interval = now; } -/** Store all our geoip statistics as entry guards into - * $DATADIR/entry-stats. */ +/** Write entry statistics to $DATADIR/stats/entry-stats. */ void geoip_entry_stats_write(time_t now) { - char *filename = get_datadir_fname("entry-stats"); + char *statsdir = NULL, *filename = NULL; char *data = NULL; char written[ISO_TIME_LEN+1]; open_file_t *open_file = NULL; @@ -1060,6 +1064,10 @@ geoip_entry_stats_write(time_t now) /* Discard all items in the client history that are too old. */ geoip_remove_old_clients(start_of_entry_stats_interval); + statsdir = get_datadir_fname("stats"); + if (check_private_dir(statsdir, CPD_CREATE) < 0) + goto done; + filename = get_datadir_fname("stats"PATH_SEPARATOR"entry-stats"); data = geoip_get_client_history_dirreq(now, GEOIP_CLIENT_CONNECT); format_iso_time(written, now); out = start_writing_to_stdio_file(filename, OPEN_FLAGS_APPEND, @@ -1079,6 +1087,7 @@ geoip_entry_stats_write(time_t now) if (open_file) abort_writing_to_file(open_file); tor_free(filename); + tor_free(statsdir); tor_free(data); } diff --git a/src/or/rephist.c b/src/or/rephist.c index 44bf940013..f7d0b2bbcf 100644 --- a/src/or/rephist.c +++ b/src/or/rephist.c @@ -1358,7 +1358,7 @@ rep_hist_exit_stats_init(time_t now) sizeof(uint32_t)); } -/** Write exit stats for the current period to disk and reset counters. */ +/** Write exit stats to $DATADIR/stats/exit-stats and reset counters. */ void rep_hist_exit_stats_write(time_t now) { @@ -1367,10 +1367,14 @@ rep_hist_exit_stats_write(time_t now) uint64_t *b, total_bytes, threshold_bytes, other_bytes; uint32_t other_streams; - char *filename = get_datadir_fname("exit-stats"); + char *statsdir = NULL, *filename = NULL; open_file_t *open_file = NULL; FILE *out = NULL; + statsdir = get_datadir_fname("stats"); + if (check_private_dir(statsdir, CPD_CREATE) < 0) + goto done; + filename = get_datadir_fname("stats"PATH_SEPARATOR"exit-stats"); format_iso_time(t, now); log_info(LD_HIST, "Writing exit port statistics to disk for period " "ending at %s.", t); @@ -1466,6 +1470,7 @@ rep_hist_exit_stats_write(time_t now) if (open_file) abort_writing_to_file(open_file); tor_free(filename); + tor_free(statsdir); } /** Note that we wrote num_bytes to an exit connection to @@ -2663,11 +2668,11 @@ _buffer_stats_compare_entries(const void **_a, const void **_b) return 0; } -/** Append buffer statistics to local file. */ +/** Write buffer statistics to $DATADIR/stats/buffer-stats. */ void rep_hist_buffer_stats_write(time_t now) { - char *filename; + char *statsdir = NULL, *filename = NULL; char written[ISO_TIME_LEN+1]; open_file_t *open_file = NULL; FILE *out; @@ -2706,7 +2711,10 @@ rep_hist_buffer_stats_write(time_t now) stat, tor_free(stat)); smartlist_clear(circuits_for_buffer_stats); /* write to file */ - filename = get_datadir_fname("buffer-stats"); + statsdir = get_datadir_fname("stats"); + if (check_private_dir(statsdir, CPD_CREATE) < 0) + goto done; + filename = get_datadir_fname("stats"PATH_SEPARATOR"buffer-stats"); out = start_writing_to_stdio_file(filename, OPEN_FLAGS_APPEND, 0600, &open_file); if (!out) @@ -2758,6 +2766,7 @@ rep_hist_buffer_stats_write(time_t now) if (open_file) abort_writing_to_file(open_file); tor_free(filename); + tor_free(statsdir); if (str_build) { SMARTLIST_FOREACH(str_build, char *, c, tor_free(c)); smartlist_free(str_build); diff --git a/src/or/router.c b/src/or/router.c index f33add8269..1c8bb1d061 100644 --- a/src/or/router.c +++ b/src/or/router.c @@ -1903,8 +1903,8 @@ extrainfo_dump_to_string(char *s, size_t maxlen, extrainfo_t *extrainfo, time_t since = time(NULL) - (24*60*60); log_info(LD_GENERAL, "Adding stats to extra-info descriptor."); if (options->DirReqStatistics && - load_stats_file("dirreq-stats", "dirreq-stats-end", since, - &contents) > 0) { + load_stats_file("stats"PATH_SEPARATOR"dirreq-stats", + "dirreq-stats-end", since, &contents) > 0) { int pos = strlen(s); if (strlcpy(s + pos, contents, maxlen - strlen(s)) != strlen(contents)) { @@ -1915,8 +1915,8 @@ extrainfo_dump_to_string(char *s, size_t maxlen, extrainfo_t *extrainfo, tor_free(contents); } if (options->EntryStatistics && - load_stats_file("entry-stats", "entry-stats-end", since, - &contents) > 0) { + load_stats_file("stats"PATH_SEPARATOR"entry-stats", + "entry-stats-end", since, &contents) > 0) { int pos = strlen(s); if (strlcpy(s + pos, contents, maxlen - strlen(s)) != strlen(contents)) { @@ -1927,8 +1927,8 @@ extrainfo_dump_to_string(char *s, size_t maxlen, extrainfo_t *extrainfo, tor_free(contents); } if (options->CellStatistics && - load_stats_file("buffer-stats", "cell-stats-end", since, - &contents) > 0) { + load_stats_file("stats"PATH_SEPARATOR"buffer-stats", + "cell-stats-end", since, &contents) > 0) { int pos = strlen(s); if (strlcpy(s + pos, contents, maxlen - strlen(s)) != strlen(contents)) { @@ -1939,8 +1939,8 @@ extrainfo_dump_to_string(char *s, size_t maxlen, extrainfo_t *extrainfo, tor_free(contents); } if (options->ExitPortStatistics && - load_stats_file("exit-stats", "exit-stats-end", since, - &contents) > 0) { + load_stats_file("stats"PATH_SEPARATOR"exit-stats", + "exit-stats-end", since, &contents) > 0) { int pos = strlen(s); if (strlcpy(s + pos, contents, maxlen - strlen(s)) != strlen(contents)) { From 52b17b6ad7c0261186b2f892ee5d17b6298aa36b Mon Sep 17 00:00:00 2001 From: Karsten Loesing Date: Wed, 19 Aug 2009 17:14:35 +0200 Subject: [PATCH 161/187] Update ChangeLog with changes from proposal 166. --- ChangeLog | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/ChangeLog b/ChangeLog index ac7beb79ea..6901cf24c4 100644 --- a/ChangeLog +++ b/ChangeLog @@ -4,23 +4,23 @@ Changes in version 0.2.2.1-alpha - 2009-0?-?? via new AccelName and AccelDir options. o New options for gathering stats safely: - - Directories that configure with --enable-dirreq-stats and set - "DirReqStatistics 1" write directory request stats to disk every - 24 hours. As compared to the --enable-geoip-stats flag in 0.2.1.x, - there are a few improvements: 1) stats are written to disk exactly - every 24 hours; 2) estimated shares of v2 and v3 requests are - determined as mean values, not at the end of a measurement period; - 3) unresolved requests are listed with country code '??'; - 4) directories also measure download times. - - Exit nodes that configure with --enable-exit-stats and set - "ExitPortStatistics 1" write statistics on the number of exit - streams and transferred bytes per port to disk every 24 hours. - - Relays that configure with --enable-buffer-stats and set - "CellStatistics 1" write statistics to disk every 24 hours on how - long cells spend in their circuit queues. - - Entry nodes that configure with --enable-entry-stats and set - "EntryStatistics 1" write statistics to disk every 24 hours on - the rough number and origins of connecting clients. + - Directories that set "DirReqStatistics 1" write directory request + stats to disk every 24 hours. As compared to the --enable-geoip-stats + flag in 0.2.1.x, there are a few improvements: 1) stats are written + to disk exactly every 24 hours; 2) estimated shares of v2 and v3 + requests are determined as mean values, not at the end of a + measurement period; 3) unresolved requests are listed with country + code '??'; 4) directories also measure download times. + - Exit nodes that set "ExitPortStatistics 1" write statistics on the + number of exit streams and transferred bytes per port to disk every + 24 hours. + - Relays that set "CellStatistics 1" write statistics to disk every + 24 hours on how long cells spend in their circuit queues. + - Entry nodes that set "EntryStatistics 1" write statistics to disk + every 24 hours on the rough number and origins of connecting clients. + - Relays that write any of the above statistics to disk and set + "ExtraInfoStatistics 1" include the past 24 hours of statistics in + their extra-info documents. o Minor features: - New --digests command-line switch to output the digests of the From 6592cbe3e1d2d3e461c1af44097fd67dd653e017 Mon Sep 17 00:00:00 2001 From: Karsten Loesing Date: Wed, 19 Aug 2009 17:15:51 +0200 Subject: [PATCH 162/187] Don't write extra-info document to debug logs. --- src/or/router.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/or/router.c b/src/or/router.c index 1c8bb1d061..d744b9bc06 100644 --- a/src/or/router.c +++ b/src/or/router.c @@ -1990,9 +1990,6 @@ extrainfo_dump_to_string(char *s, size_t maxlen, extrainfo_t *extrainfo, "We just generated an extrainfo descriptor we can't parse."); log_err(LD_BUG, "Descriptor was: <<%s>>", s); return -1; - } else { - log_debug(LD_GENERAL, "We generated an extra-info descriptor:\n%s", - s); } tor_free(s_dup); extrainfo_free(ei_tmp); From 93fd0d37559dcd4f75bba158182f486d7323b8ef Mon Sep 17 00:00:00 2001 From: Karsten Loesing Date: Wed, 19 Aug 2009 19:16:00 +0200 Subject: [PATCH 163/187] Fix a couple of bugs that showed up at runtime. --- src/or/router.c | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/src/or/router.c b/src/or/router.c index d744b9bc06..bb63dc205e 100644 --- a/src/or/router.c +++ b/src/or/router.c @@ -1842,13 +1842,15 @@ load_stats_file(const char *filename, const char *end_line, time_t after, switch (file_status(fname)) { case FN_FILE: if ((contents = read_file_to_str(fname, 0, NULL))) { + start = contents; do { - if ((start = strstr(contents, end_line))) + if (start != contents) + start++; /* Avoid infinite loops. */ + if (!(start = strstr(start, end_line))) goto err; if (strlen(start) < strlen(end_line) + sizeof(timestr)) goto err; - if (strlcpy(timestr, start + strlen(end_line), sizeof(timestr))) - goto err; + strlcpy(timestr, start + 1 + strlen(end_line), sizeof(timestr)); if (parse_iso_time(timestr, &written) < 0) goto err; } while (written < after); @@ -1906,8 +1908,7 @@ extrainfo_dump_to_string(char *s, size_t maxlen, extrainfo_t *extrainfo, load_stats_file("stats"PATH_SEPARATOR"dirreq-stats", "dirreq-stats-end", since, &contents) > 0) { int pos = strlen(s); - if (strlcpy(s + pos, contents, maxlen - strlen(s)) != - strlen(contents)) { + if (tor_snprintf(s + pos, maxlen - strlen(s), "%s\n", contents) < 0) { log_warn(LD_DIR, "Could not write dirreq-stats to extra-info " "descriptor."); s[pos] = '\0'; @@ -1918,8 +1919,7 @@ extrainfo_dump_to_string(char *s, size_t maxlen, extrainfo_t *extrainfo, load_stats_file("stats"PATH_SEPARATOR"entry-stats", "entry-stats-end", since, &contents) > 0) { int pos = strlen(s); - if (strlcpy(s + pos, contents, maxlen - strlen(s)) != - strlen(contents)) { + if (tor_snprintf(s + pos, maxlen - strlen(s), "%s\n", contents) < 0) { log_warn(LD_DIR, "Could not write entry-stats to extra-info " "descriptor."); s[pos] = '\0'; @@ -1930,8 +1930,7 @@ extrainfo_dump_to_string(char *s, size_t maxlen, extrainfo_t *extrainfo, load_stats_file("stats"PATH_SEPARATOR"buffer-stats", "cell-stats-end", since, &contents) > 0) { int pos = strlen(s); - if (strlcpy(s + pos, contents, maxlen - strlen(s)) != - strlen(contents)) { + if (tor_snprintf(s + pos, maxlen - strlen(s), "%s\n", contents) < 0) { log_warn(LD_DIR, "Could not write buffer-stats to extra-info " "descriptor."); s[pos] = '\0'; @@ -1942,8 +1941,7 @@ extrainfo_dump_to_string(char *s, size_t maxlen, extrainfo_t *extrainfo, load_stats_file("stats"PATH_SEPARATOR"exit-stats", "exit-stats-end", since, &contents) > 0) { int pos = strlen(s); - if (strlcpy(s + pos, contents, maxlen - strlen(s)) != - strlen(contents)) { + if (tor_snprintf(s + pos, maxlen - strlen(s), "%s\n", contents) < 0) { log_warn(LD_DIR, "Could not write exit-stats to extra-info " "descriptor."); s[pos] = '\0'; @@ -2007,8 +2005,6 @@ extrainfo_dump_to_string(char *s, size_t maxlen, extrainfo_t *extrainfo, "this or any future extra-info descriptors. Descriptor " "was:\n%s", s); write_stats_to_extrainfo = 0; - tor_free(s); - s = NULL; extrainfo_dump_to_string(s, maxlen, extrainfo, ident_key); } tor_free(s_dup); From 75c59d1a92e0f4956bd249844c188c432aae1712 Mon Sep 17 00:00:00 2001 From: Karsten Loesing Date: Wed, 19 Aug 2009 23:36:27 +0200 Subject: [PATCH 164/187] Some final (?) cleanups of proposal 166 implementation. --- ChangeLog | 23 ++++++------- doc/spec/proposals/000-index.txt | 4 +-- .../166-statistics-extra-info-docs.txt | 2 +- src/or/geoip.c | 33 ++++++++++++++++++- src/or/main.c | 1 - src/or/router.c | 14 ++++---- 6 files changed, 53 insertions(+), 24 deletions(-) diff --git a/ChangeLog b/ChangeLog index 6901cf24c4..97f9a3b629 100644 --- a/ChangeLog +++ b/ChangeLog @@ -4,20 +4,21 @@ Changes in version 0.2.2.1-alpha - 2009-0?-?? via new AccelName and AccelDir options. o New options for gathering stats safely: - - Directories that set "DirReqStatistics 1" write directory request - stats to disk every 24 hours. As compared to the --enable-geoip-stats - flag in 0.2.1.x, there are a few improvements: 1) stats are written - to disk exactly every 24 hours; 2) estimated shares of v2 and v3 - requests are determined as mean values, not at the end of a - measurement period; 3) unresolved requests are listed with country - code '??'; 4) directories also measure download times. + - Directories that set "DirReqStatistics 1" write statistics on + directory request to disk every 24 hours. As compared to the + --enable-geoip-stats flag in 0.2.1.x, there are a few improvements: + 1) stats are written to disk exactly every 24 hours; 2) estimated + shares of v2 and v3 requests are determined as mean values, not at + the end of a measurement period; 3) unresolved requests are listed + with country code '??'; 4) directories also measure download times. - Exit nodes that set "ExitPortStatistics 1" write statistics on the number of exit streams and transferred bytes per port to disk every 24 hours. - - Relays that set "CellStatistics 1" write statistics to disk every - 24 hours on how long cells spend in their circuit queues. - - Entry nodes that set "EntryStatistics 1" write statistics to disk - every 24 hours on the rough number and origins of connecting clients. + - Relays that set "CellStatistics 1" write statistics on how long + cells spend in their circuit queues to disk every 24 hours. + - Entry nodes that set "EntryStatistics 1" write statistics on the + rough number and origins of connecting clients to disk every 24 + hours. - Relays that write any of the above statistics to disk and set "ExtraInfoStatistics 1" include the past 24 hours of statistics in their extra-info documents. diff --git a/doc/spec/proposals/000-index.txt b/doc/spec/proposals/000-index.txt index af1f40bf6d..d2d3ca5d72 100644 --- a/doc/spec/proposals/000-index.txt +++ b/doc/spec/proposals/000-index.txt @@ -86,7 +86,7 @@ Proposals by number: 163 Detecting whether a connection comes from a client [OPEN] 164 Reporting the status of server votes [OPEN] 165 Easy migration for voting authority sets [OPEN] -166 Including Network Statistics in Extra-Info Documents [OPEN] +166 Including Network Statistics in Extra-Info Documents [ACCEPTED] Proposals by status: @@ -114,7 +114,6 @@ Proposals by status: 163 Detecting whether a connection comes from a client [for 0.2.2] 164 Reporting the status of server votes [for 0.2.2] 165 Easy migration for voting authority sets - 166 Including Network Statistics in Extra-Info Documents [for 0.2.2] ACCEPTED: 110 Avoiding infinite length circuits [for 0.2.1.x] [in 0.2.1.3-alpha] 117 IPv6 exits [for 0.2.1.x] @@ -122,6 +121,7 @@ Proposals by status: 140 Provide diffs between consensuses [for 0.2.2.x] 147 Eliminate the need for v2 directories in generating v3 directories [for 0.2.1.x] 157 Make certificate downloads specific [for 0.2.1.x] + 166 Including Network Statistics in Extra-Info Documents [for 0.2.2] META: 000 Index of Tor Proposals 001 The Tor Proposal Process diff --git a/doc/spec/proposals/166-statistics-extra-info-docs.txt b/doc/spec/proposals/166-statistics-extra-info-docs.txt index 6bc71cafd8..ab2716a71c 100644 --- a/doc/spec/proposals/166-statistics-extra-info-docs.txt +++ b/doc/spec/proposals/166-statistics-extra-info-docs.txt @@ -3,7 +3,7 @@ Title: Including Network Statistics in Extra-Info Documents Author: Karsten Loesing Created: 21-Jul-2009 Target: 0.2.2 -Status: Open +Status: Accepted Change history: diff --git a/src/or/geoip.c b/src/or/geoip.c index 786c40e5b1..622b582173 100644 --- a/src/or/geoip.c +++ b/src/or/geoip.c @@ -388,6 +388,36 @@ geoip_note_client_seen(geoip_client_action_t action, return; } + /* As a bridge that doesn't rotate request periods every 24 hours, + * possibly rotate now. */ + if (options->BridgeRelay) { + while (current_request_period_starts + REQUEST_HIST_PERIOD < now) { + if (!geoip_countries) + geoip_countries = smartlist_create(); + if (!current_request_period_starts) { + current_request_period_starts = now; + break; + } + /* Also discard all items in the client history that are too old. + * (This only works here because bridge and directory stats are + * independent. Otherwise, we'd only want to discard those items + * with action GEOIP_CLIENT_NETWORKSTATUS{_V2}.) */ + geoip_remove_old_clients(current_request_period_starts); + /* Now rotate request period */ + SMARTLIST_FOREACH(geoip_countries, geoip_country_t *, c, { + memmove(&c->n_v2_ns_requests[0], &c->n_v2_ns_requests[1], + sizeof(uint32_t)*(REQUEST_HIST_LEN-1)); + memmove(&c->n_v3_ns_requests[0], &c->n_v3_ns_requests[1], + sizeof(uint32_t)*(REQUEST_HIST_LEN-1)); + c->n_v2_ns_requests[REQUEST_HIST_LEN-1] = 0; + c->n_v3_ns_requests[REQUEST_HIST_LEN-1] = 0; + }); + current_request_period_starts += REQUEST_HIST_PERIOD; + if (n_old_request_periods < REQUEST_HIST_LEN-1) + ++n_old_request_periods; + } + } + lookup.ipaddr = addr; lookup.action = (int)action; ent = HT_FIND(clientmap, &client_history, &lookup); @@ -949,7 +979,8 @@ geoip_dirreq_stats_write(time_t now) if (!out) goto done; if (fprintf(out, "dirreq-stats-end %s (%d s)\ndirreq-v3-ips %s\n" - "dirreq-v2-ips %s\n", written, REQUEST_HIST_PERIOD, + "dirreq-v2-ips %s\n", written, + (unsigned) (now - start_of_dirreq_stats_interval), data_v3 ? data_v3 : "", data_v2 ? data_v2 : "") < 0) goto done; tor_free(data_v2); diff --git a/src/or/main.c b/src/or/main.c index 16136cb27f..fb6b1ea0e7 100644 --- a/src/or/main.c +++ b/src/or/main.c @@ -962,7 +962,6 @@ run_scheduled_events(time_t now) */ if (time_to_write_stats_files >= 0 && time_to_write_stats_files < now) { #define WRITE_STATS_INTERVAL (24*60*60) - or_options_t *options = get_options(); if (options->CellStatistics || options->DirReqStatistics || options->EntryStatistics || options->ExitPortStatistics) { if (!time_to_write_stats_files) { diff --git a/src/or/router.c b/src/or/router.c index bb63dc205e..64267f242c 100644 --- a/src/or/router.c +++ b/src/or/router.c @@ -1847,18 +1847,18 @@ load_stats_file(const char *filename, const char *end_line, time_t after, if (start != contents) start++; /* Avoid infinite loops. */ if (!(start = strstr(start, end_line))) - goto err; - if (strlen(start) < strlen(end_line) + sizeof(timestr)) - goto err; + goto notfound; + if (strlen(start) < strlen(end_line) + 1 + sizeof(timestr)) + goto notfound; strlcpy(timestr, start + 1 + strlen(end_line), sizeof(timestr)); if (parse_iso_time(timestr, &written) < 0) - goto err; - } while (written < after); + goto notfound; + } while (written <= after); *out = tor_malloc(strlen(start)); strlcpy(*out, start, strlen(start)); r = 1; } - err: + notfound: tor_free(contents); break; case FN_NOENT: @@ -2011,8 +2011,6 @@ extrainfo_dump_to_string(char *s, size_t maxlen, extrainfo_t *extrainfo, extrainfo_free(ei_tmp); } - log_info(LD_GENERAL, "Done with dumping our extra-info descriptor."); - return (int)strlen(s)+1; } From f57883a39e09e16f495a6b8706ca0d3b3a8df2a4 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 19 Aug 2009 19:21:29 -0400 Subject: [PATCH 165/187] Add basic support for SHA256. This adds an openssl 0.9.8 dependency. Let's see if anybody cares. --- src/common/crypto.c | 71 +++++++++++++++++++++++++++++++++++++-------- src/common/crypto.h | 20 +++++++++++-- src/or/test.c | 32 ++++++++++++++++++-- 3 files changed, 106 insertions(+), 17 deletions(-) diff --git a/src/common/crypto.c b/src/common/crypto.c index 57c636db68..51d4059f3e 100644 --- a/src/common/crypto.c +++ b/src/common/crypto.c @@ -1394,9 +1394,23 @@ crypto_digest(char *digest, const char *m, size_t len) return (SHA1((const unsigned char*)m,len,(unsigned char*)digest) == NULL); } +int +crypto_digest256(char *digest, const char *m, size_t len, + digest_algorithm_t algorithm) +{ + tor_assert(m); + tor_assert(digest); + tor_assert(algorithm == DIGEST_SHA256); + return (SHA256((const unsigned char*)m,len,(unsigned char*)digest) == NULL); +} + /** Intermediate information about the digest of a stream of data. */ struct crypto_digest_env_t { - SHA_CTX d; + union { + SHA_CTX sha1; + SHA256_CTX sha2; + } d; + digest_algorithm_t algorithm : 8; }; /** Allocate and return a new digest object. @@ -1406,7 +1420,19 @@ crypto_new_digest_env(void) { crypto_digest_env_t *r; r = tor_malloc(sizeof(crypto_digest_env_t)); - SHA1_Init(&r->d); + SHA1_Init(&r->d.sha1); + r->algorithm = DIGEST_SHA1; + return r; +} + +crypto_digest_env_t * +crypto_new_digest256_env(digest_algorithm_t algorithm) +{ + crypto_digest_env_t *r; + tor_assert(algorithm == DIGEST_SHA256); + r = tor_malloc(sizeof(crypto_digest_env_t)); + SHA256_Init(&r->d.sha2); + r->algorithm = algorithm; return r; } @@ -1427,30 +1453,51 @@ crypto_digest_add_bytes(crypto_digest_env_t *digest, const char *data, { tor_assert(digest); tor_assert(data); - /* Using the SHA1_*() calls directly means we don't support doing - * SHA1 in hardware. But so far the delay of getting the question + /* Using the SHA*_*() calls directly means we don't support doing + * SHA in hardware. But so far the delay of getting the question * to the hardware, and hearing the answer, is likely higher than * just doing it ourselves. Hashes are fast. */ - SHA1_Update(&digest->d, (void*)data, len); + switch (digest->algorithm) { + case DIGEST_SHA1: + SHA1_Update(&digest->d.sha1, (void*)data, len); + break; + case DIGEST_SHA256: + SHA256_Update(&digest->d.sha2, (void*)data, len); + break; + default: + tor_fragile_assert(); + break; + } } /** Compute the hash of the data that has been passed to the digest * object; write the first out_len bytes of the result to out. - * out_len must be \<= DIGEST_LEN. + * out_len must be \<= DIGEST256_LEN. */ void crypto_digest_get_digest(crypto_digest_env_t *digest, char *out, size_t out_len) { - unsigned char r[DIGEST_LEN]; - SHA_CTX tmpctx; + unsigned char r[DIGEST256_LEN]; + crypto_digest_env_t tmpenv; tor_assert(digest); tor_assert(out); - tor_assert(out_len <= DIGEST_LEN); - /* memcpy into a temporary ctx, since SHA1_Final clears the context */ - memcpy(&tmpctx, &digest->d, sizeof(SHA_CTX)); - SHA1_Final(r, &tmpctx); + /* memcpy into a temporary ctx, since SHA*_Final clears the context */ + memcpy(&tmpenv, digest, sizeof(crypto_digest_env_t)); + switch (digest->algorithm) { + case DIGEST_SHA1: + tor_assert(out_len <= DIGEST_LEN); + SHA1_Final(r, &digest->d.sha1); + break; + case DIGEST_SHA256: + tor_assert(out_len <= DIGEST256_LEN); + SHA256_Final(r, &digest->d.sha2); + break; + default: + tor_fragile_assert(); + break; + } memcpy(out, r, out_len); memset(r, 0, sizeof(r)); } diff --git a/src/common/crypto.h b/src/common/crypto.h index fa6735d788..515b870f3d 100644 --- a/src/common/crypto.h +++ b/src/common/crypto.h @@ -18,6 +18,9 @@ /** Length of the output of our message digest. */ #define DIGEST_LEN 20 +/** Length of the output of our second (improved) message digests. (For now + * this is just sha256, but any it can be any other 256-byte digest). */ +#define DIGEST256_LEN 32 /** Length of our symmetric cipher's keys. */ #define CIPHER_KEY_LEN 16 /** Length of our symmetric cipher's IV. */ @@ -27,9 +30,12 @@ /** Length of our DH keys. */ #define DH_BYTES (1024/8) -/** Length of a message digest when encoded in base64 with trailing = signs - * removed. */ +/** Length of a sha1 message digest when encoded in base64 with trailing = + * signs removed. */ #define BASE64_DIGEST_LEN 27 +/** Length of a sha256 message digest when encoded in base64 with trailing = + * signs removed. */ +#define BASE64_DIGEST256_LEN 43 /** Constants used to indicate no padding for public-key encryption */ #define PK_NO_PADDING 60000 @@ -48,6 +54,13 @@ #define FINGERPRINT_LEN 49 /** Length of hex encoding of SHA1 digest, not including final NUL. */ #define HEX_DIGEST_LEN 40 +/** Length of hex encoding of SHA256 digest, not including final NUL. */ +#define HEX_DIGEST256_LEN 64 + +typedef enum { + DIGEST_SHA1, + DIGEST_SHA256, +} digest_algorithm_t; typedef struct crypto_pk_env_t crypto_pk_env_t; typedef struct crypto_cipher_env_t crypto_cipher_env_t; @@ -145,7 +158,10 @@ int crypto_cipher_decrypt_with_iv(crypto_cipher_env_t *env, /* SHA-1 */ int crypto_digest(char *digest, const char *m, size_t len); +int crypto_digest256(char *digest, const char *m, size_t len, + digest_algorithm_t algorithm); crypto_digest_env_t *crypto_new_digest_env(void); +crypto_digest_env_t *crypto_new_digest256_env(digest_algorithm_t algorithm); void crypto_free_digest_env(crypto_digest_env_t *digest); void crypto_digest_add_bytes(crypto_digest_env_t *digest, const char *data, size_t len); diff --git a/src/or/test.c b/src/or/test.c index b86ced723a..db4b6784f4 100644 --- a/src/or/test.c +++ b/src/or/test.c @@ -618,14 +618,19 @@ test_crypto_sha(void) crypto_digest_env_t *d1 = NULL, *d2 = NULL; int i; char key[80]; - char digest[20]; + char digest[32]; char data[50]; - char d_out1[DIGEST_LEN], d_out2[DIGEST_LEN]; + char d_out1[DIGEST_LEN], d_out2[DIGEST256_LEN]; /* Test SHA-1 with a test vector from the specification. */ i = crypto_digest(data, "abc", 3); test_memeq_hex(data, "A9993E364706816ABA3E25717850C26C9CD0D89D"); + /* Test SHA-256 with a test vector from the specification. */ + i = crypto_digest256(data, "abc", 3, DIGEST_SHA256); + test_memeq_hex(data, "BA7816BF8F01CFEA414140DE5DAE2223B00361A3" + "96177A9CB410FF61F20015AD"); + /* Test HMAC-SHA-1 with test cases from RFC2202. */ /* Case 1. */ @@ -646,7 +651,7 @@ test_crypto_sha(void) test_streq(hex_str(digest, 20), "4C9007F4026250C6BC8414F9BF50C86C2D7235DA"); - /* Case . */ + /* Case 5. */ memset(key, 0xaa, 80); crypto_hmac_sha1(digest, key, 80, "Test Using Larger Than Block-Size Key - Hash Key First", @@ -672,6 +677,27 @@ test_crypto_sha(void) crypto_digest_get_digest(d1, d_out1, sizeof(d_out1)); crypto_digest(d_out2, "abcdef", 6); test_memeq(d_out1, d_out2, DIGEST_LEN); + crypto_free_digest_env(d1); + crypto_free_digest_env(d2); + + /* Incremental digest code with sha256 */ + d1 = crypto_new_digest256_env(DIGEST_SHA256); + test_assert(d1); + crypto_digest_add_bytes(d1, "abcdef", 6); + d2 = crypto_digest_dup(d1); + test_assert(d2); + crypto_digest_add_bytes(d2, "ghijkl", 6); + crypto_digest_get_digest(d2, d_out1, sizeof(d_out1)); + crypto_digest256(d_out2, "abcdefghijkl", 12, DIGEST_SHA256); + test_memeq(d_out1, d_out2, DIGEST_LEN); + crypto_digest_assign(d2, d1); + crypto_digest_add_bytes(d2, "mno", 3); + crypto_digest_get_digest(d2, d_out1, sizeof(d_out1)); + crypto_digest256(d_out2, "abcdefmno", 9, DIGEST_SHA256); + test_memeq(d_out1, d_out2, DIGEST_LEN); + crypto_digest_get_digest(d1, d_out1, sizeof(d_out1)); + crypto_digest256(d_out2, "abcdef", 6, DIGEST_SHA256); + test_memeq(d_out1, d_out2, DIGEST_LEN); done: if (d1) From d0c212995a29644e0357c81abbc0e7ea652d2504 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 20 Aug 2009 01:47:13 -0400 Subject: [PATCH 166/187] Add a SHA256 implementation for platforms that lack it. (This would be everywhere running OpenSSL 0.9.7x and earlier, including all current Macintosh users.) The code is based on Tom St Denis's LibTomCrypt implementation, modified to be way less general and use Tor's existing facilities. I picked this one because it was pretty fast and pretty free, and because Python uses it too. --- src/common/Makefile.am | 3 +- src/common/crypto.c | 22 +++ src/common/sha256.c | 331 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 355 insertions(+), 1 deletion(-) create mode 100644 src/common/sha256.c diff --git a/src/common/Makefile.am b/src/common/Makefile.am index 78f73db25f..96013845e6 100644 --- a/src/common/Makefile.am +++ b/src/common/Makefile.am @@ -1,7 +1,7 @@ noinst_LIBRARIES = libor.a libor-crypto.a libor-event.a -EXTRA_DIST = common_sha1.i +EXTRA_DIST = common_sha1.i sha256.c #CFLAGS = -Wall -Wpointer-arith -O2 @@ -29,3 +29,4 @@ common_sha1.i: $(libor_SOURCES) $(libor_crypto_a_SOURCES) $(noinst_HEADERS) fi util_codedigest.o: common_sha1.i +crypto.o: sha256.c diff --git a/src/common/crypto.c b/src/common/crypto.c index 51d4059f3e..dc645b5eda 100644 --- a/src/common/crypto.c +++ b/src/common/crypto.c @@ -62,6 +62,28 @@ #include +#if OPENSSL_VERSION_NUMBER < 0x00908000l +/* On OpenSSL versions before 0.9.8, there is no working SHA256 + * implementation, so we use Tom St Denis's nice speedy one, slightly adapted + * to our needs */ +#define SHA256_CTX sha256_state +#define SHA256_Init sha256_init +#define SHA256_Update sha256_process +#define LTC_ARGCHK(x) tor_assert(x) +#include "sha256.c" +#define SHA256_Final(a,b) sha256_done(b,a) + +static unsigned char * +SHA256(const unsigned char *m, size_t len, unsigned char *d) +{ + SHA256_CTX ctx; + SHA256_Init(&ctx); + SHA256_Update(&ctx, m, len); + SHA256_Final(d, &ctx); + return d; +} +#endif + /** Macro: is k a valid RSA public or private key? */ #define PUBLIC_KEY_OK(k) ((k) && (k)->key && (k)->key->n) /** Macro: is k a valid RSA private key? */ diff --git a/src/common/sha256.c b/src/common/sha256.c new file mode 100644 index 0000000000..d445ae4e6d --- /dev/null +++ b/src/common/sha256.c @@ -0,0 +1,331 @@ +/* Copyright (c) 2009, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ +/* This SHA256 implementation is adapted from the public domain one in + LibTomCrypt, version 1.6. Tor uses it on platforms where OpenSSL doesn't + have a SHA256. */ + + +typedef struct sha256_state { + uint64_t length; + uint32_t state[8], curlen; + unsigned char buf[64]; +} sha256_state; + +#define CRYPT_OK 0 +#define CRYPT_NOP -1 +#define CRYPT_INVALID_ARG -2 + +#define LOAD32H(x,y) STMT_BEGIN x = ntohl(get_uint32((const char*)y)); STMT_END +#define STORE32H(x,y) STMT_BEGIN set_uint32((char*)y, htonl(x)); STMT_END +#define STORE64H(x,y) STMT_BEGIN \ + set_uint32((char*)y, htonl((uint32_t)((x)>>32))); \ + set_uint32(((char*)y)+4, htonl((uint32_t)((x)&0xffffffff))); \ + STMT_END +#define RORc(x, y) ( ((((unsigned long)(x)&0xFFFFFFFFUL)>>(unsigned long)((y)&31)) | ((unsigned long)(x)<<(unsigned long)(32-((y)&31)))) & 0xFFFFFFFFUL) +#ifndef MIN + #define MIN(x, y) ( ((x)<(y))?(x):(y) ) +#endif + + +/* LibTomCrypt, modular cryptographic library -- Tom St Denis + * + * LibTomCrypt is a library that provides various cryptographic + * algorithms in a highly modular and flexible manner. + * + * The library is free for all purposes without any express + * guarantee it works. + * + * Tom St Denis, tomstdenis@gmail.com, http://libtomcrypt.com + */ + +/** + @file sha256.c + SHA256 by Tom St Denis +*/ + + +#ifdef LTC_SMALL_CODE +/* the K array */ +static const uint32_t K[64] = { + 0x428a2f98UL, 0x71374491UL, 0xb5c0fbcfUL, 0xe9b5dba5UL, 0x3956c25bUL, + 0x59f111f1UL, 0x923f82a4UL, 0xab1c5ed5UL, 0xd807aa98UL, 0x12835b01UL, + 0x243185beUL, 0x550c7dc3UL, 0x72be5d74UL, 0x80deb1feUL, 0x9bdc06a7UL, + 0xc19bf174UL, 0xe49b69c1UL, 0xefbe4786UL, 0x0fc19dc6UL, 0x240ca1ccUL, + 0x2de92c6fUL, 0x4a7484aaUL, 0x5cb0a9dcUL, 0x76f988daUL, 0x983e5152UL, + 0xa831c66dUL, 0xb00327c8UL, 0xbf597fc7UL, 0xc6e00bf3UL, 0xd5a79147UL, + 0x06ca6351UL, 0x14292967UL, 0x27b70a85UL, 0x2e1b2138UL, 0x4d2c6dfcUL, + 0x53380d13UL, 0x650a7354UL, 0x766a0abbUL, 0x81c2c92eUL, 0x92722c85UL, + 0xa2bfe8a1UL, 0xa81a664bUL, 0xc24b8b70UL, 0xc76c51a3UL, 0xd192e819UL, + 0xd6990624UL, 0xf40e3585UL, 0x106aa070UL, 0x19a4c116UL, 0x1e376c08UL, + 0x2748774cUL, 0x34b0bcb5UL, 0x391c0cb3UL, 0x4ed8aa4aUL, 0x5b9cca4fUL, + 0x682e6ff3UL, 0x748f82eeUL, 0x78a5636fUL, 0x84c87814UL, 0x8cc70208UL, + 0x90befffaUL, 0xa4506cebUL, 0xbef9a3f7UL, 0xc67178f2UL +}; +#endif + +/* Various logical functions */ +#define Ch(x,y,z) (z ^ (x & (y ^ z))) +#define Maj(x,y,z) (((x | y) & z) | (x & y)) +#define S(x, n) RORc((x),(n)) +#define R(x, n) (((x)&0xFFFFFFFFUL)>>(n)) +#define Sigma0(x) (S(x, 2) ^ S(x, 13) ^ S(x, 22)) +#define Sigma1(x) (S(x, 6) ^ S(x, 11) ^ S(x, 25)) +#define Gamma0(x) (S(x, 7) ^ S(x, 18) ^ R(x, 3)) +#define Gamma1(x) (S(x, 17) ^ S(x, 19) ^ R(x, 10)) + +/* compress 512-bits */ +#ifdef LTC_CLEAN_STACK +static int _sha256_compress(sha256_state * md, unsigned char *buf) +#else +static int sha256_compress(sha256_state * md, unsigned char *buf) +#endif +{ + uint32_t S[8], W[64], t0, t1; +#ifdef LTC_SMALL_CODE + uint32_t t; +#endif + int i; + + /* copy state into S */ + for (i = 0; i < 8; i++) { + S[i] = md->state[i]; + } + + /* copy the state into 512-bits into W[0..15] */ + for (i = 0; i < 16; i++) { + LOAD32H(W[i], buf + (4*i)); + } + + /* fill W[16..63] */ + for (i = 16; i < 64; i++) { + W[i] = Gamma1(W[i - 2]) + W[i - 7] + Gamma0(W[i - 15]) + W[i - 16]; + } + + /* Compress */ +#ifdef LTC_SMALL_CODE +#define RND(a,b,c,d,e,f,g,h,i) \ + t0 = h + Sigma1(e) + Ch(e, f, g) + K[i] + W[i]; \ + t1 = Sigma0(a) + Maj(a, b, c); \ + d += t0; \ + h = t0 + t1; + + for (i = 0; i < 64; ++i) { + RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],i); + t = S[7]; S[7] = S[6]; S[6] = S[5]; S[5] = S[4]; + S[4] = S[3]; S[3] = S[2]; S[2] = S[1]; S[1] = S[0]; S[0] = t; + } +#else +#define RND(a,b,c,d,e,f,g,h,i,ki) \ + t0 = h + Sigma1(e) + Ch(e, f, g) + ki + W[i]; \ + t1 = Sigma0(a) + Maj(a, b, c); \ + d += t0; \ + h = t0 + t1; + + RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],0,0x428a2f98); + RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],1,0x71374491); + RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],2,0xb5c0fbcf); + RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],3,0xe9b5dba5); + RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],4,0x3956c25b); + RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],5,0x59f111f1); + RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],6,0x923f82a4); + RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],7,0xab1c5ed5); + RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],8,0xd807aa98); + RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],9,0x12835b01); + RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],10,0x243185be); + RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],11,0x550c7dc3); + RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],12,0x72be5d74); + RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],13,0x80deb1fe); + RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],14,0x9bdc06a7); + RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],15,0xc19bf174); + RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],16,0xe49b69c1); + RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],17,0xefbe4786); + RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],18,0x0fc19dc6); + RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],19,0x240ca1cc); + RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],20,0x2de92c6f); + RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],21,0x4a7484aa); + RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],22,0x5cb0a9dc); + RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],23,0x76f988da); + RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],24,0x983e5152); + RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],25,0xa831c66d); + RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],26,0xb00327c8); + RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],27,0xbf597fc7); + RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],28,0xc6e00bf3); + RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],29,0xd5a79147); + RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],30,0x06ca6351); + RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],31,0x14292967); + RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],32,0x27b70a85); + RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],33,0x2e1b2138); + RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],34,0x4d2c6dfc); + RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],35,0x53380d13); + RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],36,0x650a7354); + RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],37,0x766a0abb); + RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],38,0x81c2c92e); + RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],39,0x92722c85); + RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],40,0xa2bfe8a1); + RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],41,0xa81a664b); + RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],42,0xc24b8b70); + RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],43,0xc76c51a3); + RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],44,0xd192e819); + RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],45,0xd6990624); + RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],46,0xf40e3585); + RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],47,0x106aa070); + RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],48,0x19a4c116); + RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],49,0x1e376c08); + RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],50,0x2748774c); + RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],51,0x34b0bcb5); + RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],52,0x391c0cb3); + RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],53,0x4ed8aa4a); + RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],54,0x5b9cca4f); + RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],55,0x682e6ff3); + RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],56,0x748f82ee); + RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],57,0x78a5636f); + RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],58,0x84c87814); + RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],59,0x8cc70208); + RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],60,0x90befffa); + RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],61,0xa4506ceb); + RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],62,0xbef9a3f7); + RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],63,0xc67178f2); + +#undef RND + +#endif + + /* feedback */ + for (i = 0; i < 8; i++) { + md->state[i] = md->state[i] + S[i]; + } + return CRYPT_OK; +} + +#ifdef LTC_CLEAN_STACK +static int sha256_compress(sha256_state * md, unsigned char *buf) +{ + int err; + err = _sha256_compress(md, buf); + burn_stack(sizeof(uint32_t) * 74); + return err; +} +#endif + +/** + Initialize the hash state + @param md The hash state you wish to initialize + @return CRYPT_OK if successful +*/ +static int sha256_init(sha256_state * md) +{ + LTC_ARGCHK(md != NULL); + + md->curlen = 0; + md->length = 0; + md->state[0] = 0x6A09E667UL; + md->state[1] = 0xBB67AE85UL; + md->state[2] = 0x3C6EF372UL; + md->state[3] = 0xA54FF53AUL; + md->state[4] = 0x510E527FUL; + md->state[5] = 0x9B05688CUL; + md->state[6] = 0x1F83D9ABUL; + md->state[7] = 0x5BE0CD19UL; + return CRYPT_OK; +} + +/** + Process a block of memory though the hash + @param md The hash state + @param in The data to hash + @param inlen The length of the data (octets) + @return CRYPT_OK if successful +*/ +static int sha256_process (sha256_state * md, const unsigned char *in, unsigned long inlen) +{ + unsigned long n; + int err; + LTC_ARGCHK(md != NULL); + LTC_ARGCHK(in != NULL); + if (md->curlen > sizeof(md->buf)) { + return CRYPT_INVALID_ARG; + } + while (inlen > 0) { + if (md->curlen == 0 && inlen >= 64) { + if ((err = sha256_compress (md, (unsigned char *)in)) != CRYPT_OK) { + return err; + } + md->length += 64 * 8; + in += 64; + inlen -= 64; + } else { + n = MIN(inlen, (64 - md->curlen)); + memcpy(md->buf + md->curlen, in, (size_t)n); + md->curlen += n; + in += n; + inlen -= n; + if (md->curlen == 64) { + if ((err = sha256_compress (md, md->buf)) != CRYPT_OK) { + return err; + } + md->length += 8*64; + md->curlen = 0; + } + } + } + return CRYPT_OK; +} + +/** + Terminate the hash to get the digest + @param md The hash state + @param out [out] The destination of the hash (32 bytes) + @return CRYPT_OK if successful +*/ +static int sha256_done(sha256_state * md, unsigned char *out) +{ + int i; + + LTC_ARGCHK(md != NULL); + LTC_ARGCHK(out != NULL); + + if (md->curlen >= sizeof(md->buf)) { + return CRYPT_INVALID_ARG; + } + + + /* increase the length of the message */ + md->length += md->curlen * 8; + + /* append the '1' bit */ + md->buf[md->curlen++] = (unsigned char)0x80; + + /* if the length is currently above 56 bytes we append zeros + * then compress. Then we can fall back to padding zeros and length + * encoding like normal. + */ + if (md->curlen > 56) { + while (md->curlen < 64) { + md->buf[md->curlen++] = (unsigned char)0; + } + sha256_compress(md, md->buf); + md->curlen = 0; + } + + /* pad upto 56 bytes of zeroes */ + while (md->curlen < 56) { + md->buf[md->curlen++] = (unsigned char)0; + } + + /* store length */ + STORE64H(md->length, md->buf+56); + sha256_compress(md, md->buf); + + /* copy output */ + for (i = 0; i < 8; i++) { + STORE32H(md->state[i], out+(4*i)); + } +#ifdef LTC_CLEAN_STACK + zeromem(md, sizeof(sha256_state)); +#endif + return CRYPT_OK; +} + +/* $Source: /cvs/libtom/libtomcrypt/src/hashes/sha2/sha256.c,v $ */ +/* $Revision: 1.9 $ */ +/* $Date: 2006/11/01 09:28:17 $ */ From 8771fdfda49124e46f155a396778649946a894b9 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 19 Aug 2009 22:07:32 -0400 Subject: [PATCH 167/187] Make consensus_method_is_supported() be the canonical source for its info. Add a new function to derive the declared method list from consensus_method_is_supported(). --- src/or/dirvote.c | 42 ++++++++++++++++++++++++++++++++++-------- 1 file changed, 34 insertions(+), 8 deletions(-) diff --git a/src/or/dirvote.c b/src/or/dirvote.c index aadece9096..58b02da5cb 100644 --- a/src/or/dirvote.c +++ b/src/or/dirvote.c @@ -21,6 +21,10 @@ static int dirvote_perform_vote(void); static void dirvote_clear_votes(int all_votes); static int dirvote_compute_consensus(void); static int dirvote_publish_consensus(void); +static char *make_consensus_method_list(int low, int high); + +/** The highest consensus method that we currently support. */ +#define MAX_SUPPORTED_CONSENSUS_METHOD 6 /* ===== * Voting @@ -94,6 +98,8 @@ format_networkstatus_vote(crypto_pk_env_t *private_signing_key, char vu[ISO_TIME_LEN+1]; char *flags = smartlist_join_strings(v3_ns->known_flags, " ", 0, NULL); authority_cert_t *cert = v3_ns->cert; + char *methods = + make_consensus_method_list(1, MAX_SUPPORTED_CONSENSUS_METHOD); format_iso_time(published, v3_ns->published); format_iso_time(va, v3_ns->valid_after); format_iso_time(fu, v3_ns->fresh_until); @@ -103,10 +109,7 @@ format_networkstatus_vote(crypto_pk_env_t *private_signing_key, tor_snprintf(status, len, "network-status-version 3\n" "vote-status %s\n" - /* XXX: If you change this value, you also need to - * change consensus_method_is_supported(). - * Perhaps we should unify these somehow? */ - "consensus-methods 1 2 3 4 5 6\n" + "consensus-methods %s\n" "published %s\n" "valid-after %s\n" "fresh-until %s\n" @@ -117,6 +120,7 @@ format_networkstatus_vote(crypto_pk_env_t *private_signing_key, "dir-source %s %s %s %s %d %d\n" "contact %s\n", v3_ns->type == NS_TYPE_VOTE ? "vote" : "opinion", + methods, published, va, fu, vu, v3_ns->vote_seconds, v3_ns->dist_seconds, version_lines, @@ -125,6 +129,7 @@ format_networkstatus_vote(crypto_pk_env_t *private_signing_key, ipaddr, voter->dir_port, voter->or_port, voter->contact); tor_free(flags); + tor_free(methods); outp = status + strlen(status); endp = status + len; @@ -458,10 +463,31 @@ compute_consensus_method(smartlist_t *votes) static int consensus_method_is_supported(int method) { - /* XXX: If you change this value, you also need to change - * format_networkstatus_vote(). Perhaps we should unify - * these somehow? */ - return (method >= 1) && (method <= 6); + return (method >= 1) && (method <= MAX_SUPPORTED_CONSENSUS_METHOD); +} + +/** Return a newly allocated string holding the numbers between low and high + * (inclusive) that are supported consensus methods. */ +static char * +make_consensus_method_list(int low, int high) +{ + char *list; + + char b[32]; + int i; + smartlist_t *lst; + lst = smartlist_create(); + for (i = low; i <= high; ++i) { + if (!consensus_method_is_supported(i)) + continue; + tor_snprintf(b, sizeof(b), "%d", i); + smartlist_add(lst, tor_strdup(b)); + } + list = smartlist_join_strings(lst, " ", 0, NULL); + tor_assert(list); + SMARTLIST_FOREACH(lst, char *, cp, tor_free(cp)); + smartlist_free(lst); + return list; } /** Helper: given lst, a list of version strings such that every From 5da3b45fdce8354e320df91b26a170b21291c32e Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 20 Aug 2009 12:03:32 -0400 Subject: [PATCH 168/187] Make crypto_digest_get_digest nondestructive again. Fixes bug in f57883a39. --- src/common/crypto.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/common/crypto.c b/src/common/crypto.c index dc645b5eda..4a61d3faf3 100644 --- a/src/common/crypto.c +++ b/src/common/crypto.c @@ -1510,11 +1510,11 @@ crypto_digest_get_digest(crypto_digest_env_t *digest, switch (digest->algorithm) { case DIGEST_SHA1: tor_assert(out_len <= DIGEST_LEN); - SHA1_Final(r, &digest->d.sha1); + SHA1_Final(r, &tmpenv.d.sha1); break; case DIGEST_SHA256: tor_assert(out_len <= DIGEST256_LEN); - SHA256_Final(r, &digest->d.sha2); + SHA256_Final(r, &tmpenv.d.sha2); break; default: tor_fragile_assert(); From 7f518873eb58b4d12d6db8ccbf98fb626529f859 Mon Sep 17 00:00:00 2001 From: Roger Dingledine Date: Thu, 20 Aug 2009 16:50:51 -0400 Subject: [PATCH 169/187] Notice v3 cert parsing failures If any the v3 certs we download are unparseable, we should actually notice the failure so we don't retry indefinitely. Bugfix on 0.2.0.x; reported by "rotator". --- ChangeLog | 5 +++++ src/or/directory.c | 2 ++ src/or/routerlist.c | 18 ++++++++++++------ 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/ChangeLog b/ChangeLog index ac7beb79ea..9b93d3f361 100644 --- a/ChangeLog +++ b/ChangeLog @@ -39,6 +39,11 @@ Changes in version 0.2.2.1-alpha - 2009-0?-?? as only certain clients who must have this information sooner should set this option. + o Minor bugfixes: + - If any the v3 certs we download are unparseable, we should actually + notice the failure so we don't retry indefinitely. Bugfix on + 0.2.0.x; reported by "rotator". + o Deprecated and removed features: - The controller no longer accepts the old obsolete "addr-mappings/" or "unregistered-servers-" GETINFO values. diff --git a/src/or/directory.c b/src/or/directory.c index 976c08c19a..93046489f0 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -1649,6 +1649,8 @@ connection_dir_client_reached_eof(dir_connection_t *conn) "'%s:%d'",(int) body_len, conn->_base.address, conn->_base.port); if (trusted_dirs_load_certs_from_string(body, 0, 1)<0) { log_warn(LD_DIR, "Unable to parse fetched certificates"); + /* if we fetched more than one and only some failed, the successful + * ones got flushed to disk so it's safe to call this on them */ connection_dir_download_cert_failed(conn, status_code); } else { directory_info_has_arrived(now, 0); diff --git a/src/or/routerlist.c b/src/or/routerlist.c index 42b385b101..0a32f78a69 100644 --- a/src/or/routerlist.c +++ b/src/or/routerlist.c @@ -156,21 +156,24 @@ already_have_cert(authority_cert_t *cert) /** Load a bunch of new key certificates from the string contents. If * from_store is true, the certificates are from the cache, and we - * don't need to flush them to disk. If from_store is false, we need - * to flush any changed certificates to disk. Return 0 on success, -1 on - * failure. */ + * don't need to flush them to disk. If flush is true, we need + * to flush any changed certificates to disk now. Return 0 on success, -1 + * if any certs fail to parse. */ int trusted_dirs_load_certs_from_string(const char *contents, int from_store, int flush) { trusted_dir_server_t *ds; const char *s, *eos; + int failure_code = 0; for (s = contents; *s; s = eos) { authority_cert_t *cert = authority_cert_parse_from_string(s, &eos); cert_list_t *cl; - if (!cert) + if (!cert) { + failure_code = -1; break; + } ds = trusteddirserver_get_by_v3_auth_digest( cert->cache_info.identity_digest); log_debug(LD_DIR, "Parsed certificate for %s", @@ -224,7 +227,7 @@ trusted_dirs_load_certs_from_string(const char *contents, int from_store, ds->dir_port != cert->dir_port)) { char *a = tor_dup_ip(cert->addr); log_notice(LD_DIR, "Updating address for directory authority %s " - "from %s:%d to %s:%d based on in certificate.", + "from %s:%d to %s:%d based on certificate.", ds->nickname, ds->address, (int)ds->dir_port, a, cert->dir_port); tor_free(a); @@ -241,8 +244,11 @@ trusted_dirs_load_certs_from_string(const char *contents, int from_store, if (flush) trusted_dirs_flush_certs_to_disk(); + /* call this even if failure_code is <0, since some certs might have + * succeeded. */ networkstatus_note_certs_arrived(); - return 0; + + return failure_code; } /** Save all v3 key certificates to the cached-certs file. */ From eb829cc330629da03989c9c3b466d3c69b772249 Mon Sep 17 00:00:00 2001 From: Roger Dingledine Date: Thu, 20 Aug 2009 16:56:21 -0400 Subject: [PATCH 170/187] Survive unparseable cached cert file --- ChangeLog | 1 + src/or/main.c | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index 9b93d3f361..9e68528273 100644 --- a/ChangeLog +++ b/ChangeLog @@ -43,6 +43,7 @@ Changes in version 0.2.2.1-alpha - 2009-0?-?? - If any the v3 certs we download are unparseable, we should actually notice the failure so we don't retry indefinitely. Bugfix on 0.2.0.x; reported by "rotator". + - If the cached cert file is unparseable, warn but don't exit. o Deprecated and removed features: - The controller no longer accepts the old obsolete "addr-mappings/" diff --git a/src/or/main.c b/src/or/main.c index 4969212798..52cc0282b3 100644 --- a/src/or/main.c +++ b/src/or/main.c @@ -1410,8 +1410,10 @@ do_main_loop(void) /* initialize the bootstrap status events to know we're starting up */ control_event_bootstrap(BOOTSTRAP_STATUS_STARTING, 0); - if (trusted_dirs_reload_certs()) - return -1; + if (trusted_dirs_reload_certs()) { + log_warn(LD_DIR, + "Couldn't load all cached v3 certificates. Starting anyway."); + } if (router_reload_v2_networkstatus()) { return -1; } From 978571587a85bebb37ec3cb9c2ea1fde1cecd6d7 Mon Sep 17 00:00:00 2001 From: Roger Dingledine Date: Thu, 20 Aug 2009 23:02:57 -0400 Subject: [PATCH 171/187] autoconf 2.59 appears not to support AC_PROG_SED --- configure.in | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/configure.in b/configure.in index ab7d6480cf..c21c130a05 100644 --- a/configure.in +++ b/configure.in @@ -127,7 +127,9 @@ AC_PROG_CC AC_PROG_CPP AC_PROG_MAKE_SET AC_PROG_RANLIB -AC_PROG_SED + +dnl autoconf 2.59 appears not to support AC_PROG_SED +AC_CHECK_PROG([SED],[sed],[sed],[/bin/false]) AC_PATH_PROG([SHA1SUM], [sha1sum], none) AC_PATH_PROG([OPENSSL], [openssl], none) From daa0326aaaa85a760be94ee2360cfa61a9fb5be2 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 12 Jun 2009 13:38:37 -0400 Subject: [PATCH 172/187] Add the first 8 bytes of the git commit digest to our versions. Note that unlike subversion revision numbers, it isn't meaningful to compare these for anything but equality. We define a sort-order anyway, in case one of these accidentally slips into a recommended-versions list. --- ChangeLog | 2 ++ src/or/Makefile.am | 56 +++++++++----------------------------------- src/or/config.c | 8 +++---- src/or/or.h | 3 +++ src/or/router.c | 2 -- src/or/routerparse.c | 25 ++++++++++++++++++-- src/or/test.c | 15 +++++++++++- src/or/tor_main.c | 2 +- 8 files changed, 58 insertions(+), 55 deletions(-) diff --git a/ChangeLog b/ChangeLog index 9e68528273..2428bed996 100644 --- a/ChangeLog +++ b/ChangeLog @@ -38,6 +38,8 @@ Changes in version 0.2.2.1-alpha - 2009-0?-?? setting FetchDirInfoEarly to 1. Previous behavior will stay the same as only certain clients who must have this information sooner should set this option. + - Instead of adding the svn revision to the Tor version string, report + the git commit (when we're building from a git checkout). o Minor bugfixes: - If any the v3 certs we download are unparseable, we should actually diff --git a/src/or/Makefile.am b/src/or/Makefile.am index c967a8846f..7d6c9eb0b9 100644 --- a/src/or/Makefile.am +++ b/src/or/Makefile.am @@ -57,52 +57,18 @@ config_codedigest.o: or_sha1.i tor_main.o: micro-revision.i micro-revision.i: FORCE - @svkdir=$$SVKROOT; \ - if test "x$$svkdir" = x ; then \ - svkdir=$$HOME/.svk; \ - fi; \ - if test -d ../../.git && test -x "`which git 2>&1;true`" ; then \ - if test -d ../../.git/svn && test -x "`which git-svn 2>&1;true`" ; then \ - git-svn info ../../README | \ - sed -n 's/^Revision: \([0-9][0-9]*\).*/"\1"/p' \ - > micro-revision.tmp \ - || true; \ - fi; \ - elif test -d ../../.svn && test -x "`which svn 2>&1;true`" ; then \ - svn info ../.. | \ - sed -n 's/^Revision: \([0-9][0-9]*\).*/"\1"/p' > micro-revision.tmp \ - || true; \ - elif test -x "`which svk 2>&1;true`" && test -d $$svkdir/local; then \ - location=../..; \ - rev=x; \ - while test x$$rev = xx; do \ - x=`svk info $$location | \ - sed -n 's/^Mirrored From:.*, Rev\. \([0-9][0-9]*\)/\1/p'`; \ - if test x$$x != x; then \ - rev=$$x; \ - break; \ - else \ - loc=`svk info $$location | \ - sed -n 's/^Copied From: \(.*\), Rev\. [0-9][0-9]*/\1/p' | \ - head -1`; \ - if test x$$loc = x; then \ - break; \ - else \ - location=/$$loc; \ - fi; \ - fi; \ - done; \ - if test x$$rev != xx; then \ - echo \"$$rev\" > micro-revision.tmp; \ - fi; \ - fi; \ - if test ! -f micro-revision.tmp ; then \ - if test ! -f micro-revision.i ; then \ - echo '""' > micro-revision.i; \ - fi; \ - elif test ! -f micro-revision.i || \ + @rm -f micro-revision.tmp; \ + if test -d ../../.git && test -x "`which git 2>&1;true`"; then \ + HASH="`git rev-parse --short=16 HEAD`"; \ + echo \"$$HASH\" > micro-revision.tmp; \ + fi; \ + if test ! -f micro-revision.tmp ; then \ + if test ! -f micro-revision.i ; then \ + echo '""' > micro-revision.i; \ + fi; \ + elif test ! -f micro-revision.i || \ test x"`cat micro-revision.tmp`" != x"`cat micro-revision.i`"; then \ - mv micro-revision.tmp micro-revision.i; \ + mv micro-revision.tmp micro-revision.i; \ fi; true or_sha1.i: $(tor_SOURCES) test_data.c test.c diff --git a/src/or/config.c b/src/or/config.c index de6bfcf680..43f88e537e 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -817,7 +817,7 @@ set_options(or_options_t *new_val, char **msg) return 0; } -extern const char tor_svn_revision[]; /* from tor_main.c */ +extern const char tor_git_revision[]; /* from tor_main.c */ /** The version of this Tor process, as parsed. */ static char *_version = NULL; @@ -827,10 +827,10 @@ const char * get_version(void) { if (_version == NULL) { - if (strlen(tor_svn_revision)) { - size_t len = strlen(VERSION)+strlen(tor_svn_revision)+8; + if (strlen(tor_git_revision)) { + size_t len = strlen(VERSION)+strlen(tor_git_revision)+16; _version = tor_malloc(len); - tor_snprintf(_version, len, "%s (r%s)", VERSION, tor_svn_revision); + tor_snprintf(_version, len, "%s (git-%s)", VERSION, tor_git_revision); } else { _version = tor_strdup(VERSION); } diff --git a/src/or/or.h b/src/or/or.h index 7ece877504..4d808fc3e3 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -4698,6 +4698,9 @@ typedef struct tor_version_t { int patchlevel; char status_tag[MAX_STATUS_TAG_LEN]; int svn_revision; + + int git_tag_len; + char git_tag[DIGEST_LEN]; } tor_version_t; int router_get_router_hash(const char *s, char *digest); diff --git a/src/or/router.c b/src/or/router.c index d5a2576842..f2747f5141 100644 --- a/src/or/router.c +++ b/src/or/router.c @@ -1608,8 +1608,6 @@ router_guess_address_from_dir_headers(uint32_t *guess) return -1; } -extern const char tor_svn_revision[]; /* from tor_main.c */ - /** Set platform (max length len) to a NUL-terminated short * string describing the version of Tor and the operating system we're * currently running on. diff --git a/src/or/routerparse.c b/src/or/routerparse.c index c1a7fbcfae..9736c4e000 100644 --- a/src/or/routerparse.c +++ b/src/or/routerparse.c @@ -3325,7 +3325,7 @@ tor_version_as_new_as(const char *platform, const char *cutoff) if (!*start) return 0; s = (char *)find_whitespace(start); /* also finds '\0', which is fine */ s2 = (char*)eat_whitespace(s); - if (!strcmpstart(s2, "(r")) + if (!strcmpstart(s2, "(r") || !strcmpstart(s2, "(git-")) s = (char*)find_whitespace(s2); if ((size_t)(s-start+1) >= sizeof(tmp)) /* too big, no */ @@ -3421,6 +3421,21 @@ tor_version_parse(const char *s, tor_version_t *out) if (!strcmpstart(cp, "(r")) { cp += 2; out->svn_revision = (int) strtol(cp,&eos,10); + } else if (!strcmpstart(cp, "(git-")) { + char *close_paren = strchr(cp, ')'); + int hexlen; + char digest[DIGEST_LEN]; + if (! close_paren) + return -1; + cp += 5; + hexlen = (close_paren-cp); + memset(digest, 0, sizeof(digest)); + if (hexlen > HEX_DIGEST_LEN || hexlen == 0 || (hexlen % 2) == 1) + return -1; + if (base16_decode(digest, hexlen/2, cp, hexlen)) + return -1; + memcpy(out->git_tag, digest, hexlen/2); + out->git_tag_len = hexlen/2; } return 0; @@ -3446,8 +3461,14 @@ tor_version_compare(tor_version_t *a, tor_version_t *b) return i; else if ((i = strcmp(a->status_tag, b->status_tag))) return i; + else if ((i = a->svn_revision - b->svn_revision)) + return i; + else if ((i = a->git_tag_len - b->git_tag_len)) + return i; + else if (a->git_tag_len) + return memcmp(a->git_tag, b->git_tag, a->git_tag_len); else - return a->svn_revision - b->svn_revision; + return 0; } /** Return true iff versions a and b belong to the same series. diff --git a/src/or/test.c b/src/or/test.c index db4b6784f4..0f121a84bb 100644 --- a/src/or/test.c +++ b/src/or/test.c @@ -5,7 +5,7 @@ /* Ordinarily defined in tor_main.c; this bit is just here to provide one * since we're not linking to tor_main.c */ -const char tor_svn_revision[] = ""; +const char tor_git_revision[] = ""; /** * \file test.c @@ -3212,6 +3212,19 @@ test_dir_format(void) "Tor 0.2.1.0-dev (r99)")); test_eq(1, tor_version_as_new_as("Tor 0.2.1.1", "Tor 0.2.1.0-dev (r99)")); + + /* Now try git revisions */ + test_eq(0, tor_version_parse("0.5.6.7 (git-ff00ff)", &ver1)); + test_eq(0, ver1.major); + test_eq(5, ver1.minor); + test_eq(6, ver1.micro); + test_eq(7, ver1.patchlevel); + test_eq(3, ver1.git_tag_len); + test_memeq(ver1.git_tag, "\xff\x00\xff", 3); + test_eq(-1, tor_version_parse("0.5.6.7 (git-ff00xx)", &ver1)); + test_eq(-1, tor_version_parse("0.5.6.7 (git-ff00fff)", &ver1)); + test_eq(0, tor_version_parse("0.5.6.7 (git ff00fff)", &ver1)); + done: if (r1) routerinfo_free(r1); diff --git a/src/or/tor_main.c b/src/or/tor_main.c index 137da02c41..4a6be7cddd 100644 --- a/src/or/tor_main.c +++ b/src/or/tor_main.c @@ -7,7 +7,7 @@ * built from. This string is generated by a bit of shell kludging int * src/or/Makefile.am, and is usually right. */ -const char tor_svn_revision[] = +const char tor_git_revision[] = #ifndef _MSC_VER #include "micro-revision.i" #endif From d4a75a222f8da50caaafe42464809b0856f724eb Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 21 Aug 2009 13:17:50 -0400 Subject: [PATCH 173/187] Fix a memory leak in summarizing directory request timing. Spotted by Coverity Scan. --- src/or/geoip.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/or/geoip.c b/src/or/geoip.c index 5c13154205..ef0d29e09b 100644 --- a/src/or/geoip.c +++ b/src/or/geoip.c @@ -743,7 +743,8 @@ geoip_get_dirreq_history(geoip_client_action_t action, &ent->completion_time); if (time_diff == 0) time_diff = 1; /* Avoid DIV/0; "instant" answers are impossible - * anyway by law of nature or something.. */ + * by law of nature or something, but a milisecond + * is a bit greater than "instantly" */ *bytes_per_second = 1000 * ent->response_size / time_diff; smartlist_add(dirreq_times, bytes_per_second); complete++; @@ -767,8 +768,11 @@ geoip_get_dirreq_history(geoip_client_action_t action, result = tor_malloc_zero(bufsize); written = tor_snprintf(result, bufsize, "complete=%u,timeout=%u," "running=%u", complete, timeouts, running); - if (written < 0) + if (written < 0) { + SMARTLIST_FOREACH(dirreq_times, uint32_t *, dlt, tor_free(dlt)); + smartlist_free(dirreq_times); return NULL; + } #define MIN_DIR_REQ_RESPONSES 16 if (complete >= MIN_DIR_REQ_RESPONSES) { uint32_t *dltimes = tor_malloc(sizeof(uint32_t) * complete); @@ -794,9 +798,11 @@ geoip_get_dirreq_history(geoip_client_action_t action, dltimes[9*complete/10-1], dltimes[complete-1]); tor_free(dltimes); + } else { + SMARTLIST_FOREACH(dirreq_times, uint32_t *, dlt, tor_free(dlt)); } if (written < 0) - result = NULL; + tor_free(result); smartlist_free(dirreq_times); return result; } From b51c59254709ccf5229c84eccc03011e5f0e52d1 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 21 Aug 2009 13:35:47 -0400 Subject: [PATCH 174/187] Refactor geoip_get_dirreq_history() some more. This patch avoids a bunch of allocations, and avoids using unallocated memory. --- src/or/geoip.c | 61 +++++++++++++++++++++++++++----------------------- 1 file changed, 33 insertions(+), 28 deletions(-) diff --git a/src/or/geoip.c b/src/or/geoip.c index ef0d29e09b..c61ac3c5fd 100644 --- a/src/or/geoip.c +++ b/src/or/geoip.c @@ -720,9 +720,9 @@ geoip_get_dirreq_history(geoip_client_action_t action, dirreq_type_t type) { char *result = NULL; - smartlist_t *dirreq_times = NULL; + smartlist_t *dirreq_completed = NULL; uint32_t complete = 0, timeouts = 0, running = 0; - int i = 0, bufsize = 1024, written; + int bufsize = 1024, written; dirreq_map_entry_t **ptr, **next, *ent; struct timeval now; @@ -730,7 +730,7 @@ geoip_get_dirreq_history(geoip_client_action_t action, if (action != GEOIP_CLIENT_NETWORKSTATUS && action != GEOIP_CLIENT_NETWORKSTATUS_V2) return NULL; - dirreq_times = smartlist_create(); + dirreq_completed = smartlist_create(); for (ptr = HT_START(dirreqmap, &dirreq_map); ptr; ptr = next) { ent = *ptr; if (ent->action != action || ent->type != type) { @@ -738,24 +738,17 @@ geoip_get_dirreq_history(geoip_client_action_t action, continue; } else { if (ent->completed) { - uint32_t *bytes_per_second = tor_malloc_zero(sizeof(uint32_t)); - uint32_t time_diff = (uint32_t) tv_mdiff(&ent->request_time, - &ent->completion_time); - if (time_diff == 0) - time_diff = 1; /* Avoid DIV/0; "instant" answers are impossible - * by law of nature or something, but a milisecond - * is a bit greater than "instantly" */ - *bytes_per_second = 1000 * ent->response_size / time_diff; - smartlist_add(dirreq_times, bytes_per_second); + smartlist_add(dirreq_completed, ent); complete++; + next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ptr); } else { if (tv_mdiff(&ent->request_time, &now) / 1000 > DIRREQ_TIMEOUT) timeouts++; else running++; + next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ptr); + tor_free(ent); } - next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ptr); - tor_free(ent); } } #define DIR_REQ_GRANULARITY 4 @@ -769,18 +762,29 @@ geoip_get_dirreq_history(geoip_client_action_t action, written = tor_snprintf(result, bufsize, "complete=%u,timeout=%u," "running=%u", complete, timeouts, running); if (written < 0) { - SMARTLIST_FOREACH(dirreq_times, uint32_t *, dlt, tor_free(dlt)); - smartlist_free(dirreq_times); - return NULL; + tor_free(result); + goto done; } + #define MIN_DIR_REQ_RESPONSES 16 if (complete >= MIN_DIR_REQ_RESPONSES) { - uint32_t *dltimes = tor_malloc(sizeof(uint32_t) * complete); - SMARTLIST_FOREACH(dirreq_times, uint32_t *, dlt, { - dltimes[i++] = *dlt; - tor_free(dlt); - }); - median_uint32(dltimes, complete); /* sort */ + uint32_t *dltimes; + /* We may have rounded 'completed' up. Here we want to use the + * real value. */ + complete = smartlist_len(dirreq_completed); + dltimes = tor_malloc_zero(sizeof(uint32_t) * complete); + SMARTLIST_FOREACH_BEGIN(dirreq_completed, dirreq_map_entry_t *, ent) { + uint32_t bytes_per_second; + uint32_t time_diff = (uint32_t) tv_mdiff(&ent->request_time, + &ent->completion_time); + if (time_diff == 0) + time_diff = 1; /* Avoid DIV/0; "instant" answers are impossible + * by law of nature or something, but a milisecond + * is a bit greater than "instantly" */ + bytes_per_second = 1000 * ent->response_size / time_diff; + dltimes[ent_sl_idx] = bytes_per_second; + } SMARTLIST_FOREACH_END(ent); + median_uint32(dltimes, complete); /* sorts as a side effect. */ written = tor_snprintf(result + written, bufsize - written, ",min=%u,d1=%u,d2=%u,q1=%u,d3=%u,d4=%u,md=%u," "d6=%u,d7=%u,q3=%u,d8=%u,d9=%u,max=%u", @@ -797,13 +801,14 @@ geoip_get_dirreq_history(geoip_client_action_t action, dltimes[8*complete/10-1], dltimes[9*complete/10-1], dltimes[complete-1]); + if (written<0) + tor_free(result); tor_free(dltimes); - } else { - SMARTLIST_FOREACH(dirreq_times, uint32_t *, dlt, tor_free(dlt)); } - if (written < 0) - tor_free(result); - smartlist_free(dirreq_times); + done: + SMARTLIST_FOREACH(dirreq_completed, dirreq_map_entry_t *, ent, + tor_free(ent)); + smartlist_free(dirreq_completed); return result; } #endif From 8c29b7920ae18a46ce0527806507275783d1ae42 Mon Sep 17 00:00:00 2001 From: Karsten Loesing Date: Fri, 21 Aug 2009 23:02:36 +0200 Subject: [PATCH 175/187] Add some fixes after discussion with Nick. - Refactor geoip.c by moving duplicate code into rotate_request_period(). - Don't leak memory when cleaning up cell queues. - Make sure that exit_(streams|bytes_(read|written)) are initialized in all places accessing these arrays. - Read only the last block from *stats files and ensure that its timestamp is not more than 25 hours in the past and not more than 1 hour in the future. - Stop truncating the last character when reading *stats files. The only thing that's left now is to avoid reading whole *stats files into memory. --- src/or/geoip.c | 41 ++++++++++++++++------------------ src/or/relay.c | 2 +- src/or/rephist.c | 9 ++++++++ src/or/router.c | 58 +++++++++++++++++++++++++++--------------------- 4 files changed, 62 insertions(+), 48 deletions(-) diff --git a/src/or/geoip.c b/src/or/geoip.c index 622b582173..247e16cff3 100644 --- a/src/or/geoip.c +++ b/src/or/geoip.c @@ -365,6 +365,23 @@ geoip_get_mean_shares(time_t now, double *v2_share_out, return 0; } +/* Rotate period of v2 and v3 network status requests. */ +static void +rotate_request_period(void) +{ + SMARTLIST_FOREACH(geoip_countries, geoip_country_t *, c, { + memmove(&c->n_v2_ns_requests[0], &c->n_v2_ns_requests[1], + sizeof(uint32_t)*(REQUEST_HIST_LEN-1)); + memmove(&c->n_v3_ns_requests[0], &c->n_v3_ns_requests[1], + sizeof(uint32_t)*(REQUEST_HIST_LEN-1)); + c->n_v2_ns_requests[REQUEST_HIST_LEN-1] = 0; + c->n_v3_ns_requests[REQUEST_HIST_LEN-1] = 0; + }); + current_request_period_starts += REQUEST_HIST_PERIOD; + if (n_old_request_periods < REQUEST_HIST_LEN-1) + ++n_old_request_periods; +} + /** Note that we've seen a client connect from the IP addr (host order) * at time now. Ignored by all but bridges and directories if * configured accordingly. */ @@ -404,17 +421,7 @@ geoip_note_client_seen(geoip_client_action_t action, * with action GEOIP_CLIENT_NETWORKSTATUS{_V2}.) */ geoip_remove_old_clients(current_request_period_starts); /* Now rotate request period */ - SMARTLIST_FOREACH(geoip_countries, geoip_country_t *, c, { - memmove(&c->n_v2_ns_requests[0], &c->n_v2_ns_requests[1], - sizeof(uint32_t)*(REQUEST_HIST_LEN-1)); - memmove(&c->n_v3_ns_requests[0], &c->n_v3_ns_requests[1], - sizeof(uint32_t)*(REQUEST_HIST_LEN-1)); - c->n_v2_ns_requests[REQUEST_HIST_LEN-1] = 0; - c->n_v3_ns_requests[REQUEST_HIST_LEN-1] = 0; - }); - current_request_period_starts += REQUEST_HIST_PERIOD; - if (n_old_request_periods < REQUEST_HIST_LEN-1) - ++n_old_request_periods; + rotate_request_period(); } } @@ -1046,17 +1053,7 @@ geoip_dirreq_stats_write(time_t now) open_file = NULL; /* Rotate request period */ - SMARTLIST_FOREACH(geoip_countries, geoip_country_t *, c, { - memmove(&c->n_v2_ns_requests[0], &c->n_v2_ns_requests[1], - sizeof(uint32_t)*(REQUEST_HIST_LEN-1)); - memmove(&c->n_v3_ns_requests[0], &c->n_v3_ns_requests[1], - sizeof(uint32_t)*(REQUEST_HIST_LEN-1)); - c->n_v2_ns_requests[REQUEST_HIST_LEN-1] = 0; - c->n_v3_ns_requests[REQUEST_HIST_LEN-1] = 0; - }); - current_request_period_starts += REQUEST_HIST_PERIOD; - if (n_old_request_periods < REQUEST_HIST_LEN-1) - ++n_old_request_periods; + rotate_request_period(); start_of_dirreq_stats_interval = now; diff --git a/src/or/relay.c b/src/or/relay.c index a79a4c161a..c81b8311a0 100644 --- a/src/or/relay.c +++ b/src/or/relay.c @@ -1677,7 +1677,7 @@ cell_queue_clear(cell_queue_t *queue) queue->insertion_times->first = elem->next; mp_pool_release(elem); } - queue->insertion_times = NULL; + tor_free(queue->insertion_times); } } diff --git a/src/or/rephist.c b/src/or/rephist.c index f7d0b2bbcf..8d78ac26c3 100644 --- a/src/or/rephist.c +++ b/src/or/rephist.c @@ -1371,6 +1371,9 @@ rep_hist_exit_stats_write(time_t now) open_file_t *open_file = NULL; FILE *out = NULL; + if (!exit_streams) + return; /* Not initialized */ + statsdir = get_datadir_fname("stats"); if (check_private_dir(statsdir, CPD_CREATE) < 0) goto done; @@ -1480,6 +1483,8 @@ rep_hist_note_exit_bytes_written(uint16_t port, size_t num_bytes) { if (!get_options()->ExitPortStatistics) return; + if (!exit_bytes_written) + return; /* Not initialized */ exit_bytes_written[port] += num_bytes; log_debug(LD_HIST, "Written %lu bytes to exit connection to port %d.", (unsigned long)num_bytes, port); @@ -1492,6 +1497,8 @@ rep_hist_note_exit_bytes_read(uint16_t port, size_t num_bytes) { if (!get_options()->ExitPortStatistics) return; + if (!exit_bytes_read) + return; /* Not initialized */ exit_bytes_read[port] += num_bytes; log_debug(LD_HIST, "Read %lu bytes from exit connection to port %d.", (unsigned long)num_bytes, port); @@ -1503,6 +1510,8 @@ rep_hist_note_exit_stream_opened(uint16_t port) { if (!get_options()->ExitPortStatistics) return; + if (!exit_streams) + return; /* Not initialized */ exit_streams[port]++; log_debug(LD_HIST, "Opened exit stream to port %d", port); } diff --git a/src/or/router.c b/src/or/router.c index 64267f242c..2d6734ae04 100644 --- a/src/or/router.c +++ b/src/or/router.c @@ -1826,36 +1826,40 @@ router_dump_router_to_string(char *s, size_t maxlen, routerinfo_t *router, return (int)written+1; } -/** Load the contents of filename, find the first line starting - * with end_line that has a timestamp after after, and write - * the file contents starting with that line to **contents. Return - * 1 for success, 0 if the file does not exist or does not contain a line - * matching these criteria, or -1 for failure. */ +/** Load the contents of filename, find the last line starting with + * end_line, ensure that its timestamp is not more than 25 hours in + * the past or more than 1 hour in the future with respect to now, + * and write the file contents starting with that line to **out. + * Return 1 for success, 0 if the file does not exist or does not contain + * a line matching these criteria, or -1 for failure. */ static int -load_stats_file(const char *filename, const char *end_line, time_t after, +load_stats_file(const char *filename, const char *end_line, time_t now, char **out) { int r = -1; char *fname = get_datadir_fname(filename); - char *contents, *start, timestr[ISO_TIME_LEN+1]; + char *contents, *start = NULL, *tmp, timestr[ISO_TIME_LEN+1]; time_t written; switch (file_status(fname)) { case FN_FILE: + /* X022 Find an alternative to reading the whole file to memory. */ if ((contents = read_file_to_str(fname, 0, NULL))) { - start = contents; - do { - if (start != contents) - start++; /* Avoid infinite loops. */ - if (!(start = strstr(start, end_line))) - goto notfound; - if (strlen(start) < strlen(end_line) + 1 + sizeof(timestr)) - goto notfound; - strlcpy(timestr, start + 1 + strlen(end_line), sizeof(timestr)); - if (parse_iso_time(timestr, &written) < 0) - goto notfound; - } while (written <= after); - *out = tor_malloc(strlen(start)); - strlcpy(*out, start, strlen(start)); + tmp = strstr(contents, end_line); + /* Find last block starting with end_line */ + while (tmp) { + start = tmp; + tmp = strstr(tmp + 1, end_line); + } + if (!start) + goto notfound; + if (strlen(start) < strlen(end_line) + 1 + sizeof(timestr)) + goto notfound; + strlcpy(timestr, start + 1 + strlen(end_line), sizeof(timestr)); + if (parse_iso_time(timestr, &written) < 0) + goto notfound; + if (written < now - (25*60*60) || written > now + (1*60*60)) + goto notfound; + *out = tor_strdup(start); r = 1; } notfound: @@ -1908,7 +1912,8 @@ extrainfo_dump_to_string(char *s, size_t maxlen, extrainfo_t *extrainfo, load_stats_file("stats"PATH_SEPARATOR"dirreq-stats", "dirreq-stats-end", since, &contents) > 0) { int pos = strlen(s); - if (tor_snprintf(s + pos, maxlen - strlen(s), "%s\n", contents) < 0) { + if (strlcpy(s + pos, contents, maxlen - strlen(s)) != + strlen(contents)) { log_warn(LD_DIR, "Could not write dirreq-stats to extra-info " "descriptor."); s[pos] = '\0'; @@ -1919,7 +1924,8 @@ extrainfo_dump_to_string(char *s, size_t maxlen, extrainfo_t *extrainfo, load_stats_file("stats"PATH_SEPARATOR"entry-stats", "entry-stats-end", since, &contents) > 0) { int pos = strlen(s); - if (tor_snprintf(s + pos, maxlen - strlen(s), "%s\n", contents) < 0) { + if (strlcpy(s + pos, contents, maxlen - strlen(s)) != + strlen(contents)) { log_warn(LD_DIR, "Could not write entry-stats to extra-info " "descriptor."); s[pos] = '\0'; @@ -1930,7 +1936,8 @@ extrainfo_dump_to_string(char *s, size_t maxlen, extrainfo_t *extrainfo, load_stats_file("stats"PATH_SEPARATOR"buffer-stats", "cell-stats-end", since, &contents) > 0) { int pos = strlen(s); - if (tor_snprintf(s + pos, maxlen - strlen(s), "%s\n", contents) < 0) { + if (strlcpy(s + pos, contents, maxlen - strlen(s)) != + strlen(contents)) { log_warn(LD_DIR, "Could not write buffer-stats to extra-info " "descriptor."); s[pos] = '\0'; @@ -1941,7 +1948,8 @@ extrainfo_dump_to_string(char *s, size_t maxlen, extrainfo_t *extrainfo, load_stats_file("stats"PATH_SEPARATOR"exit-stats", "exit-stats-end", since, &contents) > 0) { int pos = strlen(s); - if (tor_snprintf(s + pos, maxlen - strlen(s), "%s\n", contents) < 0) { + if (strlcpy(s + pos, contents, maxlen - strlen(s)) != + strlen(contents)) { log_warn(LD_DIR, "Could not write exit-stats to extra-info " "descriptor."); s[pos] = '\0'; From f1b5fd2aaa81daaa9065a7f0193be5855e7d4e32 Mon Sep 17 00:00:00 2001 From: Roger Dingledine Date: Tue, 25 Aug 2009 00:34:29 -0400 Subject: [PATCH 176/187] new proposals: params in consensus, and lower circwindow --- .../proposals/167-params-in-consensus.txt | 47 ++++++ doc/spec/proposals/168-reduce-circwindow.txt | 134 ++++++++++++++++++ 2 files changed, 181 insertions(+) create mode 100644 doc/spec/proposals/167-params-in-consensus.txt create mode 100644 doc/spec/proposals/168-reduce-circwindow.txt diff --git a/doc/spec/proposals/167-params-in-consensus.txt b/doc/spec/proposals/167-params-in-consensus.txt new file mode 100644 index 0000000000..7649c040cd --- /dev/null +++ b/doc/spec/proposals/167-params-in-consensus.txt @@ -0,0 +1,47 @@ +Filename: 167-params-in-consensus.txt +Title: Vote on network parameters in consensus +Author: Roger Dingledine +Created: 18-Aug-2009 +Status: Open +Target: 0.2.2 + +0. History + + +1. Overview + + Several of our new performance plans involve guessing how to tune + clients and relays, yet we won't be able to learn whether we guessed + the right tuning parameters until many people have upgraded. Instead, + we should have directory authorities vote on the parameters, and teach + Tors to read the currently recommended values out of the consensus. + +2. Design + + V3 votes should include a new "params" line after the known-flags + line. It contains key=value pairs, where value is an integer. + + Consensus documents that are generated with a sufficiently new consensus + method (7?) then include a params line that includes every key listed + in any vote, and the median value for that key (in case of ties, + we use the median closer to zero). + +2.1. Planned keys. + + The first planned parameter is "circwindow=101", which is the initial + circuit packaging window that clients and relays should use. Putting + it in the consensus will let us perform experiments with different + values once enough Tors have upgraded -- see proposal 168. + + Later parameters might include a weighting for how much to favor quiet + circuits over loud circuits in our round-robin algorithm; a weighting + for how much to prioritize relays over clients if we use an incentive + scheme like the gold-star design; and what fraction of circuits we + should throw out from proposal 151. + +2.2. What about non-integers? + + I'm not sure how we would do median on non-integer values. Further, + I don't have any non-integer values in mind yet. So I say we cross + that bridge when we get to it. + diff --git a/doc/spec/proposals/168-reduce-circwindow.txt b/doc/spec/proposals/168-reduce-circwindow.txt new file mode 100644 index 0000000000..c10cf41e2e --- /dev/null +++ b/doc/spec/proposals/168-reduce-circwindow.txt @@ -0,0 +1,134 @@ +Filename: 168-reduce-circwindow.txt +Title: Reduce default circuit window +Author: Roger Dingledine +Created: 12-Aug-2009 +Status: Open +Target: 0.2.2 + +0. History + + +1. Overview + + We should reduce the starting circuit "package window" from 1000 to + 101. The lower package window will mean that clients will only be able + to receive 101 cells (~50KB) on a circuit before they need to send a + 'sendme' acknowledgement cell to request 100 more. + + Starting with a lower package window on exit relays should save on + buffer sizes (and thus memory requirements for the exit relay), and + should save on queue sizes (and thus latency for users). + + Lowering the package window will induce an extra round-trip for every + additional 50298 bytes of the circuit. This extra step is clearly a + slow-down for large streams, but ultimately we hope that a) clients + fetching smaller streams will see better response, and b) slowing + down the large streams in this way will produce lower e2e latencies, + so the round-trips won't be so bad. + +2. Motivation + + Karsten's torperf graphs show that the median download time for a 50KB + file over Tor in mid 2009 is 7.7 seconds, whereas the median download + time for 1MB and 5MB are around 50s and 150s respectively. The 7.7 + second figure is way too high, whereas the 50s and 150s figures are + surprisingly low. + + The median round-trip latency appears to be around 2s, with 25% of + the data points taking more than 5s. That's a lot of variance. + + We designed Tor originally with the original goal of maximizing + throughput. We figured that would also optimize other network properties + like round-trip latency. Looks like we were wrong. + +3. Design + + Wherever we initialize the circuit package window, initialize it to + 101 rather than 1000. Reducing it should be safe even when interacting + with old Tors: the old Tors will receive the 101 cells and send back + a sendme ack cell. They'll still have much higher deliver windows, + but the rest of their deliver window will go unused. + + You can find the patch at arma/circwindow. It seems to work. + +3.1. Why not 100? + + Tor 0.0.0 through 0.2.1.19 have a bug where they only send the sendme + ack cell after 101 cells rather than the intended 100 cells. + + Once 0.2.1.19 is obsolete we can change it back to 100 if we like. But + hopefully we'll have moved to some datagram protocol long before + 0.2.1.19 becomes obsolete. + +3.2. What about stream packaging windows? + + Right now the stream packaging windows start at 500. The goal was to + set the stream window to half the circuit window, to provide a crude + load balancing between streams on the same circuit. Once we lower + the circuit packaging window, the stream packaging window basically + becomes redundant. + + We could leave it in -- it isn't hurting much in either case. Or we + could take it out -- people building other Tor clients would thank us + for that step. Alas, people building other Tor clients are going to + have to be compatible with current Tor clients, so in practice there's + no point taking out the stream packaging windows. + +3.3. What about variable circuit windows? + + Once upon a time we imagined adapting the circuit package window to + the network conditions. That is, we would start the window small, + and raise it based on the latency and throughput we see. + + In theory that crude imitation of TCP's windowing system would allow + us to adapt to fill the network better. In practice, I think we want + to stick with the small window and never raise it. The low cap reduces + the total throughput you can get from Tor for a given circuit. But + that's a feature, not a bug. + +4. Evaluation + + How do we know this change is actually smart? It seems intuitive that + it's helpful, and some smart systems people have agreed that it's + a good idea (or said another way, they were shocked at how big the + default package window was before). + + To get a more concrete sense of the benefit, though, Karsten has been + running torperf side-by-side on exit relays with the old package window + vs the new one. The results are mixed currently -- it is slightly faster + for fetching 40KB files, and slightly slower for fetching 50KB files. + + I think it's going to be tough to get a clear conclusion that this is + a good design just by comparing one exit relay running the patch. The + trouble is that the other hops in the circuits are still getting bogged + down by other clients introducing too much traffic into the network. + + Ultimately, we'll want to put the circwindow parameter into the + consensus so we can test a broader range of values once enough relays + have upgraded. + +5. Transition and deployment + + We should put the circwindow in the consensus (see proposal 167), + with an initial value of 101. Then as more exit relays upgrade, + clients should seamlessly get the better behavior. + + Note that upgrading the exit relay will only affect the "download" + package window. An old client that's uploading lots of bytes will + continue to use the old package window at the client side, and we + can't throttle that window at the exit side without breaking protocol. + + The real question then is what we should backport to 0.2.1. Assuming + this could be a big performance win, we can't afford to wait until + 0.2.2.x comes out before starting to see the changes here. So we have + two options as I see them: + a) once clients in 0.2.2.x know how to read the value out of the + consensus, and it's been tested for a bit, backport that part to + 0.2.1.x. + b) if it's too complex to backport, just pick a number, like 101, and + backport that number. + + Clearly choice (a) is the better one if the consensus parsing part + isn't very complex. Let's shoot for that, and fall back to (b) if the + patch turns out to be so big that we reconsider. + From 903f9ef50e418f6ec1a1ab7ae8146e9d6d2a7767 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 25 Aug 2009 10:30:54 -0400 Subject: [PATCH 177/187] Clean up a couple of style issues in the socks-client branch. --- src/or/connection.c | 2 +- src/or/reasons.c | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/or/connection.c b/src/or/connection.c index ccfbe19ed5..65ca3898e6 100644 --- a/src/or/connection.c +++ b/src/or/connection.c @@ -1386,7 +1386,7 @@ connection_proxy_connect(connection_t *conn, int type) memcpy(buf + 4, &ip4addr, 4); /* addr */ buf[8] = 0; /* userid (empty) */ - connection_write_to_buf((char *)buf, sizeof buf, conn); + connection_write_to_buf((char *)buf, sizeof(buf), conn); conn->proxy_state = PROXY_SOCKS4_WANT_CONNECT_OK; break; } diff --git a/src/or/reasons.c b/src/or/reasons.c index 5efb08bb41..78a16af10e 100644 --- a/src/or/reasons.c +++ b/src/or/reasons.c @@ -369,3 +369,4 @@ socks5_response_code_to_string(uint8_t code) return "unknown reason"; } } + From c9203749a2ead39433fa4f62452a2360e0901e42 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 26 Aug 2009 11:34:45 -0400 Subject: [PATCH 178/187] A changelog entry and a bit more documentation for socks-client --- ChangeLog | 3 +++ src/or/config.c | 3 +++ src/or/or.h | 16 ++++++++-------- src/or/reasons.c | 2 ++ 4 files changed, 16 insertions(+), 8 deletions(-) diff --git a/ChangeLog b/ChangeLog index 2428bed996..5b169956d3 100644 --- a/ChangeLog +++ b/ChangeLog @@ -2,6 +2,9 @@ Changes in version 0.2.2.1-alpha - 2009-0?-?? o Major features: - Add support for dynamic OpenSSL hardware crypto acceleration engines via new AccelName and AccelDir options. + - Tor now supports tunneling all of its outgoing connections over a SOCKS + proxy, using the SOCKS4Proxy and/or SOCKS5Proxy configuration options. + Code by Christopher Davis. o New options for gathering stats safely: - Directories that configure with --enable-dirreq-stats and set diff --git a/src/or/config.c b/src/or/config.c index c5a5b946fe..7944aa1001 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -3533,6 +3533,9 @@ options_validate(or_options_t *old_options, or_options_t *options, } } + if (options->Socks4Proxy && options->Socks5Proxy) + REJECT("You cannot specify both Socks4Proxy and SOCKS5Proxy"); + if (options->Socks5ProxyUsername) { size_t len; diff --git a/src/or/or.h b/src/or/or.h index df57f3048d..1aa40af61b 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -2392,15 +2392,15 @@ typedef struct { uint16_t HttpsProxyPort; /**< Parsed port for https proxy, if any. */ char *HttpsProxyAuthenticator; /**< username:password string, if any. */ - char *Socks4Proxy; - tor_addr_t Socks4ProxyAddr; - uint16_t Socks4ProxyPort; + char *Socks4Proxy; /**< hostname:port to use as a SOCKS4 proxy, if any. */ + tor_addr_t Socks4ProxyAddr; /**< Derived from Socks4Proxy. */ + uint16_t Socks4ProxyPort; /**< Derived from Socks4Proxy. */ - char *Socks5Proxy; - tor_addr_t Socks5ProxyAddr; - uint16_t Socks5ProxyPort; - char *Socks5ProxyUsername; - char *Socks5ProxyPassword; + char *Socks5Proxy; /**< hostname:port to use as a SOCKS5 proxy, if any. */ + tor_addr_t Socks5ProxyAddr; /**< Derived from Sock5Proxy. */ + uint16_t Socks5ProxyPort; /**< Derived from Socks5Proxy. */ + char *Socks5ProxyUsername; /**< Username for SOCKS5 authentication, if any */ + char *Socks5ProxyPassword; /**< Password for SOCKS5 authentication, if any */ /** List of configuration lines for replacement directory authorities. * If you just want to replace one class of authority at a time, diff --git a/src/or/reasons.c b/src/or/reasons.c index 78a16af10e..0544990679 100644 --- a/src/or/reasons.c +++ b/src/or/reasons.c @@ -326,6 +326,7 @@ circuit_end_reason_to_control_string(int reason) } } +/** Return a string corresponding to a SOCKS4 reponse code. */ const char * socks4_response_code_to_string(uint8_t code) { @@ -343,6 +344,7 @@ socks4_response_code_to_string(uint8_t code) } } +/** Return a string corresponding to a SOCKS5 reponse code. */ const char * socks5_response_code_to_string(uint8_t code) { From b47a27b29aef18e86d75942453da95a4cf1d0e0f Mon Sep 17 00:00:00 2001 From: Roger Dingledine Date: Wed, 26 Aug 2009 13:58:19 -0400 Subject: [PATCH 179/187] clean up the changelog for 0.2.2.1-alpha --- ChangeLog | 40 ++++++++++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/ChangeLog b/ChangeLog index 6208f23baf..0a85857c69 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,10 +1,22 @@ -Changes in version 0.2.2.1-alpha - 2009-0?-?? +Changes in version 0.2.2.1-alpha - 2009-08-26 + o New directory authorities: + - Set up urras (run by Jacob Appelbaum) as the seventh v3 directory + authority. + o Major features: - - Add support for dynamic OpenSSL hardware crypto acceleration engines - via new AccelName and AccelDir options. - - Tor now supports tunneling all of its outgoing connections over a SOCKS - proxy, using the SOCKS4Proxy and/or SOCKS5Proxy configuration options. - Code by Christopher Davis. + - New AccelName and AccelDir options add support for dynamic OpenSSL + hardware crypto acceleration engines. + - Tor now supports tunneling all of its outgoing connections over + a SOCKS proxy, using the SOCKS4Proxy and/or SOCKS5Proxy + configuration options. Code by Christopher Davis. + + o Major bugfixes: + - Send circuit or stream sendme cells when our window has decreased + by 100 cells, not when it has decreased by 101 cells. Bug uncovered + by Karsten when testing the "reduce circuit window" performance + patch. Bugfix on the 54th commit on Tor -- from July 2002, + before the release of Tor 0.0.0. This is the new winner of the + oldest-bug prize. o New options for gathering stats safely: - Directories that set "DirReqStatistics 1" write statistics on @@ -31,10 +43,10 @@ Changes in version 0.2.2.1-alpha - 2009-0?-?? source files Tor was built with. - The "torify" script now uses torsocks where available. - The memarea code now uses a sentinel value at the end of each area - to make sure nothing writes beyond the end of an area. This might + to make sure nothing writes beyond the end of an area. This might help debug some conceivable causes of bug 930. - Time and memory units in the configuration file can now be set to - fractional units. For example, "2.5 MB" is now a valid value for + fractional units. For example, "2.5 GB" is now a valid value for AccountingMax. - Certain Tor clients (such as those behind check.torproject.org) may want to fetch the consensus in an extra early manner. To enable this @@ -50,18 +62,22 @@ Changes in version 0.2.2.1-alpha - 2009-0?-?? notice the failure so we don't retry indefinitely. Bugfix on 0.2.0.x; reported by "rotator". - If the cached cert file is unparseable, warn but don't exit. + - Fix possible segmentation fault on directory authorities. Bugfix on + 0.2.1.14-rc. o Deprecated and removed features: - The controller no longer accepts the old obsolete "addr-mappings/" or "unregistered-servers-" GETINFO values. - Hidden services no longer publish version 0 descriptors, and clients - do not request or use version 0 descriptors. However, the authorities - still accept and serve version 0 descriptors when contacted by older - hidden services/clients. + do not request or use version 0 descriptors. However, the old hidden + service authorities still accept and serve version 0 descriptors + when contacted by older hidden services/clients. - The EXTENDED_EVENTS and VERBOSE_NAMES controller features are now always on; using them is necessary for correct forward-compatible controllers. - - Removal of support for .noconnect style addresses. + - Remove support for .noconnect style addresses. Nobody was using + them, and they provided another avenue for detecting Tor users + via application-level web tricks. Changes in version 0.2.1.20 - 2009-??-?? From 76108dce3fc73ea47cf965c7cdee312e95a5b2de Mon Sep 17 00:00:00 2001 From: Roger Dingledine Date: Wed, 26 Aug 2009 14:47:29 -0400 Subject: [PATCH 180/187] mark off a done proposal --- doc/TODO.022 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/TODO.022 b/doc/TODO.022 index 8bf50659fb..f4fe2ebb2a 100644 --- a/doc/TODO.022 +++ b/doc/TODO.022 @@ -58,10 +58,10 @@ NOTE 2: It's easy to list stuff like this with no time estimates and - 158: microdescriptors o Revise proposal - Implement - - 160: list bandwidth in consensus -RNM? . Finish proposal - - and actually set it reasonably - - and actually use it. + o 160: list bandwidth in consensus + o Finish proposal + o and actually set it reasonably + o and actually use it. - Proposals to improve and implement if not broken D IPv6 support. (Parts of 117, but figure out how to handle DNS From 5965a85ce05bb507f669a8de5609b7b3a649fb4b Mon Sep 17 00:00:00 2001 From: Roger Dingledine Date: Wed, 26 Aug 2009 14:48:13 -0400 Subject: [PATCH 181/187] typos in dir-spec --- doc/spec/dir-spec.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/spec/dir-spec.txt b/doc/spec/dir-spec.txt index 483a33b73c..16f121a19a 100644 --- a/doc/spec/dir-spec.txt +++ b/doc/spec/dir-spec.txt @@ -593,7 +593,7 @@ "allow-single-hop-exits" - [At most one.] + [At most once.] Present only if the router allows single-hop circuits to make exit connections. Most Tor servers do not support this: this is @@ -991,7 +991,7 @@ documents are described in section XXX below. Status documents contain a preamble, an authority section, a list of - router status entries, and one more footers signature, in that order. + router status entries, and one or more footer signature, in that order. Unlike other formats described above, a SP in these documents must be a single space character (hex 20). From 5308eceef3e7287f77d7b10065c9f8341c5dd75b Mon Sep 17 00:00:00 2001 From: Roger Dingledine Date: Wed, 26 Aug 2009 15:01:43 -0400 Subject: [PATCH 182/187] bump to 0.2.2.1-alpha --- configure.in | 2 +- contrib/tor-mingw.nsi.in | 2 +- src/win32/orconfig.h | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/configure.in b/configure.in index 1ac2cdb721..41d8b2d0e4 100644 --- a/configure.in +++ b/configure.in @@ -4,7 +4,7 @@ dnl Copyright (c) 2007-2008, The Tor Project, Inc. dnl See LICENSE for licensing information AC_INIT -AM_INIT_AUTOMAKE(tor, 0.2.2.0-alpha-dev) +AM_INIT_AUTOMAKE(tor, 0.2.2.1-alpha) AM_CONFIG_HEADER(orconfig.h) AC_CANONICAL_HOST diff --git a/contrib/tor-mingw.nsi.in b/contrib/tor-mingw.nsi.in index 10a9e353e1..9dcbc6bac5 100644 --- a/contrib/tor-mingw.nsi.in +++ b/contrib/tor-mingw.nsi.in @@ -9,7 +9,7 @@ !include "FileFunc.nsh" !insertmacro GetParameters -!define VERSION "0.2.2.0-alpha-dev" +!define VERSION "0.2.2.1-alpha" !define INSTALLER "tor-${VERSION}-win32.exe" !define WEBSITE "https://www.torproject.org/" !define LICENSE "LICENSE" diff --git a/src/win32/orconfig.h b/src/win32/orconfig.h index 563d79548d..aced487827 100644 --- a/src/win32/orconfig.h +++ b/src/win32/orconfig.h @@ -226,5 +226,5 @@ #define USING_TWOS_COMPLEMENT /* Version number of package */ -#define VERSION "0.2.2.0-alpha-dev" +#define VERSION "0.2.2.1-alpha" From b7e8a4631fecc3b3e241780bf1d735683562fd97 Mon Sep 17 00:00:00 2001 From: Roger Dingledine Date: Wed, 26 Aug 2009 15:43:18 -0400 Subject: [PATCH 183/187] changelog and spec changes for the .exit fix --- ChangeLog | 6 ++++++ doc/spec/address-spec.txt | 5 ++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 0a85857c69..a3d76ca4ce 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,10 @@ Changes in version 0.2.2.1-alpha - 2009-08-26 + o Security fixes: + - Start the process of disabling ".exit" address notation, since it + can be used for a variety of esoteric application-level attacks + on users. To reenable it, set "AllowDotExit 1" in your torrc. Fix + on 0.0.9rc5. + o New directory authorities: - Set up urras (run by Jacob Appelbaum) as the seventh v3 directory authority. diff --git a/doc/spec/address-spec.txt b/doc/spec/address-spec.txt index fdae9b82a4..2e1aff2b8a 100644 --- a/doc/spec/address-spec.txt +++ b/doc/spec/address-spec.txt @@ -33,10 +33,13 @@ "www.google.com.foo.exit=64.233.161.99.foo.exit" to speed subsequent lookups. + The .exit notation is disabled by default as of Tor 0.2.2.1-alpha, due + to potential application-level attacks. + EXAMPLES: www.example.com.exampletornode.exit - Connect to www.example.com from the node called "exampletornode." + Connect to www.example.com from the node called "exampletornode". exampletornode.exit From b2b25cf3a2d5db6c7c1da37923c34805dcd1dc21 Mon Sep 17 00:00:00 2001 From: Andrew Lewman Date: Thu, 13 Aug 2009 21:13:09 -0400 Subject: [PATCH 184/187] update fetch-all with dir auth --- contrib/directory-archive/fetch-all | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/directory-archive/fetch-all b/contrib/directory-archive/fetch-all index fabcbb8bc4..dfa5a1b3e5 100755 --- a/contrib/directory-archive/fetch-all +++ b/contrib/directory-archive/fetch-all @@ -30,8 +30,8 @@ DIRSERVERS="" DIRSERVERS="$DIRSERVERS 86.59.21.38:80" # tor26 DIRSERVERS="$DIRSERVERS 128.31.0.34:9031" # moria1 DIRSERVERS="$DIRSERVERS 128.31.0.34:9032" # moria2 -#DIRSERVERS="$DIRSERVERS 140.247.60.64:80" # lefkada DIRSERVERS="$DIRSERVERS 194.109.206.212:80" # dizum + DATEDIR=$(date "+%Y/%m/%d") TIME=$(date "+%Y%m%d-%H%M%S") From 889c07f1fc54b5bd60c7919f8a5fc784eed2d57a Mon Sep 17 00:00:00 2001 From: Karsten Loesing Date: Thu, 20 Aug 2009 18:45:03 +0200 Subject: [PATCH 185/187] When Tor fails to parse a descriptor of any kind, dump it to disk. --- ChangeLog | 2 ++ src/or/routerparse.c | 50 ++++++++++++++++++++++++++++++++++++++------ 2 files changed, 46 insertions(+), 6 deletions(-) diff --git a/ChangeLog b/ChangeLog index a3d76ca4ce..0182faa581 100644 --- a/ChangeLog +++ b/ChangeLog @@ -103,6 +103,8 @@ Changes in version 0.2.1.20 - 2009-??-?? - Fix a signed/unsigned compile warning in 0.2.1.19. - Fix possible segmentation fault on directory authorities. Bugfix on 0.2.1.14-rc. + - When Tor fails to parse a descriptor of any kind, dump it to disk. + Might help diagnosing bug 1051. Changes in version 0.2.1.19 - 2009-07-28 diff --git a/src/or/routerparse.c b/src/or/routerparse.c index 8fecd9ab47..9e5f37b4bd 100644 --- a/src/or/routerparse.c +++ b/src/or/routerparse.c @@ -545,6 +545,34 @@ static int tor_version_same_series(tor_version_t *a, tor_version_t *b); #define DUMP_AREA(a,name) STMT_NIL #endif +/** Last time we dumped a descriptor to disk. */ +static time_t last_desc_dumped = 0; + +/** For debugging purposes, dump unparseable descriptor *desc of + * type *type to file $DATADIR/unparseable-desc. Do not write more + * than one descriptor to disk per minute. If there is already such a + * file in the data directory, overwrite it. */ +static void +dump_desc(const char *desc, const char *type) +{ + time_t now = time(NULL); + tor_assert(desc); + tor_assert(type); + if (!last_desc_dumped || last_desc_dumped + 60 < now) { + char *debugfile = get_datadir_fname("unparseable-desc"); + size_t filelen = 50 + strlen(type) + strlen(desc); + char *content = tor_malloc_zero(filelen); + tor_snprintf(content, filelen, "Unable to parse descriptor of type " + "%s:\n%s", type, desc); + write_str_to_file(debugfile, content, 0); + log_info(LD_DIR, "Unable to parse descriptor of type %s. See file " + "unparseable-desc in data directory for details.", type); + tor_free(content); + tor_free(debugfile); + last_desc_dumped = now; + } +} + /** Set digest to the SHA-1 digest of the hash of the directory in * s. Return 0 on success, -1 on failure. */ @@ -734,7 +762,7 @@ router_parse_directory(const char *str) char digest[DIGEST_LEN]; time_t published_on; int r; - const char *end, *cp; + const char *end, *cp, *str_dup = str; smartlist_t *tokens = NULL; crypto_pk_env_t *declared_key = NULL; memarea_t *area = memarea_new(); @@ -807,6 +835,7 @@ router_parse_directory(const char *str) r = 0; goto done; err: + dump_desc(str_dup, "v1 directory"); r = -1; done: if (declared_key) crypto_free_pk_env(declared_key); @@ -833,7 +862,7 @@ router_parse_runningrouters(const char *str) int r = -1; crypto_pk_env_t *declared_key = NULL; smartlist_t *tokens = NULL; - const char *eos = str + strlen(str); + const char *eos = str + strlen(str), *str_dup = str; memarea_t *area = NULL; if (router_get_runningrouters_hash(str, digest)) { @@ -874,6 +903,7 @@ router_parse_runningrouters(const char *str) r = 0; err: + dump_desc(str_dup, "v1 running-routers"); if (declared_key) crypto_free_pk_env(declared_key); if (tokens) { SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); @@ -1183,7 +1213,7 @@ router_parse_entry_from_string(const char *s, const char *end, smartlist_t *tokens = NULL, *exit_policy_tokens = NULL; directory_token_t *tok; struct in_addr in; - const char *start_of_annotations, *cp; + const char *start_of_annotations, *cp, *s_dup = s; size_t prepend_len = prepend_annotations ? strlen(prepend_annotations) : 0; int ok = 1; memarea_t *area = NULL; @@ -1473,6 +1503,7 @@ router_parse_entry_from_string(const char *s, const char *end, goto done; err: + dump_desc(s_dup, "router descriptor"); routerinfo_free(router); router = NULL; done: @@ -1507,6 +1538,7 @@ extrainfo_parse_entry_from_string(const char *s, const char *end, crypto_pk_env_t *key = NULL; routerinfo_t *router = NULL; memarea_t *area = NULL; + const char *s_dup = s; if (!end) { end = s + strlen(s); @@ -1595,6 +1627,7 @@ extrainfo_parse_entry_from_string(const char *s, const char *end, goto done; err: + dump_desc(s_dup, "extra-info descriptor"); if (extrainfo) extrainfo_free(extrainfo); extrainfo = NULL; @@ -1624,6 +1657,7 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string) size_t len; int found; memarea_t *area = NULL; + const char *s_dup = s; s = eat_whitespace(s); eos = strstr(s, "\ndir-key-certification"); @@ -1777,6 +1811,7 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string) } return cert; err: + dump_desc(s_dup, "authority cert"); authority_cert_free(cert); SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); smartlist_free(tokens); @@ -1827,7 +1862,7 @@ routerstatus_parse_entry_from_string(memarea_t *area, vote_routerstatus_t *vote_rs, int consensus_method) { - const char *eos; + const char *eos, *s_dup = *s; routerstatus_t *rs = NULL; directory_token_t *tok; char timebuf[ISO_TIME_LEN+1]; @@ -2010,6 +2045,7 @@ routerstatus_parse_entry_from_string(memarea_t *area, goto done; err: + dump_desc(s_dup, "routerstatus entry"); if (rs && !vote_rs) routerstatus_free(rs); rs = NULL; @@ -2051,7 +2087,7 @@ _free_duplicate_routerstatus_entry(void *e) networkstatus_v2_t * networkstatus_v2_parse_from_string(const char *s) { - const char *eos; + const char *eos, *s_dup = s; smartlist_t *tokens = smartlist_create(); smartlist_t *footer_tokens = smartlist_create(); networkstatus_v2_t *ns = NULL; @@ -2200,6 +2236,7 @@ networkstatus_v2_parse_from_string(const char *s) goto done; err: + dump_desc(s_dup, "v2 networkstatus"); if (ns) networkstatus_v2_free(ns); ns = NULL; @@ -2226,7 +2263,7 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, networkstatus_voter_info_t *voter = NULL; networkstatus_t *ns = NULL; char ns_digest[DIGEST_LEN]; - const char *cert, *end_of_header, *end_of_footer; + const char *cert, *end_of_header, *end_of_footer, *s_dup = s; directory_token_t *tok; int ok; struct in_addr in; @@ -2584,6 +2621,7 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, goto done; err: + dump_desc(s_dup, "v3 networkstatus"); if (ns) networkstatus_vote_free(ns); ns = NULL; From 15462f72eb6c442c33d5d6be27da0802c01e6f5e Mon Sep 17 00:00:00 2001 From: Roger Dingledine Date: Wed, 26 Aug 2009 20:20:09 -0400 Subject: [PATCH 186/187] put karsten's changelog in the right place --- ChangeLog | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index 0182faa581..733e222253 100644 --- a/ChangeLog +++ b/ChangeLog @@ -70,6 +70,8 @@ Changes in version 0.2.2.1-alpha - 2009-08-26 - If the cached cert file is unparseable, warn but don't exit. - Fix possible segmentation fault on directory authorities. Bugfix on 0.2.1.14-rc. + - When Tor fails to parse a descriptor of any kind, dump it to disk. + Might help diagnosing bug 1051. o Deprecated and removed features: - The controller no longer accepts the old obsolete "addr-mappings/" @@ -103,8 +105,6 @@ Changes in version 0.2.1.20 - 2009-??-?? - Fix a signed/unsigned compile warning in 0.2.1.19. - Fix possible segmentation fault on directory authorities. Bugfix on 0.2.1.14-rc. - - When Tor fails to parse a descriptor of any kind, dump it to disk. - Might help diagnosing bug 1051. Changes in version 0.2.1.19 - 2009-07-28 From 1092fdca53ec0110b4e4c0bf51a5881ee68c4ac8 Mon Sep 17 00:00:00 2001 From: Sebastian Hahn Date: Thu, 27 Aug 2009 05:10:48 +0200 Subject: [PATCH 187/187] HiddenServiceVersion must be set to 2 currently. 0d68da2381780e2f8010b52a9e062947554699d5 removed support for Version 0, but didn't fix the manpage. --- doc/tor.1.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/tor.1.in b/doc/tor.1.in index ba703079c8..a0f8e8b0f6 100644 --- a/doc/tor.1.in +++ b/doc/tor.1.in @@ -1,4 +1,4 @@ -.TH TOR 1 "January 2009" "TOR" +.TH TOR 1 "August 2009" "TOR" .SH NAME tor \- The second-generation onion router .SH SYNOPSIS @@ -1369,7 +1369,7 @@ if you're using a Tor controller that handles hidserv publishing for you. .TP \fBHiddenServiceVersion \fR\fIversion\fR,\fIversion\fR,\fI...\fP A list of rendezvous service descriptor versions to publish for the hidden -service. Possible version numbers are 0 and 2. (Default: 0, 2) +service. Currently, only version 2 is supported. (Default: 2) .LP .TP \fBHiddenServiceAuthorizeClient \fR\fIauth-type\fR \fR\fIclient-name\fR,\fIclient-name\fR,\fI...\fP