diff --git a/ChangeLog b/ChangeLog index 436116bd7f..d368871079 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,50 @@ +Changes in version 0.2.1.17-rc - 2009-07-07 + o Major features: + - Clients now use the bandwidth values in the consensus, rather than + the bandwidth values in each relay descriptor. This approach opens + the door to more accurate bandwidth estimates once the directory + authorities start doing active measurements. Implements more of + proposal 141. + + o Major bugfixes: + - When Tor clients restart after 1-5 days, they discard all their + cached descriptors as too old, but they still use the cached + consensus document. This approach is good for robustness, but + bad for performance: since they don't know any bandwidths, they + end up choosing at random rather than weighting their choice by + speed. Fixed by the above feature of putting bandwidths in the + consensus. Bugfix on 0.2.0.x. + - Directory authorities were neglecting to mark relays down in their + internal histories if the relays fall off the routerlist without + ever being found unreachable. So there were relays in the histories + that haven't been seen for eight months, and are listed as being + up for eight months. This wreaked havoc on the "median wfu" + and "median mtbf" calculations, in turn making Guard and Stable + flags very wrong, hurting network performance. Fixes bugs 696 and + 969. Bugfix on 0.2.0.6-alpha. + + o Minor bugfixes: + - Serve the DirPortFrontPage page even when we have been approaching + our quotas recently. Fixes bug 1013; bugfix on 0.2.1.8-alpha. + - The control port would close the connection before flushing long + replies, such as the network consensus, if a QUIT command was issued + before the reply had completed. Now, the control port flushes all + pending replies before closing the connection. Also fixed a spurious + warning when a QUIT command is issued after a malformed or rejected + AUTHENTICATE command, but before the connection was closed. Patch + by Marcus Griep. Bugfix on 0.2.0.x; fixes bugs 1015 and 1016. + - When we can't find an intro key for a v2 hidden service descriptor, + fall back to the v0 hidden service descriptor and log a bug message. + Workaround for bug 1024. + - Fix a log message that did not respect the SafeLogging option. + Resolves bug 1027. + + o Minor features: + - If we're a relay and we change our IP address, be more verbose + about the reason that made us change. Should help track down + further bugs for relays on dynamic IP addresses. + + Changes in version 0.2.1.16-rc - 2009-06-20 o Security fixes: - Fix an edge case where a malicious exit relay could convince a diff --git a/LICENSE b/LICENSE index 197c92a974..bb1ebd4e48 100644 --- a/LICENSE +++ b/LICENSE @@ -14,7 +14,7 @@ Tor is distributed under this license: Copyright (c) 2001-2004, Roger Dingledine Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson -Copyright (c) 2007-2008, The Tor Project, Inc. +Copyright (c) 2007-2009, The Tor Project, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/Makefile.am b/Makefile.am index b6e559ef0c..ec8b4f1a66 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1,7 +1,7 @@ # $Id$ # Copyright (c) 2001-2004, Roger Dingledine # Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson -# Copyright (c) 2007-2008, The Tor Project, Inc. +# Copyright (c) 2007-2009, The Tor Project, Inc. # See LICENSE for licensing information AUTOMAKE_OPTIONS = foreign diff --git a/configure.in b/configure.in index 37bccce965..e704249457 100644 --- a/configure.in +++ b/configure.in @@ -5,7 +5,7 @@ dnl Copyright (c) 2007-2008, The Tor Project, Inc. dnl See LICENSE for licensing information AC_INIT -AM_INIT_AUTOMAKE(tor, 0.2.1.16-rc) +AM_INIT_AUTOMAKE(tor, 0.2.1.17-rc) AM_CONFIG_HEADER(orconfig.h) AC_CANONICAL_HOST diff --git a/contrib/tor-mingw.nsi.in b/contrib/tor-mingw.nsi.in index d65347ebe7..743e915a57 100644 --- a/contrib/tor-mingw.nsi.in +++ b/contrib/tor-mingw.nsi.in @@ -9,7 +9,7 @@ !include "FileFunc.nsh" !insertmacro GetParameters -!define VERSION "0.2.1.16-rc" +!define VERSION "0.2.1.17-rc" !define INSTALLER "tor-${VERSION}-win32.exe" !define WEBSITE "https://www.torproject.org/" !define LICENSE "LICENSE" diff --git a/doc/tor-rpm-creation.txt b/doc/tor-rpm-creation.txt index a8280fe367..3fc6bde79c 100644 --- a/doc/tor-rpm-creation.txt +++ b/doc/tor-rpm-creation.txt @@ -2,60 +2,25 @@ ## The process used to create the official rpms is as follows: -Download and Extract the latest tor source code from https://www.torproject.org/. +Download latest stable libevent from +http://www.monkey.org/~provos/libevent/ + +The first step of compiling libevent is to configure it as follows: +./configure --enable-static --disable-shared + +Complete the "make" and "make install". You will need to be root, +or sudo -s, to complete the "make install". + +Check for a successful universal binary of libevent.a in, by default, +/usr/local/lib by using the following command: + "file /usr/local/lib/libevent.a" + +Download and Extract the latest tor source code from +https://www.torproject.org/download + In the resulting directory: -./configure +LIBS=-lrt ./configure make dist-rpm You should have at least two, maybe three, rpms. There should be the binary i386.rpm, a src.rpm, and on redhat/centos machines, a debuginfo.rpm. - -## Optional customization -## -If you wish to further tune Tor binaries in rpm format beyond this list, -see the GCC doc page for further options: -http://gcc.gnu.org/onlinedocs/gcc-4.0.2/gcc/ - -The tor.spec.in file contains the basic info needed to tune the binaries -produced in rpm format. The key parameters to tune are located in the -third section of the tor.spec.in file. Locate the section similar to -this: - -## Target a specific arch and OS -# -# default is i386 linux -%define target gnu -%define target_cpu i386 -%define target_os linux - -The three parameters: target, target_cpu, and target_os are used -throughout the "make dist-rpm" process. They control the parameters -passed to "configure" and the final tuning of the binaries produced. -The default settings, as shown above, create binaries for the widest -range of Intel x86 or x86-compatible architectures. - -The parameters can be set as follows: - -The "target" parameter: -This should be "gnu", "redhat", or the short name of your linux distribution. -Other possibilities are "mandrake" or "suse". This is passed to -"configure" through the --host, --build, and --target parameters. -Therefore, this "target" parameter must be a valid OS for "configure" as -well. - -The "target_cpu" parameter: -This parameter controls the optimization and tuning of your binaries via -gcc and "configure". This parameter is passed to gcc via the -mtune= or --mcpu= options. The "configure" script will also receive this parameter -through the --host, --build, and --target parameters. Therefore, this -"target_cpu" parameter must be valid for both gcc and "configure". A -few common options for this parameter may be "athlon64, i686, pentium4" or -others. - -The "target_os" parameter: -This parameter controls the target operating system. Normally, this is -only "linux". If you wish to build rpms for a non-linux operating -system, you can replace "linux" with your operating system. - - - diff --git a/src/or/config.c b/src/or/config.c index 81999516dd..b744f8faf4 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -1381,6 +1381,13 @@ options_act(or_options_t *old_options) geoip_load_file(actual_fname, options); tor_free(actual_fname); } +#ifdef ENABLE_GEOIP_STATS + log_warn(LD_CONFIG, "We are configured to measure GeoIP statistics, but " + "the way these statistics are measured has changed " + "significantly in later versions of Tor. The results may not be " + "as expected if you are used to later versions. Be sure you " + "know what you are doing."); +#endif /* Check if we need to parse and add the EntryNodes config option. */ if (options->EntryNodes && (!old_options || diff --git a/src/or/connection_edge.c b/src/or/connection_edge.c index 1ef87dbffa..d699591cdc 100644 --- a/src/or/connection_edge.c +++ b/src/or/connection_edge.c @@ -1484,7 +1484,8 @@ connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, addresstype = parse_extended_hostname(socks->address); if (addresstype == BAD_HOSTNAME) { - log_warn(LD_APP, "Invalid hostname %s; rejecting", socks->address); + log_warn(LD_APP, "Invalid onion hostname %s; rejecting", + safe_str(socks->address)); control_event_client_status(LOG_WARN, "SOCKS_BAD_HOSTNAME HOSTNAME=%s", escaped(socks->address)); connection_mark_unattached_ap(conn, END_STREAM_REASON_TORPROTOCOL); diff --git a/src/or/control.c b/src/or/control.c index 486ccc4c75..90c99fd51a 100644 --- a/src/or/control.c +++ b/src/or/control.c @@ -2888,10 +2888,16 @@ connection_control_process_inbuf(control_connection_t *conn) --data_len; } - /* Quit is always valid. */ + /* If the connection is already closing, ignore further commands */ + if (TO_CONN(conn)->marked_for_close) { + return 0; + } + + /* Otherwise, Quit is always valid. */ if (!strcasecmp(conn->incoming_cmd, "QUIT")) { connection_write_str_to_buf("250 closing connection\r\n", conn); connection_mark_for_close(TO_CONN(conn)); + conn->_base.hold_open_until_flushed = 1; return 0; } diff --git a/src/or/directory.c b/src/or/directory.c index d783772eac..4ab2633022 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -2428,13 +2428,9 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, dlen = strlen(frontpage); /* Let's return a disclaimer page (users shouldn't use V1 anymore, and caches don't fetch '/', so this is safe). */ - if (global_write_bucket_low(TO_CONN(conn), dlen, 1)) { - log_info(LD_DIRSERV, - "Client asked for DirPortFrontPage content, but we've been " - "writing too many bytes lately. Sending 503 Dir busy."); - write_http_status_line(conn, 503, "Directory busy, try again later"); - goto done; - } + + /* [We don't check for write_bucket_low here, since we want to serve + * this page no matter what.] */ note_request(url, dlen); write_http_response_header_impl(conn, dlen, "text/html", "identity", NULL, DIRPORTFRONTPAGE_CACHE_LIFETIME); diff --git a/src/or/dirserv.c b/src/or/dirserv.c index 88afe9fccc..a64a01bb80 100644 --- a/src/or/dirserv.c +++ b/src/or/dirserv.c @@ -797,7 +797,7 @@ directory_remove_invalid(void) if (r & FP_REJECT) { log_info(LD_DIRSERV, "Router '%s' is now rejected: %s", ent->nickname, msg?msg:""); - routerlist_remove(rl, ent, 0); + routerlist_remove(rl, ent, 0, time(NULL)); i--; changed = 1; continue; @@ -951,8 +951,8 @@ dirserv_set_router_is_running(routerinfo_t *router, time_t now) answer = get_options()->AssumeReachable || now < router->last_reachable + REACHABLE_TIMEOUT; - if (router->is_running && !answer) { - /* it was running but now it's not. tell rephist. */ + if (!answer) { + /* not considered reachable. tell rephist. */ rep_hist_note_router_unreachable(router->cache_info.identity_digest, now); } diff --git a/src/or/eventdns.c b/src/or/eventdns.c index 4464148141..4ae17a40c3 100644 --- a/src/or/eventdns.c +++ b/src/or/eventdns.c @@ -1842,7 +1842,7 @@ evdns_server_request_format_response(struct server_request *req, int err) if (j > 512) { overflow: j = 512; - buf[3] |= 0x02; /* set the truncated bit. */ + buf[2] |= 0x02; /* set the truncated bit. */ } req->response_len = (size_t)j; diff --git a/src/or/main.c b/src/or/main.c index 8fc712bba3..60c42aaae3 100644 --- a/src/or/main.c +++ b/src/or/main.c @@ -903,7 +903,7 @@ run_scheduled_events(time_t now) time_to_downrate_stability = rep_hist_downrate_old_runs(now); if (authdir_mode_tests_reachability(options)) { if (time_to_save_stability < now) { - if (time_to_save_stability && rep_hist_record_mtbf_data()<0) { + if (time_to_save_stability && rep_hist_record_mtbf_data(now, 1)<0) { log_warn(LD_GENERAL, "Couldn't store mtbf data."); } #define SAVE_STABILITY_INTERVAL (30*60) @@ -1955,14 +1955,15 @@ tor_cleanup(void) /* Remove our pid file. We don't care if there was an error when we * unlink, nothing we could do about it anyways. */ if (options->command == CMD_RUN_TOR) { + time_t now = time(NULL); if (options->PidFile) unlink(options->PidFile); if (accounting_is_enabled(options)) - accounting_record_bandwidth_usage(time(NULL), get_or_state()); + accounting_record_bandwidth_usage(now, get_or_state()); or_state_mark_dirty(get_or_state(), 0); /* force an immediate save. */ - or_state_save(time(NULL)); + or_state_save(now); if (authdir_mode_tests_reachability(options)) - rep_hist_record_mtbf_data(); + rep_hist_record_mtbf_data(now, 0); } #ifdef USE_DMALLOC dmalloc_log_stats(); diff --git a/src/or/or.h b/src/or/or.h index f84485f2f4..eddeda1531 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -3970,7 +3970,7 @@ void rep_history_clean(time_t before); void rep_hist_note_router_reachable(const char *id, time_t when); void rep_hist_note_router_unreachable(const char *id, time_t when); -int rep_hist_record_mtbf_data(void); +int rep_hist_record_mtbf_data(time_t now, int missing_means_down); int rep_hist_load_mtbf_data(time_t now); time_t rep_hist_downrate_old_runs(time_t now); @@ -4411,7 +4411,8 @@ void routerinfo_free(routerinfo_t *router); void extrainfo_free(extrainfo_t *extrainfo); void routerlist_free(routerlist_t *rl); void dump_routerlist_mem_usage(int severity); -void routerlist_remove(routerlist_t *rl, routerinfo_t *ri, int make_old); +void routerlist_remove(routerlist_t *rl, routerinfo_t *ri, int make_old, + time_t now); void routerlist_free_all(void); void routerlist_reset_warnings(void); void router_set_status(const char *digest, int up); diff --git a/src/or/rendclient.c b/src/or/rendclient.c index 5b18a519b0..a5d7c1016e 100644 --- a/src/or/rendclient.c +++ b/src/or/rendclient.c @@ -94,8 +94,24 @@ rend_client_send_introduction(origin_circuit_t *introcirc, } }); if (!intro_key) { - log_warn(LD_BUG, "Internal error: could not find intro key."); - goto err; + if (rend_cache_lookup_entry(introcirc->rend_data->onion_address, + 0, &entry) > 0) { + log_warn(LD_BUG, "We have both a v0 and a v2 rend desc for this " + "service. The v2 desc doesn't contain the introduction " + "point (and key) to send an INTRODUCE1/2 cell to this " + "introduction point. Assuming the introduction point " + "is for v0 rend clients and using the service key " + "from the v0 desc instead. (This is probably a bug, " + "because we shouldn't even have both a v0 and a v2 " + "descriptor for the same service.)"); + /* See flyspray task 1024. */ + intro_key = entry->parsed->pk; + } else { + log_warn(LD_BUG, "Internal error: could not find intro key; we " + "only have a v2 rend desc with %d intro points.", + smartlist_len(entry->parsed->intro_nodes)); + goto err; + } } } if (crypto_pk_get_digest(intro_key, payload)<0) { diff --git a/src/or/rephist.c b/src/or/rephist.c index 11e040c945..13fdb58b5e 100644 --- a/src/or/rephist.c +++ b/src/or/rephist.c @@ -683,9 +683,13 @@ rep_history_clean(time_t before) } } -/** Write MTBF data to disk. Returns 0 on success, negative on failure. */ +/** Write MTBF data to disk. Return 0 on success, negative on failure. + * + * If missing_means_down, then if we're about to write an entry + * that is still considered up but isn't in our routerlist, consider it + * to be down. */ int -rep_hist_record_mtbf_data(void) +rep_hist_record_mtbf_data(time_t now, int missing_means_down) { char time_buf[ISO_TIME_LEN+1]; @@ -745,6 +749,18 @@ rep_hist_record_mtbf_data(void) hist = (or_history_t*) or_history_p; base16_encode(dbuf, sizeof(dbuf), digest, DIGEST_LEN); + + if (missing_means_down && hist->start_of_run && + !router_get_by_digest(digest)) { + /* We think this relay is running, but it's not listed in our + * routerlist. Somehow it fell out without telling us it went + * down. Complain and also correct it. */ + log_info(LD_HIST, + "Relay '%s' is listed as up in rephist, but it's not in " + "our routerlist. Correcting.", dbuf); + rep_hist_note_router_unreachable(digest, now); + } + PRINTF((f, "R %s\n", dbuf)); if (hist->start_of_run > 0) { format_iso_time(time_buf, hist->start_of_run); diff --git a/src/or/router.c b/src/or/router.c index 658c567db1..93afe4fad5 100644 --- a/src/or/router.c +++ b/src/or/router.c @@ -1509,8 +1509,8 @@ log_addr_has_changed(int severity, uint32_t prev, uint32_t cur, addrbuf_prev, addrbuf_cur, source); else log_notice(LD_GENERAL, - "Guessed our IP address as %s.", - addrbuf_cur); + "Guessed our IP address as %s (source: %s).", + addrbuf_cur, source); } /** Check whether our own address as defined by the Address configuration @@ -1533,7 +1533,7 @@ check_descriptor_ipaddress_changed(time_t now) } if (prev != cur) { - log_addr_has_changed(LOG_INFO, prev, cur, "resolve"); + log_addr_has_changed(LOG_NOTICE, prev, cur, "resolve"); ip_address_changed(0); } } diff --git a/src/or/routerlist.c b/src/or/routerlist.c index d8165e63bd..42b385b101 100644 --- a/src/or/routerlist.c +++ b/src/or/routerlist.c @@ -1523,15 +1523,12 @@ router_get_advertised_bandwidth_capped(routerinfo_t *router) return result; } -/** Eventually, the number we return will come from the directory - * consensus, so clients can dynamically update to better numbers. - * - * But for now, or in case there is no consensus available, just return - * a sufficient default. */ -static uint32_t -get_max_believable_bandwidth(void) +/** Return bw*1000, unless bw*1000 would overflow, in which case return + * INT32_MAX. */ +static INLINE int32_t +kb_to_bytes(uint32_t bw) { - return DEFAULT_MAX_BELIEVABLE_BANDWIDTH; + return (bw > (INT32_MAX/1000)) ? INT32_MAX : bw*1000; } /** Helper function: @@ -1568,7 +1565,6 @@ smartlist_choose_by_bandwidth(smartlist_t *sl, bandwidth_weight_rule_t rule, int n_unknown = 0; bitarray_t *exit_bits; bitarray_t *guard_bits; - uint32_t max_believable_bw = get_max_believable_bandwidth(); int me_idx = -1; /* Can't choose exit and guard at same time */ @@ -1591,48 +1587,50 @@ smartlist_choose_by_bandwidth(smartlist_t *sl, bandwidth_weight_rule_t rule, int32_t flags = 0; uint32_t this_bw = 0; if (statuses) { - /* need to extract router info */ status = smartlist_get(sl, i); if (router_digest_is_me(status->identity_digest)) me_idx = i; router = router_get_by_digest(status->identity_digest); is_exit = status->is_exit; is_guard = status->is_possible_guard; - if (router) { - this_bw = router_get_advertised_bandwidth(router); + if (status->has_bandwidth) { + this_bw = kb_to_bytes(status->bandwidth); } else { /* guess */ + /* XXX022 once consensuses always list bandwidths, we can take + * this guessing business out. -RD */ is_known = 0; flags = status->is_fast ? 1 : 0; flags |= is_exit ? 2 : 0; flags |= is_guard ? 4 : 0; } } else { + routerstatus_t *rs; router = smartlist_get(sl, i); + rs = router_get_consensus_status_by_id( + router->cache_info.identity_digest); if (router_digest_is_me(router->cache_info.identity_digest)) me_idx = i; is_exit = router->is_exit; is_guard = router->is_possible_guard; - this_bw = router_get_advertised_bandwidth(router); + if (rs && rs->has_bandwidth) { + this_bw = kb_to_bytes(rs->bandwidth); + } else if (rs) { /* guess; don't trust the descriptor */ + /* XXX022 once consensuses always list bandwidths, we can take + * this guessing business out. -RD */ + is_known = 0; + flags = router->is_fast ? 1 : 0; + flags |= is_exit ? 2 : 0; + flags |= is_guard ? 4 : 0; + } else /* bridge or other descriptor not in our consensus */ + this_bw = router_get_advertised_bandwidth_capped(router); } if (is_exit) bitarray_set(exit_bits, i); if (is_guard) bitarray_set(guard_bits, i); - /* if they claim something huge, don't believe it */ - if (this_bw > max_believable_bw) { - char fp[HEX_DIGEST_LEN+1]; - base16_encode(fp, sizeof(fp), statuses ? - status->identity_digest : - router->cache_info.identity_digest, - DIGEST_LEN); - log_fn(LOG_PROTOCOL_WARN, LD_DIR, - "Bandwidth %d for router %s (%s) exceeds allowed max %d, capping", - this_bw, router ? router->nickname : "(null)", - fp, max_believable_bw); - this_bw = max_believable_bw; - } if (is_known) { bandwidths[i] = (int32_t) this_bw; // safe since MAX_BELIEVABLE= 0); if (is_guard) total_guard_bw += this_bw; else @@ -2631,7 +2629,7 @@ routerlist_insert_old(routerlist_t *rl, routerinfo_t *ri) * If make_old is true, instead of deleting the router, we try adding * it to rl->old_routers. */ void -routerlist_remove(routerlist_t *rl, routerinfo_t *ri, int make_old) +routerlist_remove(routerlist_t *rl, routerinfo_t *ri, int make_old, time_t now) { routerinfo_t *ri_tmp; extrainfo_t *ei_tmp; @@ -2639,6 +2637,9 @@ routerlist_remove(routerlist_t *rl, routerinfo_t *ri, int make_old) tor_assert(0 <= idx && idx < smartlist_len(rl->routers)); tor_assert(smartlist_get(rl->routers, idx) == ri); + /* make sure the rephist module knows that it's not running */ + rep_hist_note_router_unreachable(ri->cache_info.identity_digest, now); + ri->cache_info.routerlist_index = -1; smartlist_del(rl->routers, idx); if (idx < smartlist_len(rl->routers)) { @@ -3330,7 +3331,7 @@ routerlist_remove_old_routers(void) log_info(LD_DIR, "Forgetting obsolete (too old) routerinfo for router '%s'", router->nickname); - routerlist_remove(routerlist, router, 1); + routerlist_remove(routerlist, router, 1, now); i--; } } diff --git a/src/win32/orconfig.h b/src/win32/orconfig.h index 037e869029..a13b4139c3 100644 --- a/src/win32/orconfig.h +++ b/src/win32/orconfig.h @@ -226,6 +226,6 @@ #define USING_TWOS_COMPLEMENT /* Version number of package */ -#define VERSION "0.2.1.16-rc" +#define VERSION "0.2.1.17-rc"