From 083c58f126a4390b96b0e3f14d809502d8702f3d Mon Sep 17 00:00:00 2001 From: teor Date: Sat, 20 Dec 2014 21:42:28 +1100 Subject: [PATCH 1/3] Fix TestingMinExitFlagThreshold 0 Stop requiring exits to have non-zero bandwithcapacity in a TestingTorNetwork. Instead, when TestingMinExitFlagThreshold is 0, ignore exit bandwidthcapacity. This assists in bootstrapping a testing Tor network. Fixes bugs 13718 & 13839. Makes bug 13161's TestingDirAuthVoteExit non-essential. --- .../bug13839-fix-TestingMinExitFlagThreshold | 7 ++++++ src/or/dirserv.c | 24 +++++++++++++++---- 2 files changed, 26 insertions(+), 5 deletions(-) create mode 100644 changes/bug13839-fix-TestingMinExitFlagThreshold diff --git a/changes/bug13839-fix-TestingMinExitFlagThreshold b/changes/bug13839-fix-TestingMinExitFlagThreshold new file mode 100644 index 0000000000..947614f550 --- /dev/null +++ b/changes/bug13839-fix-TestingMinExitFlagThreshold @@ -0,0 +1,7 @@ + o Minor bugfixes: + - Stop requiring exits to have non-zero bandwithcapacity in a + TestingTorNetwork. Instead, when TestingMinExitFlagThreshold is 0, + ignore exit bandwidthcapacity. + This assists in bootstrapping a testing Tor network. + Fixes bugs 13718 & 13839. + Makes bug 13161's TestingDirAuthVoteExit non-essential. diff --git a/src/or/dirserv.c b/src/or/dirserv.c index d31bb72361..a1d22b041f 100644 --- a/src/or/dirserv.c +++ b/src/or/dirserv.c @@ -887,12 +887,26 @@ static int router_is_active(const routerinfo_t *ri, const node_t *node, time_t now) { time_t cutoff = now - ROUTER_MAX_AGE_TO_PUBLISH; - if (ri->cache_info.published_on < cutoff) + if (ri->cache_info.published_on < cutoff) { return 0; - if (!node->is_running || !node->is_valid || ri->is_hibernating) + } + if (!node->is_running || !node->is_valid || ri->is_hibernating) { return 0; - if (!ri->bandwidthcapacity) + } + /* Only require bandwith capacity in non-test networks, or + * if TestingTorNetwork, and TestingMinExitFlagThreshold is non-zero */ + if (!ri->bandwidthcapacity) { + if (get_options()->TestingTorNetwork) { + if (get_options()->TestingMinExitFlagThreshold > 0) { + /* If we're in a TestingTorNetwork, and TestingMinExitFlagThreshold is, + * then require bandwidthcapacity */ + return 0; + } + } else { + /* If we're not in a TestingTorNetwork, then require bandwidthcapacity */ return 0; + } + } return 1; } @@ -1037,7 +1051,7 @@ directory_fetches_dir_info_later(const or_options_t *options) } /** Return true iff we want to fetch and keep certificates for authorities - * that we don't acknowledge as aurthorities ourself. + * that we don't acknowledge as authorities ourself. */ int directory_caches_unknown_auth_certs(const or_options_t *options) @@ -1498,7 +1512,7 @@ dirserv_compute_performance_thresholds(routerlist_t *rl, (unsigned long)guard_tk, (unsigned long)guard_bandwidth_including_exits_kb, (unsigned long)guard_bandwidth_excluding_exits_kb, - enough_mtbf_info ? "" : " don't "); + enough_mtbf_info ? "" : " don't"); tor_free(uptimes); tor_free(mtbfs); From 1ee41b3eef4b5e561c7c73e79885fe858dac6d80 Mon Sep 17 00:00:00 2001 From: teor Date: Sat, 20 Dec 2014 21:53:00 +1100 Subject: [PATCH 2/3] Allow consensus interval of 10 seconds when testing Decrease minimum consensus interval to 10 seconds when TestingTorNetwork is set. (Or 5 seconds for the first consensus.) Fix code that assumes larger interval values. This assists in quickly bootstrapping a testing Tor network. Fixes bugs 13718 & 13823. --- changes/bug13823-decrease-consensus-interval | 8 ++ src/or/config.c | 78 ++++++++++++++++---- src/or/dirvote.c | 8 +- src/or/dirvote.h | 32 +++++++- src/or/networkstatus.c | 12 +++ src/or/router.c | 13 +++- src/or/routerlist.c | 22 +++++- src/or/routerparse.c | 8 +- 8 files changed, 160 insertions(+), 21 deletions(-) create mode 100644 changes/bug13823-decrease-consensus-interval diff --git a/changes/bug13823-decrease-consensus-interval b/changes/bug13823-decrease-consensus-interval new file mode 100644 index 0000000000..1d99bd73cb --- /dev/null +++ b/changes/bug13823-decrease-consensus-interval @@ -0,0 +1,8 @@ + o Minor bugfixes: + - Decrease minimum consensus interval to 10 seconds + when TestingTorNetwork is set. (Or 5 seconds for + the first consensus.) + Fix code that assumes larger interval values. + This assists in quickly bootstrapping a testing + Tor network. + Fixes bugs 13718 & 13823. diff --git a/src/or/config.c b/src/or/config.c index 28f1df0663..a779771c86 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -468,7 +468,7 @@ static const config_var_t testing_tor_network_defaults[] = { V(V3AuthVotingInterval, INTERVAL, "5 minutes"), V(V3AuthVoteDelay, INTERVAL, "20 seconds"), V(V3AuthDistDelay, INTERVAL, "20 seconds"), - V(TestingV3AuthInitialVotingInterval, INTERVAL, "5 minutes"), + V(TestingV3AuthInitialVotingInterval, INTERVAL, "150 seconds"), V(TestingV3AuthInitialVoteDelay, INTERVAL, "20 seconds"), V(TestingV3AuthInitialDistDelay, INTERVAL, "20 seconds"), V(TestingV3AuthVotingStartOffset, INTERVAL, "0"), @@ -3397,19 +3397,68 @@ options_validate(or_options_t *old_options, or_options_t *options, if (options->V3AuthVoteDelay + options->V3AuthDistDelay >= options->V3AuthVotingInterval/2) { - REJECT("V3AuthVoteDelay plus V3AuthDistDelay must be less than half " - "V3AuthVotingInterval"); + /* + This doesn't work, but it seems like it should: + what code is preventing the interval being less than twice the lead-up? + if (options->TestingTorNetwork) { + if (options->V3AuthVoteDelay + options->V3AuthDistDelay >= + options->V3AuthVotingInterval) { + REJECT("V3AuthVoteDelay plus V3AuthDistDelay must be less than " + "V3AuthVotingInterval"); + } else { + COMPLAIN("V3AuthVoteDelay plus V3AuthDistDelay is more than half " + "V3AuthVotingInterval. This may lead to " + "consensus instability, particularly if clocks drift."); + } + } else { + */ + REJECT("V3AuthVoteDelay plus V3AuthDistDelay must be less than half " + "V3AuthVotingInterval"); + /* + } + */ + } + + if (options->V3AuthVoteDelay < MIN_VOTE_SECONDS) { + if (options->TestingTorNetwork) { + if (options->V3AuthVoteDelay < MIN_VOTE_SECONDS_TESTING) { + REJECT("V3AuthVoteDelay is way too low."); + } else { + COMPLAIN("V3AuthVoteDelay is very low. " + "This may lead to failure to vote for a consensus."); + } + } else { + REJECT("V3AuthVoteDelay is way too low."); + } + } + + if (options->V3AuthDistDelay < MIN_DIST_SECONDS) { + if (options->TestingTorNetwork) { + if (options->V3AuthDistDelay < MIN_DIST_SECONDS_TESTING) { + REJECT("V3AuthDistDelay is way too low."); + } else { + COMPLAIN("V3AuthDistDelay is very low. " + "This may lead to missing votes in a consensus."); + } + } else { + REJECT("V3AuthDistDelay is way too low."); + } } - if (options->V3AuthVoteDelay < MIN_VOTE_SECONDS) - REJECT("V3AuthVoteDelay is way too low."); - if (options->V3AuthDistDelay < MIN_DIST_SECONDS) - REJECT("V3AuthDistDelay is way too low."); if (options->V3AuthNIntervalsValid < 2) REJECT("V3AuthNIntervalsValid must be at least 2."); if (options->V3AuthVotingInterval < MIN_VOTE_INTERVAL) { - REJECT("V3AuthVotingInterval is insanely low."); + if (options->TestingTorNetwork) { + if (options->V3AuthVotingInterval < MIN_VOTE_INTERVAL_TESTING) { + REJECT("V3AuthVotingInterval is insanely low."); + } else { + COMPLAIN("V3AuthVotingInterval is very low. " + "This may lead to failure to synchronise for a consensus."); + } + } else { + REJECT("V3AuthVotingInterval is insanely low."); + } } else if (options->V3AuthVotingInterval > 24*60*60) { REJECT("V3AuthVotingInterval is insanely high."); } else if (((24*60*60) % options->V3AuthVotingInterval) != 0) { @@ -3484,26 +3533,27 @@ options_validate(or_options_t *old_options, or_options_t *options, CHECK_DEFAULT(TestingCertMaxDownloadTries); #undef CHECK_DEFAULT - if (options->TestingV3AuthInitialVotingInterval < MIN_VOTE_INTERVAL) { + if (options->TestingV3AuthInitialVotingInterval + < MIN_VOTE_INTERVAL_TESTING_INITIAL) { REJECT("TestingV3AuthInitialVotingInterval is insanely low."); } else if (((30*60) % options->TestingV3AuthInitialVotingInterval) != 0) { REJECT("TestingV3AuthInitialVotingInterval does not divide evenly into " "30 minutes."); } - if (options->TestingV3AuthInitialVoteDelay < MIN_VOTE_SECONDS) { + if (options->TestingV3AuthInitialVoteDelay < MIN_VOTE_SECONDS_TESTING) { REJECT("TestingV3AuthInitialVoteDelay is way too low."); } - if (options->TestingV3AuthInitialDistDelay < MIN_DIST_SECONDS) { + if (options->TestingV3AuthInitialDistDelay < MIN_DIST_SECONDS_TESTING) { REJECT("TestingV3AuthInitialDistDelay is way too low."); } if (options->TestingV3AuthInitialVoteDelay + options->TestingV3AuthInitialDistDelay >= - options->TestingV3AuthInitialVotingInterval/2) { + options->TestingV3AuthInitialVotingInterval) { REJECT("TestingV3AuthInitialVoteDelay plus TestingV3AuthInitialDistDelay " - "must be less than half TestingV3AuthInitialVotingInterval"); + "must be less than TestingV3AuthInitialVotingInterval"); } if (options->TestingV3AuthVotingStartOffset > @@ -3511,6 +3561,8 @@ options_validate(or_options_t *old_options, or_options_t *options, options->V3AuthVotingInterval)) { REJECT("TestingV3AuthVotingStartOffset is higher than the voting " "interval."); + } else if (options->TestingV3AuthVotingStartOffset < 0) { + REJECT("TestingV3AuthVotingStartOffset must be non-negative."); } if (options->TestingAuthDirTimeToLearnReachability < 0) { diff --git a/src/or/dirvote.c b/src/or/dirvote.c index 39505a4f9e..c6068228f9 100644 --- a/src/or/dirvote.c +++ b/src/or/dirvote.c @@ -1107,8 +1107,12 @@ networkstatus_compute_consensus(smartlist_t *votes, vote_seconds = median_int(votesec_list, n_votes); dist_seconds = median_int(distsec_list, n_votes); - tor_assert(valid_after+MIN_VOTE_INTERVAL <= fresh_until); - tor_assert(fresh_until+MIN_VOTE_INTERVAL <= valid_until); + tor_assert(valid_after + + (get_options()->TestingTorNetwork ? + MIN_VOTE_INTERVAL_TESTING : MIN_VOTE_INTERVAL) <= fresh_until); + tor_assert(fresh_until + + (get_options()->TestingTorNetwork ? + MIN_VOTE_INTERVAL_TESTING : MIN_VOTE_INTERVAL) <= valid_until); tor_assert(vote_seconds >= MIN_VOTE_SECONDS); tor_assert(dist_seconds >= MIN_DIST_SECONDS); diff --git a/src/or/dirvote.h b/src/or/dirvote.h index 5d44ba4320..b570e9d251 100644 --- a/src/or/dirvote.h +++ b/src/or/dirvote.h @@ -14,12 +14,42 @@ #include "testsupport.h" +/* + * Ideally, assuming synced clocks, we should only need 1 second for each of: + * - Vote + * - Distribute + * - Consensus Publication + * As we can gather descriptors continuously. + * (Could we even go as far as publishing the previous consensus, + * in the same second that we vote for the next one?) + * But we're not there yet: these are the lowest working values at this time. + */ + /** Lowest allowable value for VoteSeconds. */ #define MIN_VOTE_SECONDS 2 +/** Lowest allowable value for VoteSeconds when TestingTorNetwork is 1 */ +#define MIN_VOTE_SECONDS_TESTING 2 + /** Lowest allowable value for DistSeconds. */ #define MIN_DIST_SECONDS 2 -/** Smallest allowable voting interval. */ +/** Lowest allowable value for DistSeconds when TestingTorNetwork is 1 */ +#define MIN_DIST_SECONDS_TESTING 2 + +/** Lowest allowable voting interval. */ #define MIN_VOTE_INTERVAL 300 +/** Lowest allowable voting interval when TestingTorNetwork is 1: + * Voting Interval can be: + * 10, 12, 15, 18, 20, 24, 25, 30, 36, 40, 45, 50, 60, ... + * Testing Initial Voting Interval can be: + * 5, 6, 8, 9, or any of the possible values for Voting Interval, + * as they both need to evenly divide 30 minutes. + * If clock desynchronisation is an issue, use an interval of at least: + * 18 * drift in seconds, to allow for a clock slop factor */ +#define MIN_VOTE_INTERVAL_TESTING \ + (((MIN_VOTE_SECONDS_TESTING)+(MIN_DIST_SECONDS_TESTING)+1)*2) + +#define MIN_VOTE_INTERVAL_TESTING_INITIAL \ + ((MIN_VOTE_SECONDS_TESTING)+(MIN_DIST_SECONDS_TESTING)+1) /** The lowest consensus method that we currently support. */ #define MIN_SUPPORTED_CONSENSUS_METHOD 13 diff --git a/src/or/networkstatus.c b/src/or/networkstatus.c index 21efdd129d..9b24405951 100644 --- a/src/or/networkstatus.c +++ b/src/or/networkstatus.c @@ -832,6 +832,10 @@ update_consensus_networkstatus_fetch_time_impl(time_t now, int flav) a crazy-fast voting interval, though, 2 minutes may be too much. */ min_sec_before_caching = interval/16; + /* make sure we always delay by at least a second before caching */ + if (min_sec_before_caching == 0) { + min_sec_before_caching = 1; + } } if (directory_fetches_dir_info_early(options)) { @@ -863,8 +867,16 @@ update_consensus_networkstatus_fetch_time_impl(time_t now, int flav) dl_interval = (c->valid_until - start) - min_sec_before_caching; } } + /* catch low dl_interval in crazy-fast networks */ if (dl_interval < 1) dl_interval = 1; + /* catch late start in crazy-fast networks */ + if (start+dl_interval >= c->valid_until) + start = c->valid_until - dl_interval - 1; + log_debug(LD_DIR, + "fresh_until: %ld start: %ld " + "dl_interval: %ld valid_until: %ld ", + c->fresh_until, start, dl_interval, c->valid_until); /* We must not try to replace c while it's still fresh: */ tor_assert(c->fresh_until < start); /* We must download the next one before c is invalid: */ diff --git a/src/or/router.c b/src/or/router.c index 56bb909952..bad5242603 100644 --- a/src/or/router.c +++ b/src/or/router.c @@ -1223,6 +1223,11 @@ router_orport_found_reachable(void) " Publishing server descriptor." : ""); can_reach_or_port = 1; mark_my_descriptor_dirty("ORPort found reachable"); + /* This is a significant enough change to upload immediately, + * at least in a test network */ + if (get_options()->TestingTorNetwork == 1) { + reschedule_descriptor_update_check(); + } control_event_server_status(LOG_NOTICE, "REACHABILITY_SUCCEEDED ORADDRESS=%s:%d", address, me->or_port); @@ -1240,8 +1245,14 @@ router_dirport_found_reachable(void) log_notice(LD_DIRSERV,"Self-testing indicates your DirPort is reachable " "from the outside. Excellent."); can_reach_dir_port = 1; - if (decide_to_advertise_dirport(get_options(), me->dir_port)) + if (decide_to_advertise_dirport(get_options(), me->dir_port)) { mark_my_descriptor_dirty("DirPort found reachable"); + /* This is a significant enough change to upload immediately, + * at least in a test network */ + if (get_options()->TestingTorNetwork == 1) { + reschedule_descriptor_update_check(); + } + } control_event_server_status(LOG_NOTICE, "REACHABILITY_SUCCEEDED DIRADDRESS=%s:%d", address, me->dir_port); diff --git a/src/or/routerlist.c b/src/or/routerlist.c index 8379bc80b3..60d8e71a28 100644 --- a/src/or/routerlist.c +++ b/src/or/routerlist.c @@ -2210,11 +2210,29 @@ router_choose_random_node(smartlist_t *excludedsmartlist, router_add_running_nodes_to_smartlist(sl, allow_invalid, need_uptime, need_capacity, need_guard, need_desc); + log_debug(LD_CIRC, + "We found %d running nodes.", + smartlist_len(sl)); + smartlist_subtract(sl,excludednodes); - if (excludedsmartlist) + log_debug(LD_CIRC, + "We removed %d excludednodes, leaving %d nodes.", + smartlist_len(excludednodes), + smartlist_len(sl)); + + if (excludedsmartlist) { smartlist_subtract(sl,excludedsmartlist); - if (excludedset) + log_debug(LD_CIRC, + "We removed %d excludedsmartlist, leaving %d nodes.", + smartlist_len(excludedsmartlist), + smartlist_len(sl)); + } + if (excludedset) { routerset_subtract_nodes(sl,excludedset); + log_debug(LD_CIRC, + "We removed excludedset, leaving %d nodes.", + smartlist_len(sl)); + } // Always weight by bandwidth choice = node_sl_choose_by_bandwidth(sl, rule); diff --git a/src/or/routerparse.c b/src/or/routerparse.c index bc3b00226a..8176d47262 100644 --- a/src/or/routerparse.c +++ b/src/or/routerparse.c @@ -2598,11 +2598,15 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, (int) tor_parse_long(tok->args[1], 10, 0, INT_MAX, &ok, NULL); if (!ok) goto err; - if (ns->valid_after + MIN_VOTE_INTERVAL > ns->fresh_until) { + if (ns->valid_after + + (get_options()->TestingTorNetwork ? + MIN_VOTE_INTERVAL_TESTING : MIN_VOTE_INTERVAL) > ns->fresh_until) { log_warn(LD_DIR, "Vote/consensus freshness interval is too short"); goto err; } - if (ns->valid_after + MIN_VOTE_INTERVAL*2 > ns->valid_until) { + if (ns->valid_after + + (get_options()->TestingTorNetwork ? + MIN_VOTE_INTERVAL_TESTING : MIN_VOTE_INTERVAL)*2 > ns->valid_until) { log_warn(LD_DIR, "Vote/consensus liveness interval is too short"); goto err; } From 8a8797f1e45e6124ac93e3a8fb277b5758d7c935 Mon Sep 17 00:00:00 2001 From: teor Date: Sat, 20 Dec 2014 21:59:17 +1100 Subject: [PATCH 3/3] Fix If-Modified-Since in rapidly updating Tor networks When V3AuthVotingInterval is low, decrease the delay on the If-Modified-Since header passed to directory servers. This allows us to obtain consensuses promptly when the consensus interval is very short. This assists in bootstrapping a testing Tor network. Fixes bugs 13718 & 13963. --- .../bug13963-decrease-if-modified-since-delay | 7 +++++ src/or/directory.c | 27 ++++++++++++++++--- 2 files changed, 30 insertions(+), 4 deletions(-) create mode 100644 changes/bug13963-decrease-if-modified-since-delay diff --git a/changes/bug13963-decrease-if-modified-since-delay b/changes/bug13963-decrease-if-modified-since-delay new file mode 100644 index 0000000000..62371444c4 --- /dev/null +++ b/changes/bug13963-decrease-if-modified-since-delay @@ -0,0 +1,7 @@ + o Minor bugfixes: + - When V3AuthVotingInterval is low, decrease the delay on the + If-Modified-Since header passed to directory servers. + This allows us to obtain consensuses promptly when the consensus + interval is very short. + This assists in bootstrapping a testing Tor network. + Fixes bugs 13718 & 13963. diff --git a/src/or/directory.c b/src/or/directory.c index cca4b54e24..b88c9d9f10 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -433,18 +433,33 @@ directory_get_from_dirserver(uint8_t dir_purpose, uint8_t router_purpose, if (resource) flav = networkstatus_parse_flavor_name(resource); + /* DEFAULT_IF_MODIFIED_SINCE_DELAY is 1/20 of the default consensus + * period of 1 hour. + */ +#define DEFAULT_IF_MODIFIED_SINCE_DELAY (180) if (flav != -1) { /* IF we have a parsed consensus of this type, we can do an * if-modified-time based on it. */ v = networkstatus_get_latest_consensus_by_flavor(flav); - if (v) - if_modified_since = v->valid_after + 180; + if (v) { + /* In networks with particularly short V3AuthVotingIntervals, + * ask for the consensus if it's been modified since half the + * V3AuthVotingInterval of the most recent consensus. */ + time_t ims_delay = DEFAULT_IF_MODIFIED_SINCE_DELAY; + if (v->fresh_until > v->valid_after + && ims_delay > (v->fresh_until - v->valid_after)/2) { + ims_delay = (v->fresh_until - v->valid_after)/2; + } + if_modified_since = v->valid_after + ims_delay; + } } else { /* Otherwise it might be a consensus we don't parse, but which we * do cache. Look at the cached copy, perhaps. */ cached_dir_t *cd = dirserv_get_consensus(resource); + /* We have no method of determining the voting interval from an + * unparsed consensus, so we use the default. */ if (cd) - if_modified_since = cd->published + 180; + if_modified_since = cd->published + DEFAULT_IF_MODIFIED_SINCE_DELAY; } } @@ -2258,6 +2273,7 @@ write_http_status_line(dir_connection_t *conn, int status, log_warn(LD_BUG,"status line too long."); return; } + log_debug(LD_DIRSERV,"Wrote status 'HTTP/1.0 %d %s'", status, reason_phrase); connection_write_to_buf(buf, strlen(buf), TO_CONN(conn)); } @@ -2554,8 +2570,11 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, if ((header = http_get_header(headers, "If-Modified-Since: "))) { struct tm tm; if (parse_http_time(header, &tm) == 0) { - if (tor_timegm(&tm, &if_modified_since)<0) + if (tor_timegm(&tm, &if_modified_since)<0) { if_modified_since = 0; + } else { + log_debug(LD_DIRSERV, "If-Modified-Since is '%s'.", escaped(header)); + } } /* The correct behavior on a malformed If-Modified-Since header is to * act as if no If-Modified-Since header had been given. */