diff --git a/changes/bug2230_clean_1 b/changes/bug2230_clean_1 new file mode 100644 index 0000000000..a4edf9439e --- /dev/null +++ b/changes/bug2230_clean_1 @@ -0,0 +1,4 @@ + o Minor features + - Backport code from 0.2.3.x to allow directory authorities to clean + their microdescriptor caches. + diff --git a/changes/bug2230_part1 b/changes/bug2230_part1 new file mode 100644 index 0000000000..79f725410d --- /dev/null +++ b/changes/bug2230_part1 @@ -0,0 +1,7 @@ + o Minor bugfixes + - When loading the microdesc journal, remember its current size. + In 0.2.2, this helps prevent the microdesc journal from growing + without limit on authorities (who are the only ones to use it in + 0.2.2). Fixes a part of bug 2230; bugfix on 0.2.2.6-alpha. + Fix posted by "cypherpunks." + diff --git a/changes/bug2230_part2 b/changes/bug2230_part2 new file mode 100644 index 0000000000..2664ecc1a0 --- /dev/null +++ b/changes/bug2230_part2 @@ -0,0 +1,5 @@ + o Minor bugfixes + - The microdesc journal is supposed to get rebuilt only if it is + at least _half_ the length of the store, not _twice_ the length + of the store. Bugfix on 0.2.2.6-alpha; fixes part of bug 2230. + diff --git a/changes/bug2230_part4 b/changes/bug2230_part4 new file mode 100644 index 0000000000..f7721fad34 --- /dev/null +++ b/changes/bug2230_part4 @@ -0,0 +1,6 @@ + o Minor bugfixes: + - Authorities now clean their microdesc cache periodically and when + reading from disk initially, not only when adding new descriptors. + This prevents a bug where we could lose microdescriptors. Bugfix + on 0.2.2.6-alpha. + diff --git a/doc/HACKING b/doc/HACKING index b612953743..86d4a98784 100644 --- a/doc/HACKING +++ b/doc/HACKING @@ -414,10 +414,47 @@ Here are the steps Roger takes when putting out a new Tor release: and as a directory authority. See if it has any obvious bugs, and resolve those. +1.5) As applicable, merge the maint-X branch into the release-X branch. + 2) Gather the changes/* files into a changelog entry, rewriting many of them and reordering to focus on what users and funders would find interesting and understandable. + 2.1) Make sure that everything that wants a bug number has one. + 2.2) Concatenate them. + 2.3) Sort them by section. Within each section, try to make the + first entry or two and the last entry most interesting: they're + the ones that skimmers tend to read. + + 2.4) Clean them up + + Standard idioms: + "Fixes bug 9999; Bugfix on 0.3.3.3-alpha." + + One period after a space. + + Make stuff very terse + + Describe the user-visible problem right away + + Mention relevant config options by name. If they're rare or unusual, + remind people what they're for + + Avoid starting lines with open-paren + + Present and imperative tense: not past. + + If a given changes stanza showed up in a different release (e.g. + maint-0.2.1), be sure to make the stanzas identical (so people can + distinguish if these are the same change). + + 2.5) Merge them in. + + 2.6) Clean everything one last time. + + 2.7) Run it through fmt to make it pretty. + + 3) Compose a short release blurb to highlight the user-facing changes. Insert said release blurb into the ChangeLog stanza. If it's a stable release, add it to the ReleaseNotes file too. If we're adding @@ -435,6 +472,9 @@ in their approved versions list. 7) Sign and push the tarball to the website in the dist/ directory. Sign and push the git tag. + (That's either "git tag -u tor-0.2.x.y-status", then + "git push origin tag tor-0.2.x.y-status". To sign the + tarball, "gpg -ba ") 8) Edit include/versions.wmi to note the new version. Rebuild and push the website. @@ -453,3 +493,5 @@ the date in the ChangeLog. packages are up (for a stable release), and mail the release blurb and changelog to tor-talk or tor-announce. + (We might be moving to faster announcements, but don't announce until + the website is at least updated.) diff --git a/src/or/config.c b/src/or/config.c index 867182224f..666a1bdc34 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -3488,10 +3488,9 @@ options_validate(or_options_t *old_options, or_options_t *options, } if (options->CookieAuthFileGroupReadable && !options->CookieAuthFile) { - log_warn(LD_CONFIG, "You set the CookieAuthFileGroupReadable but did " - "not configure a the path for the cookie file via " - "CookieAuthFile. This means your cookie will not be group " - "readable."); + log_warn(LD_CONFIG, "CookieAuthFileGroupReadable is set, but will have " + "no effect: you must specify an explicit CookieAuthFile to " + "have it group-readable."); } if (options->UseEntryGuards && ! options->NumEntryGuards) diff --git a/src/or/main.c b/src/or/main.c index a26be39fdf..462b51e783 100644 --- a/src/or/main.c +++ b/src/or/main.c @@ -1075,6 +1075,8 @@ run_scheduled_events(time_t now) rep_history_clean(now - options->RephistTrackTime); rend_cache_clean(); rend_cache_clean_v2_descs_as_dir(); + if (authdir_mode_v3(options)) + microdesc_cache_rebuild(NULL, 0); #define CLEAN_CACHES_INTERVAL (30*60) time_to_clean_caches = now + CLEAN_CACHES_INTERVAL; } diff --git a/src/or/microdesc.c b/src/or/microdesc.c index 2c4b3435f7..73d2285009 100644 --- a/src/or/microdesc.c +++ b/src/or/microdesc.c @@ -23,6 +23,8 @@ struct microdesc_cache_t { tor_mmap_t *cache_content; /** Number of bytes used in the journal file. */ size_t journal_len; + /** Number of bytes in descriptors removed as too old. */ + size_t bytes_dropped; /** Total bytes of microdescriptor bodies we have added to this cache */ uint64_t total_len_seen; @@ -175,6 +177,8 @@ microdescs_add_list_to_cache(microdesc_cache_t *cache, if (md2->last_listed < md->last_listed) md2->last_listed = md->last_listed; microdesc_free(md); + if (where != SAVED_NOWHERE) + cache->bytes_dropped += size; continue; } @@ -204,15 +208,6 @@ microdescs_add_list_to_cache(microdesc_cache_t *cache, if (f) finish_writing_to_file(open_file); /*XXX Check me.*/ - { - size_t old_content_len = - cache->cache_content ? cache->cache_content->size : 0; - if (cache->journal_len > 16384 + old_content_len && - cache->journal_len > old_content_len * 2) { - microdesc_cache_rebuild(cache); - } - } - return added; } @@ -233,6 +228,7 @@ microdesc_cache_clear(microdesc_cache_t *cache) } cache->total_len_seen = 0; cache->n_seen = 0; + cache->bytes_dropped = 0; } /** Reload the contents of cache from disk. If it is empty, load it @@ -261,6 +257,7 @@ microdesc_cache_reload(microdesc_cache_t *cache) journal_content = read_file_to_str(cache->journal_fname, RFTS_IGNORE_MISSING, &st); if (journal_content) { + cache->journal_len = (size_t) st.st_size; added = microdescs_add_to_cache(cache, journal_content, journal_content+st.st_size, SAVED_IN_JOURNAL, 0); @@ -272,14 +269,81 @@ microdesc_cache_reload(microdesc_cache_t *cache) } log_notice(LD_DIR, "Reloaded microdescriptor cache. Found %d descriptors.", total); + + microdesc_cache_rebuild(cache, 0 /* don't force */); + return 0; } +/** By default, we remove any microdescriptors that have gone at least this + * long without appearing in a current consensus. */ +#define TOLERATE_MICRODESC_AGE (7*24*60*60) + +/** Remove all microdescriptors from cache that haven't been listed for + * a long time. Does not rebuild the cache on disk. If cutoff is + * positive, specifically remove microdescriptors that have been unlisted + * since cutoff. If force is true, remove microdescriptors even + * if we have no current live microdescriptor consensus. + */ +void +microdesc_cache_clean(microdesc_cache_t *cache, time_t cutoff, int force) +{ + microdesc_t **mdp, *victim; + int dropped=0, kept=0; + size_t bytes_dropped = 0; + time_t now = time(NULL); + + (void) force; + /* In 0.2.2, we let this proceed unconditionally: only authorities have + * microdesc caches. */ + + if (cutoff <= 0) + cutoff = now - TOLERATE_MICRODESC_AGE; + + for (mdp = HT_START(microdesc_map, &cache->map); mdp != NULL; ) { + if ((*mdp)->last_listed < cutoff) { + ++dropped; + victim = *mdp; + mdp = HT_NEXT_RMV(microdesc_map, &cache->map, mdp); + bytes_dropped += victim->bodylen; + microdesc_free(victim); + } else { + ++kept; + mdp = HT_NEXT(microdesc_map, &cache->map, mdp); + } + } + + if (dropped) { + log_notice(LD_DIR, "Removed %d/%d microdescriptors as old.", + dropped,dropped+kept); + cache->bytes_dropped += bytes_dropped; + } +} + +static int +should_rebuild_md_cache(microdesc_cache_t *cache) +{ + const size_t old_len = + cache->cache_content ? cache->cache_content->size : 0; + const size_t journal_len = cache->journal_len; + const size_t dropped = cache->bytes_dropped; + + if (journal_len < 16384) + return 0; /* Don't bother, not enough has happened yet. */ + if (dropped > (journal_len + old_len) / 3) + return 1; /* We could save 1/3 or more of the currently used space. */ + if (journal_len > old_len / 2) + return 1; /* We should append to the regular file */ + + return 0; +} + /** Regenerate the main cache file for cache, clear the journal file, * and update every microdesc_t in the cache with pointers to its new - * location. */ + * location. If force is true, do this unconditionally. If + * force is false, do it only if we expect to save space on disk. */ int -microdesc_cache_rebuild(microdesc_cache_t *cache) +microdesc_cache_rebuild(microdesc_cache_t *cache, int force) { open_file_t *open_file; FILE *f; @@ -289,7 +353,20 @@ microdesc_cache_rebuild(microdesc_cache_t *cache) off_t off = 0; int orig_size, new_size; + if (cache == NULL) { + cache = the_microdesc_cache; + if (cache == NULL) + return 0; + } + + /* Remove dead descriptors */ + microdesc_cache_clean(cache, 0/*cutoff*/, 0/*force*/); + + if (!force && !should_rebuild_md_cache(cache)) + return 0; + log_info(LD_DIR, "Rebuilding the microdescriptor cache..."); + orig_size = (int)(cache->cache_content ? cache->cache_content->size : 0); orig_size += (int)cache->journal_len; @@ -344,6 +421,7 @@ microdesc_cache_rebuild(microdesc_cache_t *cache) write_str_to_file(cache->journal_fname, "", 1); cache->journal_len = 0; + cache->bytes_dropped = 0; new_size = (int)cache->cache_content->size; log_info(LD_DIR, "Done rebuilding microdesc cache. " diff --git a/src/or/microdesc.h b/src/or/microdesc.h index b3e12f8ef0..77ce8536bc 100644 --- a/src/or/microdesc.h +++ b/src/or/microdesc.h @@ -21,7 +21,8 @@ smartlist_t *microdescs_add_list_to_cache(microdesc_cache_t *cache, smartlist_t *descriptors, saved_location_t where, int no_save); -int microdesc_cache_rebuild(microdesc_cache_t *cache); +void microdesc_cache_clean(microdesc_cache_t *cache, time_t cutoff, int force); +int microdesc_cache_rebuild(microdesc_cache_t *cache, int force); int microdesc_cache_reload(microdesc_cache_t *cache); void microdesc_cache_clear(microdesc_cache_t *cache);