From 8d41e6c47126fb467858f98b06ab7460dcc6647b Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 19 Aug 2009 20:03:40 -0400 Subject: [PATCH 01/23] Support for encoding and decoding 256-bit digests in base64 --- src/common/crypto.c | 38 ++++++++++++++++++++++++++++++++++++++ src/common/crypto.h | 2 ++ src/test/test_crypto.c | 11 +++++++++++ 3 files changed, 51 insertions(+) diff --git a/src/common/crypto.c b/src/common/crypto.c index 581d1ba5a0..21c8aed2d4 100644 --- a/src/common/crypto.c +++ b/src/common/crypto.c @@ -2274,6 +2274,44 @@ digest_from_base64(char *digest, const char *d64) #endif } +/** Base-64 encode DIGEST256_LINE bytes from digest, remove the + * trailing = and newline characters, and store the nul-terminated result in + * the first BASE64_DIGEST256_LEN+1 bytes of d64. */ +int +digest256_to_base64(char *d64, const char *digest) +{ + char buf[256]; + base64_encode(buf, sizeof(buf), digest, DIGEST256_LEN); + buf[BASE64_DIGEST256_LEN] = '\0'; + memcpy(d64, buf, BASE64_DIGEST256_LEN+1); + return 0; +} + +/** Given a base-64 encoded, nul-terminated digest in d64 (without + * trailing newline or = characters), decode it and store the result in the + * first DIGEST256_LEN bytes at digest. */ +int +digest256_from_base64(char *digest, const char *d64) +{ +#ifdef USE_OPENSSL_BASE64 + char buf_in[BASE64_DIGEST256_LEN+3]; + char buf[256]; + if (strlen(d64) != BASE64_DIGEST256_LEN) + return -1; + memcpy(buf_in, d64, BASE64_DIGEST256_LEN); + memcpy(buf_in+BASE64_DIGEST256_LEN, "=\n\0", 3); + if (base64_decode(buf, sizeof(buf), buf_in, strlen(buf_in)) != DIGEST256_LEN) + return -1; + memcpy(digest, buf, DIGEST256_LEN); + return 0; +#else + if (base64_decode(digest, DIGEST256_LEN, d64, strlen(d64)) == DIGEST256_LEN) + return 0; + else + return -1; +#endif +} + /** Implements base32 encoding as in rfc3548. Limitation: Requires * that srclen*8 is a multiple of 5. */ diff --git a/src/common/crypto.h b/src/common/crypto.h index f0958a8073..63ea96d056 100644 --- a/src/common/crypto.h +++ b/src/common/crypto.h @@ -211,6 +211,8 @@ int base32_decode(char *dest, size_t destlen, const char *src, size_t srclen); int digest_to_base64(char *d64, const char *digest); int digest_from_base64(char *digest, const char *d64); +int digest256_to_base64(char *d64, const char *digest); +int digest256_from_base64(char *digest, const char *d64); /** Length of RFC2440-style S2K specifier: the first 8 bytes are a salt, the * 9th describes how much iteration to do. */ diff --git a/src/test/test_crypto.c b/src/test/test_crypto.c index d235520d09..2edb8035bb 100644 --- a/src/test/test_crypto.c +++ b/src/test/test_crypto.c @@ -472,6 +472,17 @@ test_crypto_formats(void) test_assert(digest_from_base64(data3, "###") < 0); + /* Encoding SHA256 */ + crypto_rand(data2, DIGEST256_LEN); + memset(data2, 100, 1024); + digest256_to_base64(data2, data1); + test_eq(BASE64_DIGEST256_LEN, strlen(data2)); + test_eq(100, data2[BASE64_DIGEST256_LEN+2]); + memset(data3, 99, 1024); + test_eq(digest256_from_base64(data3, data2), 0); + test_memeq(data1, data3, DIGEST256_LEN); + test_eq(99, data3[DIGEST256_LEN+1]); + /* Base32 tests */ strlcpy(data1, "5chrs", 1024); /* bit pattern is: [35 63 68 72 73] -> From 8b2f6b27fdc03f12d092d37c42d1995ff7426916 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 14 Sep 2009 11:57:19 -0400 Subject: [PATCH 02/23] Make signature-generation code handle different key and digest lengths. --- src/or/dirserv.c | 9 ++++++--- src/or/dirvote.c | 6 +++--- src/or/or.h | 1 + src/or/rendcommon.c | 3 ++- src/or/router.c | 5 +++-- src/or/routerparse.c | 9 +++++---- 6 files changed, 20 insertions(+), 13 deletions(-) diff --git a/src/or/dirserv.c b/src/or/dirserv.c index 47dc038454..f12ef2f3d5 100644 --- a/src/or/dirserv.c +++ b/src/or/dirserv.c @@ -1091,7 +1091,8 @@ dirserv_dump_directory_to_string(char **dir_out, return -1; } note_crypto_pk_op(SIGN_DIR); - if (router_append_dirobj_signature(buf,buf_len,digest,private_key)<0) { + if (router_append_dirobj_signature(buf,buf_len,digest,DIGEST_LEN, + private_key)<0) { tor_free(buf); return -1; } @@ -1549,7 +1550,8 @@ generate_runningrouters(void) goto err; } note_crypto_pk_op(SIGN_DIR); - if (router_append_dirobj_signature(s, len, digest, private_key)<0) + if (router_append_dirobj_signature(s, len, digest, DIGEST_LEN, + private_key)<0) goto err; set_cached_dir(&the_runningrouters, s, time(NULL)); @@ -2743,7 +2745,8 @@ generate_v2_networkstatus_opinion(void) outp += strlen(outp); note_crypto_pk_op(SIGN_DIR); - if (router_append_dirobj_signature(outp,endp-outp,digest,private_key)<0) { + if (router_append_dirobj_signature(outp,endp-outp,digest,DIGEST_LEN, + private_key)<0) { log_warn(LD_BUG, "Unable to sign router status."); goto done; } diff --git a/src/or/dirvote.c b/src/or/dirvote.c index 358246ae9d..4e94eb67e6 100644 --- a/src/or/dirvote.c +++ b/src/or/dirvote.c @@ -192,7 +192,7 @@ format_networkstatus_vote(crypto_pk_env_t *private_signing_key, if (router_get_networkstatus_v3_hash(status, digest)<0) goto err; note_crypto_pk_op(SIGN_DIR); - if (router_append_dirobj_signature(outp,endp-outp,digest, + if (router_append_dirobj_signature(outp,endp-outp,digest, DIGEST_LEN, private_signing_key)<0) { log_warn(LD_BUG, "Unable to sign networkstatus vote."); goto err; @@ -1257,7 +1257,7 @@ networkstatus_compute_consensus(smartlist_t *votes, tor_snprintf(buf, sizeof(buf), "%s %s\n", fingerprint, signing_key_fingerprint); /* And the signature. */ - if (router_append_dirobj_signature(buf, sizeof(buf), digest, + if (router_append_dirobj_signature(buf, sizeof(buf), digest, DIGEST_LEN, signing_key)) { log_warn(LD_BUG, "Couldn't sign consensus networkstatus."); return NULL; /* This leaks, but it should never happen. */ @@ -1272,7 +1272,7 @@ networkstatus_compute_consensus(smartlist_t *votes, signing_key_fingerprint, 0); tor_snprintf(buf, sizeof(buf), "%s %s\n", fingerprint, signing_key_fingerprint); - if (router_append_dirobj_signature(buf, sizeof(buf), digest, + if (router_append_dirobj_signature(buf, sizeof(buf), digest, DIGEST_LEN, legacy_signing_key)) { log_warn(LD_BUG, "Couldn't sign consensus networkstatus."); return NULL; /* This leaks, but it should never happen. */ diff --git a/src/or/or.h b/src/or/or.h index f0ea25e07e..d9e883f3e8 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -4911,6 +4911,7 @@ int router_get_networkstatus_v3_hash(const char *s, char *digest); int router_get_extrainfo_hash(const char *s, char *digest); int router_append_dirobj_signature(char *buf, size_t buf_len, const char *digest, + size_t digest_len, crypto_pk_env_t *private_key); int router_parse_list_from_string(const char **s, const char *eos, smartlist_t *dest, diff --git a/src/or/rendcommon.c b/src/or/rendcommon.c index df7195e3ea..c7eb2a9d08 100644 --- a/src/or/rendcommon.c +++ b/src/or/rendcommon.c @@ -618,7 +618,8 @@ rend_encode_v2_descriptors(smartlist_t *descs_out, } if (router_append_dirobj_signature(desc_str + written, desc_len - written, - desc_digest, service_key) < 0) { + desc_digest, DIGEST_LEN, + service_key) < 0) { log_warn(LD_BUG, "Couldn't sign desc."); rend_encoded_v2_service_descriptor_free(enc); goto err; diff --git a/src/or/router.c b/src/or/router.c index 5b260de1ca..85abc8c5fb 100644 --- a/src/or/router.c +++ b/src/or/router.c @@ -1788,7 +1788,7 @@ router_dump_router_to_string(char *s, size_t maxlen, routerinfo_t *router, note_crypto_pk_op(SIGN_RTR); if (router_append_dirobj_signature(s+written,maxlen-written, - digest,ident_key)<0) { + digest,DIGEST_LEN,ident_key)<0) { log_warn(LD_BUG, "Couldn't sign router descriptor"); return -1; } @@ -1980,7 +1980,8 @@ extrainfo_dump_to_string(char *s, size_t maxlen, extrainfo_t *extrainfo, len += strlen(s+len); if (router_get_extrainfo_hash(s, digest)<0) return -1; - if (router_append_dirobj_signature(s+len, maxlen-len, digest, ident_key)<0) + if (router_append_dirobj_signature(s+len, maxlen-len, digest, DIGEST_LEN, + ident_key)<0) return -1; { diff --git a/src/or/routerparse.c b/src/or/routerparse.c index e35ece06de..02c5cdb44e 100644 --- a/src/or/routerparse.c +++ b/src/or/routerparse.c @@ -643,14 +643,15 @@ router_get_extrainfo_hash(const char *s, char *digest) */ int router_append_dirobj_signature(char *buf, size_t buf_len, const char *digest, - crypto_pk_env_t *private_key) + size_t digest_len, crypto_pk_env_t *private_key) { char *signature; size_t i; + int siglen; signature = tor_malloc(crypto_pk_keysize(private_key)); - if (crypto_pk_private_sign(private_key, signature, digest, DIGEST_LEN) < 0) { - + siglen = crypto_pk_private_sign(private_key, signature, digest, digest_len); + if (siglen < 0) { log_warn(LD_BUG,"Couldn't sign digest."); goto err; } @@ -658,7 +659,7 @@ router_append_dirobj_signature(char *buf, size_t buf_len, const char *digest, goto truncated; i = strlen(buf); - if (base64_encode(buf+i, buf_len-i, signature, 128) < 0) { + if (base64_encode(buf+i, buf_len-i, signature, siglen) < 0) { log_warn(LD_BUG,"couldn't base64-encode signature"); goto err; } From 15f4e9600cb8bb127bcf7004691fe9c59eb7505e Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 14 Sep 2009 12:02:00 -0400 Subject: [PATCH 03/23] Signature-checking code can handle longer digests. --- src/or/routerparse.c | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/src/or/routerparse.c b/src/or/routerparse.c index 02c5cdb44e..f6d6dc9a14 100644 --- a/src/or/routerparse.c +++ b/src/or/routerparse.c @@ -528,6 +528,7 @@ static directory_token_t *get_next_token(memarea_t *area, #define CST_CHECK_AUTHORITY (1<<0) #define CST_NO_CHECK_OBJTYPE (1<<1) static int check_signature_token(const char *digest, + ssize_t digest_len, directory_token_t *tok, crypto_pk_env_t *pkey, int flags, @@ -802,7 +803,7 @@ router_parse_directory(const char *str) } declared_key = find_dir_signing_key(str, str+strlen(str)); note_crypto_pk_op(VERIFY_DIR); - if (check_signature_token(digest, tok, declared_key, + if (check_signature_token(digest, DIGEST_LEN, tok, declared_key, CST_CHECK_AUTHORITY, "directory")<0) goto err; @@ -895,7 +896,7 @@ router_parse_runningrouters(const char *str) } declared_key = find_dir_signing_key(str, eos); note_crypto_pk_op(VERIFY_DIR); - if (check_signature_token(digest, tok, declared_key, + if (check_signature_token(digest, DIGEST_LEN, tok, declared_key, CST_CHECK_AUTHORITY, "running-routers") < 0) goto err; @@ -996,6 +997,7 @@ dir_signing_key_is_trusted(crypto_pk_env_t *key) */ static int check_signature_token(const char *digest, + ssize_t digest_len, directory_token_t *tok, crypto_pk_env_t *pkey, int flags, @@ -1026,14 +1028,14 @@ check_signature_token(const char *digest, signed_digest = tor_malloc(tok->object_size); if (crypto_pk_public_checksig(pkey, signed_digest, tok->object_body, tok->object_size) - != DIGEST_LEN) { + != digest_len) { log_warn(LD_DIR, "Error reading %s: invalid signature.", doctype); tor_free(signed_digest); return -1; } // log_debug(LD_DIR,"Signed %s hash starts %s", doctype, // hex_str(signed_digest,4)); - if (memcmp(digest, signed_digest, DIGEST_LEN)) { + if (memcmp(digest, signed_digest, digest_len)) { log_warn(LD_DIR, "Error reading %s: signature does not match.", doctype); tor_free(signed_digest); return -1; @@ -1489,7 +1491,7 @@ router_parse_entry_from_string(const char *s, const char *end, verified_digests = digestmap_new(); digestmap_set(verified_digests, signed_digest, (void*)(uintptr_t)1); #endif - if (check_signature_token(digest, tok, router->identity_pkey, 0, + if (check_signature_token(digest, DIGEST_LEN, tok, router->identity_pkey, 0, "router descriptor") < 0) goto err; @@ -1617,7 +1619,8 @@ extrainfo_parse_entry_from_string(const char *s, const char *end, if (key) { note_crypto_pk_op(VERIFY_RTR); - if (check_signature_token(digest, tok, key, 0, "extra-info") < 0) + if (check_signature_token(digest, DIGEST_LEN, tok, key, 0, + "extra-info") < 0) goto err; if (router) @@ -1779,7 +1782,7 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string) } } if (!found) { - if (check_signature_token(digest, tok, cert->identity_key, 0, + if (check_signature_token(digest, DIGEST_LEN, tok, cert->identity_key, 0, "key certificate")) { goto err; } @@ -1788,6 +1791,7 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string) /* XXXX Once all authorities generate cross-certified certificates, * make this field mandatory. */ if (check_signature_token(cert->cache_info.identity_digest, + DIGEST_LEN, tok, cert->signing_key, CST_NO_CHECK_OBJTYPE, @@ -2236,7 +2240,7 @@ networkstatus_v2_parse_from_string(const char *s) } note_crypto_pk_op(VERIFY_DIR); - if (check_signature_token(ns_digest, tok, ns->signing_key, 0, + if (check_signature_token(ns_digest, DIGEST_LEN, tok, ns->signing_key, 0, "network-status") < 0) goto err; @@ -2632,7 +2636,8 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, } if (ns->type != NS_TYPE_CONSENSUS) { - if (check_signature_token(ns_digest, tok, ns->cert->signing_key, 0, + if (check_signature_token(ns_digest, DIGEST_LEN, + tok, ns->cert->signing_key, 0, "network-status vote")) goto err; v->good_signature = 1; @@ -3812,7 +3817,7 @@ rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out, /* Parse and verify signature. */ tok = find_by_keyword(tokens, R_SIGNATURE); note_crypto_pk_op(VERIFY_RTR); - if (check_signature_token(desc_hash, tok, result->pk, 0, + if (check_signature_token(desc_hash, DIGEST_LEN, tok, result->pk, 0, "v2 rendezvous service descriptor") < 0) goto err; /* Verify that descriptor ID belongs to public key and secret ID part. */ From c5f7f04aff850e8e3fad28e93e6300447625fdbb Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 14 Sep 2009 12:02:49 -0400 Subject: [PATCH 04/23] Allow signed data to include other hashes later. Previously, we insisted that a valid signature must be a signature of the expected digest. Now we accept anything that starts with the expected digest. This lets us include another digest later. --- src/or/routerparse.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/or/routerparse.c b/src/or/routerparse.c index f6d6dc9a14..bfcf20b09c 100644 --- a/src/or/routerparse.c +++ b/src/or/routerparse.c @@ -1028,7 +1028,7 @@ check_signature_token(const char *digest, signed_digest = tor_malloc(tok->object_size); if (crypto_pk_public_checksig(pkey, signed_digest, tok->object_body, tok->object_size) - != digest_len) { + < digest_len) { log_warn(LD_DIR, "Error reading %s: invalid signature.", doctype); tor_free(signed_digest); return -1; From 80a7a34755e7b7163778c5475f9a6459febb1bf4 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 15 Sep 2009 13:29:48 -0400 Subject: [PATCH 05/23] Revise 162's idea of how detached signatures work with flavors The original proposal was vague and would have made older Tors reject detached-signature documents as soon as they saw one with flavors. --- doc/spec/proposals/162-consensus-flavors.txt | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/doc/spec/proposals/162-consensus-flavors.txt b/doc/spec/proposals/162-consensus-flavors.txt index 8fdf9d07bf..56a0b0e1ab 100644 --- a/doc/spec/proposals/162-consensus-flavors.txt +++ b/doc/spec/proposals/162-consensus-flavors.txt @@ -91,9 +91,18 @@ Spec modifications: 3. Document format: detached signatures. - In addition to the current detached signature format, we allow - the first line to take the form, - "consensus-digest" SP flavor SP 1*(Algname "=" Digest) NL + We amend the detached signature format to include more than one + consensus-digest line, and more than one set of signatures. + + After the consensus-digest line, we allow more lines of the form: + "additional-digest" SP flavor SP algname SP digest NL + + Before the directory-signature lines, we allow more entries of the form: + "additional-signature" SP flavor SP algname SP identity SP + signing-key-digest NL signature. + + [We do not use "consensus-digest" or "directory-signature" for flavored + consensuses, since this could confuse older Tors.] The consensus-signatures URL should contain the signatures for _all_ flavors of consensus. From bdf48393956db2827eb4340e2484cc28282b3f34 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 19 Aug 2009 20:06:59 -0400 Subject: [PATCH 06/23] Functions to encode microdescriptors and their lines. --- src/or/dirvote.c | 76 +++++++++++++++++++++++++++++++++++++++++++++--- src/or/or.h | 6 ++++ 2 files changed, 78 insertions(+), 4 deletions(-) diff --git a/src/or/dirvote.c b/src/or/dirvote.c index 4e94eb67e6..f0f92d2f60 100644 --- a/src/or/dirvote.c +++ b/src/or/dirvote.c @@ -21,7 +21,7 @@ static int dirvote_perform_vote(void); static void dirvote_clear_votes(int all_votes); static int dirvote_compute_consensus(void); static int dirvote_publish_consensus(void); -static char *make_consensus_method_list(int low, int high); +static char *make_consensus_method_list(int low, int high, const char *sep); /** The highest consensus method that we currently support. */ #define MAX_SUPPORTED_CONSENSUS_METHOD 7 @@ -102,7 +102,7 @@ format_networkstatus_vote(crypto_pk_env_t *private_signing_key, char *params; authority_cert_t *cert = v3_ns->cert; char *methods = - make_consensus_method_list(1, MAX_SUPPORTED_CONSENSUS_METHOD); + make_consensus_method_list(1, MAX_SUPPORTED_CONSENSUS_METHOD, " "); format_iso_time(published, v3_ns->published); format_iso_time(va, v3_ns->valid_after); format_iso_time(fu, v3_ns->fresh_until); @@ -480,7 +480,7 @@ consensus_method_is_supported(int method) /** Return a newly allocated string holding the numbers between low and high * (inclusive) that are supported consensus methods. */ static char * -make_consensus_method_list(int low, int high) +make_consensus_method_list(int low, int high, const char *separator) { char *list; @@ -494,7 +494,7 @@ make_consensus_method_list(int low, int high) tor_snprintf(b, sizeof(b), "%d", i); smartlist_add(lst, tor_strdup(b)); } - list = smartlist_join_strings(lst, " ", 0, NULL); + list = smartlist_join_strings(lst, separator, 0, NULL); tor_assert(list); SMARTLIST_FOREACH(lst, char *, cp, tor_free(cp)); smartlist_free(lst); @@ -2389,3 +2389,71 @@ dirvote_get_vote(const char *fp, int flags) return NULL; } +int +dirvote_create_microdescriptor(char *out, size_t outlen, + const routerinfo_t *ri) +{ + char *key = NULL, *summary = NULL, *family = NULL; + size_t keylen; + int result = -1; + char *start = out, *end = out+outlen; + + if (crypto_pk_write_public_key_to_string(ri->onion_pkey, &key, &keylen)<0) + goto done; + summary = policy_summarize(ri->exit_policy); + if (ri->declared_family) + family = smartlist_join_strings(ri->declared_family, " ", 0, NULL); + + if (tor_snprintf(out, end-out, "onion-key\n%s", key)<0) + goto done; + out += strlen(out); + if (family) { + if (tor_snprintf(out, end-out, "family %s\n", family)<0) + goto done; + out += strlen(out); + } + if (summary && strcmp(summary, "reject 1-65535")) { + if (tor_snprintf(out, end-out, "p %s\n", summary)<0) + goto done; + out += strlen(out); + } + result = out - start; + + done: + tor_free(key); + tor_free(summary); + tor_free(family); + return result; +} + +/** Lowest consensus method that generates microdescriptors */ +#define MIN_CM_MICRODESC 7 +/** Cached space-separated string to hold */ +static char *microdesc_consensus_methods = NULL; + +int +dirvote_format_microdescriptor_vote_line(char *out, size_t out_len, + const char *microdesc, + size_t microdescriptor_len) +{ + char d[DIGEST256_LEN]; + char d64[BASE64_DIGEST256_LEN]; + if (!microdesc_consensus_methods) { + microdesc_consensus_methods = + make_consensus_method_list(MIN_CM_MICRODESC, + MAX_SUPPORTED_CONSENSUS_METHOD, + ","); + tor_assert(microdesc_consensus_methods); + } + if (crypto_digest256(d, microdesc, microdescriptor_len, DIGEST_SHA256)<0) + return -1; + if (digest256_to_base64(d64, d)<0) + return -1; + + if (tor_snprintf(out, out_len, "m %s sha256=%s\n", + microdesc_consensus_methods, d64)<0) + return -1; + + return strlen(out); +} + diff --git a/src/or/or.h b/src/or/or.h index d9e883f3e8..e7e3869793 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -3823,6 +3823,12 @@ networkstatus_t * dirserv_generate_networkstatus_vote_obj(crypto_pk_env_t *private_key, authority_cert_t *cert); +int dirvote_create_microdescriptor(char *out, + size_t outlen, const routerinfo_t *ri); +int dirvote_format_microdescriptor_vote_line(char *out, size_t out_len, + const char *microdesc, + size_t microdescriptor_len); + #ifdef DIRVOTE_PRIVATE char *format_networkstatus_vote(crypto_pk_env_t *private_key, networkstatus_t *v3_ns); From a7ba02f3f1dc267be81bf36f16bc1e69b34af050 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 20 Aug 2009 11:39:22 -0400 Subject: [PATCH 07/23] Add ability to parse one or more m line from a vote. --- src/or/or.h | 14 ++++++++++++++ src/or/routerparse.c | 14 ++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/src/or/or.h b/src/or/or.h index e7e3869793..2c795421db 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -1557,6 +1557,14 @@ typedef struct routerstatus_t { } routerstatus_t; +/**DOCDOC*/ +typedef struct microdescriptor_t { + crypto_pk_env_t *onion_pkey; + smartlist_t *family; + char *exitsummary; /**< exit policy summary - + * XXX weasel: this probably should not stay a string. */ +} microdescriptor_t; + /** How many times will we try to download a router's descriptor before giving * up? */ #define MAX_ROUTERDESC_DOWNLOAD_FAILURES 8 @@ -1599,6 +1607,11 @@ typedef struct networkstatus_v2_t { * sorted by identity_digest. */ } networkstatus_v2_t; +typedef struct vote_microdesc_hash_t { + struct vote_microdesc_hash_t *next; + char *microdesc_hash_line; +} vote_microdesc_hash_t; + /** The claim about a single router, made in a vote. */ typedef struct vote_routerstatus_t { routerstatus_t status; /**< Underlying 'status' object for this router. @@ -1607,6 +1620,7 @@ typedef struct vote_routerstatus_t { * networkstatus_t.known_flags. */ char *version; /**< The version that the authority says this router is * running. */ + vote_microdesc_hash_t *microdesc; } vote_routerstatus_t; /** Information about a single voter in a vote or a consensus. */ diff --git a/src/or/routerparse.c b/src/or/routerparse.c index bfcf20b09c..dd0d32ef63 100644 --- a/src/or/routerparse.c +++ b/src/or/routerparse.c @@ -55,6 +55,7 @@ typedef enum { K_S, K_V, K_W, + K_M, K_EVENTDNS, K_EXTRA_INFO, K_EXTRA_INFO_DIGEST, @@ -321,6 +322,7 @@ static token_rule_t rtrstatus_token_table[] = { T1( "s", K_S, ARGS, NO_OBJ ), T01("v", K_V, CONCAT_ARGS, NO_OBJ ), T01("w", K_W, ARGS, NO_OBJ ), + T0N("m", K_M, CONCAT_ARGS, NO_OBJ ), T0N("opt", K_OPT, CONCAT_ARGS, OBJ_OK ), END_OF_TABLE }; @@ -2050,6 +2052,18 @@ routerstatus_parse_entry_from_string(memarea_t *area, rs->has_exitsummary = 1; } + if (vote_rs) { + SMARTLIST_FOREACH_BEGIN(tokens, directory_token_t *, t) { + if (t->tp == K_M && t->n_args) { + vote_microdesc_hash_t *line = + tor_malloc(sizeof(vote_microdesc_hash_t)); + line->next = vote_rs->microdesc; + line->microdesc_hash_line = tor_strdup(t->args[0]); + vote_rs->microdesc = line; + } + } SMARTLIST_FOREACH_END(t); + } + if (!strcasecmp(rs->nickname, UNNAMED_ROUTER_NICKNAME)) rs->is_named = 0; From a8e92ba8fd65e12dbec265ccfcf0c89ac61847f2 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 1 Sep 2009 02:00:43 -0400 Subject: [PATCH 08/23] Add a function to get the most frequent member of a list. --- src/common/container.c | 74 ++++++++++++++++++++++++++++++++++++++++++ src/common/container.h | 9 +++++ 2 files changed, 83 insertions(+) diff --git a/src/common/container.c b/src/common/container.c index 12ac2527e9..4fb94d3f7c 100644 --- a/src/common/container.c +++ b/src/common/container.c @@ -459,6 +459,42 @@ smartlist_sort(smartlist_t *sl, int (*compare)(const void **a, const void **b)) (int (*)(const void *,const void*))compare); } +/** Given a smartlist sl sorted with the function compare, + * return the most frequent member in the list. Break ties in favor of + * later elements. If the list is empty, return NULL. + */ +void * +smartlist_get_most_frequent(const smartlist_t *sl, + int (*compare)(const void **a, const void **b)) +{ + const void *most_frequent = NULL; + int most_frequent_count = 0; + + const void **cur = NULL; + int i, count=0; + + if (!sl->num_used) + return NULL; + for (i = 0; i < sl->num_used; ++i) { + const void *item = sl->list[i]; + if (cur && 0 == compare(cur, &item)) { + ++count; + } else { + if (cur && count >= most_frequent_count) { + most_frequent = *cur; + most_frequent_count = count; + } + cur = &item; + count = 1; + } + } + if (cur && count >= most_frequent_count) { + most_frequent = *cur; + most_frequent_count = count; + } + return (void*)most_frequent; +} + /** Given a sorted smartlist sl and the comparison function used to * sort it, remove all duplicate members. If free_fn is provided, calls * free_fn on each duplicate. Otherwise, just removes them. Preserves order. @@ -550,6 +586,13 @@ smartlist_sort_strings(smartlist_t *sl) smartlist_sort(sl, _compare_string_ptrs); } +/** Return the most frequent string in the sorted list sl */ +char * +smartlist_get_most_frequent_string(smartlist_t *sl) +{ + return smartlist_get_most_frequent(sl, _compare_string_ptrs); +} + /** Remove duplicate strings from a sorted list, and free them with tor_free(). */ void @@ -681,6 +724,37 @@ smartlist_uniq_digests(smartlist_t *sl) smartlist_uniq(sl, _compare_digests, _tor_free); } +/** Helper: compare two DIGEST256_LEN digests. */ +static int +_compare_digests256(const void **_a, const void **_b) +{ + return memcmp((const char*)*_a, (const char*)*_b, DIGEST256_LEN); +} + +/** Sort the list of DIGEST256_LEN-byte digests into ascending order. */ +void +smartlist_sort_digests256(smartlist_t *sl) +{ + smartlist_sort(sl, _compare_digests256); +} + +/** Return the most frequent member of the sorted list of DIGEST256_LEN + * digests in sl */ +char * +smartlist_get_most_frequent_digest256(smartlist_t *sl) +{ + return smartlist_get_most_frequent(sl, _compare_digests256); +} + +/** Remove duplicate 256-bit digests from a sorted list, and free them with + * tor_free(). + */ +void +smartlist_uniq_digests256(smartlist_t *sl) +{ + smartlist_uniq(sl, _compare_digests256, _tor_free); +} + /** Helper: Declare an entry type and a map type to implement a mapping using * ht.h. The map type will be called maptype. The key part of each * entry is declared using the C declaration keydecl. All functions diff --git a/src/common/container.h b/src/common/container.h index 4495a7a273..41c0c68705 100644 --- a/src/common/container.h +++ b/src/common/container.h @@ -93,13 +93,22 @@ void smartlist_del_keeporder(smartlist_t *sl, int idx); void smartlist_insert(smartlist_t *sl, int idx, void *val); void smartlist_sort(smartlist_t *sl, int (*compare)(const void **a, const void **b)); +void *smartlist_get_most_frequent(const smartlist_t *sl, + int (*compare)(const void **a, const void **b)); void smartlist_uniq(smartlist_t *sl, int (*compare)(const void **a, const void **b), void (*free_fn)(void *elt)); + void smartlist_sort_strings(smartlist_t *sl); void smartlist_sort_digests(smartlist_t *sl); +void smartlist_sort_digests256(smartlist_t *sl); + +char *smartlist_get_most_frequent_string(smartlist_t *sl); +char *smartlist_get_most_frequent_digest256(smartlist_t *sl); + void smartlist_uniq_strings(smartlist_t *sl); void smartlist_uniq_digests(smartlist_t *sl); +void smartlist_uniq_digests256(smartlist_t *sl); void *smartlist_bsearch(smartlist_t *sl, const void *key, int (*compare)(const void *key, const void **member)) ATTR_PURE; From e1ddee8bbe724e934fe9a4cb2d290719a7d6105c Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 24 Aug 2009 12:51:33 -0400 Subject: [PATCH 09/23] Code to generate, store, and parse microdescriptors and consensuses. The consensus documents are not signed properly, not served, and not exchanged yet. --- src/common/util.c | 7 ++ src/common/util.h | 1 + src/or/Makefile.am | 1 + src/or/dirserv.c | 35 +++++- src/or/dirvote.c | 200 +++++++++++++++++++++--------- src/or/microdesc.c | 267 +++++++++++++++++++++++++++++++++++++++++ src/or/networkstatus.c | 6 + src/or/or.h | 73 +++++++++-- src/or/routerparse.c | 206 ++++++++++++++++++++++++++++--- src/test/test_dir.c | 9 +- 10 files changed, 712 insertions(+), 93 deletions(-) create mode 100644 src/or/microdesc.c diff --git a/src/common/util.c b/src/common/util.c index d05c308fe8..139c1aaad8 100644 --- a/src/common/util.c +++ b/src/common/util.c @@ -684,6 +684,13 @@ tor_digest_is_zero(const char *digest) return tor_mem_is_zero(digest, DIGEST_LEN); } +/** Return true iff the DIGEST256_LEN bytes in digest are all zero. */ +int +tor_digest256_is_zero(const char *digest) +{ + return tor_mem_is_zero(digest, DIGEST256_LEN); +} + /* Helper: common code to check whether the result of a strtol or strtoul or * strtoll is correct. */ #define CHECK_STRTOX_RESULT() \ diff --git a/src/common/util.h b/src/common/util.h index 28ea8a0488..85234f5157 100644 --- a/src/common/util.h +++ b/src/common/util.h @@ -195,6 +195,7 @@ const char *find_whitespace(const char *s) ATTR_PURE; const char *find_whitespace_eos(const char *s, const char *eos) ATTR_PURE; int tor_mem_is_zero(const char *mem, size_t len) ATTR_PURE; int tor_digest_is_zero(const char *digest) ATTR_PURE; +int tor_digest256_is_zero(const char *digest) ATTR_PURE; char *esc_for_log(const char *string) ATTR_MALLOC; const char *escaped(const char *string); struct smartlist_t; diff --git a/src/or/Makefile.am b/src/or/Makefile.am index 097e3e24de..3dc1889a90 100644 --- a/src/or/Makefile.am +++ b/src/or/Makefile.am @@ -20,6 +20,7 @@ libtor_a_SOURCES = buffers.c circuitbuild.c circuitlist.c \ connection.c connection_edge.c connection_or.c control.c \ cpuworker.c directory.c dirserv.c dirvote.c \ dns.c dnsserv.c geoip.c hibernate.c main.c $(tor_platform_source) \ + microdesc.c \ networkstatus.c onion.c policies.c \ reasons.c relay.c rendcommon.c rendclient.c rendmid.c \ rendservice.c rephist.c router.c routerlist.c routerparse.c \ diff --git a/src/or/dirserv.c b/src/or/dirserv.c index f12ef2f3d5..e7a58edfc1 100644 --- a/src/or/dirserv.c +++ b/src/or/dirserv.c @@ -1901,10 +1901,11 @@ routerstatus_format_entry(char *buf, size_t buf_len, tor_inet_ntoa(&in, ipaddr, sizeof(ipaddr)); r = tor_snprintf(buf, buf_len, - "r %s %s %s %s %s %d %d\n", + "r %s %s %s%s%s %s %d %d\n", rs->nickname, identity64, - digest64, + (format==NS_V3_CONSENSUS_MICRODESC)?"":digest64, + (format==NS_V3_CONSENSUS_MICRODESC)?"":" ", published, ipaddr, (int)rs->or_port, @@ -1918,7 +1919,7 @@ routerstatus_format_entry(char *buf, size_t buf_len, * this here, instead of in the caller. Then we could use the * networkstatus_type_t values, with an additional control port value * added -MP */ - if (format == NS_V3_CONSENSUS) + if (format == NS_V3_CONSENSUS || format == NS_V3_CONSENSUS_MICRODESC) return 0; cp = buf + strlen(buf); @@ -2434,6 +2435,7 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_env_t *private_key, vote_timing_t timing; digestmap_t *omit_as_sybil = NULL; const int vote_on_reachability = running_long_enough_to_decide_unreachable(); + smartlist_t *microdescriptors = NULL; tor_assert(private_key); tor_assert(cert); @@ -2482,11 +2484,13 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_env_t *private_key, omit_as_sybil = get_possible_sybil_list(routers); routerstatuses = smartlist_create(); + microdescriptors = smartlist_create(); - SMARTLIST_FOREACH(routers, routerinfo_t *, ri, { + SMARTLIST_FOREACH_BEGIN(routers, routerinfo_t *, ri) { if (ri->cache_info.published_on >= cutoff) { routerstatus_t *rs; vote_routerstatus_t *vrs; + microdesc_t *md; vrs = tor_malloc_zero(sizeof(vote_routerstatus_t)); rs = &vrs->status; @@ -2501,9 +2505,30 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_env_t *private_key, rs->is_running = 0; vrs->version = version_from_platform(ri->platform); + md = dirvote_create_microdescriptor(ri); + if (md) { + char buf[128]; + vote_microdesc_hash_t *h; + dirvote_format_microdesc_vote_line(buf, sizeof(buf), md); + h = tor_malloc(sizeof(vote_microdesc_hash_t)); + h->microdesc_hash_line = tor_strdup(buf); + h->next = NULL; + vrs->microdesc = h; + md->last_listed = now; + smartlist_add(microdescriptors, md); + } + smartlist_add(routerstatuses, vrs); } - }); + } SMARTLIST_FOREACH_END(ri); + + { + smartlist_t *added = + microdescs_add_list_to_cache(get_microdesc_cache(), + microdescriptors, SAVED_NOWHERE, 0); + smartlist_free(added); + smartlist_free(microdescriptors); + } smartlist_free(routers); digestmap_free(omit_as_sybil, NULL); diff --git a/src/or/dirvote.c b/src/or/dirvote.c index f0f92d2f60..d200344ef8 100644 --- a/src/or/dirvote.c +++ b/src/or/dirvote.c @@ -24,14 +24,20 @@ static int dirvote_publish_consensus(void); static char *make_consensus_method_list(int low, int high, const char *sep); /** The highest consensus method that we currently support. */ -#define MAX_SUPPORTED_CONSENSUS_METHOD 7 +#define MAX_SUPPORTED_CONSENSUS_METHOD 8 #define MIN_METHOD_FOR_PARAMS 7 +/** Lowest consensus method that generates microdescriptors */ +#define MIN_METHOD_FOR_MICRODESC 8 + /* ===== * Voting * =====*/ +/* Overestimated. */ +#define MICRODESC_LINE_LEN 80 + /** Return a new string containing the string representation of the vote in * v3_ns, signed with our v3 signing key private_signing_key. * For v3 authorities. */ @@ -89,7 +95,7 @@ format_networkstatus_vote(crypto_pk_env_t *private_signing_key, len = 8192; len += strlen(version_lines); - len += (RS_ENTRY_LEN)*smartlist_len(rl->routers); + len += (RS_ENTRY_LEN+MICRODESC_LINE_LEN)*smartlist_len(rl->routers); len += v3_ns->cert->cache_info.signed_descriptor_len; status = tor_malloc(len); @@ -158,15 +164,25 @@ format_networkstatus_vote(crypto_pk_env_t *private_signing_key, outp += cert->cache_info.signed_descriptor_len; } - SMARTLIST_FOREACH(v3_ns->routerstatus_list, vote_routerstatus_t *, vrs, - { + SMARTLIST_FOREACH_BEGIN(v3_ns->routerstatus_list, vote_routerstatus_t *, + vrs) { + vote_microdesc_hash_t *h; if (routerstatus_format_entry(outp, endp-outp, &vrs->status, vrs->version, NS_V3_VOTE) < 0) { log_warn(LD_BUG, "Unable to print router status."); goto err; } outp += strlen(outp); - }); + + for (h = vrs->microdesc; h; h = h->next) { + size_t mlen = strlen(h->microdesc_hash_line); + if (outp+mlen >= endp) { + log_warn(LD_BUG, "Can't fit microdesc line in vote."); + } + memcpy(outp, h->microdesc_hash_line, mlen+1); + outp += strlen(outp); + } + } SMARTLIST_FOREACH_END(vrs); { char signing_key_fingerprint[FINGERPRINT_LEN+1]; @@ -189,7 +205,7 @@ format_networkstatus_vote(crypto_pk_env_t *private_signing_key, outp += strlen(outp); } - if (router_get_networkstatus_v3_hash(status, digest)<0) + if (router_get_networkstatus_v3_hash(status, digest, DIGEST_SHA1)<0) goto err; note_crypto_pk_op(SIGN_DIR); if (router_append_dirobj_signature(outp,endp-outp,digest, DIGEST_LEN, @@ -294,34 +310,8 @@ get_frequent_members(smartlist_t *out, smartlist_t *in, int min) /** Given a sorted list of strings lst, return the member that appears * most. Break ties in favor of later-occurring members. */ -static const char * -get_most_frequent_member(smartlist_t *lst) -{ - const char *most_frequent = NULL; - int most_frequent_count = 0; - - const char *cur = NULL; - int count = 0; - - SMARTLIST_FOREACH(lst, const char *, s, - { - if (cur && !strcmp(s, cur)) { - ++count; - } else { - if (count >= most_frequent_count) { - most_frequent = cur; - most_frequent_count = count; - } - cur = s; - count = 1; - } - }); - if (count >= most_frequent_count) { - most_frequent = cur; - most_frequent_count = count; - } - return most_frequent; -} +#define get_most_frequent_member(lst) \ + smartlist_get_most_frequent_string(lst) /** Return 0 if and only if a and b are routerstatuses * that come from the same routerinfo, with the same derived elements. @@ -363,7 +353,8 @@ _compare_vote_rs(const void **_a, const void **_b) * in favor of smaller descriptor digest. */ static vote_routerstatus_t * -compute_routerstatus_consensus(smartlist_t *votes) +compute_routerstatus_consensus(smartlist_t *votes, int consensus_method, + char *microdesc_digest256_out) { vote_routerstatus_t *most = NULL, *cur = NULL; int most_n = 0, cur_n = 0; @@ -399,6 +390,26 @@ compute_routerstatus_consensus(smartlist_t *votes) } tor_assert(most); + + if (consensus_method >= MIN_METHOD_FOR_MICRODESC && + microdesc_digest256_out) { + smartlist_t *digests = smartlist_create(); + const char *best_microdesc_digest; + SMARTLIST_FOREACH(votes, vote_routerstatus_t *, rs, { + char d[DIGEST256_LEN]; + if (compare_vote_rs(rs, most)) + continue; + if (!vote_routerstatus_find_microdesc_hash(d, rs, consensus_method)) + smartlist_add(digests, tor_memdup(d, sizeof(d))); + }); + smartlist_sort_digests256(digests); + best_microdesc_digest = smartlist_get_most_frequent_digest256(digests); + if (best_microdesc_digest) + memcpy(microdesc_digest256_out, best_microdesc_digest, DIGEST256_LEN); + SMARTLIST_FOREACH(digests, char *, cp, tor_free(cp)); + smartlist_free(digests); + } + return most; } @@ -615,16 +626,20 @@ networkstatus_compute_consensus(smartlist_t *votes, crypto_pk_env_t *identity_key, crypto_pk_env_t *signing_key, const char *legacy_id_key_digest, - crypto_pk_env_t *legacy_signing_key) + crypto_pk_env_t *legacy_signing_key, + consensus_flavor_t flavor) { smartlist_t *chunks; char *result = NULL; int consensus_method; - time_t valid_after, fresh_until, valid_until; int vote_seconds, dist_seconds; char *client_versions = NULL, *server_versions = NULL; smartlist_t *flags; + const routerstatus_format_type_t rs_format = + flavor == FLAV_NS ? NS_V3_CONSENSUS : NS_V3_CONSENSUS_MICRODESC; + + tor_assert(flavor == FLAV_NS || flavor == FLAV_MICRODESC); tor_assert(total_authorities >= smartlist_len(votes)); if (!smartlist_len(votes)) { @@ -955,6 +970,7 @@ networkstatus_compute_consensus(smartlist_t *votes, int n_listing = 0; int i; char buf[256]; + char microdesc_digest[DIGEST256_LEN]; /* Of the next-to-be-considered digest in each voter, which is first? */ SMARTLIST_FOREACH(votes, networkstatus_t *, v, { @@ -1021,7 +1037,9 @@ networkstatus_compute_consensus(smartlist_t *votes, /* Figure out the most popular opinion of what the most recent * routerinfo and its contents are. */ - rs = compute_routerstatus_consensus(matching_descs); + memset(microdesc_digest, 0, sizeof(microdesc_digest)); + rs = compute_routerstatus_consensus(matching_descs, consensus_method, + microdesc_digest); /* Copy bits of that into rs_out. */ tor_assert(!memcmp(lowest_id, rs->status.identity_digest, DIGEST_LEN)); memcpy(rs_out.identity_digest, lowest_id, DIGEST_LEN); @@ -1182,9 +1200,19 @@ networkstatus_compute_consensus(smartlist_t *votes, /* Okay!! Now we can write the descriptor... */ /* First line goes into "buf". */ routerstatus_format_entry(buf, sizeof(buf), &rs_out, NULL, - NS_V3_CONSENSUS); + rs_format); smartlist_add(chunks, tor_strdup(buf)); - /* Second line is all flags. The "\n" is missing. */ + /* Now an m line, if applicable. */ + if (flavor == FLAV_MICRODESC && + !tor_digest256_is_zero(microdesc_digest)) { + char m[BASE64_DIGEST256_LEN+1], *cp; + const size_t mlen = BASE64_DIGEST256_LEN+5; + digest256_to_base64(m, microdesc_digest); + cp = tor_malloc(mlen); + tor_snprintf(cp, mlen, "m %s\n", m); + smartlist_add(chunks, cp); + } + /* Next line is all flags. The "\n" is missing. */ smartlist_add(chunks, smartlist_join_strings(chosen_flags, " ", 0, NULL)); /* Now the version line. */ @@ -1206,7 +1234,7 @@ networkstatus_compute_consensus(smartlist_t *votes, }; /* Now the exitpolicy summary line. */ - if (rs_out.has_exitsummary) { + if (rs_out.has_exitsummary && flavor == FLAV_NS) { char buf[MAX_POLICY_LINE_LEN+1]; int r = tor_snprintf(buf, sizeof(buf), "p %s\n", rs_out.exitsummary); if (r<0) { @@ -2090,7 +2118,8 @@ dirvote_compute_consensus(void) consensus_body = networkstatus_compute_consensus( votes, n_voters, my_cert->identity_key, - get_my_v3_authority_signing_key(), legacy_id_digest, legacy_sign); + get_my_v3_authority_signing_key(), legacy_id_digest, legacy_sign, + FLAV_NS); } if (!consensus_body) { log_warn(LD_DIR, "Couldn't generate a consensus at all!"); @@ -2389,14 +2418,21 @@ dirvote_get_vote(const char *fp, int flags) return NULL; } -int -dirvote_create_microdescriptor(char *out, size_t outlen, - const routerinfo_t *ri) +/** Construct and return a new microdescriptor from a routerinfo ri. + * + * XXX Right now, there is only one way to generate microdescriptors from + * router descriptors. This may change in future consensus methods. If so, + * we'll need an internal way to remember which method we used, and ask for a + * particular method. + **/ +microdesc_t * +dirvote_create_microdescriptor(const routerinfo_t *ri) { + microdesc_t *result = NULL; char *key = NULL, *summary = NULL, *family = NULL; + char buf[1024]; size_t keylen; - int result = -1; - char *start = out, *end = out+outlen; + char *out = buf, *end = buf+sizeof(buf); if (crypto_pk_write_public_key_to_string(ri->onion_pkey, &key, &keylen)<0) goto done; @@ -2417,7 +2453,19 @@ dirvote_create_microdescriptor(char *out, size_t outlen, goto done; out += strlen(out); } - result = out - start; + *out = '\0'; /* Make sure it's nul-terminated. This should be a no-op */ + + { + smartlist_t *lst = microdescs_parse_from_string(buf, out, 0, 1); + if (smartlist_len(lst) != 1) { + log_warn(LD_DIR, "We generated a microdescriptor we couldn't parse."); + SMARTLIST_FOREACH(lst, microdesc_t *, md, microdesc_free(md)); + smartlist_free(lst); + goto done; + } + result = smartlist_get(lst, 0); + smartlist_free(lst); + } done: tor_free(key); @@ -2426,28 +2474,23 @@ dirvote_create_microdescriptor(char *out, size_t outlen, return result; } -/** Lowest consensus method that generates microdescriptors */ -#define MIN_CM_MICRODESC 7 /** Cached space-separated string to hold */ static char *microdesc_consensus_methods = NULL; +/** DOCDOC */ int -dirvote_format_microdescriptor_vote_line(char *out, size_t out_len, - const char *microdesc, - size_t microdescriptor_len) +dirvote_format_microdesc_vote_line(char *out, size_t out_len, + const microdesc_t *md) { - char d[DIGEST256_LEN]; char d64[BASE64_DIGEST256_LEN]; if (!microdesc_consensus_methods) { microdesc_consensus_methods = - make_consensus_method_list(MIN_CM_MICRODESC, + make_consensus_method_list(MIN_METHOD_FOR_MICRODESC, MAX_SUPPORTED_CONSENSUS_METHOD, ","); tor_assert(microdesc_consensus_methods); } - if (crypto_digest256(d, microdesc, microdescriptor_len, DIGEST_SHA256)<0) - return -1; - if (digest256_to_base64(d64, d)<0) + if (digest256_to_base64(d64, md->digest)<0) return -1; if (tor_snprintf(out, out_len, "m %s sha256=%s\n", @@ -2457,3 +2500,44 @@ dirvote_format_microdescriptor_vote_line(char *out, size_t out_len, return strlen(out); } +/** DOCDOC */ +int +vote_routerstatus_find_microdesc_hash(char *digest256_out, + const vote_routerstatus_t *vrs, + int method) +{ + /* XXXX only returns the sha256 method. */ + const vote_microdesc_hash_t *h; + char mstr[64]; + size_t mlen; + + tor_snprintf(mstr, sizeof(mstr), "%d", method); + mlen = strlen(mstr); + + for (h = vrs->microdesc; h; h = h->next) { + const char *cp = h->microdesc_hash_line; + size_t num_len; + /* cp looks like \d+(,\d+)* (digesttype=val )+ . Let's hunt for mstr in + * the first part. */ + while (1) { + num_len = strspn(cp, "1234567890"); + if (num_len == mlen && !memcmp(mstr, cp, mlen)) { + /* This is the line. */ + char buf[BASE64_DIGEST256_LEN+1]; + /* XXXX ignores extraneous stuff if the digest is too long. This + * seems harmless enough, right? */ + cp = strstr(cp, " sha256="); + if (!cp) + return -1; + cp += strlen(" sha256="); + strlcpy(buf, cp, sizeof(buf)); + return digest256_from_base64(digest256_out, buf); + } + if (num_len == 0 || cp[num_len] != ',') + break; + cp += num_len + 1; + } + } + return -1; +} + diff --git a/src/or/microdesc.c b/src/or/microdesc.c new file mode 100644 index 0000000000..0128fbbab2 --- /dev/null +++ b/src/or/microdesc.c @@ -0,0 +1,267 @@ +/* Copyright (c) 2009, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "or.h" + +/** DOCDOC everything here. */ + +#define MICRODESC_IN_CONSENSUS 1 +#define MICRODESC_IN_VOTE 2 + +struct microdesc_cache_t { + HT_HEAD(microdesc_map, microdesc_t) map; + + char *cache_fname; + char *journal_fname; + tor_mmap_t *cache_content; + size_t journal_len; +}; + +static INLINE unsigned int +_microdesc_hash(microdesc_t *md) +{ + unsigned *d = (unsigned*)md->digest; +#if SIZEOF_INT == 4 + return d[0] ^ d[1] ^ d[2] ^ d[3] ^ d[4] ^ d[5] ^ d[6] ^ d[7]; +#else + return d[0] ^ d[1] ^ d[2] ^ d[3]; +#endif +} + +static INLINE int +_microdesc_eq(microdesc_t *a, microdesc_t *b) +{ + return !memcmp(a->digest, b->digest, DIGEST256_LEN); +} + +HT_PROTOTYPE(microdesc_map, microdesc_t, node, + _microdesc_hash, _microdesc_eq); +HT_GENERATE(microdesc_map, microdesc_t, node, + _microdesc_hash, _microdesc_eq, 0.6, + _tor_malloc, _tor_realloc, _tor_free); + +static int +dump_microdescriptor(FILE *f, microdesc_t *md) +{ + /* XXXX drops unkown annotations. */ + if (md->last_listed) { + char buf[ISO_TIME_LEN+1]; + format_iso_time(buf, md->last_listed); + fprintf(f, "@last-listed %s\n", buf); + } + + md->off = (off_t) ftell(f); + fwrite(md->body, 1, md->bodylen, f); + return 0; +} + +static microdesc_cache_t *the_microdesc_cache = NULL; + +microdesc_cache_t * +get_microdesc_cache(void) +{ + if (PREDICT_UNLIKELY(the_microdesc_cache==NULL)) { + microdesc_cache_t *cache = tor_malloc_zero(sizeof(microdesc_cache_t)); + HT_INIT(microdesc_map, &cache->map); + cache->cache_fname = get_datadir_fname("cached-microdescs"); + cache->journal_fname = get_datadir_fname("cached-microdescs.new"); + microdesc_cache_reload(cache); + the_microdesc_cache = cache; + } + return the_microdesc_cache; +} + +/* There are three sources of microdescriptors: + 1) Generated us while acting as a directory authority. + 2) Loaded from the cache on disk. + 3) Downloaded. +*/ + +/* Returns list of added microdesc_t. */ +smartlist_t * +microdescs_add_to_cache(microdesc_cache_t *cache, + const char *s, const char *eos, saved_location_t where, + int no_save) +{ + /*XXXX need an argument that sets last_listed as appropriate. */ + + smartlist_t *descriptors, *added; + const int allow_annotations = (where != SAVED_NOWHERE); + const int copy_body = (where != SAVED_IN_CACHE); + + descriptors = microdescs_parse_from_string(s, eos, + allow_annotations, + copy_body); + + added = microdescs_add_list_to_cache(cache, descriptors, where, no_save); + smartlist_free(descriptors); + return added; +} + +/* Returns list of added microdesc_t. Frees any not added. */ +smartlist_t * +microdescs_add_list_to_cache(microdesc_cache_t *cache, + smartlist_t *descriptors, saved_location_t where, + int no_save) +{ + smartlist_t *added; + open_file_t *open_file = NULL; + FILE *f = NULL; + // int n_added = 0; + + if (where == SAVED_NOWHERE && !no_save) { + f = start_writing_to_stdio_file(cache->journal_fname, OPEN_FLAGS_APPEND, + 0600, &open_file); + if (!f) + log_warn(LD_DIR, "Couldn't append to journal in %s", + cache->journal_fname); + } + + added = smartlist_create(); + SMARTLIST_FOREACH_BEGIN(descriptors, microdesc_t *, md) { + microdesc_t *md2; + md2 = HT_FIND(microdesc_map, &cache->map, md); + if (md2) { + /* We already had this one. */ + if (md2->last_listed < md->last_listed) + md2->last_listed = md->last_listed; + microdesc_free(md); + continue; + } + + /* Okay, it's a new one. */ + if (f) { + dump_microdescriptor(f, md); + md->saved_location = SAVED_IN_JOURNAL; + } else { + md->saved_location = where; + } + + md->no_save = no_save; + + HT_INSERT(microdesc_map, &cache->map, md); + smartlist_add(added, md); + } SMARTLIST_FOREACH_END(md); + + finish_writing_to_file(open_file); /*XXX Check me.*/ + return added; +} + +void +microdesc_cache_clear(microdesc_cache_t *cache) +{ + microdesc_t **entry, **next; + for (entry = HT_START(microdesc_map, &cache->map); entry; entry = next) { + next = HT_NEXT_RMV(microdesc_map, &cache->map, entry); + microdesc_free(*entry); + } + if (cache->cache_content) { + tor_munmap_file(cache->cache_content); + cache->cache_content = NULL; + } +} + +int +microdesc_cache_reload(microdesc_cache_t *cache) +{ + struct stat st; + char *journal_content; + smartlist_t *added; + tor_mmap_t *mm; + int total = 0; + + microdesc_cache_clear(cache); + + mm = cache->cache_content = tor_mmap_file(cache->cache_fname); + if (mm) { + added = microdescs_add_to_cache(cache, mm->data, mm->data+mm->size, + SAVED_IN_CACHE, 0); + total += smartlist_len(added); + smartlist_free(added); + } + + journal_content = read_file_to_str(cache->journal_fname, + RFTS_IGNORE_MISSING, &st); + if (journal_content) { + added = microdescs_add_to_cache(cache, journal_content, + journal_content+st.st_size, + SAVED_IN_JOURNAL, 0); + total += smartlist_len(added); + smartlist_free(added); + tor_free(journal_content); + } + log_notice(LD_DIR, "Reloaded microdescriptor cache. Found %d descriptors.", + total); + return 0; +} + +int +microdesc_cache_rebuild(microdesc_cache_t *cache) +{ + open_file_t *open_file; + FILE *f; + microdesc_t **mdp; + smartlist_t *wrote; + + f = start_writing_to_stdio_file(cache->cache_fname, OPEN_FLAGS_REPLACE, + 0600, &open_file); + if (!f) + return -1; + + wrote = smartlist_create(); + + HT_FOREACH(mdp, microdesc_map, &cache->map) { + microdesc_t *md = *mdp; + if (md->no_save) + continue; + + dump_microdescriptor(f, md); + if (md->saved_location != SAVED_IN_CACHE) { + tor_free(md->body); + md->saved_location = SAVED_IN_CACHE; + } + + smartlist_add(wrote, md); + } + + finish_writing_to_file(open_file); /*XXX Check me.*/ + + if (cache->cache_content) + tor_munmap_file(cache->cache_content); + cache->cache_content = tor_mmap_file(cache->cache_fname); + if (!cache->cache_content && smartlist_len(wrote)) { + log_err(LD_DIR, "Couldn't map file that we just wrote to %s!", + cache->cache_fname); + return -1; + } + SMARTLIST_FOREACH_BEGIN(wrote, microdesc_t *, md) { + if (md->no_save) + continue; + tor_assert(md->saved_location == SAVED_IN_CACHE); + md->body = (char*)cache->cache_content->data + md->off; + tor_assert(!memcmp(md->body, "onion-key", 9)); + } SMARTLIST_FOREACH_END(wrote); + + smartlist_free(wrote); + + return 0; +} + +void +microdesc_free(microdesc_t *md) +{ + /* Must be removed from hash table! */ + if (md->onion_pkey) + crypto_free_pk_env(md->onion_pkey); + if (md->body && md->saved_location != SAVED_IN_CACHE) + tor_free(md->body); + + if (md->family) { + SMARTLIST_FOREACH(md->family, char *, cp, tor_free(cp)); + smartlist_free(md->family); + } + tor_free(md->exitsummary); + + tor_free(md); +} + diff --git a/src/or/networkstatus.c b/src/or/networkstatus.c index 5d1f8b24a3..752cb42124 100644 --- a/src/or/networkstatus.c +++ b/src/or/networkstatus.c @@ -242,8 +242,14 @@ router_reload_consensus_networkstatus(void) static void vote_routerstatus_free(vote_routerstatus_t *rs) { + vote_microdesc_hash_t *h, *next; tor_free(rs->version); tor_free(rs->status.exitsummary); + for (h = rs->microdesc; h; h = next) { + tor_free(h->microdesc_hash_line); + next = h->next; + tor_free(h); + } tor_free(rs); } diff --git a/src/or/or.h b/src/or/or.h index 2c795421db..6a7aec6738 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -89,6 +89,7 @@ #include "torgzip.h" #include "address.h" #include "compat_libevent.h" +#include "ht.h" /* These signals are defined to help control_signal_act work. */ @@ -1558,12 +1559,29 @@ typedef struct routerstatus_t { } routerstatus_t; /**DOCDOC*/ -typedef struct microdescriptor_t { +typedef struct microdesc_t { + HT_ENTRY(microdesc_t) node; + + /* Cache information */ + + time_t last_listed; + saved_location_t saved_location : 3; + unsigned int no_save : 1; + off_t off; + + /* The string containing the microdesc. */ + + char *body; + size_t bodylen; + char digest[DIGEST256_LEN]; + + /* Fields in the microdescriptor. */ + crypto_pk_env_t *onion_pkey; smartlist_t *family; char *exitsummary; /**< exit policy summary - - * XXX weasel: this probably should not stay a string. */ -} microdescriptor_t; + * XXX this probably should not stay a string. */ +} microdesc_t; /** How many times will we try to download a router's descriptor before giving * up? */ @@ -3748,7 +3766,8 @@ int dirserv_have_any_serverdesc(smartlist_t *fps, int spool_src); size_t dirserv_estimate_data_size(smartlist_t *fps, int is_serverdescs, int compressed); typedef enum { - NS_V2, NS_V3_CONSENSUS, NS_V3_VOTE, NS_CONTROL_PORT + NS_V2, NS_V3_CONSENSUS, NS_V3_VOTE, NS_CONTROL_PORT, + NS_V3_CONSENSUS_MICRODESC } routerstatus_format_type_t; int routerstatus_format_entry(char *buf, size_t buf_len, routerstatus_t *rs, const char *platform, @@ -3784,13 +3803,20 @@ int dirserv_read_measured_bandwidths(const char *from_file, void dirvote_free_all(void); +/** DOCDOC */ +typedef enum { + FLAV_NS, + FLAV_MICRODESC, +} consensus_flavor_t; + /* vote manipulation */ char *networkstatus_compute_consensus(smartlist_t *votes, int total_authorities, crypto_pk_env_t *identity_key, crypto_pk_env_t *signing_key, const char *legacy_identity_key_digest, - crypto_pk_env_t *legacy_signing_key); + crypto_pk_env_t *legacy_signing_key, + consensus_flavor_t flavor); int networkstatus_add_detached_signatures(networkstatus_t *target, ns_detached_signatures_t *sigs, const char **msg_out); @@ -3837,11 +3863,12 @@ networkstatus_t * dirserv_generate_networkstatus_vote_obj(crypto_pk_env_t *private_key, authority_cert_t *cert); -int dirvote_create_microdescriptor(char *out, - size_t outlen, const routerinfo_t *ri); -int dirvote_format_microdescriptor_vote_line(char *out, size_t out_len, - const char *microdesc, - size_t microdescriptor_len); +microdesc_t *dirvote_create_microdescriptor(const routerinfo_t *ri); +int dirvote_format_microdesc_vote_line(char *out, size_t out_len, + const microdesc_t *md); +int vote_routerstatus_find_microdesc_hash(char *digest256_out, + const vote_routerstatus_t *vrs, + int method); #ifdef DIRVOTE_PRIVATE char *format_networkstatus_vote(crypto_pk_env_t *private_key, @@ -4051,6 +4078,25 @@ void do_hash_password(void); int tor_init(int argc, char **argv); #endif +/********************************* microdesc.c *************************/ + +typedef struct microdesc_cache_t microdesc_cache_t; + +microdesc_cache_t *get_microdesc_cache(void); + +smartlist_t *microdescs_add_to_cache(microdesc_cache_t *cache, + const char *s, const char *eos, saved_location_t where, + int no_save); +smartlist_t *microdescs_add_list_to_cache(microdesc_cache_t *cache, + smartlist_t *descriptors, saved_location_t where, + int no_save); + +int microdesc_cache_rebuild(microdesc_cache_t *cache); +int microdesc_cache_reload(microdesc_cache_t *cache); +void microdesc_cache_clear(microdesc_cache_t *cache); + +void microdesc_free(microdesc_t *md); + /********************************* networkstatus.c *********************/ /** How old do we allow a v2 network-status to get before removing it @@ -4927,7 +4973,8 @@ int router_get_router_hash(const char *s, char *digest); int router_get_dir_hash(const char *s, char *digest); int router_get_runningrouters_hash(const char *s, char *digest); int router_get_networkstatus_v2_hash(const char *s, char *digest); -int router_get_networkstatus_v3_hash(const char *s, char *digest); +int router_get_networkstatus_v3_hash(const char *s, char *digest, + digest_algorithm_t algorithm); int router_get_extrainfo_hash(const char *s, char *digest); int router_append_dirobj_signature(char *buf, size_t buf_len, const char *digest, @@ -4971,6 +5018,10 @@ networkstatus_t *networkstatus_parse_vote_from_string(const char *s, ns_detached_signatures_t *networkstatus_parse_detached_signatures( const char *s, const char *eos); +smartlist_t *microdescs_parse_from_string(const char *s, const char *eos, + int allow_annotations, + int copy_body); + authority_cert_t *authority_cert_parse_from_string(const char *s, const char **end_of_string); int rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out, diff --git a/src/or/routerparse.c b/src/or/routerparse.c index dd0d32ef63..277c7c6c91 100644 --- a/src/or/routerparse.c +++ b/src/or/routerparse.c @@ -111,6 +111,7 @@ typedef enum { K_LEGACY_DIR_KEY, A_PURPOSE, + A_LAST_LISTED, _A_UNKNOWN, R_RENDEZVOUS_SERVICE_DESCRIPTOR, @@ -496,6 +497,14 @@ static token_rule_t networkstatus_detached_signature_token_table[] = { END_OF_TABLE }; +static token_rule_t microdesc_token_table[] = { + T1_START("onion-key", K_ONION_KEY, NO_ARGS, NEED_KEY_1024), + T01("family", K_FAMILY, ARGS, NO_OBJ ), + T01("p", K_P, CONCAT_ARGS, NO_OBJ ), + A01("@last-listed", A_LAST_LISTED, CONCAT_ARGS, NO_OBJ ), + END_OF_TABLE +}; + #undef T /* static function prototypes */ @@ -505,7 +514,8 @@ static addr_policy_t *router_parse_addr_policy_private(directory_token_t *tok); static int router_get_hash_impl(const char *s, char *digest, const char *start_str, const char *end_str, - char end_char); + char end_char, + digest_algorithm_t alg); static void token_free(directory_token_t *tok); static smartlist_t *find_all_exitpolicy(smartlist_t *s); static directory_token_t *_find_by_keyword(smartlist_t *s, @@ -586,7 +596,8 @@ int router_get_dir_hash(const char *s, char *digest) { return router_get_hash_impl(s,digest, - "signed-directory","\ndirectory-signature",'\n'); + "signed-directory","\ndirectory-signature",'\n', + DIGEST_SHA1); } /** Set digest to the SHA-1 digest of the hash of the first router in @@ -596,7 +607,8 @@ int router_get_router_hash(const char *s, char *digest) { return router_get_hash_impl(s,digest, - "router ","\nrouter-signature", '\n'); + "router ","\nrouter-signature", '\n', + DIGEST_SHA1); } /** Set digest to the SHA-1 digest of the hash of the running-routers @@ -606,7 +618,8 @@ int router_get_runningrouters_hash(const char *s, char *digest) { return router_get_hash_impl(s,digest, - "network-status","\ndirectory-signature", '\n'); + "network-status","\ndirectory-signature", '\n', + DIGEST_SHA1); } /** Set digest to the SHA-1 digest of the hash of the network-status @@ -616,17 +629,19 @@ router_get_networkstatus_v2_hash(const char *s, char *digest) { return router_get_hash_impl(s,digest, "network-status-version","\ndirectory-signature", - '\n'); + '\n', + DIGEST_SHA1); } /** Set digest to the SHA-1 digest of the hash of the network-status * string in s. Return 0 on success, -1 on failure. */ int -router_get_networkstatus_v3_hash(const char *s, char *digest) +router_get_networkstatus_v3_hash(const char *s, char *digest, + digest_algorithm_t alg) { return router_get_hash_impl(s,digest, "network-status-version","\ndirectory-signature", - ' '); + ' ', alg); } /** Set digest to the SHA-1 digest of the hash of the extrainfo @@ -634,7 +649,8 @@ router_get_networkstatus_v3_hash(const char *s, char *digest) int router_get_extrainfo_hash(const char *s, char *digest) { - return router_get_hash_impl(s,digest,"extra-info","\nrouter-signature",'\n'); + return router_get_hash_impl(s,digest,"extra-info","\nrouter-signature",'\n', + DIGEST_SHA1); } /** Helper: used to generate signatures for routers, directories and @@ -643,6 +659,8 @@ router_get_extrainfo_hash(const char *s, char *digest) * surround it with -----BEGIN/END----- pairs, and write it to the * buf_len-byte buffer at buf. Return 0 on success, -1 on * failure. + * + * DOCDOC alg */ int router_append_dirobj_signature(char *buf, size_t buf_len, const char *digest, @@ -1691,7 +1709,7 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string) goto err; } if (router_get_hash_impl(s, digest, "dir-key-certificate-version", - "\ndir-key-certification", '\n') < 0) + "\ndir-key-certification", '\n', DIGEST_SHA1) < 0) goto err; tok = smartlist_get(tokens, 0); if (tok->tp != K_DIR_KEY_CERTIFICATE_VERSION || strcmp(tok->args[0], "3")) { @@ -2298,7 +2316,7 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, if (eos_out) *eos_out = NULL; - if (router_get_networkstatus_v3_hash(s, ns_digest)) { + if (router_get_networkstatus_v3_hash(s, ns_digest, DIGEST_SHA1)) { log_warn(LD_DIR, "Unable to compute digest of network-status"); goto err; } @@ -3410,7 +3428,7 @@ find_all_exitpolicy(smartlist_t *s) return out; } -/** Compute the SHA-1 digest of the substring of s taken from the first +/** Compute the digest of the substring of s taken from the first * occurrence of start_str through the first instance of c after the * first subsequent occurrence of end_str; store the 20-byte result in * digest; return 0 on success. @@ -3420,7 +3438,8 @@ find_all_exitpolicy(smartlist_t *s) static int router_get_hash_impl(const char *s, char *digest, const char *start_str, - const char *end_str, char end_c) + const char *end_str, char end_c, + digest_algorithm_t alg) { char *start, *end; start = strstr(s, start_str); @@ -3446,14 +3465,169 @@ router_get_hash_impl(const char *s, char *digest, } ++end; - if (crypto_digest(digest, start, end-start)) { - log_warn(LD_BUG,"couldn't compute digest"); - return -1; + if (alg == DIGEST_SHA1) { + if (crypto_digest(digest, start, end-start)) { + log_warn(LD_BUG,"couldn't compute digest"); + return -1; + } + } else { + if (crypto_digest256(digest, start, end-start, alg)) { + log_warn(LD_BUG,"couldn't compute digest"); + return -1; + } } return 0; } +/** DOCDOC Assuming that s starts with a microdesc, return the start of the + * *NEXT* one. */ +static const char * +find_start_of_next_microdesc(const char *s, const char *eos) +{ + int started_with_annotations; + s = eat_whitespace_eos(s, eos); + if (!s) + return NULL; + +#define CHECK_LENGTH() STMT_BEGIN \ + if (s+32 > eos) \ + return NULL; \ + STMT_END + +#define NEXT_LINE() STMT_BEGIN \ + s = memchr(s, '\n', eos-s); \ + if (!s || s+1 >= eos) \ + return NULL; \ + s++; \ + STMT_END + + CHECK_LENGTH(); + + started_with_annotations = (*s == '@'); + + if (started_with_annotations) { + /* Start by advancing to the first non-annotation line. */ + while (*s == '@') + NEXT_LINE(); + } + CHECK_LENGTH(); + + /* Now we should be pointed at an onion-key line. If we are, then skip + * it. */ + if (!strcmpstart(s, "onion-key")) + NEXT_LINE(); + + /* Okay, now we're pointed at the first line of the microdescriptor which is + not an annotation or onion-key. The next line that _is_ an annotation or + onion-key is the start of the next microdescriptor. */ + while (s+32 < eos) { + if (*s == '@' || !strcmpstart(s, "onion-key")) + return s; + NEXT_LINE(); + } + return NULL; + +#undef CHECK_LENGTH +#undef NEXT_LINE +} + +/**DOCDOC*/ +smartlist_t * +microdescs_parse_from_string(const char *s, const char *eos, + int allow_annotations, int copy_body) +{ + smartlist_t *tokens; + smartlist_t *result; + microdesc_t *md = NULL; + memarea_t *area; + const char *start = s; + const char *start_of_next_microdesc; + int flags = allow_annotations ? TS_ANNOTATIONS_OK : 0; + + directory_token_t *tok; + + if (!eos) + eos = s + strlen(s); + + s = eat_whitespace_eos(s, eos); + area = memarea_new(); + result = smartlist_create(); + tokens = smartlist_create(); + + while (s < eos) { + start_of_next_microdesc = find_start_of_next_microdesc(s, eos); + if (!start_of_next_microdesc) + start_of_next_microdesc = eos; + + if (tokenize_string(area, s, start_of_next_microdesc, tokens, + microdesc_token_table, flags)) { + log_warn(LD_DIR, "Unparseable microdescriptor"); + goto next; + } + + md = tor_malloc_zero(sizeof(microdesc_t)); + { + const char *cp = tor_memstr(s, start_of_next_microdesc-s, + "onion-key"); + tor_assert(cp); + + md->bodylen = start_of_next_microdesc - cp; + if (copy_body) + md->body = tor_strndup(cp, md->bodylen); + else + md->body = (char*)cp; + md->off = cp - start; + } + + if ((tok = find_opt_by_keyword(tokens, A_LAST_LISTED))) { + if (parse_iso_time(tok->args[0], &md->last_listed)) { + log_warn(LD_DIR, "Bad last-listed time in microdescriptor"); + goto next; + } + } + + tok = find_by_keyword(tokens, K_ONION_KEY); + md->onion_pkey = tok->key; + tok->key = NULL; + + if ((tok = find_opt_by_keyword(tokens, K_FAMILY))) { + int i; + md->family = smartlist_create(); + for (i=0;in_args;++i) { + if (!is_legal_nickname_or_hexdigest(tok->args[i])) { + log_warn(LD_DIR, "Illegal nickname %s in family line", + escaped(tok->args[i])); + goto next; + } + smartlist_add(md->family, tor_strdup(tok->args[i])); + } + } + + if ((tok = find_opt_by_keyword(tokens, K_P))) { + md->exitsummary = tor_strdup(tok->args[0]); + } + + crypto_digest256(md->digest, md->body, md->bodylen, DIGEST_SHA256); + + smartlist_add(result, md); + + md = NULL; + next: + if (md) + microdesc_free(md); + + memarea_clear(area); + smartlist_clear(tokens); + s = start_of_next_microdesc; + } + + memarea_drop_all(area); + smartlist_free(tokens); + + return result; +} + /** Parse the Tor version of the platform string platform, * and compare it to the version in cutoff. Return 1 if * the router is at least as new as the cutoff, else return 0. @@ -3712,7 +3886,7 @@ rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out, /* Compute descriptor hash for later validation. */ if (router_get_hash_impl(desc, desc_hash, "rendezvous-service-descriptor ", - "\nsignature", '\n') < 0) { + "\nsignature", '\n', DIGEST_SHA1) < 0) { log_warn(LD_REND, "Couldn't compute descriptor hash."); goto err; } diff --git a/src/test/test_dir.c b/src/test/test_dir.c index a10493e7fa..8e566e2eec 100644 --- a/src/test/test_dir.c +++ b/src/test/test_dir.c @@ -858,7 +858,8 @@ test_dir_v3_networkstatus(void) cert3->identity_key, sign_skey_3, "AAAAAAAAAAAAAAAAAAAA", - sign_skey_leg1); + sign_skey_leg1, + FLAV_NS); test_assert(consensus_text); con = networkstatus_parse_vote_from_string(consensus_text, NULL, NS_TYPE_CONSENSUS); @@ -966,11 +967,13 @@ test_dir_v3_networkstatus(void) smartlist_shuffle(votes); consensus_text2 = networkstatus_compute_consensus(votes, 3, cert2->identity_key, - sign_skey_2, NULL,NULL); + sign_skey_2, NULL,NULL, + FLAV_NS); smartlist_shuffle(votes); consensus_text3 = networkstatus_compute_consensus(votes, 3, cert1->identity_key, - sign_skey_1, NULL,NULL); + sign_skey_1, NULL,NULL, + FLAV_NS); test_assert(consensus_text2); test_assert(consensus_text3); con2 = networkstatus_parse_vote_from_string(consensus_text2, NULL, From 3b2fc659a8ef83feedadcda32de49db06b80af10 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 16 Sep 2009 17:01:01 -0400 Subject: [PATCH 10/23] Refactor consensus signature storage for multiple digests and flavors. This patch introduces a new type called document_signature_t to represent the signature of a consensus document. Now, each consensus document can have up to one document signature per voter per digest algorithm. Also, each detached-signatures document can have up to one signature per . --- src/common/crypto.c | 33 ++++++ src/common/crypto.h | 21 +++- src/or/directory.c | 10 +- src/or/dirserv.c | 7 +- src/or/dirvote.c | 213 ++++++++++++++++++++++++-------------- src/or/networkstatus.c | 159 +++++++++++++++++++---------- src/or/or.h | 78 +++++++++----- src/or/routerlist.c | 74 +++++++------- src/or/routerparse.c | 226 +++++++++++++++++++++++++++-------------- src/test/test_dir.c | 74 ++++++++------ 10 files changed, 589 insertions(+), 306 deletions(-) diff --git a/src/common/crypto.c b/src/common/crypto.c index 21c8aed2d4..ac0e628c48 100644 --- a/src/common/crypto.c +++ b/src/common/crypto.c @@ -1448,6 +1448,39 @@ crypto_digest256(char *digest, const char *m, size_t len, return (SHA256((const unsigned char*)m,len,(unsigned char*)digest) == NULL); } +/** Set the digests_t in ds_out to contain every digest on the + * len bytes in m that we know how to compute. Return 0 on + * success, -1 on failure. */ +int +crypto_digest_all(digests_t *ds_out, const char *m, size_t len) +{ + digest_algorithm_t i; + tor_assert(ds_out); + memset(ds_out, 0, sizeof(*ds_out)); + if (crypto_digest(ds_out->d[DIGEST_SHA1], m, len) < 0) + return -1; + for (i = DIGEST_SHA256; i < N_DIGEST_ALGORITHMS; ++i) { + if (crypto_digest256(ds_out->d[i], m, len, i) < 0) + return -1; + } + return 0; +} + +/** Return the name of an algorithm, as used in directory documents. */ +const char * +crypto_digest_algorithm_get_name(digest_algorithm_t alg) +{ + switch (alg) { + case DIGEST_SHA1: + return "sha1"; + case DIGEST_SHA256: + return "sha256"; + default: + tor_fragile_assert(); + return "??unknown_digest??"; + } +} + /** Intermediate information about the digest of a stream of data. */ struct crypto_digest_env_t { union { diff --git a/src/common/crypto.h b/src/common/crypto.h index 63ea96d056..ed8468046f 100644 --- a/src/common/crypto.h +++ b/src/common/crypto.h @@ -58,9 +58,22 @@ #define HEX_DIGEST256_LEN 64 typedef enum { - DIGEST_SHA1, - DIGEST_SHA256, + DIGEST_SHA1 = 0, + DIGEST_SHA256 = 1, } digest_algorithm_t; +#define N_DIGEST_ALGORITHMS (DIGEST_SHA256+1) + +/** A set of all the digests we know how to compute, taken on a single + * string. Any digests that are shorter than 256 bits are right-padded + * with 0 bits. + * + * Note that this representation wastes 12 bytes for the SHA1 case, so + * don't use it for anything where we need to allocate a whole bunch at + * once. + **/ +typedef struct { + char d[N_DIGEST_ALGORITHMS][DIGEST256_LEN]; +} digests_t; typedef struct crypto_pk_env_t crypto_pk_env_t; typedef struct crypto_cipher_env_t crypto_cipher_env_t; @@ -158,10 +171,12 @@ int crypto_cipher_decrypt_with_iv(crypto_cipher_env_t *env, char *to, size_t tolen, const char *from, size_t fromlen); -/* SHA-1 */ +/* SHA-1 and other digests. */ int crypto_digest(char *digest, const char *m, size_t len); int crypto_digest256(char *digest, const char *m, size_t len, digest_algorithm_t algorithm); +int crypto_digest_all(digests_t *ds_out, const char *m, size_t len); +const char *crypto_digest_algorithm_get_name(digest_algorithm_t alg); crypto_digest_env_t *crypto_new_digest_env(void); crypto_digest_env_t *crypto_new_digest256_env(digest_algorithm_t algorithm); void crypto_free_digest_env(crypto_digest_env_t *digest); diff --git a/src/or/directory.c b/src/or/directory.c index 5fe2de4eee..19ef6351c2 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -2330,7 +2330,7 @@ client_likes_consensus(networkstatus_t *v, const char *want_url) dir_split_resource_into_fingerprints(want_url, want_authorities, NULL, 0, 0); need_at_least = smartlist_len(want_authorities)/2+1; - SMARTLIST_FOREACH(want_authorities, const char *, d, { + SMARTLIST_FOREACH_BEGIN(want_authorities, const char *, d) { char want_digest[DIGEST_LEN]; size_t want_len = strlen(d)/2; if (want_len > DIGEST_LEN) @@ -2341,18 +2341,18 @@ client_likes_consensus(networkstatus_t *v, const char *want_url) continue; }; - SMARTLIST_FOREACH(v->voters, networkstatus_voter_info_t *, vi, { - if (vi->signature && + SMARTLIST_FOREACH_BEGIN(v->voters, networkstatus_voter_info_t *, vi) { + if (smartlist_len(vi->sigs) && !memcmp(vi->identity_digest, want_digest, want_len)) { have++; break; }; - }); + } SMARTLIST_FOREACH_END(vi); /* early exit, if we already have enough */ if (have >= need_at_least) break; - }); + } SMARTLIST_FOREACH_END(d); SMARTLIST_FOREACH(want_authorities, char *, d, tor_free(d)); smartlist_free(want_authorities); diff --git a/src/or/dirserv.c b/src/or/dirserv.c index e7a58edfc1..326a801c41 100644 --- a/src/or/dirserv.c +++ b/src/or/dirserv.c @@ -2597,12 +2597,17 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_env_t *private_key, voter = tor_malloc_zero(sizeof(networkstatus_voter_info_t)); voter->nickname = tor_strdup(options->Nickname); memcpy(voter->identity_digest, identity_digest, DIGEST_LEN); + voter->sigs = smartlist_create(); + { + document_signature_t *sig = tor_malloc_zero(sizeof(document_signature_t)); + memcpy(sig->identity_digest, identity_digest, DIGEST_LEN); + memcpy(sig->signing_key_digest, signing_key_digest, DIGEST_LEN); + } voter->address = hostname; voter->addr = addr; voter->dir_port = options->DirPort; voter->or_port = options->ORPort; voter->contact = tor_strdup(contact); - memcpy(voter->signing_key_digest, signing_key_digest, DIGEST_LEN); if (options->V3AuthUseLegacyKey) { authority_cert_t *c = get_my_v3_legacy_cert(); if (c) { diff --git a/src/or/dirvote.c b/src/or/dirvote.c index d200344ef8..1bdf4cc403 100644 --- a/src/or/dirvote.c +++ b/src/or/dirvote.c @@ -251,6 +251,19 @@ get_voter(const networkstatus_t *vote) return smartlist_get(vote->voters, 0); } +/** DOCDOC */ +document_signature_t * +voter_get_sig_by_algorithm(const networkstatus_voter_info_t *voter, + digest_algorithm_t alg) +{ + if (!voter->sigs) + return NULL; + SMARTLIST_FOREACH(voter->sigs, document_signature_t *, sig, + if (sig->alg == alg) + return sig); + return NULL; +} + /** Temporary structure used in constructing a list of dir-source entries * for a consensus. One of these is generated for every vote, and one more * for every legacy key in each vote. */ @@ -782,8 +795,7 @@ networkstatus_compute_consensus(smartlist_t *votes, /* Add the authority sections. */ { smartlist_t *dir_sources = smartlist_create(); - SMARTLIST_FOREACH(votes, networkstatus_t *, v, - { + SMARTLIST_FOREACH_BEGIN(votes, networkstatus_t *, v) { dir_src_ent_t *e = tor_malloc_zero(sizeof(dir_src_ent_t)); e->v = v; e->digest = get_voter(v)->identity_digest; @@ -797,7 +809,7 @@ networkstatus_compute_consensus(smartlist_t *votes, e_legacy->is_legacy = 1; smartlist_add(dir_sources, e_legacy); } - }); + } SMARTLIST_FOREACH_END(v); smartlist_sort(dir_sources, _compare_dir_src_ents_by_authority_id); SMARTLIST_FOREACH(dir_sources, const dir_src_ent_t *, e, @@ -1347,84 +1359,123 @@ networkstatus_add_detached_signatures(networkstatus_t *target, const char **msg_out) { int r = 0; + const char *flavor; + smartlist_t *siglist; tor_assert(sigs); tor_assert(target); tor_assert(target->type == NS_TYPE_CONSENSUS); + flavor = networkstatus_get_flavor_name(target->flavor); + /* Do the times seem right? */ if (target->valid_after != sigs->valid_after) { + puts("A"); *msg_out = "Valid-After times do not match " "when adding detached signatures to consensus"; return -1; } if (target->fresh_until != sigs->fresh_until) { + puts("B"); *msg_out = "Fresh-until times do not match " "when adding detached signatures to consensus"; return -1; } if (target->valid_until != sigs->valid_until) { + puts("C"); *msg_out = "Valid-until times do not match " "when adding detached signatures to consensus"; return -1; } - /* Are they the same consensus? */ - if (memcmp(target->networkstatus_digest, sigs->networkstatus_digest, - DIGEST_LEN)) { - *msg_out = "Digest mismatch when adding detached signatures to consensus"; + siglist = strmap_get(sigs->signatures, flavor); + if (!siglist) { + puts("D"); + *msg_out = "No signatures for given consensus flavor"; return -1; } - /* For each voter in src... */ - SMARTLIST_FOREACH_BEGIN(sigs->signatures, networkstatus_voter_info_t *, - src_voter) { - char voter_identity[HEX_DIGEST_LEN+1]; - networkstatus_voter_info_t *target_voter = - networkstatus_get_voter_by_id(target, src_voter->identity_digest); - authority_cert_t *cert = NULL; - - base16_encode(voter_identity, sizeof(voter_identity), - src_voter->identity_digest, DIGEST_LEN); - log_info(LD_DIR, "Looking at signature from %s", voter_identity); - /* If the target doesn't know about this voter, then forget it. */ - if (!target_voter) { - log_info(LD_DIR, "We do not know about %s", voter_identity); - continue; - } - - /* If the target already has a good signature from this voter, then skip - * this one. */ - if (target_voter->good_signature) { - log_info(LD_DIR, "We already have a good signature from %s", - voter_identity); - continue; - } - - /* Try checking the signature if we haven't already. */ - if (!src_voter->good_signature && !src_voter->bad_signature) { - cert = authority_cert_get_by_digests(src_voter->identity_digest, - src_voter->signing_key_digest); - if (cert) { - networkstatus_check_voter_signature(target, src_voter, cert); + /** Make sure all the digests we know match, and at least one matches. */ + { + digests_t *digests = strmap_get(sigs->digests, flavor); + int n_matches = 0; + digest_algorithm_t alg; + if (!digests) { + puts("D"); + *msg_out = "No digests for given consensus flavor"; + return -1; + } + for (alg = DIGEST_SHA1; alg < N_DIGEST_ALGORITHMS; ++alg) { + if (!tor_mem_is_zero(digests->d[alg], DIGEST256_LEN)) { + if (!memcmp(target->digests.d[alg], digests->d[alg], DIGEST256_LEN)) { + ++n_matches; + } else { + printf("F %d\n", alg); + printf("%s\n", hex_str(target->digests.d[alg], DIGEST256_LEN)); + printf("%s\n", hex_str(digests->d[alg], DIGEST256_LEN)); + *msg_out = "Mismatched digest."; + return -1; } } + } + if (!n_matches) { + puts("G"); + *msg_out = "No regognized digests for given consensus flavor"; + } + } - /* If this signature is good, or we don't have any signature yet, - * then add it. */ - if (src_voter->good_signature || !target_voter->signature) { - log_info(LD_DIR, "Adding signature from %s", voter_identity); - ++r; - tor_free(target_voter->signature); - target_voter->signature = - tor_memdup(src_voter->signature, src_voter->signature_len); - memcpy(target_voter->signing_key_digest, src_voter->signing_key_digest, - DIGEST_LEN); - target_voter->signature_len = src_voter->signature_len; - target_voter->good_signature = src_voter->good_signature; - target_voter->bad_signature = src_voter->bad_signature; - } else { - log_info(LD_DIR, "Not adding signature from %s", voter_identity); + /* For each voter in src... */ + SMARTLIST_FOREACH_BEGIN(siglist, document_signature_t *, sig) { + char voter_identity[HEX_DIGEST_LEN+1]; + networkstatus_voter_info_t *target_voter = + networkstatus_get_voter_by_id(target, sig->identity_digest); + authority_cert_t *cert = NULL; + const char *algorithm; + document_signature_t *old_sig = NULL; + + algorithm = crypto_digest_algorithm_get_name(sig->alg); + + base16_encode(voter_identity, sizeof(voter_identity), + sig->identity_digest, DIGEST_LEN); + log_info(LD_DIR, "Looking at signature from %s using %s", voter_identity, + algorithm); + /* If the target doesn't know about this voter, then forget it. */ + if (!target_voter) { + log_info(LD_DIR, "We do not know any voter with ID %s", voter_identity); + continue; + } + + old_sig = voter_get_sig_by_algorithm(target_voter, sig->alg); + + /* If the target already has a good signature from this voter, then skip + * this one. */ + if (old_sig && old_sig->good_signature) { + log_info(LD_DIR, "We already have a good signature from %s using %s", + voter_identity, algorithm); + continue; + } + + /* Try checking the signature if we haven't already. */ + if (!sig->good_signature && !sig->bad_signature) { + cert = authority_cert_get_by_digests(sig->identity_digest, + sig->signing_key_digest); + if (cert) + networkstatus_check_document_signature(target, sig, cert); + } + + /* If this signature is good, or we don't have any signature yet, + * then maybe add it. */ + if (sig->good_signature || !old_sig || old_sig->bad_signature) { + log_info(LD_DIR, "Adding signature from %s with %s", voter_identity, + algorithm); + ++r; + if (old_sig) { + smartlist_remove(target_voter->sigs, old_sig); + document_signature_free(old_sig); } - } SMARTLIST_FOREACH_END(src_voter); + smartlist_add(target_voter->sigs, document_signature_dup(sig)); + } else { + log_info(LD_DIR, "Not adding signature from %s", voter_identity); + } + } SMARTLIST_FOREACH_END(sig); return r; } @@ -1441,6 +1492,8 @@ networkstatus_get_detached_signatures(networkstatus_t *consensus) tor_assert(consensus); tor_assert(consensus->type == NS_TYPE_CONSENSUS); + tor_assert(consensus->flavor == FLAV_NS); + elements = smartlist_create(); { @@ -1448,7 +1501,7 @@ networkstatus_get_detached_signatures(networkstatus_t *consensus) vu_buf[ISO_TIME_LEN+1]; char d[HEX_DIGEST_LEN+1]; - base16_encode(d, sizeof(d), consensus->networkstatus_digest, DIGEST_LEN); + base16_encode(d, sizeof(d), consensus->digests.d[DIGEST_SHA1], DIGEST_LEN); format_iso_time(va_buf, consensus->valid_after); format_iso_time(fu_buf, consensus->fresh_until); format_iso_time(vu_buf, consensus->valid_until); @@ -1461,26 +1514,26 @@ networkstatus_get_detached_signatures(networkstatus_t *consensus) smartlist_add(elements, tor_strdup(buf)); } - SMARTLIST_FOREACH(consensus->voters, networkstatus_voter_info_t *, v, - { + SMARTLIST_FOREACH_BEGIN(consensus->voters, networkstatus_voter_info_t *, v) { + SMARTLIST_FOREACH_BEGIN(v->sigs, document_signature_t *, sig) { char sk[HEX_DIGEST_LEN+1]; char id[HEX_DIGEST_LEN+1]; - if (!v->signature || v->bad_signature) + if (!sig->signature || sig->bad_signature || sig->alg != DIGEST_SHA1) continue; ++n_sigs; - base16_encode(sk, sizeof(sk), v->signing_key_digest, DIGEST_LEN); - base16_encode(id, sizeof(id), v->identity_digest, DIGEST_LEN); + base16_encode(sk, sizeof(sk), sig->signing_key_digest, DIGEST_LEN); + base16_encode(id, sizeof(id), sig->identity_digest, DIGEST_LEN); tor_snprintf(buf, sizeof(buf), "directory-signature %s %s\n-----BEGIN SIGNATURE-----\n", id, sk); smartlist_add(elements, tor_strdup(buf)); - base64_encode(buf, sizeof(buf), v->signature, v->signature_len); + base64_encode(buf, sizeof(buf), sig->signature, sig->signature_len); strlcat(buf, "-----END SIGNATURE-----\n", sizeof(buf)); smartlist_add(elements, tor_strdup(buf)); - }); + } SMARTLIST_FOREACH_END(sig); + } SMARTLIST_FOREACH_END(v); result = smartlist_join_strings(elements, "", 0, NULL); - SMARTLIST_FOREACH(elements, char *, cp, tor_free(cp)); smartlist_free(elements); if (!n_sigs) @@ -1493,13 +1546,15 @@ void ns_detached_signatures_free(ns_detached_signatures_t *s) { if (s->signatures) { - SMARTLIST_FOREACH(s->signatures, networkstatus_voter_info_t *, v, - { - tor_free(v->signature); - tor_free(v); - }); - smartlist_free(s->signatures); + STRMAP_FOREACH(s->signatures, flavor, smartlist_t *, sigs) { + SMARTLIST_FOREACH(sigs, document_signature_t *, sig, + document_signature_free(sig)); + smartlist_free(sigs); + } STRMAP_FOREACH_END; + strmap_free(s->signatures, NULL); + strmap_free(s->digests, _tor_free); } + tor_free(s); } @@ -1936,7 +1991,13 @@ dirvote_add_vote(const char *vote_body, const char **msg_out, int *status_out) } tor_assert(smartlist_len(vote->voters) == 1); vi = get_voter(vote); - tor_assert(vi->good_signature == 1); + { + int any_sig_good = 0; + SMARTLIST_FOREACH(vi->sigs, document_signature_t *, sig, + if (sig->good_signature) + any_sig_good = 1); + tor_assert(any_sig_good); + } ds = trusteddirserver_get_by_v3_auth_digest(vi->identity_digest); if (!ds) { char *keys = list_v3_auth_ids(); @@ -2218,8 +2279,12 @@ dirvote_add_signatures_to_pending_consensus( goto err; } - log_info(LD_DIR, "Have %d signatures for adding to consensus.", - smartlist_len(sigs->signatures)); + { + smartlist_t *sig_list = strmap_get(sigs->signatures, + networkstatus_get_flavor_name(pending_consensus->flavor)); + log_info(LD_DIR, "Have %d signatures for adding to consensus.", + sig_list ? smartlist_len(sig_list) : 0); + } r = networkstatus_add_detached_signatures(pending_consensus, sigs, msg_out); log_info(LD_DIR,"Added %d signatures to consensus.", r); @@ -2406,12 +2471,12 @@ dirvote_get_vote(const char *fp, int flags) } else { if (pending_vote_list && include_pending) { SMARTLIST_FOREACH(pending_vote_list, pending_vote_t *, pv, - if (!memcmp(pv->vote->networkstatus_digest, fp, DIGEST_LEN)) + if (!memcmp(pv->vote->digests.d[DIGEST_SHA1], fp, DIGEST_LEN)) return pv->vote_body); } if (previous_vote_list && include_previous) { SMARTLIST_FOREACH(previous_vote_list, pending_vote_t *, pv, - if (!memcmp(pv->vote->networkstatus_digest, fp, DIGEST_LEN)) + if (!memcmp(pv->vote->digests.d[DIGEST_SHA1], fp, DIGEST_LEN)) return pv->vote_body); } } diff --git a/src/or/networkstatus.c b/src/or/networkstatus.c index 752cb42124..616726525b 100644 --- a/src/or/networkstatus.c +++ b/src/or/networkstatus.c @@ -279,7 +279,25 @@ networkstatus_v2_free(networkstatus_v2_t *ns) tor_free(ns); } -/** Clear all storage held in ns. */ +/** Free all storage held in sig */ +void +document_signature_free(document_signature_t *sig) +{ + tor_free(sig->signature); + tor_free(sig); +} + +/** Return a newly allocated copy of sig */ +document_signature_t * +document_signature_dup(const document_signature_t *sig) +{ + document_signature_t *r = tor_memdup(sig, sizeof(document_signature_t)); + if (r->signature) + r->signature = tor_memdup(sig->signature, sig->signature_len); + return r; +} + +/** Free all storage held in ns. */ void networkstatus_vote_free(networkstatus_t *ns) { @@ -301,14 +319,17 @@ networkstatus_vote_free(networkstatus_t *ns) smartlist_free(ns->supported_methods); } if (ns->voters) { - SMARTLIST_FOREACH(ns->voters, networkstatus_voter_info_t *, voter, - { + SMARTLIST_FOREACH_BEGIN(ns->voters, networkstatus_voter_info_t *, voter) { tor_free(voter->nickname); tor_free(voter->address); tor_free(voter->contact); - tor_free(voter->signature); + if (voter->sigs) { + SMARTLIST_FOREACH(voter->sigs, document_signature_t *, sig, + document_signature_free(sig)); + smartlist_free(voter->sigs); + } tor_free(voter); - }); + } SMARTLIST_FOREACH_END(voter); smartlist_free(ns->voters); } if (ns->cert) @@ -347,34 +368,38 @@ networkstatus_get_voter_by_id(networkstatus_t *vote, return NULL; } -/** Check whether the signature on voter is correctly signed by - * the signing key of cert. Return -1 if cert doesn't match the +/** Check whether the signature sig is correctly signed with the + * signing key in cert. Return -1 if cert doesn't match the * signing key; otherwise set the good_signature or bad_signature flag on * voter, and return 0. */ -/* (private; exposed for testing.) */ int -networkstatus_check_voter_signature(networkstatus_t *consensus, - networkstatus_voter_info_t *voter, - authority_cert_t *cert) +networkstatus_check_document_signature(const networkstatus_t *consensus, + document_signature_t *sig, + const authority_cert_t *cert) { - char d[DIGEST_LEN]; + char key_digest[DIGEST_LEN]; + const int dlen = sig->alg == DIGEST_SHA1 ? DIGEST_LEN : DIGEST256_LEN; char *signed_digest; size_t signed_digest_len; - if (crypto_pk_get_digest(cert->signing_key, d)<0) + + if (crypto_pk_get_digest(cert->signing_key, key_digest)<0) return -1; - if (memcmp(voter->signing_key_digest, d, DIGEST_LEN)) + if (memcmp(sig->signing_key_digest, key_digest, DIGEST_LEN) || + memcmp(sig->identity_digest, cert->cache_info.identity_digest, + DIGEST_LEN)) return -1; + signed_digest_len = crypto_pk_keysize(cert->signing_key); signed_digest = tor_malloc(signed_digest_len); if (crypto_pk_public_checksig(cert->signing_key, signed_digest, - voter->signature, - voter->signature_len) != DIGEST_LEN || - memcmp(signed_digest, consensus->networkstatus_digest, DIGEST_LEN)) { + sig->signature, + sig->signature_len) < dlen || + memcmp(signed_digest, consensus->digests.d[sig->alg], dlen)) { log_warn(LD_DIR, "Got a bad signature on a networkstatus vote"); - voter->bad_signature = 1; + sig->bad_signature = 1; } else { - voter->good_signature = 1; + sig->good_signature = 1; } tor_free(signed_digest); return 0; @@ -407,37 +432,52 @@ networkstatus_check_consensus_signature(networkstatus_t *consensus, tor_assert(consensus->type == NS_TYPE_CONSENSUS); - SMARTLIST_FOREACH(consensus->voters, networkstatus_voter_info_t *, voter, - { - if (!voter->good_signature && !voter->bad_signature && voter->signature) { - /* we can try to check the signature. */ - int is_v3_auth = trusteddirserver_get_by_v3_auth_digest( - voter->identity_digest) != NULL; - authority_cert_t *cert = - authority_cert_get_by_digests(voter->identity_digest, - voter->signing_key_digest); - if (!is_v3_auth) { - smartlist_add(unrecognized, voter); - ++n_unknown; - continue; - } else if (!cert || cert->expires < now) { - smartlist_add(need_certs_from, voter); - ++n_missing_key; - continue; + SMARTLIST_FOREACH_BEGIN(consensus->voters, networkstatus_voter_info_t *, + voter) { + int good_here = 0; + int bad_here = 0; + int missing_key_here = 0; + SMARTLIST_FOREACH_BEGIN(voter->sigs, document_signature_t *, sig) { + if (!sig->good_signature && !sig->bad_signature && + sig->signature) { + /* we can try to check the signature. */ + int is_v3_auth = trusteddirserver_get_by_v3_auth_digest( + sig->identity_digest) != NULL; + authority_cert_t *cert = + authority_cert_get_by_digests(sig->identity_digest, + sig->signing_key_digest); + tor_assert(!memcmp(sig->identity_digest, voter->identity_digest, + DIGEST_LEN)); + + if (!is_v3_auth) { + smartlist_add(unrecognized, voter); + ++n_unknown; + continue; + } else if (!cert || cert->expires < now) { + smartlist_add(need_certs_from, voter); + ++missing_key_here; + continue; + } + if (networkstatus_check_document_signature(consensus, sig, cert) < 0) { + smartlist_add(need_certs_from, voter); + ++missing_key_here; + continue; + } } - if (networkstatus_check_voter_signature(consensus, voter, cert) < 0) { - smartlist_add(need_certs_from, voter); - ++n_missing_key; - continue; - } - } - if (voter->good_signature) + if (sig->good_signature) + ++good_here; + else if (sig->bad_signature) + ++bad_here; + } SMARTLIST_FOREACH_END(sig); + if (good_here) ++n_good; - else if (voter->bad_signature) + else if (bad_here) ++n_bad; + else if (missing_key_here) + ++n_missing_key; else ++n_no_signature; - }); + } SMARTLIST_FOREACH_END(voter); /* Now see whether we're missing any voters entirely. */ SMARTLIST_FOREACH(router_get_trusted_dir_servers(), @@ -1434,8 +1474,7 @@ networkstatus_set_current_consensus(const char *consensus, unsigned flags) } if (current_consensus && - !memcmp(c->networkstatus_digest, current_consensus->networkstatus_digest, - DIGEST_LEN)) { + !memcmp(&c->digests, ¤t_consensus->digests, sizeof(c->digests))) { /* We already have this one. That's a failure. */ log_info(LD_DIR, "Got a consensus we already have"); goto done; @@ -1669,10 +1708,8 @@ download_status_map_update_from_v2_networkstatus(void) v2_download_status_map = digestmap_new(); dl_status = digestmap_new(); - SMARTLIST_FOREACH(networkstatus_v2_list, networkstatus_v2_t *, ns, - { - SMARTLIST_FOREACH(ns->entries, routerstatus_t *, rs, - { + SMARTLIST_FOREACH_BEGIN(networkstatus_v2_list, networkstatus_v2_t *, ns) { + SMARTLIST_FOREACH_BEGIN(ns->entries, routerstatus_t *, rs) { const char *d = rs->descriptor_digest; download_status_t *s; if (digestmap_get(dl_status, d)) @@ -1681,8 +1718,8 @@ download_status_map_update_from_v2_networkstatus(void) s = tor_malloc_zero(sizeof(download_status_t)); } digestmap_set(dl_status, d, s); - }); - }); + } SMARTLIST_FOREACH_END(rs); + } SMARTLIST_FOREACH_END(ns); digestmap_free(v2_download_status_map, _tor_free); v2_download_status_map = dl_status; networkstatus_v2_list_has_changed = 0; @@ -1930,6 +1967,22 @@ networkstatus_get_param(networkstatus_t *ns, const char *param_name, return default_val; } +/** Return the name of the consensus flavor flav as used to identify + * the flavor in directory documents. */ +const char * +networkstatus_get_flavor_name(consensus_flavor_t flav) +{ + switch (flav) { + case FLAV_NS: + return "ns"; + case FLAV_MICRODESC: + return "microdesc"; + default: + tor_fragile_assert(); + return "??"; + } +} + /** If question is a string beginning with "ns/" in a format the * control interface expects for a GETINFO question, set *answer to a * newly-allocated string containing networkstatus lines for the appropriate diff --git a/src/or/or.h b/src/or/or.h index 6a7aec6738..e2050ecae4 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -1641,29 +1641,42 @@ typedef struct vote_routerstatus_t { vote_microdesc_hash_t *microdesc; } vote_routerstatus_t; +/** A signature of some document by an authority. */ +typedef struct document_signature_t { + /** Declared SHA-1 digest of this voter's identity key */ + char identity_digest[DIGEST_LEN]; + /** Declared SHA-1 digest of signing key used by this voter. */ + char signing_key_digest[DIGEST_LEN]; + /** Algorithm used to compute the digest of the document. */ + digest_algorithm_t alg; + /** Signature of the signed thing. */ + char *signature; + /** Length of signature */ + int signature_len; + unsigned int bad_signature : 1; /**< Set to true if we've tried to verify + * the sig, and we know it's bad. */ + unsigned int good_signature : 1; /**< Set to true if we've verified the sig + * as good. */ +} document_signature_t; + /** Information about a single voter in a vote or a consensus. */ typedef struct networkstatus_voter_info_t { + /** Declared SHA-1 digest of this voter's identity key */ + char identity_digest[DIGEST_LEN]; char *nickname; /**< Nickname of this voter */ - char identity_digest[DIGEST_LEN]; /**< Digest of this voter's identity key */ + /** Digest of this voter's "legacy" identity key, if any. In vote only; for + * consensuses, we treat legacy keys as additional signers. */ + char legacy_id_digest[DIGEST_LEN]; char *address; /**< Address of this voter, in string format. */ uint32_t addr; /**< Address of this voter, in IPv4, in host order. */ uint16_t dir_port; /**< Directory port of this voter */ uint16_t or_port; /**< OR port of this voter */ char *contact; /**< Contact information for this voter. */ char vote_digest[DIGEST_LEN]; /**< Digest of this voter's vote, as signed. */ - /** Digest of this voter's "legacy" identity key, if any. In vote only; for - * consensuses, we treat legacy keys as additional signers. */ - char legacy_id_digest[DIGEST_LEN]; /* Nothing from here on is signed. */ - char signing_key_digest[DIGEST_LEN]; /**< Declared digest of signing key - * used by this voter. */ - char *signature; /**< Signature from this voter. */ - int signature_len; /**< Length of signature */ - unsigned int bad_signature : 1; /**< Set to true if we've tried to verify - * the sig, and we know it's bad. */ - unsigned int good_signature : 1; /**< Set to true if we've verified the sig - * as good. */ + /** The signature of the document and the signature's status. */ + smartlist_t *sigs; } networkstatus_voter_info_t; /** Enumerates the possible seriousness values of a networkstatus document. */ @@ -1673,10 +1686,17 @@ typedef enum { NS_TYPE_OPINION, } networkstatus_type_t; +/** DOCDOC */ +typedef enum { + FLAV_NS, + FLAV_MICRODESC, +} consensus_flavor_t; + /** A common structure to hold a v3 network status vote, or a v3 network * status consensus. */ typedef struct networkstatus_t { - networkstatus_type_t type; /**< Vote, consensus, or opinion? */ + networkstatus_type_t type : 8; /**< Vote, consensus, or opinion? */ + consensus_flavor_t flavor : 8; /**< If a consensus, what kind? */ time_t published; /**< Vote only: Time when vote was written. */ time_t valid_after; /**< Time after which this vote or consensus applies. */ time_t fresh_until; /**< Time before which this is the most recent vote or @@ -1715,8 +1735,8 @@ typedef struct networkstatus_t { struct authority_cert_t *cert; /**< Vote only: the voter's certificate. */ - /** Digest of this document, as signed. */ - char networkstatus_digest[DIGEST_LEN]; + /** Digests of this document, as signed. */ + digests_t digests; /** List of router statuses, sorted by identity digest. For a vote, * the elements are vote_routerstatus_t; for a consensus, the elements @@ -1728,14 +1748,15 @@ typedef struct networkstatus_t { digestmap_t *desc_digest_map; } networkstatus_t; -/** A set of signatures for a networkstatus consensus. All fields are as for - * networkstatus_t. */ +/** A set of signatures for a networkstatus consensus. Unless otherwise + * noted, all fields are as for networkstatus_t. */ typedef struct ns_detached_signatures_t { time_t valid_after; time_t fresh_until; time_t valid_until; - char networkstatus_digest[DIGEST_LEN]; - smartlist_t *signatures; /* list of networkstatus_voter_info_t */ + strmap_t *digests; /**< Map from flavor name to digestset_t */ + strmap_t *signatures; /**< Map from flavor name to list of + * document_signature_t */ } ns_detached_signatures_t; /** Allowable types of desc_store_t. */ @@ -3803,12 +3824,6 @@ int dirserv_read_measured_bandwidths(const char *from_file, void dirvote_free_all(void); -/** DOCDOC */ -typedef enum { - FLAV_NS, - FLAV_MICRODESC, -} consensus_flavor_t; - /* vote manipulation */ char *networkstatus_compute_consensus(smartlist_t *votes, int total_authorities, @@ -3869,6 +3884,9 @@ int dirvote_format_microdesc_vote_line(char *out, size_t out_len, int vote_routerstatus_find_microdesc_hash(char *digest256_out, const vote_routerstatus_t *vrs, int method); +document_signature_t *voter_get_sig_by_algorithm( + const networkstatus_voter_info_t *voter, + digest_algorithm_t alg); #ifdef DIRVOTE_PRIVATE char *format_networkstatus_vote(crypto_pk_env_t *private_key, @@ -4134,9 +4152,9 @@ networkstatus_voter_info_t *networkstatus_get_voter_by_id( const char *identity); int networkstatus_check_consensus_signature(networkstatus_t *consensus, int warn); -int networkstatus_check_voter_signature(networkstatus_t *consensus, - networkstatus_voter_info_t *voter, - authority_cert_t *cert); +int networkstatus_check_document_signature(const networkstatus_t *consensus, + document_signature_t *sig, + const authority_cert_t *cert); char *networkstatus_get_cache_filename(const char *identity_digest); int router_set_networkstatus_v2(const char *s, time_t arrived_at, v2_networkstatus_source_t source, @@ -4189,6 +4207,9 @@ int32_t networkstatus_get_param(networkstatus_t *ns, const char *param_name, int32_t default_val); int getinfo_helper_networkstatus(control_connection_t *conn, const char *question, char **answer); +const char *networkstatus_get_flavor_name(consensus_flavor_t flav); +void document_signature_free(document_signature_t *sig); +document_signature_t *document_signature_dup(const document_signature_t *sig); void networkstatus_free_all(void); /********************************* ntmain.c ***************************/ @@ -4975,6 +4996,7 @@ int router_get_runningrouters_hash(const char *s, char *digest); int router_get_networkstatus_v2_hash(const char *s, char *digest); int router_get_networkstatus_v3_hash(const char *s, char *digest, digest_algorithm_t algorithm); +int router_get_networkstatus_v3_hashes(const char *s, digests_t *digests); int router_get_extrainfo_hash(const char *s, char *digest); int router_append_dirobj_signature(char *buf, size_t buf_len, const char *digest, diff --git a/src/or/routerlist.c b/src/or/routerlist.c index 0a32f78a69..5ae40ddd67 100644 --- a/src/or/routerlist.c +++ b/src/or/routerlist.c @@ -448,17 +448,18 @@ authority_certs_fetch_missing(networkstatus_t *status, time_t now) list_pending_downloads(pending, DIR_PURPOSE_FETCH_CERTIFICATE, "fp/"); if (status) { - SMARTLIST_FOREACH(status->voters, networkstatus_voter_info_t *, voter, - { - if (tor_digest_is_zero(voter->signing_key_digest)) - continue; /* This authority never signed this consensus, so don't - * go looking for a cert with key digest 0000000000. */ - if (!cache && - !trusteddirserver_get_by_v3_auth_digest(voter->identity_digest)) - continue; /* We are not a cache, and we don't know this authority.*/ - cl = get_cert_list(voter->identity_digest); + SMARTLIST_FOREACH_BEGIN(status->voters, networkstatus_voter_info_t *, + voter) { + if (!smartlist_len(voter->sigs)) + continue; /* This authority never signed this consensus, so don't + * go looking for a cert with key digest 0000000000. */ + if (!cache && + !trusteddirserver_get_by_v3_auth_digest(voter->identity_digest)) + continue; /* We are not a cache, and we don't know this authority.*/ + cl = get_cert_list(voter->identity_digest); + SMARTLIST_FOREACH_BEGIN(voter->sigs, document_signature_t *, sig) { cert = authority_cert_get_by_digests(voter->identity_digest, - voter->signing_key_digest); + sig->signing_key_digest); if (cert) { if (now < cert->expires) download_status_reset(&cl->dl_status); @@ -469,37 +470,36 @@ authority_certs_fetch_missing(networkstatus_t *status, time_t now) !digestmap_get(pending, voter->identity_digest)) { log_notice(LD_DIR, "We're missing a certificate from authority " "with signing key %s: launching request.", - hex_str(voter->signing_key_digest, DIGEST_LEN)); - smartlist_add(missing_digests, voter->identity_digest); + hex_str(sig->signing_key_digest, DIGEST_LEN)); + smartlist_add(missing_digests, sig->identity_digest); } - }); + } SMARTLIST_FOREACH_END(sig); + } SMARTLIST_FOREACH_END(voter); } - SMARTLIST_FOREACH(trusted_dir_servers, trusted_dir_server_t *, ds, - { - int found = 0; - if (!(ds->type & V3_AUTHORITY)) - continue; - if (smartlist_digest_isin(missing_digests, ds->v3_identity_digest)) - continue; - cl = get_cert_list(ds->v3_identity_digest); - SMARTLIST_FOREACH(cl->certs, authority_cert_t *, cert, - { - if (!ftime_definitely_after(now, cert->expires)) { - /* It's not expired, and we weren't looking for something to - * verify a consensus with. Call it done. */ - download_status_reset(&cl->dl_status); - found = 1; - break; - } - }); - if (!found && - download_status_is_ready(&cl->dl_status, now,MAX_CERT_DL_FAILURES) && - !digestmap_get(pending, ds->v3_identity_digest)) { - log_notice(LD_DIR, "No current certificate known for authority %s; " - "launching request.", ds->nickname); - smartlist_add(missing_digests, ds->v3_identity_digest); + SMARTLIST_FOREACH_BEGIN(trusted_dir_servers, trusted_dir_server_t *, ds) { + int found = 0; + if (!(ds->type & V3_AUTHORITY)) + continue; + if (smartlist_digest_isin(missing_digests, ds->v3_identity_digest)) + continue; + cl = get_cert_list(ds->v3_identity_digest); + SMARTLIST_FOREACH(cl->certs, authority_cert_t *, cert, { + if (!ftime_definitely_after(now, cert->expires)) { + /* It's not expired, and we weren't looking for something to + * verify a consensus with. Call it done. */ + download_status_reset(&cl->dl_status); + found = 1; + break; } }); + if (!found && + download_status_is_ready(&cl->dl_status, now,MAX_CERT_DL_FAILURES) && + !digestmap_get(pending, ds->v3_identity_digest)) { + log_notice(LD_DIR, "No current certificate known for authority %s; " + "launching request.", ds->nickname); + smartlist_add(missing_digests, ds->v3_identity_digest); + } + } SMARTLIST_FOREACH_END(ds); if (!smartlist_len(missing_digests)) { goto done; diff --git a/src/or/routerparse.c b/src/or/routerparse.c index 277c7c6c91..285a550acd 100644 --- a/src/or/routerparse.c +++ b/src/or/routerparse.c @@ -516,6 +516,9 @@ static int router_get_hash_impl(const char *s, char *digest, const char *start_str, const char *end_str, char end_char, digest_algorithm_t alg); +static int router_get_hashes_impl(const char *s, digests_t *digests, + const char *start_str, const char *end_str, + char end_char); static void token_free(directory_token_t *tok); static smartlist_t *find_all_exitpolicy(smartlist_t *s); static directory_token_t *_find_by_keyword(smartlist_t *s, @@ -633,6 +636,16 @@ router_get_networkstatus_v2_hash(const char *s, char *digest) DIGEST_SHA1); } +/** DOCDOC */ +int +router_get_networkstatus_v3_hashes(const char *s, digests_t *digests) +{ + return router_get_hashes_impl(s,digests, + "network-status-version", + "\ndirectory-signature", + ' '); +} + /** Set digest to the SHA-1 digest of the hash of the network-status * string in s. Return 0 on success, -1 on failure. */ int @@ -640,7 +653,8 @@ router_get_networkstatus_v3_hash(const char *s, char *digest, digest_algorithm_t alg) { return router_get_hash_impl(s,digest, - "network-status-version","\ndirectory-signature", + "network-status-version", + "\ndirectory-signature", ' ', alg); } @@ -2304,7 +2318,7 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, smartlist_t *rs_tokens = NULL, *footer_tokens = NULL; networkstatus_voter_info_t *voter = NULL; networkstatus_t *ns = NULL; - char ns_digest[DIGEST_LEN]; + digests_t ns_digests; const char *cert, *end_of_header, *end_of_footer, *s_dup = s; directory_token_t *tok; int ok; @@ -2316,7 +2330,7 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, if (eos_out) *eos_out = NULL; - if (router_get_networkstatus_v3_hash(s, ns_digest, DIGEST_SHA1)) { + if (router_get_networkstatus_v3_hashes(s, &ns_digests)) { log_warn(LD_DIR, "Unable to compute digest of network-status"); goto err; } @@ -2332,7 +2346,7 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, } ns = tor_malloc_zero(sizeof(networkstatus_t)); - memcpy(ns->networkstatus_digest, ns_digest, DIGEST_LEN); + memcpy(&ns->digests, &ns_digests, sizeof(ns_digests)); if (ns_type != NS_TYPE_CONSENSUS) { const char *end_of_cert = NULL; @@ -2486,8 +2500,9 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, if (voter) smartlist_add(ns->voters, voter); voter = tor_malloc_zero(sizeof(networkstatus_voter_info_t)); + voter->sigs = smartlist_create(); if (ns->type != NS_TYPE_CONSENSUS) - memcpy(voter->vote_digest, ns_digest, DIGEST_LEN); + memcpy(voter->vote_digest, ns_digests.d[DIGEST_SHA1], DIGEST_LEN); voter->nickname = tor_strdup(tok->args[0]); if (strlen(tok->args[1]) != HEX_DIGEST_LEN || @@ -2622,10 +2637,10 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, goto err; } - SMARTLIST_FOREACH(footer_tokens, directory_token_t *, _tok, - { + SMARTLIST_FOREACH_BEGIN(footer_tokens, directory_token_t *, _tok) { char declared_identity[DIGEST_LEN]; networkstatus_voter_info_t *v; + document_signature_t *sig; tok = _tok; if (tok->tp != K_DIRECTORY_SIGNATURE) continue; @@ -2650,11 +2665,15 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, "any declared directory source."); goto err; } + sig = tor_malloc_zero(sizeof(document_signature_t)); + memcpy(sig->identity_digest, v->identity_digest, DIGEST_LEN); + sig->alg = DIGEST_SHA1; if (strlen(tok->args[1]) != HEX_DIGEST_LEN || - base16_decode(v->signing_key_digest, sizeof(v->signing_key_digest), + base16_decode(sig->signing_key_digest, sizeof(sig->signing_key_digest), tok->args[1], HEX_DIGEST_LEN) < 0) { log_warn(LD_DIR, "Error decoding declared digest %s in " "network-status vote.", escaped(tok->args[1])); + tor_free(sig); goto err; } @@ -2663,32 +2682,42 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, DIGEST_LEN)) { log_warn(LD_DIR, "Digest mismatch between declared and actual on " "network-status vote."); + tor_free(sig); goto err; } } + if (voter_get_sig_by_algorithm(v, sig->alg)) { + /* We already parsed a vote with this algorithm from this voter. Use the + first one. */ + log_fn(LOG_PROTOCOL_WARN, LD_DIR, "We received a networkstatus " + "that contains two votes from the same voter with the same " + "algorithm. Ignoring the second vote."); + tor_free(sig); + continue; + } + if (ns->type != NS_TYPE_CONSENSUS) { - if (check_signature_token(ns_digest, DIGEST_LEN, + if (check_signature_token(ns_digests.d[DIGEST_SHA1], DIGEST_LEN, tok, ns->cert->signing_key, 0, - "network-status vote")) + "network-status vote")) { + tor_free(sig); goto err; - v->good_signature = 1; - } else { - if (tok->object_size >= INT_MAX) - goto err; - /* We already parsed a vote from this voter. Use the first one. */ - if (v->signature) { - log_fn(LOG_PROTOCOL_WARN, LD_DIR, "We received a networkstatus " - "that contains two votes from the same voter. Ignoring " - "the second vote."); - continue; } + sig->good_signature = 1; + } else { + if (tok->object_size >= INT_MAX) { + tor_free(sig); + goto err; + } + sig->signature = tor_memdup(tok->object_body, tok->object_size); + sig->signature_len = (int) tok->object_size; - v->signature = tor_memdup(tok->object_body, tok->object_size); - v->signature_len = (int) tok->object_size; } + smartlist_add(v->sigs, sig); + ++n_signatures; - }); + } SMARTLIST_FOREACH_END(_tok); if (! n_signatures) { log_warn(LD_DIR, "No signatures on networkstatus vote."); @@ -2714,10 +2743,14 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, smartlist_free(tokens); } if (voter) { + if (voter->sigs) { + SMARTLIST_FOREACH(voter->sigs, document_signature_t *, sig, + document_signature_free(sig)); + smartlist_free(voter->sigs); + } tor_free(voter->nickname); tor_free(voter->address); tor_free(voter->contact); - tor_free(voter->signature); tor_free(voter); } if (rs_tokens) { @@ -2747,10 +2780,19 @@ networkstatus_parse_detached_signatures(const char *s, const char *eos) * networkstatus_parse_vote_from_string(). */ directory_token_t *tok; memarea_t *area = NULL; + const char *flavor = "ns"; + digests_t *digests; + smartlist_t *sig_list; smartlist_t *tokens = smartlist_create(); ns_detached_signatures_t *sigs = tor_malloc_zero(sizeof(ns_detached_signatures_t)); + sigs->digests = strmap_new(); + sigs->signatures = strmap_new(); + digests = tor_malloc_zero(sizeof(digests_t)); + sig_list = smartlist_create(); + strmap_set(sigs->digests, flavor, digests); + strmap_set(sigs->signatures, flavor, sig_list); if (!eos) eos = s + strlen(s); @@ -2768,7 +2810,7 @@ networkstatus_parse_detached_signatures(const char *s, const char *eos) "networkstatus signatures"); goto err; } - if (base16_decode(sigs->networkstatus_digest, DIGEST_LEN, + if (base16_decode(digests->d[DIGEST_SHA1], DIGEST_LEN, tok->args[0], strlen(tok->args[0])) < 0) { log_warn(LD_DIR, "Bad encoding on on consensus-digest in detached " "networkstatus signatures"); @@ -2793,50 +2835,51 @@ networkstatus_parse_detached_signatures(const char *s, const char *eos) goto err; } - sigs->signatures = smartlist_create(); - SMARTLIST_FOREACH(tokens, directory_token_t *, _tok, - { - char id_digest[DIGEST_LEN]; - char sk_digest[DIGEST_LEN]; - networkstatus_voter_info_t *voter; + SMARTLIST_FOREACH_BEGIN(tokens, directory_token_t *, _tok) { + char id_digest[DIGEST_LEN]; + char sk_digest[DIGEST_LEN]; + document_signature_t *sig; - tok = _tok; - if (tok->tp != K_DIRECTORY_SIGNATURE) - continue; - tor_assert(tok->n_args >= 2); + tok = _tok; + if (tok->tp != K_DIRECTORY_SIGNATURE) + continue; + tor_assert(tok->n_args >= 2); - if (!tok->object_type || - strcmp(tok->object_type, "SIGNATURE") || - tok->object_size < 128 || tok->object_size > 512) { - log_warn(LD_DIR, "Bad object type or length on directory-signature"); - goto err; - } + if (!tok->object_type || + strcmp(tok->object_type, "SIGNATURE") || + tok->object_size < 128 || tok->object_size > 512) { + log_warn(LD_DIR, "Bad object type or length on directory-signature"); + goto err; + } - if (strlen(tok->args[0]) != HEX_DIGEST_LEN || - base16_decode(id_digest, sizeof(id_digest), - tok->args[0], HEX_DIGEST_LEN) < 0) { - log_warn(LD_DIR, "Error decoding declared identity %s in " - "network-status vote.", escaped(tok->args[0])); - goto err; - } - if (strlen(tok->args[1]) != HEX_DIGEST_LEN || - base16_decode(sk_digest, sizeof(sk_digest), - tok->args[1], HEX_DIGEST_LEN) < 0) { - log_warn(LD_DIR, "Error decoding declared digest %s in " - "network-status vote.", escaped(tok->args[1])); - goto err; - } + if (strlen(tok->args[0]) != HEX_DIGEST_LEN || + base16_decode(id_digest, sizeof(id_digest), + tok->args[0], HEX_DIGEST_LEN) < 0) { + log_warn(LD_DIR, "Error decoding declared identity %s in " + "network-status vote.", escaped(tok->args[0])); + goto err; + } + if (strlen(tok->args[1]) != HEX_DIGEST_LEN || + base16_decode(sk_digest, sizeof(sk_digest), + tok->args[1], HEX_DIGEST_LEN) < 0) { + log_warn(LD_DIR, "Error decoding declared digest %s in " + "network-status vote.", escaped(tok->args[1])); + goto err; + } - voter = tor_malloc_zero(sizeof(networkstatus_voter_info_t)); - memcpy(voter->identity_digest, id_digest, DIGEST_LEN); - memcpy(voter->signing_key_digest, sk_digest, DIGEST_LEN); - if (tok->object_size >= INT_MAX) - goto err; - voter->signature = tor_memdup(tok->object_body, tok->object_size); - voter->signature_len = (int) tok->object_size; + sig = tor_malloc_zero(sizeof(document_signature_t)); + sig->alg = DIGEST_SHA1; + memcpy(sig->identity_digest, id_digest, DIGEST_LEN); + memcpy(sig->signing_key_digest, sk_digest, DIGEST_LEN); + if (tok->object_size >= INT_MAX) { + tor_free(sig); + goto err; + } + sig->signature = tor_memdup(tok->object_body, tok->object_size); + sig->signature_len = (int) tok->object_size; - smartlist_add(sigs->signatures, voter); - }); + smartlist_add(sig_list, sig); + } SMARTLIST_FOREACH_END(_tok); goto done; err: @@ -3428,18 +3471,11 @@ find_all_exitpolicy(smartlist_t *s) return out; } -/** Compute the digest of the substring of s taken from the first - * occurrence of start_str through the first instance of c after the - * first subsequent occurrence of end_str; store the 20-byte result in - * digest; return 0 on success. - * - * If no such substring exists, return -1. - */ static int -router_get_hash_impl(const char *s, char *digest, - const char *start_str, - const char *end_str, char end_c, - digest_algorithm_t alg) +router_get_hash_impl_helper(const char *s, + const char *start_str, + const char *end_str, char end_c, + const char **start_out, const char **end_out) { char *start, *end; start = strstr(s, start_str); @@ -3465,6 +3501,28 @@ router_get_hash_impl(const char *s, char *digest, } ++end; + *start_out = start; + *end_out = end; + return 0; +} + +/** Compute the digest of the substring of s taken from the first + * occurrence of start_str through the first instance of c after the + * first subsequent occurrence of end_str; store the 20-byte result in + * digest; return 0 on success. + * + * If no such substring exists, return -1. + */ +static int +router_get_hash_impl(const char *s, char *digest, + const char *start_str, + const char *end_str, char end_c, + digest_algorithm_t alg) +{ + const char *start=NULL, *end=NULL; + if (router_get_hash_impl_helper(s,start_str,end_str,end_c,&start,&end)<0) + return -1; + if (alg == DIGEST_SHA1) { if (crypto_digest(digest, start, end-start)) { log_warn(LD_BUG,"couldn't compute digest"); @@ -3480,6 +3538,24 @@ router_get_hash_impl(const char *s, char *digest, return 0; } +/** As router_get_hash_impl, but compute all hashes. */ +static int +router_get_hashes_impl(const char *s, digests_t *digests, + const char *start_str, + const char *end_str, char end_c) +{ + const char *start=NULL, *end=NULL; + if (router_get_hash_impl_helper(s,start_str,end_str,end_c,&start,&end)<0) + return -1; + + if (crypto_digest_all(digests, start, end-start)) { + log_warn(LD_BUG,"couldn't compute digests"); + return -1; + } + + return 0; +} + /** DOCDOC Assuming that s starts with a microdesc, return the start of the * *NEXT* one. */ static const char * diff --git a/src/test/test_dir.c b/src/test/test_dir.c index 8e566e2eec..b1fac683cc 100644 --- a/src/test/test_dir.c +++ b/src/test/test_dir.c @@ -571,6 +571,7 @@ test_dir_v3_networkstatus(void) time_t now = time(NULL); networkstatus_voter_info_t *voter; + document_signature_t *sig; networkstatus_t *vote=NULL, *v1=NULL, *v2=NULL, *v3=NULL, *con=NULL; vote_routerstatus_t *vrs; routerstatus_t *rs; @@ -946,20 +947,25 @@ test_dir_v3_networkstatus(void) /* Check signatures. the first voter is a pseudo-entry with a legacy key. * The second one hasn't signed. The fourth one has signed: validate it. */ voter = smartlist_get(con->voters, 1); - test_assert(!voter->signature); - test_assert(!voter->good_signature); - test_assert(!voter->bad_signature); + test_eq(smartlist_len(voter->sigs), 0); +#if 0 + sig = smartlist_get(voter->sigs, 1); + test_assert(!sig->signature); + test_assert(!sig->good_signature); + test_assert(!sig->bad_signature); +#endif voter = smartlist_get(con->voters, 3); - test_assert(voter->signature); - test_assert(!voter->good_signature); - test_assert(!voter->bad_signature); - test_assert(!networkstatus_check_voter_signature(con, - smartlist_get(con->voters, 3), - cert3)); - test_assert(voter->signature); - test_assert(voter->good_signature); - test_assert(!voter->bad_signature); + test_eq(smartlist_len(voter->sigs), 1); + sig = smartlist_get(voter->sigs, 0); + test_assert(sig->signature); + test_assert(!sig->good_signature); + test_assert(!sig->bad_signature); + + test_assert(!networkstatus_check_document_signature(con, sig, cert3)); + test_assert(sig->signature); + test_assert(sig->good_signature); + test_assert(!sig->bad_signature); { const char *msg=NULL; @@ -984,10 +990,8 @@ test_dir_v3_networkstatus(void) test_assert(con3); /* All three should have the same digest. */ - test_memeq(con->networkstatus_digest, con2->networkstatus_digest, - DIGEST_LEN); - test_memeq(con->networkstatus_digest, con3->networkstatus_digest, - DIGEST_LEN); + test_memeq(&con->digests, &con2->digests, sizeof(digests_t)); + test_memeq(&con->digests, &con3->digests, sizeof(digests_t)); /* Extract a detached signature from con3. */ detached_text1 = networkstatus_get_detached_signatures(con3); @@ -1000,12 +1004,20 @@ test_dir_v3_networkstatus(void) test_eq(dsig1->valid_after, con3->valid_after); test_eq(dsig1->fresh_until, con3->fresh_until); test_eq(dsig1->valid_until, con3->valid_until); - test_memeq(dsig1->networkstatus_digest, con3->networkstatus_digest, - DIGEST_LEN); - test_eq(1, smartlist_len(dsig1->signatures)); - voter = smartlist_get(dsig1->signatures, 0); - test_memeq(voter->identity_digest, cert1->cache_info.identity_digest, - DIGEST_LEN); + { + digests_t *dsig_digests = strmap_get(dsig1->digests, "ns"); + test_assert(dsig_digests); + test_memeq(dsig_digests->d[DIGEST_SHA1], con3->digests.d[DIGEST_SHA1], + DIGEST_LEN); + } + { + smartlist_t *dsig_signatures = strmap_get(dsig1->signatures, "ns"); + test_assert(dsig_signatures); + test_eq(1, smartlist_len(dsig_signatures)); + sig = smartlist_get(dsig_signatures, 0); + test_memeq(sig->identity_digest, cert1->cache_info.identity_digest, + DIGEST_LEN); + } /* Try adding it to con2. */ detached_text2 = networkstatus_get_detached_signatures(con2); @@ -1023,7 +1035,8 @@ test_dir_v3_networkstatus(void) printf("%s\n", hd); }); */ - test_eq(2, smartlist_len(dsig2->signatures)); + test_eq(2, + smartlist_len((smartlist_t*)strmap_get(dsig2->signatures, "ns"))); /* Try adding to con2 twice; verify that nothing changes. */ test_eq(0, networkstatus_add_detached_signatures(con2, dsig1, &msg)); @@ -1031,13 +1044,14 @@ test_dir_v3_networkstatus(void) /* Add to con. */ test_eq(2, networkstatus_add_detached_signatures(con, dsig2, &msg)); /* Check signatures */ - test_assert(!networkstatus_check_voter_signature(con, - smartlist_get(con->voters, 1), - cert2)); - test_assert(!networkstatus_check_voter_signature(con, - smartlist_get(con->voters, 2), - cert1)); - + voter = smartlist_get(con->voters, 1); + sig = smartlist_get(voter->sigs, 0); + test_assert(sig); + test_assert(!networkstatus_check_document_signature(con, sig, cert2)); + voter = smartlist_get(con->voters, 2); + sig = smartlist_get(voter->sigs, 0); + test_assert(sig); + test_assert(!networkstatus_check_document_signature(con, sig, cert1)); } done: From 5576a3a094de49246e501bb24d1b3dc3a5d610b9 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 16 Sep 2009 12:34:44 -0400 Subject: [PATCH 11/23] Parse detached signature documents with multiple flavors and algorithms. --- src/common/crypto.c | 12 ++++ src/common/crypto.h | 1 + src/or/routerparse.c | 160 +++++++++++++++++++++++++++++++++++-------- 3 files changed, 146 insertions(+), 27 deletions(-) diff --git a/src/common/crypto.c b/src/common/crypto.c index ac0e628c48..f335f99aba 100644 --- a/src/common/crypto.c +++ b/src/common/crypto.c @@ -1481,6 +1481,18 @@ crypto_digest_algorithm_get_name(digest_algorithm_t alg) } } +/** DOCDOC */ +int +crypto_digest_algorithm_parse_name(const char *name) +{ + if (!strcmp(name, "sha1")) + return DIGEST_SHA1; + else if (!strcmp(name, "sha256")) + return DIGEST_SHA256; + else + return -1; +} + /** Intermediate information about the digest of a stream of data. */ struct crypto_digest_env_t { union { diff --git a/src/common/crypto.h b/src/common/crypto.h index ed8468046f..c0a4526255 100644 --- a/src/common/crypto.h +++ b/src/common/crypto.h @@ -177,6 +177,7 @@ int crypto_digest256(char *digest, const char *m, size_t len, digest_algorithm_t algorithm); int crypto_digest_all(digests_t *ds_out, const char *m, size_t len); const char *crypto_digest_algorithm_get_name(digest_algorithm_t alg); +int crypto_digest_algorithm_parse_name(const char *name); crypto_digest_env_t *crypto_new_digest_env(void); crypto_digest_env_t *crypto_new_digest256_env(digest_algorithm_t algorithm); void crypto_free_digest_env(crypto_digest_env_t *digest); diff --git a/src/or/routerparse.c b/src/or/routerparse.c index 285a550acd..21fe5f7d23 100644 --- a/src/or/routerparse.c +++ b/src/or/routerparse.c @@ -106,6 +106,8 @@ typedef enum { K_PARAMS, K_VOTE_DIGEST, K_CONSENSUS_DIGEST, + K_ADDITIONAL_DIGEST, + K_ADDITIONAL_SIGNATURE, K_CONSENSUS_METHODS, K_CONSENSUS_METHOD, K_LEGACY_DIR_KEY, @@ -490,10 +492,12 @@ static token_rule_t networkstatus_vote_footer_token_table[] = { /** List of tokens allowable in detached networkstatus signature documents. */ static token_rule_t networkstatus_detached_signature_token_table[] = { T1_START("consensus-digest", K_CONSENSUS_DIGEST, GE(1), NO_OBJ ), + T("additional-digest", K_ADDITIONAL_DIGEST,GE(3), NO_OBJ ), T1("valid-after", K_VALID_AFTER, CONCAT_ARGS, NO_OBJ ), T1("fresh-until", K_FRESH_UNTIL, CONCAT_ARGS, NO_OBJ ), T1("valid-until", K_VALID_UNTIL, CONCAT_ARGS, NO_OBJ ), - T1N("directory-signature", K_DIRECTORY_SIGNATURE, GE(2), NEED_OBJ ), + T("additional-signature", K_ADDITIONAL_SIGNATURE, GE(4), NEED_OBJ ), + T1N("directory-signature", K_DIRECTORY_SIGNATURE, GE(2), NEED_OBJ ), END_OF_TABLE }; @@ -2771,6 +2775,31 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, return ns; } +/** DOCDOC */ +static digests_t * +detached_get_digests(ns_detached_signatures_t *sigs, const char *flavor_name) +{ + digests_t *d = strmap_get(sigs->digests, flavor_name); + if (!d) { + d = tor_malloc_zero(sizeof(digests_t)); + strmap_set(sigs->digests, flavor_name, d); + } + return d; +} + +/** DOCDOC */ +static smartlist_t * +detached_get_signatures(ns_detached_signatures_t *sigs, + const char *flavor_name) +{ + smartlist_t *sl = strmap_get(sigs->signatures, flavor_name); + if (!sl) { + sl = smartlist_create(); + strmap_set(sigs->signatures, flavor_name, sl); + } + return sl; +} + /** Parse a detached v3 networkstatus signature document between s and * eos and return the result. Return -1 on failure. */ ns_detached_signatures_t * @@ -2780,19 +2809,13 @@ networkstatus_parse_detached_signatures(const char *s, const char *eos) * networkstatus_parse_vote_from_string(). */ directory_token_t *tok; memarea_t *area = NULL; - const char *flavor = "ns"; digests_t *digests; - smartlist_t *sig_list; smartlist_t *tokens = smartlist_create(); ns_detached_signatures_t *sigs = tor_malloc_zero(sizeof(ns_detached_signatures_t)); sigs->digests = strmap_new(); sigs->signatures = strmap_new(); - digests = tor_malloc_zero(sizeof(digests_t)); - sig_list = smartlist_create(); - strmap_set(sigs->digests, flavor, digests); - strmap_set(sigs->signatures, flavor, sig_list); if (!eos) eos = s + strlen(s); @@ -2804,18 +2827,57 @@ networkstatus_parse_detached_signatures(const char *s, const char *eos) goto err; } - tok = find_by_keyword(tokens, K_CONSENSUS_DIGEST); - if (strlen(tok->args[0]) != HEX_DIGEST_LEN) { - log_warn(LD_DIR, "Wrong length on consensus-digest in detached " - "networkstatus signatures"); - goto err; - } - if (base16_decode(digests->d[DIGEST_SHA1], DIGEST_LEN, - tok->args[0], strlen(tok->args[0])) < 0) { - log_warn(LD_DIR, "Bad encoding on on consensus-digest in detached " - "networkstatus signatures"); - goto err; - } + /* Grab all the digest-like tokens. */ + SMARTLIST_FOREACH_BEGIN(tokens, directory_token_t *, _tok) { + const char *algname; + digest_algorithm_t alg; + const char *flavor; + const char *hexdigest; + size_t expected_length; + + tok = _tok; + + if (tok->tp == K_CONSENSUS_DIGEST) { + algname = "sha1"; + alg = DIGEST_SHA1; + flavor = "ns"; + hexdigest = tok->args[0]; + } else if (tok->tp == K_ADDITIONAL_DIGEST) { + int a = crypto_digest_algorithm_parse_name(tok->args[1]); + if (a<0) { + log_warn(LD_DIR, "Unrecognized algorithm name %s", tok->args[0]); + continue; + } + alg = (digest_algorithm_t) a; + flavor = tok->args[0]; + algname = tok->args[1]; + hexdigest = tok->args[2]; + } else { + continue; + } + + expected_length = + (alg == DIGEST_SHA1) ? HEX_DIGEST_LEN : HEX_DIGEST256_LEN; + + if (strlen(hexdigest) != expected_length) { + log_warn(LD_DIR, "Wrong length on consensus-digest in detached " + "networkstatus signatures"); + goto err; + } + digests = detached_get_digests(sigs, flavor); + tor_assert(digests); + if (!tor_mem_is_zero(digests->d[alg], DIGEST256_LEN)) { + log_warn(LD_DIR, "Multiple digests for %s with %s on detached " + "signatures document", flavor, algname); + continue; + } + if (base16_decode(digests->d[alg], DIGEST256_LEN, + hexdigest, strlen(hexdigest)) < 0) { + log_warn(LD_DIR, "Bad encoding on on consensus-digest in detached " + "networkstatus signatures"); + goto err; + } + } SMARTLIST_FOREACH_END(_tok); tok = find_by_keyword(tokens, K_VALID_AFTER); if (parse_iso_time(tok->args[0], &sigs->valid_after)) { @@ -2836,14 +2898,43 @@ networkstatus_parse_detached_signatures(const char *s, const char *eos) } SMARTLIST_FOREACH_BEGIN(tokens, directory_token_t *, _tok) { + const char *id_hexdigest; + const char *sk_hexdigest; + const char *algname; + const char *flavor; + digest_algorithm_t alg; + char id_digest[DIGEST_LEN]; char sk_digest[DIGEST_LEN]; + smartlist_t *siglist; document_signature_t *sig; + int is_duplicate; tok = _tok; - if (tok->tp != K_DIRECTORY_SIGNATURE) + if (tok->tp == K_DIRECTORY_SIGNATURE) { + tor_assert(tok->n_args >= 2); + flavor = "ns"; + algname = "sha1"; + id_hexdigest = tok->args[0]; + sk_hexdigest = tok->args[1]; + } else if (tok->tp == K_ADDITIONAL_SIGNATURE) { + tor_assert(tok->n_args >= 4); + flavor = tok->args[0]; + algname = tok->args[1]; + id_hexdigest = tok->args[2]; + sk_hexdigest = tok->args[3]; + } else { continue; - tor_assert(tok->n_args >= 2); + } + + { + int a = crypto_digest_algorithm_parse_name(algname); + if (a<0) { + log_warn(LD_DIR, "Unrecognized algorithm name %s", algname); + continue; + } + alg = (digest_algorithm_t) a; + } if (!tok->object_type || strcmp(tok->object_type, "SIGNATURE") || @@ -2852,21 +2943,36 @@ networkstatus_parse_detached_signatures(const char *s, const char *eos) goto err; } - if (strlen(tok->args[0]) != HEX_DIGEST_LEN || + if (strlen(id_hexdigest) != HEX_DIGEST_LEN || base16_decode(id_digest, sizeof(id_digest), - tok->args[0], HEX_DIGEST_LEN) < 0) { + id_hexdigest, HEX_DIGEST_LEN) < 0) { log_warn(LD_DIR, "Error decoding declared identity %s in " - "network-status vote.", escaped(tok->args[0])); + "network-status vote.", escaped(id_hexdigest)); goto err; } if (strlen(tok->args[1]) != HEX_DIGEST_LEN || base16_decode(sk_digest, sizeof(sk_digest), - tok->args[1], HEX_DIGEST_LEN) < 0) { + sk_hexdigest, HEX_DIGEST_LEN) < 0) { log_warn(LD_DIR, "Error decoding declared digest %s in " - "network-status vote.", escaped(tok->args[1])); + "network-status vote.", escaped(sk_hexdigest)); goto err; } + siglist = detached_get_signatures(sigs, flavor); + is_duplicate = 0; + SMARTLIST_FOREACH(siglist, document_signature_t *, s, { + if (s->alg == alg && + !memcmp(id_digest, s->identity_digest, DIGEST_LEN) && + !memcmp(sk_digest, s->signing_key_digest, DIGEST_LEN)) { + is_duplicate = 1; + } + }); + if (is_duplicate) { + log_warn(LD_DIR, "Two signatures with identical keys and algorithm " + "found."); + continue; + } + sig = tor_malloc_zero(sizeof(document_signature_t)); sig->alg = DIGEST_SHA1; memcpy(sig->identity_digest, id_digest, DIGEST_LEN); @@ -2878,7 +2984,7 @@ networkstatus_parse_detached_signatures(const char *s, const char *eos) sig->signature = tor_memdup(tok->object_body, tok->object_size); sig->signature_len = (int) tok->object_size; - smartlist_add(sig_list, sig); + smartlist_add(siglist, sig); } SMARTLIST_FOREACH_END(_tok); goto done; From d9c71816b15af4325fb8ab9befa8600185d9aa90 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 22 Sep 2009 16:52:51 -0400 Subject: [PATCH 12/23] Generate all the flavors of consensuses when building consensuses. --- src/or/directory.c | 2 +- src/or/dirvote.c | 350 ++++++++++++++++++++++++++++++--------------- src/or/or.h | 9 +- 3 files changed, 241 insertions(+), 120 deletions(-) diff --git a/src/or/directory.c b/src/or/directory.c index 19ef6351c2..0feddcbae4 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -2618,7 +2618,7 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, const char *item; tor_assert(!current); /* we handle current consensus specially above, * since it wants to be spooled. */ - if ((item = dirvote_get_pending_consensus())) + if ((item = dirvote_get_pending_consensus(FLAV_NS))) smartlist_add(items, (char*)item); } else if (!current && !strcmp(url, "consensus-signatures")) { /* XXXX the spec says that we should implement diff --git a/src/or/dirvote.c b/src/or/dirvote.c index 1bdf4cc403..e3611025e9 100644 --- a/src/or/dirvote.c +++ b/src/or/dirvote.c @@ -11,15 +11,21 @@ * \brief Functions to compute directory consensus, and schedule voting. **/ -static int dirvote_add_signatures_to_pending_consensus( +typedef struct pending_consensus_t pending_consensus_t; + +static int dirvote_add_signatures_to_all_pending_consensuses( const char *detached_signatures_body, const char **msg_out); +static int dirvote_add_signatures_to_pending_consensus( + pending_consensus_t *pc, + ns_detached_signatures_t *sigs, + const char **msg_out); static char *list_v3_auth_ids(void); static void dirvote_fetch_missing_votes(void); static void dirvote_fetch_missing_signatures(void); static int dirvote_perform_vote(void); static void dirvote_clear_votes(int all_votes); -static int dirvote_compute_consensus(void); +static int dirvote_compute_consensuses(void); static int dirvote_publish_consensus(void); static char *make_consensus_method_list(int low, int high, const char *sep); @@ -1480,40 +1486,17 @@ networkstatus_add_detached_signatures(networkstatus_t *target, return r; } -/** Return a newly allocated string holding the detached-signatures document - * corresponding to the signatures on consensus. */ -char * -networkstatus_get_detached_signatures(networkstatus_t *consensus) +/** DOCDOC */ +static char * +networkstatus_format_signatures(networkstatus_t *consensus) { smartlist_t *elements; char buf[4096]; char *result = NULL; int n_sigs = 0; - tor_assert(consensus); - tor_assert(consensus->type == NS_TYPE_CONSENSUS); - - tor_assert(consensus->flavor == FLAV_NS); elements = smartlist_create(); - { - char va_buf[ISO_TIME_LEN+1], fu_buf[ISO_TIME_LEN+1], - vu_buf[ISO_TIME_LEN+1]; - char d[HEX_DIGEST_LEN+1]; - - base16_encode(d, sizeof(d), consensus->digests.d[DIGEST_SHA1], DIGEST_LEN); - format_iso_time(va_buf, consensus->valid_after); - format_iso_time(fu_buf, consensus->fresh_until); - format_iso_time(vu_buf, consensus->valid_until); - - tor_snprintf(buf, sizeof(buf), - "consensus-digest %s\n" - "valid-after %s\n" - "fresh-until %s\n" - "valid-until %s\n", d, va_buf, fu_buf, vu_buf); - smartlist_add(elements, tor_strdup(buf)); - } - SMARTLIST_FOREACH_BEGIN(consensus->voters, networkstatus_voter_info_t *, v) { SMARTLIST_FOREACH_BEGIN(v->sigs, document_signature_t *, sig) { char sk[HEX_DIGEST_LEN+1]; @@ -1541,6 +1524,53 @@ networkstatus_get_detached_signatures(networkstatus_t *consensus) return result; } +/** Return a newly allocated string holding the detached-signatures document + * corresponding to the signatures on consensus. */ +char * +networkstatus_get_detached_signatures(networkstatus_t *consensus) +{ + smartlist_t *elements; + char buf[4096]; + char *result = NULL; + tor_assert(consensus); + tor_assert(consensus->type == NS_TYPE_CONSENSUS); + + tor_assert(consensus->flavor == FLAV_NS); + + elements = smartlist_create(); + + { + char va_buf[ISO_TIME_LEN+1], fu_buf[ISO_TIME_LEN+1], + vu_buf[ISO_TIME_LEN+1]; + char d[HEX_DIGEST_LEN+1]; + + base16_encode(d, sizeof(d), consensus->digests.d[DIGEST_SHA1], DIGEST_LEN); + format_iso_time(va_buf, consensus->valid_after); + format_iso_time(fu_buf, consensus->fresh_until); + format_iso_time(vu_buf, consensus->valid_until); + + tor_snprintf(buf, sizeof(buf), + "consensus-digest %s\n" + "valid-after %s\n" + "fresh-until %s\n" + "valid-until %s\n", d, va_buf, fu_buf, vu_buf); + smartlist_add(elements, tor_strdup(buf)); + } + + { + char *sigs = networkstatus_format_signatures(consensus); + if (!sigs) + goto err; + smartlist_add(elements, sigs); + } + + result = smartlist_join_strings(elements, "", 0, NULL); + err: + SMARTLIST_FOREACH(elements, char *, cp, tor_free(cp)); + smartlist_free(elements); + return result; +} + /** Release all storage held in s. */ void ns_detached_signatures_free(ns_detached_signatures_t *s) @@ -1744,7 +1774,7 @@ dirvote_act(or_options_t *options, time_t now) if (voting_schedule.voting_ends < now && !voting_schedule.have_built_consensus) { log_notice(LD_DIR, "Time to compute a consensus."); - dirvote_compute_consensus(); + dirvote_compute_consensuses(); /* XXXX We will want to try again later if we haven't got enough * votes yet. Implement this if it turns out to ever happen. */ voting_schedule.have_built_consensus = 1; @@ -1781,14 +1811,22 @@ static smartlist_t *pending_vote_list = NULL; /** List of pending_vote_t for the previous vote. After we've used them to * build a consensus, the votes go here for the next period. */ static smartlist_t *previous_vote_list = NULL; -/** The body of the consensus that we're currently building. Once we - * have it built, it goes into dirserv.c */ -static char *pending_consensus_body = NULL; + +/** DOCDOC*/ +struct pending_consensus_t { + /** The body of the consensus that we're currently building. Once we + * have it built, it goes into dirserv.c */ + char *body; + /** The parsed in-progress consensus document. */ + networkstatus_t *consensus; +}; + +static pending_consensus_t pending_consensuses[N_CONSENSUS_FLAVORS]; + /** The detached signatures for the consensus that we're currently * building. */ static char *pending_consensus_signatures = NULL; -/** The parsed in-progress consensus document. */ -static networkstatus_t *pending_consensus = NULL; + /** List of ns_detached_signatures_t: hold signatures that get posted to us * before we have generated the consensus on our own. */ static smartlist_t *pending_consensus_signature_list = NULL; @@ -1882,15 +1920,39 @@ dirvote_fetch_missing_votes(void) static void dirvote_fetch_missing_signatures(void) { - if (!pending_consensus) + int need_any = 0; + int i; + for (i=0; i < N_CONSENSUS_FLAVORS; ++i) { + networkstatus_t *consensus = pending_consensuses[i].consensus; + if (!consensus || + networkstatus_check_consensus_signature(consensus, -1) == 1) { + /* We have no consensus, or we have one that's signed by everybody. */ + continue; + } + need_any = 1; + } + if (!need_any) return; - if (networkstatus_check_consensus_signature(pending_consensus, -1) == 1) - return; /* we have a signature from everybody. */ directory_get_from_all_authorities(DIR_PURPOSE_FETCH_DETACHED_SIGNATURES, 0, NULL); } +/** DOCDOC */ +static void +dirvote_clear_pending_consensuses(void) +{ + int i; + for (i = 0; i < N_CONSENSUS_FLAVORS; ++i) { + pending_consensus_t *pc = &pending_consensuses[i]; + tor_free(pc->body); + if (pc->consensus) { + networkstatus_vote_free(pc->consensus); + pc->consensus = NULL; + } + } +} + /** Drop all currently pending votes, consensus, and detached signatures. */ static void dirvote_clear_votes(int all_votes) @@ -1928,12 +1990,8 @@ dirvote_clear_votes(int all_votes) tor_free(cp)); smartlist_clear(pending_consensus_signature_list); } - tor_free(pending_consensus_body); tor_free(pending_consensus_signatures); - if (pending_consensus) { - networkstatus_vote_free(pending_consensus); - pending_consensus = NULL; - } + dirvote_clear_pending_consensuses(); } /** Return a newly allocated string containing the hex-encoded v3 authority @@ -2121,7 +2179,7 @@ dirvote_add_vote(const char *vote_body, const char **msg_out, int *status_out) * pending_consensus: it won't be ready to be published until we have * everybody else's signatures collected too. (V3 Authority only) */ static int -dirvote_compute_consensus(void) +dirvote_compute_consensuses(void) { /* Have we got enough votes to try? */ int n_votes, n_voters; @@ -2129,6 +2187,10 @@ dirvote_compute_consensus(void) char *consensus_body = NULL, *signatures = NULL, *votefile; networkstatus_t *consensus = NULL; authority_cert_t *my_cert; + pending_consensus_t pending[N_CONSENSUS_FLAVORS]; + int flav; + + memset(pending, 0, sizeof(pending)); if (!pending_vote_list) pending_vote_list = smartlist_create(); @@ -2168,6 +2230,7 @@ dirvote_compute_consensus(void) char legacy_dbuf[DIGEST_LEN]; crypto_pk_env_t *legacy_sign=NULL; char *legacy_id_digest = NULL; + int n_generated = 0; if (get_options()->V3AuthUseLegacyKey) { authority_cert_t *cert = get_my_v3_legacy_cert(); legacy_sign = get_my_v3_legacy_signing_key(); @@ -2176,40 +2239,59 @@ dirvote_compute_consensus(void) legacy_id_digest = legacy_dbuf; } } - consensus_body = networkstatus_compute_consensus( + + for (flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) { + const char *flavor_name = networkstatus_get_flavor_name(flav); + consensus_body = networkstatus_compute_consensus( votes, n_voters, my_cert->identity_key, get_my_v3_authority_signing_key(), legacy_id_digest, legacy_sign, - FLAV_NS); - } - if (!consensus_body) { - log_warn(LD_DIR, "Couldn't generate a consensus at all!"); - goto err; - } - consensus = networkstatus_parse_vote_from_string(consensus_body, NULL, - NS_TYPE_CONSENSUS); - if (!consensus) { - log_warn(LD_DIR, "Couldn't parse consensus we generated!"); - goto err; - } - /* 'Check' our own signature, to mark it valid. */ - networkstatus_check_consensus_signature(consensus, -1); + flav); + + if (!consensus_body) { + log_warn(LD_DIR, "Couldn't generate a %s consensus at all!", + flavor_name); + continue; + } + consensus = networkstatus_parse_vote_from_string(consensus_body, NULL, + NS_TYPE_CONSENSUS); + if (!consensus) { + log_warn(LD_DIR, "Couldn't parse %s consensus we generated!", + flavor_name); + tor_free(consensus_body); + continue; + } + + /* 'Check' our own signature, to mark it valid. */ + networkstatus_check_consensus_signature(consensus, -1); + + pending[flav].body = consensus_body; + pending[flav].consensus = consensus; + n_generated++; + consensus_body = NULL; + consensus = NULL; + } + if (!n_generated) { + log_warn(LD_DIR, "Couldn't generate any consensus flavors at all."); + goto err; + } + } + + /* XXXX NMNM NM NM wrong. */ + signatures = + networkstatus_get_detached_signatures(pending[FLAV_NS].consensus); - signatures = networkstatus_get_detached_signatures(consensus); if (!signatures) { log_warn(LD_DIR, "Couldn't extract signatures."); goto err; } - tor_free(pending_consensus_body); - pending_consensus_body = consensus_body; + dirvote_clear_pending_consensuses(); + memcpy(pending_consensuses, pending, sizeof(pending)); + tor_free(pending_consensus_signatures); pending_consensus_signatures = signatures; - if (pending_consensus) - networkstatus_vote_free(pending_consensus); - pending_consensus = consensus; - if (pending_consensus_signature_list) { int n_sigs = 0; /* we may have gotten signatures for this consensus before we built @@ -2217,7 +2299,7 @@ dirvote_compute_consensus(void) SMARTLIST_FOREACH(pending_consensus_signature_list, char *, sig, { const char *msg = NULL; - int r = dirvote_add_signatures_to_pending_consensus(sig, &msg); + int r = dirvote_add_signatures_to_all_pending_consensuses(sig, &msg); if (r >= 0) n_sigs += r; else @@ -2253,79 +2335,61 @@ dirvote_compute_consensus(void) } /** Helper: we just got the detached_signatures_body sent to us as - * signatures on the currently pending consensus. Add them to the consensus + * signatures on the currently pending consensus. Add them to pc * as appropriate. Return the number of signatures added. (?) */ static int dirvote_add_signatures_to_pending_consensus( - const char *detached_signatures_body, + pending_consensus_t *pc, + ns_detached_signatures_t *sigs, const char **msg_out) { - ns_detached_signatures_t *sigs = NULL; + const char *flavor_name; int r = -1; - tor_assert(detached_signatures_body); - tor_assert(msg_out); - /* Only call if we have a pending consensus right now. */ - tor_assert(pending_consensus); - tor_assert(pending_consensus_body); + tor_assert(pc->consensus); + tor_assert(pc->body); tor_assert(pending_consensus_signatures); + flavor_name = networkstatus_get_flavor_name(pc->consensus->flavor); *msg_out = NULL; - if (!(sigs = networkstatus_parse_detached_signatures( - detached_signatures_body, NULL))) { - *msg_out = "Couldn't parse detached signatures."; - goto err; - } - { - smartlist_t *sig_list = strmap_get(sigs->signatures, - networkstatus_get_flavor_name(pending_consensus->flavor)); - log_info(LD_DIR, "Have %d signatures for adding to consensus.", - sig_list ? smartlist_len(sig_list) : 0); + smartlist_t *sig_list = strmap_get(sigs->signatures, flavor_name); + log_info(LD_DIR, "Have %d signatures for adding to %s consensus.", + sig_list ? smartlist_len(sig_list) : 0, flavor_name); } - r = networkstatus_add_detached_signatures(pending_consensus, - sigs, msg_out); + r = networkstatus_add_detached_signatures(pc->consensus, sigs, msg_out); log_info(LD_DIR,"Added %d signatures to consensus.", r); if (r >= 1) { - char *new_detached = - networkstatus_get_detached_signatures(pending_consensus); - const char *src; + char *new_signatures = + networkstatus_format_signatures(pc->consensus); char *dst, *dst_end; size_t new_consensus_len; - if (!new_detached) { + if (!new_signatures) { *msg_out = "No signatures to add"; goto err; } new_consensus_len = - strlen(pending_consensus_body) + strlen(new_detached) + 1; - pending_consensus_body = tor_realloc(pending_consensus_body, - new_consensus_len); - dst_end = pending_consensus_body + new_consensus_len; - dst = strstr(pending_consensus_body, "directory-signature "); + strlen(pc->body) + strlen(new_signatures) + 1; + pc->body = tor_realloc(pc->body, new_consensus_len); + dst_end = pc->body + new_consensus_len; + dst = strstr(pc->body, "directory-signature "); tor_assert(dst); - src = strstr(new_detached, "directory-signature "); - tor_assert(src); - strlcpy(dst, src, dst_end-dst); + strlcpy(dst, new_signatures, dst_end-dst); /* We remove this block once it has failed to crash for a while. But * unless it shows up in profiles, we're probably better leaving it in, * just in case we break detached signature processing at some point. */ { - ns_detached_signatures_t *sigs = - networkstatus_parse_detached_signatures(new_detached, NULL); networkstatus_t *v = networkstatus_parse_vote_from_string( - pending_consensus_body, NULL, + pc->body, NULL, NS_TYPE_CONSENSUS); - tor_assert(sigs); - ns_detached_signatures_free(sigs); tor_assert(v); networkstatus_vote_free(v); } tor_free(pending_consensus_signatures); - pending_consensus_signatures = new_detached; *msg_out = "Signatures added"; } else if (r == 0) { *msg_out = "Signatures ignored"; @@ -2333,6 +2397,58 @@ dirvote_add_signatures_to_pending_consensus( goto err; } + goto done; + err: + if (!*msg_out) + *msg_out = "Unrecognized error while adding detached signatures."; + done: + return r; +} + +static int +dirvote_add_signatures_to_all_pending_consensuses( + const char *detached_signatures_body, + const char **msg_out) +{ + int r=0, i, n_added = 0, errors = 0; + ns_detached_signatures_t *sigs; + tor_assert(detached_signatures_body); + tor_assert(msg_out); + + if (!(sigs = networkstatus_parse_detached_signatures( + detached_signatures_body, NULL))) { + *msg_out = "Couldn't parse detached signatures."; + goto err; + } + + for (i = 0; i < N_CONSENSUS_FLAVORS; ++i) { + int res; + pending_consensus_t *pc = &pending_consensuses[i]; + if (!pc->consensus) + continue; + res = dirvote_add_signatures_to_pending_consensus(pc, sigs, msg_out); + if (res < 0) + errors++; + else + n_added += res; + } + + if (errors) { + r = -1; + goto err; + } + + /* Still not right XXXX NM NM*/ + if (pending_consensuses[FLAV_NS].consensus) { + char *new_detached = networkstatus_get_detached_signatures( + pending_consensuses[FLAV_NS].consensus); + if (new_detached) { + tor_free(pending_consensus_signatures); + pending_consensus_signatures = new_detached; + } + } + + r = n_added; goto done; err: if (!*msg_out) @@ -2340,6 +2456,8 @@ dirvote_add_signatures_to_pending_consensus( done: if (sigs) ns_detached_signatures_free(sigs); + /* XXXX NM Check how return is used. We can now have an error *and* + signatures added. */ return r; } @@ -2352,10 +2470,10 @@ dirvote_add_signatures(const char *detached_signatures_body, const char *source, const char **msg) { - if (pending_consensus) { + if (pending_consensuses[FLAV_NS].consensus) { log_notice(LD_DIR, "Got a signature from %s. " "Adding it to the pending consensus.", source); - return dirvote_add_signatures_to_pending_consensus( + return dirvote_add_signatures_to_all_pending_consensuses( detached_signatures_body, msg); } else { log_notice(LD_DIR, "Got a signature from %s. " @@ -2375,13 +2493,16 @@ static int dirvote_publish_consensus(void) { /* Can we actually publish it yet? */ - if (!pending_consensus || - networkstatus_check_consensus_signature(pending_consensus, 1)<0) { + if (!pending_consensuses[FLAV_NS].consensus || + networkstatus_check_consensus_signature( + pending_consensuses[FLAV_NS].consensus, 1)<0) { log_warn(LD_DIR, "Not enough info to publish pending consensus"); return -1; } - if (networkstatus_set_current_consensus(pending_consensus_body, 0)) + /* XXXXXX NMNMNM */ + if (networkstatus_set_current_consensus( + pending_consensuses[FLAV_NS].body, 0)) log_warn(LD_DIR, "Error publishing consensus"); else log_notice(LD_DIR, "Consensus published."); @@ -2400,12 +2521,8 @@ dirvote_free_all(void) smartlist_free(previous_vote_list); previous_vote_list = NULL; - tor_free(pending_consensus_body); + dirvote_clear_pending_consensuses(); tor_free(pending_consensus_signatures); - if (pending_consensus) { - networkstatus_vote_free(pending_consensus); - pending_consensus = NULL; - } if (pending_consensus_signature_list) { /* now empty as a result of clear_pending_votes. */ smartlist_free(pending_consensus_signature_list); @@ -2419,9 +2536,10 @@ dirvote_free_all(void) /** Return the body of the consensus that we're currently trying to build. */ const char * -dirvote_get_pending_consensus(void) +dirvote_get_pending_consensus(consensus_flavor_t flav) { - return pending_consensus_body; + tor_assert(((int)flav) >= 0 && flav < N_CONSENSUS_FLAVORS); + return pending_consensuses[flav].body; } /** Return the signatures that we know for the consensus that we're currently diff --git a/src/or/or.h b/src/or/or.h index e2050ecae4..b8ccac9239 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -1688,10 +1688,13 @@ typedef enum { /** DOCDOC */ typedef enum { - FLAV_NS, - FLAV_MICRODESC, + FLAV_NS = 0, + FLAV_MICRODESC = 1, } consensus_flavor_t; +/** DOCDOC */ +#define N_CONSENSUS_FLAVORS ((int)(FLAV_MICRODESC)+1) + /** A common structure to hold a v3 network status vote, or a v3 network * status consensus. */ typedef struct networkstatus_t { @@ -3863,7 +3866,7 @@ int dirvote_add_signatures(const char *detached_signatures_body, const char **msg_out); /* Item access */ -const char *dirvote_get_pending_consensus(void); +const char *dirvote_get_pending_consensus(consensus_flavor_t flav); const char *dirvote_get_pending_detached_signatures(void); #define DGV_BY_ID 1 #define DGV_INCLUDE_PENDING 2 From 3471057486a8aef0be6e74b090a3173e0794c84b Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 23 Sep 2009 03:02:32 -0400 Subject: [PATCH 13/23] Implement signatures for microdesc consensuses right. This means we need to handle the existence of multiple flavors of signature in a detached signatures document, generate them correctly, and so on. --- src/or/dirserv.c | 1 + src/or/dirvote.c | 216 +++++++++++++++++++++++++++++++++----------- src/or/or.h | 2 +- src/test/test_dir.c | 21 ++++- 4 files changed, 183 insertions(+), 57 deletions(-) diff --git a/src/or/dirserv.c b/src/or/dirserv.c index 326a801c41..17fff6110c 100644 --- a/src/or/dirserv.c +++ b/src/or/dirserv.c @@ -1876,6 +1876,7 @@ version_from_platform(const char *platform) * The format argument has three possible values: * NS_V2 - Output an entry suitable for a V2 NS opinion document * NS_V3_CONSENSUS - Output the first portion of a V3 NS consensus entry + * NS_V3_CONSENSUS_MICRODESC - DOCDOC * NS_V3_VOTE - Output a complete V3 NS vote * NS_CONTROL_PORT - Output a NS document for the control port */ diff --git a/src/or/dirvote.c b/src/or/dirvote.c index e3611025e9..d64e4d53a2 100644 --- a/src/or/dirvote.c +++ b/src/or/dirvote.c @@ -11,7 +11,14 @@ * \brief Functions to compute directory consensus, and schedule voting. **/ -typedef struct pending_consensus_t pending_consensus_t; +/** DOCDOC*/ +typedef struct pending_consensus_t { + /** The body of the consensus that we're currently building. Once we + * have it built, it goes into dirserv.c */ + char *body; + /** The parsed in-progress consensus document. */ + networkstatus_t *consensus; +} pending_consensus_t; static int dirvote_add_signatures_to_all_pending_consensuses( const char *detached_signatures_body, @@ -148,7 +155,7 @@ format_networkstatus_vote(crypto_pk_env_t *private_signing_key, flags, params, voter->nickname, fingerprint, voter->address, - ipaddr, voter->dir_port, voter->or_port, voter->contact); + ipaddr, voter->dir_port, voter->or_port, voter->contact); tor_free(params); tor_free(flags); @@ -433,14 +440,20 @@ compute_routerstatus_consensus(smartlist_t *votes, int consensus_method, } /** Given a list of strings in lst, set the DIGEST_LEN-byte digest at - * digest_out to the hash of the concatenation of those strings. */ + * digest_out to the hash of the concatenation of those strings. DOCDOC + * new arguments. */ static void -hash_list_members(char *digest_out, smartlist_t *lst) +hash_list_members(char *digest_out, size_t len_out, + smartlist_t *lst, digest_algorithm_t alg) { - crypto_digest_env_t *d = crypto_new_digest_env(); + crypto_digest_env_t *d; + if (alg == DIGEST_SHA1) + d = crypto_new_digest_env(); + else + d = crypto_new_digest256_env(alg); SMARTLIST_FOREACH(lst, const char *, cp, crypto_digest_add_bytes(d, cp, strlen(cp))); - crypto_digest_get_digest(d, digest_out, DIGEST_LEN); + crypto_digest_get_digest(d, digest_out, len_out); crypto_free_digest_env(d); } @@ -655,12 +668,15 @@ networkstatus_compute_consensus(smartlist_t *votes, int vote_seconds, dist_seconds; char *client_versions = NULL, *server_versions = NULL; smartlist_t *flags; + const char *flavor_name; const routerstatus_format_type_t rs_format = flavor == FLAV_NS ? NS_V3_CONSENSUS : NS_V3_CONSENSUS_MICRODESC; tor_assert(flavor == FLAV_NS || flavor == FLAV_MICRODESC); tor_assert(total_authorities >= smartlist_len(votes)); + flavor_name = networkstatus_get_flavor_name(flavor); + if (!smartlist_len(votes)) { log_warn(LD_DIR, "Can't compute a consensus from no votes."); return NULL; @@ -762,8 +778,12 @@ networkstatus_compute_consensus(smartlist_t *votes, format_iso_time(vu_buf, valid_until); flaglist = smartlist_join_strings(flags, " ", 0, NULL); - smartlist_add(chunks, tor_strdup("network-status-version 3\n" - "vote-status consensus\n")); + tor_snprintf(buf, sizeof(buf), "network-status-version 3%s%s\n" + "vote-status consensus\n", + flavor == FLAV_NS ? "" : " ", + flavor == FLAV_NS ? "" : flavor_name); + + smartlist_add(chunks, tor_strdup(buf)); if (consensus_method >= 2) { tor_snprintf(buf, sizeof(buf), "consensus-method %d\n", @@ -1285,25 +1305,36 @@ networkstatus_compute_consensus(smartlist_t *votes, /* Add a signature. */ { - char digest[DIGEST_LEN]; + char digest[DIGEST256_LEN]; char fingerprint[HEX_DIGEST_LEN+1]; char signing_key_fingerprint[HEX_DIGEST_LEN+1]; + digest_algorithm_t digest_alg = + flavor == FLAV_NS ? DIGEST_SHA1 : DIGEST_SHA256; + size_t digest_len = + flavor == FLAV_NS ? DIGEST_LEN : DIGEST256_LEN; + const char *algname = crypto_digest_algorithm_get_name(digest_alg); char buf[4096]; smartlist_add(chunks, tor_strdup("directory-signature ")); /* Compute the hash of the chunks. */ - hash_list_members(digest, chunks); + hash_list_members(digest, digest_len, chunks, digest_alg); /* Get the fingerprints */ crypto_pk_get_fingerprint(identity_key, fingerprint, 0); crypto_pk_get_fingerprint(signing_key, signing_key_fingerprint, 0); /* add the junk that will go at the end of the line. */ - tor_snprintf(buf, sizeof(buf), "%s %s\n", fingerprint, - signing_key_fingerprint); + if (flavor == FLAV_NS) { + tor_snprintf(buf, sizeof(buf), "%s %s\n", fingerprint, + signing_key_fingerprint); + } else { + tor_snprintf(buf, sizeof(buf), "%s %s %s\n", + algname, fingerprint, + signing_key_fingerprint); + } /* And the signature. */ - if (router_append_dirobj_signature(buf, sizeof(buf), digest, DIGEST_LEN, + if (router_append_dirobj_signature(buf, sizeof(buf), digest, digest_len, signing_key)) { log_warn(LD_BUG, "Couldn't sign consensus networkstatus."); return NULL; /* This leaks, but it should never happen. */ @@ -1316,9 +1347,15 @@ networkstatus_compute_consensus(smartlist_t *votes, legacy_id_key_digest, DIGEST_LEN); crypto_pk_get_fingerprint(legacy_signing_key, signing_key_fingerprint, 0); - tor_snprintf(buf, sizeof(buf), "%s %s\n", fingerprint, - signing_key_fingerprint); - if (router_append_dirobj_signature(buf, sizeof(buf), digest, DIGEST_LEN, + if (flavor == FLAV_NS) { + tor_snprintf(buf, sizeof(buf), "%s %s\n", fingerprint, + signing_key_fingerprint); + } else { + tor_snprintf(buf, sizeof(buf), "%s %s %s\n", + algname, fingerprint, + signing_key_fingerprint); + } + if (router_append_dirobj_signature(buf, sizeof(buf), digest, digest_len, legacy_signing_key)) { log_warn(LD_BUG, "Couldn't sign consensus networkstatus."); return NULL; /* This leaks, but it should never happen. */ @@ -1488,12 +1525,21 @@ networkstatus_add_detached_signatures(networkstatus_t *target, /** DOCDOC */ static char * -networkstatus_format_signatures(networkstatus_t *consensus) +networkstatus_format_signatures(networkstatus_t *consensus, + int for_detached_signatures) { smartlist_t *elements; char buf[4096]; char *result = NULL; int n_sigs = 0; + const consensus_flavor_t flavor = consensus->flavor; + const char *flavor_name = networkstatus_get_flavor_name(flavor); + const char *keyword; + + if (for_detached_signatures && flavor != FLAV_NS) + keyword = "additional-signature"; + else + keyword = "directory-signature"; elements = smartlist_create(); @@ -1501,14 +1547,25 @@ networkstatus_format_signatures(networkstatus_t *consensus) SMARTLIST_FOREACH_BEGIN(v->sigs, document_signature_t *, sig) { char sk[HEX_DIGEST_LEN+1]; char id[HEX_DIGEST_LEN+1]; - if (!sig->signature || sig->bad_signature || sig->alg != DIGEST_SHA1) + if (!sig->signature || sig->bad_signature) continue; ++n_sigs; base16_encode(sk, sizeof(sk), sig->signing_key_digest, DIGEST_LEN); base16_encode(id, sizeof(id), sig->identity_digest, DIGEST_LEN); - tor_snprintf(buf, sizeof(buf), - "directory-signature %s %s\n-----BEGIN SIGNATURE-----\n", - id, sk); + if (flavor == FLAV_NS) { + tor_snprintf(buf, sizeof(buf), + "%s %s %s\n-----BEGIN SIGNATURE-----\n", + keyword, id, sk); + } else { + const char *digest_name = + crypto_digest_algorithm_get_name(sig->alg); + tor_snprintf(buf, sizeof(buf), + "%s%s%s %s %s %s\n-----BEGIN SIGNATURE-----\n", + keyword, + for_detached_signatures ? " " : "", + for_detached_signatures ? flavor_name : "", + digest_name, id, sk); + } smartlist_add(elements, tor_strdup(buf)); base64_encode(buf, sizeof(buf), sig->signature, sig->signature_len); strlcat(buf, "-----END SIGNATURE-----\n", sizeof(buf)); @@ -1525,17 +1582,28 @@ networkstatus_format_signatures(networkstatus_t *consensus) } /** Return a newly allocated string holding the detached-signatures document - * corresponding to the signatures on consensus. */ + * corresponding to the signatures on consensuses, which must contain + * exactly one FLAV_NS consensus, and no more than one consensus for each + * other flavor. */ char * -networkstatus_get_detached_signatures(networkstatus_t *consensus) +networkstatus_get_detached_signatures(smartlist_t *consensuses) { smartlist_t *elements; char buf[4096]; - char *result = NULL; - tor_assert(consensus); - tor_assert(consensus->type == NS_TYPE_CONSENSUS); + char *result = NULL, *sigs = NULL; + networkstatus_t *consensus_ns = NULL; + tor_assert(consensuses); - tor_assert(consensus->flavor == FLAV_NS); + SMARTLIST_FOREACH(consensuses, networkstatus_t *, ns, { + tor_assert(ns); + tor_assert(ns->type == NS_TYPE_CONSENSUS); + if (ns && ns->flavor == FLAV_NS) + consensus_ns = ns; + }); + if (!consensus_ns) { + log_warn(LD_BUG, "No NS consensus given."); + return NULL; + } elements = smartlist_create(); @@ -1544,10 +1612,11 @@ networkstatus_get_detached_signatures(networkstatus_t *consensus) vu_buf[ISO_TIME_LEN+1]; char d[HEX_DIGEST_LEN+1]; - base16_encode(d, sizeof(d), consensus->digests.d[DIGEST_SHA1], DIGEST_LEN); - format_iso_time(va_buf, consensus->valid_after); - format_iso_time(fu_buf, consensus->fresh_until); - format_iso_time(vu_buf, consensus->valid_until); + base16_encode(d, sizeof(d), + consensus_ns->digests.d[DIGEST_SHA1], DIGEST_LEN); + format_iso_time(va_buf, consensus_ns->valid_after); + format_iso_time(fu_buf, consensus_ns->fresh_until); + format_iso_time(vu_buf, consensus_ns->valid_until); tor_snprintf(buf, sizeof(buf), "consensus-digest %s\n" @@ -1557,12 +1626,46 @@ networkstatus_get_detached_signatures(networkstatus_t *consensus) smartlist_add(elements, tor_strdup(buf)); } - { - char *sigs = networkstatus_format_signatures(consensus); - if (!sigs) + /* Get all the digests for the non-FLAV_NS consensuses */ + SMARTLIST_FOREACH_BEGIN(consensuses, networkstatus_t *, ns) { + const char *flavor_name = networkstatus_get_flavor_name(ns->flavor); + int alg; + if (ns->flavor == FLAV_NS) + continue; + + /* start with SHA256; we don't include SHA1 for anything but the basic + * consensus. */ + for (alg = DIGEST_SHA256; alg < N_DIGEST_ALGORITHMS; ++alg) { + char d[HEX_DIGEST256_LEN+1]; + const char *alg_name = + crypto_digest_algorithm_get_name(alg); + if (tor_mem_is_zero(ns->digests.d[alg], DIGEST256_LEN)) + continue; + base16_encode(d, sizeof(d), ns->digests.d[alg], DIGEST256_LEN); + tor_snprintf(buf, sizeof(buf), "additional-digest %s %s %s\n", + flavor_name, alg_name, d); + smartlist_add(elements, tor_strdup(buf)); + } + } SMARTLIST_FOREACH_END(ns); + + /* Now get all the sigs for non-FLAV_NS consensuses */ + SMARTLIST_FOREACH_BEGIN(consensuses, networkstatus_t *, ns) { + char *sigs; + if (ns->flavor == FLAV_NS) + continue; + sigs = networkstatus_format_signatures(ns, 1); + if (!sigs) { + log_warn(LD_DIR, "Couldn't format signatures"); goto err; + } smartlist_add(elements, sigs); - } + } SMARTLIST_FOREACH_END(ns); + + /* Now add the FLAV_NS consensus signatrures. */ + sigs = networkstatus_format_signatures(consensus_ns, 1); + if (!sigs) + goto err; + smartlist_add(elements, sigs); result = smartlist_join_strings(elements, "", 0, NULL); err: @@ -1571,6 +1674,23 @@ networkstatus_get_detached_signatures(networkstatus_t *consensus) return result; } +/** DOCDOC */ +static char * +get_detached_signatures_from_pending_consensuses(pending_consensus_t *pending, + int n_flavors) +{ + int flav; + char *signatures; + smartlist_t *c = smartlist_create(); + for (flav = 0; flav < n_flavors; ++flav) { + if (pending[flav].consensus) + smartlist_add(c, pending[flav].consensus); + } + signatures = networkstatus_get_detached_signatures(c); + smartlist_free(c); + return signatures; +} + /** Release all storage held in s. */ void ns_detached_signatures_free(ns_detached_signatures_t *s) @@ -1812,15 +1932,6 @@ static smartlist_t *pending_vote_list = NULL; * build a consensus, the votes go here for the next period. */ static smartlist_t *previous_vote_list = NULL; -/** DOCDOC*/ -struct pending_consensus_t { - /** The body of the consensus that we're currently building. Once we - * have it built, it goes into dirserv.c */ - char *body; - /** The parsed in-progress consensus document. */ - networkstatus_t *consensus; -}; - static pending_consensus_t pending_consensuses[N_CONSENSUS_FLAVORS]; /** The detached signatures for the consensus that we're currently @@ -2277,9 +2388,8 @@ dirvote_compute_consensuses(void) } } - /* XXXX NMNM NM NM wrong. */ - signatures = - networkstatus_get_detached_signatures(pending[FLAV_NS].consensus); + signatures = get_detached_signatures_from_pending_consensuses( + pending, N_CONSENSUS_FLAVORS); if (!signatures) { log_warn(LD_DIR, "Couldn't extract signatures."); @@ -2364,7 +2474,7 @@ dirvote_add_signatures_to_pending_consensus( if (r >= 1) { char *new_signatures = - networkstatus_format_signatures(pc->consensus); + networkstatus_format_signatures(pc->consensus, 0); char *dst, *dst_end; size_t new_consensus_len; if (!new_signatures) { @@ -2389,7 +2499,6 @@ dirvote_add_signatures_to_pending_consensus( tor_assert(v); networkstatus_vote_free(v); } - tor_free(pending_consensus_signatures); *msg_out = "Signatures added"; } else if (r == 0) { *msg_out = "Signatures ignored"; @@ -2414,6 +2523,7 @@ dirvote_add_signatures_to_all_pending_consensuses( ns_detached_signatures_t *sigs; tor_assert(detached_signatures_body); tor_assert(msg_out); + tor_assert(pending_consensus_signatures); if (!(sigs = networkstatus_parse_detached_signatures( detached_signatures_body, NULL))) { @@ -2438,10 +2548,10 @@ dirvote_add_signatures_to_all_pending_consensuses( goto err; } - /* Still not right XXXX NM NM*/ - if (pending_consensuses[FLAV_NS].consensus) { - char *new_detached = networkstatus_get_detached_signatures( - pending_consensuses[FLAV_NS].consensus); + if (n_added && pending_consensuses[FLAV_NS].consensus) { + char *new_detached = + get_detached_signatures_from_pending_consensuses( + pending_consensuses, N_CONSENSUS_FLAVORS); if (new_detached) { tor_free(pending_consensus_signatures); pending_consensus_signatures = new_detached; diff --git a/src/or/or.h b/src/or/or.h index b8ccac9239..d9b10c663d 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -3838,7 +3838,7 @@ char *networkstatus_compute_consensus(smartlist_t *votes, int networkstatus_add_detached_signatures(networkstatus_t *target, ns_detached_signatures_t *sigs, const char **msg_out); -char *networkstatus_get_detached_signatures(networkstatus_t *consensus); +char *networkstatus_get_detached_signatures(smartlist_t *consensuses); void ns_detached_signatures_free(ns_detached_signatures_t *s); /* cert manipulation */ diff --git a/src/test/test_dir.c b/src/test/test_dir.c index b1fac683cc..13b46c63c3 100644 --- a/src/test/test_dir.c +++ b/src/test/test_dir.c @@ -559,6 +559,21 @@ generate_ri_from_rs(const vote_routerstatus_t *vrs) return r; } +/** Helper: get a detached signatures document for a single FLAV_NS + * consensus. */ +static char * +get_detached_sigs(networkstatus_t *ns) +{ + char *r; + smartlist_t *sl; + tor_assert(ns && ns->flavor == FLAV_NS); + sl = smartlist_create(); + smartlist_add(sl,ns); + r = networkstatus_get_detached_signatures(sl); + smartlist_free(sl); + return r; +} + /** Run unit tests for generating and parsing V3 consensus networkstatus * documents. */ static void @@ -994,7 +1009,7 @@ test_dir_v3_networkstatus(void) test_memeq(&con->digests, &con3->digests, sizeof(digests_t)); /* Extract a detached signature from con3. */ - detached_text1 = networkstatus_get_detached_signatures(con3); + detached_text1 = get_detached_sigs(con3); tor_assert(detached_text1); /* Try to parse it. */ dsig1 = networkstatus_parse_detached_signatures(detached_text1, NULL); @@ -1020,10 +1035,10 @@ test_dir_v3_networkstatus(void) } /* Try adding it to con2. */ - detached_text2 = networkstatus_get_detached_signatures(con2); + detached_text2 = get_detached_sigs(con2); test_eq(1, networkstatus_add_detached_signatures(con2, dsig1, &msg)); tor_free(detached_text2); - detached_text2 = networkstatus_get_detached_signatures(con2); + detached_text2 = get_detached_sigs(con2); //printf("\n<%s>\n", detached_text2); dsig2 = networkstatus_parse_detached_signatures(detached_text2, NULL); test_assert(dsig2); From 0bce0161dded650ac6fa665a7b861d6faac9e91c Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 23 Sep 2009 11:45:54 -0400 Subject: [PATCH 14/23] Revise proposal 162: SHA256(x), not SHA256(SHA256(x)) The point of doing SHA256 twice is, generally, is to prevent message extension attacks where an attacker who knows H(A) can calculate H(A|B). But for attaching a signature to a document, the attacker already _knows_ A, so trying to keep them from calculating H(A|B) is pointless. --- doc/spec/proposals/162-consensus-flavors.txt | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/doc/spec/proposals/162-consensus-flavors.txt b/doc/spec/proposals/162-consensus-flavors.txt index 56a0b0e1ab..e257205bbe 100644 --- a/doc/spec/proposals/162-consensus-flavors.txt +++ b/doc/spec/proposals/162-consensus-flavors.txt @@ -148,11 +148,10 @@ Spec modifications: 4.1. The "sha256" signature format. The 'SHA256' signature format for directory objects is defined as - the RSA signature of the OAEP+-padded SHA256 digest of the SHA256 - digest of the item to be signed. When checking signatures, - the signature MUST be treated as valid if the signature material - begins with SHA256(SHA256(document)); this allows us to add other - data later. + the RSA signature of the OAEP+-padded SHA256 digest of the item to + be signed. When checking signatures, the signature MUST be treated + as valid if the signature material begins with SHA256(document); + this allows us to add other data later. Considerations: From a19981725d167927d32ef2a31c7a7968be3d95b6 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 23 Sep 2009 15:23:04 -0400 Subject: [PATCH 15/23] Parse detached signatures and microdesc networkstatuses correctly. --- src/or/networkstatus.c | 12 +++++ src/or/or.h | 1 + src/or/routerparse.c | 115 ++++++++++++++++++++++++++++++----------- src/test/test_dir.c | 87 +++++++++++++++++++++++++------ 4 files changed, 170 insertions(+), 45 deletions(-) diff --git a/src/or/networkstatus.c b/src/or/networkstatus.c index 616726525b..93a21ebb74 100644 --- a/src/or/networkstatus.c +++ b/src/or/networkstatus.c @@ -1983,6 +1983,18 @@ networkstatus_get_flavor_name(consensus_flavor_t flav) } } +/** DOCDOC return -1 on unknown */ +int +networkstatus_parse_flavor_name(const char *flavname) +{ + if (!strcmp(flavname, "ns")) + return FLAV_NS; + else if (!strcmp(flavname, "microdesc")) + return FLAV_MICRODESC; + else + return -1; +} + /** If question is a string beginning with "ns/" in a format the * control interface expects for a GETINFO question, set *answer to a * newly-allocated string containing networkstatus lines for the appropriate diff --git a/src/or/or.h b/src/or/or.h index d9b10c663d..a92b0b5e30 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -4211,6 +4211,7 @@ int32_t networkstatus_get_param(networkstatus_t *ns, const char *param_name, int getinfo_helper_networkstatus(control_connection_t *conn, const char *question, char **answer); const char *networkstatus_get_flavor_name(consensus_flavor_t flav); +int networkstatus_parse_flavor_name(const char *flavname); void document_signature_free(document_signature_t *sig); document_signature_t *document_signature_dup(const document_signature_t *sig); void networkstatus_free_all(void); diff --git a/src/or/routerparse.c b/src/or/routerparse.c index 21fe5f7d23..c6278c6efe 100644 --- a/src/or/routerparse.c +++ b/src/or/routerparse.c @@ -321,7 +321,7 @@ static token_rule_t extrainfo_token_table[] = { * documents. */ static token_rule_t rtrstatus_token_table[] = { T01("p", K_P, CONCAT_ARGS, NO_OBJ ), - T1( "r", K_R, GE(8), NO_OBJ ), + T1( "r", K_R, GE(7), NO_OBJ ), T1( "s", K_S, ARGS, NO_OBJ ), T01("v", K_V, CONCAT_ARGS, NO_OBJ ), T01("w", K_W, ARGS, NO_OBJ ), @@ -1900,22 +1900,29 @@ find_start_of_next_routerstatus(const char *s) * If consensus_method is nonzero, this routerstatus is part of a * consensus, and we should parse it according to the method used to * make that consensus. + * + * DOCDOC flav **/ static routerstatus_t * routerstatus_parse_entry_from_string(memarea_t *area, const char **s, smartlist_t *tokens, networkstatus_t *vote, vote_routerstatus_t *vote_rs, - int consensus_method) + int consensus_method, + consensus_flavor_t flav) { const char *eos, *s_dup = *s; routerstatus_t *rs = NULL; directory_token_t *tok; char timebuf[ISO_TIME_LEN+1]; struct in_addr in; + int offset = 0; tor_assert(tokens); tor_assert(bool_eq(vote, vote_rs)); + if (!consensus_method) + flav = FLAV_NS; + eos = find_start_of_next_routerstatus(*s); if (tokenize_string(area,*s, eos, tokens, rtrstatus_token_table,0)) { @@ -1927,7 +1934,15 @@ routerstatus_parse_entry_from_string(memarea_t *area, goto err; } tok = find_by_keyword(tokens, K_R); - tor_assert(tok->n_args >= 8); + tor_assert(tok->n_args >= 7); + if (flav == FLAV_NS) { + if (tok->n_args < 8) { + log_warn(LD_DIR, "Too few arguments to r"); + goto err; + } + } else { + offset = -1; + } if (vote_rs) { rs = &vote_rs->status; } else { @@ -1948,29 +1963,34 @@ routerstatus_parse_entry_from_string(memarea_t *area, goto err; } - if (digest_from_base64(rs->descriptor_digest, tok->args[2])) { - log_warn(LD_DIR, "Error decoding descriptor digest %s", - escaped(tok->args[2])); - goto err; + if (flav == FLAV_NS) { + if (digest_from_base64(rs->descriptor_digest, tok->args[2])) { + log_warn(LD_DIR, "Error decoding descriptor digest %s", + escaped(tok->args[2])); + goto err; + } } if (tor_snprintf(timebuf, sizeof(timebuf), "%s %s", - tok->args[3], tok->args[4]) < 0 || + tok->args[3+offset], tok->args[4+offset]) < 0 || parse_iso_time(timebuf, &rs->published_on)<0) { - log_warn(LD_DIR, "Error parsing time '%s %s'", - tok->args[3], tok->args[4]); + log_warn(LD_DIR, "Error parsing time '%s %s' [%d %d]", + tok->args[3+offset], tok->args[4+offset], + offset, (int)flav); goto err; } - if (tor_inet_aton(tok->args[5], &in) == 0) { + if (tor_inet_aton(tok->args[5+offset], &in) == 0) { log_warn(LD_DIR, "Error parsing router address in network-status %s", - escaped(tok->args[5])); + escaped(tok->args[5+offset])); goto err; } rs->addr = ntohl(in.s_addr); - rs->or_port =(uint16_t) tor_parse_long(tok->args[6],10,0,65535,NULL,NULL); - rs->dir_port = (uint16_t) tor_parse_long(tok->args[7],10,0,65535,NULL,NULL); + rs->or_port = (uint16_t) tor_parse_long(tok->args[6+offset], + 10,0,65535,NULL,NULL); + rs->dir_port = (uint16_t) tor_parse_long(tok->args[7+offset], + 10,0,65535,NULL,NULL); tok = find_opt_by_keyword(tokens, K_S); if (tok && vote) { @@ -2267,7 +2287,7 @@ networkstatus_v2_parse_from_string(const char *s) while (!strcmpstart(s, "r ")) { routerstatus_t *rs; if ((rs = routerstatus_parse_entry_from_string(area, &s, tokens, - NULL, NULL, 0))) + NULL, NULL, 0, 0))) smartlist_add(ns->entries, rs); } smartlist_sort(ns->entries, compare_routerstatus_entries); @@ -2329,6 +2349,8 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, struct in_addr in; int i, inorder, n_signatures = 0; memarea_t *area = NULL, *rs_area = NULL; + consensus_flavor_t flav = FLAV_NS; + tor_assert(s); if (eos_out) @@ -2352,6 +2374,22 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, ns = tor_malloc_zero(sizeof(networkstatus_t)); memcpy(&ns->digests, &ns_digests, sizeof(ns_digests)); + tok = find_by_keyword(tokens, K_NETWORK_STATUS_VERSION); + tor_assert(tok); + if (tok->n_args > 1) { + int flavor = networkstatus_parse_flavor_name(tok->args[1]); + if (flavor < 0) { + log_warn(LD_DIR, "Can't parse document with unknown flavor %s", + escaped(tok->args[2])); + goto err; + } + ns->flavor = flav = flavor; + } + if (flav != FLAV_NS && ns_type != NS_TYPE_CONSENSUS) { + log_warn(LD_DIR, "Flavor found on non-consenus networkstatus."); + goto err; + } + if (ns_type != NS_TYPE_CONSENSUS) { const char *end_of_cert = NULL; if (!(cert = strstr(s, "\ndir-key-certificate-version"))) @@ -2598,7 +2636,7 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, if (ns->type != NS_TYPE_CONSENSUS) { vote_routerstatus_t *rs = tor_malloc_zero(sizeof(vote_routerstatus_t)); if (routerstatus_parse_entry_from_string(rs_area, &s, rs_tokens, ns, - rs, 0)) + rs, 0, 0)) smartlist_add(ns->routerstatus_list, rs); else { tor_free(rs->version); @@ -2608,7 +2646,8 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, routerstatus_t *rs; if ((rs = routerstatus_parse_entry_from_string(rs_area, &s, rs_tokens, NULL, NULL, - ns->consensus_method))) + ns->consensus_method, + flav))) smartlist_add(ns->routerstatus_list, rs); } } @@ -2645,10 +2684,29 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, char declared_identity[DIGEST_LEN]; networkstatus_voter_info_t *v; document_signature_t *sig; + const char *id_hexdigest = NULL; + const char *sk_hexdigest = NULL; + digest_algorithm_t alg = DIGEST_SHA1; tok = _tok; if (tok->tp != K_DIRECTORY_SIGNATURE) continue; tor_assert(tok->n_args >= 2); + if (tok->n_args == 2) { + id_hexdigest = tok->args[0]; + sk_hexdigest = tok->args[1]; + } else { + const char *algname = tok->args[0]; + int a; + id_hexdigest = tok->args[1]; + sk_hexdigest = tok->args[2]; + a = crypto_digest_algorithm_parse_name(algname); + if (a<0) { + log_warn(LD_DIR, "Unknown digest algorithm %s; skipping", + escaped(algname)); + continue; + } + alg = a; + } if (!tok->object_type || strcmp(tok->object_type, "SIGNATURE") || @@ -2657,11 +2715,11 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, goto err; } - if (strlen(tok->args[0]) != HEX_DIGEST_LEN || + if (strlen(id_hexdigest) != HEX_DIGEST_LEN || base16_decode(declared_identity, sizeof(declared_identity), - tok->args[0], HEX_DIGEST_LEN) < 0) { + id_hexdigest, HEX_DIGEST_LEN) < 0) { log_warn(LD_DIR, "Error decoding declared identity %s in " - "network-status vote.", escaped(tok->args[0])); + "network-status vote.", escaped(id_hexdigest)); goto err; } if (!(v = networkstatus_get_voter_by_id(ns, declared_identity))) { @@ -2671,12 +2729,12 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, } sig = tor_malloc_zero(sizeof(document_signature_t)); memcpy(sig->identity_digest, v->identity_digest, DIGEST_LEN); - sig->alg = DIGEST_SHA1; - if (strlen(tok->args[1]) != HEX_DIGEST_LEN || + sig->alg = alg; + if (strlen(sk_hexdigest) != HEX_DIGEST_LEN || base16_decode(sig->signing_key_digest, sizeof(sig->signing_key_digest), - tok->args[1], HEX_DIGEST_LEN) < 0) { - log_warn(LD_DIR, "Error decoding declared digest %s in " - "network-status vote.", escaped(tok->args[1])); + sk_hexdigest, HEX_DIGEST_LEN) < 0) { + log_warn(LD_DIR, "Error decoding declared signing key digest %s in " + "network-status vote.", escaped(sk_hexdigest)); tor_free(sig); goto err; } @@ -2716,7 +2774,6 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, } sig->signature = tor_memdup(tok->object_body, tok->object_size); sig->signature_len = (int) tok->object_size; - } smartlist_add(v->sigs, sig); @@ -2950,10 +3007,10 @@ networkstatus_parse_detached_signatures(const char *s, const char *eos) "network-status vote.", escaped(id_hexdigest)); goto err; } - if (strlen(tok->args[1]) != HEX_DIGEST_LEN || + if (strlen(sk_hexdigest) != HEX_DIGEST_LEN || base16_decode(sk_digest, sizeof(sk_digest), sk_hexdigest, HEX_DIGEST_LEN) < 0) { - log_warn(LD_DIR, "Error decoding declared digest %s in " + log_warn(LD_DIR, "Error decoding declared signing key digest %s in " "network-status vote.", escaped(sk_hexdigest)); goto err; } @@ -2974,7 +3031,7 @@ networkstatus_parse_detached_signatures(const char *s, const char *eos) } sig = tor_malloc_zero(sizeof(document_signature_t)); - sig->alg = DIGEST_SHA1; + sig->alg = alg; memcpy(sig->identity_digest, id_digest, DIGEST_LEN); memcpy(sig->signing_key_digest, sk_digest, DIGEST_LEN); if (tok->object_size >= INT_MAX) { diff --git a/src/test/test_dir.c b/src/test/test_dir.c index 13b46c63c3..68dbbb45a4 100644 --- a/src/test/test_dir.c +++ b/src/test/test_dir.c @@ -559,16 +559,18 @@ generate_ri_from_rs(const vote_routerstatus_t *vrs) return r; } -/** Helper: get a detached signatures document for a single FLAV_NS - * consensus. */ +/** Helper: get a detached signatures document for one or two + * consensuses. */ static char * -get_detached_sigs(networkstatus_t *ns) +get_detached_sigs(networkstatus_t *ns, networkstatus_t *ns2) { char *r; smartlist_t *sl; tor_assert(ns && ns->flavor == FLAV_NS); sl = smartlist_create(); smartlist_add(sl,ns); + if (ns2) + smartlist_add(sl,ns2); r = networkstatus_get_detached_signatures(sl); smartlist_free(sl); return r; @@ -587,7 +589,8 @@ test_dir_v3_networkstatus(void) time_t now = time(NULL); networkstatus_voter_info_t *voter; document_signature_t *sig; - networkstatus_t *vote=NULL, *v1=NULL, *v2=NULL, *v3=NULL, *con=NULL; + networkstatus_t *vote=NULL, *v1=NULL, *v2=NULL, *v3=NULL, *con=NULL, + *con_md=NULL; vote_routerstatus_t *vrs; routerstatus_t *rs; char *v1_text=NULL, *v2_text=NULL, *v3_text=NULL, *consensus_text=NULL, *cp; @@ -596,7 +599,9 @@ test_dir_v3_networkstatus(void) /* For generating the two other consensuses. */ char *detached_text1=NULL, *detached_text2=NULL; char *consensus_text2=NULL, *consensus_text3=NULL; - networkstatus_t *con2=NULL, *con3=NULL; + char *consensus_text_md2=NULL, *consensus_text_md3=NULL; + char *consensus_text_md=NULL; + networkstatus_t *con2=NULL, *con_md2=NULL, *con3=NULL, *con_md3=NULL; ns_detached_signatures_t *dsig1=NULL, *dsig2=NULL; /* Parse certificates and keys. */ @@ -882,6 +887,17 @@ test_dir_v3_networkstatus(void) test_assert(con); //log_notice(LD_GENERAL, "<<%s>>\n<<%s>>\n<<%s>>\n", // v1_text, v2_text, v3_text); + consensus_text_md = networkstatus_compute_consensus(votes, 3, + cert3->identity_key, + sign_skey_3, + "AAAAAAAAAAAAAAAAAAAA", + sign_skey_leg1, + FLAV_MICRODESC); + test_assert(consensus_text_md); + con_md = networkstatus_parse_vote_from_string(consensus_text_md, NULL, + NS_TYPE_CONSENSUS); + test_assert(con_md); + test_eq(con_md->flavor, FLAV_MICRODESC); /* Check consensus contents. */ test_assert(con->type == NS_TYPE_CONSENSUS); @@ -956,19 +972,11 @@ test_dir_v3_networkstatus(void) test_assert(rs->is_valid); test_assert(!rs->is_named); /* XXXX check version */ - // x231 - // x213 /* Check signatures. the first voter is a pseudo-entry with a legacy key. * The second one hasn't signed. The fourth one has signed: validate it. */ voter = smartlist_get(con->voters, 1); test_eq(smartlist_len(voter->sigs), 0); -#if 0 - sig = smartlist_get(voter->sigs, 1); - test_assert(!sig->signature); - test_assert(!sig->good_signature); - test_assert(!sig->bad_signature); -#endif voter = smartlist_get(con->voters, 3); test_eq(smartlist_len(voter->sigs), 1); @@ -990,26 +998,45 @@ test_dir_v3_networkstatus(void) cert2->identity_key, sign_skey_2, NULL,NULL, FLAV_NS); + consensus_text_md2 = networkstatus_compute_consensus(votes, 3, + cert2->identity_key, + sign_skey_2, NULL,NULL, + FLAV_MICRODESC); smartlist_shuffle(votes); consensus_text3 = networkstatus_compute_consensus(votes, 3, cert1->identity_key, sign_skey_1, NULL,NULL, FLAV_NS); + consensus_text_md3 = networkstatus_compute_consensus(votes, 3, + cert1->identity_key, + sign_skey_1, NULL,NULL, + FLAV_MICRODESC); test_assert(consensus_text2); test_assert(consensus_text3); + test_assert(consensus_text_md2); + test_assert(consensus_text_md3); con2 = networkstatus_parse_vote_from_string(consensus_text2, NULL, NS_TYPE_CONSENSUS); con3 = networkstatus_parse_vote_from_string(consensus_text3, NULL, NS_TYPE_CONSENSUS); + con_md2 = networkstatus_parse_vote_from_string(consensus_text_md2, NULL, + NS_TYPE_CONSENSUS); + con_md3 = networkstatus_parse_vote_from_string(consensus_text_md3, NULL, + NS_TYPE_CONSENSUS); test_assert(con2); test_assert(con3); + test_assert(con_md2); + test_assert(con_md3); /* All three should have the same digest. */ test_memeq(&con->digests, &con2->digests, sizeof(digests_t)); test_memeq(&con->digests, &con3->digests, sizeof(digests_t)); + test_memeq(&con_md->digests, &con_md2->digests, sizeof(digests_t)); + test_memeq(&con_md->digests, &con_md3->digests, sizeof(digests_t)); + /* Extract a detached signature from con3. */ - detached_text1 = get_detached_sigs(con3); + detached_text1 = get_detached_sigs(con3, con_md3); tor_assert(detached_text1); /* Try to parse it. */ dsig1 = networkstatus_parse_detached_signatures(detached_text1, NULL); @@ -1024,6 +1051,11 @@ test_dir_v3_networkstatus(void) test_assert(dsig_digests); test_memeq(dsig_digests->d[DIGEST_SHA1], con3->digests.d[DIGEST_SHA1], DIGEST_LEN); + dsig_digests = strmap_get(dsig1->digests, "microdesc"); + test_assert(dsig_digests); + test_memeq(dsig_digests->d[DIGEST_SHA256], + con_md3->digests.d[DIGEST_SHA256], + DIGEST256_LEN); } { smartlist_t *dsig_signatures = strmap_get(dsig1->signatures, "ns"); @@ -1032,13 +1064,24 @@ test_dir_v3_networkstatus(void) sig = smartlist_get(dsig_signatures, 0); test_memeq(sig->identity_digest, cert1->cache_info.identity_digest, DIGEST_LEN); + test_eq(sig->alg, DIGEST_SHA1); + + dsig_signatures = strmap_get(dsig1->signatures, "microdesc"); + test_assert(dsig_signatures); + test_eq(1, smartlist_len(dsig_signatures)); + sig = smartlist_get(dsig_signatures, 0); + test_memeq(sig->identity_digest, cert1->cache_info.identity_digest, + DIGEST_LEN); + test_eq(sig->alg, DIGEST_SHA256); } /* Try adding it to con2. */ - detached_text2 = get_detached_sigs(con2); + detached_text2 = get_detached_sigs(con2,con_md2); test_eq(1, networkstatus_add_detached_signatures(con2, dsig1, &msg)); tor_free(detached_text2); - detached_text2 = get_detached_sigs(con2); + test_eq(1, networkstatus_add_detached_signatures(con_md2, dsig1, &msg)); + tor_free(detached_text2); + detached_text2 = get_detached_sigs(con2,con_md2); //printf("\n<%s>\n", detached_text2); dsig2 = networkstatus_parse_detached_signatures(detached_text2, NULL); test_assert(dsig2); @@ -1052,6 +1095,9 @@ test_dir_v3_networkstatus(void) */ test_eq(2, smartlist_len((smartlist_t*)strmap_get(dsig2->signatures, "ns"))); + test_eq(2, + smartlist_len((smartlist_t*)strmap_get(dsig2->signatures, + "microdesc"))); /* Try adding to con2 twice; verify that nothing changes. */ test_eq(0, networkstatus_add_detached_signatures(con2, dsig1, &msg)); @@ -1075,6 +1121,7 @@ test_dir_v3_networkstatus(void) tor_free(v2_text); tor_free(v3_text); tor_free(consensus_text); + tor_free(consensus_text_md); if (vote) networkstatus_vote_free(vote); @@ -1086,6 +1133,8 @@ test_dir_v3_networkstatus(void) networkstatus_vote_free(v3); if (con) networkstatus_vote_free(con); + if (con_md) + networkstatus_vote_free(con_md); if (sign_skey_1) crypto_free_pk_env(sign_skey_1); if (sign_skey_2) @@ -1103,12 +1152,18 @@ test_dir_v3_networkstatus(void) tor_free(consensus_text2); tor_free(consensus_text3); + tor_free(consensus_text_md2); + tor_free(consensus_text_md3); tor_free(detached_text1); tor_free(detached_text2); if (con2) networkstatus_vote_free(con2); if (con3) networkstatus_vote_free(con3); + if (con_md2) + networkstatus_vote_free(con_md2); + if (con_md3) + networkstatus_vote_free(con_md3); if (dsig1) ns_detached_signatures_free(dsig1); if (dsig2) From 851a980065e6b2df8d7cb35a22d0675b8918214b Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 13 Oct 2009 17:06:01 -0400 Subject: [PATCH 16/23] Actually remember all the consensus types when we are done generating them. --- src/or/control.c | 2 +- src/or/directory.c | 2 +- src/or/dirserv.c | 48 ++++--- src/or/dirvote.c | 35 +++-- src/or/networkstatus.c | 305 +++++++++++++++++++++++++++-------------- src/or/or.h | 17 ++- 6 files changed, 269 insertions(+), 140 deletions(-) diff --git a/src/or/control.c b/src/or/control.c index 67ee37ae52..554aef6b25 100644 --- a/src/or/control.c +++ b/src/or/control.c @@ -1506,7 +1506,7 @@ getinfo_helper_dir(control_connection_t *control_conn, } } else if (!strcmp(question, "dir/status-vote/current/consensus")) { /* v3 */ if (directory_caches_dir_info(get_options())) { - const cached_dir_t *consensus = dirserv_get_consensus(); + const cached_dir_t *consensus = dirserv_get_consensus("ns"); if (consensus) *answer = tor_strdup(consensus->dir); } diff --git a/src/or/directory.c b/src/or/directory.c index 0feddcbae4..acee78f030 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -1623,7 +1623,7 @@ connection_dir_client_reached_eof(dir_connection_t *conn) } log_info(LD_DIR,"Received consensus directory (size %d) from server " "'%s:%d'",(int) body_len, conn->_base.address, conn->_base.port); - if ((r=networkstatus_set_current_consensus(body, 0))<0) { + if ((r=networkstatus_set_current_consensus(body, "ns", 0))<0) { log_fn(r<-1?LOG_WARN:LOG_INFO, LD_DIR, "Unable to load consensus directory downloaded from " "server '%s:%d'. I'll try again soon.", diff --git a/src/or/dirserv.c b/src/or/dirserv.c index 17fff6110c..57d1a8d827 100644 --- a/src/or/dirserv.c +++ b/src/or/dirserv.c @@ -41,7 +41,7 @@ static time_t the_v2_networkstatus_is_dirty = 1; static cached_dir_t *the_directory = NULL; /** For authoritative directories: the current (v1) network status. */ -static cached_dir_t the_runningrouters = { NULL, NULL, 0, 0, 0, -1 }; +static cached_dir_t the_runningrouters; static void directory_remove_invalid(void); static cached_dir_t *dirserv_regenerate_directory(void); @@ -1211,14 +1211,14 @@ directory_too_idle_to_fetch_descriptors(or_options_t *options, time_t now) static cached_dir_t *cached_directory = NULL; /** The v1 runningrouters document we'll serve (as a cache or as an authority) * if requested. */ -static cached_dir_t cached_runningrouters = { NULL, NULL, 0, 0, 0, -1 }; +static cached_dir_t cached_runningrouters; /** Used for other dirservers' v2 network statuses. Map from hexdigest to * cached_dir_t. */ static digestmap_t *cached_v2_networkstatus = NULL; -/** The v3 consensus network status that we're currently serving. */ -static cached_dir_t *cached_v3_networkstatus = NULL; +/** Map from flavor name to the v3 consensuses that we're currently serving. */ +static strmap_t *cached_consensuses = NULL; /** Possibly replace the contents of d with the value of * directory published on when, unless when is older than @@ -1386,17 +1386,26 @@ dirserv_set_cached_networkstatus_v2(const char *networkstatus, } } -/** Replace the v3 consensus networkstatus that we're serving with - * networkstatus, published at published. No validation is - * performed. */ +/** Replace the v3 consensus networkstatus of type flavor_name that + * we're serving with networkstatus, published at published. No + * validation is performed. */ void -dirserv_set_cached_networkstatus_v3(const char *networkstatus, - time_t published) +dirserv_set_cached_consensus_networkstatus(const char *networkstatus, + const char *flavor_name, + const digests_t *digests, + time_t published) { - if (cached_v3_networkstatus) - cached_dir_decref(cached_v3_networkstatus); - cached_v3_networkstatus = new_cached_dir( - tor_strdup(networkstatus), published); + cached_dir_t *new_networkstatus; + cached_dir_t *old_networkstatus; + if (!cached_consensuses) + cached_consensuses = strmap_new(); + + new_networkstatus = new_cached_dir(tor_strdup(networkstatus), published); + memcpy(&new_networkstatus->digests, digests, sizeof(digests_t)); + old_networkstatus = strmap_set(cached_consensuses, flavor_name, + new_networkstatus); + if (old_networkstatus) + cached_dir_decref(old_networkstatus); } /** Remove any v2 networkstatus from the directory cache that was published @@ -1579,9 +1588,9 @@ dirserv_get_runningrouters(void) /** Return the latest downloaded consensus networkstatus in encoded, signed, * optionally compressed format, suitable for sending to clients. */ cached_dir_t * -dirserv_get_consensus(void) +dirserv_get_consensus(const char *flavor_name) { - return cached_v3_networkstatus; + return strmap_get(cached_consensuses, flavor_name); } /** For authoritative directories: the current (v2) network status. */ @@ -3130,8 +3139,8 @@ static cached_dir_t * lookup_cached_dir_by_fp(const char *fp) { cached_dir_t *d = NULL; - if (tor_digest_is_zero(fp) && cached_v3_networkstatus) - d = cached_v3_networkstatus; + if (tor_digest_is_zero(fp) && cached_consensuses) + d = strmap_get(cached_consensuses, "ns"); else if (router_digest_is_me(fp) && the_v2_networkstatus) d = the_v2_networkstatus; else if (cached_v2_networkstatus) @@ -3467,6 +3476,9 @@ dirserv_free_all(void) digestmap_free(cached_v2_networkstatus, _free_cached_dir); cached_v2_networkstatus = NULL; } - cached_dir_decref(cached_v3_networkstatus); + if (cached_consensuses) { + strmap_free(cached_consensuses, _free_cached_dir); + cached_consensuses = NULL; + } } diff --git a/src/or/dirvote.c b/src/or/dirvote.c index d64e4d53a2..e50a875411 100644 --- a/src/or/dirvote.c +++ b/src/or/dirvote.c @@ -421,13 +421,13 @@ compute_routerstatus_consensus(smartlist_t *votes, int consensus_method, microdesc_digest256_out) { smartlist_t *digests = smartlist_create(); const char *best_microdesc_digest; - SMARTLIST_FOREACH(votes, vote_routerstatus_t *, rs, { + SMARTLIST_FOREACH_BEGIN(votes, vote_routerstatus_t *, rs) { char d[DIGEST256_LEN]; if (compare_vote_rs(rs, most)) continue; if (!vote_routerstatus_find_microdesc_hash(d, rs, consensus_method)) smartlist_add(digests, tor_memdup(d, sizeof(d))); - }); + } SMARTLIST_FOREACH_END(rs); smartlist_sort_digests256(digests); best_microdesc_digest = smartlist_get_most_frequent_digest256(digests); if (best_microdesc_digest) @@ -2602,20 +2602,25 @@ dirvote_add_signatures(const char *detached_signatures_body, static int dirvote_publish_consensus(void) { - /* Can we actually publish it yet? */ - if (!pending_consensuses[FLAV_NS].consensus || - networkstatus_check_consensus_signature( - pending_consensuses[FLAV_NS].consensus, 1)<0) { - log_warn(LD_DIR, "Not enough info to publish pending consensus"); - return -1; - } + int i; - /* XXXXXX NMNMNM */ - if (networkstatus_set_current_consensus( - pending_consensuses[FLAV_NS].body, 0)) - log_warn(LD_DIR, "Error publishing consensus"); - else - log_notice(LD_DIR, "Consensus published."); + /* Now remember all the other consensuses as if we were a directory cache. */ + for (i = 0; i < N_CONSENSUS_FLAVORS; ++i) { + pending_consensus_t *pending = &pending_consensuses[i]; + const char *name; + name = networkstatus_get_flavor_name(i); + tor_assert(name); + if (!pending->consensus || + networkstatus_check_consensus_signature(pending->consensus, 1)<0) { + log_warn(LD_DIR, "Not enough info to publish pending %s consensus",name); + continue; + } + + if (networkstatus_set_current_consensus(pending->body, name, 0)) + log_warn(LD_DIR, "Error publishing %s consensus", name); + else + log_notice(LD_DIR, "Published %s consensus", name); + } return 0; } diff --git a/src/or/networkstatus.c b/src/or/networkstatus.c index 93a21ebb74..6481499717 100644 --- a/src/or/networkstatus.c +++ b/src/or/networkstatus.c @@ -35,16 +35,22 @@ static networkstatus_t *current_consensus = NULL; /** A v3 consensus networkstatus that we've received, but which we don't * have enough certificates to be happy about. */ -static networkstatus_t *consensus_waiting_for_certs = NULL; -/** The encoded version of consensus_waiting_for_certs. */ -static char *consensus_waiting_for_certs_body = NULL; -/** When did we set the current value of consensus_waiting_for_certs? If this - * is too recent, we shouldn't try to fetch a new consensus for a little while, - * to give ourselves time to get certificates for this one. */ -static time_t consensus_waiting_for_certs_set_at = 0; -/** Set to 1 if we've been holding on to consensus_waiting_for_certs so long - * that we should treat it as maybe being bad. */ -static int consensus_waiting_for_certs_dl_failed = 0; +typedef struct consensus_waiting_for_certs_t { + /** The consensus itself. */ + networkstatus_t *consensus; + /** The encoded version of the consensus, nul-terminated. */ + char *body; + /** When did we set the current value of consensus_waiting_for_certs? If + * this is too recent, we shouldn't try to fetch a new consensus for a + * little while, to give ourselves time to get certificates for this one. */ + time_t set_at; + /** Set to 1 if we've been holding on to it for so long we should maybe + * treat it as being bad. */ + int dl_failed; +} consensus_waiting_for_certs_t; + +static consensus_waiting_for_certs_t + consensus_waiting_for_certs[N_CONSENSUS_FLAVORS]; /** The last time we tried to download a networkstatus, or 0 for "never". We * use this to rate-limit download attempts for directory caches (including @@ -56,7 +62,7 @@ static time_t last_networkstatus_download_attempted = 0; * before the current consensus becomes invalid. */ static time_t time_to_download_next_consensus = 0; /** Download status for the current consensus networkstatus. */ -static download_status_t consensus_dl_status = { 0, 0, DL_SCHED_CONSENSUS }; +static download_status_t consensus_dl_status[N_CONSENSUS_FLAVORS]; /** True iff we have logged a warning about this OR's version being older than * listed by the authorities. */ @@ -89,6 +95,7 @@ networkstatus_reset_warnings(void) void networkstatus_reset_download_failures(void) { + int i; const smartlist_t *networkstatus_v2_list = networkstatus_get_v2_list(); SMARTLIST_FOREACH(networkstatus_v2_list, networkstatus_v2_t *, ns, SMARTLIST_FOREACH(ns->entries, routerstatus_t *, rs, @@ -97,7 +104,8 @@ networkstatus_reset_download_failures(void) rs->need_to_mirror = 1; }));; - download_status_reset(&consensus_dl_status); + for (i=0; i < N_CONSENSUS_FLAVORS; ++i) + download_status_reset(&consensus_dl_status[i]); if (v2_download_status_map) { digestmap_iter_t *iter; digestmap_t *map = v2_download_status_map; @@ -170,7 +178,7 @@ router_reload_v2_networkstatus(void) return 0; } -/** Read the cached v3 consensus networkstatus from the disk. */ +/** Read every cached v3 consensus networkstatus from the disk. */ int router_reload_consensus_networkstatus(void) { @@ -179,31 +187,46 @@ router_reload_consensus_networkstatus(void) struct stat st; or_options_t *options = get_options(); const unsigned int flags = NSSET_FROM_CACHE | NSSET_DONT_DOWNLOAD_CERTS; + int flav; /* FFFF Suppress warnings if cached consensus is bad? */ - - filename = get_datadir_fname("cached-consensus"); - s = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL); - if (s) { - if (networkstatus_set_current_consensus(s, flags) < -1) { - log_warn(LD_FS, "Couldn't load consensus networkstatus from \"%s\"", - filename); + for (flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) { + char buf[128]; + const char *flavor = networkstatus_get_flavor_name(flav); + if (flav == FLAV_NS) { + filename = get_datadir_fname("cached-consensus"); + } else { + tor_snprintf(buf, sizeof(buf), "cached-%s-consensus", flavor); + filename = get_datadir_fname(buf); } - tor_free(s); - } - tor_free(filename); + s = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL); + if (s) { + if (networkstatus_set_current_consensus(s, flavor, flags) < -1) { + log_warn(LD_FS, "Couldn't load consensus %s networkstatus from \"%s\"", + flavor, filename); + } + tor_free(s); + } + tor_free(filename); - filename = get_datadir_fname("unverified-consensus"); - s = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL); - if (s) { - if (networkstatus_set_current_consensus(s, + if (flav == FLAV_NS) { + filename = get_datadir_fname("unverified-consensus"); + } else { + tor_snprintf(buf, sizeof(buf), "unverified-%s-consensus", flavor); + filename = get_datadir_fname(buf); + } + + s = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL); + if (s) { + if (networkstatus_set_current_consensus(s, flavor, flags|NSSET_WAS_WAITING_FOR_CERTS)) { - log_info(LD_FS, "Couldn't load consensus networkstatus from \"%s\"", - filename); + log_info(LD_FS, "Couldn't load consensus %s networkstatus from \"%s\"", + flavor, filename); } - tor_free(s); + tor_free(s); + } + tor_free(filename); } - tor_free(filename); if (!current_consensus || (stat(options->FallbackNetworkstatusFile, &st)==0 && @@ -211,7 +234,7 @@ router_reload_consensus_networkstatus(void) s = read_file_to_str(options->FallbackNetworkstatusFile, RFTS_IGNORE_MISSING, NULL); if (s) { - if (networkstatus_set_current_consensus(s, + if (networkstatus_set_current_consensus(s, "ns", flags|NSSET_ACCEPT_OBSOLETE)) { log_info(LD_FS, "Couldn't load consensus networkstatus from \"%s\"", options->FallbackNetworkstatusFile); @@ -1123,27 +1146,32 @@ static void update_consensus_networkstatus_downloads(time_t now) { or_options_t *options = get_options(); + int i; if (!networkstatus_get_live_consensus(now)) time_to_download_next_consensus = now; /* No live consensus? Get one now!*/ if (time_to_download_next_consensus > now) return; /* Wait until the current consensus is older. */ if (authdir_mode_v3(options)) return; /* Authorities never fetch a consensus */ - if (!download_status_is_ready(&consensus_dl_status, now, + /* XXXXNM Microdescs: may need to download more types. */ + if (!download_status_is_ready(&consensus_dl_status[FLAV_NS], now, CONSENSUS_NETWORKSTATUS_MAX_DL_TRIES)) return; /* We failed downloading a consensus too recently. */ if (connection_get_by_type_purpose(CONN_TYPE_DIR, DIR_PURPOSE_FETCH_CONSENSUS)) return; /* There's an in-progress download.*/ - if (consensus_waiting_for_certs) { - /* XXXX make sure this doesn't delay sane downloads. */ - if (consensus_waiting_for_certs_set_at + DELAY_WHILE_FETCHING_CERTS > now) - return; /* We're still getting certs for this one. */ - else { - if (!consensus_waiting_for_certs_dl_failed) { - download_status_failed(&consensus_dl_status, 0); - consensus_waiting_for_certs_dl_failed=1; + for (i=0; i < N_CONSENSUS_FLAVORS; ++i) { + consensus_waiting_for_certs_t *waiting = &consensus_waiting_for_certs[i]; + if (waiting->consensus) { + /* XXXX make sure this doesn't delay sane downloads. */ + if (waiting->set_at + DELAY_WHILE_FETCHING_CERTS > now) + return; /* We're still getting certs for this one. */ + else { + if (!waiting->dl_failed) { + download_status_failed(&consensus_dl_status[FLAV_NS], 0); + waiting->dl_failed=1; + } } } } @@ -1159,7 +1187,8 @@ update_consensus_networkstatus_downloads(time_t now) void networkstatus_consensus_download_failed(int status_code) { - download_status_failed(&consensus_dl_status, status_code); + /* XXXXNM Microdescs: may need to handle more types. */ + download_status_failed(&consensus_dl_status[FLAV_NS], status_code); /* Retry immediately, if appropriate. */ update_consensus_networkstatus_downloads(time(NULL)); } @@ -1265,10 +1294,14 @@ update_networkstatus_downloads(time_t now) void update_certificate_downloads(time_t now) { - if (consensus_waiting_for_certs) - authority_certs_fetch_missing(consensus_waiting_for_certs, now); - else - authority_certs_fetch_missing(current_consensus, now); + int i; + for (i = 0; i < N_CONSENSUS_FLAVORS; ++i) { + if (consensus_waiting_for_certs[i].consensus) + authority_certs_fetch_missing(consensus_waiting_for_certs[i].consensus, + now); + } + + authority_certs_fetch_missing(current_consensus, now); } /** Return 1 if we have a consensus but we don't have enough certificates @@ -1276,7 +1309,8 @@ update_certificate_downloads(time_t now) int consensus_is_waiting_for_certs(void) { - return consensus_waiting_for_certs ? 1 : 0; + return consensus_waiting_for_certs[USABLE_CONSENSUS_FLAVOR].consensus + ? 1 : 0; } /** Return the network status with a given identity digest. */ @@ -1445,16 +1479,29 @@ networkstatus_copy_old_consensus_info(networkstatus_t *new_c, * user, and -2 for more serious problems. */ int -networkstatus_set_current_consensus(const char *consensus, unsigned flags) +networkstatus_set_current_consensus(const char *consensus, + const char *flavor, + unsigned flags) { - networkstatus_t *c; + networkstatus_t *c=NULL; int r, result = -1; time_t now = time(NULL); char *unverified_fname = NULL, *consensus_fname = NULL; + int flav = networkstatus_parse_flavor_name(flavor); const unsigned from_cache = flags & NSSET_FROM_CACHE; const unsigned was_waiting_for_certs = flags & NSSET_WAS_WAITING_FOR_CERTS; const unsigned dl_certs = !(flags & NSSET_DONT_DOWNLOAD_CERTS); const unsigned accept_obsolete = flags & NSSET_ACCEPT_OBSOLETE; + const unsigned require_flavor = flags & NSSET_REQUIRE_FLAVOR; + const digests_t *current_digests = NULL; + consensus_waiting_for_certs_t *waiting = NULL; + time_t current_valid_after = 0; + + if (flav < 0) { + /* XXXX we don't handle unrecognized flavors yet. */ + log_warn(LD_BUG, "Unrecognized consensus flavor %s", flavor); + return -2; + } /* Make sure it's parseable. */ c = networkstatus_parse_vote_from_string(consensus, NULL, NS_TYPE_CONSENSUS); @@ -1464,32 +1511,70 @@ networkstatus_set_current_consensus(const char *consensus, unsigned flags) goto done; } + if (c->flavor != flav) { + /* This wasn't the flavor we thought we were getting. */ + if (require_flavor) { + log_warn(LD_DIR, "Got consensus with unexpected flavor %s (wanted %s)", + networkstatus_get_flavor_name(c->flavor), flavor); + goto done; + } + flav = c->flavor; + flavor = networkstatus_get_flavor_name(flav); + } + + if (flav != USABLE_CONSENSUS_FLAVOR && + !directory_caches_dir_info(get_options())) { + /* This consensus is totally boring to us: we won't use it, and we won't + * serve it. Drop it. */ + result = -1; + goto done; + } + if (from_cache && !accept_obsolete && c->valid_until < now-OLD_ROUTER_DESC_MAX_AGE) { /* XXX022 when we try to make fallbackconsensus work again, we should * consider taking this out. Until then, believing obsolete consensuses * is causing more harm than good. See also bug 887. */ - log_info(LD_DIR, "Loaded an obsolete consensus. Discarding."); + log_info(LD_DIR, "Loaded an expired consensus. Discarding."); goto done; } - if (current_consensus && - !memcmp(&c->digests, ¤t_consensus->digests, sizeof(c->digests))) { + if (!strcmp(flavor, "ns")) { + consensus_fname = get_datadir_fname("cached-consensus"); + unverified_fname = get_datadir_fname("unverified-consensus"); + if (current_consensus) { + current_digests = ¤t_consensus->digests; + current_valid_after = current_consensus->valid_after; + } + } else { + cached_dir_t *cur; + char buf[128]; + tor_snprintf(buf, sizeof(buf), "cached-%s-consensus", flavor); + consensus_fname = get_datadir_fname(buf); + tor_snprintf(buf, sizeof(buf), "unverified-%s-consensus", flavor); + unverified_fname = get_datadir_fname(buf); + cur = dirserv_get_consensus(flavor); + if (cur) { + current_digests = &cur->digests; + current_valid_after = cur->published; + } + } + + if (current_digests && + !memcmp(&c->digests, current_digests, sizeof(c->digests))) { /* We already have this one. That's a failure. */ - log_info(LD_DIR, "Got a consensus we already have"); + log_info(LD_DIR, "Got a %s consensus we already have", flavor); goto done; } - if (current_consensus && c->valid_after <= current_consensus->valid_after) { + if (current_valid_after && c->valid_after <= current_valid_after) { /* We have a newer one. There's no point in accepting this one, * even if it's great. */ - log_info(LD_DIR, "Got a consensus at least as old as the one we have"); + log_info(LD_DIR, "Got a %s consensus at least as old as the one we have", + flavor); goto done; } - consensus_fname = get_datadir_fname("cached-consensus"); - unverified_fname = get_datadir_fname("unverified-consensus"); - /* Make sure it's signed enough. */ if ((r=networkstatus_check_consensus_signature(c, 1))<0) { if (r == -1) { @@ -1498,16 +1583,17 @@ networkstatus_set_current_consensus(const char *consensus, unsigned flags) log_info(LD_DIR, "Not enough certificates to check networkstatus consensus"); } - if (!current_consensus || - c->valid_after > current_consensus->valid_after) { - if (consensus_waiting_for_certs) - networkstatus_vote_free(consensus_waiting_for_certs); - tor_free(consensus_waiting_for_certs_body); - consensus_waiting_for_certs = c; + if (!current_valid_after || + c->valid_after > current_valid_after) { + waiting = &consensus_waiting_for_certs[flav]; + if (waiting->consensus) + networkstatus_vote_free(waiting->consensus); + tor_free(waiting->body); + waiting->consensus = c; c = NULL; /* Prevent free. */ - consensus_waiting_for_certs_body = tor_strdup(consensus); - consensus_waiting_for_certs_set_at = now; - consensus_waiting_for_certs_dl_failed = 0; + waiting->body = tor_strdup(consensus); + waiting->set_at = now; + waiting->dl_failed = 0; if (!from_cache) { write_str_to_file(unverified_fname, consensus, 0); } @@ -1536,56 +1622,65 @@ networkstatus_set_current_consensus(const char *consensus, unsigned flags) } } - if (!from_cache) + if (!from_cache && flav == USABLE_CONSENSUS_FLAVOR) control_event_client_status(LOG_NOTICE, "CONSENSUS_ARRIVED"); /* Are we missing any certificates at all? */ if (r != 1 && dl_certs) authority_certs_fetch_missing(c, now); - notify_control_networkstatus_changed(current_consensus, c); + if (flav == USABLE_CONSENSUS_FLAVOR) { + notify_control_networkstatus_changed(current_consensus, c); - if (current_consensus) { - networkstatus_copy_old_consensus_info(c, current_consensus); - networkstatus_vote_free(current_consensus); + if (current_consensus) { + networkstatus_copy_old_consensus_info(c, current_consensus); + networkstatus_vote_free(current_consensus); + } } - if (consensus_waiting_for_certs && - consensus_waiting_for_certs->valid_after <= c->valid_after) { - networkstatus_vote_free(consensus_waiting_for_certs); - consensus_waiting_for_certs = NULL; - if (consensus != consensus_waiting_for_certs_body) - tor_free(consensus_waiting_for_certs_body); + waiting = &consensus_waiting_for_certs[flav]; + if (waiting->consensus && + waiting->consensus->valid_after <= c->valid_after) { + networkstatus_vote_free(waiting->consensus); + waiting->consensus = NULL; + if (consensus != waiting->body) + tor_free(waiting->body); else - consensus_waiting_for_certs_body = NULL; - consensus_waiting_for_certs_set_at = 0; - consensus_waiting_for_certs_dl_failed = 0; + waiting->body = NULL; + waiting->set_at = 0; + waiting->dl_failed = 0; unlink(unverified_fname); } /* Reset the failure count only if this consensus is actually valid. */ if (c->valid_after <= now && now <= c->valid_until) { - download_status_reset(&consensus_dl_status); + download_status_reset(&consensus_dl_status[flav]); } else { if (!from_cache) - download_status_failed(&consensus_dl_status, 0); + download_status_failed(&consensus_dl_status[flav], 0); } - current_consensus = c; - c = NULL; /* Prevent free. */ + if (directory_caches_dir_info(get_options())) { + dirserv_set_cached_consensus_networkstatus(consensus, + flavor, + &c->digests, + current_valid_after); + } - update_consensus_networkstatus_fetch_time(now); - dirvote_recalculate_timing(get_options(), now); - routerstatus_list_update_named_server_map(); + if (flav == USABLE_CONSENSUS_FLAVOR) { + current_consensus = c; + c = NULL; /* Prevent free. */ + + /* XXXXNM Microdescs: needs a non-ns variant. */ + update_consensus_networkstatus_fetch_time(now); + dirvote_recalculate_timing(get_options(), now); + routerstatus_list_update_named_server_map(); + } if (!from_cache) { write_str_to_file(consensus_fname, consensus, 0); } - if (directory_caches_dir_info(get_options())) - dirserv_set_cached_networkstatus_v3(consensus, - current_consensus->valid_after); - if (ftime_definitely_before(now, current_consensus->valid_after)) { char tbuf[ISO_TIME_LEN+1]; char dbuf[64]; @@ -1616,13 +1711,17 @@ networkstatus_set_current_consensus(const char *consensus, unsigned flags) void networkstatus_note_certs_arrived(void) { - if (consensus_waiting_for_certs) { - if (networkstatus_check_consensus_signature( - consensus_waiting_for_certs, 0)>=0) { + int i; + for (i=0; iconsensus) + continue; + if (networkstatus_check_consensus_signature(waiting->consensus, 0)>=0) { if (!networkstatus_set_current_consensus( - consensus_waiting_for_certs_body, + waiting->body, + networkstatus_get_flavor_name(i), NSSET_WAS_WAITING_FOR_CERTS)) { - tor_free(consensus_waiting_for_certs_body); + tor_free(waiting->body); } } } @@ -2046,6 +2145,7 @@ getinfo_helper_networkstatus(control_connection_t *conn, void networkstatus_free_all(void) { + int i; if (networkstatus_v2_list) { SMARTLIST_FOREACH(networkstatus_v2_list, networkstatus_v2_t *, ns, networkstatus_v2_free(ns)); @@ -2060,11 +2160,14 @@ networkstatus_free_all(void) networkstatus_vote_free(current_consensus); current_consensus = NULL; } - if (consensus_waiting_for_certs) { - networkstatus_vote_free(consensus_waiting_for_certs); - consensus_waiting_for_certs = NULL; + for (i=0; i < N_CONSENSUS_FLAVORS; ++i) { + consensus_waiting_for_certs_t *waiting = &consensus_waiting_for_certs[i]; + if (waiting->consensus) { + networkstatus_vote_free(waiting->consensus); + waiting->consensus = NULL; + } + tor_free(waiting->body); } - tor_free(consensus_waiting_for_certs_body); if (named_server_map) { strmap_free(named_server_map, _tor_free); } diff --git a/src/or/or.h b/src/or/or.h index a92b0b5e30..b2e16a3b91 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -1282,6 +1282,7 @@ typedef struct cached_dir_t { size_t dir_len; /**< Length of dir (not counting its NUL). */ size_t dir_z_len; /**< Length of dir_z. */ time_t published; /**< When was this object published. */ + digests_t digests; /**< Digests of this object (networkstatus only) */ int refcnt; /**< Reference count for this cached_dir_t. */ } cached_dir_t; @@ -1692,6 +1693,9 @@ typedef enum { FLAV_MICRODESC = 1, } consensus_flavor_t; +/** DOCDOC */ +#define USABLE_CONSENSUS_FLAVOR FLAV_NS + /** DOCDOC */ #define N_CONSENSUS_FLAVORS ((int)(FLAV_MICRODESC)+1) @@ -3758,14 +3762,16 @@ int directory_too_idle_to_fetch_descriptors(or_options_t *options, time_t now); void directory_set_dirty(void); cached_dir_t *dirserv_get_directory(void); cached_dir_t *dirserv_get_runningrouters(void); -cached_dir_t *dirserv_get_consensus(void); +cached_dir_t *dirserv_get_consensus(const char *flavor_name); void dirserv_set_cached_directory(const char *directory, time_t when, int is_running_routers); void dirserv_set_cached_networkstatus_v2(const char *directory, const char *identity, time_t published); -void dirserv_set_cached_networkstatus_v3(const char *consensus, - time_t published); +void dirserv_set_cached_consensus_networkstatus(const char *consensus, + const char *flavor_name, + const digests_t *digests, + time_t published); void dirserv_clear_old_networkstatuses(time_t cutoff); void dirserv_clear_old_v1_info(time_t now); void dirserv_get_networkstatus_v2(smartlist_t *result, const char *key); @@ -4194,7 +4200,10 @@ networkstatus_t *networkstatus_get_reasonably_live_consensus(time_t now); #define NSSET_WAS_WAITING_FOR_CERTS 2 #define NSSET_DONT_DOWNLOAD_CERTS 4 #define NSSET_ACCEPT_OBSOLETE 8 -int networkstatus_set_current_consensus(const char *consensus, unsigned flags); +#define NSSET_REQUIRE_FLAVOR 16 +int networkstatus_set_current_consensus(const char *consensus, + const char *flavor, + unsigned flags); void networkstatus_note_certs_arrived(void); void routers_update_all_from_networkstatus(time_t now, int dir_version); void routerstatus_list_update_from_consensus_networkstatus(time_t now); From e26a79ca8aa0df0bd03b69ef18b49fb94b22943b Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 14 Oct 2009 13:57:51 -0400 Subject: [PATCH 17/23] Make start_writing_to_stdio_file() respect O_BINARY. --- src/common/util.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/common/util.c b/src/common/util.c index 139c1aaad8..06636b8274 100644 --- a/src/common/util.c +++ b/src/common/util.c @@ -1736,7 +1736,8 @@ write_str_to_file(const char *fname, const char *str, int bin) struct open_file_t { char *tempname; /**< Name of the temporary file. */ char *filename; /**< Name of the original file. */ - int rename_on_close; /**< Are we using the temporary file or not? */ + unsigned rename_on_close:1; /**< Are we using the temporary file or not? */ + unsigned binary:1; /**< Did we open in binary mode? */ int fd; /**< fd for the open file. */ FILE *stdio_file; /**< stdio wrapper for fd. */ }; @@ -1792,6 +1793,8 @@ start_writing_to_file(const char *fname, int open_flags, int mode, open_flags &= ~O_EXCL; new_file->rename_on_close = 1; } + if (open_flags & O_BINARY) + new_file->binary = 1; if ((new_file->fd = open(open_name, open_flags, mode)) < 0) { log(LOG_WARN, LD_FS, "Couldn't open \"%s\" (%s) for writing: %s", @@ -1830,7 +1833,8 @@ fdopen_file(open_file_t *file_data) if (file_data->stdio_file) return file_data->stdio_file; tor_assert(file_data->fd >= 0); - if (!(file_data->stdio_file = fdopen(file_data->fd, "a"))) { + if (!(file_data->stdio_file = fdopen(file_data->fd, + file_data->binary?"ab":"a"))) { log_warn(LD_FS, "Couldn't fdopen \"%s\" [%d]: %s", file_data->filename, file_data->fd, strerror(errno)); } From d61b5df9c1bed57cb39888a1f256cf6c234c29eb Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 14 Oct 2009 16:05:08 -0400 Subject: [PATCH 18/23] Fix various bugs in microdescriptor caching. --- src/or/main.c | 1 + src/or/microdesc.c | 105 ++++++++++++++++++++++++++++++++--------- src/or/networkstatus.c | 2 +- src/or/or.h | 1 + 4 files changed, 86 insertions(+), 23 deletions(-) diff --git a/src/or/main.c b/src/or/main.c index 25182919ae..9605b2bca8 100644 --- a/src/or/main.c +++ b/src/or/main.c @@ -1985,6 +1985,7 @@ tor_free_all(int postfork) connection_free_all(); buf_shrink_freelists(1); memarea_clear_freelist(); + microdesc_free_all(); if (!postfork) { config_free_all(); router_free_all(); diff --git a/src/or/microdesc.c b/src/or/microdesc.c index 0128fbbab2..2533564dd5 100644 --- a/src/or/microdesc.c +++ b/src/or/microdesc.c @@ -40,19 +40,28 @@ HT_GENERATE(microdesc_map, microdesc_t, node, _microdesc_hash, _microdesc_eq, 0.6, _tor_malloc, _tor_realloc, _tor_free); +/* returns n bytes written */ static int -dump_microdescriptor(FILE *f, microdesc_t *md) +dump_microdescriptor(FILE *f, microdesc_t *md, int *annotation_len_out) { + int r = 0; /* XXXX drops unkown annotations. */ if (md->last_listed) { char buf[ISO_TIME_LEN+1]; + char annotation[ISO_TIME_LEN+32]; format_iso_time(buf, md->last_listed); - fprintf(f, "@last-listed %s\n", buf); + tor_snprintf(annotation, sizeof(annotation), "@last-listed %s\n", buf); + fputs(annotation, f); + r += strlen(annotation); + *annotation_len_out = r; + } else { + *annotation_len_out = 0; } md->off = (off_t) ftell(f); fwrite(md->body, 1, md->bodylen, f); - return 0; + r += md->bodylen; + return r; } static microdesc_cache_t *the_microdesc_cache = NULL; @@ -72,7 +81,7 @@ get_microdesc_cache(void) } /* There are three sources of microdescriptors: - 1) Generated us while acting as a directory authority. + 1) Generated by us while acting as a directory authority. 2) Loaded from the cache on disk. 3) Downloaded. */ @@ -98,7 +107,8 @@ microdescs_add_to_cache(microdesc_cache_t *cache, return added; } -/* Returns list of added microdesc_t. Frees any not added. */ +/* Returns list of added microdesc_t. Frees any not added. Updates last_listed. + */ smartlist_t * microdescs_add_list_to_cache(microdesc_cache_t *cache, smartlist_t *descriptors, saved_location_t where, @@ -108,13 +118,17 @@ microdescs_add_list_to_cache(microdesc_cache_t *cache, open_file_t *open_file = NULL; FILE *f = NULL; // int n_added = 0; + size_t size = 0; if (where == SAVED_NOWHERE && !no_save) { - f = start_writing_to_stdio_file(cache->journal_fname, OPEN_FLAGS_APPEND, + f = start_writing_to_stdio_file(cache->journal_fname, + OPEN_FLAGS_APPEND|O_BINARY, 0600, &open_file); - if (!f) - log_warn(LD_DIR, "Couldn't append to journal in %s", - cache->journal_fname); + if (!f) { + log_warn(LD_DIR, "Couldn't append to journal in %s: %s", + cache->journal_fname, strerror(errno)); + return NULL; + } } added = smartlist_create(); @@ -131,8 +145,10 @@ microdescs_add_list_to_cache(microdesc_cache_t *cache, /* Okay, it's a new one. */ if (f) { - dump_microdescriptor(f, md); + int annotation_len; + size = dump_microdescriptor(f, md, &annotation_len); md->saved_location = SAVED_IN_JOURNAL; + cache->journal_len += size; } else { md->saved_location = where; } @@ -143,7 +159,18 @@ microdescs_add_list_to_cache(microdesc_cache_t *cache, smartlist_add(added, md); } SMARTLIST_FOREACH_END(md); - finish_writing_to_file(open_file); /*XXX Check me.*/ + if (f) + finish_writing_to_file(open_file); /*XXX Check me.*/ + + { + size_t old_content_len = + cache->cache_content ? cache->cache_content->size : 0; + if (cache->journal_len > 16384 + old_content_len && + cache->journal_len > old_content_len * 2) { + microdesc_cache_rebuild(cache); + } + } + return added; } @@ -152,9 +179,11 @@ microdesc_cache_clear(microdesc_cache_t *cache) { microdesc_t **entry, **next; for (entry = HT_START(microdesc_map, &cache->map); entry; entry = next) { + microdesc_t *md = *entry; next = HT_NEXT_RMV(microdesc_map, &cache->map, entry); - microdesc_free(*entry); + microdesc_free(md); } + HT_CLEAR(microdesc_map, &cache->map); if (cache->cache_content) { tor_munmap_file(cache->cache_content); cache->cache_content = NULL; @@ -176,8 +205,10 @@ microdesc_cache_reload(microdesc_cache_t *cache) if (mm) { added = microdescs_add_to_cache(cache, mm->data, mm->data+mm->size, SAVED_IN_CACHE, 0); - total += smartlist_len(added); - smartlist_free(added); + if (added) { + total += smartlist_len(added); + smartlist_free(added); + } } journal_content = read_file_to_str(cache->journal_fname, @@ -186,8 +217,10 @@ microdesc_cache_reload(microdesc_cache_t *cache) added = microdescs_add_to_cache(cache, journal_content, journal_content+st.st_size, SAVED_IN_JOURNAL, 0); - total += smartlist_len(added); - smartlist_free(added); + if (added) { + total += smartlist_len(added); + smartlist_free(added); + } tor_free(journal_content); } log_notice(LD_DIR, "Reloaded microdescriptor cache. Found %d descriptors.", @@ -202,8 +235,16 @@ microdesc_cache_rebuild(microdesc_cache_t *cache) FILE *f; microdesc_t **mdp; smartlist_t *wrote; + int size; + off_t off = 0; + int orig_size, new_size; - f = start_writing_to_stdio_file(cache->cache_fname, OPEN_FLAGS_REPLACE, + log_info(LD_DIR, "Rebuilding the microdescriptor cache..."); + orig_size = (int)(cache->cache_content ? cache->cache_content->size : 0); + orig_size += (int)cache->journal_len; + + f = start_writing_to_stdio_file(cache->cache_fname, + OPEN_FLAGS_REPLACE|O_BINARY, 0600, &open_file); if (!f) return -1; @@ -212,15 +253,17 @@ microdesc_cache_rebuild(microdesc_cache_t *cache) HT_FOREACH(mdp, microdesc_map, &cache->map) { microdesc_t *md = *mdp; + int annotation_len; if (md->no_save) continue; - dump_microdescriptor(f, md); + size = dump_microdescriptor(f, md, &annotation_len); + md->off = off + annotation_len; + off += size; if (md->saved_location != SAVED_IN_CACHE) { tor_free(md->body); md->saved_location = SAVED_IN_CACHE; } - smartlist_add(wrote, md); } @@ -229,21 +272,29 @@ microdesc_cache_rebuild(microdesc_cache_t *cache) if (cache->cache_content) tor_munmap_file(cache->cache_content); cache->cache_content = tor_mmap_file(cache->cache_fname); + if (!cache->cache_content && smartlist_len(wrote)) { log_err(LD_DIR, "Couldn't map file that we just wrote to %s!", cache->cache_fname); + smartlist_free(wrote); return -1; } SMARTLIST_FOREACH_BEGIN(wrote, microdesc_t *, md) { - if (md->no_save) - continue; tor_assert(md->saved_location == SAVED_IN_CACHE); md->body = (char*)cache->cache_content->data + md->off; tor_assert(!memcmp(md->body, "onion-key", 9)); - } SMARTLIST_FOREACH_END(wrote); + } SMARTLIST_FOREACH_END(md); smartlist_free(wrote); + write_str_to_file(cache->journal_fname, "", 1); + cache->journal_len = 0; + + new_size = (int)cache->cache_content->size; + log_info(LD_DIR, "Done rebuilding microdesc cache. " + "Saved %d bytes; %d still used.", + orig_size-new_size, new_size); + return 0; } @@ -265,3 +316,13 @@ microdesc_free(microdesc_t *md) tor_free(md); } +void +microdesc_free_all(void) +{ + if (the_microdesc_cache) { + microdesc_cache_clear(the_microdesc_cache); + tor_free(the_microdesc_cache->cache_fname); + tor_free(the_microdesc_cache->journal_fname); + tor_free(the_microdesc_cache); + } +} diff --git a/src/or/networkstatus.c b/src/or/networkstatus.c index 6481499717..37838f4b08 100644 --- a/src/or/networkstatus.c +++ b/src/or/networkstatus.c @@ -1664,7 +1664,7 @@ networkstatus_set_current_consensus(const char *consensus, dirserv_set_cached_consensus_networkstatus(consensus, flavor, &c->digests, - current_valid_after); + c->valid_after); } if (flav == USABLE_CONSENSUS_FLAVOR) { diff --git a/src/or/or.h b/src/or/or.h index b2e16a3b91..63d2eee9da 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -4123,6 +4123,7 @@ int microdesc_cache_reload(microdesc_cache_t *cache); void microdesc_cache_clear(microdesc_cache_t *cache); void microdesc_free(microdesc_t *md); +void microdesc_free_all(void); /********************************* networkstatus.c *********************/ From 200c39b66cc6e157e78a67481b85e237ccffbef3 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 15 Oct 2009 16:06:00 -0400 Subject: [PATCH 19/23] Document the microdescriptor code better. --- src/common/crypto.c | 3 ++- src/or/dirserv.c | 3 ++- src/or/dirvote.c | 48 ++++++++++++++++++++++++++++++------------ src/or/microdesc.c | 46 ++++++++++++++++++++++++++++++++-------- src/or/networkstatus.c | 3 ++- src/or/or.h | 34 +++++++++++++++++++++++++----- src/or/routerparse.c | 25 ++++++++++++++-------- 7 files changed, 122 insertions(+), 40 deletions(-) diff --git a/src/common/crypto.c b/src/common/crypto.c index f335f99aba..4ea4492898 100644 --- a/src/common/crypto.c +++ b/src/common/crypto.c @@ -1481,7 +1481,8 @@ crypto_digest_algorithm_get_name(digest_algorithm_t alg) } } -/** DOCDOC */ +/** Given the name of a digest algorithm, return its integer value, or -1 if + * the name is not recognized. */ int crypto_digest_algorithm_parse_name(const char *name) { diff --git a/src/or/dirserv.c b/src/or/dirserv.c index 57d1a8d827..251c35dd38 100644 --- a/src/or/dirserv.c +++ b/src/or/dirserv.c @@ -1885,7 +1885,8 @@ version_from_platform(const char *platform) * The format argument has three possible values: * NS_V2 - Output an entry suitable for a V2 NS opinion document * NS_V3_CONSENSUS - Output the first portion of a V3 NS consensus entry - * NS_V3_CONSENSUS_MICRODESC - DOCDOC + * NS_V3_CONSENSUS_MICRODESC - Output the first portion of a V3 microdesc + * consensus entry. * NS_V3_VOTE - Output a complete V3 NS vote * NS_CONTROL_PORT - Output a NS document for the control port */ diff --git a/src/or/dirvote.c b/src/or/dirvote.c index e50a875411..eb3c7d2cfa 100644 --- a/src/or/dirvote.c +++ b/src/or/dirvote.c @@ -11,7 +11,9 @@ * \brief Functions to compute directory consensus, and schedule voting. **/ -/** DOCDOC*/ +/** A consensus that we have built and are appending signatures to. Once it's + * time to publish it, it will become an active consensus if it accumulates + * enough signatures. */ typedef struct pending_consensus_t { /** The body of the consensus that we're currently building. Once we * have it built, it goes into dirserv.c */ @@ -264,7 +266,8 @@ get_voter(const networkstatus_t *vote) return smartlist_get(vote->voters, 0); } -/** DOCDOC */ +/** Return the signature made by voter using the algorithm + * alg, or NULL if none is found. */ document_signature_t * voter_get_sig_by_algorithm(const networkstatus_voter_info_t *voter, digest_algorithm_t alg) @@ -425,7 +428,8 @@ compute_routerstatus_consensus(smartlist_t *votes, int consensus_method, char d[DIGEST256_LEN]; if (compare_vote_rs(rs, most)) continue; - if (!vote_routerstatus_find_microdesc_hash(d, rs, consensus_method)) + if (!vote_routerstatus_find_microdesc_hash(d, rs, consensus_method, + DIGEST_SHA256)) smartlist_add(digests, tor_memdup(d, sizeof(d))); } SMARTLIST_FOREACH_END(rs); smartlist_sort_digests256(digests); @@ -439,9 +443,9 @@ compute_routerstatus_consensus(smartlist_t *votes, int consensus_method, return most; } -/** Given a list of strings in lst, set the DIGEST_LEN-byte digest at - * digest_out to the hash of the concatenation of those strings. DOCDOC - * new arguments. */ +/** Given a list of strings in lst, set the len_out-byte digest + * at digest_out to the hash of the concatenation of those strings, + * computed with the algorithm alg. */ static void hash_list_members(char *digest_out, size_t len_out, smartlist_t *lst, digest_algorithm_t alg) @@ -1523,7 +1527,11 @@ networkstatus_add_detached_signatures(networkstatus_t *target, return r; } -/** DOCDOC */ +/** Return a newly allocated string containing all the signatures on + * consensus by all voters. If for_detached_signatures is true, + * then the signatures will be put in a detached signatures document, so + * prefix any non-NS-flavored signatures with "additional-signature" rather + * than "directory-signature". */ static char * networkstatus_format_signatures(networkstatus_t *consensus, int for_detached_signatures) @@ -1674,7 +1682,9 @@ networkstatus_get_detached_signatures(smartlist_t *consensuses) return result; } -/** DOCDOC */ +/** Return a newly allocated string holding a detached-signatures document for + * all of the in-progress consensuses in the n_flavors-element array at + * pending. */ static char * get_detached_signatures_from_pending_consensuses(pending_consensus_t *pending, int n_flavors) @@ -2049,7 +2059,8 @@ dirvote_fetch_missing_signatures(void) 0, NULL); } -/** DOCDOC */ +/** Release all storage held by pending consensuses (those waiting for + * signatures). */ static void dirvote_clear_pending_consensuses(void) { @@ -2775,7 +2786,10 @@ dirvote_create_microdescriptor(const routerinfo_t *ri) /** Cached space-separated string to hold */ static char *microdesc_consensus_methods = NULL; -/** DOCDOC */ +/** Format the appropriate vote line to describe the microdescriptor md + * in a consensus vote document. Write it into the out_len-byte buffer + * in out. Return -1 on failure and the number of characters written + * on success. */ int dirvote_format_microdesc_vote_line(char *out, size_t out_len, const microdesc_t *md) @@ -2798,19 +2812,25 @@ dirvote_format_microdesc_vote_line(char *out, size_t out_len, return strlen(out); } -/** DOCDOC */ +/** If vrs has a hash made for the consensus method method with + * the digest algorithm alg, decode it and copy it into + * digest256_out and return 0. Otherwise return -1. */ int vote_routerstatus_find_microdesc_hash(char *digest256_out, const vote_routerstatus_t *vrs, - int method) + int method, + digest_algorithm_t alg) { /* XXXX only returns the sha256 method. */ const vote_microdesc_hash_t *h; char mstr[64]; size_t mlen; + char dstr[64]; tor_snprintf(mstr, sizeof(mstr), "%d", method); mlen = strlen(mstr); + tor_snprintf(dstr, sizeof(dstr), " %s=", + crypto_digest_algorithm_get_name(alg)); for (h = vrs->microdesc; h; h = h->next) { const char *cp = h->microdesc_hash_line; @@ -2824,10 +2844,10 @@ vote_routerstatus_find_microdesc_hash(char *digest256_out, char buf[BASE64_DIGEST256_LEN+1]; /* XXXX ignores extraneous stuff if the digest is too long. This * seems harmless enough, right? */ - cp = strstr(cp, " sha256="); + cp = strstr(cp, dstr); if (!cp) return -1; - cp += strlen(" sha256="); + cp += strlen(dstr); strlcpy(buf, cp, sizeof(buf)); return digest256_from_base64(digest256_out, buf); } diff --git a/src/or/microdesc.c b/src/or/microdesc.c index 2533564dd5..85070650e8 100644 --- a/src/or/microdesc.c +++ b/src/or/microdesc.c @@ -3,20 +3,26 @@ #include "or.h" -/** DOCDOC everything here. */ - -#define MICRODESC_IN_CONSENSUS 1 -#define MICRODESC_IN_VOTE 2 - +/** A data structure to hold a bunch of cached microdescriptors. There are + * two active files in the cache: a "cache file" that we mmap, and a "journal + * file" that we append to. Periodically, we rebuild the cache file to hold + * only the microdescriptors that we want to keep */ struct microdesc_cache_t { + /** Map from sha256-digest to microdesc_t for every microdesc_t in the + * cache. */ HT_HEAD(microdesc_map, microdesc_t) map; + /** Name of the cache file. */ char *cache_fname; + /** Name of the journal file. */ char *journal_fname; + /** Mmap'd contents of the cache file, or NULL if there is none. */ tor_mmap_t *cache_content; + /** Number of bytes used in the journal file. */ size_t journal_len; }; +/** Helper: computes a hash of md to place it in a hash table. */ static INLINE unsigned int _microdesc_hash(microdesc_t *md) { @@ -28,6 +34,7 @@ _microdesc_hash(microdesc_t *md) #endif } +/** Helper: compares a and for equality for hash-table purposes. */ static INLINE int _microdesc_eq(microdesc_t *a, microdesc_t *b) { @@ -40,7 +47,10 @@ HT_GENERATE(microdesc_map, microdesc_t, node, _microdesc_hash, _microdesc_eq, 0.6, _tor_malloc, _tor_realloc, _tor_free); -/* returns n bytes written */ +/** Write the body of md into f, with appropriate annotations. + * On success, return the total number of bytes written, and set + * *annotation_len_out to the number of bytes written as + * annotations. */ static int dump_microdescriptor(FILE *f, microdesc_t *md, int *annotation_len_out) { @@ -64,8 +74,11 @@ dump_microdescriptor(FILE *f, microdesc_t *md, int *annotation_len_out) return r; } +/** Holds a pointer to the current microdesc_cache_t object, or NULL if no + * such object has been allocated. */ static microdesc_cache_t *the_microdesc_cache = NULL; +/** Return a pointer to the microdescriptor cache, loading it if necessary. */ microdesc_cache_t * get_microdesc_cache(void) { @@ -86,7 +99,12 @@ get_microdesc_cache(void) 3) Downloaded. */ -/* Returns list of added microdesc_t. */ +/** Decode the microdescriptors from the string starting at s and + * ending at eos, and store them in cache. If no-save, + * mark them as non-writable to disk. If where is SAVED_IN_CACHE, + * leave their bodies as pointers to the mmap'd cache. If where is + * SAVED_NOWHERE, do not allow annotations. Return a list of the added + * microdescriptors. */ smartlist_t * microdescs_add_to_cache(microdesc_cache_t *cache, const char *s, const char *eos, saved_location_t where, @@ -107,8 +125,9 @@ microdescs_add_to_cache(microdesc_cache_t *cache, return added; } -/* Returns list of added microdesc_t. Frees any not added. Updates last_listed. - */ +/* As microdescs_add_to_cache, but takes a list of micrdescriptors instead of + * a string to encode. Frees any members of descriptors that it does + * not add. */ smartlist_t * microdescs_add_list_to_cache(microdesc_cache_t *cache, smartlist_t *descriptors, saved_location_t where, @@ -174,6 +193,7 @@ microdescs_add_list_to_cache(microdesc_cache_t *cache, return added; } +/** Remove every microdescriptor in cache. */ void microdesc_cache_clear(microdesc_cache_t *cache) { @@ -190,6 +210,8 @@ microdesc_cache_clear(microdesc_cache_t *cache) } } +/** Reload the contents of cache from disk. If it is empty, load it + * for the first time. Return 0 on success, -1 on failure. */ int microdesc_cache_reload(microdesc_cache_t *cache) { @@ -228,6 +250,9 @@ microdesc_cache_reload(microdesc_cache_t *cache) return 0; } +/** Regenerate the main cache file for cache, clear the journal file, + * and update every microdesc_t in the cache with pointers to its new + * location. */ int microdesc_cache_rebuild(microdesc_cache_t *cache) { @@ -298,6 +323,8 @@ microdesc_cache_rebuild(microdesc_cache_t *cache) return 0; } +/** Deallocate a single microdescriptor. Note: the microdescriptor MUST have + * previously been removed from the cache if it had ever been inserted. */ void microdesc_free(microdesc_t *md) { @@ -316,6 +343,7 @@ microdesc_free(microdesc_t *md) tor_free(md); } +/** Free all storage held in the microdesc.c module. */ void microdesc_free_all(void) { diff --git a/src/or/networkstatus.c b/src/or/networkstatus.c index 37838f4b08..e9e8663062 100644 --- a/src/or/networkstatus.c +++ b/src/or/networkstatus.c @@ -2082,7 +2082,8 @@ networkstatus_get_flavor_name(consensus_flavor_t flav) } } -/** DOCDOC return -1 on unknown */ +/** Return the consensus_flavor_t value for the flavor called flavname, + * or -1 if the flavor is not recongized. */ int networkstatus_parse_flavor_name(const char *flavname) { diff --git a/src/or/or.h b/src/or/or.h index 63d2eee9da..21e90a4cdf 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -1559,27 +1559,48 @@ typedef struct routerstatus_t { } routerstatus_t; -/**DOCDOC*/ +/** A microdescriptor is the smallest amount of information needed to build a + * circuit through a router. They are generated by the directory authorities, + * using information from the uploaded routerinfo documents. They are not + * self-signed, but are rather authenticated by having their hash in a signed + * networkstatus document. */ typedef struct microdesc_t { + /** Hashtable node, used to look up the microdesc by its digest. */ HT_ENTRY(microdesc_t) node; /* Cache information */ + /** When was this microdescriptor last listed in a consensus document? + * Once a microdesc has been unlisted long enough, we can drop it. + */ time_t last_listed; + /** Where is this microdescriptor currently stored? */ saved_location_t saved_location : 3; + /** If true, do not attempt to cache this microdescriptor on disk. */ unsigned int no_save : 1; + /** If saved_location == SAVED_IN_CACHE, this field holds the offset of the + * microdescriptor in the cache. */ off_t off; /* The string containing the microdesc. */ + /** A pointer to the encoded body of the microdescriptor. If the + * saved_location is SAVED_IN_CACHE, then the body is a pointer into an + * mmap'd region. Otherwise, it is a malloc'd string. The string might not + * be NUL-terminated; take the length from bodylen. */ char *body; + /** The length of the microdescriptor in body. */ size_t bodylen; + /** A SHA256-digest of the microdescriptor. */ char digest[DIGEST256_LEN]; /* Fields in the microdescriptor. */ + /** As routerinfo_t.onion_pkey */ crypto_pk_env_t *onion_pkey; + /** As routerinfo_t.family */ smartlist_t *family; + /** Encoded exit policy summary */ char *exitsummary; /**< exit policy summary - * XXX this probably should not stay a string. */ } microdesc_t; @@ -1687,16 +1708,18 @@ typedef enum { NS_TYPE_OPINION, } networkstatus_type_t; -/** DOCDOC */ +/** Enumerates recognized flavors of a consensus networkstatus document. All + * flavors of a consensus are generated from the same set of votes, but they + * present different types information to different versions of Tor. */ typedef enum { FLAV_NS = 0, FLAV_MICRODESC = 1, } consensus_flavor_t; -/** DOCDOC */ +/** Which consensus flavor do we actually want to use to build circuits? */ #define USABLE_CONSENSUS_FLAVOR FLAV_NS -/** DOCDOC */ +/** How many different consensus flavors are there? */ #define N_CONSENSUS_FLAVORS ((int)(FLAV_MICRODESC)+1) /** A common structure to hold a v3 network status vote, or a v3 network @@ -3892,7 +3915,8 @@ int dirvote_format_microdesc_vote_line(char *out, size_t out_len, const microdesc_t *md); int vote_routerstatus_find_microdesc_hash(char *digest256_out, const vote_routerstatus_t *vrs, - int method); + int method, + digest_algorithm_t alg); document_signature_t *voter_get_sig_by_algorithm( const networkstatus_voter_info_t *voter, digest_algorithm_t alg); diff --git a/src/or/routerparse.c b/src/or/routerparse.c index c6278c6efe..1f89cffa01 100644 --- a/src/or/routerparse.c +++ b/src/or/routerparse.c @@ -640,7 +640,8 @@ router_get_networkstatus_v2_hash(const char *s, char *digest) DIGEST_SHA1); } -/** DOCDOC */ +/** Set digests to all the digests of the consensus document in + * s */ int router_get_networkstatus_v3_hashes(const char *s, digests_t *digests) { @@ -677,8 +678,6 @@ router_get_extrainfo_hash(const char *s, char *digest) * surround it with -----BEGIN/END----- pairs, and write it to the * buf_len-byte buffer at buf. Return 0 on success, -1 on * failure. - * - * DOCDOC alg */ int router_append_dirobj_signature(char *buf, size_t buf_len, const char *digest, @@ -1901,7 +1900,7 @@ find_start_of_next_routerstatus(const char *s) * consensus, and we should parse it according to the method used to * make that consensus. * - * DOCDOC flav + * Parse according to the syntax used by the consensus flavor flav. **/ static routerstatus_t * routerstatus_parse_entry_from_string(memarea_t *area, @@ -2832,7 +2831,9 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, return ns; } -/** DOCDOC */ +/** Return the digests_t that holds the digests of the + * flavor_name-flavored networkstatus according to the detached + * signatures document sigs, allocating a new digests_t as neeeded. */ static digests_t * detached_get_digests(ns_detached_signatures_t *sigs, const char *flavor_name) { @@ -2844,7 +2845,9 @@ detached_get_digests(ns_detached_signatures_t *sigs, const char *flavor_name) return d; } -/** DOCDOC */ +/** Return the list of signatures of the flavor_name-flavored + * networkstatus according to the detached signatures document sigs, + * allocating a new digests_t as neeeded. */ static smartlist_t * detached_get_signatures(ns_detached_signatures_t *sigs, const char *flavor_name) @@ -3719,8 +3722,8 @@ router_get_hashes_impl(const char *s, digests_t *digests, return 0; } -/** DOCDOC Assuming that s starts with a microdesc, return the start of the - * *NEXT* one. */ +/** Assuming that s starts with a microdesc, return the start of the + * *NEXT* one. Return NULL on "not found." */ static const char * find_start_of_next_microdesc(const char *s, const char *eos) { @@ -3771,7 +3774,11 @@ find_start_of_next_microdesc(const char *s, const char *eos) #undef NEXT_LINE } -/**DOCDOC*/ +/** Parse as many microdescriptors as are found from the string starting at + * s and ending at eos. If allow_annotations is set, read any + * annotations we recognize and ignore ones we don't. If copy_body is + * true, then strdup the bodies of the microdescriptors. Return all newly + * parsed microdescriptors in a newly allocated smartlist_t. */ smartlist_t * microdescs_parse_from_string(const char *s, const char *eos, int allow_annotations, int copy_body) From a007a7c6ba39c77b36faa1e1c4864252ec1af8b7 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Sun, 18 Oct 2009 15:09:56 -0400 Subject: [PATCH 20/23] Revise consensus-flavors proposal to better URLs. The old flavored consensus URL format made it harder to decode URLs based on their prefixes, and didn't take into account our "only give it to me if it's signed by enough authorities" stuff. --- doc/spec/proposals/162-consensus-flavors.txt | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/doc/spec/proposals/162-consensus-flavors.txt b/doc/spec/proposals/162-consensus-flavors.txt index e257205bbe..e3b697afee 100644 --- a/doc/spec/proposals/162-consensus-flavors.txt +++ b/doc/spec/proposals/162-consensus-flavors.txt @@ -72,9 +72,11 @@ Spec modifications: design. In addition to the consensus currently served at - /tor/status-vote/(current|next)/consensus.z , authorities serve - another consensus of each flavor "F" from the location - /tor/status-vote/(current|next)/F/consensus.z. + /tor/status-vote/(current|next)/consensus.z and + /tor/status-vote/(current|next)/consensus/+++....z , + authorities serve another consensus of each flavor "F" from the + locations /tor/status-vote/(current|next)/consensus-F.z. and + /tor/status-vote/(current|next)/consensus-F/+....z. When caches serve these documents, they do so from the same locations. From bb22d8fc45f143666f0ee676e0aeae200104a421 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Sun, 18 Oct 2009 15:45:57 -0400 Subject: [PATCH 21/23] Add functions to serve microdescs and flavored consensuses. --- src/or/directory.c | 166 ++++++++++++++++++++++++++++++++------------ src/or/dirserv.c | 93 ++++++++++++++++++++++--- src/or/microdesc.c | 35 ++++++++++ src/or/or.h | 19 ++++- src/or/routerlist.c | 2 +- src/test/test_dir.c | 131 +++++++++++++++++++++++++++++++++- 6 files changed, 385 insertions(+), 61 deletions(-) diff --git a/src/or/directory.c b/src/or/directory.c index acee78f030..01647ceaa2 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -92,6 +92,7 @@ static void directory_initiate_command_rend(const char *address, #define ROUTERDESC_CACHE_LIFETIME (30*60) #define ROUTERDESC_BY_DIGEST_CACHE_LIFETIME (48*60*60) #define ROBOTS_CACHE_LIFETIME (24*60*60) +#define MICRODESC_CACHE_LIFETIME (48*60*60) /********* END VARIABLES ************/ @@ -610,7 +611,7 @@ connection_dir_download_networkstatus_failed(dir_connection_t *conn, * failed, and possibly retry them later.*/ smartlist_t *failed = smartlist_create(); dir_split_resource_into_fingerprints(conn->requested_resource+3, - failed, NULL, 0, 0); + failed, NULL, 0); if (smartlist_len(failed)) { dir_networkstatus_download_failed(failed, status_code); SMARTLIST_FOREACH(failed, char *, cp, tor_free(cp)); @@ -647,7 +648,7 @@ connection_dir_download_cert_failed(dir_connection_t *conn, int status) return; failed = smartlist_create(); dir_split_resource_into_fingerprints(conn->requested_resource+3, - failed, NULL, 1, 0); + failed, NULL, DSR_HEX); SMARTLIST_FOREACH(failed, char *, cp, { authority_cert_dl_failed(cp, status); @@ -1564,7 +1565,7 @@ connection_dir_client_reached_eof(dir_connection_t *conn) source = NS_FROM_DIR_BY_FP; which = smartlist_create(); dir_split_resource_into_fingerprints(conn->requested_resource+3, - which, NULL, 0, 0); + which, NULL, 0); } else if (conn->requested_resource && !strcmpstart(conn->requested_resource, "all")) { source = NS_FROM_DIR_ALL; @@ -1717,7 +1718,7 @@ connection_dir_client_reached_eof(dir_connection_t *conn) which = smartlist_create(); dir_split_resource_into_fingerprints(conn->requested_resource + (descriptor_digests ? 2 : 3), - which, NULL, 0, 0); + which, NULL, 0); n_asked_for = smartlist_len(which); } if (status_code != 200) { @@ -2328,7 +2329,7 @@ client_likes_consensus(networkstatus_t *v, const char *want_url) int need_at_least; int have = 0; - dir_split_resource_into_fingerprints(want_url, want_authorities, NULL, 0, 0); + dir_split_resource_into_fingerprints(want_url, want_authorities, NULL, 0); need_at_least = smartlist_len(want_authorities)/2+1; SMARTLIST_FOREACH_BEGIN(want_authorities, const char *, d) { char want_digest[DIGEST_LEN]; @@ -2504,6 +2505,7 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, const char *request_type = NULL; const char *key = url + strlen("/tor/status/"); long lifetime = NETWORKSTATUS_CACHE_LIFETIME; + if (!is_v3) { dirserv_get_networkstatus_v2_fingerprints(dir_fps, key); if (!strcmpstart(key, "fp/")) @@ -2518,19 +2520,44 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, } else { networkstatus_t *v = networkstatus_get_latest_consensus(); time_t now = time(NULL); + const char *want_fps = NULL; + char *flavor = NULL; #define CONSENSUS_URL_PREFIX "/tor/status-vote/current/consensus/" - if (v && - !strcmpstart(url, CONSENSUS_URL_PREFIX) && - !client_likes_consensus(v, url + strlen(CONSENSUS_URL_PREFIX))) { + #define CONSENSUS_FLAVORED_PREFIX "/tor/status-vote/current/consensus-" + /* figure out the flavor if any, and who we wanted to sign the thing */ + if (!strcmpstart(url, CONSENSUS_FLAVORED_PREFIX)) { + const char *f, *cp; + f = url + strlen(CONSENSUS_FLAVORED_PREFIX); + cp = strchr(f, '/'); + if (cp) { + want_fps = cp+1; + flavor = tor_strndup(f, cp-f); + } else { + flavor = tor_strdup(f); + } + } else { + if (!strcmpstart(url, CONSENSUS_URL_PREFIX)) + want_fps = url+strlen(CONSENSUS_URL_PREFIX); + } + + /* XXXX MICRODESC NM NM should check document of correct flavor */ + if (v && want_fps && + !client_likes_consensus(v, want_fps)) { write_http_status_line(conn, 404, "Consensus not signed by sufficient " "number of requested authorities"); smartlist_free(dir_fps); geoip_note_ns_response(act, GEOIP_REJECT_NOT_ENOUGH_SIGS); + tor_free(flavor); goto done; } - smartlist_add(dir_fps, tor_memdup("\0\0\0\0\0\0\0\0\0\0" - "\0\0\0\0\0\0\0\0\0\0", 20)); + { + char *fp = tor_malloc_zero(DIGEST_LEN); + if (flavor) + strlcpy(fp, flavor, DIGEST_LEN); + tor_free(flavor); + smartlist_add(dir_fps, fp); + } request_type = compressed?"v3.z":"v3"; lifetime = (v && v->fresh_until > now) ? v->fresh_until - now : 0; } @@ -2644,7 +2671,8 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, flags = DGV_BY_ID | (current ? DGV_INCLUDE_PREVIOUS : DGV_INCLUDE_PENDING); } - dir_split_resource_into_fingerprints(url, fps, NULL, 1, 1); + dir_split_resource_into_fingerprints(url, fps, NULL, + DSR_HEX|DSR_SORT_UNIQ); SMARTLIST_FOREACH(fps, char *, fp, { if ((d = dirvote_get_vote(fp, flags))) smartlist_add(dir_items, (cached_dir_t*)d); @@ -2697,6 +2725,41 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, goto done; } + if (!strcmpstart(url, "/tor/micro/d/")) { + smartlist_t *fps = smartlist_create(); + + dir_split_resource_into_fingerprints(url+strlen("/tor/micro/d/"), + fps, NULL, + DSR_DIGEST256|DSR_BASE64|DSR_SORT_UNIQ); + + if (!dirserv_have_any_microdesc(fps)) { + write_http_status_line(conn, 404, "Not found"); + SMARTLIST_FOREACH(fps, char *, fp, tor_free(fp)); + smartlist_free(fps); + goto done; + } + dlen = dirserv_estimate_microdesc_size(fps, compressed); + if (global_write_bucket_low(TO_CONN(conn), dlen, 2)) { + log_info(LD_DIRSERV, + "Client asked for server descriptors, but we've been " + "writing too many bytes lately. Sending 503 Dir busy."); + write_http_status_line(conn, 503, "Directory busy, try again later"); + SMARTLIST_FOREACH(fps, char *, fp, tor_free(fp)); + smartlist_free(fps); + goto done; + } + + write_http_response_header(conn, -1, compressed, MICRODESC_CACHE_LIFETIME); + conn->dir_spool_src = DIR_SPOOL_MICRODESC; + conn->fingerprint_stack = fps; + + if (compressed) + conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD); + + connection_dirserv_flushed_some(conn); + goto done; + } + if (!strcmpstart(url,"/tor/server/") || (!options->BridgeAuthoritativeDir && !options->BridgeRelay && !strcmpstart(url,"/tor/extra/"))) { @@ -2778,7 +2841,8 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, } else if (!strcmpstart(url, "/tor/keys/fp/")) { smartlist_t *fps = smartlist_create(); dir_split_resource_into_fingerprints(url+strlen("/tor/keys/fp/"), - fps, NULL, 1, 1); + fps, NULL, + DSR_HEX|DSR_SORT_UNIQ); SMARTLIST_FOREACH(fps, char *, d, { authority_cert_t *c = authority_cert_get_newest_by_id(d); if (c) smartlist_add(certs, c); @@ -2788,7 +2852,8 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, } else if (!strcmpstart(url, "/tor/keys/sk/")) { smartlist_t *fps = smartlist_create(); dir_split_resource_into_fingerprints(url+strlen("/tor/keys/sk/"), - fps, NULL, 1, 1); + fps, NULL, + DSR_HEX|DSR_SORT_UNIQ); SMARTLIST_FOREACH(fps, char *, d, { authority_cert_t *c = authority_cert_get_by_sk_digest(d); if (c) smartlist_add(certs, c); @@ -3523,19 +3588,37 @@ dir_split_resource_into_fingerprint_pairs(const char *res, /** Given a directory resource request, containing zero * or more strings separated by plus signs, followed optionally by ".z", store * the strings, in order, into fp_out. If compressed_out is - * non-NULL, set it to 1 if the resource ends in ".z", else set it to 0. If - * decode_hex is true, then delete all elements that aren't hex digests, and - * decode the rest. If sort_uniq is true, then sort the list and remove - * all duplicates. + * non-NULL, set it to 1 if the resource ends in ".z", else set it to 0. + * + * If (flags & DSR_HEX), then delete all elements that aren't hex digests, and + * decode the rest. If (flags & DSR_BASE64), then use "-" rather than "+" as + * a separator, delete all the elements that aren't base64-encoded digests, + * and decode the rest. If (flags & DSR_DIGEST256), these digests should be + * 256 bits long; else they should be 160. + * + * If (flags & DSR_SORT_UNIQ), then sort the list and remove all duplicates. */ int dir_split_resource_into_fingerprints(const char *resource, smartlist_t *fp_out, int *compressed_out, - int decode_hex, int sort_uniq) + int flags) { + const int decode_hex = flags & DSR_HEX; + const int decode_base64 = flags & DSR_BASE64; + const int digests_are_256 = flags & DSR_DIGEST256; + const int sort_uniq = flags & DSR_SORT_UNIQ; + + const int digest_len = digests_are_256 ? DIGEST256_LEN : DIGEST_LEN; + const int hex_digest_len = digests_are_256 ? + HEX_DIGEST256_LEN : HEX_DIGEST_LEN; + const int base64_digest_len = digests_are_256 ? + BASE64_DIGEST256_LEN : BASE64_DIGEST_LEN; smartlist_t *fp_tmp = smartlist_create(); + + tor_assert(!(decode_hex && decode_base64)); tor_assert(fp_out); - smartlist_split_string(fp_tmp, resource, "+", 0, 0); + + smartlist_split_string(fp_tmp, resource, decode_base64?"-":"+", 0, 0); if (compressed_out) *compressed_out = 0; if (smartlist_len(fp_tmp)) { @@ -3547,22 +3630,25 @@ dir_split_resource_into_fingerprints(const char *resource, *compressed_out = 1; } } - if (decode_hex) { + if (decode_hex || decode_base64) { + const size_t encoded_len = decode_hex ? hex_digest_len : base64_digest_len; int i; char *cp, *d = NULL; for (i = 0; i < smartlist_len(fp_tmp); ++i) { cp = smartlist_get(fp_tmp, i); - if (strlen(cp) != HEX_DIGEST_LEN) { + if (strlen(cp) != encoded_len) { log_info(LD_DIR, "Skipping digest %s with non-standard length.", escaped(cp)); smartlist_del_keeporder(fp_tmp, i--); goto again; } - d = tor_malloc_zero(DIGEST_LEN); - if (base16_decode(d, DIGEST_LEN, cp, HEX_DIGEST_LEN)<0) { - log_info(LD_DIR, "Skipping non-decodable digest %s", escaped(cp)); - smartlist_del_keeporder(fp_tmp, i--); - goto again; + d = tor_malloc_zero(digest_len); + if (decode_hex ? + (base16_decode(d, digest_len, cp, hex_digest_len)<0) : + (base64_decode(d, digest_len, cp, base64_digest_len)<0)) { + log_info(LD_DIR, "Skipping non-decodable digest %s", escaped(cp)); + smartlist_del_keeporder(fp_tmp, i--); + goto again; } smartlist_set(fp_tmp, i, d); d = NULL; @@ -3572,26 +3658,18 @@ dir_split_resource_into_fingerprints(const char *resource, } } if (sort_uniq) { - smartlist_t *fp_tmp2 = smartlist_create(); - int i; - if (decode_hex) - smartlist_sort_digests(fp_tmp); - else + if (decode_hex || decode_base64) { + if (digests_are_256) { + smartlist_sort_digests256(fp_tmp); + smartlist_uniq_digests256(fp_tmp); + } else { + smartlist_sort_digests(fp_tmp); + smartlist_uniq_digests(fp_tmp); + } + } else { smartlist_sort_strings(fp_tmp); - if (smartlist_len(fp_tmp)) - smartlist_add(fp_tmp2, smartlist_get(fp_tmp, 0)); - for (i = 1; i < smartlist_len(fp_tmp); ++i) { - char *cp = smartlist_get(fp_tmp, i); - char *last = smartlist_get(fp_tmp2, smartlist_len(fp_tmp2)-1); - - if ((decode_hex && memcmp(cp, last, DIGEST_LEN)) - || (!decode_hex && strcasecmp(cp, last))) - smartlist_add(fp_tmp2, cp); - else - tor_free(cp); + smartlist_uniq_strings(fp_tmp); } - smartlist_free(fp_tmp); - fp_tmp = fp_tmp2; } smartlist_add_all(fp_out, fp_tmp); smartlist_free(fp_tmp); diff --git a/src/or/dirserv.c b/src/or/dirserv.c index 251c35dd38..6ce6a461e1 100644 --- a/src/or/dirserv.c +++ b/src/or/dirserv.c @@ -2870,7 +2870,8 @@ dirserv_get_networkstatus_v2_fingerprints(smartlist_t *result, log_info(LD_DIRSERV, "Client requested 'all' network status objects; we have none."); } else if (!strcmpstart(key, "fp/")) { - dir_split_resource_into_fingerprints(key+3, result, NULL, 1, 1); + dir_split_resource_into_fingerprints(key+3, result, NULL, + DSR_HEX|DSR_SORT_UNIQ); } } @@ -2935,10 +2936,12 @@ dirserv_get_routerdesc_fingerprints(smartlist_t *fps_out, const char *key, } else if (!strcmpstart(key, "d/")) { by_id = 0; key += strlen("d/"); - dir_split_resource_into_fingerprints(key, fps_out, NULL, 1, 1); + dir_split_resource_into_fingerprints(key, fps_out, NULL, + DSR_HEX|DSR_SORT_UNIQ); } else if (!strcmpstart(key, "fp/")) { key += strlen("fp/"); - dir_split_resource_into_fingerprints(key, fps_out, NULL, 1, 1); + dir_split_resource_into_fingerprints(key, fps_out, NULL, + DSR_HEX|DSR_SORT_UNIQ); } else { *msg = "Key not recognized"; return -1; @@ -3003,7 +3006,8 @@ dirserv_get_routerdescs(smartlist_t *descs_out, const char *key, } else if (!strcmpstart(key, "/tor/server/d/")) { smartlist_t *digests = smartlist_create(); key += strlen("/tor/server/d/"); - dir_split_resource_into_fingerprints(key, digests, NULL, 1, 1); + dir_split_resource_into_fingerprints(key, digests, NULL, + DSR_HEX|DSR_SORT_UNIQ); SMARTLIST_FOREACH(digests, const char *, d, { signed_descriptor_t *sd = router_get_by_descriptor_digest(d); @@ -3016,7 +3020,8 @@ dirserv_get_routerdescs(smartlist_t *descs_out, const char *key, smartlist_t *digests = smartlist_create(); time_t cutoff = time(NULL) - ROUTER_MAX_AGE_TO_PUBLISH; key += strlen("/tor/server/fp/"); - dir_split_resource_into_fingerprints(key, digests, NULL, 1, 1); + dir_split_resource_into_fingerprints(key, digests, NULL, + DSR_HEX|DSR_SORT_UNIQ); SMARTLIST_FOREACH(digests, const char *, d, { if (router_digest_is_me(d)) { @@ -3132,17 +3137,20 @@ dirserv_test_reachability(time_t now, int try_all) ctr = (ctr + 1) % 128; } -/** Given a fingerprint fp which is either set if we're looking - * for a v2 status, or zeroes if we're looking for a v3 status, return - * a pointer to the appropriate cached dir object, or NULL if there isn't - * one available. */ +/** Given a fingerprint fp which is either set if we're looking for a + * v2 status, or zeroes if we're looking for a v3 status, or a NUL-padded + * flavor name if we want a flavored v3 status, return a pointer to the + * appropriate cached dir object, or NULL if there isn't one available. */ static cached_dir_t * lookup_cached_dir_by_fp(const char *fp) { cached_dir_t *d = NULL; if (tor_digest_is_zero(fp) && cached_consensuses) d = strmap_get(cached_consensuses, "ns"); - else if (router_digest_is_me(fp) && the_v2_networkstatus) + else if (memchr(fp, '\0', DIGEST_LEN) && cached_consensuses && + (d = strmap_get(cached_consensuses, fp))) { + /* this here interface is a nasty hack XXXX022 */; + } else if (router_digest_is_me(fp) && the_v2_networkstatus) d = the_v2_networkstatus; else if (cached_v2_networkstatus) d = digestmap_get(cached_v2_networkstatus, fp); @@ -3228,6 +3236,18 @@ dirserv_have_any_serverdesc(smartlist_t *fps, int spool_src) return 0; } +/** Return true iff any of the 256-bit elements in fps is the digest of + * a microdescriptor we have. */ +int +dirserv_have_any_microdesc(const smartlist_t *fps) +{ + microdesc_cache_t *cache = get_microdesc_cache(); + SMARTLIST_FOREACH(fps, const char *, fp, + if (microdesc_cache_lookup_by_digest256(cache, fp)) + return 1); + return 0; +} + /** Return an approximate estimate of the number of bytes that will * be needed to transmit the server descriptors (if is_serverdescs -- * they can be either d/ or fp/ queries) or networkstatus objects (if @@ -3259,6 +3279,17 @@ dirserv_estimate_data_size(smartlist_t *fps, int is_serverdescs, return result; } +/** Given a list of microdescriptor hashes, guess how many bytes will be + * needed to transmit them, and return the guess. */ +size_t +dirserv_estimate_microdesc_size(const smartlist_t *fps, int compressed) +{ + size_t result = smartlist_len(fps) * microdesc_average_size(NULL); + if (compressed) + result /= 2; + return result; +} + /** When we're spooling data onto our outbuf, add more whenever we dip * below this threshold. */ #define DIRSERV_BUFFER_MIN 16384 @@ -3322,6 +3353,8 @@ connection_dirserv_add_servers_to_outbuf(dir_connection_t *conn) #endif body = signed_descriptor_get_body(sd); if (conn->zlib_state) { + /* XXXX022 This 'last' business should actually happen on the last + * routerinfo, not on the last fingerprint. */ int last = ! smartlist_len(conn->fingerprint_stack); connection_write_to_buf_zlib(body, sd->signed_descriptor_len, conn, last); @@ -3345,6 +3378,44 @@ connection_dirserv_add_servers_to_outbuf(dir_connection_t *conn) return 0; } +/** Spooling helper: called when we're sending a bunch of microdescriptors, + * and the outbuf has become too empty. Pulls some entries from + * fingerprint_stack, and writes the corresponding microdescs onto outbuf. If + * we run out of entries, flushes the zlib state and sets the spool source to + * NONE. Returns 0 on success, negative on failure. + */ +static int +connection_dirserv_add_microdescs_to_outbuf(dir_connection_t *conn) +{ + microdesc_cache_t *cache = get_microdesc_cache(); + while (smartlist_len(conn->fingerprint_stack) && + buf_datalen(conn->_base.outbuf) < DIRSERV_BUFFER_MIN) { + char *fp256 = smartlist_pop_last(conn->fingerprint_stack); + microdesc_t *md = microdesc_cache_lookup_by_digest256(cache, fp256); + tor_free(fp256); + if (!md) + continue; + if (conn->zlib_state) { + /* XXXX022 This 'last' business should actually happen on the last + * routerinfo, not on the last fingerprint. */ + int last = !smartlist_len(conn->fingerprint_stack); + connection_write_to_buf_zlib(md->body, md->bodylen, conn, last); + if (last) { + tor_zlib_free(conn->zlib_state); + conn->zlib_state = NULL; + } + } else { + connection_write_to_buf(md->body, md->bodylen, TO_CONN(conn)); + } + } + if (!smartlist_len(conn->fingerprint_stack)) { + conn->dir_spool_src = DIR_SPOOL_NONE; + smartlist_free(conn->fingerprint_stack); + conn->fingerprint_stack = NULL; + } + return 0; +} + /** Spooling helper: Called when we're sending a directory or networkstatus, * and the outbuf has become too empty. Pulls some bytes from * conn-\>cached_dir-\>dir_z, uncompresses them if appropriate, and @@ -3452,6 +3523,8 @@ connection_dirserv_flushed_some(dir_connection_t *conn) case DIR_SPOOL_SERVER_BY_DIGEST: case DIR_SPOOL_SERVER_BY_FP: return connection_dirserv_add_servers_to_outbuf(conn); + case DIR_SPOOL_MICRODESC: + return connection_dirserv_add_microdescs_to_outbuf(conn); case DIR_SPOOL_CACHED_DIR: return connection_dirserv_add_dir_bytes_to_outbuf(conn); case DIR_SPOOL_NETWORKSTATUS: diff --git a/src/or/microdesc.c b/src/or/microdesc.c index 85070650e8..e10589fec9 100644 --- a/src/or/microdesc.c +++ b/src/or/microdesc.c @@ -20,6 +20,11 @@ struct microdesc_cache_t { tor_mmap_t *cache_content; /** Number of bytes used in the journal file. */ size_t journal_len; + + /** Total bytes of microdescriptor bodies we have added to this cache */ + uint64_t total_len_seen; + /** Total number of microdescriptors we have added to this cache */ + unsigned n_seen; }; /** Helper: computes a hash of md to place it in a hash table. */ @@ -176,6 +181,8 @@ microdescs_add_list_to_cache(microdesc_cache_t *cache, HT_INSERT(microdesc_map, &cache->map, md); smartlist_add(added, md); + ++cache->n_seen; + cache->total_len_seen += md->bodylen; } SMARTLIST_FOREACH_END(md); if (f) @@ -208,6 +215,8 @@ microdesc_cache_clear(microdesc_cache_t *cache) tor_munmap_file(cache->cache_content); cache->cache_content = NULL; } + cache->total_len_seen = 0; + cache->n_seen = 0; } /** Reload the contents of cache from disk. If it is empty, load it @@ -354,3 +363,29 @@ microdesc_free_all(void) tor_free(the_microdesc_cache); } } + +/** If there is a microdescriptor in cache whose sha256 digest is + * d, return it. Otherwise return NULL. */ +microdesc_t * +microdesc_cache_lookup_by_digest256(microdesc_cache_t *cache, const char *d) +{ + microdesc_t *md, search; + if (!cache) + cache = get_microdesc_cache(); + memcpy(search.digest, d, DIGEST256_LEN); + md = HT_FIND(microdesc_map, &cache->map, &search); + return md; +} + +/** Return the mean size of decriptors added to cache since it was last + * cleared. Used to estimate the size of large downloads. */ +size_t +microdesc_average_size(microdesc_cache_t *cache) +{ + if (!cache) + cache = get_microdesc_cache(); + if (!cache->n_seen) + return 512; + return (size_t)(cache->total_len_seen / cache->n_seen); +} + diff --git a/src/or/or.h b/src/or/or.h index 21e90a4cdf..0ec8029f46 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -1171,7 +1171,8 @@ typedef struct dir_connection_t { enum { DIR_SPOOL_NONE=0, DIR_SPOOL_SERVER_BY_DIGEST, DIR_SPOOL_SERVER_BY_FP, DIR_SPOOL_EXTRA_BY_DIGEST, DIR_SPOOL_EXTRA_BY_FP, - DIR_SPOOL_CACHED_DIR, DIR_SPOOL_NETWORKSTATUS + DIR_SPOOL_CACHED_DIR, DIR_SPOOL_NETWORKSTATUS, + DIR_SPOOL_MICRODESC, /* NOTE: if we add another entry, add another bit. */ } dir_spool_src : 3; /** If we're fetching descriptors, what router purpose shall we assign * to them? */ @@ -3678,9 +3679,13 @@ void directory_initiate_command(const char *address, const tor_addr_t *addr, const char *payload, size_t payload_len, time_t if_modified_since); +#define DSR_HEX (1<<0) +#define DSR_BASE64 (1<<1) +#define DSR_DIGEST256 (1<<2) +#define DSR_SORT_UNIQ (1<<3) int dir_split_resource_into_fingerprints(const char *resource, - smartlist_t *fp_out, int *compresseed_out, - int decode_hex, int sort_uniq); + smartlist_t *fp_out, int *compressed_out, + int flags); /** A pair of digests created by dir_split_resource_info_fingerprint_pairs() */ typedef struct { char first[DIGEST_LEN]; @@ -3816,8 +3821,11 @@ int authdir_wants_to_reject_router(routerinfo_t *ri, const char **msg, int dirserv_would_reject_router(routerstatus_t *rs); int dirserv_remove_old_statuses(smartlist_t *fps, time_t cutoff); int dirserv_have_any_serverdesc(smartlist_t *fps, int spool_src); +int dirserv_have_any_microdesc(const smartlist_t *fps); size_t dirserv_estimate_data_size(smartlist_t *fps, int is_serverdescs, int compressed); +size_t dirserv_estimate_microdesc_size(const smartlist_t *fps, int compressed); + typedef enum { NS_V2, NS_V3_CONSENSUS, NS_V3_VOTE, NS_CONTROL_PORT, NS_V3_CONSENSUS_MICRODESC @@ -4146,6 +4154,11 @@ int microdesc_cache_rebuild(microdesc_cache_t *cache); int microdesc_cache_reload(microdesc_cache_t *cache); void microdesc_cache_clear(microdesc_cache_t *cache); +microdesc_t *microdesc_cache_lookup_by_digest256(microdesc_cache_t *cache, + const char *d); + +size_t microdesc_average_size(microdesc_cache_t *cache); + void microdesc_free(microdesc_t *md); void microdesc_free_all(void); diff --git a/src/or/routerlist.c b/src/or/routerlist.c index 5ae40ddd67..d5e3b924e4 100644 --- a/src/or/routerlist.c +++ b/src/or/routerlist.c @@ -3832,7 +3832,7 @@ list_pending_downloads(digestmap_t *result, const char *resource = TO_DIR_CONN(conn)->requested_resource; if (!strcmpstart(resource, prefix)) dir_split_resource_into_fingerprints(resource + p_len, - tmp, NULL, 1, 0); + tmp, NULL, DSR_HEX); } }); SMARTLIST_FOREACH(tmp, char *, d, diff --git a/src/test/test_dir.c b/src/test/test_dir.c index 68dbbb45a4..ca55c115ac 100644 --- a/src/test/test_dir.c +++ b/src/test/test_dir.c @@ -361,9 +361,9 @@ test_dir_versions(void) ; } -/** Run unit tests for misc directory functions. */ +/** Run unit tests for directory fp_pair functions. */ static void -test_dir_util(void) +test_dir_fp_pairs(void) { smartlist_t *sl = smartlist_create(); fp_pair_t *pair; @@ -390,6 +390,127 @@ test_dir_util(void) smartlist_free(sl); } +static void +test_dir_split_fps(void *testdata) +{ + smartlist_t *sl = smartlist_create(); + char *mem_op_hex_tmp = NULL; + (void)testdata; + + /* Some example hex fingerprints and their base64 equivalents */ +#define HEX1 "Fe0daff89127389bc67558691231234551193EEE" +#define HEX2 "Deadbeef99999991111119999911111111f00ba4" +#define HEX3 "b33ff00db33ff00db33ff00db33ff00db33ff00d" +#define HEX256_1 \ + "f3f3f3f3fbbbbf3f3f3f3fbbbf3f3f3f3fbbbbf3f3f3f3fbbbf3f3f3f3fbbbbf" +#define HEX256_2 \ + "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccCCc" +#define HEX256_3 \ + "0123456789ABCdef0123456789ABCdef0123456789ABCdef0123456789ABCdef" +#define B64_1 "/g2v+JEnOJvGdVhpEjEjRVEZPu4" +#define B64_2 "3q2+75mZmZERERmZmRERERHwC6Q" +#define B64_3 "sz/wDbM/8A2zP/ANsz/wDbM/8A0" +#define B64_256_1 "8/Pz8/u7vz8/Pz+7vz8/Pz+7u/Pz8/P7u/Pz8/P7u78" +#define B64_256_2 "zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMw" +#define B64_256_3 "ASNFZ4mrze8BI0VniavN7wEjRWeJq83vASNFZ4mrze8" + + /* no flags set */ + dir_split_resource_into_fingerprints("A+C+B", sl, NULL, 0); + tt_int_op(smartlist_len(sl), ==, 3); + tt_str_op(smartlist_get(sl, 0), ==, "A"); + tt_str_op(smartlist_get(sl, 1), ==, "C"); + tt_str_op(smartlist_get(sl, 2), ==, "B"); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + /* uniq strings. */ + dir_split_resource_into_fingerprints("A+C+B+A+B+B", sl, NULL, DSR_SORT_UNIQ); + tt_int_op(smartlist_len(sl), ==, 3); + tt_str_op(smartlist_get(sl, 0), ==, "A"); + tt_str_op(smartlist_get(sl, 1), ==, "B"); + tt_str_op(smartlist_get(sl, 2), ==, "C"); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + /* Decode hex. */ + dir_split_resource_into_fingerprints(HEX1"+"HEX2, sl, NULL, DSR_HEX); + tt_int_op(smartlist_len(sl), ==, 2); + test_mem_op_hex(smartlist_get(sl, 0), ==, HEX1); + test_mem_op_hex(smartlist_get(sl, 1), ==, HEX2); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + /* decode hex and drop weirdness. */ + dir_split_resource_into_fingerprints(HEX1"+bogus+"HEX2"+"HEX256_1, + sl, NULL, DSR_HEX); + tt_int_op(smartlist_len(sl), ==, 2); + test_mem_op_hex(smartlist_get(sl, 0), ==, HEX1); + test_mem_op_hex(smartlist_get(sl, 1), ==, HEX2); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + /* Decode long hex */ + dir_split_resource_into_fingerprints(HEX256_1"+"HEX256_2"+"HEX2"+"HEX256_3, + sl, NULL, DSR_HEX|DSR_DIGEST256); + tt_int_op(smartlist_len(sl), ==, 3); + test_mem_op_hex(smartlist_get(sl, 0), ==, HEX256_1); + test_mem_op_hex(smartlist_get(sl, 1), ==, HEX256_2); + test_mem_op_hex(smartlist_get(sl, 2), ==, HEX256_3); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + /* Decode hex and sort. */ + dir_split_resource_into_fingerprints(HEX1"+"HEX2"+"HEX3"+"HEX2, + sl, NULL, DSR_HEX|DSR_SORT_UNIQ); + tt_int_op(smartlist_len(sl), ==, 3); + test_mem_op_hex(smartlist_get(sl, 0), ==, HEX3); + test_mem_op_hex(smartlist_get(sl, 1), ==, HEX2); + test_mem_op_hex(smartlist_get(sl, 2), ==, HEX1); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + /* Decode long hex and sort */ + dir_split_resource_into_fingerprints(HEX256_1"+"HEX256_2"+"HEX256_3 + "+"HEX256_1, + sl, NULL, + DSR_HEX|DSR_DIGEST256|DSR_SORT_UNIQ); + tt_int_op(smartlist_len(sl), ==, 3); + test_mem_op_hex(smartlist_get(sl, 0), ==, HEX256_3); + test_mem_op_hex(smartlist_get(sl, 1), ==, HEX256_2); + test_mem_op_hex(smartlist_get(sl, 2), ==, HEX256_1); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + /* Decode base64 */ + dir_split_resource_into_fingerprints(B64_1"-"B64_2, sl, NULL, DSR_BASE64); + tt_int_op(smartlist_len(sl), ==, 2); + test_mem_op_hex(smartlist_get(sl, 0), ==, HEX1); + test_mem_op_hex(smartlist_get(sl, 1), ==, HEX2); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + /* Decode long base64 */ + dir_split_resource_into_fingerprints(B64_256_1"-"B64_256_2, + sl, NULL, DSR_BASE64|DSR_DIGEST256); + tt_int_op(smartlist_len(sl), ==, 2); + test_mem_op_hex(smartlist_get(sl, 0), ==, HEX256_1); + test_mem_op_hex(smartlist_get(sl, 1), ==, HEX256_2); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + dir_split_resource_into_fingerprints(B64_256_1, + sl, NULL, DSR_BASE64|DSR_DIGEST256); + tt_int_op(smartlist_len(sl), ==, 1); + test_mem_op_hex(smartlist_get(sl, 0), ==, HEX256_1); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + done: + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_free(sl); + tor_free(mem_op_hex_tmp); +} + static void test_dir_measured_bw(void) { @@ -1173,11 +1294,15 @@ test_dir_v3_networkstatus(void) #define DIR_LEGACY(name) \ { #name, legacy_test_helper, 0, &legacy_setup, test_dir_ ## name } +#define DIR(name) \ + { #name, test_dir_##name, 0, NULL, NULL } + struct testcase_t dir_tests[] = { DIR_LEGACY(nicknames), DIR_LEGACY(formats), DIR_LEGACY(versions), - DIR_LEGACY(util), + DIR_LEGACY(fp_pairs), + DIR(split_fps), DIR_LEGACY(measured_bw), DIR_LEGACY(param_voting), DIR_LEGACY(v3_networkstatus), From 740806c453c46e9c985029a644af3a6f899be205 Mon Sep 17 00:00:00 2001 From: Sebastian Hahn Date: Mon, 19 Oct 2009 01:30:46 +0200 Subject: [PATCH 22/23] Fix compile with warnings problems on Snow Leopard --- src/or/dirvote.c | 2 +- src/or/microdesc.c | 12 ++++++------ src/or/or.h | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/or/dirvote.c b/src/or/dirvote.c index eb3c7d2cfa..ca81b2be9d 100644 --- a/src/or/dirvote.c +++ b/src/or/dirvote.c @@ -2790,7 +2790,7 @@ static char *microdesc_consensus_methods = NULL; * in a consensus vote document. Write it into the out_len-byte buffer * in out. Return -1 on failure and the number of characters written * on success. */ -int +ssize_t dirvote_format_microdesc_vote_line(char *out, size_t out_len, const microdesc_t *md) { diff --git a/src/or/microdesc.c b/src/or/microdesc.c index e10589fec9..7a65705088 100644 --- a/src/or/microdesc.c +++ b/src/or/microdesc.c @@ -56,10 +56,10 @@ HT_GENERATE(microdesc_map, microdesc_t, node, * On success, return the total number of bytes written, and set * *annotation_len_out to the number of bytes written as * annotations. */ -static int -dump_microdescriptor(FILE *f, microdesc_t *md, int *annotation_len_out) +static size_t +dump_microdescriptor(FILE *f, microdesc_t *md, size_t *annotation_len_out) { - int r = 0; + size_t r = 0; /* XXXX drops unkown annotations. */ if (md->last_listed) { char buf[ISO_TIME_LEN+1]; @@ -169,7 +169,7 @@ microdescs_add_list_to_cache(microdesc_cache_t *cache, /* Okay, it's a new one. */ if (f) { - int annotation_len; + size_t annotation_len; size = dump_microdescriptor(f, md, &annotation_len); md->saved_location = SAVED_IN_JOURNAL; cache->journal_len += size; @@ -269,7 +269,7 @@ microdesc_cache_rebuild(microdesc_cache_t *cache) FILE *f; microdesc_t **mdp; smartlist_t *wrote; - int size; + size_t size; off_t off = 0; int orig_size, new_size; @@ -287,7 +287,7 @@ microdesc_cache_rebuild(microdesc_cache_t *cache) HT_FOREACH(mdp, microdesc_map, &cache->map) { microdesc_t *md = *mdp; - int annotation_len; + size_t annotation_len; if (md->no_save) continue; diff --git a/src/or/or.h b/src/or/or.h index 0ec8029f46..2fa4a797ad 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -3919,7 +3919,7 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_env_t *private_key, authority_cert_t *cert); microdesc_t *dirvote_create_microdescriptor(const routerinfo_t *ri); -int dirvote_format_microdesc_vote_line(char *out, size_t out_len, +ssize_t dirvote_format_microdesc_vote_line(char *out, size_t out_len, const microdesc_t *md); int vote_routerstatus_find_microdesc_hash(char *digest256_out, const vote_routerstatus_t *vrs, From d40a814f4f6cceeb4af33f92372e6f5f00e7203d Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 19 Oct 2009 00:39:20 -0400 Subject: [PATCH 23/23] Changelog for microdescriptor branch. --- ChangeLog | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/ChangeLog b/ChangeLog index befb4368e4..b424837906 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,7 +1,25 @@ Changes in version 0.2.2.6-alpha - 2009-10-?? + o Major features: + - Directory authorities can now create, vote, and serve on multiple + parallel formats of directory data as part of their voting process. + This is a partial implementation of Proposal 162: "Publish the + consensus in multiple flavors." + - Directory authorities can now agree on and publish small summaries of + router information that clients can use in place of regular server + descriptors. This will eventually allow clients to use far less + bandwidth for downloading information about the network. This begins + the implementation of of Proposal 158: "Clients download a consensus + + Microdescriptors". + - The directory voting system is now extensible to use multiple hash + algorithms for signatures and resource selection. Newer formats are + signed with SHA256, with a possibility for moving to a better hash + algorithm in the future. + o Code simplifications and refactorings: - Numerous changes, bugfixes, and workarounds from Nathan Freitas to help Tor build correctly for Android phones. + - Begun converting Tor's signature and message digest logic to handle + multiple hash algorithms. o Minor bugfixes: - Fix a crash bug when trying to initialize the evdns module in