diff --git a/src/common/confline.c b/src/common/confline.c
new file mode 100644
index 0000000000..36f4c875f3
--- /dev/null
+++ b/src/common/confline.c
@@ -0,0 +1,320 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "compat.h"
+#include "confline.h"
+#include "torlog.h"
+#include "util.h"
+
+/** Helper: allocate a new configuration option mapping 'key' to 'val',
+ * append it to *lst. */
+void
+config_line_append(config_line_t **lst,
+ const char *key,
+ const char *val)
+{
+ tor_assert(lst);
+
+ config_line_t *newline;
+
+ newline = tor_malloc_zero(sizeof(config_line_t));
+ newline->key = tor_strdup(key);
+ newline->value = tor_strdup(val);
+ newline->next = NULL;
+ while (*lst)
+ lst = &((*lst)->next);
+
+ (*lst) = newline;
+}
+
+/** Return the first line in lines whose key is exactly key, or
+ * NULL if no such key exists.
+ *
+ * (In options parsing, this is for handling commandline-only options only;
+ * other options should be looked up in the appropriate data structure.) */
+const config_line_t *
+config_line_find(const config_line_t *lines,
+ const char *key)
+{
+ const config_line_t *cl;
+ for (cl = lines; cl; cl = cl->next) {
+ if (!strcmp(cl->key, key))
+ return cl;
+ }
+ return NULL;
+}
+
+/** Helper: parse the config string and strdup into key/value
+ * strings. Set *result to the list, or NULL if parsing the string
+ * failed. Return 0 on success, -1 on failure. Warn and ignore any
+ * misformatted lines.
+ *
+ * If extended is set, then treat keys beginning with / and with + as
+ * indicating "clear" and "append" respectively. */
+int
+config_get_lines(const char *string, config_line_t **result, int extended)
+{
+ config_line_t *list = NULL, **next;
+ char *k, *v;
+ const char *parse_err;
+
+ next = &list;
+ do {
+ k = v = NULL;
+ string = parse_config_line_from_str_verbose(string, &k, &v, &parse_err);
+ if (!string) {
+ log_warn(LD_CONFIG, "Error while parsing configuration: %s",
+ parse_err?parse_err:"");
+ config_free_lines(list);
+ tor_free(k);
+ tor_free(v);
+ return -1;
+ }
+ if (k && v) {
+ unsigned command = CONFIG_LINE_NORMAL;
+ if (extended) {
+ if (k[0] == '+') {
+ char *k_new = tor_strdup(k+1);
+ tor_free(k);
+ k = k_new;
+ command = CONFIG_LINE_APPEND;
+ } else if (k[0] == '/') {
+ char *k_new = tor_strdup(k+1);
+ tor_free(k);
+ k = k_new;
+ tor_free(v);
+ v = tor_strdup("");
+ command = CONFIG_LINE_CLEAR;
+ }
+ }
+ /* This list can get long, so we keep a pointer to the end of it
+ * rather than using config_line_append over and over and getting
+ * n^2 performance. */
+ *next = tor_malloc_zero(sizeof(config_line_t));
+ (*next)->key = k;
+ (*next)->value = v;
+ (*next)->next = NULL;
+ (*next)->command = command;
+ next = &((*next)->next);
+ } else {
+ tor_free(k);
+ tor_free(v);
+ }
+ } while (*string);
+
+ *result = list;
+ return 0;
+}
+
+/**
+ * Free all the configuration lines on the linked list front.
+ */
+void
+config_free_lines(config_line_t *front)
+{
+ config_line_t *tmp;
+
+ while (front) {
+ tmp = front;
+ front = tmp->next;
+
+ tor_free(tmp->key);
+ tor_free(tmp->value);
+ tor_free(tmp);
+ }
+}
+
+/** Return a newly allocated deep copy of the lines in inp. */
+config_line_t *
+config_lines_dup(const config_line_t *inp)
+{
+ return config_lines_dup_and_filter(inp, NULL);
+}
+
+/** Return a newly allocated deep copy of the lines in inp,
+ * but only the ones that match key. */
+config_line_t *
+config_lines_dup_and_filter(const config_line_t *inp,
+ const char *key)
+{
+ config_line_t *result = NULL;
+ config_line_t **next_out = &result;
+ while (inp) {
+ if (key && strcasecmpstart(inp->key, key)) {
+ inp = inp->next;
+ continue;
+ }
+ *next_out = tor_malloc_zero(sizeof(config_line_t));
+ (*next_out)->key = tor_strdup(inp->key);
+ (*next_out)->value = tor_strdup(inp->value);
+ inp = inp->next;
+ next_out = &((*next_out)->next);
+ }
+ (*next_out) = NULL;
+ return result;
+}
+
+/** Return true iff a and b contain identical keys and values in identical
+ * order. */
+int
+config_lines_eq(config_line_t *a, config_line_t *b)
+{
+ while (a && b) {
+ if (strcasecmp(a->key, b->key) || strcmp(a->value, b->value))
+ return 0;
+ a = a->next;
+ b = b->next;
+ }
+ if (a || b)
+ return 0;
+ return 1;
+}
+
+/** Return the number of lines in a whose key is key. */
+int
+config_count_key(const config_line_t *a, const char *key)
+{
+ int n = 0;
+ while (a) {
+ if (!strcasecmp(a->key, key)) {
+ ++n;
+ }
+ a = a->next;
+ }
+ return n;
+}
+
+/** Given a string containing part of a configuration file or similar format,
+ * advance past comments and whitespace and try to parse a single line. If we
+ * parse a line successfully, set *key_out to a new string holding the
+ * key portion and *value_out to a new string holding the value portion
+ * of the line, and return a pointer to the start of the next line. If we run
+ * out of data, return a pointer to the end of the string. If we encounter an
+ * error, return NULL and set *err_out (if provided) to an error
+ * message.
+ */
+const char *
+parse_config_line_from_str_verbose(const char *line, char **key_out,
+ char **value_out,
+ const char **err_out)
+{
+ /*
+ See torrc_format.txt for a description of the (silly) format this parses.
+ */
+ const char *key, *val, *cp;
+ int continuation = 0;
+
+ tor_assert(key_out);
+ tor_assert(value_out);
+
+ *key_out = *value_out = NULL;
+ key = val = NULL;
+ /* Skip until the first keyword. */
+ while (1) {
+ while (TOR_ISSPACE(*line))
+ ++line;
+ if (*line == '#') {
+ while (*line && *line != '\n')
+ ++line;
+ } else {
+ break;
+ }
+ }
+
+ if (!*line) { /* End of string? */
+ *key_out = *value_out = NULL;
+ return line;
+ }
+
+ /* Skip until the next space or \ followed by newline. */
+ key = line;
+ while (*line && !TOR_ISSPACE(*line) && *line != '#' &&
+ ! (line[0] == '\\' && line[1] == '\n'))
+ ++line;
+ *key_out = tor_strndup(key, line-key);
+
+ /* Skip until the value. */
+ while (*line == ' ' || *line == '\t')
+ ++line;
+
+ val = line;
+
+ /* Find the end of the line. */
+ if (*line == '\"') { // XXX No continuation handling is done here
+ if (!(line = unescape_string(line, value_out, NULL))) {
+ if (err_out)
+ *err_out = "Invalid escape sequence in quoted string";
+ return NULL;
+ }
+ while (*line == ' ' || *line == '\t')
+ ++line;
+ if (*line == '\r' && *(++line) == '\n')
+ ++line;
+ if (*line && *line != '#' && *line != '\n') {
+ if (err_out)
+ *err_out = "Excess data after quoted string";
+ return NULL;
+ }
+ } else {
+ /* Look for the end of the line. */
+ while (*line && *line != '\n' && (*line != '#' || continuation)) {
+ if (*line == '\\' && line[1] == '\n') {
+ continuation = 1;
+ line += 2;
+ } else if (*line == '#') {
+ do {
+ ++line;
+ } while (*line && *line != '\n');
+ if (*line == '\n')
+ ++line;
+ } else {
+ ++line;
+ }
+ }
+
+ if (*line == '\n') {
+ cp = line++;
+ } else {
+ cp = line;
+ }
+ /* Now back cp up to be the last nonspace character */
+ while (cp>val && TOR_ISSPACE(*(cp-1)))
+ --cp;
+
+ tor_assert(cp >= val);
+
+ /* Now copy out and decode the value. */
+ *value_out = tor_strndup(val, cp-val);
+ if (continuation) {
+ char *v_out, *v_in;
+ v_out = v_in = *value_out;
+ while (*v_in) {
+ if (*v_in == '#') {
+ do {
+ ++v_in;
+ } while (*v_in && *v_in != '\n');
+ if (*v_in == '\n')
+ ++v_in;
+ } else if (v_in[0] == '\\' && v_in[1] == '\n') {
+ v_in += 2;
+ } else {
+ *v_out++ = *v_in++;
+ }
+ }
+ *v_out = '\0';
+ }
+ }
+
+ if (*line == '#') {
+ do {
+ ++line;
+ } while (*line && *line != '\n');
+ }
+ while (TOR_ISSPACE(*line)) ++line;
+
+ return line;
+}
+
diff --git a/src/common/confline.h b/src/common/confline.h
new file mode 100644
index 0000000000..c695bf6b23
--- /dev/null
+++ b/src/common/confline.h
@@ -0,0 +1,47 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_CONFLINE_H
+#define TOR_CONFLINE_H
+
+/** Ordinary configuration line. */
+#define CONFIG_LINE_NORMAL 0
+/** Appends to previous configuration for the same option, even if we
+ * would ordinary replace it. */
+#define CONFIG_LINE_APPEND 1
+/* Removes all previous configuration for an option. */
+#define CONFIG_LINE_CLEAR 2
+
+/** A linked list of lines in a config file, or elsewhere */
+typedef struct config_line_t {
+ char *key;
+ char *value;
+ struct config_line_t *next;
+
+ /** What special treatment (if any) does this line require? */
+ unsigned int command:2;
+ /** If true, subsequent assignments to this linelist should replace
+ * it, not extend it. Set only on the first item in a linelist in an
+ * or_options_t. */
+ unsigned int fragile:1;
+} config_line_t;
+
+void config_line_append(config_line_t **lst,
+ const char *key, const char *val);
+config_line_t *config_lines_dup(const config_line_t *inp);
+config_line_t *config_lines_dup_and_filter(const config_line_t *inp,
+ const char *key);
+const config_line_t *config_line_find(const config_line_t *lines,
+ const char *key);
+int config_lines_eq(config_line_t *a, config_line_t *b);
+int config_count_key(const config_line_t *a, const char *key);
+int config_get_lines(const char *string, config_line_t **result, int extended);
+void config_free_lines(config_line_t *front);
+const char *parse_config_line_from_str_verbose(const char *line,
+ char **key_out, char **value_out,
+ const char **err_out);
+#endif
+
diff --git a/src/common/include.am b/src/common/include.am
index b5d6ac4ac5..d6504c7466 100644
--- a/src/common/include.am
+++ b/src/common/include.am
@@ -84,6 +84,7 @@ LIBOR_A_SRC = \
src/common/compat.c \
src/common/compat_threads.c \
src/common/compat_time.c \
+ src/common/confline.c \
src/common/container.c \
src/common/log.c \
src/common/memarea.c \
@@ -144,6 +145,7 @@ COMMONHEADERS = \
src/common/compat_openssl.h \
src/common/compat_threads.h \
src/common/compat_time.h \
+ src/common/confline.h \
src/common/container.h \
src/common/crypto.h \
src/common/crypto_curve25519.h \
diff --git a/src/common/storagedir.c b/src/common/storagedir.c
index e28a66f693..7e0be6754b 100644
--- a/src/common/storagedir.c
+++ b/src/common/storagedir.c
@@ -3,6 +3,8 @@
#include "container.h"
#include "compat.h"
+#include "confline.h"
+#include "memarea.h"
#include "sandbox.h"
#include "storagedir.h"
#include "torlog.h"
@@ -237,6 +239,40 @@ find_unused_fname(storage_dir_t *d)
return NULL;
}
+/** Helper: As storage_dir_save_bytes_to_file, but store a smartlist of
+ * sized_chunk_t rather than a single byte array. */
+static int
+storage_dir_save_chunks_to_file(storage_dir_t *d,
+ const smartlist_t *chunks,
+ int binary,
+ char **fname_out)
+{
+ uint64_t total_length = 0;
+ char *fname = find_unused_fname(d);
+ if (!fname)
+ return -1;
+
+ SMARTLIST_FOREACH(chunks, const sized_chunk_t *, ch,
+ total_length += ch->len);
+
+ char *path = NULL;
+ tor_asprintf(&path, "%s/%s", d->directory, fname);
+
+ int r = write_chunks_to_file(path, chunks, binary, 0);
+ if (r == 0) {
+ if (d->usage_known)
+ d->usage += total_length;
+ if (fname_out) {
+ *fname_out = tor_strdup(fname);
+ }
+ if (d->contents)
+ smartlist_add(d->contents, tor_strdup(fname));
+ }
+ tor_free(fname);
+ tor_free(path);
+ return r;
+}
+
/** Try to write the length bytes at data into a new file
* in d. On success, return 0 and set *fname_out to a
* newly allocated string containing the filename. On failure, return
@@ -248,25 +284,11 @@ storage_dir_save_bytes_to_file(storage_dir_t *d,
int binary,
char **fname_out)
{
- char *fname = find_unused_fname(d);
- if (!fname)
- return -1;
-
- char *path = NULL;
- tor_asprintf(&path, "%s/%s", d->directory, fname);
-
- int r = write_bytes_to_file(path, (const char *)data, length, binary);
- if (r == 0) {
- if (d->usage_known)
- d->usage += length;
- if (fname_out) {
- *fname_out = tor_strdup(fname);
- }
- if (d->contents)
- smartlist_add(d->contents, tor_strdup(fname));
- }
- tor_free(fname);
- tor_free(path);
+ smartlist_t *chunks = smartlist_new();
+ sized_chunk_t chunk = { (const char *)data, length };
+ smartlist_add(chunks, &chunk);
+ int r = storage_dir_save_chunks_to_file(d, chunks, binary, fname_out);
+ smartlist_free(chunks);
return r;
}
@@ -284,6 +306,106 @@ storage_dir_save_string_to_file(storage_dir_t *d,
(const uint8_t*)str, strlen(str), binary, fname_out);
}
+/**
+ * As storage_dir_save_bytes_to_file, but associates the data with the
+ * key-value pairs in labels. Files
+ * stored in this format can be recovered with storage_dir_map_labeled
+ * or storage_dir_read_labeled().
+ */
+int
+storage_dir_save_labeled_to_file(storage_dir_t *d,
+ const config_line_t *labels,
+ const uint8_t *data,
+ size_t length,
+ char **fname_out)
+{
+ /*
+ * The storage format is to prefix the data with the key-value pairs in
+ * labels, and a single NUL separator. But code outside this module
+ * MUST NOT rely on that format.
+ */
+
+ smartlist_t *chunks = smartlist_new();
+ memarea_t *area = memarea_new();
+ const config_line_t *line;
+ for (line = labels; line; line = line->next) {
+ sized_chunk_t *sz = memarea_alloc(area, sizeof(sized_chunk_t));
+ sz->len = strlen(line->key) + 1 + strlen(line->value) + 1;
+ const size_t allocated = sz->len + 1;
+ char *bytes = memarea_alloc(area, allocated);
+ tor_snprintf(bytes, allocated, "%s %s\n", line->key, line->value);
+ sz->bytes = bytes;
+ smartlist_add(chunks, sz);
+ }
+
+ sized_chunk_t *nul = memarea_alloc(area, sizeof(sized_chunk_t));
+ nul->len = 1;
+ nul->bytes = "\0";
+ smartlist_add(chunks, nul);
+
+ sized_chunk_t *datachunk = memarea_alloc(area, sizeof(sized_chunk_t));
+ datachunk->bytes = (const char *)data;
+ datachunk->len = length;
+ smartlist_add(chunks, datachunk);
+
+ int r = storage_dir_save_chunks_to_file(d, chunks, 1, fname_out);
+ smartlist_free(chunks);
+ memarea_drop_all(area);
+ return r;
+}
+
+/**
+ * Map a file that was created with storage_dir_save_labeled(). On failure,
+ * return NULL. On success, write a set of newly allocated labels into to
+ * *labels_out, a pointer to the into *data_out, and the data's
+ * into *sz_out. On success, also return a tor_mmap_t object whose
+ * contents should not be used -- it needs to be kept around, though, for as
+ * long as data_out is going to be valid.
+ */
+tor_mmap_t *
+storage_dir_map_labeled(storage_dir_t *dir,
+ const char *fname,
+ config_line_t **labels_out,
+ const uint8_t **data_out,
+ size_t *sz_out)
+{
+ tor_mmap_t *m = storage_dir_map(dir, fname);
+ if (! m)
+ goto err;
+ const char *nulp = memchr(m->data, '\0', m->size);
+ if (! nulp)
+ goto err;
+ if (labels_out && config_get_lines(m->data, labels_out, 0) < 0)
+ goto err;
+ size_t offset = nulp - m->data + 1;
+ tor_assert(offset <= m->size);
+ *data_out = (const uint8_t *)(m->data + offset);
+ *sz_out = m->size - offset;
+
+ return m;
+ err:
+ tor_munmap_file(m);
+ return NULL;
+}
+
+/** As storage_dir_map_labeled, but return a new byte array containing the
+ * data. */
+uint8_t *
+storage_dir_read_labeled(storage_dir_t *dir,
+ const char *fname,
+ config_line_t **labels_out,
+ size_t *sz_out)
+{
+ const uint8_t *data = NULL;
+ tor_mmap_t *m = storage_dir_map_labeled(dir, fname, labels_out,
+ &data, sz_out);
+ if (m == NULL)
+ return NULL;
+ uint8_t *result = tor_memdup(data, *sz_out);
+ tor_munmap_file(m);
+ return result;
+}
+
/**
* Remove the file called fname from d.
*/
diff --git a/src/common/storagedir.h b/src/common/storagedir.h
index 29be941b14..781194407f 100644
--- a/src/common/storagedir.h
+++ b/src/common/storagedir.h
@@ -5,7 +5,7 @@
#define TOR_STORAGEDIR_H
typedef struct storage_dir_t storage_dir_t;
-
+struct config_line_t;
struct sandbox_cfg_elem;
storage_dir_t * storage_dir_new(const char *dirname, int n_files);
@@ -26,6 +26,19 @@ int storage_dir_save_string_to_file(storage_dir_t *d,
const char *data,
int binary,
char **fname_out);
+int storage_dir_save_labeled_to_file(storage_dir_t *d,
+ const struct config_line_t *labels,
+ const uint8_t *data,
+ size_t length,
+ char **fname_out);
+tor_mmap_t *storage_dir_map_labeled(storage_dir_t *dir,
+ const char *fname,
+ struct config_line_t **labels_out,
+ const uint8_t **data_out,
+ size_t *size_out);
+uint8_t *storage_dir_read_labeled(storage_dir_t *d, const char *fname,
+ struct config_line_t **labels_out,
+ size_t *sz_out);
void storage_dir_remove_file(storage_dir_t *d,
const char *fname);
int storage_dir_shrink(storage_dir_t *d,
diff --git a/src/common/util.c b/src/common/util.c
index 93befe9d77..ca2a0c4a9c 100644
--- a/src/common/util.c
+++ b/src/common/util.c
@@ -3045,137 +3045,6 @@ unescape_string(const char *s, char **result, size_t *size_out)
}
}
-/** Given a string containing part of a configuration file or similar format,
- * advance past comments and whitespace and try to parse a single line. If we
- * parse a line successfully, set *key_out to a new string holding the
- * key portion and *value_out to a new string holding the value portion
- * of the line, and return a pointer to the start of the next line. If we run
- * out of data, return a pointer to the end of the string. If we encounter an
- * error, return NULL and set *err_out (if provided) to an error
- * message.
- */
-const char *
-parse_config_line_from_str_verbose(const char *line, char **key_out,
- char **value_out,
- const char **err_out)
-{
- /*
- See torrc_format.txt for a description of the (silly) format this parses.
- */
- const char *key, *val, *cp;
- int continuation = 0;
-
- tor_assert(key_out);
- tor_assert(value_out);
-
- *key_out = *value_out = NULL;
- key = val = NULL;
- /* Skip until the first keyword. */
- while (1) {
- while (TOR_ISSPACE(*line))
- ++line;
- if (*line == '#') {
- while (*line && *line != '\n')
- ++line;
- } else {
- break;
- }
- }
-
- if (!*line) { /* End of string? */
- *key_out = *value_out = NULL;
- return line;
- }
-
- /* Skip until the next space or \ followed by newline. */
- key = line;
- while (*line && !TOR_ISSPACE(*line) && *line != '#' &&
- ! (line[0] == '\\' && line[1] == '\n'))
- ++line;
- *key_out = tor_strndup(key, line-key);
-
- /* Skip until the value. */
- while (*line == ' ' || *line == '\t')
- ++line;
-
- val = line;
-
- /* Find the end of the line. */
- if (*line == '\"') { // XXX No continuation handling is done here
- if (!(line = unescape_string(line, value_out, NULL))) {
- if (err_out)
- *err_out = "Invalid escape sequence in quoted string";
- return NULL;
- }
- while (*line == ' ' || *line == '\t')
- ++line;
- if (*line == '\r' && *(++line) == '\n')
- ++line;
- if (*line && *line != '#' && *line != '\n') {
- if (err_out)
- *err_out = "Excess data after quoted string";
- return NULL;
- }
- } else {
- /* Look for the end of the line. */
- while (*line && *line != '\n' && (*line != '#' || continuation)) {
- if (*line == '\\' && line[1] == '\n') {
- continuation = 1;
- line += 2;
- } else if (*line == '#') {
- do {
- ++line;
- } while (*line && *line != '\n');
- if (*line == '\n')
- ++line;
- } else {
- ++line;
- }
- }
-
- if (*line == '\n') {
- cp = line++;
- } else {
- cp = line;
- }
- /* Now back cp up to be the last nonspace character */
- while (cp>val && TOR_ISSPACE(*(cp-1)))
- --cp;
-
- tor_assert(cp >= val);
-
- /* Now copy out and decode the value. */
- *value_out = tor_strndup(val, cp-val);
- if (continuation) {
- char *v_out, *v_in;
- v_out = v_in = *value_out;
- while (*v_in) {
- if (*v_in == '#') {
- do {
- ++v_in;
- } while (*v_in && *v_in != '\n');
- if (*v_in == '\n')
- ++v_in;
- } else if (v_in[0] == '\\' && v_in[1] == '\n') {
- v_in += 2;
- } else {
- *v_out++ = *v_in++;
- }
- }
- *v_out = '\0';
- }
- }
-
- if (*line == '#') {
- do {
- ++line;
- } while (*line && *line != '\n');
- }
- while (TOR_ISSPACE(*line)) ++line;
-
- return line;
-}
-
/** Expand any homedir prefix on filename; return a newly allocated
* string. */
char *
diff --git a/src/common/util.h b/src/common/util.h
index 181acb623f..cfe47f07f2 100644
--- a/src/common/util.h
+++ b/src/common/util.h
@@ -389,9 +389,6 @@ char *read_file_to_str_until_eof(int fd, size_t max_bytes_to_read,
size_t *sz_out)
ATTR_MALLOC;
const char *unescape_string(const char *s, char **result, size_t *size_out);
-const char *parse_config_line_from_str_verbose(const char *line,
- char **key_out, char **value_out,
- const char **err_out);
char *expand_filename(const char *filename);
MOCK_DECL(struct smartlist_t *, tor_listdir, (const char *dirname));
int path_is_relative(const char *filename);
diff --git a/src/or/confparse.c b/src/or/confparse.c
index ae0b3d5582..75ec92e30f 100644
--- a/src/or/confparse.c
+++ b/src/or/confparse.c
@@ -31,8 +31,6 @@ static int config_parse_msec_interval(const char *s, int *ok);
static int config_parse_interval(const char *s, int *ok);
static void config_reset(const config_format_t *fmt, void *options,
const config_var_t *var, int use_defaults);
-static config_line_t *config_lines_dup_and_filter(const config_line_t *inp,
- const char *key);
/** Allocate an empty configuration object of a given format type. */
void *
@@ -80,120 +78,6 @@ config_expand_abbrev(const config_format_t *fmt, const char *option,
return option;
}
-/** Helper: allocate a new configuration option mapping 'key' to 'val',
- * append it to *lst. */
-void
-config_line_append(config_line_t **lst,
- const char *key,
- const char *val)
-{
- config_line_t *newline;
-
- newline = tor_malloc_zero(sizeof(config_line_t));
- newline->key = tor_strdup(key);
- newline->value = tor_strdup(val);
- newline->next = NULL;
- while (*lst)
- lst = &((*lst)->next);
-
- (*lst) = newline;
-}
-
-/** Return the line in lines whose key is exactly key, or NULL
- * if no such key exists. For handling commandline-only options only; other
- * options should be looked up in the appropriate data structure. */
-const config_line_t *
-config_line_find(const config_line_t *lines,
- const char *key)
-{
- const config_line_t *cl;
- for (cl = lines; cl; cl = cl->next) {
- if (!strcmp(cl->key, key))
- return cl;
- }
- return NULL;
-}
-
-/** Helper: parse the config string and strdup into key/value
- * strings. Set *result to the list, or NULL if parsing the string
- * failed. Return 0 on success, -1 on failure. Warn and ignore any
- * misformatted lines.
- *
- * If extended is set, then treat keys beginning with / and with + as
- * indicating "clear" and "append" respectively. */
-int
-config_get_lines(const char *string, config_line_t **result, int extended)
-{
- config_line_t *list = NULL, **next;
- char *k, *v;
- const char *parse_err;
-
- next = &list;
- do {
- k = v = NULL;
- string = parse_config_line_from_str_verbose(string, &k, &v, &parse_err);
- if (!string) {
- log_warn(LD_CONFIG, "Error while parsing configuration: %s",
- parse_err?parse_err:"");
- config_free_lines(list);
- tor_free(k);
- tor_free(v);
- return -1;
- }
- if (k && v) {
- unsigned command = CONFIG_LINE_NORMAL;
- if (extended) {
- if (k[0] == '+') {
- char *k_new = tor_strdup(k+1);
- tor_free(k);
- k = k_new;
- command = CONFIG_LINE_APPEND;
- } else if (k[0] == '/') {
- char *k_new = tor_strdup(k+1);
- tor_free(k);
- k = k_new;
- tor_free(v);
- v = tor_strdup("");
- command = CONFIG_LINE_CLEAR;
- }
- }
- /* This list can get long, so we keep a pointer to the end of it
- * rather than using config_line_append over and over and getting
- * n^2 performance. */
- *next = tor_malloc_zero(sizeof(config_line_t));
- (*next)->key = k;
- (*next)->value = v;
- (*next)->next = NULL;
- (*next)->command = command;
- next = &((*next)->next);
- } else {
- tor_free(k);
- tor_free(v);
- }
- } while (*string);
-
- *result = list;
- return 0;
-}
-
-/**
- * Free all the configuration lines on the linked list front.
- */
-void
-config_free_lines(config_line_t *front)
-{
- config_line_t *tmp;
-
- while (front) {
- tmp = front;
- front = tmp->next;
-
- tor_free(tmp->key);
- tor_free(tmp->value);
- tor_free(tmp);
- }
-}
-
/** If key is a deprecated configuration option, return the message
* explaining why it is deprecated (which may be an empty string). Return NULL
* if it is not deprecated. The key field must be fully expanded. */
@@ -633,36 +517,6 @@ config_value_needs_escape(const char *value)
return 0;
}
-/** Return a newly allocated deep copy of the lines in inp. */
-config_line_t *
-config_lines_dup(const config_line_t *inp)
-{
- return config_lines_dup_and_filter(inp, NULL);
-}
-
-/** Return a newly allocated deep copy of the lines in inp,
- * but only the ones that match key. */
-static config_line_t *
-config_lines_dup_and_filter(const config_line_t *inp,
- const char *key)
-{
- config_line_t *result = NULL;
- config_line_t **next_out = &result;
- while (inp) {
- if (key && strcasecmpstart(inp->key, key)) {
- inp = inp->next;
- continue;
- }
- *next_out = tor_malloc_zero(sizeof(config_line_t));
- (*next_out)->key = tor_strdup(inp->key);
- (*next_out)->value = tor_strdup(inp->value);
- inp = inp->next;
- next_out = &((*next_out)->next);
- }
- (*next_out) = NULL;
- return result;
-}
-
/** Return newly allocated line or lines corresponding to key in the
* configuration options. If escape_val is true and a
* value needs to be quoted before it's put in a config file, quote and
@@ -1028,36 +882,6 @@ config_free(const config_format_t *fmt, void *options)
tor_free(options);
}
-/** Return true iff a and b contain identical keys and values in identical
- * order. */
-int
-config_lines_eq(config_line_t *a, config_line_t *b)
-{
- while (a && b) {
- if (strcasecmp(a->key, b->key) || strcmp(a->value, b->value))
- return 0;
- a = a->next;
- b = b->next;
- }
- if (a || b)
- return 0;
- return 1;
-}
-
-/** Return the number of lines in a whose key is key. */
-int
-config_count_key(const config_line_t *a, const char *key)
-{
- int n = 0;
- while (a) {
- if (!strcasecmp(a->key, key)) {
- ++n;
- }
- a = a->next;
- }
- return n;
-}
-
/** Return true iff the option name has the same value in o1
* and o2. Must not be called for LINELIST_S or OBSOLETE options.
*/
diff --git a/src/or/confparse.h b/src/or/confparse.h
index 5db6c8e4b6..6f5c681ba8 100644
--- a/src/or/confparse.h
+++ b/src/or/confparse.h
@@ -103,14 +103,7 @@ typedef struct config_format_t {
#define CAL_WARN_DEPRECATIONS (1u<<2)
void *config_new(const config_format_t *fmt);
-void config_line_append(config_line_t **lst,
- const char *key, const char *val);
-config_line_t *config_lines_dup(const config_line_t *inp);
-const config_line_t *config_line_find(const config_line_t *lines,
- const char *key);
void config_free(const config_format_t *fmt, void *options);
-int config_lines_eq(config_line_t *a, config_line_t *b);
-int config_count_key(const config_line_t *a, const char *key);
config_line_t *config_get_assigned_option(const config_format_t *fmt,
const void *options, const char *key,
int escape_val);
@@ -132,8 +125,6 @@ const char *config_find_deprecation(const config_format_t *fmt,
const config_var_t *config_find_option(const config_format_t *fmt,
const char *key);
-int config_get_lines(const char *string, config_line_t **result, int extended);
-void config_free_lines(config_line_t *front);
const char *config_expand_abbrev(const config_format_t *fmt,
const char *option,
int command_line, int warn_obsolete);
diff --git a/src/or/conscache.c b/src/or/conscache.c
new file mode 100644
index 0000000000..2a6e1445e3
--- /dev/null
+++ b/src/or/conscache.c
@@ -0,0 +1,506 @@
+/* Copyright (c) 2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "or.h"
+
+#include "config.h"
+#include "conscache.h"
+#include "storagedir.h"
+
+#define CCE_MAGIC 0x17162253
+
+/**
+ * A consensus_cache_entry_t is a reference-counted handle to an
+ * item in a consensus_cache_t. It can be mmapped into RAM, or not,
+ * depending whether it's currently in use.
+ */
+struct consensus_cache_entry_t {
+ uint32_t magic; /**< Must be set to CCE_MAGIC */
+ int32_t refcnt; /**< Reference count. */
+ unsigned can_remove : 1; /**< If true, we want to delete this file. */
+ /** If true, we intend to unmap this file as soon as we're done with it. */
+ unsigned release_aggressively : 1;
+
+ /** Filename for this object within the storage_dir_t */
+ char *fname;
+ /** Labels associated with this object. Immutable once the object
+ * is created. */
+ config_line_t *labels;
+ /** Pointer to the cache that includes this entry (if any). */
+ consensus_cache_t *in_cache;
+
+ /** Since what time has this object been mapped into RAM, but with the cache
+ * being the only having a reference to it? */
+ time_t unused_since;
+ /** mmaped contents of the underlying file. May be NULL */
+ tor_mmap_t *map;
+ /** Length of the body within map. */
+ size_t bodylen;
+ /** Pointer to the body within map. */
+ const uint8_t *body;
+};
+
+/**
+ * A consensus_cache_t holds a directory full of labeled items.
+ */
+struct consensus_cache_t {
+ /** Underling storage_dir_t to handle persistence */
+ storage_dir_t *dir;
+ /** List of all the entries in the directory. */
+ smartlist_t *entries;
+};
+
+static void consensus_cache_clear(consensus_cache_t *cache);
+static void consensus_cache_rescan(consensus_cache_t *);
+static void consensus_cache_entry_map(consensus_cache_t *,
+ consensus_cache_entry_t *);
+static void consensus_cache_entry_unmap(consensus_cache_entry_t *ent);
+
+/**
+ * Helper: Open a consensus cache in subdirectory subdir of the
+ * data directory, to hold up to max_entries of data.
+ */
+consensus_cache_t *
+consensus_cache_open(const char *subdir, int max_entries)
+{
+ consensus_cache_t *cache = tor_malloc_zero(sizeof(consensus_cache_t));
+ char *directory = get_datadir_fname(subdir);
+ cache->dir = storage_dir_new(directory, max_entries);
+ tor_free(directory);
+ if (!cache->dir) {
+ tor_free(cache);
+ return NULL;
+ }
+
+ consensus_cache_rescan(cache);
+ return cache;
+}
+
+/**
+ * Helper: clear all entries from cache (but do not delete
+ * any that aren't marked for removal
+ */
+static void
+consensus_cache_clear(consensus_cache_t *cache)
+{
+ consensus_cache_delete_pending(cache);
+
+ SMARTLIST_FOREACH_BEGIN(cache->entries, consensus_cache_entry_t *, ent) {
+ ent->in_cache = NULL;
+ consensus_cache_entry_decref(ent);
+ } SMARTLIST_FOREACH_END(ent);
+ smartlist_free(cache->entries);
+ cache->entries = NULL;
+}
+
+/**
+ * Drop all storage held by cache.
+ */
+void
+consensus_cache_free(consensus_cache_t *cache)
+{
+ if (! cache)
+ return;
+
+ if (cache->entries) {
+ consensus_cache_clear(cache);
+ }
+ storage_dir_free(cache->dir);
+ tor_free(cache);
+}
+
+/**
+ * Write datalen bytes of data at data into the cache,
+ * labeling that data with labels. On failure, return NULL. On
+ * success, return a newly created consensus_cache_entry_t.
+ *
+ * The returned value will be owned by the cache, and you will have a
+ * reference to it. Call consensus_cache_entry_decref() when you are
+ * done with it.
+ *
+ * The provided labels MUST have distinct keys: if they don't,
+ * this API does not specify which values (if any) for the duplicate keys
+ * will be considered.
+ */
+consensus_cache_entry_t *
+consensus_cache_add(consensus_cache_t *cache,
+ const config_line_t *labels,
+ const uint8_t *data,
+ size_t datalen)
+{
+ char *fname = NULL;
+ int r = storage_dir_save_labeled_to_file(cache->dir,
+ labels, data, datalen, &fname);
+ if (r < 0 || fname == NULL) {
+ return NULL;
+ }
+ consensus_cache_entry_t *ent =
+ tor_malloc_zero(sizeof(consensus_cache_entry_t));
+ ent->magic = CCE_MAGIC;
+ ent->fname = fname;
+ ent->labels = config_lines_dup(labels);
+ ent->in_cache = cache;
+ ent->unused_since = TIME_MAX;
+ smartlist_add(cache->entries, ent);
+ /* Start the reference count at 2: the caller owns one copy, and the
+ * cache owns another.
+ */
+ ent->refcnt = 2;
+
+ return ent;
+}
+
+/**
+ * Given a cache, return some entry for which key=value.
+ * Return NULL if no such entry exists.
+ *
+ * Does not adjust reference counts.
+ */
+consensus_cache_entry_t *
+consensus_cache_find_first(consensus_cache_t *cache,
+ const char *key,
+ const char *value)
+{
+ smartlist_t *tmp = smartlist_new();
+ consensus_cache_find_all(tmp, cache, key, value);
+ consensus_cache_entry_t *ent = NULL;
+ if (smartlist_len(tmp))
+ ent = smartlist_get(tmp, 0);
+ smartlist_free(tmp);
+ return ent;
+}
+
+/**
+ * Given a cache, add every entry to out for which
+ * key=value. If key is NULL, add every entry.
+ *
+ * Does not adjust reference counts.
+ */
+void
+consensus_cache_find_all(smartlist_t *out,
+ consensus_cache_t *cache,
+ const char *key,
+ const char *value)
+{
+ if (! key) {
+ smartlist_add_all(out, cache->entries);
+ return;
+ }
+
+ SMARTLIST_FOREACH_BEGIN(cache->entries, consensus_cache_entry_t *, ent) {
+ const char *found_val = consensus_cache_entry_get_value(ent, key);
+ if (found_val && !strcmp(value, found_val)) {
+ smartlist_add(out, ent);
+ }
+ } SMARTLIST_FOREACH_END(ent);
+}
+
+/**
+ * Given a list of consensus_cache_entry_t, remove all those entries
+ * that do not have key=value in their labels.
+ *
+ * Does not adjust reference counts.
+ */
+void
+consensus_cache_filter_list(smartlist_t *lst,
+ const char *key,
+ const char *value)
+{
+ if (BUG(lst == NULL))
+ return; // LCOV_EXCL_LINE
+ if (key == NULL)
+ return;
+ SMARTLIST_FOREACH_BEGIN(lst, consensus_cache_entry_t *, ent) {
+ const char *found_val = consensus_cache_entry_get_value(ent, key);
+ if (! found_val || strcmp(value, found_val)) {
+ SMARTLIST_DEL_CURRENT(lst, ent);
+ }
+ } SMARTLIST_FOREACH_END(ent);
+}
+
+/**
+ * If ent has a label with the given key, return its
+ * value. Otherwise return NULL.
+ *
+ * The return value is only guaranteed to be valid for as long as you
+ * hold a reference to ent.
+ */
+const char *
+consensus_cache_entry_get_value(const consensus_cache_entry_t *ent,
+ const char *key)
+{
+ const config_line_t *match = config_line_find(ent->labels, key);
+ if (match)
+ return match->value;
+ else
+ return NULL;
+}
+
+/**
+ * Return a pointer to the labels in ent.
+ *
+ * This pointer is only guaranteed to be valid for as long as you
+ * hold a reference to ent.
+ */
+const config_line_t *
+consensus_cache_entry_get_labels(const consensus_cache_entry_t *ent)
+{
+ return ent->labels;
+}
+
+/**
+ * Increase the reference count of ent.
+ */
+void
+consensus_cache_entry_incref(consensus_cache_entry_t *ent)
+{
+ if (BUG(ent->magic != CCE_MAGIC))
+ return; // LCOV_EXCL_LINE
+ ++ent->refcnt;
+ ent->unused_since = TIME_MAX;
+}
+
+/**
+ * Release a reference held to ent.
+ *
+ * If it was the last reference, ent will be freed. Therefore, you must not
+ * use ent after calling this function.
+ */
+void
+consensus_cache_entry_decref(consensus_cache_entry_t *ent)
+{
+ if (! ent)
+ return;
+ if (BUG(ent->refcnt <= 0))
+ return; // LCOV_EXCL_LINE
+ if (BUG(ent->magic != CCE_MAGIC))
+ return; // LCOV_EXCL_LINE
+
+ --ent->refcnt;
+
+ if (ent->refcnt == 1 && ent->in_cache) {
+ /* Only the cache has a reference: we don't need to keep the file
+ * mapped */
+ if (ent->map) {
+ if (ent->release_aggressively) {
+ consensus_cache_entry_unmap(ent);
+ } else {
+ ent->unused_since = approx_time();
+ }
+ }
+ return;
+ }
+
+ if (ent->refcnt > 0)
+ return;
+
+ /* Refcount is zero; we can free it. */
+ if (ent->map) {
+ consensus_cache_entry_unmap(ent);
+ }
+ tor_free(ent->fname);
+ config_free_lines(ent->labels);
+ memwipe(ent, 0, sizeof(consensus_cache_entry_t));
+ tor_free(ent);
+}
+
+/**
+ * Mark ent for deletion from the cache. Deletion will not occur
+ * until the cache is the only place that holds a reference to ent.
+ */
+void
+consensus_cache_entry_mark_for_removal(consensus_cache_entry_t *ent)
+{
+ ent->can_remove = 1;
+}
+
+/**
+ * Mark ent as the kind of entry that we don't need to keep mmap'd for
+ * any longer than we're actually using it.
+ */
+void
+consensus_cache_entry_mark_for_aggressive_release(consensus_cache_entry_t *ent)
+{
+ ent->release_aggressively = 1;
+}
+
+/**
+ * Try to read the body of ent into memory if it isn't already
+ * loaded. On success, set *body_out to the body, *sz_out
+ * to its size, and return 0. On failure return -1.
+ *
+ * The resulting body pointer will only be valid for as long as you
+ * hold a reference to ent.
+ */
+int
+consensus_cache_entry_get_body(const consensus_cache_entry_t *ent,
+ const uint8_t **body_out,
+ size_t *sz_out)
+{
+ if (BUG(ent->magic != CCE_MAGIC))
+ return -1; // LCOV_EXCL_LINE
+
+ if (! ent->map) {
+ if (! ent->in_cache)
+ return -1;
+
+ consensus_cache_entry_map((consensus_cache_t *)ent->in_cache,
+ (consensus_cache_entry_t *)ent);
+ if (! ent->map) {
+ return -1;
+ }
+ }
+
+ *body_out = ent->body;
+ *sz_out = ent->bodylen;
+ return 0;
+}
+
+/**
+ * Unmap every mmap'd element of cache that has been unused
+ * since cutoff.
+ */
+void
+consensus_cache_unmap_lazy(consensus_cache_t *cache, time_t cutoff)
+{
+ SMARTLIST_FOREACH_BEGIN(cache->entries, consensus_cache_entry_t *, ent) {
+ tor_assert_nonfatal(ent->in_cache == cache);
+ if (ent->refcnt > 1 || BUG(ent->in_cache == NULL)) {
+ /* Somebody is using this entry right now */
+ continue;
+ }
+ if (ent->unused_since > cutoff) {
+ /* Has been unused only for a little while */
+ continue;
+ }
+ if (ent->map == NULL) {
+ /* Not actually mapped. */
+ continue;
+ }
+ consensus_cache_entry_unmap(ent);
+ } SMARTLIST_FOREACH_END(ent);
+}
+
+/**
+ * Delete every element of cache has been marked with
+ * consensus_cache_entry_mark_for_removal, and which is not in use except by
+ * the cache.
+ */
+void
+consensus_cache_delete_pending(consensus_cache_t *cache)
+{
+ SMARTLIST_FOREACH_BEGIN(cache->entries, consensus_cache_entry_t *, ent) {
+ tor_assert_nonfatal(ent->in_cache == cache);
+ if (ent->refcnt > 1 || BUG(ent->in_cache == NULL)) {
+ /* Somebody is using this entry right now */
+ continue;
+ }
+ if (ent->can_remove == 0) {
+ /* Don't want to delete this. */
+ continue;
+ }
+ if (BUG(ent->refcnt <= 0)) {
+ continue; // LCOV_EXCL_LINE
+ }
+
+ SMARTLIST_DEL_CURRENT(cache->entries, ent);
+ ent->in_cache = NULL;
+ char *fname = tor_strdup(ent->fname); /* save a copy */
+ consensus_cache_entry_decref(ent);
+ storage_dir_remove_file(cache->dir, fname);
+ tor_free(fname);
+ } SMARTLIST_FOREACH_END(ent);
+}
+
+/**
+ * Internal helper: rescan cache and rebuild its list of entries.
+ */
+static void
+consensus_cache_rescan(consensus_cache_t *cache)
+{
+ if (cache->entries) {
+ consensus_cache_clear(cache);
+ }
+
+ cache->entries = smartlist_new();
+ const smartlist_t *fnames = storage_dir_list(cache->dir);
+ SMARTLIST_FOREACH_BEGIN(fnames, const char *, fname) {
+ tor_mmap_t *map = NULL;
+ config_line_t *labels = NULL;
+ const uint8_t *body;
+ size_t bodylen;
+ map = storage_dir_map_labeled(cache->dir, fname,
+ &labels, &body, &bodylen);
+ if (! map) {
+ /* Can't load this; continue */
+ log_warn(LD_FS, "Unable to map file %s from consensus cache: %s",
+ escaped(fname), strerror(errno));
+ continue;
+ }
+ consensus_cache_entry_t *ent =
+ tor_malloc_zero(sizeof(consensus_cache_entry_t));
+ ent->magic = CCE_MAGIC;
+ ent->fname = tor_strdup(fname);
+ ent->labels = labels;
+ ent->refcnt = 1;
+ ent->in_cache = cache;
+ ent->unused_since = TIME_MAX;
+ smartlist_add(cache->entries, ent);
+ tor_munmap_file(map); /* don't actually need to keep this around */
+ } SMARTLIST_FOREACH_END(fname);
+}
+
+/**
+ * Make sure that ent is mapped into RAM.
+ */
+static void
+consensus_cache_entry_map(consensus_cache_t *cache,
+ consensus_cache_entry_t *ent)
+{
+ if (ent->map)
+ return;
+
+ ent->map = storage_dir_map_labeled(cache->dir, ent->fname,
+ NULL, &ent->body, &ent->bodylen);
+ ent->unused_since = TIME_MAX;
+}
+
+/**
+ * Unmap ent from RAM.
+ *
+ * Do not call this if something other than the cache is holding a reference
+ * to ent
+ */
+static void
+consensus_cache_entry_unmap(consensus_cache_entry_t *ent)
+{
+ ent->unused_since = TIME_MAX;
+ if (!ent->map)
+ return;
+
+ tor_munmap_file(ent->map);
+ ent->map = NULL;
+ ent->body = NULL;
+ ent->bodylen = 0;
+ ent->unused_since = TIME_MAX;
+}
+
+#ifdef TOR_UNIT_TESTS
+/**
+ * Testing only: Return true iff ent is mapped into memory.
+ *
+ * (In normal operation, this information is not exposed.)
+ */
+int
+consensus_cache_entry_is_mapped(consensus_cache_entry_t *ent)
+{
+ if (ent->map) {
+ tor_assert(ent->body);
+ return 1;
+ } else {
+ tor_assert(!ent->body);
+ return 0;
+ }
+}
+#endif
+
diff --git a/src/or/conscache.h b/src/or/conscache.h
new file mode 100644
index 0000000000..94d7f15457
--- /dev/null
+++ b/src/or/conscache.h
@@ -0,0 +1,52 @@
+/* Copyright (c) 2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_CONSCACHE_H
+#define TOR_CONSCACHE_H
+
+typedef struct consensus_cache_entry_t consensus_cache_entry_t;
+typedef struct consensus_cache_t consensus_cache_t;
+
+consensus_cache_t *consensus_cache_open(const char *subdir, int max_entries);
+void consensus_cache_free(consensus_cache_t *cache);
+void consensus_cache_unmap_lazy(consensus_cache_t *cache, time_t cutoff);
+void consensus_cache_delete_pending(consensus_cache_t *cache);
+consensus_cache_entry_t *consensus_cache_add(consensus_cache_t *cache,
+ const config_line_t *labels,
+ const uint8_t *data,
+ size_t datalen);
+
+consensus_cache_entry_t *consensus_cache_find_first(
+ consensus_cache_t *cache,
+ const char *key,
+ const char *value);
+
+void consensus_cache_find_all(smartlist_t *out,
+ consensus_cache_t *cache,
+ const char *key,
+ const char *value);
+void consensus_cache_filter_list(smartlist_t *lst,
+ const char *key,
+ const char *value);
+
+const char *consensus_cache_entry_get_value(const consensus_cache_entry_t *ent,
+ const char *key);
+const config_line_t *consensus_cache_entry_get_labels(
+ const consensus_cache_entry_t *ent);
+
+void consensus_cache_entry_incref(consensus_cache_entry_t *ent);
+void consensus_cache_entry_decref(consensus_cache_entry_t *ent);
+
+void consensus_cache_entry_mark_for_removal(consensus_cache_entry_t *ent);
+void consensus_cache_entry_mark_for_aggressive_release(
+ consensus_cache_entry_t *ent);
+int consensus_cache_entry_get_body(const consensus_cache_entry_t *ent,
+ const uint8_t **body_out,
+ size_t *sz_out);
+
+#ifdef TOR_UNIT_TESTS
+int consensus_cache_entry_is_mapped(consensus_cache_entry_t *ent);
+#endif
+
+#endif
+
diff --git a/src/or/include.am b/src/or/include.am
index d65d0d3657..688ac625ea 100644
--- a/src/or/include.am
+++ b/src/or/include.am
@@ -36,6 +36,7 @@ LIBTOR_A_SOURCES = \
src/or/connection.c \
src/or/connection_edge.c \
src/or/connection_or.c \
+ src/or/conscache.c \
src/or/consdiff.c \
src/or/control.c \
src/or/cpuworker.c \
@@ -152,6 +153,7 @@ ORHEADERS = \
src/or/connection.h \
src/or/connection_edge.h \
src/or/connection_or.h \
+ src/or/conscache.h \
src/or/consdiff.h \
src/or/control.h \
src/or/cpuworker.h \
diff --git a/src/or/or.h b/src/or/or.h
index d592e4b91c..8a73cc293f 100644
--- a/src/or/or.h
+++ b/src/or/or.h
@@ -75,6 +75,7 @@
#include "address.h"
#include "compat_libevent.h"
#include "ht.h"
+#include "confline.h"
#include "replaycache.h"
#include "crypto_curve25519.h"
#include "crypto_ed25519.h"
@@ -3529,27 +3530,6 @@ typedef struct port_cfg_t {
char unix_addr[FLEXIBLE_ARRAY_MEMBER];
} port_cfg_t;
-/** Ordinary configuration line. */
-#define CONFIG_LINE_NORMAL 0
-/** Appends to previous configuration for the same option, even if we
- * would ordinary replace it. */
-#define CONFIG_LINE_APPEND 1
-/* Removes all previous configuration for an option. */
-#define CONFIG_LINE_CLEAR 2
-
-/** A linked list of lines in a config file. */
-typedef struct config_line_t {
- char *key;
- char *value;
- struct config_line_t *next;
- /** What special treatment (if any) does this line require? */
- unsigned int command:2;
- /** If true, subsequent assignments to this linelist should replace
- * it, not extend it. Set only on the first item in a linelist in an
- * or_options_t. */
- unsigned int fragile:1;
-} config_line_t;
-
typedef struct routerset_t routerset_t;
/** A magic value for the (Socks|OR|...)Port options below, telling Tor
diff --git a/src/test/include.am b/src/test/include.am
index 9670fd8189..f88f0992f2 100644
--- a/src/test/include.am
+++ b/src/test/include.am
@@ -87,6 +87,7 @@ src_test_test_SOURCES = \
src/test/test_compat_libevent.c \
src/test/test_config.c \
src/test/test_connection.c \
+ src/test/test_conscache.c \
src/test/test_consdiff.c \
src/test/test_containers.c \
src/test/test_controller.c \
diff --git a/src/test/test.c b/src/test/test.c
index fc1e3e89c0..f8c2db726d 100644
--- a/src/test/test.c
+++ b/src/test/test.c
@@ -1195,6 +1195,7 @@ struct testgroup_t testgroups[] = {
{ "compat/libevent/", compat_libevent_tests },
{ "config/", config_tests },
{ "connection/", connection_tests },
+ { "conscache/", conscache_tests },
{ "consdiff/", consdiff_tests },
{ "container/", container_tests },
{ "control/", controller_tests },
diff --git a/src/test/test.h b/src/test/test.h
index ec4d715422..2c35314dcd 100644
--- a/src/test/test.h
+++ b/src/test/test.h
@@ -190,6 +190,7 @@ extern struct testcase_t circuituse_tests[];
extern struct testcase_t compat_libevent_tests[];
extern struct testcase_t config_tests[];
extern struct testcase_t connection_tests[];
+extern struct testcase_t conscache_tests[];
extern struct testcase_t consdiff_tests[];
extern struct testcase_t container_tests[];
extern struct testcase_t controller_tests[];
diff --git a/src/test/test_conscache.c b/src/test/test_conscache.c
new file mode 100644
index 0000000000..12184f0cc6
--- /dev/null
+++ b/src/test/test_conscache.c
@@ -0,0 +1,341 @@
+/* Copyright (c) 2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "or.h"
+#include "config.h"
+#include "conscache.h"
+#include "test.h"
+
+#ifdef HAVE_UTIME_H
+#include
+#endif
+
+static void
+test_conscache_open_failure(void *arg)
+{
+ (void) arg;
+ /* Try opening a directory that doesn't exist and which we shouldn't be
+ * able to create. */
+ consensus_cache_t *cache = consensus_cache_open("a/b/c/d/e/f/g", 128);
+ tt_ptr_op(cache, OP_EQ, NULL);
+
+ done:
+ ;
+}
+
+static void
+test_conscache_simple_usage(void *arg)
+{
+ (void)arg;
+ consensus_cache_entry_t *ent = NULL, *ent2 = NULL;
+
+ /* Make a temporary datadir for these tests */
+ char *ddir_fname = tor_strdup(get_fname_rnd("datadir_cache"));
+ tor_free(get_options_mutable()->DataDirectory);
+ get_options_mutable()->DataDirectory = tor_strdup(ddir_fname);
+ check_private_dir(ddir_fname, CPD_CREATE, NULL);
+ consensus_cache_t *cache = consensus_cache_open("cons", 128);
+
+ tt_assert(cache);
+
+ /* Create object; make sure it exists. */
+ config_line_t *labels = NULL;
+ config_line_append(&labels, "Hello", "world");
+ config_line_append(&labels, "Adios", "planetas");
+ ent = consensus_cache_add(cache,
+ labels, (const uint8_t *)"A\0B\0C", 5);
+ config_free_lines(labels);
+ labels = NULL;
+ tt_assert(ent);
+
+ /* Make a second object */
+ config_line_append(&labels, "Hello", "mundo");
+ config_line_append(&labels, "Adios", "planets");
+ ent2 = consensus_cache_add(cache,
+ labels, (const uint8_t *)"xyzzy", 5);
+ config_free_lines(labels);
+ labels = NULL;
+ tt_assert(ent2);
+ tt_assert(! consensus_cache_entry_is_mapped(ent2));
+ consensus_cache_entry_decref(ent2);
+ ent2 = NULL;
+
+ /* Check get_value */
+ tt_ptr_op(NULL, OP_EQ, consensus_cache_entry_get_value(ent, "hebbo"));
+ tt_str_op("world", OP_EQ, consensus_cache_entry_get_value(ent, "Hello"));
+
+ /* Check find_first */
+ ent2 = consensus_cache_find_first(cache, "Hello", "world!");
+ tt_ptr_op(ent2, OP_EQ, NULL);
+ ent2 = consensus_cache_find_first(cache, "Hello", "world");
+ tt_ptr_op(ent2, OP_EQ, ent);
+ ent2 = consensus_cache_find_first(cache, "Hello", "mundo");
+ tt_ptr_op(ent2, OP_NE, ent);
+
+ tt_assert(! consensus_cache_entry_is_mapped(ent));
+
+ /* Check get_body */
+ const uint8_t *bp = NULL;
+ size_t sz = 0;
+ int r = consensus_cache_entry_get_body(ent, &bp, &sz);
+ tt_int_op(r, OP_EQ, 0);
+ tt_u64_op(sz, OP_EQ, 5);
+ tt_mem_op(bp, OP_EQ, "A\0B\0C", 5);
+ tt_assert(consensus_cache_entry_is_mapped(ent));
+
+ /* Free and re-create the cache, to rescan the directory. */
+ consensus_cache_free(cache);
+ consensus_cache_entry_decref(ent);
+ cache = consensus_cache_open("cons", 128);
+
+ /* Make sure the entry is still there */
+ ent = consensus_cache_find_first(cache, "Hello", "mundo");
+ tt_assert(ent);
+ ent2 = consensus_cache_find_first(cache, "Adios", "planets");
+ tt_ptr_op(ent, OP_EQ, ent2);
+ consensus_cache_entry_incref(ent);
+ tt_assert(! consensus_cache_entry_is_mapped(ent));
+ r = consensus_cache_entry_get_body(ent, &bp, &sz);
+ tt_int_op(r, OP_EQ, 0);
+ tt_u64_op(sz, OP_EQ, 5);
+ tt_mem_op(bp, OP_EQ, "xyzzy", 5);
+ tt_assert(consensus_cache_entry_is_mapped(ent));
+
+ /* There should be two entries total. */
+ smartlist_t *entries = smartlist_new();
+ consensus_cache_find_all(entries, cache, NULL, NULL);
+ int n = smartlist_len(entries);
+ smartlist_free(entries);
+ tt_int_op(n, OP_EQ, 2);
+
+ done:
+ consensus_cache_entry_decref(ent);
+ tor_free(ddir_fname);
+ consensus_cache_free(cache);
+}
+
+static void
+test_conscache_cleanup(void *arg)
+{
+ (void)arg;
+ const int N = 20;
+ consensus_cache_entry_t **ents =
+ tor_calloc(N, sizeof(consensus_cache_entry_t*));
+
+ /* Make a temporary datadir for these tests */
+ char *ddir_fname = tor_strdup(get_fname_rnd("datadir_cache"));
+ tor_free(get_options_mutable()->DataDirectory);
+ get_options_mutable()->DataDirectory = tor_strdup(ddir_fname);
+ check_private_dir(ddir_fname, CPD_CREATE, NULL);
+ consensus_cache_t *cache = consensus_cache_open("cons", 128);
+
+ tt_assert(cache);
+
+ /* Create a bunch of entries. */
+ int i;
+ for (i = 0; i < N; ++i) {
+ config_line_t *labels = NULL;
+ char num[8];
+ tor_snprintf(num, sizeof(num), "%d", i);
+ config_line_append(&labels, "test-id", "cleanup");
+ config_line_append(&labels, "index", num);
+ size_t bodylen = i * 3;
+ uint8_t *body = tor_malloc(bodylen);
+ memset(body, i, bodylen);
+ ents[i] = consensus_cache_add(cache, labels, body, bodylen);
+ tor_free(body);
+ config_free_lines(labels);
+ tt_assert(ents[i]);
+ /* We're still holding a reference to each entry at this point. */
+ }
+
+ /* Page all of the entries into RAM */
+ for (i = 0; i < N; ++i) {
+ const uint8_t *bp;
+ size_t sz;
+ tt_assert(! consensus_cache_entry_is_mapped(ents[i]));
+ consensus_cache_entry_get_body(ents[i], &bp, &sz);
+ tt_assert(consensus_cache_entry_is_mapped(ents[i]));
+ }
+
+ /* Mark some of the entries as deletable. */
+ for (i = 7; i < N; i += 7) {
+ consensus_cache_entry_mark_for_removal(ents[i]);
+ tt_assert(consensus_cache_entry_is_mapped(ents[i]));
+ }
+
+ /* Mark some of the entries as aggressively unpaged. */
+ for (i = 3; i < N; i += 3) {
+ consensus_cache_entry_mark_for_aggressive_release(ents[i]);
+ tt_assert(consensus_cache_entry_is_mapped(ents[i]));
+ }
+
+ /* Incref some of the entries again */
+ for (i = 0; i < N; i += 2) {
+ consensus_cache_entry_incref(ents[i]);
+ }
+
+ /* Now we're going to decref everything. We do so at a specific time. I'm
+ * picking the moment when I was writing this test, at 2017-04-05 12:16:48
+ * UTC. */
+ const time_t example_time = 1491394608;
+ update_approx_time(example_time);
+ for (i = 0; i < N; ++i) {
+ consensus_cache_entry_decref(ents[i]);
+ if (i % 2) {
+ ents[i] = NULL; /* We're no longer holding any reference here. */
+ }
+ }
+
+ /* At this point, the aggressively-released items with refcount 1 should
+ * be unmapped. Nothing should be deleted. */
+ consensus_cache_entry_t *e_tmp;
+ e_tmp = consensus_cache_find_first(cache, "index", "3");
+ tt_assert(e_tmp);
+ tt_assert(! consensus_cache_entry_is_mapped(e_tmp));
+ e_tmp = consensus_cache_find_first(cache, "index", "5");
+ tt_assert(e_tmp);
+ tt_assert(consensus_cache_entry_is_mapped(e_tmp));
+ e_tmp = consensus_cache_find_first(cache, "index", "6");
+ tt_assert(e_tmp);
+ tt_assert(consensus_cache_entry_is_mapped(e_tmp));
+ e_tmp = consensus_cache_find_first(cache, "index", "7");
+ tt_assert(e_tmp);
+ tt_assert(consensus_cache_entry_is_mapped(e_tmp));
+
+ /* Delete the pending-deletion items. */
+ consensus_cache_delete_pending(cache);
+ {
+ smartlist_t *entries = smartlist_new();
+ consensus_cache_find_all(entries, cache, NULL, NULL);
+ int n = smartlist_len(entries);
+ smartlist_free(entries);
+ tt_int_op(n, OP_EQ, 20 - 1); /* 1 entry was deleted */
+ }
+ e_tmp = consensus_cache_find_first(cache, "index", "7"); // refcnt == 1...
+ tt_assert(e_tmp == NULL); // so deleted.
+ e_tmp = consensus_cache_find_first(cache, "index", "14"); // refcnt == 2
+ tt_assert(e_tmp); // so, not deleted.
+
+ /* Now do lazy unmapping. */
+ // should do nothing.
+ consensus_cache_unmap_lazy(cache, example_time - 10);
+ e_tmp = consensus_cache_find_first(cache, "index", "11");
+ tt_assert(e_tmp);
+ tt_assert(consensus_cache_entry_is_mapped(e_tmp));
+ // should actually unmap
+ consensus_cache_unmap_lazy(cache, example_time + 10);
+ e_tmp = consensus_cache_find_first(cache, "index", "11");
+ tt_assert(e_tmp);
+ tt_assert(! consensus_cache_entry_is_mapped(e_tmp));
+ // This one will still be mapped, since it has a reference.
+ e_tmp = consensus_cache_find_first(cache, "index", "16");
+ tt_assert(e_tmp);
+ tt_assert(consensus_cache_entry_is_mapped(e_tmp));
+
+ for (i = 0; i < N; ++i) {
+ consensus_cache_entry_decref(ents[i]);
+ ents[i] = NULL;
+ }
+
+ /* Free and re-create the cache, to rescan the directory. Make sure the
+ * deleted thing is still deleted, along with the other deleted thing. */
+ consensus_cache_free(cache);
+ cache = consensus_cache_open("cons", 128);
+ {
+ smartlist_t *entries = smartlist_new();
+ consensus_cache_find_all(entries, cache, NULL, NULL);
+ int n = smartlist_len(entries);
+ smartlist_free(entries);
+ tt_int_op(n, OP_EQ, 18);
+ }
+
+ done:
+ for (i = 0; i < N; ++i) {
+ consensus_cache_entry_decref(ents[i]);
+ }
+ tor_free(ents);
+ tor_free(ddir_fname);
+ consensus_cache_free(cache);
+}
+
+static void
+test_conscache_filter(void *arg)
+{
+ (void)arg;
+ const int N = 30;
+ smartlist_t *lst = NULL;
+
+ /* Make a temporary datadir for these tests */
+ char *ddir_fname = tor_strdup(get_fname_rnd("datadir_cache"));
+ tor_free(get_options_mutable()->DataDirectory);
+ get_options_mutable()->DataDirectory = tor_strdup(ddir_fname);
+ check_private_dir(ddir_fname, CPD_CREATE, NULL);
+ consensus_cache_t *cache = consensus_cache_open("cons", 128);
+
+ tt_assert(cache);
+
+ /* Create a bunch of entries with different labels */
+ int i;
+ for (i = 0; i < N; ++i) {
+ config_line_t *labels = NULL;
+ char num[8];
+ tor_snprintf(num, sizeof(num), "%d", i);
+ config_line_append(&labels, "test-id", "filter");
+ config_line_append(&labels, "index", num);
+ tor_snprintf(num, sizeof(num), "%d", i % 3);
+ config_line_append(&labels, "mod3", num);
+ tor_snprintf(num, sizeof(num), "%d", i % 5);
+ config_line_append(&labels, "mod5", num);
+
+ size_t bodylen = i * 3;
+ uint8_t *body = tor_malloc(bodylen);
+ memset(body, i, bodylen);
+ consensus_cache_entry_t *ent =
+ consensus_cache_add(cache, labels, body, bodylen);
+ tor_free(body);
+ config_free_lines(labels);
+ tt_assert(ent);
+ consensus_cache_entry_decref(ent);
+ }
+
+ lst = smartlist_new();
+ /* Find nothing. */
+ consensus_cache_find_all(lst, cache, "mod5", "5");
+ tt_int_op(smartlist_len(lst), OP_EQ, 0);
+ /* Find everything. */
+ consensus_cache_find_all(lst, cache, "test-id", "filter");
+ tt_int_op(smartlist_len(lst), OP_EQ, N);
+
+ /* Now filter to find the entries that have i%3 == 1 */
+ consensus_cache_filter_list(lst, "mod3", "1");
+ tt_int_op(smartlist_len(lst), OP_EQ, 10);
+ /* Now filter to find the entries that also have i%5 == 3 */
+ consensus_cache_filter_list(lst, "mod5", "3");
+ tt_int_op(smartlist_len(lst), OP_EQ, 2);
+ /* So now we have those entries for which i%15 == 13. */
+
+ consensus_cache_entry_t *ent1 = smartlist_get(lst, 0);
+ consensus_cache_entry_t *ent2 = smartlist_get(lst, 1);
+ const char *idx1 = consensus_cache_entry_get_value(ent1, "index");
+ const char *idx2 = consensus_cache_entry_get_value(ent2, "index");
+ tt_assert( (!strcmp(idx1, "28") && !strcmp(idx2, "13")) ||
+ (!strcmp(idx1, "13") && !strcmp(idx2, "28")) );
+
+ done:
+ tor_free(ddir_fname);
+ consensus_cache_free(cache);
+ smartlist_free(lst);
+}
+
+#define ENT(name) \
+ { #name, test_conscache_ ## name, TT_FORK, NULL, NULL }
+
+struct testcase_t conscache_tests[] = {
+ ENT(open_failure),
+ ENT(simple_usage),
+ ENT(cleanup),
+ ENT(filter),
+ END_OF_TESTCASES
+};
+
diff --git a/src/test/test_storagedir.c b/src/test/test_storagedir.c
index e7e3da67eb..19e5de4ea3 100644
--- a/src/test/test_storagedir.c
+++ b/src/test/test_storagedir.c
@@ -256,6 +256,109 @@ test_storagedir_cleaning(void *arg)
}
}
+static void
+test_storagedir_save_labeled(void *arg)
+{
+ (void)arg;
+ char *dirname = tor_strdup(get_fname_rnd("store_dir"));
+ storage_dir_t *d = NULL;
+ uint8_t *inp = tor_malloc_zero(8192);
+ config_line_t *labels = NULL;
+ char *fname = NULL;
+ uint8_t *saved = NULL;
+
+ d = storage_dir_new(dirname, 10);
+ tt_assert(d);
+
+ crypto_rand((char *)inp, 8192);
+
+ config_line_append(&labels, "Foo", "bar baz");
+ config_line_append(&labels, "quux", "quuzXxz");
+ const char expected[] =
+ "Foo bar baz\n"
+ "quux quuzXxz\n";
+
+ int r = storage_dir_save_labeled_to_file(d, labels, inp, 8192, &fname);
+ tt_int_op(r, OP_EQ, 0);
+
+ size_t n;
+ saved = storage_dir_read(d, fname, 1, &n);
+ tt_assert(memchr(saved, '\0', n));
+ tt_str_op((char*)saved, OP_EQ, expected); /* NUL guarantees strcmp works */
+ tt_mem_op(saved+strlen(expected)+1, OP_EQ, inp, 8192);
+
+ done:
+ storage_dir_free(d);
+ tor_free(dirname);
+ tor_free(inp);
+ tor_free(fname);
+ config_free_lines(labels);
+ tor_free(saved);
+}
+
+static void
+test_storagedir_read_labeled(void *arg)
+{
+ (void)arg;
+ char *dirname = tor_strdup(get_fname_rnd("store_dir"));
+ storage_dir_t *d = NULL;
+ uint8_t *inp = tor_malloc_zero(8192);
+ config_line_t *labels = NULL, *labels2 = NULL;
+ char *fname = NULL;
+ tor_mmap_t *map = NULL;
+ uint8_t *as_read = NULL;
+
+ d = storage_dir_new(dirname, 10);
+ tt_assert(d);
+
+ tor_snprintf((char*)inp, 8192,
+ "Hello world\n"
+ "This is a test\n"
+ "Yadda yadda.\n");
+ size_t bodylen = 8192 - strlen((char*)inp) - 1;
+ crypto_rand((char *)inp+strlen((char*)inp)+1, bodylen);
+
+ int r = storage_dir_save_bytes_to_file(d, inp, 8192, 1, &fname);
+ tt_int_op(r, OP_EQ, 0);
+
+ /* Try mapping */
+ const uint8_t *datap = NULL;
+ size_t sz = 0;
+ map = storage_dir_map_labeled(d, fname, &labels, &datap, &sz);
+ tt_assert(map);
+ tt_assert(datap);
+ tt_u64_op(sz, OP_EQ, bodylen);
+ tt_mem_op(datap, OP_EQ, inp+strlen((char*)inp)+1, bodylen);
+ tt_assert(labels);
+ tt_str_op(labels->key, OP_EQ, "Hello");
+ tt_str_op(labels->value, OP_EQ, "world");
+ tt_assert(labels->next);
+ tt_str_op(labels->next->key, OP_EQ, "This");
+ tt_str_op(labels->next->value, OP_EQ, "is a test");
+ tt_assert(labels->next->next);
+ tt_str_op(labels->next->next->key, OP_EQ, "Yadda");
+ tt_str_op(labels->next->next->value, OP_EQ, "yadda.");
+ tt_assert(labels->next->next->next == NULL);
+
+ /* Try reading this time. */
+ sz = 0;
+ as_read = storage_dir_read_labeled(d, fname, &labels2, &sz);
+ tt_assert(as_read);
+ tt_u64_op(sz, OP_EQ, bodylen);
+ tt_mem_op(as_read, OP_EQ, inp+strlen((char*)inp)+1, bodylen);
+ tt_assert(config_lines_eq(labels, labels2));
+
+ done:
+ storage_dir_free(d);
+ tor_free(dirname);
+ tor_free(inp);
+ tor_free(fname);
+ config_free_lines(labels);
+ config_free_lines(labels2);
+ tor_munmap_file(map);
+ tor_free(as_read);
+}
+
#define ENT(name) \
{ #name, test_storagedir_ ## name, TT_FORK, NULL, NULL }
@@ -265,6 +368,8 @@ struct testcase_t storagedir_tests[] = {
ENT(deletion),
ENT(full),
ENT(cleaning),
+ ENT(save_labeled),
+ ENT(read_labeled),
END_OF_TESTCASES
};