Try to extract as many descriptors as possible from truncated http responses. (when DIR_PURPOSE_FETCH_ROUTERDESC)

svn:r5249

src/or/buffers.c

@@ -788,13 +788,16 @@ fetch_from_buf(char *string, size_t string_len, buf_t *buf)
  *  - If a headers or body doesn't fit in the arg, return -1.
  *  (We ensure that the headers or body don't exceed max len,
  *   _even if_ we're planning to discard them.)
+ *  - If force_complete is true, then succeed even if not all of the
+ *    content has arrived.
  *
  * Else, change nothing and return 0.
  */
 int
 fetch_from_buf_http(buf_t *buf,
                     char **headers_out, size_t max_headerlen,
-                    char **body_out, size_t *body_used, size_t max_bodylen)
+                    char **body_out, size_t *body_used, size_t max_bodylen,
+                    int force_complete)
 {
   char *headers, *body, *p;
   size_t headerlen, bodylen, contentlen;
@@ -840,8 +843,10 @@ fetch_from_buf_http(buf_t *buf,
     /* if content-length is malformed, then our body length is 0. fine. */
     log_fn(LOG_DEBUG,"Got a contentlen of %d.",(int)contentlen);
     if (bodylen < contentlen) {
-      log_fn(LOG_DEBUG,"body not all here yet.");
-      return 0; /* not all there yet */
+      if (!force_complete) {
+        log_fn(LOG_DEBUG,"body not all here yet.");
+        return 0; /* not all there yet */
+      }
     }
     if (bodylen > contentlen) {
       bodylen = contentlen;

src/or/connection_or.c

@@ -69,7 +69,7 @@ connection_or_read_proxy_response(connection_t *conn)
 
   switch (fetch_from_buf_http(conn->inbuf,
                               &headers, MAX_HEADERS_SIZE,
-                              NULL, NULL, 10000)) {
+                              NULL, NULL, 10000, 0)) {
     case -1: /* overflow */
       log_fn(LOG_WARN,"Your https proxy sent back an oversized response. Closing.");
       return -1;

src/or/directory.c

@@ -806,10 +806,12 @@ connection_dir_client_reached_eof(connection_t *conn)
   int compression;
   int plausible;
   int skewed=0;
+  int allow_partial = conn->purpose == DIR_PURPOSE_FETCH_SERVERDESC;
 
   switch (fetch_from_buf_http(conn->inbuf,
                               &headers, MAX_HEADERS_SIZE,
-                              &body, &body_len, MAX_DIR_SIZE)) {
+                              &body, &body_len, MAX_DIR_SIZE,
+                              allow_partial)) {
     case -1: /* overflow */
       log_fn(LOG_WARN,"'fetch' response too large (server '%s:%d'). Closing.", conn->address, conn->port);
       return -1;
@@ -878,11 +880,13 @@ connection_dir_client_reached_eof(connection_t *conn)
     }
     /* Try declared compression first if we can. */
     if (compression > 0)
-      tor_gzip_uncompress(&new_body, &new_len, body, body_len, compression, 1);
+      tor_gzip_uncompress(&new_body, &new_len, body, body_len, compression,
+                          allow_partial);
     /* Okay, if that didn't work, and we think that it was compressed
      * differently, try that. */
     if (!new_body && guessed > 0 && compression != guessed)
-      tor_gzip_uncompress(&new_body, &new_len, body, body_len, guessed, 1);
+      tor_gzip_uncompress(&new_body, &new_len, body, body_len, guessed,
+                          allow_partial);
     /* If we're pretty sure that we have a compressed directory, and
      * we didn't manage to uncompress it, then warn and bail. */
     if (!plausible && !new_body) {
@@ -1510,7 +1514,7 @@ directory_handle_command(connection_t *conn)
 
   switch (fetch_from_buf_http(conn->inbuf,
                               &headers, MAX_HEADERS_SIZE,
-                              &body, &body_len, MAX_BODY_SIZE)) {
+                              &body, &body_len, MAX_BODY_SIZE, 0)) {
     case -1: /* overflow */
       log_fn(LOG_WARN,"Invalid input from address '%s'. Closing.", conn->address);
       return -1;

src/or/main.c

@@ -580,9 +580,9 @@ run_connection_housekeeping(int i, time_t now)
     /* This check is temporary; it's to let us know whether we should consider
      * parsing partial serverdesc responses. */
     if (conn->purpose == DIR_PURPOSE_FETCH_SERVERDESC &&
-        buf_datalen(conn->inbuf)>=(24*1024)) {
-      log_fn(LOG_NOTICE, "Expired a wedged directory connection that had already downloaded %d bytes of descriptors.  If this happens often, please file a bug report.",
-             (int)buf_datalen(conn->inbuf));
+        buf_datalen(conn->inbuf)>=1024) {
+      log_fn(LOG_INFO,"Trying to extract information from wedged server desc downoad");
+      connection_dir_reached_eof(conn);
     }
     connection_mark_for_close(conn);
     return;

src/or/or.h

@@ -1362,7 +1362,8 @@ int write_to_buf(const char *string, size_t string_len, buf_t *buf);
 int fetch_from_buf(char *string, size_t string_len, buf_t *buf);
 int fetch_from_buf_http(buf_t *buf,
                         char **headers_out, size_t max_headerlen,
-                        char **body_out, size_t *body_used, size_t max_bodylen);
+                        char **body_out, size_t *body_used, size_t max_bodylen,
+                        int force_complete);
 int fetch_from_buf_socks(buf_t *buf, socks_request_t *req);
 int fetch_from_buf_control0(buf_t *buf, uint32_t *len_out, uint16_t *type_out,
                             char **body_out, int check_for_v1);

src/or/routerparse.c

@@ -636,13 +636,14 @@ check_directory_signature(const char *digest,
 /** Given a string *<b>s</b> containing a concatenated sequence of router
  * descriptors, parses them and stores the result in <b>dest</b>.  All routers
  * are marked running and verified.  Advances *s to a point immediately
- * following the last router entry.  Returns 0 on success and -1 on failure.
+ * following the last router entry.  Ignore any trailing router entries that
+ * are not complete. Returns 0 on success and -1 on failure.
  */
 int
 router_parse_list_from_string(const char **s, smartlist_t *dest)
 {
   routerinfo_t *router;
-  const char *end;
+  const char *end, *cp;
 
   tor_assert(s);
   tor_assert(*s);
@@ -654,20 +655,36 @@ router_parse_list_from_string(const char **s, smartlist_t *dest)
     if (strcmpstart(*s, "router ")!=0)
       break;
     if ((end = strstr(*s+1, "\nrouter "))) {
+      cp = end;
       end++;
     } else if ((end = strstr(*s+1, "\ndirectory-signature"))) {
+      cp = end;
       end++;
     } else {
-      end = *s+strlen(*s);
+      cp = end = *s+strlen(*s);
+    }
+
+    while (cp > *s && (!*cp || TOR_ISSPACE(*cp)))
+      --cp;
+    /* cp now points to the last non-space character in this descriptor. */
+
+    while (cp > *s  && *cp != '\n')
+      --cp;
+    /* cp now points to the first \n before the last non-bank line in this
+     * descriptor */
+
+    if (strcmpstart(cp, "\n-----END SIGNATURE-----\n")) {
+      log_fn(LOG_INFO, "Ignoring truncated router descriptor.");
+      continue;
     }
 
     router = router_parse_entry_from_string(*s, end);
+
     *s = end;
     if (!router) {
       log_fn(LOG_WARN, "Error reading router; skipping");
       continue;
     }
-
     smartlist_add(dest, router);
   }