diff --git a/src/or/circuitbuild.c b/src/or/circuitbuild.c
index 8efaa7cf4d..03b702bed9 100644
--- a/src/or/circuitbuild.c
+++ b/src/or/circuitbuild.c
@@ -1598,6 +1598,12 @@ circuit_build_times_set_timeout(circuit_build_times_t *cbt)
   long prev_timeout = tor_lround(cbt->timeout_ms/1000);
   double timeout_rate;
 
+  /*
+   * Just return if we aren't using adaptive timeouts
+   */
+  if (circuit_build_times_disabled())
+    return;
+
   if (!circuit_build_times_set_timeout_worker(cbt))
     return;