Make timeout exponential (#1583)

rust/agents/relayer/src/msg/serial_submitter.rs

@@ -210,9 +210,11 @@ impl SerialSubmitter {
         if msg.num_retries >= 16 {
             let required_duration = Duration::from_secs(match msg.num_retries {
                 i if i < 16 => unreachable!(),
-                i if (16..24).contains(&i) => 60 * 5, // wait 5 min
+                // wait 5 min
-                i if (24..32).contains(&i) => 60 * 30, // wait 30 min
+                i if (16..24).contains(&i) => 60 * 5,
-                _ => 60 * 60,                         // max timeout of 1hr beyond that
+                // exponential increase + 30 min; -21 makes it so that at i = 32 it will be
+                // ~60min timeout (64min to be more precise).
+                i => (2u64).pow(i - 21) + 60 * 30,
             });
             if Instant::now().duration_since(msg.last_attempted_at) < required_duration {
                 self.run_queue.push_back(msg);