feat(ci): rust caching in `test` workflow (#4015)

- draft: improve rust target caching in ci
- separate caches for e2e-cosmwasm, e2e-noncosmwasm, cli-e2e test
- drive-by adding `restore-keys`

test expectation:
- 1st commit of rust changes will take the full required time as it'll
have to do a fresh build and then cache
- subsequent builds of a particular rust change will be faster because
it'll pickup the rust cache from before

---------

Signed-off-by: Paul Balaji <10051819+paulbalaji@users.noreply.github.com>

.github/workflows/rust.yml

@@ -39,8 +39,7 @@ jobs:
       - name: rust cache
         uses: Swatinem/rust-cache@v2
         with:
-          prefix-key: "v2-rust"
-          shared-key: "test"
+          prefix-key: "v3-rust"
           workspaces: |
             ./rust
       - name: Free disk space
@@ -69,8 +68,7 @@ jobs:
       - name: rust cache
         uses: Swatinem/rust-cache@v2
         with:
-          prefix-key: "v2-rust"
-          shared-key: "lint"
+          prefix-key: "v3-rust"
           workspaces: |
             ./rust
       - name: Free disk space

.github/workflows/test.yml

@@ -254,6 +254,14 @@ jobs:
           toolchain: stable
           profile: minimal
 
+      - name: rust cache
+        uses: Swatinem/rust-cache@v2
+        with:
+          prefix-key: "v1-${{ runner.os }}-rust-cache"
+          shared-key: ${{ matrix.e2e-type }}
+          workspaces: |
+            ./rust
+
       - name: Free disk space
         run: |
           # Based on https://github.com/actions/runner-images/issues/2840#issuecomment-790492173
@@ -296,22 +304,15 @@ jobs:
           key: hyperlane-registry-${{ github.event.pull_request.head.sha || github.sha }}
           fail-on-cache-miss: true
 
-      - name: cargo-cache
-        uses: actions/cache@v4
-        with:
-          path: |
-            ~/.cargo
-          key: ${{ runner.os }}-cargo-cache-${{ hashFiles('./rust/Cargo.lock') }}
-
       - name: agent tests with CosmWasm
-        run: cargo test --release --package run-locally --bin run-locally --features cosmos -- cosmos::test --nocapture
+        run: cargo test --release --package run-locally --bin run-locally --features cosmos test-utils -- cosmos::test --nocapture
         if: matrix.e2e-type == 'cosmwasm'
         working-directory: ./rust
         env:
           RUST_BACKTRACE: 'full'
 
       - name: agent tests excluding CosmWasm
-        run: cargo run --release --bin run-locally
+        run: cargo run --release --bin run-locally --features test-utils
         if: matrix.e2e-type == 'non-cosmwasm'
         working-directory: ./rust
         env:
@@ -330,10 +331,60 @@ jobs:
           echo "All e2e-matrix jobs have completed."
           # You can add additional commands here to report the result as needed
 
+  prebuild-cli-e2e:
+    runs-on: larger-runner
+    if: github.event_name == 'push' || (github.event_name == 'pull_request' && github.base_ref == 'main') || github.event_name == 'merge_group'
+    needs: [checkout-registry]
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha || github.sha }}
+          submodules: recursive
+
+      - name: setup rust
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          profile: minimal
+
+      - name: rust cache
+        uses: Swatinem/rust-cache@v2
+        with:
+          prefix-key: "v1-${{ runner.os }}-rust-cache"
+          shared-key: "cli-e2e"
+          workspaces: |
+            ./rust
+
+      - name: Free disk space
+        run: |
+          # Based on https://github.com/actions/runner-images/issues/2840#issuecomment-790492173
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /opt/ghc
+          sudo rm -rf "/usr/local/share/boost"
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+
+      - name: Install mold linker
+        uses: rui314/setup-mold@v1
+        with:
+          mold-version: 2.0.0
+          make-default: true
+
+      - name: Build validator
+        run: cargo build --bin validator --features test-utils
+        working-directory: ./rust
+        env:
+          RUST_BACKTRACE: 'full'
+
+      - name: Build relayer
+        run: cargo build --bin relayer --features test-utils
+        working-directory: ./rust
+        env:
+          RUST_BACKTRACE: 'full'
+
   cli-e2e:
     runs-on: larger-runner
     if: github.event_name == 'push' || (github.event_name == 'pull_request' && github.base_ref == 'main') || github.event_name == 'merge_group'
-    needs: [yarn-build, checkout-registry]
+    needs: [yarn-build, prebuild-cli-e2e]
     strategy:
       matrix:
         include:
@@ -359,6 +410,14 @@ jobs:
           toolchain: stable
           profile: minimal
 
+      - name: rust cache
+        uses: Swatinem/rust-cache@v2
+        with:
+          prefix-key: "v1-${{ runner.os }}-rust-cache"
+          shared-key: "cli-e2e"
+          workspaces: |
+            ./rust
+
       - name: Free disk space
         run: |
           # Based on https://github.com/actions/runner-images/issues/2840#issuecomment-790492173
@@ -401,13 +460,6 @@ jobs:
           key: hyperlane-registry-${{ github.event.pull_request.head.sha || github.sha }}
           fail-on-cache-miss: true
 
-      - name: cargo-cache
-        uses: actions/cache@v4
-        with:
-          path: |
-            ~/.cargo
-          key: ${{ runner.os }}-cargo-cache-${{ hashFiles('./rust/Cargo.lock') }}
-
       - name: cli e2e tests
         run: ./typescript/cli/ci-test.sh ${{ matrix.test-type }}
 

typescript/cli/ci-test.sh

@@ -254,7 +254,7 @@ run_hyperlane_send_message() {
 
 run_validator() {
     echo -e "\nPre-building validator with cargo"
-    cargo build --bin validator
+    cargo build --bin validator --features test-utils
 
     # set some default agent env vars, used by both validators and relayer
     export HYP_CHAINS_${CHAIN1_CAPS}_BLOCKS_REORGPERIOD=0
@@ -290,7 +290,7 @@ run_validator() {
 
     echo "Validator running, sleeping to let it sync"
     # This needs to be long to allow time for the cargo build to finish
-    sleep 15
+    sleep 20
     echo "Done sleeping"
 
     for CHAIN in ${CHAIN1} ${CHAIN2}
@@ -308,7 +308,7 @@ run_validator() {
 
 run_relayer() {
     echo -e "\nPre-building relayer with cargo"
-    cargo build --bin relayer
+    cargo build --bin relayer --features test-utils
 
     echo "Running relayer"
     export CONFIG_FILES=/tmp/agent-config.json