diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d349ae85819..46b82df7991 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -97,11 +97,37 @@ jobs: runs-on: spacetimedb-new-runner-2 timeout-minutes: 15 env: - CARGO_TARGET_DIR: ${{ github.workspace }}/target RUST_BACKTRACE: full ARTIFACT_SUFFIX: linux EXE_SUFFIX: "" steps: &upload-build-artifact-steps + - &set-cargo-target-dir + name: Set Cargo target directory + shell: bash + run: | + if [[ "${RUNNER_OS}" == "Linux" ]]; then + cargo_target_dir="${HOME}/actions-runner/_work/target" + else + # Forward slashes keep the Windows path compatible with Cargo, + # PowerShell, GitHub Actions, and Git Bash filesystem commands. + cargo_target_dir="${GITHUB_WORKSPACE//\\//}/target" + fi + echo "CARGO_TARGET_DIR=${cargo_target_dir}" >>"${GITHUB_ENV}" + + - &configure-sccache + name: Configure sccache + if: runner.os == 'Linux' + shell: bash + run: | + if command -v sccache >/dev/null 2>&1; then + # sccache cannot cache incremental compilation artifacts, so + # disable Cargo's incremental mode whenever the wrapper is active. + echo "CARGO_INCREMENTAL=0" >>"${GITHUB_ENV}" + echo "RUSTC_WRAPPER=/usr/local/bin/sccache" >>"${GITHUB_ENV}" + echo "SCCACHE_DIR=${HOME}/.cache/sccache" >>"${GITHUB_ENV}" + echo "SCCACHE_CACHE_SIZE=20G" >>"${GITHUB_ENV}" + fi + - &find-git-ref name: Find Git ref env: @@ -140,13 +166,10 @@ jobs: # - V8 uses its default features and has profile-specific native output. # - OpenSSL is the vendored static build with legacy enabled. Windows uses # NASM, which is represented by a separate cache-key suffix. - # - jemalloc is an optimized static PIC build with profiling, stats, and - # its default background-thread support. - # - Zstd is an optimized static build with legacy support and dictionary - # APIs. It is reused across Cargo profiles. - &set-native-cache-keys name: Set native cache keys id: native-cache-keys + if: runner.os == 'Windows' shell: bash run: | target="$(rustc -vV | sed -n 's/^host: //p')" @@ -157,15 +180,10 @@ jobs: echo "v8-release-key=rusty-v8-v1-145.0.0-${target}" >>"$GITHUB_OUTPUT" echo "v8-debug-key=rusty-v8-debug-v1-145.0.0-${target}" >>"$GITHUB_OUTPUT" echo "openssl-key=openssl-v1-300.5.3+3.5.4-${target}${openssl_suffix}" >>"$GITHUB_OUTPUT" - if [[ "$RUNNER_OS" == "Linux" ]]; then - cc="${CC:-cc}" - cc_hash="$("$cc" --version | sha256sum | cut -c1-16)" - echo "jemalloc-key=jemalloc-v1-0.6.1+5.3.0-1-ge13ca993e8ccb9ba9847cc330696e02839f328f7-${target}-${cc_hash}" >>"$GITHUB_OUTPUT" - echo "zstd-key=zstd-v1-2.0.16+zstd.1.5.7-${target}-${cc_hash}" >>"$GITHUB_OUTPUT" - fi - &cache-rusty-v8 name: Cache rusty_v8 + if: runner.os == 'Windows' uses: actions/cache@v4 with: &v8-release-cache path: ${{ env.CARGO_TARGET_DIR }}/release/gn_out/obj @@ -174,6 +192,7 @@ jobs: - &cache-openssl name: Cache OpenSSL id: cache-openssl + if: runner.os == 'Windows' uses: actions/cache@v4 with: &openssl-cache path: ${{ github.workspace }}/.ci-cache/openssl @@ -181,43 +200,13 @@ jobs: - &configure-cached-openssl name: Configure cached OpenSSL - if: steps.cache-openssl.outputs.cache-hit == 'true' + if: runner.os == 'Windows' && steps.cache-openssl.outputs.cache-hit == 'true' shell: bash run: | # Reuse the cached vendored output as a prebuilt OpenSSL installation. echo "OPENSSL_NO_VENDOR=1" >>"$GITHUB_ENV" echo "OPENSSL_DIR=${{ github.workspace }}/.ci-cache/openssl" >>"$GITHUB_ENV" - - name: Cache jemalloc - id: cache-jemalloc - if: runner.os == 'Linux' - uses: actions/cache@v4 - with: &jemalloc-cache - path: ${{ github.workspace }}/.ci-cache/jemalloc - key: ${{ steps.native-cache-keys.outputs.jemalloc-key }} - - - &configure-cached-jemalloc - name: Configure cached jemalloc - if: runner.os == 'Linux' && steps.cache-jemalloc.outputs.cache-hit == 'true' - shell: bash - run: echo "JEMALLOC_OVERRIDE=${{ github.workspace }}/.ci-cache/jemalloc/libjemalloc_pic.a" >>"$GITHUB_ENV" - - - name: Cache Zstd - id: cache-zstd - if: runner.os == 'Linux' - uses: actions/cache@v4 - with: &zstd-cache - path: ${{ github.workspace }}/.ci-cache/zstd - key: ${{ steps.native-cache-keys.outputs.zstd-key }} - - - &configure-cached-zstd - name: Configure cached Zstd - if: runner.os == 'Linux' && steps.cache-zstd.outputs.cache-hit == 'true' - shell: bash - run: | - echo "ZSTD_SYS_USE_PKG_CONFIG=1" >>"$GITHUB_ENV" - echo "PKG_CONFIG_PATH=${{ github.workspace }}/.ci-cache/zstd/lib/pkgconfig${PKG_CONFIG_PATH:+:$PKG_CONFIG_PATH}" >>"$GITHUB_ENV" - - name: Build CLI and standalone shell: bash run: | @@ -228,65 +217,39 @@ jobs: - &prepare-openssl-cache name: Prepare OpenSSL cache - if: steps.cache-openssl.outputs.cache-hit != 'true' + if: runner.os == 'Windows' && steps.cache-openssl.outputs.cache-hit != 'true' shell: bash run: | + openssl_cache_dir="${PWD}/.ci-cache/openssl" + cd "${CARGO_TARGET_DIR}" shopt -s nullglob - openssl_installs=(target/release/build/openssl-sys-*/out/openssl-build/install) + openssl_installs=(release/build/openssl-sys-*/out/openssl-build/install) if (( ${#openssl_installs[@]} != 1 )); then echo "Expected one vendored OpenSSL installation, found ${#openssl_installs[@]}." exit 1 fi - mkdir -p .ci-cache/openssl - cp -R "${openssl_installs[0]}/." .ci-cache/openssl/ + mkdir -p "${openssl_cache_dir}" + cp -R "${openssl_installs[0]}/." "${openssl_cache_dir}/" - - name: Prepare jemalloc cache - if: runner.os == 'Linux' && steps.cache-jemalloc.outputs.cache-hit != 'true' + - name: Package build artifacts shell: bash run: | - shopt -s nullglob - jemalloc_archives=(target/release/build/tikv-jemalloc-sys-*/out/lib/libjemalloc_pic.a) - if (( ${#jemalloc_archives[@]} != 1 )); then - echo "Expected one jemalloc archive, found ${#jemalloc_archives[@]}." - exit 1 - fi - mkdir -p .ci-cache/jemalloc - cp "${jemalloc_archives[0]}" .ci-cache/jemalloc/ - echo "JEMALLOC_OVERRIDE=${{ github.workspace }}/.ci-cache/jemalloc/libjemalloc_pic.a" >>"$GITHUB_ENV" - - - name: Prepare Zstd cache - if: runner.os == 'Linux' && steps.cache-zstd.outputs.cache-hit != 'true' + archive_path="${PWD}/build-support.tar.gz" + cd "${CARGO_TARGET_DIR}" + tar -czf "${archive_path}" \ + "release/spacetimedb-cli${EXE_SUFFIX}" \ + "release/spacetimedb-standalone${EXE_SUFFIX}" + + - &show-sccache-stats + name: Show sccache statistics + if: always() && runner.os == 'Linux' shell: bash run: | - shopt -s nullglob - zstd_outputs=(target/release/build/zstd-sys-*/out) - if (( ${#zstd_outputs[@]} != 1 )); then - echo "Expected one vendored Zstd build, found ${#zstd_outputs[@]}." - exit 1 + if command -v sccache >/dev/null 2>&1; then + sccache --show-stats + else + echo "::notice::sccache is not installed on this Linux runner" fi - mkdir -p .ci-cache/zstd/lib/pkgconfig .ci-cache/zstd/include - cp "${zstd_outputs[0]}/libzstd.a" .ci-cache/zstd/lib/ - cp "${zstd_outputs[0]}"/include/*.h .ci-cache/zstd/include/ - printf '%s\n' \ - 'prefix=${pcfiledir}/../..' \ - 'libdir=${prefix}/lib' \ - 'includedir=${prefix}/include' \ - '' \ - 'Name: libzstd' \ - 'Description: Zstandard compression library' \ - 'Version: 1.5.7' \ - 'Libs: -L${libdir} -lzstd' \ - 'Cflags: -I${includedir}' \ - > .ci-cache/zstd/lib/pkgconfig/libzstd.pc - echo "ZSTD_SYS_USE_PKG_CONFIG=1" >>"$GITHUB_ENV" - echo "PKG_CONFIG_PATH=${{ github.workspace }}/.ci-cache/zstd/lib/pkgconfig${PKG_CONFIG_PATH:+:$PKG_CONFIG_PATH}" >>"$GITHUB_ENV" - - - name: Package build artifacts - shell: bash - run: | - tar -czf build-support.tar.gz \ - "target/release/spacetimedb-cli${EXE_SUFFIX}" \ - "target/release/spacetimedb-standalone${EXE_SUFFIX}" - name: Upload Cargo timing reports if: always() @@ -307,6 +270,12 @@ jobs: overwrite: true retention-days: 14 + - &show-sccache-stats + name: Show sccache statistics + if: always() && runner.os == 'Linux' + shell: bash + run: /usr/local/bin/sccache --show-stats + upload-build-artifacts-windows: needs: [merge_queue_noop, lints] if: ${{ needs.merge_queue_noop.outputs.skip != 'true' }} @@ -314,7 +283,6 @@ jobs: runs-on: spacetimedb-windows-runner timeout-minutes: 15 env: - CARGO_TARGET_DIR: ${{ github.workspace }}/target RUST_BACKTRACE: full ARTIFACT_SUFFIX: windows EXE_SUFFIX: .exe @@ -327,12 +295,14 @@ jobs: runs-on: spacetimedb-new-runner-2 timeout-minutes: 15 env: - CARGO_TARGET_DIR: ${{ github.workspace }}/target RUST_BACKTRACE: full ARTIFACT_SUFFIX: linux EXE_SUFFIX: "" SMOKETEST_SUITE: standalone steps: &smoketest-build-steps + - *set-cargo-target-dir + - *configure-sccache + - *find-git-ref - *checkout-sources @@ -344,6 +314,7 @@ jobs: - &restore-openssl name: Restore OpenSSL id: cache-openssl + if: runner.os == 'Windows' uses: actions/cache/restore@v4 with: *openssl-cache - *configure-cached-openssl @@ -356,17 +327,21 @@ jobs: run: | cargo run --timings --package ci-smoketests -- --suite "${SMOKETEST_SUITE}" archive --archive-file smoketest-nextest.tar.zst + archive_path="${PWD}/smoketest-support.tar.gz" + cd "${CARGO_TARGET_DIR}" shopt -s nullglob - precompiled_modules=(target/wasm32-unknown-unknown/release/smoketest_module_*.wasm) + precompiled_modules=(wasm32-unknown-unknown/release/smoketest_module_*.wasm) if (( ${#precompiled_modules[@]} == 0 )); then echo "No precompiled smoketest modules were produced." exit 1 fi - tar -czf smoketest-support.tar.gz \ - "target/debug/ci-smoketests${EXE_SUFFIX}" \ + tar -czf "${archive_path}" \ + "debug/ci-smoketests${EXE_SUFFIX}" \ "${precompiled_modules[@]}" + - *show-sccache-stats + - name: Upload Cargo timing reports if: always() uses: actions/upload-artifact@v4 @@ -388,13 +363,14 @@ jobs: overwrite: true retention-days: 14 + - *show-sccache-stats + smoketest_build_windows: needs: [upload-build-artifacts-windows] name: Build smoketests (Windows) runs-on: spacetimedb-windows-runner timeout-minutes: 15 env: - CARGO_TARGET_DIR: ${{ github.workspace }}/target RUST_BACKTRACE: full ARTIFACT_SUFFIX: windows EXE_SUFFIX: .exe @@ -412,12 +388,14 @@ jobs: runs-on: spacetimedb-new-runner-2 timeout-minutes: 30 env: - CARGO_TARGET_DIR: ${{ github.workspace }}/target RUST_BACKTRACE: full SPACETIMEDB_CPP_DIR: ${{ github.workspace }}/crates/bindings-cpp ARTIFACT_SUFFIX: linux PARTITION_COUNT: 1 steps: &smoketest-partition-steps + - *set-cargo-target-dir + - *configure-sccache + - *find-git-ref - *checkout-sources @@ -501,13 +479,16 @@ jobs: name: Extract build artifacts shell: bash run: | - tar -xzf build-artifacts/build-support.tar.gz + archive_path="${PWD}/build-artifacts/build-support.tar.gz" exe_suffix="" if [[ "${RUNNER_OS}" == "Windows" ]]; then exe_suffix=".exe" fi - test -f "${CARGO_TARGET_DIR}/release/spacetimedb-cli${exe_suffix}" - test -f "${CARGO_TARGET_DIR}/release/spacetimedb-standalone${exe_suffix}" + mkdir -p "${CARGO_TARGET_DIR}" + cd "${CARGO_TARGET_DIR}" + tar -xzf "${archive_path}" + test -f "release/spacetimedb-cli${exe_suffix}" + test -f "release/spacetimedb-standalone${exe_suffix}" - name: Download smoketest build uses: actions/download-artifact@v4 @@ -516,7 +497,11 @@ jobs: - name: Extract smoketest support files shell: bash - run: tar -xzf smoketest-support.tar.gz + run: | + archive_path="${PWD}/smoketest-support.tar.gz" + mkdir -p "${CARGO_TARGET_DIR}" + cd "${CARGO_TARGET_DIR}" + tar -xzf "${archive_path}" # Serial execution avoids contention between the C# tests over generated bindings. - name: Run smoketest partition (Linux) @@ -526,7 +511,7 @@ jobs: if [ -f ~/emsdk/emsdk_env.sh ]; then source ~/emsdk/emsdk_env.sh fi - ./target/debug/ci-smoketests run-archive \ + cargo ci smoketests run-archive \ --archive-file smoketest-nextest.tar.zst \ -- \ --partition hash:${{ matrix.partition }}/${{ env.PARTITION_COUNT }} @@ -539,7 +524,7 @@ jobs: if (Test-Path "$env:USERPROFILE\emsdk\emsdk_env.ps1") { & "$env:USERPROFILE\emsdk\emsdk_env.ps1" | Out-Null } - .\target\debug\ci-smoketests.exe run-archive ` + & "$env:CARGO_TARGET_DIR\debug\ci-smoketests.exe" run-archive ` --archive-file smoketest-nextest.tar.zst ` -- ` --partition hash:${{ matrix.partition }}/${{ env.PARTITION_COUNT }} @@ -562,7 +547,6 @@ jobs: runs-on: spacetimedb-windows-runner timeout-minutes: 30 env: - CARGO_TARGET_DIR: ${{ github.workspace }}/target RUST_BACKTRACE: full SPACETIMEDB_CPP_DIR: ${{ github.workspace }}/crates/bindings-cpp ARTIFACT_SUFFIX: windows @@ -596,11 +580,17 @@ jobs: runs-on: spacetimedb-new-runner-2 env: - CARGO_TARGET_DIR: ${{ github.workspace }}/target RUST_BACKTRACE: full ARTIFACT_SUFFIX: linux - SPACETIME_BIN: ${{ github.workspace }}/target/release/spacetimedb-cli steps: + - *set-cargo-target-dir + - *configure-sccache + + - &set-spacetime-bin + name: Set SpacetimeDB binary path + shell: bash + run: echo "SPACETIME_BIN=${CARGO_TARGET_DIR}/release/spacetimedb-cli" >>"${GITHUB_ENV}" + - name: Find Git ref env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -621,33 +611,6 @@ jobs: - uses: dsherret/rust-toolchain-file@v1 - name: Set default rust toolchain run: rustup default $(rustup show active-toolchain | cut -d' ' -f1) - - *set-native-cache-keys - - &restore-jemalloc - name: Restore jemalloc - id: cache-jemalloc - if: runner.os == 'Linux' - uses: actions/cache/restore@v4 - with: *jemalloc-cache - - *configure-cached-jemalloc - - &restore-zstd - name: Restore Zstd - id: cache-zstd - if: runner.os == 'Linux' - uses: actions/cache/restore@v4 - with: *zstd-cache - - *configure-cached-zstd - - &restore-rusty-v8-debug - name: Restore rusty_v8 (debug) - uses: actions/cache/restore@v4 - with: &v8-debug-cache - path: ${{ env.CARGO_TARGET_DIR }}/debug/gn_out/obj - key: ${{ steps.native-cache-keys.outputs.v8-debug-key }} - - &restore-rusty-v8 - name: Restore rusty_v8 - uses: actions/cache/restore@v4 - with: *v8-release-cache - - *restore-openssl - - *configure-cached-openssl - *download-build-artifacts - *extract-build-artifacts @@ -705,6 +668,8 @@ jobs: source ~/emsdk/emsdk_env.sh cargo ci test + - *show-sccache-stats + - name: Upload timing reports if: always() uses: actions/upload-artifact@v4 @@ -713,6 +678,8 @@ jobs: path: ${{ env.CARGO_TARGET_DIR }}/cargo-timings/ retention-days: 14 + - *show-sccache-stats + index_scan_bench: needs: [merge_queue_noop, lints] if: ${{ needs.merge_queue_noop.outputs.skip != 'true' }} @@ -723,9 +690,11 @@ jobs: queue: max timeout-minutes: 60 env: - CARGO_TARGET_DIR: ${{ github.workspace }}/target RUST_BACKTRACE: full steps: + - *set-cargo-target-dir + - *configure-sccache + - name: Find Git ref env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -747,27 +716,22 @@ jobs: - name: Set default rust toolchain run: rustup default $(rustup show active-toolchain | cut -d' ' -f1) - - *set-native-cache-keys - - *restore-jemalloc - - *configure-cached-jemalloc - - *restore-zstd - - *configure-cached-zstd - - *restore-rusty-v8 - - *restore-openssl - - *configure-cached-openssl - - name: Run index scan benchmark regression check run: cargo run --release -p spacetimedb-index-scan-gate + - *show-sccache-stats + lints: needs: [merge_queue_noop] if: ${{ needs.merge_queue_noop.outputs.skip != 'true' }} name: Lints runs-on: spacetimedb-new-runner-2 env: - CARGO_TARGET_DIR: ${{ github.workspace }}/target RUST_BACKTRACE: full steps: + - *set-cargo-target-dir + - *configure-sccache + - name: Checkout sources uses: actions/checkout@v3 @@ -776,17 +740,6 @@ jobs: run: rustup default $(rustup show active-toolchain | cut -d' ' -f1) - run: echo ::add-matcher::.github/workflows/rust_matcher.json - - *set-native-cache-keys - - *restore-jemalloc - - *configure-cached-jemalloc - - *restore-zstd - - *configure-cached-zstd - - name: Cache rusty_v8 (debug) - uses: actions/cache@v4 - with: *v8-debug-cache - - *restore-openssl - - *configure-cached-openssl - - uses: actions/setup-dotnet@v3 with: global-json-file: global.json @@ -803,6 +756,8 @@ jobs: - name: Run ci lint run: cargo ci lint + - *show-sccache-stats + - name: Upload timing reports if: always() uses: actions/upload-artifact@v4 @@ -811,6 +766,8 @@ jobs: path: ${{ env.CARGO_TARGET_DIR }}/cargo-timings/ retention-days: 14 + - *show-sccache-stats + codeowners_check: if: ${{ github.event_name == 'pull_request' }} name: CODEOWNERS check @@ -819,10 +776,12 @@ jobs: contents: read pull-requests: read env: - CARGO_TARGET_DIR: ${{ github.workspace }}/target RUST_BACKTRACE: full GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: + - *set-cargo-target-dir + - *configure-sccache + - uses: actions/checkout@v4 with: fetch-depth: 0 @@ -851,11 +810,13 @@ jobs: name: Build and test wasm bindings runs-on: spacetimedb-new-runner-2 env: - CARGO_TARGET_DIR: ${{ github.workspace }}/target RUST_BACKTRACE: full ARTIFACT_SUFFIX: linux - SPACETIME_BIN: ${{ github.workspace }}/target/release/spacetimedb-cli steps: + - *set-cargo-target-dir + - *configure-sccache + - *set-spacetime-bin + - *find-git-ref - *checkout-sources @@ -869,6 +830,8 @@ jobs: - name: Build module with latest compatible dependencies run: cargo ci module-latest-deps + - *show-sccache-stats + publish_checks: needs: [merge_queue_noop] if: ${{ needs.merge_queue_noop.outputs.skip != 'true' }} @@ -878,6 +841,9 @@ jobs: env: RUST_BACKTRACE: full steps: + - *set-cargo-target-dir + - *configure-sccache + - uses: actions/checkout@v3 - uses: dsherret/rust-toolchain-file@v1 - name: Set default rust toolchain @@ -890,7 +856,7 @@ jobs: uses: actions/upload-artifact@v4 with: name: cargo-timings-publish-checks - path: ${{ github.workspace }}/target/cargo-timings/ + path: ${{ env.CARGO_TARGET_DIR }}/cargo-timings/ retention-days: 14 update: @@ -908,18 +874,17 @@ jobs: env: RUST_BACKTRACE: full steps: - - name: Checkout - uses: actions/checkout@v3 + - *set-cargo-target-dir + - *configure-sccache + + - *find-git-ref + - *checkout-sources - name: Install Rust uses: dsherret/rust-toolchain-file@v1 - name: Set default rust toolchain run: rustup default $(rustup show active-toolchain | cut -d' ' -f1) - - *set-native-cache-keys - - *restore-zstd - - *configure-cached-zstd - - name: Install rust target run: rustup target add ${{ matrix.target }} @@ -928,12 +893,14 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: cargo ci update-flow --target=${{ matrix.target }} --github-token-auth + - *show-sccache-stats + - name: Upload timing reports if: always() uses: actions/upload-artifact@v4 with: name: cargo-timings-update-flow-${{ matrix.target }} - path: ${{ github.workspace }}/target/cargo-timings/ + path: ${{ env.CARGO_TARGET_DIR }}/cargo-timings/ retention-days: 14 # this is a no-op version of the above check with a trivially-passing body. @@ -980,6 +947,9 @@ jobs: env: RUST_BACKTRACE: full steps: + - *set-cargo-target-dir + - *configure-sccache + # Uncomment this before merging so that it will run properly if run manually through the GH actions flow. It was playing weird with rolled back # commits though. # - name: Find Git ref @@ -1055,9 +1025,11 @@ jobs: permissions: read-all runs-on: spacetimedb-new-runner-2 env: - CARGO_TARGET_DIR: ${{ github.workspace }}/target RUST_BACKTRACE: full steps: + - *set-cargo-target-dir + - *configure-sccache + - name: Find Git ref env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -1089,18 +1061,12 @@ jobs: - name: Set default rust toolchain run: rustup default $(rustup show active-toolchain | cut -d' ' -f1) - - *set-native-cache-keys - - *restore-jemalloc - - *configure-cached-jemalloc - - *restore-zstd - - *configure-cached-zstd - - *restore-openssl - - *configure-cached-openssl - - name: Check for docs change run: | cargo ci cli-docs + - *show-sccache-stats + - name: Upload timing reports if: always() uses: actions/upload-artifact@v4 @@ -1120,11 +1086,13 @@ jobs: runs-on: spacetimedb-unity-runner timeout-minutes: 30 env: - CARGO_TARGET_DIR: ${{ github.workspace }}/target RUST_BACKTRACE: full ARTIFACT_SUFFIX: linux UNITY_VERSION: 2022.3.32f1 steps: + - *set-cargo-target-dir + - *configure-sccache + - *find-git-ref - *checkout-sources @@ -1175,15 +1143,6 @@ jobs: - name: Set default rust toolchain run: rustup default $(rustup show active-toolchain | cut -d' ' -f1) - - *set-native-cache-keys - - *restore-jemalloc - - *configure-cached-jemalloc - - *restore-zstd - - *configure-cached-zstd - - *restore-rusty-v8 - - *restore-openssl - - *configure-cached-openssl - - *download-build-artifacts - *extract-build-artifacts @@ -1191,8 +1150,8 @@ jobs: name: Expose SpacetimeDB CLI on PATH shell: bash run: | - ln -sf spacetimedb-cli target/release/spacetime - echo "$GITHUB_WORKSPACE/target/release" >> "$GITHUB_PATH" + ln -sf spacetimedb-cli "${CARGO_TARGET_DIR}/release/spacetime" + echo "${CARGO_TARGET_DIR}/release" >>"${GITHUB_PATH}" - name: Generate client bindings working-directory: demo/Blackholio/server-rust @@ -1261,10 +1220,12 @@ jobs: contents: read runs-on: spacetimedb-new-runner-2 env: - CARGO_TARGET_DIR: ${{ github.workspace }}/target ARTIFACT_SUFFIX: linux UseLocalBsatnRuntime: true steps: + - *set-cargo-target-dir + - *configure-sccache + - *find-git-ref - *checkout-sources @@ -1303,15 +1264,6 @@ jobs: - name: Set default rust toolchain run: rustup default $(rustup show active-toolchain | cut -d' ' -f1) - - *set-native-cache-keys - - *restore-jemalloc - - *configure-cached-jemalloc - - *restore-zstd - - *configure-cached-zstd - - *restore-rusty-v8 - - *restore-openssl - - *configure-cached-openssl - - *download-build-artifacts - *extract-build-artifacts - *expose-spacetime-cli @@ -1363,17 +1315,21 @@ jobs: working-directory: demo/Blackholio/client-godot run: godot --headless --scene res://tests/GodotPlayModeTests.tscn + - *show-sccache-stats + csharp-testsuite: needs: [merge_queue_noop, lints, upload-build-artifacts-linux] if: ${{ needs.merge_queue_noop.outputs.skip != 'true' }} runs-on: spacetimedb-new-runner-2 timeout-minutes: 30 env: - CARGO_TARGET_DIR: ${{ github.workspace }}/target RUST_BACKTRACE: full ARTIFACT_SUFFIX: linux - SPACETIME_BIN: ${{ github.workspace }}/target/release/spacetimedb-cli steps: + - *set-cargo-target-dir + - *configure-sccache + - *set-spacetime-bin + - *find-git-ref - *checkout-sources @@ -1423,16 +1379,6 @@ jobs: - name: Set default rust toolchain run: rustup default $(rustup show active-toolchain | cut -d' ' -f1) - - *set-native-cache-keys - - *restore-jemalloc - - *configure-cached-jemalloc - - *restore-zstd - - *configure-cached-zstd - - *restore-rusty-v8-debug - - *restore-rusty-v8 - - *restore-openssl - - *configure-cached-openssl - - *download-build-artifacts - *extract-build-artifacts - *expose-spacetime-cli @@ -1485,16 +1431,21 @@ jobs: exit 1 } + - *show-sccache-stats + global_json_policy: needs: [merge_queue_noop] if: ${{ needs.merge_queue_noop.outputs.skip != 'true' }} name: Verify global.json files are symlinks - runs-on: ubuntu-latest + runs-on: spacetimedb-new-runner-2 permissions: contents: read env: RUST_BACKTRACE: full steps: + - *set-cargo-target-dir + - *configure-sccache + - name: Find Git ref env: PR_NUMBER: ${{ github.event.inputs.pr_number }} @@ -1523,19 +1474,22 @@ jobs: uses: actions/upload-artifact@v4 with: name: cargo-timings-global-json-policy - path: ${{ github.workspace }}/target/cargo-timings/ + path: ${{ env.CARGO_TARGET_DIR }}/cargo-timings/ retention-days: 14 smoketests_mod_rs_complete: needs: [merge_queue_noop] if: ${{ needs.merge_queue_noop.outputs.skip != 'true' }} name: Check smoketests/mod.rs is complete - runs-on: ubuntu-latest + runs-on: spacetimedb-new-runner-2 permissions: contents: read env: RUST_BACKTRACE: full steps: + - *set-cargo-target-dir + - *configure-sccache + - name: Find Git ref env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -1557,8 +1511,7 @@ jobs: run: rustup default $(rustup show active-toolchain | cut -d' ' -f1) - name: Verify smoketest module lists and suite constraints - run: | - cargo run -p ci-smoketest-checks + run: cargo ci smoketests check-mod-list docs-build: needs: [merge_queue_noop] @@ -1568,6 +1521,9 @@ jobs: env: RUST_BACKTRACE: full steps: + - *set-cargo-target-dir + - *configure-sccache + - name: Checkout repository uses: actions/checkout@v3 @@ -1592,20 +1548,24 @@ jobs: uses: actions/upload-artifact@v4 with: name: cargo-timings-docs-build - path: ${{ github.workspace }}/target/cargo-timings/ + path: ${{ env.CARGO_TARGET_DIR }}/cargo-timings/ retention-days: 14 + - *show-sccache-stats + typescript-test: needs: [merge_queue_noop, upload-build-artifacts-linux] if: ${{ needs.merge_queue_noop.outputs.skip != 'true' }} name: TypeScript - Tests runs-on: spacetimedb-new-runner-2 env: - CARGO_TARGET_DIR: ${{ github.workspace }}/target RUST_BACKTRACE: full ARTIFACT_SUFFIX: linux - SPACETIME_BIN: ${{ github.workspace }}/target/release/spacetimedb-cli steps: + - *set-cargo-target-dir + - *configure-sccache + - *set-spacetime-bin + - *find-git-ref - *checkout-sources @@ -1623,29 +1583,24 @@ jobs: - name: Set default rust toolchain run: rustup default $(rustup show active-toolchain | cut -d' ' -f1) - - *set-native-cache-keys - - *restore-jemalloc - - *configure-cached-jemalloc - - *restore-zstd - - *configure-cached-zstd - - *restore-rusty-v8-debug - - *restore-openssl - - *configure-cached-openssl - - *download-build-artifacts - *extract-build-artifacts - name: Run TypeScript tests run: cargo ci typescript-test + - *show-sccache-stats + - name: Upload timing reports if: always() uses: actions/upload-artifact@v4 with: name: cargo-timings-typescript-test - path: ${{ github.workspace }}/target/cargo-timings/ + path: ${{ env.CARGO_TARGET_DIR }}/cargo-timings/ retention-days: 14 + - *show-sccache-stats + # - name: Run quickstart-chat tests # working-directory: examples/quickstart-chat # run: pnpm test diff --git a/tools/ci/commands/smoketests/src/main.rs b/tools/ci/commands/smoketests/src/main.rs index 3dbf2087fe7..869a75532d2 100644 --- a/tools/ci/commands/smoketests/src/main.rs +++ b/tools/ci/commands/smoketests/src/main.rs @@ -144,8 +144,11 @@ fn run_binary_build(mut cmd: Command, failure_message: &str) -> Result<()> { // Remove cargo/rust env vars that could cause fingerprint mismatches // when the test later runs cargo build from a different environment for (key, _) in env::vars() { - let should_remove = (key.starts_with("CARGO") && key != "CARGO_HOME" && key != "CARGO_TARGET_DIR") - || key.starts_with("RUST") + let should_remove = (key.starts_with("CARGO") + && key != "CARGO_HOME" + && key != "CARGO_TARGET_DIR" + && key != "CARGO_INCREMENTAL") + || (key.starts_with("RUST") && key != "RUSTC_WRAPPER") // > The environment variable `__CARGO_FIX_YOLO` is an undocumented, internal-use-only feature // > for the Rust cargo fix command (and cargo clippy --fix) that forces the application of all // > available suggestions, including those that are marked as potentially incorrect or dangerous. diff --git a/tools/generate-client-api/src/main.rs b/tools/generate-client-api/src/main.rs index e9b794f93a2..440e06cf6b9 100644 --- a/tools/generate-client-api/src/main.rs +++ b/tools/generate-client-api/src/main.rs @@ -3,7 +3,7 @@ use anyhow::{anyhow, Context, Result}; use replace_spacetimedb::{replace_in_tree, ReplaceOptions}; use std::ffi::OsStr; use std::fs; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; use tempfile::NamedTempFile; @@ -25,6 +25,14 @@ fn run_inherit(cmd: impl AsRef, args: &[&str], cwd: Option<&Path>) -> Res Ok(()) } +fn cargo_target_dir(workspace_dir: &Path) -> PathBuf { + match std::env::var_os("CARGO_TARGET_DIR").map(PathBuf::from) { + Some(path) if path.is_absolute() => path, + Some(path) => workspace_dir.join(path), + None => workspace_dir.join("target"), + } +} + /// Run a command and return captured stdout as UTF-8 string. fn run_capture(cmd: &str, args: &[&str]) -> Result { let out = Command::new(cmd) @@ -75,8 +83,8 @@ fn main() -> Result<()> { // 4) Generate TS client run_inherit( - workspace_dir - .join("target/debug/spacetimedb-cli") + cargo_target_dir(workspace_dir) + .join("debug/spacetimedb-cli") .with_extension(std::env::consts::EXE_EXTENSION), &[ "generate",