diff --git a/.github/workflows/c-chain-reexecution-benchmark-container.yml b/.github/workflows/c-chain-reexecution-benchmark-container.yml index 522dae474129..7cfee6261a1a 100644 --- a/.github/workflows/c-chain-reexecution-benchmark-container.yml +++ b/.github/workflows/c-chain-reexecution-benchmark-container.yml @@ -1,7 +1,7 @@ name: C-Chain Re-Execution Benchmark w/ Container on: - pull_request: +# pull_request: workflow_dispatch: inputs: config: diff --git a/.github/workflows/c-chain-reexecution-benchmark-gh-native.yml b/.github/workflows/c-chain-reexecution-benchmark-gh-native.yml index 1688567b9e0d..30b83103fc99 100644 --- a/.github/workflows/c-chain-reexecution-benchmark-gh-native.yml +++ b/.github/workflows/c-chain-reexecution-benchmark-gh-native.yml @@ -1,7 +1,7 @@ name: C-Chain Re-Execution Benchmark GH Native on: - pull_request: +# pull_request: workflow_dispatch: inputs: config: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3ddcfda54dc2..e5044c78f345 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,238 +19,446 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +env: + RUST_VERSION: "1.88" + # Hardcoded versions for testing + FIREWOOD_BASELINE_VERSION: "ffi/v0.0.8" + AVALANCHEGO_BASELINE_VERSION: "92d0e99bce8fc3f7af1fa7d300e4fe1e7d210e6b" # Version of AvGO that works with v0.0.8 + FIREWOOD_CANDIDATE_VERSION: "e135deeebfd46da0cf976ccfcb4fc3a303c5b71b" + AVALANCHEGO_CANDIDATE_VERSION: "master" + START_BLOCK: "101" + END_BLOCK: "200" + jobs: - Unit: - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [macos-14, ubuntu-22.04, ubuntu-24.04, custom-arm64-jammy, custom-arm64-noble] + # Unit: + # runs-on: ${{ matrix.os }} + # strategy: + # fail-fast: false + # matrix: + # os: [macos-14, ubuntu-22.04, ubuntu-24.04, custom-arm64-jammy, custom-arm64-noble] + # steps: + # - uses: actions/checkout@v4 + # - uses: ./.github/actions/setup-go-for-project + # - name: test-unit + # shell: bash + # run: ./scripts/run_task.sh test-unit + # env: + # TIMEOUT: ${{ env.TIMEOUT }} + # Fuzz: + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v4 + # - uses: ./.github/actions/setup-go-for-project + # - name: test-fuzz + # shell: bash + # run: ./scripts/run_task.sh test-fuzz + # e2e: + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v4 + # - uses: ./.github/actions/setup-go-for-project + # - name: Run e2e tests + # uses: ./.github/actions/run-monitored-tmpnet-cmd + # with: + # run: ./scripts/run_task.sh test-e2e-ci + # artifact_prefix: e2e + # filter_by_owner: avalanchego-e2e + # prometheus_username: ${{ secrets.PROMETHEUS_ID || '' }} + # prometheus_password: ${{ secrets.PROMETHEUS_PASSWORD || '' }} + # loki_username: ${{ secrets.LOKI_ID || '' }} + # loki_password: ${{ secrets.LOKI_PASSWORD || '' }} + # e2e_post_granite: + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v4 + # - uses: ./.github/actions/setup-go-for-project + # - name: Run e2e tests + # uses: ./.github/actions/run-monitored-tmpnet-cmd + # with: + # run: ./scripts/run_task.sh test-e2e-ci -- --activate-granite + # artifact_prefix: e2e-post-granite + # filter_by_owner: avalanchego-e2e + # prometheus_username: ${{ secrets.PROMETHEUS_ID || '' }} + # prometheus_password: ${{ secrets.PROMETHEUS_PASSWORD || '' }} + # loki_username: ${{ secrets.LOKI_ID || '' }} + # loki_password: ${{ secrets.LOKI_PASSWORD || '' }} + # e2e_kube: + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v4 + # - uses: ./.github/actions/setup-go-for-project + # - uses: ./.github/actions/run-monitored-tmpnet-cmd + # with: + # run: ./scripts/run_task.sh test-e2e-kube-ci + # runtime: kube + # artifact_prefix: e2e-kube + # filter_by_owner: avalanchego-e2e + # prometheus_username: ${{ secrets.PROMETHEUS_ID || '' }} + # prometheus_password: ${{ secrets.PROMETHEUS_PASSWORD || '' }} + # loki_username: ${{ secrets.LOKI_ID || '' }} + # loki_password: ${{ secrets.LOKI_PASSWORD || '' }} + # e2e_existing_network: + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v4 + # - uses: ./.github/actions/setup-go-for-project + # - name: Run e2e tests with existing network + # uses: ./.github/actions/run-monitored-tmpnet-cmd + # with: + # run: ./scripts/run_task.sh test-e2e-existing-ci + # artifact_prefix: e2e-existing-network + # prometheus_username: ${{ secrets.PROMETHEUS_ID || '' }} + # prometheus_password: ${{ secrets.PROMETHEUS_PASSWORD || '' }} + # loki_username: ${{ secrets.LOKI_ID || '' }} + # loki_password: ${{ secrets.LOKI_PASSWORD || '' }} + # Upgrade: + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v4 + # - uses: ./.github/actions/setup-go-for-project + # - name: Run e2e tests + # uses: ./.github/actions/run-monitored-tmpnet-cmd + # with: + # run: ./scripts/run_task.sh test-upgrade + # artifact_prefix: upgrade + # prometheus_username: ${{ secrets.PROMETHEUS_ID || '' }} + # prometheus_password: ${{ secrets.PROMETHEUS_PASSWORD || '' }} + # loki_username: ${{ secrets.LOKI_ID || '' }} + # loki_password: ${{ secrets.LOKI_PASSWORD || '' }} + # Lint: + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v4 + # - uses: ./.github/actions/setup-go-for-project + # - uses: ./.github/actions/install-nix + # - name: Runs all lint checks + # shell: nix develop --command bash -x {0} + # run: ./scripts/run_task.sh lint-all-ci + # links-lint: + # name: Markdown Links Lint + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v4 + # - uses: umbrelladocs/action-linkspector@de84085e0f51452a470558693d7d308fbb2fa261 #v1.2.5 + # with: + # fail_level: any + # check_generated_protobuf: + # name: Up-to-date protobuf + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v4 + # - uses: ./.github/actions/setup-go-for-project + # # Use the dev shell instead of bufbuild/buf-action to ensure the dev shell provides the expected versions + # - uses: ./.github/actions/install-nix + # - shell: nix develop --command bash -x {0} + # run: ./scripts/run_task.sh check-generate-protobuf + # check_mockgen: + # name: Up-to-date mocks + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v4 + # - uses: ./.github/actions/setup-go-for-project + # - shell: bash + # run: ./scripts/run_task.sh check-generate-mocks + # check_canotogen: + # name: Up-to-date canoto + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v4 + # - uses: ./.github/actions/setup-go-for-project + # - shell: bash + # run: ./scripts/run_task.sh check-generate-canoto + # check_contract_bindings: + # name: Up-to-date contract bindings + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v4 + # - uses: ./.github/actions/setup-go-for-project + # - uses: ./.github/actions/install-nix + # - shell: nix develop --command bash -x {0} + # run: task check-generate-load-contract-bindings + # go_mod_tidy: + # name: Up-to-date go.mod and go.sum + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v4 + # - uses: ./.github/actions/setup-go-for-project + # - shell: bash + # run: ./scripts/run_task.sh check-go-mod-tidy + # test_build_image: + # name: Image build + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v4 + # - name: Install qemu (required for cross-platform builds) + # run: | + # sudo apt update + # sudo apt -y install qemu-system qemu-user-static + # - name: Check image build + # shell: bash + # run: ./scripts/run_task.sh test-build-image + # test_build_antithesis_avalanchego_images: + # name: Build Antithesis avalanchego images + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v4 + # - uses: ./.github/actions/setup-go-for-project + # - name: Check image build for avalanchego test setup + # shell: bash + # run: ./scripts/run_task.sh test-build-antithesis-images-avalanchego + # test_build_antithesis_xsvm_images: + # name: Build Antithesis xsvm images + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v4 + # - uses: ./.github/actions/setup-go-for-project + # - name: Check image build for xsvm test setup + # shell: bash + # run: ./scripts/run_task.sh test-build-antithesis-images-xsvm + # e2e_bootstrap_monitor: + # name: Run bootstrap monitor e2e tests + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v4 + # - uses: ./.github/actions/setup-go-for-project + # - uses: ./.github/actions/install-nix + # - name: Run e2e tests + # shell: bash + # run: nix develop --command ./scripts/run_task.sh test-bootstrap-monitor-e2e + # load: + # name: Run process-based load test + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v4 + # - uses: ./.github/actions/setup-go-for-project + # - uses: ./.github/actions/run-monitored-tmpnet-cmd + # with: + # run: ./scripts/run_task.sh test-load -- --load-timeout=30s + # artifact_prefix: load + # prometheus_username: ${{ secrets.PROMETHEUS_ID || '' }} + # prometheus_password: ${{ secrets.PROMETHEUS_PASSWORD || '' }} + # loki_username: ${{ secrets.LOKI_ID || '' }} + # loki_password: ${{ secrets.LOKI_PASSWORD || '' }} + # load_kube_kind: + # name: Run load test on kind cluster + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v4 + # - uses: ./.github/actions/setup-go-for-project + # - uses: ./.github/actions/run-monitored-tmpnet-cmd + # with: + # run: ./scripts/run_task.sh test-load-kube-kind -- --load-timeout=30s + # runtime: kube + # artifact_prefix: load-kube + # prometheus_username: ${{ secrets.PROMETHEUS_ID || '' }} + # prometheus_password: ${{ secrets.PROMETHEUS_PASSWORD || '' }} + # loki_username: ${{ secrets.LOKI_ID || '' }} + # loki_password: ${{ secrets.LOKI_PASSWORD || '' }} + # robustness: + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v4 + # - uses: ./.github/actions/setup-go-for-project + # - uses: ./.github/actions/install-nix + # # TODO(marun) Extend testing of robustness beyond deploying a suitable test environment + # - name: Deploy kind with chaos mesh + # shell: bash + # run: nix develop --command ./scripts/run_task.sh test-robustness + ReexecutionFirewoodAB-Baseline: + runs-on: avalanche-avalanchego-runner + container: + image: ghcr.io/actions/actions-runner:2.325.0 + permissions: + id-token: write + contents: write + name: "Baseline: ${{ env.FIREWOOD_BASELINE_VERSION }}" steps: - - uses: actions/checkout@v4 - - uses: ./.github/actions/setup-go-for-project - - name: test-unit + - name: Install dependencies shell: bash - run: ./scripts/run_task.sh test-unit - env: - TIMEOUT: ${{ env.TIMEOUT }} - Fuzz: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: ./.github/actions/setup-go-for-project - - name: test-fuzz - shell: bash - run: ./scripts/run_task.sh test-fuzz - e2e: - runs-on: ubuntu-latest - steps: + run: | + if ! command -v xz &> /dev/null; then + sudo apt-get update + sudo apt-get install -y xz-utils + fi - uses: actions/checkout@v4 - - uses: ./.github/actions/setup-go-for-project - - name: Run e2e tests - uses: ./.github/actions/run-monitored-tmpnet-cmd with: - run: ./scripts/run_task.sh test-e2e-ci - artifact_prefix: e2e - filter_by_owner: avalanchego-e2e - prometheus_username: ${{ secrets.PROMETHEUS_ID || '' }} - prometheus_password: ${{ secrets.PROMETHEUS_PASSWORD || '' }} - loki_username: ${{ secrets.LOKI_ID || '' }} - loki_password: ${{ secrets.LOKI_PASSWORD || '' }} - e2e_post_granite: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 + ref: ${{ env.AVALANCHEGO_BASELINE_VERSION }} - uses: ./.github/actions/setup-go-for-project - - name: Run e2e tests - uses: ./.github/actions/run-monitored-tmpnet-cmd + - uses: ./.github/actions/install-nix + # Only install Rust toolchain when building from source (non-tag versions) + - name: Setup Rust for source build + if: ${{ !startsWith(env.FIREWOOD_BASELINE_VERSION, 'v') && !startsWith(env.FIREWOOD_BASELINE_VERSION, 'ffi/') }} + uses: dtolnay/rust-toolchain@stable with: - run: ./scripts/run_task.sh test-e2e-ci -- --activate-granite - artifact_prefix: e2e-post-granite - filter_by_owner: avalanchego-e2e - prometheus_username: ${{ secrets.PROMETHEUS_ID || '' }} - prometheus_password: ${{ secrets.PROMETHEUS_PASSWORD || '' }} - loki_username: ${{ secrets.LOKI_ID || '' }} - loki_password: ${{ secrets.LOKI_PASSWORD || '' }} - e2e_kube: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: ./.github/actions/setup-go-for-project - - uses: ./.github/actions/run-monitored-tmpnet-cmd + toolchain: ${{ env.RUST_VERSION }} + - name: Setup Rust cache for source build + if: ${{ !startsWith(env.FIREWOOD_BASELINE_VERSION, 'v') && !startsWith(env.FIREWOOD_BASELINE_VERSION, 'ffi/') }} + uses: Swatinem/rust-cache@v2 + - name: Setup Firewood Baseline + shell: bash + run: | + FIREWOOD_VERSION="${{ env.FIREWOOD_BASELINE_VERSION }}" + + if [[ "$FIREWOOD_VERSION" =~ ^(v|ffi/) ]]; then + echo "Detected tag: $FIREWOOD_VERSION - using pre-built binaries" + # For tags, fetch pre-built binaries from firewood-go-ethhash repo + TEMP_DIR=$(mktemp -d) + trap "rm -rf $TEMP_DIR" EXIT + + cd "$TEMP_DIR" + git clone https://github.com/ava-labs/firewood-go-ethhash.git + cd firewood-go-ethhash + git checkout "$FIREWOOD_VERSION" + + cd "$GITHUB_WORKSPACE" + rm -rf ./ffi + cp -r "$TEMP_DIR/firewood-go-ethhash/ffi" ./ffi + else + echo "Detected commit: $FIREWOOD_VERSION - building from source" + ./scripts/build_firewood.sh "$FIREWOOD_VERSION" ./ffi + fi + + # Configure go.mod to use local FFI + go mod edit -replace github.com/ava-labs/firewood-go-ethhash/ffi=./ffi + go mod download + - name: Run Baseline Benchmark + uses: ./.github/actions/c-chain-reexecution-benchmark with: - run: ./scripts/run_task.sh test-e2e-kube-ci - runtime: kube - artifact_prefix: e2e-kube - filter_by_owner: avalanchego-e2e - prometheus_username: ${{ secrets.PROMETHEUS_ID || '' }} - prometheus_password: ${{ secrets.PROMETHEUS_PASSWORD || '' }} - loki_username: ${{ secrets.LOKI_ID || '' }} - loki_password: ${{ secrets.LOKI_PASSWORD || '' }} - e2e_existing_network: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: ./.github/actions/setup-go-for-project - - name: Run e2e tests with existing network - uses: ./.github/actions/run-monitored-tmpnet-cmd + config: firewood + start-block: ${{ env.START_BLOCK }} + end-block: ${{ env.END_BLOCK }} + prometheus-username: ${{ secrets.PROMETHEUS_ID || '' }} + prometheus-password: ${{ secrets.PROMETHEUS_PASSWORD || '' }} + push-github-action-benchmark: false + aws-role: ${{ secrets.AWS_S3_READ_ONLY_ROLE }} + aws-region: us-east-2 + github-token: ${{ secrets.GITHUB_TOKEN }} + push-post-state: '' + runner_name: 'baseline-${{ env.FIREWOOD_BASELINE_VERSION }}' + - name: Upload baseline results + uses: actions/upload-artifact@v4 with: - run: ./scripts/run_task.sh test-e2e-existing-ci - artifact_prefix: e2e-existing-network - prometheus_username: ${{ secrets.PROMETHEUS_ID || '' }} - prometheus_password: ${{ secrets.PROMETHEUS_PASSWORD || '' }} - loki_username: ${{ secrets.LOKI_ID || '' }} - loki_password: ${{ secrets.LOKI_PASSWORD || '' }} - Upgrade: - runs-on: ubuntu-latest + name: baseline-benchmark-results + path: | + output.txt + benchmark-results/ + retention-days: 1 + ReexecutionFirewoodAB-Candidate: + runs-on: avalanche-avalanchego-runner + container: + image: ghcr.io/actions/actions-runner:2.325.0 + permissions: + id-token: write + contents: write + name: "Candidate: ${{ env.FIREWOOD_CANDIDATE_VERSION }}" steps: + - name: Install dependencies + shell: bash + run: | + if ! command -v xz &> /dev/null; then + sudo apt-get update + sudo apt-get install -y xz-utils + fi - uses: actions/checkout@v4 - - uses: ./.github/actions/setup-go-for-project - - name: Run e2e tests - uses: ./.github/actions/run-monitored-tmpnet-cmd with: - run: ./scripts/run_task.sh test-upgrade - artifact_prefix: upgrade - prometheus_username: ${{ secrets.PROMETHEUS_ID || '' }} - prometheus_password: ${{ secrets.PROMETHEUS_PASSWORD || '' }} - loki_username: ${{ secrets.LOKI_ID || '' }} - loki_password: ${{ secrets.LOKI_PASSWORD || '' }} - Lint: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 + ref: ${{ env.AVALANCHEGO_CANDIDATE_VERSION }} - uses: ./.github/actions/setup-go-for-project - uses: ./.github/actions/install-nix - - name: Runs all lint checks - shell: nix develop --command bash -x {0} - run: ./scripts/run_task.sh lint-all-ci - links-lint: - name: Markdown Links Lint - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: umbrelladocs/action-linkspector@de84085e0f51452a470558693d7d308fbb2fa261 #v1.2.5 + # Only install Rust toolchain when building from source (non-tag versions) + - name: Setup Rust for source build + if: ${{ !startsWith(env.FIREWOOD_CANDIDATE_VERSION, 'v') && !startsWith(env.FIREWOOD_CANDIDATE_VERSION, 'ffi/') }} + uses: dtolnay/rust-toolchain@stable with: - fail_level: any - check_generated_protobuf: - name: Up-to-date protobuf - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: ./.github/actions/setup-go-for-project - # Use the dev shell instead of bufbuild/buf-action to ensure the dev shell provides the expected versions - - uses: ./.github/actions/install-nix - - shell: nix develop --command bash -x {0} - run: ./scripts/run_task.sh check-generate-protobuf - check_mockgen: - name: Up-to-date mocks - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: ./.github/actions/setup-go-for-project - - shell: bash - run: ./scripts/run_task.sh check-generate-mocks - check_canotogen: - name: Up-to-date canoto - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: ./.github/actions/setup-go-for-project - - shell: bash - run: ./scripts/run_task.sh check-generate-canoto - check_contract_bindings: - name: Up-to-date contract bindings - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: ./.github/actions/setup-go-for-project - - uses: ./.github/actions/install-nix - - shell: nix develop --command bash -x {0} - run: task check-generate-load-contract-bindings - go_mod_tidy: - name: Up-to-date go.mod and go.sum - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: ./.github/actions/setup-go-for-project - - shell: bash - run: ./scripts/run_task.sh check-go-mod-tidy - test_build_image: - name: Image build - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Install qemu (required for cross-platform builds) - run: | - sudo apt update - sudo apt -y install qemu-system qemu-user-static - - name: Check image build - shell: bash - run: ./scripts/run_task.sh test-build-image - test_build_antithesis_avalanchego_images: - name: Build Antithesis avalanchego images - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: ./.github/actions/setup-go-for-project - - name: Check image build for avalanchego test setup - shell: bash - run: ./scripts/run_task.sh test-build-antithesis-images-avalanchego - test_build_antithesis_xsvm_images: - name: Build Antithesis xsvm images - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: ./.github/actions/setup-go-for-project - - name: Check image build for xsvm test setup - shell: bash - run: ./scripts/run_task.sh test-build-antithesis-images-xsvm - e2e_bootstrap_monitor: - name: Run bootstrap monitor e2e tests - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: ./.github/actions/setup-go-for-project - - uses: ./.github/actions/install-nix - - name: Run e2e tests + toolchain: ${{ env.RUST_VERSION }} + - name: Setup Rust cache for source build + if: ${{ !startsWith(env.FIREWOOD_CANDIDATE_VERSION, 'v') && !startsWith(env.FIREWOOD_CANDIDATE_VERSION, 'ffi/') }} + uses: Swatinem/rust-cache@v2 + - name: Setup Firewood Candidate shell: bash - run: nix develop --command ./scripts/run_task.sh test-bootstrap-monitor-e2e - load: - name: Run process-based load test - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: ./.github/actions/setup-go-for-project - - uses: ./.github/actions/run-monitored-tmpnet-cmd + run: | + FIREWOOD_VERSION="${{ env.FIREWOOD_CANDIDATE_VERSION }}" + + if [[ "$FIREWOOD_VERSION" =~ ^(v|ffi/) ]]; then + echo "Detected tag: $FIREWOOD_VERSION - using pre-built binaries" + # For tags, fetch pre-built binaries from firewood-go-ethhash repo + TEMP_DIR=$(mktemp -d) + trap "rm -rf $TEMP_DIR" EXIT + + cd "$TEMP_DIR" + git clone https://github.com/ava-labs/firewood-go-ethhash.git + cd firewood-go-ethhash + git checkout "$FIREWOOD_VERSION" + + cd "$GITHUB_WORKSPACE" + rm -rf ./ffi + cp -r "$TEMP_DIR/firewood-go-ethhash/ffi" ./ffi + else + echo "Detected commit: $FIREWOOD_VERSION - building from source" + ./scripts/build_firewood.sh "$FIREWOOD_VERSION" ./ffi + fi + + # Configure go.mod to use local FFI + go mod edit -replace github.com/ava-labs/firewood-go-ethhash/ffi=./ffi + go mod download + - name: Run Candidate Benchmark + uses: ./.github/actions/c-chain-reexecution-benchmark with: - run: ./scripts/run_task.sh test-load -- --load-timeout=30s - artifact_prefix: load - prometheus_username: ${{ secrets.PROMETHEUS_ID || '' }} - prometheus_password: ${{ secrets.PROMETHEUS_PASSWORD || '' }} - loki_username: ${{ secrets.LOKI_ID || '' }} - loki_password: ${{ secrets.LOKI_PASSWORD || '' }} - load_kube_kind: - name: Run load test on kind cluster - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: ./.github/actions/setup-go-for-project - - uses: ./.github/actions/run-monitored-tmpnet-cmd + config: firewood + start-block: ${{ env.START_BLOCK }} + end-block: ${{ env.END_BLOCK }} + prometheus-username: ${{ secrets.PROMETHEUS_ID || '' }} + prometheus-password: ${{ secrets.PROMETHEUS_PASSWORD || '' }} + push-github-action-benchmark: false + aws-role: ${{ secrets.AWS_S3_READ_ONLY_ROLE }} + aws-region: us-east-2 + github-token: ${{ secrets.GITHUB_TOKEN }} + push-post-state: '' + runner_name: 'candidate-${{ env.FIREWOOD_CANDIDATE_VERSION }}' + - name: Upload candidate results + uses: actions/upload-artifact@v4 with: - run: ./scripts/run_task.sh test-load-kube-kind -- --load-timeout=30s - runtime: kube - artifact_prefix: load-kube - prometheus_username: ${{ secrets.PROMETHEUS_ID || '' }} - prometheus_password: ${{ secrets.PROMETHEUS_PASSWORD || '' }} - loki_username: ${{ secrets.LOKI_ID || '' }} - loki_password: ${{ secrets.LOKI_PASSWORD || '' }} - robustness: + name: candidate-benchmark-results + path: | + output.txt + benchmark-results/ + retention-days: 1 + compare-results: + needs: [ ReexecutionFirewoodAB-Baseline, ReexecutionFirewoodAB-Candidate ] runs-on: ubuntu-latest + if: always() steps: - - uses: actions/checkout@v4 - - uses: ./.github/actions/setup-go-for-project - - uses: ./.github/actions/install-nix - # TODO(marun) Extend testing of robustness beyond deploying a suitable test environment - - name: Deploy kind with chaos mesh + - name: Download baseline results + uses: actions/download-artifact@v4 + with: + name: baseline-benchmark-results + path: ./baseline-results + continue-on-error: true + - name: Download candidate results + uses: actions/download-artifact@v4 + with: + name: candidate-benchmark-results + path: ./candidate-results + continue-on-error: true + - name: Compare Results shell: bash - run: nix develop --command ./scripts/run_task.sh test-robustness + run: | + echo "# Firewood Performance Comparison (Test)" >> $GITHUB_STEP_SUMMARY + echo "**Baseline:** ${{ env.FIREWOOD_BASELINE_VERSION }}" >> $GITHUB_STEP_SUMMARY + echo "**Candidate:** ${{ env.FIREWOOD_CANDIDATE_VERSION }}" >> $GITHUB_STEP_SUMMARY + echo "**Blocks:** ${{ env.START_BLOCK }} - ${{ env.END_BLOCK }}" >> $GITHUB_STEP_SUMMARY + + if [ -f "./baseline-results/output.txt" ] && [ -f "./candidate-results/output.txt" ]; then + BASELINE_MGAS=$(grep -o '[0-9]\+\.[0-9]\+ mgas/s' ./baseline-results/output.txt | head -1 | cut -d' ' -f1 || echo "N/A") + CANDIDATE_MGAS=$(grep -o '[0-9]\+\.[0-9]\+ mgas/s' ./candidate-results/output.txt | head -1 | cut -d' ' -f1 || echo "N/A") + + echo "- Baseline: ${BASELINE_MGAS} mgas/s" >> $GITHUB_STEP_SUMMARY + echo "- Candidate: ${CANDIDATE_MGAS} mgas/s" >> $GITHUB_STEP_SUMMARY + + if [ "$BASELINE_MGAS" != "N/A" ] && [ "$CANDIDATE_MGAS" != "N/A" ]; then + CHANGE=$(echo "scale=2; ($CANDIDATE_MGAS - $BASELINE_MGAS) / $BASELINE_MGAS * 100" | bc -l) + echo "- Change: ${CHANGE}%" >> $GITHUB_STEP_SUMMARY + fi + else + [ -f "./baseline-results/output.txt" ] && echo "- Baseline: Completed" >> $GITHUB_STEP_SUMMARY || echo "- Baseline: Failed" >> $GITHUB_STEP_SUMMARY + [ -f "./candidate-results/output.txt" ] && echo "- Candidate: Completed" >> $GITHUB_STEP_SUMMARY || echo "- Candidate: Failed" >> $GITHUB_STEP_SUMMARY + fi diff --git a/.github/workflows/l.yml b/.github/workflows/l.yml new file mode 100644 index 000000000000..1939c042b6fb --- /dev/null +++ b/.github/workflows/l.yml @@ -0,0 +1,235 @@ +# Firewood Comparison Benchmark +# +# Compares performance between two Firewood versions by running C-Chain re-execution benchmarks. +# Supports both tagged releases (uses pre-built binaries when available) and source builds. +# Each version runs with its specified AvalancheGo commit for compatibility. + +name: Firewood Comparison Benchmark + +env: + RUST_VERSION: "1.88" + +on: + workflow_dispatch: + inputs: + firewood-baseline-version: + description: 'Baseline Firewood version (tag like ffi/v0.0.12 or commit hash)' + required: true + avalanchego-baseline-version: + description: 'AvalancheGo version (tag like v1.13.0 or commit hash)' + required: true + default: 'master' + firewood-candidate-version: + description: 'Candidate Firewood version (tag or commit hash)' + required: true + avalanchego-candidate-version: + description: 'AvalancheGo version (tag like v1.13.0 or commit hash)' + required: true + start-block: + description: 'Start block for benchmark' + required: false + default: '101' + end-block: + description: 'End block for benchmark' + required: false + default: '200' + +jobs: + baseline-benchmark: + runs-on: avalanche-avalanchego-runner + container: + image: ghcr.io/actions/actions-runner:2.325.0 + permissions: + id-token: write + contents: write + name: "Baseline: ${{ github.event.inputs.firewood-baseline-version }}" + steps: + - name: Install dependencies + shell: bash + run: | + if ! command -v xz &> /dev/null; then + sudo apt-get update + sudo apt-get install -y xz-utils + fi + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.inputs.avalanchego-baseline-version }} + - uses: ./.github/actions/setup-go-for-project + - uses: ./.github/actions/install-nix + # Only install Rust toolchain when building from source (non-tag versions) + - name: Setup Rust for source build + if: ${{ !startsWith(github.event.inputs.firewood-baseline-version, 'v') && !startsWith(github.event.inputs.firewood-baseline-version, 'ffi/') }} + uses: dtolnay/rust-toolchain@stable + with: + toolchain: ${{ env.RUST_VERSION }} + - name: Setup Rust cache for source build + if: ${{ !startsWith(github.event.inputs.firewood-baseline-version, 'v') && !startsWith(github.event.inputs.firewood-baseline-version, 'ffi/') }} + uses: Swatinem/rust-cache@v2 + - name: Setup Firewood Baseline + shell: bash + run: | + FIREWOOD_VERSION="${{ github.event.inputs.firewood-baseline-version }}" + + if [[ "$FIREWOOD_VERSION" =~ ^(v|ffi/) ]]; then + echo "Detected tag: $FIREWOOD_VERSION - using pre-built binaries" + # For tags, fetch pre-built binaries from firewood-go-ethhash repo + TEMP_DIR=$(mktemp -d) + trap "rm -rf $TEMP_DIR" EXIT + + cd "$TEMP_DIR" + git clone https://github.com/ava-labs/firewood-go-ethhash.git + cd firewood-go-ethhash + git checkout "$FIREWOOD_VERSION" + + cd "$GITHUB_WORKSPACE" + rm -rf ./ffi + cp -r "$TEMP_DIR/firewood-go-ethhash/ffi" ./ffi + else + echo "Detected commit: $FIREWOOD_VERSION - building from source" + ./scripts/build_firewood.sh "$FIREWOOD_VERSION" ./ffi + fi + + # Configure go.mod to use local FFI + go mod edit -replace github.com/ava-labs/firewood-go-ethhash/ffi=./ffi + go mod download + - name: Run Baseline Benchmark + uses: ./.github/actions/c-chain-reexecution-benchmark + with: + config: firewood + start-block: ${{ github.event.inputs.start-block }} + end-block: ${{ github.event.inputs.end-block }} + prometheus-username: ${{ secrets.PROMETHEUS_ID || '' }} + prometheus-password: ${{ secrets.PROMETHEUS_PASSWORD || '' }} + push-github-action-benchmark: false + aws-role: ${{ secrets.AWS_S3_READ_ONLY_ROLE }} + aws-region: us-east-2 + github-token: ${{ secrets.GITHUB_TOKEN }} + push-post-state: '' + runner_name: 'baseline-${{ github.event.inputs.firewood-baseline-version }}' + - name: Upload baseline results + uses: actions/upload-artifact@v4 + with: + name: baseline-benchmark-results + path: | + output.txt + benchmark-results/ + retention-days: 7 + candidate-benchmark: + runs-on: avalanche-avalanchego-runner + container: + image: ghcr.io/actions/actions-runner:2.325.0 + permissions: + id-token: write + contents: write + name: "Candidate: ${{ github.event.inputs.firewood-candidate-version }}" + steps: + - name: Install dependencies + shell: bash + run: | + if ! command -v xz &> /dev/null; then + sudo apt-get update + sudo apt-get install -y xz-utils + fi + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.inputs.avalanchego-candidate-version }} + - uses: ./.github/actions/setup-go-for-project + - uses: ./.github/actions/install-nix + # Only install Rust toolchain when building from source (non-tag versions) + - name: Setup Rust for source build + if: ${{ !startsWith(github.event.inputs.firewood-candidate-version, 'v') && !startsWith(github.event.inputs.firewood-candidate-version, 'ffi/') }} + uses: dtolnay/rust-toolchain@stable + with: + toolchain: ${{ env.RUST_VERSION }} + - name: Setup Rust cache for source build + if: ${{ !startsWith(github.event.inputs.firewood-candidate-version, 'v') && !startsWith(github.event.inputs.firewood-candidate-version, 'ffi/') }} + uses: Swatinem/rust-cache@v2 + - name: Setup Firewood Candidate + shell: bash + run: | + FIREWOOD_VERSION="${{ github.event.inputs.firewood-candidate-version }}" + + if [[ "$FIREWOOD_VERSION" =~ ^(v|ffi/) ]]; then + echo "Detected tag: $FIREWOOD_VERSION - using pre-built binaries" + # For tags, fetch pre-built binaries from firewood-go-ethhash repo + TEMP_DIR=$(mktemp -d) + trap "rm -rf $TEMP_DIR" EXIT + + cd "$TEMP_DIR" + git clone https://github.com/ava-labs/firewood-go-ethhash.git + cd firewood-go-ethhash + git checkout "$FIREWOOD_VERSION" + + cd "$GITHUB_WORKSPACE" + rm -rf ./ffi + cp -r "$TEMP_DIR/firewood-go-ethhash/ffi" ./ffi + else + echo "Detected commit: $FIREWOOD_VERSION - building from source" + ./scripts/build_firewood.sh "$FIREWOOD_VERSION" ./ffi + fi + + # Configure go.mod to use local FFI + go mod edit -replace github.com/ava-labs/firewood-go-ethhash/ffi=./ffi + go mod download + - name: Run Candidate Benchmark + uses: ./.github/actions/c-chain-reexecution-benchmark + with: + config: firewood + start-block: ${{ github.event.inputs.start-block }} + end-block: ${{ github.event.inputs.end-block }} + prometheus-username: ${{ secrets.PROMETHEUS_ID || '' }} + prometheus-password: ${{ secrets.PROMETHEUS_PASSWORD || '' }} + push-github-action-benchmark: false + aws-role: ${{ secrets.AWS_S3_READ_ONLY_ROLE }} + aws-region: us-east-2 + github-token: ${{ secrets.GITHUB_TOKEN }} + push-post-state: '' + runner_name: 'candidate-${{ github.event.inputs.firewood-candidate-version }}' + - name: Upload candidate results + uses: actions/upload-artifact@v4 + with: + name: candidate-benchmark-results + path: | + output.txt + benchmark-results/ + retention-days: 7 + compare-results: + needs: [baseline-benchmark, candidate-benchmark] + runs-on: ubuntu-latest + if: always() + steps: + - name: Download baseline results + uses: actions/download-artifact@v4 + with: + name: baseline-benchmark-results + path: ./baseline-results + continue-on-error: true + - name: Download candidate results + uses: actions/download-artifact@v4 + with: + name: candidate-benchmark-results + path: ./candidate-results + continue-on-error: true + - name: Compare Results + shell: bash + run: | + echo "# Firewood Performance Comparison" >> $GITHUB_STEP_SUMMARY + echo "**Baseline:** ${{ github.event.inputs.firewood-baseline-version }}" >> $GITHUB_STEP_SUMMARY + echo "**Candidate:** ${{ github.event.inputs.firewood-candidate-version }}" >> $GITHUB_STEP_SUMMARY + echo "**Blocks:** ${{ github.event.inputs.start-block }} - ${{ github.event.inputs.end-block }}" >> $GITHUB_STEP_SUMMARY + + if [ -f "./baseline-results/output.txt" ] && [ -f "./candidate-results/output.txt" ]; then + BASELINE_MGAS=$(grep -o '[0-9]\+\.[0-9]\+ mgas/s' ./baseline-results/output.txt | head -1 | cut -d' ' -f1 || echo "N/A") + CANDIDATE_MGAS=$(grep -o '[0-9]\+\.[0-9]\+ mgas/s' ./candidate-results/output.txt | head -1 | cut -d' ' -f1 || echo "N/A") + + echo "- Baseline: ${BASELINE_MGAS} mgas/s" >> $GITHUB_STEP_SUMMARY + echo "- Candidate: ${CANDIDATE_MGAS} mgas/s" >> $GITHUB_STEP_SUMMARY + + if [ "$BASELINE_MGAS" != "N/A" ] && [ "$CANDIDATE_MGAS" != "N/A" ]; then + CHANGE=$(echo "scale=2; ($CANDIDATE_MGAS - $BASELINE_MGAS) / $BASELINE_MGAS * 100" | bc -l) + echo "- Change: ${CHANGE}%" >> $GITHUB_STEP_SUMMARY + fi + else + [ -f "./baseline-results/output.txt" ] && echo "- Baseline: Completed" >> $GITHUB_STEP_SUMMARY || echo "- Baseline: Failed" >> $GITHUB_STEP_SUMMARY + [ -f "./candidate-results/output.txt" ] && echo "- Candidate: Completed" >> $GITHUB_STEP_SUMMARY || echo "- Candidate: Failed" >> $GITHUB_STEP_SUMMARY + fi