Skip to content
Closed
Changes from 1 commit
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
e3b95ee
ci: c-chain-reexecution-firewood v0.0.8 and v0.0.12 on ARC
Elvis339 Sep 9, 2025
4911d98
ci: use different runner
Elvis339 Sep 10, 2025
1caaaad
ci: use different runner x2
Elvis339 Sep 10, 2025
bc95a9a
ci: run reexecution bench with different firewood versions
Elvis339 Sep 11, 2025
cbf8600
ci: set container
Elvis339 Sep 11, 2025
d309fb2
Merge branch 'master' into reexecution-bench-firewood
Elvis339 Sep 11, 2025
6beee7a
ci: install build essential and gcc
Elvis339 Sep 11, 2025
52b4afb
Merge remote-tracking branch 'origin/reexecution-bench-firewood' into…
Elvis339 Sep 11, 2025
9d80db6
ci: pinpoint avgo to 66ce7a7701db0c4b370f97b339478d5b5ebe919a matchin…
Elvis339 Sep 11, 2025
d324d64
ci: pin to 20051efe ref
Elvis339 Sep 15, 2025
5395d3e
Merge branch 'master' into reexecution-bench-firewood
Elvis339 Sep 15, 2025
d2490ca
ci: update ref to eabe3e0d
Elvis339 Sep 15, 2025
622e6b0
ci: update ref
Elvis339 Sep 15, 2025
743483f
Merge branch 'master' into reexecution-bench-firewood
Elvis339 Sep 15, 2025
23a958a
ci: update ref to 0c94b286
Elvis339 Sep 15, 2025
17ae8a3
ci: update ref to 92d0e99b
Elvis339 Sep 15, 2025
377cb9b
ci
Elvis339 Sep 18, 2025
4608cc1
Merge branch 'master' into reexecution-bench-firewood
Elvis339 Sep 18, 2025
abdd774
ci: fmt
Elvis339 Sep 18, 2025
56b1097
Merge remote-tracking branch 'origin/reexecution-bench-firewood' into…
Elvis339 Sep 18, 2025
40dc032
ci: fmt
Elvis339 Sep 18, 2025
a9d6350
ci: fmt
Elvis339 Sep 18, 2025
9c15213
ci: cleanup
Elvis339 Sep 18, 2025
6053b4c
ci: disable c-chain-reexecution-benchmark
Elvis339 Sep 18, 2025
5f2fe25
ci: rename
Elvis339 Sep 18, 2025
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
ci: fmt
  • Loading branch information
Elvis339 committed Sep 18, 2025
commit 40dc032e579e52a52cbd1be043e63db5094b33c7
378 changes: 189 additions & 189 deletions .github/workflows/c-chain-firewood-reexecution-bench-a-b-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -264,204 +264,204 @@ jobs:
# - name: Deploy kind with chaos mesh
# shell: bash
# run: nix develop --command ./scripts/run_task.sh test-robustness
# Add this job to your existing Tests workflow
# Add this job to your existing Tests workflow
# Firewood v0.0.8 job - uses older AvalancheGo commit
baseline-benchmark:
runs-on: avalanche-avalanchego-runner
container:
image: ghcr.io/actions/actions-runner:2.325.0
permissions:
id-token: write
contents: write
name: "Baseline: ${{ env.FIREWOOD_BASELINE_VERSION }}"
steps:
- name: Install dependencies
shell: bash
run: |
if ! command -v xz &> /dev/null; then
sudo apt-get update
sudo apt-get install -y xz-utils
fi
- uses: actions/checkout@v4
with:
ref: ${{ env.AVALANCHEGO_BASELINE_VERSION }}
- uses: ./.github/actions/setup-go-for-project
- uses: ./.github/actions/install-nix
# Only install Rust toolchain when building from source (non-tag versions)
- name: Setup Rust for source build
if: ${{ !startsWith(env.FIREWOOD_BASELINE_VERSION, 'v') && !startsWith(env.FIREWOOD_BASELINE_VERSION, 'ffi/') }}
uses: dtolnay/rust-toolchain@stable
with:
toolchain: ${{ env.RUST_VERSION }}
- name: Setup Rust cache for source build
if: ${{ !startsWith(env.FIREWOOD_BASELINE_VERSION, 'v') && !startsWith(env.FIREWOOD_BASELINE_VERSION, 'ffi/') }}
uses: Swatinem/rust-cache@v2
- name: Setup Firewood Baseline
shell: bash
run: |
FIREWOOD_VERSION="${{ env.FIREWOOD_BASELINE_VERSION }}"

if [[ "$FIREWOOD_VERSION" =~ ^(v|ffi/) ]]; then
echo "Detected tag: $FIREWOOD_VERSION - using pre-built binaries"
# For tags, fetch pre-built binaries from firewood-go-ethhash repo
TEMP_DIR=$(mktemp -d)
trap "rm -rf $TEMP_DIR" EXIT
baseline-benchmark:
runs-on: avalanche-avalanchego-runner
container:
image: ghcr.io/actions/actions-runner:2.325.0
permissions:
id-token: write
contents: write
name: "Baseline: ${{ env.FIREWOOD_BASELINE_VERSION }}"
steps:
- name: Install dependencies
shell: bash
run: |
if ! command -v xz &> /dev/null; then
sudo apt-get update
sudo apt-get install -y xz-utils
fi
- uses: actions/checkout@v4
with:
ref: ${{ env.AVALANCHEGO_BASELINE_VERSION }}
- uses: ./.github/actions/setup-go-for-project
- uses: ./.github/actions/install-nix
# Only install Rust toolchain when building from source (non-tag versions)
- name: Setup Rust for source build
if: ${{ !startsWith(env.FIREWOOD_BASELINE_VERSION, 'v') && !startsWith(env.FIREWOOD_BASELINE_VERSION, 'ffi/') }}
uses: dtolnay/rust-toolchain@stable
with:
toolchain: ${{ env.RUST_VERSION }}
- name: Setup Rust cache for source build
if: ${{ !startsWith(env.FIREWOOD_BASELINE_VERSION, 'v') && !startsWith(env.FIREWOOD_BASELINE_VERSION, 'ffi/') }}
uses: Swatinem/rust-cache@v2
- name: Setup Firewood Baseline
shell: bash
run: |
FIREWOOD_VERSION="${{ env.FIREWOOD_BASELINE_VERSION }}"

cd "$TEMP_DIR"
git clone https://github.com/ava-labs/firewood-go-ethhash.git
cd firewood-go-ethhash
git checkout "$FIREWOOD_VERSION"
if [[ "$FIREWOOD_VERSION" =~ ^(v|ffi/) ]]; then
echo "Detected tag: $FIREWOOD_VERSION - using pre-built binaries"
# For tags, fetch pre-built binaries from firewood-go-ethhash repo
TEMP_DIR=$(mktemp -d)
trap "rm -rf $TEMP_DIR" EXIT

cd "$GITHUB_WORKSPACE"
rm -rf ./ffi
cp -r "$TEMP_DIR/firewood-go-ethhash/ffi" ./ffi
else
echo "Detected commit: $FIREWOOD_VERSION - building from source"
./scripts/build_firewood.sh "$FIREWOOD_VERSION" ./ffi
fi
cd "$TEMP_DIR"
git clone https://github.com/ava-labs/firewood-go-ethhash.git
cd firewood-go-ethhash
git checkout "$FIREWOOD_VERSION"

# Configure go.mod to use local FFI
go mod edit -replace github.com/ava-labs/firewood-go-ethhash/ffi=./ffi
go mod download
- name: Run Baseline Benchmark
uses: ./.github/actions/c-chain-reexecution-benchmark
with:
config: firewood
start-block: ${{ env.START_BLOCK }}
end-block: ${{ env.END_BLOCK }}
prometheus-username: ${{ secrets.PROMETHEUS_ID || '' }}
prometheus-password: ${{ secrets.PROMETHEUS_PASSWORD || '' }}
push-github-action-benchmark: false
aws-role: ${{ secrets.AWS_S3_READ_ONLY_ROLE }}
aws-region: us-east-2
github-token: ${{ secrets.GITHUB_TOKEN }}
push-post-state: ''
runner_name: 'baseline-${{ env.FIREWOOD_BASELINE_VERSION }}'
- name: Upload baseline results
uses: actions/upload-artifact@v4
with:
name: baseline-benchmark-results
path: |
output.txt
benchmark-results/
retention-days: 1
candidate-benchmark:
runs-on: avalanche-avalanchego-runner
container:
image: ghcr.io/actions/actions-runner:2.325.0
permissions:
id-token: write
contents: write
name: "Candidate: ${{ env.FIREWOOD_CANDIDATE_VERSION }}"
steps:
- name: Install dependencies
shell: bash
run: |
if ! command -v xz &> /dev/null; then
sudo apt-get update
sudo apt-get install -y xz-utils
fi
- uses: actions/checkout@v4
with:
ref: ${{ env.AVALANCHEGO_CANDIDATE_VERSION }}
- uses: ./.github/actions/setup-go-for-project
- uses: ./.github/actions/install-nix
# Only install Rust toolchain when building from source (non-tag versions)
- name: Setup Rust for source build
if: ${{ !startsWith(env.FIREWOOD_CANDIDATE_VERSION, 'v') && !startsWith(env.FIREWOOD_CANDIDATE_VERSION, 'ffi/') }}
uses: dtolnay/rust-toolchain@stable
with:
toolchain: ${{ env.RUST_VERSION }}
- name: Setup Rust cache for source build
if: ${{ !startsWith(env.FIREWOOD_CANDIDATE_VERSION, 'v') && !startsWith(env.FIREWOOD_CANDIDATE_VERSION, 'ffi/') }}
uses: Swatinem/rust-cache@v2
- name: Setup Firewood Candidate
shell: bash
run: |
FIREWOOD_VERSION="${{ env.FIREWOOD_CANDIDATE_VERSION }}"
cd "$GITHUB_WORKSPACE"
rm -rf ./ffi
cp -r "$TEMP_DIR/firewood-go-ethhash/ffi" ./ffi
else
echo "Detected commit: $FIREWOOD_VERSION - building from source"
./scripts/build_firewood.sh "$FIREWOOD_VERSION" ./ffi
fi

if [[ "$FIREWOOD_VERSION" =~ ^(v|ffi/) ]]; then
echo "Detected tag: $FIREWOOD_VERSION - using pre-built binaries"
# For tags, fetch pre-built binaries from firewood-go-ethhash repo
TEMP_DIR=$(mktemp -d)
trap "rm -rf $TEMP_DIR" EXIT
# Configure go.mod to use local FFI
go mod edit -replace github.com/ava-labs/firewood-go-ethhash/ffi=./ffi
go mod download
- name: Run Baseline Benchmark
uses: ./.github/actions/c-chain-reexecution-benchmark
with:
config: firewood
start-block: ${{ env.START_BLOCK }}
end-block: ${{ env.END_BLOCK }}
prometheus-username: ${{ secrets.PROMETHEUS_ID || '' }}
prometheus-password: ${{ secrets.PROMETHEUS_PASSWORD || '' }}
push-github-action-benchmark: false
aws-role: ${{ secrets.AWS_S3_READ_ONLY_ROLE }}
aws-region: us-east-2
github-token: ${{ secrets.GITHUB_TOKEN }}
push-post-state: ''
runner_name: 'baseline-${{ env.FIREWOOD_BASELINE_VERSION }}'
- name: Upload baseline results
uses: actions/upload-artifact@v4
with:
name: baseline-benchmark-results
path: |
output.txt
benchmark-results/
retention-days: 1

cd "$TEMP_DIR"
git clone https://github.com/ava-labs/firewood-go-ethhash.git
cd firewood-go-ethhash
git checkout "$FIREWOOD_VERSION"
candidate-benchmark:
runs-on: avalanche-avalanchego-runner
container:
image: ghcr.io/actions/actions-runner:2.325.0
permissions:
id-token: write
contents: write
name: "Candidate: ${{ env.FIREWOOD_CANDIDATE_VERSION }}"
steps:
- name: Install dependencies
shell: bash
run: |
if ! command -v xz &> /dev/null; then
sudo apt-get update
sudo apt-get install -y xz-utils
fi
- uses: actions/checkout@v4
with:
ref: ${{ env.AVALANCHEGO_CANDIDATE_VERSION }}
- uses: ./.github/actions/setup-go-for-project
- uses: ./.github/actions/install-nix
# Only install Rust toolchain when building from source (non-tag versions)
- name: Setup Rust for source build
if: ${{ !startsWith(env.FIREWOOD_CANDIDATE_VERSION, 'v') && !startsWith(env.FIREWOOD_CANDIDATE_VERSION, 'ffi/') }}
uses: dtolnay/rust-toolchain@stable
with:
toolchain: ${{ env.RUST_VERSION }}
- name: Setup Rust cache for source build
if: ${{ !startsWith(env.FIREWOOD_CANDIDATE_VERSION, 'v') && !startsWith(env.FIREWOOD_CANDIDATE_VERSION, 'ffi/') }}
uses: Swatinem/rust-cache@v2
- name: Setup Firewood Candidate
shell: bash
run: |
FIREWOOD_VERSION="${{ env.FIREWOOD_CANDIDATE_VERSION }}"

cd "$GITHUB_WORKSPACE"
rm -rf ./ffi
cp -r "$TEMP_DIR/firewood-go-ethhash/ffi" ./ffi
else
echo "Detected commit: $FIREWOOD_VERSION - building from source"
./scripts/build_firewood.sh "$FIREWOOD_VERSION" ./ffi
fi
if [[ "$FIREWOOD_VERSION" =~ ^(v|ffi/) ]]; then
echo "Detected tag: $FIREWOOD_VERSION - using pre-built binaries"
# For tags, fetch pre-built binaries from firewood-go-ethhash repo
TEMP_DIR=$(mktemp -d)
trap "rm -rf $TEMP_DIR" EXIT

cd "$TEMP_DIR"
git clone https://github.com/ava-labs/firewood-go-ethhash.git
cd firewood-go-ethhash
git checkout "$FIREWOOD_VERSION"

cd "$GITHUB_WORKSPACE"
rm -rf ./ffi
cp -r "$TEMP_DIR/firewood-go-ethhash/ffi" ./ffi
else
echo "Detected commit: $FIREWOOD_VERSION - building from source"
./scripts/build_firewood.sh "$FIREWOOD_VERSION" ./ffi
fi

# Configure go.mod to use local FFI
go mod edit -replace github.com/ava-labs/firewood-go-ethhash/ffi=./ffi
go mod download
- name: Run Candidate Benchmark
uses: ./.github/actions/c-chain-reexecution-benchmark
with:
config: firewood
start-block: ${{ env.START_BLOCK }}
end-block: ${{ env.END_BLOCK }}
prometheus-username: ${{ secrets.PROMETHEUS_ID || '' }}
prometheus-password: ${{ secrets.PROMETHEUS_PASSWORD || '' }}
push-github-action-benchmark: false
aws-role: ${{ secrets.AWS_S3_READ_ONLY_ROLE }}
aws-region: us-east-2
github-token: ${{ secrets.GITHUB_TOKEN }}
push-post-state: ''
runner_name: 'candidate-${{ env.FIREWOOD_CANDIDATE_VERSION }}'
- name: Upload candidate results
uses: actions/upload-artifact@v4
with:
name: candidate-benchmark-results
path: |
output.txt
benchmark-results/
retention-days: 1

# Configure go.mod to use local FFI
go mod edit -replace github.com/ava-labs/firewood-go-ethhash/ffi=./ffi
go mod download
- name: Run Candidate Benchmark
uses: ./.github/actions/c-chain-reexecution-benchmark
with:
config: firewood
start-block: ${{ env.START_BLOCK }}
end-block: ${{ env.END_BLOCK }}
prometheus-username: ${{ secrets.PROMETHEUS_ID || '' }}
prometheus-password: ${{ secrets.PROMETHEUS_PASSWORD || '' }}
push-github-action-benchmark: false
aws-role: ${{ secrets.AWS_S3_READ_ONLY_ROLE }}
aws-region: us-east-2
github-token: ${{ secrets.GITHUB_TOKEN }}
push-post-state: ''
runner_name: 'candidate-${{ env.FIREWOOD_CANDIDATE_VERSION }}'
- name: Upload candidate results
uses: actions/upload-artifact@v4
with:
name: candidate-benchmark-results
path: |
output.txt
benchmark-results/
retention-days: 1
compare-results:
needs: [ baseline-benchmark, candidate-benchmark ]
runs-on: ubuntu-latest
if: always()
steps:
- name: Download baseline results
uses: actions/download-artifact@v4
with:
name: baseline-benchmark-results
path: ./baseline-results
continue-on-error: true
- name: Download candidate results
uses: actions/download-artifact@v4
with:
name: candidate-benchmark-results
path: ./candidate-results
continue-on-error: true
- name: Compare Results
shell: bash
run: |
echo "# Firewood Performance Comparison (Test)" >> $GITHUB_STEP_SUMMARY
echo "**Baseline:** ${{ env.FIREWOOD_BASELINE_VERSION }}" >> $GITHUB_STEP_SUMMARY
echo "**Candidate:** ${{ env.FIREWOOD_CANDIDATE_VERSION }}" >> $GITHUB_STEP_SUMMARY
echo "**Blocks:** ${{ env.START_BLOCK }} - ${{ env.END_BLOCK }}" >> $GITHUB_STEP_SUMMARY
compare-results:
needs: [baseline-benchmark, candidate-benchmark]
runs-on: ubuntu-latest
if: always()
steps:
- name: Download baseline results
uses: actions/download-artifact@v4
with:
name: baseline-benchmark-results
path: ./baseline-results
continue-on-error: true
- name: Download candidate results
uses: actions/download-artifact@v4
with:
name: candidate-benchmark-results
path: ./candidate-results
continue-on-error: true
- name: Compare Results
shell: bash
run: |
echo "# Firewood Performance Comparison (Test)" >> $GITHUB_STEP_SUMMARY
echo "**Baseline:** ${{ env.FIREWOOD_BASELINE_VERSION }}" >> $GITHUB_STEP_SUMMARY
echo "**Candidate:** ${{ env.FIREWOOD_CANDIDATE_VERSION }}" >> $GITHUB_STEP_SUMMARY
echo "**Blocks:** ${{ env.START_BLOCK }} - ${{ env.END_BLOCK }}" >> $GITHUB_STEP_SUMMARY

if [ -f "./baseline-results/output.txt" ] && [ -f "./candidate-results/output.txt" ]; then
BASELINE_MGAS=$(grep -o '[0-9]\+\.[0-9]\+ mgas/s' ./baseline-results/output.txt | head -1 | cut -d' ' -f1 || echo "N/A")
CANDIDATE_MGAS=$(grep -o '[0-9]\+\.[0-9]\+ mgas/s' ./candidate-results/output.txt | head -1 | cut -d' ' -f1 || echo "N/A")
if [ -f "./baseline-results/output.txt" ] && [ -f "./candidate-results/output.txt" ]; then
BASELINE_MGAS=$(grep -o '[0-9]\+\.[0-9]\+ mgas/s' ./baseline-results/output.txt | head -1 | cut -d' ' -f1 || echo "N/A")
CANDIDATE_MGAS=$(grep -o '[0-9]\+\.[0-9]\+ mgas/s' ./candidate-results/output.txt | head -1 | cut -d' ' -f1 || echo "N/A")

echo "- Baseline: ${BASELINE_MGAS} mgas/s" >> $GITHUB_STEP_SUMMARY
echo "- Candidate: ${CANDIDATE_MGAS} mgas/s" >> $GITHUB_STEP_SUMMARY
echo "- Baseline: ${BASELINE_MGAS} mgas/s" >> $GITHUB_STEP_SUMMARY
echo "- Candidate: ${CANDIDATE_MGAS} mgas/s" >> $GITHUB_STEP_SUMMARY

if [ "$BASELINE_MGAS" != "N/A" ] && [ "$CANDIDATE_MGAS" != "N/A" ]; then
CHANGE=$(echo "scale=2; ($CANDIDATE_MGAS - $BASELINE_MGAS) / $BASELINE_MGAS * 100" | bc -l)
echo "- Change: ${CHANGE}%" >> $GITHUB_STEP_SUMMARY
fi
else
[ -f "./baseline-results/output.txt" ] && echo "- Baseline: Completed" >> $GITHUB_STEP_SUMMARY || echo "- Baseline: Failed" >> $GITHUB_STEP_SUMMARY
[ -f "./candidate-results/output.txt" ] && echo "- Candidate: Completed" >> $GITHUB_STEP_SUMMARY || echo "- Candidate: Failed" >> $GITHUB_STEP_SUMMARY
if [ "$BASELINE_MGAS" != "N/A" ] && [ "$CANDIDATE_MGAS" != "N/A" ]; then
CHANGE=$(echo "scale=2; ($CANDIDATE_MGAS - $BASELINE_MGAS) / $BASELINE_MGAS * 100" | bc -l)
echo "- Change: ${CHANGE}%" >> $GITHUB_STEP_SUMMARY
fi
else
[ -f "./baseline-results/output.txt" ] && echo "- Baseline: Completed" >> $GITHUB_STEP_SUMMARY || echo "- Baseline: Failed" >> $GITHUB_STEP_SUMMARY
[ -f "./candidate-results/output.txt" ] && echo "- Candidate: Completed" >> $GITHUB_STEP_SUMMARY || echo "- Candidate: Failed" >> $GITHUB_STEP_SUMMARY
fi
Loading