Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .gitlab/pipeline/short-benchmarks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ short-benchmark-westend: &short-bench
tags:
- benchmark
script:
- ./artifacts/polkadot benchmark pallet --chain $RUNTIME-dev --pallet "*" --extrinsic "*" --steps 2 --repeat 1
- ./artifacts/polkadot benchmark pallet --chain $RUNTIME-dev --pallet "*" --extrinsic "*" --steps 2 --repeat 1 --sanity-weight-check ignore

# run short-benchmarks for system parachain runtimes from cumulus

Expand All @@ -47,7 +47,7 @@ short-benchmark-westend: &short-bench
tags:
- benchmark
script:
- ./artifacts/polkadot-parachain benchmark pallet --chain $RUNTIME_CHAIN --pallet "*" --extrinsic "*" --steps 2 --repeat 1
- ./artifacts/polkadot-parachain benchmark pallet --chain $RUNTIME_CHAIN --pallet "*" --extrinsic "*" --steps 2 --repeat 1 --sanity-weight-check ignore

short-benchmark-asset-hub-rococo:
<<: *short-bench-cumulus
Expand Down
3 changes: 1 addition & 2 deletions .gitlab/pipeline/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,6 @@ test-linux-stable-runtime-benchmarks:
RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings"
script:
- time cargo nextest run --workspace --features runtime-benchmarks benchmark --locked --cargo-profile testnet

# can be used to run all tests
# test-linux-stable-all:
# stage: test
Expand Down Expand Up @@ -320,7 +319,7 @@ quick-benchmarks:
WASM_BUILD_NO_COLOR: 1
WASM_BUILD_RUSTFLAGS: "-C debug-assertions -D warnings"
script:
- time cargo run --locked --release -p staging-node-cli --bin substrate-node --features runtime-benchmarks -- benchmark pallet --execution wasm --wasm-execution compiled --chain dev --pallet "*" --extrinsic "*" --steps 2 --repeat 1
- time cargo run --locked --release -p staging-node-cli --bin substrate-node --features runtime-benchmarks -- benchmark pallet --execution wasm --wasm-execution compiled --chain dev --pallet "*" --extrinsic "*" --steps 2 --repeat 1 --sanity-weight-check ignore

test-frame-examples-compile-to-wasm:
# into one job
Expand Down
1 change: 1 addition & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

6 changes: 5 additions & 1 deletion cumulus/parachain-template/runtime/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -707,6 +707,8 @@ impl_runtime_apis! {
fn benchmark_metadata(extra: bool) -> (
Vec<frame_benchmarking::BenchmarkList>,
Vec<frame_support::traits::StorageInfo>,
Weight,
frame_support::weights::RuntimeDbWeight,
) {
use frame_benchmarking::{Benchmarking, BenchmarkList};
use frame_support::traits::StorageInfoTrait;
Expand All @@ -717,7 +719,9 @@ impl_runtime_apis! {
list_benchmarks!(list, extra);

let storage_info = AllPalletsWithSystem::storage_info();
(list, storage_info)
let max_extrinsic_weight = RuntimeBlockWeights::get().per_class.get(DispatchClass::Normal).max_extrinsic.unwrap();
let db_weight: frame_support::weights::RuntimeDbWeight = <Self as frame_system::Config>::DbWeight::get();
(list, storage_info, max_extrinsic_weight, db_weight)
}

fn dispatch_benchmark(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1294,6 +1294,8 @@ impl_runtime_apis! {
fn benchmark_metadata(extra: bool) -> (
Vec<frame_benchmarking::BenchmarkList>,
Vec<frame_support::traits::StorageInfo>,
Weight,
frame_support::weights::RuntimeDbWeight,
) {
use frame_benchmarking::{Benchmarking, BenchmarkList};
use frame_support::traits::StorageInfoTrait;
Expand Down Expand Up @@ -1322,7 +1324,9 @@ impl_runtime_apis! {
list_benchmarks!(list, extra);

let storage_info = AllPalletsWithSystem::storage_info();
(list, storage_info)
let max_extrinsic_weight = RuntimeBlockWeights::get().per_class.get(DispatchClass::Normal).max_extrinsic.unwrap();
let db_weight: frame_support::weights::RuntimeDbWeight = <Self as frame_system::Config>::DbWeight::get();
(list, storage_info, max_extrinsic_weight, db_weight)
}

fn dispatch_benchmark(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1372,6 +1372,8 @@ impl_runtime_apis! {
fn benchmark_metadata(extra: bool) -> (
Vec<frame_benchmarking::BenchmarkList>,
Vec<frame_support::traits::StorageInfo>,
Weight,
frame_support::weights::RuntimeDbWeight,
) {
use frame_benchmarking::{Benchmarking, BenchmarkList};
use frame_support::traits::StorageInfoTrait;
Expand Down Expand Up @@ -1400,7 +1402,9 @@ impl_runtime_apis! {
list_benchmarks!(list, extra);

let storage_info = AllPalletsWithSystem::storage_info();
(list, storage_info)
let max_extrinsic_weight = RuntimeBlockWeights::get().per_class.get(DispatchClass::Normal).max_extrinsic.unwrap();
let db_weight: frame_support::weights::RuntimeDbWeight = <Self as frame_system::Config>::DbWeight::get();
(list, storage_info, max_extrinsic_weight, db_weight)
}

fn dispatch_benchmark(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1059,6 +1059,8 @@ impl_runtime_apis! {
fn benchmark_metadata(extra: bool) -> (
Vec<frame_benchmarking::BenchmarkList>,
Vec<frame_support::traits::StorageInfo>,
Weight,
frame_support::weights::RuntimeDbWeight,
) {
use frame_benchmarking::{Benchmarking, BenchmarkList};
use frame_support::traits::StorageInfoTrait;
Expand All @@ -1083,7 +1085,9 @@ impl_runtime_apis! {
list_benchmarks!(list, extra);

let storage_info = AllPalletsWithSystem::storage_info();
(list, storage_info)
let max_extrinsic_weight = RuntimeBlockWeights::get().per_class.get(DispatchClass::Normal).max_extrinsic.unwrap();
let db_weight: frame_support::weights::RuntimeDbWeight = <Self as frame_system::Config>::DbWeight::get();
(list, storage_info, max_extrinsic_weight, db_weight)
}

fn dispatch_benchmark(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -949,6 +949,8 @@ impl_runtime_apis! {
fn benchmark_metadata(extra: bool) -> (
Vec<frame_benchmarking::BenchmarkList>,
Vec<frame_support::traits::StorageInfo>,
Weight,
frame_support::weights::RuntimeDbWeight,
) {
use frame_benchmarking::{Benchmarking, BenchmarkList};
use frame_support::traits::StorageInfoTrait;
Expand All @@ -960,7 +962,9 @@ impl_runtime_apis! {
list_benchmarks!(list, extra);

let storage_info = AllPalletsWithSystem::storage_info();
(list, storage_info)
let max_extrinsic_weight = RuntimeBlockWeights::get().per_class.get(DispatchClass::Normal).max_extrinsic.unwrap();
let db_weight: frame_support::weights::RuntimeDbWeight = <Self as frame_system::Config>::DbWeight::get();
(list, storage_info, max_extrinsic_weight, db_weight)
}

fn dispatch_benchmark(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -675,6 +675,8 @@ impl_runtime_apis! {
fn benchmark_metadata(extra: bool) -> (
Vec<frame_benchmarking::BenchmarkList>,
Vec<frame_support::traits::StorageInfo>,
Weight,
frame_support::weights::RuntimeDbWeight,
) {
use frame_benchmarking::{Benchmarking, BenchmarkList};
use frame_support::traits::StorageInfoTrait;
Expand All @@ -686,7 +688,9 @@ impl_runtime_apis! {
list_benchmarks!(list, extra);

let storage_info = AllPalletsWithSystem::storage_info();
(list, storage_info)
let max_extrinsic_weight = RuntimeBlockWeights::get().per_class.get(DispatchClass::Normal).max_extrinsic.unwrap();
let db_weight: frame_support::weights::RuntimeDbWeight = <Self as frame_system::Config>::DbWeight::get();
(list, storage_info, max_extrinsic_weight, db_weight)
}

fn dispatch_benchmark(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -652,6 +652,8 @@ impl_runtime_apis! {
fn benchmark_metadata(extra: bool) -> (
Vec<frame_benchmarking::BenchmarkList>,
Vec<frame_support::traits::StorageInfo>,
Weight,
frame_support::weights::RuntimeDbWeight,
) {
use frame_benchmarking::{Benchmarking, BenchmarkList};
use frame_support::traits::StorageInfoTrait;
Expand All @@ -669,7 +671,9 @@ impl_runtime_apis! {
list_benchmarks!(list, extra);

let storage_info = AllPalletsWithSystem::storage_info();
(list, storage_info)
let max_extrinsic_weight = RuntimeBlockWeights::get().per_class.get(DispatchClass::Normal).max_extrinsic.unwrap();
let db_weight: frame_support::weights::RuntimeDbWeight = <Self as frame_system::Config>::DbWeight::get();
(list, storage_info, max_extrinsic_weight, db_weight)
}

fn dispatch_benchmark(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -435,17 +435,21 @@ impl_runtime_apis! {
fn benchmark_metadata(extra: bool) -> (
Vec<frame_benchmarking::BenchmarkList>,
Vec<frame_support::traits::StorageInfo>,
Weight,
frame_support::weights::RuntimeDbWeight,
) {
use frame_benchmarking::{Benchmarking, BenchmarkList};
use frame_support::traits::StorageInfoTrait;
use frame_system_benchmarking::Pallet as SystemBench;
use sp_core::Get;

let mut list = Vec::<BenchmarkList>::new();
list_benchmarks!(list, extra);

let storage_info = AllPalletsWithSystem::storage_info();

(list, storage_info)
let max_extrinsic_weight = RuntimeBlockWeights::get().per_class.get(DispatchClass::Normal).max_extrinsic.unwrap();
let db_weight: frame_support::weights::RuntimeDbWeight = <Self as frame_system::Config>::DbWeight::get();
(list, storage_info, max_extrinsic_weight, db_weight)
}

fn dispatch_benchmark(
Expand Down
6 changes: 5 additions & 1 deletion cumulus/parachains/runtimes/people/people-westend/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -638,6 +638,8 @@ impl_runtime_apis! {
fn benchmark_metadata(extra: bool) -> (
Vec<frame_benchmarking::BenchmarkList>,
Vec<frame_support::traits::StorageInfo>,
Weight,
frame_support::weights::RuntimeDbWeight,
) {
use frame_benchmarking::{Benchmarking, BenchmarkList};
use frame_support::traits::StorageInfoTrait;
Expand All @@ -655,7 +657,9 @@ impl_runtime_apis! {
list_benchmarks!(list, extra);

let storage_info = AllPalletsWithSystem::storage_info();
(list, storage_info)
let max_extrinsic_weight = RuntimeBlockWeights::get().per_class.get(DispatchClass::Normal).max_extrinsic.unwrap();
let db_weight: frame_support::weights::RuntimeDbWeight = <Self as frame_system::Config>::DbWeight::get();
(list, storage_info, max_extrinsic_weight, db_weight)
}

fn dispatch_benchmark(
Expand Down
6 changes: 5 additions & 1 deletion cumulus/parachains/runtimes/testing/penpal/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -847,6 +847,8 @@ impl_runtime_apis! {
fn benchmark_metadata(extra: bool) -> (
Vec<frame_benchmarking::BenchmarkList>,
Vec<frame_support::traits::StorageInfo>,
Weight,
frame_support::weights::RuntimeDbWeight,
) {
use frame_benchmarking::{Benchmarking, BenchmarkList};
use frame_support::traits::StorageInfoTrait;
Expand All @@ -857,7 +859,9 @@ impl_runtime_apis! {
list_benchmarks!(list, extra);

let storage_info = AllPalletsWithSystem::storage_info();
(list, storage_info)
let max_extrinsic_weight = RuntimeBlockWeights::get().per_class.get(DispatchClass::Normal).max_extrinsic.unwrap();
let db_weight: frame_support::weights::RuntimeDbWeight = <Self as frame_system::Config>::DbWeight::get();
(list, storage_info, max_extrinsic_weight, db_weight)
}

fn dispatch_benchmark(
Expand Down
6 changes: 5 additions & 1 deletion polkadot/runtime/rococo/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2229,6 +2229,8 @@ sp_api::impl_runtime_apis! {
fn benchmark_metadata(extra: bool) -> (
Vec<frame_benchmarking::BenchmarkList>,
Vec<frame_support::traits::StorageInfo>,
Weight,
frame_support::weights::RuntimeDbWeight,
) {
use frame_benchmarking::{Benchmarking, BenchmarkList};
use frame_support::traits::StorageInfoTrait;
Expand All @@ -2242,7 +2244,9 @@ sp_api::impl_runtime_apis! {
list_benchmarks!(list, extra);

let storage_info = AllPalletsWithSystem::storage_info();
return (list, storage_info)
let max_extrinsic_weight = BlockWeights::get().per_class.get(DispatchClass::Normal).max_extrinsic.unwrap();
let db_weight: frame_support::weights::RuntimeDbWeight = <Self as frame_system::Config>::DbWeight::get();
(list, storage_info, max_extrinsic_weight, db_weight)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I hoped that we could somehow around doing a breaking change for all runtimes, but it seems unavoidable, unless we want to squeeze this into into any of the vectors.
Maybe you can create a single struct that holds the Weight and RuntimeDbWeight. Then we can extend that in the future and avoid more breaking changes.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Creating a single struct for Weight and RuntimeDbWeight is either way a better solution. Would you prefer to have it included in BenchmarkList / StorageInfo and make it backwards compatible or add an additional parameter which results in a breaking change?

Copy link
Member

@ggwpez ggwpez Feb 5, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think it would need one breaking change, yes. But from then on we can avoid further breaking changes by using the following pattern:

struct AllBenchmarkData { // not a good name
  list: Vec<frame_benchmarking::BenchmarkList>,
  info: Vec<frame_support::traits::StorageInfo>,
  max_ext_weight: Option<..>,
  db_weights: Option<..>
}

and having Default on that thing. The runtime devs can use ..Default::default() to stay forward compatible and we can keep adding Options without directly breaking their code.

}

fn dispatch_benchmark(
Expand Down
6 changes: 5 additions & 1 deletion polkadot/runtime/westend/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2243,6 +2243,8 @@ sp_api::impl_runtime_apis! {
fn benchmark_metadata(extra: bool) -> (
Vec<frame_benchmarking::BenchmarkList>,
Vec<frame_support::traits::StorageInfo>,
Weight,
frame_support::weights::RuntimeDbWeight,
) {
use frame_benchmarking::{Benchmarking, BenchmarkList};
use frame_support::traits::StorageInfoTrait;
Expand All @@ -2261,7 +2263,9 @@ sp_api::impl_runtime_apis! {
list_benchmarks!(list, extra);

let storage_info = AllPalletsWithSystem::storage_info();
return (list, storage_info)
let max_extrinsic_weight = BlockWeights::get().per_class.get(DispatchClass::Normal).max_extrinsic.unwrap();
let db_weight: frame_support::weights::RuntimeDbWeight = <Self as frame_system::Config>::DbWeight::get();
(list, storage_info, max_extrinsic_weight, db_weight)
}

fn dispatch_benchmark(
Expand Down
13 changes: 9 additions & 4 deletions substrate/bin/node-template/runtime/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,9 @@ use sp_version::RuntimeVersion;
use frame_support::genesis_builder_helper::{build_config, create_default_config};
// A few exports that help ease life for downstream crates.
pub use frame_support::{
construct_runtime, derive_impl, parameter_types,
construct_runtime, derive_impl,
dispatch::DispatchClass,
parameter_types,
traits::{
ConstBool, ConstU128, ConstU32, ConstU64, ConstU8, KeyOwnerProofSystem, Randomness,
StorageInfo,
Expand Down Expand Up @@ -257,7 +259,7 @@ impl pallet_template::Config for Runtime {

// Create the runtime by composing the FRAME pallets that were previously configured.
construct_runtime!(
pub enum Runtime {
pub struct Runtime {
System: frame_system,
Timestamp: pallet_timestamp,
Aura: pallet_aura,
Expand Down Expand Up @@ -494,6 +496,8 @@ impl_runtime_apis! {
fn benchmark_metadata(extra: bool) -> (
Vec<frame_benchmarking::BenchmarkList>,
Vec<frame_support::traits::StorageInfo>,
Weight,
frame_support::weights::RuntimeDbWeight,
) {
use frame_benchmarking::{baseline, Benchmarking, BenchmarkList};
use frame_support::traits::StorageInfoTrait;
Expand All @@ -504,8 +508,9 @@ impl_runtime_apis! {
list_benchmarks!(list, extra);

let storage_info = AllPalletsWithSystem::storage_info();

(list, storage_info)
let max_extrinsic_weight = BlockWeights::get().per_class.get(DispatchClass::Normal).max_extrinsic.unwrap();
let db_weight: frame_support::weights::RuntimeDbWeight = <Self as frame_system::Config>::DbWeight::get();
(list, storage_info, max_extrinsic_weight, db_weight)
}

fn dispatch_benchmark(
Expand Down
6 changes: 5 additions & 1 deletion substrate/bin/node/runtime/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2848,6 +2848,8 @@ impl_runtime_apis! {
fn benchmark_metadata(extra: bool) -> (
Vec<frame_benchmarking::BenchmarkList>,
Vec<frame_support::traits::StorageInfo>,
Weight,
frame_support::weights::RuntimeDbWeight,
) {
use frame_benchmarking::{baseline, Benchmarking, BenchmarkList};
use frame_support::traits::StorageInfoTrait;
Expand All @@ -2866,8 +2868,10 @@ impl_runtime_apis! {
list_benchmarks!(list, extra);

let storage_info = AllPalletsWithSystem::storage_info();
let max_extrinsic_weight = RuntimeBlockWeights::get().per_class.get(DispatchClass::Normal).max_extrinsic.unwrap();
let db_weight: frame_support::weights::RuntimeDbWeight = <Self as frame_system::Config>::DbWeight::get();

(list, storage_info)
(list, storage_info, max_extrinsic_weight, db_weight)
}

fn dispatch_benchmark(
Expand Down
16 changes: 16 additions & 0 deletions substrate/client/cli/src/arg_enums.rs
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,22 @@ impl std::fmt::Display for WasmExecutionMethod {
}
}

/// How to output the result of the sanity weight check and what to do when it fails.
#[allow(missing_docs)]
#[derive(Debug, Clone, Copy, ValueEnum, PartialEq)]
#[value(rename_all = "kebab-case")]
pub enum SanityWeightCheck {
/// Prints the results in the terminal and on failing returns an error.
Error,
/// Prints the results in the terminal.
Warning,
/// Sanity weight check is ignored.
Ignore,
}

/// The default [`SanityWeightCheck`].
pub const DEFAULT_SANITY_WEIGHT_CHECK: SanityWeightCheck = SanityWeightCheck::Error;

/// Converts the execution method and instantiation strategy command line arguments
/// into an execution method which can be used internally.
pub fn execution_method_from_cli(
Expand Down
12 changes: 11 additions & 1 deletion substrate/client/cli/src/params/shared_params.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.

use crate::arg_enums::TracingReceiver;
use crate::{arg_enums::TracingReceiver, SanityWeightCheck, DEFAULT_SANITY_WEIGHT_CHECK};
use clap::Args;
use sc_service::config::BasePath;
use std::path::PathBuf;
Expand Down Expand Up @@ -87,6 +87,16 @@ pub struct SharedParams {
/// Receiver to process tracing messages.
#[arg(long, value_name = "RECEIVER", value_enum, ignore_case = true, default_value_t = TracingReceiver::Log)]
pub tracing_receiver: TracingReceiver,

/// Sanity weight check for benchmarks. Checks that no benchmark function has a weight larger
/// than the (`DispatchClass::Normal`) max. normal class extrinsic weight.
#[arg(
long,
value_name = "ERROR-LEVEL",
value_enum,
default_value_t = DEFAULT_SANITY_WEIGHT_CHECK,
)]
pub sanity_weight_check: SanityWeightCheck,
}

impl SharedParams {
Expand Down
Loading