Skip to content
This repository was archived by the owner on Nov 15, 2023. It is now read-only.

Batch benchmarks together with * notation. #5436

Merged
gavofyork merged 7 commits into from
Mar 31, 2020
Merged

Batch benchmarks together with * notation. #5436

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
72592e4
Batch benchmarks together with `*` notation.
gavofyork Mar 27, 2020
3a30c76
Merge branch 'master' into gav-fast-benchmarks
shawntabrizi Mar 28, 2020
4568da7
Fix short structopt conflict
shawntabrizi Mar 28, 2020
6fdb322
Merge branch 'gav-fast-benchmarks' of https://github.com/paritytech/s…
shawntabrizi Mar 28, 2020
9a5e9dd
Return error if `batches` is empty
shawntabrizi Mar 28, 2020
ff98bb9
Move fast benchmarks macro into `frame_benchmarking` (#5445)
shawntabrizi Mar 31, 2020
1570f24
Merge branch 'master' into gav-fast-benchmarks
shawntabrizi Mar 31, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
97 changes: 18 additions & 79 deletions bin/node/runtime/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -858,95 +858,34 @@ impl_runtime_apis! {
#[cfg(feature = "runtime-benchmarks")]
impl frame_benchmarking::Benchmark<Block> for Runtime {
fn dispatch_benchmark(
module: Vec<u8>,
extrinsic: Vec<u8>,
pallet: Vec<u8>,
benchmark: Vec<u8>,
lowest_range_values: Vec<u32>,
highest_range_values: Vec<u32>,
steps: Vec<u32>,
repeat: u32,
) -> Result<Vec<frame_benchmarking::BenchmarkResults>, sp_runtime::RuntimeString> {
use frame_benchmarking::Benchmarking;
) -> Result<Vec<frame_benchmarking::BenchmarkBatch>, sp_runtime::RuntimeString> {
use frame_benchmarking::{Benchmarking, BenchmarkBatch, add_benchmark};
// Trying to add benchmarks directly to the Session Pallet caused cyclic dependency issues.
// To get around that, we separated the Session benchmarks into its own crate, which is why
// we need these two lines below.
use pallet_session_benchmarking::Module as SessionBench;
impl pallet_session_benchmarking::Trait for Runtime {}

let result = match module.as_slice() {
b"pallet-balances" | b"balances" => Balances::run_benchmark(
extrinsic,
lowest_range_values,
highest_range_values,
steps,
repeat,
),
b"pallet-im-online" | b"im-online" => ImOnline::run_benchmark(
extrinsic,
lowest_range_values,
highest_range_values,
steps,
repeat,
),
b"pallet-identity" | b"identity" => Identity::run_benchmark(
extrinsic,
lowest_range_values,
highest_range_values,
steps,
repeat,
),
b"pallet-session" | b"session" => SessionBench::<Runtime>::run_benchmark(
extrinsic,
lowest_range_values,
highest_range_values,
steps,
repeat,
),
b"pallet-staking" | b"staking" => Staking::run_benchmark(
extrinsic,
lowest_range_values,
highest_range_values,
steps,
repeat,
),
b"pallet-timestamp" | b"timestamp" => Timestamp::run_benchmark(
extrinsic,
lowest_range_values,
highest_range_values,
steps,
repeat,
),
b"pallet-treasury" | b"treasury" => Treasury::run_benchmark(
extrinsic,
lowest_range_values,
highest_range_values,
steps,
repeat,
),
b"pallet-vesting" | b"vesting" => Vesting::run_benchmark(
extrinsic,
lowest_range_values,
highest_range_values,
steps,
repeat,
),
b"pallet-democracy" | b"democracy" => Democracy::run_benchmark(
extrinsic,
lowest_range_values,
highest_range_values,
steps,
repeat,
),
b"pallet-collective" | b"collective" => Council::run_benchmark(
extrinsic,
lowest_range_values,
highest_range_values,
steps,
repeat,
),
_ => Err("Benchmark not found for this pallet."),
};

result.map_err(|e| e.into())
let mut batches = Vec::<BenchmarkBatch>::new();
let params = (&pallet, &benchmark, &lowest_range_values, &highest_range_values, &steps, repeat);
add_benchmark!(params, batches, b"balances", Balances);
add_benchmark!(params, batches, b"im-online", ImOnline);
add_benchmark!(params, batches, b"identity", Identity);
add_benchmark!(params, batches, b"session", SessionBench::<Runtime>);
add_benchmark!(params, batches, b"staking", Staking);
add_benchmark!(params, batches, b"timestamp", Timestamp);
add_benchmark!(params, batches, b"treasury", Treasury);
add_benchmark!(params, batches, b"vesting", Vesting);
add_benchmark!(params, batches, b"democracy", Democracy);
add_benchmark!(params, batches, b"collective", Council);
if batches.is_empty() { return Err("Benchmark not found for this pallet.".into()) }
Ok(batches)
}
}
}
Expand Down
90 changes: 80 additions & 10 deletions frame/benchmarking/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -505,15 +505,19 @@ macro_rules! impl_benchmark {
NO_INSTANCE $( $name:ident ),*
) => {
impl<T: Trait> $crate::Benchmarking<$crate::BenchmarkResults> for Module<T> {
fn benchmarks() -> Vec<&'static [u8]> {
vec![ $( stringify!($name).as_ref() ),* ]
}

fn run_benchmark(
extrinsic: Vec<u8>,
lowest_range_values: Vec<u32>,
highest_range_values: Vec<u32>,
steps: Vec<u32>,
extrinsic: &[u8],
lowest_range_values: &[u32],
highest_range_values: &[u32],
steps: &[u32],
repeat: u32,
) -> Result<Vec<$crate::BenchmarkResults>, &'static str> {
// Map the input to the selected benchmark.
let extrinsic = sp_std::str::from_utf8(extrinsic.as_slice())
let extrinsic = sp_std::str::from_utf8(extrinsic)
.map_err(|_| "`extrinsic` is not a valid utf8 string!")?;
let selected_benchmark = match extrinsic {
$( stringify!($name) => SelectedBenchmark::$name, )*
Expand Down Expand Up @@ -597,15 +601,19 @@ macro_rules! impl_benchmark {
INSTANCE $( $name:ident ),*
) => {
impl<T: Trait<I>, I: Instance> $crate::Benchmarking<$crate::BenchmarkResults> for Module<T, I> {
fn benchmarks() -> Vec<&'static [u8]> {
vec![ $( stringify!($name).as_ref() ),* ]
}

fn run_benchmark(
extrinsic: Vec<u8>,
lowest_range_values: Vec<u32>,
highest_range_values: Vec<u32>,
steps: Vec<u32>,
extrinsic: &[u8],
lowest_range_values: &[u32],
highest_range_values: &[u32],
steps: &[u32],
repeat: u32,
) -> Result<Vec<$crate::BenchmarkResults>, &'static str> {
// Map the input to the selected benchmark.
let extrinsic = sp_std::str::from_utf8(extrinsic.as_slice())
let extrinsic = sp_std::str::from_utf8(extrinsic)
.map_err(|_| "`extrinsic` is not a valid utf8 string!")?;
let selected_benchmark = match extrinsic {
$( stringify!($name) => SelectedBenchmark::$name, )*
Expand Down Expand Up @@ -686,3 +694,65 @@ macro_rules! impl_benchmark {
}
}
}


/// This macro adds pallet benchmarks to a `Vec<BenchmarkBatch>` object.
///
/// First create an object that holds in the input parameters for the benchmark:
///
/// ```ignore
/// let params = (&pallet, &benchmark, &lowest_range_values, &highest_range_values, &steps, repeat);
/// ```
///
/// Then define a mutable local variable to hold your `BenchmarkBatch` object:
///
/// ```ignore
/// let mut batches = Vec::<BenchmarkBatch>::new();
/// ````
///
/// Then add the pallets you want to benchmark to this object, including the string
/// you want to use target a particular pallet:
///
/// ```ignore
/// add_benchmark!(params, batches, b"balances", Balances);
/// add_benchmark!(params, batches, b"identity", Identity);
/// add_benchmark!(params, batches, b"session", SessionBench::<Runtime>);
/// ...
/// ```
///
/// At the end of `dispatch_benchmark`, you should return this batches object.
#[macro_export]
macro_rules! add_benchmark {
( $params:ident, $batches:ident, $name:literal, $( $location:tt )* ) => (
let (pallet, benchmark, lowest_range_values, highest_range_values, steps, repeat) = $params;
if &pallet[..] == &$name[..] || &pallet[..] == &b"*"[..] {
if &pallet[..] == &b"*"[..] || &benchmark[..] == &b"*"[..] {
for benchmark in $( $location )*::benchmarks().into_iter() {
$batches.push($crate::BenchmarkBatch {
results: $( $location )*::run_benchmark(
benchmark,
&lowest_range_values[..],
&highest_range_values[..],
&steps[..],
repeat,
)?,
pallet: pallet.to_vec(),
benchmark: benchmark.to_vec(),
});
}
} else {
$batches.push($crate::BenchmarkBatch {
results: $( $location )*::run_benchmark(
&benchmark[..],
&lowest_range_values[..],
&highest_range_values[..],
&steps[..],
repeat,
)?,
pallet: pallet.to_vec(),
benchmark: benchmark.clone(),
});
}
}
)
}
34 changes: 25 additions & 9 deletions frame/benchmarking/src/utils.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,13 +22,24 @@ use sp_io::hashing::blake2_256;
use sp_runtime::RuntimeString;

/// An alphabet of possible parameters to use for benchmarking.
#[derive(codec::Encode, codec::Decode, Clone, Copy, PartialEq, Debug)]
#[derive(Encode, Decode, Clone, Copy, PartialEq, Debug)]
#[allow(missing_docs)]
#[allow(non_camel_case_types)]
pub enum BenchmarkParameter {
a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t, u, v, w, x, y, z,
}

/// The results of a single of benchmark.
#[derive(Encode, Decode, Clone, PartialEq, Debug)]
pub struct BenchmarkBatch {
/// The pallet containing this benchmark.
pub pallet: Vec<u8>,
/// The extrinsic (or benchmark name) of this benchmark.
pub benchmark: Vec<u8>,
/// The results from this benchmark.
pub results: Vec<BenchmarkResults>,
}

/// Results from running benchmarks on a FRAME pallet.
/// Contains duration of the function call in nanoseconds along with the benchmark parameters
/// used for that benchmark result.
Expand All @@ -39,13 +50,13 @@ sp_api::decl_runtime_apis! {
pub trait Benchmark {
/// Dispatch the given benchmark.
fn dispatch_benchmark(
module: Vec<u8>,
extrinsic: Vec<u8>,
pallet: Vec<u8>,
Copy link
Contributor

@kianenigma kianenigma Mar 27, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
pallet: Vec<u8>,
pallet: &[u8],

Shouldn't these also use slice?

Copy link
Member

@shawntabrizi shawntabrizi Mar 28, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These values from from structopt, so I think they will be Vecs.

benchmark: Vec<u8>,
lowest_range_values: Vec<u32>,
highest_range_values: Vec<u32>,
steps: Vec<u32>,
repeat: u32,
) -> Result<Vec<BenchmarkResults>, RuntimeString>;
) -> Result<Vec<BenchmarkBatch>, RuntimeString>;
}
}

Expand Down Expand Up @@ -75,19 +86,24 @@ pub trait Benchmarking {

/// The pallet benchmarking trait.
pub trait Benchmarking<T> {
/// Get the benchmarks available for this pallet. Generally there is one benchmark per
/// extrinsic, so these are sometimes just called "extrinsics".
fn benchmarks() -> Vec<&'static [u8]>;

/// Run the benchmarks for this pallet.
///
/// Parameters
/// - `extrinsic`: The name of extrinsic function you want to benchmark encoded as bytes.
/// - `name`: The name of extrinsic function or benchmark you want to benchmark encoded as
/// bytes.
/// - `steps`: The number of sample points you want to take across the range of parameters.
/// - `lowest_range_values`: The lowest number for each range of parameters.
/// - `highest_range_values`: The highest number for each range of parameters.
/// - `repeat`: The number of times you want to repeat a benchmark.
fn run_benchmark(
extrinsic: Vec<u8>,
lowest_range_values: Vec<u32>,
highest_range_values: Vec<u32>,
steps: Vec<u32>,
name: &[u8],
lowest_range_values: &[u32],
highest_range_values: &[u32],
steps: &[u32],
repeat: u32,
) -> Result<Vec<T>, &'static str>;
}
Expand Down
Loading