diff --git a/frame/benchmarking/src/analysis.rs b/frame/benchmarking/src/analysis.rs index c17e206c34c6c..6963d84ee614e 100644 --- a/frame/benchmarking/src/analysis.rs +++ b/frame/benchmarking/src/analysis.rs @@ -37,7 +37,35 @@ pub enum BenchmarkSelector { } impl Analysis { + // Useful for when there are no components, and we just need an median value of the benchmark results. + // Note: We choose the median value because it is more robust to outliers. + fn median_value(r: &Vec, selector: BenchmarkSelector) -> Option { + if r.is_empty() { return None } + + let mut values: Vec = r.iter().map(|result| + match selector { + BenchmarkSelector::ExtrinsicTime => result.extrinsic_time, + BenchmarkSelector::StorageRootTime => result.storage_root_time, + BenchmarkSelector::Reads => result.reads.into(), + BenchmarkSelector::Writes => result.writes.into(), + } + ).collect(); + + values.sort(); + let mid = values.len() / 2; + + Some(Self { + base: values[mid], + slopes: Vec::new(), + names: Vec::new(), + value_dists: None, + model: None, + }) + } + pub fn median_slopes(r: &Vec, selector: BenchmarkSelector) -> Option { + if r[0].components.is_empty() { return Self::median_value(r, selector) } + let results = r[0].components.iter().enumerate().map(|(i, &(param, _))| { let mut counted = BTreeMap::, usize>::new(); for result in r.iter() { @@ -114,6 +142,8 @@ impl Analysis { } pub fn min_squares_iqr(r: &Vec, selector: BenchmarkSelector) -> Option { + if r[0].components.is_empty() { return Self::median_value(r, selector) } + let mut results = BTreeMap::, Vec>::new(); for result in r.iter() { let p = result.components.iter().map(|x| x.1).collect::>(); diff --git a/frame/benchmarking/src/lib.rs b/frame/benchmarking/src/lib.rs index 532cb273c989d..f306cb5139130 100644 --- a/frame/benchmarking/src/lib.rs +++ b/frame/benchmarking/src/lib.rs @@ -794,100 +794,114 @@ macro_rules! impl_benchmark { // Default number of steps for a component. let mut prev_steps = 10; - // Select the component we will be benchmarking. Each component will be benchmarked. - for (idx, (name, low, high)) in components.iter().enumerate() { - // Get the number of steps for this component. - let steps = steps.get(idx).cloned().unwrap_or(prev_steps); - prev_steps = steps; - - // Skip this loop if steps is zero - if steps == 0 { continue } - - let lowest = lowest_range_values.get(idx).cloned().unwrap_or(*low); - let highest = highest_range_values.get(idx).cloned().unwrap_or(*high); - - let diff = highest - lowest; + let repeat_benchmark = | + repeat: u32, + c: Vec<($crate::BenchmarkParameter, u32)>, + results: &mut Vec<$crate::BenchmarkResults>, + | -> Result<(), &'static str> { + // Run the benchmark `repeat` times. + for _ in 0..repeat { + // Set up the externalities environment for the setup we want to + // benchmark. + let closure_to_benchmark = < + SelectedBenchmark as $crate::BenchmarkingSetup + >::instance(&selected_benchmark, &c)?; - // Create up to `STEPS` steps for that component between high and low. - let step_size = (diff / steps).max(1); - let num_of_steps = diff / step_size + 1; + // Set the block number to at least 1 so events are deposited. + if $crate::Zero::is_zero(&frame_system::Module::::block_number()) { + frame_system::Module::::set_block_number(1.into()); + } - for s in 0..num_of_steps { - // This is the value we will be testing for component `name` - let component_value = lowest + step_size * s; + // Commit the externalities to the database, flushing the DB cache. + // This will enable worst case scenario for reading from the database. + $crate::benchmarking::commit_db(); + + // Reset the read/write counter so we don't count operations in the setup process. + $crate::benchmarking::reset_read_write_count(); + + // Time the extrinsic logic. + frame_support::debug::trace!( + target: "benchmark", + "Start Benchmark: {:?}", c + ); + + let start_extrinsic = $crate::benchmarking::current_time(); + closure_to_benchmark()?; + let finish_extrinsic = $crate::benchmarking::current_time(); + let elapsed_extrinsic = finish_extrinsic - start_extrinsic; + // Commit the changes to get proper write count + $crate::benchmarking::commit_db(); + frame_support::debug::trace!( + target: "benchmark", + "End Benchmark: {} ns", elapsed_extrinsic + ); + let read_write_count = $crate::benchmarking::read_write_count(); + frame_support::debug::trace!( + target: "benchmark", + "Read/Write Count {:?}", read_write_count + ); + + // Time the storage root recalculation. + let start_storage_root = $crate::benchmarking::current_time(); + $crate::storage_root(); + let finish_storage_root = $crate::benchmarking::current_time(); + let elapsed_storage_root = finish_storage_root - start_storage_root; + + results.push($crate::BenchmarkResults { + components: c.clone(), + extrinsic_time: elapsed_extrinsic, + storage_root_time: elapsed_storage_root, + reads: read_write_count.0, + repeat_reads: read_write_count.1, + writes: read_write_count.2, + repeat_writes: read_write_count.3, + }); + + // Wipe the DB back to the genesis state. + $crate::benchmarking::wipe_db(); + } - // Select the max value for all the other components. - let c: Vec<($crate::BenchmarkParameter, u32)> = components.iter() - .enumerate() - .map(|(idx, (n, _, h))| - if n == name { - (*n, component_value) - } else { - (*n, *highest_range_values.get(idx).unwrap_or(h)) - } - ) - .collect(); + Ok(()) + }; - // Run the benchmark `repeat` times. - for _ in 0..repeat { - // Set up the externalities environment for the setup we want to - // benchmark. - let closure_to_benchmark = < - SelectedBenchmark as $crate::BenchmarkingSetup - >::instance(&selected_benchmark, &c)?; - - // Set the block number to at least 1 so events are deposited. - if $crate::Zero::is_zero(&frame_system::Module::::block_number()) { - frame_system::Module::::set_block_number(1.into()); - } - - // Commit the externalities to the database, flushing the DB cache. - // This will enable worst case scenario for reading from the database. - $crate::benchmarking::commit_db(); - - // Reset the read/write counter so we don't count operations in the setup process. - $crate::benchmarking::reset_read_write_count(); - - // Time the extrinsic logic. - frame_support::debug::trace!( - target: "benchmark", - "Start Benchmark: {:?} {:?}", name, component_value - ); - - let start_extrinsic = $crate::benchmarking::current_time(); - closure_to_benchmark()?; - let finish_extrinsic = $crate::benchmarking::current_time(); - let elapsed_extrinsic = finish_extrinsic - start_extrinsic; - // Commit the changes to get proper write count - $crate::benchmarking::commit_db(); - frame_support::debug::trace!( - target: "benchmark", - "End Benchmark: {} ns", elapsed_extrinsic - ); - let read_write_count = $crate::benchmarking::read_write_count(); - frame_support::debug::trace!( - target: "benchmark", - "Read/Write Count {:?}", read_write_count - ); - - // Time the storage root recalculation. - let start_storage_root = $crate::benchmarking::current_time(); - $crate::storage_root(); - let finish_storage_root = $crate::benchmarking::current_time(); - let elapsed_storage_root = finish_storage_root - start_storage_root; - - results.push($crate::BenchmarkResults { - components: c.clone(), - extrinsic_time: elapsed_extrinsic, - storage_root_time: elapsed_storage_root, - reads: read_write_count.0, - repeat_reads: read_write_count.1, - writes: read_write_count.2, - repeat_writes: read_write_count.3, - }); - - // Wipe the DB back to the genesis state. - $crate::benchmarking::wipe_db(); + if components.is_empty() { + repeat_benchmark(repeat, Default::default(), &mut results)?; + } else { + // Select the component we will be benchmarking. Each component will be benchmarked. + for (idx, (name, low, high)) in components.iter().enumerate() { + // Get the number of steps for this component. + let steps = steps.get(idx).cloned().unwrap_or(prev_steps); + prev_steps = steps; + + // Skip this loop if steps is zero + if steps == 0 { continue } + + let lowest = lowest_range_values.get(idx).cloned().unwrap_or(*low); + let highest = highest_range_values.get(idx).cloned().unwrap_or(*high); + + let diff = highest - lowest; + + // Create up to `STEPS` steps for that component between high and low. + let step_size = (diff / steps).max(1); + let num_of_steps = diff / step_size + 1; + + for s in 0..num_of_steps { + // This is the value we will be testing for component `name` + let component_value = lowest + step_size * s; + + // Select the max value for all the other components. + let c: Vec<($crate::BenchmarkParameter, u32)> = components.iter() + .enumerate() + .map(|(idx, (n, _, h))| + if n == name { + (*n, component_value) + } else { + (*n, *highest_range_values.get(idx).unwrap_or(h)) + } + ) + .collect(); + + repeat_benchmark(repeat, c, &mut results)?; } } } @@ -938,99 +952,117 @@ macro_rules! impl_benchmark { // Default number of steps for a component. let mut prev_steps = 10; - // Select the component we will be benchmarking. Each component will be benchmarked. - for (idx, (name, low, high)) in components.iter().enumerate() { - // Get the number of steps for this component. - let steps = steps.get(idx).cloned().unwrap_or(prev_steps); - prev_steps = steps; + let repeat_benchmark = | + repeat: u32, + c: Vec<($crate::BenchmarkParameter, u32)>, + results: &mut Vec<$crate::BenchmarkResults>, + | -> Result<(), &'static str> { + // Run the benchmark `repeat` times. + for _ in 0..repeat { + // Set up the externalities environment for the setup we want to + // benchmark. + let closure_to_benchmark = < + SelectedBenchmark as $crate::BenchmarkingSetupInstance + >::instance(&selected_benchmark, &c)?; - // Skip this loop if steps is zero - if steps == 0 { continue } - - let lowest = lowest_range_values.get(idx).cloned().unwrap_or(*low); - let highest = highest_range_values.get(idx).cloned().unwrap_or(*high); - - let diff = highest - lowest; - - // Create up to `STEPS` steps for that component between high and low. - let step_size = (diff / steps).max(1); - let num_of_steps = diff / step_size + 1; + // Set the block number to at least 1 so events are deposited. + if $crate::Zero::is_zero(&frame_system::Module::::block_number()) { + frame_system::Module::::set_block_number(1.into()); + } - for s in 0..num_of_steps { - // This is the value we will be testing for component `name` - let component_value = lowest + step_size * s; + // Commit the externalities to the database, flushing the DB cache. + // This will enable worst case scenario for reading from the database. + $crate::benchmarking::commit_db(); + + // Reset the read/write counter so we don't count operations in the setup process. + $crate::benchmarking::reset_read_write_count(); + + // Time the extrinsic logic. + frame_support::debug::trace!( + target: "benchmark", + "Start Benchmark: {:?}", + c, + ); + + let start_extrinsic = $crate::benchmarking::current_time(); + closure_to_benchmark()?; + let finish_extrinsic = $crate::benchmarking::current_time(); + let elapsed_extrinsic = finish_extrinsic - start_extrinsic; + // Commit the changes to get proper write count + $crate::benchmarking::commit_db(); + frame_support::debug::trace!( + target: "benchmark", + "End Benchmark: {} ns", + elapsed_extrinsic, + ); + let read_write_count = $crate::benchmarking::read_write_count(); + frame_support::debug::trace!( + target: "benchmark", + "Read/Write Count {:?}", + read_write_count, + ); + + // Time the storage root recalculation. + let start_storage_root = $crate::benchmarking::current_time(); + $crate::storage_root(); + let finish_storage_root = $crate::benchmarking::current_time(); + let elapsed_storage_root = finish_storage_root - start_storage_root; + + results.push($crate::BenchmarkResults { + components: c.clone(), + extrinsic_time: elapsed_extrinsic, + storage_root_time: elapsed_storage_root, + reads: read_write_count.0, + repeat_reads: read_write_count.1, + writes: read_write_count.2, + repeat_writes: read_write_count.3, + }); + + // Wipe the DB back to the genesis state. + $crate::benchmarking::wipe_db(); + } - // Select the max value for all the other components. - let c: Vec<($crate::BenchmarkParameter, u32)> = components.iter() - .enumerate() - .map(|(idx, (n, _, h))| - if n == name { - (*n, component_value) - } else { - (*n, *highest_range_values.get(idx).unwrap_or(h)) - } - ) - .collect(); + Ok(()) + }; - // Run the benchmark `repeat` times. - for _ in 0..repeat { - // Set up the externalities environment for the setup we want to benchmark. - let closure_to_benchmark = < - SelectedBenchmark as $crate::BenchmarkingSetupInstance - >::instance(&selected_benchmark, &c)?; - - // Set the block number to at least 1 so events are deposited. - if $crate::Zero::is_zero(&frame_system::Module::::block_number()) { - frame_system::Module::::set_block_number(1.into()); - } - - // Commit the externalities to the database, flushing the DB cache. - // This will enable worst case scenario for reading from the database. - $crate::benchmarking::commit_db(); - - // Reset the read/write counter so we don't count operations in the setup process. - $crate::benchmarking::reset_read_write_count(); - - // Time the extrinsic logic. - frame_support::debug::trace!( - target: "benchmark", - "Start Benchmark: {:?} {:?}", name, component_value - ); - - let start_extrinsic = $crate::benchmarking::current_time(); - closure_to_benchmark()?; - let finish_extrinsic = $crate::benchmarking::current_time(); - let elapsed_extrinsic = finish_extrinsic - start_extrinsic; - // Commit the changes to get proper write count - $crate::benchmarking::commit_db(); - frame_support::debug::trace!( - target: "benchmark", - "End Benchmark: {} ns", elapsed_extrinsic - ); - let read_write_count = $crate::benchmarking::read_write_count(); - frame_support::debug::trace!( - target: "benchmark", - "Read/Write Count {:?}", read_write_count - ); - - // Time the storage root recalculation. - let start_storage_root = $crate::benchmarking::current_time(); - $crate::storage_root(); - let finish_storage_root = $crate::benchmarking::current_time(); - let elapsed_storage_root = finish_storage_root - start_storage_root; - - results.push($crate::BenchmarkResults { - components: c.clone(), - extrinsic_time: elapsed_extrinsic, - storage_root_time: elapsed_storage_root, - reads: read_write_count.0, - repeat_reads: read_write_count.1, - writes: read_write_count.2, - repeat_writes: read_write_count.3, - }); - - // Wipe the DB back to the genesis state. - $crate::benchmarking::wipe_db(); + if components.is_empty() { + repeat_benchmark(repeat, Default::default(), &mut results)?; + } else { + // Select the component we will be benchmarking. Each component will be benchmarked. + for (idx, (name, low, high)) in components.iter().enumerate() { + // Get the number of steps for this component. + let steps = steps.get(idx).cloned().unwrap_or(prev_steps); + prev_steps = steps; + + // Skip this loop if steps is zero + if steps == 0 { continue } + + let lowest = lowest_range_values.get(idx).cloned().unwrap_or(*low); + let highest = highest_range_values.get(idx).cloned().unwrap_or(*high); + + let diff = highest - lowest; + + // Create up to `STEPS` steps for that component between high and low. + let step_size = (diff / steps).max(1); + let num_of_steps = diff / step_size + 1; + + for s in 0..num_of_steps { + // This is the value we will be testing for component `name` + let component_value = lowest + step_size * s; + + // Select the max value for all the other components. + let c: Vec<($crate::BenchmarkParameter, u32)> = components.iter() + .enumerate() + .map(|(idx, (n, _, h))| + if n == name { + (*n, component_value) + } else { + (*n, *highest_range_values.get(idx).unwrap_or(h)) + } + ) + .collect(); + + repeat_benchmark(repeat, c, &mut results)?; } } } @@ -1060,40 +1092,48 @@ macro_rules! impl_benchmark_test { SelectedBenchmark as $crate::BenchmarkingSetup >::components(&selected_benchmark); - assert!( - components.len() != 0, - "You need to add components to your benchmark!", - ); - for (_, (name, low, high)) in components.iter().enumerate() { - // Test only the low and high value, assuming values in the middle won't break - for component_value in vec![low, high] { - // Select the max value for all the other components. - let c: Vec<($crate::BenchmarkParameter, u32)> = components.iter() - .enumerate() - .map(|(_, (n, _, h))| - if n == name { - (*n, *component_value) - } else { - (*n, *h) - } - ) - .collect(); + let execute_benchmark = | + c: Vec<($crate::BenchmarkParameter, u32)> + | -> Result<(), &'static str> { + // Set up the verification state + let closure_to_verify = < + SelectedBenchmark as $crate::BenchmarkingSetup + >::verify(&selected_benchmark, &c)?; + + // Set the block number to at least 1 so events are deposited. + if $crate::Zero::is_zero(&frame_system::Module::::block_number()) { + frame_system::Module::::set_block_number(1.into()); + } - // Set up the verification state - let closure_to_verify = < - SelectedBenchmark as $crate::BenchmarkingSetup - >::verify(&selected_benchmark, &c)?; + // Run verification + closure_to_verify()?; - // Set the block number to at least 1 so events are deposited. - if $crate::Zero::is_zero(&frame_system::Module::::block_number()) { - frame_system::Module::::set_block_number(1.into()); - } + // Reset the state + $crate::benchmarking::wipe_db(); - // Run verification - closure_to_verify()?; + Ok(()) + }; - // Reset the state - $crate::benchmarking::wipe_db(); + if components.is_empty() { + execute_benchmark(Default::default())?; + } else { + for (_, (name, low, high)) in components.iter().enumerate() { + // Test only the low and high value, assuming values in the middle won't break + for component_value in vec![low, high] { + // Select the max value for all the other components. + let c: Vec<($crate::BenchmarkParameter, u32)> = components.iter() + .enumerate() + .map(|(_, (n, _, h))| + if n == name { + (*n, *component_value) + } else { + (*n, *h) + } + ) + .collect(); + + execute_benchmark(c)?; + } } } Ok(()) @@ -1114,6 +1154,28 @@ macro_rules! impl_benchmark_test { SelectedBenchmark as $crate::BenchmarkingSetupInstance >::components(&selected_benchmark); + let execute_benchmark = | + c: Vec<($crate::BenchmarkParameter, u32)> + | -> Result<(), &'static str> { + // Set up the verification state + let closure_to_verify = < + SelectedBenchmark as $crate::BenchmarkingSetupInstance + >::verify(&selected_benchmark, &c)?; + + // Set the block number to at least 1 so events are deposited. + if $crate::Zero::is_zero(&frame_system::Module::::block_number()) { + frame_system::Module::::set_block_number(1.into()); + } + + // Run verification + closure_to_verify()?; + + // Reset the state + $crate::benchmarking::wipe_db(); + + Ok(()) + }; + for (_, (name, low, high)) in components.iter().enumerate() { // Test only the low and high value, assuming values in the middle won't break for component_value in vec![low, high] { @@ -1129,21 +1191,7 @@ macro_rules! impl_benchmark_test { ) .collect(); - // Set up the verification state - let closure_to_verify = < - SelectedBenchmark as $crate::BenchmarkingSetupInstance - >::verify(&selected_benchmark, &c)?; - - // Set the block number to at least 1 so events are deposited. - if $crate::Zero::is_zero(&frame_system::Module::::block_number()) { - frame_system::Module::::set_block_number(1.into()); - } - - // Run verification - closure_to_verify()?; - - // Reset the state - $crate::benchmarking::wipe_db(); + execute_benchmark(c)?; } } Ok(()) diff --git a/frame/benchmarking/src/tests.rs b/frame/benchmarking/src/tests.rs index 56e6da149438c..6a4dc7eee4ee7 100644 --- a/frame/benchmarking/src/tests.rs +++ b/frame/benchmarking/src/tests.rs @@ -165,6 +165,10 @@ benchmarks!{ verify { ensure!(m[0] == 0, "You forgot to sort!") } + + no_components { + let caller = account::("caller", 0, 0); + }: set_value(RawOrigin::Signed(caller), 0) } #[test] @@ -240,5 +244,6 @@ fn benchmarks_generate_unit_tests() { assert_ok!(test_benchmark_sort_vector::()); assert_err!(test_benchmark_bad_origin::(), "Bad origin"); assert_err!(test_benchmark_bad_verify::(), "You forgot to sort!"); + assert_ok!(test_benchmark_no_components::()); }); } diff --git a/utils/frame/benchmarking-cli/src/command.rs b/utils/frame/benchmarking-cli/src/command.rs index 09b246e4766a3..550d2c12c83e5 100644 --- a/utils/frame/benchmarking-cli/src/command.rs +++ b/utils/frame/benchmarking-cli/src/command.rs @@ -89,78 +89,81 @@ impl BenchmarkCmd { let results = , String> as Decode>::decode(&mut &result[..]) .map_err(|e| format!("Failed to decode benchmark results: {:?}", e))?; - if self.output { - if self.weight_trait { - let mut file = crate::writer::open_file("traits.rs")?; - crate::writer::write_trait(&mut file, results.clone())?; - } else { - let mut file = crate::writer::open_file("benchmarks.rs")?; - crate::writer::write_results(&mut file, results.clone())?; - } - } - match results { - Ok(batches) => for batch in batches.into_iter() { - // Print benchmark metadata - println!( - "Pallet: {:?}, Extrinsic: {:?}, Lowest values: {:?}, Highest values: {:?}, Steps: {:?}, Repeat: {:?}", - String::from_utf8(batch.pallet).expect("Encoded from String; qed"), - String::from_utf8(batch.benchmark).expect("Encoded from String; qed"), - self.lowest_range_values, - self.highest_range_values, - self.steps, - self.repeat, - ); - - // Skip raw data + analysis if there are no results - if batch.results.len() == 0 { continue } - - if self.raw_data { - // Print the table header - batch.results[0].components.iter().for_each(|param| print!("{:?},", param.0)); - - print!("extrinsic_time,storage_root_time,reads,repeat_reads,writes,repeat_writes\n"); - // Print the values - batch.results.iter().for_each(|result| { - let parameters = &result.components; - parameters.iter().for_each(|param| print!("{:?},", param.1)); - // Print extrinsic time and storage root time - print!("{:?},{:?},{:?},{:?},{:?},{:?}\n", - result.extrinsic_time, - result.storage_root_time, - result.reads, - result.repeat_reads, - result.writes, - result.repeat_writes, - ); - }); - - println!(); - } - - // Conduct analysis. - if !self.no_median_slopes { - println!("Median Slopes Analysis\n========"); - if let Some(analysis) = Analysis::median_slopes(&batch.results, BenchmarkSelector::ExtrinsicTime) { - println!("-- Extrinsic Time --\n{}", analysis); - } - if let Some(analysis) = Analysis::median_slopes(&batch.results, BenchmarkSelector::Reads) { - println!("Reads = {:?}", analysis); - } - if let Some(analysis) = Analysis::median_slopes(&batch.results, BenchmarkSelector::Writes) { - println!("Writes = {:?}", analysis); + Ok(batches) => { + // If we are going to output results to a file... + if self.output { + if self.weight_trait { + let mut file = crate::writer::open_file("traits.rs")?; + crate::writer::write_trait(&mut file, batches.clone())?; + } else { + let mut file = crate::writer::open_file("benchmarks.rs")?; + crate::writer::write_results(&mut file, batches.clone())?; } } - if !self.no_min_squares { - println!("Min Squares Analysis\n========"); - if let Some(analysis) = Analysis::min_squares_iqr(&batch.results, BenchmarkSelector::ExtrinsicTime) { - println!("-- Extrinsic Time --\n{}", analysis); + + for batch in batches.into_iter() { + // Print benchmark metadata + println!( + "Pallet: {:?}, Extrinsic: {:?}, Lowest values: {:?}, Highest values: {:?}, Steps: {:?}, Repeat: {:?}", + String::from_utf8(batch.pallet).expect("Encoded from String; qed"), + String::from_utf8(batch.benchmark).expect("Encoded from String; qed"), + self.lowest_range_values, + self.highest_range_values, + self.steps, + self.repeat, + ); + + // Skip raw data + analysis if there are no results + if batch.results.is_empty() { continue } + + if self.raw_data { + // Print the table header + batch.results[0].components.iter().for_each(|param| print!("{:?},", param.0)); + + print!("extrinsic_time,storage_root_time,reads,repeat_reads,writes,repeat_writes\n"); + // Print the values + batch.results.iter().for_each(|result| { + let parameters = &result.components; + parameters.iter().for_each(|param| print!("{:?},", param.1)); + // Print extrinsic time and storage root time + print!("{:?},{:?},{:?},{:?},{:?},{:?}\n", + result.extrinsic_time, + result.storage_root_time, + result.reads, + result.repeat_reads, + result.writes, + result.repeat_writes, + ); + }); + + println!(); } - if let Some(analysis) = Analysis::min_squares_iqr(&batch.results, BenchmarkSelector::Reads) { - println!("Reads = {:?}", analysis); + + // Conduct analysis. + if !self.no_median_slopes { + println!("Median Slopes Analysis\n========"); + if let Some(analysis) = Analysis::median_slopes(&batch.results, BenchmarkSelector::ExtrinsicTime) { + println!("-- Extrinsic Time --\n{}", analysis); + } + if let Some(analysis) = Analysis::median_slopes(&batch.results, BenchmarkSelector::Reads) { + println!("Reads = {:?}", analysis); + } + if let Some(analysis) = Analysis::median_slopes(&batch.results, BenchmarkSelector::Writes) { + println!("Writes = {:?}", analysis); + } } - if let Some(analysis) = Analysis::min_squares_iqr(&batch.results, BenchmarkSelector::Writes) { - println!("Writes = {:?}", analysis); + if !self.no_min_squares { + println!("Min Squares Analysis\n========"); + if let Some(analysis) = Analysis::min_squares_iqr(&batch.results, BenchmarkSelector::ExtrinsicTime) { + println!("-- Extrinsic Time --\n{}", analysis); + } + if let Some(analysis) = Analysis::min_squares_iqr(&batch.results, BenchmarkSelector::Reads) { + println!("Reads = {:?}", analysis); + } + if let Some(analysis) = Analysis::min_squares_iqr(&batch.results, BenchmarkSelector::Writes) { + println!("Writes = {:?}", analysis); + } } } }, diff --git a/utils/frame/benchmarking-cli/src/writer.rs b/utils/frame/benchmarking-cli/src/writer.rs index bd411b536a826..199dbb795e581 100644 --- a/utils/frame/benchmarking-cli/src/writer.rs +++ b/utils/frame/benchmarking-cli/src/writer.rs @@ -30,12 +30,15 @@ pub fn open_file(path: &str) -> Result { .open(path) } -pub fn write_trait(file: &mut File, batches: Result, String>) -> Result<(), std::io::Error> { - let batches = batches.unwrap(); - +pub fn write_trait(file: &mut File, batches: Vec) -> Result<(), std::io::Error> { let mut current_pallet = Vec::::new(); - batches.iter().for_each(|batch| { + // Skip writing if there are no batches + if batches.is_empty() { return Ok(()) } + + for batch in &batches { + // Skip writing if there are no results + if batch.results.is_empty() { continue } let pallet_string = String::from_utf8(batch.pallet.clone()).unwrap(); let benchmark_string = String::from_utf8(batch.benchmark.clone()).unwrap(); @@ -55,7 +58,7 @@ pub fn write_trait(file: &mut File, batches: Result, String> } // function name - write!(file, " fn {}(", benchmark_string).unwrap(); + write!(file, "\tfn {}(", benchmark_string).unwrap(); // params let components = &batch.results[0].components; @@ -64,7 +67,7 @@ pub fn write_trait(file: &mut File, batches: Result, String> } // return value write!(file, ") -> Weight;\n").unwrap(); - }); + } // final close trait write!(file, "}}\n").unwrap(); @@ -72,7 +75,8 @@ pub fn write_trait(file: &mut File, batches: Result, String> // Reset current_pallet = Vec::::new(); - batches.iter().for_each(|batch| { + for batch in &batches { + if batch.results.is_empty() { continue } let benchmark_string = String::from_utf8(batch.benchmark.clone()).unwrap(); @@ -91,7 +95,7 @@ pub fn write_trait(file: &mut File, batches: Result, String> } // function name - write!(file, " fn {}(", benchmark_string).unwrap(); + write!(file, "\tfn {}(", benchmark_string).unwrap(); // params let components = &batch.results[0].components; @@ -100,7 +104,7 @@ pub fn write_trait(file: &mut File, batches: Result, String> } // return value write!(file, ") -> Weight {{ 1_000_000_000 }}\n").unwrap(); - }); + } // final close trait write!(file, "}}\n").unwrap(); @@ -108,15 +112,18 @@ pub fn write_trait(file: &mut File, batches: Result, String> Ok(()) } -pub fn write_results(file: &mut File, batches: Result, String>) -> Result<(), std::io::Error> { - let batches = batches.unwrap(); - +pub fn write_results(file: &mut File, batches: Vec) -> Result<(), std::io::Error> { let mut current_pallet = Vec::::new(); + // Skip writing if there are no batches + if batches.is_empty() { return Ok(()) } + // general imports write!(file, "use frame_support::weights::{{Weight, constants::RocksDbWeight as DbWeight}};\n").unwrap(); - batches.iter().for_each(|batch| { + for batch in &batches { + // Skip writing if there are no results + if batch.results.is_empty() { continue } let pallet_string = String::from_utf8(batch.pallet.clone()).unwrap(); let benchmark_string = String::from_utf8(batch.benchmark.clone()).unwrap(); @@ -143,7 +150,7 @@ pub fn write_results(file: &mut File, batches: Result, Strin } // function name - write!(file, " fn {}(", benchmark_string).unwrap(); + write!(file, "\tfn {}(", benchmark_string).unwrap(); // params let components = &batch.results[0].components; @@ -154,35 +161,35 @@ pub fn write_results(file: &mut File, batches: Result, Strin write!(file, ") -> Weight {{\n").unwrap(); let extrinsic_time = Analysis::min_squares_iqr(&batch.results, BenchmarkSelector::ExtrinsicTime).unwrap(); - write!(file, " ({} as Weight)\n", extrinsic_time.base.saturating_mul(1000)).unwrap(); + write!(file, "\t\t({} as Weight)\n", extrinsic_time.base.saturating_mul(1000)).unwrap(); extrinsic_time.slopes.iter().zip(extrinsic_time.names.iter()).for_each(|(slope, name)| { - write!(file, " .saturating_add(({} as Weight).saturating_mul({} as Weight))\n", + write!(file, "\t\t\t.saturating_add(({} as Weight).saturating_mul({} as Weight))\n", slope.saturating_mul(1000), name, ).unwrap(); }); let reads = Analysis::min_squares_iqr(&batch.results, BenchmarkSelector::Reads).unwrap(); - write!(file, " .saturating_add(DbWeight::get().reads({} as Weight))\n", reads.base).unwrap(); + write!(file, "\t\t\t.saturating_add(DbWeight::get().reads({} as Weight))\n", reads.base).unwrap(); reads.slopes.iter().zip(reads.names.iter()).for_each(|(slope, name)| { - write!(file, " .saturating_add(DbWeight::get().reads(({} as Weight).saturating_mul({} as Weight)))\n", + write!(file, "\t\t\t.saturating_add(DbWeight::get().reads(({} as Weight).saturating_mul({} as Weight)))\n", slope, name, ).unwrap(); }); let writes = Analysis::min_squares_iqr(&batch.results, BenchmarkSelector::Writes).unwrap(); - write!(file, " .saturating_add(DbWeight::get().writes({} as Weight))\n", writes.base).unwrap(); + write!(file, "\t\t\t.saturating_add(DbWeight::get().writes({} as Weight))\n", writes.base).unwrap(); writes.slopes.iter().zip(writes.names.iter()).for_each(|(slope, name)| { - write!(file, " .saturating_add(DbWeight::get().writes(({} as Weight).saturating_mul({} as Weight)))\n", + write!(file, "\t\t\t.saturating_add(DbWeight::get().writes(({} as Weight).saturating_mul({} as Weight)))\n", slope, name, ).unwrap(); }); // close function - write!(file, " }}\n").unwrap(); - }); + write!(file, "\t}}\n").unwrap(); + } // final close trait write!(file, "}}\n").unwrap();