use benchmark list in cli

bin/node/runtime/src/lib.rs

@@ -1535,18 +1535,54 @@ impl_runtime_apis! {
 	#[cfg(feature = "runtime-benchmarks")]
 	impl frame_benchmarking::Benchmark<Block> for Runtime {
 		fn benchmarks(extra: bool) -> Vec<frame_benchmarking::BenchmarkList> {
-			use frame_benchmarking::{Benchmarking, BenchmarkList};
-
-			let benchmarks = Assets::benchmarks(extra)
-				.iter()
-				.map(|b| b.to_vec())
-				.collect::<Vec<_>>();
-			let list = BenchmarkList {
-				pallet: b"Balances".to_vec(),
-				benchmarks: benchmarks.to_vec(),
-			};
-
-			return vec![list]
+			use frame_benchmarking::{list_benchmark, Benchmarking, BenchmarkList};
+
+			// Trying to add benchmarks directly to the Session Pallet caused cyclic dependency
+			// issues. To get around that, we separated the Session benchmarks into its own crate,
+			// which is why we need these two lines below.
+			use pallet_session_benchmarking::Pallet as SessionBench;
+			use pallet_offences_benchmarking::Pallet as OffencesBench;
+			use frame_system_benchmarking::Pallet as SystemBench;
+
+			impl pallet_session_benchmarking::Config for Runtime {}
+			impl pallet_offences_benchmarking::Config for Runtime {}
+			impl frame_system_benchmarking::Config for Runtime {}
+
+			let mut list = Vec::<BenchmarkList>::new();
+
+			list_benchmark!(list, extra, pallet_assets, Assets);
+			list_benchmark!(list, extra, pallet_babe, Babe);
+			list_benchmark!(list, extra, pallet_balances, Balances);
+			list_benchmark!(list, extra, pallet_bounties, Bounties);
+			list_benchmark!(list, extra, pallet_collective, Council);
+			list_benchmark!(list, extra, pallet_contracts, Contracts);
+			list_benchmark!(list, extra, pallet_democracy, Democracy);
+			list_benchmark!(list, extra, pallet_election_provider_multi_phase, ElectionProviderMultiPhase);
+			list_benchmark!(list, extra, pallet_elections_phragmen, Elections);
+			list_benchmark!(list, extra, pallet_gilt, Gilt);
+			list_benchmark!(list, extra, pallet_grandpa, Grandpa);
+			list_benchmark!(list, extra, pallet_identity, Identity);
+			list_benchmark!(list, extra, pallet_im_online, ImOnline);
+			list_benchmark!(list, extra, pallet_indices, Indices);
+			list_benchmark!(list, extra, pallet_lottery, Lottery);
+			list_benchmark!(list, extra, pallet_membership, TechnicalMembership);
+			list_benchmark!(list, extra, pallet_mmr, Mmr);
+			list_benchmark!(list, extra, pallet_multisig, Multisig);
+			list_benchmark!(list, extra, pallet_offences, OffencesBench::<Runtime>);
+			list_benchmark!(list, extra, pallet_proxy, Proxy);
+			list_benchmark!(list, extra, pallet_scheduler, Scheduler);
+			list_benchmark!(list, extra, pallet_session, SessionBench::<Runtime>);
+			list_benchmark!(list, extra, pallet_staking, Staking);
+			list_benchmark!(list, extra, frame_system, SystemBench::<Runtime>);
+			list_benchmark!(list, extra, pallet_timestamp, Timestamp);
+			list_benchmark!(list, extra, pallet_tips, Tips);
+			list_benchmark!(list, extra, pallet_transaction_storage, TransactionStorage);
+			list_benchmark!(list, extra, pallet_treasury, Treasury);
+			list_benchmark!(list, extra, pallet_uniques, Uniques);
+			list_benchmark!(list, extra, pallet_utility, Utility);
+			list_benchmark!(list, extra, pallet_vesting, Vesting);
+
+			return list
 		}
 
 		fn dispatch_benchmark(
@@ -1565,10 +1601,6 @@ impl_runtime_apis! {
 			use pallet_offences_benchmarking::Pallet as OffencesBench;
 			use frame_system_benchmarking::Pallet as SystemBench;
 
-			impl pallet_session_benchmarking::Config for Runtime {}
-			impl pallet_offences_benchmarking::Config for Runtime {}
-			impl frame_system_benchmarking::Config for Runtime {}
-
 			let whitelist: Vec<TrackedStorageKey> = vec![
 				// Block Number
 				hex_literal::hex!("26aa394eea5630e07c48ae0c9558cef702a5c1b19ab7a04f536c519aca4983ac").to_vec().into(),

frame/benchmarking/src/lib.rs

@@ -1344,62 +1344,50 @@ macro_rules! add_benchmark {
 			verify,
 			extra,
 		} = config;
-		if &pallet[..] == &name_string[..] || &pallet[..] == &b"*"[..] {
-			if &pallet[..] == &b"*"[..] || &benchmark[..] == &b"*"[..] {
-				for benchmark in $( $location )*::benchmarks(*extra).into_iter() {
-					$batches.push($crate::BenchmarkBatch {
-						pallet: name_string.to_vec(),
-						instance: instance_string.to_vec(),
-						benchmark: benchmark.to_vec(),
-						results: $( $location )*::run_benchmark(
-							benchmark,
-							&lowest_range_values[..],
-							&highest_range_values[..],
-							*steps,
-							*repeat,
-							whitelist,
-							*verify,
-						).map_err(|e| {
-							$crate::show_benchmark_debug_info(
-								instance_string,
-								benchmark,
-								lowest_range_values,
-								highest_range_values,
-								steps,
-								repeat,
-								verify,
-								e,
-							)
-						})?,
-					});
-				}
-			} else {
-				$batches.push($crate::BenchmarkBatch {
-					pallet: name_string.to_vec(),
-					instance: instance_string.to_vec(),
-					benchmark: benchmark.clone(),
-					results: $( $location )*::run_benchmark(
-						&benchmark[..],
-						&lowest_range_values[..],
-						&highest_range_values[..],
-						*steps,
-						*repeat,
-						whitelist,
-						*verify,
-					).map_err(|e| {
-						$crate::show_benchmark_debug_info(
-							instance_string,
-							benchmark,
-							lowest_range_values,
-							highest_range_values,
-							steps,
-							repeat,
-							verify,
-							e,
-						)
-					})?,
-				});
-			}
+		if &pallet[..] == &name_string[..] {
+			$batches.push($crate::BenchmarkBatch {
+				pallet: name_string.to_vec(),
+				instance: instance_string.to_vec(),
+				benchmark: benchmark.clone(),
+				results: $( $location )*::run_benchmark(
+					&benchmark[..],
+					&lowest_range_values[..],
+					&highest_range_values[..],
+					*steps,
+					*repeat,
+					whitelist,
+					*verify,
+				).map_err(|e| {
+					$crate::show_benchmark_debug_info(
+						instance_string,
+						benchmark,
+						lowest_range_values,
+						highest_range_values,
+						steps,
+						repeat,
+						verify,
+						e,
+					)
+				})?
+			});
 		}
 	)
 }
+
+#[macro_export]
+macro_rules! list_benchmark {
+	( $list:ident, $extra:ident, $name:path, $( $location:tt )* ) => (
+		let pallet_string = stringify!($name).as_bytes();
+		let instance_string = stringify!( $( $location )* ).as_bytes();
+		let benchmarks = $( $location )*::benchmarks($extra)
+			.iter()
+			.map(|b| b.to_vec())
+			.collect::<Vec<_>>();
+		let pallet_benchmarks = BenchmarkList {
+			pallet: pallet_string.to_vec(),
+			instance: instance_string.to_vec(),
+			benchmarks: benchmarks.to_vec(),
+		};
+		$list.push(pallet_benchmarks)
+	)
+}

frame/benchmarking/src/utils.rs

@@ -116,6 +116,7 @@ pub struct BenchmarkConfig {
 #[derive(Encode, Decode, Default, Clone, PartialEq, Debug)]
 pub struct BenchmarkList {
 	pub pallet: Vec<u8>,
+	pub instance: Vec<u8>,
 	pub benchmarks: Vec<Vec<u8>>,
 }
 

utils/frame/benchmarking-cli/src/command.rs

@@ -100,6 +100,8 @@ impl BenchmarkCmd {
 		let spec = config.chain_spec;
 		let wasm_method = self.wasm_method.into();
 		let strategy = self.execution.unwrap_or(ExecutionStrategy::Native);
+		let pallet = self.pallet.as_bytes();
+		let extrinsic = self.extrinsic.as_bytes();
 
 		let genesis_storage = spec.build_storage()?;
 		let mut changes = Default::default();
@@ -114,88 +116,103 @@ impl BenchmarkCmd {
 		let mut batches = Vec::new();
 		let mut storage_info = Vec::new();
 
-		for r in 0..self.repeat {
+		let extensions = || -> Extensions {
 			let mut extensions = Extensions::default();
 			extensions.register(KeystoreExt(Arc::new(KeyStore::new()) as SyncCryptoStorePtr));
 			let (offchain, _) = TestOffchainExt::new();
 			let (pool, _) = TestTransactionPoolExt::new();
 			extensions.register(OffchainWorkerExt::new(offchain.clone()));
 			extensions.register(OffchainDbExt::new(offchain));
 			extensions.register(TransactionPoolExt::new(pool));
+			return extensions
+		};
+
+		// Get Benchmark List
+		let result = StateMachine::<_, _, NumberFor<BB>, _>::new(
+			&state,
+			None,
+			&mut changes,
+			&executor,
+			"Benchmark_benchmarks",
+			&(self.extra).encode(),
+			extensions(),
+			&sp_state_machine::backend::BackendRuntimeCode::new(&state).runtime_code()?,
+			sp_core::testing::TaskExecutor::new(),
+		)
+		.execute(strategy.into())
+		.map_err(|e| format!("Error getting benchmark list: {:?}", e))?;
+
+		let list =
+			<Vec<frame_benchmarking::BenchmarkList> as Decode>::decode(&mut &result[..])
+				.map_err(|e| format!("Failed to decode benchmark list: {:?}", e))?;
+
+		if self.list {
+			// Show the list and exit early
+			for item in list {
+				println!("{}", String::from_utf8(item.pallet).unwrap());
+				for benchmark in item.benchmarks {
+					println!("- {}", String::from_utf8(benchmark).unwrap());
+				}
+			}
+			return Ok(())
+		}
 
-			if self.list {
+		// Use the benchmark list and the user input to determine the set of benchmarks to run.
+		let mut benchmarks_to_run = Vec::new();
+		for item in list {
+			if pallet == &item.pallet[..] || pallet == &b"*"[..] {
+				if &pallet[..] == &b"*"[..] || &extrinsic[..] == &b"*"[..] {
+					for benchmark in item.benchmarks {
+						benchmarks_to_run.push((item.pallet.clone(), benchmark));
+					}
+				} else {
+					benchmarks_to_run.push((pallet.to_vec(), extrinsic.to_vec()));
+				}
+			}
+		}
+
+		// Run the benchmarks
+		for (pallet, extrinsic) in benchmarks_to_run {
+			for r in 0..self.repeat {
 				let result = StateMachine::<_, _, NumberFor<BB>, _>::new(
 					&state,
 					None,
 					&mut changes,
 					&executor,
-					"Benchmark_benchmarks",
-					&(self.extra).encode(),
-					extensions,
+					"Benchmark_dispatch_benchmark",
+					&(
+						&pallet,
+						&extrinsic,
+						self.lowest_range_values.clone(),
+						self.highest_range_values.clone(),
+						(self.steps, self.steps),
+						(r, self.repeat),
+						!self.no_verify,
+						self.extra,
+					)
+						.encode(),
+					extensions(),
 					&sp_state_machine::backend::BackendRuntimeCode::new(&state).runtime_code()?,
 					sp_core::testing::TaskExecutor::new(),
 				)
 				.execute(strategy.into())
-				.map_err(|e| format!("Error getting benchmark list: {:?}", e))?;
-
-				let lists =
-					<Vec<frame_benchmarking::BenchmarkList> as Decode>::decode(&mut &result[..])
-						.map_err(|e| format!("Failed to decode benchmark list: {:?}", e))?;
+				.map_err(|e| format!("Error executing runtime benchmark: {:?}", e))?;
 
-				for list in lists {
-					println!("{}", String::from_utf8(list.pallet).unwrap());
-					for benchmark in list.benchmarks {
-						println!("- {}", String::from_utf8(benchmark).unwrap());
-					}
-				}
+				let (batch, last_storage_info) = <std::result::Result<
+					(Vec<BenchmarkBatch>, Vec<StorageInfo>),
+					String,
+				> as Decode>::decode(&mut &result[..])
+				.map_err(|e| format!("Failed to decode benchmark results: {:?}", e))??;
 
-				return Ok(())
+				batches.extend(batch);
+				storage_info = last_storage_info;
 			}
-
-			let result = StateMachine::<_, _, NumberFor<BB>, _>::new(
-				&state,
-				None,
-				&mut changes,
-				&executor,
-				"Benchmark_dispatch_benchmark",
-				&(
-					&self.pallet,
-					&self.extrinsic,
-					self.lowest_range_values.clone(),
-					self.highest_range_values.clone(),
-					(self.steps, self.steps),
-					(r, self.repeat),
-					!self.no_verify,
-					self.extra,
-				)
-					.encode(),
-				extensions,
-				&sp_state_machine::backend::BackendRuntimeCode::new(&state).runtime_code()?,
-				sp_core::testing::TaskExecutor::new(),
-			)
-			.execute(strategy.into())
-			.map_err(|e| format!("Error executing runtime benchmark: {:?}", e))?;
-
-			let (batch, last_storage_info) = <std::result::Result<
-				(Vec<BenchmarkBatch>, Vec<StorageInfo>),
-				String,
-			> as Decode>::decode(&mut &result[..])
-			.map_err(|e| format!("Failed to decode benchmark results: {:?}", e))??;
-
-			batches.extend(batch);
-			storage_info = last_storage_info;
 		}
 
+		// Combine all of the benchmark results, so that benchmarks of the same pallet/function
+		// are together.
 		let batches = combine_batches(batches);
 
-		for b in batches.clone() {
-			println!(
-				"{:?}: {:?}",
-				String::from_utf8(b.pallet).unwrap(),
-				String::from_utf8(b.benchmark).unwrap(),
-			);
-		}
-
 		if let Some(output_path) = &self.output {
 			crate::writer::write_results(&batches, &storage_info, output_path, self)?;
 		}
@@ -297,33 +314,3 @@ impl CliConfiguration for BenchmarkCmd {
 		})
 	}
 }
-
-// fn list<BB>(extra: bool) -> Result<()>
-// where
-// 	BB: BlockT + Debug,
-// 	<<<BB as BlockT>::Header as HeaderT>::Number as std::str::FromStr>::Err: std::fmt::Debug,
-// 	<BB as BlockT>::Hash: std::str::FromStr,
-// {
-// 	let result = StateMachine::<_, _, NumberFor<BB>, _>::new(
-// 		&state,
-// 		None,
-// 		&mut changes,
-// 		&executor,
-// 		"Benchmark_benchmarks",
-// 		&(extra).encode(),
-// 		extensions,
-// 		&sp_state_machine::backend::BackendRuntimeCode::new(&state).runtime_code()?,
-// 		sp_core::testing::TaskExecutor::new(),
-// 	)
-// 	.execute(strategy.into())
-// 	.map_err(|e| format!("Error getting benchmark list: {:?}", e))?;
-
-// 	let list = <std::result::Result<
-// 			Vec<frame_benchmarking::BenchmarkList>,
-// 			String,
-// 		> as Decode>::decode(&mut &result[..])
-// 		.map_err(|e| format!("Failed to decode benchmark list: {:?}", e))??;
-
-// 	println!("{:?}", list);
-// 	Ok(())
-// }