Benchmarking externalities

bin/node-template/src/cli.rs

@@ -7,6 +7,7 @@ use sc_cli::{display_role, informant, parse_and_prepare, ParseAndPrepare, NoCust
 use sc_service::{AbstractService, Roles as ServiceRoles, Configuration};
 use sp_consensus_aura::sr25519::{AuthorityPair as AuraPair};
 use crate::chain_spec;
+use node_template_runtime::opaque::Block;
 use log::info;
 
 /// Parse command line arguments into service configuration.
@@ -51,7 +52,7 @@ pub fn run<I, T, E>(args: I, exit: E, version: VersionInfo) -> error::Result<()>
 		ParseAndPrepare::CheckBlock(cmd) => cmd.run_with_builder(|config: Config<_>|
 			Ok(new_full_start!(config).0), load_spec, exit),
 		ParseAndPrepare::PurgeChain(cmd) => cmd.run(load_spec),
-		ParseAndPrepare::Benchmark(cmd) => cmd.run(load_spec),
+		ParseAndPrepare::Benchmark(cmd) => cmd.run::<Block, service::Executor, _, _, _>(load_spec),
 		ParseAndPrepare::RevertChain(cmd) => cmd.run_with_builder(|config: Config<_>|
 			Ok(new_full_start!(config).0), load_spec),
 		ParseAndPrepare::CustomCommand(_) => Ok(())

bin/node/cli/src/cli.rs

@@ -24,6 +24,7 @@ use sc_cli::{display_role, parse_and_prepare, GetSharedParams, ParseAndPrepare};
 use crate::{service, ChainSpec, load_spec};
 use crate::factory_impl::FactoryState;
 use node_transaction_factory::RuntimeAdapter;
+use node_runtime::Block;
 use futures::{channel::oneshot, future::{select, Either}};
 
 /// Custom subcommands.
@@ -136,7 +137,7 @@ pub fn run<I, T, E>(args: I, exit: E, version: sc_cli::VersionInfo) -> error::Re
 		ParseAndPrepare::CheckBlock(cmd) => cmd.run_with_builder(|config: Config<_, _>|
 			Ok(new_full_start!(config).0), load_spec, exit),
 		ParseAndPrepare::PurgeChain(cmd) => cmd.run(load_spec),
-		ParseAndPrepare::Benchmark(cmd) => cmd.run(load_spec),
+		ParseAndPrepare::Benchmark(cmd) => cmd.run::<Block, node_executor::Executor, _, _, _>(load_spec),
 		ParseAndPrepare::RevertChain(cmd) => cmd.run_with_builder(|config: Config<_, _>|
 			Ok(new_full_start!(config).0), load_spec),
 		ParseAndPrepare::CustomCommand(CustomSubcommands::Factory(cli_args)) => {

client/cli/src/lib.rs

@@ -245,7 +245,7 @@ where
 			ParseAndPreparePurge { params, version }
 		),
 		params::CoreParams::Benchmark(params) => ParseAndPrepare::Benchmark(
-			ParseAndPrepareBenchmark { params, version }
+			ParseAndPrepareBenchmark { params }
 		),
 		params::CoreParams::Revert(params) => ParseAndPrepare::RevertChain(
 			ParseAndPrepareRevert { params, version }
@@ -283,7 +283,7 @@ pub enum ParseAndPrepare<'a, CC, RP> {
 	/// Command ready to purge the chain.
 	PurgeChain(ParseAndPreparePurge<'a>),
 	/// Command ready to benchmark the chain.
-	Benchmark(ParseAndPrepareBenchmark<'a>),
+	Benchmark(ParseAndPrepareBenchmark),
 	/// Command ready to revert the chain.
 	RevertChain(ParseAndPrepareRevert<'a>),
 	/// An additional custom command passed to `parse_and_prepare`.
@@ -300,7 +300,7 @@ impl<'a, CC, RP> ParseAndPrepare<'a, CC, RP> where CC: GetSharedParams {
 			ParseAndPrepare::ImportBlocks(c) => Some(&c.params.shared_params),
 			ParseAndPrepare::CheckBlock(c) => Some(&c.params.shared_params),
 			ParseAndPrepare::PurgeChain(c) => Some(&c.params.shared_params),
-			ParseAndPrepare::Benchmark(c) => Some(&c.params.shared_params),
+			ParseAndPrepare::Benchmark(_) => None,
 			ParseAndPrepare::RevertChain(c) => Some(&c.params.shared_params),
 			ParseAndPrepare::CustomCommand(c) => c.shared_params(),
 		}
@@ -367,13 +367,8 @@ impl<'a, CC, RP> ParseAndPrepare<'a, CC, RP> {
 					c.version,
 					default_base_path,
 				)).transpose(),
-			ParseAndPrepare::Benchmark(c) =>
-				Some(create_config_with_db_path(
-					spec_factory,
-					&c.params.shared_params,
-					c.version,
-					default_base_path,
-				)).transpose(),
+			ParseAndPrepare::Benchmark(_) =>
+				Ok(None),
 			ParseAndPrepare::RevertChain(c) =>
 				Some(create_config_with_db_path(
 					spec_factory,
@@ -729,44 +724,33 @@ impl<'a> ParseAndPreparePurge<'a> {
 }
 
 /// Command ready to benchmark the runtime.
-pub struct ParseAndPrepareBenchmark<'a> {
+pub struct ParseAndPrepareBenchmark {
 	params: BenchmarkCmd,
-	version: &'a VersionInfo,
 }
 
-impl<'a> ParseAndPrepareBenchmark<'a> {
+impl ParseAndPrepareBenchmark {
 	/// Runs the command and benchmarks the chain.
-	pub fn run<G, E, S>(
+	pub fn run<B, D, G, E, S>(
 		self,
 		spec_factory: S
 	) -> error::Result<()> where
+		B: BlockT,
+		D: sc_service::NativeExecutionDispatch + 'static,
 		S: FnOnce(&str) -> Result<Option<ChainSpec<G, E>>, String>,
 		G: RuntimeGenesis,
 		E: ChainSpecExtension,
 	{
-		let mut config = create_config_with_db_path::<(), _, _, _>(
-			spec_factory,
-			&self.params.shared_params,
-			self.version,
-			None,
-		)?;
-		fill_config_keystore_in_memory(&mut config)?;
-		let db_path = match config.database {
-			DatabaseConfig::Path { path, .. } => path,
-			_ => {
-				eprintln!("Cannot purge custom database implementation");
-				return Ok(());
-			}
+		let shared_params = SharedParams {
+			chain: self.params.chain,
+			dev: false,
+			base_path: None,
+			log: self.params.log
 		};
-
-		if self.params.pallet.is_some() {
-			print!("Input contains: {:?}", self.params.pallet.unwrap())
-		} else {
-			print!("Input is Empty");
-		}
-
-		pallet_identity::run_benchmarks();
-
+		init_logger(shared_params.log.as_ref().map(|v| v.as_ref()).unwrap_or(""));
+		let spec = load_spec(&shared_params, spec_factory)?;
+		let execution_strategy = self.params.execution.unwrap_or(ExecutionStrategy::Native).into();
+		let wasm_method = self.params.wasm_method.into();
+		sc_service::chain_ops::benchmark_runtime::<B, D>(execution_strategy, wasm_method)?;
 		Ok(())
 	}
 }

client/cli/src/params.rs

@@ -861,9 +861,32 @@ pub struct BenchmarkCmd {
 	#[structopt(short, long)]
 	pub pallet: Option<String>,
 
-	#[allow(missing_docs)]
-	#[structopt(flatten)]
-	pub shared_params: SharedParams,
+	/// The execution strategy that should be used for benchmarks
+	#[structopt(
+		long = "execution",
+		value_name = "STRATEGY",
+		possible_values = &ExecutionStrategy::variants(),
+		case_insensitive = true,
+	)]
+	pub execution: Option<ExecutionStrategy>,
+
+	/// Method for executing Wasm runtime code.
+	#[structopt(
+		long = "wasm-execution",
+		value_name = "METHOD",
+		possible_values = &WasmExecutionMethod::enabled_variants(),
+		case_insensitive = true,
+		default_value = "Interpreted"
+	)]
+	pub wasm_method: WasmExecutionMethod,
+
+	/// Specify the chain specification (one of dev, local or staging).
+	#[structopt(long = "chain", value_name = "CHAIN_SPEC")]
+	pub chain: Option<String>,
+
+	/// Sets a custom logging filter.
+	#[structopt(short = "l", long = "log", value_name = "LOG_PATTERN")]
+	pub log: Option<String>,
 }
 
 /// All core commands that are provided by default.

client/db/Cargo.toml

@@ -7,6 +7,7 @@ edition = "2018"
 [dependencies]
 parking_lot = "0.9.0"
 log = "0.4.8"
+rand = "0.7"
 kvdb = "0.3.0"
 kvdb-rocksdb = { version = "0.4", optional = true }
 kvdb-memorydb = "0.3.0"

client/db/src/lib.rs

@@ -28,6 +28,7 @@
 
 pub mod light;
 pub mod offchain;
+pub mod bench;
 
 mod children;
 mod cache;
@@ -79,6 +80,7 @@ use crate::storage_cache::{CachingState, SharedCache, new_shared_cache};
 use crate::stats::StateUsageStats;
 use log::{trace, debug, warn};
 pub use sc_state_db::PruningMode;
+pub use bench::BenchmarkingState;
 
 #[cfg(feature = "test-helpers")]
 use sc_client::in_mem::Backend as InMemoryBackend;
@@ -104,6 +106,7 @@ pub use kvdb;
 pub struct RefTrackingState<Block: BlockT> {
 	state: DbState<Block>,
 	storage: Arc<StorageDb<Block>>,
+
 	parent_hash: Option<Block::Hash>,
 }
 

client/service/src/chain_ops.rs

@@ -27,9 +27,11 @@ use sp_runtime::traits::{
 };
 use sp_runtime::generic::{BlockId, SignedBlock};
 use codec::{Decode, Encode, IoReader};
-use sc_client::Client;
+use sc_client::{Client, ExecutionStrategy, StateMachine};
+use sc_client_db::BenchmarkingState;
 use sp_consensus::import_queue::{IncomingBlock, Link, BlockImportError, BlockImportResult, ImportQueue};
 use sp_consensus::BlockOrigin;
+use sc_executor::{NativeExecutor, NativeExecutionDispatch, WasmExecutionMethod};
 
 use std::{io::{Read, Write, Seek}, pin::Pin};
 
@@ -43,6 +45,30 @@ pub fn build_spec<G, E>(spec: ChainSpec<G, E>, raw: bool) -> error::Result<Strin
 	Ok(spec.to_json(raw)?)
 }
 
+pub fn benchmark_runtime<TBl: BlockT, TExecDisp: NativeExecutionDispatch + 'static>(
+	strategy: ExecutionStrategy,
+	wasm_method: WasmExecutionMethod,
+) -> error::Result<()> {
+	let mut changes = Default::default();
+	let state = BenchmarkingState::<TBl>::new()?;
+	let executor = NativeExecutor::<TExecDisp>::new(
+		wasm_method,
+		None, // heap pages
+	);
+	let _ = StateMachine::<_, _, NumberFor<TBl>, _>::new(
+		&state,
+		None,
+		&mut changes,
+		&executor,
+		"run_benchmark",
+		&[],
+		Default::default(),
+	).execute(strategy).map_err(|e| format!("Error executing runtime benchmark: {:?}", e))?;
+	info!("Done.");
+	Ok(())
+}
+
+
 impl<
 	TBl, TRtApi, TCfg, TGen, TCSExt, TBackend,
 	TExec, TFchr, TSc, TImpQu, TFprb, TFpp, TNetP,

client/service/src/lib.rs

@@ -65,6 +65,7 @@ pub use sp_transaction_pool::{TransactionPool, InPoolTransaction, error::IntoPoo
 pub use sc_transaction_pool::txpool::Options as TransactionPoolOptions;
 pub use sc_client::FinalityNotifications;
 pub use sc_rpc::Metadata as RpcMetadata;
+pub use sc_executor::NativeExecutionDispatch;
 #[doc(hidden)]
 pub use std::{ops::Deref, result::Result, sync::Arc};
 #[doc(hidden)]

client/src/lib.rs

@@ -104,4 +104,4 @@ pub use crate::{
 	},
 	leaves::LeafSet,
 };
-pub use sp_state_machine::{ExecutionStrategy, StorageProof};
+pub use sp_state_machine::{ExecutionStrategy, StorageProof, StateMachine};

primitives/externalities/src/lib.rs

@@ -202,6 +202,15 @@ pub trait Externalities: ExtensionStore {
 	///
 	/// Returns the SCALE encoded hash.
 	fn storage_changes_root(&mut self, parent: &[u8]) -> Result<Option<Vec<u8>>, ()>;
+
+
+	fn wipe(&mut self) {
+		unimplemented!()
+	}
+
+	fn commit(&mut self) {
+		unimplemented!()
+	}
 }
 
 /// Extension for the [`Externalities`] trait.

primitives/io/src/lib.rs

@@ -784,6 +784,16 @@ pub trait Benchmarking {
 				.expect("Unix time doesn't go backwards; qed")
 				.as_nanos()
 		}
+
+		/// Reset state to empty.
+		fn wipe_db(&mut self) {
+			self.wipe()
+		}
+
+		/// Commit pending storage changes to the trie database.
+		fn commit_db(&mut self) {
+			self.commit()
+		}
 }
 
 /// Wasm-only interface that provides functions for interacting with the sandbox.

primitives/state-machine/src/backend.rs

@@ -213,6 +213,16 @@ pub trait Backend<H: Hasher>: std::fmt::Debug {
 	fn usage_info(&self) -> UsageInfo {
 		UsageInfo::empty()
 	}
+
+	/// Wipe the state database.
+	fn wipe(&mut self) -> Result<(), Self::Error> {
+		unimplemented!()
+	}
+
+	/// Commit given transaction to storage.
+	fn commit(&mut self, _storage_root: H::Out, _transaction: Self::Transaction) -> Result<(), Self::Error> {
+		unimplemented!()
+	}
 }
 
 impl<'a, T: Backend<H>, H: Hasher> Backend<H> for &'a T {

primitives/state-machine/src/ext.rs

@@ -582,6 +582,24 @@ where
 
 		root.map(|r| r.map(|o| o.encode()))
 	}
+
+	fn wipe(&mut self) {
+		self.overlay.drain_storage_changes(&self.backend, None, Default::default(), self.storage_transaction_cache)
+			.expect(EXT_NOT_ALLOWED_TO_FAIL);
+		self.storage_transaction_cache.reset();
+		self.backend.wipe().expect(EXT_NOT_ALLOWED_TO_FAIL)
+	}
+
+	fn commit(&mut self) {
+		self.overlay.commit_prospective();
+		self.overlay.drain_storage_changes(&self.backend, None, Default::default(), self.storage_transaction_cache)
+			.expect(EXT_NOT_ALLOWED_TO_FAIL);
+		self.backend.commit(
+			self.storage_transaction_cache.transaction_storage_root.take().expect(EXT_NOT_ALLOWED_TO_FAIL),
+			self.storage_transaction_cache.transaction.take().expect(EXT_NOT_ALLOWED_TO_FAIL),
+		).expect(EXT_NOT_ALLOWED_TO_FAIL);
+		self.storage_transaction_cache.reset();
+	}
 }
 
 impl<'a, H, B, N> sp_externalities::ExtensionStore for Ext<'a, H, N, B>