From 092b82905c814855229724507f96b0a72b97d5d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Drwi=C4=99ga?= Date: Mon, 28 Oct 2019 16:11:24 +0100 Subject: [PATCH 01/10] Pass Extensions instead of individual objects. --- Cargo.lock | 1 + core/client/Cargo.toml | 4 ++- core/client/db/src/lib.rs | 2 +- core/client/src/call_executor.rs | 45 +++++++++----------------- core/client/src/client.rs | 27 +++++++++++----- core/client/src/genesis.rs | 18 ++++------- core/client/src/light/call_executor.rs | 22 ++++++------- core/client/src/light/mod.rs | 2 +- core/state-machine/src/lib.rs | 42 ++++++++++-------------- core/test-client/src/lib.rs | 2 +- core/test-runtime/client/src/lib.rs | 2 +- 11 files changed, 76 insertions(+), 91 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3343942cc12a1..d9f944ba236c8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4912,6 +4912,7 @@ dependencies = [ "sr-version 2.0.0", "substrate-consensus-common 2.0.0", "substrate-executor 2.0.0", + "substrate-externalities 2.0.0", "substrate-header-metadata 2.0.0", "substrate-inherents 2.0.0", "substrate-keyring 2.0.0", diff --git a/core/client/Cargo.toml b/core/client/Cargo.toml index 5a0afd2d6f72a..c73f454d4f3e6 100644 --- a/core/client/Cargo.toml +++ b/core/client/Cargo.toml @@ -28,6 +28,7 @@ rstd = { package = "sr-std", path = "../sr-std", default-features = false } inherents = { package = "substrate-inherents", path = "../inherents", default-features = false } sr-api-macros = { path = "../sr-api-macros" } header-metadata = { package = "substrate-header-metadata", path = "header-metadata", optional = true } +externalities = { package = "substrate-externalities", path = "../externalities", optional = true } [dev-dependencies] env_logger = "0.7.0" @@ -60,5 +61,6 @@ std = [ "keyring", "trie", "substrate-telemetry", - "kvdb" + "kvdb", + "externalities", ] diff --git a/core/client/db/src/lib.rs b/core/client/db/src/lib.rs index 0f765506a31ef..ff559b33215d1 100644 --- a/core/client/db/src/lib.rs +++ b/core/client/db/src/lib.rs @@ -228,7 +228,7 @@ pub fn new_client( S: BuildStorage, { let backend = Arc::new(Backend::new(settings, CANONICALIZATION_DELAY)?); - let executor = client::LocalCallExecutor::new(backend.clone(), executor, keystore); + let executor = client::LocalCallExecutor::new(backend.clone(), executor); Ok(( client::Client::new(backend.clone(), executor, genesis_storage, fork_blocks, execution_strategies)?, backend, diff --git a/core/client/src/call_executor.rs b/core/client/src/call_executor.rs index 05a2a8eba1c3e..ac5b6047945d7 100644 --- a/core/client/src/call_executor.rs +++ b/core/client/src/call_executor.rs @@ -24,10 +24,11 @@ use state_machine::{ backend::Backend as _, ChangesTrieTransaction, }; use executor::{RuntimeVersion, RuntimeInfo, NativeVersion}; +use externalities::Extensions; use hash_db::Hasher; use primitives::{ - offchain::OffchainExt, H256, Blake2Hasher, NativeOrEncoded, NeverNativeValue, - traits::{CodeExecutor, KeystoreExt}, + H256, Blake2Hasher, NativeOrEncoded, NeverNativeValue, + traits::CodeExecutor, }; use crate::runtime_api::{ProofRecorder, InitializeBlock}; @@ -53,7 +54,7 @@ where method: &str, call_data: &[u8], strategy: ExecutionStrategy, - side_effects_handler: Option, + extensions: Option, ) -> Result, error::Error>; /// Execute a contextual call on top of state in a block of a given hash. @@ -80,9 +81,8 @@ where initialize_block: InitializeBlock<'a, B>, execution_manager: ExecutionManager, native_call: Option, - side_effects_handler: Option, proof_recorder: &Option>>>, - enable_keystore: bool, + extensions: Option, ) -> error::Result> where ExecutionManager: Clone; /// Extract RuntimeVersion of given block @@ -108,7 +108,7 @@ where call_data: &[u8], manager: ExecutionManager, native_call: Option, - side_effects_handler: Option, + extensions: Option, ) -> Result< ( NativeOrEncoded, @@ -156,7 +156,6 @@ where pub struct LocalCallExecutor { backend: Arc, executor: E, - keystore: Option, } impl LocalCallExecutor { @@ -164,12 +163,10 @@ impl LocalCallExecutor { pub fn new( backend: Arc, executor: E, - keystore: Option, ) -> Self { LocalCallExecutor { backend, executor, - keystore, } } } @@ -179,7 +176,6 @@ impl Clone for LocalCallExecutor where E: Clone { LocalCallExecutor { backend: self.backend.clone(), executor: self.executor.clone(), - keystore: self.keystore.clone(), } } } @@ -198,19 +194,18 @@ where method: &str, call_data: &[u8], strategy: ExecutionStrategy, - side_effects_handler: Option, + extensions: Option, ) -> error::Result> { let mut changes = OverlayedChanges::default(); let state = self.backend.state_at(*id)?; let return_data = StateMachine::new( &state, self.backend.changes_trie_storage(), - side_effects_handler, &mut changes, &self.executor, method, call_data, - self.keystore.clone().map(KeystoreExt), + extensions.unwrap_or_default(), ).execute_using_consensus_failure_handler::<_, NeverNativeValue, fn() -> _>( strategy.get_manager(), false, @@ -240,9 +235,8 @@ where initialize_block: InitializeBlock<'a, Block>, execution_manager: ExecutionManager, native_call: Option, - side_effects_handler: Option, recorder: &Option>>>, - enable_keystore: bool, + extensions: Option, ) -> Result, error::Error> where ExecutionManager: Clone { match initialize_block { InitializeBlock::Do(ref init_block) @@ -253,12 +247,6 @@ where _ => {}, } - let keystore = if enable_keystore { - self.keystore.clone().map(KeystoreExt) - } else { - None - }; - let mut state = self.backend.state_at(*at)?; let result = match recorder { @@ -277,12 +265,11 @@ where StateMachine::new( &backend, self.backend.changes_trie_storage(), - side_effects_handler, &mut *changes.borrow_mut(), &self.executor, method, call_data, - keystore, + extensions.unwrap_or_default(), ) .execute_using_consensus_failure_handler( execution_manager, @@ -295,12 +282,11 @@ where None => StateMachine::new( &state, self.backend.changes_trie_storage(), - side_effects_handler, &mut *changes.borrow_mut(), &self.executor, method, call_data, - keystore, + extensions.unwrap_or_default(), ) .execute_using_consensus_failure_handler( execution_manager, @@ -343,7 +329,7 @@ where call_data: &[u8], manager: ExecutionManager, native_call: Option, - side_effects_handler: Option, + extensions: Option, ) -> error::Result<( NativeOrEncoded, (S::Transaction, ::Out), @@ -352,12 +338,11 @@ where StateMachine::new( state, self.backend.changes_trie_storage(), - side_effects_handler, changes, &self.executor, method, call_data, - self.keystore.clone().map(KeystoreExt), + extensions.unwrap_or_default(), ).execute_using_consensus_failure_handler( manager, true, @@ -384,7 +369,9 @@ where &self.executor, method, call_data, - self.keystore.clone().map(KeystoreExt), + // TODO [ToDr] Do we really need keystore here? + // self.keystore.clone().map(KeystoreExt), + None, ) .map_err(Into::into) } diff --git a/core/client/src/client.rs b/core/client/src/client.rs index d853d851c546c..d9fcae519465f 100644 --- a/core/client/src/client.rs +++ b/core/client/src/client.rs @@ -46,6 +46,7 @@ use state_machine::{ OverlayedChanges, BackendTrustLevel, }; use executor::{RuntimeVersion, RuntimeInfo}; +use externalities::Extensions; use consensus::{ Error as ConsensusError, BlockStatus, BlockImportParams, BlockCheckParams, ImportResult, BlockOrigin, ForkChoiceStrategy, @@ -269,7 +270,7 @@ pub fn new_with_backend( Block: BlockT, B: backend::LocalBackend { - let call_executor = LocalCallExecutor::new(backend.clone(), executor, keystore); + let call_executor = LocalCallExecutor::new(backend.clone(), executor); Client::new(backend, call_executor, build_genesis_storage, Default::default(), Default::default()) } @@ -1468,11 +1469,22 @@ impl CallRuntimeAt for Client where self.execution_strategies.other.get_manager(), }; - let capabilities = context.capabilities(); - let offchain_extensions = if let ExecutionContext::OffchainCall(Some(ext)) = context { - Some(OffchainExt::new(offchain::LimitedExternalities::new(capabilities, ext.0))) - } else { - None + let extensions = { + let mut extensions = Extensions::new(); + let capabilities = context.capabilities(); + + // TODO [ToDr] Keystore + // if capabilities.has(offchain::Capability::Keystore) { + // extensions.register(self.keystore.clone()); + // } + + if let ExecutionContext::OffchainCall(Some(ext)) = context { + extensions.register( + OffchainExt::new(offchain::LimitedExternalities::new(capabilities, ext.0)) + ) + } + + extensions }; self.executor.contextual_call::<_, fn(_,_) -> _,_,_>( @@ -1484,9 +1496,8 @@ impl CallRuntimeAt for Client where initialize_block, manager, native_call, - offchain_extensions, recorder, - capabilities.has(offchain::Capability::Keystore), + Some(extensions), ) } diff --git a/core/client/src/genesis.rs b/core/client/src/genesis.rs index 031b2dc0ad7df..94822466bbff5 100644 --- a/core/client/src/genesis.rs +++ b/core/client/src/genesis.rs @@ -93,12 +93,11 @@ mod tests { StateMachine::new( backend, Some(&InMemoryChangesTrieStorage::<_, u64>::new()), - None, &mut overlay, &executor(), "Core_initialize_block", &header.encode(), - None, + Default::default(), ).execute( ExecutionStrategy::NativeElseWasm, ).unwrap(); @@ -107,12 +106,11 @@ mod tests { StateMachine::new( backend, Some(&InMemoryChangesTrieStorage::<_, u64>::new()), - None, &mut overlay, &executor(), "BlockBuilder_apply_extrinsic", &tx.encode(), - None, + Default::default(), ).execute( ExecutionStrategy::NativeElseWasm, ).unwrap(); @@ -121,12 +119,11 @@ mod tests { let (ret_data, _, _) = StateMachine::new( backend, Some(&InMemoryChangesTrieStorage::<_, u64>::new()), - None, &mut overlay, &executor(), "BlockBuilder_finalize_block", &[], - None, + Default::default(), ).execute( ExecutionStrategy::NativeElseWasm, ).unwrap(); @@ -169,12 +166,11 @@ mod tests { let _ = StateMachine::new( &backend, Some(&InMemoryChangesTrieStorage::<_, u64>::new()), - None, &mut overlay, &executor(), "Core_execute_block", &b1data, - None, + Default::default(), ).execute( ExecutionStrategy::NativeElseWasm, ).unwrap(); @@ -199,12 +195,11 @@ mod tests { let _ = StateMachine::new( &backend, Some(&InMemoryChangesTrieStorage::<_, u64>::new()), - None, &mut overlay, &executor(), "Core_execute_block", &b1data, - None, + Default::default(), ).execute( ExecutionStrategy::AlwaysWasm, ).unwrap(); @@ -229,12 +224,11 @@ mod tests { let r = StateMachine::new( &backend, Some(&InMemoryChangesTrieStorage::<_, u64>::new()), - None, &mut overlay, &executor(), "Core_execute_block", &b1data, - None, + Default::default(), ).execute( ExecutionStrategy::NativeElseWasm, ); diff --git a/core/client/src/light/call_executor.rs b/core/client/src/light/call_executor.rs index 65776bcfe08f3..f5103f05379d7 100644 --- a/core/client/src/light/call_executor.rs +++ b/core/client/src/light/call_executor.rs @@ -23,12 +23,13 @@ use std::{ use codec::{Encode, Decode}; use primitives::{ - offchain::OffchainExt, H256, Blake2Hasher, convert_hash, NativeOrEncoded, + H256, Blake2Hasher, convert_hash, NativeOrEncoded, traits::CodeExecutor, }; use sr_primitives::{ generic::BlockId, traits::{One, Block as BlockT, Header as HeaderT, NumberFor}, }; +use externalities::Extensions; use state_machine::{ self, Backend as StateBackend, OverlayedChanges, ExecutionStrategy, create_proof_check_backend, execution_proof_check_on_trie_backend, ExecutionManager, ChangesTrieTransaction, @@ -81,10 +82,10 @@ impl CallExecutor for method: &str, call_data: &[u8], strategy: ExecutionStrategy, - side_effects_handler: Option, + extensions: Option, ) -> ClientResult> { match self.backend.is_local_state_available(id) { - true => self.local.call(id, method, call_data, strategy, side_effects_handler), + true => self.local.call(id, method, call_data, strategy, extensions), false => Err(ClientError::NotAvailableOnLightClient), } } @@ -108,9 +109,8 @@ impl CallExecutor for initialize_block: InitializeBlock<'a, Block>, _manager: ExecutionManager, native_call: Option, - side_effects_handler: Option, recorder: &Option>>>, - enable_keystore: bool, + extensions: Option, ) -> ClientResult> where ExecutionManager: Clone { // there's no actual way/need to specify native/wasm execution strategy on light node // => we can safely ignore passed values @@ -134,9 +134,8 @@ impl CallExecutor for initialize_block, ExecutionManager::NativeWhenPossible, native_call, - side_effects_handler, recorder, - enable_keystore, + extensions, ).map_err(|e| ClientError::Execution(Box::new(e.to_string()))), false => Err(ClientError::NotAvailableOnLightClient), } @@ -164,7 +163,7 @@ impl CallExecutor for _call_data: &[u8], _manager: ExecutionManager, _native_call: Option, - _side_effects_handler: Option, + _extensions: Option, ) -> ClientResult<( NativeOrEncoded, (S::Transaction, ::Out), @@ -314,7 +313,7 @@ mod tests { _method: &str, _call_data: &[u8], _strategy: ExecutionStrategy, - _side_effects_handler: Option, + _extensions: Option, ) -> Result, ClientError> { Ok(vec![42]) } @@ -338,9 +337,8 @@ mod tests { _initialize_block: InitializeBlock<'a, Block>, _execution_manager: ExecutionManager, _native_call: Option, - _side_effects_handler: Option, _proof_recorder: &Option>>>, - _enable_keystore: bool, + _extensions: Option, ) -> ClientResult> where ExecutionManager: Clone { unreachable!() } @@ -364,7 +362,7 @@ mod tests { _call_data: &[u8], _manager: ExecutionManager, _native_call: Option, - _side_effects_handler: Option, + _extensions: Option, ) -> Result< ( NativeOrEncoded, diff --git a/core/client/src/light/mod.rs b/core/client/src/light/mod.rs index d06a9ae9dd93c..4577955b9da83 100644 --- a/core/client/src/light/mod.rs +++ b/core/client/src/light/mod.rs @@ -65,7 +65,7 @@ pub fn new_light( GS: BuildStorage, E: CodeExecutor + RuntimeInfo, { - let local_executor = LocalCallExecutor::new(backend.clone(), code_executor, None); + let local_executor = LocalCallExecutor::new(backend.clone(), code_executor); let executor = GenesisCallExecutor::new(backend.clone(), local_executor); Client::new(backend, executor, genesis_storage, Default::default(), Default::default()) } diff --git a/core/state-machine/src/lib.rs b/core/state-machine/src/lib.rs index c3092367f0646..2d27ebd99b7c7 100644 --- a/core/state-machine/src/lib.rs +++ b/core/state-machine/src/lib.rs @@ -23,7 +23,7 @@ use log::{warn, trace}; use hash_db::Hasher; use codec::{Decode, Encode}; use primitives::{ - storage::well_known_keys, NativeOrEncoded, NeverNativeValue, offchain::OffchainExt, + storage::well_known_keys, NativeOrEncoded, NeverNativeValue, traits::{KeystoreExt, CodeExecutor}, hexdisplay::HexDisplay, hash::H256, }; use overlayed_changes::OverlayedChangeSet; @@ -185,23 +185,12 @@ impl<'a, B, H, N, T, Exec> StateMachine<'a, B, H, N, T, Exec> where pub fn new( backend: &'a B, changes_trie_storage: Option<&'a T>, - offchain_ext: Option, overlay: &'a mut OverlayedChanges, exec: &'a Exec, method: &'a str, call_data: &'a [u8], - keystore: Option, + extensions: Extensions, ) -> Self { - let mut extensions = Extensions::new(); - - if let Some(keystore) = keystore { - extensions.register(keystore); - } - - if let Some(offchain) = offchain_ext { - extensions.register(offchain); - } - Self { backend, exec, @@ -497,8 +486,12 @@ where Exec: CodeExecutor, { let proving_backend = proving_backend::ProvingBackend::new(trie_backend); + let mut extensions = Extensions::new(); + if let Some(keystore) = keystore { + extensions.register(keystore); + } let mut sm = StateMachine::<_, H, _, InMemoryChangesTrieStorage, Exec>::new( - &proving_backend, None, None, overlay, exec, method, call_data, keystore, + &proving_backend, None, overlay, exec, method, call_data, extensions, ); let (result, _, _) = sm.execute_using_consensus_failure_handler::<_, NeverNativeValue, fn() -> _>( @@ -542,8 +535,12 @@ where H: Hasher, Exec: CodeExecutor, { + let mut extensions = Extensions::new(); + if let Some(keystore) = keystore { + extensions.register(keystore); + } let mut sm = StateMachine::<_, H, _, InMemoryChangesTrieStorage, Exec>::new( - trie_backend, None, None, overlay, exec, method, call_data, keystore, + trie_backend, None, overlay, exec, method, call_data, extensions, ); sm.execute_using_consensus_failure_handler::<_, NeverNativeValue, fn() -> _>( @@ -820,7 +817,6 @@ mod tests { let mut state_machine = StateMachine::new( &backend, Some(&changes_trie_storage), - None, &mut overlayed_changes, &DummyCodeExecutor { change_changes_trie_config: false, @@ -830,7 +826,7 @@ mod tests { }, "test", &[], - None, + Default::default(), ); assert_eq!( @@ -849,7 +845,6 @@ mod tests { let mut state_machine = StateMachine::new( &backend, Some(&changes_trie_storage), - None, &mut overlayed_changes, &DummyCodeExecutor { change_changes_trie_config: false, @@ -859,7 +854,7 @@ mod tests { }, "test", &[], - None, + Default::default(), ); assert_eq!(state_machine.execute(ExecutionStrategy::NativeElseWasm).unwrap().0, vec![66]); @@ -875,7 +870,6 @@ mod tests { let mut state_machine = StateMachine::new( &backend, Some(&changes_trie_storage), - None, &mut overlayed_changes, &DummyCodeExecutor { change_changes_trie_config: false, @@ -885,7 +879,7 @@ mod tests { }, "test", &[], - None, + Default::default(), ); assert!( @@ -1085,7 +1079,6 @@ mod tests { let mut state_machine = StateMachine::new( &backend, Some(&changes_trie_storage), - None, &mut overlayed_changes, &DummyCodeExecutor { change_changes_trie_config: true, @@ -1095,7 +1088,7 @@ mod tests { }, "test", &[], - None, + Default::default(), ); assert!(state_machine.execute(ExecutionStrategy::NativeWhenPossible).is_err()); @@ -1110,7 +1103,6 @@ mod tests { let mut state_machine = StateMachine::new( &backend, Some(&changes_trie_storage), - None, &mut overlayed_changes, &DummyCodeExecutor { change_changes_trie_config: true, @@ -1120,7 +1112,7 @@ mod tests { }, "test", &[], - None, + Default::default(), ); assert!(state_machine.execute(ExecutionStrategy::NativeElseWasm).is_err()); diff --git a/core/test-client/src/lib.rs b/core/test-client/src/lib.rs index dbe4431456a74..22c3fed1eb31e 100644 --- a/core/test-client/src/lib.rs +++ b/core/test-client/src/lib.rs @@ -223,7 +223,7 @@ impl TestClientBuilder< let executor = executor.into().unwrap_or_else(|| NativeExecutor::new(WasmExecutionMethod::Interpreted, None) ); - let executor = LocalCallExecutor::new(self.backend.clone(), executor, self.keystore.take()); + let executor = LocalCallExecutor::new(self.backend.clone(), executor); self.build_with_executor(executor) } diff --git a/core/test-runtime/client/src/lib.rs b/core/test-runtime/client/src/lib.rs index affbae62c22e7..b970e97a451e1 100644 --- a/core/test-runtime/client/src/lib.rs +++ b/core/test-runtime/client/src/lib.rs @@ -262,7 +262,7 @@ pub fn new_light() -> ( let blockchain = Arc::new(client::light::blockchain::Blockchain::new(storage)); let backend = Arc::new(LightBackend::new(blockchain.clone())); let executor = NativeExecutor::new(WasmExecutionMethod::Interpreted, None); - let local_call_executor = client::LocalCallExecutor::new(backend.clone(), executor, None); + let local_call_executor = client::LocalCallExecutor::new(backend.clone(), executor); let call_executor = LightExecutor::new( backend.clone(), local_call_executor, From acb35dc4f28bc88db0988fdeb8f08a9645e5a09c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Drwi=C4=99ga?= Date: Sat, 9 Nov 2019 17:43:55 +0100 Subject: [PATCH 02/10] Move TransactionPool to a separate ExternalitiesExtension. --- core/client/db/src/lib.rs | 7 +- core/client/src/call_executor.rs | 4 +- core/client/src/client.rs | 84 +++---------- core/client/src/execution_extensions.rs | 161 ++++++++++++++++++++++++ core/client/src/lib.rs | 33 +++-- core/offchain/src/api.rs | 12 +- core/offchain/src/testing.rs | 10 +- core/primitives/src/offchain.rs | 27 ++-- core/primitives/src/traits.rs | 8 +- core/service/src/builder.rs | 14 ++- core/sr-io/with_std.rs | 22 ++-- core/sr-primitives/src/offchain/mod.rs | 25 +++- core/state-machine/src/lib.rs | 3 +- core/test-client/src/lib.rs | 9 +- core/transaction-pool/graph/src/pool.rs | 22 ++++ node/runtime/src/lib.rs | 4 +- 16 files changed, 308 insertions(+), 137 deletions(-) create mode 100644 core/client/src/execution_extensions.rs diff --git a/core/client/db/src/lib.rs b/core/client/db/src/lib.rs index c85d35cb39547..1b9a6954dd452 100644 --- a/core/client/db/src/lib.rs +++ b/core/client/db/src/lib.rs @@ -40,7 +40,7 @@ use std::collections::{HashMap, HashSet}; use client::backend::NewBlockState; use client::blockchain::{well_known_cache_keys, HeaderBackend}; -use client::{ForkBlocks, ExecutionStrategies}; +use client::{ForkBlocks, ExecutionExtensions}; use client::backend::{StorageCollection, ChildStorageCollection}; use client::error::{Result as ClientResult, Error as ClientError}; use codec::{Decode, Encode}; @@ -224,8 +224,7 @@ pub fn new_client( executor: E, genesis_storage: S, fork_blocks: ForkBlocks, - execution_strategies: ExecutionStrategies, - keystore: Option, + execution_extensions: ExecutionExtensions, ) -> Result<( client::Client< Backend, @@ -245,7 +244,7 @@ pub fn new_client( let backend = Arc::new(Backend::new(settings, CANONICALIZATION_DELAY)?); let executor = client::LocalCallExecutor::new(backend.clone(), executor); Ok(( - client::Client::new(backend.clone(), executor, genesis_storage, fork_blocks, execution_strategies)?, + client::Client::new(backend.clone(), executor, genesis_storage, fork_blocks, execution_extensions)?, backend, )) } diff --git a/core/client/src/call_executor.rs b/core/client/src/call_executor.rs index e5acc7023df2c..787d083f3129e 100644 --- a/core/client/src/call_executor.rs +++ b/core/client/src/call_executor.rs @@ -369,8 +369,8 @@ where &self.executor, method, call_data, - // TODO [ToDr] Do we really need keystore here? - // self.keystore.clone().map(KeystoreExt), + // Passing `None` here, since we don't really want to prove anything + // about our local keys. None, ) .map_err(Into::into) diff --git a/core/client/src/client.rs b/core/client/src/client.rs index f8a23c93c7c35..42a8a43466e72 100644 --- a/core/client/src/client.rs +++ b/core/client/src/client.rs @@ -26,9 +26,10 @@ use parking_lot::{Mutex, RwLock}; use codec::{Encode, Decode}; use hash_db::{Hasher, Prefix}; use primitives::{ - Blake2Hasher, H256, ChangesTrieConfiguration, convert_hash, NeverNativeValue, ExecutionContext, - NativeOrEncoded, storage::{StorageKey, StorageData, well_known_keys}, - offchain::{OffchainExt, self}, traits::CodeExecutor, + Blake2Hasher, H256, ChangesTrieConfiguration, convert_hash, + NeverNativeValue, ExecutionContext, NativeOrEncoded, + storage::{StorageKey, StorageData, well_known_keys}, + traits::CodeExecutor, }; use substrate_telemetry::{telemetry, SUBSTRATE_INFO}; use sr_primitives::{ @@ -46,7 +47,6 @@ use state_machine::{ OverlayedChanges, BackendTrustLevel, StorageProof, merge_storage_proofs, }; use executor::{RuntimeVersion, RuntimeInfo}; -use externalities::Extensions; use consensus::{ Error as ConsensusError, BlockStatus, BlockImportParams, BlockCheckParams, ImportResult, BlockOrigin, ForkChoiceStrategy, @@ -69,6 +69,7 @@ use crate::{ well_known_cache_keys::Id as CacheKeyId, }, call_executor::{CallExecutor, LocalCallExecutor}, + execution_extensions::ExecutionExtensions, notifications::{StorageNotifications, StorageEventStream}, light::{call_executor::prove_execution, fetcher::ChangesProof}, block_builder::{self, api::BlockBuilder as BlockBuilderAPI}, @@ -94,33 +95,6 @@ type ChangesUpdate = ChangesTrieTransaction = Option, ::Hash>>; -/// Execution strategies settings. -#[derive(Debug, Clone)] -pub struct ExecutionStrategies { - /// Execution strategy used when syncing. - pub syncing: ExecutionStrategy, - /// Execution strategy used when importing blocks. - pub importing: ExecutionStrategy, - /// Execution strategy used when constructing blocks. - pub block_construction: ExecutionStrategy, - /// Execution strategy used for offchain workers. - pub offchain_worker: ExecutionStrategy, - /// Execution strategy used in other cases. - pub other: ExecutionStrategy, -} - -impl Default for ExecutionStrategies { - fn default() -> ExecutionStrategies { - ExecutionStrategies { - syncing: ExecutionStrategy::NativeElseWasm, - importing: ExecutionStrategy::NativeElseWasm, - block_construction: ExecutionStrategy::AlwaysWasm, - offchain_worker: ExecutionStrategy::NativeWhenPossible, - other: ExecutionStrategy::NativeElseWasm, - } - } -} - /// Substrate Client pub struct Client where Block: BlockT { backend: Arc, @@ -131,7 +105,7 @@ pub struct Client where Block: BlockT { // holds the block hash currently being imported. TODO: replace this with block queue importing_block: RwLock>, fork_blocks: ForkBlocks, - execution_strategies: ExecutionStrategies, + execution_extensions: ExecutionExtensions, _phantom: PhantomData, } @@ -299,7 +273,7 @@ impl Client where executor: E, build_genesis_storage: S, fork_blocks: ForkBlocks, - execution_strategies: ExecutionStrategies + execution_extensions: ExecutionExtensions, ) -> error::Result { if backend.blockchain().header(BlockId::Number(Zero::zero()))?.is_none() { let (genesis_storage, children_genesis_storage) = build_genesis_storage.build_storage()?; @@ -328,14 +302,14 @@ impl Client where finality_notification_sinks: Default::default(), importing_block: Default::default(), fork_blocks, - execution_strategies, + execution_extensions, _phantom: Default::default(), }) } - /// Get a reference to the execution strategies. - pub fn execution_strategies(&self) -> &ExecutionStrategies { - &self.execution_strategies + /// Get a reference to the execution extensions. + pub fn execution_extensions(&self) -> &ExecutionExtensions { + &self.execution_extensions } /// Get a reference to the state at a given block. @@ -1064,9 +1038,9 @@ impl Client where &encoded_block, match origin { BlockOrigin::NetworkInitialSync => get_execution_manager( - self.execution_strategies().syncing, + self.execution_extensions().strategies().syncing, ), - _ => get_execution_manager(self.execution_strategies().importing), + _ => get_execution_manager(self.execution_extensions().strategies().importing), }, None, None, @@ -1447,37 +1421,7 @@ impl CallRuntimeAt for Client where context: ExecutionContext, recorder: &Option>>>, ) -> error::Result> { - let manager = match context { - ExecutionContext::BlockConstruction => - self.execution_strategies.block_construction.get_manager(), - ExecutionContext::Syncing => - self.execution_strategies.syncing.get_manager(), - ExecutionContext::Importing => - self.execution_strategies.importing.get_manager(), - ExecutionContext::OffchainCall(Some((_, capabilities))) if capabilities.has_all() => - self.execution_strategies.offchain_worker.get_manager(), - ExecutionContext::OffchainCall(_) => - self.execution_strategies.other.get_manager(), - }; - - let extensions = { - let mut extensions = Extensions::new(); - let capabilities = context.capabilities(); - - // TODO [ToDr] Keystore - // if capabilities.has(offchain::Capability::Keystore) { - // extensions.register(self.keystore.clone()); - // } - - if let ExecutionContext::OffchainCall(Some(ext)) = context { - extensions.register( - OffchainExt::new(offchain::LimitedExternalities::new(capabilities, ext.0)) - ) - } - - extensions - }; - + let (manager, extensions) = self.execution_extensions.manager_and_extensions(at, context); self.executor.contextual_call::<_, fn(_,_) -> _,_,_>( || core_api.initialize_block(at, &self.prepare_environment_block(at)?), at, diff --git a/core/client/src/execution_extensions.rs b/core/client/src/execution_extensions.rs new file mode 100644 index 0000000000000..2e296a1a466f7 --- /dev/null +++ b/core/client/src/execution_extensions.rs @@ -0,0 +1,161 @@ +// Copyright 2019 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use std::sync::{Weak, Arc}; +use codec::Decode; +use primitives::{ + ExecutionContext, + offchain::{self, OffchainExt, TransactionPoolExt}, + traits::{BareCryptoStorePtr, KeystoreExt}, +}; +use sr_primitives::{ + generic::BlockId, + traits, + offchain::{TransactionPool}, +}; +use state_machine::{ExecutionStrategy, ExecutionManager, DefaultHandler}; +use externalities::Extensions; +use parking_lot::RwLock; + +/// Execution strategies settings. +#[derive(Debug, Clone)] +pub struct ExecutionStrategies { + /// Execution strategy used when syncing. + pub syncing: ExecutionStrategy, + /// Execution strategy used when importing blocks. + pub importing: ExecutionStrategy, + /// Execution strategy used when constructing blocks. + pub block_construction: ExecutionStrategy, + /// Execution strategy used for offchain workers. + pub offchain_worker: ExecutionStrategy, + /// Execution strategy used in other cases. + pub other: ExecutionStrategy, +} + +impl Default for ExecutionStrategies { + fn default() -> ExecutionStrategies { + ExecutionStrategies { + syncing: ExecutionStrategy::NativeElseWasm, + importing: ExecutionStrategy::NativeElseWasm, + block_construction: ExecutionStrategy::AlwaysWasm, + offchain_worker: ExecutionStrategy::NativeWhenPossible, + other: ExecutionStrategy::NativeElseWasm, + } + } +} + +pub struct ExecutionExtensions { + strategies: ExecutionStrategies, + keystore: Option, + transaction_pool: RwLock>>>, +} + +impl Default for ExecutionExtensions { + fn default() -> Self { + Self { + strategies: Default::default(), + keystore: None, + transaction_pool: RwLock::new(None), + } + } +} + +impl ExecutionExtensions { + pub fn new( + strategies: ExecutionStrategies, + keystore: Option, + ) -> Self { + let transaction_pool = RwLock::new(None); + Self { strategies, keystore, transaction_pool } + } + + /// Get a reference to the execution strategies. + pub fn strategies(&self) -> &ExecutionStrategies { + &self.strategies + } + + pub fn register_transaction_pool(&self, pool: Weak>) { + *self.transaction_pool.write() = Some(pool); + } + + pub fn manager_and_extensions( + &self, + at: &BlockId, + context: ExecutionContext, + ) -> ( + ExecutionManager>, + Extensions, + ) { + let manager = match context { + ExecutionContext::BlockConstruction => + self.strategies.block_construction.get_manager(), + ExecutionContext::Syncing => + self.strategies.syncing.get_manager(), + ExecutionContext::Importing => + self.strategies.importing.get_manager(), + ExecutionContext::OffchainCall(Some((_, capabilities))) if capabilities.has_all() => + self.strategies.offchain_worker.get_manager(), + ExecutionContext::OffchainCall(_) => + self.strategies.other.get_manager(), + }; + + let capabilities = context.capabilities(); + + let mut extensions = Extensions::new(); + + if capabilities.has(offchain::Capability::Keystore) { + if let Some(keystore) = self.keystore.as_ref() { + extensions.register(KeystoreExt(keystore.clone())); + } + } + + if capabilities.has(offchain::Capability::TransactionPool) { + if let Some(pool) = self.transaction_pool.read().as_ref().and_then(|x| x.upgrade()) { + extensions.register(TransactionPoolExt(Box::new(TransactionPoolAdapter { + at: *at, + pool, + }) as _)); + } + } + + if let ExecutionContext::OffchainCall(Some(ext)) = context { + extensions.register( + OffchainExt::new(offchain::LimitedExternalities::new(capabilities, ext.0)) + ) + } + + (manager, extensions) + } +} + +struct TransactionPoolAdapter { + at: BlockId, + pool: Arc>, +} + +impl offchain::TransactionPool for TransactionPoolAdapter { + fn submit_transaction(&mut self, data: Vec) -> Result<(), ()> { + let xt = match Block::Extrinsic::decode(&mut &*data) { + Ok(xt) => xt, + Err(e) => { + log::warn!("Unable to decode extrinsic: {:?}: {}", data, e.what()); + return Err(()); + }, + }; + + self.pool.submit_at(&self.at, xt) + } +} diff --git a/core/client/src/lib.rs b/core/client/src/lib.rs index 57147eb18b181..11fd5c4aca37d 100644 --- a/core/client/src/lib.rs +++ b/core/client/src/lib.rs @@ -103,30 +103,29 @@ mod call_executor; #[cfg(feature = "std")] mod client; #[cfg(feature = "std")] +mod execution_extensions; +#[cfg(feature = "std")] mod notifications; #[cfg(feature = "std")] -pub use crate::blockchain::Info as ChainInfo; -#[cfg(feature = "std")] -pub use crate::call_executor::{CallExecutor, LocalCallExecutor}; -#[cfg(feature = "std")] -pub use crate::client::{ - new_with_backend, - new_in_mem, - BlockBody, ImportNotifications, FinalityNotifications, BlockchainEvents, - BlockImportNotification, Client, ClientInfo, ExecutionStrategies, FinalityNotification, - LongestChain, BlockOf, ProvideUncles, ForkBlocks, - utils, apply_aux, +pub use crate::{ + blockchain::{Info as ChainInfo, well_known_cache_keys}, + call_executor::{CallExecutor, LocalCallExecutor}, + execution_extensions::{ExecutionStrategies, ExecutionExtensions}, + client::{ + new_with_backend, + new_in_mem, + BlockBody, ImportNotifications, FinalityNotifications, BlockchainEvents, + BlockImportNotification, Client, ClientInfo, FinalityNotification, + LongestChain, BlockOf, ProvideUncles, ForkBlocks, + utils, apply_aux, + }, + notifications::{StorageEventStream, StorageChangeSet}, + leaves::LeafSet, }; #[cfg(feature = "std")] -pub use crate::notifications::{StorageEventStream, StorageChangeSet}; -#[cfg(feature = "std")] pub use state_machine::{ExecutionStrategy, StorageProof}; -#[cfg(feature = "std")] -pub use crate::leaves::LeafSet; -#[cfg(feature = "std")] -pub use crate::blockchain::well_known_cache_keys; #[doc(inline)] pub use sr_api_macros::{decl_runtime_apis, impl_runtime_apis}; diff --git a/core/offchain/src/api.rs b/core/offchain/src/api.rs index 35b6e20df2b29..bb36133282ec9 100644 --- a/core/offchain/src/api.rs +++ b/core/offchain/src/api.rs @@ -82,12 +82,12 @@ where self.is_validator } - fn submit_transaction(&mut self, ext: Vec) -> Result<(), ()> { - self.sender - .unbounded_send(ExtMessage::SubmitExtrinsic(ext)) - .map(|_| ()) - .map_err(|_| ()) - } + // fn submit_transaction(&mut self, ext: Vec) -> Result<(), ()> { + // self.sender + // .unbounded_send(ExtMessage::SubmitExtrinsic(ext)) + // .map(|_| ()) + // .map_err(|_| ()) + // } fn network_state(&self) -> Result { let external_addresses = self.network_state.external_addresses(); diff --git a/core/offchain/src/testing.rs b/core/offchain/src/testing.rs index e1cc7f71a3811..bc65b75dc9ad8 100644 --- a/core/offchain/src/testing.rs +++ b/core/offchain/src/testing.rs @@ -143,11 +143,11 @@ impl offchain::Externalities for TestOffchainExt { unimplemented!("not needed in tests so far") } - fn submit_transaction(&mut self, ex: Vec) -> Result<(), ()> { - let mut state = self.0.write(); - state.transactions.push(ex); - Ok(()) - } + // fn submit_transaction(&mut self, ex: Vec) -> Result<(), ()> { + // let mut state = self.0.write(); + // state.transactions.push(ex); + // Ok(()) + // } fn network_state(&self) -> Result { Ok(OpaqueNetworkState { diff --git a/core/primitives/src/offchain.rs b/core/primitives/src/offchain.rs index 27bd29a00df8d..afe4f38ee083a 100644 --- a/core/primitives/src/offchain.rs +++ b/core/primitives/src/offchain.rs @@ -295,10 +295,6 @@ pub trait Externalities { /// Even if this function returns `true`, it does not mean that any keys are configured /// and that the validator is registered in the chain. fn is_validator(&self) -> bool; - /// Submit transaction. - /// - /// The transaction will end up in the pool and be propagated to others. - fn submit_transaction(&mut self, extrinsic: Vec) -> Result<(), ()>; /// Returns information about the local node's network state. fn network_state(&self) -> Result; @@ -465,10 +461,6 @@ impl Externalities for Box { (& **self).is_validator() } - fn submit_transaction(&mut self, ex: Vec) -> Result<(), ()> { - (&mut **self).submit_transaction(ex) - } - fn network_state(&self) -> Result { (& **self).network_state() } @@ -568,11 +560,6 @@ impl Externalities for LimitedExternalities { self.externalities.is_validator() } - fn submit_transaction(&mut self, ex: Vec) -> Result<(), ()> { - self.check(Capability::TransactionPool, "submit_transaction"); - self.externalities.submit_transaction(ex) - } - fn network_state(&self) -> Result { self.check(Capability::NetworkState, "network_state"); self.externalities.network_state() @@ -669,6 +656,20 @@ impl OffchainExt { } } +#[cfg(feature = "std")] +pub trait TransactionPool { + /// Submit transaction. + /// + /// The transaction will end up in the pool and be propagated to others. + fn submit_transaction(&mut self, extrinsic: Vec) -> Result<(), ()>; +} + +#[cfg(feature = "std")] +externalities::decl_extension! { + pub struct TransactionPoolExt(Box); +} + + #[cfg(test)] mod tests { use super::*; diff --git a/core/primitives/src/traits.rs b/core/primitives/src/traits.rs index 1ef665032eed4..f1b13408d1f0c 100644 --- a/core/primitives/src/traits.rs +++ b/core/primitives/src/traits.rs @@ -18,7 +18,11 @@ use crate::{crypto::KeyTypeId, ed25519, sr25519}; -use std::{fmt::{Debug, Display}, panic::UnwindSafe}; +use std::{ + fmt::{Debug, Display}, + panic::UnwindSafe, + sync::Arc, +}; pub use externalities::{Externalities, ExternalitiesExt}; @@ -68,7 +72,7 @@ pub trait BareCryptoStore: Send + Sync { } /// A pointer to the key store. -pub type BareCryptoStorePtr = std::sync::Arc>; +pub type BareCryptoStorePtr = Arc>; externalities::decl_extension! { /// The keystore extension to register/retrieve from the externalities. diff --git a/core/service/src/builder.rs b/core/service/src/builder.rs index e39610b702372..98821a10bec6a 100644 --- a/core/service/src/builder.rs +++ b/core/service/src/builder.rs @@ -184,14 +184,18 @@ where TGen: RuntimeGenesis, TCSExt: Extension { client_db::DatabaseSettingsSrc::Custom(db.clone()), }, }; - + + let extensions = client::ExecutionExtensions::new( + config.execution_strategies.clone(), + Some(keystore.clone()), + ); + client_db::new_client( db_config, executor, &config.chain_spec, fork_blocks, - config.execution_strategies.clone(), - Some(keystore.clone()), + extensions, )? }; @@ -820,6 +824,10 @@ ServiceBuilder< "best" => ?chain_info.best_hash ); + // make transaction pool available for off-chain runtime calls. + client.execution_extensions() + .register_transaction_pool(Arc::downgrade(&transaction_pool) as _); + let transaction_pool_adapter = Arc::new(TransactionPoolAdapter { imports_external_transactions: !config.roles.is_light(), pool: transaction_pool.clone(), diff --git a/core/sr-io/with_std.rs b/core/sr-io/with_std.rs index fdd32124c129c..5776962ef64cd 100644 --- a/core/sr-io/with_std.rs +++ b/core/sr-io/with_std.rs @@ -16,8 +16,10 @@ use primitives::{ blake2_128, blake2_256, twox_128, twox_256, twox_64, ed25519, Blake2Hasher, sr25519, Pair, H256, - traits::KeystoreExt, storage::ChildStorageKey, hexdisplay::HexDisplay, Hasher, - offchain::{self, OffchainExt}, + traits::KeystoreExt, + storage::ChildStorageKey, + hexdisplay::HexDisplay, Hasher, + offchain::{self, OffchainExt, TransactionPoolExt, TransactionPool}, }; // Switch to this after PoC-3 // pub use primitives::BlakeHasher; @@ -347,18 +349,22 @@ fn with_offchain(f: impl FnOnce(&mut dyn offchain::Externalities) -> R, msg: } impl OffchainApi for () { + fn submit_transaction(data: Vec) -> Result<(), ()> { + with_externalities(|ext| ext + .extension::() + .map(|ext| ext.submit_transaction(data)) + .expect(r#"submit_transaction can be called only in the offchain context + with TransactionPool capabilities"# + ) + ).expect("submit_transaction cannot be called outside of an Externalities-provided environment.") + } + fn is_validator() -> bool { with_offchain(|ext| { ext.is_validator() }, "is_validator can be called only in the offchain worker context") } - fn submit_transaction(data: Vec) -> Result<(), ()> { - with_offchain(|ext| { - ext.submit_transaction(data) - }, "submit_transaction can be called only in the offchain worker context") - } - fn network_state() -> Result { with_offchain(|ext| { ext.network_state() diff --git a/core/sr-primitives/src/offchain/mod.rs b/core/sr-primitives/src/offchain/mod.rs index 6b82f77111999..51dc19bc12d60 100644 --- a/core/sr-primitives/src/offchain/mod.rs +++ b/core/sr-primitives/src/offchain/mod.rs @@ -14,6 +14,29 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -//! A collection of higher lever helpers for offchain workers. +//! A collection of higher lever helpers for offchain calls. + +use crate::{ + traits, + generic::BlockId, +}; pub mod http; + +/// An abstraction for transaction pool. +/// +/// This trait is used by offchain calls to be able to submit transactions. +/// The main use case is for offchain workers, to feed back the results of computations, +/// but since the transaction pool access is a separate `ExternalitiesExtension` it can +/// be also used in context of other offchain calls. For one may generate and submit +/// a transaction for some misbehavior reports (say equivocation). +pub trait TransactionPool: Send + Sync { + /// Submit transaction. + /// + /// The transaction will end up in the pool and be propagated to others. + fn submit_at( + &self, + at: &BlockId, + extrinsic: Block::Extrinsic, + ) -> Result<(), ()>; +} diff --git a/core/state-machine/src/lib.rs b/core/state-machine/src/lib.rs index c7c089abb66ab..7e014b0e18688 100644 --- a/core/state-machine/src/lib.rs +++ b/core/state-machine/src/lib.rs @@ -68,7 +68,8 @@ pub use error::{Error, ExecutionError}; type CallResult = Result, E>; -type DefaultHandler = fn(CallResult, CallResult) -> CallResult; +/// Default handler of the execution manager. +pub type DefaultHandler = fn(CallResult, CallResult) -> CallResult; /// Type of changes trie transaction. pub type ChangesTrieTransaction = ( diff --git a/core/test-client/src/lib.rs b/core/test-client/src/lib.rs index 22c3fed1eb31e..80d4fce1961b5 100644 --- a/core/test-client/src/lib.rs +++ b/core/test-client/src/lib.rs @@ -20,7 +20,7 @@ pub mod client_ext; -pub use client::{ExecutionStrategies, blockchain, backend, self}; +pub use client::{ExecutionStrategies, ExecutionExtensions, blockchain, backend, self}; pub use client_db::{Backend, self}; pub use client_ext::ClientExt; pub use consensus; @@ -188,7 +188,10 @@ impl TestClientBuilder executor, storage, Default::default(), - self.execution_strategies, + ExecutionExtensions::new( + self.execution_strategies, + self.keystore.clone(), + ) ).expect("Creates new client"); let longest_chain = client::LongestChain::new(self.backend); @@ -204,7 +207,7 @@ impl TestClientBuilder< > { /// Build the test client with the given native executor. pub fn build_with_native_executor( - mut self, + self, executor: I, ) -> ( client::Client< diff --git a/core/transaction-pool/graph/src/pool.rs b/core/transaction-pool/graph/src/pool.rs index dcb54f710f175..b0a89faa8d15f 100644 --- a/core/transaction-pool/graph/src/pool.rs +++ b/core/transaction-pool/graph/src/pool.rs @@ -382,6 +382,28 @@ impl Clone for Pool { } } +impl sr_primitives::offchain::TransactionPool for Pool { + fn submit_at( + &self, + at: &BlockId, + extrinsic: ::Extrinsic, + ) -> Result<(), ()> { + let result = futures::executor::block_on( + self.submit_one(&at, extrinsic) + ); + + result + .map(|_| ()) + .map_err(|e| { + log::warn!( + target: "txpool", + "(offchain call) Error submitting a transaction to the pool: {:?}", + e + ) + }) + } +} + #[cfg(test)] mod tests { use std::{ diff --git a/node/runtime/src/lib.rs b/node/runtime/src/lib.rs index a0ee20988eaf8..ef49d5138a362 100644 --- a/node/runtime/src/lib.rs +++ b/node/runtime/src/lib.rs @@ -468,7 +468,7 @@ impl system::offchain::CreateTransaction for Runtim type Public = ::Signer; type Signature = Signature; - fn create_transaction>( + fn create_transaction>( call: Call, public: Self::Public, account: AccountId, @@ -487,7 +487,7 @@ impl system::offchain::CreateTransaction for Runtim Default::default(), ); let raw_payload = SignedPayload::new(call, extra).ok()?; - let signature = F::sign(public, &raw_payload)?; + let signature = TSigner::sign(public, &raw_payload)?; let address = Indices::unlookup(account); let (call, extra, _) = raw_payload.deconstruct(); Some((call, (address, signature, extra))) From 530d36083155c4d188128d7c5508da74e9cda0ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Drwi=C4=99ga?= Date: Fri, 15 Nov 2019 18:51:51 +0100 Subject: [PATCH 03/10] Fix compilation.? --- Cargo.lock | 1 + client/api/Cargo.toml | 1 + client/api/src/call_executor.rs | 2 +- primitives/core/src/offchain.rs | 2 +- primitives/sr-io/src/lib.rs | 7 +++++-- 5 files changed, 9 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 937aab9414101..f727b5f6f9d42 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5453,6 +5453,7 @@ dependencies = [ "substrate-client-db 2.0.0", "substrate-consensus-common 2.0.0", "substrate-executor 2.0.0", + "substrate-externalities 2.0.0", "substrate-header-metadata 2.0.0", "substrate-inherents 2.0.0", "substrate-keyring 2.0.0", diff --git a/client/api/Cargo.toml b/client/api/Cargo.toml index 9a122ad0f8591..243fdf0c41723 100644 --- a/client/api/Cargo.toml +++ b/client/api/Cargo.toml @@ -10,6 +10,7 @@ codec = { package = "parity-scale-codec", version = "1.0.0", default-features = consensus = { package = "substrate-consensus-common", path = "../../primitives/consensus/common" } derive_more = { version = "0.15.0" } executor = { package = "substrate-executor", path = "../executor" } +externalities = { package = "substrate-externalities", path = "../../primitives/externalities" } fnv = { version = "1.0.6" } futures = { version = "0.1.29" } futures03 = { package = "futures-preview", version = "0.3.0-alpha.19", features = ["compat"] } diff --git a/client/api/src/call_executor.rs b/client/api/src/call_executor.rs index e40c32f5bd1df..1a4430d112314 100644 --- a/client/api/src/call_executor.rs +++ b/client/api/src/call_executor.rs @@ -26,7 +26,7 @@ use state_machine::{ use executor::{RuntimeVersion, NativeVersion}; use externalities::Extensions; use hash_db::Hasher; -use primitives::{offchain::OffchainExt, Blake2Hasher, NativeOrEncoded}; +use primitives::{Blake2Hasher, NativeOrEncoded}; use sr_api::{ProofRecorder, InitializeBlock}; use crate::error; diff --git a/primitives/core/src/offchain.rs b/primitives/core/src/offchain.rs index 26f7c8143c8b7..8bb0a03578a48 100644 --- a/primitives/core/src/offchain.rs +++ b/primitives/core/src/offchain.rs @@ -667,7 +667,7 @@ pub trait TransactionPool { #[cfg(feature = "std")] externalities::decl_extension! { - pub struct TransactionPoolExt(Box); + pub struct TransactionPoolExt(Box); } diff --git a/primitives/sr-io/src/lib.rs b/primitives/sr-io/src/lib.rs index 080d0b245b738..6f11ff881d5f5 100644 --- a/primitives/sr-io/src/lib.rs +++ b/primitives/sr-io/src/lib.rs @@ -35,7 +35,10 @@ use rstd::ops::Deref; #[cfg(feature = "std")] use primitives::{ - crypto::Pair, traits::KeystoreExt, offchain::OffchainExt, hexdisplay::HexDisplay, + crypto::Pair, + traits::KeystoreExt, + offchain::{OffchainExt, TransactionPoolExt}, + hexdisplay::HexDisplay, storage::ChildStorageKey, }; @@ -424,7 +427,7 @@ pub trait Offchain { /// /// The transaction will end up in the pool. fn submit_transaction(&mut self, data: Vec) -> Result<(), ()> { - self.extension::() + self.extension::() .expect("submit_transaction can be called only in the offchain worker context") .submit_transaction(data) } From 8dd80908f6d0e6df7940cd52eff4f172724b6c8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Drwi=C4=99ga?= Date: Mon, 18 Nov 2019 17:12:43 +0100 Subject: [PATCH 04/10] Clean up. --- Cargo.lock | 1 - client/api/src/client.rs | 28 ------ client/api/src/lib.rs | 1 + client/cli/src/lib.rs | 2 +- client/db/src/lib.rs | 5 +- client/offchain/Cargo.toml | 1 - client/offchain/src/api.rs | 87 +++---------------- client/offchain/src/lib.rs | 11 +-- client/service/src/builder.rs | 6 +- client/src/client.rs | 9 +- test/utils/client/src/lib.rs | 6 +- .../runtime/client/src/block_builder_ext.rs | 2 +- 12 files changed, 35 insertions(+), 124 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f727b5f6f9d42..0d1762afcbff2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5914,7 +5914,6 @@ dependencies = [ "substrate-offchain-primitives 2.0.0", "substrate-primitives 2.0.0", "substrate-test-runtime-client 2.0.0", - "substrate-transaction-pool 2.0.0", "threadpool 1.7.1 (registry+https://github.com/rust-lang/crates.io-index)", "tokio 0.1.22 (registry+https://github.com/rust-lang/crates.io-index)", ] diff --git a/client/api/src/client.rs b/client/api/src/client.rs index a51fc9f03f875..1809d0a30763a 100644 --- a/client/api/src/client.rs +++ b/client/api/src/client.rs @@ -17,7 +17,6 @@ use std::collections::HashMap; use futures03::channel::mpsc; use primitives::storage::StorageKey; -use state_machine::ExecutionStrategy; use sr_primitives::{ traits::{Block as BlockT, NumberFor}, generic::BlockId @@ -39,33 +38,6 @@ pub type FinalityNotifications = mpsc::UnboundedReceiver = Option, ::Hash>>; -/// Execution strategies settings. -#[derive(Debug, Clone)] -pub struct ExecutionStrategies { - /// Execution strategy used when syncing. - pub syncing: ExecutionStrategy, - /// Execution strategy used when importing blocks. - pub importing: ExecutionStrategy, - /// Execution strategy used when constructing blocks. - pub block_construction: ExecutionStrategy, - /// Execution strategy used for offchain workers. - pub offchain_worker: ExecutionStrategy, - /// Execution strategy used in other cases. - pub other: ExecutionStrategy, -} - -impl Default for ExecutionStrategies { - fn default() -> ExecutionStrategies { - ExecutionStrategies { - syncing: ExecutionStrategy::NativeElseWasm, - importing: ExecutionStrategy::NativeElseWasm, - block_construction: ExecutionStrategy::AlwaysWasm, - offchain_worker: ExecutionStrategy::NativeWhenPossible, - other: ExecutionStrategy::NativeElseWasm, - } - } -} - /// Figure out the block type for a given type (for now, just a `Client`). pub trait BlockOf { /// The type of the block. diff --git a/client/api/src/lib.rs b/client/api/src/lib.rs index 6d3bcb44578f1..4369eb1efe276 100644 --- a/client/api/src/lib.rs +++ b/client/api/src/lib.rs @@ -15,6 +15,7 @@ // along with Substrate. If not, see . //! Substrate client interfaces. +#![warn(missing_docs)] pub mod backend; pub mod blockchain; diff --git a/client/cli/src/lib.rs b/client/cli/src/lib.rs index 1209518cbaf65..99d9052649dea 100644 --- a/client/cli/src/lib.rs +++ b/client/cli/src/lib.rs @@ -26,7 +26,7 @@ mod execution_strategy; pub mod error; pub mod informant; -use client_api::ExecutionStrategies; +use client_api::execution_extensions::ExecutionStrategies; use service::{ config::{Configuration, DatabaseConfig}, ServiceBuilderExport, ServiceBuilderImport, ServiceBuilderRevert, diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index 55ec34b06913f..155d2aaeab7b5 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -39,11 +39,12 @@ use std::path::PathBuf; use std::io; use std::collections::{HashMap, HashSet}; +use client_api::ForkBlocks; use client_api::backend::NewBlockState; -use client_api::blockchain::{well_known_cache_keys, HeaderBackend}; -use client_api::{ForkBlocks, ExecutionStrategies}; use client_api::backend::{StorageCollection, ChildStorageCollection}; +use client_api::blockchain::{well_known_cache_keys, HeaderBackend}; use client_api::error::{Result as ClientResult, Error as ClientError}; +use client_api::execution_extensions::ExecutionExtensions; use codec::{Decode, Encode}; use hash_db::{Hasher, Prefix}; use kvdb::{KeyValueDB, DBTransaction}; diff --git a/client/offchain/Cargo.toml b/client/offchain/Cargo.toml index 29621d8502bc1..63ec8b486ea0c 100644 --- a/client/offchain/Cargo.toml +++ b/client/offchain/Cargo.toml @@ -23,7 +23,6 @@ parking_lot = "0.9.0" primitives = { package = "substrate-primitives", path = "../../primitives/core" } rand = "0.7.2" sr-primitives = { path = "../../primitives/sr-primitives" } -transaction_pool = { package = "substrate-transaction-pool", path = "../transaction-pool" } network = { package = "substrate-network", path = "../network" } keystore = { package = "substrate-keystore", path = "../keystore" } diff --git a/client/offchain/src/api.rs b/client/offchain/src/api.rs index 8c86e0618d1c9..9228e62d0950a 100644 --- a/client/offchain/src/api.rs +++ b/client/offchain/src/api.rs @@ -22,8 +22,8 @@ use std::{ }; use client_api::OffchainStorage; -use futures::{StreamExt as _, Future, FutureExt as _, future, channel::mpsc}; -use log::{info, debug, warn, error}; +use futures::Future; +use log::error; use network::{PeerId, Multiaddr, NetworkStateInfo}; use codec::{Encode, Decode}; use primitives::offchain::{ @@ -31,8 +31,6 @@ use primitives::offchain::{ OpaqueNetworkState, OpaquePeerId, OpaqueMultiaddr, StorageKind, }; pub use offchain_primitives::STORAGE_PREFIX; -use sr_primitives::{generic::BlockId, traits::{self, Extrinsic}}; -use transaction_pool::txpool::{Pool, ChainApi}; #[cfg(not(target_os = "unknown"))] mod http; @@ -44,19 +42,14 @@ mod http_dummy; mod timestamp; -/// A message between the offchain extension and the processing thread. -enum ExtMessage { - SubmitExtrinsic(Vec), -} - /// Asynchronous offchain API. /// /// NOTE this is done to prevent recursive calls into the runtime (which are not supported currently). -pub(crate) struct Api { - sender: mpsc::UnboundedSender, +pub(crate) struct Api { + /// Offchain Workers database. db: Storage, + /// A NetworkState provider. network_state: Arc, - _at: BlockId, /// Is this node a potential validator? is_validator: bool, /// Everything HTTP-related is handled by a different struct. @@ -73,22 +66,11 @@ fn unavailable_yet(name: &str) -> R { const LOCAL_DB: &str = "LOCAL (fork-aware) DB"; -impl OffchainExt for Api -where - Storage: OffchainStorage, - Block: traits::Block, -{ +impl OffchainExt for Api { fn is_validator(&self) -> bool { self.is_validator } - // fn submit_transaction(&mut self, ext: Vec) -> Result<(), ()> { - // self.sender - // .unbounded_send(ExtMessage::SubmitExtrinsic(ext)) - // .map(|_| ()) - // .map_err(|_| ()) - // } - fn network_state(&self) -> Result { let external_addresses = self.network_state.external_addresses(); @@ -260,40 +242,28 @@ impl TryFrom for NetworkState { /// Offchain extensions implementation API /// /// This is the asynchronous processing part of the API. -pub(crate) struct AsyncApi { - receiver: Option>, - transaction_pool: Arc>, - at: BlockId, +pub(crate) struct AsyncApi { /// Everything HTTP-related is handled by a different struct. http: Option, } -impl AsyncApi { +impl AsyncApi { /// Creates new Offchain extensions API implementation an the asynchronous processing part. pub fn new( - transaction_pool: Arc>, db: S, - at: BlockId, network_state: Arc, is_validator: bool, - ) -> (Api, AsyncApi) { - let (sender, rx) = mpsc::unbounded(); - + ) -> (Api, AsyncApi) { let (http_api, http_worker) = http::http(); let api = Api { - sender, db, network_state, - _at: at, is_validator, http: http_api, }; let async_api = AsyncApi { - receiver: Some(rx), - transaction_pool, - at, http: Some(http_worker), }; @@ -302,35 +272,9 @@ impl AsyncApi { /// Run a processing task for the API pub fn process(mut self) -> impl Future { - let receiver = self.receiver.take().expect("Take invoked only once."); let http = self.http.take().expect("Take invoked only once."); - let extrinsics = receiver.for_each(move |msg| { - match msg { - ExtMessage::SubmitExtrinsic(ext) => self.submit_extrinsic(ext), - } - }); - - future::join(extrinsics, http) - .map(|((), ())| ()) - } - - fn submit_extrinsic(&mut self, ext: Vec) -> impl Future { - let xt = match ::Extrinsic::decode(&mut &*ext) { - Ok(xt) => xt, - Err(e) => { - warn!("Unable to decode extrinsic: {:?}: {}", ext, e.what()); - return future::Either::Left(future::ready(())) - }, - }; - - info!("Submitting transaction to the pool: {:?} (isSigned: {:?})", xt, xt.is_signed()); - future::Either::Right(self.transaction_pool - .submit_one(&self.at, xt.clone()) - .map(|result| match result { - Ok(hash) => { debug!("[{:?}] Offchain transaction added to the pool.", hash); }, - Err(e) => { warn!("Couldn't submit offchain transaction: {:?}", e); }, - })) + http } } @@ -338,10 +282,8 @@ impl AsyncApi { mod tests { use super::*; use std::{convert::{TryFrom, TryInto}, time::SystemTime}; - use sr_primitives::traits::Zero; use client_db::offchain::LocalStorage; use network::PeerId; - use test_client::runtime::Block; struct MockNetworkStateInfo(); @@ -355,19 +297,14 @@ mod tests { } } - fn offchain_api() -> (Api, AsyncApi) { + fn offchain_api() -> (Api, AsyncApi) { let _ = env_logger::try_init(); let db = LocalStorage::new_test(); let client = Arc::new(test_client::new()); - let pool = Arc::new( - Pool::new(Default::default(), transaction_pool::FullChainApi::new(client.clone())) - ); - let mock = Arc::new(MockNetworkStateInfo()); + AsyncApi::new( - pool, db, - BlockId::Number(Zero::zero()), mock, false, ) diff --git a/client/offchain/src/lib.rs b/client/offchain/src/lib.rs index e253a6d9234fb..579b0f8831327 100644 --- a/client/offchain/src/lib.rs +++ b/client/offchain/src/lib.rs @@ -43,7 +43,6 @@ use log::{debug, warn}; use network::NetworkStateInfo; use primitives::{offchain, ExecutionContext}; use sr_primitives::{generic::BlockId, traits::{self, ProvideRuntimeApi}}; -use transaction_pool::txpool::{Pool, ChainApi}; use client_api::{OffchainStorage}; mod api; @@ -94,13 +93,12 @@ impl OffchainWorkers< { /// Start the offchain workers after given block. #[must_use] - pub fn on_block_imported( + pub fn on_block_imported( &self, number: &::Number, - pool: &Arc>, network_state: Arc, is_validator: bool, - ) -> impl Future where A: ChainApi + 'static { + ) -> impl Future { let runtime = self.client.runtime_api(); let at = BlockId::number(*number); let has_api = runtime.has_api::>(&at); @@ -108,9 +106,7 @@ impl OffchainWorkers< if has_api.unwrap_or(false) { let (api, runner) = api::AsyncApi::new( - pool.clone(), self.db.clone(), - at.clone(), network_state.clone(), is_validator, ); @@ -166,12 +162,13 @@ mod tests { } } + + // TODO [ToDr] Move this test somewhere higher level. #[test] fn should_call_into_runtime_and_produce_extrinsic() { // given let _ = env_logger::try_init(); let client = Arc::new(test_client::new()); - let pool = Arc::new(Pool::new(Default::default(), transaction_pool::FullChainApi::new(client.clone()))); let db = client_db::offchain::LocalStorage::new_test(); let network_state = Arc::new(MockNetworkStateInfo()); diff --git a/client/service/src/builder.rs b/client/service/src/builder.rs index 4968bfc0c7549..db559b69aa5d7 100644 --- a/client/service/src/builder.rs +++ b/client/service/src/builder.rs @@ -193,7 +193,7 @@ where TGen: RuntimeGenesis, TCSExt: Extension { }, }; - let extensions = client::ExecutionExtensions::new( + let extensions = client_api::execution_extensions::ExecutionExtensions::new( config.execution_strategies.clone(), Some(keystore.clone()), ); @@ -921,8 +921,8 @@ ServiceBuilder< } let offchain = offchain.as_ref().and_then(|o| o.upgrade()); - if let (Some(txpool), Some(offchain)) = (txpool, offchain) { - let future = offchain.on_block_imported(&number, &txpool, network_state_info.clone(), is_validator) + if let Some(offchain) = offchain { + let future = offchain.on_block_imported(&number, network_state_info.clone(), is_validator) .map(|()| Ok(())); let _ = to_spawn_tx_.unbounded_send(Box::new(Compat::new(future))); } diff --git a/client/src/client.rs b/client/src/client.rs index 92ef54579910e..abcf8fd2d1188 100644 --- a/client/src/client.rs +++ b/client/src/client.rs @@ -69,14 +69,14 @@ pub use client_api::{ }, client::{ ImportNotifications, FinalityNotification, FinalityNotifications, BlockImportNotification, - ClientInfo, BlockchainEvents, BlockBody, ProvideUncles, ExecutionStrategies, ForkBlocks, + ClientInfo, BlockchainEvents, BlockBody, ProvideUncles, ForkBlocks, BlockOf, }, - execution_extensions::ExecutionExtensions, + execution_extensions::{ExecutionExtensions, ExecutionStrategies}, notifications::{StorageNotifications, StorageEventStream}, error::Error, error, - CallExecutor + CallExecutor, }; use crate::{ @@ -174,7 +174,8 @@ pub fn new_with_backend( B: backend::LocalBackend { let call_executor = LocalCallExecutor::new(backend.clone(), executor); - Client::new(backend, call_executor, build_genesis_storage, Default::default(), Default::default()) + let extensions = ExecutionExtensions::new(Default::default(), keystore); + Client::new(backend, call_executor, build_genesis_storage, Default::default(), extensions) } impl BlockOf for Client where diff --git a/test/utils/client/src/lib.rs b/test/utils/client/src/lib.rs index bb0de4c2c8f2c..584f41c253984 100644 --- a/test/utils/client/src/lib.rs +++ b/test/utils/client/src/lib.rs @@ -20,7 +20,11 @@ pub mod client_ext; -pub use client::{ExecutionStrategies, ExecutionExtensions, blockchain, self}; +pub use client_api::{ + blockchain, + execution_extensions::{ExecutionStrategies, ExecutionExtensions}, + self, +}; pub use client_db::{Backend, self}; pub use client_ext::ClientExt; pub use consensus; diff --git a/test/utils/runtime/client/src/block_builder_ext.rs b/test/utils/runtime/client/src/block_builder_ext.rs index 6bd3ed18ebeb6..40270f94e171f 100644 --- a/test/utils/runtime/client/src/block_builder_ext.rs +++ b/test/utils/runtime/client/src/block_builder_ext.rs @@ -18,7 +18,7 @@ use runtime; use sr_primitives::traits::ProvideRuntimeApi; -use generic_test_client::client; +use generic_test_client::client_api as client; use block_builder::BlockBuilderApi; From dd4bfe0dde17c03fb6b8365643209ac1b3c64b7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Drwi=C4=99ga?= Date: Tue, 19 Nov 2019 15:31:38 +0100 Subject: [PATCH 05/10] Refactor testing utilities. --- Cargo.lock | 2 +- client/api/src/backend.rs | 2 +- client/api/src/lib.rs | 2 - client/db/src/offchain.rs | 2 +- client/offchain/Cargo.toml | 3 +- client/offchain/src/api.rs | 1 - client/offchain/src/lib.rs | 16 +++-- client/src/in_mem.rs | 6 +- client/src/light/backend.rs | 6 +- paint/im-online/Cargo.toml | 3 - paint/im-online/src/tests.rs | 4 ++ .../core/src/{offchain.rs => offchain/mod.rs} | 32 ++++++++- .../core/src/offchain/storage.rs | 25 ++----- .../core/src/offchain}/testing.rs | 66 ++++++++++++++----- test/utils/client/src/lib.rs | 7 +- .../runtime/client/src/block_builder_ext.rs | 2 +- 16 files changed, 112 insertions(+), 67 deletions(-) rename primitives/core/src/{offchain.rs => offchain/mod.rs} (95%) rename client/api/src/offchain.rs => primitives/core/src/offchain/storage.rs (74%) rename {client/offchain/src => primitives/core/src/offchain}/testing.rs (85%) diff --git a/Cargo.lock b/Cargo.lock index 3cc1c720bd035..72a58a22411d4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3452,7 +3452,6 @@ dependencies = [ "sr-staking-primitives 2.0.0", "sr-std 2.0.0", "substrate-application-crypto 2.0.0", - "substrate-offchain 2.0.0", "substrate-primitives 2.0.0", ] @@ -5914,6 +5913,7 @@ dependencies = [ "substrate-offchain-primitives 2.0.0", "substrate-primitives 2.0.0", "substrate-test-runtime-client 2.0.0", + "substrate-transaction-pool 2.0.0", "threadpool 1.7.1 (registry+https://github.com/rust-lang/crates.io-index)", "tokio 0.1.22 (registry+https://github.com/rust-lang/crates.io-index)", ] diff --git a/client/api/src/backend.rs b/client/api/src/backend.rs index 03cd183ad2c21..1ab76a0681fd8 100644 --- a/client/api/src/backend.rs +++ b/client/api/src/backend.rs @@ -19,6 +19,7 @@ use std::sync::Arc; use std::collections::HashMap; use primitives::ChangesTrieConfiguration; +use primitives::offchain::OffchainStorage; use sr_primitives::{generic::BlockId, Justification, StorageOverlay, ChildrenStorageOverlay}; use sr_primitives::traits::{Block as BlockT, NumberFor}; use state_machine::backend::Backend as StateBackend; @@ -27,7 +28,6 @@ use crate::{ blockchain::{ Backend as BlockchainBackend, well_known_cache_keys }, - offchain::OffchainStorage, error, light::RemoteBlockchain, }; diff --git a/client/api/src/lib.rs b/client/api/src/lib.rs index 4369eb1efe276..04abc86308545 100644 --- a/client/api/src/lib.rs +++ b/client/api/src/lib.rs @@ -25,7 +25,6 @@ pub mod error; pub mod execution_extensions; pub mod light; pub mod notifications; -pub mod offchain; // TODO: avoid re-exports pub use backend::*; @@ -35,7 +34,6 @@ pub use client::*; pub use error::*; pub use light::*; pub use notifications::*; -pub use offchain::*; pub use state_machine::{StorageProof, ExecutionStrategy}; diff --git a/client/db/src/offchain.rs b/client/db/src/offchain.rs index 09578f5f6b1d0..7d179ffa0284e 100644 --- a/client/db/src/offchain.rs +++ b/client/db/src/offchain.rs @@ -56,7 +56,7 @@ impl LocalStorage { } } -impl client_api::OffchainStorage for LocalStorage { +impl primitives::offchain::OffchainStorage for LocalStorage { fn set(&mut self, prefix: &[u8], key: &[u8], value: &[u8]) { let key: Vec = prefix.iter().chain(key).cloned().collect(); let mut tx = self.db.transaction(); diff --git a/client/offchain/Cargo.toml b/client/offchain/Cargo.toml index 63ec8b486ea0c..c6de81317fa9c 100644 --- a/client/offchain/Cargo.toml +++ b/client/offchain/Cargo.toml @@ -31,10 +31,11 @@ hyper = "0.12.35" hyper-rustls = "0.17.1" [dev-dependencies] -env_logger = "0.7.0" client-db = { package = "substrate-client-db", path = "../db/", default-features = true } +env_logger = "0.7.0" test-client = { package = "substrate-test-runtime-client", path = "../../test/utils/runtime/client" } tokio = "0.1.22" +transaction_pool = { package = "substrate-transaction-pool", path = "../transaction-pool" } [features] default = [] diff --git a/client/offchain/src/api.rs b/client/offchain/src/api.rs index 9228e62d0950a..d000b1ae712a5 100644 --- a/client/offchain/src/api.rs +++ b/client/offchain/src/api.rs @@ -300,7 +300,6 @@ mod tests { fn offchain_api() -> (Api, AsyncApi) { let _ = env_logger::try_init(); let db = LocalStorage::new_test(); - let client = Arc::new(test_client::new()); let mock = Arc::new(MockNetworkStateInfo()); AsyncApi::new( diff --git a/client/offchain/src/lib.rs b/client/offchain/src/lib.rs index 579b0f8831327..9866ce53f99f6 100644 --- a/client/offchain/src/lib.rs +++ b/client/offchain/src/lib.rs @@ -43,12 +43,10 @@ use log::{debug, warn}; use network::NetworkStateInfo; use primitives::{offchain, ExecutionContext}; use sr_primitives::{generic::BlockId, traits::{self, ProvideRuntimeApi}}; -use client_api::{OffchainStorage}; +use client_api::OffchainStorage; mod api; -pub mod testing; - pub use offchain_primitives::{OffchainWorkerApi, STORAGE_PREFIX}; /// An offchain workers manager. @@ -149,6 +147,8 @@ impl OffchainWorkers< mod tests { use super::*; use network::{Multiaddr, PeerId}; + use std::sync::Arc; + use transaction_pool::txpool::Pool; struct MockNetworkStateInfo(); @@ -162,19 +162,23 @@ mod tests { } } - - // TODO [ToDr] Move this test somewhere higher level. #[test] fn should_call_into_runtime_and_produce_extrinsic() { // given let _ = env_logger::try_init(); let client = Arc::new(test_client::new()); + let pool = Arc::new(Pool::new( + Default::default(), + transaction_pool::FullChainApi::new(client.clone()) + )); + client.execution_extensions() + .register_transaction_pool(Arc::downgrade(&pool.clone()) as _); let db = client_db::offchain::LocalStorage::new_test(); let network_state = Arc::new(MockNetworkStateInfo()); // when let offchain = OffchainWorkers::new(client, db); - futures::executor::block_on(offchain.on_block_imported(&0u64, &pool, network_state, false)); + futures::executor::block_on(offchain.on_block_imported(&0u64, network_state, false)); // then assert_eq!(pool.status().ready, 1); diff --git a/client/src/in_mem.rs b/client/src/in_mem.rs index 4c74943189020..4ff0aae3e483b 100644 --- a/client/src/in_mem.rs +++ b/client/src/in_mem.rs @@ -20,6 +20,9 @@ use std::collections::{HashMap, HashSet}; use std::sync::Arc; use parking_lot::{RwLock, Mutex}; use primitives::{ChangesTrieConfiguration, storage::well_known_keys}; +use primitives::offchain::storage::{ + InMemOffchainStorage as OffchainStorage +}; use sr_primitives::generic::{BlockId, DigestItem}; use sr_primitives::traits::{Block as BlockT, Header as HeaderT, Zero, NumberFor}; use sr_primitives::{Justification, StorageOverlay, ChildrenStorageOverlay}; @@ -35,9 +38,6 @@ use client_api::{ blockchain::{ self, BlockStatus, HeaderBackend, well_known_cache_keys::Id as CacheKeyId }, - offchain::{ - InMemOffchainStorage as OffchainStorage - } }; use crate::leaves::LeafSet; diff --git a/client/src/light/backend.rs b/client/src/light/backend.rs index ec4543191896e..5db0c55d5e979 100644 --- a/client/src/light/backend.rs +++ b/client/src/light/backend.rs @@ -21,8 +21,9 @@ use std::collections::HashMap; use std::sync::Arc; use parking_lot::{RwLock, Mutex}; -use sr_primitives::{generic::BlockId, Justification, StorageOverlay, ChildrenStorageOverlay}; use state_machine::{Backend as StateBackend, TrieBackend, backend::InMemory as InMemoryState, ChangesTrieTransaction}; +use primitives::offchain::storage::InMemOffchainStorage; +use sr_primitives::{generic::BlockId, Justification, StorageOverlay, ChildrenStorageOverlay}; use sr_primitives::traits::{Block as BlockT, NumberFor, Zero, Header}; use crate::in_mem::{self, check_genesis_storage}; use client_api::{ @@ -37,9 +38,8 @@ use client_api::{ Error as ClientError, Result as ClientResult }, light::Storage as BlockchainStorage, - InMemOffchainStorage, }; -use crate::light::blockchain::{Blockchain}; +use crate::light::blockchain::Blockchain; use hash_db::Hasher; use trie::MemoryDB; diff --git a/paint/im-online/Cargo.toml b/paint/im-online/Cargo.toml index e509620f6e037..bfb9a0c4b125b 100644 --- a/paint/im-online/Cargo.toml +++ b/paint/im-online/Cargo.toml @@ -18,9 +18,6 @@ sr-staking-primitives = { path = "../../primitives/sr-staking-primitives", defau support = { package = "paint-support", path = "../support", default-features = false } system = { package = "paint-system", path = "../system", default-features = false } -[dev-dependencies] -offchain = { package = "substrate-offchain", path = "../../client/offchain" } - [features] default = ["std", "session/historical"] std = [ diff --git a/paint/im-online/src/tests.rs b/paint/im-online/src/tests.rs index 382eb4f1d1f04..e54a1378f990c 100644 --- a/paint/im-online/src/tests.rs +++ b/paint/im-online/src/tests.rs @@ -285,7 +285,11 @@ fn should_not_send_a_report_if_already_online() { let mut ext = new_test_ext(); let (offchain, state) = TestOffchainExt::new(); + let (pool, pool_state) = TestTransactionPoolExt::new(); ext.register_extension(OffchainExt::new(offchain)); + ext.register_extension(primitives::offchain::TransactionPoolExt( + Box::new(pool as _) + )); ext.execute_with(|| { advance_session(); diff --git a/primitives/core/src/offchain.rs b/primitives/core/src/offchain/mod.rs similarity index 95% rename from primitives/core/src/offchain.rs rename to primitives/core/src/offchain/mod.rs index 8bb0a03578a48..27936416dae11 100644 --- a/primitives/core/src/offchain.rs +++ b/primitives/core/src/offchain/mod.rs @@ -23,6 +23,31 @@ use runtime_interface::pass_by::{PassByCodec, PassByInner, PassByEnum}; pub use crate::crypto::KeyTypeId; +#[cfg(feature = "std")] +pub mod storage; +#[cfg(feature = "std")] +pub mod testing; + +/// Offchain workers local storage. +pub trait OffchainStorage: Clone + Send + Sync { + /// Persist a value in storage under given key and prefix. + fn set(&mut self, prefix: &[u8], key: &[u8], value: &[u8]); + + /// Retrieve a value from storage under given key and prefix. + fn get(&self, prefix: &[u8], key: &[u8]) -> Option>; + + /// Replace the value in storage if given old_value matches the current one. + /// + /// Returns `true` if the value has been set and false otherwise. + fn compare_and_set( + &mut self, + prefix: &[u8], + key: &[u8], + old_value: Option<&[u8]>, + new_value: &[u8], + ) -> bool; +} + /// A type of supported crypto. #[derive(Clone, Copy, PartialEq, Eq, Encode, Decode, RuntimeDebug, PassByEnum)] #[repr(C)] @@ -657,6 +682,11 @@ impl OffchainExt { } } +/// Abstraction over transaction pool. +/// +/// This trait is currently used within the `ExternalitiesExtension` +/// to provide offchain calls with access to the transaction pool without +/// tight coupling with any pool implementation. #[cfg(feature = "std")] pub trait TransactionPool { /// Submit transaction. @@ -667,10 +697,10 @@ pub trait TransactionPool { #[cfg(feature = "std")] externalities::decl_extension! { + /// An externalities extension to submit transactions to the pool. pub struct TransactionPoolExt(Box); } - #[cfg(test)] mod tests { use super::*; diff --git a/client/api/src/offchain.rs b/primitives/core/src/offchain/storage.rs similarity index 74% rename from client/api/src/offchain.rs rename to primitives/core/src/offchain/storage.rs index 761241efe7996..60339f70055f2 100644 --- a/client/api/src/offchain.rs +++ b/primitives/core/src/offchain/storage.rs @@ -14,27 +14,10 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -use std::collections::hash_map::{HashMap, Entry}; - -/// Offchain workers local storage. -pub trait OffchainStorage: Clone + Send + Sync { - /// Persist a value in storage under given key and prefix. - fn set(&mut self, prefix: &[u8], key: &[u8], value: &[u8]); - - /// Retrieve a value from storage under given key and prefix. - fn get(&self, prefix: &[u8], key: &[u8]) -> Option>; +//! In-memory implementation of offchain workers database. - /// Replace the value in storage if given old_value matches the current one. - /// - /// Returns `true` if the value has been set and false otherwise. - fn compare_and_set( - &mut self, - prefix: &[u8], - key: &[u8], - old_value: Option<&[u8]>, - new_value: &[u8], - ) -> bool; -} +use std::collections::hash_map::{HashMap, Entry}; +use crate::offchain::OffchainStorage; /// In-memory storage for offchain workers. #[derive(Debug, Clone, Default)] @@ -74,4 +57,4 @@ impl OffchainStorage for InMemOffchainStorage { _ => false, } } -} \ No newline at end of file +} diff --git a/client/offchain/src/testing.rs b/primitives/core/src/offchain/testing.rs similarity index 85% rename from client/offchain/src/testing.rs rename to primitives/core/src/offchain/testing.rs index ddb5c923b89e0..0b15c2f314ab3 100644 --- a/client/offchain/src/testing.rs +++ b/primitives/core/src/offchain/testing.rs @@ -14,23 +14,28 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -//! Offchain Externalities implementation for tests. +//! Utilities for offchain calls testing. +//! +//! Namely all ExecutionExtensions that allow mocking +//! the extra APIs. use std::{ collections::BTreeMap, sync::Arc, }; -use client_api::{OffchainStorage, InMemOffchainStorage}; -use parking_lot::RwLock; -use primitives::offchain::{ +use crate::offchain::{ self, + storage::InMemOffchainStorage, HttpError, HttpRequestId as RequestId, HttpRequestStatus as RequestStatus, Timestamp, StorageKind, OpaqueNetworkState, + TransactionPool, + OffchainStorage, }; +use parking_lot::RwLock; /// Pending request. #[derive(Debug, Default, PartialEq, Eq)] @@ -59,7 +64,7 @@ pub struct PendingRequest { /// /// This can be used in tests to respond or assert stuff about interactions. #[derive(Debug, Default)] -pub struct State { +pub struct OffchainState { /// A list of pending requests. pub requests: BTreeMap, expected_requests: BTreeMap, @@ -67,11 +72,9 @@ pub struct State { pub persistent_storage: InMemOffchainStorage, /// Local storage pub local_storage: InMemOffchainStorage, - /// A vector of transactions submitted from the runtime. - pub transactions: Vec>, } -impl State { +impl OffchainState { /// Asserts that pending request has been submitted and fills it's response. pub fn fulfill_pending_request( &mut self, @@ -117,7 +120,7 @@ impl State { } } -impl Drop for State { +impl Drop for OffchainState { fn drop(&mut self) { // If we panic! while we are already in a panic, the test dies with an illegal instruction. if !self.expected_requests.is_empty() && !std::thread::panicking() { @@ -128,11 +131,11 @@ impl Drop for State { /// Implementation of offchain externalities used for tests. #[derive(Clone, Default, Debug)] -pub struct TestOffchainExt(pub Arc>); +pub struct TestOffchainExt(pub Arc>); impl TestOffchainExt { /// Create new `TestOffchainExt` and a reference to the internal state. - pub fn new() -> (Self, Arc>) { + pub fn new() -> (Self, Arc>) { let ext = Self::default(); let state = ext.0.clone(); (ext, state) @@ -144,12 +147,6 @@ impl offchain::Externalities for TestOffchainExt { unimplemented!("not needed in tests so far") } - // fn submit_transaction(&mut self, ex: Vec) -> Result<(), ()> { - // let mut state = self.0.write(); - // state.transactions.push(ex); - // Ok(()) - // } - fn network_state(&self) -> Result { Ok(OpaqueNetworkState { peer_id: Default::default(), @@ -305,3 +302,38 @@ impl offchain::Externalities for TestOffchainExt { } } } + +/// The internal state of the fake transaction pool. +#[derive(Default)] +pub struct PoolState { + /// A vector of transactions submitted from the runtime. + pub transactions: Vec>, +} + +/// Implementation of transaction pool used for test. +/// +/// Note that this implementation does not verify correctness +/// of sent extrinsics. It's meant to be used in contexts +/// where an actual runtime is not known. +/// +/// It's advised to write integration tests that include the +/// actual transaction pool to make sure the produced +/// transactions are valid. +#[derive(Default)] +pub struct TestTransactionPoolExt(Arc>); + +impl TestTransactionPoolExt { + /// Create new `TestTransactionPoolExt` and a reference to the internal state. + pub fn new() -> (Self, Arc>) { + let ext = Self::default(); + let state = ext.0.clone(); + (ext, state) + } +} + +impl TransactionPool for TestTransactionPoolExt { + fn submit_transaction(&mut self, extrinsic: Vec) -> Result<(), ()> { + self.0.write().transactions.push(extrinsic); + Ok(()) + } +} diff --git a/test/utils/client/src/lib.rs b/test/utils/client/src/lib.rs index 584f41c253984..c4f97aec1daad 100644 --- a/test/utils/client/src/lib.rs +++ b/test/utils/client/src/lib.rs @@ -20,11 +20,8 @@ pub mod client_ext; -pub use client_api::{ - blockchain, - execution_extensions::{ExecutionStrategies, ExecutionExtensions}, - self, -}; +pub use client::{blockchain, self}; +pub use client_api::execution_extensions::{ExecutionStrategies, ExecutionExtensions}; pub use client_db::{Backend, self}; pub use client_ext::ClientExt; pub use consensus; diff --git a/test/utils/runtime/client/src/block_builder_ext.rs b/test/utils/runtime/client/src/block_builder_ext.rs index 40270f94e171f..6bd3ed18ebeb6 100644 --- a/test/utils/runtime/client/src/block_builder_ext.rs +++ b/test/utils/runtime/client/src/block_builder_ext.rs @@ -18,7 +18,7 @@ use runtime; use sr_primitives::traits::ProvideRuntimeApi; -use generic_test_client::client_api as client; +use generic_test_client::client; use block_builder::BlockBuilderApi; From bd4f0b09b23b3435ef1af7cfd53c52943014ac8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Drwi=C4=99ga?= Date: Tue, 19 Nov 2019 15:40:42 +0100 Subject: [PATCH 06/10] Add docs, fix tests. --- Cargo.lock | 1 - client/api/src/execution_extensions.rs | 23 +++++++++++++++++++ client/db/src/offchain.rs | 2 +- client/executor/src/integration_tests/mod.rs | 6 ++--- client/offchain/src/api.rs | 2 +- client/offchain/src/lib.rs | 3 +-- client/src/in_mem.rs | 2 +- paint/im-online/src/tests.rs | 22 ++++++++++-------- primitives/core/src/offchain/mod.rs | 9 ++++++++ primitives/sr-primitives/Cargo.toml | 1 - primitives/sr-primitives/src/offchain/http.rs | 6 +++-- 11 files changed, 56 insertions(+), 21 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 72a58a22411d4..03d4f6d7e8b42 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5086,7 +5086,6 @@ dependencies = [ "sr-std 2.0.0", "substrate-application-crypto 2.0.0", "substrate-inherents 2.0.0", - "substrate-offchain 2.0.0", "substrate-primitives 2.0.0", ] diff --git a/client/api/src/execution_extensions.rs b/client/api/src/execution_extensions.rs index 2e296a1a466f7..83d70998e16aa 100644 --- a/client/api/src/execution_extensions.rs +++ b/client/api/src/execution_extensions.rs @@ -14,6 +14,12 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . +//! Execution extensions for runtime calls. +//! +//! This module is responsible for defining the execution +//! strategy for the runtime calls and provide the right `Externalities` +//! extensions to support APIs for particular execution context & capabilities. + use std::sync::{Weak, Arc}; use codec::Decode; use primitives::{ @@ -57,6 +63,11 @@ impl Default for ExecutionStrategies { } } +/// A producer of execution extensions for offchain calls. +/// +/// This crate aggregates extensions available for the offchain calls +/// and is responsbile to produce a right `Extensions` object +/// for each call, based on required `Capabilities`. pub struct ExecutionExtensions { strategies: ExecutionStrategies, keystore: Option, @@ -74,6 +85,7 @@ impl Default for ExecutionExtensions { } impl ExecutionExtensions { + /// Create new `ExecutionExtensions` given a `keystore` and `ExecutionStrategies`. pub fn new( strategies: ExecutionStrategies, keystore: Option, @@ -87,10 +99,20 @@ impl ExecutionExtensions { &self.strategies } + /// Register transaction pool extension. + /// + /// To break retain cycle between `Client` and `TransactionPool` we require this + /// extension to be a `Weak` reference. + /// That's also the reason why it's being registered lazily instead of + /// during initialisation. pub fn register_transaction_pool(&self, pool: Weak>) { *self.transaction_pool.write() = Some(pool); } + /// Create `ExecutionManager` and `Extensions` for given offchain call. + /// + /// Based on the execution context and capabilities it produces + /// the right manager and extensions object to support desired set of APIs. pub fn manager_and_extensions( &self, at: &BlockId, @@ -141,6 +163,7 @@ impl ExecutionExtensions { } } +/// A wrapper type to pass `BlockId` to the actual transaction pool. struct TransactionPoolAdapter { at: BlockId, pool: Arc>, diff --git a/client/db/src/offchain.rs b/client/db/src/offchain.rs index 7d179ffa0284e..0435a1c865bda 100644 --- a/client/db/src/offchain.rs +++ b/client/db/src/offchain.rs @@ -117,7 +117,7 @@ impl primitives::offchain::OffchainStorage for LocalStorage { #[cfg(test)] mod tests { use super::*; - use client_api::OffchainStorage; + use primitives::offchain::OffchainStorage; #[test] fn should_compare_and_set_and_clear_the_locks_map() { diff --git a/client/executor/src/integration_tests/mod.rs b/client/executor/src/integration_tests/mod.rs index 71df0b8d8cc9e..32424c6065ef7 100644 --- a/client/executor/src/integration_tests/mod.rs +++ b/client/executor/src/integration_tests/mod.rs @@ -19,12 +19,12 @@ mod sandbox; use codec::{Encode, Decode}; use hex_literal::hex; use primitives::{ - Blake2Hasher, blake2_128, blake2_256, ed25519, sr25519, map, Pair, offchain::OffchainExt, + Blake2Hasher, blake2_128, blake2_256, ed25519, sr25519, map, Pair, + offchain::{OffchainExt, testing}, traits::Externalities, }; use runtime_test::WASM_BINARY; use state_machine::TestExternalities as CoreTestExternalities; -use substrate_offchain::testing; use test_case::test_case; use trie::{TrieConfiguration, trie_types::Layout}; @@ -401,7 +401,7 @@ fn ordered_trie_root_should_work(wasm_method: WasmExecutionMethod) { #[test_case(WasmExecutionMethod::Interpreted)] #[cfg_attr(feature = "wasmtime", test_case(WasmExecutionMethod::Compiled))] fn offchain_local_storage_should_work(wasm_method: WasmExecutionMethod) { - use client_api::OffchainStorage; + use primitives::offchain::OffchainStorage; let mut ext = TestExternalities::default(); let (offchain, state) = testing::TestOffchainExt::new(); diff --git a/client/offchain/src/api.rs b/client/offchain/src/api.rs index d000b1ae712a5..ff2a5a433a354 100644 --- a/client/offchain/src/api.rs +++ b/client/offchain/src/api.rs @@ -21,7 +21,7 @@ use std::{ thread::sleep, }; -use client_api::OffchainStorage; +use primitives::offchain::OffchainStorage; use futures::Future; use log::error; use network::{PeerId, Multiaddr, NetworkStateInfo}; diff --git a/client/offchain/src/lib.rs b/client/offchain/src/lib.rs index 9866ce53f99f6..a1db2456bd9e7 100644 --- a/client/offchain/src/lib.rs +++ b/client/offchain/src/lib.rs @@ -41,9 +41,8 @@ use sr_api::ApiExt; use futures::future::Future; use log::{debug, warn}; use network::NetworkStateInfo; -use primitives::{offchain, ExecutionContext}; +use primitives::{offchain::{self, OffchainStorage}, ExecutionContext}; use sr_primitives::{generic::BlockId, traits::{self, ProvideRuntimeApi}}; -use client_api::OffchainStorage; mod api; diff --git a/client/src/in_mem.rs b/client/src/in_mem.rs index 4ff0aae3e483b..efab895794eea 100644 --- a/client/src/in_mem.rs +++ b/client/src/in_mem.rs @@ -811,7 +811,7 @@ pub fn check_genesis_storage(top: &StorageOverlay, children: &ChildrenStorageOve #[cfg(test)] mod tests { - use client_api::offchain::{OffchainStorage, InMemOffchainStorage}; + use primitives::offchain::{OffchainStorage, storage::InMemOffchainStorage}; use std::sync::Arc; use test_client; use primitives::Blake2Hasher; diff --git a/paint/im-online/src/tests.rs b/paint/im-online/src/tests.rs index e54a1378f990c..edfdc34bc00d2 100644 --- a/paint/im-online/src/tests.rs +++ b/paint/im-online/src/tests.rs @@ -20,8 +20,12 @@ use super::*; use crate::mock::*; -use offchain::testing::TestOffchainExt; -use primitives::offchain::{OpaquePeerId, OffchainExt}; +use primitives::offchain::{ + OpaquePeerId, + OffchainExt, + TransactionPoolExt, + testing::{TestOffchainExt, TestTransactionPoolExt}, +}; use support::{dispatch, assert_noop}; use sr_primitives::testing::UintAuthorityId; @@ -183,8 +187,10 @@ fn late_heartbeat_should_fail() { #[test] fn should_generate_heartbeats() { let mut ext = new_test_ext(); - let (offchain, state) = TestOffchainExt::new(); + let (offchain, _state) = TestOffchainExt::new(); + let (pool, state) = TestTransactionPoolExt::new(); ext.register_extension(OffchainExt::new(offchain)); + ext.register_extension(TransactionPoolExt::new(pool)); ext.execute_with(|| { // given @@ -284,12 +290,10 @@ fn should_not_send_a_report_if_already_online() { use authorship::EventHandler; let mut ext = new_test_ext(); - let (offchain, state) = TestOffchainExt::new(); + let (offchain, _state) = TestOffchainExt::new(); let (pool, pool_state) = TestTransactionPoolExt::new(); ext.register_extension(OffchainExt::new(offchain)); - ext.register_extension(primitives::offchain::TransactionPoolExt( - Box::new(pool as _) - )); + ext.register_extension(TransactionPoolExt::new(pool)); ext.execute_with(|| { advance_session(); @@ -308,9 +312,9 @@ fn should_not_send_a_report_if_already_online() { ImOnline::offchain(4); // then - let transaction = state.write().transactions.pop().unwrap(); + let transaction = pool_state.write().transactions.pop().unwrap(); // All validators have `0` as their session key, but we should only produce 1 hearbeat. - assert_eq!(state.read().transactions.len(), 0); + assert_eq!(pool_state.read().transactions.len(), 0); // check stuff about the transaction. let ex: Extrinsic = Decode::decode(&mut &*transaction).unwrap(); let heartbeat = match ex.1 { diff --git a/primitives/core/src/offchain/mod.rs b/primitives/core/src/offchain/mod.rs index 27936416dae11..8afabc0392a2a 100644 --- a/primitives/core/src/offchain/mod.rs +++ b/primitives/core/src/offchain/mod.rs @@ -701,6 +701,15 @@ externalities::decl_extension! { pub struct TransactionPoolExt(Box); } +#[cfg(feature = "std")] +impl TransactionPoolExt { + /// Create a new instance of `TransactionPoolExt`. + pub fn new(pool: O) -> Self { + Self(Box::new(pool)) + } +} + + #[cfg(test)] mod tests { use super::*; diff --git a/primitives/sr-primitives/Cargo.toml b/primitives/sr-primitives/Cargo.toml index e1412cf53407e..bd709c68bd4d9 100644 --- a/primitives/sr-primitives/Cargo.toml +++ b/primitives/sr-primitives/Cargo.toml @@ -21,7 +21,6 @@ inherents = { package = "substrate-inherents", path = "../inherents", default-fe [dev-dependencies] serde_json = "1.0.41" rand = "0.7.2" -substrate-offchain = { path = "../../client/offchain" } [features] bench = [] diff --git a/primitives/sr-primitives/src/offchain/http.rs b/primitives/sr-primitives/src/offchain/http.rs index 8024437075924..50d536fc14dbb 100644 --- a/primitives/sr-primitives/src/offchain/http.rs +++ b/primitives/sr-primitives/src/offchain/http.rs @@ -516,8 +516,10 @@ impl<'a> HeadersIterator<'a> { mod tests { use super::*; use runtime_io::TestExternalities; - use substrate_offchain::testing; - use primitives::offchain::OffchainExt; + use primitives::offchain::{ + OffchainExt, + testing, + }; #[test] fn should_send_a_basic_request_and_get_response() { From fdb356b16f0107404d2e37470d6529c54eae338d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Drwi=C4=99ga?= Date: Tue, 19 Nov 2019 16:15:03 +0100 Subject: [PATCH 07/10] Fix doctest. --- Cargo.lock | 14 ++++++-------- client/src/lib.rs | 1 - 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 03d4f6d7e8b42..c603406461e8a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2470,11 +2470,11 @@ dependencies = [ ] [[package]] -name = "lru-cache" -version = "0.1.2" +name = "lru" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "linked-hash-map 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", + "hashbrown 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -5792,7 +5792,7 @@ dependencies = [ name = "substrate-header-metadata" version = "2.0.0" dependencies = [ - "lru-cache 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "lru 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", "sr-primitives 2.0.0", ] @@ -5853,7 +5853,7 @@ dependencies = [ "linked-hash-map 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", "linked_hash_set 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", - "lru-cache 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "lru 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", "quickcheck 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -5939,9 +5939,7 @@ version = "2.0.0" dependencies = [ "futures-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)", "libp2p 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)", - "linked-hash-map 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", - "lru-cache 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.41 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -7850,7 +7848,7 @@ dependencies = [ "checksum lock_api 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f8912e782533a93a167888781b836336a6ca5da6175c05944c86cf28c31104dc" "checksum log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "e19e8d5c34a3e0e2223db8e060f9e8264aeeb5c5fc64a4ee9965c062211c024b" "checksum log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)" = "14b6052be84e6b71ab17edffc2eeabf5c2c3ae1fdb464aae35ac50c67a44e1f7" -"checksum lru-cache 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "31e24f1ad8321ca0e8a1e0ac13f23cb668e6f5466c2c57319f6a5cf1cc8e3b1c" +"checksum lru 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "26b0dca4ac5b5083c5169ab12205e6473df1c7659940e4978b94f363c6b54b22" "checksum mach 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "86dd2487cdfea56def77b88438a2c915fb45113c5319bfe7e14306ca4cd0b0e1" "checksum malloc_size_of_derive 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e37c5d4cd9473c5f4c9c111f033f15d4df9bd378fdf615944e360a4f55a05f0b" "checksum matches 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" diff --git a/client/src/lib.rs b/client/src/lib.rs index e8ad90fccad18..71127d3cb061e 100644 --- a/client/src/lib.rs +++ b/client/src/lib.rs @@ -63,7 +63,6 @@ //! LocalCallExecutor::new( //! backend.clone(), //! NativeExecutor::::new(WasmExecutionMethod::Interpreted, None), -//! None, //! ), //! // This parameter provides the storage for the chain genesis. //! <(StorageOverlay, ChildrenStorageOverlay)>::default(), From e7f2b75fc0705294f39258db9b84ecf6631b64c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Drwi=C4=99ga?= Date: Tue, 19 Nov 2019 18:20:55 +0100 Subject: [PATCH 08/10] Fix formatting and add some logs. --- client/transaction-pool/graph/src/pool.rs | 23 ++++++++++++----------- primitives/sr-io/src/lib.rs | 3 ++- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/client/transaction-pool/graph/src/pool.rs b/client/transaction-pool/graph/src/pool.rs index b58f3ed175df5..514ce174a1ef2 100644 --- a/client/transaction-pool/graph/src/pool.rs +++ b/client/transaction-pool/graph/src/pool.rs @@ -394,19 +394,20 @@ impl sr_primitives::offchain::TransactionPool for Pool at: &BlockId, extrinsic: ::Extrinsic, ) -> Result<(), ()> { - let result = futures::executor::block_on( - self.submit_one(&at, extrinsic) + log::debug!( + target: "txpool", + "(offchain call) Submitting a transaction to the pool: {:?}", + extrinsic ); - result - .map(|_| ()) - .map_err(|e| { - log::warn!( - target: "txpool", - "(offchain call) Error submitting a transaction to the pool: {:?}", - e - ) - }) + let result = futures::executor::block_on(self.submit_one(&at, extrinsic)); + + result.map(|_| ()) + .map_err(|e| log::warn!( + target: "txpool", + "(offchain call) Error submitting a transaction to the pool: {:?}", + e + )) } } diff --git a/primitives/sr-io/src/lib.rs b/primitives/sr-io/src/lib.rs index 6f11ff881d5f5..2b40670c450cb 100644 --- a/primitives/sr-io/src/lib.rs +++ b/primitives/sr-io/src/lib.rs @@ -428,7 +428,8 @@ pub trait Offchain { /// The transaction will end up in the pool. fn submit_transaction(&mut self, data: Vec) -> Result<(), ()> { self.extension::() - .expect("submit_transaction can be called only in the offchain worker context") + .expect("submit_transaction can be called only in the offchain call context with + TransactionPool capabilities enabled") .submit_transaction(data) } From d28a521530bea834a2a612e8e48f00b18a1efd26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Drwi=C4=99ga?= Date: Tue, 19 Nov 2019 18:26:55 +0100 Subject: [PATCH 09/10] Add some docs. --- client/api/src/backend.rs | 13 +++++++++++++ client/api/src/call_executor.rs | 2 ++ client/api/src/client.rs | 2 ++ 3 files changed, 17 insertions(+) diff --git a/client/api/src/backend.rs b/client/api/src/backend.rs index 1ab76a0681fd8..a00c5a3ac89f5 100644 --- a/client/api/src/backend.rs +++ b/client/api/src/backend.rs @@ -41,12 +41,22 @@ pub type StorageCollection = Vec<(Vec, Option>)>; /// In memory arrays of storage values for multiple child tries. pub type ChildStorageCollection = Vec<(Vec, StorageCollection)>; +/// Import operation summary. +/// +/// Contains information about the block that just got imported, +/// including storage changes, reorged blocks, etc. pub struct ImportSummary { + /// Block hash of the imported block. pub hash: Block::Hash, + /// Import origin. pub origin: BlockOrigin, + /// Header of the imported block. pub header: Block::Header, + /// Is this block a new best block. pub is_new_best: bool, + /// Optional storage changes. pub storage_changes: Option<(StorageCollection, ChildStorageCollection)>, + /// Blocks that got retracted because of this one got imported. pub retracted: Vec, } @@ -56,8 +66,11 @@ pub struct ClientImportOperation< H: Hasher, B: Backend, > { + /// DB Operation. pub op: B::BlockImportOperation, + /// Summary of imported block. pub notify_imported: Option>, + /// A list of hashes of blocks that got finalized. pub notify_finalized: Vec, } diff --git a/client/api/src/call_executor.rs b/client/api/src/call_executor.rs index 1a4430d112314..73273336330ae 100644 --- a/client/api/src/call_executor.rs +++ b/client/api/src/call_executor.rs @@ -14,6 +14,8 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . +//! A method call executor interface. + use std::{cmp::Ord, panic::UnwindSafe, result, cell::RefCell}; use codec::{Encode, Decode}; use sr_primitives::{ diff --git a/client/api/src/client.rs b/client/api/src/client.rs index 1809d0a30763a..675a53974c8d8 100644 --- a/client/api/src/client.rs +++ b/client/api/src/client.rs @@ -14,6 +14,8 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . +//! A set of APIs supported by the client along with their primitives. + use std::collections::HashMap; use futures03::channel::mpsc; use primitives::storage::StorageKey; From 2f06ac4cdddeaf94623644fabe098764424f7b51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Drwi=C4=99ga?= Date: Wed, 20 Nov 2019 11:53:31 +0100 Subject: [PATCH 10/10] Remove unused files. --- primitives/sr-io/with_std.rs | 559 -------------- primitives/sr-io/without_std.rs | 1241 ------------------------------- 2 files changed, 1800 deletions(-) delete mode 100644 primitives/sr-io/with_std.rs delete mode 100644 primitives/sr-io/without_std.rs diff --git a/primitives/sr-io/with_std.rs b/primitives/sr-io/with_std.rs deleted file mode 100644 index dd0acbd49fce2..0000000000000 --- a/primitives/sr-io/with_std.rs +++ /dev/null @@ -1,559 +0,0 @@ -// Copyright 2017-2019 Parity Technologies (UK) Ltd. -// This file is part of Substrate. - -// Substrate is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Substrate is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Substrate. If not, see . - -use primitives::{ - blake2_128, blake2_256, twox_128, twox_256, twox_64, ed25519, Blake2Hasher, sr25519, Pair, H256, - traits::KeystoreExt, - storage::ChildStorageKey, - hexdisplay::HexDisplay, Hasher, - offchain::{self, OffchainExt, TransactionPoolExt, TransactionPool}, -}; -// Switch to this after PoC-3 -// pub use primitives::BlakeHasher; -pub use substrate_state_machine::{BasicExternalities, TestExternalities}; - -use trie::{TrieConfiguration, trie_types::Layout}; - -use std::{collections::HashMap, convert::TryFrom}; - -use externalities::{with_externalities, set_and_run_with_externalities, ExternalitiesExt}; - -/// Additional bounds for `Hasher` trait for with_std. -pub trait HasherBounds {} -impl HasherBounds for T {} - -/// Returns a `ChildStorageKey` if the given `storage_key` slice is a valid storage -/// key or panics otherwise. -/// -/// Panicking here is aligned with what the `without_std` environment would do -/// in the case of an invalid child storage key. -fn child_storage_key_or_panic(storage_key: &[u8]) -> ChildStorageKey { - match ChildStorageKey::from_slice(storage_key) { - Some(storage_key) => storage_key, - None => panic!("child storage key is invalid"), - } -} - -impl StorageApi for () { - fn storage(key: &[u8]) -> Option> { - with_externalities(|ext| ext.storage(key).map(|s| s.to_vec())) - .expect("storage cannot be called outside of an Externalities-provided environment.") - } - - fn read_storage(key: &[u8], value_out: &mut [u8], value_offset: usize) -> Option { - with_externalities(|ext| ext.storage(key).map(|value| { - let data = &value[value_offset.min(value.len())..]; - let written = std::cmp::min(data.len(), value_out.len()); - value_out[..written].copy_from_slice(&data[..written]); - value.len() - })).expect("read_storage cannot be called outside of an Externalities-provided environment.") - } - - fn child_storage(storage_key: &[u8], key: &[u8]) -> Option> { - with_externalities(|ext| { - let storage_key = child_storage_key_or_panic(storage_key); - ext.child_storage(storage_key, key).map(|s| s.to_vec()) - }) - .expect("storage cannot be called outside of an Externalities-provided environment.") - } - - fn set_storage(key: &[u8], value: &[u8]) { - with_externalities(|ext| - ext.set_storage(key.to_vec(), value.to_vec()) - ); - } - - fn read_child_storage( - storage_key: &[u8], - key: &[u8], - value_out: &mut [u8], - value_offset: usize, - ) -> Option { - with_externalities(|ext| { - let storage_key = child_storage_key_or_panic(storage_key); - ext.child_storage(storage_key, key) - .map(|value| { - let data = &value[value_offset.min(value.len())..]; - let written = std::cmp::min(data.len(), value_out.len()); - value_out[..written].copy_from_slice(&data[..written]); - value.len() - }) - }) - .expect("read_child_storage cannot be called outside of an Externalities-provided environment.") - } - - fn set_child_storage(storage_key: &[u8], key: &[u8], value: &[u8]) { - with_externalities(|ext| { - let storage_key = child_storage_key_or_panic(storage_key); - ext.set_child_storage(storage_key, key.to_vec(), value.to_vec()) - }); - } - - fn clear_storage(key: &[u8]) { - with_externalities(|ext| - ext.clear_storage(key) - ); - } - - fn clear_child_storage(storage_key: &[u8], key: &[u8]) { - with_externalities(|ext| { - let storage_key = child_storage_key_or_panic(storage_key); - ext.clear_child_storage(storage_key, key) - }); - } - - fn kill_child_storage(storage_key: &[u8]) { - with_externalities(|ext| { - let storage_key = child_storage_key_or_panic(storage_key); - ext.kill_child_storage(storage_key) - }); - } - - fn exists_storage(key: &[u8]) -> bool { - with_externalities(|ext| - ext.exists_storage(key) - ).unwrap_or(false) - } - - fn exists_child_storage(storage_key: &[u8], key: &[u8]) -> bool { - with_externalities(|ext| { - let storage_key = child_storage_key_or_panic(storage_key); - ext.exists_child_storage(storage_key, key) - }).unwrap_or(false) - } - - fn clear_prefix(prefix: &[u8]) { - with_externalities(|ext| ext.clear_prefix(prefix)); - } - - fn clear_child_prefix(storage_key: &[u8], prefix: &[u8]) { - with_externalities(|ext| { - let storage_key = child_storage_key_or_panic(storage_key); - ext.clear_child_prefix(storage_key, prefix) - }); - } - - fn storage_root() -> [u8; 32] { - with_externalities(|ext| - ext.storage_root() - ).unwrap_or(H256::zero()).into() - } - - fn child_storage_root(storage_key: &[u8]) -> Vec { - with_externalities(|ext| { - let storage_key = child_storage_key_or_panic(storage_key); - ext.child_storage_root(storage_key) - }).expect("child_storage_root cannot be called outside of an Externalities-provided environment.") - } - - fn storage_changes_root(parent_hash: [u8; 32]) -> Option<[u8; 32]> { - with_externalities(|ext| - ext.storage_changes_root(parent_hash.into()).map(|h| h.map(|h| h.into())) - ).unwrap_or(Ok(None)).expect("Invalid parent hash passed to storage_changes_root") - } - - fn blake2_256_trie_root(input: Vec<(Vec, Vec)>) -> H256 { - Layout::::trie_root(input) - } - - fn blake2_256_ordered_trie_root(input: Vec>) -> H256 { - Layout::::ordered_trie_root(input) - } -} - -impl OtherApi for () { - fn chain_id() -> u64 { - with_externalities(|ext| - ext.chain_id() - ).unwrap_or(0) - } - - fn print_num(val: u64) { - log::debug!(target: "runtime", "{}", val); - } - - fn print_utf8(utf8: &[u8]) { - if let Ok(data) = std::str::from_utf8(utf8) { - log::debug!(target: "runtime", "{}", data) - } - } - - fn print_hex(data: &[u8]) { - log::debug!(target: "runtime", "{}", HexDisplay::from(&data)); - } - - fn log( - level: LogLevel, - target: &[u8], - message: &[u8], - ) { - let target = std::str::from_utf8(target).unwrap_or("invalid utf8"); - let msg = std::str::from_utf8(message).unwrap_or("invalid utf8"); - - log::log!( - target: target, - log::Level::from(level), - "{}", - msg, - ) - } -} - -impl CryptoApi for () { - fn ed25519_public_keys(id: KeyTypeId) -> Vec { - with_externalities(|ext| { - ext.extension::() - .expect("No `keystore` associated for the current context!") - .read() - .ed25519_public_keys(id) - }).expect("`ed25519_public_keys` cannot be called outside of an Externalities-provided environment.") - } - - fn ed25519_generate(id: KeyTypeId, seed: Option<&str>) -> ed25519::Public { - with_externalities(|ext| { - ext.extension::() - .expect("No `keystore` associated for the current context!") - .write() - .ed25519_generate_new(id, seed) - .expect("`ed25519_generate` failed") - }).expect("`ed25519_generate` cannot be called outside of an Externalities-provided environment.") - } - - fn ed25519_sign( - id: KeyTypeId, - pubkey: &ed25519::Public, - msg: &[u8], - ) -> Option { - let pub_key = ed25519::Public::try_from(pubkey.as_ref()).ok()?; - - with_externalities(|ext| { - ext.extension::() - .expect("No `keystore` associated for the current context!") - .read() - .ed25519_key_pair(id, &pub_key) - .map(|k| k.sign(msg)) - }).expect("`ed25519_sign` cannot be called outside of an Externalities-provided environment.") - } - - fn ed25519_verify(sig: &ed25519::Signature, msg: &[u8], pubkey: &ed25519::Public) -> bool { - ed25519::Pair::verify(sig, msg, pubkey) - } - - fn sr25519_public_keys(id: KeyTypeId) -> Vec { - with_externalities(|ext| { - ext.extension::() - .expect("No `keystore` associated for the current context!") - .read() - .sr25519_public_keys(id) - }).expect("`sr25519_public_keys` cannot be called outside of an Externalities-provided environment.") - } - - fn sr25519_generate(id: KeyTypeId, seed: Option<&str>) -> sr25519::Public { - with_externalities(|ext| { - ext.extension::() - .expect("No `keystore` associated for the current context!") - .write() - .sr25519_generate_new(id, seed) - .expect("`sr25519_generate` failed") - }).expect("`sr25519_generate` cannot be called outside of an Externalities-provided environment.") - } - - fn sr25519_sign( - id: KeyTypeId, - pubkey: &sr25519::Public, - msg: &[u8], - ) -> Option { - let pub_key = sr25519::Public::try_from(pubkey.as_ref()).ok()?; - - with_externalities(|ext| { - ext.extension::() - .expect("No `keystore` associated for the current context!") - .read() - .sr25519_key_pair(id, &pub_key) - .map(|k| k.sign(msg)) - }).expect("`sr25519_sign` cannot be called outside of an Externalities-provided environment.") - } - - fn sr25519_verify(sig: &sr25519::Signature, msg: &[u8], pubkey: &sr25519::Public) -> bool { - sr25519::Pair::verify(sig, msg, pubkey) - } - - fn secp256k1_ecdsa_recover(sig: &[u8; 65], msg: &[u8; 32]) -> Result<[u8; 64], EcdsaVerifyError> { - let rs = secp256k1::Signature::parse_slice(&sig[0..64]) - .map_err(|_| EcdsaVerifyError::BadRS)?; - let v = secp256k1::RecoveryId::parse(if sig[64] > 26 { sig[64] - 27 } else { sig[64] } as u8) - .map_err(|_| EcdsaVerifyError::BadV)?; - let pubkey = secp256k1::recover(&secp256k1::Message::parse(msg), &rs, &v) - .map_err(|_| EcdsaVerifyError::BadSignature)?; - let mut res = [0u8; 64]; - res.copy_from_slice(&pubkey.serialize()[1..65]); - Ok(res) - } - - fn secp256k1_ecdsa_recover_compressed(sig: &[u8; 65], msg: &[u8; 32]) -> Result<[u8; 33], EcdsaVerifyError> { - let rs = secp256k1::Signature::parse_slice(&sig[0..64]) - .map_err(|_| EcdsaVerifyError::BadRS)?; - let v = secp256k1::RecoveryId::parse(if sig[64] > 26 { sig[64] - 27 } else { sig[64] } as u8) - .map_err(|_| EcdsaVerifyError::BadV)?; - let pubkey = secp256k1::recover(&secp256k1::Message::parse(msg), &rs, &v) - .map_err(|_| EcdsaVerifyError::BadSignature)?; - Ok(pubkey.serialize_compressed()) - } -} - -impl HashingApi for () { - fn keccak_256(data: &[u8]) -> [u8; 32] { - tiny_keccak::keccak256(data) - } - - fn blake2_128(data: &[u8]) -> [u8; 16] { - blake2_128(data) - } - - fn blake2_256(data: &[u8]) -> [u8; 32] { - blake2_256(data) - } - - fn twox_256(data: &[u8]) -> [u8; 32] { - twox_256(data) - } - - fn twox_128(data: &[u8]) -> [u8; 16] { - twox_128(data) - } - - fn twox_64(data: &[u8]) -> [u8; 8] { - twox_64(data) - } -} - -fn with_offchain(f: impl FnOnce(&mut dyn offchain::Externalities) -> R, msg: &'static str) -> R { - with_externalities(|ext| ext - .extension::() - .map(|ext| f(&mut **ext)) - .expect(msg) - ).expect("offchain-worker functions cannot be called outside of an Externalities-provided environment.") -} - -impl OffchainApi for () { - fn submit_transaction(data: Vec) -> Result<(), ()> { - with_externalities(|ext| ext - .extension::() - .map(|ext| ext.submit_transaction(data)) - .expect(r#"submit_transaction can be called only in the offchain context - with TransactionPool capabilities"# - ) - ).expect("submit_transaction cannot be called outside of an Externalities-provided environment.") - } - - fn is_validator() -> bool { - with_offchain(|ext| { - ext.is_validator() - }, "is_validator can be called only in the offchain worker context") - } - - fn network_state() -> Result { - with_offchain(|ext| { - ext.network_state() - }, "network_state can be called only in the offchain worker context") - } - - fn timestamp() -> offchain::Timestamp { - with_offchain(|ext| { - ext.timestamp() - }, "timestamp can be called only in the offchain worker context") - } - - fn sleep_until(deadline: offchain::Timestamp) { - with_offchain(|ext| { - ext.sleep_until(deadline) - }, "sleep_until can be called only in the offchain worker context") - } - - fn random_seed() -> [u8; 32] { - with_offchain(|ext| { - ext.random_seed() - }, "random_seed can be called only in the offchain worker context") - } - - fn local_storage_set(kind: offchain::StorageKind, key: &[u8], value: &[u8]) { - with_offchain(|ext| { - ext.local_storage_set(kind, key, value) - }, "local_storage_set can be called only in the offchain worker context") - } - - fn local_storage_compare_and_set( - kind: offchain::StorageKind, - key: &[u8], - old_value: Option<&[u8]>, - new_value: &[u8], - ) -> bool { - with_offchain(|ext| { - ext.local_storage_compare_and_set(kind, key, old_value, new_value) - }, "local_storage_compare_and_set can be called only in the offchain worker context") - } - - fn local_storage_get(kind: offchain::StorageKind, key: &[u8]) -> Option> { - with_offchain(|ext| { - ext.local_storage_get(kind, key) - }, "local_storage_get can be called only in the offchain worker context") - } - - fn http_request_start( - method: &str, - uri: &str, - meta: &[u8], - ) -> Result { - with_offchain(|ext| { - ext.http_request_start(method, uri, meta) - }, "http_request_start can be called only in the offchain worker context") - } - - fn http_request_add_header( - request_id: offchain::HttpRequestId, - name: &str, - value: &str, - ) -> Result<(), ()> { - with_offchain(|ext| { - ext.http_request_add_header(request_id, name, value) - }, "http_request_add_header can be called only in the offchain worker context") - } - - fn http_request_write_body( - request_id: offchain::HttpRequestId, - chunk: &[u8], - deadline: Option, - ) -> Result<(), offchain::HttpError> { - with_offchain(|ext| { - ext.http_request_write_body(request_id, chunk, deadline) - }, "http_request_write_body can be called only in the offchain worker context") - } - - fn http_response_wait( - ids: &[offchain::HttpRequestId], - deadline: Option, - ) -> Vec { - with_offchain(|ext| { - ext.http_response_wait(ids, deadline) - }, "http_response_wait can be called only in the offchain worker context") - } - - fn http_response_headers( - request_id: offchain::HttpRequestId, - ) -> Vec<(Vec, Vec)> { - with_offchain(|ext| { - ext.http_response_headers(request_id) - }, "http_response_headers can be called only in the offchain worker context") - } - - fn http_response_read_body( - request_id: offchain::HttpRequestId, - buffer: &mut [u8], - deadline: Option, - ) -> Result { - with_offchain(|ext| { - ext.http_response_read_body(request_id, buffer, deadline) - }, "http_response_read_body can be called only in the offchain worker context") - } -} - -impl Api for () {} - -/// A set of key value pairs for storage. -pub type StorageOverlay = HashMap, Vec>; - -/// A set of key value pairs for children storage; -pub type ChildrenStorageOverlay = HashMap, StorageOverlay>; - -/// Execute the given closure with global functions available whose functionality routes into -/// externalities that draw from and populate `storage` and `children_storage`. -/// Forwards the value that the closure returns. -pub fn with_storage R>( - storage: &mut (StorageOverlay, ChildrenStorageOverlay), - f: F -) -> R { - let mut alt_storage = Default::default(); - rstd::mem::swap(&mut alt_storage, storage); - - let mut ext = BasicExternalities::new(alt_storage.0, alt_storage.1); - let r = set_and_run_with_externalities(&mut ext, f); - - *storage = ext.into_storages(); - - r -} - -#[cfg(test)] -mod std_tests { - use super::*; - use primitives::map; - - #[test] - fn storage_works() { - let mut t = BasicExternalities::default(); - assert!(set_and_run_with_externalities(&mut t, || { - assert_eq!(storage(b"hello"), None); - set_storage(b"hello", b"world"); - assert_eq!(storage(b"hello"), Some(b"world".to_vec())); - assert_eq!(storage(b"foo"), None); - set_storage(b"foo", &[1, 2, 3][..]); - true - })); - - t = BasicExternalities::new(map![b"foo".to_vec() => b"bar".to_vec()], map![]); - - assert!(!set_and_run_with_externalities(&mut t, || { - assert_eq!(storage(b"hello"), None); - assert_eq!(storage(b"foo"), Some(b"bar".to_vec())); - false - })); - } - - #[test] - fn read_storage_works() { - let mut t = BasicExternalities::new(map![ - b":test".to_vec() => b"\x0b\0\0\0Hello world".to_vec() - ], map![]); - - set_and_run_with_externalities(&mut t, || { - let mut v = [0u8; 4]; - assert!(read_storage(b":test", &mut v[..], 0).unwrap() >= 4); - assert_eq!(v, [11u8, 0, 0, 0]); - let mut w = [0u8; 11]; - assert!(read_storage(b":test", &mut w[..], 4).unwrap() >= 11); - assert_eq!(&w, b"Hello world"); - }); - } - - #[test] - fn clear_prefix_works() { - let mut t = BasicExternalities::new(map![ - b":a".to_vec() => b"\x0b\0\0\0Hello world".to_vec(), - b":abcd".to_vec() => b"\x0b\0\0\0Hello world".to_vec(), - b":abc".to_vec() => b"\x0b\0\0\0Hello world".to_vec(), - b":abdd".to_vec() => b"\x0b\0\0\0Hello world".to_vec() - ], map![]); - - set_and_run_with_externalities(&mut t, || { - clear_prefix(b":abc"); - - assert!(storage(b":a").is_some()); - assert!(storage(b":abdd").is_some()); - assert!(storage(b":abcd").is_none()); - assert!(storage(b":abc").is_none()); - }); - } -} diff --git a/primitives/sr-io/without_std.rs b/primitives/sr-io/without_std.rs deleted file mode 100644 index c3f7d62031b93..0000000000000 --- a/primitives/sr-io/without_std.rs +++ /dev/null @@ -1,1241 +0,0 @@ -// Copyright 2017-2019 Parity Technologies (UK) Ltd. -// This file is part of Substrate. - -// Substrate is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Substrate is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Substrate. If not, see . - -#[doc(hidden)] -pub use rstd; -pub use rstd::{mem, slice}; - -use core::{intrinsics, panic::PanicInfo}; -use rstd::{vec::Vec, cell::Cell, convert::TryInto}; -use primitives::offchain; -use codec::Decode; - -#[cfg(not(feature = "no_panic_handler"))] -#[panic_handler] -#[no_mangle] -pub fn panic(info: &PanicInfo) -> ! { - unsafe { - let message = rstd::alloc::format!("{}", info); - extern_functions_host_impl::ext_print_utf8(message.as_ptr() as *const u8, message.len() as u32); - intrinsics::abort() - } -} - -#[cfg(not(feature = "no_oom"))] -#[alloc_error_handler] -pub extern fn oom(_: core::alloc::Layout) -> ! { - static OOM_MSG: &str = "Runtime memory exhausted. Aborting"; - - unsafe { - extern_functions_host_impl::ext_print_utf8(OOM_MSG.as_ptr(), OOM_MSG.len() as u32); - intrinsics::abort(); - } -} - -/// External (Host) APIs -pub mod ext { - use super::*; - - /// The state of an exchangeable function. - #[derive(Clone, Copy)] - enum ExchangeableFunctionState { - /// Original function is present - Original, - /// The function has been replaced. - Replaced, - } - - /// A function which implementation can be exchanged. - /// - /// Internally this works by swapping function pointers. - pub struct ExchangeableFunction(Cell<(T, ExchangeableFunctionState)>); - - impl ExchangeableFunction { - /// Create a new instance of `ExchangeableFunction`. - pub const fn new(impl_: T) -> Self { - Self(Cell::new((impl_, ExchangeableFunctionState::Original))) - } - } - - impl ExchangeableFunction { - /// Replace the implementation with `new_impl`. - /// - /// # Panics - /// - /// Panics when trying to replace an already replaced implementation. - /// - /// # Returns - /// - /// Returns the original implementation wrapped in [`RestoreImplementation`]. - pub fn replace_implementation(&'static self, new_impl: T) -> RestoreImplementation { - if let ExchangeableFunctionState::Replaced = self.0.get().1 { - panic!("Trying to replace an already replaced implementation!") - } - - let old = self.0.replace((new_impl, ExchangeableFunctionState::Replaced)); - - RestoreImplementation(self, Some(old.0)) - } - - /// Restore the original implementation. - fn restore_orig_implementation(&self, orig: T) { - self.0.set((orig, ExchangeableFunctionState::Original)); - } - - /// Returns the internal function pointer. - pub fn get(&self) -> T { - self.0.get().0 - } - } - - // WASM does not support threads, so this is safe; qed. - unsafe impl Sync for ExchangeableFunction {} - - /// Restores a function implementation on drop. - /// - /// Stores a static reference to the function object and the original implementation. - pub struct RestoreImplementation(&'static ExchangeableFunction, Option); - - impl Drop for RestoreImplementation { - fn drop(&mut self) { - self.0.restore_orig_implementation(self.1.take().expect("Value is only taken on drop; qed")); - } - } - - /// Declare extern functions - macro_rules! extern_functions { - ( - $( - $( #[$attr:meta] )* - fn $name:ident ( $( $arg:ident : $arg_ty:ty ),* $(,)? ) $( -> $ret:ty )?; - )* - ) => { - $( - $( #[$attr] )* - #[allow(non_upper_case_globals)] - pub static $name: ExchangeableFunction $ret )?> = - ExchangeableFunction::new(extern_functions_host_impl::$name); - )* - - /// The exchangeable extern functions host implementations. - pub(crate) mod extern_functions_host_impl { - $( - pub unsafe fn $name ( $( $arg : $arg_ty ),* ) $( -> $ret )? { - implementation::$name ( $( $arg ),* ) - } - )* - - mod implementation { - extern "C" { - $( - pub fn $name ( $( $arg : $arg_ty ),* ) $( -> $ret )?; - )* - } - } - } - }; - } - - /// Host functions, provided by the executor. - /// A WebAssembly runtime module would "import" these to access the execution environment - /// (most importantly, storage) or perform heavy hash calculations. - /// See also "ext_" functions in sr-sandbox and sr-std - extern_functions! { - /// Host functions for printing, useful for debugging. - fn ext_print_utf8(utf8_data: *const u8, utf8_len: u32); - /// Print data as hex. - fn ext_print_hex(data: *const u8, len: u32); - /// Print a number - fn ext_print_num(value: u64); - /// Print a log line if logging for given level and target is enabled. - fn ext_log( - level: u32, - target_data: *const u8, - target_len: u32, - message_data: *const u8, - message_len: u32, - ); - - /// Set value for key in storage. - fn ext_set_storage(key_data: *const u8, key_len: u32, value_data: *const u8, value_len: u32); - /// Remove key and value from storage. - fn ext_clear_storage(key_data: *const u8, key_len: u32); - /// Checks if the given key exists in the storage. - /// - /// # Returns - /// - /// - `1` if the value exists. - /// - `0` if the value does not exists. - fn ext_exists_storage(key_data: *const u8, key_len: u32) -> u32; - /// Remove storage entries which key starts with given prefix. - fn ext_clear_prefix(prefix_data: *const u8, prefix_len: u32); - /// Remove child storage entries which key starts with given prefix. - fn ext_clear_child_prefix( - storage_key_data: *const u8, - storage_key_len: u32, - prefix_data: *const u8, - prefix_len: u32, - ); - /// Gets the value of the given key from storage. - /// - /// The host allocates the memory for storing the value. - /// - /// # Returns - /// - /// - `0` if no value exists to the given key. `written_out` is set to `u32::max_value()`. - /// - Otherwise, pointer to the value in memory. `written_out` contains the length of the value. - fn ext_get_allocated_storage(key_data: *const u8, key_len: u32, written_out: *mut u32) -> *mut u8; - /// Gets the value of the given key from storage. - /// - /// The value is written into `value` starting at `value_offset`. - /// - /// If the value length is greater than `value_len - value_offset`, the value is written partially. - /// - /// # Returns - /// - /// - `u32::max_value()` if the value does not exists. - /// - /// - Otherwise, the number of bytes written for value. - fn ext_get_storage_into( - key_data: *const u8, - key_len: u32, - value_data: *mut u8, - value_len: u32, - value_offset: u32, - ) -> u32; - /// Gets the trie root of the storage. - fn ext_storage_root(result: *mut u8); - /// Get the change trie root of the current storage overlay at a block with given parent. - /// - /// # Returns - /// - /// - `1` if the change trie root was found. - /// - `0` if the change trie root was not found. - fn ext_storage_changes_root( - parent_hash_data: *const u8, - parent_hash_len: u32, - result: *mut u8, - ) -> u32; - - /// A child storage function. - /// - /// See [`ext_set_storage`] for details. - /// - /// A child storage is used e.g. by a contract. - fn ext_set_child_storage( - storage_key_data: *const u8, - storage_key_len: u32, - key_data: *const u8, - key_len: u32, - value_data: *const u8, - value_len: u32, - ); - /// A child storage function. - /// - /// See [`ext_clear_storage`] for details. - /// - /// A child storage is used e.g. by a contract. - fn ext_clear_child_storage( - storage_key_data: *const u8, - storage_key_len: u32, - key_data: *const u8, - key_len: u32, - ); - /// A child storage function. - /// - /// See [`ext_exists_storage`] for details. - /// - /// A child storage is used e.g. by a contract. - fn ext_exists_child_storage( - storage_key_data: *const u8, - storage_key_len: u32, - key_data: *const u8, - key_len: u32, - ) -> u32; - /// A child storage function. - /// - /// See [`ext_kill_storage`] for details. - /// - /// A child storage is used e.g. by a contract. - fn ext_kill_child_storage(storage_key_data: *const u8, storage_key_len: u32); - /// A child storage function. - /// - /// See [`ext_get_allocated_storage`] for details. - /// - /// A child storage is used e.g. by a contract. - fn ext_get_allocated_child_storage( - storage_key_data: *const u8, - storage_key_len: u32, - key_data: *const u8, - key_len: u32, - written_out: *mut u32, - ) -> *mut u8; - /// A child storage function. - /// - /// See [`ext_get_storage_into`] for details. - /// - /// A child storage is used e.g. by a contract. - fn ext_get_child_storage_into( - storage_key_data: *const u8, - storage_key_len: u32, - key_data: *const u8, - key_len: u32, - value_data: *mut u8, - value_len: u32, - value_offset: u32, - ) -> u32; - /// Commits all changes and calculates the child-storage root. - /// - /// A child storage is used e.g. by a contract. - /// - /// # Returns - /// - /// - The pointer to the result vector and `written_out` contains its length. - fn ext_child_storage_root( - storage_key_data: *const u8, - storage_key_len: u32, - written_out: *mut u32 - ) -> *mut u8; - - /// The current relay chain identifier. - fn ext_chain_id() -> u64; - - /// Calculate a blake2_256 merkle trie root. - fn ext_blake2_256_enumerated_trie_root( - values_data: *const u8, - lens_data: *const u32, - lens_len: u32, - result: *mut u8 - ); - /// BLAKE2_128 hash - fn ext_blake2_128(data: *const u8, len: u32, out: *mut u8); - /// BLAKE2_256 hash - fn ext_blake2_256(data: *const u8, len: u32, out: *mut u8); - /// XX64 hash - fn ext_twox_64(data: *const u8, len: u32, out: *mut u8); - /// XX128 hash - fn ext_twox_128(data: *const u8, len: u32, out: *mut u8); - /// XX256 hash - fn ext_twox_256(data: *const u8, len: u32, out: *mut u8); - /// Keccak256 hash - fn ext_keccak_256(data: *const u8, len: u32, out: *mut u8); - - /// Returns all `ed25519` public keys for the given key type from the keystore. - fn ext_ed25519_public_keys(id: *const u8, result_len: *mut u32) -> *mut u8; - - /// Note: `ext_ed25519_verify` returns `0` if the signature is correct, nonzero otherwise. - fn ext_ed25519_verify( - msg_data: *const u8, - msg_len: u32, - sig_data: *const u8, - pubkey_data: *const u8, - ) -> u32; - - /// Generate an `ed25519` key pair for the given key type id and store the public key - /// in `out`. - fn ext_ed25519_generate(id: *const u8, seed: *const u8, seed_len: u32, out: *mut u8); - - /// Sign the given `msg` with the `ed25519` key pair that corresponds to then given key - /// type id and public key. The raw signature is stored in `out`. - /// - /// # Returns - /// - /// - `0` on success - /// - nonezero if something failed, e.g. retrieving of the key. - fn ext_ed25519_sign( - id: *const u8, - pubkey: *const u8, - msg: *const u8, - msg_len: u32, - out: *mut u8, - ) -> u32; - - /// Returns all `sr25519` public keys for the given key type from the keystore. - fn ext_sr25519_public_keys(id: *const u8, result_len: *mut u32) -> *mut u8; - - /// Note: `ext_sr25519_verify` returns 0 if the signature is correct, nonzero otherwise. - fn ext_sr25519_verify( - msg_data: *const u8, - msg_len: u32, - sig_data: *const u8, - pubkey_data: *const u8, - ) -> u32; - - /// Generate an `sr25519` key pair for the given key type id and store the public - /// key in `out`. - fn ext_sr25519_generate(id: *const u8, seed: *const u8, seed_len: u32, out: *mut u8); - - /// Sign the given `msg` with the `sr25519` key pair that corresponds to then given key - /// type id and public key. The raw signature is stored in `out`. - /// - /// # Returns - /// - /// - `0` on success - /// - nonezero if something failed, e.g. retrieving of the key. - fn ext_sr25519_sign( - id: *const u8, - pubkey: *const u8, - msg: *const u8, - msg_len: u32, - out: *mut u8, - ) -> u32; - - /// Note: ext_secp256k1_ecdsa_recover returns 0 if the signature is correct, nonzero otherwise. - /// - /// pubkey_data must point to 64 bytes. - fn ext_secp256k1_ecdsa_recover( - msg_data: *const u8, - sig_data: *const u8, - pubkey_data: *mut u8, - ) -> u32; - - /// Note: ext_secp256k1_ecdsa_recover_compressed returns 0 if the signature is correct, nonzero otherwise. - /// - /// pubkey_data must point to 33 bytes. - fn ext_secp256k1_ecdsa_recover_compressed( - msg_data: *const u8, - sig_data: *const u8, - pubkey_data: *mut u8, - ) -> u32; - - //================================ - // Offchain-worker Context - //================================ - - /// Returns if the local node is a potential validator. - /// - /// - `1` == `true` - /// - `0` == `false` - fn ext_is_validator() -> u32; - - /// Submit transaction. - /// - /// # Returns - /// - /// - 0 if it was successfuly added to the pool - /// - nonzero otherwise. - fn ext_submit_transaction(data: *const u8, len: u32) -> u32; - - /// Returns information about the local node's network state. - /// - /// # Returns - /// - /// The encoded `Result`. - /// `written_out` contains the length of the message. - /// - /// The ownership of the returned buffer is transferred to the runtime - /// code and the runtime is responsible for freeing it. This is always - /// a properly allocated pointer (which cannot be NULL), hence the - /// runtime code can always rely on it. - fn ext_network_state(written_out: *mut u32) -> *mut u8; - - /// Returns current UNIX timestamp (milliseconds) - fn ext_timestamp() -> u64; - - /// Pause execution until given timestamp (milliseconds; `deadline`) is reached. - /// - /// The deadline is obtained by querying the current timestamp via `ext_timestamp` - /// and then adding some time to it. - fn ext_sleep_until(deadline: u64); - - /// Generate a random seed - /// - /// `data` has to be a pointer to a slice of 32 bytes. - fn ext_random_seed(data: *mut u8); - - /// Write a value to local storage. - fn ext_local_storage_set(kind: u32, key: *const u8, key_len: u32, value: *const u8, value_len: u32); - - /// Write a value to local storage in atomic fashion. - /// - /// # Returns - /// - `0` in case the value has been set - /// - `1` if the `old_value` didn't match - fn ext_local_storage_compare_and_set( - kind: u32, - key: *const u8, - key_len: u32, - old_value: *const u8, - old_value_len: u32, - new_value: *const u8, - new_value_len: u32, - ) -> u32; - - /// Read a value from local storage. - /// - /// - /// # Returns - /// - /// - 0 if the value has not been found, the `value_len` is set to `u32::max_value`. - /// - Otherwise, pointer to the value in memory. `value_len` contains the length of the value. - fn ext_local_storage_get(kind: u32, key: *const u8, key_len: u32, value_len: *mut u32) -> *mut u8; - - /// Initiates a http request. - /// - /// `meta` is parity-scale-codec encoded additional parameters to the request (like redirection policy, - /// timeouts, certificates policy, etc). The format is not yet specified and the field is currently - /// only reserved for future use. - /// - /// # Returns - /// - /// `RequestId(u16)` of initiated request, any value beyond `u16::max_value` - /// signifies an error. - fn ext_http_request_start( - method: *const u8, - method_len: u32, - url: *const u8, - url_len: u32, - meta: *const u8, - meta_len: u32, - ) -> u32; - - /// Add a header to the request. - /// - /// # Returns - /// - /// - `0` if successful (and the request id exists) - /// - nonzero otherwise - fn ext_http_request_add_header( - request_id: u32, - name: *const u8, - name_len: u32, - value: *const u8, - value_len: u32, - ) -> u32; - - /// Write a chunk of request body. - /// - /// Writing an empty chunks finalises the request. - /// Passing `0` as deadline blocks forever. - /// - /// # Returns - /// - /// - `0` if successful, - /// - nonzero otherwise (see HttpError for the codes) - fn ext_http_request_write_body( - request_id: u32, - chunk: *const u8, - chunk_len: u32, - deadline: u64, - ) -> u32; - - /// Block and wait for the responses for given requests. - /// - /// Note that if deadline is 0 the method will block indefinitely, - /// otherwise unready responses will produce `DeadlineReached` status. - /// (see #primitives::offchain::HttpRequestStatus) - /// - /// Make sure that `statuses` have the same length as ids. - fn ext_http_response_wait( - ids: *const u32, - ids_len: u32, - statuses: *mut u32, - deadline: u64, - ); - - /// Read all response headers. - /// - /// Note the headers are only available before response body is fully consumed. - /// - /// # Returns - /// - /// - A pointer to parity-scale-codec encoded vector of pairs `(HeaderKey, HeaderValue)`. - /// - In case invalid `id` is passed it returns a pointer to parity-encoded empty vector. - fn ext_http_response_headers( - id: u32, - written_out: *mut u32, - ) -> *mut u8; - - /// Read a chunk of body response to given buffer. - /// - /// Passing `0` as deadline blocks forever. - /// - /// # Returns - /// - /// The number of bytes written if successful, - /// - if it's `0` it means response has been fully consumed, - /// - if it's greater than `u32::max_value() - 255` it means reading body failed. - /// - /// In case of failure, the error code should be mapped to `HttpError` - /// in a following manner: - /// - `u32::max_value()` HttpError code 1 (DeadlineReached) - /// - `u32::max_value() - 1` HttpError code 2 (IoError) - /// The rest is reserved for potential future errors. - fn ext_http_response_read_body( - id: u32, - buffer: *mut u8, - buffer_len: u32, - deadline: u64, - ) -> u32; - } -} - -pub use self::ext::*; - -impl StorageApi for () { - fn storage(key: &[u8]) -> Option> { - let mut length: u32 = 0; - unsafe { - let ptr = ext_get_allocated_storage.get()(key.as_ptr(), key.len() as u32, &mut length); - from_raw_parts(ptr, length) - } - } - - fn child_storage(storage_key: &[u8], key: &[u8]) -> Option> { - let mut length: u32 = 0; - unsafe { - let ptr = ext_get_allocated_child_storage.get()( - storage_key.as_ptr(), - storage_key.len() as u32, - key.as_ptr(), - key.len() as u32, - &mut length - ); - from_raw_parts(ptr, length) - } - } - - fn read_storage(key: &[u8], value_out: &mut [u8], value_offset: usize) -> Option { - unsafe { - match ext_get_storage_into.get()( - key.as_ptr(), - key.len() as u32, - value_out.as_mut_ptr(), - value_out.len() as u32, - value_offset as u32, - ) { - none if none == u32::max_value() => None, - length => Some(length as usize), - } - } - } - - fn read_child_storage(storage_key: &[u8], key: &[u8], value_out: &mut [u8], value_offset: usize) -> Option { - unsafe { - match ext_get_child_storage_into.get()( - storage_key.as_ptr(), storage_key.len() as u32, - key.as_ptr(), key.len() as u32, - value_out.as_mut_ptr(), value_out.len() as u32, - value_offset as u32 - ) { - none if none == u32::max_value() => None, - length => Some(length as usize), - } - } - } - - fn set_storage(key: &[u8], value: &[u8]) { - unsafe { - ext_set_storage.get()( - key.as_ptr(), key.len() as u32, - value.as_ptr(), value.len() as u32 - ); - } - } - - fn set_child_storage(storage_key: &[u8], key: &[u8], value: &[u8]) { - unsafe { - ext_set_child_storage.get()( - storage_key.as_ptr(), storage_key.len() as u32, - key.as_ptr(), key.len() as u32, - value.as_ptr(), value.len() as u32 - ); - } - } - - fn clear_storage(key: &[u8]) { - unsafe { - ext_clear_storage.get()( - key.as_ptr(), key.len() as u32 - ); - } - } - - fn clear_child_storage(storage_key: &[u8], key: &[u8]) { - unsafe { - ext_clear_child_storage.get()( - storage_key.as_ptr(), storage_key.len() as u32, - key.as_ptr(), key.len() as u32 - ); - } - } - - fn exists_storage(key: &[u8]) -> bool { - unsafe { - ext_exists_storage.get()( - key.as_ptr(), key.len() as u32 - ) != 0 - } - } - - fn exists_child_storage(storage_key: &[u8], key: &[u8]) -> bool { - unsafe { - ext_exists_child_storage.get()( - storage_key.as_ptr(), storage_key.len() as u32, - key.as_ptr(), key.len() as u32 - ) != 0 - } - } - - fn clear_prefix(prefix: &[u8]) { - unsafe { - ext_clear_prefix.get()( - prefix.as_ptr(), - prefix.len() as u32 - ); - } - } - - fn clear_child_prefix(storage_key: &[u8], prefix: &[u8]) { - unsafe { - ext_clear_child_prefix.get()( - storage_key.as_ptr(), storage_key.len() as u32, - prefix.as_ptr(), prefix.len() as u32 - ); - } - } - - fn kill_child_storage(storage_key: &[u8]) { - unsafe { - ext_kill_child_storage.get()( - storage_key.as_ptr(), - storage_key.len() as u32 - ); - } - } - - fn storage_root() -> [u8; 32] { - let mut result: [u8; 32] = Default::default(); - unsafe { - ext_storage_root.get()(result.as_mut_ptr()); - } - result - } - - fn child_storage_root(storage_key: &[u8]) -> Vec { - let mut length: u32 = 0; - unsafe { - let ptr = ext_child_storage_root.get()( - storage_key.as_ptr(), - storage_key.len() as u32, - &mut length - ); - from_raw_parts(ptr, length).expect("ext_child_storage_root never returns u32::max_value; qed") - } - } - - fn storage_changes_root(parent_hash: [u8; 32]) -> Option<[u8; 32]> { - let mut result: [u8; 32] = Default::default(); - let is_set = unsafe { - ext_storage_changes_root.get()(parent_hash.as_ptr(), parent_hash.len() as u32, result.as_mut_ptr()) - }; - - if is_set != 0 { - Some(result) - } else { - None - } - } - - - fn blake2_256_trie_root(_input: Vec<(Vec, Vec)>) -> H256 { - unimplemented!() - } - - fn blake2_256_ordered_trie_root(input: Vec>) -> H256 { - let mut values = Vec::with_capacity(input.len()); - let mut lengths = Vec::with_capacity(input.len()); - for v in input { - values.extend_from_slice(&v); - lengths.push((v.len() as u32).to_le()); - } - let mut result: [u8; 32] = Default::default(); - unsafe { - ext_blake2_256_enumerated_trie_root.get()( - values.as_ptr(), - lengths.as_ptr(), - lengths.len() as u32, - result.as_mut_ptr(), - ); - } - result.into() - } -} - -impl OtherApi for () { - fn chain_id() -> u64 { - unsafe { - ext_chain_id.get()() - } - } - - fn print_num(val: u64) { - unsafe { - ext_print_num.get()(val); - } - } - - fn print_utf8(utf8: &[u8]) { - unsafe { - ext_print_utf8.get()(utf8.as_ptr(), utf8.len() as u32); - } - } - - fn print_hex(data: &[u8]) { - unsafe { - ext_print_hex.get()(data.as_ptr(), data.len() as u32); - } - } - - fn log( - level: LogLevel, - target: &[u8], - message: &[u8] - ) { - unsafe { - ext_log.get()( - level as u32, - target.as_ptr(), - target.len() as u32, - message.as_ptr(), - message.len() as u32, - ) - } - } -} - -impl HashingApi for () { - fn keccak_256(data: &[u8]) -> [u8; 32] { - let mut result: [u8; 32] = Default::default(); - unsafe { - ext_keccak_256.get()(data.as_ptr(), data.len() as u32, result.as_mut_ptr()); - } - result - } - - fn blake2_128(data: &[u8]) -> [u8; 16] { - let mut result: [u8; 16] = Default::default(); - unsafe { - ext_blake2_128.get()(data.as_ptr(), data.len() as u32, result.as_mut_ptr()); - } - result - } - - fn blake2_256(data: &[u8]) -> [u8; 32] { - let mut result: [u8; 32] = Default::default(); - unsafe { - ext_blake2_256.get()(data.as_ptr(), data.len() as u32, result.as_mut_ptr()); - } - result - } - - fn twox_256(data: &[u8]) -> [u8; 32] { - let mut result: [u8; 32] = Default::default(); - unsafe { - ext_twox_256.get()(data.as_ptr(), data.len() as u32, result.as_mut_ptr()); - } - result - } - - fn twox_128(data: &[u8]) -> [u8; 16] { - let mut result: [u8; 16] = Default::default(); - unsafe { - ext_twox_128.get()(data.as_ptr(), data.len() as u32, result.as_mut_ptr()); - } - result - } - - fn twox_64(data: &[u8]) -> [u8; 8] { - let mut result: [u8; 8] = Default::default(); - unsafe { - ext_twox_64.get()(data.as_ptr(), data.len() as u32, result.as_mut_ptr()); - } - result - } -} - -impl CryptoApi for () { - fn ed25519_public_keys(id: KeyTypeId) -> Vec { - let mut res_len = 0u32; - unsafe { - let res_ptr = ext_ed25519_public_keys.get()(id.0.as_ptr(), &mut res_len); - Vec::decode(&mut rstd::slice::from_raw_parts(res_ptr, res_len as usize)).unwrap_or_default() - } - } - - fn ed25519_generate(id: KeyTypeId, seed: Option<&str>) -> ed25519::Public { - let mut res = [0u8; 32]; - let seed = seed.as_ref().map(|s| s.as_bytes()).unwrap_or(&[]); - unsafe { - ext_ed25519_generate.get()(id.0.as_ptr(), seed.as_ptr(), seed.len() as u32, res.as_mut_ptr()) - }; - ed25519::Public(res) - } - - fn ed25519_sign( - id: KeyTypeId, - pubkey: &ed25519::Public, - msg: &[u8], - ) -> Option { - let mut res = [0u8; 64]; - let success = unsafe { - ext_ed25519_sign.get()( - id.0.as_ptr(), - pubkey.0.as_ptr(), - msg.as_ptr(), - msg.len() as u32, - res.as_mut_ptr(), - ) == 0 - }; - - if success { - Some(ed25519::Signature(res)) - } else { - None - } - } - - fn ed25519_verify(sig: &ed25519::Signature, msg: &[u8], pubkey: &ed25519::Public) -> bool { - unsafe { - ext_ed25519_verify.get()( - msg.as_ptr(), - msg.len() as u32, - sig.0.as_ptr(), - pubkey.0.as_ptr(), - ) == 0 - } - } - - fn sr25519_public_keys(id: KeyTypeId) -> Vec { - let mut res_len = 0u32; - unsafe { - let res_ptr = ext_sr25519_public_keys.get()(id.0.as_ptr(), &mut res_len); - Vec::decode(&mut rstd::slice::from_raw_parts(res_ptr, res_len as usize)).unwrap_or_default() - } - } - - fn sr25519_generate(id: KeyTypeId, seed: Option<&str>) -> sr25519::Public { - let mut res = [0u8;32]; - let seed = seed.as_ref().map(|s| s.as_bytes()).unwrap_or(&[]); - unsafe { - ext_sr25519_generate.get()(id.0.as_ptr(), seed.as_ptr(), seed.len() as u32, res.as_mut_ptr()) - }; - sr25519::Public(res) - } - - fn sr25519_sign( - id: KeyTypeId, - pubkey: &sr25519::Public, - msg: &[u8], - ) -> Option { - let mut res = [0u8; 64]; - let success = unsafe { - ext_sr25519_sign.get()( - id.0.as_ptr(), - pubkey.0.as_ptr(), - msg.as_ptr(), - msg.len() as u32, - res.as_mut_ptr(), - ) == 0 - }; - - if success { - Some(sr25519::Signature(res)) - } else { - None - } - } - - fn sr25519_verify(sig: &sr25519::Signature, msg: &[u8], pubkey: &sr25519::Public) -> bool { - unsafe { - ext_sr25519_verify.get()( - msg.as_ptr(), - msg.len() as u32, - sig.0.as_ptr(), - pubkey.0.as_ptr(), - ) == 0 - } - } - - fn secp256k1_ecdsa_recover(sig: &[u8; 65], msg: &[u8; 32]) -> Result<[u8; 64], EcdsaVerifyError> { - let mut pubkey = [0u8; 64]; - match unsafe { - ext_secp256k1_ecdsa_recover.get()(msg.as_ptr(), sig.as_ptr(), pubkey.as_mut_ptr()) - } { - 0 => Ok(pubkey), - 1 => Err(EcdsaVerifyError::BadRS), - 2 => Err(EcdsaVerifyError::BadV), - 3 => Err(EcdsaVerifyError::BadSignature), - _ => unreachable!("`ext_secp256k1_ecdsa_recover` only returns 0, 1, 2 or 3; qed"), - } - } - - fn secp256k1_ecdsa_recover_compressed(sig: &[u8; 65], msg: &[u8; 32]) -> Result<[u8; 33], EcdsaVerifyError> { - let mut pubkey = [0u8; 33]; - match unsafe { - ext_secp256k1_ecdsa_recover_compressed.get()(msg.as_ptr(), sig.as_ptr(), pubkey.as_mut_ptr()) - } { - 0 => Ok(pubkey), - 1 => Err(EcdsaVerifyError::BadRS), - 2 => Err(EcdsaVerifyError::BadV), - 3 => Err(EcdsaVerifyError::BadSignature), - _ => unreachable!("`ext_secp256k1_ecdsa_recover_compressed` only returns 0, 1, 2 or 3; qed"), - } - } -} - -impl OffchainApi for () { - fn is_validator() -> bool { - unsafe { ext_is_validator.get()() == 1 } - } - - fn submit_transaction(data: Vec) -> Result<(), ()> { - let ret = unsafe { - ext_submit_transaction.get()(data.as_ptr(), data.len() as u32) - }; - - if ret == 0 { - Ok(()) - } else { - Err(()) - } - } - - fn network_state() -> Result { - let mut len = 0_u32; - let raw_result = unsafe { - let ptr = ext_network_state.get()(&mut len); - - from_raw_parts(ptr, len) - }; - - match raw_result { - Some(raw_result) => codec::Decode::decode(&mut &*raw_result).unwrap_or(Err(())), - None => Err(()) - } - } - - fn timestamp() -> offchain::Timestamp { - offchain::Timestamp::from_unix_millis(unsafe { - ext_timestamp.get()() - }) - } - - fn sleep_until(deadline: offchain::Timestamp) { - unsafe { - ext_sleep_until.get()(deadline.unix_millis()) - } - } - - fn random_seed() -> [u8; 32] { - let mut result = [0_u8; 32]; - unsafe { - ext_random_seed.get()(result.as_mut_ptr()) - } - result - } - - fn local_storage_set(kind: offchain::StorageKind, key: &[u8], value: &[u8]) { - unsafe { - ext_local_storage_set.get()( - kind.into(), - key.as_ptr(), - key.len() as u32, - value.as_ptr(), - value.len() as u32, - ) - } - } - - fn local_storage_compare_and_set( - kind: offchain::StorageKind, - key: &[u8], - old_value: Option<&[u8]>, - new_value: &[u8], - ) -> bool { - let (ptr, len) = match old_value { - Some(old_value) => ( - old_value.as_ptr(), - old_value.len() as u32, - ), - None => (0 as *const u8, u32::max_value()), - }; - - unsafe { - ext_local_storage_compare_and_set.get()( - kind.into(), - key.as_ptr(), - key.len() as u32, - ptr, - len, - new_value.as_ptr(), - new_value.len() as u32, - ) == 0 - } - } - - fn local_storage_get(kind: offchain::StorageKind, key: &[u8]) -> Option> { - let mut len = 0u32; - unsafe { - let ptr = ext_local_storage_get.get()( - kind.into(), - key.as_ptr(), - key.len() as u32, - &mut len, - ); - - from_raw_parts(ptr, len) - } - } - - fn http_request_start(method: &str, url: &str, meta: &[u8]) -> Result { - let method = method.as_bytes(); - let url = url.as_bytes(); - - let result = unsafe { - ext_http_request_start.get()( - method.as_ptr(), - method.len() as u32, - url.as_ptr(), - url.len() as u32, - meta.as_ptr(), - meta.len() as u32, - ) - }; - - if result > u16::max_value() as u32 { - Err(()) - } else { - Ok(offchain::HttpRequestId(result as u16)) - } - } - - fn http_request_add_header(request_id: offchain::HttpRequestId, name: &str, value: &str) -> Result<(), ()> { - let name = name.as_bytes(); - let value = value.as_bytes(); - - let result = unsafe { - ext_http_request_add_header.get()( - request_id.into(), - name.as_ptr(), - name.len() as u32, - value.as_ptr(), - value.len() as u32, - ) - }; - - if result == 0 { - Ok(()) - } else { - Err(()) - } - } - - fn http_request_write_body( - request_id: offchain::HttpRequestId, - chunk: &[u8], - deadline: Option - ) -> Result<(), offchain::HttpError> { - let res = unsafe { - ext_http_request_write_body.get()( - request_id.into(), - chunk.as_ptr(), - chunk.len() as u32, - deadline.map_or(0, |x| x.unix_millis()), - ) - }; - - if res == 0 { - Ok(()) - } else { - Err(res.try_into().unwrap_or(offchain::HttpError::IoError)) - } - } - - fn http_response_wait( - ids: &[offchain::HttpRequestId], - deadline: Option - ) -> Vec { - let ids = ids.iter().map(|x| x.0 as u32).collect::>(); - let mut statuses = Vec::new(); - statuses.resize(ids.len(), 0u32); - - unsafe { - ext_http_response_wait.get()( - ids.as_ptr(), - ids.len() as u32, - statuses.as_mut_ptr(), - deadline.map_or(0, |x| x.unix_millis()), - ) - } - - statuses - .into_iter() - .map(|status| status.try_into().unwrap_or(offchain::HttpRequestStatus::Invalid)) - .collect() - } - - fn http_response_headers( - request_id: offchain::HttpRequestId, - ) -> Vec<(Vec, Vec)> { - let mut len = 0u32; - let raw_result = unsafe { - let ptr = ext_http_response_headers.get()( - request_id.into(), - &mut len, - ); - - from_raw_parts(ptr, len).expect("ext_http_response_headers never return u32::max_value; qed") - }; - - codec::Decode::decode(&mut &*raw_result).unwrap_or_default() - } - - fn http_response_read_body( - request_id: offchain::HttpRequestId, - buffer: &mut [u8], - deadline: Option, - ) -> Result { - let res = unsafe { - ext_http_response_read_body.get()( - request_id.into(), - buffer.as_mut_ptr(), - buffer.len() as u32, - deadline.map_or(0, |x| x.unix_millis()), - ) - }; - - if res >= u32::max_value() - 255 { - let code = (u32::max_value() - res) + 1; - code.try_into().map_err(|_| offchain::HttpError::IoError) - } else { - Ok(res as usize) - } - } -} - -unsafe fn from_raw_parts(ptr: *mut u8, len: u32) -> Option> { - if len == u32::max_value() { - None - } else { - // Invariants required by Vec::from_raw_parts are not formally fulfilled. - // We don't allocate via String/Vec, but use a custom allocator instead. - // See #300 for more details. - Some(>::from_raw_parts(ptr, len as usize, len as usize)) - } -} - -impl Api for () {}