From 6a779f3aad62c21e34810a42369be03dd6f1c88b Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 22 Apr 2020 09:35:40 +0200 Subject: [PATCH 01/16] impl offchain index Ref #3722 --- Cargo.lock | 2 + bin/node/testing/src/bench.rs | 1 + client/api/src/backend.rs | 10 +- client/api/src/call_executor.rs | 3 +- client/api/src/execution_extensions.rs | 4 +- client/cli/src/arg_enums.rs | 12 + client/cli/src/commands/mod.rs | 8 +- client/cli/src/commands/runcmd.rs | 41 +--- client/cli/src/config.rs | 21 +- client/cli/src/params/mod.rs | 2 + .../cli/src/params/offchain_worker_params.rs | 77 ++++++ client/db/src/lib.rs | 26 +- client/executor/runtime-test/src/lib.rs | 6 + client/executor/src/integration_tests/mod.rs | 22 ++ client/service/src/builder.rs | 14 +- client/service/src/config.rs | 10 +- client/service/test/src/lib.rs | 2 +- client/src/call_executor.rs | 23 +- client/src/client.rs | 49 +++- client/src/genesis.rs | 14 ++ client/src/lib.rs | 4 +- client/src/light/call_executor.rs | 5 +- client/src/light/mod.rs | 5 +- .../api/proc-macro/src/decl_runtime_apis.rs | 3 + .../api/proc-macro/src/impl_runtime_apis.rs | 6 + primitives/api/src/lib.rs | 6 + primitives/core/src/offchain/mod.rs | 6 +- primitives/core/src/offchain/storage.rs | 228 ++++++++++++++++++ primitives/externalities/src/lib.rs | 3 + primitives/io/src/lib.rs | 17 ++ primitives/offchain/Cargo.toml | 5 + primitives/offchain/src/lib.rs | 4 +- primitives/state-machine/src/basic.rs | 2 + primitives/state-machine/src/ext.rs | 67 ++++- primitives/state-machine/src/lib.rs | 22 ++ .../state-machine/src/overlayed_changes.rs | 11 + primitives/state-machine/src/testing.rs | 7 + test-utils/client/src/lib.rs | 3 +- test-utils/runtime/client/src/lib.rs | 2 +- utils/frame/benchmarking-cli/src/command.rs | 2 + 40 files changed, 674 insertions(+), 81 deletions(-) create mode 100644 client/cli/src/params/offchain_worker_params.rs diff --git a/Cargo.lock b/Cargo.lock index 7b36a7d08a475..1017d3eaa265e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7522,7 +7522,9 @@ name = "sp-offchain" version = "2.0.0-dev" dependencies = [ "sp-api", + "sp-core", "sp-runtime", + "sp-state-machine", ] [[package]] diff --git a/bin/node/testing/src/bench.rs b/bin/node/testing/src/bench.rs index ea48f02650649..91d0cfb55a0e8 100644 --- a/bin/node/testing/src/bench.rs +++ b/bin/node/testing/src/bench.rs @@ -230,6 +230,7 @@ impl BenchDb { ExecutionExtensions::new(profile.into_execution_strategies(), None), Box::new(TaskExecutor::new()), None, + Default::default(), ).expect("Should not fail"); (client, backend) diff --git a/client/api/src/backend.rs b/client/api/src/backend.rs index 33a370c7cb2c5..3fe9bd96be5e2 100644 --- a/client/api/src/backend.rs +++ b/client/api/src/backend.rs @@ -19,7 +19,7 @@ use std::sync::Arc; use std::collections::HashMap; use sp_core::ChangesTrieConfigurationRange; -use sp_core::offchain::OffchainStorage; +use sp_core::offchain::{OffchainStorage,storage::OffchainOverlayedChanges}; use sp_runtime::{generic::BlockId, Justification, Storage}; use sp_runtime::traits::{Block as BlockT, NumberFor, HashFor}; use sp_state_machine::{ @@ -148,6 +148,14 @@ pub trait BlockImportOperation { child_update: ChildStorageCollection, ) -> sp_blockchain::Result<()>; + /// Set offchain storage changes. + fn update_offchain_storage( + &mut self, + _offchain_update: OffchainOverlayedChanges, + ) -> sp_blockchain::Result<()> { + Ok(()) + } + /// Inject changes trie data into the database. fn update_changes_trie( &mut self, diff --git a/client/api/src/call_executor.rs b/client/api/src/call_executor.rs index 3afd29be8d49c..00711e83b75ac 100644 --- a/client/api/src/call_executor.rs +++ b/client/api/src/call_executor.rs @@ -26,7 +26,7 @@ use sp_state_machine::{ }; use sc_executor::{RuntimeVersion, NativeVersion}; use sp_externalities::Extensions; -use sp_core::NativeOrEncoded; +use sp_core::{NativeOrEncoded,offchain::storage::OffchainOverlayedChanges}; use sp_api::{ProofRecorder, InitializeBlock, StorageTransactionCache}; use crate::execution_extensions::ExecutionExtensions; @@ -84,6 +84,7 @@ pub trait CallExecutor { method: &str, call_data: &[u8], changes: &RefCell, + offchain_changes: &RefCell, storage_transaction_cache: Option<&RefCell< StorageTransactionCache>::State>, >>, diff --git a/client/api/src/execution_extensions.rs b/client/api/src/execution_extensions.rs index 10d33c20e679c..b1903e1ee5f8d 100644 --- a/client/api/src/execution_extensions.rs +++ b/client/api/src/execution_extensions.rs @@ -177,7 +177,7 @@ impl ExecutionExtensions { if let ExecutionContext::OffchainCall(Some(ext)) = context { extensions.register( OffchainExt::new(offchain::LimitedExternalities::new(capabilities, ext.0)) - ) + ); } (manager, extensions) @@ -202,4 +202,4 @@ impl offchain::TransactionPool for TransactionPoolAdapter< self.pool.submit_at(&self.at, xt) } -} +} \ No newline at end of file diff --git a/client/cli/src/arg_enums.rs b/client/cli/src/arg_enums.rs index 2adccdd879ed3..f0eeca4c8bef1 100644 --- a/client/cli/src/arg_enums.rs +++ b/client/cli/src/arg_enums.rs @@ -136,6 +136,18 @@ arg_enum! { } } + +arg_enum! { + /// Whether off-chain workers are enabled. + #[allow(missing_docs)] + #[derive(Debug, Clone)] + pub enum OffchainWorkerEnabled { + Always, + Never, + WhenValidating, + } +} + /// Default value for the `--execution-syncing` parameter. pub const DEFAULT_EXECUTION_SYNCING: ExecutionStrategy = ExecutionStrategy::NativeElseWasm; /// Default value for the `--execution-import-block` parameter. diff --git a/client/cli/src/commands/mod.rs b/client/cli/src/commands/mod.rs index ec36fb3b64be4..143ff491aef95 100644 --- a/client/cli/src/commands/mod.rs +++ b/client/cli/src/commands/mod.rs @@ -144,6 +144,12 @@ macro_rules! substrate_cli_subcommands { } } + fn offchain_worker_params(&self) -> Option<&$crate::OffchainWorkerParams> { + match self { + $($enum::$variant(cmd) => cmd.offchain_worker_params()),* + } + } + fn base_path(&self) -> $crate::Result<::std::option::Option<::std::path::PathBuf>> { match self { $($enum::$variant(cmd) => cmd.base_path()),* @@ -327,7 +333,7 @@ macro_rules! substrate_cli_subcommands { } } - fn offchain_worker(&self, role: &::sc_service::Role) -> $crate::Result { + fn offchain_worker(&self, role: &::sc_service::Role) -> $crate::Result<::sc_service::config::OffchainWorkerConfig> { match self { $($enum::$variant(cmd) => cmd.offchain_worker(role)),* } diff --git a/client/cli/src/commands/runcmd.rs b/client/cli/src/commands/runcmd.rs index 3a3a17ea3fabb..a24cadcd4f78b 100644 --- a/client/cli/src/commands/runcmd.rs +++ b/client/cli/src/commands/runcmd.rs @@ -20,6 +20,7 @@ use crate::params::KeystoreParams; use crate::params::NetworkParams; use crate::params::SharedParams; use crate::params::TransactionPoolParams; +use crate::params::OffchainWorkerParams; use crate::CliConfiguration; use regex::Regex; use sc_service::{ @@ -28,18 +29,7 @@ use sc_service::{ }; use sc_telemetry::TelemetryEndpoints; use std::net::{IpAddr, Ipv4Addr, SocketAddr}; -use structopt::{clap::arg_enum, StructOpt}; - -arg_enum! { - /// Whether off-chain workers are enabled. - #[allow(missing_docs)] - #[derive(Debug, Clone)] - pub enum OffchainWorkerEnabled { - Always, - Never, - WhenValidating, - } -} +use structopt::StructOpt; /// The `run` command used to run a node. #[derive(Debug, StructOpt, Clone)] @@ -173,17 +163,9 @@ pub struct RunCmd { #[structopt(long = "telemetry-url", value_name = "URL VERBOSITY", parse(try_from_str = parse_telemetry_endpoints))] pub telemetry_endpoints: Vec<(String, u8)>, - /// Should execute offchain workers on every block. - /// - /// By default it's only enabled for nodes that are authoring new blocks. - #[structopt( - long = "offchain-worker", - value_name = "ENABLED", - possible_values = &OffchainWorkerEnabled::variants(), - case_insensitive = true, - default_value = "WhenValidating" - )] - pub offchain_worker: OffchainWorkerEnabled, + #[allow(missing_docs)] + #[structopt(flatten)] + pub offchain_worker_params: OffchainWorkerParams, #[allow(missing_docs)] #[structopt(flatten)] @@ -300,6 +282,10 @@ impl CliConfiguration for RunCmd { Some(&self.keystore_params) } + fn offchain_worker_params(&self) -> Option<&OffchainWorkerParams> { + Some(&self.offchain_worker_params) + } + fn node_name(&self) -> Result { let name: String = match (self.name.as_ref(), self.get_keyring()) { (Some(name), _) => name.to_string(), @@ -439,15 +425,6 @@ impl CliConfiguration for RunCmd { Ok(self.unsafe_rpc_expose) } - fn offchain_worker(&self, role: &Role) -> Result { - Ok(match (&self.offchain_worker, role) { - (OffchainWorkerEnabled::WhenValidating, Role::Authority { .. }) => true, - (OffchainWorkerEnabled::Always, _) => true, - (OffchainWorkerEnabled::Never, _) => false, - (OffchainWorkerEnabled::WhenValidating, _) => false, - }) - } - fn transaction_pool(&self) -> Result { Ok(self.pool_config.transaction_pool()) } diff --git a/client/cli/src/config.rs b/client/cli/src/config.rs index 9de2022a592f4..9cdbdf59274a7 100644 --- a/client/cli/src/config.rs +++ b/client/cli/src/config.rs @@ -19,15 +19,15 @@ use crate::error::Result; use crate::{ init_logger, ImportParams, KeystoreParams, NetworkParams, NodeKeyParams, - PruningParams, SharedParams, SubstrateCli, + OffchainWorkerParams, PruningParams, SharedParams, SubstrateCli, }; use crate::arg_enums::Database; use app_dirs::{AppDataType, AppInfo}; use names::{Generator, Name}; use sc_service::config::{ Configuration, DatabaseConfig, ExecutionStrategies, ExtTransport, KeystoreConfig, - NetworkConfiguration, NodeKeyConfig, PrometheusConfig, PruningMode, Role, TelemetryEndpoints, - TransactionPoolOptions, WasmExecutionMethod, + NetworkConfiguration, NodeKeyConfig, OffchainWorkerConfig, PrometheusConfig, PruningMode, + Role, TelemetryEndpoints, TransactionPoolOptions, WasmExecutionMethod, }; use sc_service::{ChainSpec, TracingReceiver}; use std::future::Future; @@ -67,6 +67,11 @@ pub trait CliConfiguration: Sized { None } + /// Get the OffchainWorkerParams for this object + fn offchain_worker_params(&self) -> Option<&OffchainWorkerParams> { + None + } + /// Get the NodeKeyParams for this object fn node_key_params(&self) -> Option<&NodeKeyParams> { self.network_params() @@ -301,11 +306,13 @@ pub trait CliConfiguration: Sized { Ok(Default::default()) } - /// Returns `Ok(true)` if offchain worker should be used + /// Returns a offchain worker config wrapped in `Ok(_)` /// - /// By default this is `false`. - fn offchain_worker(&self, _role: &Role) -> Result { - Ok(Default::default()) + /// By default offchain workers are disabled. + fn offchain_worker(&self, role: &Role) -> Result { + self.offchain_worker_params() + .map(|x| x.offchain_worker(role)) + .unwrap_or_else(|| { Ok(OffchainWorkerConfig::default()) }) } /// Returns `Ok(true)` if authoring should be forced diff --git a/client/cli/src/params/mod.rs b/client/cli/src/params/mod.rs index 9097bf8589919..af94a0c6f6b3f 100644 --- a/client/cli/src/params/mod.rs +++ b/client/cli/src/params/mod.rs @@ -21,6 +21,7 @@ mod node_key_params; mod pruning_params; mod shared_params; mod transaction_pool_params; +mod offchain_worker_params; use std::fmt::Debug; use std::str::FromStr; @@ -29,6 +30,7 @@ pub use crate::params::import_params::*; pub use crate::params::keystore_params::*; pub use crate::params::network_params::*; pub use crate::params::node_key_params::*; +pub use crate::params::offchain_worker_params::*; pub use crate::params::pruning_params::*; pub use crate::params::shared_params::*; pub use crate::params::transaction_pool_params::*; diff --git a/client/cli/src/params/offchain_worker_params.rs b/client/cli/src/params/offchain_worker_params.rs new file mode 100644 index 0000000000000..dcdb78befee2a --- /dev/null +++ b/client/cli/src/params/offchain_worker_params.rs @@ -0,0 +1,77 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + + +//! Offchain worker related configuration parameters. +//! +//! A subset of configuration parameters which are relevant to +//! the inner working of offchain workers. The usage is solely +//! targeted at handling input parameter parsing providing +//! a reasonable abstraction. + +use structopt::StructOpt; +use sc_service::config::OffchainWorkerConfig; +use sc_network::config::Role; + +use crate::error; +use crate::OffchainWorkerEnabled; + + +/// Offchain worker related parameters. +#[derive(Debug, StructOpt, Clone)] +pub struct OffchainWorkerParams { + /// Should execute offchain workers on every block. + /// + /// By default it's only enabled for nodes that are authoring new blocks. + #[structopt( + long = "offchain-worker", + value_name = "ENABLED", + possible_values = &OffchainWorkerEnabled::variants(), + case_insensitive = true, + default_value = "WhenValidating" + )] + pub enabled: OffchainWorkerEnabled, + + /// Allow access to offchain workers indexing API + /// + /// Enables a runtime to write directly to a offchain workers + /// DB during block import. + #[structopt( + long = "enable-offchain-indexing", + value_name = "ENABLE_OFFCHAIN_INDEXING" + )] + pub indexing_enabled: bool, +} + +impl OffchainWorkerParams { + /// Load spec to `Configuration` from `OffchainWorkerParams` and spec factory. + pub fn offchain_worker( + &self, + role: &Role, + ) -> error::Result + { + let enabled = match (&self.enabled, role) { + (OffchainWorkerEnabled::WhenValidating, Role::Authority { .. }) => true, + (OffchainWorkerEnabled::Always, _) => true, + (OffchainWorkerEnabled::Never, _) => false, + (OffchainWorkerEnabled::WhenValidating, _) => false, + }; + + let indexing_enabled = enabled && self.indexing_enabled; + + Ok(OffchainWorkerConfig { enabled, indexing_enabled}) + } +} diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index 579ea2db4ad8d..c72d289e83480 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -50,6 +50,7 @@ use std::path::{Path, PathBuf}; use std::io; use std::collections::HashMap; + use sc_client_api::{ ForkBlocks, UsageInfo, MemoryInfo, BadBlocks, IoInfo, MemorySize, CloneableSpawn, execution_extensions::ExecutionExtensions, @@ -65,6 +66,7 @@ use sp_trie::{MemoryDB, PrefixedMemoryDB, prefixed_key}; use sp_database::Transaction; use parking_lot::RwLock; use sp_core::{ChangesTrieConfiguration, traits::CodeExecutor}; +use sp_core::offchain::storage::{OffchainOverlayedChange,OffchainOverlayedChanges}; use sp_core::storage::{well_known_keys, ChildInfo}; use sp_runtime::{ generic::BlockId, Justification, Storage, @@ -324,6 +326,7 @@ pub fn new_client( execution_extensions: ExecutionExtensions, spawn_handle: Box, prometheus_registry: Option, + config: sc_client::ClientConfig, ) -> Result<( sc_client::Client< Backend, @@ -340,7 +343,7 @@ pub fn new_client( E: CodeExecutor + RuntimeInfo, { let backend = Arc::new(Backend::new(settings, CANONICALIZATION_DELAY)?); - let executor = sc_client::LocalCallExecutor::new(backend.clone(), executor, spawn_handle); + let executor = sc_client::LocalCallExecutor::new(backend.clone(), executor, spawn_handle, config.clone()); Ok(( sc_client::Client::new( backend.clone(), @@ -350,6 +353,7 @@ pub fn new_client( bad_blocks, execution_extensions, prometheus_registry, + config, )?, backend, )) @@ -558,6 +562,7 @@ pub struct BlockImportOperation { db_updates: PrefixedMemoryDB>, storage_updates: StorageCollection, child_storage_updates: ChildStorageCollection, + offchain_storage_updates: OffchainOverlayedChanges, changes_trie_updates: MemoryDB>, changes_trie_build_cache_update: Option>>, changes_trie_config_update: Option>, @@ -569,6 +574,15 @@ pub struct BlockImportOperation { } impl BlockImportOperation { + fn apply_offchain(&mut self, transaction: &mut Transaction) { + for (key, value_operation) in self.offchain_storage_updates.drain() { + match value_operation { + OffchainOverlayedChange::SetValue(val) => transaction.set_from_vec(columns::OFFCHAIN, &key, val), + OffchainOverlayedChange::Remove => transaction.remove(columns::OFFCHAIN, &key), + } + } + } + fn apply_aux(&mut self, transaction: &mut Transaction) { for (key, maybe_val) in self.aux_ops.drain(..) { match maybe_val { @@ -675,6 +689,14 @@ impl sc_client_api::backend::BlockImportOperation for Bloc Ok(()) } + fn update_offchain_storage( + &mut self, + offchain_update: OffchainOverlayedChanges, + ) -> ClientResult<()> { + self.offchain_storage_updates = offchain_update; + Ok(()) + } + fn mark_finalized( &mut self, block: BlockId, @@ -1017,6 +1039,7 @@ impl Backend { let mut finalization_displaced_leaves = None; operation.apply_aux(&mut transaction); + operation.apply_offchain(&mut transaction); let mut meta_updates = Vec::with_capacity(operation.finalized_blocks.len()); let mut last_finalized_hash = self.blockchain.meta.read().finalized_hash; @@ -1360,6 +1383,7 @@ impl sc_client_api::backend::Backend for Backend { db_updates: PrefixedMemoryDB::default(), storage_updates: Default::default(), child_storage_updates: Default::default(), + offchain_storage_updates: Default::default(), changes_trie_config_update: None, changes_trie_updates: MemoryDB::default(), changes_trie_build_cache_update: None, diff --git a/client/executor/runtime-test/src/lib.rs b/client/executor/runtime-test/src/lib.rs index 38a16ae39ea01..15a4177048a40 100644 --- a/client/executor/runtime-test/src/lib.rs +++ b/client/executor/runtime-test/src/lib.rs @@ -183,6 +183,12 @@ sp_core::wasm_export_functions! { } } + + fn test_offchain_index_set() { + sp_io::offchain_index::set(b"k", b"v"); + } + + fn test_offchain_local_storage() -> bool { let kind = sp_core::offchain::StorageKind::PERSISTENT; assert_eq!(sp_io::offchain::local_storage_get(kind, b"test"), None); diff --git a/client/executor/src/integration_tests/mod.rs b/client/executor/src/integration_tests/mod.rs index e1ef18a0853ff..4973d1a887e7b 100644 --- a/client/executor/src/integration_tests/mod.rs +++ b/client/executor/src/integration_tests/mod.rs @@ -450,6 +450,28 @@ fn ordered_trie_root_should_work(wasm_method: WasmExecutionMethod) { ); } +#[test_case(WasmExecutionMethod::Interpreted)] +#[cfg_attr(feature = "wasmtime", test_case(WasmExecutionMethod::Compiled))] +fn offchain_index(wasm_method: WasmExecutionMethod) { + let mut ext = TestExternalities::default(); + let (offchain, _state) = testing::TestOffchainExt::new(); + ext.register_extension(OffchainExt::new(offchain)); + call_in_wasm( + "test_offchain_index_set", + &[0], + wasm_method, + &mut ext.ext(), + ).unwrap(); + + use sp_core::offchain::storage::OffchainOverlayedChange; + assert_eq!( + ext.ext() + .get_offchain_storage_changes() + .get(sp_core::offchain::STORAGE_PREFIX, b"k"), + Some(OffchainOverlayedChange::SetValue(b"v".to_vec())) + ); +} + #[test_case(WasmExecutionMethod::Interpreted)] #[cfg_attr(feature = "wasmtime", test_case(WasmExecutionMethod::Compiled))] fn offchain_local_storage_should_work(wasm_method: WasmExecutionMethod) { diff --git a/client/service/src/builder.rs b/client/service/src/builder.rs index 1cdebe8b14bf3..37ccae4ec8f3c 100644 --- a/client/service/src/builder.rs +++ b/client/service/src/builder.rs @@ -17,7 +17,7 @@ use crate::{Service, NetworkStatus, NetworkState, error::Error, DEFAULT_PROTOCOL_ID, MallocSizeOfWasm}; use crate::{TaskManagerBuilder, start_rpc_servers, build_network_future, TransactionPoolAdapter}; use crate::status_sinks; -use crate::config::{Configuration, KeystoreConfig, PrometheusConfig}; +use crate::config::{Configuration, KeystoreConfig, PrometheusConfig, OffchainWorkerConfig}; use crate::metrics::MetricsService; use sc_client_api::{ self, @@ -27,7 +27,7 @@ use sc_client_api::{ ExecutorProvider, CallExecutor }; use sp_utils::mpsc::{tracing_unbounded, TracingUnboundedSender}; -use sc_client::Client; +use sc_client::{Client, ClientConfig}; use sc_chain_spec::get_extension; use sp_consensus::import_queue::ImportQueue; use futures::{ @@ -215,6 +215,10 @@ fn new_full_parts( extensions, Box::new(tasks_builder.spawn_handle()), config.prometheus_config.as_ref().map(|config| config.registry.clone()), + ClientConfig { + offchain_worker_enabled : config.offchain_worker.enabled , + offchain_indexing_api: config.offchain_worker.indexing_enabled, + }, )? }; @@ -834,11 +838,11 @@ ServiceBuilder< let network_status_sinks = Arc::new(Mutex::new(status_sinks::StatusSinks::new())); let offchain_storage = backend.offchain_storage(); - let offchain_workers = match (config.offchain_worker, offchain_storage.clone()) { - (true, Some(db)) => { + let offchain_workers = match (config.offchain_worker.clone(), offchain_storage.clone()) { + (OffchainWorkerConfig {enabled: true, .. }, Some(db)) => { Some(Arc::new(sc_offchain::OffchainWorkers::new(client.clone(), db))) }, - (true, None) => { + (OffchainWorkerConfig {enabled: true, .. }, None) => { warn!("Offchain workers disabled, due to lack of offchain storage support in backend."); None }, diff --git a/client/service/src/config.rs b/client/service/src/config.rs index affb4aabfc632..70db6e6ba071b 100644 --- a/client/service/src/config.rs +++ b/client/service/src/config.rs @@ -79,7 +79,7 @@ pub struct Configuration { /// The default number of 64KB pages to allocate for Wasm execution pub default_heap_pages: Option, /// Should offchain workers be executed. - pub offchain_worker: bool, + pub offchain_worker: OffchainWorkerConfig, /// Enable authoring even when offline. pub force_authoring: bool, /// Disable GRANDPA when running in validator mode @@ -125,6 +125,14 @@ impl KeystoreConfig { } } } +/// Configuration of the database of the client. +#[derive(Clone, Default)] +pub struct OffchainWorkerConfig { + /// If this is allowed. + pub enabled: bool, + /// allow writes from the runtime to the offchain worker database + pub indexing_enabled: bool, +} /// Configuration of the Prometheus endpoint. #[derive(Clone)] diff --git a/client/service/test/src/lib.rs b/client/service/test/src/lib.rs index 9b9140dd8d3ff..c53c36ca6a79e 100644 --- a/client/service/test/src/lib.rs +++ b/client/service/test/src/lib.rs @@ -193,7 +193,7 @@ fn node_config { backend: Arc, executor: E, spawn_handle: Box, + client_config: ClientConfig, } impl LocalCallExecutor { @@ -43,11 +45,13 @@ impl LocalCallExecutor { backend: Arc, executor: E, spawn_handle: Box, + client_config: ClientConfig, ) -> Self { LocalCallExecutor { backend, executor, spawn_handle, + client_config, } } } @@ -58,6 +62,7 @@ impl Clone for LocalCallExecutor where E: Clone { backend: self.backend.clone(), executor: self.executor.clone(), spawn_handle: self.spawn_handle.clone(), + client_config: self.client_config.clone(), } } } @@ -81,6 +86,11 @@ where extensions: Option, ) -> sp_blockchain::Result> { let mut changes = OverlayedChanges::default(); + let mut offchain_changes = if self.client_config.offchain_indexing_api { + OffchainOverlayedChanges::enabled() + } else { + OffchainOverlayedChanges::disabled() + }; let changes_trie = backend::changes_tries_state_at_block( id, self.backend.changes_trie_storage() )?; @@ -90,6 +100,7 @@ where &state, changes_trie, &mut changes, + &mut offchain_changes, &self.executor, method, call_data, @@ -120,6 +131,7 @@ where method: &str, call_data: &[u8], changes: &RefCell, + offchain_changes: &RefCell, storage_transaction_cache: Option<&RefCell< StorageTransactionCache >>, @@ -143,6 +155,9 @@ where let mut state = self.backend.state_at(*at)?; + let changes = &mut *changes.borrow_mut(); + let offchain_changes = &mut *offchain_changes.borrow_mut(); + match recorder { Some(recorder) => { let trie_state = state.as_trie_backend() @@ -160,11 +175,11 @@ where recorder.clone(), ); - let changes = &mut *changes.borrow_mut(); let mut state_machine = StateMachine::new( &backend, changes_trie_state, changes, + offchain_changes, &self.executor, method, call_data, @@ -179,11 +194,11 @@ where None => { let state_runtime_code = sp_state_machine::backend::BackendRuntimeCode::new(&state); let runtime_code = state_runtime_code.runtime_code()?; - let changes = &mut *changes.borrow_mut(); let mut state_machine = StateMachine::new( &state, changes_trie_state, changes, + offchain_changes, &self.executor, method, call_data, @@ -198,6 +213,7 @@ where fn runtime_version(&self, id: &BlockId) -> sp_blockchain::Result { let mut overlay = OverlayedChanges::default(); + let mut offchain_overlay = OffchainOverlayedChanges::default(); let changes_trie_state = backend::changes_tries_state_at_block( id, self.backend.changes_trie_storage(), @@ -206,6 +222,7 @@ where let mut cache = StorageTransactionCache::::default(); let mut ext = Ext::new( &mut overlay, + &mut offchain_overlay, &mut cache, &state, changes_trie_state, diff --git a/client/src/client.rs b/client/src/client.rs index 2a8040febf3ef..ce1945f72da03 100644 --- a/client/src/client.rs +++ b/client/src/client.rs @@ -99,6 +99,7 @@ pub struct Client where Block: BlockT { importing_block: RwLock>, block_rules: BlockRules, execution_extensions: ExecutionExtensions, + config: ClientConfig, _phantom: PhantomData, } @@ -136,6 +137,7 @@ pub fn new_in_mem( keystore: Option, prometheus_registry: Option, spawn_handle: Box, + config: ClientConfig, ) -> sp_blockchain::Result, LocalCallExecutor, E>, @@ -146,7 +148,24 @@ pub fn new_in_mem( S: BuildStorage, Block: BlockT, { - new_with_backend(Arc::new(in_mem::Backend::new()), executor, genesis_storage, keystore, spawn_handle, prometheus_registry) + new_with_backend( + Arc::new(in_mem::Backend::new()), + executor, + genesis_storage, + keystore, + spawn_handle, + prometheus_registry, + config, + ) +} + +/// Relevant client configuration items relevant for the client. +#[derive(Debug,Clone,Default)] +pub struct ClientConfig { + /// Enable the offchain worker db. + pub offchain_worker_enabled: bool, + /// If true, allows access from the runtime to write into offchain worker db. + pub offchain_indexing_api: bool, } /// Create a client with the explicitly provided backend. @@ -158,6 +177,7 @@ pub fn new_with_backend( keystore: Option, spawn_handle: Box, prometheus_registry: Option, + config: ClientConfig, ) -> sp_blockchain::Result, Block, RA>> where E: CodeExecutor + RuntimeInfo, @@ -165,7 +185,7 @@ pub fn new_with_backend( Block: BlockT, B: backend::LocalBackend + 'static, { - let call_executor = LocalCallExecutor::new(backend.clone(), executor, spawn_handle); + let call_executor = LocalCallExecutor::new(backend.clone(), executor, spawn_handle, config.clone()); let extensions = ExecutionExtensions::new(Default::default(), keystore); Client::new( backend, @@ -175,6 +195,7 @@ pub fn new_with_backend( Default::default(), extensions, prometheus_registry, + config, ) } @@ -243,6 +264,7 @@ impl Client where B: backend::Backend, E: CallExecutor, Block: BlockT, + Block::Header: Clone, { /// Creates new Substrate Client with given blockchain and code executor. pub fn new( @@ -253,6 +275,7 @@ impl Client where bad_blocks: BadBlocks, execution_extensions: ExecutionExtensions, _prometheus_registry: Option, + config: ClientConfig, ) -> sp_blockchain::Result { if backend.blockchain().header(BlockId::Number(Zero::zero()))?.is_none() { let genesis_storage = build_genesis_storage.build_storage()?; @@ -282,6 +305,7 @@ impl Client where importing_block: Default::default(), block_rules: BlockRules::new(fork_blocks, bad_blocks), execution_extensions, + config, _phantom: Default::default(), }) } @@ -614,7 +638,7 @@ impl Client where if let Ok(ImportResult::Imported(ref aux)) = result { if aux.is_new_best { use rand::Rng; - + // don't send telemetry block import events during initial sync for every // block to avoid spamming the telemetry server, these events will be randomly // sent at a rate of 1/10. @@ -696,7 +720,22 @@ impl Client where operation.op.update_cache(new_cache); - let (main_sc, child_sc, tx, _, changes_trie_tx) = storage_changes.into_inner(); + let ( + main_sc, + child_sc, + offchain_sc, + tx, _, + changes_trie_tx, + ) = storage_changes.into_inner(); + + if self.config.offchain_indexing_api { + // if let Some(mut offchain_storage) = self.backend.offchain_storage() { + // offchain_sc.iter().for_each(|(k,v)| { + // offchain_storage.set(b"block-import-info", k,v) + // }); + // } + operation.op.update_offchain_storage(offchain_sc)?; + } operation.op.update_db_storage(tx)?; operation.op.update_storage(main_sc.clone(), child_sc.clone())?; @@ -1554,6 +1593,7 @@ impl CallApiAt for Client where params.function, ¶ms.arguments, params.overlayed_changes, + params.offchain_changes, Some(params.storage_transaction_cache), params.initialize_block, manager, @@ -3493,6 +3533,7 @@ pub(crate) mod tests { None, None, sp_core::tasks::executor(), + Default::default(), ) .unwrap(); diff --git a/client/src/genesis.rs b/client/src/genesis.rs index 2c84ff1e4331a..c5dd44404c9b1 100644 --- a/client/src/genesis.rs +++ b/client/src/genesis.rs @@ -55,6 +55,7 @@ mod tests { }; use sp_runtime::traits::BlakeTwo256; use sp_core::tasks::executor as tasks_executor; + use sp_core::offchain::storage::OffchainOverlayedChanges; use hex_literal::*; native_executor_instance!( @@ -90,6 +91,7 @@ mod tests { }; let hash = header.hash(); let mut overlay = OverlayedChanges::default(); + let mut offchain_overlay = OffchainOverlayedChanges::default(); let backend_runtime_code = sp_state_machine::backend::BackendRuntimeCode::new(&backend); let runtime_code = backend_runtime_code.runtime_code().expect("Code is part of the backend"); @@ -97,6 +99,7 @@ mod tests { backend, sp_state_machine::disabled_changes_trie_state::<_, u64>(), &mut overlay, + &mut offchain_overlay, &executor(), "Core_initialize_block", &header.encode(), @@ -112,6 +115,7 @@ mod tests { backend, sp_state_machine::disabled_changes_trie_state::<_, u64>(), &mut overlay, + &mut offchain_overlay, &executor(), "BlockBuilder_apply_extrinsic", &tx.encode(), @@ -127,6 +131,7 @@ mod tests { backend, sp_state_machine::disabled_changes_trie_state::<_, u64>(), &mut overlay, + &mut offchain_overlay, &executor(), "BlockBuilder_finalize_block", &[], @@ -174,10 +179,13 @@ mod tests { let runtime_code = backend_runtime_code.runtime_code().expect("Code is part of the backend"); let mut overlay = OverlayedChanges::default(); + let mut offchain_overlay = OffchainOverlayedChanges::default(); + let _ = StateMachine::new( &backend, sp_state_machine::disabled_changes_trie_state::<_, u64>(), &mut overlay, + &mut offchain_overlay, &executor(), "Core_execute_block", &b1data, @@ -206,10 +214,13 @@ mod tests { let runtime_code = backend_runtime_code.runtime_code().expect("Code is part of the backend"); let mut overlay = OverlayedChanges::default(); + let mut offchain_overlay = OffchainOverlayedChanges::default(); + let _ = StateMachine::new( &backend, sp_state_machine::disabled_changes_trie_state::<_, u64>(), &mut overlay, + &mut offchain_overlay, &executor(), "Core_execute_block", &b1data, @@ -238,10 +249,13 @@ mod tests { let runtime_code = backend_runtime_code.runtime_code().expect("Code is part of the backend"); let mut overlay = OverlayedChanges::default(); + let mut offchain_overlay = OffchainOverlayedChanges::default(); + let r = StateMachine::new( &backend, sp_state_machine::disabled_changes_trie_state::<_, u64>(), &mut overlay, + &mut offchain_overlay, &executor(), "Core_execute_block", &b1data, diff --git a/client/src/lib.rs b/client/src/lib.rs index 20a3ed058aa7c..aa7e3388f9373 100644 --- a/client/src/lib.rs +++ b/client/src/lib.rs @@ -99,8 +99,8 @@ pub use crate::{ client::{ new_with_backend, new_in_mem, - BlockBackend, ImportNotifications, FinalityNotifications, BlockchainEvents, LockImportRun, - BlockImportNotification, Client, ClientInfo, ExecutionStrategies, FinalityNotification, + ImportNotifications, FinalityNotifications, BlockchainEvents, LockImportRun, + BlockImportNotification, Client, ClientConfig, ClientInfo, ExecutionStrategies, FinalityNotification, LongestChain, BlockOf, ProvideUncles, BadBlocks, ForkBlocks, apply_aux, }, leaves::LeafSet, diff --git a/client/src/light/call_executor.rs b/client/src/light/call_executor.rs index b439a268d2fe1..3a91f2dc23dac 100644 --- a/client/src/light/call_executor.rs +++ b/client/src/light/call_executor.rs @@ -21,7 +21,7 @@ use std::{ }; use codec::{Encode, Decode}; -use sp_core::{convert_hash, NativeOrEncoded, traits::CodeExecutor}; +use sp_core::{convert_hash, NativeOrEncoded, traits::CodeExecutor, offchain::storage::OffchainOverlayedChanges}; use sp_runtime::{ generic::BlockId, traits::{One, Block as BlockT, Header as HeaderT, HashFor}, }; @@ -108,6 +108,7 @@ impl CallExecutor for method: &str, call_data: &[u8], changes: &RefCell, + offchain_changes: &RefCell, _: Option<&RefCell>>, initialize_block: InitializeBlock<'a, Block>, _manager: ExecutionManager, @@ -134,6 +135,7 @@ impl CallExecutor for method, call_data, changes, + offchain_changes, None, initialize_block, ExecutionManager::NativeWhenPossible, @@ -338,6 +340,7 @@ mod tests { _method: &str, _call_data: &[u8], _changes: &RefCell, + _offchain_changes: &RefCell, _storage_transaction_cache: Option<&RefCell< StorageTransactionCache< Block, diff --git a/client/src/light/mod.rs b/client/src/light/mod.rs index 2bb6c85376859..60bb87394f95a 100644 --- a/client/src/light/mod.rs +++ b/client/src/light/mod.rs @@ -31,7 +31,7 @@ use sp_blockchain::Result as ClientResult; use prometheus_endpoint::Registry; use crate::call_executor::LocalCallExecutor; -use crate::client::Client; +use crate::client::{Client,ClientConfig}; use sc_client_api::{ light::Storage as BlockchainStorage, CloneableSpawn, }; @@ -77,7 +77,7 @@ pub fn new_light( S: BlockchainStorage + 'static, E: CodeExecutor + RuntimeInfo + Clone + 'static, { - let local_executor = LocalCallExecutor::new(backend.clone(), code_executor, spawn_handle.clone()); + let local_executor = LocalCallExecutor::new(backend.clone(), code_executor, spawn_handle.clone(), ClientConfig::default()); let executor = GenesisCallExecutor::new(backend.clone(), local_executor); Client::new( backend, @@ -87,6 +87,7 @@ pub fn new_light( Default::default(), Default::default(), prometheus_registry, + ClientConfig::default(), ) } diff --git a/primitives/api/proc-macro/src/decl_runtime_apis.rs b/primitives/api/proc-macro/src/decl_runtime_apis.rs index ef50bd840a7cc..440176bc46858 100644 --- a/primitives/api/proc-macro/src/decl_runtime_apis.rs +++ b/primitives/api/proc-macro/src/decl_runtime_apis.rs @@ -407,6 +407,7 @@ fn generate_call_api_at_calls(decl: &ItemTrait) -> Result { at: &#crate_::BlockId, args: Vec, changes: &std::cell::RefCell<#crate_::OverlayedChanges>, + offchain_changes: &std::cell::RefCell<#crate_::OffchainOverlayedChanges>, storage_transaction_cache: &std::cell::RefCell< #crate_::StorageTransactionCache >, @@ -436,6 +437,7 @@ fn generate_call_api_at_calls(decl: &ItemTrait) -> Result { native_call: None, arguments: args, overlayed_changes: changes, + offchain_changes, storage_transaction_cache, initialize_block, context, @@ -456,6 +458,7 @@ fn generate_call_api_at_calls(decl: &ItemTrait) -> Result { native_call, arguments: args, overlayed_changes: changes, + offchain_changes, storage_transaction_cache, initialize_block, context, diff --git a/primitives/api/proc-macro/src/impl_runtime_apis.rs b/primitives/api/proc-macro/src/impl_runtime_apis.rs index 7def6aa0fb73a..7423dbec559c8 100644 --- a/primitives/api/proc-macro/src/impl_runtime_apis.rs +++ b/primitives/api/proc-macro/src/impl_runtime_apis.rs @@ -207,6 +207,7 @@ fn generate_runtime_api_base_structures() -> Result { commit_on_success: std::cell::RefCell, initialized_block: std::cell::RefCell>>, changes: std::cell::RefCell<#crate_::OverlayedChanges>, + offchain_changes: std::cell::RefCell<#crate_::OffchainOverlayedChanges>, storage_transaction_cache: std::cell::RefCell< #crate_::StorageTransactionCache >, @@ -335,6 +336,7 @@ fn generate_runtime_api_base_structures() -> Result { commit_on_success: true.into(), initialized_block: None.into(), changes: Default::default(), + offchain_changes: Default::default(), recorder: Default::default(), storage_transaction_cache: Default::default(), }.into() @@ -353,6 +355,7 @@ fn generate_runtime_api_base_structures() -> Result { &C, &Self, &std::cell::RefCell<#crate_::OverlayedChanges>, + &std::cell::RefCell<#crate_::OffchainOverlayedChanges>, &std::cell::RefCell<#crate_::StorageTransactionCache>, &std::cell::RefCell>>, &Option<#crate_::ProofRecorder>, @@ -366,6 +369,7 @@ fn generate_runtime_api_base_structures() -> Result { &self.call, self, &self.changes, + &self.offchain_changes, &self.storage_transaction_cache, &self.initialized_block, &self.recorder, @@ -517,6 +521,7 @@ impl<'a> Fold for ApiRuntimeImplToApiRuntimeApiImpl<'a> { call_runtime_at, core_api, changes, + offchain_changes, storage_transaction_cache, initialized_block, recorder @@ -527,6 +532,7 @@ impl<'a> Fold for ApiRuntimeImplToApiRuntimeApiImpl<'a> { at, params_encoded, changes, + offchain_changes, storage_transaction_cache, initialized_block, params.map(|p| { diff --git a/primitives/api/src/lib.rs b/primitives/api/src/lib.rs index a3fc15ba7e249..f8a22c45665ff 100644 --- a/primitives/api/src/lib.rs +++ b/primitives/api/src/lib.rs @@ -27,6 +27,8 @@ //! //! Besides the macros and the [`Core`] runtime api, this crates provides the [`Metadata`] runtime //! api, the [`ApiExt`] trait, the [`CallApiAt`] trait and the [`ConstructRuntimeApi`] trait. +//! +//! On a meta level this implies, the client calls the generated API from the client perspective. #![cfg_attr(not(feature = "std"), no_std)] @@ -44,6 +46,8 @@ pub use sp_core::NativeOrEncoded; #[doc(hidden)] #[cfg(feature = "std")] pub use hash_db::Hasher; +#[cfg(feature = "std")] +pub use sp_core::offchain::storage::OffchainOverlayedChanges; #[doc(hidden)] #[cfg(not(feature = "std"))] pub use sp_core::to_substrate_wasm_fn_return_value; @@ -431,6 +435,8 @@ pub struct CallApiAtParams<'a, Block: BlockT, C, NC, Backend: StateBackend, /// The overlayed changes that are on top of the state. pub overlayed_changes: &'a RefCell, + /// The overlayed changes to be applied to the offchain worker database. + pub offchain_changes: &'a RefCell, /// The cache for storage transactions. pub storage_transaction_cache: &'a RefCell>, /// Determines if the function requires that `initialize_block` should be called before calling diff --git a/primitives/core/src/offchain/mod.rs b/primitives/core/src/offchain/mod.rs index c393b0f9f8882..e792d71afca3b 100644 --- a/primitives/core/src/offchain/mod.rs +++ b/primitives/core/src/offchain/mod.rs @@ -28,6 +28,9 @@ pub mod storage; #[cfg(feature = "std")] pub mod testing; +/// Local storage prefix used by the Offchain Worker API to +pub const STORAGE_PREFIX : &'static [u8] = b"storage"; + /// Offchain workers local storage. pub trait OffchainStorage: Clone + Send + Sync { /// Persist a value in storage under given key and prefix. @@ -482,8 +485,8 @@ pub trait Externalities: Send { buffer: &mut [u8], deadline: Option ) -> Result; - } + impl Externalities for Box { fn is_validator(&self) -> bool { (& **self).is_validator() @@ -557,6 +560,7 @@ impl Externalities for Box { (&mut **self).http_response_read_body(request_id, buffer, deadline) } } + /// An `OffchainExternalities` implementation with limited capabilities. pub struct LimitedExternalities { capabilities: Capabilities, diff --git a/primitives/core/src/offchain/storage.rs b/primitives/core/src/offchain/storage.rs index 31b6423e5d81c..247d7b5204888 100644 --- a/primitives/core/src/offchain/storage.rs +++ b/primitives/core/src/offchain/storage.rs @@ -18,6 +18,7 @@ use std::collections::hash_map::{HashMap, Entry}; use crate::offchain::OffchainStorage; +use std::iter::Iterator; /// In-memory storage for offchain workers. #[derive(Debug, Clone, Default)] @@ -25,6 +26,24 @@ pub struct InMemOffchainStorage { storage: HashMap, Vec>, } +impl InMemOffchainStorage { + /// Consume the offchain storage and iterate over all key value pairs. + pub fn into_iter(self) -> impl Iterator,Vec)> { + self.storage.into_iter() + } + + /// Iterate over all key value pairs by reference. + pub fn iter<'a>(&'a self) -> impl Iterator,&'a Vec)> { + self.storage.iter() + } + + /// Remove a key and its associated value from the offchain database. + pub fn remove(&mut self, prefix: &[u8], key: &[u8]) { + let key: Vec = prefix.iter().chain(key).cloned().collect(); + let _ = self.storage.remove(&key); + } +} + impl OffchainStorage for InMemOffchainStorage { fn set(&mut self, prefix: &[u8], key: &[u8], value: &[u8]) { let key = prefix.iter().chain(key).cloned().collect(); @@ -58,3 +77,212 @@ impl OffchainStorage for InMemOffchainStorage { } } } + + + + +/// Change to be applied to the offchain worker db in regards to a key. +#[derive(Debug,Clone,Hash,Eq,PartialEq)] +pub enum OffchainOverlayedChange { + /// Remove the data associated with the key + Remove, + /// Overwrite the value of an associated key + SetValue(Vec), +} + +/// In-memory storage for offchain workers recoding changes for the actual offchain storage implementation. +#[derive(Debug, Clone)] +pub enum OffchainOverlayedChanges { + /// Writing overlay changes to the offchain worker database is disabled by configuration. + Disabled, + /// Overlay changes can be recorded using the inner collection of this variant. + Enabled(HashMap, OffchainOverlayedChange>), +} + +impl Default for OffchainOverlayedChanges { + fn default() -> Self { + Self::Disabled + } +} + +impl OffchainOverlayedChanges { + /// Create the disabled variant. + pub fn disabled() -> Self { + Self::Disabled + } + + /// Create the enabled variant. + pub fn enabled() -> Self { + Self::Enabled(HashMap::new()) + } + + /// Consume the offchain storage and iterate over all key value pairs. + pub fn into_iter(self) -> OffchainOverlayedChangesIntoIter { + OffchainOverlayedChangesIntoIter::new(self) + } + + /// Iterate over all key value pairs by reference. + pub fn iter<'a>(&'a self) -> OffchainOverlayedChangesIter { + OffchainOverlayedChangesIter::new(&self) + } + + /// Drain all elements of changeset. + pub fn drain<'a,'d>(&'a mut self) -> OffchainOverlayedChangesDrain<'d> where 'a: 'd { + OffchainOverlayedChangesDrain::new(self) + } + + /// Remove a key and its associated value from the offchain database. + pub fn remove(&mut self, prefix: &[u8], key: &[u8]) { + if let Self::Enabled(ref mut storage) = self { + let key: Vec = prefix.iter().chain(key).cloned().collect(); + let _ = storage.insert(key, OffchainOverlayedChange::Remove); + } + } + + /// Set the value associated with a key under a prefix to the value provided. + pub fn set(&mut self, prefix: &[u8], key: &[u8], value: &[u8]) { + if let Self::Enabled(ref mut storage) = self { + let key = prefix.iter().chain(key).cloned().collect(); + let _ = storage.insert(key, OffchainOverlayedChange::SetValue(value.to_vec())); + } + } + + /// Obtain a associated value to the given key in storage with prefix. + pub fn get(&self, prefix: &[u8], key: &[u8]) -> Option { + if let Self::Enabled(ref storage) = self { + let key: Vec = prefix.iter().chain(key).cloned().collect(); + storage.get(&key).cloned() + } else { + None + } + } +} + +use std::collections::hash_map; + +/// Iterate by reference over the prepared offchain worker storage changes. +pub struct OffchainOverlayedChangesIter<'i> { + inner: Option,OffchainOverlayedChange>>, +} + +impl<'i> Iterator for OffchainOverlayedChangesIter<'i> { + type Item = (&'i Vec, &'i OffchainOverlayedChange); + fn next(&mut self) -> Option { + if let Some(ref mut iter) = self.inner { + iter.next() + } else { + None + } + } +} + +impl<'i> OffchainOverlayedChangesIter<'i> { + /// Create a new iterator based on a refernce to the parent container. + pub fn new(container: &'i OffchainOverlayedChanges) -> Self { + match container { + OffchainOverlayedChanges::Enabled(inner) => Self { + inner: Some(inner.iter()) + }, + OffchainOverlayedChanges::Disabled => Self { inner: None, }, + } + } +} + + +/// Iterate by value over the prepared offchain worker storage changes. +pub struct OffchainOverlayedChangesIntoIter { + inner: Option,OffchainOverlayedChange>>, +} + +impl Iterator for OffchainOverlayedChangesIntoIter { + type Item = (Vec, OffchainOverlayedChange); + fn next(&mut self) -> Option { + if let Some(ref mut iter) = self.inner { + iter.next() + } else { + None + } + } +} + +impl OffchainOverlayedChangesIntoIter { + /// Create a new iterator by consuming the collection. + pub fn new(container: OffchainOverlayedChanges) -> Self { + match container { + OffchainOverlayedChanges::Enabled(inner) => Self { + inner: Some(inner.into_iter()) + }, + OffchainOverlayedChanges::Disabled => Self { inner: None, }, + } + } +} + +/// Iterate over all items while draining them from the collection. +pub struct OffchainOverlayedChangesDrain<'d> { + inner: Option,OffchainOverlayedChange>>, +} + +impl<'d> Iterator for OffchainOverlayedChangesDrain<'d> { + type Item = (Vec, OffchainOverlayedChange); + fn next(&mut self) -> Option { + if let Some(ref mut iter) = self.inner { + iter.next() + } else { + None + } + } +} + +impl<'d> OffchainOverlayedChangesDrain<'d> { + /// Create a new iterator by taking a mut reference to the collection, + /// for the lifetime of the created drain iterator. + pub fn new(container: &'d mut OffchainOverlayedChanges) -> Self { + match container { + OffchainOverlayedChanges::Enabled(ref mut inner) => Self { + inner: Some(inner.drain()) + }, + OffchainOverlayedChanges::Disabled => Self { inner: None, }, + } + } +} + + +#[cfg(test)] +mod test { + use super::*; + use super::super::STORAGE_PREFIX; + + #[test] + fn test_drain() { + let mut ooc = OffchainOverlayedChanges::enabled(); + ooc.set(STORAGE_PREFIX,b"kkk", b"vvv"); + let drained = ooc.drain().count(); + assert_eq!(drained, 1); + let leftover = ooc.iter().count(); + assert_eq!(leftover, 0); + + ooc.set(STORAGE_PREFIX, b"a", b"v"); + ooc.set(STORAGE_PREFIX, b"b", b"v"); + ooc.set(STORAGE_PREFIX, b"c", b"v"); + ooc.set(STORAGE_PREFIX, b"d", b"v"); + ooc.set(STORAGE_PREFIX, b"e", b"v"); + assert_eq!(ooc.iter().count(), 5); + } + + #[test] + fn test_accumulated_set_remove_set() { + let mut ooc = OffchainOverlayedChanges::enabled(); + ooc.set(STORAGE_PREFIX, b"ppp", b"qqq"); + ooc.remove(STORAGE_PREFIX, b"ppp"); + // keys are equiv, so it will overwrite the value and the overlay will contain + // one item + assert_eq!(ooc.iter().count(), 1); + + ooc.set(STORAGE_PREFIX, b"ppp", b"rrr"); + let mut iter = ooc.into_iter(); + let mut k = STORAGE_PREFIX.to_vec(); + k.extend_from_slice(&b"ppp"[..]); + assert_eq!(iter.next(), Some((k, OffchainOverlayedChange::SetValue(b"rrr".to_vec())))); + assert_eq!(iter.next(), None); + } +} diff --git a/primitives/externalities/src/lib.rs b/primitives/externalities/src/lib.rs index 96f17579a29f5..f00742b2e8e6b 100644 --- a/primitives/externalities/src/lib.rs +++ b/primitives/externalities/src/lib.rs @@ -47,6 +47,9 @@ pub enum Error { /// /// Provides access to the storage and to other registered extensions. pub trait Externalities: ExtensionStore { + /// Write a key value pair to the offchain storage database. + fn set_offchain_storage(&mut self, key: &[u8], value: Option<&[u8]>); + /// Read runtime storage. fn storage(&self, key: &[u8]) -> Option>; diff --git a/primitives/io/src/lib.rs b/primitives/io/src/lib.rs index b5827202333e2..af0e6c9b68309 100644 --- a/primitives/io/src/lib.rs +++ b/primitives/io/src/lib.rs @@ -602,6 +602,20 @@ pub trait Hashing { } } +/// Interface that provides functions to access the offchain database access. +#[runtime_interface] +pub trait OffchainIndex { + /// Write a key value pair to the offchain worker database in a buffered fashion. + fn set(&mut self, key: &[u8], value: &[u8]) { + self.set_offchain_storage(key, Some(value)); + } + + /// Remove a key and its associated value from the offchain worker database. + fn clear(&mut self, key: &[u8]) { + self.set_offchain_storage(key, None); + } +} + #[cfg(feature = "std")] sp_externalities::decl_extension! { /// The keystore extension to register/retrieve from the externalities. @@ -609,6 +623,8 @@ sp_externalities::decl_extension! { } /// Interface that provides functions to access the offchain functionality. +/// +/// These functions are being made available to the runtime and are called by the runtime. #[runtime_interface] pub trait Offchain { /// Returns if the local node is a potential validator. @@ -984,6 +1000,7 @@ pub type SubstrateHostFunctions = ( logging::HostFunctions, sandbox::HostFunctions, crate::trie::HostFunctions, + offchain_index::HostFunctions, ); #[cfg(test)] diff --git a/primitives/offchain/Cargo.toml b/primitives/offchain/Cargo.toml index 2f87cb1b168ad..ddac54372ea98 100644 --- a/primitives/offchain/Cargo.toml +++ b/primitives/offchain/Cargo.toml @@ -12,12 +12,17 @@ repository = "https://github.com/paritytech/substrate/" targets = ["x86_64-unknown-linux-gnu"] [dependencies] +sp-core = { version = "2.0.0-dev", default-features = false, path = "../core" } sp-api = { version = "2.0.0-dev", default-features = false, path = "../api" } sp-runtime = { version = "2.0.0-dev", default-features = false, path = "../runtime" } +[dev-dependencies] +sp-state-machine = { version = "0.8.0-alpha.5", default-features = false, path = "../state-machine" } + [features] default = ["std"] std = [ + "sp-core/std", "sp-api/std", "sp-runtime/std" ] diff --git a/primitives/offchain/src/lib.rs b/primitives/offchain/src/lib.rs index ae02fed496768..8f043d712a840 100644 --- a/primitives/offchain/src/lib.rs +++ b/primitives/offchain/src/lib.rs @@ -19,8 +19,8 @@ #![cfg_attr(not(feature = "std"), no_std)] #![warn(missing_docs)] -/// Local Storage Prefix used by the Offchain Worker API to -pub const STORAGE_PREFIX: &[u8] = b"storage"; +/// Re-export of parent module scope storage prefix. +pub use sp_core::offchain::STORAGE_PREFIX as STORAGE_PREFIX; sp_api::decl_runtime_apis! { /// The offchain worker api. diff --git a/primitives/state-machine/src/basic.rs b/primitives/state-machine/src/basic.rs index 7f26085958e97..163737946da97 100644 --- a/primitives/state-machine/src/basic.rs +++ b/primitives/state-machine/src/basic.rs @@ -140,6 +140,8 @@ impl From> for BasicExternalities { } impl Externalities for BasicExternalities { + fn set_offchain_storage(&mut self, _key: &[u8], _value: Option<&[u8]>) {} + fn storage(&self, key: &[u8]) -> Option { self.inner.top.get(key).cloned() } diff --git a/primitives/state-machine/src/ext.rs b/primitives/state-machine/src/ext.rs index 399bfc69d864f..6bc5eb450e120 100644 --- a/primitives/state-machine/src/ext.rs +++ b/primitives/state-machine/src/ext.rs @@ -24,6 +24,7 @@ use crate::{ use hash_db::Hasher; use sp_core::{ + offchain::storage::OffchainOverlayedChanges, storage::{well_known_keys::is_child_storage_key, ChildInfo}, traits::Externalities, hexdisplay::HexDisplay, }; @@ -74,6 +75,8 @@ pub struct Ext<'a, H, N, B> { /// The overlayed changes to write to. overlay: &'a mut OverlayedChanges, + /// The overlayed changes destined for the offchain worker database. + offchain_overlay: &'a mut OffchainOverlayedChanges, /// The storage backend to read from. backend: &'a B, /// The cache for the storage transactions. @@ -99,13 +102,15 @@ where /// Create a new `Ext` from overlayed changes and read-only backend pub fn new( overlay: &'a mut OverlayedChanges, + offchain_overlay: &'a mut OffchainOverlayedChanges, storage_transaction_cache: &'a mut StorageTransactionCache, backend: &'a B, changes_trie_state: Option>, extensions: Option<&'a mut Extensions>, ) -> Self { - Ext { + Self { overlay, + offchain_overlay, backend, changes_trie_state, storage_transaction_cache, @@ -121,6 +126,11 @@ where fn mark_dirty(&mut self) { self.storage_transaction_cache.reset(); } + + /// Read only accessor for the scheduled overlay changes. + pub fn get_offchain_storage_changes(&self) -> &OffchainOverlayedChanges { + &*self.offchain_overlay + } } #[cfg(test)] @@ -152,6 +162,15 @@ where B: 'a + Backend, N: crate::changes_trie::BlockNumber, { + + fn set_offchain_storage(&mut self, key: &[u8], value: Option<&[u8]>) { + use ::sp_core::offchain::STORAGE_PREFIX; + match value { + Some(value) => self.offchain_overlay.set(STORAGE_PREFIX, key, value), + None => self.offchain_overlay.remove(STORAGE_PREFIX, key), + } + } + fn storage(&self, key: &[u8]) -> Option { let _guard = sp_panic_handler::AbortGuard::force_abort(); let result = self.overlay.storage(key).map(|x| x.map(|x| x.to_vec())).unwrap_or_else(|| @@ -572,14 +591,23 @@ mod tests { use hex_literal::hex; use num_traits::Zero; use codec::Encode; - use sp_core::{H256, Blake2Hasher, storage::well_known_keys::EXTRINSIC_INDEX, map}; + use sp_core::{ + H256, + Blake2Hasher, + map, + offchain, + storage::{ + Storage, + StorageChild, + well_known_keys::EXTRINSIC_INDEX, + }, + }; use crate::{ changes_trie::{ Configuration as ChangesTrieConfiguration, InMemoryStorage as TestChangesTrieStorage, }, InMemoryBackend, overlayed_changes::OverlayedValue, }; - use sp_core::storage::{Storage, StorageChild}; type TestBackend = InMemoryBackend; type TestExt<'a> = Ext<'a, Blake2Hasher, u64, TestBackend>; @@ -602,6 +630,13 @@ mod tests { } } + fn prepare_offchain_overlay_with_changes() -> OffchainOverlayedChanges { + let mut ooc = OffchainOverlayedChanges::enabled(); + ooc.set(offchain::STORAGE_PREFIX, b"k1", b"v1"); + ooc.set(offchain::STORAGE_PREFIX, b"k2", b"v2"); + ooc + } + fn changes_trie_config() -> ChangesTrieConfiguration { ChangesTrieConfiguration { digest_interval: 0, @@ -612,29 +647,32 @@ mod tests { #[test] fn storage_changes_root_is_none_when_storage_is_not_provided() { let mut overlay = prepare_overlay_with_changes(); + let mut offchain_overlay = prepare_offchain_overlay_with_changes(); let mut cache = StorageTransactionCache::default(); let backend = TestBackend::default(); - let mut ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); + let mut ext = TestExt::new(&mut overlay, &mut offchain_overlay, &mut cache, &backend, None, None); assert_eq!(ext.storage_changes_root(&H256::default().encode()).unwrap(), None); } #[test] fn storage_changes_root_is_none_when_state_is_not_provided() { let mut overlay = prepare_overlay_with_changes(); + let mut offchain_overlay = prepare_offchain_overlay_with_changes(); let mut cache = StorageTransactionCache::default(); let backend = TestBackend::default(); - let mut ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); + let mut ext = TestExt::new(&mut overlay, &mut offchain_overlay, &mut cache, &backend, None, None); assert_eq!(ext.storage_changes_root(&H256::default().encode()).unwrap(), None); } #[test] fn storage_changes_root_is_some_when_extrinsic_changes_are_non_empty() { let mut overlay = prepare_overlay_with_changes(); + let mut offchain_overlay = prepare_offchain_overlay_with_changes(); let mut cache = StorageTransactionCache::default(); let storage = TestChangesTrieStorage::with_blocks(vec![(99, Default::default())]); let state = Some(ChangesTrieState::new(changes_trie_config(), Zero::zero(), &storage)); let backend = TestBackend::default(); - let mut ext = TestExt::new(&mut overlay, &mut cache, &backend, state, None); + let mut ext = TestExt::new(&mut overlay, &mut offchain_overlay, &mut cache, &backend, state, None); assert_eq!( ext.storage_changes_root(&H256::default().encode()).unwrap(), Some(hex!("bb0c2ef6e1d36d5490f9766cfcc7dfe2a6ca804504c3bb206053890d6dd02376").to_vec()), @@ -644,12 +682,13 @@ mod tests { #[test] fn storage_changes_root_is_some_when_extrinsic_changes_are_empty() { let mut overlay = prepare_overlay_with_changes(); + let mut offchain_overlay = prepare_offchain_overlay_with_changes(); let mut cache = StorageTransactionCache::default(); overlay.prospective.top.get_mut(&vec![1]).unwrap().value = None; let storage = TestChangesTrieStorage::with_blocks(vec![(99, Default::default())]); let state = Some(ChangesTrieState::new(changes_trie_config(), Zero::zero(), &storage)); let backend = TestBackend::default(); - let mut ext = TestExt::new(&mut overlay, &mut cache, &backend, state, None); + let mut ext = TestExt::new(&mut overlay, &mut offchain_overlay, &mut cache, &backend, state, None); assert_eq!( ext.storage_changes_root(&H256::default().encode()).unwrap(), Some(hex!("96f5aae4690e7302737b6f9b7f8567d5bbb9eac1c315f80101235a92d9ec27f4").to_vec()), @@ -662,6 +701,7 @@ mod tests { let mut overlay = OverlayedChanges::default(); overlay.set_storage(vec![20], None); overlay.set_storage(vec![30], Some(vec![31])); + let mut offchain_overlay = prepare_offchain_overlay_with_changes(); let backend = Storage { top: map![ vec![10] => vec![10], @@ -671,7 +711,7 @@ mod tests { children_default: map![] }.into(); - let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); + let ext = TestExt::new(&mut overlay, &mut offchain_overlay, &mut cache, &backend, None, None); // next_backend < next_overlay assert_eq!(ext.next_storage_key(&[5]), Some(vec![10])); @@ -687,7 +727,7 @@ mod tests { drop(ext); overlay.set_storage(vec![50], Some(vec![50])); - let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); + let ext = TestExt::new(&mut overlay, &mut offchain_overlay, &mut cache, &backend, None, None); // next_overlay exist but next_backend doesn't exist assert_eq!(ext.next_storage_key(&[40]), Some(vec![50])); @@ -717,7 +757,9 @@ mod tests { }.into(); - let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); + let mut offchain_overlay = prepare_offchain_overlay_with_changes(); + + let ext = TestExt::new(&mut overlay, &mut offchain_overlay, &mut cache, &backend, None, None); // next_backend < next_overlay assert_eq!(ext.next_child_storage_key(child_info, &[5]), Some(vec![10])); @@ -733,7 +775,7 @@ mod tests { drop(ext); overlay.set_child_storage(child_info, vec![50], Some(vec![50])); - let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); + let ext = TestExt::new(&mut overlay, &mut offchain_overlay, &mut cache, &backend, None, None); // next_overlay exist but next_backend doesn't exist assert_eq!(ext.next_child_storage_key(child_info, &[40]), Some(vec![50])); @@ -747,6 +789,7 @@ mod tests { let mut overlay = OverlayedChanges::default(); overlay.set_child_storage(child_info, vec![20], None); overlay.set_child_storage(child_info, vec![30], Some(vec![31])); + let mut offchain_overlay = prepare_offchain_overlay_with_changes(); let backend = Storage { top: map![], children_default: map![ @@ -761,7 +804,7 @@ mod tests { ], }.into(); - let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); + let ext = TestExt::new(&mut overlay, &mut offchain_overlay, &mut cache, &backend, None, None); assert_eq!(ext.child_storage(child_info, &[10]), Some(vec![10])); assert_eq!( diff --git a/primitives/state-machine/src/lib.rs b/primitives/state-machine/src/lib.rs index 1c0007c5f9108..c121da34a7897 100644 --- a/primitives/state-machine/src/lib.rs +++ b/primitives/state-machine/src/lib.rs @@ -23,6 +23,7 @@ use log::{warn, trace}; use hash_db::Hasher; use codec::{Decode, Encode, Codec}; use sp_core::{ + offchain::storage::OffchainOverlayedChanges, storage::ChildInfo, NativeOrEncoded, NeverNativeValue, hexdisplay::HexDisplay, traits::{CodeExecutor, CallInWasmExt, RuntimeCode}, }; @@ -187,6 +188,7 @@ pub struct StateMachine<'a, B, H, N, Exec> method: &'a str, call_data: &'a [u8], overlay: &'a mut OverlayedChanges, + offchain_overlay: &'a mut OffchainOverlayedChanges, extensions: Extensions, changes_trie_state: Option>, storage_transaction_cache: Option<&'a mut StorageTransactionCache>, @@ -216,6 +218,7 @@ impl<'a, B, H, N, Exec> StateMachine<'a, B, H, N, Exec> where backend: &'a B, changes_trie_state: Option>, overlay: &'a mut OverlayedChanges, + offchain_overlay: &'a mut OffchainOverlayedChanges, exec: &'a Exec, method: &'a str, call_data: &'a [u8], @@ -233,6 +236,7 @@ impl<'a, B, H, N, Exec> StateMachine<'a, B, H, N, Exec> where call_data, extensions, overlay, + offchain_overlay, changes_trie_state, storage_transaction_cache: None, runtime_code, @@ -290,6 +294,7 @@ impl<'a, B, H, N, Exec> StateMachine<'a, B, H, N, Exec> where let mut ext = Ext::new( self.overlay, + self.offchain_overlay, cache, self.backend, self.changes_trie_state.clone(), @@ -500,11 +505,13 @@ where Exec: CodeExecutor + 'static + Clone, N: crate::changes_trie::BlockNumber, { + let mut offchain_overlay = OffchainOverlayedChanges::default(); let proving_backend = proving_backend::ProvingBackend::new(trie_backend); let mut sm = StateMachine::<_, H, N, Exec>::new( &proving_backend, None, overlay, + &mut offchain_overlay, exec, method, call_data, @@ -566,10 +573,12 @@ where Exec: CodeExecutor + Clone + 'static, N: crate::changes_trie::BlockNumber, { + let mut offchain_overlay = OffchainOverlayedChanges::default(); let mut sm = StateMachine::<_, H, N, Exec>::new( trie_backend, None, overlay, + &mut offchain_overlay, exec, method, call_data, @@ -820,12 +829,14 @@ mod tests { fn execute_works() { let backend = trie_backend::tests::test_trie(); let mut overlayed_changes = Default::default(); + let mut offchain_overlayed_changes = Default::default(); let wasm_code = RuntimeCode::empty(); let mut state_machine = StateMachine::new( &backend, changes_trie::disabled_state::<_, u64>(), &mut overlayed_changes, + &mut offchain_overlayed_changes, &DummyCodeExecutor { change_changes_trie_config: false, native_available: true, @@ -850,12 +861,14 @@ mod tests { fn execute_works_with_native_else_wasm() { let backend = trie_backend::tests::test_trie(); let mut overlayed_changes = Default::default(); + let mut offchain_overlayed_changes = Default::default(); let wasm_code = RuntimeCode::empty(); let mut state_machine = StateMachine::new( &backend, changes_trie::disabled_state::<_, u64>(), &mut overlayed_changes, + &mut offchain_overlayed_changes, &DummyCodeExecutor { change_changes_trie_config: false, native_available: true, @@ -877,12 +890,14 @@ mod tests { let mut consensus_failed = false; let backend = trie_backend::tests::test_trie(); let mut overlayed_changes = Default::default(); + let mut offchain_overlayed_changes = Default::default(); let wasm_code = RuntimeCode::empty(); let mut state_machine = StateMachine::new( &backend, changes_trie::disabled_state::<_, u64>(), &mut overlayed_changes, + &mut offchain_overlayed_changes, &DummyCodeExecutor { change_changes_trie_config: false, native_available: true, @@ -968,11 +983,14 @@ mod tests { ], ..Default::default() }; + let mut offchain_overlay = Default::default(); + { let mut cache = StorageTransactionCache::default(); let mut ext = Ext::new( &mut overlay, + &mut offchain_overlay, &mut cache, backend, changes_trie::disabled_state::<_, u64>(), @@ -1003,9 +1021,11 @@ mod tests { let mut state = InMemoryBackend::::default(); let backend = state.as_trie_backend().unwrap(); let mut overlay = OverlayedChanges::default(); + let mut offchain_overlay = OffchainOverlayedChanges::default(); let mut cache = StorageTransactionCache::default(); let mut ext = Ext::new( &mut overlay, + &mut offchain_overlay, &mut cache, backend, changes_trie::disabled_state::<_, u64>(), @@ -1099,12 +1119,14 @@ mod tests { use crate::trie_backend::tests::test_trie; let mut overlay = OverlayedChanges::default(); + let mut offchain_overlay = OffchainOverlayedChanges::default(); let mut transaction = { let backend = test_trie(); let mut cache = StorageTransactionCache::default(); let mut ext = Ext::new( &mut overlay, + &mut offchain_overlay, &mut cache, &backend, changes_trie::disabled_state::<_, u64>(), diff --git a/primitives/state-machine/src/overlayed_changes.rs b/primitives/state-machine/src/overlayed_changes.rs index f57d13ee3ffec..cf7b582843ed6 100644 --- a/primitives/state-machine/src/overlayed_changes.rs +++ b/primitives/state-machine/src/overlayed_changes.rs @@ -30,6 +30,7 @@ use std::iter::FromIterator; use std::collections::{HashMap, BTreeMap, BTreeSet}; use codec::{Decode, Encode}; use sp_core::storage::{well_known_keys::EXTRINSIC_INDEX, ChildInfo}; +use sp_core::offchain::storage::OffchainOverlayedChanges; use std::{mem, ops}; use hash_db::Hasher; @@ -94,9 +95,12 @@ pub struct StorageChanges { pub main_storage_changes: StorageCollection, /// All changes to the child storages. pub child_storage_changes: ChildStorageCollection, + /// Offchain state changes to write to the offchain database. + pub offchain_storage_changes: OffchainOverlayedChanges, /// A transaction for the backend that contains all changes from /// [`main_storage_changes`](Self::main_storage_changes) and from /// [`child_storage_changes`](Self::child_storage_changes). + /// [`offchain_storage_changes`](Self::offchain_storage_changes). pub transaction: Transaction, /// The storage root after applying the transaction. pub transaction_storage_root: H::Out, @@ -111,6 +115,7 @@ impl StorageChanges { pub fn into_inner(self) -> ( StorageCollection, ChildStorageCollection, + OffchainOverlayedChanges, Transaction, H::Out, Option>, @@ -118,6 +123,7 @@ impl StorageChanges { ( self.main_storage_changes, self.child_storage_changes, + self.offchain_storage_changes, self.transaction, self.transaction_storage_root, self.changes_trie_transaction, @@ -162,6 +168,7 @@ impl Default for StorageChanges Self { main_storage_changes: Default::default(), child_storage_changes: Default::default(), + offchain_storage_changes: Default::default(), transaction: Default::default(), transaction_storage_root: Default::default(), changes_trie_transaction: None, @@ -503,11 +510,13 @@ impl OverlayedChanges { .take() .expect("Changes trie transaction was generated by `changes_trie_root`; qed"); + let offchain_storage_changes = Default::default(); let (main_storage_changes, child_storage_changes) = self.drain_committed(); Ok(StorageChanges { main_storage_changes: main_storage_changes.collect(), child_storage_changes: child_storage_changes.map(|(sk, it)| (sk, it.0.collect())).collect(), + offchain_storage_changes, transaction, transaction_storage_root, changes_trie_transaction, @@ -745,9 +754,11 @@ mod tests { ..Default::default() }; + let mut offchain_overlay = Default::default(); let mut cache = StorageTransactionCache::default(); let mut ext = Ext::new( &mut overlay, + &mut offchain_overlay, &mut cache, &backend, crate::changes_trie::disabled_state::<_, u64>(), diff --git a/primitives/state-machine/src/testing.rs b/primitives/state-machine/src/testing.rs index 70a96c623adae..5b7b4332b863e 100644 --- a/primitives/state-machine/src/testing.rs +++ b/primitives/state-machine/src/testing.rs @@ -30,6 +30,7 @@ use crate::{ }, }; use sp_core::{ + offchain::storage::OffchainOverlayedChanges, storage::{ well_known_keys::{CHANGES_TRIE_CONFIG, CODE, HEAP_PAGES, is_child_storage_key}, Storage, @@ -44,6 +45,7 @@ where H::Out: codec::Codec, { overlay: OverlayedChanges, + offchain_overlay: OffchainOverlayedChanges, storage_transaction_cache: StorageTransactionCache< as Backend>::Transaction, H, N >, @@ -61,6 +63,7 @@ impl TestExternalities pub fn ext(&mut self) -> Ext> { Ext::new( &mut self.overlay, + &mut self.offchain_overlay, &mut self.storage_transaction_cache, &self.backend, match self.changes_trie_config.clone() { @@ -98,11 +101,15 @@ impl TestExternalities storage.top.insert(HEAP_PAGES.to_vec(), 8u64.encode()); storage.top.insert(CODE.to_vec(), code.to_vec()); + let offchain_overlay = OffchainOverlayedChanges::enabled(); + let mut extensions = Extensions::default(); extensions.register(sp_core::traits::TaskExecutorExt(sp_core::tasks::executor())); + TestExternalities { overlay, + offchain_overlay, changes_trie_config, extensions, changes_trie_storage: ChangesTrieInMemoryStorage::new(), diff --git a/test-utils/client/src/lib.rs b/test-utils/client/src/lib.rs index 4880b296c7048..bf3f8c6878212 100644 --- a/test-utils/client/src/lib.rs +++ b/test-utils/client/src/lib.rs @@ -214,6 +214,7 @@ impl TestClientBuilder TestClientBuilder< let executor = executor.into().unwrap_or_else(|| NativeExecutor::new(WasmExecutionMethod::Interpreted, None, 8) ); - let executor = LocalCallExecutor::new(self.backend.clone(), executor, tasks_executor()); + let executor = LocalCallExecutor::new(self.backend.clone(), executor, tasks_executor(), Default::default()); self.build_with_executor(executor) } diff --git a/test-utils/runtime/client/src/lib.rs b/test-utils/runtime/client/src/lib.rs index 10360c8076338..a2b38a342bc6d 100644 --- a/test-utils/runtime/client/src/lib.rs +++ b/test-utils/runtime/client/src/lib.rs @@ -352,7 +352,7 @@ pub fn new_light() -> ( let blockchain = Arc::new(sc_client::light::blockchain::Blockchain::new(storage)); let backend = Arc::new(LightBackend::new(blockchain.clone())); let executor = new_native_executor(); - let local_call_executor = sc_client::LocalCallExecutor::new(backend.clone(), executor, sp_core::tasks::executor()); + let local_call_executor = sc_client::LocalCallExecutor::new(backend.clone(), executor, sp_core::tasks::executor(), Default::default()); let call_executor = LightExecutor::new( backend.clone(), local_call_executor, diff --git a/utils/frame/benchmarking-cli/src/command.rs b/utils/frame/benchmarking-cli/src/command.rs index 5e35d57cdaee9..5352b7f008110 100644 --- a/utils/frame/benchmarking-cli/src/command.rs +++ b/utils/frame/benchmarking-cli/src/command.rs @@ -44,6 +44,7 @@ impl BenchmarkCmd { let genesis_storage = spec.build_storage()?; let mut changes = Default::default(); + let mut offchain_changes = Default::default(); let cache_size = Some(self.database_cache_size as usize); let state = BenchmarkingState::::new(genesis_storage, cache_size)?; let executor = NativeExecutor::::new( @@ -59,6 +60,7 @@ impl BenchmarkCmd { &state, None, &mut changes, + &mut offchain_changes, &executor, "Benchmark_dispatch_benchmark", &( From 75cbe3df5828bdf12e1b97f98e4695b2b3cd7506 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 22 Apr 2020 14:22:01 +0200 Subject: [PATCH 02/16] Update client/cli/src/params/offchain_worker_params.rs Co-Authored-By: Kian Paimani <5588131+kianenigma@users.noreply.github.com> --- client/cli/src/params/offchain_worker_params.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/cli/src/params/offchain_worker_params.rs b/client/cli/src/params/offchain_worker_params.rs index dcdb78befee2a..6d69e0a749b7d 100644 --- a/client/cli/src/params/offchain_worker_params.rs +++ b/client/cli/src/params/offchain_worker_params.rs @@ -72,6 +72,6 @@ impl OffchainWorkerParams { let indexing_enabled = enabled && self.indexing_enabled; - Ok(OffchainWorkerConfig { enabled, indexing_enabled}) + Ok(OffchainWorkerConfig { enabled, indexing_enabled }) } } From f82ff29249d82e6a1896df13b13e652ca8653303 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 22 Apr 2020 14:22:14 +0200 Subject: [PATCH 03/16] Update client/api/src/execution_extensions.rs Co-Authored-By: Kian Paimani <5588131+kianenigma@users.noreply.github.com> --- client/api/src/execution_extensions.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/api/src/execution_extensions.rs b/client/api/src/execution_extensions.rs index b1903e1ee5f8d..55ffc3794c4ea 100644 --- a/client/api/src/execution_extensions.rs +++ b/client/api/src/execution_extensions.rs @@ -202,4 +202,4 @@ impl offchain::TransactionPool for TransactionPoolAdapter< self.pool.submit_at(&self.at, xt) } -} \ No newline at end of file +} From 43fefee3d9a00741745b5ae94f03f3973014cc85 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 22 Apr 2020 14:23:02 +0200 Subject: [PATCH 04/16] Update client/executor/src/integration_tests/mod.rs Co-Authored-By: Kian Paimani <5588131+kianenigma@users.noreply.github.com> --- client/executor/src/integration_tests/mod.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/client/executor/src/integration_tests/mod.rs b/client/executor/src/integration_tests/mod.rs index 4973d1a887e7b..4c972f8f02846 100644 --- a/client/executor/src/integration_tests/mod.rs +++ b/client/executor/src/integration_tests/mod.rs @@ -457,11 +457,11 @@ fn offchain_index(wasm_method: WasmExecutionMethod) { let (offchain, _state) = testing::TestOffchainExt::new(); ext.register_extension(OffchainExt::new(offchain)); call_in_wasm( - "test_offchain_index_set", - &[0], - wasm_method, - &mut ext.ext(), - ).unwrap(); + "test_offchain_index_set", + &[0], + wasm_method, + &mut ext.ext(), + ).unwrap(); use sp_core::offchain::storage::OffchainOverlayedChange; assert_eq!( From 08853b0eef392fdd346ae38d9043a05b1dc85a79 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 22 Apr 2020 14:34:10 +0200 Subject: [PATCH 05/16] Update client/service/src/config.rs Co-Authored-By: Kian Paimani <5588131+kianenigma@users.noreply.github.com> --- client/service/src/config.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/service/src/config.rs b/client/service/src/config.rs index 70db6e6ba071b..4ebe7553949ab 100644 --- a/client/service/src/config.rs +++ b/client/service/src/config.rs @@ -130,7 +130,7 @@ impl KeystoreConfig { pub struct OffchainWorkerConfig { /// If this is allowed. pub enabled: bool, - /// allow writes from the runtime to the offchain worker database + /// allow writes from the runtime to the offchain worker database. pub indexing_enabled: bool, } From 64a5dac11f216f8bd02a984579c83129dd33aaaf Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 22 Apr 2020 14:35:25 +0200 Subject: [PATCH 06/16] Update primitives/core/src/offchain/storage.rs Co-Authored-By: Kian Paimani <5588131+kianenigma@users.noreply.github.com> --- primitives/core/src/offchain/storage.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/primitives/core/src/offchain/storage.rs b/primitives/core/src/offchain/storage.rs index 247d7b5204888..3e31896285fd2 100644 --- a/primitives/core/src/offchain/storage.rs +++ b/primitives/core/src/offchain/storage.rs @@ -162,7 +162,7 @@ use std::collections::hash_map; /// Iterate by reference over the prepared offchain worker storage changes. pub struct OffchainOverlayedChangesIter<'i> { - inner: Option,OffchainOverlayedChange>>, + inner: Option, OffchainOverlayedChange>>, } impl<'i> Iterator for OffchainOverlayedChangesIter<'i> { From 8769de92efe2498d5289d9146482076bef50cd59 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 22 Apr 2020 14:37:00 +0200 Subject: [PATCH 07/16] Update primitives/core/src/offchain/storage.rs Co-Authored-By: Kian Paimani <5588131+kianenigma@users.noreply.github.com> --- primitives/core/src/offchain/storage.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/primitives/core/src/offchain/storage.rs b/primitives/core/src/offchain/storage.rs index 3e31896285fd2..830c25392b742 100644 --- a/primitives/core/src/offchain/storage.rs +++ b/primitives/core/src/offchain/storage.rs @@ -127,7 +127,7 @@ impl OffchainOverlayedChanges { } /// Drain all elements of changeset. - pub fn drain<'a,'d>(&'a mut self) -> OffchainOverlayedChangesDrain<'d> where 'a: 'd { + pub fn drain<'a, 'd>(&'a mut self) -> OffchainOverlayedChangesDrain<'d> where 'a: 'd { OffchainOverlayedChangesDrain::new(self) } From 701ea047c355ef273d03791bdefce8cc0e2609dc Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 22 Apr 2020 14:42:53 +0200 Subject: [PATCH 08/16] fix doctest --- client/src/lib.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/client/src/lib.rs b/client/src/lib.rs index aa7e3388f9373..97cd7df78c7a6 100644 --- a/client/src/lib.rs +++ b/client/src/lib.rs @@ -63,6 +63,7 @@ //! backend.clone(), //! NativeExecutor::::new(WasmExecutionMethod::Interpreted, None, 8), //! sp_core::tasks::executor(), +//! Default::default(), //! ), //! // This parameter provides the storage for the chain genesis. //! &::default(), @@ -70,6 +71,7 @@ //! Default::default(), //! Default::default(), //! None, +//! Default::default(), //! ); //! ``` //! From 1626cc197d75d50497e8308aacc0d2a7c6ce74e2 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 23 Apr 2020 14:52:03 +0200 Subject: [PATCH 09/16] Update primitives/state-machine/src/ext.rs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Tomasz Drwięga --- primitives/state-machine/src/ext.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/primitives/state-machine/src/ext.rs b/primitives/state-machine/src/ext.rs index 6bc5eb450e120..efe14d8d4698c 100644 --- a/primitives/state-machine/src/ext.rs +++ b/primitives/state-machine/src/ext.rs @@ -75,7 +75,7 @@ pub struct Ext<'a, H, N, B> { /// The overlayed changes to write to. overlay: &'a mut OverlayedChanges, - /// The overlayed changes destined for the offchain worker database. + /// The overlayed changes destined for the Offchain DB. offchain_overlay: &'a mut OffchainOverlayedChanges, /// The storage backend to read from. backend: &'a B, From 798695d3c33a0eb04e5fb282031d486e10a37ef4 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 23 Apr 2020 14:52:46 +0200 Subject: [PATCH 10/16] Update primitives/io/src/lib.rs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Tomasz Drwięga --- primitives/io/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/primitives/io/src/lib.rs b/primitives/io/src/lib.rs index af0e6c9b68309..539190733bda5 100644 --- a/primitives/io/src/lib.rs +++ b/primitives/io/src/lib.rs @@ -605,7 +605,7 @@ pub trait Hashing { /// Interface that provides functions to access the offchain database access. #[runtime_interface] pub trait OffchainIndex { - /// Write a key value pair to the offchain worker database in a buffered fashion. + /// Write a key value pair to the Offchain DB database in a buffered fashion. fn set(&mut self, key: &[u8], value: &[u8]) { self.set_offchain_storage(key, Some(value)); } From eb5bc86450a5ff69ebd852c442a4fd694a692809 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 23 Apr 2020 14:53:02 +0200 Subject: [PATCH 11/16] Update primitives/io/src/lib.rs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Tomasz Drwięga --- primitives/io/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/primitives/io/src/lib.rs b/primitives/io/src/lib.rs index 539190733bda5..4081535d0d7ff 100644 --- a/primitives/io/src/lib.rs +++ b/primitives/io/src/lib.rs @@ -602,7 +602,7 @@ pub trait Hashing { } } -/// Interface that provides functions to access the offchain database access. +/// Interface that provides functions to access the Offchain DB. #[runtime_interface] pub trait OffchainIndex { /// Write a key value pair to the Offchain DB database in a buffered fashion. From a7d990a56292a54e14455ac87b524a8a0a19f4ee Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 23 Apr 2020 14:53:13 +0200 Subject: [PATCH 12/16] Update client/cli/src/params/offchain_worker_params.rs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Tomasz Drwięga --- client/cli/src/params/offchain_worker_params.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/cli/src/params/offchain_worker_params.rs b/client/cli/src/params/offchain_worker_params.rs index 6d69e0a749b7d..eb3cc74ad900c 100644 --- a/client/cli/src/params/offchain_worker_params.rs +++ b/client/cli/src/params/offchain_worker_params.rs @@ -45,7 +45,7 @@ pub struct OffchainWorkerParams { )] pub enabled: OffchainWorkerEnabled, - /// Allow access to offchain workers indexing API + /// Enable Offchain Indexing API, which allows block import to write to Offchain DB. /// /// Enables a runtime to write directly to a offchain workers /// DB during block import. From 9b0012cddd202541c645a9a33686e5044e039b13 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 23 Apr 2020 14:53:24 +0200 Subject: [PATCH 13/16] Update primitives/io/src/lib.rs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Tomasz Drwięga --- primitives/io/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/primitives/io/src/lib.rs b/primitives/io/src/lib.rs index 4081535d0d7ff..7b61e19b48f82 100644 --- a/primitives/io/src/lib.rs +++ b/primitives/io/src/lib.rs @@ -610,7 +610,7 @@ pub trait OffchainIndex { self.set_offchain_storage(key, Some(value)); } - /// Remove a key and its associated value from the offchain worker database. + /// Remove a key and its associated value from the Offchain DB. fn clear(&mut self, key: &[u8]) { self.set_offchain_storage(key, None); } From b27ff76b2cd97ee28b3db2f3bc75b02c786dcf85 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 23 Apr 2020 14:53:37 +0200 Subject: [PATCH 14/16] Update client/cli/src/config.rs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Tomasz Drwięga --- client/cli/src/config.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/cli/src/config.rs b/client/cli/src/config.rs index 9cdbdf59274a7..9ea22f9536a2b 100644 --- a/client/cli/src/config.rs +++ b/client/cli/src/config.rs @@ -306,7 +306,7 @@ pub trait CliConfiguration: Sized { Ok(Default::default()) } - /// Returns a offchain worker config wrapped in `Ok(_)` + /// Returns an offchain worker config wrapped in `Ok(_)` /// /// By default offchain workers are disabled. fn offchain_worker(&self, role: &Role) -> Result { From 32ae2ed0f46c5d4b9b8278c159211c5a9642ed77 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 23 Apr 2020 14:53:56 +0200 Subject: [PATCH 15/16] Update client/cli/src/config.rs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Tomasz Drwięga --- client/cli/src/config.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/cli/src/config.rs b/client/cli/src/config.rs index 9ea22f9536a2b..1d07355f0ac93 100644 --- a/client/cli/src/config.rs +++ b/client/cli/src/config.rs @@ -67,7 +67,7 @@ pub trait CliConfiguration: Sized { None } - /// Get the OffchainWorkerParams for this object + /// Get a reference to `OffchainWorkerParams` for this object. fn offchain_worker_params(&self) -> Option<&OffchainWorkerParams> { None } From 62ccc603a41d74c753f42b04eca46d452e9c00e3 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 23 Apr 2020 14:54:11 +0200 Subject: [PATCH 16/16] Update client/api/src/backend.rs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Tomasz Drwięga --- client/api/src/backend.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/api/src/backend.rs b/client/api/src/backend.rs index 3fe9bd96be5e2..ad697a35932ab 100644 --- a/client/api/src/backend.rs +++ b/client/api/src/backend.rs @@ -148,7 +148,7 @@ pub trait BlockImportOperation { child_update: ChildStorageCollection, ) -> sp_blockchain::Result<()>; - /// Set offchain storage changes. + /// Write offchain storage changes to the database. fn update_offchain_storage( &mut self, _offchain_update: OffchainOverlayedChanges,