From 9ccf26e4f6bf73cd8261a61d11cb5454471760cd Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 18 Nov 2019 20:18:23 +0100 Subject: [PATCH 01/27] In progress, runtime io must switch to future proof root + child_specific (unique id) + u32 type. --- paint/contracts/src/account_db.rs | 16 +- paint/contracts/src/lib.rs | 42 +++- paint/contracts/src/rent.rs | 10 +- paint/support/src/storage/child.rs | 132 ++++++++--- primitives/core/storage/src/lib.rs | 80 +++++++ primitives/externalities/src/lib.rs | 81 +++++-- primitives/sr-io/src/lib.rs | 71 ++++-- primitives/sr-io/with_std.rs | 47 ++-- primitives/sr-io/without_std.rs | 35 ++- primitives/state-machine/src/backend.rs | 222 +++++++++++++----- primitives/state-machine/src/basic.rs | 63 +++-- .../state-machine/src/changes_trie/build.rs | 18 +- primitives/state-machine/src/ext.rs | 148 ++++++++---- primitives/state-machine/src/lib.rs | 14 +- .../state-machine/src/overlayed_changes.rs | 28 ++- .../state-machine/src/proving_backend.rs | 47 +++- primitives/state-machine/src/trie_backend.rs | 30 ++- test/utils/runtime/src/lib.rs | 6 +- 18 files changed, 840 insertions(+), 250 deletions(-) diff --git a/paint/contracts/src/account_db.rs b/paint/contracts/src/account_db.rs index 5aa3a64fd9bc8..f3e53fc601681 100644 --- a/paint/contracts/src/account_db.rs +++ b/paint/contracts/src/account_db.rs @@ -128,7 +128,7 @@ impl AccountDb for DirectAccountDb { trie_id: Option<&TrieId>, location: &StorageKey ) -> Option> { - trie_id.and_then(|id| child::get_raw(id, &blake2_256(location))) + trie_id.and_then(|id| child::get_raw(id, crate::trie_unique_id(&id[..]), &blake2_256(location))) } fn get_code_hash(&self, account: &T::AccountId) -> Option> { >::get(account).and_then(|i| i.as_alive().map(|i| i.code_hash)) @@ -173,13 +173,13 @@ impl AccountDb for DirectAccountDb { (false, Some(info), _) => info, // Existing contract is being removed. (true, Some(info), None) => { - child::kill_storage(&info.trie_id); + child::kill_storage(&info.trie_id, info.child_trie_unique_id()); >::remove(&address); continue; } // Existing contract is being replaced by a new one. (true, Some(info), Some(code_hash)) => { - child::kill_storage(&info.trie_id); + child::kill_storage(&info.trie_id, info.child_trie_unique_id()); AliveContractInfo:: { code_hash, storage_size: T::StorageSizeOffset::get(), @@ -217,14 +217,18 @@ impl AccountDb for DirectAccountDb { } for (k, v) in changed.storage.into_iter() { - if let Some(value) = child::get_raw(&new_info.trie_id[..], &blake2_256(&k)) { + if let Some(value) = child::get_raw( + &new_info.trie_id[..], + new_info.child_trie_unique_id(), + &blake2_256(&k), + ) { new_info.storage_size -= value.len() as u32; } if let Some(value) = v { new_info.storage_size += value.len() as u32; - child::put_raw(&new_info.trie_id[..], &blake2_256(&k), &value[..]); + child::put_raw(&new_info.trie_id[..], new_info.child_trie_unique_id(), &blake2_256(&k), &value[..]); } else { - child::kill(&new_info.trie_id[..], &blake2_256(&k)); + child::kill(&new_info.trie_id[..], new_info.child_trie_unique_id(), &blake2_256(&k)); } } diff --git a/paint/contracts/src/lib.rs b/paint/contracts/src/lib.rs index a4b59226dcdf4..8b2e018230589 100644 --- a/paint/contracts/src/lib.rs +++ b/paint/contracts/src/lib.rs @@ -223,6 +223,19 @@ pub struct RawAliveContractInfo { pub last_write: Option, } +impl RawAliveContractInfo { + /// Associated child trie unique id is built from the hash part of the trie id. + pub fn child_trie_unique_id(&self) -> Option<&[u8]> { + trie_unique_id(&self.trie_id[..]) + } +} + +/// Associated child trie unique id is built from the hash part of the trie id. +pub(crate) fn trie_unique_id(trie_id: &[u8]) -> Option<&[u8]> { + let start = CHILD_STORAGE_KEY_PREFIX.len() + b"default:".len(); + Some(&trie_id[start ..]) +} + pub type TombstoneContractInfo = RawTombstoneContractInfo<::Hash, ::Hashing>; @@ -793,8 +806,17 @@ impl Module { let key_values_taken = delta.iter() .filter_map(|key| { - child::get_raw(&origin_contract.trie_id, &blake2_256(key)).map(|value| { - child::kill(&origin_contract.trie_id, &blake2_256(key)); + child::get_raw( + &origin_contract.trie_id, + origin_contract.child_trie_unique_id(), + &blake2_256(key), + ).map(|value| { + child::kill( + &origin_contract.trie_id, + origin_contract.child_trie_unique_id(), + &blake2_256(key), + ); + (key, value) }) }) @@ -803,13 +825,21 @@ impl Module { let tombstone = >::new( // This operation is cheap enough because last_write (delta not included) // is not this block as it has been checked earlier. - &runtime_io::storage::child_root(&origin_contract.trie_id)[..], - code_hash, + &child::child_root( + &origin_contract.trie_id, + origin_contract.child_trie_unique_id(), + )[..], + code_hash, ); if tombstone != dest_tombstone { for (key, value) in key_values_taken { - child::put_raw(&origin_contract.trie_id, &blake2_256(key), &value); + child::put_raw( + &origin_contract.trie_id, + origin_contract.child_trie_unique_id(), + &blake2_256(key), + &value, + ); } return Err("Tombstones don't match"); @@ -887,7 +917,7 @@ decl_storage! { impl OnFreeBalanceZero for Module { fn on_free_balance_zero(who: &T::AccountId) { if let Some(ContractInfo::Alive(info)) = >::take(who) { - child::kill_storage(&info.trie_id); + child::kill_storage(&info.trie_id, info.child_trie_unique_id()); } } } diff --git a/paint/contracts/src/rent.rs b/paint/contracts/src/rent.rs index e286ce307fcc8..701e7c820321c 100644 --- a/paint/contracts/src/rent.rs +++ b/paint/contracts/src/rent.rs @@ -19,6 +19,7 @@ use sr_primitives::traits::{Bounded, CheckedDiv, CheckedMul, Saturating, Zero, SaturatedConversion}; use support::traits::{Currency, ExistenceRequirement, Get, WithdrawReason, OnUnbalanced}; use support::StorageMap; +use support::storage::child; #[derive(PartialEq, Eq, Copy, Clone)] #[must_use] @@ -99,7 +100,7 @@ fn try_evict_or_and_pay_rent( if balance < subsistence_threshold { // The contract cannot afford to leave a tombstone, so remove the contract info altogether. >::remove(account); - runtime_io::storage::child_storage_kill(&contract.trie_id); + child::kill_storage(&contract.trie_id, contract.child_trie_unique_id()); return (RentOutcome::Evicted, None); } @@ -146,7 +147,10 @@ fn try_evict_or_and_pay_rent( // threshold, so it leaves a tombstone. // Note: this operation is heavy. - let child_storage_root = runtime_io::storage::child_root(&contract.trie_id); + let child_storage_root = child::child_root( + &contract.trie_id, + contract.child_trie_unique_id(), + ); let tombstone = >::new( &child_storage_root[..], @@ -155,7 +159,7 @@ fn try_evict_or_and_pay_rent( let tombstone_info = ContractInfo::Tombstone(tombstone); >::insert(account, &tombstone_info); - runtime_io::storage::child_storage_kill(&contract.trie_id); + child::kill_storage(&contract.trie_id, contract.child_trie_unique_id()); return (RentOutcome::Evicted, Some(tombstone_info)); } diff --git a/paint/support/src/storage/child.rs b/paint/support/src/storage/child.rs index d43c2e896f3d8..e1aeed4becef7 100644 --- a/paint/support/src/storage/child.rs +++ b/paint/support/src/storage/child.rs @@ -19,97 +19,175 @@ //! This module is a currently only a variant of unhashed with additional `storage_key`. //! Note that `storage_key` must be unique and strong (strong in the sense of being long enough to //! avoid collision from a resistant hash function (which unique implies)). +//! +//! A key collision free unique id is required as parameter this is use to avoid key collision between child tries. +//! This unique id management and generation responsability is delegated to paint module. +//! In the future this id could be generated and stored out of chain state, and using None as a parameter will result into this +//! mode of operation. +//! +//! To be able to live with managed unique id it is very important that paint module uses real +//! unique id. If paint use somehow weak id, care should be taken to not conflict to future scheme +//! of managed unique id which will be compact SCALE encoded integer. +//! Empty unique id is also a reserved value (will be use internally for managed unique id as a +//! parameter in runtime io). // NOTE: could replace unhashed by having only one kind of storage (root being null storage key (storage_key can become Option<&[u8]>). use crate::rstd::prelude::*; use codec::{Codec, Encode, Decode}; +/// Unique id managed by the system is not implemented yet. +const SOME: &'static str = "Managed child unique ids are not yet implemented"; /// Return the value of the item in storage under `key`, or `None` if there is no explicit entry. -pub fn get(storage_key: &[u8], key: &[u8]) -> Option { - runtime_io::storage::child_get(storage_key, key).map(|v| { +pub fn get( + storage_key: &[u8], + unique_id: Option<&[u8]>, + key: &[u8], +) -> Option { + runtime_io::storage::child_get(storage_key, unique_id.expect(SOME), key).map(|v| { Decode::decode(&mut &v[..]).expect("storage is not null, therefore must be a valid type") }) } /// Return the value of the item in storage under `key`, or the type's default if there is no /// explicit entry. -pub fn get_or_default(storage_key: &[u8], key: &[u8]) -> T { - get(storage_key, key).unwrap_or_else(Default::default) +pub fn get_or_default( + storage_key: &[u8], + unique_id: Option<&[u8]>, + key: &[u8], +) -> T { + get(storage_key, unique_id, key).unwrap_or_else(Default::default) } /// Return the value of the item in storage under `key`, or `default_value` if there is no /// explicit entry. -pub fn get_or(storage_key: &[u8], key: &[u8], default_value: T) -> T { - get(storage_key, key).unwrap_or(default_value) +pub fn get_or( + storage_key: &[u8], + unique_id: Option<&[u8]>, + key: &[u8], + default_value: T, +) -> T { + get(storage_key, unique_id, key).unwrap_or(default_value) } /// Return the value of the item in storage under `key`, or `default_value()` if there is no /// explicit entry. pub fn get_or_else T>( storage_key: &[u8], + unique_id: Option<&[u8]>, key: &[u8], default_value: F, ) -> T { - get(storage_key, key).unwrap_or_else(default_value) + get(storage_key, unique_id, key).unwrap_or_else(default_value) } /// Put `value` in storage under `key`. -pub fn put(storage_key: &[u8], key: &[u8], value: &T) { - value.using_encoded(|slice| runtime_io::storage::child_set(storage_key, key, slice)); +pub fn put( + storage_key: &[u8], + unique_id: Option<&[u8]>, + key: &[u8], + value: &T, +) { + value.using_encoded(|slice| + runtime_io::storage::child_set(storage_key, unique_id.expect(SOME), key, slice) + ); } /// Remove `key` from storage, returning its value if it had an explicit entry or `None` otherwise. -pub fn take(storage_key: &[u8], key: &[u8]) -> Option { - let r = get(storage_key, key); +pub fn take( + storage_key: &[u8], + unique_id: Option<&[u8]>, + key: &[u8], +) -> Option { + let r = get(storage_key, unique_id, key); if r.is_some() { - kill(storage_key, key); + kill(storage_key, unique_id, key); } r } /// Remove `key` from storage, returning its value, or, if there was no explicit entry in storage, /// the default for its type. -pub fn take_or_default(storage_key: &[u8], key: &[u8]) -> T { - take(storage_key, key).unwrap_or_else(Default::default) +pub fn take_or_default( + storage_key: &[u8], + unique_id: Option<&[u8]>, + key: &[u8], +) -> T { + take(storage_key, unique_id, key).unwrap_or_else(Default::default) } /// Return the value of the item in storage under `key`, or `default_value` if there is no /// explicit entry. Ensure there is no explicit entry on return. -pub fn take_or(storage_key: &[u8],key: &[u8], default_value: T) -> T { - take(storage_key, key).unwrap_or(default_value) +pub fn take_or( + storage_key: &[u8], + unique_id: Option<&[u8]>, + key: &[u8], + default_value: T, +) -> T { + take(storage_key, unique_id, key).unwrap_or(default_value) } /// Return the value of the item in storage under `key`, or `default_value()` if there is no /// explicit entry. Ensure there is no explicit entry on return. pub fn take_or_else T>( storage_key: &[u8], + unique_id: Option<&[u8]>, key: &[u8], default_value: F, ) -> T { - take(storage_key, key).unwrap_or_else(default_value) + take(storage_key, unique_id, key).unwrap_or_else(default_value) } /// Check to see if `key` has an explicit entry in storage. -pub fn exists(storage_key: &[u8], key: &[u8]) -> bool { - runtime_io::storage::child_read(storage_key, key, &mut [0;0][..], 0).is_some() +pub fn exists( + storage_key: &[u8], + unique_id: Option<&[u8]>, + key: &[u8], +) -> bool { + runtime_io::storage::child_read( + storage_key, unique_id.expect(SOME), key, &mut [0;0][..], 0, + ).is_some() } /// Remove all `storage_key` key/values -pub fn kill_storage(storage_key: &[u8]) { - runtime_io::storage::child_storage_kill(storage_key) +pub fn kill_storage( + storage_key: &[u8], + unique_id: Option<&[u8]>, +) { + runtime_io::storage::child_storage_kill(storage_key, unique_id.expect(SOME)) } /// Ensure `key` has no explicit entry in storage. -pub fn kill(storage_key: &[u8], key: &[u8]) { - runtime_io::storage::child_clear(storage_key, key); +pub fn kill( + storage_key: &[u8], + unique_id: Option<&[u8]>, + key: &[u8], +) { + runtime_io::storage::child_clear(storage_key, unique_id.expect(SOME), key); } /// Get a Vec of bytes from storage. -pub fn get_raw(storage_key: &[u8], key: &[u8]) -> Option> { - runtime_io::storage::child_get(storage_key, key) +pub fn get_raw( + storage_key: &[u8], + unique_id: Option<&[u8]>, + key: &[u8], +) -> Option> { + runtime_io::storage::child_get(storage_key, unique_id.expect(SOME), key) } /// Put a raw byte slice into storage. -pub fn put_raw(storage_key: &[u8], key: &[u8], value: &[u8]) { - runtime_io::storage::child_set(storage_key, key, value) +pub fn put_raw( + storage_key: &[u8], + unique_id: Option<&[u8]>, + key: &[u8], + value: &[u8], +) { + runtime_io::storage::child_set(storage_key, unique_id.expect(SOME), key, value) +} + +/// Calculate current child root value. +pub fn child_root( + storage_key: &[u8], + unique_id: Option<&[u8]>, +) -> Vec { + runtime_io::storage::child_root(storage_key, unique_id.expect(SOME)) } diff --git a/primitives/core/storage/src/lib.rs b/primitives/core/storage/src/lib.rs index ba36e2c80f81e..08f2005428503 100644 --- a/primitives/core/storage/src/lib.rs +++ b/primitives/core/storage/src/lib.rs @@ -156,3 +156,83 @@ impl<'a> ChildStorageKey<'a> { self.storage_key.into_owned() } } + +/// Information related to a child trie query. +pub enum ChildInfo<'a> { + Default(ChildTrie<'a>), +} + +/// Owned version of ChildInfo +#[derive(Debug, Clone)] +#[cfg_attr(feature = "std", derive(PartialEq, Eq, Hash, PartialOrd, Ord))] +pub enum OwnedChildInfo { + Default(OwnedChildTrie), +} + +impl<'a> ChildInfo<'a> { + /// Instantiate info for a default child trie. + pub const fn new_default(unique_id: &'a[u8], root: Option<&'a[u8]>) -> Self { + ChildInfo::Default(ChildTrie { + unique_id, + root, + }) + } + + /// Create owned child info. + pub fn to_owned(&self) -> OwnedChildInfo { + match self { + ChildInfo::Default(ChildTrie { unique_id, root }) + => OwnedChildInfo::Default(OwnedChildTrie { + unique_id: unique_id.to_vec(), + root: root.as_ref().map(|s| s.to_vec()), + }), + } + } + +} + +impl OwnedChildInfo { + /// Instantiate info for a default child trie. + pub fn new_default(unique_id: Vec, root: Option>) -> Self { + OwnedChildInfo::Default(OwnedChildTrie { + unique_id, + root, + }) + } + + /// Get reference to child info. + pub fn as_ref(&self) -> ChildInfo { + match self { + OwnedChildInfo::Default(OwnedChildTrie { unique_id, root }) + => ChildInfo::Default(ChildTrie { + unique_id: unique_id.as_slice(), + root: root.as_ref().map(Vec::as_slice), + }), + } + } +} + +/// A child trie of default type. +/// It share its trie node storage with any kind of key, +/// and its unique id needs to be collision free (eg strong +/// crypto hash). +pub struct ChildTrie<'a> { + /// If root was fetch it can be memoïzed in this field + /// to avoid querying it explicitly. + pub root: Option<&'a[u8]>, + + /// Unique id must but unique and free of any possible key collision. + pub unique_id: &'a[u8], +} + +/// Owned version of default child trie. +#[derive(Debug, Clone)] +#[cfg_attr(feature = "std", derive(PartialEq, Eq, Hash, PartialOrd, Ord))] +pub struct OwnedChildTrie { + /// If root was fetch it can be memoïzed in this field + /// to avoid querying it explicitly. + pub root: Option>, + + /// Unique id must but unique and free of any possible key collision. + pub unique_id: Vec, +} diff --git a/primitives/externalities/src/lib.rs b/primitives/externalities/src/lib.rs index 4efbc54a4edc9..6609b542a58b9 100644 --- a/primitives/externalities/src/lib.rs +++ b/primitives/externalities/src/lib.rs @@ -26,7 +26,7 @@ use primitive_types::H256; use std::any::{Any, TypeId}; -use primitives_storage::ChildStorageKey; +use primitives_storage::{ChildStorageKey, ChildInfo}; pub use scope_limited::{set_and_run_with_externalities, with_externalities}; pub use extensions::{Extension, Extensions, ExtensionStore}; @@ -45,13 +45,23 @@ pub trait Externalities: ExtensionStore { fn storage_hash(&self, key: &[u8]) -> Option; /// Get child storage value hash. This may be optimized for large values. - fn child_storage_hash(&self, storage_key: ChildStorageKey, key: &[u8]) -> Option; + fn child_storage_hash( + &self, + storage_key: ChildStorageKey, + child_info: ChildInfo, + key: &[u8], + ) -> Option; /// Read original runtime storage, ignoring any overlayed changes. fn original_storage(&self, key: &[u8]) -> Option>; /// Read original runtime child storage, ignoring any overlayed changes. - fn original_child_storage(&self, storage_key: ChildStorageKey, key: &[u8]) -> Option>; + fn original_child_storage( + &self, + storage_key: ChildStorageKey, + child_info: ChildInfo, + key: &[u8], + ) -> Option>; /// Get original storage value hash, ignoring any overlayed changes. /// This may be optimized for large values. @@ -59,10 +69,20 @@ pub trait Externalities: ExtensionStore { /// Get original child storage value hash, ignoring any overlayed changes. /// This may be optimized for large values. - fn original_child_storage_hash(&self, storage_key: ChildStorageKey, key: &[u8]) -> Option; + fn original_child_storage_hash( + &self, + storage_key: ChildStorageKey, + child_info: ChildInfo, + key: &[u8], + ) -> Option; /// Read child runtime storage. - fn child_storage(&self, storage_key: ChildStorageKey, key: &[u8]) -> Option>; + fn child_storage( + &self, + storage_key: ChildStorageKey, + child_info: ChildInfo, + key: &[u8], + ) -> Option>; /// Set storage entry `key` of current contract being called (effective immediately). fn set_storage(&mut self, key: Vec, value: Vec) { @@ -70,8 +90,14 @@ pub trait Externalities: ExtensionStore { } /// Set child storage entry `key` of current contract being called (effective immediately). - fn set_child_storage(&mut self, storage_key: ChildStorageKey, key: Vec, value: Vec) { - self.place_child_storage(storage_key, key, Some(value)) + fn set_child_storage( + &mut self, + storage_key: ChildStorageKey, + child_info: ChildInfo, + key: Vec, + value: Vec, + ) { + self.place_child_storage(storage_key, child_info, key, Some(value)) } /// Clear a storage entry (`key`) of current contract being called (effective immediately). @@ -80,8 +106,13 @@ pub trait Externalities: ExtensionStore { } /// Clear a child storage entry (`key`) of current contract being called (effective immediately). - fn clear_child_storage(&mut self, storage_key: ChildStorageKey, key: &[u8]) { - self.place_child_storage(storage_key, key.to_vec(), None) + fn clear_child_storage( + &mut self, + storage_key: ChildStorageKey, + child_info: ChildInfo, + key: &[u8], + ) { + self.place_child_storage(storage_key, child_info, key.to_vec(), None) } /// Whether a storage entry exists. @@ -90,24 +121,40 @@ pub trait Externalities: ExtensionStore { } /// Whether a child storage entry exists. - fn exists_child_storage(&self, storage_key: ChildStorageKey, key: &[u8]) -> bool { - self.child_storage(storage_key, key).is_some() + fn exists_child_storage( + &self, + storage_key: ChildStorageKey, + child_info: ChildInfo, + key: &[u8], + ) -> bool { + self.child_storage(storage_key, child_info, key).is_some() } /// Clear an entire child storage. - fn kill_child_storage(&mut self, storage_key: ChildStorageKey); + fn kill_child_storage(&mut self, storage_key: ChildStorageKey, child_info: ChildInfo); /// Clear storage entries which keys are start with the given prefix. fn clear_prefix(&mut self, prefix: &[u8]); /// Clear child storage entries which keys are start with the given prefix. - fn clear_child_prefix(&mut self, storage_key: ChildStorageKey, prefix: &[u8]); + fn clear_child_prefix( + &mut self, + storage_key: ChildStorageKey, + child_info: ChildInfo, + prefix: &[u8], + ); /// Set or clear a storage entry (`key`) of current contract being called (effective immediately). fn place_storage(&mut self, key: Vec, value: Option>); /// Set or clear a child storage entry. Return whether the operation succeeds. - fn place_child_storage(&mut self, storage_key: ChildStorageKey, key: Vec, value: Option>); + fn place_child_storage( + &mut self, + storage_key: ChildStorageKey, + child_info: ChildInfo, + key: Vec, + value: Option>, + ); /// Get the identity of the chain. fn chain_id(&self) -> u64; @@ -120,7 +167,11 @@ pub trait Externalities: ExtensionStore { /// storage keys in the top-level storage map. /// If the storage root equals the default hash as defined by the trie, the key in the top-level /// storage map will be removed. - fn child_storage_root(&mut self, storage_key: ChildStorageKey) -> Vec; + fn child_storage_root( + &mut self, + storage_key: ChildStorageKey, + child_info: ChildInfo, + ) -> Vec; /// Get the change trie root of the current storage overlay at a block with given parent. fn storage_changes_root(&mut self, parent: H256) -> Result, ()>; diff --git a/primitives/sr-io/src/lib.rs b/primitives/sr-io/src/lib.rs index 080d0b245b738..a3be958a2a720 100644 --- a/primitives/sr-io/src/lib.rs +++ b/primitives/sr-io/src/lib.rs @@ -89,16 +89,21 @@ pub trait Storage { } /// Returns the data for `key` in the child storage or `None` if the key can not be found. - fn child_get(&self, child_storage_key: &[u8], key: &[u8]) -> Option> { + /// A key collision free unique id is required as parameter. + fn child_get( + &self, + child_storage_key: &[u8], + unique_id: &[u8], + key: &[u8], + ) -> Option> { let storage_key = child_storage_key_or_panic(child_storage_key); - self.child_storage(storage_key, key).map(|s| s.to_vec()) + self.child_storage(storage_key, unique_id, key).map(|s| s.to_vec()) } /// Get `key` from storage, placing the value into `value_out` and return the number of /// bytes that the entry in storage has beyond the offset or `None` if the storage entry /// doesn't exist at all. - /// If `value_out` length is smaller than the returned length, only `value_out` length bytes - /// are copied into `value_out`. + /// A key collision free unique id is required as parameter. fn read(&self, key: &[u8], value_out: &mut [u8], value_offset: u32) -> Option { self.storage(key).map(|value| { let value_offset = value_offset as usize; @@ -112,17 +117,19 @@ pub trait Storage { /// Get `key` from child storage, placing the value into `value_out` and return the number /// of bytes that the entry in storage has beyond the offset or `None` if the storage entry /// doesn't exist at all. + /// A key collision free unique id is required as parameter. /// If `value_out` length is smaller than the returned length, only `value_out` length bytes /// are copied into `value_out`. fn child_read( &self, child_storage_key: &[u8], + unique_id: &[u8], key: &[u8], value_out: &mut [u8], value_offset: u32, ) -> Option { let storage_key = child_storage_key_or_panic(child_storage_key); - self.child_storage(storage_key, key) + self.child_storage(storage_key, unique_id, key) .map(|value| { let value_offset = value_offset as usize; let data = &value[value_offset.min(value.len())..]; @@ -138,9 +145,16 @@ pub trait Storage { } /// Set `key` to `value` in the child storage denoted by `child_storage_key`. - fn child_set(&mut self, child_storage_key: &[u8], key: &[u8], value: &[u8]) { + /// A key collision free unique id is required as parameter. + fn child_set( + &mut self, + child_storage_key: &[u8], + unique_id: &[u8], + key: &[u8], + value: &[u8], + ) { let storage_key = child_storage_key_or_panic(child_storage_key); - self.set_child_storage(storage_key, key.to_vec(), value.to_vec()); + self.set_child_storage(storage_key, unique_id, key.to_vec(), value.to_vec()); } /// Clear the storage of the given `key` and its value. @@ -149,15 +163,24 @@ pub trait Storage { } /// Clear the given child storage of the given `key` and its value. - fn child_clear(&mut self, child_storage_key: &[u8], key: &[u8]) { + fn child_clear( + &mut self, + child_storage_key: &[u8], + unique_id: &[u8], + key: &[u8], + ) { let storage_key = child_storage_key_or_panic(child_storage_key); - self.clear_child_storage(storage_key, key); + self.clear_child_storage(storage_key, unique_id, key); } /// Clear an entire child storage. - fn child_storage_kill(&mut self, child_storage_key: &[u8]) { + fn child_storage_kill( + &mut self, + child_storage_key: &[u8], + unique_id: &[u8], + ) { let storage_key = child_storage_key_or_panic(child_storage_key); - self.kill_child_storage(storage_key); + self.kill_child_storage(storage_key, unique_id); } /// Check whether the given `key` exists in storage. @@ -166,9 +189,14 @@ pub trait Storage { } /// Check whether the given `key` exists in storage. - fn child_exists(&self, child_storage_key: &[u8], key: &[u8]) -> bool { + fn child_exists( + &self, + child_storage_key: &[u8], + unique_id: &[u8], + key: &[u8], + ) -> bool { let storage_key = child_storage_key_or_panic(child_storage_key); - self.exists_child_storage(storage_key, key) + self.exists_child_storage(storage_key, unique_id, key) } /// Clear the storage of each key-value pair where the key starts with the given `prefix`. @@ -177,9 +205,14 @@ pub trait Storage { } /// Clear the child storage of each key-value pair where the key starts with the given `prefix`. - fn child_clear_prefix(&mut self, child_storage_key: &[u8], prefix: &[u8]) { + fn child_clear_prefix( + &mut self, + child_storage_key: &[u8], + unique_id: &[u8], + prefix: &[u8], + ) { let storage_key = child_storage_key_or_panic(child_storage_key); - self.clear_child_prefix(storage_key, prefix); + self.clear_child_prefix(storage_key, unique_id, prefix); } /// "Commit" all existing operations and compute the resulting storage root. @@ -188,9 +221,13 @@ pub trait Storage { } /// "Commit" all existing operations and compute the resulting child storage root. - fn child_root(&mut self, child_storage_key: &[u8]) -> Vec { + fn child_root( + &mut self, + child_storage_key: &[u8], + unique_id: &[u8], + ) -> Vec { let storage_key = child_storage_key_or_panic(child_storage_key); - self.child_storage_root(storage_key) + self.child_storage_root(storage_key, unique_id) } /// "Commit" all existing operations and get the resulting storage change root. diff --git a/primitives/sr-io/with_std.rs b/primitives/sr-io/with_std.rs index 7e0504c37aafc..72a4f1951d912 100644 --- a/primitives/sr-io/with_std.rs +++ b/primitives/sr-io/with_std.rs @@ -17,7 +17,7 @@ use primitives::{ blake2_128, blake2_256, twox_128, twox_256, twox_64, ed25519, Blake2Hasher, sr25519, Pair, H256, traits::KeystoreExt, storage::ChildStorageKey, hexdisplay::HexDisplay, Hasher, - offchain::{self, OffchainExt}, + offchain::{self, OffchainExt}, storage::OwnedChildInfo, }; // Switch to this after PoC-3 // pub use primitives::BlakeHasher; @@ -60,10 +60,14 @@ impl StorageApi for () { })).expect("read_storage cannot be called outside of an Externalities-provided environment.") } - fn child_storage(storage_key: &[u8], key: &[u8]) -> Option> { + fn child_storage( + storage_key: &[u8], + unique_id: &[u8], + key: &[u8], + ) -> Option> { with_externalities(|ext| { let storage_key = child_storage_key_or_panic(storage_key); - ext.child_storage(storage_key, key).map(|s| s.to_vec()) + ext.child_storage(storage_key, unique_id, key).map(|s| s.to_vec()) }) .expect("storage cannot be called outside of an Externalities-provided environment.") } @@ -76,13 +80,14 @@ impl StorageApi for () { fn read_child_storage( storage_key: &[u8], + unique_id: &[u8], key: &[u8], value_out: &mut [u8], value_offset: usize, ) -> Option { with_externalities(|ext| { let storage_key = child_storage_key_or_panic(storage_key); - ext.child_storage(storage_key, key) + ext.child_storage(storage_key, unique_id, key) .map(|value| { let data = &value[value_offset.min(value.len())..]; let written = std::cmp::min(data.len(), value_out.len()); @@ -93,10 +98,15 @@ impl StorageApi for () { .expect("read_child_storage cannot be called outside of an Externalities-provided environment.") } - fn set_child_storage(storage_key: &[u8], key: &[u8], value: &[u8]) { + fn set_child_storage( + storage_key: &[u8], + unique_id: &[u8], + key: &[u8], + value: &[u8], + ) { with_externalities(|ext| { let storage_key = child_storage_key_or_panic(storage_key); - ext.set_child_storage(storage_key, key.to_vec(), value.to_vec()) + ext.set_child_storage(storage_key, unique_id, key.to_vec(), value.to_vec()) }); } @@ -106,17 +116,17 @@ impl StorageApi for () { ); } - fn clear_child_storage(storage_key: &[u8], key: &[u8]) { + fn clear_child_storage(storage_key: &[u8], key: &[u8], unique_id: &[u8]) { with_externalities(|ext| { let storage_key = child_storage_key_or_panic(storage_key); - ext.clear_child_storage(storage_key, key) + ext.clear_child_storage(storage_key, unique_id, key) }); } - fn kill_child_storage(storage_key: &[u8]) { + fn kill_child_storage(storage_key: &[u8], unique_id: &[u8]) { with_externalities(|ext| { let storage_key = child_storage_key_or_panic(storage_key); - ext.kill_child_storage(storage_key) + ext.kill_child_storage(storage_key, unique_id) }); } @@ -126,10 +136,10 @@ impl StorageApi for () { ).unwrap_or(false) } - fn exists_child_storage(storage_key: &[u8], key: &[u8]) -> bool { + fn exists_child_storage(storage_key: &[u8], unique_id: &[u8], key: &[u8]) -> bool { with_externalities(|ext| { let storage_key = child_storage_key_or_panic(storage_key); - ext.exists_child_storage(storage_key, key) + ext.exists_child_storage(storage_key, unique_id, key) }).unwrap_or(false) } @@ -137,10 +147,10 @@ impl StorageApi for () { with_externalities(|ext| ext.clear_prefix(prefix)); } - fn clear_child_prefix(storage_key: &[u8], prefix: &[u8]) { + fn clear_child_prefix(storage_key: &[u8], unique_id: &[u8], prefix: &[u8]) { with_externalities(|ext| { let storage_key = child_storage_key_or_panic(storage_key); - ext.clear_child_prefix(storage_key, prefix) + ext.clear_child_prefix(storage_key, unique_id, prefix) }); } @@ -150,10 +160,10 @@ impl StorageApi for () { ).unwrap_or(H256::zero()).into() } - fn child_storage_root(storage_key: &[u8]) -> Vec { + fn child_storage_root(storage_key: &[u8], unique_id: &[u8]) -> Vec { with_externalities(|ext| { let storage_key = child_storage_key_or_panic(storage_key); - ext.child_storage_root(storage_key) + ext.child_storage_root(storage_key, unique_id) }).expect("child_storage_root cannot be called outside of an Externalities-provided environment.") } @@ -469,8 +479,9 @@ impl Api for () {} /// A set of key value pairs for storage. pub type StorageOverlay = HashMap, Vec>; -/// A set of key value pairs for children storage; -pub type ChildrenStorageOverlay = HashMap, StorageOverlay>; +/// Sets of key value pairs for children storage, stored at parent key location, +/// with an associated unique id. +pub type ChildrenStorageOverlay = HashMap, (StorageOverlay, OwnedChildInfo)>; /// Execute the given closure with global functions available whose functionality routes into /// externalities that draw from and populate `storage` and `children_storage`. diff --git a/primitives/sr-io/without_std.rs b/primitives/sr-io/without_std.rs index c3f7d62031b93..d1ce1860f418a 100644 --- a/primitives/sr-io/without_std.rs +++ b/primitives/sr-io/without_std.rs @@ -594,12 +594,14 @@ impl StorageApi for () { } } - fn child_storage(storage_key: &[u8], key: &[u8]) -> Option> { + fn child_storage(storage_key: &[u8], unique_id: &[u8], key: &[u8]) -> Option> { let mut length: u32 = 0; unsafe { let ptr = ext_get_allocated_child_storage.get()( storage_key.as_ptr(), storage_key.len() as u32, + unique_id.as_ptr(), + unique_id.len() as u32, key.as_ptr(), key.len() as u32, &mut length @@ -623,10 +625,17 @@ impl StorageApi for () { } } - fn read_child_storage(storage_key: &[u8], key: &[u8], value_out: &mut [u8], value_offset: usize) -> Option { + fn read_child_storage( + storage_key: &[u8], + unique_id: &[u8], + key: &[u8], + value_out: &mut [u8], + value_offset: usize, + ) -> Option { unsafe { match ext_get_child_storage_into.get()( storage_key.as_ptr(), storage_key.len() as u32, + unique_id.as_ptr(), unique_id.len() as u32, key.as_ptr(), key.len() as u32, value_out.as_mut_ptr(), value_out.len() as u32, value_offset as u32 @@ -646,10 +655,11 @@ impl StorageApi for () { } } - fn set_child_storage(storage_key: &[u8], key: &[u8], value: &[u8]) { + fn set_child_storage(storage_key: &[u8], unique_id: &[u8], key: &[u8], value: &[u8]) { unsafe { ext_set_child_storage.get()( storage_key.as_ptr(), storage_key.len() as u32, + unique_id.as_ptr(), unique_id.len() as u32, key.as_ptr(), key.len() as u32, value.as_ptr(), value.len() as u32 ); @@ -664,10 +674,11 @@ impl StorageApi for () { } } - fn clear_child_storage(storage_key: &[u8], key: &[u8]) { + fn clear_child_storage(storage_key: &[u8], unique_id: &[u8], key: &[u8]) { unsafe { ext_clear_child_storage.get()( storage_key.as_ptr(), storage_key.len() as u32, + unique_id.as_ptr(), unique_id.len() as u32, key.as_ptr(), key.len() as u32 ); } @@ -681,10 +692,11 @@ impl StorageApi for () { } } - fn exists_child_storage(storage_key: &[u8], key: &[u8]) -> bool { + fn exists_child_storage(storage_key: &[u8], unique_id: &[u8], key: &[u8]) -> bool { unsafe { ext_exists_child_storage.get()( storage_key.as_ptr(), storage_key.len() as u32, + unique_id.as_ptr(), unique_id.len() as u32, key.as_ptr(), key.len() as u32 ) != 0 } @@ -699,20 +711,23 @@ impl StorageApi for () { } } - fn clear_child_prefix(storage_key: &[u8], prefix: &[u8]) { + fn clear_child_prefix(storage_key: &[u8], unique_id: &[u8], prefix: &[u8]) { unsafe { ext_clear_child_prefix.get()( storage_key.as_ptr(), storage_key.len() as u32, + unique_id.as_ptr(), unique_id.len() as u32, prefix.as_ptr(), prefix.len() as u32 ); } } - fn kill_child_storage(storage_key: &[u8]) { + fn kill_child_storage(storage_key: &[u8], unique_id) { unsafe { ext_kill_child_storage.get()( storage_key.as_ptr(), - storage_key.len() as u32 + storage_key.len() as u32, + unique_id.as_ptr(), + unique_id.len() as u32, ); } } @@ -725,12 +740,14 @@ impl StorageApi for () { result } - fn child_storage_root(storage_key: &[u8]) -> Vec { + fn child_storage_root(storage_key: &[u8], unique_id: &[u8]) -> Vec { let mut length: u32 = 0; unsafe { let ptr = ext_child_storage_root.get()( storage_key.as_ptr(), storage_key.len() as u32, + unique_id.as_ptr(), + unique_id.len() as u32, &mut length ); from_raw_parts(ptr, length).expect("ext_child_storage_root never returns u32::max_value; qed") diff --git a/primitives/state-machine/src/backend.rs b/primitives/state-machine/src/backend.rs index 5fbda1aa32f45..9952a4ef74983 100644 --- a/primitives/state-machine/src/backend.rs +++ b/primitives/state-machine/src/backend.rs @@ -25,6 +25,7 @@ use trie::{ TrieMut, MemoryDB, child_trie_root, default_child_trie_root, TrieConfiguration, trie_types::{TrieDBMut, Layout}, }; +use primitives::storage::{ChildInfo, OwnedChildInfo}; /// A state backend is used to read state data and can have changes committed /// to it. @@ -49,11 +50,21 @@ pub trait Backend: std::fmt::Debug { } /// Get keyed child storage or None if there is nothing associated. - fn child_storage(&self, storage_key: &[u8], key: &[u8]) -> Result>, Self::Error>; + fn child_storage( + &self, + storage_key: &[u8], + child_info: ChildInfo, + key: &[u8], + ) -> Result>, Self::Error>; /// Get child keyed storage value hash or None if there is nothing associated. - fn child_storage_hash(&self, storage_key: &[u8], key: &[u8]) -> Result, Self::Error> { - self.child_storage(storage_key, key).map(|v| v.map(|v| H::hash(&v))) + fn child_storage_hash( + &self, + storage_key: &[u8], + child_info: ChildInfo, + key: &[u8], + ) -> Result, Self::Error> { + self.child_storage(storage_key, child_info, key).map(|v| v.map(|v| H::hash(&v))) } /// true if a key exists in storage. @@ -62,12 +73,22 @@ pub trait Backend: std::fmt::Debug { } /// true if a key exists in child storage. - fn exists_child_storage(&self, storage_key: &[u8], key: &[u8]) -> Result { - Ok(self.child_storage(storage_key, key)?.is_some()) + fn exists_child_storage( + &self, + storage_key: &[u8], + child_info: ChildInfo, + key: &[u8], + ) -> Result { + Ok(self.child_storage(storage_key, child_info, key)?.is_some()) } /// Retrieve all entries keys of child storage and call `f` for each of those keys. - fn for_keys_in_child_storage(&self, storage_key: &[u8], f: F); + fn for_keys_in_child_storage( + &self, + storage_key: &[u8], + child_info: ChildInfo, + f: F, + ); /// Retrieve all entries keys which start with the given prefix and /// call `f` for each of those keys. @@ -82,7 +103,13 @@ pub trait Backend: std::fmt::Debug { /// Retrieve all child entries keys which start with the given prefix and /// call `f` for each of those keys. - fn for_child_keys_with_prefix(&self, storage_key: &[u8], prefix: &[u8], f: F); + fn for_child_keys_with_prefix( + &self, + storage_key: &[u8], + child_info: ChildInfo, + prefix: &[u8], + f: F, + ); /// Calculate the storage root, with given delta over what is already stored in /// the backend, and produce a "transaction" that can be used to commit. @@ -95,7 +122,12 @@ pub trait Backend: std::fmt::Debug { /// Calculate the child storage root, with given delta over what is already stored in /// the backend, and produce a "transaction" that can be used to commit. The second argument /// is true if child storage root equals default storage root. - fn child_storage_root(&self, storage_key: &[u8], delta: I) -> (Vec, bool, Self::Transaction) + fn child_storage_root( + &self, + storage_key: &[u8], + child_info: ChildInfo, + delta: I, + ) -> (Vec, bool, Self::Transaction) where I: IntoIterator, Option>)>, H::Out: Ord; @@ -111,9 +143,14 @@ pub trait Backend: std::fmt::Debug { } /// Get all keys of child storage with given prefix - fn child_keys(&self, child_storage_key: &[u8], prefix: &[u8]) -> Vec> { + fn child_keys( + &self, + storage_key: &[u8], + child_info: ChildInfo, + prefix: &[u8], + ) -> Vec> { let mut all = Vec::new(); - self.for_child_keys_with_prefix(child_storage_key, prefix, |k| all.push(k.to_vec())); + self.for_child_keys_with_prefix(storage_key, child_info, prefix, |k| all.push(k.to_vec())); all } @@ -133,15 +170,15 @@ pub trait Backend: std::fmt::Debug { where I1: IntoIterator, Option>)>, I2i: IntoIterator, Option>)>, - I2: IntoIterator, I2i)>, + I2: IntoIterator, I2i, OwnedChildInfo)>, ::Out: Ord, { let mut txs: Self::Transaction = Default::default(); let mut child_roots: Vec<_> = Default::default(); // child first - for (storage_key, child_delta) in child_deltas { + for (storage_key, child_delta, child_info) in child_deltas { let (child_root, empty, child_txs) = - self.child_storage_root(&storage_key[..], child_delta); + self.child_storage_root(&storage_key[..], child_info.as_ref(), child_delta); txs.consolidate(child_txs); if empty { child_roots.push((storage_key, None)); @@ -166,20 +203,36 @@ impl<'a, T: Backend, H: Hasher> Backend for &'a T { (*self).storage(key) } - fn child_storage(&self, storage_key: &[u8], key: &[u8]) -> Result>, Self::Error> { - (*self).child_storage(storage_key, key) + fn child_storage( + &self, + storage_key: &[u8], + child_info: ChildInfo, + key: &[u8], + ) -> Result>, Self::Error> { + (*self).child_storage(storage_key, child_info, key) } - fn for_keys_in_child_storage(&self, storage_key: &[u8], f: F) { - (*self).for_keys_in_child_storage(storage_key, f) + fn for_keys_in_child_storage( + &self, + storage_key: &[u8], + child_info: ChildInfo, + f: F, + ) { + (*self).for_keys_in_child_storage(storage_key, child_info, f) } fn for_keys_with_prefix(&self, prefix: &[u8], f: F) { (*self).for_keys_with_prefix(prefix, f) } - fn for_child_keys_with_prefix(&self, storage_key: &[u8], prefix: &[u8], f: F) { - (*self).for_child_keys_with_prefix(storage_key, prefix, f) + fn for_child_keys_with_prefix( + &self, + storage_key: &[u8], + child_info: ChildInfo, + prefix: &[u8], + f: F, + ) { + (*self).for_child_keys_with_prefix(storage_key, child_info, prefix, f) } fn storage_root(&self, delta: I) -> (H::Out, Self::Transaction) @@ -190,12 +243,17 @@ impl<'a, T: Backend, H: Hasher> Backend for &'a T { (*self).storage_root(delta) } - fn child_storage_root(&self, storage_key: &[u8], delta: I) -> (Vec, bool, Self::Transaction) + fn child_storage_root( + &self, + storage_key: &[u8], + child_info: ChildInfo, + delta: I, + ) -> (Vec, bool, Self::Transaction) where I: IntoIterator, Option>)>, H::Out: Ord, { - (*self).child_storage_root(storage_key, delta) + (*self).child_storage_root(storage_key, child_info, delta) } fn pairs(&self) -> Vec<(Vec, Vec)> { @@ -219,7 +277,10 @@ impl Consolidate for () { } } -impl Consolidate for Vec<(Option>, Vec, Option>)> { +impl Consolidate for Vec<( + Option<(Vec, OwnedChildInfo)>, + Vec<(Vec, Option>)>, + )> { fn consolidate(&mut self, mut other: Self) { self.append(&mut other); } @@ -249,7 +310,7 @@ impl error::Error for Void { /// In-memory backend. Fully recomputes tries each time `as_trie_backend` is called but useful for /// tests and proof checking. pub struct InMemory { - inner: HashMap>, HashMap, Vec>>, + inner: HashMap, OwnedChildInfo)>, HashMap, Vec>>, // This field is only needed for returning reference in `as_trie_backend`. trie: Option, H>>, _hasher: PhantomData, @@ -291,19 +352,21 @@ impl InMemory { /// Copy the state, with applied updates pub fn update(&self, changes: >::Transaction) -> Self { let mut inner: HashMap<_, _> = self.inner.clone(); - for (storage_key, key, val) in changes { - match val { - Some(v) => { inner.entry(storage_key).or_default().insert(key, v); }, - None => { inner.entry(storage_key).or_default().remove(&key); }, + for (child_info, key_values) in changes { + let mut entry = inner.entry(child_info).or_default(); + for (key, val) in key_values { + match val { + Some(v) => { entry.insert(key, v); }, + None => { entry.remove(&key); }, + } } } - inner.into() } } -impl From>, HashMap, Vec>>> for InMemory { - fn from(inner: HashMap>, HashMap, Vec>>) -> Self { +impl From, OwnedChildInfo)>, HashMap, Vec>>> for InMemory { + fn from(inner: HashMap, OwnedChildInfo)>, HashMap, Vec>>) -> Self { InMemory { inner: inner, trie: None, @@ -314,14 +377,14 @@ impl From>, HashMap, Vec>>> for In impl From<( HashMap, Vec>, - HashMap, HashMap, Vec>>, + HashMap, (HashMap, Vec>, OwnedChildInfo)>, )> for InMemory { fn from(inners: ( HashMap, Vec>, - HashMap, HashMap, Vec>>, + HashMap, (HashMap, Vec>, OwnedChildInfo)>, )) -> Self { - let mut inner: HashMap>, HashMap, Vec>> - = inners.1.into_iter().map(|(k, v)| (Some(k), v)).collect(); + let mut inner: HashMap, OwnedChildInfo)>, HashMap, Vec>> + = inners.1.into_iter().map(|(k, (v, ci))| (Some((k, ci)), v)).collect(); inner.insert(None, inners.0); InMemory { inner: inner, @@ -343,12 +406,18 @@ impl From, Vec>> for InMemory { } } -impl From>, Vec, Option>)>> for InMemory { - fn from(inner: Vec<(Option>, Vec, Option>)>) -> Self { - let mut expanded: HashMap>, HashMap, Vec>> = HashMap::new(); - for (child_key, key, value) in inner { - if let Some(value) = value { - expanded.entry(child_key).or_default().insert(key, value); +impl From, OwnedChildInfo)>, Vec<(Vec, Option>)>)>> + for InMemory { + fn from( + inner: Vec<(Option<(Vec, OwnedChildInfo)>, Vec<(Vec, Option>)>)>, + ) -> Self { + let mut expanded: HashMap, OwnedChildInfo)>, HashMap, Vec>> = HashMap::new(); + for (child_info, key_values) in inner { + let mut entry = expanded.entry(child_info).or_default(); + for (key, value) in key_values { + if let Some(value) = value { + entry.insert(key, value); + } } } expanded.into() @@ -357,22 +426,33 @@ impl From>, Vec, Option>)>> for InMem impl InMemory { /// child storage key iterator - pub fn child_storage_keys(&self) -> impl Iterator { - self.inner.iter().filter_map(|item| item.0.as_ref().map(|v|&v[..])) + pub fn child_storage_keys(&self) -> impl Iterator { + self.inner.iter().filter_map(|item| + item.0.as_ref().map(|v|(&v.0[..], v.1.as_ref())) + ) } } impl Backend for InMemory { type Error = Void; - type Transaction = Vec<(Option>, Vec, Option>)>; + type Transaction = Vec<( + Option<(Vec, OwnedChildInfo)>, + Vec<(Vec, Option>)>, + )>; type TrieBackendStorage = MemoryDB; fn storage(&self, key: &[u8]) -> Result>, Self::Error> { Ok(self.inner.get(&None).and_then(|map| map.get(key).map(Clone::clone))) } - fn child_storage(&self, storage_key: &[u8], key: &[u8]) -> Result>, Self::Error> { - Ok(self.inner.get(&Some(storage_key.to_vec())).and_then(|map| map.get(key).map(Clone::clone))) + fn child_storage( + &self, + storage_key: &[u8], + child_info: ChildInfo, + key: &[u8], + ) -> Result>, Self::Error> { + Ok(self.inner.get(&Some((storage_key.to_vec(), child_info.to_owned()))) + .and_then(|map| map.get(key).map(Clone::clone))) } fn exists_storage(&self, key: &[u8]) -> Result { @@ -388,12 +468,24 @@ impl Backend for InMemory { .for_each(|(k, v)| f(k, v))); } - fn for_keys_in_child_storage(&self, storage_key: &[u8], mut f: F) { - self.inner.get(&Some(storage_key.to_vec())).map(|map| map.keys().for_each(|k| f(&k))); + fn for_keys_in_child_storage( + &self, + storage_key: &[u8], + child_info: ChildInfo, + mut f: F, + ) { + self.inner.get(&Some((storage_key.to_vec(), child_info.to_owned()))) + .map(|map| map.keys().for_each(|k| f(&k))); } - fn for_child_keys_with_prefix(&self, storage_key: &[u8], prefix: &[u8], f: F) { - self.inner.get(&Some(storage_key.to_vec())) + fn for_child_keys_with_prefix( + &self, + storage_key: &[u8], + child_info: ChildInfo, + prefix: &[u8], + f: F, + ) { + self.inner.get(&Some((storage_key.to_vec(), child_info.to_owned()))) .map(|map| map.keys().filter(|key| key.starts_with(prefix)).map(|k| &**k).for_each(f)); } @@ -413,19 +505,26 @@ impl Backend for InMemory { .filter_map(|(k, maybe_val)| maybe_val.map(|val| (k, val))) ); - let full_transaction = transaction.into_iter().map(|(k, v)| (None, k, v)).collect(); + let full_transaction = transaction.into_iter().collect(); - (root, full_transaction) + (root, vec![(None, full_transaction)]) } - fn child_storage_root(&self, storage_key: &[u8], delta: I) -> (Vec, bool, Self::Transaction) + fn child_storage_root( + &self, + storage_key: &[u8], + child_info: ChildInfo, + delta: I, + ) -> (Vec, bool, Self::Transaction) where I: IntoIterator, Option>)>, H::Out: Ord { let storage_key = storage_key.to_vec(); + let child_info = Some((storage_key.clone(), child_info.to_owned())); - let existing_pairs = self.inner.get(&Some(storage_key.clone())) + + let existing_pairs = self.inner.get(&child_info) .into_iter() .flat_map(|map| map.iter().map(|(k, v)| (k.clone(), Some(v.clone())))); @@ -438,11 +537,11 @@ impl Backend for InMemory { .filter_map(|(k, maybe_val)| maybe_val.map(|val| (k, val))) ); - let full_transaction = transaction.into_iter().map(|(k, v)| (Some(storage_key.clone()), k, v)).collect(); + let full_transaction = transaction.into_iter().collect(); let is_default = root == default_child_trie_root::>(&storage_key); - (root, is_default, full_transaction) + (root, is_default, vec![(child_info, full_transaction)]) } fn pairs(&self) -> Vec<(Vec, Vec)> { @@ -459,8 +558,13 @@ impl Backend for InMemory { .collect() } - fn child_keys(&self, storage_key: &[u8], prefix: &[u8]) -> Vec> { - self.inner.get(&Some(storage_key.to_vec())) + fn child_keys( + &self, + storage_key: &[u8], + child_info: ChildInfo, + prefix: &[u8], + ) -> Vec> { + self.inner.get(&Some((storage_key.to_vec(), child_info.to_owned()))) .into_iter() .flat_map(|map| map.keys().filter(|k| k.starts_with(prefix)).cloned()) .collect() @@ -470,8 +574,8 @@ impl Backend for InMemory { let mut mdb = MemoryDB::default(); let mut new_child_roots = Vec::new(); let mut root_map = None; - for (storage_key, map) in &self.inner { - if let Some(storage_key) = storage_key.as_ref() { + for (child_info, map) in &self.inner { + if let Some((storage_key, child_info)) = child_info.as_ref() { let ch = insert_into_memory_db::(&mut mdb, map.clone().into_iter())?; new_child_roots.push((storage_key.clone(), ch.as_ref().into())); } else { diff --git a/primitives/state-machine/src/basic.rs b/primitives/state-machine/src/basic.rs index e758b3dd3b814..8d2cc08aba4ae 100644 --- a/primitives/state-machine/src/basic.rs +++ b/primitives/state-machine/src/basic.rs @@ -24,7 +24,7 @@ use trie::trie_types::Layout; use primitives::{ storage::{ well_known_keys::is_child_storage_key, ChildStorageKey, StorageOverlay, - ChildrenStorageOverlay + ChildrenStorageOverlay, ChildInfo, }, traits::Externalities, Blake2Hasher, hash::H256, }; @@ -130,20 +130,40 @@ impl Externalities for BasicExternalities { self.storage_hash(key) } - fn child_storage(&self, storage_key: ChildStorageKey, key: &[u8]) -> Option> { + fn child_storage( + &self, + storage_key: ChildStorageKey, + child_info: ChildInfo, + key: &[u8], + ) -> Option> { self.children.get(storage_key.as_ref()).and_then(|child| child.get(key)).cloned() } - fn child_storage_hash(&self, storage_key: ChildStorageKey, key: &[u8]) -> Option { - self.child_storage(storage_key, key).map(|v| Blake2Hasher::hash(&v)) + fn child_storage_hash( + &self, + storage_key: ChildStorageKey, + child_info: ChildInfo, + key: &[u8], + ) -> Option { + self.child_storage(storage_key, child_info, key).map(|v| Blake2Hasher::hash(&v)) } - fn original_child_storage_hash(&self, storage_key: ChildStorageKey, key: &[u8]) -> Option { - self.child_storage_hash(storage_key, key) + fn original_child_storage_hash( + &self, + storage_key: ChildStorageKey, + child_info: ChildInfo, + key: &[u8], + ) -> Option { + self.child_storage_hash(storage_key, child_info, key) } - fn original_child_storage(&self, storage_key: ChildStorageKey, key: &[u8]) -> Option> { - Externalities::child_storage(self, storage_key, key) + fn original_child_storage( + &self, + storage_key: ChildStorageKey, + child_info: ChildInfo, + key: &[u8], + ) -> Option> { + Externalities::child_storage(self, storage_key, child_info, key) } fn place_storage(&mut self, key: Vec, maybe_value: Option>) { @@ -161,6 +181,7 @@ impl Externalities for BasicExternalities { fn place_child_storage( &mut self, storage_key: ChildStorageKey, + child_info: ChildInfo, key: Vec, value: Option>, ) { @@ -172,7 +193,11 @@ impl Externalities for BasicExternalities { } } - fn kill_child_storage(&mut self, storage_key: ChildStorageKey) { + fn kill_child_storage( + &mut self, + storage_key: ChildStorageKey, + child_info: ChildInfo, + ) { self.children.remove(storage_key.as_ref()); } @@ -188,7 +213,12 @@ impl Externalities for BasicExternalities { self.top.retain(|key, _| !key.starts_with(prefix)); } - fn clear_child_prefix(&mut self, storage_key: ChildStorageKey, prefix: &[u8]) { + fn clear_child_prefix( + &mut self, + storage_key: ChildStorageKey, + child_info: ChildInfo, + prefix: &[u8], + ) { if let Some(child) = self.children.get_mut(storage_key.as_ref()) { child.retain(|key, _| !key.starts_with(prefix)); } @@ -198,15 +228,15 @@ impl Externalities for BasicExternalities { fn storage_root(&mut self) -> H256 { let mut top = self.top.clone(); - let keys: Vec<_> = self.children.keys().map(|k| k.to_vec()).collect(); // Single child trie implementation currently allows using the same child // empty root for all child trie. Using null storage key until multiple // type of child trie support. let empty_hash = default_child_trie_root::>(&[]); - for storage_key in keys { + for (storage_key, (_values, child_info)) in self.children.iter() { let child_root = self.child_storage_root( ChildStorageKey::from_slice(storage_key.as_slice()) .expect("Map only feed by valid keys; qed"), + Some(&child_info[..]), ); if &empty_hash[..] == &child_root[..] { top.remove(&storage_key); @@ -218,11 +248,16 @@ impl Externalities for BasicExternalities { Layout::::trie_root(self.top.clone()) } - fn child_storage_root(&mut self, storage_key: ChildStorageKey) -> Vec { + fn child_storage_root( + &mut self, + storage_key: ChildStorageKey, + child_info: ChildInfo, + ) -> Vec { if let Some(child) = self.children.get(storage_key.as_ref()) { let delta = child.clone().into_iter().map(|(k, v)| (k, Some(v))); - InMemory::::default().child_storage_root(storage_key.as_ref(), delta).0 + InMemory::::default() + .child_storage_root(storage_key.as_ref(), child_info, delta).0 } else { default_child_trie_root::>(storage_key.as_ref()) } diff --git a/primitives/state-machine/src/changes_trie/build.rs b/primitives/state-machine/src/changes_trie/build.rs index 10c38a41e2650..0680372de5794 100644 --- a/primitives/state-machine/src/changes_trie/build.rs +++ b/primitives/state-machine/src/changes_trie/build.rs @@ -132,10 +132,15 @@ fn prepare_extrinsics_input_inner<'a, B, H, Number>( H: Hasher, Number: BlockNumber, { - let (committed, prospective) = if let Some(sk) = storage_key.as_ref() { - (changes.committed.children.get(sk), changes.prospective.children.get(sk)) + let (committed, prospective, child_info) = if let Some(sk) = storage_key.as_ref() { + let child_info = changes.child_info(sk).cloned(); + ( + changes.committed.children.get(sk).map(|c| &c.0), + changes.prospective.children.get(sk).map(|c| &c.0), + child_info, + ) } else { - (Some(&changes.committed.top), Some(&changes.prospective.top)) + (Some(&changes.committed.top), Some(&changes.prospective.top), None) }; committed.iter().flat_map(|c| c.iter()) .chain(prospective.iter().flat_map(|c| c.iter())) @@ -147,8 +152,11 @@ fn prepare_extrinsics_input_inner<'a, B, H, Number>( // AND are not in storage at the beginning of operation if let Some(sk) = storage_key.as_ref() { if !changes.child_storage(sk, k).map(|v| v.is_some()).unwrap_or_default() { - if !backend.exists_child_storage(sk, k).map_err(|e| format!("{}", e))? { - return Ok(map); + if let Some(child_info) = child_info.as_ref() { + if !backend.exists_child_storage(sk, child_info.as_ref(), k) + .map_err(|e| format!("{}", e))? { + return Ok(map); + } } } } else { diff --git a/primitives/state-machine/src/ext.rs b/primitives/state-machine/src/ext.rs index 0e93302a95a54..2913523d74b3f 100644 --- a/primitives/state-machine/src/ext.rs +++ b/primitives/state-machine/src/ext.rs @@ -25,7 +25,7 @@ use crate::{ use hash_db::Hasher; use primitives::{ - storage::{ChildStorageKey, well_known_keys::is_child_storage_key}, + storage::{ChildStorageKey, well_known_keys::is_child_storage_key, ChildInfo}, traits::Externalities, hexdisplay::HexDisplay, hash::H256, }; use trie::{trie_types::Layout, MemoryDB, default_child_trie_root}; @@ -227,13 +227,19 @@ where result } - fn child_storage(&self, storage_key: ChildStorageKey, key: &[u8]) -> Option> { + fn child_storage( + &self, + storage_key: ChildStorageKey, + child_info: ChildInfo, + key: &[u8], + ) -> Option> { let _guard = panic_handler::AbortGuard::force_abort(); let result = self.overlay .child_storage(storage_key.as_ref(), key) .map(|x| x.map(|x| x.to_vec())) .unwrap_or_else(|| - self.backend.child_storage(storage_key.as_ref(), key).expect(EXT_NOT_ALLOWED_TO_FAIL) + self.backend.child_storage(storage_key.as_ref(), child_info, key) + .expect(EXT_NOT_ALLOWED_TO_FAIL) ); trace!(target: "state-trace", "{:04x}: GetChild({}) {}={:?}", @@ -246,7 +252,12 @@ where result } - fn child_storage_hash(&self, storage_key: ChildStorageKey, key: &[u8]) -> Option { + fn child_storage_hash( + &self, + storage_key: ChildStorageKey, + child_info: ChildInfo, + key: &[u8], + ) -> Option { let _guard = panic_handler::AbortGuard::force_abort(); let result = self.overlay .child_storage(storage_key.as_ref(), key) @@ -265,10 +276,15 @@ where result } - fn original_child_storage(&self, storage_key: ChildStorageKey, key: &[u8]) -> Option> { + fn original_child_storage( + &self, + storage_key: ChildStorageKey, + child_info: ChildInfo, + key: &[u8], + ) -> Option> { let _guard = panic_handler::AbortGuard::force_abort(); let result = self.backend - .child_storage(storage_key.as_ref(), key) + .child_storage(storage_key.as_ref(), child_info, key) .expect(EXT_NOT_ALLOWED_TO_FAIL); trace!(target: "state-trace", "{:04x}: ChildOriginal({}) {}={:?}", @@ -280,10 +296,15 @@ where result } - fn original_child_storage_hash(&self, storage_key: ChildStorageKey, key: &[u8]) -> Option { + fn original_child_storage_hash( + &self, + storage_key: ChildStorageKey, + child_info: ChildInfo, + key: &[u8], + ) -> Option { let _guard = panic_handler::AbortGuard::force_abort(); let result = self.backend - .child_storage_hash(storage_key.as_ref(), key) + .child_storage_hash(storage_key.as_ref(), child_info, key) .expect(EXT_NOT_ALLOWED_TO_FAIL); trace!(target: "state-trace", "{}: ChildHashOriginal({}) {}={:?}", @@ -311,13 +332,18 @@ where } - fn exists_child_storage(&self, storage_key: ChildStorageKey, key: &[u8]) -> bool { + fn exists_child_storage( + &self, + storage_key: ChildStorageKey, + child_info: ChildInfo, + key: &[u8], + ) -> bool { let _guard = panic_handler::AbortGuard::force_abort(); let result = match self.overlay.child_storage(storage_key.as_ref(), key) { Some(x) => x.is_some(), _ => self.backend - .exists_child_storage(storage_key.as_ref(), key) + .exists_child_storage(storage_key.as_ref(), child_info, key) .expect(EXT_NOT_ALLOWED_TO_FAIL), }; @@ -349,6 +375,7 @@ where fn place_child_storage( &mut self, storage_key: ChildStorageKey, + child_info: ChildInfo, key: Vec, value: Option>, ) { @@ -364,7 +391,11 @@ where self.overlay.set_child_storage(storage_key.into_owned(), key, value); } - fn kill_child_storage(&mut self, storage_key: ChildStorageKey) { + fn kill_child_storage( + &mut self, + storage_key: ChildStorageKey, + child_info: ChildInfo, + ) { trace!(target: "state-trace", "{:04x}: KillChild({})", self.id, HexDisplay::from(&storage_key.as_ref()), @@ -373,7 +404,7 @@ where self.mark_dirty(); self.overlay.clear_child_storage(storage_key.as_ref()); - self.backend.for_keys_in_child_storage(storage_key.as_ref(), |key| { + self.backend.for_keys_in_child_storage(storage_key.as_ref(), child_info, |key| { self.overlay.set_child_storage(storage_key.as_ref().to_vec(), key.to_vec(), None); }); } @@ -396,7 +427,12 @@ where }); } - fn clear_child_prefix(&mut self, storage_key: ChildStorageKey, prefix: &[u8]) { + fn clear_child_prefix( + &mut self, + storage_key: ChildStorageKey, + child_info: ChildInfo, + prefix: &[u8], + ) { trace!(target: "state-trace", "{:04x}: ClearChildPrefix({}) {}", self.id, HexDisplay::from(&storage_key.as_ref()), @@ -406,7 +442,7 @@ where self.mark_dirty(); self.overlay.clear_child_prefix(storage_key.as_ref(), prefix); - self.backend.for_child_keys_with_prefix(storage_key.as_ref(), prefix, |key| { + self.backend.for_child_keys_with_prefix(storage_key.as_ref(), child_info, prefix, |key| { self.overlay.set_child_storage(storage_key.as_ref().to_vec(), key.to_vec(), None); }); } @@ -425,16 +461,21 @@ where return root.clone(); } - let child_storage_keys = - self.overlay.prospective.children.keys() + let child_storage_keys = self.overlay.prospective.children.keys() .chain(self.overlay.committed.children.keys()); let child_delta_iter = child_storage_keys.map(|storage_key| - (storage_key.clone(), self.overlay.committed.children.get(storage_key) - .into_iter() - .flat_map(|map| map.iter().map(|(k, v)| (k.clone(), v.value.clone()))) - .chain(self.overlay.prospective.children.get(storage_key) + ( + storage_key.clone(), + self.overlay.committed.children.get(storage_key) .into_iter() - .flat_map(|map| map.iter().map(|(k, v)| (k.clone(), v.value.clone())))))); + .flat_map(|(map, _)| map.iter().map(|(k, v)| (k.clone(), v.value.clone()))) + .chain(self.overlay.prospective.children.get(storage_key) + .into_iter() + .flat_map(|(map, _)| map.iter().map(|(k, v)| (k.clone(), v.value.clone())))), + self.overlay.child_info(storage_key).cloned() + .expect("child info initialized in either committed or prospective"), + ) + ); // compute and memoize @@ -450,7 +491,12 @@ where root } - fn child_storage_root(&mut self, storage_key: ChildStorageKey) -> Vec { + // TODO EMCH seems like child_info parameter is useless here!!! + fn child_storage_root( + &mut self, + storage_key: ChildStorageKey, + _child_info: ChildInfo, + ) -> Vec { let _guard = panic_handler::AbortGuard::force_abort(); if self.storage_transaction.is_some() { let root = self @@ -467,30 +513,44 @@ where } else { let storage_key = storage_key.as_ref(); - let (root, is_empty, _) = { - let delta = self.overlay.committed.children.get(storage_key) - .into_iter() - .flat_map(|map| map.clone().into_iter().map(|(k, v)| (k, v.value))) - .chain(self.overlay.prospective.children.get(storage_key) - .into_iter() - .flat_map(|map| map.clone().into_iter().map(|(k, v)| (k, v.value)))); - - self.backend.child_storage_root(storage_key, delta) - }; - - if is_empty { - self.overlay.set_storage(storage_key.into(), None); + if let Some(child_info) = self.overlay.child_info(storage_key).cloned() { + let (root, is_empty, _) = { + let delta = self.overlay.committed.children.get(storage_key) + .into_iter() + .flat_map(|(map, _)| map.clone().into_iter().map(|(k, v)| (k, v.value))) + .chain(self.overlay.prospective.children.get(storage_key) + .into_iter() + .flat_map(|(map, _)| map.clone().into_iter().map(|(k, v)| (k, v.value)))); + + self.backend.child_storage_root(storage_key, child_info.as_ref(), delta) + }; + + if is_empty { + self.overlay.set_storage(storage_key.into(), None); + } else { + self.overlay.set_storage(storage_key.into(), Some(root.clone())); + } + + trace!(target: "state-trace", "{:04x}: ChildRoot({}) {}", + self.id, + HexDisplay::from(&storage_key.as_ref()), + HexDisplay::from(&root.as_ref()), + ); + root } else { - self.overlay.set_storage(storage_key.into(), Some(root.clone())); + // empty overlay + let root = self + .storage(storage_key.as_ref()) + .unwrap_or( + default_child_trie_root::>(storage_key.as_ref()) + ); + trace!(target: "state-trace", "{:04x}: ChildRoot({}) (no change) {}", + self.id, + HexDisplay::from(&storage_key.as_ref()), + HexDisplay::from(&root.as_ref()), + ); + root } - - trace!(target: "state-trace", "{:04x}: ChildRoot({}) {}", - self.id, - HexDisplay::from(&storage_key.as_ref()), - HexDisplay::from(&root.as_ref()), - ); - root - } } diff --git a/primitives/state-machine/src/lib.rs b/primitives/state-machine/src/lib.rs index 85fa02f7ffe8b..3d21f1d38829e 100644 --- a/primitives/state-machine/src/lib.rs +++ b/primitives/state-machine/src/lib.rs @@ -23,7 +23,7 @@ use log::{warn, trace}; use hash_db::Hasher; use codec::{Decode, Encode}; use primitives::{ - storage::well_known_keys, NativeOrEncoded, NeverNativeValue, offchain::OffchainExt, + storage::{well_known_keys, ChildInfo}, NativeOrEncoded, NeverNativeValue, offchain::OffchainExt, traits::{KeystoreExt, CodeExecutor}, hexdisplay::HexDisplay, hash::H256, }; use overlayed_changes::OverlayedChangeSet; @@ -576,6 +576,7 @@ where pub fn prove_child_read( mut backend: B, storage_key: &[u8], + child_info: ChildInfo, keys: I, ) -> Result> where @@ -587,7 +588,7 @@ where { let trie_backend = backend.as_trie_backend() .ok_or_else(|| Box::new(ExecutionError::UnableToGenerateProof) as Box)?; - prove_child_read_on_trie_backend(trie_backend, storage_key, keys) + prove_child_read_on_trie_backend(trie_backend, storage_key, child_info, keys) } /// Generate storage read proof on pre-created trie backend. @@ -615,6 +616,7 @@ where pub fn prove_child_read_on_trie_backend( trie_backend: &TrieBackend, storage_key: &[u8], + child_info: ChildInfo, keys: I, ) -> Result> where @@ -627,7 +629,7 @@ where let proving_backend = proving_backend::ProvingBackend::<_, H>::new(trie_backend); for key in keys.into_iter() { proving_backend - .child_storage(storage_key, key.as_ref()) + .child_storage(storage_key, child_info, key.as_ref()) .map_err(|e| Box::new(e) as Box)?; } Ok(proving_backend.extract_proof()) @@ -702,7 +704,9 @@ where H: Hasher, H::Out: Ord, { - proving_backend.child_storage(storage_key, key).map_err(|e| Box::new(e) as Box) + // Not a prefixed memory db, using empty unique id and include root resolution. + proving_backend.child_storage(storage_key, ChildInfo::new_default(&[], None), key) + .map_err(|e| Box::new(e) as Box) } /// Sets overlayed changes' changes trie configuration. Returns error if configuration @@ -764,6 +768,8 @@ mod tests { fallback_succeeds: bool, } + const CHILD_INFO_1: ChildInfo<'static> = ChildInfo::new_default(b"unique_id_1", None); + impl CodeExecutor for DummyCodeExecutor { type Error = u8; diff --git a/primitives/state-machine/src/overlayed_changes.rs b/primitives/state-machine/src/overlayed_changes.rs index 73cb8b604a1c2..e98bd2941883a 100644 --- a/primitives/state-machine/src/overlayed_changes.rs +++ b/primitives/state-machine/src/overlayed_changes.rs @@ -21,7 +21,7 @@ use std::iter::FromIterator; use std::collections::{HashMap, BTreeSet}; use codec::Decode; use crate::changes_trie::{NO_EXTRINSIC_INDEX, Configuration as ChangesTrieConfig}; -use primitives::storage::well_known_keys::EXTRINSIC_INDEX; +use primitives::storage::{well_known_keys::EXTRINSIC_INDEX, OwnedChildInfo}; /// The overlayed changes to state to be queried on top of the backend. /// @@ -56,7 +56,7 @@ pub struct OverlayedChangeSet { /// Top level storage changes. pub top: HashMap, OverlayedValue>, /// Child storage changes. - pub children: HashMap, HashMap, OverlayedValue>>, + pub children: HashMap, (HashMap, OverlayedValue>, OwnedChildInfo)>, } #[cfg(test)] @@ -285,10 +285,12 @@ impl OverlayedChanges { .extend(prospective_extrinsics); } } - for (storage_key, mut map) in self.prospective.children.drain() { - let map_dest = self.committed.children.entry(storage_key).or_default(); + for (storage_key, (mut map, child_info)) in self.prospective.children.drain() { + let child_content = self.committed.children.entry(storage_key) + .or_insert_with(|| (Default::default(), child_info)); + // No update to child info at this point (will be needed for deletion). for (key, val) in map.drain() { - let entry = map_dest.entry(key).or_default(); + let entry = child_content.0.entry(key).or_default(); entry.value = val.value; if let Some(prospective_extrinsics) = val.extrinsics { @@ -306,12 +308,12 @@ impl OverlayedChanges { /// Will panic if there are any uncommitted prospective changes. pub fn into_committed(self) -> ( impl Iterator, Option>)>, - impl Iterator, impl Iterator, Option>)>)>, + impl Iterator, (impl Iterator, Option>)>, OwnedChildInfo))>, ){ assert!(self.prospective.is_empty()); (self.committed.top.into_iter().map(|(k, v)| (k, v.value)), self.committed.children.into_iter() - .map(|(sk, v)| (sk, v.into_iter().map(|(k, v)| (k, v.value))))) + .map(|(sk, (v, ci))| (sk, (v.into_iter().map(|(k, v)| (k, v.value)), ci)))) } /// Inserts storage entry responsible for current extrinsic index. @@ -339,6 +341,18 @@ impl OverlayedChanges { false => None, } } + + /// Get child info for a storage key. + /// Take the latest value so prospective first. + pub fn child_info(&self, storage_key: &[u8]) -> Option<&OwnedChildInfo> { + if let Some((_, ci)) = self.prospective.children.get(storage_key) { + return Some(&ci); + } + if let Some((_, ci)) = self.committed.children.get(storage_key) { + return Some(&ci); + } + None + } } #[cfg(test)] diff --git a/primitives/state-machine/src/proving_backend.rs b/primitives/state-machine/src/proving_backend.rs index 2ebd82f0c43c0..7163527086e39 100644 --- a/primitives/state-machine/src/proving_backend.rs +++ b/primitives/state-machine/src/proving_backend.rs @@ -32,6 +32,7 @@ use crate::trie_backend_essence::{Ephemeral, TrieBackendEssence, TrieBackendStor use crate::{Error, ExecutionError, Backend}; use std::collections::{HashMap, HashSet}; use crate::DBValue; +use primitives::storage::ChildInfo; /// Patricia trie-based backend specialized in get value proofs. pub struct ProvingBackendRecorder<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> { @@ -264,12 +265,22 @@ impl<'a, S, H> Backend for ProvingBackend<'a, S, H> self.0.storage(key) } - fn child_storage(&self, storage_key: &[u8], key: &[u8]) -> Result>, Self::Error> { - self.0.child_storage(storage_key, key) + fn child_storage( + &self, + storage_key: &[u8], + child_info: ChildInfo, + key: &[u8], + ) -> Result>, Self::Error> { + self.0.child_storage(storage_key, child_info, key) } - fn for_keys_in_child_storage(&self, storage_key: &[u8], f: F) { - self.0.for_keys_in_child_storage(storage_key, f) + fn for_keys_in_child_storage( + &self, + storage_key: &[u8], + child_info: ChildInfo, + f: F, + ) { + self.0.for_keys_in_child_storage(storage_key, child_info, f) } fn for_keys_with_prefix(&self, prefix: &[u8], f: F) { @@ -280,8 +291,14 @@ impl<'a, S, H> Backend for ProvingBackend<'a, S, H> self.0.for_key_values_with_prefix(prefix, f) } - fn for_child_keys_with_prefix(&self, storage_key: &[u8], prefix: &[u8], f: F) { - self.0.for_child_keys_with_prefix(storage_key, prefix, f) + fn for_child_keys_with_prefix( + &self, + storage_key: &[u8], + child_info: ChildInfo, + prefix: &[u8], + f: F, + ) { + self.0.for_child_keys_with_prefix(storage_key, child_info, prefix, f) } fn pairs(&self) -> Vec<(Vec, Vec)> { @@ -292,8 +309,13 @@ impl<'a, S, H> Backend for ProvingBackend<'a, S, H> self.0.keys(prefix) } - fn child_keys(&self, child_storage_key: &[u8], prefix: &[u8]) -> Vec> { - self.0.child_keys(child_storage_key, prefix) + fn child_keys( + &self, + storage_key: &[u8], + child_info: ChildInfo, + prefix: &[u8], + ) -> Vec> { + self.0.child_keys(storage_key, child_info, prefix) } fn storage_root(&self, delta: I) -> (H::Out, Self::Transaction) @@ -302,12 +324,17 @@ impl<'a, S, H> Backend for ProvingBackend<'a, S, H> self.0.storage_root(delta) } - fn child_storage_root(&self, storage_key: &[u8], delta: I) -> (Vec, bool, Self::Transaction) + fn child_storage_root( + &self, + storage_key: &[u8], + child_info: ChildInfo, + delta: I, + ) -> (Vec, bool, Self::Transaction) where I: IntoIterator, Option>)>, H::Out: Ord { - self.0.child_storage_root(storage_key, delta) + self.0.child_storage_root(storage_key, child_info, delta) } } diff --git a/primitives/state-machine/src/trie_backend.rs b/primitives/state-machine/src/trie_backend.rs index 432ccf3e75f0e..c26306b2b4557 100644 --- a/primitives/state-machine/src/trie_backend.rs +++ b/primitives/state-machine/src/trie_backend.rs @@ -22,6 +22,7 @@ use trie::{Trie, delta_trie_root, default_child_trie_root, child_delta_trie_root use trie::trie_types::{TrieDB, TrieError, Layout}; use crate::trie_backend_essence::{TrieBackendEssence, TrieBackendStorage, Ephemeral}; use crate::Backend; +use primitives::storage::ChildInfo; /// Patricia trie-based backend. Transaction type is an overlay of changes to commit. pub struct TrieBackend, H: Hasher> { @@ -74,7 +75,12 @@ impl, H: Hasher> Backend for TrieBackend where self.essence.storage(key) } - fn child_storage(&self, storage_key: &[u8], key: &[u8]) -> Result>, Self::Error> { + fn child_storage( + &self, + storage_key: &[u8], + child_info: ChildInfo, + key: &[u8], + ) -> Result>, Self::Error> { self.essence.child_storage(storage_key, key) } @@ -86,11 +92,22 @@ impl, H: Hasher> Backend for TrieBackend where self.essence.for_key_values_with_prefix(prefix, f) } - fn for_keys_in_child_storage(&self, storage_key: &[u8], f: F) { + fn for_keys_in_child_storage( + &self, + storage_key: &[u8], + child_info: ChildInfo, + f: F, + ) { self.essence.for_keys_in_child_storage(storage_key, f) } - fn for_child_keys_with_prefix(&self, storage_key: &[u8], prefix: &[u8], f: F) { + fn for_child_keys_with_prefix( + &self, + storage_key: &[u8], + child_info: ChildInfo, + prefix: &[u8], + f: F, + ) { self.essence.for_child_keys_with_prefix(storage_key, prefix, f) } @@ -159,7 +176,12 @@ impl, H: Hasher> Backend for TrieBackend where (root, write_overlay) } - fn child_storage_root(&self, storage_key: &[u8], delta: I) -> (Vec, bool, Self::Transaction) + fn child_storage_root( + &self, + storage_key: &[u8], + child_info: ChildInfo, + delta: I, + ) -> (Vec, bool, Self::Transaction) where I: IntoIterator, Option>)>, H::Out: Ord diff --git a/test/utils/runtime/src/lib.rs b/test/utils/runtime/src/lib.rs index 2ec109938889d..476c674f18e9a 100644 --- a/test/utils/runtime/src/lib.rs +++ b/test/utils/runtime/src/lib.rs @@ -910,12 +910,14 @@ fn test_read_storage() { fn test_read_child_storage() { const CHILD_KEY: &[u8] = b":child_storage:default:read_child_storage"; + const UNIQUE_ID: &[u8] = b":unique_id"; const KEY: &[u8] = b":read_child_storage"; - runtime_io::storage::child_set(CHILD_KEY, KEY, b"test"); + runtime_io::storage::child_set(CHILD_KEY, UNIQUE_ID, KEY, b"test"); let mut v = [0u8; 4]; let r = runtime_io::storage::child_read( CHILD_KEY, + UNIQUE_ID, KEY, &mut v, 0 @@ -924,7 +926,7 @@ fn test_read_child_storage() { assert_eq!(&v, b"test"); let mut v = [0u8; 4]; - let r = runtime_io::storage::child_read(CHILD_KEY, KEY, &mut v, 8); + let r = runtime_io::storage::child_read(CHILD_KEY, UNIQUE_ID, KEY, &mut v, 8); assert_eq!(r, Some(4)); assert_eq!(&v, &[0, 0, 0, 0]); } From a0f3818f4923ceb4cf17a52f8cc308f99ca02020 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 19 Nov 2019 19:33:03 +0100 Subject: [PATCH 02/27] Switch interface, sr-io seems ok, rpc could use similar interface to sr-io, genesis json broken if there is child trie in existing encoding genesis. --- client/api/src/light.rs | 6 +- client/chain-spec/src/chain_spec.rs | 35 ++++++---- client/db/src/lib.rs | 64 +++++++++++++----- client/db/src/storage_cache.rs | 56 ++++++++++++---- .../executor/src/deprecated_host_interface.rs | 65 ++++++++++++++++--- client/finality-grandpa/src/tests.rs | 2 +- client/network/src/chain.rs | 7 +- client/network/src/protocol.rs | 5 +- client/network/src/protocol/light_dispatch.rs | 5 +- client/network/src/protocol/message.rs | 2 + client/rpc/api/src/state/mod.rs | 4 ++ client/rpc/src/state/mod.rs | 18 +++-- client/rpc/src/state/state_full.rs | 27 ++++++-- client/rpc/src/state/state_light.rs | 6 +- client/rpc/src/state/tests.rs | 19 ++++-- client/src/cht.rs | 4 +- client/src/client.rs | 22 ++++--- client/src/in_mem.rs | 4 +- client/src/light/backend.rs | 44 +++++++++---- client/src/light/fetcher.rs | 10 ++- paint/support/src/storage/child.rs | 53 ++++++++++++--- primitives/core/storage/src/lib.rs | 49 +++++++++++++- primitives/sr-io/src/lib.rs | 58 ++++++++++++----- primitives/sr-io/with_std.rs | 11 +--- primitives/sr-primitives/src/lib.rs | 3 +- primitives/state-machine/src/backend.rs | 33 ++++++---- primitives/state-machine/src/basic.rs | 63 +++++++++--------- .../state-machine/src/changes_trie/build.rs | 16 +++-- primitives/state-machine/src/ext.rs | 12 ++-- primitives/state-machine/src/lib.rs | 9 ++- .../state-machine/src/overlayed_changes.rs | 52 ++++++++++----- .../state-machine/src/proving_backend.rs | 28 ++++---- primitives/state-machine/src/testing.rs | 35 +++++----- primitives/state-machine/src/trie_backend.rs | 8 +-- test/utils/client/src/lib.rs | 14 ++-- test/utils/runtime/client/src/lib.rs | 11 ++-- test/utils/runtime/src/genesismap.rs | 19 +++--- test/utils/runtime/src/lib.rs | 21 +++++- 38 files changed, 634 insertions(+), 266 deletions(-) diff --git a/client/api/src/light.rs b/client/api/src/light.rs index b82f8ba745912..4c271167df9fd 100644 --- a/client/api/src/light.rs +++ b/client/api/src/light.rs @@ -26,7 +26,7 @@ use sr_primitives::{ }, generic::BlockId }; -use primitives::{ChangesTrieConfiguration}; +use primitives::ChangesTrieConfiguration; use state_machine::StorageProof; use header_metadata::HeaderMetadata; use crate::{ @@ -86,6 +86,8 @@ pub struct RemoteReadChildRequest { pub header: Header, /// Storage key for child. pub storage_key: Vec, + /// Child trie associated unique id. + pub unique_id: Vec, /// Child storage key to read. pub keys: Vec>, /// Number of times to retry request. None means that default RETRY_COUNT is used. @@ -332,4 +334,4 @@ pub mod tests { not_implemented_in_tests() } } -} \ No newline at end of file +} diff --git a/client/chain-spec/src/chain_spec.rs b/client/chain-spec/src/chain_spec.rs index 0f69654b9e8f6..1dbfbeec554f9 100644 --- a/client/chain-spec/src/chain_spec.rs +++ b/client/chain-spec/src/chain_spec.rs @@ -22,7 +22,7 @@ use std::fs::File; use std::path::PathBuf; use std::rc::Rc; use serde::{Serialize, Deserialize}; -use primitives::storage::{StorageKey, StorageData}; +use primitives::storage::{StorageKey, StorageData, ChildInfo}; use sr_primitives::{BuildStorage, StorageOverlay, ChildrenStorageOverlay}; use serde_json as json; use crate::RuntimeGenesis; @@ -76,10 +76,17 @@ impl<'a, G: RuntimeGenesis, E> BuildStorage for &'a ChainSpec { Genesis::Runtime(gc) => gc.build_storage(), Genesis::Raw(map, children_map) => Ok(( map.into_iter().map(|(k, v)| (k.0, v.0)).collect(), - children_map.into_iter().map(|(sk, map)| ( - sk.0, - map.into_iter().map(|(k, v)| (k.0, v.0)).collect(), - )).collect(), + children_map.into_iter().map(|(sk, child_content)| { + let child_info = ChildInfo::resolve_child_info( + child_content.2, + child_content.1.as_slice(), + ).expect("chainspec contains correct content").to_owned(); + ( + sk.0, + (child_content.0.into_iter().map(|(k, v)| (k.0, v.0)).collect(), + child_info), + ) + }).collect(), )), } } @@ -99,7 +106,7 @@ enum Genesis { Runtime(G), Raw( HashMap, - HashMap>, + HashMap, Vec, u32)>, ), } @@ -259,12 +266,18 @@ impl ChainSpec { .map(|(k, v)| (StorageKey(k), StorageData(v))) .collect(); let children = storage.1.into_iter() - .map(|(sk, child)| ( + .map(|(sk, child)| { + let (info, ci_type) = child.1.as_ref().info(); + ( StorageKey(sk), - child.into_iter() - .map(|(k, v)| (StorageKey(k), StorageData(v))) - .collect(), - )) + ( + child.0.into_iter() + .map(|(k, v)| (StorageKey(k), StorageData(v))) + .collect(), + info, + ci_type, + ), + )}) .collect(); Genesis::Raw(top, children) diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index 90d8aadccc1ae..c6de17356b65b 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -50,7 +50,7 @@ use kvdb::{KeyValueDB, DBTransaction}; use trie::{MemoryDB, PrefixedMemoryDB, prefixed_key}; use parking_lot::{Mutex, RwLock}; use primitives::{H256, Blake2Hasher, ChangesTrieConfiguration, convert_hash, traits::CodeExecutor}; -use primitives::storage::well_known_keys; +use primitives::storage::{well_known_keys, ChildInfo}; use sr_primitives::{ generic::{BlockId, DigestItem}, Justification, StorageOverlay, ChildrenStorageOverlay, BuildStorage, @@ -133,16 +133,26 @@ impl StateBackend for RefTrackingState { self.state.storage_hash(key) } - fn child_storage(&self, storage_key: &[u8], key: &[u8]) -> Result>, Self::Error> { - self.state.child_storage(storage_key, key) + fn child_storage( + &self, + storage_key: &[u8], + child_info: ChildInfo, + key: &[u8], + ) -> Result>, Self::Error> { + self.state.child_storage(storage_key, child_info, key) } fn exists_storage(&self, key: &[u8]) -> Result { self.state.exists_storage(key) } - fn exists_child_storage(&self, storage_key: &[u8], key: &[u8]) -> Result { - self.state.exists_child_storage(storage_key, key) + fn exists_child_storage( + &self, + storage_key: &[u8], + child_info: ChildInfo, + key: &[u8], + ) -> Result { + self.state.exists_child_storage(storage_key, child_info, key) } fn for_keys_with_prefix(&self, prefix: &[u8], f: F) { @@ -153,12 +163,23 @@ impl StateBackend for RefTrackingState { self.state.for_key_values_with_prefix(prefix, f) } - fn for_keys_in_child_storage(&self, storage_key: &[u8], f: F) { - self.state.for_keys_in_child_storage(storage_key, f) + fn for_keys_in_child_storage( + &self, + storage_key: &[u8], + child_info: ChildInfo, + f: F, + ) { + self.state.for_keys_in_child_storage(storage_key, child_info, f) } - fn for_child_keys_with_prefix(&self, storage_key: &[u8], prefix: &[u8], f: F) { - self.state.for_child_keys_with_prefix(storage_key, prefix, f) + fn for_child_keys_with_prefix( + &self, + storage_key: &[u8], + child_info: ChildInfo, + prefix: &[u8], + f: F, + ) { + self.state.for_child_keys_with_prefix(storage_key, child_info, prefix, f) } fn storage_root(&self, delta: I) -> (H256, Self::Transaction) @@ -168,11 +189,16 @@ impl StateBackend for RefTrackingState { self.state.storage_root(delta) } - fn child_storage_root(&self, storage_key: &[u8], delta: I) -> (Vec, bool, Self::Transaction) + fn child_storage_root( + &self, + storage_key: &[u8], + child_info: ChildInfo, + delta: I, + ) -> (Vec, bool, Self::Transaction) where I: IntoIterator, Option>)>, { - self.state.child_storage_root(storage_key, delta) + self.state.child_storage_root(storage_key, child_info, delta) } fn pairs(&self) -> Vec<(Vec, Vec)> { @@ -183,8 +209,13 @@ impl StateBackend for RefTrackingState { self.state.keys(prefix) } - fn child_keys(&self, child_key: &[u8], prefix: &[u8]) -> Vec> { - self.state.child_keys(child_key, prefix) + fn child_keys( + &self, + storage_key: &[u8], + child_info: ChildInfo, + prefix: &[u8], + ) -> Vec> { + self.state.child_keys(storage_key, child_info, prefix) } fn as_trie_backend(&mut self) -> Option<&state_machine::TrieBackend> { @@ -523,8 +554,8 @@ impl client_api::backend::BlockImportOperation } let child_delta = children.into_iter() - .map(|(storage_key, child_overlay)| - (storage_key, child_overlay.into_iter().map(|(k, v)| (k, Some(v))))); + .map(|(storage_key, child_content)| + (storage_key, child_content.0.into_iter().map(|(k, v)| (k, Some(v))), child_content.1)); let (root, transaction) = self.old_state.full_storage_root( top.into_iter().map(|(k, v)| (k, Some(v))), @@ -888,7 +919,8 @@ impl> Backend { }; let mut op = inmem.begin_operation().unwrap(); op.set_block_data(header, body, justification, new_block_state).unwrap(); - op.update_db_storage(state.into_iter().map(|(k, v)| (None, k, Some(v))).collect()).unwrap(); + op.update_db_storage(vec![(None, state.into_iter().map(|(k, v)| (k, Some(v))).collect())]) + .unwrap(); inmem.commit_operation(op).unwrap(); } diff --git a/client/db/src/storage_cache.rs b/client/db/src/storage_cache.rs index 9888dfbe647a5..4111b0aba5467 100644 --- a/client/db/src/storage_cache.rs +++ b/client/db/src/storage_cache.rs @@ -23,6 +23,7 @@ use linked_hash_map::{LinkedHashMap, Entry}; use hash_db::Hasher; use sr_primitives::traits::{Block as BlockT, Header}; use primitives::hexdisplay::HexDisplay; +use primitives::storage::ChildInfo; use state_machine::{backend::Backend as StateBackend, TrieBackend}; use log::trace; use client_api::backend::{StorageCollection, ChildStorageCollection}; @@ -517,7 +518,12 @@ impl, B: BlockT> StateBackend for CachingState< Ok(hash) } - fn child_storage(&self, storage_key: &[u8], key: &[u8]) -> Result>, Self::Error> { + fn child_storage( + &self, + storage_key: &[u8], + child_info: ChildInfo, + key: &[u8], + ) -> Result>, Self::Error> { let key = (storage_key.to_vec(), key.to_vec()); let local_cache = self.cache.local_cache.upgradable_read(); if let Some(entry) = local_cache.child_storage.get(&key).cloned() { @@ -532,7 +538,7 @@ impl, B: BlockT> StateBackend for CachingState< } } trace!("Cache miss: {:?}", key); - let value = self.state.child_storage(storage_key, &key.1[..])?; + let value = self.state.child_storage(storage_key, child_info, &key.1[..])?; RwLockUpgradableReadGuard::upgrade(local_cache).child_storage.insert(key, value.clone()); Ok(value) } @@ -541,12 +547,22 @@ impl, B: BlockT> StateBackend for CachingState< Ok(self.storage(key)?.is_some()) } - fn exists_child_storage(&self, storage_key: &[u8], key: &[u8]) -> Result { - self.state.exists_child_storage(storage_key, key) + fn exists_child_storage( + &self, + storage_key: &[u8], + child_info: ChildInfo, + key: &[u8], + ) -> Result { + self.state.exists_child_storage(storage_key, child_info, key) } - fn for_keys_in_child_storage(&self, storage_key: &[u8], f: F) { - self.state.for_keys_in_child_storage(storage_key, f) + fn for_keys_in_child_storage( + &self, + storage_key: &[u8], + child_info: ChildInfo, + f: F, + ) { + self.state.for_keys_in_child_storage(storage_key, child_info, f) } fn for_keys_with_prefix(&self, prefix: &[u8], f: F) { @@ -557,8 +573,14 @@ impl, B: BlockT> StateBackend for CachingState< self.state.for_key_values_with_prefix(prefix, f) } - fn for_child_keys_with_prefix(&self, storage_key: &[u8], prefix: &[u8], f: F) { - self.state.for_child_keys_with_prefix(storage_key, prefix, f) + fn for_child_keys_with_prefix( + &self, + storage_key: &[u8], + child_info: ChildInfo, + prefix: &[u8], + f: F, + ) { + self.state.for_child_keys_with_prefix(storage_key, child_info, prefix, f) } fn storage_root(&self, delta: I) -> (H::Out, Self::Transaction) @@ -569,12 +591,17 @@ impl, B: BlockT> StateBackend for CachingState< self.state.storage_root(delta) } - fn child_storage_root(&self, storage_key: &[u8], delta: I) -> (Vec, bool, Self::Transaction) + fn child_storage_root( + &self, + storage_key: &[u8], + child_info: ChildInfo, + delta: I, + ) -> (Vec, bool, Self::Transaction) where I: IntoIterator, Option>)>, H::Out: Ord { - self.state.child_storage_root(storage_key, delta) + self.state.child_storage_root(storage_key, child_info, delta) } fn pairs(&self) -> Vec<(Vec, Vec)> { @@ -585,8 +612,13 @@ impl, B: BlockT> StateBackend for CachingState< self.state.keys(prefix) } - fn child_keys(&self, child_key: &[u8], prefix: &[u8]) -> Vec> { - self.state.child_keys(child_key, prefix) + fn child_keys( + &self, + storage_key: &[u8], + child_info: ChildInfo, + prefix: &[u8], + ) -> Vec> { + self.state.child_keys(storage_key, child_info, prefix) } fn as_trie_backend(&mut self) -> Option<&TrieBackend> { diff --git a/client/executor/src/deprecated_host_interface.rs b/client/executor/src/deprecated_host_interface.rs index 223b13367aa87..5182cdfbfe0d7 100644 --- a/client/executor/src/deprecated_host_interface.rs +++ b/client/executor/src/deprecated_host_interface.rs @@ -20,7 +20,7 @@ use codec::Encode; use std::{convert::TryFrom, str}; use primitives::{ blake2_128, blake2_256, twox_64, twox_128, twox_256, ed25519, sr25519, keccak_256, Blake2Hasher, Pair, - crypto::KeyTypeId, offchain, + crypto::KeyTypeId, offchain, storage::ChildType, }; use trie::{TrieConfiguration, trie_types::Layout}; use wasm_interface::{ @@ -232,7 +232,14 @@ impl_wasm_host_interface! { let value = context.read_memory(value_data, value_len) .map_err(|_| "Invalid attempt to determine value in ext_set_child_storage")?; - Ok(runtime_io::storage::child_set(&storage_key, &key, &value)) + // This can only work with archive mode. + Ok(runtime_io::storage::child_set( + &storage_key, + &[], + ChildType::CryptoUniqueId as u32, + &key, + &value, + )) } ext_clear_child_storage( @@ -246,7 +253,13 @@ impl_wasm_host_interface! { let key = context.read_memory(key_data, key_len) .map_err(|_| "Invalid attempt to determine key in ext_clear_child_storage")?; - Ok(runtime_io::storage::child_clear(&storage_key, &key)) + // This can only work with archive mode. + Ok(runtime_io::storage::child_clear( + &storage_key, + &[], + ChildType::CryptoUniqueId as u32, + &key, + )) } ext_clear_storage(key_data: Pointer, key_len: WordSize) { @@ -272,7 +285,13 @@ impl_wasm_host_interface! { let key = context.read_memory(key_data, key_len) .map_err(|_| "Invalid attempt to determine key in ext_exists_child_storage")?; - Ok(if runtime_io::storage::child_exists(&storage_key, &key) { 1 } else { 0 }) + // This can only work with archive mode. + Ok(if runtime_io::storage::child_exists( + &storage_key, + &[], + ChildType::CryptoUniqueId as u32, + &key, + ) { 1 } else { 0 }) } ext_clear_prefix(prefix_data: Pointer, prefix_len: WordSize) { @@ -291,13 +310,24 @@ impl_wasm_host_interface! { .map_err(|_| "Invalid attempt to determine storage_key in ext_clear_child_prefix")?; let prefix = context.read_memory(prefix_data, prefix_len) .map_err(|_| "Invalid attempt to determine prefix in ext_clear_child_prefix")?; - Ok(runtime_io::storage::child_clear_prefix(&storage_key, &prefix)) + // This can only work with archive mode. + Ok(runtime_io::storage::child_clear_prefix( + &storage_key, + &[], + ChildType::CryptoUniqueId as u32, + &prefix, + )) } ext_kill_child_storage(storage_key_data: Pointer, storage_key_len: WordSize) { let storage_key = context.read_memory(storage_key_data, storage_key_len) .map_err(|_| "Invalid attempt to determine storage_key in ext_kill_child_storage")?; - Ok(runtime_io::storage::child_storage_kill(&storage_key)) + // This can only work with archive mode. + Ok(runtime_io::storage::child_storage_kill( + &storage_key, + &[], + ChildType::CryptoUniqueId as u32, + )) } ext_get_allocated_storage( @@ -334,7 +364,13 @@ impl_wasm_host_interface! { let key = context.read_memory(key_data, key_len) .map_err(|_| "Invalid attempt to determine key in ext_get_allocated_child_storage")?; - if let Some(value) = runtime_io::storage::child_get(&storage_key, &key) { + // This can only work with archive mode. + if let Some(value) = runtime_io::storage::child_get( + &storage_key, + &[], + ChildType::CryptoUniqueId as u32, + &key, + ) { let offset = context.allocate_memory(value.len() as u32)?; context.write_memory(offset, &value) .map_err(|_| "Invalid attempt to set memory in ext_get_allocated_child_storage")?; @@ -383,7 +419,13 @@ impl_wasm_host_interface! { let key = context.read_memory(key_data, key_len) .map_err(|_| "Invalid attempt to get key in ext_get_child_storage_into")?; - if let Some(value) = runtime_io::storage::child_get(&storage_key, &key) { + // This can only work with archive mode. + if let Some(value) = runtime_io::storage::child_get( + &storage_key, + &[], + ChildType::CryptoUniqueId as u32, + &key, + ) { let data = &value[value.len().min(value_offset as usize)..]; let written = std::cmp::min(value_len as usize, data.len()); context.write_memory(value_data, &data[..written]) @@ -406,7 +448,12 @@ impl_wasm_host_interface! { ) -> Pointer { let storage_key = context.read_memory(storage_key_data, storage_key_len) .map_err(|_| "Invalid attempt to determine storage_key in ext_child_storage_root")?; - let value = runtime_io::storage::child_root(&storage_key); + // This can only work with archive mode. + let value = runtime_io::storage::child_root( + &storage_key, + &[], + ChildType::CryptoUniqueId as u32, + ); let offset = context.allocate_memory(value.len() as u32)?; context.write_memory(offset, &value) diff --git a/client/finality-grandpa/src/tests.rs b/client/finality-grandpa/src/tests.rs index 6d4d439a9fb34..6c9f1abcdee28 100644 --- a/client/finality-grandpa/src/tests.rs +++ b/client/finality-grandpa/src/tests.rs @@ -290,7 +290,7 @@ impl AuthoritySetForFinalityProver for TestApi { fn prove_authorities(&self, block: &BlockId) -> Result { let authorities = self.authorities(block)?; let backend = >::from(vec![ - (None, b"authorities".to_vec(), Some(authorities.encode())) + (None, vec![(b"authorities".to_vec(), Some(authorities.encode()))]) ]); let proof = prove_read(backend, vec![b"authorities"]) .expect("failure proving read from in-memory storage backend"); diff --git a/client/network/src/chain.rs b/client/network/src/chain.rs index 5207a24bf9d30..11185158a4e50 100644 --- a/client/network/src/chain.rs +++ b/client/network/src/chain.rs @@ -23,7 +23,8 @@ use consensus::{BlockImport, BlockStatus, Error as ConsensusError}; use sr_primitives::traits::{Block as BlockT, Header as HeaderT}; use sr_primitives::generic::{BlockId}; use sr_primitives::Justification; -use primitives::{H256, Blake2Hasher, storage::StorageKey}; +use primitives::{H256, Blake2Hasher}; +use primitives::storage::{StorageKey, ChildInfo}; /// Local client abstraction for the network. pub trait Client: Send + Sync { @@ -57,6 +58,7 @@ pub trait Client: Send + Sync { &self, block: &Block::Hash, storage_key: &[u8], + child_info: ChildInfo, keys: &[Vec], ) -> Result; @@ -135,10 +137,11 @@ impl Client for SubstrateClient where &self, block: &Block::Hash, storage_key: &[u8], + child_info: ChildInfo, keys: &[Vec], ) -> Result { (self as &SubstrateClient) - .read_child_proof(&BlockId::Hash(block.clone()), storage_key, keys) + .read_child_proof(&BlockId::Hash(block.clone()), storage_key, child_info, keys) } fn execution_proof(&self, block: &Block::Hash, method: &str, data: &[u8]) -> Result<(Vec, StorageProof), Error> { diff --git a/client/network/src/protocol.rs b/client/network/src/protocol.rs index 780a071505a65..a1bd5f5adfbc2 100644 --- a/client/network/src/protocol.rs +++ b/client/network/src/protocol.rs @@ -23,7 +23,7 @@ use libp2p::{Multiaddr, PeerId}; use libp2p::core::{ConnectedPoint, nodes::Substream, muxing::StreamMuxerBox}; use libp2p::swarm::{ProtocolsHandler, IntoProtocolsHandler}; use libp2p::swarm::{NetworkBehaviour, NetworkBehaviourAction, PollParameters}; -use primitives::storage::StorageKey; +use primitives::storage::{StorageKey, ChildInfo}; use consensus::{ BlockOrigin, block_validation::BlockAnnounceValidator, @@ -218,12 +218,14 @@ impl<'a, B: BlockT> LightDispatchNetwork for LightDispatchIn<'a> { id: RequestId, block: ::Hash, storage_key: Vec, + unique_id: Vec, keys: Vec>, ) { let message: Message = message::generic::Message::RemoteReadChildRequest(message::RemoteReadChildRequest { id, block, storage_key, + unique_id, keys, }); @@ -1463,6 +1465,7 @@ impl, H: ExHashT> Protocol { let proof = match self.context_data.chain.read_child_proof( &request.block, &request.storage_key, + ChildInfo::new_default(&request.unique_id[..], None), &request.keys, ) { Ok(proof) => proof, diff --git a/client/network/src/protocol/light_dispatch.rs b/client/network/src/protocol/light_dispatch.rs index 280495c0c41b6..e5f35d6016c56 100644 --- a/client/network/src/protocol/light_dispatch.rs +++ b/client/network/src/protocol/light_dispatch.rs @@ -68,6 +68,7 @@ pub trait LightDispatchNetwork { id: RequestId, block: ::Hash, storage_key: Vec, + unique_id: Vec, keys: Vec>, ); @@ -621,6 +622,7 @@ impl Request { self.id, data.block, data.storage_key.clone(), + data.unique_id.clone(), data.keys.clone(), ), RequestData::RemoteCall(ref data, _) => @@ -807,7 +809,7 @@ pub mod tests { fn send_header_request(&mut self, _: &PeerId, _: RequestId, _: <::Header as HeaderT>::Number) {} fn send_read_request(&mut self, _: &PeerId, _: RequestId, _: ::Hash, _: Vec>) {} fn send_read_child_request(&mut self, _: &PeerId, _: RequestId, _: ::Hash, _: Vec, - _: Vec>) {} + _: Vec, _: Vec>) {} fn send_call_request(&mut self, _: &PeerId, _: RequestId, _: ::Hash, _: String, _: Vec) {} fn send_changes_request(&mut self, _: &PeerId, _: RequestId, _: ::Hash, _: ::Hash, _: ::Hash, _: ::Hash, _: Option>, _: Vec) {} @@ -1030,6 +1032,7 @@ pub mod tests { header: dummy_header(), block: Default::default(), storage_key: b":child_storage:sub".to_vec(), + unique_id: b"unique_id_1".to_vec(), keys: vec![b":key".to_vec()], retry_count: None, }, tx)); diff --git a/client/network/src/protocol/message.rs b/client/network/src/protocol/message.rs index 847c03f680d7c..12b86db7caabb 100644 --- a/client/network/src/protocol/message.rs +++ b/client/network/src/protocol/message.rs @@ -368,6 +368,8 @@ pub mod generic { pub block: H, /// Child Storage key. pub storage_key: Vec, + /// Child trie associated unique id. + pub unique_id: Vec, /// Storage key. pub keys: Vec>, } diff --git a/client/rpc/api/src/state/mod.rs b/client/rpc/api/src/state/mod.rs index 0d06092ca1659..deb561b6e3737 100644 --- a/client/rpc/api/src/state/mod.rs +++ b/client/rpc/api/src/state/mod.rs @@ -60,6 +60,7 @@ pub trait StateApi { fn child_storage_keys( &self, child_storage_key: StorageKey, + unique_id: StorageKey, prefix: StorageKey, hash: Option ) -> FutureResult>; @@ -69,6 +70,7 @@ pub trait StateApi { fn child_storage( &self, child_storage_key: StorageKey, + unique_id: StorageKey, key: StorageKey, hash: Option ) -> FutureResult>; @@ -78,6 +80,7 @@ pub trait StateApi { fn child_storage_hash( &self, child_storage_key: StorageKey, + unique_id: StorageKey, key: StorageKey, hash: Option ) -> FutureResult>; @@ -87,6 +90,7 @@ pub trait StateApi { fn child_storage_size( &self, child_storage_key: StorageKey, + unique_id: StorageKey, key: StorageKey, hash: Option ) -> FutureResult>; diff --git a/client/rpc/src/state/mod.rs b/client/rpc/src/state/mod.rs index 8e15c488070fd..62534b89c68e7 100644 --- a/client/rpc/src/state/mod.rs +++ b/client/rpc/src/state/mod.rs @@ -95,6 +95,7 @@ pub trait StateBackend: Send + Sync + 'static &self, block: Option, child_storage_key: StorageKey, + unique_id: StorageKey, prefix: StorageKey, ) -> FutureResult>; @@ -103,6 +104,7 @@ pub trait StateBackend: Send + Sync + 'static &self, block: Option, child_storage_key: StorageKey, + unique_id: StorageKey, key: StorageKey, ) -> FutureResult>; @@ -111,6 +113,7 @@ pub trait StateBackend: Send + Sync + 'static &self, block: Option, child_storage_key: StorageKey, + unique_id: StorageKey, key: StorageKey, ) -> FutureResult>; @@ -119,9 +122,10 @@ pub trait StateBackend: Send + Sync + 'static &self, block: Option, child_storage_key: StorageKey, + unique_id: StorageKey, key: StorageKey, ) -> FutureResult> { - Box::new(self.child_storage(block, child_storage_key, key) + Box::new(self.child_storage(block, child_storage_key, unique_id, key) .map(|x| x.map(|x| x.0.len() as u64))) } @@ -256,37 +260,41 @@ impl StateApi for State fn child_storage( &self, child_storage_key: StorageKey, + unique_id: StorageKey, key: StorageKey, block: Option ) -> FutureResult> { - self.backend.child_storage(block, child_storage_key, key) + self.backend.child_storage(block, child_storage_key, unique_id, key) } fn child_storage_keys( &self, child_storage_key: StorageKey, + unique_id: StorageKey, key_prefix: StorageKey, block: Option ) -> FutureResult> { - self.backend.child_storage_keys(block, child_storage_key, key_prefix) + self.backend.child_storage_keys(block, child_storage_key, unique_id, key_prefix) } fn child_storage_hash( &self, child_storage_key: StorageKey, + unique_id: StorageKey, key: StorageKey, block: Option ) -> FutureResult> { - self.backend.child_storage_hash(block, child_storage_key, key) + self.backend.child_storage_hash(block, child_storage_key, unique_id, key) } fn child_storage_size( &self, child_storage_key: StorageKey, + unique_id: StorageKey, key: StorageKey, block: Option ) -> FutureResult> { - self.backend.child_storage_size(block, child_storage_key, key) + self.backend.child_storage_size(block, child_storage_key, unique_id, key) } fn metadata(&self, block: Option) -> FutureResult { diff --git a/client/rpc/src/state/state_full.rs b/client/rpc/src/state/state_full.rs index 7c4382210ce8f..c9eeb2abe4597 100644 --- a/client/rpc/src/state/state_full.rs +++ b/client/rpc/src/state/state_full.rs @@ -33,7 +33,8 @@ use client::{ Client, CallExecutor, BlockchainEvents, }; use primitives::{ - H256, Blake2Hasher, Bytes, storage::{well_known_keys, StorageKey, StorageData, StorageChangeSet}, + H256, Blake2Hasher, Bytes, + storage::{well_known_keys, StorageKey, StorageData, StorageChangeSet, ChildInfo}, }; use runtime_version::RuntimeVersion; use state_machine::ExecutionStrategy; @@ -294,11 +295,17 @@ impl StateBackend for FullState, child_storage_key: StorageKey, + unique_id: StorageKey, prefix: StorageKey, ) -> FutureResult> { Box::new(result( self.block_or_best(block) - .and_then(|block| self.client.child_storage_keys(&BlockId::Hash(block), &child_storage_key, &prefix)) + .and_then(|block| self.client.child_storage_keys( + &BlockId::Hash(block), + &child_storage_key, + ChildInfo::new_default(&unique_id.0[..], None), + &prefix, + )) .map_err(client_err))) } @@ -306,11 +313,17 @@ impl StateBackend for FullState, child_storage_key: StorageKey, + unique_id: StorageKey, key: StorageKey, ) -> FutureResult> { Box::new(result( self.block_or_best(block) - .and_then(|block| self.client.child_storage(&BlockId::Hash(block), &child_storage_key, &key)) + .and_then(|block| self.client.child_storage( + &BlockId::Hash(block), + &child_storage_key, + ChildInfo::new_default(&unique_id.0[..], None), + &key, + )) .map_err(client_err))) } @@ -318,11 +331,17 @@ impl StateBackend for FullState, child_storage_key: StorageKey, + unique_id: StorageKey, key: StorageKey, ) -> FutureResult> { Box::new(result( self.block_or_best(block) - .and_then(|block| self.client.child_storage_hash(&BlockId::Hash(block), &child_storage_key, &key)) + .and_then(|block| self.client.child_storage_hash( + &BlockId::Hash(block), + &child_storage_key, + ChildInfo::new_default(&unique_id.0[..], None), + &key, + )) .map_err(client_err))) } diff --git a/client/rpc/src/state/state_light.rs b/client/rpc/src/state/state_light.rs index 2c8e59c5b6aac..8b295d514f443 100644 --- a/client/rpc/src/state/state_light.rs +++ b/client/rpc/src/state/state_light.rs @@ -236,6 +236,7 @@ impl StateBackend for LightState, _child_storage_key: StorageKey, + _unique_id: StorageKey, _prefix: StorageKey, ) -> FutureResult> { Box::new(result(Err(client_err(ClientError::NotAvailableOnLightClient)))) @@ -245,6 +246,7 @@ impl StateBackend for LightState, child_storage_key: StorageKey, + unique_id: StorageKey, key: StorageKey, ) -> FutureResult> { let block = self.block_or_best(block); @@ -255,6 +257,7 @@ impl StateBackend for LightState StateBackend for LightState, child_storage_key: StorageKey, + unique_id: StorageKey, key: StorageKey, ) -> FutureResult> { Box::new(self - .child_storage(block, child_storage_key, key) + .child_storage(block, child_storage_key, unique_id, key) .and_then(|maybe_storage| result(Ok(maybe_storage.map(|storage| Blake2Hasher::hash(&storage.0)))) ) diff --git a/client/rpc/src/state/tests.rs b/client/rpc/src/state/tests.rs index 2e3690f3058a2..777f028a5bfe9 100644 --- a/client/rpc/src/state/tests.rs +++ b/client/rpc/src/state/tests.rs @@ -21,7 +21,7 @@ use self::error::Error; use std::sync::Arc; use assert_matches::assert_matches; use futures::stream::Stream; -use primitives::storage::well_known_keys; +use primitives::storage::{well_known_keys, ChildInfo}; use sr_io::hashing::blake2_256; use test_client::{ prelude::*, @@ -29,6 +29,9 @@ use test_client::{ runtime, }; +const CHILD_UUID: &[u8] = b"unique_id"; +const CHILD_INFO: ChildInfo<'static> = ChildInfo::new_default(CHILD_UUID, None); + #[test] fn should_return_storage() { const KEY: &[u8] = b":mock"; @@ -39,12 +42,13 @@ fn should_return_storage() { let mut core = tokio::runtime::Runtime::new().unwrap(); let client = TestClientBuilder::new() .add_extra_storage(KEY.to_vec(), VALUE.to_vec()) - .add_extra_child_storage(STORAGE_KEY.to_vec(), KEY.to_vec(), CHILD_VALUE.to_vec()) + .add_extra_child_storage(STORAGE_KEY.to_vec(), CHILD_INFO, KEY.to_vec(), CHILD_VALUE.to_vec()) .build(); let genesis_hash = client.genesis_hash(); let client = new_full(Arc::new(client), Subscriptions::new(Arc::new(core.executor()))); let key = StorageKey(KEY.to_vec()); let storage_key = StorageKey(STORAGE_KEY.to_vec()); + let unique_id = StorageKey(CHILD_UUID.to_vec()); assert_eq!( client.storage(key.clone(), Some(genesis_hash).into()).wait() @@ -62,7 +66,7 @@ fn should_return_storage() { ); assert_eq!( core.block_on( - client.child_storage(storage_key, key, Some(genesis_hash).into()) + client.child_storage(storage_key, unique_id, key, Some(genesis_hash).into()) .map(|x| x.map(|x| x.0.len())) ).unwrap().unwrap() as usize, CHILD_VALUE.len(), @@ -72,9 +76,10 @@ fn should_return_storage() { #[test] fn should_return_child_storage() { + let unique_id = StorageKey(CHILD_UUID.to_vec()); let core = tokio::runtime::Runtime::new().unwrap(); let client = Arc::new(test_client::TestClientBuilder::new() - .add_child_storage("test", "key", vec![42_u8]) + .add_child_storage("test", "key", CHILD_INFO, vec![42_u8]) .build()); let genesis_hash = client.genesis_hash(); let client = new_full(client, Subscriptions::new(Arc::new(core.executor()))); @@ -83,16 +88,16 @@ fn should_return_child_storage() { assert_matches!( - client.child_storage(child_key.clone(), key.clone(), Some(genesis_hash).into()).wait(), + client.child_storage(child_key.clone(), unique_id.clone(), key.clone(), Some(genesis_hash).into()).wait(), Ok(Some(StorageData(ref d))) if d[0] == 42 && d.len() == 1 ); assert_matches!( - client.child_storage_hash(child_key.clone(), key.clone(), Some(genesis_hash).into()) + client.child_storage_hash(child_key.clone(), unique_id.clone(), key.clone(), Some(genesis_hash).into()) .wait().map(|x| x.is_some()), Ok(true) ); assert_matches!( - client.child_storage_size(child_key.clone(), key.clone(), None).wait(), + client.child_storage_size(child_key.clone(), unique_id.clone(), key.clone(), None).wait(), Ok(Some(1)) ); } diff --git a/client/src/cht.rs b/client/src/cht.rs index 874ea02c2a1a3..a9bd8991601d2 100644 --- a/client/src/cht.rs +++ b/client/src/cht.rs @@ -98,9 +98,9 @@ pub fn build_proof( { let transaction = build_pairs::(cht_size, cht_num, hashes)? .into_iter() - .map(|(k, v)| (None, k, Some(v))) + .map(|(k, v)| (k, Some(v))) .collect::>(); - let mut storage = InMemoryState::::default().update(transaction); + let mut storage = InMemoryState::::default().update(vec![(None, transaction)]); let trie_storage = storage.as_trie_backend() .expect("InMemoryState::as_trie_backend always returns Some; qed"); prove_read_on_trie_backend( diff --git a/client/src/client.rs b/client/src/client.rs index def038fb3e022..aa51fad09a404 100644 --- a/client/src/client.rs +++ b/client/src/client.rs @@ -27,7 +27,7 @@ use codec::{Encode, Decode}; use hash_db::{Hasher, Prefix}; use primitives::{ Blake2Hasher, H256, ChangesTrieConfiguration, convert_hash, NeverNativeValue, ExecutionContext, - NativeOrEncoded, storage::{StorageKey, StorageData, well_known_keys}, + NativeOrEncoded, storage::{StorageKey, StorageData, well_known_keys, ChildInfo}, offchain::{OffchainExt, self}, traits::CodeExecutor, }; use substrate_telemetry::{telemetry, SUBSTRATE_INFO}; @@ -265,10 +265,11 @@ impl Client where &self, id: &BlockId, child_storage_key: &StorageKey, + child_info: ChildInfo, key_prefix: &StorageKey ) -> error::Result> { let keys = self.state_at(id)? - .child_keys(&child_storage_key.0, &key_prefix.0) + .child_keys(&child_storage_key.0, child_info, &key_prefix.0) .into_iter() .map(StorageKey) .collect(); @@ -279,11 +280,13 @@ impl Client where pub fn child_storage( &self, id: &BlockId, - child_storage_key: &StorageKey, + storage_key: &StorageKey, + child_info: ChildInfo, key: &StorageKey ) -> error::Result> { Ok(self.state_at(id)? - .child_storage(&child_storage_key.0, &key.0).map_err(|e| error::Error::from_state(Box::new(e)))? + .child_storage(&storage_key.0, child_info, &key.0) + .map_err(|e| error::Error::from_state(Box::new(e)))? .map(StorageData)) } @@ -291,11 +294,13 @@ impl Client where pub fn child_storage_hash( &self, id: &BlockId, - child_storage_key: &StorageKey, + storage_key: &StorageKey, + child_info: ChildInfo, key: &StorageKey ) -> error::Result> { Ok(self.state_at(id)? - .child_storage_hash(&child_storage_key.0, &key.0).map_err(|e| error::Error::from_state(Box::new(e)))? + .child_storage_hash(&storage_key.0, child_info, &key.0) + .map_err(|e| error::Error::from_state(Box::new(e)))? ) } @@ -332,13 +337,14 @@ impl Client where &self, id: &BlockId, storage_key: &[u8], + child_info: ChildInfo, keys: I, ) -> error::Result where I: IntoIterator, I::Item: AsRef<[u8]>, { self.state_at(id) - .and_then(|state| prove_child_read(state, storage_key, keys) + .and_then(|state| prove_child_read(state, storage_key, child_info, keys) .map_err(Into::into)) } @@ -1004,7 +1010,7 @@ impl Client where overlay.commit_prospective(); let (top, children) = overlay.into_committed(); - let children = children.map(|(sk, it)| (sk, it.collect())).collect(); + let children = children.map(|(sk, it)| (sk, it.0.collect())).collect(); if import_headers.post().state_root() != &storage_update.1 { return Err(error::Error::InvalidStateRoot); } diff --git a/client/src/in_mem.rs b/client/src/in_mem.rs index 4c74943189020..bddb2e9c3bb8a 100644 --- a/client/src/in_mem.rs +++ b/client/src/in_mem.rs @@ -510,8 +510,8 @@ where check_genesis_storage(&top, &children)?; let child_delta = children.into_iter() - .map(|(storage_key, child_overlay)| - (storage_key, child_overlay.into_iter().map(|(k, v)| (k, Some(v))))); + .map(|(storage_key, child_content)| + (storage_key, child_content.0.into_iter().map(|(k, v)| (k, Some(v))), child_content.1)); let (root, transaction) = self.old_state.full_storage_root( top.into_iter().map(|(k, v)| (k, Some(v))), diff --git a/client/src/light/backend.rs b/client/src/light/backend.rs index ec4543191896e..068fd9bc9f45e 100644 --- a/client/src/light/backend.rs +++ b/client/src/light/backend.rs @@ -21,6 +21,7 @@ use std::collections::HashMap; use std::sync::Arc; use parking_lot::{RwLock, Mutex}; +use primitives::storage::{ChildInfo, OwnedChildInfo}; use sr_primitives::{generic::BlockId, Justification, StorageOverlay, ChildrenStorageOverlay}; use state_machine::{Backend as StateBackend, TrieBackend, backend::InMemory as InMemoryState, ChangesTrieTransaction}; use sr_primitives::traits::{Block as BlockT, NumberFor, Zero, Header}; @@ -284,18 +285,17 @@ where check_genesis_storage(&top, &children)?; // this is only called when genesis block is imported => shouldn't be performance bottleneck - let mut storage: HashMap>, StorageOverlay> = HashMap::new(); + let mut storage: HashMap, OwnedChildInfo)>, StorageOverlay> = HashMap::new(); storage.insert(None, top); // create a list of children keys to re-compute roots for - let child_delta = children.keys() - .cloned() - .map(|storage_key| (storage_key, None)) + let child_delta = children.iter() + .map(|(storage_key, (_m, child_info))| (storage_key.clone(), None, child_info.clone())) .collect::>(); // make sure to persist the child storage - for (child_key, child_storage) in children { - storage.insert(Some(child_key), child_storage); + for (child_key, (child_storage, child_info)) in children { + storage.insert(Some((child_key, child_info)), child_storage); } let storage_update: InMemoryState = storage.into(); @@ -357,10 +357,15 @@ impl StateBackend for GenesisOrUnavailableState } } - fn child_storage(&self, storage_key: &[u8], key: &[u8]) -> ClientResult>> { + fn child_storage( + &self, + storage_key: &[u8], + child_info: ChildInfo, + key: &[u8], + ) -> ClientResult>> { match *self { GenesisOrUnavailableState::Genesis(ref state) => - Ok(state.child_storage(storage_key, key).expect(IN_MEMORY_EXPECT_PROOF)), + Ok(state.child_storage(storage_key, child_info, key).expect(IN_MEMORY_EXPECT_PROOF)), GenesisOrUnavailableState::Unavailable => Err(ClientError::NotAvailableOnLightClient), } } @@ -379,10 +384,15 @@ impl StateBackend for GenesisOrUnavailableState } } - - fn for_keys_in_child_storage(&self, storage_key: &[u8], action: A) { + fn for_keys_in_child_storage( + &self, + storage_key: &[u8], + child_info: ChildInfo, + action: A, + ) { match *self { - GenesisOrUnavailableState::Genesis(ref state) => state.for_keys_in_child_storage(storage_key, action), + GenesisOrUnavailableState::Genesis(ref state) => + state.for_keys_in_child_storage(storage_key, child_info, action), GenesisOrUnavailableState::Unavailable => (), } } @@ -390,12 +400,13 @@ impl StateBackend for GenesisOrUnavailableState fn for_child_keys_with_prefix( &self, storage_key: &[u8], + child_info: ChildInfo, prefix: &[u8], action: A, ) { match *self { GenesisOrUnavailableState::Genesis(ref state) => - state.for_child_keys_with_prefix(storage_key, prefix, action), + state.for_child_keys_with_prefix(storage_key, child_info, prefix, action), GenesisOrUnavailableState::Unavailable => (), } } @@ -411,13 +422,18 @@ impl StateBackend for GenesisOrUnavailableState } } - fn child_storage_root(&self, key: &[u8], delta: I) -> (Vec, bool, Self::Transaction) + fn child_storage_root( + &self, + storage_key: &[u8], + child_info: ChildInfo, + delta: I, + ) -> (Vec, bool, Self::Transaction) where I: IntoIterator, Option>)> { match *self { GenesisOrUnavailableState::Genesis(ref state) => { - let (root, is_equal, _) = state.child_storage_root(key, delta); + let (root, is_equal, _) = state.child_storage_root(storage_key, child_info, delta); (root, is_equal, ()) }, GenesisOrUnavailableState::Unavailable => (H::Out::default().as_ref().to_vec(), true, ()), diff --git a/client/src/light/fetcher.rs b/client/src/light/fetcher.rs index e8b88e93b9bab..788f614ffb85a 100644 --- a/client/src/light/fetcher.rs +++ b/client/src/light/fetcher.rs @@ -343,11 +343,14 @@ pub mod tests { use crate::light::fetcher::{FetchChecker, LightDataChecker, RemoteHeaderRequest}; use crate::light::blockchain::tests::{DummyStorage, DummyBlockchain}; use primitives::{blake2_256, Blake2Hasher, H256}; - use primitives::storage::{well_known_keys, StorageKey}; + use primitives::storage::{well_known_keys, StorageKey, ChildInfo}; use sr_primitives::generic::BlockId; use state_machine::Backend; use super::*; + const CHILD_UUID_1: &'static [u8] = b"unique_id_1"; + const CHILD_INFO_1: ChildInfo<'static> = ChildInfo::new_default(CHILD_UUID_1, None); + type TestChecker = LightDataChecker< NativeExecutor, Blake2Hasher, @@ -398,7 +401,7 @@ pub mod tests { use test_client::TestClientBuilderExt; // prepare remote client let remote_client = test_client::TestClientBuilder::new() - .add_extra_child_storage(b":child_storage:default:child1".to_vec(), b"key1".to_vec(), b"value1".to_vec()) + .add_extra_child_storage(b":child_storage:default:child1".to_vec(), CHILD_INFO_1, b"key1".to_vec(), b"value1".to_vec()) .build(); let remote_block_id = BlockId::Number(0); let remote_block_hash = remote_client.block_hash(0).unwrap().unwrap(); @@ -410,12 +413,14 @@ pub mod tests { let child_value = remote_client.child_storage( &remote_block_id, &StorageKey(b":child_storage:default:child1".to_vec()), + CHILD_INFO_1, &StorageKey(b"key1".to_vec()), ).unwrap().unwrap().0; assert_eq!(b"value1"[..], child_value[..]); let remote_read_proof = remote_client.read_child_proof( &remote_block_id, b":child_storage:default:child1", + CHILD_INFO_1, &[b"key1"], ).unwrap(); @@ -496,6 +501,7 @@ pub mod tests { block: remote_block_header.hash(), header: remote_block_header, storage_key: b":child_storage:default:child1".to_vec(), + unique_id: CHILD_UUID_1.to_vec(), keys: vec![b"key1".to_vec()], retry_count: None, }, diff --git a/paint/support/src/storage/child.rs b/paint/support/src/storage/child.rs index e1aeed4becef7..466f860cbfe85 100644 --- a/paint/support/src/storage/child.rs +++ b/paint/support/src/storage/child.rs @@ -43,7 +43,12 @@ pub fn get( unique_id: Option<&[u8]>, key: &[u8], ) -> Option { - runtime_io::storage::child_get(storage_key, unique_id.expect(SOME), key).map(|v| { + runtime_io::storage::child_get( + storage_key, + unique_id.expect(SOME), + primitives::storage::ChildType::CryptoUniqueId as u32, + key, + ).map(|v| { Decode::decode(&mut &v[..]).expect("storage is not null, therefore must be a valid type") }) } @@ -88,7 +93,13 @@ pub fn put( value: &T, ) { value.using_encoded(|slice| - runtime_io::storage::child_set(storage_key, unique_id.expect(SOME), key, slice) + runtime_io::storage::child_set( + storage_key, + unique_id.expect(SOME), + primitives::storage::ChildType::CryptoUniqueId as u32, + key, + slice, + ) ); } @@ -144,7 +155,9 @@ pub fn exists( key: &[u8], ) -> bool { runtime_io::storage::child_read( - storage_key, unique_id.expect(SOME), key, &mut [0;0][..], 0, + storage_key, unique_id.expect(SOME), + primitives::storage::ChildType::CryptoUniqueId as u32, + key, &mut [0;0][..], 0, ).is_some() } @@ -153,7 +166,11 @@ pub fn kill_storage( storage_key: &[u8], unique_id: Option<&[u8]>, ) { - runtime_io::storage::child_storage_kill(storage_key, unique_id.expect(SOME)) + runtime_io::storage::child_storage_kill( + storage_key, + unique_id.expect(SOME), + primitives::storage::ChildType::CryptoUniqueId as u32, + ) } /// Ensure `key` has no explicit entry in storage. @@ -162,7 +179,12 @@ pub fn kill( unique_id: Option<&[u8]>, key: &[u8], ) { - runtime_io::storage::child_clear(storage_key, unique_id.expect(SOME), key); + runtime_io::storage::child_clear( + storage_key, + unique_id.expect(SOME), + primitives::storage::ChildType::CryptoUniqueId as u32, + key, + ); } /// Get a Vec of bytes from storage. @@ -171,7 +193,12 @@ pub fn get_raw( unique_id: Option<&[u8]>, key: &[u8], ) -> Option> { - runtime_io::storage::child_get(storage_key, unique_id.expect(SOME), key) + runtime_io::storage::child_get( + storage_key, + unique_id.expect(SOME), + primitives::storage::ChildType::CryptoUniqueId as u32, + key, + ) } /// Put a raw byte slice into storage. @@ -181,7 +208,13 @@ pub fn put_raw( key: &[u8], value: &[u8], ) { - runtime_io::storage::child_set(storage_key, unique_id.expect(SOME), key, value) + runtime_io::storage::child_set( + storage_key, + unique_id.expect(SOME), + primitives::storage::ChildType::CryptoUniqueId as u32, + key, + value, + ) } /// Calculate current child root value. @@ -189,5 +222,9 @@ pub fn child_root( storage_key: &[u8], unique_id: Option<&[u8]>, ) -> Vec { - runtime_io::storage::child_root(storage_key, unique_id.expect(SOME)) + runtime_io::storage::child_root( + storage_key, + unique_id.expect(SOME), + primitives::storage::ChildType::CryptoUniqueId as u32, + ) } diff --git a/primitives/core/storage/src/lib.rs b/primitives/core/storage/src/lib.rs index 08f2005428503..94d37a2593c34 100644 --- a/primitives/core/storage/src/lib.rs +++ b/primitives/core/storage/src/lib.rs @@ -46,7 +46,10 @@ pub type StorageOverlay = std::collections::HashMap, Vec>; /// A set of key value pairs for children storage; #[cfg(feature = "std")] -pub type ChildrenStorageOverlay = std::collections::HashMap, StorageOverlay>; +pub type ChildrenStorageOverlay = std::collections::HashMap< + Vec, + (StorageOverlay, OwnedChildInfo), +>; /// Storage change set #[derive(RuntimeDebug)] @@ -157,6 +160,7 @@ impl<'a> ChildStorageKey<'a> { } } +#[derive(Clone, Copy)] /// Information related to a child trie query. pub enum ChildInfo<'a> { Default(ChildTrie<'a>), @@ -189,6 +193,48 @@ impl<'a> ChildInfo<'a> { } } + /// Create child info from a linear byte encoded value and a type. + pub fn resolve_child_info(child_type: u32, data: &'a[u8]) -> Option { + match child_type { + x if x == ChildType::CryptoUniqueId as u32 => Some(ChildInfo::new_default(data, None)), + x if x == ChildType::CryptoUniqueIdRoot32Api as u32 => if data.len() >= 32 { + Some(ChildInfo::new_default(&data[32..], Some(&data[..32]))) + } else { None }, + _ => None, + } + } + + /// Return a single byte vector containing child info content and this child info type. + /// This can be use as input for `resolve_child_info`. + pub fn info(&self) -> (Vec, u32) { + match self { + ChildInfo::Default(ChildTrie { + root, + unique_id, + }) => { + if let Some(root) = root { + if root.len() == 32 { + let mut buf = root.to_vec(); + buf.extend_from_slice(unique_id); + return (buf, ChildType::CryptoUniqueIdRoot32Api as u32); + } + } + (unique_id.to_vec(), ChildType::CryptoUniqueId as u32) + }, + } + } + +} + +/// Type of child, this can be different child usage +/// of api variant, it is not strictly speaking different +/// child kind. +#[repr(u32)] +pub enum ChildType { + /// Default, it uses a cryptographic strong unique id as input. + CryptoUniqueId = 1, + /// Default, but with root send when querying. Root is a 32 byte hash. + CryptoUniqueIdRoot32Api = 2, } impl OwnedChildInfo { @@ -216,6 +262,7 @@ impl OwnedChildInfo { /// It share its trie node storage with any kind of key, /// and its unique id needs to be collision free (eg strong /// crypto hash). +#[derive(Clone, Copy)] pub struct ChildTrie<'a> { /// If root was fetch it can be memoïzed in this field /// to avoid querying it explicitly. diff --git a/primitives/sr-io/src/lib.rs b/primitives/sr-io/src/lib.rs index a3be958a2a720..da190fac01e3f 100644 --- a/primitives/sr-io/src/lib.rs +++ b/primitives/sr-io/src/lib.rs @@ -36,7 +36,7 @@ use rstd::ops::Deref; #[cfg(feature = "std")] use primitives::{ crypto::Pair, traits::KeystoreExt, offchain::OffchainExt, hexdisplay::HexDisplay, - storage::ChildStorageKey, + storage::ChildStorageKey, storage::ChildInfo, }; use primitives::{ @@ -93,11 +93,14 @@ pub trait Storage { fn child_get( &self, child_storage_key: &[u8], - unique_id: &[u8], + child_definition: &[u8], + child_type: u32, key: &[u8], ) -> Option> { let storage_key = child_storage_key_or_panic(child_storage_key); - self.child_storage(storage_key, unique_id, key).map(|s| s.to_vec()) + let child_info = ChildInfo::resolve_child_info(child_type, child_definition) + .expect("Invalid child definition"); + self.child_storage(storage_key, child_info, key).map(|s| s.to_vec()) } /// Get `key` from storage, placing the value into `value_out` and return the number of @@ -123,13 +126,16 @@ pub trait Storage { fn child_read( &self, child_storage_key: &[u8], - unique_id: &[u8], + child_definition: &[u8], + child_type: u32, key: &[u8], value_out: &mut [u8], value_offset: u32, ) -> Option { let storage_key = child_storage_key_or_panic(child_storage_key); - self.child_storage(storage_key, unique_id, key) + let child_info = ChildInfo::resolve_child_info(child_type, child_definition) + .expect("Invalid child definition"); + self.child_storage(storage_key, child_info, key) .map(|value| { let value_offset = value_offset as usize; let data = &value[value_offset.min(value.len())..]; @@ -149,12 +155,15 @@ pub trait Storage { fn child_set( &mut self, child_storage_key: &[u8], - unique_id: &[u8], + child_definition: &[u8], + child_type: u32, key: &[u8], value: &[u8], ) { let storage_key = child_storage_key_or_panic(child_storage_key); - self.set_child_storage(storage_key, unique_id, key.to_vec(), value.to_vec()); + let child_info = ChildInfo::resolve_child_info(child_type, child_definition) + .expect("Invalid child definition"); + self.set_child_storage(storage_key, child_info, key.to_vec(), value.to_vec()); } /// Clear the storage of the given `key` and its value. @@ -166,21 +175,27 @@ pub trait Storage { fn child_clear( &mut self, child_storage_key: &[u8], - unique_id: &[u8], + child_definition: &[u8], + child_type: u32, key: &[u8], ) { let storage_key = child_storage_key_or_panic(child_storage_key); - self.clear_child_storage(storage_key, unique_id, key); + let child_info = ChildInfo::resolve_child_info(child_type, child_definition) + .expect("Invalid child definition"); + self.clear_child_storage(storage_key, child_info, key); } /// Clear an entire child storage. fn child_storage_kill( &mut self, child_storage_key: &[u8], - unique_id: &[u8], + child_definition: &[u8], + child_type: u32, ) { let storage_key = child_storage_key_or_panic(child_storage_key); - self.kill_child_storage(storage_key, unique_id); + let child_info = ChildInfo::resolve_child_info(child_type, child_definition) + .expect("Invalid child definition"); + self.kill_child_storage(storage_key, child_info); } /// Check whether the given `key` exists in storage. @@ -192,11 +207,14 @@ pub trait Storage { fn child_exists( &self, child_storage_key: &[u8], - unique_id: &[u8], + child_definition: &[u8], + child_type: u32, key: &[u8], ) -> bool { let storage_key = child_storage_key_or_panic(child_storage_key); - self.exists_child_storage(storage_key, unique_id, key) + let child_info = ChildInfo::resolve_child_info(child_type, child_definition) + .expect("Invalid child definition"); + self.exists_child_storage(storage_key, child_info, key) } /// Clear the storage of each key-value pair where the key starts with the given `prefix`. @@ -208,11 +226,14 @@ pub trait Storage { fn child_clear_prefix( &mut self, child_storage_key: &[u8], - unique_id: &[u8], + child_definition: &[u8], + child_type: u32, prefix: &[u8], ) { let storage_key = child_storage_key_or_panic(child_storage_key); - self.clear_child_prefix(storage_key, unique_id, prefix); + let child_info = ChildInfo::resolve_child_info(child_type, child_definition) + .expect("Invalid child definition"); + self.clear_child_prefix(storage_key, child_info, prefix); } /// "Commit" all existing operations and compute the resulting storage root. @@ -224,10 +245,13 @@ pub trait Storage { fn child_root( &mut self, child_storage_key: &[u8], - unique_id: &[u8], + child_definition: &[u8], + child_type: u32, ) -> Vec { let storage_key = child_storage_key_or_panic(child_storage_key); - self.child_storage_root(storage_key, unique_id) + let child_info = ChildInfo::resolve_child_info(child_type, child_definition) + .expect("Invalid child definition"); + self.child_storage_root(storage_key, child_info) } /// "Commit" all existing operations and get the resulting storage change root. diff --git a/primitives/sr-io/with_std.rs b/primitives/sr-io/with_std.rs index 72a4f1951d912..403cced74b673 100644 --- a/primitives/sr-io/with_std.rs +++ b/primitives/sr-io/with_std.rs @@ -16,8 +16,8 @@ use primitives::{ blake2_128, blake2_256, twox_128, twox_256, twox_64, ed25519, Blake2Hasher, sr25519, Pair, H256, - traits::KeystoreExt, storage::ChildStorageKey, hexdisplay::HexDisplay, Hasher, - offchain::{self, OffchainExt}, storage::OwnedChildInfo, + traits::KeystoreExt, hexdisplay::HexDisplay, Hasher, offchain::{self, OffchainExt}, + storage::{ChildStorageKey, OwnedChildInfo, StorageOverlay, ChildrenStorageOverlay}, }; // Switch to this after PoC-3 // pub use primitives::BlakeHasher; @@ -476,13 +476,6 @@ impl OffchainApi for () { impl Api for () {} -/// A set of key value pairs for storage. -pub type StorageOverlay = HashMap, Vec>; - -/// Sets of key value pairs for children storage, stored at parent key location, -/// with an associated unique id. -pub type ChildrenStorageOverlay = HashMap, (StorageOverlay, OwnedChildInfo)>; - /// Execute the given closure with global functions available whose functionality routes into /// externalities that draw from and populate `storage` and `children_storage`. /// Forwards the value that the closure returns. diff --git a/primitives/sr-primitives/src/lib.rs b/primitives/sr-primitives/src/lib.rs index 01ace2c004d0b..e7bd563c66ca7 100644 --- a/primitives/sr-primitives/src/lib.rs +++ b/primitives/sr-primitives/src/lib.rs @@ -151,7 +151,8 @@ impl BuildStorage for (StorageOverlay, ChildrenStorageOverlay) { for (k, other_map) in self.1.iter() { let k = k.clone(); if let Some(map) = storage.1.get_mut(&k) { - map.extend(other_map.iter().map(|(k, v)| (k.clone(), v.clone()))); + map.0.extend(other_map.0.iter().map(|(k, v)| (k.clone(), v.clone()))); + debug_assert!(map.1 == other_map.1, "No child change implemented"); } else { storage.1.insert(k, other_map.clone()); } diff --git a/primitives/state-machine/src/backend.rs b/primitives/state-machine/src/backend.rs index 9952a4ef74983..5a5de985a0e6e 100644 --- a/primitives/state-machine/src/backend.rs +++ b/primitives/state-machine/src/backend.rs @@ -27,6 +27,12 @@ use trie::{ }; use primitives::storage::{ChildInfo, OwnedChildInfo}; + +pub(crate) type StorageTuple = ( + HashMap, Vec>, + HashMap, (HashMap, Vec>, OwnedChildInfo)>, +); + /// A state backend is used to read state data and can have changes committed /// to it. /// @@ -353,7 +359,7 @@ impl InMemory { pub fn update(&self, changes: >::Transaction) -> Self { let mut inner: HashMap<_, _> = self.inner.clone(); for (child_info, key_values) in changes { - let mut entry = inner.entry(child_info).or_default(); + let entry = inner.entry(child_info).or_default(); for (key, val) in key_values { match val { Some(v) => { entry.insert(key, v); }, @@ -375,14 +381,8 @@ impl From, OwnedChildInfo)>, HashMap, } } -impl From<( - HashMap, Vec>, - HashMap, (HashMap, Vec>, OwnedChildInfo)>, -)> for InMemory { - fn from(inners: ( - HashMap, Vec>, - HashMap, (HashMap, Vec>, OwnedChildInfo)>, - )) -> Self { +impl From for InMemory { + fn from(inners: StorageTuple) -> Self { let mut inner: HashMap, OwnedChildInfo)>, HashMap, Vec>> = inners.1.into_iter().map(|(k, (v, ci))| (Some((k, ci)), v)).collect(); inner.insert(None, inners.0); @@ -413,7 +413,7 @@ impl From, OwnedChildInfo)>, Vec<(Vec, Optio ) -> Self { let mut expanded: HashMap, OwnedChildInfo)>, HashMap, Vec>> = HashMap::new(); for (child_info, key_values) in inner { - let mut entry = expanded.entry(child_info).or_default(); + let entry = expanded.entry(child_info).or_default(); for (key, value) in key_values { if let Some(value) = value { entry.insert(key, value); @@ -575,7 +575,7 @@ impl Backend for InMemory { let mut new_child_roots = Vec::new(); let mut root_map = None; for (child_info, map) in &self.inner { - if let Some((storage_key, child_info)) = child_info.as_ref() { + if let Some((storage_key, _child_info)) = child_info.as_ref() { let ch = insert_into_memory_db::(&mut mdb, map.clone().into_iter())?; new_child_roots.push((storage_key.clone(), ch.as_ref().into())); } else { @@ -620,16 +620,21 @@ pub(crate) fn insert_into_memory_db(mdb: &mut MemoryDB, input: I) -> Op #[cfg(test)] mod tests { use super::*; - + /// Assert in memory backend with only child trie keys works as trie backend. #[test] fn in_memory_with_child_trie_only() { let storage = InMemory::::default(); + let child_info = OwnedChildInfo::new_default(b"unique_id_1".to_vec(), None); let mut storage = storage.update( - vec![(Some(b"1".to_vec()), b"2".to_vec(), Some(b"3".to_vec()))] + vec![( + Some((b"1".to_vec(), child_info.clone())), + vec![(b"2".to_vec(), Some(b"3".to_vec()))] + )] ); let trie_backend = storage.as_trie_backend().unwrap(); - assert_eq!(trie_backend.child_storage(b"1", b"2").unwrap(), Some(b"3".to_vec())); + assert_eq!(trie_backend.child_storage(b"1", child_info.as_ref(), b"2").unwrap(), + Some(b"3".to_vec())); assert!(trie_backend.storage(b"1").unwrap().is_some()); } } diff --git a/primitives/state-machine/src/basic.rs b/primitives/state-machine/src/basic.rs index 8d2cc08aba4ae..a1c81c47c71e4 100644 --- a/primitives/state-machine/src/basic.rs +++ b/primitives/state-machine/src/basic.rs @@ -17,14 +17,14 @@ //! Basic implementation for Externalities. use std::{collections::HashMap, any::{TypeId, Any}, iter::FromIterator}; -use crate::backend::{Backend, InMemory}; +use crate::backend::{Backend, InMemory, StorageTuple}; use hash_db::Hasher; use trie::{TrieConfiguration, default_child_trie_root}; use trie::trie_types::Layout; use primitives::{ storage::{ well_known_keys::is_child_storage_key, ChildStorageKey, StorageOverlay, - ChildrenStorageOverlay, ChildInfo, + ChildInfo, ChildrenStorageOverlay, }, traits::Externalities, Blake2Hasher, hash::H256, }; @@ -52,10 +52,7 @@ impl BasicExternalities { } /// Consume self and returns inner storages - pub fn into_storages(self) -> ( - HashMap, Vec>, - HashMap, HashMap, Vec>>, - ) { + pub fn into_storages(self) -> StorageTuple { (self.top, self.children) } @@ -133,10 +130,10 @@ impl Externalities for BasicExternalities { fn child_storage( &self, storage_key: ChildStorageKey, - child_info: ChildInfo, + _child_info: ChildInfo, key: &[u8], ) -> Option> { - self.children.get(storage_key.as_ref()).and_then(|child| child.get(key)).cloned() + self.children.get(storage_key.as_ref()).and_then(|child| child.0.get(key)).cloned() } fn child_storage_hash( @@ -185,18 +182,19 @@ impl Externalities for BasicExternalities { key: Vec, value: Option>, ) { - let child_map = self.children.entry(storage_key.into_owned()).or_default(); + let child_map = self.children.entry(storage_key.into_owned()) + .or_insert_with(|| (Default::default(), child_info.to_owned())); if let Some(value) = value { - child_map.insert(key, value); + child_map.0.insert(key, value); } else { - child_map.remove(&key); + child_map.0.remove(&key); } } fn kill_child_storage( &mut self, storage_key: ChildStorageKey, - child_info: ChildInfo, + _child_info: ChildInfo, ) { self.children.remove(storage_key.as_ref()); } @@ -216,11 +214,11 @@ impl Externalities for BasicExternalities { fn clear_child_prefix( &mut self, storage_key: ChildStorageKey, - child_info: ChildInfo, + _child_info: ChildInfo, prefix: &[u8], ) { if let Some(child) = self.children.get_mut(storage_key.as_ref()) { - child.retain(|key, _| !key.starts_with(prefix)); + child.0.retain(|key, _| !key.starts_with(prefix)); } } @@ -228,20 +226,23 @@ impl Externalities for BasicExternalities { fn storage_root(&mut self) -> H256 { let mut top = self.top.clone(); + let keys: Vec<_> = self.children.keys().map(|k| k.to_vec()).collect(); // Single child trie implementation currently allows using the same child // empty root for all child trie. Using null storage key until multiple // type of child trie support. let empty_hash = default_child_trie_root::>(&[]); - for (storage_key, (_values, child_info)) in self.children.iter() { + for storage_key in keys { + let child_info = self.children.get(storage_key.as_slice()).as_ref() + .expect("key fetch above").1.to_owned(); let child_root = self.child_storage_root( ChildStorageKey::from_slice(storage_key.as_slice()) .expect("Map only feed by valid keys; qed"), - Some(&child_info[..]), + child_info.as_ref(), ); if &empty_hash[..] == &child_root[..] { - top.remove(&storage_key); + top.remove(storage_key.as_slice()); } else { - top.insert(storage_key, child_root); + top.insert(storage_key.clone(), child_root); } } @@ -251,13 +252,13 @@ impl Externalities for BasicExternalities { fn child_storage_root( &mut self, storage_key: ChildStorageKey, - child_info: ChildInfo, + _child_info: ChildInfo, ) -> Vec { if let Some(child) = self.children.get(storage_key.as_ref()) { - let delta = child.clone().into_iter().map(|(k, v)| (k, Some(v))); + let delta = child.0.clone().into_iter().map(|(k, v)| (k, Some(v))); InMemory::::default() - .child_storage_root(storage_key.as_ref(), child_info, delta).0 + .child_storage_root(storage_key.as_ref(), child.1.as_ref(), delta).0 } else { default_child_trie_root::>(storage_key.as_ref()) } @@ -282,6 +283,8 @@ mod tests { use primitives::storage::well_known_keys::CODE; use hex_literal::hex; + const CHILD_INFO_1: ChildInfo<'static> = ChildInfo::new_default(b"unique_id_1", None); + #[test] fn commit_should_work() { let mut ext = BasicExternalities::default(); @@ -310,24 +313,24 @@ mod tests { let mut ext = BasicExternalities::new( Default::default(), map![ - child_storage.clone() => map![ + child_storage.clone() => (map![ b"doe".to_vec() => b"reindeer".to_vec() - ] + ], CHILD_INFO_1.to_owned()), ] ); let child = || ChildStorageKey::from_vec(child_storage.clone()).unwrap(); - assert_eq!(ext.child_storage(child(), b"doe"), Some(b"reindeer".to_vec())); + assert_eq!(ext.child_storage(child(), CHILD_INFO_1, b"doe"), Some(b"reindeer".to_vec())); - ext.set_child_storage(child(), b"dog".to_vec(), b"puppy".to_vec()); - assert_eq!(ext.child_storage(child(), b"dog"), Some(b"puppy".to_vec())); + ext.set_child_storage(child(), CHILD_INFO_1, b"dog".to_vec(), b"puppy".to_vec()); + assert_eq!(ext.child_storage(child(), CHILD_INFO_1, b"dog"), Some(b"puppy".to_vec())); - ext.clear_child_storage(child(), b"dog"); - assert_eq!(ext.child_storage(child(), b"dog"), None); + ext.clear_child_storage(child(), CHILD_INFO_1, b"dog"); + assert_eq!(ext.child_storage(child(), CHILD_INFO_1, b"dog"), None); - ext.kill_child_storage(child()); - assert_eq!(ext.child_storage(child(), b"doe"), None); + ext.kill_child_storage(child(), CHILD_INFO_1); + assert_eq!(ext.child_storage(child(), CHILD_INFO_1, b"doe"), None); } #[test] diff --git a/primitives/state-machine/src/changes_trie/build.rs b/primitives/state-machine/src/changes_trie/build.rs index 0680372de5794..9d27e752cdd43 100644 --- a/primitives/state-machine/src/changes_trie/build.rs +++ b/primitives/state-machine/src/changes_trie/build.rs @@ -339,12 +339,16 @@ mod test { use codec::Encode; use primitives::Blake2Hasher; use primitives::storage::well_known_keys::{EXTRINSIC_INDEX}; + use primitives::storage::ChildInfo; use crate::backend::InMemory; use crate::changes_trie::{RootsStorage, Configuration, storage::InMemoryStorage}; use crate::changes_trie::build_cache::{IncompleteCacheAction, IncompleteCachedBuildData}; use crate::overlayed_changes::{OverlayedValue, OverlayedChangeSet}; use super::*; + const CHILD_INFO_1: ChildInfo<'static> = ChildInfo::new_default(b"unique_id_1", None); + const CHILD_INFO_2: ChildInfo<'static> = ChildInfo::new_default(b"unique_id_2", None); + fn prepare_for_build(zero: u64) -> ( InMemory, InMemoryStorage, @@ -423,18 +427,18 @@ mod test { }), ].into_iter().collect(), children: vec![ - (child_trie_key1.clone(), vec![ + (child_trie_key1.clone(), (vec![ (vec![100], OverlayedValue { value: Some(vec![200]), extrinsics: Some(vec![0, 2].into_iter().collect()) }) - ].into_iter().collect()), - (child_trie_key2, vec![ + ].into_iter().collect(), CHILD_INFO_1.to_owned())), + (child_trie_key2, (vec![ (vec![100], OverlayedValue { value: Some(vec![200]), extrinsics: Some(vec![0, 2].into_iter().collect()) }) - ].into_iter().collect()), + ].into_iter().collect(), CHILD_INFO_2.to_owned())), ].into_iter().collect() }, committed: OverlayedChangeSet { top: vec![ @@ -452,12 +456,12 @@ mod test { }), ].into_iter().collect(), children: vec![ - (child_trie_key1, vec![ + (child_trie_key1, (vec![ (vec![100], OverlayedValue { value: Some(vec![202]), extrinsics: Some(vec![3].into_iter().collect()) }) - ].into_iter().collect()), + ].into_iter().collect(), CHILD_INFO_1.to_owned())), ].into_iter().collect(), }, changes_trie_config: Some(config.clone()), diff --git a/primitives/state-machine/src/ext.rs b/primitives/state-machine/src/ext.rs index 2913523d74b3f..cd2ea0a4d2848 100644 --- a/primitives/state-machine/src/ext.rs +++ b/primitives/state-machine/src/ext.rs @@ -255,7 +255,7 @@ where fn child_storage_hash( &self, storage_key: ChildStorageKey, - child_info: ChildInfo, + _child_info: ChildInfo, key: &[u8], ) -> Option { let _guard = panic_handler::AbortGuard::force_abort(); @@ -388,7 +388,7 @@ where let _guard = panic_handler::AbortGuard::force_abort(); self.mark_dirty(); - self.overlay.set_child_storage(storage_key.into_owned(), key, value); + self.overlay.set_child_storage(storage_key.into_owned(), child_info, key, value); } fn kill_child_storage( @@ -403,9 +403,9 @@ where let _guard = panic_handler::AbortGuard::force_abort(); self.mark_dirty(); - self.overlay.clear_child_storage(storage_key.as_ref()); + self.overlay.clear_child_storage(storage_key.as_ref(), child_info); self.backend.for_keys_in_child_storage(storage_key.as_ref(), child_info, |key| { - self.overlay.set_child_storage(storage_key.as_ref().to_vec(), key.to_vec(), None); + self.overlay.set_child_storage(storage_key.as_ref().to_vec(), child_info, key.to_vec(), None); }); } @@ -441,9 +441,9 @@ where let _guard = panic_handler::AbortGuard::force_abort(); self.mark_dirty(); - self.overlay.clear_child_prefix(storage_key.as_ref(), prefix); + self.overlay.clear_child_prefix(storage_key.as_ref(), child_info, prefix); self.backend.for_child_keys_with_prefix(storage_key.as_ref(), child_info, prefix, |key| { - self.overlay.set_child_storage(storage_key.as_ref().to_vec(), key.to_vec(), None); + self.overlay.set_child_storage(storage_key.as_ref().to_vec(), child_info, key.to_vec(), None); }); } diff --git a/primitives/state-machine/src/lib.rs b/primitives/state-machine/src/lib.rs index 3d21f1d38829e..ddabf4030b112 100644 --- a/primitives/state-machine/src/lib.rs +++ b/primitives/state-machine/src/lib.rs @@ -629,7 +629,7 @@ where let proving_backend = proving_backend::ProvingBackend::<_, H>::new(trie_backend); for key in keys.into_iter() { proving_backend - .child_storage(storage_key, child_info, key.as_ref()) + .child_storage(storage_key, child_info.clone(), key.as_ref()) .map_err(|e| Box::new(e) as Box)?; } Ok(proving_backend.extract_proof()) @@ -1007,22 +1007,26 @@ mod tests { ext.set_child_storage( ChildStorageKey::from_slice(b":child_storage:default:testchild").unwrap(), + CHILD_INFO_1, b"abc".to_vec(), b"def".to_vec() ); assert_eq!( ext.child_storage( ChildStorageKey::from_slice(b":child_storage:default:testchild").unwrap(), + CHILD_INFO_1, b"abc" ), Some(b"def".to_vec()) ); ext.kill_child_storage( - ChildStorageKey::from_slice(b":child_storage:default:testchild").unwrap() + ChildStorageKey::from_slice(b":child_storage:default:testchild").unwrap(), + CHILD_INFO_1, ); assert_eq!( ext.child_storage( ChildStorageKey::from_slice(b":child_storage:default:testchild").unwrap(), + CHILD_INFO_1, b"abc" ), None @@ -1058,6 +1062,7 @@ mod tests { let remote_proof = prove_child_read( remote_backend, b":child_storage:default:sub1", + CHILD_INFO_1, &[b"value3"], ).unwrap(); let local_result1 = read_child_proof_check::( diff --git a/primitives/state-machine/src/overlayed_changes.rs b/primitives/state-machine/src/overlayed_changes.rs index e98bd2941883a..2c20290d48c1f 100644 --- a/primitives/state-machine/src/overlayed_changes.rs +++ b/primitives/state-machine/src/overlayed_changes.rs @@ -21,7 +21,7 @@ use std::iter::FromIterator; use std::collections::{HashMap, BTreeSet}; use codec::Decode; use crate::changes_trie::{NO_EXTRINSIC_INDEX, Configuration as ChangesTrieConfig}; -use primitives::storage::{well_known_keys::EXTRINSIC_INDEX, OwnedChildInfo}; +use primitives::storage::{well_known_keys::EXTRINSIC_INDEX, OwnedChildInfo, ChildInfo}; /// The overlayed changes to state to be queried on top of the backend. /// @@ -118,13 +118,13 @@ impl OverlayedChanges { /// value has been set. pub fn child_storage(&self, storage_key: &[u8], key: &[u8]) -> Option> { if let Some(map) = self.prospective.children.get(storage_key) { - if let Some(val) = map.get(key) { + if let Some(val) = map.0.get(key) { return Some(val.value.as_ref().map(AsRef::as_ref)); } } if let Some(map) = self.committed.children.get(storage_key) { - if let Some(val) = map.get(key) { + if let Some(val) = map.0.get(key) { return Some(val.value.as_ref().map(AsRef::as_ref)); } } @@ -149,10 +149,17 @@ impl OverlayedChanges { /// Inserts the given key-value pair into the prospective child change set. /// /// `None` can be used to delete a value specified by the given key. - pub(crate) fn set_child_storage(&mut self, storage_key: Vec, key: Vec, val: Option>) { + pub(crate) fn set_child_storage( + &mut self, + storage_key: Vec, + child_info: ChildInfo, + key: Vec, + val: Option>, + ) { let extrinsic_index = self.extrinsic_index(); - let map_entry = self.prospective.children.entry(storage_key).or_default(); - let entry = map_entry.entry(key).or_default(); + let map_entry = self.prospective.children.entry(storage_key) + .or_insert_with(|| (Default::default(), child_info.to_owned())); + let entry = map_entry.0.entry(key).or_default(); entry.value = val; if let Some(extrinsic) = extrinsic_index { @@ -167,11 +174,16 @@ impl OverlayedChanges { /// change set, and still can be reverted by [`discard_prospective`]. /// /// [`discard_prospective`]: #method.discard_prospective - pub(crate) fn clear_child_storage(&mut self, storage_key: &[u8]) { + pub(crate) fn clear_child_storage( + &mut self, + storage_key: &[u8], + child_info: ChildInfo, + ) { let extrinsic_index = self.extrinsic_index(); - let map_entry = self.prospective.children.entry(storage_key.to_vec()).or_default(); + let map_entry = self.prospective.children.entry(storage_key.to_vec()) + .or_insert_with(|| (Default::default(), child_info.to_owned())); - map_entry.values_mut().for_each(|e| { + map_entry.0.values_mut().for_each(|e| { if let Some(extrinsic) = extrinsic_index { e.extrinsics.get_or_insert_with(Default::default) .insert(extrinsic); @@ -180,10 +192,10 @@ impl OverlayedChanges { e.value = None; }); - if let Some(committed_map) = self.committed.children.get(storage_key) { + if let Some((committed_map, _child_info)) = self.committed.children.get(storage_key) { for (key, value) in committed_map.iter() { - if !map_entry.contains_key(key) { - map_entry.insert(key.clone(), OverlayedValue { + if !map_entry.0.contains_key(key) { + map_entry.0.insert(key.clone(), OverlayedValue { value: None, extrinsics: extrinsic_index.map(|i| { let mut e = value.extrinsics.clone() @@ -234,11 +246,17 @@ impl OverlayedChanges { } } - pub(crate) fn clear_child_prefix(&mut self, storage_key: &[u8], prefix: &[u8]) { + pub(crate) fn clear_child_prefix( + &mut self, + storage_key: &[u8], + child_info: ChildInfo, + prefix: &[u8], + ) { let extrinsic_index = self.extrinsic_index(); - let map_entry = self.prospective.children.entry(storage_key.to_vec()).or_default(); + let map_entry = self.prospective.children.entry(storage_key.to_vec()) + .or_insert_with(|| (Default::default(), child_info.to_owned())); - for (key, entry) in map_entry.iter_mut() { + for (key, entry) in map_entry.0.iter_mut() { if key.starts_with(prefix) { entry.value = None; @@ -249,12 +267,12 @@ impl OverlayedChanges { } } - if let Some(child_committed) = self.committed.children.get(storage_key) { + if let Some((child_committed, _child_info)) = self.committed.children.get(storage_key) { // Then do the same with keys from commited changes. // NOTE that we are making changes in the prospective change set. for key in child_committed.keys() { if key.starts_with(prefix) { - let entry = map_entry.entry(key.clone()).or_default(); + let entry = map_entry.0.entry(key.clone()).or_default(); entry.value = None; if let Some(extrinsic) = extrinsic_index { diff --git a/primitives/state-machine/src/proving_backend.rs b/primitives/state-machine/src/proving_backend.rs index 7163527086e39..5c00b7031850f 100644 --- a/primitives/state-machine/src/proving_backend.rs +++ b/primitives/state-machine/src/proving_backend.rs @@ -377,6 +377,9 @@ mod tests { use primitives::{Blake2Hasher, storage::ChildStorageKey}; use crate::proving_backend::create_proof_check_backend; + const CHILD_INFO_1: ChildInfo<'static> = ChildInfo::new_default(b"unique_id_1", None); + const CHILD_INFO_2: ChildInfo<'static> = ChildInfo::new_default(b"unique_id_2", None); + fn test_proving<'a>( trie_backend: &'a TrieBackend,Blake2Hasher>, ) -> ProvingBackend<'a, PrefixedMemoryDB, Blake2Hasher> { @@ -422,9 +425,9 @@ mod tests { #[test] fn proof_recorded_and_checked() { - let contents = (0..64).map(|i| (None, vec![i], Some(vec![i]))).collect::>(); + let contents = (0..64).map(|i| (vec![i], Some(vec![i]))).collect::>(); let in_memory = InMemory::::default(); - let mut in_memory = in_memory.update(contents); + let mut in_memory = in_memory.update(vec![(None, contents)]); let in_memory_root = in_memory.storage_root(::std::iter::empty()).0; (0..64).for_each(|i| assert_eq!(in_memory.storage(&[i]).unwrap().unwrap(), vec![i])); @@ -448,26 +451,29 @@ mod tests { let subtrie2 = ChildStorageKey::from_slice(b":child_storage:default:sub2").unwrap(); let own1 = subtrie1.into_owned(); let own2 = subtrie2.into_owned(); - let contents = (0..64).map(|i| (None, vec![i], Some(vec![i]))) - .chain((28..65).map(|i| (Some(own1.clone()), vec![i], Some(vec![i])))) - .chain((10..15).map(|i| (Some(own2.clone()), vec![i], Some(vec![i])))) - .collect::>(); + let contents = vec![ + (None, (0..64).map(|i| (vec![i], Some(vec![i]))).collect()), + (Some((own1.clone(), CHILD_INFO_1.to_owned())), + (28..65).map(|i| (vec![i], Some(vec![i]))).collect()), + (Some((own2.clone(), CHILD_INFO_1.to_owned())), + (10..15).map(|i| (vec![i], Some(vec![i]))).collect()), + ]; let in_memory = InMemory::::default(); let mut in_memory = in_memory.update(contents); let in_memory_root = in_memory.full_storage_root::<_, Vec<_>, _>( ::std::iter::empty(), - in_memory.child_storage_keys().map(|k|(k.to_vec(), Vec::new())) + in_memory.child_storage_keys().map(|k|(k.0.to_vec(), Vec::new(), k.1.to_owned())) ).0; (0..64).for_each(|i| assert_eq!( in_memory.storage(&[i]).unwrap().unwrap(), vec![i] )); (28..65).for_each(|i| assert_eq!( - in_memory.child_storage(&own1[..], &[i]).unwrap().unwrap(), + in_memory.child_storage(&own1[..], CHILD_INFO_1, &[i]).unwrap().unwrap(), vec![i] )); (10..15).for_each(|i| assert_eq!( - in_memory.child_storage(&own2[..], &[i]).unwrap().unwrap(), + in_memory.child_storage(&own2[..], CHILD_INFO_2, &[i]).unwrap().unwrap(), vec![i] )); @@ -495,7 +501,7 @@ mod tests { assert_eq!(proof_check.storage(&[64]).unwrap(), None); let proving = ProvingBackend::new(trie); - assert_eq!(proving.child_storage(&own1[..], &[64]), Ok(Some(vec![64]))); + assert_eq!(proving.child_storage(&own1[..], CHILD_INFO_1, &[64]), Ok(Some(vec![64]))); let proof = proving.extract_proof(); let proof_check = create_proof_check_backend::( @@ -503,7 +509,7 @@ mod tests { proof ).unwrap(); assert_eq!( - proof_check.child_storage(&own1[..], &[64]).unwrap().unwrap(), + proof_check.child_storage(&own1[..], CHILD_INFO_1, &[64]).unwrap().unwrap(), vec![64] ); } diff --git a/primitives/state-machine/src/testing.rs b/primitives/state-machine/src/testing.rs index 61b338bc81acd..953ddb5d90b49 100644 --- a/primitives/state-machine/src/testing.rs +++ b/primitives/state-machine/src/testing.rs @@ -16,10 +16,10 @@ //! Test implementation for Externalities. -use std::{collections::HashMap, any::{Any, TypeId}}; +use std::any::{Any, TypeId}; use hash_db::Hasher; use crate::{ - backend::{InMemory, Backend}, OverlayedChanges, + backend::{InMemory, Backend, StorageTuple}, OverlayedChanges, changes_trie::{ InMemoryStorage as ChangesTrieInMemoryStorage, BlockNumber as ChangesTrieBlockNumber, @@ -35,8 +35,6 @@ use primitives::{ use codec::Encode; use externalities::{Extensions, Extension}; -type StorageTuple = (HashMap, Vec>, HashMap, HashMap, Vec>>); - /// Simple HashMap-based Externalities impl. pub struct TestExternalities=Blake2Hasher, N: ChangesTrieBlockNumber=u64> { overlay: OverlayedChanges, @@ -77,22 +75,17 @@ impl, N: ChangesTrieBlockNumber> TestExternalities { storage.0.insert(HEAP_PAGES.to_vec(), 8u64.encode()); storage.0.insert(CODE.to_vec(), code.to_vec()); - let backend: HashMap<_, _> = storage.1.into_iter() - .map(|(keyspace, map)| (Some(keyspace), map)) - .chain(Some((None, storage.0)).into_iter()) - .collect(); - TestExternalities { overlay, changes_trie_storage: ChangesTrieInMemoryStorage::new(), - backend: backend.into(), + backend: storage.into(), extensions: Default::default(), } } /// Insert key/value into backend pub fn insert(&mut self, k: Vec, v: Vec) { - self.backend = self.backend.update(vec![(None, k, Some(v))]); + self.backend = self.backend.update(vec![(None, vec![(k, Some(v))])]); } /// Registers the given extension for this instance. @@ -107,19 +100,23 @@ impl, N: ChangesTrieBlockNumber> TestExternalities { /// Return a new backend with all pending value. pub fn commit_all(&self) -> InMemory { - let top = self.overlay.committed.top.clone().into_iter() + let top: Vec<_> = self.overlay.committed.top.clone().into_iter() .chain(self.overlay.prospective.top.clone().into_iter()) - .map(|(k, v)| (None, k, v.value)); + .map(|(k, v)| (k, v.value)).collect(); + let mut transaction = vec![(None, top)]; - let children = self.overlay.committed.children.clone().into_iter() + self.overlay.committed.children.clone().into_iter() .chain(self.overlay.prospective.children.clone().into_iter()) - .flat_map(|(keyspace, map)| { - map.into_iter() - .map(|(k, v)| (Some(keyspace.clone()), k, v.value)) - .collect::>() + .for_each(|(keyspace, (map, child_info))| { + transaction.push(( + Some((keyspace, child_info)), + map.into_iter() + .map(|(k, v)| (k, v.value)) + .collect::>(), + )) }); - self.backend.update(top.chain(children).collect()) + self.backend.update(transaction) } /// Execute the given closure while `self` is set as externalities. diff --git a/primitives/state-machine/src/trie_backend.rs b/primitives/state-machine/src/trie_backend.rs index c26306b2b4557..7ff5a681d3ddc 100644 --- a/primitives/state-machine/src/trie_backend.rs +++ b/primitives/state-machine/src/trie_backend.rs @@ -78,7 +78,7 @@ impl, H: Hasher> Backend for TrieBackend where fn child_storage( &self, storage_key: &[u8], - child_info: ChildInfo, + _child_info: ChildInfo, key: &[u8], ) -> Result>, Self::Error> { self.essence.child_storage(storage_key, key) @@ -95,7 +95,7 @@ impl, H: Hasher> Backend for TrieBackend where fn for_keys_in_child_storage( &self, storage_key: &[u8], - child_info: ChildInfo, + _child_info: ChildInfo, f: F, ) { self.essence.for_keys_in_child_storage(storage_key, f) @@ -104,7 +104,7 @@ impl, H: Hasher> Backend for TrieBackend where fn for_child_keys_with_prefix( &self, storage_key: &[u8], - child_info: ChildInfo, + _child_info: ChildInfo, prefix: &[u8], f: F, ) { @@ -179,7 +179,7 @@ impl, H: Hasher> Backend for TrieBackend where fn child_storage_root( &self, storage_key: &[u8], - child_info: ChildInfo, + _child_info: ChildInfo, delta: I, ) -> (Vec, bool, Self::Transaction) where diff --git a/test/utils/client/src/lib.rs b/test/utils/client/src/lib.rs index 7143b67599199..e356ea1f788e0 100644 --- a/test/utils/client/src/lib.rs +++ b/test/utils/client/src/lib.rs @@ -37,7 +37,7 @@ pub use state_machine::ExecutionStrategy; use std::sync::Arc; use std::collections::HashMap; use hash_db::Hasher; -use primitives::storage::well_known_keys; +use primitives::storage::{well_known_keys, OwnedChildInfo, ChildInfo}; use sr_primitives::traits::Block as BlockT; use client::LocalCallExecutor; @@ -63,7 +63,7 @@ impl GenesisInit for () { pub struct TestClientBuilder { execution_strategies: ExecutionStrategies, genesis_init: G, - child_storage_extension: HashMap, Vec<(Vec, Vec)>>, + child_storage_extension: HashMap, (Vec<(Vec, Vec)>, OwnedChildInfo)>, backend: Arc, _executor: std::marker::PhantomData, keystore: Option, @@ -135,10 +135,12 @@ impl TestClientBuilder mut self, key: impl AsRef<[u8]>, child_key: impl AsRef<[u8]>, + child_info: ChildInfo, value: impl AsRef<[u8]>, ) -> Self { - let entry = self.child_storage_extension.entry(key.as_ref().to_vec()).or_insert_with(Vec::new); - entry.push((child_key.as_ref().to_vec(), value.as_ref().to_vec())); + let entry = self.child_storage_extension.entry(key.as_ref().to_vec()) + .or_insert_with(|| (Vec::new(), child_info.to_owned())); + entry.0.push((child_key.as_ref().to_vec(), value.as_ref().to_vec())); self } @@ -179,10 +181,10 @@ impl TestClientBuilder let mut storage = self.genesis_init.genesis_storage(); // Add some child storage keys. - for (key, value) in self.child_storage_extension { + for (key, child_content) in self.child_storage_extension { storage.1.insert( well_known_keys::CHILD_STORAGE_KEY_PREFIX.iter().cloned().chain(key).collect(), - value.into_iter().collect(), + (child_content.0.into_iter().collect(), child_content.1), ); } diff --git a/test/utils/runtime/client/src/lib.rs b/test/utils/runtime/client/src/lib.rs index b5f3833e52f07..4c808f767eeb4 100644 --- a/test/utils/runtime/client/src/lib.rs +++ b/test/utils/runtime/client/src/lib.rs @@ -29,6 +29,7 @@ pub use generic_test_client::*; pub use runtime; use primitives::sr25519; +use primitives::storage::{ChildInfo, OwnedChildInfo}; use runtime::genesismap::{GenesisConfig, additional_storage_with_genesis}; use sr_primitives::traits::{Block as BlockT, Header as HeaderT, Hash as HashT}; @@ -90,7 +91,7 @@ pub struct GenesisParameters { support_changes_trie: bool, heap_pages_override: Option, extra_storage: HashMap, Vec>, - child_extra_storage: HashMap, HashMap, Vec>>, + child_extra_storage: HashMap, (HashMap, Vec>, OwnedChildInfo)>, } impl GenesisParameters { @@ -122,7 +123,7 @@ impl generic_test_client::GenesisInit for GenesisParameters { let child_roots = storage.1.iter().map(|(sk, child_map)| { let state_root = <<::Header as HeaderT>::Hashing as HashT>::trie_root( - child_map.clone().into_iter().collect() + child_map.0.clone().into_iter().collect() ); (sk.clone(), state_root.encode()) }); @@ -181,6 +182,7 @@ pub trait TestClientBuilderExt: Sized { fn add_extra_child_storage>, K: Into>, V: Into>>( self, storage_key: SK, + child_info: ChildInfo, key: K, value: V, ) -> Self; @@ -227,6 +229,7 @@ impl TestClientBuilderExt for TestClientBuilder< fn add_extra_child_storage>, K: Into>, V: Into>>( mut self, storage_key: SK, + child_info: ChildInfo, key: K, value: V, ) -> Self { @@ -236,8 +239,8 @@ impl TestClientBuilderExt for TestClientBuilder< assert!(!key.is_empty()); self.genesis_init_mut().child_extra_storage .entry(storage_key) - .or_insert_with(Default::default) - .insert(key, value.into()); + .or_insert_with(|| (Default::default(), child_info.to_owned())) + .0.insert(key, value.into()); self } diff --git a/test/utils/runtime/src/genesismap.rs b/test/utils/runtime/src/genesismap.rs index 79cba52323e58..6a86c149438bd 100644 --- a/test/utils/runtime/src/genesismap.rs +++ b/test/utils/runtime/src/genesismap.rs @@ -20,7 +20,8 @@ use std::collections::HashMap; use runtime_io::hashing::{blake2_256, twox_128}; use super::{AuthorityId, AccountId, WASM_BINARY, system}; use codec::{Encode, KeyedVec, Joiner}; -use primitives::{ChangesTrieConfiguration, map, storage::well_known_keys}; +use primitives::{ChangesTrieConfiguration, map}; +use primitives::storage::{well_known_keys, OwnedChildInfo}; use sr_primitives::traits::{Block as BlockT, Hash as HashT, Header as HeaderT}; /// Configuration of a general Substrate test genesis block. @@ -31,7 +32,7 @@ pub struct GenesisConfig { heap_pages_override: Option, /// Additional storage key pairs that will be added to the genesis map. extra_storage: HashMap, Vec>, - child_extra_storage: HashMap, HashMap, Vec>>, + child_extra_storage: HashMap, (HashMap, Vec>, OwnedChildInfo)>, } impl GenesisConfig { @@ -42,7 +43,7 @@ impl GenesisConfig { balance: u64, heap_pages_override: Option, extra_storage: HashMap, Vec>, - child_extra_storage: HashMap, HashMap, Vec>>, + child_extra_storage: HashMap, (HashMap, Vec>, OwnedChildInfo)>, ) -> Self { GenesisConfig { changes_trie_config: match support_changes_trie { @@ -59,7 +60,7 @@ impl GenesisConfig { pub fn genesis_map(&self) -> ( HashMap, Vec>, - HashMap, HashMap, Vec>>, + HashMap, (HashMap, Vec>, OwnedChildInfo)>, ) { let wasm_runtime = WASM_BINARY.to_vec(); let mut map: HashMap, Vec> = self.balances.iter() @@ -93,17 +94,19 @@ impl GenesisConfig { pub fn insert_genesis_block( storage: &mut ( HashMap, Vec>, - HashMap, HashMap, Vec>>, + HashMap, (HashMap, Vec>, OwnedChildInfo)>, ) ) -> primitives::hash::H256 { - let child_roots = storage.1.iter().map(|(sk, child_map)| { + let child_roots = storage.1.iter().map(|(sk, child_content)| { let state_root = <<::Header as HeaderT>::Hashing as HashT>::trie_root( - child_map.clone().into_iter().collect(), + child_content.0.clone().into_iter().collect(), ); (sk.clone(), state_root.encode()) }); + // add child roots to storage + storage.0.extend(child_roots); let state_root = <<::Header as HeaderT>::Hashing as HashT>::trie_root( - storage.0.clone().into_iter().chain(child_roots).collect() + storage.0.clone().into_iter().collect() ); let block: crate::Block = substrate_client::genesis::construct_genesis_block(state_root); let genesis_hash = block.header.hash(); diff --git a/test/utils/runtime/src/lib.rs b/test/utils/runtime/src/lib.rs index 476c674f18e9a..7d6e60ee78820 100644 --- a/test/utils/runtime/src/lib.rs +++ b/test/utils/runtime/src/lib.rs @@ -50,6 +50,7 @@ use runtime_version::NativeVersion; use runtime_support::{impl_outer_origin, parameter_types}; use inherents::{CheckInherentsResult, InherentData}; use cfg_if::cfg_if; +use primitives::storage::ChildType; // Ensure Babe and Aura use the same crypto to simplify things a bit. pub use babe_primitives::AuthorityId; @@ -912,21 +913,35 @@ fn test_read_child_storage() { const CHILD_KEY: &[u8] = b":child_storage:default:read_child_storage"; const UNIQUE_ID: &[u8] = b":unique_id"; const KEY: &[u8] = b":read_child_storage"; - runtime_io::storage::child_set(CHILD_KEY, UNIQUE_ID, KEY, b"test"); + runtime_io::storage::child_set( + CHILD_KEY, + UNIQUE_ID, + ChildType::CryptoUniqueId as u32, + KEY, + b"test", + ); let mut v = [0u8; 4]; let r = runtime_io::storage::child_read( CHILD_KEY, UNIQUE_ID, + ChildType::CryptoUniqueId as u32, KEY, &mut v, - 0 + 0, ); assert_eq!(r, Some(4)); assert_eq!(&v, b"test"); let mut v = [0u8; 4]; - let r = runtime_io::storage::child_read(CHILD_KEY, UNIQUE_ID, KEY, &mut v, 8); + let r = runtime_io::storage::child_read( + CHILD_KEY, + UNIQUE_ID, + ChildType::CryptoUniqueId as u32, + KEY, + &mut v, + 8, + ); assert_eq!(r, Some(4)); assert_eq!(&v, &[0, 0, 0, 0]); } From 41c502058c72a681bdc519076543d66d6a56189d Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 20 Nov 2019 12:05:46 +0100 Subject: [PATCH 03/27] test from previous implementation. --- primitives/state-machine/src/lib.rs | 35 ++++++++++++++++++++ primitives/state-machine/src/trie_backend.rs | 17 +++++++++- 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/primitives/state-machine/src/lib.rs b/primitives/state-machine/src/lib.rs index ddabf4030b112..4fbea08c74300 100644 --- a/primitives/state-machine/src/lib.rs +++ b/primitives/state-machine/src/lib.rs @@ -1112,6 +1112,41 @@ mod tests { assert!(state_machine.execute(ExecutionStrategy::NativeWhenPossible).is_err()); } + #[test] + fn child_storage_uuid() { + const CHILD_INFO_1: ChildInfo<'static> = ChildInfo::new_default(b"unique_id_1", None); + const CHILD_INFO_2: ChildInfo<'static> = ChildInfo::new_default(b"unique_id_2", None); + use crate::trie_backend::tests::test_trie; + let mut overlay = OverlayedChanges::default(); + + let subtrie1 = ChildStorageKey::from_slice(b":child_storage:default:sub_test1").unwrap(); + let subtrie2 = ChildStorageKey::from_slice(b":child_storage:default:sub_test2").unwrap(); + let mut transaction = { + let backend = test_trie(); + let changes_trie_storage = InMemoryChangesTrieStorage::::new(); + let mut ext = Ext::new( + &mut overlay, + &backend, + Some(&changes_trie_storage), + None, + ); + ext.set_child_storage(subtrie1, CHILD_INFO_1, b"abc".to_vec(), b"def".to_vec()); + ext.set_child_storage(subtrie2, CHILD_INFO_2, b"abc".to_vec(), b"def".to_vec()); + ext.storage_root(); + (ext.transaction().0).0 + }; + let mut duplicate = false; + for (k, (value, rc)) in transaction.drain().iter() { + // look for a key inserted twice: transaction rc is 2 + if *rc == 2 { + duplicate = true; + println!("test duplicate for {:?} {:?}", k, value); + } + } + // TODO EMCH revert this condition when implemented!!! + assert!(duplicate); + } + #[test] fn cannot_change_changes_trie_config_with_native_else_wasm() { let backend = trie_backend::tests::test_trie(); diff --git a/primitives/state-machine/src/trie_backend.rs b/primitives/state-machine/src/trie_backend.rs index 7ff5a681d3ddc..4971e33b87898 100644 --- a/primitives/state-machine/src/trie_backend.rs +++ b/primitives/state-machine/src/trie_backend.rs @@ -232,10 +232,16 @@ pub mod tests { use trie::{TrieMut, PrefixedMemoryDB, trie_types::TrieDBMut}; use super::*; + const CHILD_KEY_1: &[u8] = b":child_storage:default:sub1"; + + const CHILD_UUID_1: &[u8] = b"unique_id_1"; + const CHILD_INFO_1: ChildInfo<'static> = ChildInfo::new_default(CHILD_UUID_1, None); + fn test_db() -> (PrefixedMemoryDB, H256) { let mut root = H256::default(); let mut mdb = PrefixedMemoryDB::::default(); { + // TODO EMCH let mut mdb = KeySpacedDBMut::new(&mut mdb, Some(&keyspace1)); let mut trie = TrieDBMut::new(&mut mdb, &mut root); trie.insert(b"value3", &[142]).expect("insert failed"); trie.insert(b"value4", &[124]).expect("insert failed"); @@ -245,7 +251,7 @@ pub mod tests { let mut sub_root = Vec::new(); root.encode_to(&mut sub_root); let mut trie = TrieDBMut::new(&mut mdb, &mut root); - trie.insert(b":child_storage:default:sub1", &sub_root).expect("insert failed"); + trie.insert(CHILD_KEY_1, &sub_root[..]).expect("insert failed"); trie.insert(b"key", b"value").expect("insert failed"); trie.insert(b"value1", &[42]).expect("insert failed"); trie.insert(b"value2", &[24]).expect("insert failed"); @@ -267,6 +273,15 @@ pub mod tests { assert_eq!(test_trie().storage(b"key").unwrap(), Some(b"value".to_vec())); } + #[test] + fn read_from_child_storage_returns_some() { + let test_trie = test_trie(); + assert_eq!( + test_trie.child_storage(CHILD_KEY_1, CHILD_INFO_1, b"value3").unwrap(), + Some(vec![142u8]), + ); + } + #[test] fn read_from_storage_returns_none() { assert_eq!(test_trie().storage(b"non-existing-key").unwrap(), None); From 6eb041491bab16fb45077552c348cd9b4aad74db Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 20 Nov 2019 12:11:05 +0100 Subject: [PATCH 04/27] fix proving test. --- primitives/state-machine/src/proving_backend.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/primitives/state-machine/src/proving_backend.rs b/primitives/state-machine/src/proving_backend.rs index 5c00b7031850f..6a6a7d72f6dc3 100644 --- a/primitives/state-machine/src/proving_backend.rs +++ b/primitives/state-machine/src/proving_backend.rs @@ -455,7 +455,7 @@ mod tests { (None, (0..64).map(|i| (vec![i], Some(vec![i]))).collect()), (Some((own1.clone(), CHILD_INFO_1.to_owned())), (28..65).map(|i| (vec![i], Some(vec![i]))).collect()), - (Some((own2.clone(), CHILD_INFO_1.to_owned())), + (Some((own2.clone(), CHILD_INFO_2.to_owned())), (10..15).map(|i| (vec![i], Some(vec![i]))).collect()), ]; let in_memory = InMemory::::default(); From e8e72fcfc194262a7980aea561d3eeed99eae905 Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 20 Nov 2019 16:21:37 +0100 Subject: [PATCH 05/27] Restore Keyspacedb from other branch, only apply to child trie. --- primitives/core/storage/src/lib.rs | 9 ++ primitives/state-machine/src/backend.rs | 2 + primitives/state-machine/src/lib.rs | 3 +- .../state-machine/src/proving_backend.rs | 2 + primitives/state-machine/src/trie_backend.rs | 15 +-- .../state-machine/src/trie_backend_essence.rs | 19 ++- primitives/trie/src/lib.rs | 115 +++++++++++++++++- 7 files changed, 148 insertions(+), 17 deletions(-) diff --git a/primitives/core/storage/src/lib.rs b/primitives/core/storage/src/lib.rs index 94d37a2593c34..c453ce46c4941 100644 --- a/primitives/core/storage/src/lib.rs +++ b/primitives/core/storage/src/lib.rs @@ -224,6 +224,15 @@ impl<'a> ChildInfo<'a> { } } + /// Return byte sequence (keyspace) that can be use for underlying db to isolate keys. + pub fn keyspace(&self) -> &[u8] { + match self { + ChildInfo::Default(ChildTrie { + root: _root, + unique_id, + }) => &unique_id[..], + } + } } /// Type of child, this can be different child usage diff --git a/primitives/state-machine/src/backend.rs b/primitives/state-machine/src/backend.rs index 5a5de985a0e6e..bf087075312cb 100644 --- a/primitives/state-machine/src/backend.rs +++ b/primitives/state-machine/src/backend.rs @@ -576,6 +576,8 @@ impl Backend for InMemory { let mut root_map = None; for (child_info, map) in &self.inner { if let Some((storage_key, _child_info)) = child_info.as_ref() { + // no need to use child_info at this point because we use a MemoryDB for + // proof (with PrefixedMemoryDB it would be needed). let ch = insert_into_memory_db::(&mut mdb, map.clone().into_iter())?; new_child_roots.push((storage_key.clone(), ch.as_ref().into())); } else { diff --git a/primitives/state-machine/src/lib.rs b/primitives/state-machine/src/lib.rs index 4fbea08c74300..475ab6fcc9cf3 100644 --- a/primitives/state-machine/src/lib.rs +++ b/primitives/state-machine/src/lib.rs @@ -1143,8 +1143,7 @@ mod tests { println!("test duplicate for {:?} {:?}", k, value); } } - // TODO EMCH revert this condition when implemented!!! - assert!(duplicate); + assert!(!duplicate); } #[test] diff --git a/primitives/state-machine/src/proving_backend.rs b/primitives/state-machine/src/proving_backend.rs index 6a6a7d72f6dc3..2dbffd7252906 100644 --- a/primitives/state-machine/src/proving_backend.rs +++ b/primitives/state-machine/src/proving_backend.rs @@ -143,6 +143,7 @@ impl<'a, S, H> ProvingBackendRecorder<'a, S, H> pub fn child_storage( &mut self, storage_key: &[u8], + child_info: ChildInfo, key: &[u8] ) -> Result>, String> { let root = self.storage(storage_key)? @@ -158,6 +159,7 @@ impl<'a, S, H> ProvingBackendRecorder<'a, S, H> read_child_trie_value_with::, _, _>( storage_key, + child_info.keyspace(), &eph, &root, key, diff --git a/primitives/state-machine/src/trie_backend.rs b/primitives/state-machine/src/trie_backend.rs index 4971e33b87898..bf9afd365dd29 100644 --- a/primitives/state-machine/src/trie_backend.rs +++ b/primitives/state-machine/src/trie_backend.rs @@ -78,10 +78,10 @@ impl, H: Hasher> Backend for TrieBackend where fn child_storage( &self, storage_key: &[u8], - _child_info: ChildInfo, + child_info: ChildInfo, key: &[u8], ) -> Result>, Self::Error> { - self.essence.child_storage(storage_key, key) + self.essence.child_storage(storage_key, child_info, key) } fn for_keys_with_prefix(&self, prefix: &[u8], f: F) { @@ -95,10 +95,10 @@ impl, H: Hasher> Backend for TrieBackend where fn for_keys_in_child_storage( &self, storage_key: &[u8], - _child_info: ChildInfo, + child_info: ChildInfo, f: F, ) { - self.essence.for_keys_in_child_storage(storage_key, f) + self.essence.for_keys_in_child_storage(storage_key, child_info, f) } fn for_child_keys_with_prefix( @@ -179,7 +179,7 @@ impl, H: Hasher> Backend for TrieBackend where fn child_storage_root( &self, storage_key: &[u8], - _child_info: ChildInfo, + child_info: ChildInfo, delta: I, ) -> (Vec, bool, Self::Transaction) where @@ -205,6 +205,7 @@ impl, H: Hasher> Backend for TrieBackend where match child_delta_trie_root::, _, _, _, _>( storage_key, + child_info.keyspace(), &mut eph, root.clone(), delta @@ -229,7 +230,7 @@ pub mod tests { use std::collections::HashSet; use primitives::{Blake2Hasher, H256}; use codec::Encode; - use trie::{TrieMut, PrefixedMemoryDB, trie_types::TrieDBMut}; + use trie::{TrieMut, PrefixedMemoryDB, trie_types::TrieDBMut, KeySpacedDBMut}; use super::*; const CHILD_KEY_1: &[u8] = b":child_storage:default:sub1"; @@ -241,7 +242,7 @@ pub mod tests { let mut root = H256::default(); let mut mdb = PrefixedMemoryDB::::default(); { - // TODO EMCH let mut mdb = KeySpacedDBMut::new(&mut mdb, Some(&keyspace1)); + let mut mdb = KeySpacedDBMut::new(&mut mdb, CHILD_UUID_1); let mut trie = TrieDBMut::new(&mut mdb, &mut root); trie.insert(b"value3", &[142]).expect("insert failed"); trie.insert(b"value4", &[124]).expect("insert failed"); diff --git a/primitives/state-machine/src/trie_backend_essence.rs b/primitives/state-machine/src/trie_backend_essence.rs index 5a5431963448c..f4529fe037e76 100644 --- a/primitives/state-machine/src/trie_backend_essence.rs +++ b/primitives/state-machine/src/trie_backend_essence.rs @@ -26,6 +26,7 @@ use trie::{Trie, MemoryDB, PrefixedMemoryDB, DBValue, for_keys_in_child_trie}; use trie::trie_types::{TrieDB, TrieError, Layout}; use crate::backend::Consolidate; +use primitives::storage::ChildInfo; /// Patricia trie-based storage trait. pub trait Storage: Send + Sync { @@ -77,7 +78,12 @@ impl, H: Hasher> TrieBackendEssence { } /// Get the value of child storage at given key. - pub fn child_storage(&self, storage_key: &[u8], key: &[u8]) -> Result>, String> { + pub fn child_storage( + &self, + storage_key: &[u8], + child_info: ChildInfo, + key: &[u8], + ) -> Result>, String> { let root = self.storage(storage_key)? .unwrap_or(default_child_trie_root::>(storage_key)); @@ -89,11 +95,17 @@ impl, H: Hasher> TrieBackendEssence { let map_e = |e| format!("Trie lookup error: {}", e); - read_child_trie_value::, _>(storage_key, &eph, &root, key).map_err(map_e) + read_child_trie_value::, _>(storage_key, child_info.keyspace(), &eph, &root, key) + .map_err(map_e) } /// Retrieve all entries keys of child storage and call `f` for each of those keys. - pub fn for_keys_in_child_storage(&self, storage_key: &[u8], f: F) { + pub fn for_keys_in_child_storage( + &self, + storage_key: &[u8], + child_info: ChildInfo, + f: F, + ) { let root = match self.storage(storage_key) { Ok(v) => v.unwrap_or(default_child_trie_root::>(storage_key)), Err(e) => { @@ -110,6 +122,7 @@ impl, H: Hasher> TrieBackendEssence { if let Err(e) = for_keys_in_child_trie::, _, Ephemeral>( storage_key, + child_info.keyspace(), &eph, &root, f, diff --git a/primitives/trie/src/lib.rs b/primitives/trie/src/lib.rs index eab05bab28af0..24c86448e2362 100644 --- a/primitives/trie/src/lib.rs +++ b/primitives/trie/src/lib.rs @@ -24,8 +24,9 @@ mod node_codec; mod trie_stream; use rstd::boxed::Box; +use rstd::marker::PhantomData; use rstd::vec::Vec; -use hash_db::Hasher; +use hash_db::{Hasher, Prefix}; /// Our `NodeCodec`-specific error. pub use error::Error; /// The Substrate format implementation of `TrieStream`. @@ -186,6 +187,7 @@ pub fn child_trie_root(_storage_key: &[u8], input /// but a generic implementation may ignore this type parameter and use other hashers. pub fn child_delta_trie_root( _storage_key: &[u8], + keyspace: &[u8], db: &mut DB, root_vec: Vec, delta: I @@ -202,7 +204,8 @@ pub fn child_delta_trie_root( root.as_mut().copy_from_slice(&root_vec); { - let mut trie = TrieDBMut::::from_existing(&mut *db, &mut root)?; + let mut db = KeySpacedDBMut::new(&mut *db, keyspace); + let mut trie = TrieDBMut::::from_existing(&mut db, &mut root)?; for (key, change) in delta { match change { @@ -218,6 +221,7 @@ pub fn child_delta_trie_root( /// Call `f` for all keys in a child trie. pub fn for_keys_in_child_trie( _storage_key: &[u8], + keyspace: &[u8], db: &DB, root_slice: &[u8], mut f: F @@ -230,7 +234,8 @@ pub fn for_keys_in_child_trie( // root is fetched from DB, not writable by runtime, so it's always valid. root.as_mut().copy_from_slice(root_slice); - let trie = TrieDB::::new(&*db, &root)?; + let db = KeySpacedDB::new(&*db, keyspace); + let trie = TrieDB::::new(&db, &root)?; let iter = trie.iter()?; for x in iter { @@ -267,6 +272,7 @@ pub fn record_all_keys( /// Read a value from the child trie. pub fn read_child_trie_value( _storage_key: &[u8], + keyspace: &[u8], db: &DB, root_slice: &[u8], key: &[u8] @@ -279,12 +285,14 @@ pub fn read_child_trie_value( // root is fetched from DB, not writable by runtime, so it's always valid. root.as_mut().copy_from_slice(root_slice); - Ok(TrieDB::::new(&*db, &root)?.get(key).map(|x| x.map(|val| val.to_vec()))?) + let db = KeySpacedDB::new(&*db, keyspace); + Ok(TrieDB::::new(&db, &root)?.get(key).map(|x| x.map(|val| val.to_vec()))?) } /// Read a value from the child trie with given query. pub fn read_child_trie_value_with, DB>( _storage_key: &[u8], + keyspace: &[u8], db: &DB, root_slice: &[u8], key: &[u8], @@ -298,7 +306,104 @@ pub fn read_child_trie_value_with::new(&*db, &root)?.get_with(key, query).map(|x| x.map(|val| val.to_vec()))?) + let db = KeySpacedDB::new(&*db, keyspace); + Ok(TrieDB::::new(&db, &root)?.get_with(key, query).map(|x| x.map(|val| val.to_vec()))?) +} + +/// `HashDB` implementation that append a encoded prefix (unique id bytes) in addition to the +/// prefix of every key value. +pub struct KeySpacedDB<'a, DB, H>(&'a DB, &'a [u8], PhantomData); + +/// `HashDBMut` implementation that append a encoded prefix (unique id bytes) in addition to the +/// prefix of every key value. +/// +/// Mutable variant of `KeySpacedDB`, see [`KeySpacedDB`]. +pub struct KeySpacedDBMut<'a, DB, H>(&'a mut DB, &'a [u8], PhantomData); + +/// Utility function used to merge some byte data (keyspace) and `prefix` data +/// before calling key value database primitives. +fn keyspace_as_prefix_alloc(ks: &[u8], prefix: Prefix) -> (Vec, Option) { + let mut result = rstd::vec![0; ks.len() + prefix.0.len()]; + result[..ks.len()].copy_from_slice(ks); + result[ks.len()..].copy_from_slice(prefix.0); + (result, prefix.1) +} + +impl<'a, DB, H> KeySpacedDB<'a, DB, H> where + H: Hasher, +{ + /// instantiate new keyspaced db + pub fn new(db: &'a DB, ks: &'a [u8]) -> Self { + KeySpacedDB(db, ks, PhantomData) + } +} + +impl<'a, DB, H> KeySpacedDBMut<'a, DB, H> where + H: Hasher, +{ + /// instantiate new keyspaced db + pub fn new(db: &'a mut DB, ks: &'a [u8]) -> Self { + KeySpacedDBMut(db, ks, PhantomData) + } +} + +impl<'a, DB, H, T> hash_db::HashDBRef for KeySpacedDB<'a, DB, H> where + DB: hash_db::HashDBRef, + H: Hasher, + T: From<&'static [u8]>, +{ + fn get(&self, key: &H::Out, prefix: Prefix) -> Option { + let derived_prefix = keyspace_as_prefix_alloc(self.1, prefix); + self.0.get(key, (&derived_prefix.0, derived_prefix.1)) + } + + fn contains(&self, key: &H::Out, prefix: Prefix) -> bool { + let derived_prefix = keyspace_as_prefix_alloc(self.1, prefix); + self.0.contains(key, (&derived_prefix.0, derived_prefix.1)) + } +} + +impl<'a, DB, H, T> hash_db::HashDB for KeySpacedDBMut<'a, DB, H> where + DB: hash_db::HashDB, + H: Hasher, + T: Default + PartialEq + for<'b> From<&'b [u8]> + Clone + Send + Sync, +{ + fn get(&self, key: &H::Out, prefix: Prefix) -> Option { + let derived_prefix = keyspace_as_prefix_alloc(self.1, prefix); + self.0.get(key, (&derived_prefix.0, derived_prefix.1)) + } + + fn contains(&self, key: &H::Out, prefix: Prefix) -> bool { + let derived_prefix = keyspace_as_prefix_alloc(self.1, prefix); + self.0.contains(key, (&derived_prefix.0, derived_prefix.1)) + } + + fn insert(&mut self, prefix: Prefix, value: &[u8]) -> H::Out { + let derived_prefix = keyspace_as_prefix_alloc(self.1, prefix); + self.0.insert((&derived_prefix.0, derived_prefix.1), value) + } + + fn emplace(&mut self, key: H::Out, prefix: Prefix, value: T) { + let derived_prefix = keyspace_as_prefix_alloc(self.1, prefix); + self.0.emplace(key, (&derived_prefix.0, derived_prefix.1), value) + } + + fn remove(&mut self, key: &H::Out, prefix: Prefix) { + let derived_prefix = keyspace_as_prefix_alloc(self.1, prefix); + self.0.remove(key, (&derived_prefix.0, derived_prefix.1)) + } +} + +impl<'a, DB, H, T> hash_db::AsHashDB for KeySpacedDBMut<'a, DB, H> where + DB: hash_db::HashDB, + H: Hasher, + T: Default + PartialEq + for<'b> From<&'b [u8]> + Clone + Send + Sync, +{ + fn as_hash_db(&self) -> &dyn hash_db::HashDB { &*self } + + fn as_hash_db_mut<'b>(&'b mut self) -> &'b mut (dyn hash_db::HashDB + 'b) { + &mut *self + } } /// Constants used into trie simplification codec. From 88ffbc49ca7738640c356ca41953fd0276e57e31 Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 20 Nov 2019 17:05:02 +0100 Subject: [PATCH 06/27] Removing unneeded child_info from child root (child info are stored if things changed, otherwhise the root does not change). --- client/executor/src/deprecated_host_interface.rs | 2 -- paint/contracts/src/lib.rs | 1 - paint/contracts/src/rent.rs | 1 - paint/support/src/storage/child.rs | 3 --- primitives/externalities/src/lib.rs | 1 - primitives/sr-io/src/lib.rs | 6 +----- primitives/state-machine/src/basic.rs | 4 ---- primitives/state-machine/src/ext.rs | 2 -- 8 files changed, 1 insertion(+), 19 deletions(-) diff --git a/client/executor/src/deprecated_host_interface.rs b/client/executor/src/deprecated_host_interface.rs index 5182cdfbfe0d7..a47fd80989b17 100644 --- a/client/executor/src/deprecated_host_interface.rs +++ b/client/executor/src/deprecated_host_interface.rs @@ -451,8 +451,6 @@ impl_wasm_host_interface! { // This can only work with archive mode. let value = runtime_io::storage::child_root( &storage_key, - &[], - ChildType::CryptoUniqueId as u32, ); let offset = context.allocate_memory(value.len() as u32)?; diff --git a/paint/contracts/src/lib.rs b/paint/contracts/src/lib.rs index 8b2e018230589..91a743b5025d4 100644 --- a/paint/contracts/src/lib.rs +++ b/paint/contracts/src/lib.rs @@ -827,7 +827,6 @@ impl Module { // is not this block as it has been checked earlier. &child::child_root( &origin_contract.trie_id, - origin_contract.child_trie_unique_id(), )[..], code_hash, ); diff --git a/paint/contracts/src/rent.rs b/paint/contracts/src/rent.rs index 701e7c820321c..d8af01eb883a7 100644 --- a/paint/contracts/src/rent.rs +++ b/paint/contracts/src/rent.rs @@ -149,7 +149,6 @@ fn try_evict_or_and_pay_rent( // Note: this operation is heavy. let child_storage_root = child::child_root( &contract.trie_id, - contract.child_trie_unique_id(), ); let tombstone = >::new( diff --git a/paint/support/src/storage/child.rs b/paint/support/src/storage/child.rs index 466f860cbfe85..a63e1107d5043 100644 --- a/paint/support/src/storage/child.rs +++ b/paint/support/src/storage/child.rs @@ -220,11 +220,8 @@ pub fn put_raw( /// Calculate current child root value. pub fn child_root( storage_key: &[u8], - unique_id: Option<&[u8]>, ) -> Vec { runtime_io::storage::child_root( storage_key, - unique_id.expect(SOME), - primitives::storage::ChildType::CryptoUniqueId as u32, ) } diff --git a/primitives/externalities/src/lib.rs b/primitives/externalities/src/lib.rs index 6609b542a58b9..923497255f4c2 100644 --- a/primitives/externalities/src/lib.rs +++ b/primitives/externalities/src/lib.rs @@ -170,7 +170,6 @@ pub trait Externalities: ExtensionStore { fn child_storage_root( &mut self, storage_key: ChildStorageKey, - child_info: ChildInfo, ) -> Vec; /// Get the change trie root of the current storage overlay at a block with given parent. diff --git a/primitives/sr-io/src/lib.rs b/primitives/sr-io/src/lib.rs index da190fac01e3f..c39e1a776059e 100644 --- a/primitives/sr-io/src/lib.rs +++ b/primitives/sr-io/src/lib.rs @@ -245,13 +245,9 @@ pub trait Storage { fn child_root( &mut self, child_storage_key: &[u8], - child_definition: &[u8], - child_type: u32, ) -> Vec { let storage_key = child_storage_key_or_panic(child_storage_key); - let child_info = ChildInfo::resolve_child_info(child_type, child_definition) - .expect("Invalid child definition"); - self.child_storage_root(storage_key, child_info) + self.child_storage_root(storage_key) } /// "Commit" all existing operations and get the resulting storage change root. diff --git a/primitives/state-machine/src/basic.rs b/primitives/state-machine/src/basic.rs index a1c81c47c71e4..01d769c9d7c5a 100644 --- a/primitives/state-machine/src/basic.rs +++ b/primitives/state-machine/src/basic.rs @@ -232,12 +232,9 @@ impl Externalities for BasicExternalities { // type of child trie support. let empty_hash = default_child_trie_root::>(&[]); for storage_key in keys { - let child_info = self.children.get(storage_key.as_slice()).as_ref() - .expect("key fetch above").1.to_owned(); let child_root = self.child_storage_root( ChildStorageKey::from_slice(storage_key.as_slice()) .expect("Map only feed by valid keys; qed"), - child_info.as_ref(), ); if &empty_hash[..] == &child_root[..] { top.remove(storage_key.as_slice()); @@ -252,7 +249,6 @@ impl Externalities for BasicExternalities { fn child_storage_root( &mut self, storage_key: ChildStorageKey, - _child_info: ChildInfo, ) -> Vec { if let Some(child) = self.children.get(storage_key.as_ref()) { let delta = child.0.clone().into_iter().map(|(k, v)| (k, Some(v))); diff --git a/primitives/state-machine/src/ext.rs b/primitives/state-machine/src/ext.rs index cd2ea0a4d2848..c710b94b138e2 100644 --- a/primitives/state-machine/src/ext.rs +++ b/primitives/state-machine/src/ext.rs @@ -491,11 +491,9 @@ where root } - // TODO EMCH seems like child_info parameter is useless here!!! fn child_storage_root( &mut self, storage_key: ChildStorageKey, - _child_info: ChildInfo, ) -> Vec { let _guard = panic_handler::AbortGuard::force_abort(); if self.storage_transaction.is_some() { From 435fcb2931c07a4e4d5ee56abd0693549dd0b4af Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 20 Nov 2019 18:37:15 +0100 Subject: [PATCH 07/27] Switch rpc to use same format as ext: more future proof. --- client/api/src/light.rs | 7 ++- client/network/src/protocol.rs | 53 ++++++++++++------- client/network/src/protocol/light_dispatch.rs | 13 +++-- client/network/src/protocol/message.rs | 6 ++- client/rpc/api/src/state/mod.rs | 12 +++-- client/rpc/src/state/mod.rs | 40 +++++++++----- client/rpc/src/state/state_full.rs | 20 ++++--- client/rpc/src/state/state_light.rs | 14 +++-- client/rpc/src/state/tests.rs | 17 +++--- client/src/light/fetcher.rs | 7 +-- 10 files changed, 122 insertions(+), 67 deletions(-) diff --git a/client/api/src/light.rs b/client/api/src/light.rs index 4c271167df9fd..6f5aea9052b51 100644 --- a/client/api/src/light.rs +++ b/client/api/src/light.rs @@ -86,8 +86,11 @@ pub struct RemoteReadChildRequest { pub header: Header, /// Storage key for child. pub storage_key: Vec, - /// Child trie associated unique id. - pub unique_id: Vec, + /// Child trie encoded information + pub child_info: Vec, + /// Child type needed to resolve child info + /// and choose child implementation. + pub child_type: u32, /// Child storage key to read. pub keys: Vec>, /// Number of times to retry request. None means that default RETRY_COUNT is used. diff --git a/client/network/src/protocol.rs b/client/network/src/protocol.rs index a1bd5f5adfbc2..87fbc37adfb4d 100644 --- a/client/network/src/protocol.rs +++ b/client/network/src/protocol.rs @@ -218,14 +218,16 @@ impl<'a, B: BlockT> LightDispatchNetwork for LightDispatchIn<'a> { id: RequestId, block: ::Hash, storage_key: Vec, - unique_id: Vec, + child_info: Vec, + child_type: u32, keys: Vec>, ) { let message: Message = message::generic::Message::RemoteReadChildRequest(message::RemoteReadChildRequest { id, block, storage_key, - unique_id, + child_info, + child_type, keys, }); @@ -1462,24 +1464,37 @@ impl, H: ExHashT> Protocol { trace!(target: "sync", "Remote read child request {} from {} ({} {} at {})", request.id, who, request.storage_key.to_hex::(), keys_str(), request.block); - let proof = match self.context_data.chain.read_child_proof( - &request.block, - &request.storage_key, - ChildInfo::new_default(&request.unique_id[..], None), - &request.keys, - ) { - Ok(proof) => proof, - Err(error) => { - trace!(target: "sync", "Remote read child request {} from {} ({} {} at {}) failed with: {}", - request.id, - who, - request.storage_key.to_hex::(), - keys_str(), - request.block, - error - ); - StorageProof::empty() + let proof = if let Some(child_info) = ChildInfo::resolve_child_info(request.child_type, &request.child_info[..]) { + match self.context_data.chain.read_child_proof( + &request.block, + &request.storage_key, + child_info, + &request.keys, + ) { + Ok(proof) => proof, + Err(error) => { + trace!(target: "sync", "Remote read child request {} from {} ({} {} at {}) failed with: {}", + request.id, + who, + request.storage_key.to_hex::(), + keys_str(), + request.block, + error + ); + StorageProof::empty() + } } + } else { + trace!(target: "sync", "Remote read child request {} from {} ({} {} at {}) failed with: {}", + request.id, + who, + request.storage_key.to_hex::(), + keys_str(), + request.block, + "invalid child info and type", + ); + + StorageProof::empty() }; self.send_message( &who, diff --git a/client/network/src/protocol/light_dispatch.rs b/client/network/src/protocol/light_dispatch.rs index e5f35d6016c56..b7730de2adb49 100644 --- a/client/network/src/protocol/light_dispatch.rs +++ b/client/network/src/protocol/light_dispatch.rs @@ -68,7 +68,8 @@ pub trait LightDispatchNetwork { id: RequestId, block: ::Hash, storage_key: Vec, - unique_id: Vec, + child_info: Vec, + child_type: u32, keys: Vec>, ); @@ -622,7 +623,8 @@ impl Request { self.id, data.block, data.storage_key.clone(), - data.unique_id.clone(), + data.child_info.clone(), + data.child_type, data.keys.clone(), ), RequestData::RemoteCall(ref data, _) => @@ -678,6 +680,7 @@ pub mod tests { use std::sync::Arc; use std::time::Instant; use futures::{Future, sync::oneshot}; + use primitives::storage::ChildInfo; use sr_primitives::traits::{Block as BlockT, NumberFor, Header as HeaderT}; use client_api::{error::{Error as ClientError, Result as ClientResult}}; use client_api::{FetchChecker, RemoteHeaderRequest, @@ -809,7 +812,7 @@ pub mod tests { fn send_header_request(&mut self, _: &PeerId, _: RequestId, _: <::Header as HeaderT>::Number) {} fn send_read_request(&mut self, _: &PeerId, _: RequestId, _: ::Hash, _: Vec>) {} fn send_read_child_request(&mut self, _: &PeerId, _: RequestId, _: ::Hash, _: Vec, - _: Vec, _: Vec>) {} + _: Vec, _: u32, _: Vec>) {} fn send_call_request(&mut self, _: &PeerId, _: RequestId, _: ::Hash, _: String, _: Vec) {} fn send_changes_request(&mut self, _: &PeerId, _: RequestId, _: ::Hash, _: ::Hash, _: ::Hash, _: ::Hash, _: Option>, _: Vec) {} @@ -1028,11 +1031,13 @@ pub mod tests { light_dispatch.on_connect(&mut network_interface, peer0.clone(), Roles::FULL, 1000); let (tx, response) = oneshot::channel(); + let child_info = ChildInfo::new_default(b"unique_id_1", None).info(); light_dispatch.add_request(&mut network_interface, RequestData::RemoteReadChild(RemoteReadChildRequest { header: dummy_header(), block: Default::default(), storage_key: b":child_storage:sub".to_vec(), - unique_id: b"unique_id_1".to_vec(), + child_info: child_info.0.to_vec(), + child_type: child_info.1, keys: vec![b":key".to_vec()], retry_count: None, }, tx)); diff --git a/client/network/src/protocol/message.rs b/client/network/src/protocol/message.rs index 12b86db7caabb..c524eeddcf31a 100644 --- a/client/network/src/protocol/message.rs +++ b/client/network/src/protocol/message.rs @@ -368,8 +368,10 @@ pub mod generic { pub block: H, /// Child Storage key. pub storage_key: Vec, - /// Child trie associated unique id. - pub unique_id: Vec, + /// Child trie associated info. + pub child_info: Vec, + /// Child trie associated type. + pub child_type: u32, /// Storage key. pub keys: Vec>, } diff --git a/client/rpc/api/src/state/mod.rs b/client/rpc/api/src/state/mod.rs index deb561b6e3737..9a549b00c4304 100644 --- a/client/rpc/api/src/state/mod.rs +++ b/client/rpc/api/src/state/mod.rs @@ -60,7 +60,8 @@ pub trait StateApi { fn child_storage_keys( &self, child_storage_key: StorageKey, - unique_id: StorageKey, + child_info: StorageKey, + child_type: u32, prefix: StorageKey, hash: Option ) -> FutureResult>; @@ -70,7 +71,8 @@ pub trait StateApi { fn child_storage( &self, child_storage_key: StorageKey, - unique_id: StorageKey, + child_info: StorageKey, + child_type: u32, key: StorageKey, hash: Option ) -> FutureResult>; @@ -80,7 +82,8 @@ pub trait StateApi { fn child_storage_hash( &self, child_storage_key: StorageKey, - unique_id: StorageKey, + child_info: StorageKey, + child_type: u32, key: StorageKey, hash: Option ) -> FutureResult>; @@ -90,7 +93,8 @@ pub trait StateApi { fn child_storage_size( &self, child_storage_key: StorageKey, - unique_id: StorageKey, + child_info: StorageKey, + child_type: u32, key: StorageKey, hash: Option ) -> FutureResult>; diff --git a/client/rpc/src/state/mod.rs b/client/rpc/src/state/mod.rs index 62534b89c68e7..265783f552edc 100644 --- a/client/rpc/src/state/mod.rs +++ b/client/rpc/src/state/mod.rs @@ -95,7 +95,8 @@ pub trait StateBackend: Send + Sync + 'static &self, block: Option, child_storage_key: StorageKey, - unique_id: StorageKey, + child_info: StorageKey, + child_type: u32, prefix: StorageKey, ) -> FutureResult>; @@ -104,7 +105,8 @@ pub trait StateBackend: Send + Sync + 'static &self, block: Option, child_storage_key: StorageKey, - unique_id: StorageKey, + child_info: StorageKey, + child_type: u32, key: StorageKey, ) -> FutureResult>; @@ -113,7 +115,8 @@ pub trait StateBackend: Send + Sync + 'static &self, block: Option, child_storage_key: StorageKey, - unique_id: StorageKey, + child_info: StorageKey, + child_type: u32, key: StorageKey, ) -> FutureResult>; @@ -122,10 +125,11 @@ pub trait StateBackend: Send + Sync + 'static &self, block: Option, child_storage_key: StorageKey, - unique_id: StorageKey, + child_info: StorageKey, + child_type: u32, key: StorageKey, ) -> FutureResult> { - Box::new(self.child_storage(block, child_storage_key, unique_id, key) + Box::new(self.child_storage(block, child_storage_key, child_info, child_type, key) .map(|x| x.map(|x| x.0.len() as u64))) } @@ -260,41 +264,45 @@ impl StateApi for State fn child_storage( &self, child_storage_key: StorageKey, - unique_id: StorageKey, + child_info: StorageKey, + child_type: u32, key: StorageKey, block: Option ) -> FutureResult> { - self.backend.child_storage(block, child_storage_key, unique_id, key) + self.backend.child_storage(block, child_storage_key, child_info, child_type, key) } fn child_storage_keys( &self, child_storage_key: StorageKey, - unique_id: StorageKey, + child_info: StorageKey, + child_type: u32, key_prefix: StorageKey, block: Option ) -> FutureResult> { - self.backend.child_storage_keys(block, child_storage_key, unique_id, key_prefix) + self.backend.child_storage_keys(block, child_storage_key, child_info, child_type, key_prefix) } fn child_storage_hash( &self, child_storage_key: StorageKey, - unique_id: StorageKey, + child_info: StorageKey, + child_type: u32, key: StorageKey, block: Option ) -> FutureResult> { - self.backend.child_storage_hash(block, child_storage_key, unique_id, key) + self.backend.child_storage_hash(block, child_storage_key, child_info, child_type, key) } fn child_storage_size( &self, child_storage_key: StorageKey, - unique_id: StorageKey, + child_info: StorageKey, + child_type: u32, key: StorageKey, block: Option ) -> FutureResult> { - self.backend.child_storage_size(block, child_storage_key, unique_id, key) + self.backend.child_storage_size(block, child_storage_key, child_info, child_type, key) } fn metadata(&self, block: Option) -> FutureResult { @@ -343,3 +351,9 @@ impl StateApi for State fn client_err(err: client::error::Error) -> Error { Error::Client(Box::new(err)) } + +const CHILD_RESOLUTION_ERROR: &str = "Unexpected child info and type"; + +fn child_resolution_error() -> client::error::Error { + client::error::Error::Msg(CHILD_RESOLUTION_ERROR.to_string()) +} diff --git a/client/rpc/src/state/state_full.rs b/client/rpc/src/state/state_full.rs index c9eeb2abe4597..ea573884b7e19 100644 --- a/client/rpc/src/state/state_full.rs +++ b/client/rpc/src/state/state_full.rs @@ -45,7 +45,7 @@ use sr_primitives::{ use sr_api::Metadata; -use super::{StateBackend, error::{FutureResult, Error, Result}, client_err}; +use super::{StateBackend, error::{FutureResult, Error, Result}, client_err, child_resolution_error}; /// Ranges to query in state_queryStorage. struct QueryStorageRange { @@ -295,7 +295,8 @@ impl StateBackend for FullState, child_storage_key: StorageKey, - unique_id: StorageKey, + child_info: StorageKey, + child_type: u32, prefix: StorageKey, ) -> FutureResult> { Box::new(result( @@ -303,7 +304,8 @@ impl StateBackend for FullState StateBackend for FullState, child_storage_key: StorageKey, - unique_id: StorageKey, + child_info: StorageKey, + child_type: u32, key: StorageKey, ) -> FutureResult> { Box::new(result( @@ -321,7 +324,8 @@ impl StateBackend for FullState StateBackend for FullState, child_storage_key: StorageKey, - unique_id: StorageKey, + child_info: StorageKey, + child_type: u32, key: StorageKey, ) -> FutureResult> { Box::new(result( @@ -339,7 +344,8 @@ impl StateBackend for FullState StateBackend for LightState, _child_storage_key: StorageKey, - _unique_id: StorageKey, + _child_info: StorageKey, + _child_type: u32, _prefix: StorageKey, ) -> FutureResult> { Box::new(result(Err(client_err(ClientError::NotAvailableOnLightClient)))) @@ -246,7 +247,8 @@ impl StateBackend for LightState, child_storage_key: StorageKey, - unique_id: StorageKey, + child_info: StorageKey, + child_type: u32, key: StorageKey, ) -> FutureResult> { let block = self.block_or_best(block); @@ -257,7 +259,8 @@ impl StateBackend for LightState StateBackend for LightState, child_storage_key: StorageKey, - unique_id: StorageKey, + child_info: StorageKey, + child_type: u32, key: StorageKey, ) -> FutureResult> { Box::new(self - .child_storage(block, child_storage_key, unique_id, key) + .child_storage(block, child_storage_key, child_info, child_type, key) .and_then(|maybe_storage| result(Ok(maybe_storage.map(|storage| Blake2Hasher::hash(&storage.0)))) ) diff --git a/client/rpc/src/state/tests.rs b/client/rpc/src/state/tests.rs index 777f028a5bfe9..2559af0045640 100644 --- a/client/rpc/src/state/tests.rs +++ b/client/rpc/src/state/tests.rs @@ -29,8 +29,7 @@ use test_client::{ runtime, }; -const CHILD_UUID: &[u8] = b"unique_id"; -const CHILD_INFO: ChildInfo<'static> = ChildInfo::new_default(CHILD_UUID, None); +const CHILD_INFO: ChildInfo<'static> = ChildInfo::new_default(b"unique_id", None); #[test] fn should_return_storage() { @@ -48,7 +47,8 @@ fn should_return_storage() { let client = new_full(Arc::new(client), Subscriptions::new(Arc::new(core.executor()))); let key = StorageKey(KEY.to_vec()); let storage_key = StorageKey(STORAGE_KEY.to_vec()); - let unique_id = StorageKey(CHILD_UUID.to_vec()); + let (child_info, child_type) = CHILD_INFO.info(); + let child_info = StorageKey(child_info); assert_eq!( client.storage(key.clone(), Some(genesis_hash).into()).wait() @@ -66,7 +66,7 @@ fn should_return_storage() { ); assert_eq!( core.block_on( - client.child_storage(storage_key, unique_id, key, Some(genesis_hash).into()) + client.child_storage(storage_key, child_info, child_type, key, Some(genesis_hash).into()) .map(|x| x.map(|x| x.0.len())) ).unwrap().unwrap() as usize, CHILD_VALUE.len(), @@ -76,7 +76,8 @@ fn should_return_storage() { #[test] fn should_return_child_storage() { - let unique_id = StorageKey(CHILD_UUID.to_vec()); + let (child_info, child_type) = CHILD_INFO.info(); + let child_info = StorageKey(child_info); let core = tokio::runtime::Runtime::new().unwrap(); let client = Arc::new(test_client::TestClientBuilder::new() .add_child_storage("test", "key", CHILD_INFO, vec![42_u8]) @@ -88,16 +89,16 @@ fn should_return_child_storage() { assert_matches!( - client.child_storage(child_key.clone(), unique_id.clone(), key.clone(), Some(genesis_hash).into()).wait(), + client.child_storage(child_key.clone(), child_info.clone(), child_type, key.clone(), Some(genesis_hash).into()).wait(), Ok(Some(StorageData(ref d))) if d[0] == 42 && d.len() == 1 ); assert_matches!( - client.child_storage_hash(child_key.clone(), unique_id.clone(), key.clone(), Some(genesis_hash).into()) + client.child_storage_hash(child_key.clone(), child_info.clone(), child_type, key.clone(), Some(genesis_hash).into()) .wait().map(|x| x.is_some()), Ok(true) ); assert_matches!( - client.child_storage_size(child_key.clone(), unique_id.clone(), key.clone(), None).wait(), + client.child_storage_size(child_key.clone(), child_info.clone(), child_type, key.clone(), None).wait(), Ok(Some(1)) ); } diff --git a/client/src/light/fetcher.rs b/client/src/light/fetcher.rs index 788f614ffb85a..2265ead200a25 100644 --- a/client/src/light/fetcher.rs +++ b/client/src/light/fetcher.rs @@ -348,8 +348,7 @@ pub mod tests { use state_machine::Backend; use super::*; - const CHILD_UUID_1: &'static [u8] = b"unique_id_1"; - const CHILD_INFO_1: ChildInfo<'static> = ChildInfo::new_default(CHILD_UUID_1, None); + const CHILD_INFO_1: ChildInfo<'static> = ChildInfo::new_default(b"unique_id_1", None); type TestChecker = LightDataChecker< NativeExecutor, @@ -496,12 +495,14 @@ pub mod tests { remote_read_proof, result, ) = prepare_for_read_child_proof_check(); + let child_infos = CHILD_INFO_1.info(); assert_eq!((&local_checker as &dyn FetchChecker).check_read_child_proof( &RemoteReadChildRequest::
{ block: remote_block_header.hash(), header: remote_block_header, storage_key: b":child_storage:default:child1".to_vec(), - unique_id: CHILD_UUID_1.to_vec(), + child_info: child_infos.0, + child_type: child_infos.1, keys: vec![b"key1".to_vec()], retry_count: None, }, From b8c7e37c05e7712df74141fbe433897e59b9e9aa Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 20 Nov 2019 20:35:11 +0100 Subject: [PATCH 08/27] use root from child info for trie backend essence. --- primitives/core/storage/src/lib.rs | 14 ++++++++- .../state-machine/src/trie_backend_essence.rs | 30 +++++++++++++------ 2 files changed, 34 insertions(+), 10 deletions(-) diff --git a/primitives/core/storage/src/lib.rs b/primitives/core/storage/src/lib.rs index c453ce46c4941..82e44ce757aba 100644 --- a/primitives/core/storage/src/lib.rs +++ b/primitives/core/storage/src/lib.rs @@ -230,9 +230,21 @@ impl<'a> ChildInfo<'a> { ChildInfo::Default(ChildTrie { root: _root, unique_id, - }) => &unique_id[..], + }) => unique_id, } } + + /// Return the child reference to state if it is already known. + /// For a child trie it is its root. + pub fn root(&self) -> Option<&[u8]> { + match self { + ChildInfo::Default(ChildTrie { + root, + unique_id: _unique_id, + }) => *root, + } + } + } /// Type of child, this can be different child usage diff --git a/primitives/state-machine/src/trie_backend_essence.rs b/primitives/state-machine/src/trie_backend_essence.rs index f4529fe037e76..fcef0345c8d57 100644 --- a/primitives/state-machine/src/trie_backend_essence.rs +++ b/primitives/state-machine/src/trie_backend_essence.rs @@ -84,8 +84,14 @@ impl, H: Hasher> TrieBackendEssence { child_info: ChildInfo, key: &[u8], ) -> Result>, String> { - let root = self.storage(storage_key)? - .unwrap_or(default_child_trie_root::>(storage_key)); + let fetched_root; + let root = if let Some(root) = child_info.root() { + root + } else { + fetched_root = self.storage(storage_key)? + .unwrap_or(default_child_trie_root::>(storage_key)); + &fetched_root + }; let mut read_overlay = S::Overlay::default(); let eph = Ephemeral { @@ -95,7 +101,7 @@ impl, H: Hasher> TrieBackendEssence { let map_e = |e| format!("Trie lookup error: {}", e); - read_child_trie_value::, _>(storage_key, child_info.keyspace(), &eph, &root, key) + read_child_trie_value::, _>(storage_key, child_info.keyspace(), &eph, root, key) .map_err(map_e) } @@ -106,12 +112,18 @@ impl, H: Hasher> TrieBackendEssence { child_info: ChildInfo, f: F, ) { - let root = match self.storage(storage_key) { - Ok(v) => v.unwrap_or(default_child_trie_root::>(storage_key)), - Err(e) => { - debug!(target: "trie", "Error while iterating child storage: {}", e); - return; - } + let fetched_root; + let root = if let Some(root) = child_info.root() { + root + } else { + fetched_root = match self.storage(storage_key) { + Ok(v) => v.unwrap_or(default_child_trie_root::>(storage_key)), + Err(e) => { + debug!(target: "trie", "Error while iterating child storage: {}", e); + return; + } + }; + &fetched_root }; let mut read_overlay = S::Overlay::default(); From ee43955e3d31f84b350fc14ca0203e7b53733d65 Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 20 Nov 2019 20:52:19 +0100 Subject: [PATCH 09/27] Breaking long lines. --- client/rpc/src/state/tests.rs | 29 ++++++++++++++++++++++++----- client/src/light/fetcher.rs | 8 ++++++-- paint/support/src/storage/child.rs | 7 ++++--- 3 files changed, 34 insertions(+), 10 deletions(-) diff --git a/client/rpc/src/state/tests.rs b/client/rpc/src/state/tests.rs index 2559af0045640..251c14e0678dc 100644 --- a/client/rpc/src/state/tests.rs +++ b/client/rpc/src/state/tests.rs @@ -84,21 +84,40 @@ fn should_return_child_storage() { .build()); let genesis_hash = client.genesis_hash(); let client = new_full(client, Subscriptions::new(Arc::new(core.executor()))); - let child_key = StorageKey(well_known_keys::CHILD_STORAGE_KEY_PREFIX.iter().chain(b"test").cloned().collect()); + let child_key = StorageKey( + well_known_keys::CHILD_STORAGE_KEY_PREFIX.iter().chain(b"test").cloned().collect() + ); let key = StorageKey(b"key".to_vec()); assert_matches!( - client.child_storage(child_key.clone(), child_info.clone(), child_type, key.clone(), Some(genesis_hash).into()).wait(), + client.child_storage( + child_key.clone(), + child_info.clone(), + child_type, + key.clone(), + Some(genesis_hash).into(), + ).wait(), Ok(Some(StorageData(ref d))) if d[0] == 42 && d.len() == 1 ); assert_matches!( - client.child_storage_hash(child_key.clone(), child_info.clone(), child_type, key.clone(), Some(genesis_hash).into()) - .wait().map(|x| x.is_some()), + client.child_storage_hash( + child_key.clone(), + child_info.clone(), + child_type, + key.clone(), + Some(genesis_hash).into(), + ).wait().map(|x| x.is_some()), Ok(true) ); assert_matches!( - client.child_storage_size(child_key.clone(), child_info.clone(), child_type, key.clone(), None).wait(), + client.child_storage_size( + child_key.clone(), + child_info.clone(), + child_type, + key.clone(), + None, + ).wait(), Ok(Some(1)) ); } diff --git a/client/src/light/fetcher.rs b/client/src/light/fetcher.rs index 2265ead200a25..3abd8f6bda359 100644 --- a/client/src/light/fetcher.rs +++ b/client/src/light/fetcher.rs @@ -400,8 +400,12 @@ pub mod tests { use test_client::TestClientBuilderExt; // prepare remote client let remote_client = test_client::TestClientBuilder::new() - .add_extra_child_storage(b":child_storage:default:child1".to_vec(), CHILD_INFO_1, b"key1".to_vec(), b"value1".to_vec()) - .build(); + .add_extra_child_storage( + b":child_storage:default:child1".to_vec(), + CHILD_INFO_1, + b"key1".to_vec(), + b"value1".to_vec(), + ).build(); let remote_block_id = BlockId::Number(0); let remote_block_hash = remote_client.block_hash(0).unwrap().unwrap(); let mut remote_block_header = remote_client.header(&remote_block_id).unwrap().unwrap(); diff --git a/paint/support/src/storage/child.rs b/paint/support/src/storage/child.rs index a63e1107d5043..2763c8523cc56 100644 --- a/paint/support/src/storage/child.rs +++ b/paint/support/src/storage/child.rs @@ -20,10 +20,11 @@ //! Note that `storage_key` must be unique and strong (strong in the sense of being long enough to //! avoid collision from a resistant hash function (which unique implies)). //! -//! A key collision free unique id is required as parameter this is use to avoid key collision between child tries. +//! A key collision free unique id is required as parameter this is use to avoid key collision +//! between child tries. //! This unique id management and generation responsability is delegated to paint module. -//! In the future this id could be generated and stored out of chain state, and using None as a parameter will result into this -//! mode of operation. +//! In the future this id could be generated and stored out of chain state, and using None as a +//! parameter will result into this mode of operation. //! //! To be able to live with managed unique id it is very important that paint module uses real //! unique id. If paint use somehow weak id, care should be taken to not conflict to future scheme From a8818d93c08faa1817c632bb43b9147ce62193fa Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 21 Nov 2019 11:35:31 +0100 Subject: [PATCH 10/27] Update doc and clean pr a bit. --- client/api/src/light.rs | 6 ++-- client/network/src/protocol/message.rs | 5 +-- palette/support/src/storage/child.rs | 8 +---- primitives/core/storage/src/lib.rs | 43 ++++++++++++++----------- primitives/sr-io/src/lib.rs | 3 -- primitives/state-machine/src/backend.rs | 3 +- primitives/state-machine/src/ext.rs | 14 +++++--- 7 files changed, 42 insertions(+), 40 deletions(-) diff --git a/client/api/src/light.rs b/client/api/src/light.rs index 6f5aea9052b51..31afe4265fb60 100644 --- a/client/api/src/light.rs +++ b/client/api/src/light.rs @@ -86,10 +86,10 @@ pub struct RemoteReadChildRequest { pub header: Header, /// Storage key for child. pub storage_key: Vec, - /// Child trie encoded information + /// Child trie source information. pub child_info: Vec, - /// Child type needed to resolve child info - /// and choose child implementation. + /// Child type, its required to resolve `child_info` + /// content and choose child implementation. pub child_type: u32, /// Child storage key to read. pub keys: Vec>, diff --git a/client/network/src/protocol/message.rs b/client/network/src/protocol/message.rs index c524eeddcf31a..8ba237eecda8e 100644 --- a/client/network/src/protocol/message.rs +++ b/client/network/src/protocol/message.rs @@ -368,9 +368,10 @@ pub mod generic { pub block: H, /// Child Storage key. pub storage_key: Vec, - /// Child trie associated info. + /// Child trie source information. pub child_info: Vec, - /// Child trie associated type. + /// Child type, its required to resolve `child_info` + /// content and choose child implementation. pub child_type: u32, /// Storage key. pub keys: Vec>, diff --git a/palette/support/src/storage/child.rs b/palette/support/src/storage/child.rs index 5f5357a7142b3..77cc3d9c1fa6f 100644 --- a/palette/support/src/storage/child.rs +++ b/palette/support/src/storage/child.rs @@ -20,17 +20,11 @@ //! Note that `storage_key` must be unique and strong (strong in the sense of being long enough to //! avoid collision from a resistant hash function (which unique implies)). //! -//! A key collision free unique id is required as parameter this is use to avoid key collision +//! A **key collision free** unique id is required as parameter to avoid key collision //! between child tries. //! This unique id management and generation responsability is delegated to pallet module. //! In the future this id could be generated and stored out of chain state, and using None as a //! parameter will result into this mode of operation. -//! -//! To be able to live with managed unique id it is very important that pallet module uses real -//! unique id. If pallet uses somehow weak id, care should be taken to not conflict to future scheme -//! of managed unique id which will be compact SCALE encoded integer. -//! Empty unique id is also a reserved value (will be use internally for managed unique id as a -//! parameter in runtime io). // NOTE: could replace unhashed by having only one kind of storage (root being null storage key (storage_key can become Option<&[u8]>). use crate::rstd::prelude::*; diff --git a/primitives/core/storage/src/lib.rs b/primitives/core/storage/src/lib.rs index 82e44ce757aba..73285acab0d15 100644 --- a/primitives/core/storage/src/lib.rs +++ b/primitives/core/storage/src/lib.rs @@ -161,12 +161,13 @@ impl<'a> ChildStorageKey<'a> { } #[derive(Clone, Copy)] -/// Information related to a child trie query. +/// Information related to a child state. pub enum ChildInfo<'a> { Default(ChildTrie<'a>), } -/// Owned version of ChildInfo +/// Owned version of `ChildInfo`. +/// To be use in persistence layers. #[derive(Debug, Clone)] #[cfg_attr(feature = "std", derive(PartialEq, Eq, Hash, PartialOrd, Ord))] pub enum OwnedChildInfo { @@ -174,7 +175,7 @@ pub enum OwnedChildInfo { } impl<'a> ChildInfo<'a> { - /// Instantiate info for a default child trie. + /// Instantiates information for a default child trie. pub const fn new_default(unique_id: &'a[u8], root: Option<&'a[u8]>) -> Self { ChildInfo::Default(ChildTrie { unique_id, @@ -182,7 +183,7 @@ impl<'a> ChildInfo<'a> { }) } - /// Create owned child info. + /// Instantiates a owned version of this child info. pub fn to_owned(&self) -> OwnedChildInfo { match self { ChildInfo::Default(ChildTrie { unique_id, root }) @@ -193,7 +194,7 @@ impl<'a> ChildInfo<'a> { } } - /// Create child info from a linear byte encoded value and a type. + /// Create child info from a linear byte packed value and a given type. pub fn resolve_child_info(child_type: u32, data: &'a[u8]) -> Option { match child_type { x if x == ChildType::CryptoUniqueId as u32 => Some(ChildInfo::new_default(data, None)), @@ -204,7 +205,7 @@ impl<'a> ChildInfo<'a> { } } - /// Return a single byte vector containing child info content and this child info type. + /// Return a single byte vector containing packed child info content and its child info type. /// This can be use as input for `resolve_child_info`. pub fn info(&self) -> (Vec, u32) { match self { @@ -224,7 +225,9 @@ impl<'a> ChildInfo<'a> { } } - /// Return byte sequence (keyspace) that can be use for underlying db to isolate keys. + /// Return byte sequence (keyspace) that can be use by underlying db to isolate keys. + /// This is a unique id of the child trie. The collision resistance of this value + /// depends on the type of child info use. For `ChildInfo::Default` it is and need to be. pub fn keyspace(&self) -> &[u8] { match self { ChildInfo::Default(ChildTrie { @@ -235,6 +238,8 @@ impl<'a> ChildInfo<'a> { } /// Return the child reference to state if it is already known. + /// If it returns `None` the information will need to be fetch by + /// the caller. /// For a child trie it is its root. pub fn root(&self) -> Option<&[u8]> { match self { @@ -247,19 +252,20 @@ impl<'a> ChildInfo<'a> { } -/// Type of child, this can be different child usage -/// of api variant, it is not strictly speaking different -/// child kind. +/// Type of child. +/// It does not strictly define different child type, it can also +/// be related to technical consideration or api variant. #[repr(u32)] pub enum ChildType { /// Default, it uses a cryptographic strong unique id as input. CryptoUniqueId = 1, - /// Default, but with root send when querying. Root is a 32 byte hash. + /// Default, but with a root registerd to skip root fetch when querying. + /// Root is a 32 byte hash. CryptoUniqueIdRoot32Api = 2, } impl OwnedChildInfo { - /// Instantiate info for a default child trie. + /// Instantiates info for a default child trie. pub fn new_default(unique_id: Vec, root: Option>) -> Self { OwnedChildInfo::Default(OwnedChildTrie { unique_id, @@ -267,7 +273,7 @@ impl OwnedChildInfo { }) } - /// Get reference to child info. + /// Get `ChildInfo` reference to this owned child info. pub fn as_ref(&self) -> ChildInfo { match self { OwnedChildInfo::Default(OwnedChildTrie { unique_id, root }) @@ -280,6 +286,7 @@ impl OwnedChildInfo { } /// A child trie of default type. +/// Default is the same implementation as the top trie. /// It share its trie node storage with any kind of key, /// and its unique id needs to be collision free (eg strong /// crypto hash). @@ -289,18 +296,18 @@ pub struct ChildTrie<'a> { /// to avoid querying it explicitly. pub root: Option<&'a[u8]>, - /// Unique id must but unique and free of any possible key collision. + /// Unique id must but unique and free of any possible key collision + /// (depending on its storage behavior). pub unique_id: &'a[u8], } -/// Owned version of default child trie. +/// Owned version of default child trie `ChildTrie`. #[derive(Debug, Clone)] #[cfg_attr(feature = "std", derive(PartialEq, Eq, Hash, PartialOrd, Ord))] pub struct OwnedChildTrie { - /// If root was fetch it can be memoïzed in this field - /// to avoid querying it explicitly. + /// See `ChildTrie` reference field documentation. pub root: Option>, - /// Unique id must but unique and free of any possible key collision. + /// See `ChildTrie` reference field documentation. pub unique_id: Vec, } diff --git a/primitives/sr-io/src/lib.rs b/primitives/sr-io/src/lib.rs index c39e1a776059e..77969f4a59e6c 100644 --- a/primitives/sr-io/src/lib.rs +++ b/primitives/sr-io/src/lib.rs @@ -106,7 +106,6 @@ pub trait Storage { /// Get `key` from storage, placing the value into `value_out` and return the number of /// bytes that the entry in storage has beyond the offset or `None` if the storage entry /// doesn't exist at all. - /// A key collision free unique id is required as parameter. fn read(&self, key: &[u8], value_out: &mut [u8], value_offset: u32) -> Option { self.storage(key).map(|value| { let value_offset = value_offset as usize; @@ -120,7 +119,6 @@ pub trait Storage { /// Get `key` from child storage, placing the value into `value_out` and return the number /// of bytes that the entry in storage has beyond the offset or `None` if the storage entry /// doesn't exist at all. - /// A key collision free unique id is required as parameter. /// If `value_out` length is smaller than the returned length, only `value_out` length bytes /// are copied into `value_out`. fn child_read( @@ -151,7 +149,6 @@ pub trait Storage { } /// Set `key` to `value` in the child storage denoted by `child_storage_key`. - /// A key collision free unique id is required as parameter. fn child_set( &mut self, child_storage_key: &[u8], diff --git a/primitives/state-machine/src/backend.rs b/primitives/state-machine/src/backend.rs index bf087075312cb..707601e5ffa20 100644 --- a/primitives/state-machine/src/backend.rs +++ b/primitives/state-machine/src/backend.rs @@ -27,7 +27,6 @@ use trie::{ }; use primitives::storage::{ChildInfo, OwnedChildInfo}; - pub(crate) type StorageTuple = ( HashMap, Vec>, HashMap, (HashMap, Vec>, OwnedChildInfo)>, @@ -622,7 +621,7 @@ pub(crate) fn insert_into_memory_db(mdb: &mut MemoryDB, input: I) -> Op #[cfg(test)] mod tests { use super::*; - + /// Assert in memory backend with only child trie keys works as trie backend. #[test] fn in_memory_with_child_trie_only() { diff --git a/primitives/state-machine/src/ext.rs b/primitives/state-machine/src/ext.rs index c710b94b138e2..65bd0a6f17c76 100644 --- a/primitives/state-machine/src/ext.rs +++ b/primitives/state-machine/src/ext.rs @@ -469,9 +469,11 @@ where self.overlay.committed.children.get(storage_key) .into_iter() .flat_map(|(map, _)| map.iter().map(|(k, v)| (k.clone(), v.value.clone()))) - .chain(self.overlay.prospective.children.get(storage_key) - .into_iter() - .flat_map(|(map, _)| map.iter().map(|(k, v)| (k.clone(), v.value.clone())))), + .chain( + self.overlay.prospective.children.get(storage_key) + .into_iter() + .flat_map(|(map, _)| map.iter().map(|(k, v)| (k.clone(), v.value.clone()))) + ), self.overlay.child_info(storage_key).cloned() .expect("child info initialized in either committed or prospective"), ) @@ -516,9 +518,11 @@ where let delta = self.overlay.committed.children.get(storage_key) .into_iter() .flat_map(|(map, _)| map.clone().into_iter().map(|(k, v)| (k, v.value))) - .chain(self.overlay.prospective.children.get(storage_key) + .chain( + self.overlay.prospective.children.get(storage_key) .into_iter() - .flat_map(|(map, _)| map.clone().into_iter().map(|(k, v)| (k, v.value)))); + .flat_map(|(map, _)| map.clone().into_iter().map(|(k, v)| (k, v.value))) + ); self.backend.child_storage_root(storage_key, child_info.as_ref(), delta) }; From cac9330406b762b79fe745e46e7e6b1817449f0c Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 27 Nov 2019 22:24:22 +0100 Subject: [PATCH 11/27] fix error type --- client/rpc/src/state/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/rpc/src/state/mod.rs b/client/rpc/src/state/mod.rs index 70eb872811edf..b3180f739e4a3 100644 --- a/client/rpc/src/state/mod.rs +++ b/client/rpc/src/state/mod.rs @@ -354,6 +354,6 @@ fn client_err(err: sp_blockchain::Error) -> Error { const CHILD_RESOLUTION_ERROR: &str = "Unexpected child info and type"; -fn child_resolution_error() -> client::error::Error { - client::error::Error::Msg(CHILD_RESOLUTION_ERROR.to_string()) +fn child_resolution_error() -> sp_blockchain::Error { + sp_blockchain::Error::Msg(CHILD_RESOLUTION_ERROR.to_string()) } From c05bc80e13b7cf4e3811d1f04903b25f43c3f572 Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 28 Nov 2019 16:20:50 +0100 Subject: [PATCH 12/27] Restore removed doc on merge and update sr-io doc. --- frame/support/src/storage/child.rs | 2 -- primitives/sr-io/src/lib.rs | 29 +++++++++++++++++++++++++++-- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/frame/support/src/storage/child.rs b/frame/support/src/storage/child.rs index 46101f0c8bab2..147d95ec6f897 100644 --- a/frame/support/src/storage/child.rs +++ b/frame/support/src/storage/child.rs @@ -23,8 +23,6 @@ //! A **key collision free** unique id is required as parameter to avoid key collision //! between child tries. //! This unique id management and generation responsability is delegated to pallet module. -//! In the future this id could be generated and stored out of chain state, and using None as a -//! parameter will result into this mode of operation. // NOTE: could replace unhashed by having only one kind of storage (root being null storage key (storage_key can become Option<&[u8]>). use crate::rstd::prelude::*; diff --git a/primitives/sr-io/src/lib.rs b/primitives/sr-io/src/lib.rs index 50b4bf2fc5d03..97904be192233 100644 --- a/primitives/sr-io/src/lib.rs +++ b/primitives/sr-io/src/lib.rs @@ -91,8 +91,17 @@ pub trait Storage { self.storage(key).map(|s| s.to_vec()) } - /// Returns the data for `key` in the child storage or `None` if the key can not be found. - /// A key collision free unique id is required as parameter. + /// All Child api uses : + /// - A `child_storage_key` to define the anchor point for the child proof + /// (commonly the location where the child root is stored in its parent trie). + /// - A `child_storage_types` to identify the kind of the child type and how its + /// `child definition` parameter is encoded. + /// - A `child_definition_parameter` which is the additional information required + /// to use the child trie. For instance defaults child tries requires this to + /// contain a collision free unique id. + /// + /// This function specifically returns the data for `key` in the child storage or `None` + /// if the key can not be found. fn child_get( &self, child_storage_key: &[u8], @@ -109,6 +118,8 @@ pub trait Storage { /// Get `key` from storage, placing the value into `value_out` and return the number of /// bytes that the entry in storage has beyond the offset or `None` if the storage entry /// doesn't exist at all. + /// If `value_out` length is smaller than the returned length, only `value_out` length bytes + /// are copied into `value_out`. fn read(&self, key: &[u8], value_out: &mut [u8], value_offset: u32) -> Option { self.storage(key).map(|value| { let value_offset = value_offset as usize; @@ -124,6 +135,8 @@ pub trait Storage { /// doesn't exist at all. /// If `value_out` length is smaller than the returned length, only `value_out` length bytes /// are copied into `value_out`. + /// + /// See `child_get` for common child api parameters. fn child_read( &self, child_storage_key: &[u8], @@ -152,6 +165,8 @@ pub trait Storage { } /// Set `key` to `value` in the child storage denoted by `child_storage_key`. + /// + /// See `child_get` for common child api parameters. fn child_set( &mut self, child_storage_key: &[u8], @@ -172,6 +187,8 @@ pub trait Storage { } /// Clear the given child storage of the given `key` and its value. + /// + /// See `child_get` for common child api parameters. fn child_clear( &mut self, child_storage_key: &[u8], @@ -186,6 +203,8 @@ pub trait Storage { } /// Clear an entire child storage. + /// + /// See `child_get` for common child api parameters. fn child_storage_kill( &mut self, child_storage_key: &[u8], @@ -204,6 +223,8 @@ pub trait Storage { } /// Check whether the given `key` exists in storage. + /// + /// See `child_get` for common child api parameters. fn child_exists( &self, child_storage_key: &[u8], @@ -223,6 +244,8 @@ pub trait Storage { } /// Clear the child storage of each key-value pair where the key starts with the given `prefix`. + /// + /// See `child_get` for common child api parameters. fn child_clear_prefix( &mut self, child_storage_key: &[u8], @@ -250,6 +273,8 @@ pub trait Storage { /// The hashing algorithm is defined by the `Block`. /// /// Returns the SCALE encoded hash. + /// + /// See `child_get` for common child api parameters. fn child_root( &mut self, child_storage_key: &[u8], From bdff8437187ee75ad351afe35ef0f9782e363d72 Mon Sep 17 00:00:00 2001 From: cheme Date: Fri, 29 Nov 2019 13:23:50 +0100 Subject: [PATCH 13/27] Switch child storage api to use directly unique id, if managed id where to be put in place, the api will change at this time. --- frame/contracts/src/lib.rs | 6 ++-- frame/support/src/storage/child.rs | 44 ++++++++++++++---------------- 2 files changed, 24 insertions(+), 26 deletions(-) diff --git a/frame/contracts/src/lib.rs b/frame/contracts/src/lib.rs index a48b7ecff7589..1fe13059a5161 100644 --- a/frame/contracts/src/lib.rs +++ b/frame/contracts/src/lib.rs @@ -225,15 +225,15 @@ pub struct RawAliveContractInfo { impl RawAliveContractInfo { /// Associated child trie unique id is built from the hash part of the trie id. - pub fn child_trie_unique_id(&self) -> Option<&[u8]> { + pub fn child_trie_unique_id(&self) -> &[u8] { trie_unique_id(&self.trie_id[..]) } } /// Associated child trie unique id is built from the hash part of the trie id. -pub(crate) fn trie_unique_id(trie_id: &[u8]) -> Option<&[u8]> { +pub(crate) fn trie_unique_id(trie_id: &[u8]) -> &[u8] { let start = CHILD_STORAGE_KEY_PREFIX.len() + b"default:".len(); - Some(&trie_id[start ..]) + &trie_id[start ..] } pub type TombstoneContractInfo = diff --git a/frame/support/src/storage/child.rs b/frame/support/src/storage/child.rs index 147d95ec6f897..030f747f33c16 100644 --- a/frame/support/src/storage/child.rs +++ b/frame/support/src/storage/child.rs @@ -28,17 +28,15 @@ use crate::rstd::prelude::*; use codec::{Codec, Encode, Decode}; -/// Unique id managed by the system is not implemented yet. -const SOME: &'static str = "Managed child unique ids are not yet implemented"; /// Return the value of the item in storage under `key`, or `None` if there is no explicit entry. pub fn get( storage_key: &[u8], - unique_id: Option<&[u8]>, + unique_id: &[u8], key: &[u8], ) -> Option { runtime_io::storage::child_get( storage_key, - unique_id.expect(SOME), + unique_id, primitives::storage::ChildType::CryptoUniqueId as u32, key, ).and_then(|v| { @@ -54,7 +52,7 @@ pub fn get( /// explicit entry. pub fn get_or_default( storage_key: &[u8], - unique_id: Option<&[u8]>, + unique_id: &[u8], key: &[u8], ) -> T { get(storage_key, unique_id, key).unwrap_or_else(Default::default) @@ -64,7 +62,7 @@ pub fn get_or_default( /// explicit entry. pub fn get_or( storage_key: &[u8], - unique_id: Option<&[u8]>, + unique_id: &[u8], key: &[u8], default_value: T, ) -> T { @@ -75,7 +73,7 @@ pub fn get_or( /// explicit entry. pub fn get_or_else T>( storage_key: &[u8], - unique_id: Option<&[u8]>, + unique_id: &[u8], key: &[u8], default_value: F, ) -> T { @@ -85,14 +83,14 @@ pub fn get_or_else T>( /// Put `value` in storage under `key`. pub fn put( storage_key: &[u8], - unique_id: Option<&[u8]>, + unique_id: &[u8], key: &[u8], value: &T, ) { value.using_encoded(|slice| runtime_io::storage::child_set( storage_key, - unique_id.expect(SOME), + unique_id, primitives::storage::ChildType::CryptoUniqueId as u32, key, slice, @@ -103,7 +101,7 @@ pub fn put( /// Remove `key` from storage, returning its value if it had an explicit entry or `None` otherwise. pub fn take( storage_key: &[u8], - unique_id: Option<&[u8]>, + unique_id: &[u8], key: &[u8], ) -> Option { let r = get(storage_key, unique_id, key); @@ -117,7 +115,7 @@ pub fn take( /// the default for its type. pub fn take_or_default( storage_key: &[u8], - unique_id: Option<&[u8]>, + unique_id: &[u8], key: &[u8], ) -> T { take(storage_key, unique_id, key).unwrap_or_else(Default::default) @@ -127,7 +125,7 @@ pub fn take_or_default( /// explicit entry. Ensure there is no explicit entry on return. pub fn take_or( storage_key: &[u8], - unique_id: Option<&[u8]>, + unique_id: &[u8], key: &[u8], default_value: T, ) -> T { @@ -138,7 +136,7 @@ pub fn take_or( /// explicit entry. Ensure there is no explicit entry on return. pub fn take_or_else T>( storage_key: &[u8], - unique_id: Option<&[u8]>, + unique_id: &[u8], key: &[u8], default_value: F, ) -> T { @@ -148,11 +146,11 @@ pub fn take_or_else T>( /// Check to see if `key` has an explicit entry in storage. pub fn exists( storage_key: &[u8], - unique_id: Option<&[u8]>, + unique_id: &[u8], key: &[u8], ) -> bool { runtime_io::storage::child_read( - storage_key, unique_id.expect(SOME), + storage_key, unique_id, primitives::storage::ChildType::CryptoUniqueId as u32, key, &mut [0;0][..], 0, ).is_some() @@ -161,11 +159,11 @@ pub fn exists( /// Remove all `storage_key` key/values pub fn kill_storage( storage_key: &[u8], - unique_id: Option<&[u8]>, + unique_id: &[u8], ) { runtime_io::storage::child_storage_kill( storage_key, - unique_id.expect(SOME), + unique_id, primitives::storage::ChildType::CryptoUniqueId as u32, ) } @@ -173,12 +171,12 @@ pub fn kill_storage( /// Ensure `key` has no explicit entry in storage. pub fn kill( storage_key: &[u8], - unique_id: Option<&[u8]>, + unique_id: &[u8], key: &[u8], ) { runtime_io::storage::child_clear( storage_key, - unique_id.expect(SOME), + unique_id, primitives::storage::ChildType::CryptoUniqueId as u32, key, ); @@ -187,12 +185,12 @@ pub fn kill( /// Get a Vec of bytes from storage. pub fn get_raw( storage_key: &[u8], - unique_id: Option<&[u8]>, + unique_id: &[u8], key: &[u8], ) -> Option> { runtime_io::storage::child_get( storage_key, - unique_id.expect(SOME), + unique_id, primitives::storage::ChildType::CryptoUniqueId as u32, key, ) @@ -201,13 +199,13 @@ pub fn get_raw( /// Put a raw byte slice into storage. pub fn put_raw( storage_key: &[u8], - unique_id: Option<&[u8]>, + unique_id: &[u8], key: &[u8], value: &[u8], ) { runtime_io::storage::child_set( storage_key, - unique_id.expect(SOME), + unique_id, primitives::storage::ChildType::CryptoUniqueId as u32, key, value, From 9ce77bbd7b64869522a95602038eff2a600029d2 Mon Sep 17 00:00:00 2001 From: cheme Date: Fri, 29 Nov 2019 14:21:49 +0100 Subject: [PATCH 14/27] Clean deprecated host interface from child. --- .../executor/src/deprecated_host_interface.rs | 180 +----------------- .../state-machine/src/overlayed_changes.rs | 1 + 2 files changed, 2 insertions(+), 179 deletions(-) diff --git a/client/executor/src/deprecated_host_interface.rs b/client/executor/src/deprecated_host_interface.rs index 4bdeb655f2276..e7dbc384a44c3 100644 --- a/client/executor/src/deprecated_host_interface.rs +++ b/client/executor/src/deprecated_host_interface.rs @@ -20,7 +20,7 @@ use codec::Encode; use std::{convert::TryFrom, str}; use primitives::{ blake2_128, blake2_256, twox_64, twox_128, twox_256, ed25519, sr25519, keccak_256, Blake2Hasher, Pair, - crypto::KeyTypeId, offchain, storage::ChildType, + crypto::KeyTypeId, offchain, }; use trie::{TrieConfiguration, trie_types::Layout}; use wasm_interface::{ @@ -217,51 +217,6 @@ impl_wasm_host_interface! { Ok(runtime_io::storage::set(&key, &value)) } - ext_set_child_storage( - storage_key_data: Pointer, - storage_key_len: WordSize, - key_data: Pointer, - key_len: WordSize, - value_data: Pointer, - value_len: WordSize, - ) { - let storage_key = context.read_memory(storage_key_data, storage_key_len) - .map_err(|_| "Invalid attempt to determine storage_key in ext_set_child_storage")?; - let key = context.read_memory(key_data, key_len) - .map_err(|_| "Invalid attempt to determine key in ext_set_child_storage")?; - let value = context.read_memory(value_data, value_len) - .map_err(|_| "Invalid attempt to determine value in ext_set_child_storage")?; - - // This can only work with archive mode. - Ok(runtime_io::storage::child_set( - &storage_key, - &[], - ChildType::CryptoUniqueId as u32, - &key, - &value, - )) - } - - ext_clear_child_storage( - storage_key_data: Pointer, - storage_key_len: WordSize, - key_data: Pointer, - key_len: WordSize, - ) { - let storage_key = context.read_memory(storage_key_data, storage_key_len) - .map_err(|_| "Invalid attempt to determine storage_key in ext_clear_child_storage")?; - let key = context.read_memory(key_data, key_len) - .map_err(|_| "Invalid attempt to determine key in ext_clear_child_storage")?; - - // This can only work with archive mode. - Ok(runtime_io::storage::child_clear( - &storage_key, - &[], - ChildType::CryptoUniqueId as u32, - &key, - )) - } - ext_clear_storage(key_data: Pointer, key_len: WordSize) { let key = context.read_memory(key_data, key_len) .map_err(|_| "Invalid attempt to determine key in ext_clear_storage")?; @@ -274,62 +229,12 @@ impl_wasm_host_interface! { Ok(if runtime_io::storage::exists(&key) { 1 } else { 0 }) } - ext_exists_child_storage( - storage_key_data: Pointer, - storage_key_len: WordSize, - key_data: Pointer, - key_len: WordSize, - ) -> u32 { - let storage_key = context.read_memory(storage_key_data, storage_key_len) - .map_err(|_| "Invalid attempt to determine storage_key in ext_exists_child_storage")?; - let key = context.read_memory(key_data, key_len) - .map_err(|_| "Invalid attempt to determine key in ext_exists_child_storage")?; - - // This can only work with archive mode. - Ok(if runtime_io::storage::child_exists( - &storage_key, - &[], - ChildType::CryptoUniqueId as u32, - &key, - ) { 1 } else { 0 }) - } - ext_clear_prefix(prefix_data: Pointer, prefix_len: WordSize) { let prefix = context.read_memory(prefix_data, prefix_len) .map_err(|_| "Invalid attempt to determine prefix in ext_clear_prefix")?; Ok(runtime_io::storage::clear_prefix(&prefix)) } - ext_clear_child_prefix( - storage_key_data: Pointer, - storage_key_len: WordSize, - prefix_data: Pointer, - prefix_len: WordSize, - ) { - let storage_key = context.read_memory(storage_key_data, storage_key_len) - .map_err(|_| "Invalid attempt to determine storage_key in ext_clear_child_prefix")?; - let prefix = context.read_memory(prefix_data, prefix_len) - .map_err(|_| "Invalid attempt to determine prefix in ext_clear_child_prefix")?; - // This can only work with archive mode. - Ok(runtime_io::storage::child_clear_prefix( - &storage_key, - &[], - ChildType::CryptoUniqueId as u32, - &prefix, - )) - } - - ext_kill_child_storage(storage_key_data: Pointer, storage_key_len: WordSize) { - let storage_key = context.read_memory(storage_key_data, storage_key_len) - .map_err(|_| "Invalid attempt to determine storage_key in ext_kill_child_storage")?; - // This can only work with archive mode. - Ok(runtime_io::storage::child_storage_kill( - &storage_key, - &[], - ChildType::CryptoUniqueId as u32, - )) - } - ext_get_allocated_storage( key_data: Pointer, key_len: WordSize, @@ -352,38 +257,6 @@ impl_wasm_host_interface! { } } - ext_get_allocated_child_storage( - storage_key_data: Pointer, - storage_key_len: WordSize, - key_data: Pointer, - key_len: WordSize, - written_out: Pointer, - ) -> Pointer { - let storage_key = context.read_memory(storage_key_data, storage_key_len) - .map_err(|_| "Invalid attempt to determine storage_key in ext_get_allocated_child_storage")?; - let key = context.read_memory(key_data, key_len) - .map_err(|_| "Invalid attempt to determine key in ext_get_allocated_child_storage")?; - - // This can only work with archive mode. - if let Some(value) = runtime_io::storage::child_get( - &storage_key, - &[], - ChildType::CryptoUniqueId as u32, - &key, - ) { - let offset = context.allocate_memory(value.len() as u32)?; - context.write_memory(offset, &value) - .map_err(|_| "Invalid attempt to set memory in ext_get_allocated_child_storage")?; - context.write_primitive(written_out, value.len() as u32) - .map_err(|_| "Invalid attempt to write written_out in ext_get_allocated_child_storage")?; - Ok(offset) - } else { - context.write_primitive(written_out, u32::max_value()) - .map_err(|_| "Invalid attempt to write failed written_out in ext_get_allocated_child_storage")?; - Ok(Pointer::null()) - } - } - ext_get_storage_into( key_data: Pointer, key_len: WordSize, @@ -405,62 +278,11 @@ impl_wasm_host_interface! { } } - ext_get_child_storage_into( - storage_key_data: Pointer, - storage_key_len: WordSize, - key_data: Pointer, - key_len: WordSize, - value_data: Pointer, - value_len: WordSize, - value_offset: WordSize, - ) -> WordSize { - let storage_key = context.read_memory(storage_key_data, storage_key_len) - .map_err(|_| "Invalid attempt to determine storage_key in ext_get_child_storage_into")?; - let key = context.read_memory(key_data, key_len) - .map_err(|_| "Invalid attempt to get key in ext_get_child_storage_into")?; - - // This can only work with archive mode. - if let Some(value) = runtime_io::storage::child_get( - &storage_key, - &[], - ChildType::CryptoUniqueId as u32, - &key, - ) { - let data = &value[value.len().min(value_offset as usize)..]; - let written = std::cmp::min(value_len as usize, data.len()); - context.write_memory(value_data, &data[..written]) - .map_err(|_| "Invalid attempt to get value in ext_get_child_storage_into")?; - Ok(value.len() as u32) - } else { - Ok(u32::max_value()) - } - } - ext_storage_root(result: Pointer) { context.write_memory(result, runtime_io::storage::root().as_ref()) .map_err(|_| "Invalid attempt to set memory in ext_storage_root".into()) } - ext_child_storage_root( - storage_key_data: Pointer, - storage_key_len: WordSize, - written_out: Pointer, - ) -> Pointer { - let storage_key = context.read_memory(storage_key_data, storage_key_len) - .map_err(|_| "Invalid attempt to determine storage_key in ext_child_storage_root")?; - // This can only work with archive mode. - let value = runtime_io::storage::child_root( - &storage_key, - ); - - let offset = context.allocate_memory(value.len() as u32)?; - context.write_memory(offset, &value) - .map_err(|_| "Invalid attempt to set memory in ext_child_storage_root")?; - context.write_primitive(written_out, value.len() as u32) - .map_err(|_| "Invalid attempt to write written_out in ext_child_storage_root")?; - Ok(offset) - } - ext_storage_changes_root( parent_hash_data: Pointer, _len: WordSize, diff --git a/primitives/state-machine/src/overlayed_changes.rs b/primitives/state-machine/src/overlayed_changes.rs index d4d3622b869c4..fc4d42191a28d 100644 --- a/primitives/state-machine/src/overlayed_changes.rs +++ b/primitives/state-machine/src/overlayed_changes.rs @@ -159,6 +159,7 @@ impl OverlayedChanges { let extrinsic_index = self.extrinsic_index(); let map_entry = self.prospective.children.entry(storage_key) .or_insert_with(|| (Default::default(), child_info.to_owned())); + debug_assert!(map_entry.1 == child_info, "No child change implemented"); let entry = map_entry.0.entry(key).or_default(); entry.value = val; From 92c7864a1dd9d05f0944b942172e0dea82057747 Mon Sep 17 00:00:00 2001 From: cheme Date: Fri, 29 Nov 2019 15:11:18 +0100 Subject: [PATCH 15/27] Removing assertion on child info (can fail depending on root memoization). --- primitives/sr-primitives/src/lib.rs | 1 - primitives/state-machine/src/overlayed_changes.rs | 1 - 2 files changed, 2 deletions(-) diff --git a/primitives/sr-primitives/src/lib.rs b/primitives/sr-primitives/src/lib.rs index 649142820f9ce..788e27a6e3d85 100644 --- a/primitives/sr-primitives/src/lib.rs +++ b/primitives/sr-primitives/src/lib.rs @@ -154,7 +154,6 @@ impl BuildStorage for (StorageOverlay, ChildrenStorageOverlay) { let k = k.clone(); if let Some(map) = storage.1.get_mut(&k) { map.0.extend(other_map.0.iter().map(|(k, v)| (k.clone(), v.clone()))); - debug_assert!(map.1 == other_map.1, "No child change implemented"); } else { storage.1.insert(k, other_map.clone()); } diff --git a/primitives/state-machine/src/overlayed_changes.rs b/primitives/state-machine/src/overlayed_changes.rs index fc4d42191a28d..d4d3622b869c4 100644 --- a/primitives/state-machine/src/overlayed_changes.rs +++ b/primitives/state-machine/src/overlayed_changes.rs @@ -159,7 +159,6 @@ impl OverlayedChanges { let extrinsic_index = self.extrinsic_index(); let map_entry = self.prospective.children.entry(storage_key) .or_insert_with(|| (Default::default(), child_info.to_owned())); - debug_assert!(map_entry.1 == child_info, "No child change implemented"); let entry = map_entry.0.entry(key).or_default(); entry.value = val; From fc91068c19aa5b5b6ade72f371fee133b5c681d5 Mon Sep 17 00:00:00 2001 From: cheme Date: Fri, 29 Nov 2019 15:52:01 +0100 Subject: [PATCH 16/27] merging child info in the overlay when possible. --- primitives/core/storage/src/lib.rs | 27 +++++++++++++++++++ primitives/sr-primitives/src/lib.rs | 3 +++ primitives/state-machine/src/basic.rs | 2 +- .../state-machine/src/overlayed_changes.rs | 7 +++++ 4 files changed, 38 insertions(+), 1 deletion(-) diff --git a/primitives/core/storage/src/lib.rs b/primitives/core/storage/src/lib.rs index 73285acab0d15..56eb15785b2de 100644 --- a/primitives/core/storage/src/lib.rs +++ b/primitives/core/storage/src/lib.rs @@ -273,6 +273,14 @@ impl OwnedChildInfo { }) } + /// Try to update with another instance, return false if both instance + /// are not compatible. + pub fn try_update(&mut self, other: ChildInfo) -> bool { + match self { + OwnedChildInfo::Default(owned_child_trie) => owned_child_trie.try_update(other), + } + } + /// Get `ChildInfo` reference to this owned child info. pub fn as_ref(&self) -> ChildInfo { match self { @@ -311,3 +319,22 @@ pub struct OwnedChildTrie { /// See `ChildTrie` reference field documentation. pub unique_id: Vec, } + +impl OwnedChildTrie { + /// Try to update with another instance, return false if both instance + /// are not compatible. + fn try_update(&mut self, other: ChildInfo) -> bool { + match other { + ChildInfo::Default(other) => { + if self.unique_id != other.unique_id { + return false; + } + if self.root.is_none() { + self.root = other.root.as_ref().map(|s| s.to_vec()); + } + debug_assert!(self.root.as_ref().map(|v| v.as_slice()) == other.root); + }, + } + true + } +} diff --git a/primitives/sr-primitives/src/lib.rs b/primitives/sr-primitives/src/lib.rs index 788e27a6e3d85..49edfc0ba11f4 100644 --- a/primitives/sr-primitives/src/lib.rs +++ b/primitives/sr-primitives/src/lib.rs @@ -154,6 +154,9 @@ impl BuildStorage for (StorageOverlay, ChildrenStorageOverlay) { let k = k.clone(); if let Some(map) = storage.1.get_mut(&k) { map.0.extend(other_map.0.iter().map(|(k, v)| (k.clone(), v.clone()))); + if !map.1.try_update(other_map.1.as_ref()) { + return Err("Incompatible child info update".to_string()); + } } else { storage.1.insert(k, other_map.clone()); } diff --git a/primitives/state-machine/src/basic.rs b/primitives/state-machine/src/basic.rs index 6ea32db8bc2b7..cd1db2419799c 100644 --- a/primitives/state-machine/src/basic.rs +++ b/primitives/state-machine/src/basic.rs @@ -240,7 +240,7 @@ impl Externalities for BasicExternalities { if &empty_hash[..] == &child_root[..] { top.remove(storage_key.as_slice()); } else { - top.insert(storage_key.clone(), child_root); + top.insert(storage_key, child_root); } } diff --git a/primitives/state-machine/src/overlayed_changes.rs b/primitives/state-machine/src/overlayed_changes.rs index d4d3622b869c4..2044429aed593 100644 --- a/primitives/state-machine/src/overlayed_changes.rs +++ b/primitives/state-machine/src/overlayed_changes.rs @@ -159,6 +159,9 @@ impl OverlayedChanges { let extrinsic_index = self.extrinsic_index(); let map_entry = self.prospective.children.entry(storage_key) .or_insert_with(|| (Default::default(), child_info.to_owned())); + let updatable = map_entry.1.try_update(child_info); + debug_assert!(updatable); + let entry = map_entry.0.entry(key).or_default(); entry.value = val; @@ -182,6 +185,8 @@ impl OverlayedChanges { let extrinsic_index = self.extrinsic_index(); let map_entry = self.prospective.children.entry(storage_key.to_vec()) .or_insert_with(|| (Default::default(), child_info.to_owned())); + let updatable = map_entry.1.try_update(child_info); + debug_assert!(updatable); map_entry.0.values_mut().for_each(|e| { if let Some(extrinsic) = extrinsic_index { @@ -255,6 +260,8 @@ impl OverlayedChanges { let extrinsic_index = self.extrinsic_index(); let map_entry = self.prospective.children.entry(storage_key.to_vec()) .or_insert_with(|| (Default::default(), child_info.to_owned())); + let updatable = map_entry.1.try_update(child_info); + debug_assert!(updatable); for (key, entry) in map_entry.0.iter_mut() { if key.starts_with(prefix) { From cdabca3543055a0068c34c9a9873d1df37eba7ff Mon Sep 17 00:00:00 2001 From: cheme Date: Fri, 29 Nov 2019 16:12:00 +0100 Subject: [PATCH 17/27] child iteration by prefix using child_info. --- primitives/state-machine/src/trie_backend.rs | 4 +- .../state-machine/src/trie_backend_essence.rs | 44 ++++++++++++------- 2 files changed, 31 insertions(+), 17 deletions(-) diff --git a/primitives/state-machine/src/trie_backend.rs b/primitives/state-machine/src/trie_backend.rs index e60ac23d48c3c..9980f964c9dc7 100644 --- a/primitives/state-machine/src/trie_backend.rs +++ b/primitives/state-machine/src/trie_backend.rs @@ -105,11 +105,11 @@ impl, H: Hasher> Backend for TrieBackend where fn for_child_keys_with_prefix( &self, storage_key: &[u8], - _child_info: ChildInfo, + child_info: ChildInfo, prefix: &[u8], f: F, ) { - self.essence.for_child_keys_with_prefix(storage_key, prefix, f) + self.essence.for_child_keys_with_prefix(storage_key, child_info, prefix, f) } fn pairs(&self) -> Vec<(Vec, Vec)> { diff --git a/primitives/state-machine/src/trie_backend_essence.rs b/primitives/state-machine/src/trie_backend_essence.rs index 5d65e73d74ef7..0ccc504d13638 100644 --- a/primitives/state-machine/src/trie_backend_essence.rs +++ b/primitives/state-machine/src/trie_backend_essence.rs @@ -23,7 +23,7 @@ use log::{debug, warn}; use hash_db::{self, Hasher, EMPTY_PREFIX, Prefix}; use trie::{Trie, MemoryDB, PrefixedMemoryDB, DBValue, default_child_trie_root, read_trie_value, read_child_trie_value, - for_keys_in_child_trie}; + for_keys_in_child_trie, KeySpacedDB}; use trie::trie_types::{TrieDB, TrieError, Layout}; use crate::backend::Consolidate; use primitives::storage::ChildInfo; @@ -148,25 +148,32 @@ impl, H: Hasher> TrieBackendEssence where H::Out: pub fn for_child_keys_with_prefix( &self, storage_key: &[u8], + child_info: ChildInfo, prefix: &[u8], mut f: F, ) { - let root_vec = match self.storage(storage_key) { - Ok(v) => v.unwrap_or(default_child_trie_root::>(storage_key).encode()), - Err(e) => { - debug!(target: "trie", "Error while iterating child storage: {}", e); - return; - } + let fetched_root; + let root_vec = if let Some(root) = child_info.root() { + root + } else { + fetched_root = match self.storage(storage_key) { + Ok(v) => v.unwrap_or(default_child_trie_root::>(storage_key).encode()), + Err(e) => { + debug!(target: "trie", "Error while iterating child storage with prefix: {}", e); + return; + } + }; + &fetched_root }; + let mut root = H::Out::default(); root.as_mut().copy_from_slice(&root_vec); - - self.keys_values_with_prefix_inner(&root, prefix, |k, _v| f(k)) + self.keys_values_with_prefix_inner(&root, prefix, |k, _v| f(k), Some(child_info)) } /// Execute given closure for all keys starting with prefix. pub fn for_keys_with_prefix(&self, prefix: &[u8], mut f: F) { - self.keys_values_with_prefix_inner(&self.root, prefix, |k, _v| f(k)) + self.keys_values_with_prefix_inner(&self.root, prefix, |k, _v| f(k), None) } @@ -175,15 +182,16 @@ impl, H: Hasher> TrieBackendEssence where H::Out: root: &H::Out, prefix: &[u8], mut f: F, + child_info: Option, ) { let mut read_overlay = S::Overlay::default(); let eph = Ephemeral { storage: &self.storage, overlay: &mut read_overlay, }; - - let mut iter = move || -> Result<(), Box>> { - let trie = TrieDB::::new(&eph, root)?; + + let mut iter = move |db| -> Result<(), Box>> { + let trie = TrieDB::::new(db, root)?; let mut iter = trie.iter()?; iter.seek(prefix)?; @@ -201,14 +209,20 @@ impl, H: Hasher> TrieBackendEssence where H::Out: Ok(()) }; - if let Err(e) = iter() { + let result = if let Some(child_info) = child_info { + let db = KeySpacedDB::new(&eph, child_info.keyspace()); + iter(&db) + } else { + iter(&eph) + }; + if let Err(e) = result { debug!(target: "trie", "Error while iterating by prefix: {}", e); } } /// Execute given closure for all key and values starting with prefix. pub fn for_key_values_with_prefix(&self, prefix: &[u8], f: F) { - self.keys_values_with_prefix_inner(&self.root, prefix, f) + self.keys_values_with_prefix_inner(&self.root, prefix, f, None) } } From d4c73345bf35b387b4aefa68920f988b12c34c35 Mon Sep 17 00:00:00 2001 From: Emeric Chevalier Date: Mon, 2 Dec 2019 12:51:48 +0100 Subject: [PATCH 18/27] Using ChainInfo in frame support. ChainInfo gets redesign to avoid buffers allocation on every calls. --- client/chain-spec/src/chain_spec.rs | 5 +- client/network/src/protocol/light_dispatch.rs | 7 +- client/rpc/src/state/tests.rs | 6 +- client/src/light/fetcher.rs | 4 +- frame/contracts/src/lib.rs | 6 +- frame/support/src/storage/child.rs | 79 ++++++++-------- primitives/core/storage/src/lib.rs | 90 +++++++++++-------- primitives/state-machine/src/basic.rs | 2 +- .../state-machine/src/changes_trie/build.rs | 4 +- primitives/state-machine/src/lib.rs | 8 +- .../state-machine/src/proving_backend.rs | 4 +- primitives/state-machine/src/trie_backend.rs | 2 +- 12 files changed, 121 insertions(+), 96 deletions(-) diff --git a/client/chain-spec/src/chain_spec.rs b/client/chain-spec/src/chain_spec.rs index 1dbfbeec554f9..68cfff8d2abd9 100644 --- a/client/chain-spec/src/chain_spec.rs +++ b/client/chain-spec/src/chain_spec.rs @@ -267,14 +267,15 @@ impl ChainSpec { .collect(); let children = storage.1.into_iter() .map(|(sk, child)| { - let (info, ci_type) = child.1.as_ref().info(); + let info = child.1.as_ref(); + let (info, ci_type) = info.info(); ( StorageKey(sk), ( child.0.into_iter() .map(|(k, v)| (StorageKey(k), StorageData(v))) .collect(), - info, + info.to_vec(), ci_type, ), )}) diff --git a/client/network/src/protocol/light_dispatch.rs b/client/network/src/protocol/light_dispatch.rs index e7cde0fc2ff24..ef2e0c628de35 100644 --- a/client/network/src/protocol/light_dispatch.rs +++ b/client/network/src/protocol/light_dispatch.rs @@ -1031,13 +1031,14 @@ pub mod tests { light_dispatch.on_connect(&mut network_interface, peer0.clone(), Roles::FULL, 1000); let (tx, response) = oneshot::channel(); - let child_info = ChildInfo::new_default(b"unique_id_1", None).info(); + let child_info = ChildInfo::new_default(b"unique_id_1"); + let (child_info, child_type) = child_info.info(); light_dispatch.add_request(&mut network_interface, RequestData::RemoteReadChild(RemoteReadChildRequest { header: dummy_header(), block: Default::default(), storage_key: b":child_storage:sub".to_vec(), - child_info: child_info.0.to_vec(), - child_type: child_info.1, + child_info: child_info.to_vec(), + child_type, keys: vec![b":key".to_vec()], retry_count: None, }, tx)); diff --git a/client/rpc/src/state/tests.rs b/client/rpc/src/state/tests.rs index 24477a44092ad..ced3f4e61df4b 100644 --- a/client/rpc/src/state/tests.rs +++ b/client/rpc/src/state/tests.rs @@ -29,7 +29,7 @@ use test_client::{ runtime, }; -const CHILD_INFO: ChildInfo<'static> = ChildInfo::new_default(b"unique_id", None); +const CHILD_INFO: ChildInfo<'static> = ChildInfo::new_default(b"unique_id"); #[test] fn should_return_storage() { @@ -48,7 +48,7 @@ fn should_return_storage() { let key = StorageKey(KEY.to_vec()); let storage_key = StorageKey(STORAGE_KEY.to_vec()); let (child_info, child_type) = CHILD_INFO.info(); - let child_info = StorageKey(child_info); + let child_info = StorageKey(child_info.to_vec()); assert_eq!( client.storage(key.clone(), Some(genesis_hash).into()).wait() @@ -77,7 +77,7 @@ fn should_return_storage() { #[test] fn should_return_child_storage() { let (child_info, child_type) = CHILD_INFO.info(); - let child_info = StorageKey(child_info); + let child_info = StorageKey(child_info.to_vec()); let core = tokio::runtime::Runtime::new().unwrap(); let client = Arc::new(test_client::TestClientBuilder::new() .add_child_storage("test", "key", CHILD_INFO, vec![42_u8]) diff --git a/client/src/light/fetcher.rs b/client/src/light/fetcher.rs index e8799e84e0d4e..ab003ffab9fa0 100644 --- a/client/src/light/fetcher.rs +++ b/client/src/light/fetcher.rs @@ -344,7 +344,7 @@ pub mod tests { use state_machine::Backend; use super::*; - const CHILD_INFO_1: ChildInfo<'static> = ChildInfo::new_default(b"unique_id_1", None); + const CHILD_INFO_1: ChildInfo<'static> = ChildInfo::new_default(b"unique_id_1"); type TestChecker = LightDataChecker< NativeExecutor, @@ -501,7 +501,7 @@ pub mod tests { block: remote_block_header.hash(), header: remote_block_header, storage_key: b":child_storage:default:child1".to_vec(), - child_info: child_infos.0, + child_info: child_infos.0.to_vec(), child_type: child_infos.1, keys: vec![b"key1".to_vec()], retry_count: None, diff --git a/frame/contracts/src/lib.rs b/frame/contracts/src/lib.rs index 1fe13059a5161..aeba0c916f8a8 100644 --- a/frame/contracts/src/lib.rs +++ b/frame/contracts/src/lib.rs @@ -225,15 +225,15 @@ pub struct RawAliveContractInfo { impl RawAliveContractInfo { /// Associated child trie unique id is built from the hash part of the trie id. - pub fn child_trie_unique_id(&self) -> &[u8] { + pub fn child_trie_unique_id(&self) -> child::ChildInfo { trie_unique_id(&self.trie_id[..]) } } /// Associated child trie unique id is built from the hash part of the trie id. -pub(crate) fn trie_unique_id(trie_id: &[u8]) -> &[u8] { +pub(crate) fn trie_unique_id(trie_id: &[u8]) -> child::ChildInfo { let start = CHILD_STORAGE_KEY_PREFIX.len() + b"default:".len(); - &trie_id[start ..] + child::ChildInfo::new_default(&trie_id[start ..]) } pub type TombstoneContractInfo = diff --git a/frame/support/src/storage/child.rs b/frame/support/src/storage/child.rs index 030f747f33c16..940659101f96e 100644 --- a/frame/support/src/storage/child.rs +++ b/frame/support/src/storage/child.rs @@ -27,17 +27,19 @@ use crate::rstd::prelude::*; use codec::{Codec, Encode, Decode}; +pub use primitives::storage::ChildInfo; /// Return the value of the item in storage under `key`, or `None` if there is no explicit entry. pub fn get( storage_key: &[u8], - unique_id: &[u8], + child_info: ChildInfo, key: &[u8], ) -> Option { + let (data, child_type) = child_info.info(); runtime_io::storage::child_get( storage_key, - unique_id, - primitives::storage::ChildType::CryptoUniqueId as u32, + data, + child_type, key, ).and_then(|v| { Decode::decode(&mut &v[..]).map(Some).unwrap_or_else(|_| { @@ -52,46 +54,47 @@ pub fn get( /// explicit entry. pub fn get_or_default( storage_key: &[u8], - unique_id: &[u8], + child_info: ChildInfo, key: &[u8], ) -> T { - get(storage_key, unique_id, key).unwrap_or_else(Default::default) + get(storage_key, child_info, key).unwrap_or_else(Default::default) } /// Return the value of the item in storage under `key`, or `default_value` if there is no /// explicit entry. pub fn get_or( storage_key: &[u8], - unique_id: &[u8], + child_info: ChildInfo, key: &[u8], default_value: T, ) -> T { - get(storage_key, unique_id, key).unwrap_or(default_value) + get(storage_key, child_info, key).unwrap_or(default_value) } /// Return the value of the item in storage under `key`, or `default_value()` if there is no /// explicit entry. pub fn get_or_else T>( storage_key: &[u8], - unique_id: &[u8], + child_info: ChildInfo, key: &[u8], default_value: F, ) -> T { - get(storage_key, unique_id, key).unwrap_or_else(default_value) + get(storage_key, child_info, key).unwrap_or_else(default_value) } /// Put `value` in storage under `key`. pub fn put( storage_key: &[u8], - unique_id: &[u8], + child_info: ChildInfo, key: &[u8], value: &T, ) { + let (data, child_type) = child_info.info(); value.using_encoded(|slice| runtime_io::storage::child_set( storage_key, - unique_id, - primitives::storage::ChildType::CryptoUniqueId as u32, + data, + child_type, key, slice, ) @@ -101,12 +104,12 @@ pub fn put( /// Remove `key` from storage, returning its value if it had an explicit entry or `None` otherwise. pub fn take( storage_key: &[u8], - unique_id: &[u8], + child_info: ChildInfo, key: &[u8], ) -> Option { - let r = get(storage_key, unique_id, key); + let r = get(storage_key, child_info, key); if r.is_some() { - kill(storage_key, unique_id, key); + kill(storage_key, child_info, key); } r } @@ -115,43 +118,43 @@ pub fn take( /// the default for its type. pub fn take_or_default( storage_key: &[u8], - unique_id: &[u8], + child_info: ChildInfo, key: &[u8], ) -> T { - take(storage_key, unique_id, key).unwrap_or_else(Default::default) + take(storage_key, child_info, key).unwrap_or_else(Default::default) } /// Return the value of the item in storage under `key`, or `default_value` if there is no /// explicit entry. Ensure there is no explicit entry on return. pub fn take_or( storage_key: &[u8], - unique_id: &[u8], + child_info: ChildInfo, key: &[u8], default_value: T, ) -> T { - take(storage_key, unique_id, key).unwrap_or(default_value) + take(storage_key, child_info, key).unwrap_or(default_value) } /// Return the value of the item in storage under `key`, or `default_value()` if there is no /// explicit entry. Ensure there is no explicit entry on return. pub fn take_or_else T>( storage_key: &[u8], - unique_id: &[u8], + child_info: ChildInfo, key: &[u8], default_value: F, ) -> T { - take(storage_key, unique_id, key).unwrap_or_else(default_value) + take(storage_key, child_info, key).unwrap_or_else(default_value) } /// Check to see if `key` has an explicit entry in storage. pub fn exists( storage_key: &[u8], - unique_id: &[u8], + child_info: ChildInfo, key: &[u8], ) -> bool { + let (data, child_type) = child_info.info(); runtime_io::storage::child_read( - storage_key, unique_id, - primitives::storage::ChildType::CryptoUniqueId as u32, + storage_key, data, child_type, key, &mut [0;0][..], 0, ).is_some() } @@ -159,25 +162,27 @@ pub fn exists( /// Remove all `storage_key` key/values pub fn kill_storage( storage_key: &[u8], - unique_id: &[u8], + child_info: ChildInfo, ) { + let (data, child_type) = child_info.info(); runtime_io::storage::child_storage_kill( storage_key, - unique_id, - primitives::storage::ChildType::CryptoUniqueId as u32, + data, + child_type, ) } /// Ensure `key` has no explicit entry in storage. pub fn kill( storage_key: &[u8], - unique_id: &[u8], + child_info: ChildInfo, key: &[u8], ) { + let (data, child_type) = child_info.info(); runtime_io::storage::child_clear( storage_key, - unique_id, - primitives::storage::ChildType::CryptoUniqueId as u32, + data, + child_type, key, ); } @@ -185,13 +190,14 @@ pub fn kill( /// Get a Vec of bytes from storage. pub fn get_raw( storage_key: &[u8], - unique_id: &[u8], + child_info: ChildInfo, key: &[u8], ) -> Option> { + let (data, child_type) = child_info.info(); runtime_io::storage::child_get( storage_key, - unique_id, - primitives::storage::ChildType::CryptoUniqueId as u32, + data, + child_type, key, ) } @@ -199,14 +205,15 @@ pub fn get_raw( /// Put a raw byte slice into storage. pub fn put_raw( storage_key: &[u8], - unique_id: &[u8], + child_info: ChildInfo, key: &[u8], value: &[u8], ) { + let (data, child_type) = child_info.info(); runtime_io::storage::child_set( storage_key, - unique_id, - primitives::storage::ChildType::CryptoUniqueId as u32, + data, + child_type, key, value, ) diff --git a/primitives/core/storage/src/lib.rs b/primitives/core/storage/src/lib.rs index 56eb15785b2de..b32b6b57b329d 100644 --- a/primitives/core/storage/src/lib.rs +++ b/primitives/core/storage/src/lib.rs @@ -176,20 +176,20 @@ pub enum OwnedChildInfo { impl<'a> ChildInfo<'a> { /// Instantiates information for a default child trie. - pub const fn new_default(unique_id: &'a[u8], root: Option<&'a[u8]>) -> Self { + pub const fn new_default(unique_id: &'a[u8]) -> Self { ChildInfo::Default(ChildTrie { - unique_id, - root, + root: 0, + data: unique_id, }) } /// Instantiates a owned version of this child info. pub fn to_owned(&self) -> OwnedChildInfo { match self { - ChildInfo::Default(ChildTrie { unique_id, root }) + ChildInfo::Default(ChildTrie { data, root }) => OwnedChildInfo::Default(OwnedChildTrie { - unique_id: unique_id.to_vec(), - root: root.as_ref().map(|s| s.to_vec()), + root: *root, + data: data.to_vec(), }), } } @@ -197,9 +197,12 @@ impl<'a> ChildInfo<'a> { /// Create child info from a linear byte packed value and a given type. pub fn resolve_child_info(child_type: u32, data: &'a[u8]) -> Option { match child_type { - x if x == ChildType::CryptoUniqueId as u32 => Some(ChildInfo::new_default(data, None)), + x if x == ChildType::CryptoUniqueId as u32 => Some(ChildInfo::new_default(data)), x if x == ChildType::CryptoUniqueIdRoot32Api as u32 => if data.len() >= 32 { - Some(ChildInfo::new_default(&data[32..], Some(&data[..32]))) + Some(ChildInfo::Default(ChildTrie { + root: 32, + data, + })) } else { None }, _ => None, } @@ -207,20 +210,18 @@ impl<'a> ChildInfo<'a> { /// Return a single byte vector containing packed child info content and its child info type. /// This can be use as input for `resolve_child_info`. - pub fn info(&self) -> (Vec, u32) { + pub fn info(&self) -> (&[u8], u32) { match self { ChildInfo::Default(ChildTrie { root, - unique_id, + data, }) => { - if let Some(root) = root { - if root.len() == 32 { - let mut buf = root.to_vec(); - buf.extend_from_slice(unique_id); - return (buf, ChildType::CryptoUniqueIdRoot32Api as u32); + if *root > 0 { + if *root == 32 { + return (data, ChildType::CryptoUniqueIdRoot32Api as u32); } } - (unique_id.to_vec(), ChildType::CryptoUniqueId as u32) + (data, ChildType::CryptoUniqueId as u32) }, } } @@ -231,9 +232,9 @@ impl<'a> ChildInfo<'a> { pub fn keyspace(&self) -> &[u8] { match self { ChildInfo::Default(ChildTrie { - root: _root, - unique_id, - }) => unique_id, + root, + data, + }) => &data[*root..], } } @@ -245,8 +246,12 @@ impl<'a> ChildInfo<'a> { match self { ChildInfo::Default(ChildTrie { root, - unique_id: _unique_id, - }) => *root, + data, + }) => if *root > 0 { + Some(&data[..*root]) + } else { + None + } } } @@ -266,10 +271,16 @@ pub enum ChildType { impl OwnedChildInfo { /// Instantiates info for a default child trie. - pub fn new_default(unique_id: Vec, root: Option>) -> Self { + pub fn new_default(mut unique_id: Vec, root: Option>) -> Self { + debug_assert!(root.as_ref().map(|r| r.len()).unwrap_or(32) == 32); OwnedChildInfo::Default(OwnedChildTrie { - unique_id, - root, + root: root.as_ref().map(|r| r.len()).unwrap_or(0), + data: if let Some(mut root) = root { + root.append(&mut unique_id); + root + } else { + unique_id + } }) } @@ -284,10 +295,10 @@ impl OwnedChildInfo { /// Get `ChildInfo` reference to this owned child info. pub fn as_ref(&self) -> ChildInfo { match self { - OwnedChildInfo::Default(OwnedChildTrie { unique_id, root }) + OwnedChildInfo::Default(OwnedChildTrie { data, root }) => ChildInfo::Default(ChildTrie { - unique_id: unique_id.as_slice(), - root: root.as_ref().map(Vec::as_slice), + data: data.as_slice(), + root: *root, }), } } @@ -300,13 +311,16 @@ impl OwnedChildInfo { /// crypto hash). #[derive(Clone, Copy)] pub struct ChildTrie<'a> { - /// If root was fetch it can be memoïzed in this field - /// to avoid querying it explicitly. - pub root: Option<&'a[u8]>, + /// If root was fetch it can be memoïzed as first part of + /// the encoded data to avoid querying it explicitly. + /// This field keep trace of the stored root size. + root: usize, + /// Data containing unique id. /// Unique id must but unique and free of any possible key collision /// (depending on its storage behavior). - pub unique_id: &'a[u8], + /// Data can also contain root in first position. + data: &'a[u8], } /// Owned version of default child trie `ChildTrie`. @@ -314,10 +328,10 @@ pub struct ChildTrie<'a> { #[cfg_attr(feature = "std", derive(PartialEq, Eq, Hash, PartialOrd, Ord))] pub struct OwnedChildTrie { /// See `ChildTrie` reference field documentation. - pub root: Option>, + root: usize, /// See `ChildTrie` reference field documentation. - pub unique_id: Vec, + data: Vec, } impl OwnedChildTrie { @@ -326,13 +340,15 @@ impl OwnedChildTrie { fn try_update(&mut self, other: ChildInfo) -> bool { match other { ChildInfo::Default(other) => { - if self.unique_id != other.unique_id { + if self.data[self.root..] != other.data[other.root..] { return false; } - if self.root.is_none() { - self.root = other.root.as_ref().map(|s| s.to_vec()); + if self.root == 0 { + self.root = other.root; + self.data = other.data.to_vec(); + } else { + debug_assert!(self.data[..self.root] == other.data[..other.root]); } - debug_assert!(self.root.as_ref().map(|v| v.as_slice()) == other.root); }, } true diff --git a/primitives/state-machine/src/basic.rs b/primitives/state-machine/src/basic.rs index cd1db2419799c..2f188bf93c9a7 100644 --- a/primitives/state-machine/src/basic.rs +++ b/primitives/state-machine/src/basic.rs @@ -280,7 +280,7 @@ mod tests { use primitives::storage::well_known_keys::CODE; use hex_literal::hex; - const CHILD_INFO_1: ChildInfo<'static> = ChildInfo::new_default(b"unique_id_1", None); + const CHILD_INFO_1: ChildInfo<'static> = ChildInfo::new_default(b"unique_id_1"); #[test] fn commit_should_work() { diff --git a/primitives/state-machine/src/changes_trie/build.rs b/primitives/state-machine/src/changes_trie/build.rs index d51a36f52d3ae..3f64451c54953 100644 --- a/primitives/state-machine/src/changes_trie/build.rs +++ b/primitives/state-machine/src/changes_trie/build.rs @@ -347,8 +347,8 @@ mod test { use crate::overlayed_changes::{OverlayedValue, OverlayedChangeSet}; use super::*; - const CHILD_INFO_1: ChildInfo<'static> = ChildInfo::new_default(b"unique_id_1", None); - const CHILD_INFO_2: ChildInfo<'static> = ChildInfo::new_default(b"unique_id_2", None); + const CHILD_INFO_1: ChildInfo<'static> = ChildInfo::new_default(b"unique_id_1"); + const CHILD_INFO_2: ChildInfo<'static> = ChildInfo::new_default(b"unique_id_2"); fn prepare_for_build(zero: u64) -> ( InMemory, diff --git a/primitives/state-machine/src/lib.rs b/primitives/state-machine/src/lib.rs index 298c2460b43ed..53703029e9362 100644 --- a/primitives/state-machine/src/lib.rs +++ b/primitives/state-machine/src/lib.rs @@ -691,7 +691,7 @@ where H::Out: Ord + Codec, { // Not a prefixed memory db, using empty unique id and include root resolution. - proving_backend.child_storage(storage_key, ChildInfo::new_default(&[], None), key) + proving_backend.child_storage(storage_key, ChildInfo::new_default(&[]), key) .map_err(|e| Box::new(e) as Box) } @@ -754,7 +754,7 @@ mod tests { fallback_succeeds: bool, } - const CHILD_INFO_1: ChildInfo<'static> = ChildInfo::new_default(b"unique_id_1", None); + const CHILD_INFO_1: ChildInfo<'static> = ChildInfo::new_default(b"unique_id_1"); impl CodeExecutor for DummyCodeExecutor { type Error = u8; @@ -1094,8 +1094,8 @@ mod tests { #[test] fn child_storage_uuid() { - const CHILD_INFO_1: ChildInfo<'static> = ChildInfo::new_default(b"unique_id_1", None); - const CHILD_INFO_2: ChildInfo<'static> = ChildInfo::new_default(b"unique_id_2", None); + const CHILD_INFO_1: ChildInfo<'static> = ChildInfo::new_default(b"unique_id_1"); + const CHILD_INFO_2: ChildInfo<'static> = ChildInfo::new_default(b"unique_id_2"); use crate::trie_backend::tests::test_trie; let mut overlay = OverlayedChanges::default(); diff --git a/primitives/state-machine/src/proving_backend.rs b/primitives/state-machine/src/proving_backend.rs index 88ad8a76d29dc..945a10c0b1ca9 100644 --- a/primitives/state-machine/src/proving_backend.rs +++ b/primitives/state-machine/src/proving_backend.rs @@ -384,8 +384,8 @@ mod tests { use primitives::{Blake2Hasher, storage::ChildStorageKey}; use crate::proving_backend::create_proof_check_backend; - const CHILD_INFO_1: ChildInfo<'static> = ChildInfo::new_default(b"unique_id_1", None); - const CHILD_INFO_2: ChildInfo<'static> = ChildInfo::new_default(b"unique_id_2", None); + const CHILD_INFO_1: ChildInfo<'static> = ChildInfo::new_default(b"unique_id_1"); + const CHILD_INFO_2: ChildInfo<'static> = ChildInfo::new_default(b"unique_id_2"); fn test_proving<'a>( trie_backend: &'a TrieBackend,Blake2Hasher>, diff --git a/primitives/state-machine/src/trie_backend.rs b/primitives/state-machine/src/trie_backend.rs index 9980f964c9dc7..fea27148b31f9 100644 --- a/primitives/state-machine/src/trie_backend.rs +++ b/primitives/state-machine/src/trie_backend.rs @@ -238,7 +238,7 @@ pub mod tests { const CHILD_KEY_1: &[u8] = b":child_storage:default:sub1"; const CHILD_UUID_1: &[u8] = b"unique_id_1"; - const CHILD_INFO_1: ChildInfo<'static> = ChildInfo::new_default(CHILD_UUID_1, None); + const CHILD_INFO_1: ChildInfo<'static> = ChildInfo::new_default(CHILD_UUID_1); fn test_db() -> (PrefixedMemoryDB, H256) { let mut root = H256::default(); From 64f4084011efa67b5e4a859d860e6fbb68f51ec3 Mon Sep 17 00:00:00 2001 From: Emeric Chevalier Date: Mon, 2 Dec 2019 16:11:07 +0100 Subject: [PATCH 19/27] Add length of root to the data of child info. --- Cargo.lock | 1 + primitives/core/storage/Cargo.toml | 1 + primitives/core/storage/src/lib.rs | 117 ++++++++++++++++++++--------- 3 files changed, 84 insertions(+), 35 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2a5b8cde28036..923aa8bce6b86 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6284,6 +6284,7 @@ name = "substrate-primitives-storage" version = "2.0.0" dependencies = [ "impl-serde 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", "sr-std 2.0.0", "substrate-debug-derive 2.0.0", diff --git a/primitives/core/storage/Cargo.toml b/primitives/core/storage/Cargo.toml index 1e5d7ee8b4548..d027e4304b396 100644 --- a/primitives/core/storage/Cargo.toml +++ b/primitives/core/storage/Cargo.toml @@ -10,6 +10,7 @@ rstd = { package = "sr-std", path = "../../sr-std", default-features = false } serde = { version = "1.0.101", optional = true, features = ["derive"] } impl-serde = { version = "0.2.3", optional = true } substrate-debug-derive = { version = "2.0.0", path = "../debug-derive" } +codec = { package = "parity-scale-codec", version = "1.0.0", default-features = false, features = ["derive"] } [features] default = [ "std" ] diff --git a/primitives/core/storage/src/lib.rs b/primitives/core/storage/src/lib.rs index b32b6b57b329d..13634c6d5f2b4 100644 --- a/primitives/core/storage/src/lib.rs +++ b/primitives/core/storage/src/lib.rs @@ -23,6 +23,7 @@ use serde::{Serialize, Deserialize}; use substrate_debug_derive::RuntimeDebug; use rstd::{vec::Vec, borrow::Cow}; +use codec::{Encode, Decode}; /// Storage key. #[derive(PartialEq, Eq, RuntimeDebug)] @@ -178,7 +179,8 @@ impl<'a> ChildInfo<'a> { /// Instantiates information for a default child trie. pub const fn new_default(unique_id: &'a[u8]) -> Self { ChildInfo::Default(ChildTrie { - root: 0, + offset: 0, + root_end: 0, data: unique_id, }) } @@ -186,9 +188,10 @@ impl<'a> ChildInfo<'a> { /// Instantiates a owned version of this child info. pub fn to_owned(&self) -> OwnedChildInfo { match self { - ChildInfo::Default(ChildTrie { data, root }) + ChildInfo::Default(ChildTrie { data, root_end, offset }) => OwnedChildInfo::Default(OwnedChildTrie { - root: *root, + offset: *offset, + root_end: *root_end, data: data.to_vec(), }), } @@ -198,12 +201,22 @@ impl<'a> ChildInfo<'a> { pub fn resolve_child_info(child_type: u32, data: &'a[u8]) -> Option { match child_type { x if x == ChildType::CryptoUniqueId as u32 => Some(ChildInfo::new_default(data)), - x if x == ChildType::CryptoUniqueIdRoot32Api as u32 => if data.len() >= 32 { - Some(ChildInfo::Default(ChildTrie { - root: 32, - data, - })) - } else { None }, + x if x == ChildType::CryptoUniqueIdRootApi as u32 => { + let data_cursor = &mut data.clone(); + // u32 is considered enough for a root size. + let number: u32 = match Decode::decode(data_cursor) { + Ok(number) => number, + Err(_) => return None, + }; + let offset = data.len() - data_cursor.len(); + if data.len() >= number as usize { + Some(ChildInfo::Default(ChildTrie { + offset, + root_end: number as usize + offset, + data, + })) + } else { None } + }, _ => None, } } @@ -213,13 +226,12 @@ impl<'a> ChildInfo<'a> { pub fn info(&self) -> (&[u8], u32) { match self { ChildInfo::Default(ChildTrie { - root, + root_end, data, + .. }) => { - if *root > 0 { - if *root == 32 { - return (data, ChildType::CryptoUniqueIdRoot32Api as u32); - } + if *root_end > 0 { + return (data, ChildType::CryptoUniqueIdRootApi as u32); } (data, ChildType::CryptoUniqueId as u32) }, @@ -232,9 +244,10 @@ impl<'a> ChildInfo<'a> { pub fn keyspace(&self) -> &[u8] { match self { ChildInfo::Default(ChildTrie { - root, + root_end, data, - }) => &data[*root..], + .. + }) => &data[*root_end..], } } @@ -245,10 +258,11 @@ impl<'a> ChildInfo<'a> { pub fn root(&self) -> Option<&[u8]> { match self { ChildInfo::Default(ChildTrie { - root, + root_end, data, - }) => if *root > 0 { - Some(&data[..*root]) + offset, + }) => if *root_end > 0 { + Some(&data[*offset..*root_end]) } else { None } @@ -265,19 +279,27 @@ pub enum ChildType { /// Default, it uses a cryptographic strong unique id as input. CryptoUniqueId = 1, /// Default, but with a root registerd to skip root fetch when querying. - /// Root is a 32 byte hash. - CryptoUniqueIdRoot32Api = 2, + /// Root is variable length, its length is SCALE encoded at the start. + CryptoUniqueIdRootApi = 2, } impl OwnedChildInfo { /// Instantiates info for a default child trie. pub fn new_default(mut unique_id: Vec, root: Option>) -> Self { - debug_assert!(root.as_ref().map(|r| r.len()).unwrap_or(32) == 32); + let (offset, root_end, encoded) = if let Some(mut root) = root { + let mut encoded = Encode::encode(&(root.len() as u32)); + let offset = encoded.len(); + encoded.append(&mut root); + (offset, encoded.len(), Some(encoded)) + } else { + (0, 0, None) + }; OwnedChildInfo::Default(OwnedChildTrie { - root: root.as_ref().map(|r| r.len()).unwrap_or(0), - data: if let Some(mut root) = root { - root.append(&mut unique_id); - root + root_end, + offset, + data: if let Some(mut encoded) = encoded { + encoded.append(&mut unique_id); + encoded } else { unique_id } @@ -295,10 +317,11 @@ impl OwnedChildInfo { /// Get `ChildInfo` reference to this owned child info. pub fn as_ref(&self) -> ChildInfo { match self { - OwnedChildInfo::Default(OwnedChildTrie { data, root }) + OwnedChildInfo::Default(OwnedChildTrie { data, root_end, offset }) => ChildInfo::Default(ChildTrie { data: data.as_slice(), - root: *root, + root_end: *root_end, + offset: *offset, }), } } @@ -311,10 +334,12 @@ impl OwnedChildInfo { /// crypto hash). #[derive(Clone, Copy)] pub struct ChildTrie<'a> { + /// Encoded data offset (correspond to size of encoded size of root if any). + offset: usize, /// If root was fetch it can be memoïzed as first part of /// the encoded data to avoid querying it explicitly. - /// This field keep trace of the stored root size. - root: usize, + /// This field keep trace of the position of the end of root. + root_end: usize, /// Data containing unique id. /// Unique id must but unique and free of any possible key collision @@ -328,7 +353,10 @@ pub struct ChildTrie<'a> { #[cfg_attr(feature = "std", derive(PartialEq, Eq, Hash, PartialOrd, Ord))] pub struct OwnedChildTrie { /// See `ChildTrie` reference field documentation. - root: usize, + offset: usize, + + /// See `ChildTrie` reference field documentation. + root_end: usize, /// See `ChildTrie` reference field documentation. data: Vec, @@ -340,17 +368,36 @@ impl OwnedChildTrie { fn try_update(&mut self, other: ChildInfo) -> bool { match other { ChildInfo::Default(other) => { - if self.data[self.root..] != other.data[other.root..] { + if self.data[self.root_end..] != other.data[other.root_end..] { return false; } - if self.root == 0 { - self.root = other.root; + if self.root_end == 0 { + self.root_end = other.root_end; self.data = other.data.to_vec(); } else { - debug_assert!(self.data[..self.root] == other.data[..other.root]); + debug_assert!(self.data[..self.root_end] == other.data[..other.root_end]); } }, } true } } + +#[test] +fn test_encode() { + let root = vec![1; 297]; + let unique_id = vec![2; 16]; + let owned_child = OwnedChildInfo::new_default(unique_id.clone(), Some(root.clone())); + let child = owned_child.as_ref(); + assert_eq!(child.keyspace(), &unique_id[..]); + assert_eq!(child.root(), Some(&root[..])); + + let owned_child = OwnedChildInfo::new_default(unique_id.clone(), None); + let child = owned_child.as_ref(); + assert_eq!(child.keyspace(), &unique_id[..]); + assert_eq!(child.root(), None); + + let child = ChildInfo::new_default(&unique_id[..]); + assert_eq!(child.keyspace(), &unique_id[..]); + assert_eq!(child.root(), None); +} From a87bd4809b0bfc023d12c218fef329c5e94e202e Mon Sep 17 00:00:00 2001 From: Emeric Chevalier Date: Mon, 2 Dec 2019 16:19:05 +0100 Subject: [PATCH 20/27] comments --- client/chain-spec/src/chain_spec.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/client/chain-spec/src/chain_spec.rs b/client/chain-spec/src/chain_spec.rs index 68cfff8d2abd9..25476d7be8a03 100644 --- a/client/chain-spec/src/chain_spec.rs +++ b/client/chain-spec/src/chain_spec.rs @@ -105,7 +105,12 @@ impl<'a, G: RuntimeGenesis, E> BuildStorage for &'a ChainSpec { enum Genesis { Runtime(G), Raw( + // Map of values in the top trie. HashMap, + // Map of child tries. + // Each child tries contains its map of values + // and a child info encoded as a byte array with the + // type of this child info as a u32. HashMap, Vec, u32)>, ), } From 68712f95a76f77312689585a8867f50caade35a3 Mon Sep 17 00:00:00 2001 From: Emeric Chevalier Date: Mon, 2 Dec 2019 18:09:17 +0100 Subject: [PATCH 21/27] Encode compact. --- primitives/core/storage/src/lib.rs | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/primitives/core/storage/src/lib.rs b/primitives/core/storage/src/lib.rs index 13634c6d5f2b4..fdc5ce7a75754 100644 --- a/primitives/core/storage/src/lib.rs +++ b/primitives/core/storage/src/lib.rs @@ -23,7 +23,7 @@ use serde::{Serialize, Deserialize}; use substrate_debug_derive::RuntimeDebug; use rstd::{vec::Vec, borrow::Cow}; -use codec::{Encode, Decode}; +use codec::{Encode, Decode, Compact}; /// Storage key. #[derive(PartialEq, Eq, RuntimeDebug)] @@ -202,10 +202,10 @@ impl<'a> ChildInfo<'a> { match child_type { x if x == ChildType::CryptoUniqueId as u32 => Some(ChildInfo::new_default(data)), x if x == ChildType::CryptoUniqueIdRootApi as u32 => { - let data_cursor = &mut data.clone(); + let data_cursor = &mut &data[..]; // u32 is considered enough for a root size. - let number: u32 = match Decode::decode(data_cursor) { - Ok(number) => number, + let number = match Compact::::decode(data_cursor) { + Ok(number) => number.0, Err(_) => return None, }; let offset = data.len() - data_cursor.len(); @@ -287,7 +287,8 @@ impl OwnedChildInfo { /// Instantiates info for a default child trie. pub fn new_default(mut unique_id: Vec, root: Option>) -> Self { let (offset, root_end, encoded) = if let Some(mut root) = root { - let mut encoded = Encode::encode(&(root.len() as u32)); + let mut encoded = Encode::encode(&Compact(root.len() as u32)); + let offset = encoded.len(); encoded.append(&mut root); (offset, encoded.len(), Some(encoded)) @@ -389,15 +390,28 @@ fn test_encode() { let unique_id = vec![2; 16]; let owned_child = OwnedChildInfo::new_default(unique_id.clone(), Some(root.clone())); let child = owned_child.as_ref(); + let (child_info, child_type) = child.info(); + let child_info = child_info.to_vec(); + assert_eq!(child.keyspace(), &unique_id[..]); + assert_eq!(child.root(), Some(&root[..])); + + let child = ChildInfo::resolve_child_info(child_type, &child_info[..]).unwrap(); assert_eq!(child.keyspace(), &unique_id[..]); assert_eq!(child.root(), Some(&root[..])); let owned_child = OwnedChildInfo::new_default(unique_id.clone(), None); let child = owned_child.as_ref(); + let (child_info, child_type) = child.info(); + let child_info = child_info.to_vec(); assert_eq!(child.keyspace(), &unique_id[..]); assert_eq!(child.root(), None); - + + let child = ChildInfo::resolve_child_info(child_type, &child_info[..]).unwrap(); + assert_eq!(child.keyspace(), &unique_id[..]); + assert_eq!(child.root(), None); + let child = ChildInfo::new_default(&unique_id[..]); assert_eq!(child.keyspace(), &unique_id[..]); assert_eq!(child.root(), None); + } From 3db467e38f11756bf7711cee87907ee73c7c092a Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 3 Dec 2019 10:41:59 +0100 Subject: [PATCH 22/27] Remove child info with root. --- Cargo.lock | 1 - primitives/core/storage/Cargo.toml | 1 - primitives/core/storage/src/lib.rs | 141 +----------------- primitives/state-machine/src/backend.rs | 2 +- .../state-machine/src/trie_backend_essence.rs | 49 ++---- 5 files changed, 23 insertions(+), 171 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 923aa8bce6b86..2a5b8cde28036 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6284,7 +6284,6 @@ name = "substrate-primitives-storage" version = "2.0.0" dependencies = [ "impl-serde 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", - "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", "sr-std 2.0.0", "substrate-debug-derive 2.0.0", diff --git a/primitives/core/storage/Cargo.toml b/primitives/core/storage/Cargo.toml index d027e4304b396..1e5d7ee8b4548 100644 --- a/primitives/core/storage/Cargo.toml +++ b/primitives/core/storage/Cargo.toml @@ -10,7 +10,6 @@ rstd = { package = "sr-std", path = "../../sr-std", default-features = false } serde = { version = "1.0.101", optional = true, features = ["derive"] } impl-serde = { version = "0.2.3", optional = true } substrate-debug-derive = { version = "2.0.0", path = "../debug-derive" } -codec = { package = "parity-scale-codec", version = "1.0.0", default-features = false, features = ["derive"] } [features] default = [ "std" ] diff --git a/primitives/core/storage/src/lib.rs b/primitives/core/storage/src/lib.rs index fdc5ce7a75754..7529055963271 100644 --- a/primitives/core/storage/src/lib.rs +++ b/primitives/core/storage/src/lib.rs @@ -23,7 +23,6 @@ use serde::{Serialize, Deserialize}; use substrate_debug_derive::RuntimeDebug; use rstd::{vec::Vec, borrow::Cow}; -use codec::{Encode, Decode, Compact}; /// Storage key. #[derive(PartialEq, Eq, RuntimeDebug)] @@ -179,8 +178,6 @@ impl<'a> ChildInfo<'a> { /// Instantiates information for a default child trie. pub const fn new_default(unique_id: &'a[u8]) -> Self { ChildInfo::Default(ChildTrie { - offset: 0, - root_end: 0, data: unique_id, }) } @@ -188,10 +185,8 @@ impl<'a> ChildInfo<'a> { /// Instantiates a owned version of this child info. pub fn to_owned(&self) -> OwnedChildInfo { match self { - ChildInfo::Default(ChildTrie { data, root_end, offset }) + ChildInfo::Default(ChildTrie { data }) => OwnedChildInfo::Default(OwnedChildTrie { - offset: *offset, - root_end: *root_end, data: data.to_vec(), }), } @@ -201,22 +196,6 @@ impl<'a> ChildInfo<'a> { pub fn resolve_child_info(child_type: u32, data: &'a[u8]) -> Option { match child_type { x if x == ChildType::CryptoUniqueId as u32 => Some(ChildInfo::new_default(data)), - x if x == ChildType::CryptoUniqueIdRootApi as u32 => { - let data_cursor = &mut &data[..]; - // u32 is considered enough for a root size. - let number = match Compact::::decode(data_cursor) { - Ok(number) => number.0, - Err(_) => return None, - }; - let offset = data.len() - data_cursor.len(); - if data.len() >= number as usize { - Some(ChildInfo::Default(ChildTrie { - offset, - root_end: number as usize + offset, - data, - })) - } else { None } - }, _ => None, } } @@ -226,15 +205,8 @@ impl<'a> ChildInfo<'a> { pub fn info(&self) -> (&[u8], u32) { match self { ChildInfo::Default(ChildTrie { - root_end, data, - .. - }) => { - if *root_end > 0 { - return (data, ChildType::CryptoUniqueIdRootApi as u32); - } - (data, ChildType::CryptoUniqueId as u32) - }, + }) => (data, ChildType::CryptoUniqueId as u32), } } @@ -244,31 +216,10 @@ impl<'a> ChildInfo<'a> { pub fn keyspace(&self) -> &[u8] { match self { ChildInfo::Default(ChildTrie { - root_end, data, - .. - }) => &data[*root_end..], + }) => &data[..], } } - - /// Return the child reference to state if it is already known. - /// If it returns `None` the information will need to be fetch by - /// the caller. - /// For a child trie it is its root. - pub fn root(&self) -> Option<&[u8]> { - match self { - ChildInfo::Default(ChildTrie { - root_end, - data, - offset, - }) => if *root_end > 0 { - Some(&data[*offset..*root_end]) - } else { - None - } - } - } - } /// Type of child. @@ -278,32 +229,13 @@ impl<'a> ChildInfo<'a> { pub enum ChildType { /// Default, it uses a cryptographic strong unique id as input. CryptoUniqueId = 1, - /// Default, but with a root registerd to skip root fetch when querying. - /// Root is variable length, its length is SCALE encoded at the start. - CryptoUniqueIdRootApi = 2, } impl OwnedChildInfo { /// Instantiates info for a default child trie. - pub fn new_default(mut unique_id: Vec, root: Option>) -> Self { - let (offset, root_end, encoded) = if let Some(mut root) = root { - let mut encoded = Encode::encode(&Compact(root.len() as u32)); - - let offset = encoded.len(); - encoded.append(&mut root); - (offset, encoded.len(), Some(encoded)) - } else { - (0, 0, None) - }; + pub fn new_default(unique_id: Vec) -> Self { OwnedChildInfo::Default(OwnedChildTrie { - root_end, - offset, - data: if let Some(mut encoded) = encoded { - encoded.append(&mut unique_id); - encoded - } else { - unique_id - } + data: unique_id, }) } @@ -318,11 +250,9 @@ impl OwnedChildInfo { /// Get `ChildInfo` reference to this owned child info. pub fn as_ref(&self) -> ChildInfo { match self { - OwnedChildInfo::Default(OwnedChildTrie { data, root_end, offset }) + OwnedChildInfo::Default(OwnedChildTrie { data }) => ChildInfo::Default(ChildTrie { data: data.as_slice(), - root_end: *root_end, - offset: *offset, }), } } @@ -335,17 +265,9 @@ impl OwnedChildInfo { /// crypto hash). #[derive(Clone, Copy)] pub struct ChildTrie<'a> { - /// Encoded data offset (correspond to size of encoded size of root if any). - offset: usize, - /// If root was fetch it can be memoïzed as first part of - /// the encoded data to avoid querying it explicitly. - /// This field keep trace of the position of the end of root. - root_end: usize, - /// Data containing unique id. /// Unique id must but unique and free of any possible key collision /// (depending on its storage behavior). - /// Data can also contain root in first position. data: &'a[u8], } @@ -353,12 +275,6 @@ pub struct ChildTrie<'a> { #[derive(Debug, Clone)] #[cfg_attr(feature = "std", derive(PartialEq, Eq, Hash, PartialOrd, Ord))] pub struct OwnedChildTrie { - /// See `ChildTrie` reference field documentation. - offset: usize, - - /// See `ChildTrie` reference field documentation. - root_end: usize, - /// See `ChildTrie` reference field documentation. data: Vec, } @@ -368,50 +284,7 @@ impl OwnedChildTrie { /// are not compatible. fn try_update(&mut self, other: ChildInfo) -> bool { match other { - ChildInfo::Default(other) => { - if self.data[self.root_end..] != other.data[other.root_end..] { - return false; - } - if self.root_end == 0 { - self.root_end = other.root_end; - self.data = other.data.to_vec(); - } else { - debug_assert!(self.data[..self.root_end] == other.data[..other.root_end]); - } - }, + ChildInfo::Default(other) => self.data[..] != other.data[..], } - true } } - -#[test] -fn test_encode() { - let root = vec![1; 297]; - let unique_id = vec![2; 16]; - let owned_child = OwnedChildInfo::new_default(unique_id.clone(), Some(root.clone())); - let child = owned_child.as_ref(); - let (child_info, child_type) = child.info(); - let child_info = child_info.to_vec(); - assert_eq!(child.keyspace(), &unique_id[..]); - assert_eq!(child.root(), Some(&root[..])); - - let child = ChildInfo::resolve_child_info(child_type, &child_info[..]).unwrap(); - assert_eq!(child.keyspace(), &unique_id[..]); - assert_eq!(child.root(), Some(&root[..])); - - let owned_child = OwnedChildInfo::new_default(unique_id.clone(), None); - let child = owned_child.as_ref(); - let (child_info, child_type) = child.info(); - let child_info = child_info.to_vec(); - assert_eq!(child.keyspace(), &unique_id[..]); - assert_eq!(child.root(), None); - - let child = ChildInfo::resolve_child_info(child_type, &child_info[..]).unwrap(); - assert_eq!(child.keyspace(), &unique_id[..]); - assert_eq!(child.root(), None); - - let child = ChildInfo::new_default(&unique_id[..]); - assert_eq!(child.keyspace(), &unique_id[..]); - assert_eq!(child.root(), None); - -} diff --git a/primitives/state-machine/src/backend.rs b/primitives/state-machine/src/backend.rs index 0a811e369d1bb..87621e39222aa 100644 --- a/primitives/state-machine/src/backend.rs +++ b/primitives/state-machine/src/backend.rs @@ -627,7 +627,7 @@ mod tests { #[test] fn in_memory_with_child_trie_only() { let storage = InMemory::::default(); - let child_info = OwnedChildInfo::new_default(b"unique_id_1".to_vec(), None); + let child_info = OwnedChildInfo::new_default(b"unique_id_1".to_vec()); let mut storage = storage.update( vec![( Some((b"1".to_vec(), child_info.clone())), diff --git a/primitives/state-machine/src/trie_backend_essence.rs b/primitives/state-machine/src/trie_backend_essence.rs index 0ccc504d13638..848a281a188b1 100644 --- a/primitives/state-machine/src/trie_backend_essence.rs +++ b/primitives/state-machine/src/trie_backend_essence.rs @@ -85,14 +85,8 @@ impl, H: Hasher> TrieBackendEssence where H::Out: child_info: ChildInfo, key: &[u8], ) -> Result>, String> { - let fetched_root; - let root = if let Some(root) = child_info.root() { - root - } else { - fetched_root = self.storage(storage_key)? - .unwrap_or(default_child_trie_root::>(storage_key).encode()); - &fetched_root - }; + let root = self.storage(storage_key)? + .unwrap_or(default_child_trie_root::>(storage_key).encode()); let mut read_overlay = S::Overlay::default(); let eph = Ephemeral { @@ -102,7 +96,7 @@ impl, H: Hasher> TrieBackendEssence where H::Out: let map_e = |e| format!("Trie lookup error: {}", e); - read_child_trie_value::, _>(storage_key, child_info.keyspace(), &eph, root, key) + read_child_trie_value::, _>(storage_key, child_info.keyspace(), &eph, &root, key) .map_err(map_e) } @@ -113,18 +107,12 @@ impl, H: Hasher> TrieBackendEssence where H::Out: child_info: ChildInfo, f: F, ) { - let fetched_root; - let root = if let Some(root) = child_info.root() { - root - } else { - fetched_root = match self.storage(storage_key) { - Ok(v) => v.unwrap_or(default_child_trie_root::>(storage_key).encode()), - Err(e) => { - debug!(target: "trie", "Error while iterating child storage: {}", e); - return; - } - }; - &fetched_root + let root = match self.storage(storage_key) { + Ok(v) => v.unwrap_or(default_child_trie_root::>(storage_key).encode()), + Err(e) => { + debug!(target: "trie", "Error while iterating child storage: {}", e); + return; + } }; let mut read_overlay = S::Overlay::default(); @@ -152,20 +140,13 @@ impl, H: Hasher> TrieBackendEssence where H::Out: prefix: &[u8], mut f: F, ) { - let fetched_root; - let root_vec = if let Some(root) = child_info.root() { - root - } else { - fetched_root = match self.storage(storage_key) { - Ok(v) => v.unwrap_or(default_child_trie_root::>(storage_key).encode()), - Err(e) => { - debug!(target: "trie", "Error while iterating child storage with prefix: {}", e); - return; - } - }; - &fetched_root + let root_vec = match self.storage(storage_key) { + Ok(v) => v.unwrap_or(default_child_trie_root::>(storage_key).encode()), + Err(e) => { + debug!(target: "trie", "Error while iterating child storage: {}", e); + return; + } }; - let mut root = H::Out::default(); root.as_mut().copy_from_slice(&root_vec); self.keys_values_with_prefix_inner(&root, prefix, |k, _v| f(k), Some(child_info)) From 8e2b8b9846421effa5a32500085117f8f65df778 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 3 Dec 2019 11:29:01 +0100 Subject: [PATCH 23/27] Fix try_update condition. --- primitives/core/storage/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/primitives/core/storage/src/lib.rs b/primitives/core/storage/src/lib.rs index 23a8e308bc730..f4a745b1eaeff 100644 --- a/primitives/core/storage/src/lib.rs +++ b/primitives/core/storage/src/lib.rs @@ -284,7 +284,7 @@ impl OwnedChildTrie { /// are not compatible. fn try_update(&mut self, other: ChildInfo) -> bool { match other { - ChildInfo::Default(other) => self.data[..] != other.data[..], + ChildInfo::Default(other) => self.data[..] == other.data[..], } } } From 5d74151c5fbaa0d7cb14c9035ae40f23a153ee20 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 3 Dec 2019 13:51:59 +0100 Subject: [PATCH 24/27] Comment Ext child root caching. --- primitives/state-machine/src/ext.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/primitives/state-machine/src/ext.rs b/primitives/state-machine/src/ext.rs index 1c537d911fbdc..58f46f4832d68 100644 --- a/primitives/state-machine/src/ext.rs +++ b/primitives/state-machine/src/ext.rs @@ -531,6 +531,11 @@ where }; let root = root.encode(); + // We store update in the overlay in order to be able to use 'self.storage_transaction' + // cache. This is brittle as it rely on Ext only querying the trie backend for + // storage root. + // A better design would be to manage 'child_storage_transaction' in a + // similar way as 'storage_transaction' but for each child trie. if is_empty { self.overlay.set_storage(storage_key.into(), None); } else { From 0ba3da59c9ed50abef383c4df095b81be09dff0c Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 5 Dec 2019 15:42:51 +0100 Subject: [PATCH 25/27] Replace tuples by struct with field --- bin/node/executor/src/lib.rs | 185 ++++++++++-------- bin/node/testing/src/client.rs | 2 +- client/api/src/backend.rs | 4 +- client/chain-spec/src/chain_spec.rs | 78 +++++--- client/db/src/lib.rs | 28 +-- client/executor/src/integration_tests/mod.rs | 14 +- client/src/client.rs | 4 +- client/src/genesis.rs | 11 +- client/src/in_mem.rs | 18 +- client/src/light/backend.rs | 20 +- .../src/storage/genesis_config/mod.rs | 19 +- frame/support/test/tests/instance.rs | 7 +- frame/system/src/lib.rs | 13 +- primitives/core/storage/src/lib.rs | 34 +++- primitives/sr-io/src/lib.rs | 22 ++- primitives/sr-primitives/src/lib.rs | 28 +-- primitives/state-machine/src/backend.rs | 9 +- primitives/state-machine/src/basic.rs | 103 +++++----- primitives/state-machine/src/testing.rs | 10 +- test/utils/client/src/lib.rs | 24 ++- test/utils/runtime/client/src/lib.rs | 26 +-- test/utils/runtime/src/genesismap.rs | 33 ++-- test/utils/runtime/src/system.rs | 8 +- 23 files changed, 381 insertions(+), 319 deletions(-) diff --git a/bin/node/executor/src/lib.rs b/bin/node/executor/src/lib.rs index b2410ed15edeb..70a9853fc6b15 100644 --- a/bin/node/executor/src/lib.rs +++ b/bin/node/executor/src/lib.rs @@ -42,7 +42,7 @@ mod tests { use state_machine::TestExternalities as CoreTestExternalities; use primitives::{ Blake2Hasher, NeverNativeValue, NativeOrEncoded, map, - traits::{CodeExecutor, Externalities}, storage::well_known_keys, + traits::{CodeExecutor, Externalities}, storage::{well_known_keys, Storage}, }; use sp_runtime::{ Fixed64, traits::{Header as HeaderT, Hash as HashT, Convert}, ApplyExtrinsicResult, @@ -143,20 +143,23 @@ mod tests { #[test] fn panic_execution_with_foreign_code_gives_error() { - let mut t = TestExternalities::::new_with_code(BLOATY_CODE, (map![ - >::hashed_key_for(alice()) => { - 69_u128.encode() - }, - >::hashed_key().to_vec() => { - 69_u128.encode() - }, - >::hashed_key().to_vec() => { - 0_u128.encode() - }, - >::hashed_key_for(0) => { - vec![0u8; 32] - } - ], map![])); + let mut t = TestExternalities::::new_with_code(BLOATY_CODE, Storage { + top: map![ + >::hashed_key_for(alice()) => { + 69_u128.encode() + }, + >::hashed_key().to_vec() => { + 69_u128.encode() + }, + >::hashed_key().to_vec() => { + 0_u128.encode() + }, + >::hashed_key_for(0) => { + vec![0u8; 32] + } + ], + children: map![], + }); let r = executor_call:: _>( &mut t, @@ -179,20 +182,23 @@ mod tests { #[test] fn bad_extrinsic_with_native_equivalent_code_gives_error() { - let mut t = TestExternalities::::new_with_code(COMPACT_CODE, (map![ - >::hashed_key_for(alice()) => { - 69_u128.encode() - }, - >::hashed_key().to_vec() => { - 69_u128.encode() - }, - >::hashed_key().to_vec() => { - 0_u128.encode() - }, - >::hashed_key_for(0) => { - vec![0u8; 32] - } - ], map![])); + let mut t = TestExternalities::::new_with_code(COMPACT_CODE, Storage { + top: map![ + >::hashed_key_for(alice()) => { + 69_u128.encode() + }, + >::hashed_key().to_vec() => { + 69_u128.encode() + }, + >::hashed_key().to_vec() => { + 0_u128.encode() + }, + >::hashed_key_for(0) => { + vec![0u8; 32] + } + ], + children: map![], + }); let r = executor_call:: _>( &mut t, @@ -215,16 +221,19 @@ mod tests { #[test] fn successful_execution_with_native_equivalent_code_gives_ok() { - let mut t = TestExternalities::::new_with_code(COMPACT_CODE, (map![ - >::hashed_key_for(alice()) => { - (111 * DOLLARS).encode() - }, - >::hashed_key().to_vec() => { - (111 * DOLLARS).encode() - }, - >::hashed_key().to_vec() => vec![0u8; 16], - >::hashed_key_for(0) => vec![0u8; 32] - ], map![])); + let mut t = TestExternalities::::new_with_code(COMPACT_CODE, Storage { + top: map![ + >::hashed_key_for(alice()) => { + (111 * DOLLARS).encode() + }, + >::hashed_key().to_vec() => { + (111 * DOLLARS).encode() + }, + >::hashed_key().to_vec() => vec![0u8; 16], + >::hashed_key_for(0) => vec![0u8; 32] + ], + children: map![], + }); let r = executor_call:: _>( &mut t, @@ -254,16 +263,19 @@ mod tests { #[test] fn successful_execution_with_foreign_code_gives_ok() { - let mut t = TestExternalities::::new_with_code(BLOATY_CODE, (map![ - >::hashed_key_for(alice()) => { - (111 * DOLLARS).encode() - }, - >::hashed_key().to_vec() => { - (111 * DOLLARS).encode() - }, - >::hashed_key().to_vec() => vec![0u8; 16], - >::hashed_key_for(0) => vec![0u8; 32] - ], map![])); + let mut t = TestExternalities::::new_with_code(BLOATY_CODE, Storage { + top: map![ + >::hashed_key_for(alice()) => { + (111 * DOLLARS).encode() + }, + >::hashed_key().to_vec() => { + (111 * DOLLARS).encode() + }, + >::hashed_key().to_vec() => vec![0u8; 16], + >::hashed_key_for(0) => vec![0u8; 32] + ], + children: map![], + }); let r = executor_call:: _>( &mut t, @@ -828,16 +840,19 @@ mod tests { #[test] fn panic_execution_gives_error() { - let mut t = TestExternalities::::new_with_code(BLOATY_CODE, (map![ - >::hashed_key_for(alice()) => { - 0_u128.encode() - }, - >::hashed_key().to_vec() => { - 0_u128.encode() - }, - >::hashed_key().to_vec() => vec![0u8; 16], - >::hashed_key_for(0) => vec![0u8; 32] - ], map![])); + let mut t = TestExternalities::::new_with_code(BLOATY_CODE, Storage { + top: map![ + >::hashed_key_for(alice()) => { + 0_u128.encode() + }, + >::hashed_key().to_vec() => { + 0_u128.encode() + }, + >::hashed_key().to_vec() => vec![0u8; 16], + >::hashed_key_for(0) => vec![0u8; 32] + ], + children: map![], + }); let r = executor_call:: _>( &mut t, @@ -860,16 +875,19 @@ mod tests { #[test] fn successful_execution_gives_ok() { - let mut t = TestExternalities::::new_with_code(COMPACT_CODE, (map![ - >::hashed_key_for(alice()) => { - (111 * DOLLARS).encode() - }, - >::hashed_key().to_vec() => { - (111 * DOLLARS).encode() - }, - >::hashed_key().to_vec() => vec![0u8; 16], - >::hashed_key_for(0) => vec![0u8; 32] - ], map![])); + let mut t = TestExternalities::::new_with_code(COMPACT_CODE, Storage { + top: map![ + >::hashed_key_for(alice()) => { + (111 * DOLLARS).encode() + }, + >::hashed_key().to_vec() => { + (111 * DOLLARS).encode() + }, + >::hashed_key().to_vec() => vec![0u8; 16], + >::hashed_key_for(0) => vec![0u8; 32] + ], + children: map![], + }); let r = executor_call:: _>( &mut t, @@ -1037,19 +1055,22 @@ mod tests { // - 1 MILLICENTS in substrate node. // - 1 milli-dot based on current polkadot runtime. // (this baed on assigning 0.1 CENT to the cheapest tx with `weight = 100`) - let mut t = TestExternalities::::new_with_code(COMPACT_CODE, (map![ - >::hashed_key_for(alice()) => { - (100 * DOLLARS).encode() - }, - >::hashed_key_for(bob()) => { - (10 * DOLLARS).encode() - }, - >::hashed_key().to_vec() => { - (110 * DOLLARS).encode() - }, - >::hashed_key().to_vec() => vec![0u8; 16], - >::hashed_key_for(0) => vec![0u8; 32] - ], map![])); + let mut t = TestExternalities::::new_with_code(COMPACT_CODE, Storage { + top: map![ + >::hashed_key_for(alice()) => { + (100 * DOLLARS).encode() + }, + >::hashed_key_for(bob()) => { + (10 * DOLLARS).encode() + }, + >::hashed_key().to_vec() => { + (110 * DOLLARS).encode() + }, + >::hashed_key().to_vec() => vec![0u8; 16], + >::hashed_key_for(0) => vec![0u8; 32] + ], + children: map![], + }); let tip = 1_000_000; let xt = sign(CheckedExtrinsic { diff --git a/bin/node/testing/src/client.rs b/bin/node/testing/src/client.rs index b865a407fac4d..a6964b39156d0 100644 --- a/bin/node/testing/src/client.rs +++ b/bin/node/testing/src/client.rs @@ -42,7 +42,7 @@ pub struct GenesisParameters { } impl test_client::GenesisInit for GenesisParameters { - fn genesis_storage(&self) -> (StorageOverlay, ChildrenStorageOverlay) { + fn genesis_storage(&self) -> Storage { crate::genesis::config(self.support_changes_trie, None).build_storage().unwrap() } } diff --git a/client/api/src/backend.rs b/client/api/src/backend.rs index 488cab6bb01d8..a71ffff74b56e 100644 --- a/client/api/src/backend.rs +++ b/client/api/src/backend.rs @@ -20,7 +20,7 @@ use std::sync::Arc; use std::collections::HashMap; use primitives::ChangesTrieConfiguration; use primitives::offchain::OffchainStorage; -use sp_runtime::{generic::BlockId, Justification, StorageOverlay, ChildrenStorageOverlay}; +use sp_runtime::{generic::BlockId, Justification, Storage}; use sp_runtime::traits::{Block as BlockT, NumberFor}; use state_machine::backend::Backend as StateBackend; use state_machine::{ChangesTrieStorage as StateChangesTrieStorage, ChangesTrieTransaction}; @@ -134,7 +134,7 @@ pub trait BlockImportOperation where fn update_db_storage(&mut self, update: >::Transaction) -> sp_blockchain::Result<()>; /// Inject storage data into the database replacing any existing data. - fn reset_storage(&mut self, top: StorageOverlay, children: ChildrenStorageOverlay) -> sp_blockchain::Result; + fn reset_storage(&mut self, storage: Storage) -> sp_blockchain::Result; /// Set storage changes. fn update_storage( diff --git a/client/chain-spec/src/chain_spec.rs b/client/chain-spec/src/chain_spec.rs index edc4f89d38663..3c4cc7a54abb7 100644 --- a/client/chain-spec/src/chain_spec.rs +++ b/client/chain-spec/src/chain_spec.rs @@ -22,8 +22,8 @@ use std::fs::File; use std::path::PathBuf; use std::rc::Rc; use serde::{Serialize, Deserialize}; -use primitives::storage::{StorageKey, StorageData, ChildInfo}; -use sp_runtime::{BuildStorage, StorageOverlay, ChildrenStorageOverlay}; +use primitives::storage::{StorageKey, StorageData, ChildInfo, Storage, StorageChild}; +use sp_runtime::BuildStorage; use serde_json as json; use crate::RuntimeGenesis; use network::Multiaddr; @@ -71,48 +71,62 @@ impl GenesisSource { } impl<'a, G: RuntimeGenesis, E> BuildStorage for &'a ChainSpec { - fn build_storage(&self) -> Result<(StorageOverlay, ChildrenStorageOverlay), String> { + fn build_storage(&self) -> Result { match self.genesis.resolve()? { Genesis::Runtime(gc) => gc.build_storage(), - Genesis::Raw(map, children_map) => Ok(( - map.into_iter().map(|(k, v)| (k.0, v.0)).collect(), - children_map.into_iter().map(|(sk, child_content)| { + Genesis::Raw(RawGenesis { top: map, children: children_map }) => Ok(Storage { + top: map.into_iter().map(|(k, v)| (k.0, v.0)).collect(), + children: children_map.into_iter().map(|(sk, child_content)| { let child_info = ChildInfo::resolve_child_info( - child_content.2, - child_content.1.as_slice(), + child_content.child_type, + child_content.child_info.as_slice(), ).expect("chainspec contains correct content").to_owned(); ( sk.0, - (child_content.0.into_iter().map(|(k, v)| (k.0, v.0)).collect(), - child_info), + StorageChild { + data: child_content.data.into_iter().map(|(k, v)| (k.0, v.0)).collect(), + child_info, + }, ) }).collect(), - )), + }), } } fn assimilate_storage( &self, - _: &mut (StorageOverlay, ChildrenStorageOverlay) + _: &mut Storage, ) -> Result<(), String> { Err("`assimilate_storage` not implemented for `ChainSpec`.".into()) } } +type GenesisStorage = HashMap; + +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[serde(deny_unknown_fields)] +struct ChildRawStorage { + data: GenesisStorage, + child_info: Vec, + child_type: u32, +} + +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[serde(deny_unknown_fields)] +/// Storage content for genesis block. +struct RawGenesis { + pub top: GenesisStorage, + pub children: HashMap, +} + #[derive(Serialize, Deserialize)] #[serde(rename_all = "camelCase")] #[serde(deny_unknown_fields)] enum Genesis { Runtime(G), - Raw( - // Map of values in the top trie. - HashMap, - // Map of child tries. - // Each child tries contains its map of values - // and a child info encoded as a byte array with the - // type of this child info as a u32. - HashMap, Vec, u32)>, - ), + Raw(RawGenesis), } #[derive(Serialize, Deserialize, Clone, Debug)] @@ -267,26 +281,26 @@ impl ChainSpec { let genesis = match (raw, self.genesis.resolve()?) { (true, Genesis::Runtime(g)) => { let storage = g.build_storage()?; - let top = storage.0.into_iter() + let top = storage.top.into_iter() .map(|(k, v)| (StorageKey(k), StorageData(v))) .collect(); - let children = storage.1.into_iter() + let children = storage.children.into_iter() .map(|(sk, child)| { - let info = child.1.as_ref(); + let info = child.child_info.as_ref(); let (info, ci_type) = info.info(); ( StorageKey(sk), - ( - child.0.into_iter() + ChildRawStorage { + data: child.data.into_iter() .map(|(k, v)| (StorageKey(k), StorageData(v))) .collect(), - info.to_vec(), - ci_type, - ), + child_info: info.to_vec(), + child_type: ci_type, + }, )}) .collect(); - Genesis::Raw(top, children) + Genesis::Raw(RawGenesis { top, children }) }, (_, genesis) => genesis, }; @@ -309,9 +323,9 @@ mod tests { impl BuildStorage for Genesis { fn assimilate_storage( &self, - storage: &mut (StorageOverlay, ChildrenStorageOverlay), + storage: &mut Storage, ) -> Result<(), String> { - storage.0.extend( + storage.top.extend( self.0.iter().map(|(a, b)| (a.clone().into_bytes(), b.clone().into_bytes())) ); Ok(()) diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index d544f3a23d669..ecd8550610e22 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -54,7 +54,7 @@ use parking_lot::{Mutex, RwLock}; use primitives::{H256, Blake2Hasher, ChangesTrieConfiguration, convert_hash, traits::CodeExecutor}; use primitives::storage::{well_known_keys, ChildInfo}; use sp_runtime::{ - generic::{BlockId, DigestItem}, Justification, StorageOverlay, ChildrenStorageOverlay, + generic::{BlockId, DigestItem}, Justification, Storage, BuildStorage, }; use sp_runtime::traits::{ @@ -546,26 +546,26 @@ impl client_api::backend::BlockImportOperation fn reset_storage( &mut self, - top: StorageOverlay, - children: ChildrenStorageOverlay + storage: Storage, ) -> ClientResult { - if top.iter().any(|(k, _)| well_known_keys::is_child_storage_key(k)) { + if storage.top.iter().any(|(k, _)| well_known_keys::is_child_storage_key(k)) { return Err(sp_blockchain::Error::GenesisInvalid.into()); } - for child_key in children.keys() { + for child_key in storage.children.keys() { if !well_known_keys::is_child_storage_key(&child_key) { return Err(sp_blockchain::Error::GenesisInvalid.into()); } } - let child_delta = children.into_iter() - .map(|(storage_key, child_content)| - (storage_key, child_content.0.into_iter().map(|(k, v)| (k, Some(v))), child_content.1)); + let child_delta = storage.children.into_iter().map(|(storage_key, child_content)| ( + storage_key, + child_content.data.into_iter().map(|(k, v)| (k, Some(v))), child_content.child_info), + ); let (root, transaction) = self.old_state.full_storage_root( - top.into_iter().map(|(k, v)| (k, Some(v))), + storage.top.into_iter().map(|(k, v)| (k, Some(v))), child_delta ); @@ -1735,7 +1735,10 @@ mod tests { ).0.into(); let hash = header.hash(); - op.reset_storage(storage.iter().cloned().collect(), Default::default()).unwrap(); + op.reset_storage(Storage { + top: storage.iter().cloned().collect(), + children: Default::default(), + }).unwrap(); op.set_block_data( header.clone(), Some(vec![]), @@ -1817,7 +1820,10 @@ mod tests { ).0.into(); let hash = header.hash(); - op.reset_storage(storage.iter().cloned().collect(), Default::default()).unwrap(); + op.reset_storage(Storage { + top: storage.iter().cloned().collect(), + children: Default::default(), + }).unwrap(); key = op.db_updates.insert(EMPTY_PREFIX, b"hello"); op.set_block_data( diff --git a/client/executor/src/integration_tests/mod.rs b/client/executor/src/integration_tests/mod.rs index fce39bb4006c3..c6b61f110f14c 100644 --- a/client/executor/src/integration_tests/mod.rs +++ b/client/executor/src/integration_tests/mod.rs @@ -128,11 +128,14 @@ fn storage_should_work(wasm_method: WasmExecutionMethod) { assert_eq!(output, b"all ok!".to_vec().encode()); } - let expected = TestExternalities::new((map![ + let expected = TestExternalities::new(primitives::storage::Storage { + top: map![ b"input".to_vec() => b"Hello world".to_vec(), b"foo".to_vec() => b"bar".to_vec(), b"baz".to_vec() => b"bar".to_vec() - ], map![])); + ], + children: map![], + }); assert_eq!(ext, expected); } @@ -162,11 +165,14 @@ fn clear_prefix_should_work(wasm_method: WasmExecutionMethod) { assert_eq!(output, b"all ok!".to_vec().encode()); } - let expected = TestExternalities::new((map![ + let expected = TestExternalities::new(primitives::storage::Storage { + top: map![ b"aaa".to_vec() => b"1".to_vec(), b"aab".to_vec() => b"2".to_vec(), b"bbb".to_vec() => b"5".to_vec() - ], map![])); + ], + children: map![], + }); assert_eq!(expected, ext); } diff --git a/client/src/client.rs b/client/src/client.rs index 936eb9c3a510a..fe3dfbead346b 100644 --- a/client/src/client.rs +++ b/client/src/client.rs @@ -199,10 +199,10 @@ impl Client where execution_extensions: ExecutionExtensions, ) -> sp_blockchain::Result { if backend.blockchain().header(BlockId::Number(Zero::zero()))?.is_none() { - let (genesis_storage, children_genesis_storage) = build_genesis_storage.build_storage()?; + let genesis_storage = build_genesis_storage.build_storage()?; let mut op = backend.begin_operation()?; backend.begin_state_operation(&mut op, BlockId::Hash(Default::default()))?; - let state_root = op.reset_storage(genesis_storage, children_genesis_storage)?; + let state_root = op.reset_storage(genesis_storage)?; let genesis_block = genesis::construct_genesis_block::(state_root.into()); info!("Initializing Genesis block/state (state: {}, header-hash: {})", genesis_block.header().state_root(), diff --git a/client/src/genesis.rs b/client/src/genesis.rs index d2743167422aa..a080a87cf46c5 100644 --- a/client/src/genesis.rs +++ b/client/src/genesis.rs @@ -53,7 +53,7 @@ mod tests { runtime::{Hash, Transfer, Block, BlockNumber, Header, Digest}, AccountKeyring, Sr25519Keyring, }; - use primitives::{Blake2Hasher, map}; + use primitives::Blake2Hasher; use hex_literal::*; native_executor_instance!( @@ -154,8 +154,7 @@ mod tests { vec![AccountKeyring::One.into(), AccountKeyring::Two.into()], 1000, None, - map![], - map![], + Default::default(), ).genesis_map(); let genesis_hash = insert_genesis_block(&mut storage); @@ -183,8 +182,7 @@ mod tests { vec![AccountKeyring::One.into(), AccountKeyring::Two.into()], 1000, None, - map![], - map![], + Default::default(), ).genesis_map(); let genesis_hash = insert_genesis_block(&mut storage); @@ -212,8 +210,7 @@ mod tests { vec![AccountKeyring::One.into(), AccountKeyring::Two.into()], 68, None, - map![], - map![], + Default::default(), ).genesis_map(); let genesis_hash = insert_genesis_block(&mut storage); diff --git a/client/src/in_mem.rs b/client/src/in_mem.rs index e28b6dcdae13f..85bdd954c80ba 100644 --- a/client/src/in_mem.rs +++ b/client/src/in_mem.rs @@ -25,7 +25,7 @@ use primitives::offchain::storage::{ }; use sp_runtime::generic::{BlockId, DigestItem}; use sp_runtime::traits::{Block as BlockT, Header as HeaderT, Zero, NumberFor}; -use sp_runtime::{Justification, StorageOverlay, ChildrenStorageOverlay}; +use sp_runtime::{Justification, Storage}; use state_machine::backend::{Backend as StateBackend, InMemory}; use state_machine::{self, InMemoryChangesTrieStorage, ChangesTrieAnchorBlockId, ChangesTrieTransaction}; use hash_db::{Hasher, Prefix}; @@ -505,15 +505,15 @@ where Ok(()) } - fn reset_storage(&mut self, top: StorageOverlay, children: ChildrenStorageOverlay) -> sp_blockchain::Result { - check_genesis_storage(&top, &children)?; + fn reset_storage(&mut self, storage: Storage) -> sp_blockchain::Result { + check_genesis_storage(&storage)?; - let child_delta = children.into_iter() + let child_delta = storage.children.into_iter() .map(|(storage_key, child_content)| - (storage_key, child_content.0.into_iter().map(|(k, v)| (k, Some(v))), child_content.1)); + (storage_key, child_content.data.into_iter().map(|(k, v)| (k, Some(v))), child_content.child_info)); let (root, transaction) = self.old_state.full_storage_root( - top.into_iter().map(|(k, v)| (k, Some(v))), + storage.top.into_iter().map(|(k, v)| (k, Some(v))), child_delta ); @@ -796,12 +796,12 @@ impl state_machine::ChangesTrieStorage> for Change } /// Check that genesis storage is valid. -pub fn check_genesis_storage(top: &StorageOverlay, children: &ChildrenStorageOverlay) -> sp_blockchain::Result<()> { - if top.iter().any(|(k, _)| well_known_keys::is_child_storage_key(k)) { +pub fn check_genesis_storage(storage: &Storage) -> sp_blockchain::Result<()> { + if storage.top.iter().any(|(k, _)| well_known_keys::is_child_storage_key(k)) { return Err(sp_blockchain::Error::GenesisInvalid.into()); } - if children.keys().any(|child_key| !well_known_keys::is_child_storage_key(&child_key)) { + if storage.children.keys().any(|child_key| !well_known_keys::is_child_storage_key(&child_key)) { return Err(sp_blockchain::Error::GenesisInvalid.into()); } diff --git a/client/src/light/backend.rs b/client/src/light/backend.rs index fc6b8724ba3ca..41548fd18cf93 100644 --- a/client/src/light/backend.rs +++ b/client/src/light/backend.rs @@ -26,7 +26,7 @@ use state_machine::{ Backend as StateBackend, TrieBackend, backend::InMemory as InMemoryState, ChangesTrieTransaction }; use primitives::offchain::storage::InMemOffchainStorage; -use sp_runtime::{generic::BlockId, Justification, StorageOverlay, ChildrenStorageOverlay}; +use sp_runtime::{generic::BlockId, Justification, Storage}; use sp_runtime::traits::{Block as BlockT, NumberFor, Zero, Header}; use crate::in_mem::{self, check_genesis_storage}; use sp_blockchain::{ Error as ClientError, Result as ClientResult }; @@ -281,21 +281,21 @@ where Ok(()) } - fn reset_storage(&mut self, top: StorageOverlay, children: ChildrenStorageOverlay) -> ClientResult { - check_genesis_storage(&top, &children)?; + fn reset_storage(&mut self, input: Storage) -> ClientResult { + check_genesis_storage(&input)?; // this is only called when genesis block is imported => shouldn't be performance bottleneck - let mut storage: HashMap, OwnedChildInfo)>, StorageOverlay> = HashMap::new(); - storage.insert(None, top); + let mut storage: HashMap, OwnedChildInfo)>, _> = HashMap::new(); + storage.insert(None, input.top); // create a list of children keys to re-compute roots for - let child_delta = children.iter() - .map(|(storage_key, (_m, child_info))| (storage_key.clone(), None, child_info.clone())) + let child_delta = input.children.iter() + .map(|(storage_key, storage_child)| (storage_key.clone(), None, storage_child.child_info.clone())) .collect::>(); // make sure to persist the child storage - for (child_key, (child_storage, child_info)) in children { - storage.insert(Some((child_key, child_info)), child_storage); + for (child_key, storage_child) in input.children { + storage.insert(Some((child_key, storage_child.child_info)), storage_child.data); } let storage_update: InMemoryState = storage.into(); @@ -478,7 +478,7 @@ mod tests { let backend: Backend<_, Blake2Hasher> = Backend::new(Arc::new(DummyBlockchain::new(DummyStorage::new()))); let mut op = backend.begin_operation().unwrap(); op.set_block_data(header0, None, None, NewBlockState::Final).unwrap(); - op.reset_storage(Default::default(), Default::default()).unwrap(); + op.reset_storage(Default::default()).unwrap(); backend.commit_operation(op).unwrap(); match backend.state_at(BlockId::Number(0)).unwrap() { diff --git a/frame/support/procedural/src/storage/genesis_config/mod.rs b/frame/support/procedural/src/storage/genesis_config/mod.rs index 4102513cce852..4e91ccd1df13e 100644 --- a/frame/support/procedural/src/storage/genesis_config/mod.rs +++ b/frame/support/procedural/src/storage/genesis_config/mod.rs @@ -139,13 +139,10 @@ fn impl_build_storage( #[cfg(feature = "std")] impl#genesis_impl GenesisConfig#genesis_struct #genesis_where_clause { pub fn build_storage #fn_generic (&self) -> std::result::Result< - ( - #scrate::sp_runtime::StorageOverlay, - #scrate::sp_runtime::ChildrenStorageOverlay, - ), + #scrate::sp_runtime::Storage, String > #fn_where_clause { - let mut storage = (Default::default(), Default::default()); + let mut storage = Default::default(); self.assimilate_storage::<#fn_traitinstance>(&mut storage)?; Ok(storage) } @@ -153,12 +150,9 @@ fn impl_build_storage( /// Assimilate the storage for this module into pre-existing overlays. pub fn assimilate_storage #fn_generic ( &self, - tuple_storage: &mut ( - #scrate::sp_runtime::StorageOverlay, - #scrate::sp_runtime::ChildrenStorageOverlay, - ), + storage: &mut #scrate::sp_runtime::Storage, ) -> std::result::Result<(), String> #fn_where_clause { - #scrate::BasicExternalities::execute_with_storage(tuple_storage, || { + #scrate::BasicExternalities::execute_with_storage(storage, || { #( #builder_blocks )* Ok(()) }) @@ -171,10 +165,7 @@ fn impl_build_storage( { fn build_module_genesis_storage( &self, - storage: &mut ( - #scrate::sp_runtime::StorageOverlay, - #scrate::sp_runtime::ChildrenStorageOverlay, - ), + storage: &mut #scrate::sp_runtime::Storage, ) -> std::result::Result<(), String> { self.assimilate_storage::<#fn_traitinstance> (storage) } diff --git a/frame/support/test/tests/instance.rs b/frame/support/test/tests/instance.rs index a3ef6ba86881d..af60de0fb90c9 100644 --- a/frame/support/test/tests/instance.rs +++ b/frame/support/test/tests/instance.rs @@ -300,7 +300,10 @@ fn new_test_ext() -> runtime_io::TestExternalities { #[test] fn storage_instance_independance() { - let mut storage = (std::collections::HashMap::new(), std::collections::HashMap::new()); + let mut storage = primitives::storage::Storage { + top: std::collections::HashMap::new(), + children: std::collections::HashMap::new() + }; state_machine::BasicExternalities::execute_with_storage(&mut storage, || { module2::Value::::put(0); module2::Value::::put(0); @@ -320,7 +323,7 @@ fn storage_instance_independance() { module2::DoubleMap::::insert(&0, &0, &0); }); // 16 storage values + 4 linked_map head. - assert_eq!(storage.0.len(), 16 + 4); + assert_eq!(storage.top.len(), 16 + 4); } #[test] diff --git a/frame/system/src/lib.rs b/frame/system/src/lib.rs index cce3468639a72..09d75829184fb 100644 --- a/frame/system/src/lib.rs +++ b/frame/system/src/lib.rs @@ -703,11 +703,14 @@ impl Module { /// Get the basic externalities for this module, useful for tests. #[cfg(any(feature = "std", test))] pub fn externalities() -> TestExternalities { - TestExternalities::new((map![ - >::hashed_key_for(T::BlockNumber::zero()) => [69u8; 32].encode(), - >::hashed_key().to_vec() => T::BlockNumber::one().encode(), - >::hashed_key().to_vec() => [69u8; 32].encode() - ], map![])) + TestExternalities::new(primitives::storage::Storage { + top: map![ + >::hashed_key_for(T::BlockNumber::zero()) => [69u8; 32].encode(), + >::hashed_key().to_vec() => T::BlockNumber::one().encode(), + >::hashed_key().to_vec() => [69u8; 32].encode() + ], + children: map![], + }) } /// Set the block number to something in particular. Can be used as an alternative to diff --git a/primitives/core/storage/src/lib.rs b/primitives/core/storage/src/lib.rs index f4a745b1eaeff..5622364e784aa 100644 --- a/primitives/core/storage/src/lib.rs +++ b/primitives/core/storage/src/lib.rs @@ -40,16 +40,40 @@ pub struct StorageData( pub Vec, ); + +/// Map of data to use in a storage, it is a collection of +/// byte key and values. +#[cfg(feature = "std")] +pub type StorageMap = std::collections::HashMap, Vec>; + +#[cfg(feature = "std")] +#[derive(Debug, PartialEq, Eq, Clone)] +/// Child trie storage data. +pub struct StorageChild { + /// Child data for storage. + pub data: StorageMap, + /// Associated child info for a child + /// trie. + pub child_info: OwnedChildInfo, +} + +#[cfg(feature = "std")] +#[derive(Default, Debug, Clone)] +/// Struct containing data needed for a storage. +pub struct Storage { + /// Top trie storage data. + pub top: StorageMap, + /// Children trie storage data by storage key. + pub children: std::collections::HashMap, StorageChild>, +} + /// A set of key value pairs for storage. #[cfg(feature = "std")] -pub type StorageOverlay = std::collections::HashMap, Vec>; +pub type StorageOverlay = StorageMap; /// A set of key value pairs for children storage; #[cfg(feature = "std")] -pub type ChildrenStorageOverlay = std::collections::HashMap< - Vec, - (StorageOverlay, OwnedChildInfo), ->; +pub type ChildrenStorageOverlay = std::collections::HashMap, StorageChild>; /// Storage change set #[derive(RuntimeDebug)] diff --git a/primitives/sr-io/src/lib.rs b/primitives/sr-io/src/lib.rs index 3fa998fab2f17..3e40673ef6b6f 100644 --- a/primitives/sr-io/src/lib.rs +++ b/primitives/sr-io/src/lib.rs @@ -883,6 +883,7 @@ mod tests { use super::*; use primitives::map; use sp_state_machine::BasicExternalities; + use primitives::storage::Storage; #[test] fn storage_works() { @@ -895,7 +896,10 @@ mod tests { storage::set(b"foo", &[1, 2, 3][..]); }); - t = BasicExternalities::new(map![b"foo".to_vec() => b"bar".to_vec()], map![]); + t = BasicExternalities::new(Storage { + top: map![b"foo".to_vec() => b"bar".to_vec()], + children: map![], + }); t.execute_with(|| { assert_eq!(storage::get(b"hello"), None); @@ -905,10 +909,10 @@ mod tests { #[test] fn read_storage_works() { - let mut t = BasicExternalities::new( - map![b":test".to_vec() => b"\x0b\0\0\0Hello world".to_vec()], - map![], - ); + let mut t = BasicExternalities::new(Storage { + top: map![b":test".to_vec() => b"\x0b\0\0\0Hello world".to_vec()], + children: map![], + }); t.execute_with(|| { let mut v = [0u8; 4]; @@ -922,15 +926,15 @@ mod tests { #[test] fn clear_prefix_works() { - let mut t = BasicExternalities::new( - map![ + let mut t = BasicExternalities::new(Storage { + top: map![ b":a".to_vec() => b"\x0b\0\0\0Hello world".to_vec(), b":abcd".to_vec() => b"\x0b\0\0\0Hello world".to_vec(), b":abc".to_vec() => b"\x0b\0\0\0Hello world".to_vec(), b":abdd".to_vec() => b"\x0b\0\0\0Hello world".to_vec() ], - map![], - ); + children: map![], + }); t.execute_with(|| { storage::clear_prefix(b":abc"); diff --git a/primitives/sr-primitives/src/lib.rs b/primitives/sr-primitives/src/lib.rs index 38940b563c2d6..041e871de8a52 100644 --- a/primitives/sr-primitives/src/lib.rs +++ b/primitives/sr-primitives/src/lib.rs @@ -38,7 +38,7 @@ pub use paste; pub use app_crypto; #[cfg(feature = "std")] -pub use primitives::storage::{StorageOverlay, ChildrenStorageOverlay}; +pub use primitives::storage::{Storage, StorageChild}; use rstd::prelude::*; use rstd::convert::TryFrom; @@ -121,15 +121,15 @@ use crate::traits::IdentifyAccount; #[cfg(feature = "std")] pub trait BuildStorage: Sized { /// Build the storage out of this builder. - fn build_storage(&self) -> Result<(StorageOverlay, ChildrenStorageOverlay), String> { - let mut storage = (Default::default(), Default::default()); + fn build_storage(&self) -> Result { + let mut storage = Default::default(); self.assimilate_storage(&mut storage)?; Ok(storage) } /// Assimilate the storage for this module into pre-existing overlays. fn assimilate_storage( &self, - storage: &mut (StorageOverlay, ChildrenStorageOverlay), + storage: &mut primitives::storage::Storage, ) -> Result<(), String>; } @@ -139,26 +139,26 @@ pub trait BuildModuleGenesisStorage: Sized { /// Create the module genesis storage into the given `storage` and `child_storage`. fn build_module_genesis_storage( &self, - storage: &mut (StorageOverlay, ChildrenStorageOverlay), + storage: &mut primitives::storage::Storage, ) -> Result<(), String>; } #[cfg(feature = "std")] -impl BuildStorage for (StorageOverlay, ChildrenStorageOverlay) { +impl BuildStorage for primitives::storage::Storage { fn assimilate_storage( &self, - storage: &mut (StorageOverlay, ChildrenStorageOverlay), + storage: &mut primitives::storage::Storage, )-> Result<(), String> { - storage.0.extend(self.0.iter().map(|(k, v)| (k.clone(), v.clone()))); - for (k, other_map) in self.1.iter() { + storage.top.extend(self.top.iter().map(|(k, v)| (k.clone(), v.clone()))); + for (k, other_map) in self.children.iter() { let k = k.clone(); - if let Some(map) = storage.1.get_mut(&k) { - map.0.extend(other_map.0.iter().map(|(k, v)| (k.clone(), v.clone()))); - if !map.1.try_update(other_map.1.as_ref()) { + if let Some(map) = storage.children.get_mut(&k) { + map.data.extend(other_map.data.iter().map(|(k, v)| (k.clone(), v.clone()))); + if !map.child_info.try_update(other_map.child_info.as_ref()) { return Err("Incompatible child info update".to_string()); } } else { - storage.1.insert(k, other_map.clone()); + storage.children.insert(k, other_map.clone()); } } Ok(()) @@ -535,7 +535,7 @@ macro_rules! impl_outer_config { impl $crate::BuildStorage for $main { fn assimilate_storage( &self, - storage: &mut ($crate::StorageOverlay, $crate::ChildrenStorageOverlay), + storage: &mut $crate::Storage, ) -> std::result::Result<(), String> { $( if let Some(ref extra) = self.[< $snake $(_ $instance )? >] { diff --git a/primitives/state-machine/src/backend.rs b/primitives/state-machine/src/backend.rs index 87621e39222aa..daf7e62fc6d27 100644 --- a/primitives/state-machine/src/backend.rs +++ b/primitives/state-machine/src/backend.rs @@ -28,10 +28,7 @@ use trie::{ use codec::{Encode, Codec}; use primitives::storage::{ChildInfo, OwnedChildInfo}; -pub(crate) type StorageTuple = ( - HashMap, Vec>, - HashMap, (HashMap, Vec>, OwnedChildInfo)>, -); +pub(crate) type StorageTuple = primitives::storage::Storage; /// A state backend is used to read state data and can have changes committed /// to it. @@ -384,8 +381,8 @@ impl From, OwnedChildInfo)>, HashMap, impl From for InMemory { fn from(inners: StorageTuple) -> Self { let mut inner: HashMap, OwnedChildInfo)>, HashMap, Vec>> - = inners.1.into_iter().map(|(k, (v, ci))| (Some((k, ci)), v)).collect(); - inner.insert(None, inners.0); + = inners.children.into_iter().map(|(k, c)| (Some((k, c.child_info)), c.data)).collect(); + inner.insert(None, inners.top); InMemory { inner: inner, trie: None, diff --git a/primitives/state-machine/src/basic.rs b/primitives/state-machine/src/basic.rs index 2f188bf93c9a7..a38bb16f63a3d 100644 --- a/primitives/state-machine/src/basic.rs +++ b/primitives/state-machine/src/basic.rs @@ -23,8 +23,8 @@ use trie::{TrieConfiguration, default_child_trie_root}; use trie::trie_types::Layout; use primitives::{ storage::{ - well_known_keys::is_child_storage_key, ChildStorageKey, StorageOverlay, - ChildInfo, ChildrenStorageOverlay, + well_known_keys::is_child_storage_key, ChildStorageKey, Storage, + ChildInfo, StorageChild, }, traits::Externalities, Blake2Hasher, }; @@ -34,40 +34,36 @@ use codec::Encode; /// Simple HashMap-based Externalities impl. #[derive(Debug)] pub struct BasicExternalities { - top: StorageOverlay, - children: ChildrenStorageOverlay, + inner: Storage, } impl BasicExternalities { /// Create a new instance of `BasicExternalities` - pub fn new(top: StorageOverlay, children: ChildrenStorageOverlay) -> Self { - BasicExternalities { - top, - children, - } + pub fn new(inner: Storage) -> Self { + BasicExternalities { inner } } /// Insert key/value pub fn insert(&mut self, k: Vec, v: Vec) -> Option> { - self.top.insert(k, v) + self.inner.top.insert(k, v) } /// Consume self and returns inner storages pub fn into_storages(self) -> StorageTuple { - (self.top, self.children) + self.inner } /// Execute the given closure `f` with the externalities set and initialized with `storage`. /// /// Returns the result of the closure and updates `storage` with all changes. pub fn execute_with_storage( - storage: &mut (StorageOverlay, ChildrenStorageOverlay), + storage: &mut primitives::storage::Storage, f: impl FnOnce() -> R, ) -> R { - let mut ext = Self { - top: storage.0.drain().collect(), - children: storage.1.drain().collect(), - }; + let mut ext = Self { inner: Storage { + top: std::mem::replace(&mut storage.top, Default::default()), + children: std::mem::replace(&mut storage.children, Default::default()), + }}; let r = ext.execute_with(f); @@ -86,34 +82,35 @@ impl BasicExternalities { impl PartialEq for BasicExternalities { fn eq(&self, other: &BasicExternalities) -> bool { - self.top.eq(&other.top) && self.children.eq(&other.children) + self.inner.top.eq(&other.inner.top) + && self.inner.children.eq(&other.inner.children) } } impl FromIterator<(Vec, Vec)> for BasicExternalities { fn from_iter, Vec)>>(iter: I) -> Self { let mut t = Self::default(); - t.top.extend(iter); + t.inner.top.extend(iter); t } } impl Default for BasicExternalities { - fn default() -> Self { Self::new(Default::default(), Default::default()) } + fn default() -> Self { Self::new(Default::default()) } } impl From, Vec>> for BasicExternalities { fn from(hashmap: HashMap, Vec>) -> Self { - BasicExternalities { + BasicExternalities { inner: Storage { top: hashmap, children: Default::default(), - } + }} } } impl Externalities for BasicExternalities { fn storage(&self, key: &[u8]) -> Option> { - self.top.get(key).cloned() + self.inner.top.get(key).cloned() } fn storage_hash(&self, key: &[u8]) -> Option> { @@ -134,7 +131,7 @@ impl Externalities for BasicExternalities { _child_info: ChildInfo, key: &[u8], ) -> Option> { - self.children.get(storage_key.as_ref()).and_then(|child| child.0.get(key)).cloned() + self.inner.children.get(storage_key.as_ref()).and_then(|child| child.data.get(key)).cloned() } fn child_storage_hash( @@ -171,8 +168,8 @@ impl Externalities for BasicExternalities { } match maybe_value { - Some(value) => { self.top.insert(key, value); } - None => { self.top.remove(&key); } + Some(value) => { self.inner.top.insert(key, value); } + None => { self.inner.top.remove(&key); } } } @@ -183,12 +180,15 @@ impl Externalities for BasicExternalities { key: Vec, value: Option>, ) { - let child_map = self.children.entry(storage_key.into_owned()) - .or_insert_with(|| (Default::default(), child_info.to_owned())); + let child_map = self.inner.children.entry(storage_key.into_owned()) + .or_insert_with(|| StorageChild { + data: Default::default(), + child_info: child_info.to_owned(), + }); if let Some(value) = value { - child_map.0.insert(key, value); + child_map.data.insert(key, value); } else { - child_map.0.remove(&key); + child_map.data.remove(&key); } } @@ -197,7 +197,7 @@ impl Externalities for BasicExternalities { storage_key: ChildStorageKey, _child_info: ChildInfo, ) { - self.children.remove(storage_key.as_ref()); + self.inner.children.remove(storage_key.as_ref()); } fn clear_prefix(&mut self, prefix: &[u8]) { @@ -209,7 +209,7 @@ impl Externalities for BasicExternalities { return; } - self.top.retain(|key, _| !key.starts_with(prefix)); + self.inner.top.retain(|key, _| !key.starts_with(prefix)); } fn clear_child_prefix( @@ -218,16 +218,16 @@ impl Externalities for BasicExternalities { _child_info: ChildInfo, prefix: &[u8], ) { - if let Some(child) = self.children.get_mut(storage_key.as_ref()) { - child.0.retain(|key, _| !key.starts_with(prefix)); + if let Some(child) = self.inner.children.get_mut(storage_key.as_ref()) { + child.data.retain(|key, _| !key.starts_with(prefix)); } } fn chain_id(&self) -> u64 { 42 } fn storage_root(&mut self) -> Vec { - let mut top = self.top.clone(); - let keys: Vec<_> = self.children.keys().map(|k| k.to_vec()).collect(); + let mut top = self.inner.top.clone(); + let keys: Vec<_> = self.inner.children.keys().map(|k| k.to_vec()).collect(); // Single child trie implementation currently allows using the same child // empty root for all child trie. Using null storage key until multiple // type of child trie support. @@ -244,18 +244,18 @@ impl Externalities for BasicExternalities { } } - Layout::::trie_root(self.top.clone()).as_ref().into() + Layout::::trie_root(self.inner.top.clone()).as_ref().into() } fn child_storage_root( &mut self, storage_key: ChildStorageKey, ) -> Vec { - if let Some(child) = self.children.get(storage_key.as_ref()) { - let delta = child.0.clone().into_iter().map(|(k, v)| (k, Some(v))); + if let Some(child) = self.inner.children.get(storage_key.as_ref()) { + let delta = child.data.clone().into_iter().map(|(k, v)| (k, Some(v))); InMemory::::default() - .child_storage_root(storage_key.as_ref(), child.1.as_ref(), delta).0 + .child_storage_root(storage_key.as_ref(), child.child_info.as_ref(), delta).0 } else { default_child_trie_root::>(storage_key.as_ref()) }.encode() @@ -277,6 +277,7 @@ impl externalities::ExtensionStore for BasicExternalities { mod tests { use super::*; use primitives::map; + use primitives::storage::{Storage, StorageChild}; use primitives::storage::well_known_keys::CODE; use hex_literal::hex; @@ -307,14 +308,15 @@ mod tests { fn children_works() { let child_storage = b":child_storage:default:test".to_vec(); - let mut ext = BasicExternalities::new( - Default::default(), - map![ - child_storage.clone() => (map![ - b"doe".to_vec() => b"reindeer".to_vec() - ], CHILD_INFO_1.to_owned()), + let mut ext = BasicExternalities::new(Storage { + top: Default::default(), + children: map![ + child_storage.clone() => StorageChild { + data: map![ b"doe".to_vec() => b"reindeer".to_vec() ], + child_info: CHILD_INFO_1.to_owned(), + } ] - ); + }); let child = || ChildStorageKey::from_vec(child_storage.clone()).unwrap(); @@ -333,11 +335,8 @@ mod tests { #[test] fn basic_externalities_is_empty() { // Make sure no values are set by default in `BasicExternalities`. - let (storage, child_storage) = BasicExternalities::new( - Default::default(), - Default::default(), - ).into_storages(); - assert!(storage.is_empty()); - assert!(child_storage.is_empty()); + let storage = BasicExternalities::new(Default::default()).into_storages(); + assert!(storage.top.is_empty()); + assert!(storage.children.is_empty()); } } diff --git a/primitives/state-machine/src/testing.rs b/primitives/state-machine/src/testing.rs index fb35055d855ce..79499e4db1bb0 100644 --- a/primitives/state-machine/src/testing.rs +++ b/primitives/state-machine/src/testing.rs @@ -63,17 +63,17 @@ impl, N: ChangesTrieBlockNumber> TestExternalities { pub fn new_with_code(code: &[u8], mut storage: StorageTuple) -> Self { let mut overlay = OverlayedChanges::default(); - assert!(storage.0.keys().all(|key| !is_child_storage_key(key))); - assert!(storage.1.keys().all(|key| is_child_storage_key(key))); + assert!(storage.top.keys().all(|key| !is_child_storage_key(key))); + assert!(storage.children.keys().all(|key| is_child_storage_key(key))); super::set_changes_trie_config( &mut overlay, - storage.0.get(&CHANGES_TRIE_CONFIG.to_vec()).cloned(), + storage.top.get(&CHANGES_TRIE_CONFIG.to_vec()).cloned(), false, ).expect("changes trie configuration is correct in test env; qed"); - storage.0.insert(HEAP_PAGES.to_vec(), 8u64.encode()); - storage.0.insert(CODE.to_vec(), code.to_vec()); + storage.top.insert(HEAP_PAGES.to_vec(), 8u64.encode()); + storage.top.insert(CODE.to_vec(), code.to_vec()); TestExternalities { overlay, diff --git a/test/utils/client/src/lib.rs b/test/utils/client/src/lib.rs index d4c179b205d57..c6067e5d7c557 100644 --- a/test/utils/client/src/lib.rs +++ b/test/utils/client/src/lib.rs @@ -32,13 +32,13 @@ pub use keyring::{ sr25519::Keyring as Sr25519Keyring, }; pub use primitives::{Blake2Hasher, traits::BareCryptoStorePtr}; -pub use sp_runtime::{StorageOverlay, ChildrenStorageOverlay}; +pub use sp_runtime::{Storage, StorageChild}; pub use state_machine::ExecutionStrategy; use std::sync::Arc; use std::collections::HashMap; use hash_db::Hasher; -use primitives::storage::{well_known_keys, OwnedChildInfo, ChildInfo}; +use primitives::storage::{well_known_keys, ChildInfo}; use sp_runtime::traits::Block as BlockT; use client::LocalCallExecutor; @@ -51,11 +51,11 @@ pub type LightBackend = client::light::backend::Backend< /// A genesis storage initialisation trait. pub trait GenesisInit: Default { /// Construct genesis storage. - fn genesis_storage(&self) -> (StorageOverlay, ChildrenStorageOverlay); + fn genesis_storage(&self) -> Storage; } impl GenesisInit for () { - fn genesis_storage(&self) -> (StorageOverlay, ChildrenStorageOverlay) { + fn genesis_storage(&self) -> Storage { Default::default() } } @@ -64,7 +64,7 @@ impl GenesisInit for () { pub struct TestClientBuilder { execution_strategies: ExecutionStrategies, genesis_init: G, - child_storage_extension: HashMap, (Vec<(Vec, Vec)>, OwnedChildInfo)>, + child_storage_extension: HashMap, StorageChild>, backend: Arc, _executor: std::marker::PhantomData, keystore: Option, @@ -140,8 +140,11 @@ impl TestClientBuilder value: impl AsRef<[u8]>, ) -> Self { let entry = self.child_storage_extension.entry(key.as_ref().to_vec()) - .or_insert_with(|| (Vec::new(), child_info.to_owned())); - entry.0.push((child_key.as_ref().to_vec(), value.as_ref().to_vec())); + .or_insert_with(|| StorageChild { + data: Default::default(), + child_info: child_info.to_owned(), + }); + entry.data.insert(child_key.as_ref().to_vec(), value.as_ref().to_vec()); self } @@ -183,9 +186,12 @@ impl TestClientBuilder // Add some child storage keys. for (key, child_content) in self.child_storage_extension { - storage.1.insert( + storage.children.insert( well_known_keys::CHILD_STORAGE_KEY_PREFIX.iter().cloned().chain(key).collect(), - (child_content.0.into_iter().collect(), child_content.1), + StorageChild { + data: child_content.data.into_iter().collect(), + child_info: child_content.child_info, + }, ); } diff --git a/test/utils/runtime/client/src/lib.rs b/test/utils/runtime/client/src/lib.rs index 1c5134977cf15..13d9b19553700 100644 --- a/test/utils/runtime/client/src/lib.rs +++ b/test/utils/runtime/client/src/lib.rs @@ -29,7 +29,7 @@ pub use generic_test_client::*; pub use runtime; use primitives::sr25519; -use primitives::storage::{ChildInfo, OwnedChildInfo}; +use primitives::storage::{ChildInfo, Storage, StorageChild}; use runtime::genesismap::{GenesisConfig, additional_storage_with_genesis}; use sp_runtime::traits::{Block as BlockT, Header as HeaderT, Hash as HashT, NumberFor}; use client::{ @@ -98,8 +98,7 @@ pub type LightExecutor = client::light::call_executor::GenesisCallExecutor< pub struct GenesisParameters { support_changes_trie: bool, heap_pages_override: Option, - extra_storage: HashMap, Vec>, - child_extra_storage: HashMap, (HashMap, Vec>, OwnedChildInfo)>, + extra_storage: Storage, } impl GenesisParameters { @@ -119,27 +118,26 @@ impl GenesisParameters { 1000, self.heap_pages_override, self.extra_storage.clone(), - self.child_extra_storage.clone(), ) } } impl generic_test_client::GenesisInit for GenesisParameters { - fn genesis_storage(&self) -> (StorageOverlay, ChildrenStorageOverlay) { + fn genesis_storage(&self) -> Storage { use codec::Encode; let mut storage = self.genesis_config().genesis_map(); - let child_roots = storage.1.iter().map(|(sk, child_map)| { + let child_roots = storage.children.iter().map(|(sk, child_content)| { let state_root = <<::Header as HeaderT>::Hashing as HashT>::trie_root( - child_map.0.clone().into_iter().collect() + child_content.data.clone().into_iter().collect() ); (sk.clone(), state_root.encode()) }); let state_root = <<::Header as HeaderT>::Hashing as HashT>::trie_root( - storage.0.clone().into_iter().chain(child_roots).collect() + storage.top.clone().into_iter().chain(child_roots).collect() ); let block: runtime::Block = client::genesis::construct_genesis_block(state_root); - storage.0.extend(additional_storage_with_genesis(&block)); + storage.top.extend(additional_storage_with_genesis(&block)); storage } @@ -230,7 +228,7 @@ impl TestClientBuilderExt for TestClientBuilder< fn add_extra_storage>, V: Into>>(mut self, key: K, value: V) -> Self { let key = key.into(); assert!(!key.is_empty()); - self.genesis_init_mut().extra_storage.insert(key, value.into()); + self.genesis_init_mut().extra_storage.top.insert(key, value.into()); self } @@ -245,10 +243,12 @@ impl TestClientBuilderExt for TestClientBuilder< let key = key.into(); assert!(!storage_key.is_empty()); assert!(!key.is_empty()); - self.genesis_init_mut().child_extra_storage + self.genesis_init_mut().extra_storage.children .entry(storage_key) - .or_insert_with(|| (Default::default(), child_info.to_owned())) - .0.insert(key, value.into()); + .or_insert_with(|| StorageChild { + data: Default::default(), + child_info: child_info.to_owned(), + }).data.insert(key, value.into()); self } diff --git a/test/utils/runtime/src/genesismap.rs b/test/utils/runtime/src/genesismap.rs index 010c1a661e0f7..ff3b1fc786d0a 100644 --- a/test/utils/runtime/src/genesismap.rs +++ b/test/utils/runtime/src/genesismap.rs @@ -21,7 +21,7 @@ use runtime_io::hashing::{blake2_256, twox_128}; use super::{AuthorityId, AccountId, WASM_BINARY, system}; use codec::{Encode, KeyedVec, Joiner}; use primitives::{ChangesTrieConfiguration, map}; -use primitives::storage::{well_known_keys, OwnedChildInfo}; +use primitives::storage::{well_known_keys, Storage}; use sp_runtime::traits::{Block as BlockT, Hash as HashT, Header as HeaderT}; /// Configuration of a general Substrate test genesis block. @@ -31,8 +31,7 @@ pub struct GenesisConfig { balances: Vec<(AccountId, u64)>, heap_pages_override: Option, /// Additional storage key pairs that will be added to the genesis map. - extra_storage: HashMap, Vec>, - child_extra_storage: HashMap, (HashMap, Vec>, OwnedChildInfo)>, + extra_storage: Storage, } impl GenesisConfig { @@ -42,8 +41,7 @@ impl GenesisConfig { endowed_accounts: Vec, balance: u64, heap_pages_override: Option, - extra_storage: HashMap, Vec>, - child_extra_storage: HashMap, (HashMap, Vec>, OwnedChildInfo)>, + extra_storage: Storage, ) -> Self { GenesisConfig { changes_trie_config: match support_changes_trie { @@ -54,14 +52,10 @@ impl GenesisConfig { balances: endowed_accounts.into_iter().map(|a| (a, balance)).collect(), heap_pages_override, extra_storage, - child_extra_storage, } } - pub fn genesis_map(&self) -> ( - HashMap, Vec>, - HashMap, (HashMap, Vec>, OwnedChildInfo)>, - ) { + pub fn genesis_map(&self) -> Storage { let wasm_runtime = WASM_BINARY.to_vec(); let mut map: HashMap, Vec> = self.balances.iter() .map(|&(ref account, balance)| (account.to_keyed_vec(b"balance:"), vec![].and(&balance))) @@ -79,10 +73,10 @@ impl GenesisConfig { } map.insert(twox_128(&b"sys:auth"[..])[..].to_vec(), self.authorities.encode()); // Add the extra storage entries. - map.extend(self.extra_storage.clone().into_iter()); + map.extend(self.extra_storage.top.clone().into_iter()); // Assimilate the system genesis config. - let mut storage = (map, self.child_extra_storage.clone()); + let mut storage = Storage { top: map, children: self.extra_storage.children.clone()}; let mut config = system::GenesisConfig::default(); config.authorities = self.authorities.clone(); config.assimilate_storage(&mut storage).expect("Adding `system::GensisConfig` to the genesis"); @@ -92,25 +86,22 @@ impl GenesisConfig { } pub fn insert_genesis_block( - storage: &mut ( - HashMap, Vec>, - HashMap, (HashMap, Vec>, OwnedChildInfo)>, - ) + storage: &mut Storage, ) -> primitives::hash::H256 { - let child_roots = storage.1.iter().map(|(sk, child_content)| { + let child_roots = storage.children.iter().map(|(sk, child_content)| { let state_root = <<::Header as HeaderT>::Hashing as HashT>::trie_root( - child_content.0.clone().into_iter().collect(), + child_content.data.clone().into_iter().collect(), ); (sk.clone(), state_root.encode()) }); // add child roots to storage - storage.0.extend(child_roots); + storage.top.extend(child_roots); let state_root = <<::Header as HeaderT>::Hashing as HashT>::trie_root( - storage.0.clone().into_iter().collect() + storage.top.clone().into_iter().collect() ); let block: crate::Block = sc_client::genesis::construct_genesis_block(state_root); let genesis_hash = block.header.hash(); - storage.0.extend(additional_storage_with_genesis(&block)); + storage.top.extend(additional_storage_with_genesis(&block)); genesis_hash } diff --git a/test/utils/runtime/src/system.rs b/test/utils/runtime/src/system.rs index a0c1a2769ec09..cf26b2e08d522 100644 --- a/test/utils/runtime/src/system.rs +++ b/test/utils/runtime/src/system.rs @@ -361,16 +361,16 @@ mod tests { ]; TestExternalities::new_with_code( WASM_BINARY, - ( - map![ + primitives::storage::Storage { + top: map![ twox_128(b"latest").to_vec() => vec![69u8; 32], twox_128(b"sys:auth").to_vec() => authorities.encode(), blake2_256(&AccountKeyring::Alice.to_raw_public().to_keyed_vec(b"balance:")).to_vec() => { vec![111u8, 0, 0, 0, 0, 0, 0, 0] } ], - map![], - ) + children: map![], + }, ) } From d9241871194a3e24ec538119bc2d311de612ff27 Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 5 Dec 2019 15:47:35 +0100 Subject: [PATCH 26/27] remove StorageTuple alias. --- primitives/state-machine/src/backend.rs | 8 +++----- primitives/state-machine/src/basic.rs | 4 ++-- primitives/state-machine/src/testing.rs | 13 +++++++------ 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/primitives/state-machine/src/backend.rs b/primitives/state-machine/src/backend.rs index daf7e62fc6d27..0186fa67e048f 100644 --- a/primitives/state-machine/src/backend.rs +++ b/primitives/state-machine/src/backend.rs @@ -26,9 +26,7 @@ use trie::{ trie_types::{TrieDBMut, Layout}, }; use codec::{Encode, Codec}; -use primitives::storage::{ChildInfo, OwnedChildInfo}; - -pub(crate) type StorageTuple = primitives::storage::Storage; +use primitives::storage::{ChildInfo, OwnedChildInfo, Storage}; /// A state backend is used to read state data and can have changes committed /// to it. @@ -378,8 +376,8 @@ impl From, OwnedChildInfo)>, HashMap, } } -impl From for InMemory { - fn from(inners: StorageTuple) -> Self { +impl From for InMemory { + fn from(inners: Storage) -> Self { let mut inner: HashMap, OwnedChildInfo)>, HashMap, Vec>> = inners.children.into_iter().map(|(k, c)| (Some((k, c.child_info)), c.data)).collect(); inner.insert(None, inners.top); diff --git a/primitives/state-machine/src/basic.rs b/primitives/state-machine/src/basic.rs index a38bb16f63a3d..41e81043c4de4 100644 --- a/primitives/state-machine/src/basic.rs +++ b/primitives/state-machine/src/basic.rs @@ -17,7 +17,7 @@ //! Basic implementation for Externalities. use std::{collections::HashMap, any::{TypeId, Any}, iter::FromIterator}; -use crate::backend::{Backend, InMemory, StorageTuple}; +use crate::backend::{Backend, InMemory}; use hash_db::Hasher; use trie::{TrieConfiguration, default_child_trie_root}; use trie::trie_types::Layout; @@ -49,7 +49,7 @@ impl BasicExternalities { } /// Consume self and returns inner storages - pub fn into_storages(self) -> StorageTuple { + pub fn into_storages(self) -> Storage { self.inner } diff --git a/primitives/state-machine/src/testing.rs b/primitives/state-machine/src/testing.rs index 79499e4db1bb0..a37cd8caef0fb 100644 --- a/primitives/state-machine/src/testing.rs +++ b/primitives/state-machine/src/testing.rs @@ -19,7 +19,7 @@ use std::any::{Any, TypeId}; use hash_db::Hasher; use crate::{ - backend::{InMemory, Backend, StorageTuple}, OverlayedChanges, + backend::{InMemory, Backend}, OverlayedChanges, changes_trie::{ InMemoryStorage as ChangesTrieInMemoryStorage, BlockNumber as ChangesTrieBlockNumber, @@ -28,7 +28,8 @@ use crate::{ }; use primitives::{ storage::{ - well_known_keys::{CHANGES_TRIE_CONFIG, CODE, HEAP_PAGES, is_child_storage_key} + well_known_keys::{CHANGES_TRIE_CONFIG, CODE, HEAP_PAGES, is_child_storage_key}, + Storage, }, hash::H256, Blake2Hasher, }; @@ -55,12 +56,12 @@ impl, N: ChangesTrieBlockNumber> TestExternalities { } /// Create a new instance of `TestExternalities` with storage. - pub fn new(storage: StorageTuple) -> Self { + pub fn new(storage: Storage) -> Self { Self::new_with_code(&[], storage) } /// Create a new instance of `TestExternalities` with code and storage. - pub fn new_with_code(code: &[u8], mut storage: StorageTuple) -> Self { + pub fn new_with_code(code: &[u8], mut storage: Storage) -> Self { let mut overlay = OverlayedChanges::default(); assert!(storage.top.keys().all(|key| !is_child_storage_key(key))); @@ -146,8 +147,8 @@ impl, N: ChangesTrieBlockNumber> Default for TestExternaliti fn default() -> Self { Self::new(Default::default()) } } -impl, N: ChangesTrieBlockNumber> From for TestExternalities { - fn from(storage: StorageTuple) -> Self { +impl, N: ChangesTrieBlockNumber> From for TestExternalities { + fn from(storage: Storage) -> Self { Self::new(storage) } } From ead585619adeac0e1de5561de8dd7935fb97565a Mon Sep 17 00:00:00 2001 From: cheme Date: Fri, 6 Dec 2019 09:38:51 +0100 Subject: [PATCH 27/27] Fix doc tests, and remove StorageOverlay and ChildStorageOverlay aliases. --- client/src/lib.rs | 4 ++-- primitives/core/storage/src/lib.rs | 8 -------- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/client/src/lib.rs b/client/src/lib.rs index 00a0d8c26900c..364733f1a4155 100644 --- a/client/src/lib.rs +++ b/client/src/lib.rs @@ -48,7 +48,7 @@ //! use std::sync::Arc; //! use sc_client::{Client, in_mem::Backend, LocalCallExecutor}; //! use primitives::Blake2Hasher; -//! use sp_runtime::{StorageOverlay, ChildrenStorageOverlay}; +//! use sp_runtime::Storage; //! use executor::{NativeExecutor, WasmExecutionMethod}; //! //! // In this example, we're using the `Block` and `RuntimeApi` types from the @@ -65,7 +65,7 @@ //! NativeExecutor::::new(WasmExecutionMethod::Interpreted, None), //! ), //! // This parameter provides the storage for the chain genesis. -//! <(StorageOverlay, ChildrenStorageOverlay)>::default(), +//! ::default(), //! Default::default(), //! Default::default(), //! ); diff --git a/primitives/core/storage/src/lib.rs b/primitives/core/storage/src/lib.rs index 5622364e784aa..6b2bd978ef7ff 100644 --- a/primitives/core/storage/src/lib.rs +++ b/primitives/core/storage/src/lib.rs @@ -67,14 +67,6 @@ pub struct Storage { pub children: std::collections::HashMap, StorageChild>, } -/// A set of key value pairs for storage. -#[cfg(feature = "std")] -pub type StorageOverlay = StorageMap; - -/// A set of key value pairs for children storage; -#[cfg(feature = "std")] -pub type ChildrenStorageOverlay = std::collections::HashMap, StorageChild>; - /// Storage change set #[derive(RuntimeDebug)] #[cfg_attr(feature = "std", derive(Serialize, Deserialize, PartialEq, Eq))]