Skip to content
This repository was archived by the owner on Nov 15, 2023. It is now read-only.
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
42 commits
Select commit Hold shift + click to select a range
9ccf26e
In progress, runtime io must switch to future proof root +
cheme Nov 18, 2019
a0f3818
Switch interface, sr-io seems ok, rpc could use similar interface to
cheme Nov 19, 2019
41c5020
test from previous implementation.
cheme Nov 20, 2019
6eb0414
fix proving test.
cheme Nov 20, 2019
e8e72fc
Restore Keyspacedb from other branch, only apply to child trie.
cheme Nov 20, 2019
88ffbc4
Removing unneeded child_info from child root (child info are stored
cheme Nov 20, 2019
435fcb2
Switch rpc to use same format as ext: more future proof.
cheme Nov 20, 2019
b8c7e37
use root from child info for trie backend essence.
cheme Nov 20, 2019
ee43955
Breaking long lines.
cheme Nov 20, 2019
fe285ae
Merge branch 'master' into child_uuid_step1
cheme Nov 21, 2019
a8818d9
Update doc and clean pr a bit.
cheme Nov 21, 2019
9873dcb
Merge branch 'master' into child_uuid_step1
cheme Nov 23, 2019
609a847
Merge branch 'master' into child_uuid_step1
cheme Nov 25, 2019
7254f6c
Merge branch 'master' into child_uuid_step1
cheme Nov 27, 2019
cac9330
fix error type
cheme Nov 27, 2019
c3b943c
Merge branch 'master' into child_uuid_step1
cheme Nov 28, 2019
c05bc80
Restore removed doc on merge and update sr-io doc.
cheme Nov 28, 2019
cd1e343
Merge branch 'master' into child_uuid_step1
cheme Nov 29, 2019
bdff843
Switch child storage api to use directly unique id, if managed id
cheme Nov 29, 2019
9ce77bb
Clean deprecated host interface from child.
cheme Nov 29, 2019
92c7864
Removing assertion on child info (can fail depending on root
cheme Nov 29, 2019
fc91068
merging child info in the overlay when possible.
cheme Nov 29, 2019
cdabca3
child iteration by prefix using child_info.
cheme Nov 29, 2019
783f389
Merge branch 'master' into child_uuid_step1
cheme Dec 2, 2019
d4c7334
Using ChainInfo in frame support. ChainInfo gets redesign to avoid
cheme Dec 2, 2019
64f4084
Add length of root to the data of child info.
cheme Dec 2, 2019
a87bd48
comments
cheme Dec 2, 2019
68712f9
Encode compact.
cheme Dec 2, 2019
3db467e
Remove child info with root.
cheme Dec 3, 2019
a7425cc
Merge branch 'master' into child_uuid_step1
cheme Dec 3, 2019
8e2b8b9
Fix try_update condition.
cheme Dec 3, 2019
5d74151
Comment Ext child root caching.
cheme Dec 3, 2019
0ba3da5
Replace tuples by struct with field
cheme Dec 5, 2019
d924187
remove StorageTuple alias.
cheme Dec 5, 2019
43da346
Merge branch 'master' into child_uuid_step1
cheme Dec 5, 2019
ead5856
Fix doc tests, and remove StorageOverlay and ChildStorageOverlay
cheme Dec 6, 2019
9e05542
Merge branch 'master' into child_uuid_step1
cheme Dec 8, 2019
6f107d0
Merge branch 'master' into child_uuid_step1
cheme Dec 10, 2019
17dd44d
Merge branch 'master' into child_uuid_step1
cheme Dec 11, 2019
0aaeefa
Merge branch 'master' into child_uuid_step1
cheme Dec 11, 2019
489c9f9
Merge branch 'master' into child_uuid_step1
cheme Dec 11, 2019
c58b539
Merge branch 'master' into child_uuid_step1
cheme Dec 13, 2019
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Remove child info with root.
  • Loading branch information
cheme committed Dec 3, 2019
commit 3db467e38f11756bf7711cee87907ee73c7c092a
1 change: 0 additions & 1 deletion Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 0 additions & 1 deletion primitives/core/storage/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ rstd = { package = "sr-std", path = "../../sr-std", default-features = false }
serde = { version = "1.0.101", optional = true, features = ["derive"] }
impl-serde = { version = "0.2.3", optional = true }
substrate-debug-derive = { version = "2.0.0", path = "../debug-derive" }
codec = { package = "parity-scale-codec", version = "1.0.0", default-features = false, features = ["derive"] }

[features]
default = [ "std" ]
Expand Down
141 changes: 7 additions & 134 deletions primitives/core/storage/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@ use serde::{Serialize, Deserialize};
use substrate_debug_derive::RuntimeDebug;

use rstd::{vec::Vec, borrow::Cow};
use codec::{Encode, Decode, Compact};

/// Storage key.
#[derive(PartialEq, Eq, RuntimeDebug)]
Expand Down Expand Up @@ -179,19 +178,15 @@ impl<'a> ChildInfo<'a> {
/// Instantiates information for a default child trie.
pub const fn new_default(unique_id: &'a[u8]) -> Self {
ChildInfo::Default(ChildTrie {
offset: 0,
root_end: 0,
data: unique_id,
})
}

/// Instantiates a owned version of this child info.
pub fn to_owned(&self) -> OwnedChildInfo {
match self {
ChildInfo::Default(ChildTrie { data, root_end, offset })
ChildInfo::Default(ChildTrie { data })
=> OwnedChildInfo::Default(OwnedChildTrie {
offset: *offset,
root_end: *root_end,
data: data.to_vec(),
}),
}
Expand All @@ -201,22 +196,6 @@ impl<'a> ChildInfo<'a> {
pub fn resolve_child_info(child_type: u32, data: &'a[u8]) -> Option<Self> {
match child_type {
x if x == ChildType::CryptoUniqueId as u32 => Some(ChildInfo::new_default(data)),
x if x == ChildType::CryptoUniqueIdRootApi as u32 => {
let data_cursor = &mut &data[..];
// u32 is considered enough for a root size.
let number = match Compact::<u32>::decode(data_cursor) {
Ok(number) => number.0,
Err(_) => return None,
};
let offset = data.len() - data_cursor.len();
if data.len() >= number as usize {
Some(ChildInfo::Default(ChildTrie {
offset,
root_end: number as usize + offset,
data,
}))
} else { None }
},
_ => None,
}
}
Expand All @@ -226,15 +205,8 @@ impl<'a> ChildInfo<'a> {
pub fn info(&self) -> (&[u8], u32) {
match self {
ChildInfo::Default(ChildTrie {
root_end,
data,
..
}) => {
if *root_end > 0 {
return (data, ChildType::CryptoUniqueIdRootApi as u32);
}
(data, ChildType::CryptoUniqueId as u32)
},
}) => (data, ChildType::CryptoUniqueId as u32),
}
}

Expand All @@ -244,31 +216,10 @@ impl<'a> ChildInfo<'a> {
pub fn keyspace(&self) -> &[u8] {
match self {
ChildInfo::Default(ChildTrie {
root_end,
data,
..
}) => &data[*root_end..],
}) => &data[..],
}
}

/// Return the child reference to state if it is already known.
/// If it returns `None` the information will need to be fetch by
/// the caller.
/// For a child trie it is its root.
pub fn root(&self) -> Option<&[u8]> {
match self {
ChildInfo::Default(ChildTrie {
root_end,
data,
offset,
}) => if *root_end > 0 {
Some(&data[*offset..*root_end])
} else {
None
}
}
}

}

/// Type of child.
Expand All @@ -278,32 +229,13 @@ impl<'a> ChildInfo<'a> {
pub enum ChildType {
/// Default, it uses a cryptographic strong unique id as input.
CryptoUniqueId = 1,
/// Default, but with a root registerd to skip root fetch when querying.
/// Root is variable length, its length is SCALE encoded at the start.
CryptoUniqueIdRootApi = 2,
}

impl OwnedChildInfo {
/// Instantiates info for a default child trie.
pub fn new_default(mut unique_id: Vec<u8>, root: Option<Vec<u8>>) -> Self {
let (offset, root_end, encoded) = if let Some(mut root) = root {
let mut encoded = Encode::encode(&Compact(root.len() as u32));

let offset = encoded.len();
encoded.append(&mut root);
(offset, encoded.len(), Some(encoded))
} else {
(0, 0, None)
};
pub fn new_default(unique_id: Vec<u8>) -> Self {
OwnedChildInfo::Default(OwnedChildTrie {
root_end,
offset,
data: if let Some(mut encoded) = encoded {
encoded.append(&mut unique_id);
encoded
} else {
unique_id
}
data: unique_id,
})
}

Expand All @@ -318,11 +250,9 @@ impl OwnedChildInfo {
/// Get `ChildInfo` reference to this owned child info.
pub fn as_ref(&self) -> ChildInfo {
match self {
OwnedChildInfo::Default(OwnedChildTrie { data, root_end, offset })
OwnedChildInfo::Default(OwnedChildTrie { data })
=> ChildInfo::Default(ChildTrie {
data: data.as_slice(),
root_end: *root_end,
offset: *offset,
}),
}
}
Expand All @@ -335,30 +265,16 @@ impl OwnedChildInfo {
/// crypto hash).
#[derive(Clone, Copy)]
pub struct ChildTrie<'a> {
/// Encoded data offset (correspond to size of encoded size of root if any).
offset: usize,
/// If root was fetch it can be memoïzed as first part of
/// the encoded data to avoid querying it explicitly.
/// This field keep trace of the position of the end of root.
root_end: usize,

/// Data containing unique id.
/// Unique id must but unique and free of any possible key collision
/// (depending on its storage behavior).
/// Data can also contain root in first position.
data: &'a[u8],
}

/// Owned version of default child trie `ChildTrie`.
#[derive(Debug, Clone)]
#[cfg_attr(feature = "std", derive(PartialEq, Eq, Hash, PartialOrd, Ord))]
pub struct OwnedChildTrie {
/// See `ChildTrie` reference field documentation.
offset: usize,

/// See `ChildTrie` reference field documentation.
root_end: usize,

/// See `ChildTrie` reference field documentation.
data: Vec<u8>,
}
Expand All @@ -368,50 +284,7 @@ impl OwnedChildTrie {
/// are not compatible.
fn try_update(&mut self, other: ChildInfo) -> bool {
match other {
ChildInfo::Default(other) => {
if self.data[self.root_end..] != other.data[other.root_end..] {
return false;
}
if self.root_end == 0 {
self.root_end = other.root_end;
self.data = other.data.to_vec();
} else {
debug_assert!(self.data[..self.root_end] == other.data[..other.root_end]);
}
},
ChildInfo::Default(other) => self.data[..] != other.data[..],
}
true
}
}

#[test]
fn test_encode() {
let root = vec![1; 297];
let unique_id = vec![2; 16];
let owned_child = OwnedChildInfo::new_default(unique_id.clone(), Some(root.clone()));
let child = owned_child.as_ref();
let (child_info, child_type) = child.info();
let child_info = child_info.to_vec();
assert_eq!(child.keyspace(), &unique_id[..]);
assert_eq!(child.root(), Some(&root[..]));

let child = ChildInfo::resolve_child_info(child_type, &child_info[..]).unwrap();
assert_eq!(child.keyspace(), &unique_id[..]);
assert_eq!(child.root(), Some(&root[..]));

let owned_child = OwnedChildInfo::new_default(unique_id.clone(), None);
let child = owned_child.as_ref();
let (child_info, child_type) = child.info();
let child_info = child_info.to_vec();
assert_eq!(child.keyspace(), &unique_id[..]);
assert_eq!(child.root(), None);

let child = ChildInfo::resolve_child_info(child_type, &child_info[..]).unwrap();
assert_eq!(child.keyspace(), &unique_id[..]);
assert_eq!(child.root(), None);

let child = ChildInfo::new_default(&unique_id[..]);
assert_eq!(child.keyspace(), &unique_id[..]);
assert_eq!(child.root(), None);

}
2 changes: 1 addition & 1 deletion primitives/state-machine/src/backend.rs
Original file line number Diff line number Diff line change
Expand Up @@ -627,7 +627,7 @@ mod tests {
#[test]
fn in_memory_with_child_trie_only() {
let storage = InMemory::<primitives::Blake2Hasher>::default();
let child_info = OwnedChildInfo::new_default(b"unique_id_1".to_vec(), None);
let child_info = OwnedChildInfo::new_default(b"unique_id_1".to_vec());
let mut storage = storage.update(
vec![(
Some((b"1".to_vec(), child_info.clone())),
Expand Down
49 changes: 15 additions & 34 deletions primitives/state-machine/src/trie_backend_essence.rs
Original file line number Diff line number Diff line change
Expand Up @@ -85,14 +85,8 @@ impl<S: TrieBackendStorage<H>, H: Hasher> TrieBackendEssence<S, H> where H::Out:
child_info: ChildInfo,
key: &[u8],
) -> Result<Option<Vec<u8>>, String> {
let fetched_root;
let root = if let Some(root) = child_info.root() {
root
} else {
fetched_root = self.storage(storage_key)?
.unwrap_or(default_child_trie_root::<Layout<H>>(storage_key).encode());
&fetched_root
};
let root = self.storage(storage_key)?
.unwrap_or(default_child_trie_root::<Layout<H>>(storage_key).encode());

let mut read_overlay = S::Overlay::default();
let eph = Ephemeral {
Expand All @@ -102,7 +96,7 @@ impl<S: TrieBackendStorage<H>, H: Hasher> TrieBackendEssence<S, H> where H::Out:

let map_e = |e| format!("Trie lookup error: {}", e);

read_child_trie_value::<Layout<H>, _>(storage_key, child_info.keyspace(), &eph, root, key)
read_child_trie_value::<Layout<H>, _>(storage_key, child_info.keyspace(), &eph, &root, key)
.map_err(map_e)
}

Expand All @@ -113,18 +107,12 @@ impl<S: TrieBackendStorage<H>, H: Hasher> TrieBackendEssence<S, H> where H::Out:
child_info: ChildInfo,
f: F,
) {
let fetched_root;
let root = if let Some(root) = child_info.root() {
root
} else {
fetched_root = match self.storage(storage_key) {
Ok(v) => v.unwrap_or(default_child_trie_root::<Layout<H>>(storage_key).encode()),
Err(e) => {
debug!(target: "trie", "Error while iterating child storage: {}", e);
return;
}
};
&fetched_root
let root = match self.storage(storage_key) {
Ok(v) => v.unwrap_or(default_child_trie_root::<Layout<H>>(storage_key).encode()),
Err(e) => {
debug!(target: "trie", "Error while iterating child storage: {}", e);
return;
}
};

let mut read_overlay = S::Overlay::default();
Expand Down Expand Up @@ -152,20 +140,13 @@ impl<S: TrieBackendStorage<H>, H: Hasher> TrieBackendEssence<S, H> where H::Out:
prefix: &[u8],
mut f: F,
) {
let fetched_root;
let root_vec = if let Some(root) = child_info.root() {
root
} else {
fetched_root = match self.storage(storage_key) {
Ok(v) => v.unwrap_or(default_child_trie_root::<Layout<H>>(storage_key).encode()),
Err(e) => {
debug!(target: "trie", "Error while iterating child storage with prefix: {}", e);
return;
}
};
&fetched_root
let root_vec = match self.storage(storage_key) {
Ok(v) => v.unwrap_or(default_child_trie_root::<Layout<H>>(storage_key).encode()),
Err(e) => {
debug!(target: "trie", "Error while iterating child storage: {}", e);
return;
}
};

let mut root = H::Out::default();
root.as_mut().copy_from_slice(&root_vec);
self.keys_values_with_prefix_inner(&root, prefix, |k, _v| f(k), Some(child_info))
Expand Down