Skip to content
This repository was archived by the owner on Nov 15, 2023. It is now read-only.
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Start using trie db
  • Loading branch information
bkchr committed Apr 3, 2019
commit 2c6bf07e7d5bd8a1452324cd7de9126c00f77fb6
1,793 changes: 918 additions & 875 deletions Cargo.lock

Large diffs are not rendered by default.

11 changes: 11 additions & 0 deletions runtime/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,14 @@ edition = "2018"
codec = { package = "parity-codec", version = "3.1", default-features = false, features = [ "derive" ] }
rstd = { package = "sr-std", git = "https://github.com/paritytech/substrate", default-features = false, branch = "polkadot-master" }
runtime-primitives = { package = "sr-primitives", git = "https://github.com/paritytech/substrate", default-features = false, branch = "polkadot-master" }
primitives = { package = "substrate-primitives", git = "https://github.com/paritytech/substrate", default-features = false, branch = "polkadot-master" }
rio = { package = "sr-io", git = "https://github.com/paritytech/substrate", default-features = false, branch = "polkadot-master" }
executive = { package = "srml-executive", git = "https://github.com/paritytech/substrate", default-features = false, branch = "polkadot-master" }
substrate-trie = { git = "https://github.com/paritytech/substrate", default-features = false, branch = "polkadot-master" }
memory-db = { version = "0.12.2", default-features = false }
hash-db = { version = "0.12.2", default-features = false }
trie-db = { version = "0.12.2", default-features = false }
hashbrown = "0.1"

[dev-dependencies]
keyring = { package = "substrate-keyring", git = "https://github.com/paritytech/substrate", branch = "polkadot-master" }
Expand All @@ -24,5 +30,10 @@ std = [
"rstd/std",
"rio/std",
"runtime-primitives/std",
"primitives/std",
"executive/std",
"memory-db/std",
"hash-db/std",
"trie-db/std",
]
wasm = ["hashbrown/nightly"]
18 changes: 11 additions & 7 deletions runtime/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@

#![cfg_attr(not(feature = "std"), no_std)]

use rstd::{vec::Vec, collections::btree_map::BTreeMap};
use rstd::vec::Vec;
use codec::{Encode, Decode};
use runtime_primitives::traits::Block as BlockT;

Expand All @@ -27,31 +27,35 @@ pub use rstd::slice;
#[macro_use]
pub mod validate_block;

type WitnessData = BTreeMap<Vec<u8>, Vec<u8>>;
/// The type of the witness data.
type WitnessData = Vec<Vec<u8>>;

/// The parachain block that is created on a collator and validated by a validator.
#[derive(Encode, Decode)]
struct ParachainBlock<B: BlockT> {
struct ParachainBlockData<B: BlockT> {
extrinsics: Vec<<B as BlockT>::Extrinsic>,
/// The data that is required to emulate the storage accesses executed by all extrinsics.
witness_data: WitnessData,
}

impl<B: BlockT> ParachainBlock<B> {
impl<B: BlockT> ParachainBlockData<B> {
#[cfg(test)]
fn new(extrinsics: Vec<<B as BlockT>::Extrinsic>, witness_data: WitnessData) -> Self {
fn new(
extrinsics: Vec<<B as BlockT>::Extrinsic>,
witness_data: WitnessData
) -> Self {
Self {
extrinsics,
witness_data,
}
}
}

impl<B: BlockT> Default for ParachainBlock<B> {
impl<B: BlockT> Default for ParachainBlockData<B> {
fn default() -> Self {
Self {
extrinsics: Vec::default(),
witness_data: BTreeMap::default(),
witness_data: WitnessData::default(),
}
}
}
205 changes: 205 additions & 0 deletions runtime/src/validate_block/implementation.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,205 @@
// Copyright 2019 Parity Technologies (UK) Ltd.
// This file is part of Substrate.

// Substrate is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.

// Substrate is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.

// You should have received a copy of the GNU General Public License
// along with Cumulus. If not, see <http://www.gnu.org/licenses/>.

//! The actual implementation of the validate block functionality.

use crate::WitnessData;
use runtime_primitives::traits::{
Block as BlockT, Header as HeaderT, One, Hash as HashT
};
use executive::ExecuteBlock;

use substrate_trie::{MemoryDB, read_trie_value};

use rstd::{slice, ptr, cmp, vec::Vec, boxed::Box, mem};

use hash_db::HashDB;

/// Extract the hashing algorithm type from the given block type.
type HashingOf<B> = <<B as BlockT>::Header as HeaderT>::Hashing;
/// Extract the hash type from the given block type.
type HashOf<B> = <<B as BlockT>::Header as HeaderT>::Hash;

/// Abstract the storage into a trait without `Block` generic.
trait StorageT {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

a bit weird to call it StorageT. why not just Storage?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Because I have a struct that is called Storage in the same module 😅

/// Retrieve the value for the given key.
fn get(&self, key: &[u8]) -> Option<Vec<u8>>;

/// Insert the given key and value.
fn insert(&mut self, key: &[u8], value: &[u8]);

/// Remove key and value.
fn remove(&mut self, key: &[u8]);
}

static mut STORAGE: Option<Box<StorageT>> = None;
/// The message to use as expect message while accessing the `STORAGE`.
const STORAGE_SET_EXPECT: &str =
"`STORAGE` needs to be set before calling this function.";

/// Validate a given parachain block on a validator.
#[cfg(not(feature = "std"))]
#[doc(hidden)]
pub fn validate_block<B: BlockT, E: ExecuteBlock<B>>(
mut block: &[u8],
mut prev_head: &[u8]
) {
use codec::Decode;

let block = crate::ParachainBlockData::<B>::decode(&mut block)
.expect("Could not decode parachain block.");
let parent_header = <<B as BlockT>::Header as Decode>::decode(&mut prev_head)
.expect("Could not decode parent header.");
let storage = Storage::<B>::new(
block.witness_data,
parent_header.state_root().clone()
).expect("Create storage out of witness data.");

let _guard = unsafe {
STORAGE = Some(Box::new(storage));
(
// Replace storage calls with our own implementations
rio::ext_get_allocated_storage.replace_implementation(ext_get_allocated_storage),
rio::ext_get_storage_into.replace_implementation(ext_get_storage_into),
rio::ext_set_storage.replace_implementation(ext_set_storage),
rio::ext_exists_storage.replace_implementation(ext_exists_storage),
rio::ext_clear_storage.replace_implementation(ext_clear_storage),
)
};

let block_number = *parent_header.number() + One::one();
//E::execute_extrinsics_without_checks(block_number, block.extrinsics);
}

/// The storage implementation used when validating a block.
struct Storage<B: BlockT> {
witness_data: MemoryDB<<HashingOf<B> as HashT>::Hasher>,
overlay: hashbrown::HashMap<Vec<u8>, Option<Vec<u8>>>,
storage_root: HashOf<B>,
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

B::Hash is easier

}

impl<B: BlockT> Storage<B> {
/// Initialize from the given witness data and storage root.
///
/// Returns an error if given storage root was not found in the witness data.
fn new(
data: WitnessData,
storage_root: HashOf<B>
) -> Result<Self, &'static str> {
let mut db = MemoryDB::default();
data.into_iter().for_each(|i| { db.insert(&[], &i); });
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

what does the &[] mean here?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't really know. I copied this code from substrate. I think that &[] is the prefix. Maybe @cheme can help us here?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That is the prefix for the MemoryDB, I believe here if MemoryDB is used from substrate trie crate, there is no key transformation with the prefix so it is safe to put nothing (a variant called PrefixedMemoryDB does insert the prefix before the key when inserting/querying the keyvaluedb to avoid collision in trie with unhashed value).

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

hm, maybe we need this when using a chain with child tries

Copy link
Contributor

@cheme cheme Apr 15, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

there is the key conflict with child trie, but this way of doing things cannot solve it directly (for two identical child trie there is the sametrie related prefix). I was also wondering about exposing this key transformation trait to insert a child trie specific prefix with a specialized implementation (paritytech/trie#12 last comment), but using a hashdb specific implementation would achieve the same thing.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I assume that we then need to adapt this? This is basically the functionality I use to generate the proof. I assume we would need to extend the proof with the prefix somehow?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If you use child trie I think you need paritytech/substrate#2209 (I should split it in two has I believe the api change would not be merged or at least requires some consensus but the proof side of the pr seems ok).
But otherwhise the prefix thing should only apply to avoid reference count in db, so the Recorder of trie does not use the prefix so it should not matter (that is the good thing about the way Arkadiy solves the issue).


if !db.contains(&storage_root, &[]) {
return Err("Witness data does not contain given storage root.")
}

Ok(Self {
witness_data: db,
overlay: Default::default(),
storage_root,
})
}
}

impl<B: BlockT> StorageT for Storage<B> {
fn get(&self, key: &[u8]) -> Option<Vec<u8>> {
self.overlay.get(key).cloned().or_else(|| {
read_trie_value(
&self.witness_data,
&self.storage_root, key
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

key should go on new line.

).ok()
}).unwrap_or(None)
}

fn insert(&mut self, key: &[u8], value: &[u8]) {
self.overlay.insert(key.to_vec(), Some(value.to_vec()));
}

fn remove(&mut self, key: &[u8]) {
self.overlay.insert(key.to_vec(), None);
}
}

pub unsafe fn ext_get_allocated_storage(
key_data: *const u8,
key_len: u32,
written_out: *mut u32,
) -> *mut u8 {
let key = slice::from_raw_parts(key_data, key_len as usize);
match STORAGE.as_mut().expect(STORAGE_SET_EXPECT).get(key) {
Some(mut value) => {
*written_out = value.len() as u32;
let ptr = value.as_mut_ptr();
mem::forget(ptr);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

what's the purpose of forgetting ptr (which is copy) and then using it right after?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Was incorrect and is fixed now.

ptr
},
None => {
*written_out = u32::max_value();
ptr::null_mut()
}
}
}

pub unsafe fn ext_set_storage(
key_data: *const u8,
key_len: u32,
value_data: *const u8,
value_len: u32,
) {
let key = slice::from_raw_parts(key_data, key_len as usize);
let value = slice::from_raw_parts(value_data, value_len as usize);

STORAGE.as_mut().expect(STORAGE_SET_EXPECT).insert(key, value);
}

pub unsafe fn ext_get_storage_into(
key_data: *const u8,
key_len: u32,
value_data: *mut u8,
value_len: u32,
value_offset: u32,
) -> u32 {
let key = slice::from_raw_parts(key_data, key_len as usize);
let out_value = slice::from_raw_parts_mut(value_data, value_len as usize);

match STORAGE.as_mut().expect(STORAGE_SET_EXPECT).get(key) {
Some(value) => {
let value = &value[value_offset as usize..];
let len = cmp::min(value_len as usize, value.len());
out_value[..len].copy_from_slice(&value[..len]);
len as u32
},
None => {
u32::max_value()
}
}
}

pub unsafe fn ext_exists_storage(key_data: *const u8, key_len: u32) -> u32 {
let key = slice::from_raw_parts(key_data, key_len as usize);

if STORAGE.as_mut().expect(STORAGE_SET_EXPECT).get(key).is_some() {
1
} else {
0
}
}

pub unsafe fn ext_clear_storage(key_data: *const u8, key_len: u32) {
let key = slice::from_raw_parts(key_data, key_len as usize);

STORAGE.as_mut().expect(STORAGE_SET_EXPECT).remove(key);
}
39 changes: 4 additions & 35 deletions runtime/src/validate_block/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,16 +16,11 @@

//! A module that enables a runtime to work as parachain.

#[cfg(not(feature = "std"))]
use runtime_primitives::traits::{Block as BlockT, Header as HeaderT, One};
#[cfg(not(feature = "std"))]
use executive::ExecuteBlock;

#[cfg(not(feature = "std"))]
#[doc(hidden)]
pub mod storage_functions;
#[cfg(test)]
mod tests;
#[cfg(not(feature = "std"))]
#[doc(hidden)]
pub mod implementation;

/// Register the `validate_block` function that is used by parachains to validate blocks on a validator.
///
Expand Down Expand Up @@ -65,7 +60,7 @@ macro_rules! register_validate_block_impl {
let block = $crate::slice::from_raw_parts(block, block_len as usize);
let prev_head = $crate::slice::from_raw_parts(prev_head, prev_head_len as usize);

$crate::validate_block::validate_block::<$block, $block_executor>(block, prev_head);
$crate::validate_block::implementation::validate_block::<$block, $block_executor>(block, prev_head);
}
}
};
Expand All @@ -77,30 +72,4 @@ macro_rules! register_validate_block_impl {
#[macro_export]
macro_rules! register_validate_block_impl {
($block:ty, $block_executor:ty) => {};
}

/// Validate a given parachain block on a validator.
#[cfg(not(feature = "std"))]
#[doc(hidden)]
pub fn validate_block<Block: BlockT, E: ExecuteBlock<Block>>(mut block: &[u8], mut prev_head: &[u8]) {
use codec::Decode;

let block = crate::ParachainBlock::<Block>::decode(&mut block).expect("Could not decode parachain block.");
let parent_header = <<Block as BlockT>::Header as Decode>::decode(&mut prev_head).expect("Could not decode parent header.");

let _guard = unsafe {
use storage_functions as storage;
storage::STORAGE = Some(block.witness_data);
(
// Replace storage calls with our own implementations
rio::ext_get_allocated_storage.replace_implementation(storage::ext_get_allocated_storage),
rio::ext_get_storage_into.replace_implementation(storage::ext_get_storage_into),
rio::ext_set_storage.replace_implementation(storage::ext_set_storage),
rio::ext_exists_storage.replace_implementation(storage::ext_exists_storage),
rio::ext_clear_storage.replace_implementation(storage::ext_clear_storage),
)
};

let block_number = *parent_header.number() + One::one();
E::execute_extrinsics_without_checks(block_number, block.extrinsics);
}
Loading