changs_trie: continue

demo/executor/src/lib.rs

@@ -47,14 +47,14 @@ mod tests {
 	use codec::{Encode, Decode, Joiner};
 	use keyring::Keyring;
 	use runtime_support::{Hashable, StorageValue, StorageMap};
-	use state_machine::{CodeExecutor, TestExternalities};
+	use state_machine::{CodeExecutor, Externalities, TestExternalities};
 	use primitives::{twox_128, KeccakHasher, RlpCodec};
 	use demo_primitives::{Hash, BlockNumber, AccountId};
 	use runtime_primitives::traits::Header as HeaderT;
 	use runtime_primitives::{ApplyOutcome, ApplyError, ApplyResult, MaybeUnsigned};
 	use {staking, system, consensus};
 	use demo_runtime::{Header, Block, UncheckedExtrinsic, Extrinsic, Call, Concrete, Staking,
-		BuildStorage, GenesisConfig, SessionConfig, StakingConfig, BareExtrinsic};
+		BuildStorage, GenesisConfig, SessionConfig, StakingConfig, BareExtrinsic, SystemConfig};
 	use ed25519::{Public, Pair};
 
 	const BLOATY_CODE: &[u8] = include_bytes!("../../runtime/wasm/target/wasm32-unknown-unknown/release/demo_runtime.wasm");
@@ -183,12 +183,16 @@ mod tests {
 		});
 	}
 
-	fn new_test_ext() -> TestExternalities<KeccakHasher, RlpCodec> {
+	fn new_test_ext(support_changes_trie: bool) -> TestExternalities<KeccakHasher, RlpCodec> {
 		use keyring::Keyring::*;
 		let three = [3u8; 32].into();
 		TestExternalities::new(GenesisConfig {
 			consensus: Some(Default::default()),
-			system: Some(Default::default()),
+			system: Some(SystemConfig {
+				changes_trie_enabled: support_changes_trie,
+				changes_trie_digest_interval: Some(2),
+				changes_trie_digest_levels: Some(2),
+			}),
 			session: Some(SessionConfig {
 				session_length: 2,
 				validators: vec![One.to_raw_public().into(), Two.to_raw_public().into(), three],
@@ -245,12 +249,20 @@ mod tests {
 		(Block { header, extrinsics }.encode(), hash.into())
 	}
 
-	fn block1() -> (Vec<u8>, Hash) {
+	fn block1(support_changes_trie: bool) -> (Vec<u8>, Hash) {
 		construct_block(
 			1,
 			[69u8; 32].into(),
-			hex!("c563199c60df7d914262b1775b284870f3a5da2f24b56d2c6288b37c815a6cd9").into(),
-			Some(hex!("0000000000000000000000000000000000000000000000000000000000000000").into()),
+			if support_changes_trie {
+				hex!("a4ffa8ace317d34db3e19318a4a63c60f1cc225c54fb9fb5ac490e1d6f668006").into()
+			} else {
+				hex!("7dce77b1be773a182f8717f037acd5a20ae5944df2c1088b7b349f52bccae642").into()
+			},
+			if support_changes_trie {
+				Some(hex!("2d2248964aaeb0781010933e38e3fcbdf02abf110c232169f8899297acc743df").into())
+			} else {
+				None
+			},
 			vec![BareExtrinsic {
 				signed: alice(),
 				index: 0,
@@ -262,9 +274,9 @@ mod tests {
 	fn block2() -> (Vec<u8>, Hash) {
 		construct_block(
 			2,
-			block1().1,
-			hex!("2c5d1fa1e0304212ddf8f63545dada6e85c8ea80655461f3d0cc83c4a2c9d3ad").into(),
-			Some(hex!("0000000000000000000000000000000000000000000000000000000000000000").into()),
+			block1(false).1,
+			hex!("54de316e8eb537f422607d5af675b438fc5fa358cc2a98a4c40bd4d3b42e126f").into(),
+			None,
 			vec![
 				BareExtrinsic {
 					signed: bob(),
@@ -284,8 +296,8 @@ mod tests {
 		construct_block(
 			1,
 			[69u8; 32].into(),
-			hex!("06d026c0d687ec583660a6052de6f89acdb24ea964d06be3831c837c3c426966").into(),
-			Some(hex!("0000000000000000000000000000000000000000000000000000000000000000").into()),
+			hex!("e4d976e01b40fe6c03782d9f0aa0d150bde2f73d40a1508555f88dcd9c13925e").into(),
+			None,
 			vec![BareExtrinsic {
 				signed: alice(),
 				index: 0,
@@ -296,9 +308,9 @@ mod tests {
 
 	#[test]
 	fn full_native_block_import_works() {
-		let mut t = new_test_ext();
+		let mut t = new_test_ext(false);
 
-		executor().call(&mut t, COMPACT_CODE, "execute_block", &block1().0, true).0.unwrap();
+		executor().call(&mut t, COMPACT_CODE, "execute_block", &block1(false).0, true).0.unwrap();
 
 		runtime_io::with_externalities(&mut t, || {
 			assert_eq!(Staking::voting_balance(&alice()), 41);
@@ -315,10 +327,8 @@ mod tests {
 
 	#[test]
 	fn full_wasm_block_import_works() {
-		::env_logger::init().ok();
-
-		let mut t = new_test_ext();
-		WasmExecutor::new(8).call(&mut t, COMPACT_CODE, "execute_block", &block1().0).unwrap();
+		let mut t = new_test_ext(false);
+		WasmExecutor::new(8).call(&mut t, COMPACT_CODE, "execute_block", &block1(false).0).unwrap();
 
 		runtime_io::with_externalities(&mut t, || {
 			assert_eq!(Staking::voting_balance(&alice()), 41);
@@ -335,23 +345,23 @@ mod tests {
 
 	#[test]
 	fn wasm_big_block_import_fails() {
-		let mut t = new_test_ext();
+		let mut t = new_test_ext(false);
 
 		let r = WasmExecutor::new(8).call(&mut t, COMPACT_CODE, "execute_block", &block1big().0);
 		assert!(!r.is_ok());
 	}
 
 	#[test]
 	fn native_big_block_import_succeeds() {
-		let mut t = new_test_ext();
+		let mut t = new_test_ext(false);
 
 		let r = Executor::with_heap_pages(8).call(&mut t, COMPACT_CODE, "execute_block", &block1big().0, true).0;
 		assert!(r.is_ok());
 	}
 
 	#[test]
 	fn native_big_block_import_fails_on_fallback() {
-		let mut t = new_test_ext();
+		let mut t = new_test_ext(false);
 
 		let r = Executor::with_heap_pages(8).call(&mut t, COMPACT_CODE, "execute_block", &block1big().0, false).0;
 		assert!(!r.is_ok());
@@ -401,4 +411,20 @@ mod tests {
 			assert_eq!(Staking::voting_balance(&bob()), 69);
 		});
 	}
+
+	#[test]
+	fn full_native_block_import_works_with_changes_trie() {
+		let mut t = new_test_ext(true);
+		executor().call(&mut t, COMPACT_CODE, "execute_block", &block1(true).0, true).0.unwrap();
+
+		assert!(t.storage_changes_root().is_some());
+	}
+
+	#[test]
+	fn full_wasm_block_import_works_with_changes_trie() {
+		let mut t = new_test_ext(true);
+		WasmExecutor::new(8).call(&mut t, COMPACT_CODE, "execute_block", &block1(true).0).unwrap();
+
+		assert!(t.storage_changes_root().is_some());
+	}
 }

substrate/client/Cargo.toml

@@ -27,6 +27,7 @@ substrate-telemetry = { path = "../telemetry" }
 hashdb = { git = "https://github.com/paritytech/parity-common" }
 patricia-trie = { git = "https://github.com/paritytech/parity-common" }
 rlp = { git = "https://github.com/paritytech/parity-common" }
+memorydb = { git = "https://github.com/paritytech/parity-common" }
 
 [dev-dependencies]
 substrate-test-client = { path = "../test-client" }

substrate/client/db/src/lib.rs

@@ -68,7 +68,7 @@ pub use state_db::PruningMode;
 const FINALIZATION_WINDOW: u64 = 32;
 
 /// DB-backed patricia trie state, transaction type is an overlay of changes to commit.
-pub type DbState = state_machine::TrieBackend<KeccakHasher, RlpCodec>;
+pub type DbState = state_machine::TrieBackend<Arc<state_machine::Storage<KeccakHasher>>, KeccakHasher, RlpCodec>;
 
 /// Database settings.
 pub struct DatabaseSettings {
@@ -289,7 +289,24 @@ impl<Block: BlockT> state_db::HashDb for StorageDb<Block> {
 	}
 }
 
-struct DbChangesTrieStorage<Block: BlockT> {
+struct GenesisStorage(pub H256);
+
+impl GenesisStorage {
+	pub fn new() -> Self {
+		let mut root = H256::default();
+		let mut mdb = MemoryDB::<KeccakHasher>::new();
+		state_machine::TrieDBMut::<KeccakHasher, RlpCodec>::new(&mut mdb, &mut root);
+		GenesisStorage(root)
+	}
+}
+
+impl state_machine::Storage<KeccakHasher> for GenesisStorage {
+	fn get(&self, _key: &H256) -> Result<Option<DBValue>, String> {
+		Ok(None)
+	}
+}
+
+pub struct DbChangesTrieStorage<Block: BlockT> {
 	db: Arc<KeyValueDB>,
 	_phantom: ::std::marker::PhantomData<Block>,
 }
@@ -313,7 +330,7 @@ impl<Block: BlockT> state_machine::ChangesTrieStorage<KeccakHasher> for DbChange
 /// Otherwise, trie nodes are kept only from the most recent block.
 pub struct Backend<Block: BlockT> {
 	storage: Arc<StorageDb<Block>>,
-	tries_change_storage: Arc<DbChangesTrieStorage<Block>>,
+	tries_change_storage: DbChangesTrieStorage<Block>,
 	blockchain: BlockchainDb<Block>,
 	finalization_window: u64,
 }
@@ -350,7 +367,7 @@ impl<Block: BlockT> Backend<Block> {
 
 		Ok(Backend {
 			storage: Arc::new(storage_db),
-			tries_change_storage: Arc::new(tries_change_storage),
+			tries_change_storage: tries_change_storage,
 			blockchain,
 			finalization_window,
 		})
@@ -372,10 +389,17 @@ fn apply_state_commit(transaction: &mut DBTransaction, commit: state_db::CommitS
 	}
 }
 
+fn apply_changes_trie_commit(transaction: &mut DBTransaction, mut commit: MemoryDB<KeccakHasher>) {
+	for (key, (val, _)) in commit.drain() {
+		transaction.put(columns::CHANGES_TRIE, &key[..], &val);
+	}
+}
+
 impl<Block> client::backend::Backend<Block, KeccakHasher, RlpCodec> for Backend<Block> where Block: BlockT {
 	type BlockImportOperation = BlockImportOperation<Block, KeccakHasher>;
 	type Blockchain = BlockchainDb<Block>;
 	type State = DbState;
+	type ChangesTrieStorage = DbChangesTrieStorage<Block>;
 
 	fn begin_operation(&self, block: BlockId<Block>) -> Result<Self::BlockImportOperation, client::error::Error> {
 		let state = self.state_at(block)?;
@@ -416,6 +440,7 @@ impl<Block> client::backend::Backend<Block, KeccakHasher, RlpCodec> for Backend<
 			let number_u64 = number.as_().into();
 			let commit = self.storage.state_db.insert_block(&hash, number_u64, &pending_block.header.parent_hash(), changeset);
 			apply_state_commit(&mut transaction, commit);
+			apply_changes_trie_commit(&mut transaction, operation.changes_trie_updates);
 
 			//finalize an older block
 			if number_u64 > self.finalization_window {
@@ -443,6 +468,10 @@ impl<Block> client::backend::Backend<Block, KeccakHasher, RlpCodec> for Backend<
 		Ok(())
 	}
 
+	fn changes_trie_storage(&self) -> Option<&Self::ChangesTrieStorage> {
+		Some(&self.tries_change_storage)
+	}
+
 	fn revert(&self, n: NumberFor<Block>) -> Result<NumberFor<Block>, client::error::Error> {
 		use client::blockchain::HeaderBackend;
 		let mut best = self.blockchain.info()?.best_number;
@@ -482,14 +511,17 @@ impl<Block> client::backend::Backend<Block, KeccakHasher, RlpCodec> for Backend<
 
 		// special case for genesis initialization
 		match block {
-			BlockId::Hash(h) if h == Default::default() =>
-				return Ok(DbState::with_storage_for_genesis(self.storage.clone(), Some(self.tries_change_storage.clone()))),
+			BlockId::Hash(h) if h == Default::default() => {
+				let genesis_storage = GenesisStorage::new();
+				let root = genesis_storage.0.clone();
+				return Ok(DbState::new(Arc::new(genesis_storage), root));
+			},
 			_ => {}
 		}
 
 		self.blockchain.header(block).and_then(|maybe_hdr| maybe_hdr.map(|hdr| {
 			let root: H256 = H256::from_slice(hdr.state_root().as_ref());
-			DbState::with_storage(self.storage.clone(), root, Some(self.tries_change_storage.clone()))
+			DbState::new(self.storage.clone(), root)
 		}).ok_or_else(|| client::error::ErrorKind::UnknownBlock(format!("{:?}", block)).into()))
 	}
 }

substrate/client/src/backend.rs

@@ -22,8 +22,10 @@ use runtime_primitives::bft::Justification;
 use runtime_primitives::generic::BlockId;
 use runtime_primitives::traits::{Block as BlockT, NumberFor};
 use state_machine::backend::Backend as StateBackend;
+use state_machine::ChangesTrieStorage as StateChangesTrieStorage;
 use patricia_trie::NodeCodec;
 use hashdb::Hasher;
+use memorydb::MemoryDB;
 
 /// Block insertion operation. Keeps hold if the inserted block state and data.
 pub trait BlockImportOperation<Block, H, C>
@@ -54,7 +56,7 @@ where
 	/// Inject storage data into the database replacing any existing data.
 	fn reset_storage<I: Iterator<Item=(Vec<u8>, Vec<u8>)>>(&mut self, iter: I) -> error::Result<()>;
 	/// Inject changes trie data into the database.
-	fn update_changes_trie(&mut self, update: <Self::State as StateBackend<H, C>>::ChangesTrieTransaction) -> error::Result<()>;
+	fn update_changes_trie(&mut self, update: MemoryDB<H>) -> error::Result<()>;
 }
 
 /// Client backend. Manages the data layer.
@@ -77,6 +79,8 @@ where
 	type Blockchain: ::blockchain::Backend<Block>;
 	/// Associated state backend type.
 	type State: StateBackend<H, C>;
+	/// Changes trie storage/
+	type ChangesTrieStorage: StateChangesTrieStorage<H>;
 
 	/// Begin a new block insertion transaction with given parent block id.
 	/// When constructing the genesis, this is called with all-zero hash.
@@ -85,6 +89,8 @@ where
 	fn commit_operation(&self, transaction: Self::BlockImportOperation) -> error::Result<()>;
 	/// Returns reference to blockchain backend.
 	fn blockchain(&self) -> &Self::Blockchain;
+	/// Returns reference to changes trie storage.
+	fn changes_trie_storage(&self) -> Option<&Self::ChangesTrieStorage>;
 	/// Returns state backend with post-state of given block.
 	fn state_at(&self, block: BlockId<Block>) -> error::Result<Self::State>;
 	/// Attempts to revert the chain by `n` blocks. Returns the number of blocks that were

substrate/client/src/call_executor.rs

@@ -26,6 +26,7 @@ use patricia_trie::NodeCodec;
 use primitives::{KeccakHasher, RlpCodec};
 use hashdb::Hasher;
 use rlp::Encodable;
+use memorydb::MemoryDB;
 
 use backend;
 use error;
@@ -76,7 +77,7 @@ where
 		method: &str,
 		call_data: &[u8],
 		manager: ExecutionManager<F>
-	) -> Result<(Vec<u8>, S::StorageTransaction, Option<S::ChangesTrieTransaction>), error::Error>;
+	) -> Result<(Vec<u8>, S::StorageTransaction, Option<MemoryDB<H>>), error::Error>;
 
 	/// Execute a call to a contract on top of given state, gathering execution proof.
 	///
@@ -142,7 +143,8 @@ where
 	fn runtime_version(&self, id: &BlockId<Block>) -> error::Result<RuntimeVersion> {
 		let mut overlay = OverlayedChanges::default();
 		let state = self.backend.state_at(*id)?;
-		let mut externalities = Ext::new(&mut overlay, &state);
+		let changes_trie_storage = self.backend.changes_trie_storage();
+		let mut externalities = Ext::new(&mut overlay, &state, changes_trie_storage);
 		let code = externalities.storage(b":code").ok_or(error::ErrorKind::VersionInvalid)?
 			.to_vec();
 
@@ -159,9 +161,10 @@ where
 		method: &str,
 		call_data: &[u8],
 		manager: ExecutionManager<F>,
-	) -> error::Result<(Vec<u8>, S::StorageTransaction, Option<S::ChangesTrieTransaction>)> {
+	) -> error::Result<(Vec<u8>, S::StorageTransaction, Option<MemoryDB<KeccakHasher>>)> {
 		state_machine::execute_using_consensus_failure_handler(
 			state,
+			self.backend.changes_trie_storage(),
 			changes,
 			&self.executor,
 			method,

substrate/client/src/client.rs

@@ -23,7 +23,7 @@ use primitives::AuthorityId;
 use runtime_primitives::{bft::Justification, generic::{BlockId, SignedBlock, Block as RuntimeBlock}};
 use runtime_primitives::traits::{Block as BlockT, Header as HeaderT, Zero, One, As, NumberFor};
 use runtime_primitives::BuildStorage;
-use primitives::{KeccakHasher, RlpCodec};
+use primitives::{KeccakHasher, RlpCodec, H256};
 use primitives::storage::{StorageKey, StorageData};
 use codec::Decode;
 use state_machine::{
@@ -169,6 +169,7 @@ pub fn new_in_mem<E, Block, S>(
 		E: CodeExecutor<KeccakHasher> + RuntimeInfo,
 		S: BuildStorage,
 		Block: BlockT,
+		H256: From<Block::Hash>,
 {
 	let backend = Arc::new(in_mem::Backend::new());
 	let executor = LocalCallExecutor::new(backend.clone(), executor);
@@ -281,7 +282,7 @@ impl<B, E, Block> Client<B, E, Block> where
 		overlay: &mut OverlayedChanges,
 		f: F
 	) -> error::Result<T> {
-		Ok(runtime_io::with_externalities(&mut Ext::new(overlay, &self.state_at(id)?), f))
+		Ok(runtime_io::with_externalities(&mut Ext::new(overlay, &self.state_at(id)?, self.backend.changes_trie_storage()), f))
 	}
 
 	/// Create a new block, built on the head of the chain.

substrate/client/src/error.rs

@@ -94,6 +94,12 @@ error_chain! {
 			display("This method is not currently available when running in light client mode"),
 		}
 
+		/// Unable to generate execution proof.
+		UnableToGenerateExecutionProof {
+			description("unable to generate execution proof"),
+			display("The node has no capabilities to generate execution proof"),
+		}
+
 		/// Invalid remote execution proof.
 		InvalidExecutionProof {
 			description("invalid execution proof"),