use changes tries in query_storage RPC

Author: svyatonik

core/client/db/src/lib.rs

@@ -430,6 +430,21 @@ impl<Block: BlockT> DbChangesTrieStorage<Block> {
 	}
 }
 
+impl<Block: BlockT> client::backend::PrunableStateChangesTrieStorage<Blake2Hasher> for DbChangesTrieStorage<Block> {
+	fn oldest_changes_trie_block(
+		&self,
+		config: &ChangesTrieConfiguration,
+		best_finalized_block: u64
+	) -> u64 {
+		let min_blocks_to_keep = match self.min_blocks_to_keep {
+			Some(min_blocks_to_keep) => min_blocks_to_keep,
+			None => return 1,
+		};
+
+		state_machine::oldest_non_pruned_changes_trie(config, min_blocks_to_keep, best_finalized_block)
+	}
+}
+
 impl<Block: BlockT> state_machine::ChangesTrieRootsStorage<Blake2Hasher> for DbChangesTrieStorage<Block> {
 	fn root(&self, anchor: &state_machine::ChangesTrieAnchorBlockId<H256>, block: u64) -> Result<Option<H256>, String> {
 		// check API requirement

core/client/src/backend.rs

@@ -17,7 +17,7 @@
 //! Substrate Client data backend
 
 use error;
-use primitives::AuthorityId;
+use primitives::{AuthorityId, ChangesTrieConfiguration};
 use runtime_primitives::{generic::BlockId, Justification, StorageMap, ChildrenStorageMap};
 use runtime_primitives::traits::{Block as BlockT, NumberFor};
 use state_machine::backend::Backend as StateBackend;
@@ -98,7 +98,7 @@ where
 	/// Associated state backend type.
 	type State: StateBackend<H>;
 	/// Changes trie storage.
-	type ChangesTrieStorage: StateChangesTrieStorage<H>;
+	type ChangesTrieStorage: PrunableStateChangesTrieStorage<H>;
 
 	/// Begin a new block insertion transaction with given parent block id.
 	/// When constructing the genesis, this is called with all-zero hash.
@@ -123,6 +123,12 @@ where
 	fn get_aux(&self, key: &[u8]) -> error::Result<Option<Vec<u8>>>;
 }
 
+/// Changes trie storage that supports pruning.
+pub trait PrunableStateChangesTrieStorage<H: Hasher>: StateChangesTrieStorage<H> {
+	/// Get number block of oldest, non-pruned changes trie.
+	fn oldest_changes_trie_block(&self, config: &ChangesTrieConfiguration, best_finalized: u64) -> u64;
+}
+
 /// Mark for all Backend implementations, that are making use of state data, stored locally.
 pub trait LocalBackend<Block, H>: Backend<Block, H>
 where

core/client/src/client.rs

@@ -36,11 +36,12 @@ use primitives::storage::well_known_keys;
 use codec::{Encode, Decode};
 use state_machine::{
 	Backend as StateBackend, CodeExecutor,
-	ExecutionStrategy, ExecutionManager, ChangesTrieAnchorBlockId,
-	prove_read, key_changes, key_changes_proof, OverlayedChanges
+	ExecutionStrategy, ExecutionManager, prove_read,
+	key_changes, key_changes_proof,
+	ChangesTrieAnchorBlockId, OverlayedChanges
 };
 
-use backend::{self, BlockImportOperation};
+use backend::{self, BlockImportOperation, PrunableStateChangesTrieStorage};
 use blockchain::{self, Info as ChainInfo, Backend as ChainBackend, HeaderBackend as ChainHeaderBackend};
 use call_executor::{CallExecutor, LocalCallExecutor};
 use executor::{RuntimeVersion, RuntimeInfo};
@@ -336,33 +337,56 @@ impl<B, E, Block> Client<B, E, Block> where
 		Ok((header, proof))
 	}
 
+	/// Get longest range within [first; last] that is possible to use in `key_changes`
+	/// and `key_changes_proof` calls.
+	/// Range could be shortened from the beginning if some changes tries have been pruned.
+	/// Returns None if changes trues are not supported.
+	pub fn max_key_changes_range(
+		&self,
+		first: NumberFor<Block>,
+		last: BlockId<Block>,
+	) -> error::Result<Option<(NumberFor<Block>, BlockId<Block>)>> {
+		let (config, storage) = match self.require_changes_trie().ok() {
+			Some((config, storage)) => (config, storage),
+			None => return Ok(None),
+		};
+
+		let first = first.as_();
+		let last_num = self.require_block_number_from_id(&last)?.as_();
+		if first > last_num {
+			return Err(error::ErrorKind::ChangesTrieAccessFailed("Invalid changes trie range".into()).into());
+		}
+
+		let finalized_number = self.backend.blockchain().info()?.finalized_number;
+		let oldest = storage.oldest_changes_trie_block(config, finalized_number.as_());
+		let first = As::sa(::std::cmp::max(first, oldest));
+		Ok(Some((first, last)))
+	}
+
 	/// Get pairs of (block, extrinsic) where key has been changed at given blocks range.
 	/// Works only for runtimes that are supporting changes tries.
+	/// Changes are returned in descending order.
 	pub fn key_changes(
 		&self,
-		first: Block::Hash,
-		last: Block::Hash,
-		key: &[u8]
+		first: NumberFor<Block>,
+		last: BlockId<Block>,
+		key: &StorageKey,
 	) -> error::Result<Vec<(NumberFor<Block>, u32)>> {
-		let config = self.changes_trie_config.as_ref();
-		let storage = self.backend.changes_trie_storage();
-		let (config, storage) = match (config, storage) {
-			(Some(config), Some(storage)) => (config, storage),
-			_ => return Err(error::ErrorKind::ChangesTriesNotSupported.into()),
-		};
-
-		key_changes::<_, Blake2Hasher>(
+		let (config, storage) = self.require_changes_trie()?;
+		let last_number = self.require_block_number_from_id(&last)?.as_();
+		let last_hash = self.require_block_hash_from_id(&last)?;
+		key_changes(
 			config,
-			storage,
-			self.require_block_number_from_id(&BlockId::Hash(first))?.as_(),
+			&*storage,
+			first.as_(),
 			&ChangesTrieAnchorBlockId {
-				hash: convert_hash(&last),
-				number: self.require_block_number_from_id(&BlockId::Hash(last))?.as_(),
+				hash: convert_hash(&last_hash),
+				number: last_number,
 			},
 			self.backend.blockchain().info()?.best_number.as_(),
-			key)
+			&key.0)
+		.and_then(|r| r.map(|r| r.map(|(block, tx)| (As::sa(block), tx))).collect::<Result<_, _>>())
 		.map_err(|err| error::ErrorKind::ChangesTrieAccessFailed(err).into())
-		.map(|r| r.into_iter().map(|(b, e)| (As::sa(b), e)).collect())
 	}
 
 	/// Get proof for computation of (block, extrinsic) pairs where key has been changed at given blocks range.
@@ -374,33 +398,37 @@ impl<B, E, Block> Client<B, E, Block> where
 		first: Block::Hash,
 		last: Block::Hash,
 		max: Block::Hash,
-		key: &[u8]
+		key: &StorageKey,
 	) -> error::Result<(NumberFor<Block>, Vec<Vec<u8>>)> {
-		let config = self.changes_trie_config.as_ref();
-		let storage = self.backend.changes_trie_storage();
-		let (config, storage) = match (config, storage) {
-			(Some(config), Some(storage)) => (config, storage),
-			_ => return Err(error::ErrorKind::ChangesTriesNotSupported.into()),
-		};
-
+		let (config, storage) = self.require_changes_trie()?;
 		let max_number = ::std::cmp::min(
 			self.backend.blockchain().info()?.best_number,
 			self.require_block_number_from_id(&BlockId::Hash(max))?,
 		);
-		key_changes_proof::<_, Blake2Hasher>(
+		key_changes_proof(
 			config,
-			storage,
+			&*storage,
 			self.require_block_number_from_id(&BlockId::Hash(first))?.as_(),
 			&ChangesTrieAnchorBlockId {
 				hash: convert_hash(&last),
 				number: self.require_block_number_from_id(&BlockId::Hash(last))?.as_(),
 			},
 			max_number.as_(),
-			key)
+			&key.0)
 		.map_err(|err| error::ErrorKind::ChangesTrieAccessFailed(err).into())
 		.map(|proof| (max_number, proof))
 	}
 
+	/// Returns changes trie configuration and storage or an error if it is not supported.
+	fn require_changes_trie(&self) -> error::Result<(&ChangesTrieConfiguration, &B::ChangesTrieStorage)> {
+		let config = self.changes_trie_config.as_ref();
+		let storage = self.backend.changes_trie_storage();
+		match (config, storage) {
+			(Some(config), Some(storage)) => Ok((config, storage)),
+			_ => Err(error::ErrorKind::ChangesTriesNotSupported.into()),
+		}
+	}
+
 	/// Create a new block, built on the head of the chain.
 	pub fn new_block(&self) -> error::Result<block_builder::BlockBuilder<B, E, Block, Blake2Hasher>>
 	where E: Clone
@@ -750,7 +778,13 @@ impl<B, E, Block> Client<B, E, Block> where
 		}
 	}
 
-	/// Convert an arbitrary block ID into a block hash.
+	/// Convert an arbitrary block ID into a block number, returning error if the block is unknown.
+	fn require_block_hash_from_id(&self, id: &BlockId<Block>) -> error::Result<Block::Hash> {
+		self.block_hash_from_id(id)
+			.and_then(|n| n.ok_or_else(|| error::ErrorKind::UnknownBlock(format!("{}", id)).into()))
+	}
+
+	/// Convert an arbitrary block ID into a block number.
 	pub fn block_number_from_id(&self, id: &BlockId<Block>) -> error::Result<Option<NumberFor<Block>>> {
 		match *id {
 			BlockId::Hash(_) => Ok(self.header(id)?.map(|h| h.number().clone())),
@@ -1586,9 +1620,7 @@ pub(crate) mod tests {
 		let (client, _, test_cases) = prepare_client_with_key_changes();
 
 		for (index, (begin, end, key, expected_result)) in test_cases.into_iter().enumerate() {
-			let begin = client.block_hash(begin).unwrap().unwrap();
-			let end = client.block_hash(end).unwrap().unwrap();
-			let actual_result = client.key_changes(begin, end, &key).unwrap();
+			let actual_result = client.key_changes(begin, BlockId::Number(end), &StorageKey(key)).unwrap();
 			match actual_result == expected_result {
 				true => (),
 				false => panic!(format!("Failed test {}: actual = {:?}, expected = {:?}",

core/client/src/in_mem.rs

@@ -22,14 +22,14 @@ use parking_lot::RwLock;
 use error;
 use backend::{self, NewBlockState};
 use light;
-use primitives::{AuthorityId, storage::well_known_keys};
+use primitives::{AuthorityId, ChangesTrieConfiguration, storage::well_known_keys};
 use runtime_primitives::generic::BlockId;
 use runtime_primitives::traits::{Block as BlockT, Header as HeaderT, Zero,
 	NumberFor, As, Digest, DigestItem};
 use runtime_primitives::{Justification, StorageMap, ChildrenStorageMap};
 use blockchain::{self, BlockStatus, HeaderBackend};
 use state_machine::backend::{Backend as StateBackend, InMemory, Consolidate};
-use state_machine::InMemoryChangesTrieStorage;
+use state_machine::{self, InMemoryChangesTrieStorage, ChangesTrieAnchorBlockId};
 use hash_db::Hasher;
 use heapsize::HeapSizeOf;
 use leaves::LeafSet;
@@ -436,7 +436,7 @@ where
 	H::Out: HeapSizeOf + Ord,
 {
 	states: RwLock<HashMap<Block::Hash, InMemory<H>>>,
-	changes_trie_storage: InMemoryChangesTrieStorage<H>,
+	changes_trie_storage: ChangesTrieStorage<H>,
 	blockchain: Blockchain<Block>,
 	aux: RwLock<HashMap<Vec<u8>, Vec<u8>>>,
 }
@@ -451,7 +451,7 @@ where
 	pub fn new() -> Backend<Block, H> {
 		Backend {
 			states: RwLock::new(HashMap::new()),
-			changes_trie_storage: InMemoryChangesTrieStorage::new(),
+			changes_trie_storage: ChangesTrieStorage(InMemoryChangesTrieStorage::new()),
 			blockchain: Blockchain::new(),
 			aux: RwLock::new(HashMap::new()),
 		}
@@ -467,7 +467,7 @@ where
 	type BlockImportOperation = BlockImportOperation<Block, H>;
 	type Blockchain = Blockchain<Block>;
 	type State = InMemory<H>;
-	type ChangesTrieStorage = InMemoryChangesTrieStorage<H>;
+	type ChangesTrieStorage = ChangesTrieStorage<H>;
 
 	fn begin_operation(&self, block: BlockId<Block>) -> error::Result<Self::BlockImportOperation> {
 		let state = match block {
@@ -498,7 +498,7 @@ where
 			if let Some(changes_trie_root) = changes_trie_root {
 				if let Some(changes_trie_update) = operation.changes_trie_update {
 					let changes_trie_root: H::Out = changes_trie_root.into();
-					self.changes_trie_storage.insert(header.number().as_(), changes_trie_root, changes_trie_update);
+					self.changes_trie_storage.0.insert(header.number().as_(), changes_trie_root, changes_trie_update);
 				}
 			}
 
@@ -574,6 +574,27 @@ impl<Block: BlockT> blockchain::Cache<Block> for Cache<Block> {
 	}
 }
 
+/// Prunable in-memory changes trie storage.
+pub struct ChangesTrieStorage<H: Hasher>(InMemoryChangesTrieStorage<H>) where H::Out: HeapSizeOf;
+
+impl<H: Hasher> backend::PrunableStateChangesTrieStorage<H> for ChangesTrieStorage<H> where H::Out: HeapSizeOf {
+	fn oldest_changes_trie_block(&self, _config: &ChangesTrieConfiguration, _best_finalized: u64) -> u64 {
+		0
+	}
+}
+
+impl<H: Hasher> state_machine::ChangesTrieRootsStorage<H> for ChangesTrieStorage<H> where H::Out: HeapSizeOf {
+	fn root(&self, anchor: &ChangesTrieAnchorBlockId<H::Out>, block: u64) -> Result<Option<H::Out>, String> {
+		self.0.root(anchor, block)
+	}
+}
+
+impl<H: Hasher> state_machine::ChangesTrieStorage<H> for ChangesTrieStorage<H> where H::Out: HeapSizeOf {
+	fn get(&self, key: &H::Out) -> Result<Option<state_machine::DBValue>, String> {
+		self.0.get(key)
+	}
+}
+
 /// Insert authorities entry into in-memory blockchain cache. Extracted as a separate function to use it in tests.
 pub fn cache_authorities_at<Block: BlockT>(
 	blockchain: &Blockchain<Block>,

core/client/src/light/backend.rs

@@ -23,7 +23,7 @@ use parking_lot::RwLock;
 
 use primitives::AuthorityId;
 use runtime_primitives::{generic::BlockId, Justification, StorageMap, ChildrenStorageMap};
-use state_machine::{Backend as StateBackend, InMemoryChangesTrieStorage, TrieBackend};
+use state_machine::{Backend as StateBackend, TrieBackend};
 use runtime_primitives::traits::{Block as BlockT, NumberFor};
 
 use in_mem;
@@ -79,7 +79,7 @@ impl<S, F, Block, H> ClientBackend<Block, H> for Backend<S, F> where
 	type BlockImportOperation = ImportOperation<Block, S, F>;
 	type Blockchain = Blockchain<S, F>;
 	type State = OnDemandState<Block, S, F>;
-	type ChangesTrieStorage = InMemoryChangesTrieStorage<H>;
+	type ChangesTrieStorage = in_mem::ChangesTrieStorage<H>;
 
 	fn begin_operation(&self, _block: BlockId<Block>) -> ClientResult<Self::BlockImportOperation> {
 		Ok(ImportOperation {

core/client/src/light/fetcher.rs

@@ -275,7 +275,7 @@ pub mod tests {
 	use light::fetcher::{Fetcher, FetchChecker, LightDataChecker,
 		RemoteCallRequest, RemoteHeaderRequest};
 	use primitives::{Blake2Hasher};
-	use primitives::storage::well_known_keys;
+	use primitives::storage::{StorageKey, well_known_keys};
 	use runtime_primitives::generic::BlockId;
 	use state_machine::Backend;
 	use super::*;
@@ -419,6 +419,7 @@ pub mod tests {
 			let end_hash = remote_client.block_hash(end).unwrap().unwrap();
 
 			// 'fetch' changes proof from remote node
+			let key = StorageKey(key);
 			let (remote_max, remote_proof) = remote_client.key_changes_proof(
 				begin_hash, end_hash, max_hash, &key
 			).unwrap();
@@ -431,7 +432,7 @@ pub mod tests {
 				last_block: (end, end_hash),
 				max_block: (max, max_hash),
 				tries_roots: local_roots_range,
-				key: key,
+				key: key.0,
 				retry_count: None,
 			};
 			let local_result = local_checker.check_changes_proof(
@@ -460,6 +461,7 @@ pub mod tests {
 		let end_hash = remote_client.block_hash(end).unwrap().unwrap();
 
 		// 'fetch' changes proof from remote node
+		let key = StorageKey(key);
 		let (remote_max, mut remote_proof) = remote_client.key_changes_proof(
 			begin_hash, end_hash, max_hash, &key).unwrap();
 		let local_roots_range = local_roots.clone()[(begin - 1) as usize..].to_vec();
@@ -469,7 +471,7 @@ pub mod tests {
 			last_block: (end, end_hash),
 			max_block: (max, max_hash),
 			tries_roots: local_roots_range.clone(),
-			key: key,
+			key: key.0,
 			retry_count: None,
 		};
 

core/network/src/chain.rs

@@ -23,7 +23,7 @@ use runtime_primitives::traits::{Block as BlockT, Header as HeaderT, NumberFor};
 use runtime_primitives::generic::{BlockId};
 use consensus::{ImportBlock, ImportResult};
 use runtime_primitives::Justification;
-use primitives::{H256, Blake2Hasher, AuthorityId};
+use primitives::{H256, Blake2Hasher, AuthorityId, storage::StorageKey};
 
 /// Local client abstraction for the network.
 pub trait Client<Block: BlockT>: Send + Sync {
@@ -64,7 +64,7 @@ pub trait Client<Block: BlockT>: Send + Sync {
 		first: Block::Hash,
 		last: Block::Hash,
 		max: Block::Hash,
-		key: &[u8]
+		key: &StorageKey,
 	) -> Result<(NumberFor<Block>, Vec<Vec<u8>>), Error>;
 }
 
@@ -121,7 +121,7 @@ impl<B, E, Block> Client<Block> for SubstrateClient<B, E, Block> where
 		first: Block::Hash,
 		last: Block::Hash,
 		max: Block::Hash,
-		key: &[u8]
+		key: &StorageKey
 	) -> Result<(NumberFor<Block>, Vec<Vec<u8>>), Error> {
 		(self as &SubstrateClient<B, E, Block>).key_changes_proof(first, last, max, key)
 	}