Cursor API; force limits

frame/support/src/lib.rs

@@ -1097,12 +1097,12 @@ pub mod tests {
 			DoubleMap::insert(&(key1 + 1), &key2, &4u64);
 			DoubleMap::insert(&(key1 + 1), &(key2 + 1), &4u64);
 			assert!(matches!(
-				DoubleMap::clear_prefix(&key1, None),
+				DoubleMap::clear_prefix(&key1, u32::max_value(), None),
 				// Note this is the incorrect answer (for now), since we are using v2 of
 				// `clear_prefix`.
 				// When we switch to v3, then this will become:
 				//   sp_io::ClearPrefixResult::NoneLeft { db: 0, total: 2 },
-				sp_io::ClearPrefixResult::NoneLeft { db: 0, total: 0 },
+				sp_io::ClearPrefixResult { maybe_cursor: None, db: 0, total: 0, loops: 0 },
 			));
 			assert_eq!(DoubleMap::get(&key1, &key2), 0u64);
 			assert_eq!(DoubleMap::get(&key1, &(key2 + 1)), 0u64);

frame/support/src/storage/child.rs

@@ -144,34 +144,51 @@ pub fn kill_storage(child_info: &ChildInfo, limit: Option<u32>) -> KillStorageRe
 	}
 }
 
-/// Remove all `storage_key` key/values
+/// Partially clear the child storage of each key-value pair.
 ///
-/// Deletes all keys from the overlay and up to `limit` keys from the backend if
-/// it is set to `Some`. No limit is applied when `limit` is set to `None`.
+/// # Limit
 ///
-/// The limit can be used to partially delete a child trie in case it is too large
-/// to delete in one go (block).
+/// A *limit* should always be provided through `maybe_limit`. This is one fewer than the
+/// maximum number of backend iterations which may be done by this operation and as such
+/// represents the maximum number of backend deletions which may happen. A *limit* of zero
+/// implies that no keys will be deleted, through there may be a single iteration done.
 ///
-/// # Note
+/// The limit can be used to partially delete storage items in case it is too large or costly
+/// to delete all in a single operation.
 ///
-/// Please note that keys that are residing in the overlay for that child trie when
-/// issuing this call are all deleted without counting towards the `limit`. Only keys
-/// written during the current block are part of the overlay. Deleting with a `limit`
-/// mostly makes sense with an empty overlay for that child trie.
+/// # Cursor
 ///
-/// Calling this function multiple times per block for the same `storage_key` does
-/// not make much sense because it is not cumulative when called inside the same block.
-/// Use this function to distribute the deletion of a single child trie across multiple
-/// blocks.
-pub fn clear_storage(child_info: &ChildInfo, limit: Option<u32>) -> ClearPrefixResult {
+/// A *cursor* may be passed in to this operation with `maybe_cursor`. `None` should only be
+/// passed once (in the initial call) for any attempt to clear storage. Subsequent calls
+/// operating on the same prefix should always pass `Some`, and this should be equal to the
+/// previous call result's `maybe_prefix` field.
+///
+/// Returns [`ClearPrefixResult`] to inform about the result. Once the resultant `maybe_prefix`
+/// field is `None`, then no further items remain to be deleted.
+///
+/// NOTE: After the initial call for any given child storage, it is important that no keys further
+/// keys are inserted. If so, then they may or may not be deleted by subsequent calls.
+///
+/// # Note
+///
+/// Please note that keys which are residing in the overlay for the child are deleted without
+/// counting towards the `limit`.
+pub fn clear_storage(
+	child_info: &ChildInfo,
+	maybe_limit: Option<u32>,
+	_maybe_cursor: Option<&[u8]>,
+) -> ClearPrefixResult {
+	// TODO: Once the network has upgraded to include the new host functions, this code can be
+	// enabled.
+	// sp_io::default_child_storage::storage_kill(prefix, maybe_limit, maybe_cursor)
 	let r = match child_info.child_type() {
 		ChildType::ParentKeyId =>
-			sp_io::default_child_storage::storage_kill(child_info.storage_key(), limit),
+			sp_io::default_child_storage::storage_kill(child_info.storage_key(), maybe_limit),
 	};
-	use sp_io::{ClearPrefixResult::*, KillStorageResult::*};
+	use sp_io::KillStorageResult::*;
 	match r {
-		AllRemoved(db) => NoneLeft { db, total: db },
-		SomeRemaining(db) => SomeLeft { db, total: db },
+		AllRemoved(db) => ClearPrefixResult { maybe_cursor: None, db, total: db, loops: db },
+		SomeRemaining(db) => ClearPrefixResult { maybe_cursor: None, db, total: db, loops: db },
 	}
 }
 

frame/support/src/storage/generator/double_map.rs

@@ -202,18 +202,26 @@ where
 		unhashed::kill(&Self::storage_double_map_final_key(k1, k2))
 	}
 
-	fn remove_prefix<KArg1>(k1: KArg1, limit: Option<u32>) -> sp_io::KillStorageResult
+	fn remove_prefix<KArg1>(k1: KArg1, maybe_limit: Option<u32>) -> sp_io::KillStorageResult
 	where
 		KArg1: EncodeLike<K1>,
 	{
-		Self::clear_prefix(k1, limit).into()
+		unhashed::clear_prefix(
+			Self::storage_double_map_final_key1(k1).as_ref(),
+			maybe_limit,
+			None,
+		).into()
 	}
 
-	fn clear_prefix<KArg1>(k1: KArg1, limit: Option<u32>) -> sp_io::ClearPrefixResult
+	fn clear_prefix<KArg1>(k1: KArg1, limit: u32, maybe_cursor: Option<&[u8]>) -> sp_io::ClearPrefixResult
 	where
 		KArg1: EncodeLike<K1>,
 	{
-		unhashed::clear_prefix(Self::storage_double_map_final_key1(k1).as_ref(), limit)
+		unhashed::clear_prefix(
+			Self::storage_double_map_final_key1(k1).as_ref(),
+			Some(limit),
+			maybe_cursor,
+		).into()
 	}
 
 	fn iter_prefix_values<KArg1>(k1: KArg1) -> storage::PrefixIterator<V>

frame/support/src/storage/generator/nmap.rs

@@ -183,14 +183,14 @@ where
 	where
 		K: HasKeyPrefix<KP>,
 	{
-		Self::clear_prefix(partial_key, limit).into()
+		unhashed::clear_prefix(&Self::storage_n_map_partial_key(partial_key), limit, None).into()
 	}
 
-	fn clear_prefix<KP>(partial_key: KP, limit: Option<u32>) -> sp_io::ClearPrefixResult
+	fn clear_prefix<KP>(partial_key: KP, limit: u32, maybe_cursor: Option<&[u8]>) -> sp_io::ClearPrefixResult
 	where
 		K: HasKeyPrefix<KP>,
 	{
-		unhashed::clear_prefix(&Self::storage_n_map_partial_key(partial_key), limit)
+		unhashed::clear_prefix(&Self::storage_n_map_partial_key(partial_key), Some(limit), maybe_cursor)
 	}
 
 	fn iter_prefix_values<KP>(partial_key: KP) -> PrefixIterator<V>

frame/support/src/storage/migration.rs

@@ -256,12 +256,42 @@ pub fn put_storage_value<T: Encode>(module: &[u8], item: &[u8], hash: &[u8], val
 
 /// Remove all items under a storage prefix by the `module`, the map's `item` name and the key
 /// `hash`.
+#[deprecated = "Use `clear_storage_prefix` instead"]
 pub fn remove_storage_prefix(module: &[u8], item: &[u8], hash: &[u8]) {
 	let mut key = vec![0u8; 32 + hash.len()];
 	let storage_prefix = storage_prefix(module, item);
 	key[0..32].copy_from_slice(&storage_prefix);
 	key[32..].copy_from_slice(hash);
-	frame_support::storage::unhashed::clear_prefix(&key, None);
+	let _ = frame_support::storage::unhashed::clear_prefix(&key, None, None);
+}
+
+/// Attmept to remove all values under a storage prefix by the `module`, the map's `item` name and
+/// the key `hash`.
+///
+/// All values in the client overlay will be deleted, if `maybe_limit` is `Some` then up to
+/// that number of values are deleted from the client backend, otherwise all values in the
+/// client backend are deleted.
+///
+/// ## Cursors
+///
+/// The `maybe_cursor` parameter should be `None` for the first call to initial removal.
+/// If the resultant `maybe_cursor` is `Some`, then another call is required to complete the
+/// removal operation. This value must be passed in as the subsequent call's `maybe_cursor`
+/// parameter. If the resultant `maybe_cursor` is `None`, then the operation is complete and no
+/// items remain in storage provided that no items were added between the first calls and the
+/// final call.
+pub fn clear_storage_prefix(
+	module: &[u8],
+	item: &[u8],
+	hash: &[u8],
+	maybe_limit: Option<u32>,
+	maybe_cursor: Option<&[u8]>
+) -> sp_io::ClearPrefixResult {
+	let mut key = vec![0u8; 32 + hash.len()];
+	let storage_prefix = storage_prefix(module, item);
+	key[0..32].copy_from_slice(&storage_prefix);
+	key[32..].copy_from_slice(hash);
+	frame_support::storage::unhashed::clear_prefix(&key, maybe_limit, maybe_cursor)
 }
 
 /// Take a particular item in storage by the `module`, the map's `item` name and the key `hash`.

frame/support/src/storage/mod.rs

@@ -519,19 +519,26 @@ pub trait StorageDoubleMap<K1: FullEncode, K2: FullEncode, V: FullCodec> {
 	where
 		KArg1: ?Sized + EncodeLike<K1>;
 
-	/// Remove all values under the first key `k1` in the overlay and up to `limit` in the
+	/// Remove all values under the first key `k1` in the overlay and up to `maybe_limit` in the
 	/// backend.
 	///
-	/// All values in the client overlay will be deleted, if there is some `limit` then up to
-	/// `limit` values are deleted from the client backend, if `limit` is none then all values in
-	/// the client backend are deleted.
-	///
-	/// # Note
-	///
-	/// Calling this multiple times per block with a `limit` set leads always to the same keys being
-	/// removed and the same result being returned. This happens because the keys to delete in the
-	/// overlay are not taken into account when deleting keys in the backend.
-	fn clear_prefix<KArg1>(k1: KArg1, limit: Option<u32>) -> sp_io::ClearPrefixResult
+	/// All values in the client overlay will be deleted, if `maybe_limit` is `Some` then up to
+	/// that number of values are deleted from the client backend, otherwise all values in the
+	/// client backend are deleted.
+	///
+	/// ## Cursors
+	///
+	/// The `maybe_cursor` parameter should be `None` for the first call to initial removal.
+	/// If the resultant `maybe_cursor` is `Some`, then another call is required to complete the
+	/// removal operation. This value must be passed in as the subsequent call's `maybe_cursor`
+	/// parameter. If the resultant `maybe_cursor` is `None`, then the operation is complete and no
+	/// items remain in storage provided that no items were added between the first calls and the
+	/// final call.
+	fn clear_prefix<KArg1>(
+		k1: KArg1,
+		limit: u32,
+		maybe_cursor: Option<&[u8]>,
+	) -> sp_io::ClearPrefixResult
 	where
 		KArg1: ?Sized + EncodeLike<K1>;
 
@@ -677,19 +684,34 @@ pub trait StorageNMap<K: KeyGenerator, V: FullCodec> {
 	where
 		K: HasKeyPrefix<KP>;
 
-	/// Remove all values starting with `partial_key` in the overlay and up to `limit` in the
-	/// backend.
+	/// Attempt to remove items from the map matching a `partial_key` prefix.
 	///
-	/// All values in the client overlay will be deleted, if there is some `limit` then up to
-	/// `limit` values are deleted from the client backend, if `limit` is none then all values in
-	/// the client backend are deleted.
+	/// Returns [`ClearPrefixResult`] to inform about the result. Once the resultant `maybe_cursor`
+	/// field is `None`, then no further items remain to be deleted.
 	///
-	/// # Note
+	/// NOTE: After the initial call for any given map, it is important that no further items
+	/// are inserted into the map which match the `partial key`. If so, then the map may not be
+	/// empty when the resultant `maybe_cursor` is `None`.
 	///
-	/// Calling this multiple times per block with a `limit` set leads always to the same keys being
-	/// removed and the same result being returned. This happens because the keys to delete in the
-	/// overlay are not taken into account when deleting keys in the backend.
-	fn clear_prefix<KP>(partial_key: KP, limit: Option<u32>) -> sp_io::ClearPrefixResult
+	/// # Limit
+	///
+	/// A `limit` must be provided in order to cap the maximum
+	/// amount of deletions done in a single call. This is one fewer than the
+	/// maximum number of backend iterations which may be done by this operation and as such
+	/// represents the maximum number of backend deletions which may happen. A *limit* of zero
+	/// implies that no keys will be deleted, through there may be a single iteration done.
+	///
+	/// # Cursor
+	///
+	/// A *cursor* may be passed in to this operation with `maybe_cursor`. `None` should only be
+	/// passed once (in the initial call) for any given storage map and `partial_key`. Subsequent
+	/// calls operating on the same map/`partial_key` should always pass `Some`, and this should be
+	/// equal to the previous call result's `maybe_cursor` field.
+	fn clear_prefix<KP>(
+		partial_key: KP,
+		limit: u32,
+		maybe_cursor: Option<&[u8]>,
+	) -> sp_io::ClearPrefixResult
 	where
 		K: HasKeyPrefix<KP>;
 
@@ -1145,22 +1167,34 @@ pub trait StoragePrefixedMap<Value: FullCodec> {
 	/// overlay are not taken into account when deleting keys in the backend.
 	#[deprecated = "Use `clear` instead"]
 	fn remove_all(limit: Option<u32>) -> sp_io::KillStorageResult {
-		Self::clear(limit).into()
+		unhashed::clear_prefix(&Self::final_prefix(), limit, None).into()
 	}
 
-	/// Remove all values in the overlay and up to `limit` in the backend.
+	/// Attempt to remove all items from the map.
 	///
-	/// All values in the client overlay will be deleted, if there is some `limit` then up to
-	/// `limit` values are deleted from the client backend, if `limit` is none then all values in
-	/// the client backend are deleted.
+	/// Returns [`ClearPrefixResult`] to inform about the result. Once the resultant `maybe_cursor`
+	/// field is `None`, then no further items remain to be deleted.
 	///
-	/// # Note
+	/// NOTE: After the initial call for any given map, it is important that no further items
+	/// are inserted into the map. If so, then the map may not be empty when the resultant
+	/// `maybe_cursor` is `None`.
 	///
-	/// Calling this multiple times per block with a `limit` set leads always to the same keys being
-	/// removed and the same result being returned. This happens because the keys to delete in the
-	/// overlay are not taken into account when deleting keys in the backend.
-	fn clear(limit: Option<u32>) -> sp_io::ClearPrefixResult {
-		unhashed::clear_prefix(&Self::final_prefix(), limit)
+	/// # Limit
+	///
+	/// A `limit` must always be provided through in order to cap the maximum
+	/// amount of deletions done in a single call. This is one fewer than the
+	/// maximum number of backend iterations which may be done by this operation and as such
+	/// represents the maximum number of backend deletions which may happen. A *limit* of zero
+	/// implies that no keys will be deleted, through there may be a single iteration done.
+	///
+	/// # Cursor
+	///
+	/// A *cursor* may be passed in to this operation with `maybe_cursor`. `None` should only be
+	/// passed once (in the initial call) for any given storage map. Subsequent calls
+	/// operating on the same map should always pass `Some`, and this should be equal to the
+	/// previous call result's `maybe_cursor` field.
+	fn clear(limit: u32, maybe_cursor: Option<&[u8]>) -> sp_io::ClearPrefixResult {
+		unhashed::clear_prefix(&Self::final_prefix(), Some(limit), maybe_cursor)
 	}
 
 	/// Iter over all value of the storage.
@@ -1475,7 +1509,7 @@ mod test {
 			assert_eq!(MyStorage::iter_values().collect::<Vec<_>>(), vec![1, 2, 3, 4]);
 
 			// test removal
-			MyStorage::clear(None);
+			let _ = MyStorage::clear(u32::max_value(), None);
 			assert!(MyStorage::iter_values().collect::<Vec<_>>().is_empty());
 
 			// test migration
@@ -1485,7 +1519,7 @@ mod test {
 			assert!(MyStorage::iter_values().collect::<Vec<_>>().is_empty());
 			MyStorage::translate_values(|v: u32| Some(v as u64));
 			assert_eq!(MyStorage::iter_values().collect::<Vec<_>>(), vec![1, 2]);
-			MyStorage::clear(None);
+			let _ = MyStorage::clear(u32::max_value(), None);
 
 			// test migration 2
 			unhashed::put(&[&k[..], &vec![1][..]].concat(), &1u128);
@@ -1497,7 +1531,7 @@ mod test {
 			assert_eq!(MyStorage::iter_values().collect::<Vec<_>>(), vec![1, 2, 3]);
 			MyStorage::translate_values(|v: u128| Some(v as u64));
 			assert_eq!(MyStorage::iter_values().collect::<Vec<_>>(), vec![1, 2, 3]);
-			MyStorage::clear(None);
+			let _ = MyStorage::clear(u32::max_value(), None);
 
 			// test that other values are not modified.
 			assert_eq!(unhashed::get(&key_before[..]), Some(32u64));