Skip to content
This repository was archived by the owner on Nov 15, 2023. It is now read-only.
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
40 commits
Select commit Hold shift + click to select a range
0e48dc9
Fix overlay prefix removal result
gavofyork May 18, 2022
ae9b285
Second part of the overlay prefix removal fix.
gavofyork May 18, 2022
7f80f12
Report only items deleted from storage in clear_prefix
gavofyork May 20, 2022
3e580e5
Fix kill_prefix
gavofyork May 20, 2022
4a5ff62
Formatting
gavofyork May 20, 2022
8d56c13
Remove unused code
gavofyork May 20, 2022
91362cd
Fixes
gavofyork May 20, 2022
4767538
Fixes
gavofyork May 20, 2022
08b898c
Introduce clear_prefix host function v3
gavofyork May 20, 2022
aee7993
Formatting
gavofyork May 20, 2022
a1982d1
Use v2 for now
gavofyork May 20, 2022
ed26985
Fixes
gavofyork May 20, 2022
72f8fdd
Formatting
gavofyork May 20, 2022
d97aa89
Docs
gavofyork May 20, 2022
f4bbf8d
Child prefix removal should also hide v3 for now
gavofyork May 20, 2022
e82b1ca
Fixes
gavofyork May 20, 2022
73294b0
Fixes
gavofyork May 20, 2022
8a8efb1
Formatting
gavofyork May 20, 2022
a626d54
Fixes
gavofyork May 21, 2022
6867534
apply_to_keys_whle takes start_at
gavofyork May 21, 2022
94e885e
apply_to_keys_whle takes start_at
gavofyork May 21, 2022
2566a4d
apply_to_keys_whle takes start_at
gavofyork May 23, 2022
a71f9a0
Cursor API; force limits
gavofyork May 23, 2022
f435107
Use unsafe deprecated functions
gavofyork May 23, 2022
bc181a0
Formatting
gavofyork May 23, 2022
9317f6a
Merge remote-tracking branch 'origin/master' into gav-clear-prefix-v2
gavofyork May 23, 2022
0e51dbe
Fixes
gavofyork May 23, 2022
ee53604
Grumbles
gavofyork May 25, 2022
1fb2ef5
Fixes
gavofyork May 25, 2022
0338fce
Docs
gavofyork May 25, 2022
73d1fc6
Some nitpicks :see_no_evil:
bkchr May 25, 2022
ef7f475
Update primitives/externalities/src/lib.rs
gavofyork May 25, 2022
7db73b1
Formatting
gavofyork May 25, 2022
6aeb3a3
Fixes
KiChjang May 25, 2022
4692d06
cargo fmt
KiChjang May 25, 2022
7e8a8c2
Fixes
KiChjang May 25, 2022
c22b924
Fixes
gavofyork May 25, 2022
edf7ce0
Update primitives/io/src/lib.rs
gavofyork May 25, 2022
33224f2
Formatting
gavofyork May 25, 2022
a5f349b
Fixes
gavofyork May 26, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Formatting
  • Loading branch information
gavofyork committed May 23, 2022
commit bc181a0a829ebdc9f38d08210d84b4a3ce2e5983
3 changes: 1 addition & 2 deletions frame/contracts/src/storage.rs
Original file line number Diff line number Diff line change
Expand Up @@ -267,8 +267,7 @@ where
// Cannot panic due to loop condition
let trie = &mut queue[0];
#[allow(deprecated)]
let outcome =
child::kill_storage(&child_trie_info(&trie.trie_id), Some(remaining_key_budget));
let outcome = child::kill_storage(&child_trie_info(&trie.trie_id), Some(remaining_key_budget));
let keys_removed = match outcome {
// This happens when our budget wasn't large enough to remove all keys.
KillStorageResult::SomeRemaining(c) => c,
Expand Down
16 changes: 9 additions & 7 deletions frame/support/src/storage/generator/double_map.rs
Original file line number Diff line number Diff line change
Expand Up @@ -206,22 +206,24 @@ where
where
KArg1: EncodeLike<K1>,
{
unhashed::clear_prefix(
Self::storage_double_map_final_key1(k1).as_ref(),
maybe_limit,
None,
).into()
unhashed::clear_prefix(Self::storage_double_map_final_key1(k1).as_ref(), maybe_limit, None)
.into()
}

fn clear_prefix<KArg1>(k1: KArg1, limit: u32, maybe_cursor: Option<&[u8]>) -> sp_io::ClearPrefixResult
fn clear_prefix<KArg1>(
k1: KArg1,
limit: u32,
maybe_cursor: Option<&[u8]>,
) -> sp_io::ClearPrefixResult
where
KArg1: EncodeLike<K1>,
{
unhashed::clear_prefix(
Self::storage_double_map_final_key1(k1).as_ref(),
Some(limit),
maybe_cursor,
).into()
)
.into()
}

fn iter_prefix_values<KArg1>(k1: KArg1) -> storage::PrefixIterator<V>
Expand Down
12 changes: 10 additions & 2 deletions frame/support/src/storage/generator/nmap.rs
Original file line number Diff line number Diff line change
Expand Up @@ -186,11 +186,19 @@ where
unhashed::clear_prefix(&Self::storage_n_map_partial_key(partial_key), limit, None).into()
}

fn clear_prefix<KP>(partial_key: KP, limit: u32, maybe_cursor: Option<&[u8]>) -> sp_io::ClearPrefixResult
fn clear_prefix<KP>(
partial_key: KP,
limit: u32,
maybe_cursor: Option<&[u8]>,
) -> sp_io::ClearPrefixResult
where
K: HasKeyPrefix<KP>,
{
unhashed::clear_prefix(&Self::storage_n_map_partial_key(partial_key), Some(limit), maybe_cursor)
unhashed::clear_prefix(
&Self::storage_n_map_partial_key(partial_key),
Some(limit),
maybe_cursor,
)
}

fn iter_prefix_values<KP>(partial_key: KP) -> PrefixIterator<V>
Expand Down
2 changes: 1 addition & 1 deletion frame/support/src/storage/migration.rs
Original file line number Diff line number Diff line change
Expand Up @@ -285,7 +285,7 @@ pub fn clear_storage_prefix(
item: &[u8],
hash: &[u8],
maybe_limit: Option<u32>,
maybe_cursor: Option<&[u8]>
maybe_cursor: Option<&[u8]>,
) -> sp_io::ClearPrefixResult {
let mut key = vec![0u8; 32 + hash.len()];
let storage_prefix = storage_prefix(module, item);
Expand Down
12 changes: 10 additions & 2 deletions frame/support/src/storage/types/double_map.rs
Original file line number Diff line number Diff line change
Expand Up @@ -261,11 +261,19 @@ where
/// passed once (in the initial call) for any given storage map and `first_key`. Subsequent
/// calls operating on the same map/`first_key` should always pass `Some`, and this should be
/// equal to the previous call result's `maybe_cursor` field.
pub fn clear_prefix<KArg1>(first_key: KArg1, limit: u32, maybe_cursor: Option<&[u8]>) -> sp_io::ClearPrefixResult
pub fn clear_prefix<KArg1>(
first_key: KArg1,
limit: u32,
maybe_cursor: Option<&[u8]>,
) -> sp_io::ClearPrefixResult
where
KArg1: ?Sized + EncodeLike<Key1>,
{
<Self as crate::storage::StorageDoubleMap<Key1, Key2, Value>>::clear_prefix(first_key, limit, maybe_cursor)
<Self as crate::storage::StorageDoubleMap<Key1, Key2, Value>>::clear_prefix(
first_key,
limit,
maybe_cursor,
)
}

/// Iterate over values that share the first key.
Expand Down
12 changes: 10 additions & 2 deletions frame/support/src/storage/types/nmap.rs
Original file line number Diff line number Diff line change
Expand Up @@ -217,11 +217,19 @@ where
/// passed once (in the initial call) for any given storage map and `partial_key`. Subsequent
/// calls operating on the same map/`partial_key` should always pass `Some`, and this should be
/// equal to the previous call result's `maybe_cursor` field.
pub fn clear_prefix<KP>(partial_key: KP, limit: u32, maybe_cursor: Option<&[u8]>) -> sp_io::ClearPrefixResult
pub fn clear_prefix<KP>(
partial_key: KP,
limit: u32,
maybe_cursor: Option<&[u8]>,
) -> sp_io::ClearPrefixResult
where
Key: HasKeyPrefix<KP>,
{
<Self as crate::storage::StorageNMap<Key, Value>>::clear_prefix(partial_key, limit, maybe_cursor)
<Self as crate::storage::StorageNMap<Key, Value>>::clear_prefix(
partial_key,
limit,
maybe_cursor,
)
}

/// Iterate over values that share the first key.
Expand Down
17 changes: 12 additions & 5 deletions primitives/io/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -200,7 +200,8 @@ pub trait Storage {
/// backend.
#[version(2)]
fn clear_prefix(&mut self, prefix: &[u8], limit: Option<u32>) -> KillStorageResult {
let (maybe_cursor, num_removed, ..) = Externalities::clear_prefix(*self, prefix, limit, None);
let (maybe_cursor, num_removed, ..) =
Externalities::clear_prefix(*self, prefix, limit, None);
match maybe_cursor {
None => KillStorageResult::AllRemoved(num_removed),
Some(_) => KillStorageResult::SomeRemaining(num_removed),
Expand Down Expand Up @@ -243,7 +244,7 @@ pub trait Storage {
&mut self,
maybe_prefix: &[u8],
maybe_limit: Option<u32>,
maybe_cursor: Option<Vec<u8>>, //< TODO Make work or just Option<Vec<u8>>?
maybe_cursor: Option<Vec<u8>>, //< TODO Make work or just Option<Vec<u8>>?
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do you mean here that this should work with Option<&[u8]>?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, unfortunately it seems non-trivial right now as it's not natively supported by the hist interface codec.

) -> ClearPrefixResult {
let (maybe_cursor, db, total, loops) = Externalities::clear_prefix(
*self,
Expand Down Expand Up @@ -425,12 +426,17 @@ pub trait DefaultChildStorage {
///
/// See `Storage` module `clear_prefix` documentation for `limit` usage.
#[version(4, register_only)]
fn storage_kill(&mut self, storage_key: &[u8], maybe_limit: Option<u32>, maybe_cursor: Option<Vec<u8>>) -> ClearPrefixResult {
fn storage_kill(
&mut self,
storage_key: &[u8],
maybe_limit: Option<u32>,
maybe_cursor: Option<Vec<u8>>,
) -> ClearPrefixResult {
let child_info = ChildInfo::new_default(storage_key);
let (maybe_cursor, db, total, loops) = self.kill_child_storage(
&child_info,
maybe_limit,
maybe_cursor.as_ref().map(|x| &x[..])
maybe_cursor.as_ref().map(|x| &x[..]),
);
ClearPrefixResult { maybe_cursor, db, total, loops }
}
Expand Down Expand Up @@ -462,7 +468,8 @@ pub trait DefaultChildStorage {
limit: Option<u32>,
) -> KillStorageResult {
let child_info = ChildInfo::new_default(storage_key);
let (maybe_cursor, num_removed, ..) = self.clear_child_prefix(&child_info, prefix, limit, None);
let (maybe_cursor, num_removed, ..) =
self.clear_child_prefix(&child_info, prefix, limit, None);
match maybe_cursor {
None => KillStorageResult::AllRemoved(num_removed),
Some(..) => KillStorageResult::SomeRemaining(num_removed),
Expand Down
51 changes: 28 additions & 23 deletions primitives/state-machine/src/ext.rs
Original file line number Diff line number Diff line change
Expand Up @@ -449,7 +449,7 @@ where
self.mark_dirty();
let overlay_count = self.overlay.clear_child_storage(child_info);
let (next_cursor, backend_count, iterations) =
self.limit_remove_from_backend(Some(child_info), None, maybe_limit, maybe_cursor);
self.limit_remove_from_backend(Some(child_info), None, maybe_limit, maybe_cursor);
(next_cursor, backend_count, overlay_count + backend_count, iterations)
}

Expand Down Expand Up @@ -500,8 +500,12 @@ where

self.mark_dirty();
let overlay_count = self.overlay.clear_child_prefix(child_info, prefix);
let (next_cursor, backend_count, iterations) =
self.limit_remove_from_backend(Some(child_info), Some(prefix), maybe_limit, maybe_cursor);
let (next_cursor, backend_count, iterations) = self.limit_remove_from_backend(
Some(child_info),
Some(prefix),
maybe_limit,
maybe_cursor,
);
(next_cursor, backend_count, backend_count + overlay_count, iterations)
}

Expand Down Expand Up @@ -753,27 +757,28 @@ where
let mut delete_count: u32 = 0;
let mut loop_count: u32 = 0;
let mut maybe_next_key = None;
self.backend.apply_to_keys_while(maybe_child, maybe_prefix, maybe_cursor, |key| {
if maybe_limit.map_or(false, |limit| loop_count == limit) {
maybe_next_key = Some(key.to_vec());
return false
}
let overlay = match maybe_child {
Some(child_info) => self.overlay.child_storage(child_info, key),
None => self.overlay.storage(key),
};
if !matches!(overlay, Some(None)) {
// already pending deletion from the backend - no need to delete it again.
if let Some(child_info) = maybe_child {
self.overlay.set_child_storage(child_info, key.to_vec(), None);
} else {
self.overlay.set_storage(key.to_vec(), None);
self.backend
.apply_to_keys_while(maybe_child, maybe_prefix, maybe_cursor, |key| {
if maybe_limit.map_or(false, |limit| loop_count == limit) {
maybe_next_key = Some(key.to_vec());
return false
}
delete_count = delete_count.saturating_add(1);
}
loop_count = loop_count.saturating_add(1);
true
});
let overlay = match maybe_child {
Some(child_info) => self.overlay.child_storage(child_info, key),
None => self.overlay.storage(key),
};
if !matches!(overlay, Some(None)) {
// already pending deletion from the backend - no need to delete it again.
if let Some(child_info) = maybe_child {
self.overlay.set_child_storage(child_info, key.to_vec(), None);
} else {
self.overlay.set_storage(key.to_vec(), None);
}
delete_count = delete_count.saturating_add(1);
}
loop_count = loop_count.saturating_add(1);
true
});
(maybe_next_key, delete_count, loop_count)
}
}
Expand Down
2 changes: 1 addition & 1 deletion primitives/state-machine/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1348,6 +1348,7 @@ mod execution {
mod tests {
use super::{ext::Ext, *};
use crate::execution::CallResult;
use assert_matches::assert_matches;
use codec::{Decode, Encode};
use sp_core::{
map,
Expand All @@ -1357,7 +1358,6 @@ mod tests {
NativeOrEncoded, NeverNativeValue,
};
use sp_runtime::traits::BlakeTwo256;
use assert_matches::assert_matches;
use std::{
collections::{BTreeMap, HashMap},
panic::UnwindSafe,
Expand Down
5 changes: 4 additions & 1 deletion primitives/state-machine/src/testing.rs
Original file line number Diff line number Diff line change
Expand Up @@ -381,7 +381,10 @@ mod tests {
{
let mut ext = ext.ext();

assert!(ext.kill_child_storage(&child_info, Some(2), None).0.is_some(), "Should not delete all keys");
assert!(
ext.kill_child_storage(&child_info, Some(2), None).0.is_some(),
"Should not delete all keys"
);

assert!(ext.child_storage(&child_info, &b"doe"[..]).is_none());
assert!(ext.child_storage(&child_info, &b"dog"[..]).is_none());
Expand Down
3 changes: 2 additions & 1 deletion primitives/state-machine/src/trie_backend_essence.rs
Original file line number Diff line number Diff line change
Expand Up @@ -342,7 +342,8 @@ where
let trie = TrieDB::<H>::new(db, root)?;
let prefix = maybe_prefix.unwrap_or(&[]);
let iter = match maybe_start_at {
Some(start_at) => TrieDBKeyIterator::new_prefixed_then_seek(&trie, prefix, start_at),
Some(start_at) =>
TrieDBKeyIterator::new_prefixed_then_seek(&trie, prefix, start_at),
None => TrieDBKeyIterator::new_prefixed(&trie, prefix),
}?;

Expand Down