Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
37 commits
Select commit Hold shift + click to select a range
0af325f
Make local-cache unlimited
alexggh Feb 20, 2025
f326e8c
Remove unrelated changes
alexggh Feb 24, 2025
7799138
Create unlimited cache only in onchain context
alexggh Feb 24, 2025
a8dc8c1
Move writing back to the shared_cache from drop to a worker thread
alexggh Feb 25, 2025
c03011a
Fix build
alexggh Mar 11, 2025
6be7f6d
Add manual trie flush
alexggh Mar 12, 2025
94c7f8d
Fix on import
alexggh Mar 12, 2025
37a5a87
Merge branch 'master' into alexggh/local_cache_unlimited
alexggh Mar 12, 2025
c90d5eb
Some fixes
alexggh Mar 12, 2025
96fab66
Cosmetic changes
alexggh Mar 13, 2025
296e6f5
Merge remote-tracking branch 'origin/master' into alexggh/local_cache…
alexggh Apr 3, 2025
ad1aa3f
Remove unneded changes
alexggh Apr 3, 2025
c267986
Add metrics to the trie cache
alexggh Apr 8, 2025
596eaa2
Voluntarily give the lock in untrusted paths
alexggh Apr 8, 2025
b1ad910
Add unittest for cache
alexggh Apr 9, 2025
39c0ad4
Merge remote-tracking branch 'origin/master' into alexggh/local_cache…
alexggh Apr 9, 2025
99c758a
Update substrate/client/api/src/backend.rs
alexggh Apr 16, 2025
3e495e3
Update substrate/client/api/src/backend.rs
alexggh Apr 16, 2025
3ad7962
Update substrate/client/api/src/backend.rs
alexggh Apr 16, 2025
6c3964f
Update substrate/primitives/trie/src/cache/shared_cache.rs
alexggh Apr 16, 2025
bc17fa0
Merge remote-tracking branch 'origin/master' into alexggh/local_cache…
alexggh Apr 16, 2025
d1e73c5
Make cache trusted
alexggh Apr 17, 2025
ca8f97e
Rename local_cache to untrusted
alexggh Apr 17, 2025
5228fbf
Rename default in untrusted
alexggh Apr 17, 2025
9e2e204
Review suggestions
alexggh Apr 17, 2025
f14dbbd
Move snapshots structs in metrics
alexggh Apr 17, 2025
8245a80
Use a trusted local cache
alexggh Apr 17, 2025
d4504d0
Drop cli flag use_trusted_local_cache
alexggh Apr 17, 2025
c983020
Update basic_authorship.rs
alexggh Apr 17, 2025
0906d98
Make cargo fmt happy
alexggh Apr 17, 2025
418ab9c
Make taplo happy
alexggh Apr 17, 2025
d2661f9
Add prdoc
alexggh Apr 17, 2025
858b9dc
Merge remote-tracking branch 'origin/master' into alexggh/local_cache…
alexggh Apr 23, 2025
448b498
Merge branch 'master' into alexggh/local_cache_unlimited
alexggh Apr 25, 2025
9ca77ea
Update substrate/primitives/trie/src/cache/shared_cache.rs
alexggh Apr 28, 2025
08c74e5
Address review feedback
alexggh Apr 28, 2025
f37537a
Merge branch 'master' into alexggh/local_cache_unlimited
alexggh May 14, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Rename local_cache to untrusted
Signed-off-by: Alexandru Gheorghe <alexandru.gheorghe@parity.io>
  • Loading branch information
alexggh committed Apr 17, 2025
commit ca8f97e97334f6eba78509101f5be4ba6a4acc95
Original file line number Diff line number Diff line change
Expand Up @@ -202,7 +202,7 @@ mod tests {
let cache_for_test: SharedTrieCache<sp_core::Blake2Hasher> =
SharedTrieCache::new(CacheSize::new(1024 * 5), None);
{
let local_cache = cache_for_test.local_cache();
let local_cache = cache_for_test.local_cache_untrusted();
let mut trie_cache_for_reference = local_cache.as_trie_db_cache(root);
let mut reference_trie_recorder = reference_recorder.as_trie_recorder(root);
let reference_trie =
Expand All @@ -211,7 +211,7 @@ mod tests {
.with_cache(&mut trie_cache_for_reference)
.build();

let local_cache_for_test = reference_cache.local_cache();
let local_cache_for_test = reference_cache.local_cache_untrusted();
let mut trie_cache_for_test = local_cache_for_test.as_trie_db_cache(root);
let mut trie_recorder_under_test = recorder_for_test.as_trie_recorder(root);
let test_trie =
Expand Down
2 changes: 1 addition & 1 deletion substrate/client/db/src/bench.rs
Original file line number Diff line number Diff line change
Expand Up @@ -199,7 +199,7 @@ impl<Hasher: Hash> BenchmarkingState<Hasher> {
*self.state.borrow_mut() = Some(
DbStateBuilder::<Hasher>::new(storage_db, self.root.get())
.with_optional_recorder(self.proof_recorder.clone())
.with_cache(self.shared_trie_cache.local_cache())
.with_cache(self.shared_trie_cache.local_cache_untrusted())
.build(),
);
Ok(())
Expand Down
6 changes: 3 additions & 3 deletions substrate/client/db/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2002,7 +2002,7 @@ impl<Block: BlockT> Backend<Block> {
fn empty_state(&self) -> RecordStatsState<RefTrackingState<Block>, Block> {
let root = EmptyStorage::<Block>::new().0; // Empty trie
let db_state = DbStateBuilder::<HashingFor<Block>>::new(self.storage.clone(), root)
.with_optional_cache(self.shared_trie_cache.as_ref().map(|c| c.local_cache()))
.with_optional_cache(self.shared_trie_cache.as_ref().map(|c| c.local_cache_untrusted()))
.build();
let state = RefTrackingState::new(db_state, self.storage.clone(), None);
RecordStatsState::new(state, None, self.state_usage.clone())
Expand Down Expand Up @@ -2499,7 +2499,7 @@ impl<Block: BlockT> sc_client_api::backend::Backend<Block> for Backend<Block> {
let db_state =
DbStateBuilder::<HashingFor<Block>>::new(genesis_state.clone(), root)
.with_optional_cache(
self.shared_trie_cache.as_ref().map(|c| c.local_cache()),
self.shared_trie_cache.as_ref().map(|c| c.local_cache_untrusted()),
)
.build();

Expand Down Expand Up @@ -2528,7 +2528,7 @@ impl<Block: BlockT> sc_client_api::backend::Backend<Block> for Backend<Block> {
{
c.local_cache_trusted()
} else {
c.local_cache()
c.local_cache_untrusted()
}
}))
.build();
Expand Down
18 changes: 9 additions & 9 deletions substrate/primitives/state-machine/src/trie_backend.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1161,7 +1161,7 @@ pub mod tests {

let proving = TrieBackendBuilder::wrap(&trie)
.with_recorder(Recorder::default())
.with_optional_cache(cache.as_ref().map(|c| c.local_cache()))
.with_optional_cache(cache.as_ref().map(|c| c.local_cache_untrusted()))
.build();
assert_eq!(proving.storage(&[42]).unwrap().unwrap(), vec![42; size_content]);

Expand Down Expand Up @@ -1206,7 +1206,7 @@ pub mod tests {

let proving = TrieBackendBuilder::wrap(&trie)
.with_recorder(Recorder::default())
.with_optional_cache(cache.as_ref().map(|c| c.local_cache()))
.with_optional_cache(cache.as_ref().map(|c| c.local_cache_untrusted()))
.build();

(0..63).for_each(|i| {
Expand Down Expand Up @@ -1278,7 +1278,7 @@ pub mod tests {

let proving = TrieBackendBuilder::wrap(&trie)
.with_recorder(Recorder::default())
.with_optional_cache(cache.as_ref().map(|c| c.local_cache()))
.with_optional_cache(cache.as_ref().map(|c| c.local_cache_untrusted()))
.build();
assert_eq!(proving.storage(&[42]).unwrap().unwrap(), vec![42]);

Expand All @@ -1295,7 +1295,7 @@ pub mod tests {

let proving = TrieBackendBuilder::wrap(&trie)
.with_recorder(Recorder::default())
.with_optional_cache(cache.as_ref().map(|c| c.local_cache()))
.with_optional_cache(cache.as_ref().map(|c| c.local_cache_untrusted()))
.build();
assert_eq!(proving.child_storage(child_info_1, &[64]), Ok(Some(vec![64])));
assert_eq!(proving.child_storage(child_info_1, &[25]), Ok(None));
Expand Down Expand Up @@ -1392,7 +1392,7 @@ pub mod tests {

let cache = SharedTrieCache::<BlakeTwo256>::new(CacheSize::unlimited(), None);
{
let local_cache = cache.local_cache();
let local_cache = cache.local_cache_untrusted();
let mut trie_cache = local_cache.as_trie_db_cache(child_1_root);

// Put the value/node into the cache.
Expand All @@ -1409,7 +1409,7 @@ pub mod tests {
// Record the access
let proving = TrieBackendBuilder::wrap(&trie)
.with_recorder(Recorder::default())
.with_cache(cache.local_cache())
.with_cache(cache.local_cache_untrusted())
.build();
assert_eq!(proving.child_storage(child_info_1, &[65]), Ok(Some(vec![65; 128])));

Expand Down Expand Up @@ -1491,11 +1491,11 @@ pub mod tests {
];

let new_root = {
let trie = test_trie(StateVersion::V1, Some(shared_cache.local_cache()), None);
let trie = test_trie(StateVersion::V1, Some(shared_cache.local_cache_untrusted()), None);
trie.storage_root(new_data.clone().into_iter(), StateVersion::V1).0
};

let local_cache = shared_cache.local_cache();
let local_cache = shared_cache.local_cache_untrusted();
let mut cache = local_cache.as_trie_db_cache(new_root);
// All the data should be cached now
for (key, value) in new_data {
Expand Down Expand Up @@ -1566,7 +1566,7 @@ pub mod tests {

let proving = TrieBackendBuilder::wrap(&trie)
.with_recorder(Recorder::default())
.with_optional_cache(cache.as_ref().map(|c| c.local_cache()))
.with_optional_cache(cache.as_ref().map(|c| c.local_cache_untrusted()))
.build();
assert_eq!(proving.storage(&key).unwrap().unwrap(), top_trie_val);
assert_eq!(
Expand Down
22 changes: 11 additions & 11 deletions substrate/primitives/trie/src/cache/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -906,7 +906,7 @@ mod tests {
let (db, root) = create_trie();

let shared_cache = Cache::new(CACHE_SIZE, None);
let local_cache = shared_cache.local_cache();
let local_cache = shared_cache.local_cache_untrusted();

{
let mut cache = local_cache.as_trie_db_cache(root);
Expand All @@ -933,7 +933,7 @@ mod tests {

let fake_data = Bytes::from(&b"fake_data"[..]);

let local_cache = shared_cache.local_cache();
let local_cache = shared_cache.local_cache_untrusted();
shared_cache.write_lock_inner().unwrap().value_cache_mut().lru.insert(
ValueCacheKey::new_value(TEST_DATA[1].0, root),
(fake_data.clone(), Default::default()).into(),
Expand All @@ -960,7 +960,7 @@ mod tests {
let mut new_root = root;

{
let local_cache = shared_cache.local_cache();
let local_cache = shared_cache.local_cache_untrusted();

let mut cache = local_cache.as_trie_db_mut_cache();

Expand Down Expand Up @@ -1001,7 +1001,7 @@ mod tests {
shared_cache.reset_value_cache();
}

let local_cache = shared_cache.local_cache();
let local_cache = shared_cache.local_cache_untrusted();
let recorder = Recorder::default();

{
Expand Down Expand Up @@ -1048,7 +1048,7 @@ mod tests {
}

let recorder = Recorder::default();
let local_cache = shared_cache.local_cache();
let local_cache = shared_cache.local_cache_untrusted();
let mut new_root = root;

{
Expand Down Expand Up @@ -1090,7 +1090,7 @@ mod tests {
let shared_cache = Cache::new(CACHE_SIZE, None);

{
let local_cache = shared_cache.local_cache();
let local_cache = shared_cache.local_cache_untrusted();

let mut cache = local_cache.as_trie_db_cache(root);
let trie = TrieDBBuilder::<Layout>::new(&db, &root).with_cache(&mut cache).build();
Expand All @@ -1114,7 +1114,7 @@ mod tests {
// The second run is using an empty value cache to ensure that we access the nodes.
for _ in 0..2 {
{
let local_cache = shared_cache.local_cache();
let local_cache = shared_cache.local_cache_untrusted();

let mut cache = local_cache.as_trie_db_cache(root);
let trie = TrieDBBuilder::<Layout>::new(&db, &root).with_cache(&mut cache).build();
Expand Down Expand Up @@ -1148,7 +1148,7 @@ mod tests {
.collect::<Vec<_>>();

{
let local_cache = shared_cache.local_cache();
let local_cache = shared_cache.local_cache_untrusted();

let mut cache = local_cache.as_trie_db_cache(root);
let trie = TrieDBBuilder::<Layout>::new(&db, &root).with_cache(&mut cache).build();
Expand Down Expand Up @@ -1177,7 +1177,7 @@ mod tests {

let shared_cache = Cache::new(CACHE_SIZE, None);
{
let local_cache = shared_cache.local_cache();
let local_cache = shared_cache.local_cache_untrusted();

let mut new_root = root;

Expand Down Expand Up @@ -1221,7 +1221,7 @@ mod tests {
// Populate the trie cache with, use a local untrusted cache and confirm not everything ends
// up in the shared trie cache.
let root = {
let local_cache = shared_cache.local_cache();
let local_cache = shared_cache.local_cache_untrusted();

let mut new_root = root;

Expand Down Expand Up @@ -1308,7 +1308,7 @@ mod tests {
keys: &Vec<Vec<u8>>,
expected_value: Vec<u8>,
) -> TrieHitStatsSnapshot {
let local_cache = shared_cache.local_cache();
let local_cache = shared_cache.local_cache_untrusted();
let mut cache = local_cache.as_trie_db_cache(root);
let trie = TrieDBBuilder::<Layout>::new(db, &root).with_cache(&mut cache).build();

Expand Down
4 changes: 2 additions & 2 deletions substrate/primitives/trie/src/cache/shared_cache.rs
Original file line number Diff line number Diff line change
Expand Up @@ -710,7 +710,7 @@ impl<H: Hasher> SharedTrieCache<H> {
}

/// Create a new [`LocalTrieCache`](super::LocalTrieCache) instance from this shared cache.
pub fn local_cache(&self) -> super::LocalTrieCache<H> {
pub fn local_cache_untrusted(&self) -> super::LocalTrieCache<H> {
let local_value_cache_config = LocalValueCacheConfig::default();
let local_node_cache_config = LocalNodeCacheConfig::default();
tracing::debug!(
Expand Down Expand Up @@ -738,7 +738,7 @@ impl<H: Hasher> SharedTrieCache<H> {
}
}

/// Creates a TrieCache that allows the local_caches to grow to indefinitely.
/// Creates a TrieCache that allows the local_caches to grow to indefinitely.
///
/// This is safe to be used only for trusted paths because it removes all limits on cache
/// growth and promotion, which could lead to excessive memory usage if used in untrusted or
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ impl StorageCmd {
let pov_recorder = enable_pov_recorder.then(|| Default::default());

DbStateBuilder::<HashingFor<Block>>::new(storage.clone(), original_root)
.with_optional_cache(shared_trie_cache.as_ref().map(|c| c.local_cache()))
.with_optional_cache(shared_trie_cache.as_ref().map(|c| c.local_cache_untrusted()))
.with_optional_recorder(pov_recorder)
.build()
};
Expand Down