Skip to content
This repository was archived by the owner on Nov 15, 2023. It is now read-only.

Mmr client gadget - support pallet reset #12999

Merged
paritytech-processbot merged 5 commits into from
Dec 22, 2022
Merged

Mmr client gadget - support pallet reset #12999

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
e580800
Remove unneeded code
serban300 Dec 20, 2022
5c6335f
Moving some code
serban300 Dec 20, 2022
b8f6654
Support pallet-mmr reset
serban300 Dec 21, 2022
924499c
Rename update_first_mmr_block
serban300 Dec 22, 2022
af9aa2e
Renamings
serban300 Dec 22, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
74 changes: 42 additions & 32 deletions client/merkle-mountain-range/src/aux_schema.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ pub(crate) fn write_gadget_state<B: Block, BE: AuxStore>(
backend.insert_aux(&[(GADGET_STATE, state.encode().as_slice())], &[])
}

fn load_decode<B: AuxStore, T: Decode>(backend: &B, key: &[u8]) -> ClientResult<Option<T>> {
fn load_typed<B: AuxStore, T: Decode>(backend: &B, key: &[u8]) -> ClientResult<Option<T>> {
match backend.get_aux(key)? {
None => Ok(None),
Some(t) => T::decode(&mut &t[..])
Expand All @@ -54,17 +54,17 @@ fn load_decode<B: AuxStore, T: Decode>(backend: &B, key: &[u8]) -> ClientResult<
}
}

/// Load or initialize persistent data from backend.
pub(crate) fn load_persistent<B, BE>(backend: &BE) -> ClientResult<Option<PersistedState<B>>>
/// Load persistent data from backend.
pub(crate) fn load_state<B, BE>(backend: &BE) -> ClientResult<Option<PersistedState<B>>>
where
B: Block,
BE: AuxStore,
{
let version: Option<u32> = load_decode(backend, VERSION_KEY)?;
let version: Option<u32> = load_typed(backend, VERSION_KEY)?;

match version {
None => (),
Some(1) => return load_decode::<_, PersistedState<B>>(backend, GADGET_STATE),
Some(1) => return load_typed::<_, PersistedState<B>>(backend, GADGET_STATE),
other =>
return Err(ClientError::Backend(format!("Unsupported MMR aux DB version: {:?}", other))),
}
Expand All @@ -73,15 +73,36 @@ where
Ok(None)
}

/// Load or initialize persistent data from backend.
pub(crate) fn load_or_init_state<B, BE>(
backend: &BE,
default: NumberFor<B>,
) -> sp_blockchain::Result<NumberFor<B>>
where
B: Block,
BE: AuxStore,
{
// Initialize gadget best_canon from AUX DB or from pallet genesis.
if let Some(best) = load_state::<B, BE>(backend)? {
info!(target: LOG_TARGET, "Loading MMR best canonicalized state from db: {:?}.", best);
Ok(best)
} else {
info!(
target: LOG_TARGET,
"Loading MMR from pallet genesis on what appears to be the first startup: {:?}.",
default
);
write_current_version(backend)?;
write_gadget_state::<B, BE>(backend, &default)?;
Ok(default)
}
}

#[cfg(test)]
pub(crate) mod tests {
use super::*;
use crate::test_utils::{
run_test_with_mmr_gadget_pre_post_using_client, MmrBlock, MockClient, OffchainKeyType,
};
use crate::test_utils::{run_test_with_mmr_gadget_pre_post_using_client, MmrBlock, MockClient};
use parking_lot::Mutex;
use sp_core::offchain::{DbExternalities, StorageKind};
use sp_mmr_primitives::utils::NodesUtils;
use sp_runtime::generic::BlockId;
use std::{sync::Arc, time::Duration};
use substrate_test_runtime_client::{runtime::Block, Backend};
Expand All @@ -92,15 +113,15 @@ pub(crate) mod tests {
let backend = &*client.backend;

// version not available in db -> None
assert_eq!(load_persistent::<Block, Backend>(backend).unwrap(), None);
assert_eq!(load_state::<Block, Backend>(backend).unwrap(), None);

// populate version in db
write_current_version(backend).unwrap();
// verify correct version is retrieved
assert_eq!(load_decode(backend, VERSION_KEY).unwrap(), Some(CURRENT_VERSION));
assert_eq!(load_typed(backend, VERSION_KEY).unwrap(), Some(CURRENT_VERSION));

// version is available in db but state isn't -> None
assert_eq!(load_persistent::<Block, Backend>(backend).unwrap(), None);
assert_eq!(load_state::<Block, Backend>(backend).unwrap(), None);
}

#[test]
Expand All @@ -111,9 +132,9 @@ pub(crate) mod tests {
let backend = client.backend.clone();

// version not available in db -> None
assert_eq!(load_decode::<Backend, Option<u32>>(&*backend, VERSION_KEY).unwrap(), None);
assert_eq!(load_typed::<Backend, Option<u32>>(&*backend, VERSION_KEY).unwrap(), None);
// state not available in db -> None
assert_eq!(load_persistent::<Block, Backend>(&*backend).unwrap(), None);
assert_eq!(load_state::<Block, Backend>(&*backend).unwrap(), None);
// run the gadget while importing and finalizing 3 blocks
run_test_with_mmr_gadget_pre_post_using_client(
client.clone(),
Expand All @@ -135,8 +156,8 @@ pub(crate) mod tests {
|client| async move {
let backend = &*client.backend;
// check there is both version and best canon available in db before running gadget
assert_eq!(load_decode(backend, VERSION_KEY).unwrap(), Some(CURRENT_VERSION));
assert_eq!(load_persistent::<Block, Backend>(backend).unwrap(), Some(3));
assert_eq!(load_typed(backend, VERSION_KEY).unwrap(), Some(CURRENT_VERSION));
assert_eq!(load_state::<Block, Backend>(backend).unwrap(), Some(3));
},
|client| async move {
let a4 = client.import_block(&BlockId::Number(3), b"a4", Some(3)).await;
Expand All @@ -148,7 +169,7 @@ pub(crate) mod tests {
// a4, a5, a6 were canonicalized
client.assert_canonicalized(&[&a4, &a5, &a6]);
// check persisted best canon was updated
assert_eq!(load_persistent::<Block, Backend>(&*client.backend).unwrap(), Some(6));
assert_eq!(load_state::<Block, Backend>(&*client.backend).unwrap(), Some(6));
},
);
}
Expand Down Expand Up @@ -177,19 +198,8 @@ pub(crate) mod tests {
client.assert_canonicalized(&slice);

// now manually move them back to non-canon/temp location
let mut offchain_db = client.offchain_db();
for mmr_block in slice {
for node in NodesUtils::right_branch_ending_in_leaf(mmr_block.leaf_idx.unwrap())
{
let canon_key = mmr_block.get_offchain_key(node, OffchainKeyType::Canon);
let val = offchain_db
.local_storage_get(StorageKind::PERSISTENT, &canon_key)
.unwrap();
offchain_db.local_storage_clear(StorageKind::PERSISTENT, &canon_key);

let temp_key = mmr_block.get_offchain_key(node, OffchainKeyType::Temp);
offchain_db.local_storage_set(StorageKind::PERSISTENT, &temp_key, &val);
}
client.undo_block_canonicalization(mmr_block)
}
},
);
Expand All @@ -203,7 +213,7 @@ pub(crate) mod tests {
let slice: Vec<&MmrBlock> = blocks.iter().collect();

// verify persisted state says a1, a2, a3 were canonicalized,
assert_eq!(load_persistent::<Block, Backend>(&*client.backend).unwrap(), Some(3));
assert_eq!(load_state::<Block, Backend>(&*client.backend).unwrap(), Some(3));
// but actually they are NOT canon (we manually reverted them earlier).
client.assert_not_canonicalized(&slice);
},
Expand All @@ -221,7 +231,7 @@ pub(crate) mod tests {
// but a4, a5, a6 were canonicalized
client.assert_canonicalized(&[&a4, &a5, &a6]);
// check persisted best canon was updated
assert_eq!(load_persistent::<Block, Backend>(&*client.backend).unwrap(), Some(6));
assert_eq!(load_state::<Block, Backend>(&*client.backend).unwrap(), Some(6));
},
);
}
Expand Down
137 changes: 74 additions & 63 deletions client/merkle-mountain-range/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ use crate::offchain_mmr::OffchainMmr;
use beefy_primitives::MmrRootHash;
use futures::StreamExt;
use log::{debug, error, trace, warn};
use sc_client_api::{Backend, BlockchainEvents, FinalityNotifications};
use sc_client_api::{Backend, BlockchainEvents, FinalityNotification, FinalityNotifications};
use sc_offchain::OffchainDb;
use sp_api::ProvideRuntimeApi;
use sp_blockchain::{HeaderBackend, HeaderMetadata};
Expand All @@ -60,6 +60,62 @@ use std::{marker::PhantomData, sync::Arc};
/// Logging target for the mmr gadget.
pub const LOG_TARGET: &str = "mmr";

/// A convenience MMR client trait that defines all the type bounds a MMR client
/// has to satisfy and defines some helper methods.
pub trait MmrClient<B, BE>:
BlockchainEvents<B> + HeaderBackend<B> + HeaderMetadata<B> + ProvideRuntimeApi<B>
where
B: Block,
BE: Backend<B>,
Self::Api: MmrApi<B, MmrRootHash, NumberFor<B>>,
{
/// Get the block number where the mmr pallet was added to the runtime.
fn first_mmr_block_num(&self, notification: &FinalityNotification<B>) -> Option<NumberFor<B>> {
let best_block = *notification.header.number();
match self.runtime_api().mmr_leaf_count(&BlockId::number(best_block)) {
Ok(Ok(mmr_leaf_count)) => {
match utils::first_mmr_block_num::<B::Header>(best_block, mmr_leaf_count) {
Ok(first_mmr_block) => {
debug!(
target: LOG_TARGET,
"pallet-mmr detected at block {:?} with genesis at block {:?}",
best_block,
first_mmr_block
);
Some(first_mmr_block)
},
Err(e) => {
error!(
target: LOG_TARGET,
"Error calculating the first mmr block: {:?}", e
);
None
},
}
},
_ => {
trace!(
target: LOG_TARGET,
"pallet-mmr not detected at block {:?} ... (best finalized {:?})",
best_block,
notification.header.number()
);
None
},
}
}
}

impl<B, BE, T> MmrClient<B, BE> for T
where
B: Block,
BE: Backend<B>,
T: BlockchainEvents<B> + HeaderBackend<B> + HeaderMetadata<B> + ProvideRuntimeApi<B>,
T::Api: MmrApi<B, MmrRootHash, NumberFor<B>>,
{
// empty
}

struct OffchainMmrBuilder<B: Block, BE: Backend<B>, C> {
backend: Arc<BE>,
client: Arc<C>,
Expand All @@ -73,74 +129,29 @@ impl<B, BE, C> OffchainMmrBuilder<B, BE, C>
where
B: Block,
BE: Backend<B>,
C: ProvideRuntimeApi<B> + HeaderBackend<B> + HeaderMetadata<B>,
C: MmrClient<B, BE>,
C::Api: MmrApi<B, MmrRootHash, NumberFor<B>>,
{
async fn try_build(
self,
finality_notifications: &mut FinalityNotifications<B>,
) -> Option<OffchainMmr<B, BE, C>> {
while let Some(notification) = finality_notifications.next().await {
let best_block = *notification.header.number();
match self.client.runtime_api().mmr_leaf_count(&BlockId::number(best_block)) {
Ok(Ok(mmr_leaf_count)) => {
debug!(
target: LOG_TARGET,
"pallet-mmr detected at block {:?} with mmr size {:?}",
best_block,
mmr_leaf_count
);
match utils::first_mmr_block_num::<B::Header>(best_block, mmr_leaf_count) {
Ok(first_mmr_block) => {
debug!(
target: LOG_TARGET,
"pallet-mmr genesis computed at block {:?}", first_mmr_block,
);
let best_canonicalized =
match offchain_mmr::load_or_init_best_canonicalized::<B, BE>(
&*self.backend,
first_mmr_block,
) {
Ok(best) => best,
Err(e) => {
error!(
target: LOG_TARGET,
"Error loading state from aux db: {:?}", e
);
return None
},
};
let mut offchain_mmr = OffchainMmr {
backend: self.backend,
client: self.client,
offchain_db: self.offchain_db,
indexing_prefix: self.indexing_prefix,
first_mmr_block,
best_canonicalized,
};
// We need to make sure all blocks leading up to current notification
// have also been canonicalized.
offchain_mmr.canonicalize_catch_up(&notification);
// We have to canonicalize and prune the blocks in the finality
// notification that lead to building the offchain-mmr as well.
offchain_mmr.canonicalize_and_prune(notification);
return Some(offchain_mmr)
},
Err(e) => {
error!(
target: LOG_TARGET,
"Error calculating the first mmr block: {:?}", e
);
},
}
},
_ => {
trace!(
target: LOG_TARGET,
"Waiting for MMR pallet to become available... (best finalized {:?})",
notification.header.number()
);
},
if let Some(first_mmr_block_num) = self.client.first_mmr_block_num(&notification) {
let mut offchain_mmr = OffchainMmr::new(
self.backend,
self.client,
self.offchain_db,
self.indexing_prefix,
first_mmr_block_num,
)?;
// We need to make sure all blocks leading up to current notification
// have also been canonicalized.
offchain_mmr.canonicalize_catch_up(&notification);
// We have to canonicalize and prune the blocks in the finality
// notification that lead to building the offchain-mmr as well.
offchain_mmr.canonicalize_and_prune(notification);
return Some(offchain_mmr)
}
}

Expand All @@ -165,7 +176,7 @@ where
B: Block,
<B::Header as Header>::Number: Into<LeafIndex>,
BE: Backend<B>,
C: BlockchainEvents<B> + HeaderBackend<B> + HeaderMetadata<B> + ProvideRuntimeApi<B>,
C: MmrClient<B, BE>,
C::Api: MmrApi<B, MmrRootHash, NumberFor<B>>,
{
async fn run(mut self, builder: OffchainMmrBuilder<B, BE, C>) {
Expand Down
Loading