diff --git a/Cargo.lock b/Cargo.lock index 741abb459a06..bfc60401eebe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7197,6 +7197,7 @@ name = "polkadot-node-network-protocol" version = "0.9.37" dependencies = [ "async-trait", + "bitvec 1.0.1", "derive_more", "fatality", "futures", @@ -7801,6 +7802,7 @@ version = "0.9.37" dependencies = [ "arrayvec 0.5.2", "assert_matches", + "bitvec 1.0.1", "fatality", "futures", "futures-timer", @@ -7810,9 +7812,11 @@ dependencies = [ "polkadot-node-primitives", "polkadot-node-subsystem", "polkadot-node-subsystem-test-helpers", + "polkadot-node-subsystem-types", "polkadot-node-subsystem-util", "polkadot-primitives", "polkadot-primitives-test-helpers", + "rand_chacha 0.3.1", "sc-keystore", "sc-network", "sp-application-crypto", @@ -8126,14 +8130,14 @@ dependencies = [ [[package]] name = "pretty_assertions" -version = "1.2.1" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c89f989ac94207d048d92db058e4f6ec7342b0971fc58d1271ca148b799b3563" +checksum = "a25e9bcb20aa780fd0bb16b72403a9064d6b3f22f026946029acb941a50af755" dependencies = [ - "ansi_term", "ctor", "diff", "output_vt100", + "yansi", ] [[package]] @@ -13951,6 +13955,12 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "yansi" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec" + [[package]] name = "yasna" version = "0.5.1" diff --git a/node/core/backing/src/lib.rs b/node/core/backing/src/lib.rs index a640acf6d62c..157181a9d940 100644 --- a/node/core/backing/src/lib.rs +++ b/node/core/backing/src/lib.rs @@ -80,16 +80,16 @@ use futures::{ use error::{Error, FatalResult}; use polkadot_node_primitives::{ - AvailableData, InvalidCandidate, PoV, SignedFullStatementWithPVD, StatementWithPVD, - ValidationResult, BACKING_EXECUTION_TIMEOUT, + minimum_votes, AvailableData, InvalidCandidate, PoV, SignedFullStatementWithPVD, + StatementWithPVD, ValidationResult, BACKING_EXECUTION_TIMEOUT, }; use polkadot_node_subsystem::{ messages::{ AvailabilityDistributionMessage, AvailabilityStoreMessage, CanSecondRequest, CandidateBackingMessage, CandidateValidationMessage, CollatorProtocolMessage, - HypotheticalCandidate, HypotheticalFrontierRequest, ProspectiveParachainsMessage, - ProvisionableData, ProvisionerMessage, RuntimeApiMessage, RuntimeApiRequest, - StatementDistributionMessage, + HypotheticalCandidate, HypotheticalFrontierRequest, IntroduceCandidateRequest, + ProspectiveParachainsMessage, ProvisionableData, ProvisionerMessage, RuntimeApiMessage, + RuntimeApiRequest, StatementDistributionMessage, }, overseer, ActiveLeavesUpdate, FromOrchestra, OverseerSignal, SpawnedSubsystem, SubsystemError, }; @@ -374,13 +374,6 @@ struct AttestingData { backing: Vec, } -/// How many votes we need to consider a candidate backed. -/// -/// WARNING: This has to be kept in sync with the runtime check in the inclusion module. -fn minimum_votes(n_validators: usize) -> usize { - std::cmp::min(2, n_validators) -} - #[derive(Default)] struct TableContext { validator: Option, @@ -1514,10 +1507,12 @@ async fn import_statement( if !per_candidate.contains_key(&candidate_hash) { if rp_state.prospective_parachains_mode.is_enabled() { let (tx, rx) = oneshot::channel(); - ctx.send_message(ProspectiveParachainsMessage::CandidateSeconded( - candidate.descriptor().para_id, - candidate.clone(), - pvd.clone(), + ctx.send_message(ProspectiveParachainsMessage::IntroduceCandidate( + IntroduceCandidateRequest { + candidate_para: candidate.descriptor().para_id, + candidate_receipt: candidate.clone(), + persisted_validation_data: pvd.clone(), + }, tx, )) .await; @@ -1536,6 +1531,12 @@ async fn import_statement( return Err(Error::RejectedByProspectiveParachains) }, } + + ctx.send_message(ProspectiveParachainsMessage::CandidateSeconded( + candidate.descriptor().para_id, + candidate_hash, + )) + .await; } // Only save the candidate if it was approved by prospective parachains. diff --git a/node/core/backing/src/tests/prospective_parachains.rs b/node/core/backing/src/tests/prospective_parachains.rs index c29a3382510e..fe935c2746e5 100644 --- a/node/core/backing/src/tests/prospective_parachains.rs +++ b/node/core/backing/src/tests/prospective_parachains.rs @@ -420,18 +420,27 @@ fn seconding_sanity_check_allowed() { assert_matches!( virtual_overseer.recv().await, AllMessages::ProspectiveParachains( - ProspectiveParachainsMessage::CandidateSeconded( - candidate_para, - candidate_receipt, - _pvd, + ProspectiveParachainsMessage::IntroduceCandidate( + req, tx, ), - ) if candidate_receipt == candidate && candidate_para == para_id && pvd == _pvd => { + ) if + req.candidate_receipt == candidate + && req.candidate_para == para_id + && pvd == req.persisted_validation_data => { // Any non-empty response will do. tx.send(vec![(leaf_a_hash, vec![0, 1, 2, 3])]).unwrap(); } ); + assert_matches!( + virtual_overseer.recv().await, + AllMessages::ProspectiveParachains(ProspectiveParachainsMessage::CandidateSeconded( + _, + _ + )) + ); + assert_matches!( virtual_overseer.recv().await, AllMessages::StatementDistribution( @@ -557,18 +566,27 @@ fn seconding_sanity_check_disallowed() { assert_matches!( virtual_overseer.recv().await, AllMessages::ProspectiveParachains( - ProspectiveParachainsMessage::CandidateSeconded( - candidate_para, - candidate_receipt, - _pvd, + ProspectiveParachainsMessage::IntroduceCandidate( + req, tx, ), - ) if candidate_receipt == candidate && candidate_para == para_id && pvd == _pvd => { + ) if + req.candidate_receipt == candidate + && req.candidate_para == para_id + && pvd == req.persisted_validation_data => { // Any non-empty response will do. tx.send(vec![(leaf_a_hash, vec![0, 2, 3])]).unwrap(); } ); + assert_matches!( + virtual_overseer.recv().await, + AllMessages::ProspectiveParachains(ProspectiveParachainsMessage::CandidateSeconded( + _, + _ + )) + ); + assert_matches!( virtual_overseer.recv().await, AllMessages::StatementDistribution( @@ -757,13 +775,14 @@ fn prospective_parachains_reject_candidate() { assert_matches!( virtual_overseer.recv().await, AllMessages::ProspectiveParachains( - ProspectiveParachainsMessage::CandidateSeconded( - candidate_para, - candidate_receipt, - _pvd, + ProspectiveParachainsMessage::IntroduceCandidate( + req, tx, ), - ) if candidate_receipt == candidate && candidate_para == para_id && pvd == _pvd => { + ) if + req.candidate_receipt == candidate + && req.candidate_para == para_id + && pvd == req.persisted_validation_data => { // Reject it. tx.send(Vec::new()).unwrap(); } @@ -808,18 +827,27 @@ fn prospective_parachains_reject_candidate() { assert_matches!( virtual_overseer.recv().await, AllMessages::ProspectiveParachains( - ProspectiveParachainsMessage::CandidateSeconded( - candidate_para, - candidate_receipt, - _pvd, + ProspectiveParachainsMessage::IntroduceCandidate( + req, tx, ), - ) if candidate_receipt == candidate && candidate_para == para_id && pvd == _pvd => { + ) if + req.candidate_receipt == candidate + && req.candidate_para == para_id + && pvd == req.persisted_validation_data => { // Any non-empty response will do. tx.send(vec![(leaf_a_hash, vec![0, 2, 3])]).unwrap(); } ); + assert_matches!( + virtual_overseer.recv().await, + AllMessages::ProspectiveParachains(ProspectiveParachainsMessage::CandidateSeconded( + _, + _ + )) + ); + assert_matches!( virtual_overseer.recv().await, AllMessages::StatementDistribution( @@ -938,19 +966,28 @@ fn second_multiple_candidates_per_relay_parent() { .await; // Prospective parachains are notified. + assert_matches!( + virtual_overseer.recv().await, + AllMessages::ProspectiveParachains( + ProspectiveParachainsMessage::IntroduceCandidate( + req, + tx, + ), + ) if + &req.candidate_receipt == candidate + && req.candidate_para == para_id + && pvd == req.persisted_validation_data + => { + // Any non-empty response will do. + tx.send(vec![(leaf_hash, vec![0, 2, 3])]).unwrap(); + } + ); + assert_matches!( virtual_overseer.recv().await, AllMessages::ProspectiveParachains( - ProspectiveParachainsMessage::CandidateSeconded( - candidate_para, - candidate_receipt, - _pvd, - tx, - ), - ) if &candidate_receipt == candidate && candidate_para == para_id && pvd == _pvd => { - // Any non-empty response will do. - tx.send(vec![(leaf_hash, vec![0, 2, 3])]).unwrap(); - } + ProspectiveParachainsMessage::CandidateSeconded(_, _) + ) ); assert_matches!( @@ -1072,18 +1109,27 @@ fn backing_works() { assert_matches!( virtual_overseer.recv().await, AllMessages::ProspectiveParachains( - ProspectiveParachainsMessage::CandidateSeconded( - candidate_para, - candidate_receipt, - _pvd, + ProspectiveParachainsMessage::IntroduceCandidate( + req, tx, ), - ) if candidate_receipt == candidate_a && candidate_para == para_id && pvd == _pvd => { + ) if + req.candidate_receipt == candidate_a + && req.candidate_para == para_id + && pvd == req.persisted_validation_data => { // Any non-empty response will do. tx.send(vec![(leaf_hash, vec![0, 2, 3])]).unwrap(); } ); + assert_matches!( + virtual_overseer.recv().await, + AllMessages::ProspectiveParachains(ProspectiveParachainsMessage::CandidateSeconded( + _, + _ + )) + ); + assert_validate_seconded_candidate( &mut virtual_overseer, candidate_a.descriptor().relay_parent, @@ -1276,10 +1322,13 @@ fn concurrent_dependent_candidates() { // Order is not guaranteed since we have 2 statements being handled concurrently. match msg { AllMessages::ProspectiveParachains( - ProspectiveParachainsMessage::CandidateSeconded(.., tx), + ProspectiveParachainsMessage::IntroduceCandidate(_, tx), ) => { tx.send(vec![(leaf_hash, vec![0, 2, 3])]).unwrap(); }, + AllMessages::ProspectiveParachains( + ProspectiveParachainsMessage::CandidateSeconded(_, _), + ) => {}, AllMessages::RuntimeApi(RuntimeApiMessage::Request( _, RuntimeApiRequest::ValidationCodeByHash(_, tx), @@ -1472,19 +1521,28 @@ fn seconding_sanity_check_occupy_same_depth() { .await; // Prospective parachains are notified. + assert_matches!( + virtual_overseer.recv().await, + AllMessages::ProspectiveParachains( + ProspectiveParachainsMessage::IntroduceCandidate( + req, + tx, + ), + ) if + &req.candidate_receipt == candidate + && &req.candidate_para == para_id + && pvd == req.persisted_validation_data + => { + // Any non-empty response will do. + tx.send(vec![(leaf_hash, vec![0, 2, 3])]).unwrap(); + } + ); + assert_matches!( virtual_overseer.recv().await, AllMessages::ProspectiveParachains( - ProspectiveParachainsMessage::CandidateSeconded( - candidate_para, - candidate_receipt, - _pvd, - tx, - ), - ) if &candidate_receipt == candidate && candidate_para == *para_id && pvd == _pvd => { - // Any non-empty response will do. - tx.send(vec![(leaf_hash, vec![0, 2, 3])]).unwrap(); - } + ProspectiveParachainsMessage::CandidateSeconded(_, _) + ) ); assert_matches!( @@ -1610,18 +1668,28 @@ fn occupied_core_assignment() { assert_matches!( virtual_overseer.recv().await, AllMessages::ProspectiveParachains( - ProspectiveParachainsMessage::CandidateSeconded( - candidate_para, - candidate_receipt, - _pvd, + ProspectiveParachainsMessage::IntroduceCandidate( + req, tx, ), - ) if candidate_receipt == candidate && candidate_para == para_id && pvd == _pvd => { + ) if + req.candidate_receipt == candidate + && req.candidate_para == para_id + && pvd == req.persisted_validation_data + => { // Any non-empty response will do. tx.send(vec![(leaf_a_hash, vec![0, 1, 2, 3])]).unwrap(); } ); + assert_matches!( + virtual_overseer.recv().await, + AllMessages::ProspectiveParachains(ProspectiveParachainsMessage::CandidateSeconded( + _, + _ + )) + ); + assert_matches!( virtual_overseer.recv().await, AllMessages::StatementDistribution( diff --git a/node/core/prospective-parachains/src/fragment_tree.rs b/node/core/prospective-parachains/src/fragment_tree.rs index eea0f6969291..1ded5fa44fdb 100644 --- a/node/core/prospective-parachains/src/fragment_tree.rs +++ b/node/core/prospective-parachains/src/fragment_tree.rs @@ -144,7 +144,6 @@ impl CandidateStorage { } /// Remove a candidate from the store. - #[allow(dead_code)] pub fn remove_candidate(&mut self, candidate_hash: &CandidateHash) { if let Some(entry) = self.by_candidate_hash.remove(candidate_hash) { let parent_head_hash = entry.candidate.persisted_validation_data.parent_head.hash(); @@ -158,7 +157,6 @@ impl CandidateStorage { } /// Note that an existing candidate has been seconded. - #[allow(dead_code)] pub fn mark_seconded(&mut self, candidate_hash: &CandidateHash) { if let Some(entry) = self.by_candidate_hash.get_mut(candidate_hash) { if entry.state != CandidateState::Backed { @@ -251,7 +249,6 @@ enum CandidateState { /// is not necessarily backed. Introduced, /// The candidate has been seconded. - #[allow(dead_code)] Seconded, /// The candidate has been completely backed by the group. Backed, diff --git a/node/core/prospective-parachains/src/lib.rs b/node/core/prospective-parachains/src/lib.rs index 01d45f6a0e2e..f0e6eaaa14c8 100644 --- a/node/core/prospective-parachains/src/lib.rs +++ b/node/core/prospective-parachains/src/lib.rs @@ -37,7 +37,7 @@ use futures::{channel::oneshot, prelude::*}; use polkadot_node_subsystem::{ messages::{ ChainApiMessage, FragmentTreeMembership, HypotheticalCandidate, - HypotheticalFrontierRequest, ProspectiveParachainsMessage, + HypotheticalFrontierRequest, IntroduceCandidateRequest, ProspectiveParachainsMessage, ProspectiveValidationDataRequest, RuntimeApiMessage, RuntimeApiRequest, }, overseer, ActiveLeavesUpdate, FromOrchestra, OverseerSignal, SpawnedSubsystem, SubsystemError, @@ -47,8 +47,7 @@ use polkadot_node_subsystem_util::{ runtime::{prospective_parachains_mode, ProspectiveParachainsMode}, }; use polkadot_primitives::vstaging::{ - BlockNumber, CandidateHash, CommittedCandidateReceipt, CoreState, Hash, Id as ParaId, - PersistedValidationData, + BlockNumber, CandidateHash, CoreState, Hash, Id as ParaId, PersistedValidationData, }; use crate::{ @@ -136,8 +135,10 @@ async fn run_iteration( }, FromOrchestra::Signal(OverseerSignal::BlockFinalized(..)) => {}, FromOrchestra::Communication { msg } => match msg { - ProspectiveParachainsMessage::CandidateSeconded(para, candidate, pvd, tx) => - handle_candidate_seconded(&mut *ctx, view, para, candidate, pvd, tx).await?, + ProspectiveParachainsMessage::IntroduceCandidate(request, tx) => + handle_candidate_introduced(&mut *ctx, view, request, tx).await?, + ProspectiveParachainsMessage::CandidateSeconded(para, candidate_hash) => + handle_candidate_seconded(view, para, candidate_hash), ProspectiveParachainsMessage::CandidateBacked(para, candidate_hash) => handle_candidate_backed(&mut *ctx, view, para, candidate_hash).await?, ProspectiveParachainsMessage::GetBackableCandidate( @@ -289,14 +290,18 @@ fn prune_view_candidate_storage(view: &mut View, metrics: &Metrics) { } #[overseer::contextbounds(ProspectiveParachains, prefix = self::overseer)] -async fn handle_candidate_seconded( +async fn handle_candidate_introduced( _ctx: &mut Context, view: &mut View, - para: ParaId, - candidate: CommittedCandidateReceipt, - pvd: PersistedValidationData, + request: IntroduceCandidateRequest, tx: oneshot::Sender, ) -> JfyiErrorResult<()> { + let IntroduceCandidateRequest { + candidate_para: para, + candidate_receipt: candidate, + persisted_validation_data: pvd, + } = request; + // Add the candidate to storage. // Then attempt to add it to all trees. let storage = match view.candidate_storage.get_mut(¶) { @@ -316,8 +321,9 @@ async fn handle_candidate_seconded( let candidate_hash = match storage.add_candidate(candidate, pvd) { Ok(c) => c, - Err(crate::fragment_tree::CandidateStorageInsertionError::CandidateAlreadyKnown(_)) => { - let _ = tx.send(Vec::new()); + Err(crate::fragment_tree::CandidateStorageInsertionError::CandidateAlreadyKnown(c)) => { + // Candidate known - return existing fragment tree membership. + let _ = tx.send(fragment_tree_membership(&view.active_leaves, para, c)); return Ok(()) }, Err( @@ -347,11 +353,45 @@ async fn handle_candidate_seconded( } } } + + if membership.is_empty() { + storage.remove_candidate(&candidate_hash); + } + let _ = tx.send(membership); Ok(()) } +fn handle_candidate_seconded(view: &mut View, para: ParaId, candidate_hash: CandidateHash) { + let storage = match view.candidate_storage.get_mut(¶) { + None => { + gum::warn!( + target: LOG_TARGET, + para_id = ?para, + ?candidate_hash, + "Received instruction to second unknown candidate", + ); + + return + }, + Some(storage) => storage, + }; + + if !storage.contains(&candidate_hash) { + gum::warn!( + target: LOG_TARGET, + para_id = ?para, + ?candidate_hash, + "Received instruction to second unknown candidate", + ); + + return + } + + storage.mark_seconded(&candidate_hash); +} + #[overseer::contextbounds(ProspectiveParachains, prefix = self::overseer)] async fn handle_candidate_backed( _ctx: &mut Context, @@ -365,7 +405,7 @@ async fn handle_candidate_backed( target: LOG_TARGET, para_id = ?para, ?candidate_hash, - "Received instruction to back candidate", + "Received instruction to back unknown candidate", ); return Ok(()) @@ -378,7 +418,7 @@ async fn handle_candidate_backed( target: LOG_TARGET, para_id = ?para, ?candidate_hash, - "Received instruction to mark unknown candidate as backed.", + "Received instruction to back unknown candidate", ); return Ok(()) diff --git a/node/core/prospective-parachains/src/tests.rs b/node/core/prospective-parachains/src/tests.rs index d8d99eba1bff..1936a482e685 100644 --- a/node/core/prospective-parachains/src/tests.rs +++ b/node/core/prospective-parachains/src/tests.rs @@ -15,7 +15,6 @@ // along with Polkadot. If not, see . use super::*; -use ::polkadot_primitives_test_helpers::{dummy_candidate_receipt_bad_sig, dummy_hash}; use assert_matches::assert_matches; use polkadot_node_subsystem::{ errors::RuntimeApiError, @@ -27,12 +26,11 @@ use polkadot_node_subsystem::{ use polkadot_node_subsystem_test_helpers as test_helpers; use polkadot_node_subsystem_types::{jaeger, ActivatedLeaf, LeafStatus}; use polkadot_primitives::{ - v2::{ - CandidateCommitments, HeadData, Header, PersistedValidationData, ScheduledCore, - ValidationCodeHash, - }, vstaging::{AsyncBackingParameters, Constraints, InboundHrmpLimitations}, + CommittedCandidateReceipt, HeadData, Header, PersistedValidationData, ScheduledCore, + ValidationCodeHash, }; +use polkadot_primitives_test_helpers::make_candidate; use std::sync::Arc; const ALLOWED_ANCESTRY_LEN: u32 = 3; @@ -59,7 +57,7 @@ fn dummy_constraints( ump_remaining: 10, ump_remaining_bytes: 1_000, max_ump_num_per_candidate: 10, - dmp_remaining_messages: vec![10], + dmp_remaining_messages: vec![], hrmp_inbound: InboundHrmpLimitations { valid_watermarks }, hrmp_channels_out: vec![], max_hrmp_num_per_candidate: 0, @@ -70,42 +68,6 @@ fn dummy_constraints( } } -fn dummy_pvd(parent_head: HeadData, relay_parent_number: u32) -> PersistedValidationData { - PersistedValidationData { - parent_head, - relay_parent_number, - max_pov_size: MAX_POV_SIZE, - relay_parent_storage_root: dummy_hash(), - } -} - -fn make_candidate( - leaf: &TestLeaf, - para_id: ParaId, - parent_head: HeadData, - head_data: HeadData, - validation_code_hash: ValidationCodeHash, -) -> (CommittedCandidateReceipt, PersistedValidationData) { - let pvd = dummy_pvd(parent_head, leaf.number); - let commitments = CandidateCommitments { - head_data, - horizontal_messages: Default::default(), - upward_messages: Default::default(), - new_validation_code: None, - processed_downward_messages: 0, - hrmp_watermark: leaf.number, - }; - - let mut candidate = dummy_candidate_receipt_bad_sig(leaf.hash, Some(Default::default())); - candidate.commitments_hash = commitments.hash(); - candidate.descriptor.para_id = para_id; - candidate.descriptor.persisted_validation_data_hash = pvd.hash(); - candidate.descriptor.validation_code_hash = validation_code_hash; - let candidate = CommittedCandidateReceipt { descriptor: candidate.descriptor, commitments }; - - (candidate, pvd) -} - struct TestState { availability_cores: Vec, validation_code_hash: ValidationCodeHash, @@ -334,25 +296,36 @@ async fn deactivate_leaf(virtual_overseer: &mut VirtualOverseer, hash: Hash) { .await; } -async fn second_candidate( +async fn introduce_candidate( virtual_overseer: &mut VirtualOverseer, candidate: CommittedCandidateReceipt, pvd: PersistedValidationData, - expected_candidate_response: Vec<(Hash, Vec)>, ) { - let (tx, rx) = oneshot::channel(); + let req = IntroduceCandidateRequest { + candidate_para: candidate.descriptor().para_id, + candidate_receipt: candidate, + persisted_validation_data: pvd, + }; + let (tx, _) = oneshot::channel(); + virtual_overseer + .send(overseer::FromOrchestra::Communication { + msg: ProspectiveParachainsMessage::IntroduceCandidate(req, tx), + }) + .await; +} + +async fn second_candidate( + virtual_overseer: &mut VirtualOverseer, + candidate: CommittedCandidateReceipt, +) { virtual_overseer .send(overseer::FromOrchestra::Communication { msg: ProspectiveParachainsMessage::CandidateSeconded( candidate.descriptor.para_id, - candidate, - pvd, - tx, + candidate.hash(), ), }) .await; - let resp = rx.await.unwrap(); - assert_eq!(resp, expected_candidate_response); } async fn back_candidate( @@ -414,6 +387,7 @@ async fn get_hypothetical_frontier( receipt: CommittedCandidateReceipt, persisted_validation_data: PersistedValidationData, fragment_tree_relay_parent: Hash, + backed_in_path_only: bool, expected_depths: Vec, ) { let hypothetical_candidate = HypotheticalCandidate::Complete { @@ -424,7 +398,7 @@ async fn get_hypothetical_frontier( let request = HypotheticalFrontierRequest { candidates: vec![hypothetical_candidate.clone()], fragment_tree_relay_parent: Some(fragment_tree_relay_parent), - backed_in_path_only: false, + backed_in_path_only, }; let (tx, rx) = oneshot::channel(); virtual_overseer @@ -433,8 +407,11 @@ async fn get_hypothetical_frontier( }) .await; let resp = rx.await.unwrap(); - let expected_frontier = - vec![(hypothetical_candidate, vec![(fragment_tree_relay_parent, expected_depths)])]; + let expected_frontier = if expected_depths.is_empty() { + vec![(hypothetical_candidate, vec![])] + } else { + vec![(hypothetical_candidate, vec![(fragment_tree_relay_parent, expected_depths)])] + }; assert_eq!(resp, expected_frontier); } @@ -540,7 +517,8 @@ fn send_candidates_and_check_if_found() { // Candidate A1 let (candidate_a1, pvd_a1) = make_candidate( - &leaf_a, + leaf_a.hash, + leaf_a.number, 1.into(), HeadData(vec![1, 2, 3]), HeadData(vec![1]), @@ -551,7 +529,8 @@ fn send_candidates_and_check_if_found() { // Candidate A2 let (candidate_a2, pvd_a2) = make_candidate( - &leaf_a, + leaf_a.hash, + leaf_a.number, 2.into(), HeadData(vec![2, 3, 4]), HeadData(vec![2]), @@ -562,7 +541,8 @@ fn send_candidates_and_check_if_found() { // Candidate B let (candidate_b, pvd_b) = make_candidate( - &leaf_b, + leaf_b.hash, + leaf_b.number, 1.into(), HeadData(vec![3, 4, 5]), HeadData(vec![3]), @@ -573,7 +553,8 @@ fn send_candidates_and_check_if_found() { // Candidate C let (candidate_c, pvd_c) = make_candidate( - &leaf_c, + leaf_c.hash, + leaf_c.number, 2.into(), HeadData(vec![6, 7, 8]), HeadData(vec![4]), @@ -582,11 +563,11 @@ fn send_candidates_and_check_if_found() { let candidate_hash_c = candidate_c.hash(); let response_c = vec![(leaf_c.hash, vec![0])]; - // Second candidates. - second_candidate(&mut virtual_overseer, candidate_a1, pvd_a1, response_a1.clone()).await; - second_candidate(&mut virtual_overseer, candidate_a2, pvd_a2, response_a2.clone()).await; - second_candidate(&mut virtual_overseer, candidate_b, pvd_b, response_b.clone()).await; - second_candidate(&mut virtual_overseer, candidate_c, pvd_c, response_c.clone()).await; + // Introduce candidates. + introduce_candidate(&mut virtual_overseer, candidate_a1, pvd_a1).await; + introduce_candidate(&mut virtual_overseer, candidate_a2, pvd_a2).await; + introduce_candidate(&mut virtual_overseer, candidate_b, pvd_b).await; + introduce_candidate(&mut virtual_overseer, candidate_c, pvd_c).await; // Check candidate tree membership. get_membership(&mut virtual_overseer, 1.into(), candidate_hash_a1, response_a1).await; @@ -650,29 +631,30 @@ fn check_candidate_parent_leaving_view() { // Candidate A1 let (candidate_a1, pvd_a1) = make_candidate( - &leaf_a, + leaf_a.hash, + leaf_a.number, 1.into(), HeadData(vec![1, 2, 3]), HeadData(vec![1]), test_state.validation_code_hash, ); let candidate_hash_a1 = candidate_a1.hash(); - let response_a1 = vec![(leaf_a.hash, vec![0])]; // Candidate A2 let (candidate_a2, pvd_a2) = make_candidate( - &leaf_a, + leaf_a.hash, + leaf_a.number, 2.into(), HeadData(vec![2, 3, 4]), HeadData(vec![2]), test_state.validation_code_hash, ); let candidate_hash_a2 = candidate_a2.hash(); - let response_a2 = vec![(leaf_a.hash, vec![0])]; // Candidate B let (candidate_b, pvd_b) = make_candidate( - &leaf_b, + leaf_b.hash, + leaf_b.number, 1.into(), HeadData(vec![3, 4, 5]), HeadData(vec![3]), @@ -683,7 +665,8 @@ fn check_candidate_parent_leaving_view() { // Candidate C let (candidate_c, pvd_c) = make_candidate( - &leaf_c, + leaf_c.hash, + leaf_c.number, 2.into(), HeadData(vec![6, 7, 8]), HeadData(vec![4]), @@ -692,11 +675,11 @@ fn check_candidate_parent_leaving_view() { let candidate_hash_c = candidate_c.hash(); let response_c = vec![(leaf_c.hash, vec![0])]; - // Second candidates. - second_candidate(&mut virtual_overseer, candidate_a1, pvd_a1, response_a1.clone()).await; - second_candidate(&mut virtual_overseer, candidate_a2, pvd_a2, response_a2.clone()).await; - second_candidate(&mut virtual_overseer, candidate_b, pvd_b, response_b.clone()).await; - second_candidate(&mut virtual_overseer, candidate_c, pvd_c, response_c.clone()).await; + // Introduce candidates. + introduce_candidate(&mut virtual_overseer, candidate_a1, pvd_a1).await; + introduce_candidate(&mut virtual_overseer, candidate_a2, pvd_a2).await; + introduce_candidate(&mut virtual_overseer, candidate_b, pvd_b).await; + introduce_candidate(&mut virtual_overseer, candidate_c, pvd_c).await; // Deactivate leaf A. deactivate_leaf(&mut virtual_overseer, leaf_a.hash).await; @@ -772,7 +755,8 @@ fn check_candidate_on_multiple_forks() { // Candidate on leaf A. let (candidate_a, pvd_a) = make_candidate( - &leaf_a, + leaf_a.hash, + leaf_a.number, 1.into(), HeadData(vec![1, 2, 3]), HeadData(vec![1]), @@ -783,7 +767,8 @@ fn check_candidate_on_multiple_forks() { // Candidate on leaf B. let (candidate_b, pvd_b) = make_candidate( - &leaf_b, + leaf_b.hash, + leaf_b.number, 1.into(), HeadData(vec![3, 4, 5]), HeadData(vec![1]), @@ -794,7 +779,8 @@ fn check_candidate_on_multiple_forks() { // Candidate on leaf C. let (candidate_c, pvd_c) = make_candidate( - &leaf_c, + leaf_c.hash, + leaf_c.number, 1.into(), HeadData(vec![5, 6, 7]), HeadData(vec![1]), @@ -803,28 +789,10 @@ fn check_candidate_on_multiple_forks() { let candidate_hash_c = candidate_c.hash(); let response_c = vec![(leaf_c.hash, vec![0])]; - // Second candidate on all three leaves. - second_candidate( - &mut virtual_overseer, - candidate_a.clone(), - pvd_a.clone(), - response_a.clone(), - ) - .await; - second_candidate( - &mut virtual_overseer, - candidate_b.clone(), - pvd_b.clone(), - response_b.clone(), - ) - .await; - second_candidate( - &mut virtual_overseer, - candidate_c.clone(), - pvd_c.clone(), - response_c.clone(), - ) - .await; + // Introduce candidates on all three leaves. + introduce_candidate(&mut virtual_overseer, candidate_a.clone(), pvd_a).await; + introduce_candidate(&mut virtual_overseer, candidate_b.clone(), pvd_b).await; + introduce_candidate(&mut virtual_overseer, candidate_c.clone(), pvd_c).await; // Check candidate tree membership. get_membership(&mut virtual_overseer, 1.into(), candidate_hash_a, response_a).await; @@ -861,18 +829,19 @@ fn check_backable_query() { // Candidate A let (candidate_a, pvd_a) = make_candidate( - &leaf_a, + leaf_a.hash, + leaf_a.number, 1.into(), HeadData(vec![1, 2, 3]), HeadData(vec![1]), test_state.validation_code_hash, ); let candidate_hash_a = candidate_a.hash(); - let response_a = vec![(leaf_a.hash, vec![0])]; // Candidate B let (mut candidate_b, pvd_b) = make_candidate( - &leaf_a, + leaf_a.hash, + leaf_a.number, 1.into(), HeadData(vec![1]), HeadData(vec![2]), @@ -881,24 +850,25 @@ fn check_backable_query() { // Set a field to make this candidate unique. candidate_b.descriptor.para_head = Hash::from_low_u64_le(1000); let candidate_hash_b = candidate_b.hash(); - let response_b = vec![(leaf_a.hash, vec![1])]; - // Second candidates. - second_candidate( - &mut virtual_overseer, - candidate_a.clone(), - pvd_a.clone(), - response_a.clone(), - ) - .await; - second_candidate( + // Introduce candidates. + introduce_candidate(&mut virtual_overseer, candidate_a.clone(), pvd_a).await; + introduce_candidate(&mut virtual_overseer, candidate_b.clone(), pvd_b).await; + + // Should not get any backable candidates. + get_backable_candidate( &mut virtual_overseer, - candidate_b.clone(), - pvd_b.clone(), - response_b.clone(), + &leaf_a, + 1.into(), + vec![candidate_hash_a], + None, ) .await; + // Second candidates. + second_candidate(&mut virtual_overseer, candidate_a.clone()).await; + second_candidate(&mut virtual_overseer, candidate_b.clone()).await; + // Should not get any backable candidates. get_backable_candidate( &mut virtual_overseer, @@ -953,7 +923,7 @@ fn check_backable_query() { // Test depth query. #[test] -fn check_depth_query() { +fn check_hypothetical_frontier_query() { let test_state = TestState::default(); let view = test_harness(|mut virtual_overseer| async move { // Leaf A @@ -971,36 +941,36 @@ fn check_depth_query() { // Candidate A. let (candidate_a, pvd_a) = make_candidate( - &leaf_a, + leaf_a.hash, + leaf_a.number, 1.into(), HeadData(vec![1, 2, 3]), HeadData(vec![1]), test_state.validation_code_hash, ); let candidate_hash_a = candidate_a.hash(); - let response_a = vec![(leaf_a.hash, vec![0])]; // Candidate B. let (candidate_b, pvd_b) = make_candidate( - &leaf_a, + leaf_a.hash, + leaf_a.number, 1.into(), HeadData(vec![1]), HeadData(vec![2]), test_state.validation_code_hash, ); let candidate_hash_b = candidate_b.hash(); - let response_b = vec![(leaf_a.hash, vec![1])]; // Candidate C. let (candidate_c, pvd_c) = make_candidate( - &leaf_a, + leaf_a.hash, + leaf_a.number, 1.into(), HeadData(vec![2]), HeadData(vec![3]), test_state.validation_code_hash, ); let candidate_hash_c = candidate_c.hash(); - let response_c = vec![(leaf_a.hash, vec![2])]; // Get hypothetical frontier of candidate A before adding it. get_hypothetical_frontier( @@ -1009,19 +979,25 @@ fn check_depth_query() { candidate_a.clone(), pvd_a.clone(), leaf_a.hash, + false, vec![0], ) .await; - - // Add candidate A. - second_candidate( + // Should work with `backed_in_path_only: true`, too. + get_hypothetical_frontier( &mut virtual_overseer, + candidate_hash_a, candidate_a.clone(), pvd_a.clone(), - response_a.clone(), + leaf_a.hash, + true, + vec![0], ) .await; + // Add candidate A. + introduce_candidate(&mut virtual_overseer, candidate_a.clone(), pvd_a.clone()).await; + // Get frontier of candidate A after adding it. get_hypothetical_frontier( &mut virtual_overseer, @@ -1029,6 +1005,7 @@ fn check_depth_query() { candidate_a.clone(), pvd_a.clone(), leaf_a.hash, + false, vec![0], ) .await; @@ -1040,18 +1017,13 @@ fn check_depth_query() { candidate_b.clone(), pvd_b.clone(), leaf_a.hash, + false, vec![1], ) .await; // Add candidate B. - second_candidate( - &mut virtual_overseer, - candidate_b.clone(), - pvd_b.clone(), - response_b.clone(), - ) - .await; + introduce_candidate(&mut virtual_overseer, candidate_b.clone(), pvd_b.clone()).await; // Get frontier of candidate B after adding it. get_hypothetical_frontier( @@ -1060,6 +1032,7 @@ fn check_depth_query() { candidate_b, pvd_b.clone(), leaf_a.hash, + false, vec![1], ) .await; @@ -1071,29 +1044,47 @@ fn check_depth_query() { candidate_c.clone(), pvd_c.clone(), leaf_a.hash, + false, vec![2], ) .await; - - // Add candidate C. - second_candidate( + // Should be empty with `backed_in_path_only` because we haven't backed anything. + get_hypothetical_frontier( &mut virtual_overseer, + candidate_hash_c, candidate_c.clone(), pvd_c.clone(), - response_c.clone(), + leaf_a.hash, + true, + vec![], ) .await; + // Add candidate C. + introduce_candidate(&mut virtual_overseer, candidate_c.clone(), pvd_c.clone()).await; + // Get frontier of candidate C after adding it. get_hypothetical_frontier( &mut virtual_overseer, candidate_hash_c, - candidate_c, + candidate_c.clone(), pvd_c.clone(), leaf_a.hash, + false, vec![2], ) .await; + // Should be empty with `backed_in_path_only` because we haven't backed anything. + get_hypothetical_frontier( + &mut virtual_overseer, + candidate_hash_c, + candidate_c.clone(), + pvd_c.clone(), + leaf_a.hash, + true, + vec![], + ) + .await; virtual_overseer }); @@ -1121,33 +1112,33 @@ fn check_pvd_query() { // Candidate A. let (candidate_a, pvd_a) = make_candidate( - &leaf_a, + leaf_a.hash, + leaf_a.number, 1.into(), HeadData(vec![1, 2, 3]), HeadData(vec![1]), test_state.validation_code_hash, ); - let response_a = vec![(leaf_a.hash, vec![0])]; // Candidate B. let (candidate_b, pvd_b) = make_candidate( - &leaf_a, + leaf_a.hash, + leaf_a.number, 1.into(), HeadData(vec![1]), HeadData(vec![2]), test_state.validation_code_hash, ); - let response_b = vec![(leaf_a.hash, vec![1])]; // Candidate C. let (candidate_c, pvd_c) = make_candidate( - &leaf_a, + leaf_a.hash, + leaf_a.number, 1.into(), HeadData(vec![2]), HeadData(vec![3]), test_state.validation_code_hash, ); - let response_c = vec![(leaf_a.hash, vec![2])]; // Get pvd of candidate A before adding it. get_pvd( @@ -1160,13 +1151,7 @@ fn check_pvd_query() { .await; // Add candidate A. - second_candidate( - &mut virtual_overseer, - candidate_a.clone(), - pvd_a.clone(), - response_a.clone(), - ) - .await; + introduce_candidate(&mut virtual_overseer, candidate_a.clone(), pvd_a.clone()).await; back_candidate(&mut virtual_overseer, &candidate_a, candidate_a.hash()).await; // Get pvd of candidate A after adding it. @@ -1190,8 +1175,7 @@ fn check_pvd_query() { .await; // Add candidate B. - second_candidate(&mut virtual_overseer, candidate_b, pvd_b.clone(), response_b.clone()) - .await; + introduce_candidate(&mut virtual_overseer, candidate_b, pvd_b.clone()).await; // Get pvd of candidate B after adding it. get_pvd( @@ -1214,8 +1198,7 @@ fn check_pvd_query() { .await; // Add candidate C. - second_candidate(&mut virtual_overseer, candidate_c, pvd_c.clone(), response_c.clone()) - .await; + introduce_candidate(&mut virtual_overseer, candidate_c, pvd_c.clone()).await; // Get pvd of candidate C after adding it. get_pvd( diff --git a/node/network/protocol/Cargo.toml b/node/network/protocol/Cargo.toml index e170d76a713b..8d840413dcc3 100644 --- a/node/network/protocol/Cargo.toml +++ b/node/network/protocol/Cargo.toml @@ -22,6 +22,7 @@ fatality = "0.0.6" rand = "0.8" derive_more = "0.99" gum = { package = "tracing-gum", path = "../../gum" } +bitvec = "1" [dev-dependencies] rand_chacha = "0.3.1" diff --git a/node/network/protocol/src/lib.rs b/node/network/protocol/src/lib.rs index 25c16de5ccdf..9f50bf29406e 100644 --- a/node/network/protocol/src/lib.rs +++ b/node/network/protocol/src/lib.rs @@ -589,11 +589,12 @@ pub mod v1 { /// vstaging network protocol types. pub mod vstaging { + use bitvec::{order::Lsb0, slice::BitSlice, vec::BitVec}; use parity_scale_codec::{Decode, Encode}; use polkadot_primitives::vstaging::{ - CandidateHash, CandidateIndex, CollatorId, CollatorSignature, Hash, Id as ParaId, - UncheckedSignedAvailabilityBitfield, + CandidateHash, CandidateIndex, CollatorId, CollatorSignature, GroupIndex, Hash, + Id as ParaId, UncheckedSignedAvailabilityBitfield, UncheckedSignedStatement, }; use polkadot_node_primitives::{ @@ -609,17 +610,155 @@ pub mod vstaging { Bitfield(Hash, UncheckedSignedAvailabilityBitfield), } + /// Bitfields indicating the statements that are known or undesired + /// about a candidate. + #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] + pub struct StatementFilter { + /// Seconded statements. '1' is known or undesired. + pub seconded_in_group: BitVec, + /// Valid statements. '1' is known or undesired. + pub validated_in_group: BitVec, + } + + impl StatementFilter { + /// Create a new blank filter with the given group size. + pub fn blank(group_size: usize) -> Self { + StatementFilter { + seconded_in_group: BitVec::repeat(false, group_size), + validated_in_group: BitVec::repeat(false, group_size), + } + } + + /// Create a new full filter with the given group size. + pub fn full(group_size: usize) -> Self { + StatementFilter { + seconded_in_group: BitVec::repeat(true, group_size), + validated_in_group: BitVec::repeat(true, group_size), + } + } + + /// Whether the filter has a specific expected length, consistent across both + /// bitfields. + pub fn has_len(&self, len: usize) -> bool { + self.seconded_in_group.len() == len && self.validated_in_group.len() == len + } + + /// Determine the number of backing validators in the statement filter. + pub fn backing_validators(&self) -> usize { + self.seconded_in_group + .iter() + .by_vals() + .zip(self.validated_in_group.iter().by_vals()) + .filter(|&(s, v)| s || v) // no double-counting + .count() + } + + /// Whether the statement filter has at least one seconded statement. + pub fn has_seconded(&self) -> bool { + self.seconded_in_group.iter().by_vals().any(|x| x) + } + + /// Mask out `Seconded` statements in `self` according to the provided + /// bitvec. Bits appearing in `mask` will not appear in `self` afterwards. + pub fn mask_seconded(&mut self, mask: &BitSlice) { + for (mut x, mask) in self + .seconded_in_group + .iter_mut() + .zip(mask.iter().by_vals().chain(std::iter::repeat(false))) + { + // (x, mask) => x + // (true, true) => false + // (true, false) => true + // (false, true) => false + // (false, false) => false + *x = *x && !mask; + } + } + + /// Mask out `Valid` statements in `self` according to the provided + /// bitvec. Bits appearing in `mask` will not appear in `self` afterwards. + pub fn mask_valid(&mut self, mask: &BitSlice) { + for (mut x, mask) in self + .validated_in_group + .iter_mut() + .zip(mask.iter().by_vals().chain(std::iter::repeat(false))) + { + // (x, mask) => x + // (true, true) => false + // (true, false) => true + // (false, true) => false + // (false, false) => false + *x = *x && !mask; + } + } + } + + /// A manifest of a known backed candidate, along with a description + /// of the statements backing it. + #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] + pub struct BackedCandidateManifest { + /// The relay-parent of the candidate. + pub relay_parent: Hash, + /// The hash of the candidate. + pub candidate_hash: CandidateHash, + /// The group index backing the candidate at the relay-parent. + pub group_index: GroupIndex, + /// The para ID of the candidate. It is illegal for this to + /// be a para ID which is not assigned to the group indicated + /// in this manifest. + pub para_id: ParaId, + /// The head-data corresponding to the candidate. + pub parent_head_data_hash: Hash, + /// A statement filter which indicates which validators in the + /// para's group at the relay-parent have validated this candidate + /// and issued statements about it, to the advertiser's knowledge. + /// + /// This MUST have exactly the minimum amount of bytes + /// necessary to represent the number of validators in the assigned + /// backing group as-of the relay-parent. + pub statement_knowledge: StatementFilter, + } + + /// An acknowledgement of a backed candidate being known. + #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] + pub struct BackedCandidateAcknowledgement { + /// The hash of the candidate. + pub candidate_hash: CandidateHash, + /// A statement filter which indicates which validators in the + /// para's group at the relay-parent have validated this candidate + /// and issued statements about it, to the advertiser's knowledge. + /// + /// This MUST have exactly the minimum amount of bytes + /// necessary to represent the number of validators in the assigned + /// backing group as-of the relay-parent. + pub statement_knowledge: StatementFilter, + } + /// Network messages used by the statement distribution subsystem. #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] pub enum StatementDistributionMessage { - // TODO [now]: notifications for v2 + /// A notification of a signed statement in compact form. + #[codec(index = 0)] + Statement(Hash, UncheckedSignedStatement), + + /// A notification of a backed candidate being known by the + /// sending node, for the purpose of being requested by the receiving node + /// if needed. + #[codec(index = 1)] + BackedCandidateManifest(BackedCandidateManifest), + + /// A notification of a backed candidate being known by the sending node, + /// for the purpose of informing a receiving node which already has the candidate. + #[codec(index = 2)] + BackedCandidateKnown(BackedCandidateAcknowledgement), + /// All messages for V1 for compatibility with the statement distribution /// protocol, for relay-parents that don't support asynchronous backing. /// /// These are illegal to send to V1 peers, and illegal to send concerning relay-parents /// which support asynchronous backing. This backwards compatibility should be /// considered immediately deprecated and can be removed once the node software - /// is not required to support asynchronous backing anymore. + /// is not required to support logic from before asynchronous backing anymore. #[codec(index = 255)] V1Compatibility(crate::v1::StatementDistributionMessage), } diff --git a/node/network/protocol/src/request_response/mod.rs b/node/network/protocol/src/request_response/mod.rs index 21d3c5bed478..83e2ac12df96 100644 --- a/node/network/protocol/src/request_response/mod.rs +++ b/node/network/protocol/src/request_response/mod.rs @@ -52,10 +52,10 @@ pub use outgoing::{OutgoingRequest, OutgoingResult, Recipient, Requests, Respons ///// Multiplexer for incoming requests. // pub mod multiplexer; -/// Actual versioned requests and responses, that are sent over the wire. +/// Actual versioned requests and responses that are sent over the wire. pub mod v1; -/// Staging requests to be sent over the wire. +/// Actual versioned requests and responses that are sent over the wire. pub mod vstaging; /// A protocol per subsystem seems to make the most sense, this way we don't need any dispatching @@ -76,6 +76,10 @@ pub enum Protocol { StatementFetchingV1, /// Sending of dispute statements with application level confirmations. DisputeSendingV1, + + /// Protocol for requesting candidates with attestations in statement distribution + /// when async backing is enabled. + AttestedCandidateVStaging, } /// Minimum bandwidth we expect for validators - 500Mbit/s is the recommendation, so approximately @@ -107,12 +111,30 @@ const POV_REQUEST_TIMEOUT_CONNECTED: Duration = Duration::from_millis(1200); /// fit statement distribution within a block of 6 seconds.) const STATEMENTS_TIMEOUT: Duration = Duration::from_secs(1); +/// We want attested candidate requests to time out relatively fast, +/// because slow requests will bottleneck the backing system. Ideally, we'd have +/// an adaptive timeout based on the candidate size, because there will be a lot of variance +/// in candidate sizes: candidates with no code and no messages vs candidates with code +/// and messages. +/// +/// We supply leniency because there are often large candidates and asynchronous +/// backing allows them to be included over a longer window of time. Exponential back-off +/// up to a maximum of 10 seconds would be ideal, but isn't supported by the +/// infrastructure here yet: see https://github.com/paritytech/polkadot/issues/6009 +const ATTESTED_CANDIDATE_TIMEOUT: Duration = Duration::from_millis(2500); + /// We don't want a slow peer to slow down all the others, at the same time we want to get out the /// data quickly in full to at least some peers (as this will reduce load on us as they then can /// start serving the data). So this value is a trade-off. 3 seems to be sensible. So we would need /// to have 3 slow nodes connected, to delay transfer for others by `STATEMENTS_TIMEOUT`. pub const MAX_PARALLEL_STATEMENT_REQUESTS: u32 = 3; +/// We don't want a slow peer to slow down all the others, at the same time we want to get out the +/// data quickly in full to at least some peers (as this will reduce load on us as they then can +/// start serving the data). So this value is a tradeoff. 5 seems to be sensible. So we would need +/// to have 5 slow nodes connected, to delay transfer for others by `ATTESTED_CANDIDATE_TIMEOUT`. +pub const MAX_PARALLEL_ATTESTED_CANDIDATE_REQUESTS: u32 = 5; + /// Response size limit for responses of POV like data. /// /// This is larger than `MAX_POV_SIZE` to account for protocol overhead and for additional data in @@ -126,6 +148,12 @@ const POV_RESPONSE_SIZE: u64 = MAX_POV_SIZE as u64 + 10_000; /// This is `MAX_CODE_SIZE` plus some additional space for protocol overhead. const STATEMENT_RESPONSE_SIZE: u64 = MAX_CODE_SIZE as u64 + 10_000; +/// Maximum response sizes for `AttestedCandidateVStaging`. +/// +/// This is `MAX_CODE_SIZE` plus some additional space for protocol overhead and +/// additional backing statements. +const ATTESTED_CANDIDATE_RESPONSE_SIZE: u64 = MAX_CODE_SIZE as u64 + 100_000; + /// We can have relative large timeouts here, there is no value of hitting a /// timeout as we want to get statements through to each node in any case. pub const DISPUTE_REQUEST_TIMEOUT: Duration = Duration::from_secs(12); @@ -227,6 +255,14 @@ impl Protocol { request_timeout: DISPUTE_REQUEST_TIMEOUT, inbound_queue: tx, }, + Protocol::AttestedCandidateVStaging => RequestResponseConfig { + name, + fallback_names, + max_request_size: 1_000, + max_response_size: ATTESTED_CANDIDATE_RESPONSE_SIZE, + request_timeout: ATTESTED_CANDIDATE_TIMEOUT, + inbound_queue: tx, + }, } } @@ -271,24 +307,46 @@ impl Protocol { // average, so something in the ballpark of 100 should be fine. Nodes will retry on // failure, so having a good value here is mostly about performance tuning. Protocol::DisputeSendingV1 => 100, + + Protocol::AttestedCandidateVStaging => { + // We assume we can utilize up to 70% of the available bandwidth for statements. + // This is just a guess/estimate, with the following considerations: If we are + // faster than that, queue size will stay low anyway, even if not - requesters will + // get an immediate error, but if we are slower, requesters will run in a timeout - + // wasting precious time. + let available_bandwidth = 7 * MIN_BANDWIDTH_BYTES / 10; + let size = u64::saturating_sub( + ATTESTED_CANDIDATE_TIMEOUT.as_millis() as u64 * available_bandwidth / + (1000 * MAX_CODE_SIZE as u64), + MAX_PARALLEL_ATTESTED_CANDIDATE_REQUESTS as u64, + ); + debug_assert!( + size > 0, + "We should have a channel size greater zero, otherwise we won't accept any requests." + ); + size as usize + }, } } /// Fallback protocol names of this protocol, as understood by substrate networking. fn get_fallback_names(self) -> Vec { - std::iter::once(self.get_legacy_name().into()).collect() + self.get_legacy_name().into_iter().map(Into::into).collect() } - /// Legacy protocol name associated with each peer set. - const fn get_legacy_name(self) -> &'static str { + /// Legacy protocol name associated with each peer set, if any. + const fn get_legacy_name(self) -> Option<&'static str> { match self { - Protocol::ChunkFetchingV1 => "/polkadot/req_chunk/1", - Protocol::CollationFetchingV1 => "/polkadot/req_collation/1", - Protocol::CollationFetchingVStaging => "/polkadot/req_collation/2", - Protocol::PoVFetchingV1 => "/polkadot/req_pov/1", - Protocol::AvailableDataFetchingV1 => "/polkadot/req_available_data/1", - Protocol::StatementFetchingV1 => "/polkadot/req_statement/1", - Protocol::DisputeSendingV1 => "/polkadot/send_dispute/1", + Protocol::ChunkFetchingV1 => Some("/polkadot/req_chunk/1"), + Protocol::CollationFetchingV1 => Some("/polkadot/req_collation/1"), + Protocol::PoVFetchingV1 => Some("/polkadot/req_pov/1"), + Protocol::AvailableDataFetchingV1 => Some("/polkadot/req_available_data/1"), + Protocol::StatementFetchingV1 => Some("/polkadot/req_statement/1"), + Protocol::DisputeSendingV1 => Some("/polkadot/send_dispute/1"), + + // Introduced after legacy names became legacy. + Protocol::AttestedCandidateVStaging => None, + Protocol::CollationFetchingVStaging => None, } } } @@ -340,11 +398,13 @@ impl ReqProtocolNames { let short_name = match protocol { Protocol::ChunkFetchingV1 => "/req_chunk/1", Protocol::CollationFetchingV1 => "/req_collation/1", - Protocol::CollationFetchingVStaging => "/req_collation/2", Protocol::PoVFetchingV1 => "/req_pov/1", Protocol::AvailableDataFetchingV1 => "/req_available_data/1", Protocol::StatementFetchingV1 => "/req_statement/1", Protocol::DisputeSendingV1 => "/send_dispute/1", + + Protocol::CollationFetchingVStaging => "/req_collation/2", + Protocol::AttestedCandidateVStaging => "/req_attested_candidate/2", }; format!("{}{}", prefix, short_name).into() diff --git a/node/network/protocol/src/request_response/outgoing.rs b/node/network/protocol/src/request_response/outgoing.rs index 79e9f37b60fe..e5aa117ff654 100644 --- a/node/network/protocol/src/request_response/outgoing.rs +++ b/node/network/protocol/src/request_response/outgoing.rs @@ -41,6 +41,8 @@ pub enum Requests { /// Requests for notifying about an ongoing dispute. DisputeSendingV1(OutgoingRequest), + /// Request a candidate and attestations. + AttestedCandidateVStaging(OutgoingRequest), /// Fetch a collation from a collator which previously announced it. /// Compared to V1 it requires specifying which candidate is requested by its hash. CollationFetchingVStaging(OutgoingRequest), @@ -57,6 +59,7 @@ impl Requests { Self::AvailableDataFetchingV1(_) => Protocol::AvailableDataFetchingV1, Self::StatementFetchingV1(_) => Protocol::StatementFetchingV1, Self::DisputeSendingV1(_) => Protocol::DisputeSendingV1, + Self::AttestedCandidateVStaging(_) => Protocol::AttestedCandidateVStaging, } } @@ -76,6 +79,7 @@ impl Requests { Self::AvailableDataFetchingV1(r) => r.encode_request(), Self::StatementFetchingV1(r) => r.encode_request(), Self::DisputeSendingV1(r) => r.encode_request(), + Self::AttestedCandidateVStaging(r) => r.encode_request(), } } } diff --git a/node/network/protocol/src/request_response/vstaging.rs b/node/network/protocol/src/request_response/vstaging.rs index 0b8d223e3aee..f84de9505534 100644 --- a/node/network/protocol/src/request_response/vstaging.rs +++ b/node/network/protocol/src/request_response/vstaging.rs @@ -14,10 +14,50 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . +//! Requests and responses as sent over the wire for the individual protocols. + use parity_scale_codec::{Decode, Encode}; -use polkadot_primitives::v2::{CandidateHash, Hash, Id as ParaId}; + +use polkadot_primitives::vstaging::{ + CandidateHash, CommittedCandidateReceipt, Hash, Id as ParaId, PersistedValidationData, + UncheckedSignedStatement, +}; use super::{IsRequest, Protocol}; +use crate::vstaging::StatementFilter; + +/// Request a candidate with statements. +#[derive(Debug, Clone, Encode, Decode)] +pub struct AttestedCandidateRequest { + /// Hash of the candidate we want to request. + pub candidate_hash: CandidateHash, + /// Statement filter with 'OR' semantics, indicating which validators + /// not to send statements for. + /// + /// The filter must have exactly the minimum size required to + /// fit all validators from the backing group. + /// + /// The response may not contain any statements masked out by this mask. + pub mask: StatementFilter, +} + +/// Response to an `AttestedCandidateRequest`. +#[derive(Debug, Clone, Encode, Decode)] +pub struct AttestedCandidateResponse { + /// The candidate receipt, with commitments. + pub candidate_receipt: CommittedCandidateReceipt, + /// The [`PersistedValidationData`] corresponding to the candidate. + pub persisted_validation_data: PersistedValidationData, + /// All known statements about the candidate, in compact form, + /// omitting `Seconded` statements which were intended to be masked + /// out. + pub statements: Vec, +} + +impl IsRequest for AttestedCandidateRequest { + type Response = AttestedCandidateResponse; + const PROTOCOL: Protocol = Protocol::AttestedCandidateVStaging; +} /// Responses as sent by collators. pub type CollationFetchingResponse = super::v1::CollationFetchingResponse; diff --git a/node/network/statement-distribution/Cargo.toml b/node/network/statement-distribution/Cargo.toml index 5dcb2a75d3f5..dc5d9b15d7b3 100644 --- a/node/network/statement-distribution/Cargo.toml +++ b/node/network/statement-distribution/Cargo.toml @@ -20,10 +20,11 @@ indexmap = "1.9.1" parity-scale-codec = { version = "3.3.0", default-features = false, features = ["derive"] } thiserror = "1.0.31" fatality = "0.0.6" +bitvec = "1" [dev-dependencies] -polkadot-node-subsystem-test-helpers = { path = "../../subsystem-test-helpers" } assert_matches = "1.4.0" +polkadot-node-subsystem-test-helpers = { path = "../../subsystem-test-helpers" } sp-authority-discovery = { git = "https://github.com/paritytech/substrate", branch = "master" } sp-keyring = { git = "https://github.com/paritytech/substrate", branch = "master" } sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" } @@ -34,3 +35,5 @@ sc-keystore = { git = "https://github.com/paritytech/substrate", branch = "maste sc-network = { git = "https://github.com/paritytech/substrate", branch = "master" } futures-timer = "3.0.2" polkadot-primitives-test-helpers = { path = "../../../primitives/test-helpers" } +rand_chacha = "0.3" +polkadot-node-subsystem-types = { path = "../../subsystem-types" } diff --git a/node/network/statement-distribution/src/error.rs b/node/network/statement-distribution/src/error.rs index 70dadc43ed8c..c49b208adbfb 100644 --- a/node/network/statement-distribution/src/error.rs +++ b/node/network/statement-distribution/src/error.rs @@ -19,8 +19,10 @@ use polkadot_node_network_protocol::PeerId; use polkadot_node_subsystem::{RuntimeApiError, SubsystemError}; -use polkadot_node_subsystem_util::runtime; -use polkadot_primitives::{CandidateHash, Hash, Id as ParaId}; +use polkadot_node_subsystem_util::{ + backing_implicit_view::FetchError as ImplicitViewFetchError, runtime, +}; +use polkadot_primitives::v2::{CandidateHash, Hash, Id as ParaId}; use futures::channel::oneshot; @@ -64,6 +66,21 @@ pub enum Error { #[error("Fetching persisted validation data for para {0:?}, {1:?}")] FetchPersistedValidationData(ParaId, RuntimeApiError), + #[error("Fetching session index failed {0:?}")] + FetchSessionIndex(RuntimeApiError), + + #[error("Fetching session info failed {0:?}")] + FetchSessionInfo(RuntimeApiError), + + #[error("Fetching availability cores failed {0:?}")] + FetchAvailabilityCores(RuntimeApiError), + + #[error("Fetching validator groups failed {0:?}")] + FetchValidatorGroups(RuntimeApiError), + + #[error("Attempted to share statement when not a validator or not assigned")] + InvalidShare, + #[error("Relay parent could not be found in active heads")] NoSuchHead(Hash), @@ -84,6 +101,10 @@ pub enum Error { // Responder no longer waits for our data. (Should not happen right now.) #[error("Oneshot `GetData` channel closed")] ResponderGetDataCanceled, + + // Failed to activate leaf due to a fetch error. + #[error("Implicit view failure while activating leaf")] + ActivateLeafFailure(ImplicitViewFetchError), } /// Utility for eating top level errors and log them. diff --git a/node/network/statement-distribution/src/legacy_v1/mod.rs b/node/network/statement-distribution/src/legacy_v1/mod.rs index 752385e1aea6..f5c174f28184 100644 --- a/node/network/statement-distribution/src/legacy_v1/mod.rs +++ b/node/network/statement-distribution/src/legacy_v1/mod.rs @@ -426,7 +426,7 @@ impl PeerRelayParentKnowledge { } } -struct PeerData { +pub struct PeerData { view: View, protocol_version: ValidationVersion, view_knowledge: HashMap, @@ -1439,6 +1439,16 @@ async fn handle_incoming_message<'a, Context>( Versioned::VStaging(protocol_vstaging::StatementDistributionMessage::V1Compatibility( m, )) => m, + Versioned::VStaging(_) => { + // The higher-level subsystem code is supposed to filter out + // all non v1 messages. + gum::debug!( + target: LOG_TARGET, + "Legacy statement-distribution code received unintended v2 message" + ); + + return None + }, }; let relay_parent = message.get_relay_parent(); diff --git a/node/network/statement-distribution/src/legacy_v1/responder.rs b/node/network/statement-distribution/src/legacy_v1/responder.rs index 8db38385e581..e9e45f56fe68 100644 --- a/node/network/statement-distribution/src/legacy_v1/responder.rs +++ b/node/network/statement-distribution/src/legacy_v1/responder.rs @@ -48,8 +48,8 @@ pub enum ResponderMessage { /// A fetching task, taking care of fetching large statements via request/response. /// -/// A fetch task does not know about a particular `Statement` instead it just tries fetching a -/// `CommittedCandidateReceipt` from peers, whether this can be used to re-assemble one ore +/// A fetch task does not know about a particular `Statement`, instead it just tries fetching a +/// `CommittedCandidateReceipt` from peers, whether this can be used to re-assemble one or /// many `SignedFullStatement`s needs to be verified by the caller. pub async fn respond( mut receiver: IncomingRequestReceiver, diff --git a/node/network/statement-distribution/src/legacy_v1/tests.rs b/node/network/statement-distribution/src/legacy_v1/tests.rs index f768a7370363..0764040921cd 100644 --- a/node/network/statement-distribution/src/legacy_v1/tests.rs +++ b/node/network/statement-distribution/src/legacy_v1/tests.rs @@ -762,11 +762,13 @@ fn receiving_from_one_sends_to_another_and_to_candidate_backing() { let req_protocol_names = ReqProtocolNames::new(&GENESIS_HASH, None); let (statement_req_receiver, _) = IncomingRequest::get_config_receiver(&req_protocol_names); + let (candidate_req_receiver, _) = IncomingRequest::get_config_receiver(&req_protocol_names); let bg = async move { let s = StatementDistributionSubsystem::new( Arc::new(LocalKeystore::in_memory()), statement_req_receiver, + candidate_req_receiver, Default::default(), AlwaysZeroRng, ); @@ -786,6 +788,17 @@ fn receiving_from_one_sends_to_another_and_to_candidate_backing() { ))) .await; + assert_matches!( + handle.recv().await, + AllMessages::RuntimeApi( + RuntimeApiMessage::Request(r, RuntimeApiRequest::StagingAsyncBackingParameters(tx)) + ) + if r == hash_a + => { + let _ = tx.send(Err(polkadot_node_subsystem::RuntimeApiError::NotSupported{runtime_api_name: "async_backing_parameters"})); + } + ); + assert_matches!( handle.recv().await, AllMessages::RuntimeApi( @@ -984,11 +997,13 @@ fn receiving_large_statement_from_one_sends_to_another_and_to_candidate_backing( let req_protocol_names = ReqProtocolNames::new(&GENESIS_HASH, None); let (statement_req_receiver, mut req_cfg) = IncomingRequest::get_config_receiver(&req_protocol_names); + let (candidate_req_receiver, _) = IncomingRequest::get_config_receiver(&req_protocol_names); let bg = async move { let s = StatementDistributionSubsystem::new( make_ferdie_keystore(), statement_req_receiver, + candidate_req_receiver, Default::default(), AlwaysZeroRng, ); @@ -1008,6 +1023,17 @@ fn receiving_large_statement_from_one_sends_to_another_and_to_candidate_backing( ))) .await; + assert_matches!( + handle.recv().await, + AllMessages::RuntimeApi( + RuntimeApiMessage::Request(r, RuntimeApiRequest::StagingAsyncBackingParameters(tx)) + ) + if r == hash_a + => { + let _ = tx.send(Err(polkadot_node_subsystem::RuntimeApiError::NotSupported{runtime_api_name: "async_backing_parameters"})); + } + ); + assert_matches!( handle.recv().await, AllMessages::RuntimeApi( @@ -1512,11 +1538,13 @@ fn share_prioritizes_backing_group() { let req_protocol_names = ReqProtocolNames::new(&GENESIS_HASH, None); let (statement_req_receiver, mut req_cfg) = IncomingRequest::get_config_receiver(&req_protocol_names); + let (candidate_req_receiver, _) = IncomingRequest::get_config_receiver(&req_protocol_names); let bg = async move { let s = StatementDistributionSubsystem::new( make_ferdie_keystore(), statement_req_receiver, + candidate_req_receiver, Default::default(), AlwaysZeroRng, ); @@ -1536,6 +1564,17 @@ fn share_prioritizes_backing_group() { ))) .await; + assert_matches!( + handle.recv().await, + AllMessages::RuntimeApi( + RuntimeApiMessage::Request(r, RuntimeApiRequest::StagingAsyncBackingParameters(tx)) + ) + if r == hash_a + => { + let _ = tx.send(Err(polkadot_node_subsystem::RuntimeApiError::NotSupported{runtime_api_name: "async_backing_parameters"})); + } + ); + assert_matches!( handle.recv().await, AllMessages::RuntimeApi( @@ -1817,10 +1856,12 @@ fn peer_cant_flood_with_large_statements() { let req_protocol_names = ReqProtocolNames::new(&GENESIS_HASH, None); let (statement_req_receiver, _) = IncomingRequest::get_config_receiver(&req_protocol_names); + let (candidate_req_receiver, _) = IncomingRequest::get_config_receiver(&req_protocol_names); let bg = async move { let s = StatementDistributionSubsystem::new( make_ferdie_keystore(), statement_req_receiver, + candidate_req_receiver, Default::default(), AlwaysZeroRng, ); @@ -1840,6 +1881,17 @@ fn peer_cant_flood_with_large_statements() { ))) .await; + assert_matches!( + handle.recv().await, + AllMessages::RuntimeApi( + RuntimeApiMessage::Request(r, RuntimeApiRequest::StagingAsyncBackingParameters(tx)) + ) + if r == hash_a + => { + let _ = tx.send(Err(polkadot_node_subsystem::RuntimeApiError::NotSupported{runtime_api_name: "async_backing_parameters"})); + } + ); + assert_matches!( handle.recv().await, AllMessages::RuntimeApi( @@ -2023,11 +2075,13 @@ fn handle_multiple_seconded_statements() { let req_protocol_names = ReqProtocolNames::new(&GENESIS_HASH, None); let (statement_req_receiver, _) = IncomingRequest::get_config_receiver(&req_protocol_names); + let (candidate_req_receiver, _) = IncomingRequest::get_config_receiver(&req_protocol_names); let virtual_overseer_fut = async move { let s = StatementDistributionSubsystem::new( Arc::new(LocalKeystore::in_memory()), statement_req_receiver, + candidate_req_receiver, Default::default(), AlwaysZeroRng, ); @@ -2047,6 +2101,17 @@ fn handle_multiple_seconded_statements() { ))) .await; + assert_matches!( + handle.recv().await, + AllMessages::RuntimeApi( + RuntimeApiMessage::Request(r, RuntimeApiRequest::StagingAsyncBackingParameters(tx)) + ) + if r == relay_parent_hash + => { + let _ = tx.send(Err(polkadot_node_subsystem::RuntimeApiError::NotSupported{runtime_api_name: "async_backing_parameters"})); + } + ); + assert_matches!( handle.recv().await, AllMessages::RuntimeApi( diff --git a/node/network/statement-distribution/src/lib.rs b/node/network/statement-distribution/src/lib.rs index c1ab9bd50821..11d765d7aa95 100644 --- a/node/network/statement-distribution/src/lib.rs +++ b/node/network/statement-distribution/src/lib.rs @@ -25,7 +25,9 @@ use error::{log_error, FatalResult}; use polkadot_node_network_protocol::{ - request_response::{v1 as request_v1, IncomingRequestReceiver}, + request_response::{ + v1 as request_v1, vstaging::AttestedCandidateRequest, IncomingRequestReceiver, + }, vstaging as protocol_vstaging, Versioned, }; use polkadot_node_primitives::StatementWithPVD; @@ -56,6 +58,8 @@ use legacy_v1::{ ResponderMessage as V1ResponderMessage, }; +mod vstaging; + const LOG_TARGET: &str = "parachain::statement-distribution"; /// The statement distribution subsystem. @@ -64,6 +68,8 @@ pub struct StatementDistributionSubsystem { keystore: SyncCryptoStorePtr, /// Receiver for incoming large statement requests. v1_req_receiver: Option>, + /// Receiver for incoming candidate requests. + req_receiver: Option>, /// Prometheus metrics metrics: Metrics, /// Pseudo-random generator for peers selection logic @@ -93,25 +99,41 @@ enum MuxedMessage { V1Requester(Option), /// Messages from spawned v1 (legacy) responder background task. V1Responder(Option), + /// Messages from candidate responder background task. + Responder(Option), + /// Messages from answered requests. + Response(vstaging::UnhandledResponse), } #[overseer::contextbounds(StatementDistribution, prefix = self::overseer)] impl MuxedMessage { async fn receive( ctx: &mut Context, + state: &mut vstaging::State, from_v1_requester: &mut mpsc::Receiver, from_v1_responder: &mut mpsc::Receiver, + from_responder: &mut mpsc::Receiver, ) -> MuxedMessage { // We are only fusing here to make `select` happy, in reality we will quit if one of those // streams end: let from_orchestra = ctx.recv().fuse(); let from_v1_requester = from_v1_requester.next(); let from_v1_responder = from_v1_responder.next(); - futures::pin_mut!(from_orchestra, from_v1_requester, from_v1_responder); + let from_responder = from_responder.next(); + let receive_response = vstaging::receive_response(state).fuse(); + futures::pin_mut!( + from_orchestra, + from_v1_requester, + from_v1_responder, + from_responder, + receive_response + ); futures::select! { msg = from_orchestra => MuxedMessage::Subsystem(msg.map_err(FatalError::SubsystemReceive)), msg = from_v1_requester => MuxedMessage::V1Requester(msg), msg = from_v1_responder => MuxedMessage::V1Responder(msg), + msg = from_responder => MuxedMessage::Responder(msg), + msg = receive_response => MuxedMessage::Response(msg), } } } @@ -122,14 +144,22 @@ impl StatementDistributionSubsystem { pub fn new( keystore: SyncCryptoStorePtr, v1_req_receiver: IncomingRequestReceiver, + req_receiver: IncomingRequestReceiver, metrics: Metrics, rng: R, ) -> Self { - Self { keystore, v1_req_receiver: Some(v1_req_receiver), metrics, rng } + Self { + keystore, + v1_req_receiver: Some(v1_req_receiver), + req_receiver: Some(req_receiver), + metrics, + rng, + } } async fn run(mut self, mut ctx: Context) -> std::result::Result<(), FatalError> { let mut legacy_v1_state = crate::legacy_v1::State::new(self.keystore.clone()); + let mut state = crate::vstaging::State::new(self.keystore.clone()); // Sender/Receiver for getting news from our statement fetching tasks. let (v1_req_sender, mut v1_req_receiver) = mpsc::channel(1); @@ -146,14 +176,34 @@ impl StatementDistributionSubsystem { ) .map_err(FatalError::SpawnTask)?; + // Sender/receiver for getting news from our candidate responder task. + let (res_sender, mut res_receiver) = mpsc::channel(1); + + ctx.spawn( + "candidate-responder", + vstaging::respond_task( + self.req_receiver.take().expect("Mandatory argument to new. qed"), + res_sender.clone(), + ) + .boxed(), + ) + .map_err(FatalError::SpawnTask)?; + loop { - let message = - MuxedMessage::receive(&mut ctx, &mut v1_req_receiver, &mut v1_res_receiver).await; + let message = MuxedMessage::receive( + &mut ctx, + &mut state, + &mut v1_req_receiver, + &mut v1_res_receiver, + &mut res_receiver, + ) + .await; match message { MuxedMessage::Subsystem(result) => { let result = self .handle_subsystem_message( &mut ctx, + &mut state, &mut legacy_v1_state, &v1_req_sender, result?, @@ -185,7 +235,18 @@ impl StatementDistributionSubsystem { .await; log_error(result.map_err(From::from), "handle_responder_message")?; }, + MuxedMessage::Responder(result) => { + vstaging::answer_request( + &mut state, + result.ok_or(FatalError::RequesterReceiverFinished)?, + ); + }, + MuxedMessage::Response(result) => { + vstaging::handle_response(&mut ctx, &mut state, result).await; + }, }; + + vstaging::dispatch_requests(&mut ctx, &mut state).await; } Ok(()) } @@ -193,6 +254,7 @@ impl StatementDistributionSubsystem { async fn handle_subsystem_message( &mut self, ctx: &mut Context, + state: &mut vstaging::State, legacy_v1_state: &mut legacy_v1::State, v1_req_sender: &mpsc::Sender, message: FromOrchestra, @@ -206,70 +268,112 @@ impl StatementDistributionSubsystem { })) => { let _timer = metrics.time_active_leaves_update(); - for deactivated in deactivated { - crate::legacy_v1::handle_deactivate_leaf(legacy_v1_state, deactivated); - } - - if let Some(activated) = activated { - // Legacy, activate only if no prospective parachains support. + // vstaging should handle activated first because of implicit view. + if let Some(ref activated) = activated { let mode = prospective_parachains_mode(ctx.sender(), activated.hash).await?; - if let ProspectiveParachainsMode::Disabled = mode { - crate::legacy_v1::handle_activated_leaf(ctx, legacy_v1_state, activated) - .await?; + if let ProspectiveParachainsMode::Enabled { .. } = mode { + vstaging::handle_active_leaves_update(ctx, state, activated, mode).await?; + } else if let ProspectiveParachainsMode::Disabled = mode { + for deactivated in &deactivated { + crate::legacy_v1::handle_deactivate_leaf(legacy_v1_state, *deactivated); + } + + crate::legacy_v1::handle_activated_leaf( + ctx, + legacy_v1_state, + activated.clone(), + ) + .await?; + } + } else { + for deactivated in &deactivated { + crate::legacy_v1::handle_deactivate_leaf(legacy_v1_state, *deactivated); } + vstaging::handle_deactivate_leaves(state, &deactivated); } }, FromOrchestra::Signal(OverseerSignal::BlockFinalized(..)) => { // do nothing }, FromOrchestra::Signal(OverseerSignal::Conclude) => return Ok(true), - FromOrchestra::Communication { msg } => - match msg { - StatementDistributionMessage::Share(relay_parent, statement) => { - let _timer = metrics.time_share(); - - // pass to legacy if legacy state contains head. - if legacy_v1_state.contains_relay_parent(&relay_parent) { - crate::legacy_v1::share_local_statement( - ctx, - legacy_v1_state, - relay_parent, - StatementWithPVD::drop_pvd_from_signed(statement), - &mut self.rng, - metrics, - ) + FromOrchestra::Communication { msg } => match msg { + StatementDistributionMessage::Share(relay_parent, statement) => { + let _timer = metrics.time_share(); + + // pass to legacy if legacy state contains head. + if legacy_v1_state.contains_relay_parent(&relay_parent) { + crate::legacy_v1::share_local_statement( + ctx, + legacy_v1_state, + relay_parent, + StatementWithPVD::drop_pvd_from_signed(statement), + &mut self.rng, + metrics, + ) + .await?; + } else { + vstaging::share_local_statement(ctx, state, relay_parent, statement) .await?; + } + }, + StatementDistributionMessage::NetworkBridgeUpdate(event) => { + // pass all events to both protocols except for messages, + // which are filtered. + enum VersionTarget { + Legacy, + Current, + Both, + } + + impl VersionTarget { + fn targets_legacy(&self) -> bool { + match self { + &VersionTarget::Legacy | &VersionTarget::Both => true, + _ => false, + } } - }, - StatementDistributionMessage::NetworkBridgeUpdate(event) => { - // pass to legacy, but not if the message isn't - // v1. - let legacy = match &event { - NetworkBridgeEvent::PeerMessage(_, message) => match message { - Versioned::VStaging(protocol_vstaging::StatementDistributionMessage::V1Compatibility(_)) => true, - Versioned::V1(_) => true, - // TODO [now]: _ => false, - }, - _ => true, - }; - - if legacy { - crate::legacy_v1::handle_network_update( - ctx, - legacy_v1_state, - v1_req_sender, - event, - &mut self.rng, - metrics, - ) - .await; + + fn targets_current(&self) -> bool { + match self { + &VersionTarget::Current | &VersionTarget::Both => true, + _ => false, + } } + } - // TODO [now]: pass to vstaging, but not if the message is - // v1 or the connecting peer is v1. - }, - StatementDistributionMessage::Backed(_candidate_hash) => {}, + let target = match &event { + NetworkBridgeEvent::PeerMessage(_, message) => match message { + Versioned::VStaging( + protocol_vstaging::StatementDistributionMessage::V1Compatibility(_), + ) => VersionTarget::Legacy, + Versioned::V1(_) => VersionTarget::Legacy, + Versioned::VStaging(_) => VersionTarget::Current, + }, + _ => VersionTarget::Both, + }; + + if target.targets_legacy() { + crate::legacy_v1::handle_network_update( + ctx, + legacy_v1_state, + v1_req_sender, + event.clone(), + &mut self.rng, + metrics, + ) + .await; + } + + if target.targets_current() { + // pass to vstaging. + vstaging::handle_network_update(ctx, state, event).await; + } + }, + StatementDistributionMessage::Backed(candidate_hash) => { + crate::vstaging::handle_backed_candidate_message(ctx, state, candidate_hash) + .await; }, + }, } Ok(false) } diff --git a/node/network/statement-distribution/src/vstaging/candidates.rs b/node/network/statement-distribution/src/vstaging/candidates.rs new file mode 100644 index 000000000000..804da987ba6d --- /dev/null +++ b/node/network/statement-distribution/src/vstaging/candidates.rs @@ -0,0 +1,1297 @@ +// Copyright 2022 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! The [`Candidates`] store tracks information about advertised candidates +//! as well as which peers have advertised them. +//! +//! Due to the request-oriented nature of this protocol, we often learn +//! about candidates just as a hash, alongside claimed properties that the +//! receipt would commit to. However, it is only later on that we can +//! confirm those claimed properties. This store lets us keep track of +//! all candidates which are currently 'relevant' after spam-protection, and +//! gives us the ability to detect mis-advertisements after the fact +//! and punish them accordingly. + +use polkadot_node_network_protocol::PeerId; +use polkadot_node_subsystem::messages::HypotheticalCandidate; +use polkadot_primitives::vstaging::{ + CandidateHash, CommittedCandidateReceipt, GroupIndex, Hash, Id as ParaId, + PersistedValidationData, +}; + +use std::{ + collections::{ + hash_map::{Entry, HashMap}, + HashSet, + }, + sync::Arc, +}; + +/// This encapsulates the correct and incorrect advertisers +/// post-confirmation of a candidate. +#[derive(Debug, Default, PartialEq)] +pub struct PostConfirmationReckoning { + /// Peers which advertised correctly. + pub correct: HashSet, + /// Peers which advertised the candidate incorrectly. + pub incorrect: HashSet, +} + +/// Outputs generated by initial confirmation of a candidate. +#[derive(Debug, PartialEq)] +pub struct PostConfirmation { + /// The hypothetical candidate used to determine importability and membership + /// in the hypothetical frontier. + pub hypothetical: HypotheticalCandidate, + /// A "reckoning" of peers who have advertised the candidate previously, + /// either accurately or inaccurately. + pub reckoning: PostConfirmationReckoning, +} + +/// A tracker for all known candidates in the view. +/// +/// See module docs for more info. +#[derive(Default)] +pub struct Candidates { + candidates: HashMap, + by_parent: HashMap<(Hash, ParaId), HashSet>, +} + +impl Candidates { + /// Insert an advertisement. + /// + /// This should be invoked only after performing + /// spam protection and only for advertisements that + /// are valid within the current view. [`Candidates`] never prunes + /// candidate by peer ID, to avoid peers skirting misbehavior + /// reports by disconnecting intermittently. Therefore, this presumes + /// that spam protection limits the peers which can send advertisements + /// about unconfirmed candidates. + /// + /// It returns either `Ok(())` or an immediate error in the + /// case that the candidate is already known and reality conflicts + /// with the advertisement. + pub fn insert_unconfirmed( + &mut self, + peer: PeerId, + candidate_hash: CandidateHash, + claimed_relay_parent: Hash, + claimed_group_index: GroupIndex, + claimed_parent_hash_and_id: Option<(Hash, ParaId)>, + ) -> Result<(), BadAdvertisement> { + let entry = self.candidates.entry(candidate_hash).or_insert_with(|| { + CandidateState::Unconfirmed(UnconfirmedCandidate { + claims: Vec::new(), + parent_claims: HashMap::new(), + unconfirmed_importable_under: HashSet::new(), + }) + }); + + match entry { + CandidateState::Confirmed(ref c) => { + if c.relay_parent() != claimed_relay_parent { + return Err(BadAdvertisement) + } + + if c.group_index() != claimed_group_index { + return Err(BadAdvertisement) + } + + if let Some((claimed_parent_hash, claimed_id)) = claimed_parent_hash_and_id { + if c.parent_head_data_hash() != claimed_parent_hash { + return Err(BadAdvertisement) + } + + if c.para_id() != claimed_id { + return Err(BadAdvertisement) + } + } + }, + CandidateState::Unconfirmed(ref mut c) => { + c.add_claims( + peer, + CandidateClaims { + relay_parent: claimed_relay_parent, + group_index: claimed_group_index, + parent_hash_and_id: claimed_parent_hash_and_id, + }, + ); + + if let Some(parent_claims) = claimed_parent_hash_and_id { + self.by_parent.entry(parent_claims).or_default().insert(candidate_hash); + } + }, + } + + Ok(()) + } + + /// Note that a candidate has been confirmed. If the candidate has just been + /// confirmed (previous state was `Unconfirmed`), then this returns `Some`. Otherwise, `None`. + /// + /// If we are confirming for the first time, then remove any outdated claims, and generate a + /// reckoning of which peers advertised correctly and incorrectly. + /// + /// This does no sanity-checking of input data, and will overwrite already-confirmed candidates. + pub fn confirm_candidate( + &mut self, + candidate_hash: CandidateHash, + candidate_receipt: CommittedCandidateReceipt, + persisted_validation_data: PersistedValidationData, + assigned_group: GroupIndex, + ) -> Option { + let parent_hash = persisted_validation_data.parent_head.hash(); + let relay_parent = candidate_receipt.descriptor().relay_parent; + let para_id = candidate_receipt.descriptor().para_id; + + let prev_state = self.candidates.insert( + candidate_hash, + CandidateState::Confirmed(ConfirmedCandidate { + receipt: Arc::new(candidate_receipt), + persisted_validation_data, + assigned_group, + parent_hash, + importable_under: HashSet::new(), + }), + ); + let new_confirmed = + match self.candidates.get_mut(&candidate_hash).expect("just inserted; qed") { + CandidateState::Confirmed(x) => x, + _ => panic!("just inserted as confirmed; qed"), + }; + + self.by_parent.entry((parent_hash, para_id)).or_default().insert(candidate_hash); + + match prev_state { + None => Some(PostConfirmation { + reckoning: Default::default(), + hypothetical: new_confirmed.to_hypothetical(candidate_hash), + }), + Some(CandidateState::Confirmed(_)) => None, + Some(CandidateState::Unconfirmed(u)) => Some({ + let mut reckoning = PostConfirmationReckoning::default(); + + for (leaf_hash, x) in u.unconfirmed_importable_under { + if x.relay_parent == relay_parent && + x.parent_hash == parent_hash && + x.para_id == para_id + { + new_confirmed.importable_under.insert(leaf_hash); + } + } + + for (peer, claims) in u.claims { + // Update the by-parent-hash index not to store any outdated + // claims. + if let Some((claimed_parent_hash, claimed_id)) = claims.parent_hash_and_id { + if claimed_parent_hash != parent_hash || claimed_id != para_id { + if let Entry::Occupied(mut e) = + self.by_parent.entry((claimed_parent_hash, claimed_id)) + { + e.get_mut().remove(&candidate_hash); + if e.get().is_empty() { + e.remove(); + } + } + } + } + + if claims.check(relay_parent, assigned_group, parent_hash, para_id) { + reckoning.correct.insert(peer); + } else { + reckoning.incorrect.insert(peer); + } + } + + PostConfirmation { + reckoning, + hypothetical: new_confirmed.to_hypothetical(candidate_hash), + } + }), + } + } + + /// Whether a candidate is confirmed. + pub fn is_confirmed(&self, candidate_hash: &CandidateHash) -> bool { + match self.candidates.get(candidate_hash) { + Some(CandidateState::Confirmed(_)) => true, + _ => false, + } + } + + /// Get a reference to the candidate, if it's known and confirmed. + pub fn get_confirmed(&self, candidate_hash: &CandidateHash) -> Option<&ConfirmedCandidate> { + match self.candidates.get(candidate_hash) { + Some(CandidateState::Confirmed(ref c)) => Some(c), + _ => None, + } + } + + /// Whether statements from a candidate are importable. + /// + /// This is only true when the candidate is known, confirmed, + /// and is importable in a fragment tree. + pub fn is_importable(&self, candidate_hash: &CandidateHash) -> bool { + self.get_confirmed(candidate_hash).map_or(false, |c| c.is_importable(None)) + } + + /// Note that a candidate is importable in a fragment tree indicated by the given + /// leaf hash. + pub fn note_importable_under(&mut self, candidate: &HypotheticalCandidate, leaf_hash: Hash) { + match candidate { + HypotheticalCandidate::Incomplete { + candidate_hash, + candidate_para, + parent_head_data_hash, + candidate_relay_parent, + } => { + let u = UnconfirmedImportable { + relay_parent: *candidate_relay_parent, + parent_hash: *parent_head_data_hash, + para_id: *candidate_para, + }; + + if let Some(&mut CandidateState::Unconfirmed(ref mut c)) = + self.candidates.get_mut(&candidate_hash) + { + c.note_maybe_importable_under(leaf_hash, u); + } + }, + HypotheticalCandidate::Complete { candidate_hash, .. } => { + if let Some(&mut CandidateState::Confirmed(ref mut c)) = + self.candidates.get_mut(&candidate_hash) + { + c.importable_under.insert(leaf_hash); + } + }, + } + } + + /// Get all hypothetical candidates which should be tested + /// for inclusion in the frontier. + /// + /// Provide optional parent parablock information to filter hypotheticals to only + /// potential children of that parent. + pub fn frontier_hypotheticals( + &self, + parent: Option<(Hash, ParaId)>, + ) -> Vec { + fn extend_hypotheticals<'a>( + v: &mut Vec, + i: impl IntoIterator, + maybe_required_parent: Option<(Hash, ParaId)>, + ) { + for (c_hash, candidate) in i { + match candidate { + CandidateState::Unconfirmed(u) => + u.extend_hypotheticals(*c_hash, v, maybe_required_parent), + CandidateState::Confirmed(c) => v.push(c.to_hypothetical(*c_hash)), + } + } + } + + let mut v = Vec::new(); + if let Some(parent) = parent { + let maybe_children = self.by_parent.get(&parent); + let i = maybe_children + .into_iter() + .flatten() + .filter_map(|c_hash| self.candidates.get_key_value(c_hash)); + + extend_hypotheticals(&mut v, i, Some(parent)); + } else { + extend_hypotheticals(&mut v, self.candidates.iter(), None); + } + v + } + + /// Prune all candidates according to the relay-parent predicate + /// provided. + pub fn on_deactivate_leaves( + &mut self, + leaves: &[Hash], + relay_parent_live: impl Fn(&Hash) -> bool, + ) { + let by_parent = &mut self.by_parent; + let mut remove_parent_claims = |c_hash, parent_hash, id| { + if let Entry::Occupied(mut e) = by_parent.entry((parent_hash, id)) { + e.get_mut().remove(&c_hash); + if e.get().is_empty() { + e.remove(); + } + } + }; + self.candidates.retain(|c_hash, state| match state { + CandidateState::Confirmed(ref mut c) => + if !relay_parent_live(&c.relay_parent()) { + remove_parent_claims(*c_hash, c.parent_head_data_hash(), c.para_id()); + false + } else { + for leaf_hash in leaves { + c.importable_under.remove(leaf_hash); + } + true + }, + CandidateState::Unconfirmed(ref mut c) => { + c.on_deactivate_leaves( + leaves, + |parent_hash, id| remove_parent_claims(*c_hash, parent_hash, id), + &relay_parent_live, + ); + c.has_claims() + }, + }) + } +} + +/// A bad advertisement was recognized. +#[derive(Debug, PartialEq)] +pub struct BadAdvertisement; + +#[derive(Debug, PartialEq)] +enum CandidateState { + Unconfirmed(UnconfirmedCandidate), + Confirmed(ConfirmedCandidate), +} + +/// Claims made alongside the advertisement of a candidate. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +struct CandidateClaims { + /// The relay-parent committed to by the candidate. + relay_parent: Hash, + /// The group index assigned to this candidate. + group_index: GroupIndex, + /// The hash of the parent head-data and the ParaId. This is optional, + /// as only some types of advertisements include this data. + parent_hash_and_id: Option<(Hash, ParaId)>, +} + +impl CandidateClaims { + fn check( + &self, + relay_parent: Hash, + group_index: GroupIndex, + parent_hash: Hash, + para_id: ParaId, + ) -> bool { + self.relay_parent == relay_parent && + self.group_index == group_index && + self.parent_hash_and_id.map_or(true, |p| p == (parent_hash, para_id)) + } +} + +// properties of an unconfirmed but hypothetically importable candidate. +#[derive(Debug, Hash, PartialEq, Eq)] +struct UnconfirmedImportable { + relay_parent: Hash, + parent_hash: Hash, + para_id: ParaId, +} + +// An unconfirmed candidate may have have been advertised under +// multiple identifiers. We track here, on the basis of unique identifier, +// the peers which advertised each candidate in a specific way. +#[derive(Debug, PartialEq)] +struct UnconfirmedCandidate { + claims: Vec<(PeerId, CandidateClaims)>, + // ref-counted + parent_claims: HashMap<(Hash, ParaId), Vec<(Hash, usize)>>, + unconfirmed_importable_under: HashSet<(Hash, UnconfirmedImportable)>, +} + +impl UnconfirmedCandidate { + fn add_claims(&mut self, peer: PeerId, claims: CandidateClaims) { + // This does no deduplication, but this is only called after + // spam prevention is already done. In practice we expect that + // each peer will be able to announce the same candidate about 1 time per live relay-parent, + // but in doing so it limits the amount of other candidates it can advertise. on balance, + // memory consumption is bounded in the same way. + if let Some(parent_claims) = claims.parent_hash_and_id { + let sub_claims = self.parent_claims.entry(parent_claims).or_default(); + match sub_claims.iter().position(|x| x.0 == claims.relay_parent) { + Some(p) => sub_claims[p].1 += 1, + None => sub_claims.push((claims.relay_parent, 1)), + } + } + self.claims.push((peer, claims)); + } + + fn note_maybe_importable_under( + &mut self, + active_leaf: Hash, + unconfirmed_importable: UnconfirmedImportable, + ) { + self.unconfirmed_importable_under.insert((active_leaf, unconfirmed_importable)); + } + + fn on_deactivate_leaves( + &mut self, + leaves: &[Hash], + mut remove_parent_index: impl FnMut(Hash, ParaId), + relay_parent_live: impl Fn(&Hash) -> bool, + ) { + self.claims.retain(|c| { + if relay_parent_live(&c.1.relay_parent) { + true + } else { + if let Some(parent_claims) = c.1.parent_hash_and_id { + if let Entry::Occupied(mut e) = self.parent_claims.entry(parent_claims) { + if let Some(p) = e.get().iter().position(|x| x.0 == c.1.relay_parent) { + let sub_claims = e.get_mut(); + sub_claims[p].1 -= 1; + if sub_claims[p].1 == 0 { + sub_claims.remove(p); + } + }; + + if e.get().is_empty() { + remove_parent_index(parent_claims.0, parent_claims.1); + e.remove(); + } + } + } + + false + } + }); + + self.unconfirmed_importable_under + .retain(|(l, props)| leaves.contains(l) && relay_parent_live(&props.relay_parent)); + } + + fn extend_hypotheticals( + &self, + candidate_hash: CandidateHash, + v: &mut Vec, + required_parent: Option<(Hash, ParaId)>, + ) { + fn extend_hypotheticals_inner<'a>( + candidate_hash: CandidateHash, + v: &mut Vec, + i: impl IntoIterator)>, + ) { + for ((parent_head_hash, para_id), possible_relay_parents) in i { + for (relay_parent, _rc) in possible_relay_parents { + v.push(HypotheticalCandidate::Incomplete { + candidate_hash, + candidate_para: *para_id, + parent_head_data_hash: *parent_head_hash, + candidate_relay_parent: *relay_parent, + }); + } + } + } + + match required_parent { + Some(parent) => extend_hypotheticals_inner( + candidate_hash, + v, + self.parent_claims.get_key_value(&parent), + ), + None => extend_hypotheticals_inner(candidate_hash, v, self.parent_claims.iter()), + } + } + + fn has_claims(&self) -> bool { + !self.claims.is_empty() + } +} + +/// A confirmed candidate. +#[derive(Debug, PartialEq)] +pub struct ConfirmedCandidate { + receipt: Arc, + persisted_validation_data: PersistedValidationData, + assigned_group: GroupIndex, + parent_hash: Hash, + // active leaves statements about this candidate are importable under. + importable_under: HashSet, +} + +impl ConfirmedCandidate { + /// Get the relay-parent of the candidate. + pub fn relay_parent(&self) -> Hash { + self.receipt.descriptor().relay_parent + } + + /// Get the para-id of the candidate. + pub fn para_id(&self) -> ParaId { + self.receipt.descriptor().para_id + } + + /// Get the underlying candidate receipt. + pub fn candidate_receipt(&self) -> &Arc { + &self.receipt + } + + /// Get the persisted validation data. + pub fn persisted_validation_data(&self) -> &PersistedValidationData { + &self.persisted_validation_data + } + + /// Whether the candidate is importable. + pub fn is_importable<'a>(&self, under_active_leaf: impl Into>) -> bool { + match under_active_leaf.into() { + Some(h) => self.importable_under.contains(h), + None => !self.importable_under.is_empty(), + } + } + + /// Get the parent head data hash. + pub fn parent_head_data_hash(&self) -> Hash { + self.parent_hash + } + + /// Get the group index of the assigned group. Note that this is in the context + /// of the state of the chain at the candidate's relay parent and its para-id. + pub fn group_index(&self) -> GroupIndex { + self.assigned_group + } + + fn to_hypothetical(&self, candidate_hash: CandidateHash) -> HypotheticalCandidate { + HypotheticalCandidate::Complete { + candidate_hash, + receipt: self.receipt.clone(), + persisted_validation_data: self.persisted_validation_data.clone(), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use polkadot_primitives::HeadData; + use polkadot_primitives_test_helpers::make_candidate; + + #[test] + fn inserting_unconfirmed_rejects_on_incompatible_claims() { + let relay_head_data_a = HeadData(vec![1, 2, 3]); + let relay_head_data_b = HeadData(vec![4, 5, 6]); + let relay_hash_a = relay_head_data_a.hash(); + let relay_hash_b = relay_head_data_b.hash(); + + let para_id_a = 1.into(); + let para_id_b = 2.into(); + + let (candidate_a, pvd_a) = make_candidate( + relay_hash_a, + 1, + para_id_a, + relay_head_data_a, + HeadData(vec![1]), + Hash::from_low_u64_be(1000).into(), + ); + + let candidate_hash_a = candidate_a.hash(); + + let peer = PeerId::random(); + + let group_index_a = 100.into(); + let group_index_b = 200.into(); + + let mut candidates = Candidates::default(); + + // Confirm a candidate first. + candidates.confirm_candidate(candidate_hash_a, candidate_a, pvd_a, group_index_a); + + // Relay parent does not match. + assert_eq!( + candidates.insert_unconfirmed( + peer, + candidate_hash_a, + relay_hash_b, + group_index_a, + Some((relay_hash_a, para_id_a)), + ), + Err(BadAdvertisement) + ); + + // Group index does not match. + assert_eq!( + candidates.insert_unconfirmed( + peer, + candidate_hash_a, + relay_hash_a, + group_index_b, + Some((relay_hash_a, para_id_a)), + ), + Err(BadAdvertisement) + ); + + // Parent head data does not match. + assert_eq!( + candidates.insert_unconfirmed( + peer, + candidate_hash_a, + relay_hash_a, + group_index_a, + Some((relay_hash_b, para_id_a)), + ), + Err(BadAdvertisement) + ); + + // Para ID does not match. + assert_eq!( + candidates.insert_unconfirmed( + peer, + candidate_hash_a, + relay_hash_a, + group_index_a, + Some((relay_hash_a, para_id_b)), + ), + Err(BadAdvertisement) + ); + + // Everything matches. + assert_eq!( + candidates.insert_unconfirmed( + peer, + candidate_hash_a, + relay_hash_a, + group_index_a, + Some((relay_hash_a, para_id_a)), + ), + Ok(()) + ); + } + + // Tests that: + // + // - When the advertisement matches, confirming does not change the parent hash index. + // - When it doesn't match, confirming updates the index. Specifically, confirming should prune + // unconfirmed claims. + #[test] + fn confirming_maintains_parent_hash_index() { + let relay_head_data = HeadData(vec![1, 2, 3]); + let relay_hash = relay_head_data.hash(); + + let candidate_head_data_a = HeadData(vec![1]); + let candidate_head_data_b = HeadData(vec![2]); + let candidate_head_data_c = HeadData(vec![3]); + let candidate_head_data_d = HeadData(vec![4]); + let candidate_head_data_hash_a = candidate_head_data_a.hash(); + let candidate_head_data_hash_b = candidate_head_data_b.hash(); + let candidate_head_data_hash_c = candidate_head_data_c.hash(); + + let (candidate_a, pvd_a) = make_candidate( + relay_hash, + 1, + 1.into(), + relay_head_data, + candidate_head_data_a.clone(), + Hash::from_low_u64_be(1000).into(), + ); + let (candidate_b, pvd_b) = make_candidate( + relay_hash, + 1, + 1.into(), + candidate_head_data_a, + candidate_head_data_b.clone(), + Hash::from_low_u64_be(2000).into(), + ); + let (candidate_c, _) = make_candidate( + relay_hash, + 1, + 1.into(), + candidate_head_data_b.clone(), + candidate_head_data_c.clone(), + Hash::from_low_u64_be(3000).into(), + ); + let (candidate_d, pvd_d) = make_candidate( + relay_hash, + 1, + 1.into(), + candidate_head_data_c.clone(), + candidate_head_data_d, + Hash::from_low_u64_be(4000).into(), + ); + + let candidate_hash_a = candidate_a.hash(); + let candidate_hash_b = candidate_b.hash(); + let candidate_hash_c = candidate_c.hash(); + let candidate_hash_d = candidate_d.hash(); + + let peer = PeerId::random(); + let group_index = 100.into(); + + let mut candidates = Candidates::default(); + + // Insert some unconfirmed candidates. + + // Advertise A without parent hash. + candidates + .insert_unconfirmed(peer, candidate_hash_a, relay_hash, group_index, None) + .ok() + .unwrap(); + assert_eq!(candidates.by_parent, HashMap::default()); + + // Advertise A with parent hash and ID. + candidates + .insert_unconfirmed( + peer, + candidate_hash_a, + relay_hash, + group_index, + Some((relay_hash, 1.into())), + ) + .ok() + .unwrap(); + assert_eq!( + candidates.by_parent, + HashMap::from([((relay_hash, 1.into()), HashSet::from([candidate_hash_a]))]) + ); + + // Advertise B with parent A. + candidates + .insert_unconfirmed( + peer, + candidate_hash_b, + relay_hash, + group_index, + Some((candidate_head_data_hash_a, 1.into())), + ) + .ok() + .unwrap(); + assert_eq!( + candidates.by_parent, + HashMap::from([ + ((relay_hash, 1.into()), HashSet::from([candidate_hash_a])), + ((candidate_head_data_hash_a, 1.into()), HashSet::from([candidate_hash_b])) + ]) + ); + + // Advertise C with parent A. + candidates + .insert_unconfirmed( + peer, + candidate_hash_c, + relay_hash, + group_index, + Some((candidate_head_data_hash_a, 1.into())), + ) + .ok() + .unwrap(); + assert_eq!( + candidates.by_parent, + HashMap::from([ + ((relay_hash, 1.into()), HashSet::from([candidate_hash_a])), + ( + (candidate_head_data_hash_a, 1.into()), + HashSet::from([candidate_hash_b, candidate_hash_c]) + ) + ]) + ); + + // Advertise D with parent A. + candidates + .insert_unconfirmed( + peer, + candidate_hash_d, + relay_hash, + group_index, + Some((candidate_head_data_hash_a, 1.into())), + ) + .ok() + .unwrap(); + assert_eq!( + candidates.by_parent, + HashMap::from([ + ((relay_hash, 1.into()), HashSet::from([candidate_hash_a])), + ( + (candidate_head_data_hash_a, 1.into()), + HashSet::from([candidate_hash_b, candidate_hash_c, candidate_hash_d]) + ) + ]) + ); + + // Insert confirmed candidates and check parent hash index. + + // Confirmation matches advertisement. Index should be unchanged. + candidates.confirm_candidate(candidate_hash_a, candidate_a, pvd_a, group_index); + assert_eq!( + candidates.by_parent, + HashMap::from([ + ((relay_hash, 1.into()), HashSet::from([candidate_hash_a])), + ( + (candidate_head_data_hash_a, 1.into()), + HashSet::from([candidate_hash_b, candidate_hash_c, candidate_hash_d]) + ) + ]) + ); + candidates.confirm_candidate(candidate_hash_b, candidate_b, pvd_b, group_index); + assert_eq!( + candidates.by_parent, + HashMap::from([ + ((relay_hash, 1.into()), HashSet::from([candidate_hash_a])), + ( + (candidate_head_data_hash_a, 1.into()), + HashSet::from([candidate_hash_b, candidate_hash_c, candidate_hash_d]) + ) + ]) + ); + + // Confirmation does not match advertisement. Index should be updated. + candidates.confirm_candidate(candidate_hash_d, candidate_d, pvd_d, group_index); + assert_eq!( + candidates.by_parent, + HashMap::from([ + ((relay_hash, 1.into()), HashSet::from([candidate_hash_a])), + ( + (candidate_head_data_hash_a, 1.into()), + HashSet::from([candidate_hash_b, candidate_hash_c]) + ), + ((candidate_head_data_hash_c, 1.into()), HashSet::from([candidate_hash_d])) + ]) + ); + + // Make a new candidate for C with a different para ID. + let (new_candidate_c, new_pvd_c) = make_candidate( + relay_hash, + 1, + 2.into(), + candidate_head_data_b, + candidate_head_data_c.clone(), + Hash::from_low_u64_be(3000).into(), + ); + candidates.confirm_candidate(candidate_hash_c, new_candidate_c, new_pvd_c, group_index); + assert_eq!( + candidates.by_parent, + HashMap::from([ + ((relay_hash, 1.into()), HashSet::from([candidate_hash_a])), + ((candidate_head_data_hash_a, 1.into()), HashSet::from([candidate_hash_b])), + ((candidate_head_data_hash_b, 2.into()), HashSet::from([candidate_hash_c])), + ((candidate_head_data_hash_c, 1.into()), HashSet::from([candidate_hash_d])) + ]) + ); + } + + #[test] + fn test_returned_post_confirmation() { + let relay_head_data = HeadData(vec![1, 2, 3]); + let relay_hash = relay_head_data.hash(); + + let candidate_head_data_a = HeadData(vec![1]); + let candidate_head_data_b = HeadData(vec![2]); + let candidate_head_data_c = HeadData(vec![3]); + let candidate_head_data_d = HeadData(vec![4]); + let candidate_head_data_hash_a = candidate_head_data_a.hash(); + let candidate_head_data_hash_b = candidate_head_data_b.hash(); + + let (candidate_a, pvd_a) = make_candidate( + relay_hash, + 1, + 1.into(), + relay_head_data, + candidate_head_data_a.clone(), + Hash::from_low_u64_be(1000).into(), + ); + let (candidate_b, pvd_b) = make_candidate( + relay_hash, + 1, + 1.into(), + candidate_head_data_a.clone(), + candidate_head_data_b.clone(), + Hash::from_low_u64_be(2000).into(), + ); + let (candidate_c, _) = make_candidate( + relay_hash, + 1, + 1.into(), + candidate_head_data_a.clone(), + candidate_head_data_c.clone(), + Hash::from_low_u64_be(3000).into(), + ); + let (candidate_d, pvd_d) = make_candidate( + relay_hash, + 1, + 1.into(), + candidate_head_data_b.clone(), + candidate_head_data_d, + Hash::from_low_u64_be(4000).into(), + ); + + let candidate_hash_a = candidate_a.hash(); + let candidate_hash_b = candidate_b.hash(); + let candidate_hash_c = candidate_c.hash(); + let candidate_hash_d = candidate_d.hash(); + + let peer_a = PeerId::random(); + let peer_b = PeerId::random(); + let peer_c = PeerId::random(); + let peer_d = PeerId::random(); + + let group_index = 100.into(); + + let mut candidates = Candidates::default(); + + // Insert some unconfirmed candidates. + + // Advertise A without parent hash. + candidates + .insert_unconfirmed(peer_a, candidate_hash_a, relay_hash, group_index, None) + .ok() + .unwrap(); + + // Advertise A with parent hash and ID. + candidates + .insert_unconfirmed( + peer_a, + candidate_hash_a, + relay_hash, + group_index, + Some((relay_hash, 1.into())), + ) + .ok() + .unwrap(); + + // (Correctly) advertise B with parent A. Do it from a couple of peers. + candidates + .insert_unconfirmed( + peer_a, + candidate_hash_b, + relay_hash, + group_index, + Some((candidate_head_data_hash_a, 1.into())), + ) + .ok() + .unwrap(); + candidates + .insert_unconfirmed( + peer_b, + candidate_hash_b, + relay_hash, + group_index, + Some((candidate_head_data_hash_a, 1.into())), + ) + .ok() + .unwrap(); + + // (Wrongly) advertise C with parent A. Do it from a couple peers. + candidates + .insert_unconfirmed( + peer_b, + candidate_hash_c, + relay_hash, + group_index, + Some((candidate_head_data_hash_a, 1.into())), + ) + .ok() + .unwrap(); + candidates + .insert_unconfirmed( + peer_c, + candidate_hash_c, + relay_hash, + group_index, + Some((candidate_head_data_hash_a, 1.into())), + ) + .ok() + .unwrap(); + + // Advertise D. Do it correctly from one peer (parent B) and wrongly from another (parent A). + candidates + .insert_unconfirmed( + peer_c, + candidate_hash_d, + relay_hash, + group_index, + Some((candidate_head_data_hash_b, 1.into())), + ) + .ok() + .unwrap(); + candidates + .insert_unconfirmed( + peer_d, + candidate_hash_d, + relay_hash, + group_index, + Some((candidate_head_data_hash_a, 1.into())), + ) + .ok() + .unwrap(); + + assert_eq!( + candidates.by_parent, + HashMap::from([ + ((relay_hash, 1.into()), HashSet::from([candidate_hash_a])), + ( + (candidate_head_data_hash_a, 1.into()), + HashSet::from([candidate_hash_b, candidate_hash_c, candidate_hash_d]) + ), + ((candidate_head_data_hash_b, 1.into()), HashSet::from([candidate_hash_d])) + ]) + ); + + // Insert confirmed candidates and check parent hash index. + + // Confirmation matches advertisement. + let post_confirmation = candidates.confirm_candidate( + candidate_hash_a, + candidate_a.clone(), + pvd_a.clone(), + group_index, + ); + assert_eq!( + post_confirmation, + Some(PostConfirmation { + hypothetical: HypotheticalCandidate::Complete { + candidate_hash: candidate_hash_a, + receipt: Arc::new(candidate_a), + persisted_validation_data: pvd_a, + }, + reckoning: PostConfirmationReckoning { + correct: HashSet::from([peer_a]), + incorrect: HashSet::from([]), + }, + }) + ); + + let post_confirmation = candidates.confirm_candidate( + candidate_hash_b, + candidate_b.clone(), + pvd_b.clone(), + group_index, + ); + assert_eq!( + post_confirmation, + Some(PostConfirmation { + hypothetical: HypotheticalCandidate::Complete { + candidate_hash: candidate_hash_b, + receipt: Arc::new(candidate_b), + persisted_validation_data: pvd_b, + }, + reckoning: PostConfirmationReckoning { + correct: HashSet::from([peer_a, peer_b]), + incorrect: HashSet::from([]), + }, + }) + ); + + // Confirm candidate with two wrong peers (different group index). + let (new_candidate_c, new_pvd_c) = make_candidate( + relay_hash, + 1, + 2.into(), + candidate_head_data_b, + candidate_head_data_c.clone(), + Hash::from_low_u64_be(3000).into(), + ); + let post_confirmation = candidates.confirm_candidate( + candidate_hash_c, + new_candidate_c.clone(), + new_pvd_c.clone(), + group_index, + ); + assert_eq!( + post_confirmation, + Some(PostConfirmation { + hypothetical: HypotheticalCandidate::Complete { + candidate_hash: candidate_hash_c, + receipt: Arc::new(new_candidate_c), + persisted_validation_data: new_pvd_c, + }, + reckoning: PostConfirmationReckoning { + correct: HashSet::from([]), + incorrect: HashSet::from([peer_b, peer_c]), + }, + }) + ); + + // Confirm candidate with one wrong peer (different parent head data). + let post_confirmation = candidates.confirm_candidate( + candidate_hash_d, + candidate_d.clone(), + pvd_d.clone(), + group_index, + ); + assert_eq!( + post_confirmation, + Some(PostConfirmation { + hypothetical: HypotheticalCandidate::Complete { + candidate_hash: candidate_hash_d, + receipt: Arc::new(candidate_d), + persisted_validation_data: pvd_d, + }, + reckoning: PostConfirmationReckoning { + correct: HashSet::from([peer_c]), + incorrect: HashSet::from([peer_d]), + }, + }) + ); + } + + #[test] + fn test_hypothetical_frontiers() { + let relay_head_data = HeadData(vec![1, 2, 3]); + let relay_hash = relay_head_data.hash(); + + let candidate_head_data_a = HeadData(vec![1]); + let candidate_head_data_b = HeadData(vec![2]); + let candidate_head_data_c = HeadData(vec![3]); + let candidate_head_data_d = HeadData(vec![4]); + let candidate_head_data_hash_a = candidate_head_data_a.hash(); + let candidate_head_data_hash_b = candidate_head_data_b.hash(); + let candidate_head_data_hash_d = candidate_head_data_d.hash(); + + let (candidate_a, pvd_a) = make_candidate( + relay_hash, + 1, + 1.into(), + relay_head_data, + candidate_head_data_a.clone(), + Hash::from_low_u64_be(1000).into(), + ); + let (candidate_b, _) = make_candidate( + relay_hash, + 1, + 1.into(), + candidate_head_data_a.clone(), + candidate_head_data_b.clone(), + Hash::from_low_u64_be(2000).into(), + ); + let (candidate_c, _) = make_candidate( + relay_hash, + 1, + 1.into(), + candidate_head_data_a.clone(), + candidate_head_data_c.clone(), + Hash::from_low_u64_be(3000).into(), + ); + let (candidate_d, _) = make_candidate( + relay_hash, + 1, + 1.into(), + candidate_head_data_b.clone(), + candidate_head_data_d, + Hash::from_low_u64_be(4000).into(), + ); + + let candidate_hash_a = candidate_a.hash(); + let candidate_hash_b = candidate_b.hash(); + let candidate_hash_c = candidate_c.hash(); + let candidate_hash_d = candidate_d.hash(); + + let peer = PeerId::random(); + let group_index = 100.into(); + + let mut candidates = Candidates::default(); + + // Confirm A. + candidates.confirm_candidate( + candidate_hash_a, + candidate_a.clone(), + pvd_a.clone(), + group_index, + ); + + // Advertise B with parent A. + candidates + .insert_unconfirmed( + peer, + candidate_hash_b, + relay_hash, + group_index, + Some((candidate_head_data_hash_a, 1.into())), + ) + .ok() + .unwrap(); + + // Advertise C with parent A. + candidates + .insert_unconfirmed( + peer, + candidate_hash_c, + relay_hash, + group_index, + Some((candidate_head_data_hash_a, 1.into())), + ) + .ok() + .unwrap(); + + // Advertise D with parent B. + candidates + .insert_unconfirmed( + peer, + candidate_hash_d, + relay_hash, + group_index, + Some((candidate_head_data_hash_b, 1.into())), + ) + .ok() + .unwrap(); + + assert_eq!( + candidates.by_parent, + HashMap::from([ + ((relay_hash, 1.into()), HashSet::from([candidate_hash_a])), + ( + (candidate_head_data_hash_a, 1.into()), + HashSet::from([candidate_hash_b, candidate_hash_c]) + ), + ((candidate_head_data_hash_b, 1.into()), HashSet::from([candidate_hash_d])) + ]) + ); + + let hypothetical_a = HypotheticalCandidate::Complete { + candidate_hash: candidate_hash_a, + receipt: Arc::new(candidate_a), + persisted_validation_data: pvd_a, + }; + let hypothetical_b = HypotheticalCandidate::Incomplete { + candidate_hash: candidate_hash_b, + candidate_para: 1.into(), + parent_head_data_hash: candidate_head_data_hash_a, + candidate_relay_parent: relay_hash, + }; + let hypothetical_c = HypotheticalCandidate::Incomplete { + candidate_hash: candidate_hash_c, + candidate_para: 1.into(), + parent_head_data_hash: candidate_head_data_hash_a, + candidate_relay_parent: relay_hash, + }; + let hypothetical_d = HypotheticalCandidate::Incomplete { + candidate_hash: candidate_hash_d, + candidate_para: 1.into(), + parent_head_data_hash: candidate_head_data_hash_b, + candidate_relay_parent: relay_hash, + }; + + let hypotheticals = candidates.frontier_hypotheticals(Some((relay_hash, 1.into()))); + assert_eq!(hypotheticals.len(), 1); + assert!(hypotheticals.contains(&hypothetical_a)); + + let hypotheticals = + candidates.frontier_hypotheticals(Some((candidate_head_data_hash_a, 2.into()))); + assert_eq!(hypotheticals.len(), 0); + + let hypotheticals = + candidates.frontier_hypotheticals(Some((candidate_head_data_hash_a, 1.into()))); + assert_eq!(hypotheticals.len(), 2); + assert!(hypotheticals.contains(&hypothetical_b)); + assert!(hypotheticals.contains(&hypothetical_c)); + + let hypotheticals = + candidates.frontier_hypotheticals(Some((candidate_head_data_hash_d, 1.into()))); + assert_eq!(hypotheticals.len(), 0); + + let hypotheticals = candidates.frontier_hypotheticals(None); + assert_eq!(hypotheticals.len(), 4); + assert!(hypotheticals.contains(&hypothetical_a)); + assert!(hypotheticals.contains(&hypothetical_b)); + assert!(hypotheticals.contains(&hypothetical_c)); + assert!(hypotheticals.contains(&hypothetical_d)); + } +} diff --git a/node/network/statement-distribution/src/vstaging/cluster.rs b/node/network/statement-distribution/src/vstaging/cluster.rs new file mode 100644 index 000000000000..49852a912d2f --- /dev/null +++ b/node/network/statement-distribution/src/vstaging/cluster.rs @@ -0,0 +1,1203 @@ +// Copyright 2022 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! Direct distribution of statements within a cluster, +//! even those concerning candidates which are not yet backed. +//! +//! Members of a validation group assigned to a para at a given relay-parent +//! always distribute statements directly to each other. +//! +//! The main way we limit the amount of candidates that have to be handled by +//! the system is to limit the amount of `Seconded` messages that we allow +//! each validator to issue at each relay-parent. Since the amount of relay-parents +//! that we have to deal with at any time is itself bounded, this lets us bound +//! the memory and work that we have here. Bounding `Seconded` statements is enough +//! because they imply a bounded amount of `Valid` statements about the same candidate +//! which may follow. +//! +//! The motivation for this piece of code is that the statements that each validator +//! sees may differ. i.e. even though a validator is allowed to issue X `Seconded` +//! statements at a relay-parent, they may in fact issue X*2 and issue one set to +//! one partition of the backing group and one set to another. Of course, in practice +//! these types of partitions will not exist, but in the worst case each validator in the +//! group would see an entirely different set of X `Seconded` statements from some validator +//! and each validator is in its own partition. After that partition resolves, we'd have to +//! deal with up to `limit*group_size` `Seconded` statements from that validator. And then +//! if every validator in the group does the same thing, we're dealing with something like +//! `limit*group_size^2` `Seconded` statements in total. +//! +//! Given that both our group sizes and our limits per relay-parent are small, this is +//! quite manageable, and the utility here lets us deal with it in only a few kilobytes +//! of memory. +//! +//! It's also worth noting that any case where a validator issues more than the legal limit +//! of `Seconded` statements at a relay parent is trivially slashable on-chain, which means +//! the 'worst case' adversary that this code defends against is effectively lighting money +//! on fire. Nevertheless, we handle the case here to ensure that the behavior of the +//! system is well-defined even if an adversary is willing to be slashed. +//! +//! More concretely, this module exposes a [`ClusterTracker`] utility which allows us to determine +//! whether to accept or reject messages from other validators in the same group as we +//! are in, based on _the most charitable possible interpretation of our protocol rules_, +//! and to keep track of what we have sent to other validators in the group and what we may +//! continue to send them. + +use polkadot_primitives::vstaging::{CandidateHash, CompactStatement, ValidatorIndex}; + +use std::collections::{HashMap, HashSet}; + +#[derive(Hash, PartialEq, Eq)] +struct ValidStatementManifest { + remote: ValidatorIndex, + originator: ValidatorIndex, + candidate_hash: CandidateHash, +} + +// A piece of knowledge about a candidate +#[derive(Hash, Clone, PartialEq, Eq)] +enum Knowledge { + // General knowledge. + General(CandidateHash), + // Specific knowledge of a given statement (with its originator) + Specific(CompactStatement, ValidatorIndex), +} + +// Knowledge paired with its source. +#[derive(Hash, Clone, PartialEq, Eq)] +enum TaggedKnowledge { + // Knowledge we have received from the validator on the p2p layer. + IncomingP2P(Knowledge), + // Knowledge we have sent to the validator on the p2p layer. + OutgoingP2P(Knowledge), + // Knowledge of candidates the validator has seconded. + // This is limited only to `Seconded` statements we have accepted + // _without prejudice_. + Seconded(CandidateHash), +} + +/// Utility for keeping track of limits on direct statements within a group. +/// +/// See module docs for more details. +pub struct ClusterTracker { + validators: Vec, + seconding_limit: usize, + knowledge: HashMap>, + // Statements known locally which haven't been sent to particular validators. + // maps target validator to (originator, statement) pairs. + pending: HashMap>, +} + +impl ClusterTracker { + /// Instantiate a new `ClusterTracker` tracker. Fails if `cluster_validators` is empty + pub fn new(cluster_validators: Vec, seconding_limit: usize) -> Option { + if cluster_validators.is_empty() { + return None + } + Some(ClusterTracker { + validators: cluster_validators, + seconding_limit, + knowledge: HashMap::new(), + pending: HashMap::new(), + }) + } + + /// Query whether we can receive some statement from the given validator. + /// + /// This does no deduplication of `Valid` statements. + pub fn can_receive( + &self, + sender: ValidatorIndex, + originator: ValidatorIndex, + statement: CompactStatement, + ) -> Result { + if !self.is_in_group(sender) || !self.is_in_group(originator) { + return Err(RejectIncoming::NotInGroup) + } + + if self.they_sent(sender, Knowledge::Specific(statement.clone(), originator)) { + return Err(RejectIncoming::Duplicate) + } + + match statement { + CompactStatement::Seconded(candidate_hash) => { + // check whether the sender has not sent too many seconded statements for the originator. + // we know by the duplicate check above that this iterator doesn't include the + // statement itself. + let other_seconded_for_orig_from_remote = self + .knowledge + .get(&sender) + .into_iter() + .flat_map(|v_knowledge| v_knowledge.iter()) + .filter(|k| match k { + TaggedKnowledge::IncomingP2P(Knowledge::Specific( + CompactStatement::Seconded(_), + orig, + )) if orig == &originator => true, + _ => false, + }) + .count(); + + if other_seconded_for_orig_from_remote == self.seconding_limit { + return Err(RejectIncoming::ExcessiveSeconded) + } + + // at this point, it doesn't seem like the remote has done anything wrong. + if self.seconded_already_or_within_limit(originator, candidate_hash) { + Ok(Accept::Ok) + } else { + Ok(Accept::WithPrejudice) + } + }, + CompactStatement::Valid(candidate_hash) => { + if !self.knows_candidate(sender, candidate_hash) { + return Err(RejectIncoming::CandidateUnknown) + } + + Ok(Accept::Ok) + }, + } + } + + /// Note that we issued a statement. This updates internal structures. + pub fn note_issued(&mut self, originator: ValidatorIndex, statement: CompactStatement) { + for cluster_member in &self.validators { + if !self.they_know_statement(*cluster_member, originator, statement.clone()) { + // add the statement to pending knowledge for all peers + // which don't know the statement. + self.pending + .entry(*cluster_member) + .or_default() + .insert((originator, statement.clone())); + } + } + } + + /// Note that we accepted an incoming statement. This updates internal structures. + /// + /// Should only be called after a successful `can_receive` call. + pub fn note_received( + &mut self, + sender: ValidatorIndex, + originator: ValidatorIndex, + statement: CompactStatement, + ) { + for cluster_member in &self.validators { + if cluster_member == &sender { + if let Some(pending) = self.pending.get_mut(&sender) { + pending.remove(&(originator, statement.clone())); + } + } else if !self.they_know_statement(*cluster_member, originator, statement.clone()) { + // add the statement to pending knowledge for all peers + // which don't know the statement. + self.pending + .entry(*cluster_member) + .or_default() + .insert((originator, statement.clone())); + } + } + + { + let sender_knowledge = self.knowledge.entry(sender).or_default(); + sender_knowledge.insert(TaggedKnowledge::IncomingP2P(Knowledge::Specific( + statement.clone(), + originator, + ))); + + if let CompactStatement::Seconded(candidate_hash) = statement.clone() { + sender_knowledge + .insert(TaggedKnowledge::IncomingP2P(Knowledge::General(candidate_hash))); + } + } + + if let CompactStatement::Seconded(candidate_hash) = statement { + // since we accept additional `Seconded` statements beyond the limits + // 'with prejudice', we must respect the limit here. + if self.seconded_already_or_within_limit(originator, candidate_hash) { + let originator_knowledge = self.knowledge.entry(originator).or_default(); + originator_knowledge.insert(TaggedKnowledge::Seconded(candidate_hash)); + } + } + } + + /// Query whether we can send a statement to a given validator. + pub fn can_send( + &self, + target: ValidatorIndex, + originator: ValidatorIndex, + statement: CompactStatement, + ) -> Result<(), RejectOutgoing> { + if !self.is_in_group(target) || !self.is_in_group(originator) { + return Err(RejectOutgoing::NotInGroup) + } + + if self.they_know_statement(target, originator, statement.clone()) { + return Err(RejectOutgoing::Known) + } + + match statement { + CompactStatement::Seconded(candidate_hash) => { + // we send the same `Seconded` statements to all our peers, and only the first `k` from + // each originator. + if !self.seconded_already_or_within_limit(originator, candidate_hash) { + return Err(RejectOutgoing::ExcessiveSeconded) + } + + Ok(()) + }, + CompactStatement::Valid(candidate_hash) => { + if !self.knows_candidate(target, candidate_hash) { + return Err(RejectOutgoing::CandidateUnknown) + } + + Ok(()) + }, + } + } + + /// Note that we sent an outgoing statement to a peer in the group. + /// This must be preceded by a successful `can_send` call. + pub fn note_sent( + &mut self, + target: ValidatorIndex, + originator: ValidatorIndex, + statement: CompactStatement, + ) { + { + let target_knowledge = self.knowledge.entry(target).or_default(); + target_knowledge.insert(TaggedKnowledge::OutgoingP2P(Knowledge::Specific( + statement.clone(), + originator, + ))); + + if let CompactStatement::Seconded(candidate_hash) = statement.clone() { + target_knowledge + .insert(TaggedKnowledge::OutgoingP2P(Knowledge::General(candidate_hash))); + } + } + + if let CompactStatement::Seconded(candidate_hash) = statement { + let originator_knowledge = self.knowledge.entry(originator).or_default(); + originator_knowledge.insert(TaggedKnowledge::Seconded(candidate_hash)); + } + + if let Some(pending) = self.pending.get_mut(&target) { + pending.remove(&(originator, statement)); + } + } + + /// Get all targets as validator-indices. This doesn't attempt to filter + /// out the local validator index. + pub fn targets(&self) -> &[ValidatorIndex] { + &self.validators + } + + /// Get all possible senders for the given originator. + /// Returns the empty slice in the case that the originator + /// is not part of the cluster. + // note: this API is future-proofing for a case where we may + // extend clusters beyond just the assigned group, for optimization + // purposes. + pub fn senders_for_originator(&self, originator: ValidatorIndex) -> &[ValidatorIndex] { + if self.validators.contains(&originator) { + &self.validators[..] + } else { + &[] + } + } + + /// Whether a validator knows the candidate is `Seconded`. + pub fn knows_candidate( + &self, + validator: ValidatorIndex, + candidate_hash: CandidateHash, + ) -> bool { + // we sent, they sent, or they signed and we received from someone else. + + self.we_sent_seconded(validator, candidate_hash) || + self.they_sent_seconded(validator, candidate_hash) || + self.validator_seconded(validator, candidate_hash) + } + + /// Returns a Vec of pending statements to be sent to a particular validator + /// index. `Seconded` statements are sorted to the front of the vector. + /// + /// Pending statements have the form (originator, compact statement). + pub fn pending_statements_for( + &self, + target: ValidatorIndex, + ) -> Vec<(ValidatorIndex, CompactStatement)> { + let mut v = self + .pending + .get(&target) + .map(|x| x.iter().cloned().collect::>()) + .unwrap_or_default(); + + v.sort_by_key(|(_, s)| match s { + CompactStatement::Seconded(_) => 0u8, + CompactStatement::Valid(_) => 1u8, + }); + + v + } + + // returns true if it's legal to accept a new `Seconded` message from this validator. + // This is either + // 1. because we've already accepted it. + // 2. because there's space for more seconding. + fn seconded_already_or_within_limit( + &self, + validator: ValidatorIndex, + candidate_hash: CandidateHash, + ) -> bool { + let seconded_other_candidates = self + .knowledge + .get(&validator) + .into_iter() + .flat_map(|v_knowledge| v_knowledge.iter()) + .filter(|k| match k { + TaggedKnowledge::Seconded(c) if c != &candidate_hash => true, + _ => false, + }) + .count(); + + // This fulfills both properties by under-counting when the validator is at the limit + // but _has_ seconded the candidate already. + seconded_other_candidates < self.seconding_limit + } + + fn they_know_statement( + &self, + validator: ValidatorIndex, + originator: ValidatorIndex, + statement: CompactStatement, + ) -> bool { + let knowledge = Knowledge::Specific(statement, originator); + self.we_sent(validator, knowledge.clone()) || self.they_sent(validator, knowledge) + } + + fn they_sent(&self, validator: ValidatorIndex, knowledge: Knowledge) -> bool { + self.knowledge + .get(&validator) + .map_or(false, |k| k.contains(&TaggedKnowledge::IncomingP2P(knowledge))) + } + + fn we_sent(&self, validator: ValidatorIndex, knowledge: Knowledge) -> bool { + self.knowledge + .get(&validator) + .map_or(false, |k| k.contains(&TaggedKnowledge::OutgoingP2P(knowledge))) + } + + fn we_sent_seconded(&self, validator: ValidatorIndex, candidate_hash: CandidateHash) -> bool { + self.we_sent(validator, Knowledge::General(candidate_hash)) + } + + fn they_sent_seconded(&self, validator: ValidatorIndex, candidate_hash: CandidateHash) -> bool { + self.they_sent(validator, Knowledge::General(candidate_hash)) + } + + fn validator_seconded(&self, validator: ValidatorIndex, candidate_hash: CandidateHash) -> bool { + self.knowledge + .get(&validator) + .map_or(false, |k| k.contains(&TaggedKnowledge::Seconded(candidate_hash))) + } + + fn is_in_group(&self, validator: ValidatorIndex) -> bool { + self.validators.contains(&validator) + } +} + +/// Incoming statement was accepted. +#[derive(Debug, PartialEq)] +pub enum Accept { + /// Neither the peer nor the originator have apparently exceeded limits. + /// Candidate or statement may already be known. + Ok, + /// Accept the message; the peer hasn't exceeded limits but the originator has. + WithPrejudice, +} + +/// Incoming statement was rejected. +#[derive(Debug, PartialEq)] +pub enum RejectIncoming { + /// Peer sent excessive `Seconded` statements. + ExcessiveSeconded, + /// Sender or originator is not in the group. + NotInGroup, + /// Candidate is unknown to us. Only applies to `Valid` statements. + CandidateUnknown, + /// Statement is duplicate. + Duplicate, +} + +/// Outgoing statement was rejected. +#[derive(Debug, PartialEq)] +pub enum RejectOutgoing { + /// Candidate was unknown. Only applies to `Valid` statements. + CandidateUnknown, + /// We attempted to send excessive `Seconded` statements. + /// indicates a bug on the local node's code. + ExcessiveSeconded, + /// The statement was already known to the peer. + Known, + /// Target or originator not in the group. + NotInGroup, +} + +#[cfg(test)] +mod tests { + use super::*; + use polkadot_primitives::vstaging::Hash; + + #[test] + fn rejects_incoming_outside_of_group() { + let group = + vec![ValidatorIndex(5), ValidatorIndex(200), ValidatorIndex(24), ValidatorIndex(146)]; + + let seconding_limit = 2; + + let tracker = ClusterTracker::new(group.clone(), seconding_limit).expect("not empty"); + + assert_eq!( + tracker.can_receive( + ValidatorIndex(100), + ValidatorIndex(5), + CompactStatement::Seconded(CandidateHash(Hash::repeat_byte(1))), + ), + Err(RejectIncoming::NotInGroup), + ); + + assert_eq!( + tracker.can_receive( + ValidatorIndex(5), + ValidatorIndex(100), + CompactStatement::Seconded(CandidateHash(Hash::repeat_byte(1))), + ), + Err(RejectIncoming::NotInGroup), + ); + } + + #[test] + fn begrudgingly_accepts_too_many_seconded_from_multiple_peers() { + let group = + vec![ValidatorIndex(5), ValidatorIndex(200), ValidatorIndex(24), ValidatorIndex(146)]; + + let seconding_limit = 2; + let hash_a = CandidateHash(Hash::repeat_byte(1)); + let hash_b = CandidateHash(Hash::repeat_byte(2)); + let hash_c = CandidateHash(Hash::repeat_byte(3)); + + let mut tracker = ClusterTracker::new(group.clone(), seconding_limit).expect("not empty"); + + assert_eq!( + tracker.can_receive( + ValidatorIndex(5), + ValidatorIndex(5), + CompactStatement::Seconded(hash_a), + ), + Ok(Accept::Ok), + ); + tracker.note_received( + ValidatorIndex(5), + ValidatorIndex(5), + CompactStatement::Seconded(hash_a), + ); + + assert_eq!( + tracker.can_receive( + ValidatorIndex(5), + ValidatorIndex(5), + CompactStatement::Seconded(hash_b), + ), + Ok(Accept::Ok), + ); + tracker.note_received( + ValidatorIndex(5), + ValidatorIndex(5), + CompactStatement::Seconded(hash_b), + ); + + assert_eq!( + tracker.can_receive( + ValidatorIndex(5), + ValidatorIndex(5), + CompactStatement::Seconded(hash_c), + ), + Err(RejectIncoming::ExcessiveSeconded), + ); + } + + #[test] + fn rejects_too_many_seconded_from_sender() { + let group = + vec![ValidatorIndex(5), ValidatorIndex(200), ValidatorIndex(24), ValidatorIndex(146)]; + + let seconding_limit = 2; + let hash_a = CandidateHash(Hash::repeat_byte(1)); + let hash_b = CandidateHash(Hash::repeat_byte(2)); + let hash_c = CandidateHash(Hash::repeat_byte(3)); + + let mut tracker = ClusterTracker::new(group.clone(), seconding_limit).expect("not empty"); + + assert_eq!( + tracker.can_receive( + ValidatorIndex(5), + ValidatorIndex(5), + CompactStatement::Seconded(hash_a), + ), + Ok(Accept::Ok), + ); + tracker.note_received( + ValidatorIndex(5), + ValidatorIndex(5), + CompactStatement::Seconded(hash_a), + ); + + assert_eq!( + tracker.can_receive( + ValidatorIndex(5), + ValidatorIndex(5), + CompactStatement::Seconded(hash_b), + ), + Ok(Accept::Ok), + ); + tracker.note_received( + ValidatorIndex(5), + ValidatorIndex(5), + CompactStatement::Seconded(hash_b), + ); + + assert_eq!( + tracker.can_receive( + ValidatorIndex(200), + ValidatorIndex(5), + CompactStatement::Seconded(hash_c), + ), + Ok(Accept::WithPrejudice), + ); + } + + #[test] + fn rejects_duplicates() { + let group = + vec![ValidatorIndex(5), ValidatorIndex(200), ValidatorIndex(24), ValidatorIndex(146)]; + + let seconding_limit = 2; + let hash_a = CandidateHash(Hash::repeat_byte(1)); + + let mut tracker = ClusterTracker::new(group, seconding_limit).expect("not empty"); + + tracker.note_received( + ValidatorIndex(5), + ValidatorIndex(5), + CompactStatement::Seconded(hash_a), + ); + + tracker.note_received( + ValidatorIndex(5), + ValidatorIndex(200), + CompactStatement::Valid(hash_a), + ); + + assert_eq!( + tracker.can_receive( + ValidatorIndex(5), + ValidatorIndex(5), + CompactStatement::Seconded(hash_a), + ), + Err(RejectIncoming::Duplicate), + ); + + assert_eq!( + tracker.can_receive( + ValidatorIndex(5), + ValidatorIndex(200), + CompactStatement::Valid(hash_a), + ), + Err(RejectIncoming::Duplicate), + ); + } + + #[test] + fn rejects_incoming_valid_without_seconded() { + let group = + vec![ValidatorIndex(5), ValidatorIndex(200), ValidatorIndex(24), ValidatorIndex(146)]; + + let seconding_limit = 2; + + let tracker = ClusterTracker::new(group, seconding_limit).expect("not empty"); + + let hash_a = CandidateHash(Hash::repeat_byte(1)); + + assert_eq!( + tracker.can_receive( + ValidatorIndex(5), + ValidatorIndex(5), + CompactStatement::Valid(hash_a), + ), + Err(RejectIncoming::CandidateUnknown), + ); + } + + #[test] + fn accepts_incoming_valid_after_receiving_seconded() { + let group = + vec![ValidatorIndex(5), ValidatorIndex(200), ValidatorIndex(24), ValidatorIndex(146)]; + + let seconding_limit = 2; + + let mut tracker = ClusterTracker::new(group.clone(), seconding_limit).expect("not empty"); + let hash_a = CandidateHash(Hash::repeat_byte(1)); + + tracker.note_received( + ValidatorIndex(5), + ValidatorIndex(200), + CompactStatement::Seconded(hash_a), + ); + + assert_eq!( + tracker.can_receive( + ValidatorIndex(5), + ValidatorIndex(5), + CompactStatement::Valid(hash_a), + ), + Ok(Accept::Ok) + ); + } + + #[test] + fn accepts_incoming_valid_after_outgoing_seconded() { + let group = + vec![ValidatorIndex(5), ValidatorIndex(200), ValidatorIndex(24), ValidatorIndex(146)]; + + let seconding_limit = 2; + + let mut tracker = ClusterTracker::new(group.clone(), seconding_limit).expect("not empty"); + let hash_a = CandidateHash(Hash::repeat_byte(1)); + + tracker.note_sent( + ValidatorIndex(5), + ValidatorIndex(200), + CompactStatement::Seconded(hash_a), + ); + + assert_eq!( + tracker.can_receive( + ValidatorIndex(5), + ValidatorIndex(5), + CompactStatement::Valid(hash_a), + ), + Ok(Accept::Ok) + ); + } + + #[test] + fn cannot_send_too_many_seconded_even_to_multiple_peers() { + let group = + vec![ValidatorIndex(5), ValidatorIndex(200), ValidatorIndex(24), ValidatorIndex(146)]; + + let seconding_limit = 2; + + let mut tracker = ClusterTracker::new(group.clone(), seconding_limit).expect("not empty"); + let hash_a = CandidateHash(Hash::repeat_byte(1)); + let hash_b = CandidateHash(Hash::repeat_byte(2)); + let hash_c = CandidateHash(Hash::repeat_byte(3)); + + tracker.note_sent( + ValidatorIndex(200), + ValidatorIndex(5), + CompactStatement::Seconded(hash_a), + ); + + tracker.note_sent( + ValidatorIndex(200), + ValidatorIndex(5), + CompactStatement::Seconded(hash_b), + ); + + assert_eq!( + tracker.can_send( + ValidatorIndex(200), + ValidatorIndex(5), + CompactStatement::Seconded(hash_c), + ), + Err(RejectOutgoing::ExcessiveSeconded), + ); + + assert_eq!( + tracker.can_send( + ValidatorIndex(24), + ValidatorIndex(5), + CompactStatement::Seconded(hash_c), + ), + Err(RejectOutgoing::ExcessiveSeconded), + ); + } + + #[test] + fn cannot_send_duplicate() { + let group = + vec![ValidatorIndex(5), ValidatorIndex(200), ValidatorIndex(24), ValidatorIndex(146)]; + + let seconding_limit = 2; + + let mut tracker = ClusterTracker::new(group.clone(), seconding_limit).expect("not empty"); + let hash_a = CandidateHash(Hash::repeat_byte(1)); + + tracker.note_sent( + ValidatorIndex(200), + ValidatorIndex(5), + CompactStatement::Seconded(hash_a), + ); + + assert_eq!( + tracker.can_send( + ValidatorIndex(200), + ValidatorIndex(5), + CompactStatement::Seconded(hash_a), + ), + Err(RejectOutgoing::Known), + ); + } + + #[test] + fn cannot_send_what_was_received() { + let group = + vec![ValidatorIndex(5), ValidatorIndex(200), ValidatorIndex(24), ValidatorIndex(146)]; + + let seconding_limit = 2; + + let mut tracker = ClusterTracker::new(group.clone(), seconding_limit).expect("not empty"); + let hash_a = CandidateHash(Hash::repeat_byte(1)); + + tracker.note_received( + ValidatorIndex(200), + ValidatorIndex(5), + CompactStatement::Seconded(hash_a), + ); + + assert_eq!( + tracker.can_send( + ValidatorIndex(200), + ValidatorIndex(5), + CompactStatement::Seconded(hash_a), + ), + Err(RejectOutgoing::Known), + ); + } + + // Ensure statements received with prejudice don't prevent sending later. + #[test] + fn can_send_statements_received_with_prejudice() { + let group = + vec![ValidatorIndex(5), ValidatorIndex(200), ValidatorIndex(24), ValidatorIndex(146)]; + + let seconding_limit = 1; + + let mut tracker = ClusterTracker::new(group.clone(), seconding_limit).expect("not empty"); + let hash_a = CandidateHash(Hash::repeat_byte(1)); + let hash_b = CandidateHash(Hash::repeat_byte(2)); + + assert_eq!( + tracker.can_receive( + ValidatorIndex(200), + ValidatorIndex(5), + CompactStatement::Seconded(hash_a), + ), + Ok(Accept::Ok), + ); + + tracker.note_received( + ValidatorIndex(200), + ValidatorIndex(5), + CompactStatement::Seconded(hash_a), + ); + + assert_eq!( + tracker.can_receive( + ValidatorIndex(24), + ValidatorIndex(5), + CompactStatement::Seconded(hash_b), + ), + Ok(Accept::WithPrejudice), + ); + + tracker.note_received( + ValidatorIndex(24), + ValidatorIndex(5), + CompactStatement::Seconded(hash_b), + ); + + assert_eq!( + tracker.can_send( + ValidatorIndex(24), + ValidatorIndex(5), + CompactStatement::Seconded(hash_a), + ), + Ok(()), + ); + } + + // Test that the `pending_statements` are set whenever we receive a fresh statement. + // + // Also test that pending statements are sorted, with `Seconded` statements in the front. + #[test] + fn pending_statements_set_when_receiving_fresh_statements() { + let group = + vec![ValidatorIndex(5), ValidatorIndex(200), ValidatorIndex(24), ValidatorIndex(146)]; + + let seconding_limit = 1; + + let mut tracker = ClusterTracker::new(group.clone(), seconding_limit).expect("not empty"); + let hash_a = CandidateHash(Hash::repeat_byte(1)); + let hash_b = CandidateHash(Hash::repeat_byte(2)); + + // Receive a 'Seconded' statement for candidate A. + { + assert_eq!( + tracker.can_receive( + ValidatorIndex(200), + ValidatorIndex(5), + CompactStatement::Seconded(hash_a), + ), + Ok(Accept::Ok), + ); + tracker.note_received( + ValidatorIndex(200), + ValidatorIndex(5), + CompactStatement::Seconded(hash_a), + ); + + assert_eq!( + tracker.pending_statements_for(ValidatorIndex(5)), + vec![(ValidatorIndex(5), CompactStatement::Seconded(hash_a))] + ); + assert_eq!(tracker.pending_statements_for(ValidatorIndex(200)), vec![]); + assert_eq!( + tracker.pending_statements_for(ValidatorIndex(24)), + vec![(ValidatorIndex(5), CompactStatement::Seconded(hash_a))] + ); + assert_eq!( + tracker.pending_statements_for(ValidatorIndex(146)), + vec![(ValidatorIndex(5), CompactStatement::Seconded(hash_a))] + ); + } + + // Receive a 'Valid' statement for candidate A. + { + // First, send a `Seconded` statement for the candidate. + assert_eq!( + tracker.can_send( + ValidatorIndex(24), + ValidatorIndex(200), + CompactStatement::Seconded(hash_a) + ), + Ok(()) + ); + tracker.note_sent( + ValidatorIndex(24), + ValidatorIndex(200), + CompactStatement::Seconded(hash_a), + ); + + // We have to see that the candidate is known by the sender, e.g. we sent them 'Seconded' + // above. + assert_eq!( + tracker.can_receive( + ValidatorIndex(24), + ValidatorIndex(200), + CompactStatement::Valid(hash_a), + ), + Ok(Accept::Ok), + ); + tracker.note_received( + ValidatorIndex(24), + ValidatorIndex(200), + CompactStatement::Valid(hash_a), + ); + + assert_eq!( + tracker.pending_statements_for(ValidatorIndex(5)), + vec![ + (ValidatorIndex(5), CompactStatement::Seconded(hash_a)), + (ValidatorIndex(200), CompactStatement::Valid(hash_a)) + ] + ); + assert_eq!( + tracker.pending_statements_for(ValidatorIndex(200)), + vec![(ValidatorIndex(200), CompactStatement::Valid(hash_a))] + ); + assert_eq!( + tracker.pending_statements_for(ValidatorIndex(24)), + vec![(ValidatorIndex(5), CompactStatement::Seconded(hash_a))] + ); + assert_eq!( + tracker.pending_statements_for(ValidatorIndex(146)), + vec![ + (ValidatorIndex(5), CompactStatement::Seconded(hash_a)), + (ValidatorIndex(200), CompactStatement::Valid(hash_a)) + ] + ); + } + + // Receive a 'Seconded' statement for candidate B. + { + assert_eq!( + tracker.can_receive( + ValidatorIndex(5), + ValidatorIndex(146), + CompactStatement::Seconded(hash_b), + ), + Ok(Accept::Ok), + ); + tracker.note_received( + ValidatorIndex(5), + ValidatorIndex(146), + CompactStatement::Seconded(hash_b), + ); + + assert_eq!( + tracker.pending_statements_for(ValidatorIndex(5)), + vec![ + (ValidatorIndex(5), CompactStatement::Seconded(hash_a)), + (ValidatorIndex(200), CompactStatement::Valid(hash_a)) + ] + ); + assert_eq!( + tracker.pending_statements_for(ValidatorIndex(200)), + vec![ + (ValidatorIndex(146), CompactStatement::Seconded(hash_b)), + (ValidatorIndex(200), CompactStatement::Valid(hash_a)), + ] + ); + { + let mut pending_statements = tracker.pending_statements_for(ValidatorIndex(24)); + pending_statements.sort(); + assert_eq!( + pending_statements, + vec![ + (ValidatorIndex(5), CompactStatement::Seconded(hash_a)), + (ValidatorIndex(146), CompactStatement::Seconded(hash_b)) + ], + ); + } + { + let mut pending_statements = tracker.pending_statements_for(ValidatorIndex(146)); + pending_statements.sort(); + assert_eq!( + pending_statements, + vec![ + (ValidatorIndex(5), CompactStatement::Seconded(hash_a)), + (ValidatorIndex(146), CompactStatement::Seconded(hash_b)), + (ValidatorIndex(200), CompactStatement::Valid(hash_a)), + ] + ); + } + } + } + + // Test that the `pending_statements` are updated when we send or receive statements from others + // in the cluster. + #[test] + fn pending_statements_updated_when_sending_statements() { + let group = + vec![ValidatorIndex(5), ValidatorIndex(200), ValidatorIndex(24), ValidatorIndex(146)]; + + let seconding_limit = 1; + + let mut tracker = ClusterTracker::new(group.clone(), seconding_limit).expect("not empty"); + let hash_a = CandidateHash(Hash::repeat_byte(1)); + let hash_b = CandidateHash(Hash::repeat_byte(2)); + + // Receive a 'Seconded' statement for candidate A. + { + assert_eq!( + tracker.can_receive( + ValidatorIndex(200), + ValidatorIndex(5), + CompactStatement::Seconded(hash_a), + ), + Ok(Accept::Ok), + ); + tracker.note_received( + ValidatorIndex(200), + ValidatorIndex(5), + CompactStatement::Seconded(hash_a), + ); + + // Pending statements should be updated. + assert_eq!( + tracker.pending_statements_for(ValidatorIndex(5)), + vec![(ValidatorIndex(5), CompactStatement::Seconded(hash_a))] + ); + assert_eq!(tracker.pending_statements_for(ValidatorIndex(200)), vec![]); + assert_eq!( + tracker.pending_statements_for(ValidatorIndex(24)), + vec![(ValidatorIndex(5), CompactStatement::Seconded(hash_a))] + ); + assert_eq!( + tracker.pending_statements_for(ValidatorIndex(146)), + vec![(ValidatorIndex(5), CompactStatement::Seconded(hash_a))] + ); + } + + // Receive a 'Valid' statement for candidate B. + { + // First, send a `Seconded` statement for the candidate. + assert_eq!( + tracker.can_send( + ValidatorIndex(24), + ValidatorIndex(200), + CompactStatement::Seconded(hash_b) + ), + Ok(()) + ); + tracker.note_sent( + ValidatorIndex(24), + ValidatorIndex(200), + CompactStatement::Seconded(hash_b), + ); + + // We have to see the candidate is known by the sender, e.g. we sent them 'Seconded'. + assert_eq!( + tracker.can_receive( + ValidatorIndex(24), + ValidatorIndex(200), + CompactStatement::Valid(hash_b), + ), + Ok(Accept::Ok), + ); + tracker.note_received( + ValidatorIndex(24), + ValidatorIndex(200), + CompactStatement::Valid(hash_b), + ); + + // Pending statements should be updated. + assert_eq!( + tracker.pending_statements_for(ValidatorIndex(5)), + vec![ + (ValidatorIndex(5), CompactStatement::Seconded(hash_a)), + (ValidatorIndex(200), CompactStatement::Valid(hash_b)) + ] + ); + assert_eq!( + tracker.pending_statements_for(ValidatorIndex(200)), + vec![(ValidatorIndex(200), CompactStatement::Valid(hash_b))] + ); + assert_eq!( + tracker.pending_statements_for(ValidatorIndex(24)), + vec![(ValidatorIndex(5), CompactStatement::Seconded(hash_a))] + ); + assert_eq!( + tracker.pending_statements_for(ValidatorIndex(146)), + vec![ + (ValidatorIndex(5), CompactStatement::Seconded(hash_a)), + (ValidatorIndex(200), CompactStatement::Valid(hash_b)) + ] + ); + } + + // Send a 'Seconded' statement. + { + assert_eq!( + tracker.can_send( + ValidatorIndex(5), + ValidatorIndex(5), + CompactStatement::Seconded(hash_a) + ), + Ok(()) + ); + tracker.note_sent( + ValidatorIndex(5), + ValidatorIndex(5), + CompactStatement::Seconded(hash_a), + ); + + // Pending statements should be updated. + assert_eq!( + tracker.pending_statements_for(ValidatorIndex(5)), + vec![(ValidatorIndex(200), CompactStatement::Valid(hash_b))] + ); + assert_eq!( + tracker.pending_statements_for(ValidatorIndex(200)), + vec![(ValidatorIndex(200), CompactStatement::Valid(hash_b))] + ); + assert_eq!( + tracker.pending_statements_for(ValidatorIndex(24)), + vec![(ValidatorIndex(5), CompactStatement::Seconded(hash_a))] + ); + assert_eq!( + tracker.pending_statements_for(ValidatorIndex(146)), + vec![ + (ValidatorIndex(5), CompactStatement::Seconded(hash_a)), + (ValidatorIndex(200), CompactStatement::Valid(hash_b)) + ] + ); + } + + // Send a 'Valid' statement. + { + // First, send a `Seconded` statement for the candidate. + assert_eq!( + tracker.can_send( + ValidatorIndex(5), + ValidatorIndex(200), + CompactStatement::Seconded(hash_b) + ), + Ok(()) + ); + tracker.note_sent( + ValidatorIndex(5), + ValidatorIndex(200), + CompactStatement::Seconded(hash_b), + ); + + // We have to see that the candidate is known by the sender, e.g. we sent them 'Seconded' + // above. + assert_eq!( + tracker.can_send( + ValidatorIndex(5), + ValidatorIndex(200), + CompactStatement::Valid(hash_b) + ), + Ok(()) + ); + tracker.note_sent( + ValidatorIndex(5), + ValidatorIndex(200), + CompactStatement::Valid(hash_b), + ); + + // Pending statements should be updated. + assert_eq!(tracker.pending_statements_for(ValidatorIndex(5)), vec![]); + assert_eq!( + tracker.pending_statements_for(ValidatorIndex(200)), + vec![(ValidatorIndex(200), CompactStatement::Valid(hash_b))] + ); + assert_eq!( + tracker.pending_statements_for(ValidatorIndex(24)), + vec![(ValidatorIndex(5), CompactStatement::Seconded(hash_a))] + ); + assert_eq!( + tracker.pending_statements_for(ValidatorIndex(146)), + vec![ + (ValidatorIndex(5), CompactStatement::Seconded(hash_a)), + (ValidatorIndex(200), CompactStatement::Valid(hash_b)) + ] + ); + } + } +} diff --git a/node/network/statement-distribution/src/vstaging/grid.rs b/node/network/statement-distribution/src/vstaging/grid.rs new file mode 100644 index 000000000000..bdcbabffd3e5 --- /dev/null +++ b/node/network/statement-distribution/src/vstaging/grid.rs @@ -0,0 +1,2248 @@ +// Copyright 2022 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! Utilities for handling distribution of backed candidates along the grid (outside the group to +//! the rest of the network). +//! +//! The grid uses the gossip topology defined in [`polkadot_node_network_protocol::grid_topology`]. +//! It defines how messages and statements are forwarded between validators. +//! +//! # Protocol +//! +//! - Once the candidate is backed, produce a 'backed candidate packet' +//! `(CommittedCandidateReceipt, Statements)`. +//! - Members of a backing group produce an announcement of a fully-backed candidate +//! (aka "full manifest") when they are finished. +//! - `BackedCandidateManifest` +//! - Manifests are sent along the grid topology to peers who have the relay-parent +//! in their implicit view. +//! - Only sent by 1st-hop nodes after downloading the backed candidate packet. +//! - The grid topology is a 2-dimensional grid that provides either a 1 +//! or 2-hop path from any originator to any recipient - 1st-hop nodes +//! are those which share either a row or column with the originator, +//! and 2nd-hop nodes are those which share a column or row with that +//! 1st-hop node. +//! - Note that for the purposes of statement distribution, we actually +//! take the union of the routing paths from each validator in a group +//! to the local node to determine the sending and receiving paths. +//! - Ignored when received out-of-topology +//! - On every local view change, members of the backing group rebroadcast the +//! manifest for all candidates under every new relay-parent across the grid. +//! - Nodes should send a `BackedCandidateAcknowledgement(CandidateHash, +//! StatementFilter)` notification to any peer which has sent a manifest, and +//! the candidate has been acquired by other means. +//! - Request/response for the candidate + votes. +//! - Ignore if they are inconsistent with the manifest. +//! - A malicious backing group is capable of producing an unbounded number of +//! backed candidates. +//! - We request the candidate only if the candidate has a hypothetical depth in +//! any of our fragment trees, and: +//! - the seconding validators have not seconded any other candidates at that +//! depth in any of those fragment trees +//! - All members of the group attempt to circulate all statements (in compact form) +//! from the rest of the group on candidates that have already been backed. +//! - They do this via the grid topology. +//! - They add the statements to their backed candidate packet for future +//! requestors, and also: +//! - send the statement to any peer, which: +//! - we advertised the backed candidate to (sent manifest), and: +//! - has previously & successfully requested the backed candidate packet, +//! or: +//! - which has sent a `BackedCandidateAcknowledgement` +//! - 1st-hop nodes do the same thing + +use polkadot_node_network_protocol::{ + grid_topology::SessionGridTopology, vstaging::StatementFilter, +}; +use polkadot_primitives::vstaging::{ + CandidateHash, CompactStatement, GroupIndex, Hash, ValidatorIndex, +}; + +use std::collections::{ + hash_map::{Entry, HashMap}, + HashSet, +}; + +use bitvec::{order::Lsb0, slice::BitSlice}; + +use super::{groups::Groups, LOG_TARGET}; + +/// Our local view of a subset of the grid topology organized around a specific validator +/// group. +/// +/// This tracks which authorities we expect to communicate with concerning +/// candidates from the group. This includes both the authorities we are +/// expected to send to as well as the authorities we expect to receive from. +/// +/// In the case that this group is the group that we are locally assigned to, +/// the 'receiving' side will be empty. +#[derive(Debug, PartialEq)] +struct GroupSubView { + // validators we are 'sending' to. + sending: HashSet, + // validators we are 'receiving' from. + receiving: HashSet, +} + +/// Our local view of the topology for a session, as it pertains to backed +/// candidate distribution. +#[derive(Debug)] +pub struct SessionTopologyView { + group_views: HashMap, +} + +impl SessionTopologyView { + /// Returns an iterator over all validator indices from the group who are allowed to + /// send us manifests of the given kind. + pub fn iter_sending_for_group( + &self, + group: GroupIndex, + kind: ManifestKind, + ) -> impl Iterator + '_ { + self.group_views.get(&group).into_iter().flat_map(move |sub| match kind { + ManifestKind::Full => sub.receiving.iter().cloned(), + ManifestKind::Acknowledgement => sub.sending.iter().cloned(), + }) + } +} + +/// Build a view of the topology for the session. +/// For groups that we are part of: we receive from nobody and send to our X/Y peers. +/// For groups that we are not part of: we receive from any validator in the group we share a slice with +/// and send to the corresponding X/Y slice in the other dimension. +/// For any validators we don't share a slice with, we receive from the nodes +/// which share a slice with them. +pub fn build_session_topology<'a>( + groups: impl IntoIterator>, + topology: &SessionGridTopology, + our_index: Option, +) -> SessionTopologyView { + let mut view = SessionTopologyView { group_views: HashMap::new() }; + + let our_index = match our_index { + None => return view, + Some(i) => i, + }; + + let our_neighbors = match topology.compute_grid_neighbors_for(our_index) { + None => { + gum::warn!(target: LOG_TARGET, ?our_index, "our index unrecognized in topology?"); + + return view + }, + Some(n) => n, + }; + + for (i, group) in groups.into_iter().enumerate() { + let mut sub_view = GroupSubView { sending: HashSet::new(), receiving: HashSet::new() }; + + if group.contains(&our_index) { + sub_view.sending.extend(our_neighbors.validator_indices_x.iter().cloned()); + sub_view.sending.extend(our_neighbors.validator_indices_y.iter().cloned()); + + // remove all other same-group validators from this set, they are + // in the cluster. + // TODO [now]: test this behavior. + for v in group { + sub_view.sending.remove(v); + } + } else { + for &group_val in group { + // If the validator shares a slice with us, we expect to + // receive from them and send to our neighbors in the other + // dimension. + + if our_neighbors.validator_indices_x.contains(&group_val) { + sub_view.receiving.insert(group_val); + sub_view.sending.extend( + our_neighbors + .validator_indices_y + .iter() + .filter(|v| !group.contains(v)) + .cloned(), + ); + + continue + } + + if our_neighbors.validator_indices_y.contains(&group_val) { + sub_view.receiving.insert(group_val); + sub_view.sending.extend( + our_neighbors + .validator_indices_x + .iter() + .filter(|v| !group.contains(v)) + .cloned(), + ); + + continue + } + + // If they don't share a slice with us, we don't send to anybody + // but receive from any peers sharing a dimension with both of us + let their_neighbors = match topology.compute_grid_neighbors_for(group_val) { + None => { + gum::warn!( + target: LOG_TARGET, + index = ?group_val, + "validator index unrecognized in topology?" + ); + + continue + }, + Some(n) => n, + }; + + // their X, our Y + for potential_link in &their_neighbors.validator_indices_x { + if our_neighbors.validator_indices_y.contains(potential_link) { + sub_view.receiving.insert(*potential_link); + break // one max + } + } + + // their Y, our X + for potential_link in &their_neighbors.validator_indices_y { + if our_neighbors.validator_indices_x.contains(potential_link) { + sub_view.receiving.insert(*potential_link); + break // one max + } + } + } + } + + view.group_views.insert(GroupIndex(i as _), sub_view); + } + + view +} + +/// The kind of backed candidate manifest we should send to a remote peer. +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum ManifestKind { + /// Full manifests contain information about the candidate and should be sent + /// to peers which aren't guaranteed to have the candidate already. + Full, + /// Acknowledgement manifests omit information which is implicit in the candidate + /// itself, and should be sent to peers which are guaranteed to have the candidate + /// already. + Acknowledgement, +} + +/// A tracker of knowledge from authorities within the grid for a particular +/// relay-parent. +#[derive(Default)] +pub struct GridTracker { + received: HashMap, + confirmed_backed: HashMap, + unconfirmed: HashMap>, + pending_manifests: HashMap>, + + // maps target to (originator, statement) pairs. + pending_statements: HashMap>, +} + +impl GridTracker { + /// Attempt to import a manifest advertised by a remote peer. + /// + /// This checks whether the peer is allowed to send us manifests + /// about this group at this relay-parent. This also does sanity + /// checks on the format of the manifest and the amount of votes + /// it contains. It has effects on the stored state only when successful. + /// + /// This returns a `bool` on success, which if true indicates that an acknowledgement is + /// to be sent in response to the received manifest. This only occurs when the + /// candidate is already known to be confirmed and backed. + pub fn import_manifest( + &mut self, + session_topology: &SessionTopologyView, + groups: &Groups, + candidate_hash: CandidateHash, + seconding_limit: usize, + manifest: ManifestSummary, + kind: ManifestKind, + sender: ValidatorIndex, + ) -> Result { + let claimed_group_index = manifest.claimed_group_index; + + let group_topology = match session_topology.group_views.get(&manifest.claimed_group_index) { + None => return Err(ManifestImportError::Disallowed), + Some(g) => g, + }; + + let receiving_from = group_topology.receiving.contains(&sender); + let sending_to = group_topology.sending.contains(&sender); + let manifest_allowed = match kind { + // Peers can send manifests _if_: + // * They are in the receiving set for the group AND the manifest is full OR + // * They are in the sending set for the group AND we have sent them + // a manifest AND the received manifest is partial. + ManifestKind::Full => receiving_from, + ManifestKind::Acknowledgement => + sending_to && + self.confirmed_backed + .get(&candidate_hash) + .map_or(false, |c| c.has_sent_manifest_to(sender)), + }; + + if !manifest_allowed { + return Err(ManifestImportError::Disallowed) + } + + let (group_size, backing_threshold) = + match groups.get_size_and_backing_threshold(manifest.claimed_group_index) { + Some(x) => x, + None => return Err(ManifestImportError::Malformed), + }; + + let remote_knowledge = manifest.statement_knowledge.clone(); + + if !remote_knowledge.has_len(group_size) { + return Err(ManifestImportError::Malformed) + } + + if !remote_knowledge.has_seconded() { + return Err(ManifestImportError::Malformed) + } + + // ensure votes are sufficient to back. + let votes = remote_knowledge.backing_validators(); + + if votes < backing_threshold { + return Err(ManifestImportError::Insufficient) + } + + self.received.entry(sender).or_default().import_received( + group_size, + seconding_limit, + candidate_hash, + manifest, + )?; + + let mut ack = false; + if let Some(confirmed) = self.confirmed_backed.get_mut(&candidate_hash) { + if receiving_from && !confirmed.has_sent_manifest_to(sender) { + // due to checks above, the manifest `kind` is guaranteed to be `Full` + self.pending_manifests + .entry(sender) + .or_default() + .insert(candidate_hash, ManifestKind::Acknowledgement); + + ack = true; + } + + // add all statements in local_knowledge & !remote_knowledge + // to `pending_statements` for this validator. + confirmed.manifest_received_from(sender, remote_knowledge); + if let Some(pending_statements) = confirmed.pending_statements(sender) { + self.pending_statements.entry(sender).or_default().extend( + decompose_statement_filter( + groups, + claimed_group_index, + candidate_hash, + &pending_statements, + ), + ); + } + } else { + // `received` prevents conflicting manifests so this is max 1 per validator. + self.unconfirmed + .entry(candidate_hash) + .or_default() + .push((sender, claimed_group_index)) + } + + Ok(ack) + } + + /// Add a new backed candidate to the tracker. This yields + /// a list of validators which we should either advertise to + /// or signal that we know the candidate, along with the corresponding + /// type of manifest we should send. + pub fn add_backed_candidate( + &mut self, + session_topology: &SessionTopologyView, + candidate_hash: CandidateHash, + group_index: GroupIndex, + local_knowledge: StatementFilter, + ) -> Vec<(ValidatorIndex, ManifestKind)> { + let c = match self.confirmed_backed.entry(candidate_hash) { + Entry::Occupied(_) => return Vec::new(), + Entry::Vacant(v) => v.insert(KnownBackedCandidate { + group_index, + mutual_knowledge: HashMap::new(), + local_knowledge, + }), + }; + + // Populate the entry with previously unconfirmed manifests. + for (v, claimed_group_index) in + self.unconfirmed.remove(&candidate_hash).into_iter().flatten() + { + if claimed_group_index != group_index { + // This is misbehavior, but is handled more comprehensively elsewhere + continue + } + + let statement_filter = self + .received + .get(&v) + .and_then(|r| r.candidate_statement_filter(&candidate_hash)) + .expect("unconfirmed is only populated by validators who have sent manifest; qed"); + + // No need to send direct statements, because our local knowledge is `None` + c.manifest_received_from(v, statement_filter); + } + + let group_topology = match session_topology.group_views.get(&group_index) { + None => return Vec::new(), + Some(g) => g, + }; + + // advertise onwards and accept received advertisements + + let sending_group_manifests = + group_topology.sending.iter().map(|v| (*v, ManifestKind::Full)); + + let receiving_group_manifests = group_topology.receiving.iter().filter_map(|v| { + if c.has_received_manifest_from(*v) { + Some((*v, ManifestKind::Acknowledgement)) + } else { + None + } + }); + + // Note that order is important: if a validator is part of both the sending + // and receiving groups, we may overwrite a `Full` manifest with a `Acknowledgement` + // one. + for (v, manifest_mode) in sending_group_manifests.chain(receiving_group_manifests) { + self.pending_manifests + .entry(v) + .or_default() + .insert(candidate_hash, manifest_mode); + } + + self.pending_manifests + .iter() + .filter_map(|(v, x)| x.get(&candidate_hash).map(|k| (*v, *k))) + .collect() + } + + /// Note that a backed candidate has been advertised to a + /// given validator. + pub fn manifest_sent_to( + &mut self, + groups: &Groups, + validator_index: ValidatorIndex, + candidate_hash: CandidateHash, + local_knowledge: StatementFilter, + ) { + if let Some(c) = self.confirmed_backed.get_mut(&candidate_hash) { + c.manifest_sent_to(validator_index, local_knowledge); + + if let Some(pending_statements) = c.pending_statements(validator_index) { + self.pending_statements.entry(validator_index).or_default().extend( + decompose_statement_filter( + groups, + c.group_index, + candidate_hash, + &pending_statements, + ), + ); + } + } + + if let Some(x) = self.pending_manifests.get_mut(&validator_index) { + x.remove(&candidate_hash); + } + } + + /// Returns a vector of all candidates pending manifests for the specific validator, and + /// the type of manifest we should send. + pub fn pending_manifests_for( + &self, + validator_index: ValidatorIndex, + ) -> Vec<(CandidateHash, ManifestKind)> { + self.pending_manifests + .get(&validator_index) + .into_iter() + .flat_map(|pending| pending.iter().map(|(c, m)| (*c, *m))) + .collect() + } + + /// Returns a statement filter indicating statements that a given peer + /// is awaiting concerning the given candidate, constrained by the statements + /// we have ourselves. + pub fn pending_statements_for( + &self, + validator_index: ValidatorIndex, + candidate_hash: CandidateHash, + ) -> Option { + self.confirmed_backed + .get(&candidate_hash) + .and_then(|x| x.pending_statements(validator_index)) + } + + /// Returns a vector of all pending statements to the validator, sorted with + /// `Seconded` statements at the front. + /// + /// Statements are in the form `(Originator, Statement Kind)`. + pub fn all_pending_statements_for( + &self, + validator_index: ValidatorIndex, + ) -> Vec<(ValidatorIndex, CompactStatement)> { + let mut v = self + .pending_statements + .get(&validator_index) + .map(|x| x.iter().cloned().collect()) + .unwrap_or(Vec::new()); + + v.sort_by_key(|(_, s)| match s { + CompactStatement::Seconded(_) => 0u32, + CompactStatement::Valid(_) => 1u32, + }); + + v + } + + /// Whether a validator can request a manifest from us. + pub fn can_request(&self, validator: ValidatorIndex, candidate_hash: CandidateHash) -> bool { + self.confirmed_backed.get(&candidate_hash).map_or(false, |c| { + c.has_sent_manifest_to(validator) && !c.has_received_manifest_from(validator) + }) + } + + /// Determine the validators which can send a statement to us by direct broadcast. + pub fn direct_statement_providers( + &self, + groups: &Groups, + originator: ValidatorIndex, + statement: &CompactStatement, + ) -> Vec { + let (g, c_h, kind, in_group) = + match extract_statement_and_group_info(groups, originator, statement) { + None => return Vec::new(), + Some(x) => x, + }; + + self.confirmed_backed + .get(&c_h) + .map(|k| k.direct_statement_senders(g, in_group, kind)) + .unwrap_or_default() + } + + /// Determine the validators which can receive a statement from us by direct + /// broadcast. + pub fn direct_statement_targets( + &self, + groups: &Groups, + originator: ValidatorIndex, + statement: &CompactStatement, + ) -> Vec { + let (g, c_h, kind, in_group) = + match extract_statement_and_group_info(groups, originator, statement) { + None => return Vec::new(), + Some(x) => x, + }; + + self.confirmed_backed + .get(&c_h) + .map(|k| k.direct_statement_recipients(g, in_group, kind)) + .unwrap_or_default() + } + + /// Note that we have learned about a statement. This will update + /// `pending_statements_for` for any relevant validators if actually + /// fresh. + pub fn learned_fresh_statement( + &mut self, + groups: &Groups, + session_topology: &SessionTopologyView, + originator: ValidatorIndex, + statement: &CompactStatement, + ) { + let (g, c_h, kind, in_group) = + match extract_statement_and_group_info(groups, originator, statement) { + None => return, + Some(x) => x, + }; + + let known = match self.confirmed_backed.get_mut(&c_h) { + None => return, + Some(x) => x, + }; + + if !known.note_fresh_statement(in_group, kind) { + return + } + + // Add to `pending_statements` for all validators we communicate with + // who have exchanged manifests. + let all_group_validators = session_topology + .group_views + .get(&g) + .into_iter() + .flat_map(|g| g.sending.iter().chain(g.receiving.iter())); + + for v in all_group_validators { + if known.is_pending_statement(*v, in_group, kind) { + self.pending_statements + .entry(*v) + .or_default() + .insert((originator, statement.clone())); + } + } + } + + /// Note that a direct statement about a given candidate was sent to or + /// received from the given validator. + pub fn sent_or_received_direct_statement( + &mut self, + groups: &Groups, + originator: ValidatorIndex, + counterparty: ValidatorIndex, + statement: &CompactStatement, + ) { + if let Some((_, c_h, kind, in_group)) = + extract_statement_and_group_info(groups, originator, statement) + { + if let Some(known) = self.confirmed_backed.get_mut(&c_h) { + known.sent_or_received_direct_statement(counterparty, in_group, kind); + + if let Some(pending) = self.pending_statements.get_mut(&counterparty) { + pending.remove(&(originator, statement.clone())); + } + } + } + } + + /// Get the advertised statement filter of a validator for a candidate. + pub fn advertised_statements( + &self, + validator: ValidatorIndex, + candidate_hash: &CandidateHash, + ) -> Option { + self.received.get(&validator)?.candidate_statement_filter(candidate_hash) + } + + #[cfg(test)] + fn is_manifest_pending_for( + &self, + validator: ValidatorIndex, + candidate_hash: &CandidateHash, + ) -> Option { + self.pending_manifests + .get(&validator) + .and_then(|m| m.get(candidate_hash)) + .map(|x| *x) + } +} + +fn extract_statement_and_group_info( + groups: &Groups, + originator: ValidatorIndex, + statement: &CompactStatement, +) -> Option<(GroupIndex, CandidateHash, StatementKind, usize)> { + let (statement_kind, candidate_hash) = match statement { + CompactStatement::Seconded(h) => (StatementKind::Seconded, h), + CompactStatement::Valid(h) => (StatementKind::Valid, h), + }; + + let group = match groups.by_validator_index(originator) { + None => return None, + Some(g) => g, + }; + + let index_in_group = groups.get(group)?.iter().position(|v| v == &originator)?; + + Some((group, *candidate_hash, statement_kind, index_in_group)) +} + +fn decompose_statement_filter<'a>( + groups: &'a Groups, + group_index: GroupIndex, + candidate_hash: CandidateHash, + statement_filter: &'a StatementFilter, +) -> impl Iterator + 'a { + groups.get(group_index).into_iter().flat_map(move |g| { + let s = statement_filter + .seconded_in_group + .iter_ones() + .map(|i| g[i]) + .map(move |i| (i, CompactStatement::Seconded(candidate_hash))); + + let v = statement_filter + .validated_in_group + .iter_ones() + .map(|i| g[i]) + .map(move |i| (i, CompactStatement::Valid(candidate_hash))); + + s.chain(v) + }) +} + +/// A summary of a manifest being sent by a counterparty. +#[derive(Clone)] +pub struct ManifestSummary { + /// The claimed parent head data hash of the candidate. + pub claimed_parent_hash: Hash, + /// The claimed group index assigned to the candidate. + pub claimed_group_index: GroupIndex, + /// A statement filter sent alongisde the candidate, communicating + /// knowledge. + pub statement_knowledge: StatementFilter, +} + +/// Errors in importing a manifest. +#[derive(Debug, Clone)] +pub enum ManifestImportError { + /// The manifest conflicts with another, previously sent manifest. + Conflicting, + /// The manifest has overflowed beyond the limits of what the + /// counterparty was allowed to send us. + Overflow, + /// The manifest claims insufficient attestations to achieve the backing + /// threshold. + Insufficient, + /// The manifest is malformed. + Malformed, + /// The manifest was not allowed to be sent. + Disallowed, +} + +/// The knowledge we are aware of counterparties having of manifests. +#[derive(Default)] +struct ReceivedManifests { + received: HashMap, + // group -> seconded counts. + seconded_counts: HashMap>, +} + +impl ReceivedManifests { + fn candidate_statement_filter( + &self, + candidate_hash: &CandidateHash, + ) -> Option { + self.received.get(candidate_hash).map(|m| m.statement_knowledge.clone()) + } + + /// Attempt to import a received manifest from a counterparty. + /// + /// This will reject manifests which are either duplicate, conflicting, + /// or imply an irrational amount of `Seconded` statements. + /// + /// This assumes that the manifest has already been checked for + /// validity - i.e. that the bitvecs match the claimed group in size + /// and that the manifest includes at least one `Seconded` + /// attestation and includes enough attestations for the candidate + /// to be backed. + /// + /// This also should only be invoked when we are intended to track + /// the knowledge of this peer as determined by the [`SessionTopology`]. + fn import_received( + &mut self, + group_size: usize, + seconding_limit: usize, + candidate_hash: CandidateHash, + manifest_summary: ManifestSummary, + ) -> Result<(), ManifestImportError> { + match self.received.entry(candidate_hash) { + Entry::Occupied(mut e) => { + // occupied entry. + + // filter out clearly conflicting data. + { + let prev = e.get(); + if prev.claimed_group_index != manifest_summary.claimed_group_index { + return Err(ManifestImportError::Conflicting) + } + + if prev.claimed_parent_hash != manifest_summary.claimed_parent_hash { + return Err(ManifestImportError::Conflicting) + } + + if !manifest_summary + .statement_knowledge + .seconded_in_group + .contains(&prev.statement_knowledge.seconded_in_group) + { + return Err(ManifestImportError::Conflicting) + } + + if !manifest_summary + .statement_knowledge + .validated_in_group + .contains(&prev.statement_knowledge.validated_in_group) + { + return Err(ManifestImportError::Conflicting) + } + + let mut fresh_seconded = + manifest_summary.statement_knowledge.seconded_in_group.clone(); + fresh_seconded |= &prev.statement_knowledge.seconded_in_group; + + let within_limits = updating_ensure_within_seconding_limit( + &mut self.seconded_counts, + manifest_summary.claimed_group_index, + group_size, + seconding_limit, + &fresh_seconded, + ); + + if !within_limits { + return Err(ManifestImportError::Overflow) + } + } + + // All checks passed. Overwrite: guaranteed to be + // superset. + *e.get_mut() = manifest_summary; + Ok(()) + }, + Entry::Vacant(e) => { + let within_limits = updating_ensure_within_seconding_limit( + &mut self.seconded_counts, + manifest_summary.claimed_group_index, + group_size, + seconding_limit, + &manifest_summary.statement_knowledge.seconded_in_group, + ); + + if within_limits { + e.insert(manifest_summary); + Ok(()) + } else { + Err(ManifestImportError::Overflow) + } + }, + } + } +} + +// updates validator-seconded records but only if the new statements +// are OK. returns `true` if alright and `false` otherwise. +// +// The seconding limit is a per-validator limit. It ensures an upper bound on the total number of +// candidates entering the system. +fn updating_ensure_within_seconding_limit( + seconded_counts: &mut HashMap>, + group_index: GroupIndex, + group_size: usize, + seconding_limit: usize, + new_seconded: &BitSlice, +) -> bool { + if seconding_limit == 0 { + return false + } + + // due to the check above, if this was non-existent this function will + // always return `true`. + let counts = seconded_counts.entry(group_index).or_insert_with(|| vec![0; group_size]); + + for i in new_seconded.iter_ones() { + if counts[i] == seconding_limit { + return false + } + } + + for i in new_seconded.iter_ones() { + counts[i] += 1; + } + + true +} + +#[derive(Debug, Clone, Copy)] +enum StatementKind { + Seconded, + Valid, +} + +trait FilterQuery { + fn contains(&self, index: usize, statement_kind: StatementKind) -> bool; + fn set(&mut self, index: usize, statement_kind: StatementKind); +} + +impl FilterQuery for StatementFilter { + fn contains(&self, index: usize, statement_kind: StatementKind) -> bool { + match statement_kind { + StatementKind::Seconded => self.seconded_in_group.get(index).map_or(false, |x| *x), + StatementKind::Valid => self.validated_in_group.get(index).map_or(false, |x| *x), + } + } + + fn set(&mut self, index: usize, statement_kind: StatementKind) { + let b = match statement_kind { + StatementKind::Seconded => self.seconded_in_group.get_mut(index), + StatementKind::Valid => self.validated_in_group.get_mut(index), + }; + + if let Some(mut b) = b { + *b = true; + } + } +} + +/// Knowledge that we have about a remote peer concerning a candidate, and that they have about us +/// concerning the candidate. +#[derive(Debug, Clone)] +struct MutualKnowledge { + /// Knowledge the remote peer has about the candidate, as far as we're aware. + /// `Some` only if they have advertised, acknowledged, or requested the candidate. + remote_knowledge: Option, + /// Knowledge we have indicated to the remote peer about the candidate. + /// `Some` only if we have advertised, acknowledged, or requested the candidate + /// from them. + local_knowledge: Option, +} + +// A utility struct for keeping track of metadata about candidates +// we have confirmed as having been backed. +#[derive(Debug, Clone)] +struct KnownBackedCandidate { + group_index: GroupIndex, + local_knowledge: StatementFilter, + mutual_knowledge: HashMap, +} + +impl KnownBackedCandidate { + fn has_received_manifest_from(&self, validator: ValidatorIndex) -> bool { + self.mutual_knowledge + .get(&validator) + .map_or(false, |k| k.remote_knowledge.is_some()) + } + + fn has_sent_manifest_to(&self, validator: ValidatorIndex) -> bool { + self.mutual_knowledge + .get(&validator) + .map_or(false, |k| k.local_knowledge.is_some()) + } + + fn manifest_sent_to(&mut self, validator: ValidatorIndex, local_knowledge: StatementFilter) { + let k = self + .mutual_knowledge + .entry(validator) + .or_insert_with(|| MutualKnowledge { remote_knowledge: None, local_knowledge: None }); + + k.local_knowledge = Some(local_knowledge); + } + + fn manifest_received_from( + &mut self, + validator: ValidatorIndex, + remote_knowledge: StatementFilter, + ) { + let k = self + .mutual_knowledge + .entry(validator) + .or_insert_with(|| MutualKnowledge { remote_knowledge: None, local_knowledge: None }); + + k.remote_knowledge = Some(remote_knowledge); + } + + fn direct_statement_senders( + &self, + group_index: GroupIndex, + originator_index_in_group: usize, + statement_kind: StatementKind, + ) -> Vec { + if group_index != self.group_index { + return Vec::new() + } + + self.mutual_knowledge + .iter() + .filter(|(_, k)| k.remote_knowledge.is_some()) + .filter(|(_, k)| { + k.local_knowledge + .as_ref() + .map_or(false, |r| !r.contains(originator_index_in_group, statement_kind)) + }) + .map(|(v, _)| *v) + .collect() + } + + fn direct_statement_recipients( + &self, + group_index: GroupIndex, + originator_index_in_group: usize, + statement_kind: StatementKind, + ) -> Vec { + if group_index != self.group_index { + return Vec::new() + } + + self.mutual_knowledge + .iter() + .filter(|(_, k)| k.local_knowledge.is_some()) + .filter(|(_, k)| { + k.remote_knowledge + .as_ref() + .map_or(false, |r| !r.contains(originator_index_in_group, statement_kind)) + }) + .map(|(v, _)| *v) + .collect() + } + + fn note_fresh_statement( + &mut self, + statement_index_in_group: usize, + statement_kind: StatementKind, + ) -> bool { + let really_fresh = !self.local_knowledge.contains(statement_index_in_group, statement_kind); + self.local_knowledge.set(statement_index_in_group, statement_kind); + + really_fresh + } + + fn sent_or_received_direct_statement( + &mut self, + validator: ValidatorIndex, + statement_index_in_group: usize, + statement_kind: StatementKind, + ) { + if let Some(k) = self.mutual_knowledge.get_mut(&validator) { + if let (Some(r), Some(l)) = (k.remote_knowledge.as_mut(), k.local_knowledge.as_mut()) { + r.set(statement_index_in_group, statement_kind); + l.set(statement_index_in_group, statement_kind); + } + } + } + + fn is_pending_statement( + &self, + validator: ValidatorIndex, + statement_index_in_group: usize, + statement_kind: StatementKind, + ) -> bool { + // existence of both remote & local knowledge indicate we have exchanged + // manifests. + // then, everything that is not in the remote knowledge is pending + self.mutual_knowledge + .get(&validator) + .filter(|k| k.local_knowledge.is_some()) + .and_then(|k| k.remote_knowledge.as_ref()) + .map(|k| !k.contains(statement_index_in_group, statement_kind)) + .unwrap_or(false) + } + + fn pending_statements(&self, validator: ValidatorIndex) -> Option { + // existence of both remote & local knowledge indicate we have exchanged + // manifests. + // then, everything that is not in the remote knowledge is pending, and we + // further limit this by what is in the local knowledge itself. we use the + // full local knowledge, as the local knowledge stored here may be outdated. + let full_local = &self.local_knowledge; + + self.mutual_knowledge + .get(&validator) + .filter(|k| k.local_knowledge.is_some()) + .and_then(|k| k.remote_knowledge.as_ref()) + .map(|remote| StatementFilter { + seconded_in_group: full_local.seconded_in_group.clone() & + !remote.seconded_in_group.clone(), + validated_in_group: full_local.validated_in_group.clone() & + !remote.validated_in_group.clone(), + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use assert_matches::assert_matches; + use polkadot_node_network_protocol::grid_topology::TopologyPeerInfo; + use sp_authority_discovery::AuthorityPair as AuthorityDiscoveryPair; + use sp_core::crypto::Pair as PairT; + + fn dummy_groups(group_size: usize) -> Groups { + let groups = vec![(0..(group_size as u32)).map(ValidatorIndex).collect()].into(); + + Groups::new(groups) + } + + #[test] + fn topology_empty_for_no_index() { + let base_topology = SessionGridTopology::new( + vec![0, 1, 2], + vec![ + TopologyPeerInfo { + peer_ids: Vec::new(), + validator_index: ValidatorIndex(0), + discovery_id: AuthorityDiscoveryPair::generate().0.public(), + }, + TopologyPeerInfo { + peer_ids: Vec::new(), + validator_index: ValidatorIndex(1), + discovery_id: AuthorityDiscoveryPair::generate().0.public(), + }, + TopologyPeerInfo { + peer_ids: Vec::new(), + validator_index: ValidatorIndex(2), + discovery_id: AuthorityDiscoveryPair::generate().0.public(), + }, + ], + ); + + let t = build_session_topology( + &[vec![ValidatorIndex(0)], vec![ValidatorIndex(1)], vec![ValidatorIndex(2)]], + &base_topology, + None, + ); + + assert!(t.group_views.is_empty()); + } + + #[test] + fn topology_setup() { + let base_topology = SessionGridTopology::new( + (0..9).collect(), + (0..9) + .map(|i| TopologyPeerInfo { + peer_ids: Vec::new(), + validator_index: ValidatorIndex(i), + discovery_id: AuthorityDiscoveryPair::generate().0.public(), + }) + .collect(), + ); + + let t = build_session_topology( + &[ + vec![ValidatorIndex(0), ValidatorIndex(3), ValidatorIndex(6)], + vec![ValidatorIndex(4), ValidatorIndex(2), ValidatorIndex(7)], + vec![ValidatorIndex(8), ValidatorIndex(5), ValidatorIndex(1)], + ], + &base_topology, + Some(ValidatorIndex(0)), + ); + + assert_eq!(t.group_views.len(), 3); + + // 0 1 2 + // 3 4 5 + // 6 7 8 + + // our group: we send to all row/column neighbors which are not in our + // group and receive nothing. + assert_eq!( + t.group_views.get(&GroupIndex(0)).unwrap().sending, + vec![1, 2].into_iter().map(ValidatorIndex).collect::>(), + ); + assert_eq!(t.group_views.get(&GroupIndex(0)).unwrap().receiving, HashSet::new(),); + + // we share a row with '2' and have indirect connections to '4' and '7'. + + assert_eq!( + t.group_views.get(&GroupIndex(1)).unwrap().sending, + vec![3, 6].into_iter().map(ValidatorIndex).collect::>(), + ); + assert_eq!( + t.group_views.get(&GroupIndex(1)).unwrap().receiving, + vec![1, 2, 3, 6].into_iter().map(ValidatorIndex).collect::>(), + ); + + // we share a row with '1' and have indirect connections to '5' and '8'. + + assert_eq!( + t.group_views.get(&GroupIndex(2)).unwrap().sending, + vec![3, 6].into_iter().map(ValidatorIndex).collect::>(), + ); + assert_eq!( + t.group_views.get(&GroupIndex(2)).unwrap().receiving, + vec![1, 2, 3, 6].into_iter().map(ValidatorIndex).collect::>(), + ); + } + + #[test] + fn knowledge_rejects_conflicting_manifest() { + let mut knowledge = ReceivedManifests::default(); + + let expected_manifest_summary = ManifestSummary { + claimed_parent_hash: Hash::repeat_byte(2), + claimed_group_index: GroupIndex(0), + statement_knowledge: StatementFilter { + seconded_in_group: bitvec::bitvec![u8, Lsb0; 1, 1, 0], + validated_in_group: bitvec::bitvec![u8, Lsb0; 0, 1, 1], + }, + }; + + knowledge + .import_received( + 3, + 2, + CandidateHash(Hash::repeat_byte(1)), + expected_manifest_summary.clone(), + ) + .unwrap(); + + // conflicting group + + let mut s = expected_manifest_summary.clone(); + s.claimed_group_index = GroupIndex(1); + assert_matches!( + knowledge.import_received(3, 2, CandidateHash(Hash::repeat_byte(1)), s,), + Err(ManifestImportError::Conflicting) + ); + + // conflicting parent hash + + let mut s = expected_manifest_summary.clone(); + s.claimed_parent_hash = Hash::repeat_byte(3); + assert_matches!( + knowledge.import_received(3, 2, CandidateHash(Hash::repeat_byte(1)), s,), + Err(ManifestImportError::Conflicting) + ); + + // conflicting seconded statements bitfield + + let mut s = expected_manifest_summary.clone(); + s.statement_knowledge.seconded_in_group = bitvec::bitvec![u8, Lsb0; 0, 1, 0]; + assert_matches!( + knowledge.import_received(3, 2, CandidateHash(Hash::repeat_byte(1)), s,), + Err(ManifestImportError::Conflicting) + ); + + // conflicting valid statements bitfield + + let mut s = expected_manifest_summary.clone(); + s.statement_knowledge.validated_in_group = bitvec::bitvec![u8, Lsb0; 0, 1, 0]; + assert_matches!( + knowledge.import_received(3, 2, CandidateHash(Hash::repeat_byte(1)), s,), + Err(ManifestImportError::Conflicting) + ); + } + + // Make sure we don't import manifests that would put a validator in a group over the limit of + // candidates they are allowed to second (aka seconding limit). + #[test] + fn reject_overflowing_manifests() { + let mut knowledge = ReceivedManifests::default(); + knowledge + .import_received( + 3, + 2, + CandidateHash(Hash::repeat_byte(1)), + ManifestSummary { + claimed_parent_hash: Hash::repeat_byte(0xA), + claimed_group_index: GroupIndex(0), + statement_knowledge: StatementFilter { + seconded_in_group: bitvec::bitvec![u8, Lsb0; 1, 1, 0], + validated_in_group: bitvec::bitvec![u8, Lsb0; 0, 1, 1], + }, + }, + ) + .unwrap(); + + knowledge + .import_received( + 3, + 2, + CandidateHash(Hash::repeat_byte(2)), + ManifestSummary { + claimed_parent_hash: Hash::repeat_byte(0xB), + claimed_group_index: GroupIndex(0), + statement_knowledge: StatementFilter { + seconded_in_group: bitvec::bitvec![u8, Lsb0; 1, 0, 1], + validated_in_group: bitvec::bitvec![u8, Lsb0; 0, 1, 1], + }, + }, + ) + .unwrap(); + + // Reject a seconding validator that is already at the seconding limit. Seconding counts for + // the validators should not be applied. + assert_matches!( + knowledge.import_received( + 3, + 2, + CandidateHash(Hash::repeat_byte(3)), + ManifestSummary { + claimed_parent_hash: Hash::repeat_byte(0xC), + claimed_group_index: GroupIndex(0), + statement_knowledge: StatementFilter { + seconded_in_group: bitvec::bitvec![u8, Lsb0; 1, 1, 1], + validated_in_group: bitvec::bitvec![u8, Lsb0; 0, 1, 1], + } + }, + ), + Err(ManifestImportError::Overflow) + ); + + // Don't reject validators that have seconded less than the limit so far. + knowledge + .import_received( + 3, + 2, + CandidateHash(Hash::repeat_byte(3)), + ManifestSummary { + claimed_parent_hash: Hash::repeat_byte(0xC), + claimed_group_index: GroupIndex(0), + statement_knowledge: StatementFilter { + seconded_in_group: bitvec::bitvec![u8, Lsb0; 0, 1, 1], + validated_in_group: bitvec::bitvec![u8, Lsb0; 0, 1, 1], + }, + }, + ) + .unwrap(); + } + + #[test] + fn reject_disallowed_manifest() { + let mut tracker = GridTracker::default(); + let session_topology = SessionTopologyView { + group_views: vec![( + GroupIndex(0), + GroupSubView { + sending: HashSet::new(), + receiving: vec![ValidatorIndex(0)].into_iter().collect(), + }, + )] + .into_iter() + .collect(), + }; + + let groups = dummy_groups(3); + + let candidate_hash = CandidateHash(Hash::repeat_byte(42)); + + assert_eq!(groups.get_size_and_backing_threshold(GroupIndex(0)), Some((3, 2)),); + + // Known group, disallowed receiving validator. + + assert_matches!( + tracker.import_manifest( + &session_topology, + &groups, + candidate_hash, + 3, + ManifestSummary { + claimed_parent_hash: Hash::repeat_byte(0), + claimed_group_index: GroupIndex(0), + statement_knowledge: StatementFilter { + seconded_in_group: bitvec::bitvec![u8, Lsb0; 0, 1, 0], + validated_in_group: bitvec::bitvec![u8, Lsb0; 1, 0, 1], + } + }, + ManifestKind::Full, + ValidatorIndex(1), + ), + Err(ManifestImportError::Disallowed) + ); + + // Unknown group + + assert_matches!( + tracker.import_manifest( + &session_topology, + &groups, + candidate_hash, + 3, + ManifestSummary { + claimed_parent_hash: Hash::repeat_byte(0), + claimed_group_index: GroupIndex(1), + statement_knowledge: StatementFilter { + seconded_in_group: bitvec::bitvec![u8, Lsb0; 0, 1, 0], + validated_in_group: bitvec::bitvec![u8, Lsb0; 1, 0, 1], + } + }, + ManifestKind::Full, + ValidatorIndex(0), + ), + Err(ManifestImportError::Disallowed) + ); + } + + #[test] + fn reject_malformed_wrong_group_size() { + let mut tracker = GridTracker::default(); + let session_topology = SessionTopologyView { + group_views: vec![( + GroupIndex(0), + GroupSubView { + sending: HashSet::new(), + receiving: vec![ValidatorIndex(0)].into_iter().collect(), + }, + )] + .into_iter() + .collect(), + }; + + let groups = dummy_groups(3); + + let candidate_hash = CandidateHash(Hash::repeat_byte(42)); + + assert_eq!(groups.get_size_and_backing_threshold(GroupIndex(0)), Some((3, 2)),); + + assert_matches!( + tracker.import_manifest( + &session_topology, + &groups, + candidate_hash, + 3, + ManifestSummary { + claimed_parent_hash: Hash::repeat_byte(0), + claimed_group_index: GroupIndex(0), + statement_knowledge: StatementFilter { + seconded_in_group: bitvec::bitvec![u8, Lsb0; 0, 1, 0, 1], + validated_in_group: bitvec::bitvec![u8, Lsb0; 1, 0, 1], + } + }, + ManifestKind::Full, + ValidatorIndex(0), + ), + Err(ManifestImportError::Malformed) + ); + + assert_matches!( + tracker.import_manifest( + &session_topology, + &groups, + candidate_hash, + 3, + ManifestSummary { + claimed_parent_hash: Hash::repeat_byte(0), + claimed_group_index: GroupIndex(0), + statement_knowledge: StatementFilter { + seconded_in_group: bitvec::bitvec![u8, Lsb0; 0, 1, 0], + validated_in_group: bitvec::bitvec![u8, Lsb0; 1, 0, 1, 0], + } + }, + ManifestKind::Full, + ValidatorIndex(0), + ), + Err(ManifestImportError::Malformed) + ); + } + + #[test] + fn reject_malformed_no_seconders() { + let mut tracker = GridTracker::default(); + let session_topology = SessionTopologyView { + group_views: vec![( + GroupIndex(0), + GroupSubView { + sending: HashSet::new(), + receiving: vec![ValidatorIndex(0)].into_iter().collect(), + }, + )] + .into_iter() + .collect(), + }; + + let groups = dummy_groups(3); + + let candidate_hash = CandidateHash(Hash::repeat_byte(42)); + + assert_eq!(groups.get_size_and_backing_threshold(GroupIndex(0)), Some((3, 2)),); + + assert_matches!( + tracker.import_manifest( + &session_topology, + &groups, + candidate_hash, + 3, + ManifestSummary { + claimed_parent_hash: Hash::repeat_byte(0), + claimed_group_index: GroupIndex(0), + statement_knowledge: StatementFilter { + seconded_in_group: bitvec::bitvec![u8, Lsb0; 0, 0, 0], + validated_in_group: bitvec::bitvec![u8, Lsb0; 1, 1, 1], + } + }, + ManifestKind::Full, + ValidatorIndex(0), + ), + Err(ManifestImportError::Malformed) + ); + } + + #[test] + fn reject_insufficient_below_threshold() { + let mut tracker = GridTracker::default(); + let session_topology = SessionTopologyView { + group_views: vec![( + GroupIndex(0), + GroupSubView { + sending: HashSet::new(), + receiving: HashSet::from([ValidatorIndex(0)]), + }, + )] + .into_iter() + .collect(), + }; + + let groups = dummy_groups(3); + + let candidate_hash = CandidateHash(Hash::repeat_byte(42)); + + assert_eq!(groups.get_size_and_backing_threshold(GroupIndex(0)), Some((3, 2)),); + + // only one vote + + assert_matches!( + tracker.import_manifest( + &session_topology, + &groups, + candidate_hash, + 3, + ManifestSummary { + claimed_parent_hash: Hash::repeat_byte(0), + claimed_group_index: GroupIndex(0), + statement_knowledge: StatementFilter { + seconded_in_group: bitvec::bitvec![u8, Lsb0; 0, 0, 1], + validated_in_group: bitvec::bitvec![u8, Lsb0; 0, 0, 0], + } + }, + ManifestKind::Full, + ValidatorIndex(0), + ), + Err(ManifestImportError::Insufficient) + ); + + // seconding + validating still not enough to reach '2' threshold + + assert_matches!( + tracker.import_manifest( + &session_topology, + &groups, + candidate_hash, + 3, + ManifestSummary { + claimed_parent_hash: Hash::repeat_byte(0), + claimed_group_index: GroupIndex(0), + statement_knowledge: StatementFilter { + seconded_in_group: bitvec::bitvec![u8, Lsb0; 0, 0, 1], + validated_in_group: bitvec::bitvec![u8, Lsb0; 0, 0, 1], + } + }, + ManifestKind::Full, + ValidatorIndex(0), + ), + Err(ManifestImportError::Insufficient) + ); + + // finally good. + + assert_matches!( + tracker.import_manifest( + &session_topology, + &groups, + candidate_hash, + 3, + ManifestSummary { + claimed_parent_hash: Hash::repeat_byte(0), + claimed_group_index: GroupIndex(0), + statement_knowledge: StatementFilter { + seconded_in_group: bitvec::bitvec![u8, Lsb0; 0, 0, 1], + validated_in_group: bitvec::bitvec![u8, Lsb0; 0, 1, 0], + } + }, + ManifestKind::Full, + ValidatorIndex(0), + ), + Ok(false) + ); + } + + // Test that when we add a candidate as backed and advertise it to the sending group, they can + // provide an acknowledgement manifest in response. + #[test] + fn senders_can_provide_manifests_in_acknowledgement() { + let validator_index = ValidatorIndex(0); + + let mut tracker = GridTracker::default(); + let session_topology = SessionTopologyView { + group_views: vec![( + GroupIndex(0), + GroupSubView { + sending: HashSet::from([validator_index]), + receiving: HashSet::from([ValidatorIndex(1)]), + }, + )] + .into_iter() + .collect(), + }; + + let candidate_hash = CandidateHash(Hash::repeat_byte(42)); + let group_index = GroupIndex(0); + let group_size = 3; + let local_knowledge = StatementFilter::blank(group_size); + + let groups = dummy_groups(group_size); + + // Add the candidate as backed. + let receivers = tracker.add_backed_candidate( + &session_topology, + candidate_hash, + group_index, + local_knowledge.clone(), + ); + // Validator 0 is in the sending group. Advertise onward to it. + // + // Validator 1 is in the receiving group, but we have not received from it, so we're not + // expected to send it an acknowledgement. + assert_eq!(receivers, vec![(validator_index, ManifestKind::Full)]); + + // Note the manifest as 'sent' to validator 0. + tracker.manifest_sent_to(&groups, validator_index, candidate_hash, local_knowledge); + + // Import manifest of kind `Acknowledgement` from validator 0. + let ack = tracker.import_manifest( + &session_topology, + &groups, + candidate_hash, + 3, + ManifestSummary { + claimed_parent_hash: Hash::repeat_byte(0), + claimed_group_index: group_index, + statement_knowledge: StatementFilter { + seconded_in_group: bitvec::bitvec![u8, Lsb0; 0, 1, 0], + validated_in_group: bitvec::bitvec![u8, Lsb0; 1, 0, 1], + }, + }, + ManifestKind::Acknowledgement, + validator_index, + ); + assert_matches!(ack, Ok(false)); + } + + // Check that pending communication is set correctly when receiving a manifest on a confirmed candidate. + // + // It should also overwrite any existing `Full` ManifestKind. + #[test] + fn pending_communication_receiving_manifest_on_confirmed_candidate() { + let validator_index = ValidatorIndex(0); + + let mut tracker = GridTracker::default(); + let session_topology = SessionTopologyView { + group_views: vec![( + GroupIndex(0), + GroupSubView { + sending: HashSet::from([validator_index]), + receiving: HashSet::from([ValidatorIndex(1)]), + }, + )] + .into_iter() + .collect(), + }; + + let candidate_hash = CandidateHash(Hash::repeat_byte(42)); + let group_index = GroupIndex(0); + let group_size = 3; + let local_knowledge = StatementFilter::blank(group_size); + + let groups = dummy_groups(group_size); + + // Manifest should not be pending yet. + let pending_manifest = tracker.is_manifest_pending_for(validator_index, &candidate_hash); + assert_eq!(pending_manifest, None); + + // Add the candidate as backed. + tracker.add_backed_candidate( + &session_topology, + candidate_hash, + group_index, + local_knowledge.clone(), + ); + + // Manifest should be pending as `Full`. + let pending_manifest = tracker.is_manifest_pending_for(validator_index, &candidate_hash); + assert_eq!(pending_manifest, Some(ManifestKind::Full)); + + // Note the manifest as 'sent' to validator 0. + tracker.manifest_sent_to(&groups, validator_index, candidate_hash, local_knowledge); + + // Import manifest. + // + // Should overwrite existing `Full` manifest. + let ack = tracker.import_manifest( + &session_topology, + &groups, + candidate_hash, + 3, + ManifestSummary { + claimed_parent_hash: Hash::repeat_byte(0), + claimed_group_index: group_index, + statement_knowledge: StatementFilter { + seconded_in_group: bitvec::bitvec![u8, Lsb0; 0, 1, 0], + validated_in_group: bitvec::bitvec![u8, Lsb0; 1, 0, 1], + }, + }, + ManifestKind::Acknowledgement, + validator_index, + ); + assert_matches!(ack, Ok(false)); + + let pending_manifest = tracker.is_manifest_pending_for(validator_index, &candidate_hash); + assert_eq!(pending_manifest, None); + } + + // Check that pending communication is cleared correctly in `manifest_sent_to` + // + // Also test a scenario where manifest import returns `Ok(true)` (should acknowledge). + #[test] + fn pending_communication_is_cleared() { + let validator_index = ValidatorIndex(0); + + let mut tracker = GridTracker::default(); + let session_topology = SessionTopologyView { + group_views: vec![( + GroupIndex(0), + GroupSubView { + sending: HashSet::new(), + receiving: HashSet::from([validator_index]), + }, + )] + .into_iter() + .collect(), + }; + + let candidate_hash = CandidateHash(Hash::repeat_byte(42)); + let group_index = GroupIndex(0); + let group_size = 3; + let local_knowledge = StatementFilter::blank(group_size); + + let groups = dummy_groups(group_size); + + // Add the candidate as backed. + tracker.add_backed_candidate( + &session_topology, + candidate_hash, + group_index, + local_knowledge.clone(), + ); + + // Manifest should not be pending yet. + let pending_manifest = tracker.is_manifest_pending_for(validator_index, &candidate_hash); + assert_eq!(pending_manifest, None); + + // Import manifest. The candidate is confirmed backed and we are expected to receive from + // validator 0, so send it an acknowledgement. + let ack = tracker.import_manifest( + &session_topology, + &groups, + candidate_hash, + 3, + ManifestSummary { + claimed_parent_hash: Hash::repeat_byte(0), + claimed_group_index: group_index, + statement_knowledge: StatementFilter { + seconded_in_group: bitvec::bitvec![u8, Lsb0; 0, 1, 0], + validated_in_group: bitvec::bitvec![u8, Lsb0; 1, 0, 1], + }, + }, + ManifestKind::Full, + validator_index, + ); + assert_matches!(ack, Ok(true)); + + // Acknowledgement manifest should be pending. + let pending_manifest = tracker.is_manifest_pending_for(validator_index, &candidate_hash); + assert_eq!(pending_manifest, Some(ManifestKind::Acknowledgement)); + + // Note the candidate as advertised. + tracker.manifest_sent_to(&groups, validator_index, candidate_hash, local_knowledge); + + // Pending manifest should be cleared. + let pending_manifest = tracker.is_manifest_pending_for(validator_index, &candidate_hash); + assert_eq!(pending_manifest, None); + } + + /// A manifest exchange means that both `manifest_sent_to` and `manifest_received_from` have + /// been invoked. + /// + /// In practice, it means that one of three things have happened: + /// + /// - They announced, we acknowledged + /// + /// - We announced, they acknowledged + /// + /// - We announced, they announced (not sure if this can actually happen; it would happen if 2 + /// nodes had each other in their sending set and they sent manifests at the same time. The + /// code accounts for this anyway) + #[test] + fn pending_statements_are_updated_after_manifest_exchange() { + let send_to = ValidatorIndex(0); + let receive_from = ValidatorIndex(1); + + let mut tracker = GridTracker::default(); + let session_topology = SessionTopologyView { + group_views: vec![( + GroupIndex(0), + GroupSubView { + sending: HashSet::from([send_to]), + receiving: HashSet::from([receive_from]), + }, + )] + .into_iter() + .collect(), + }; + + let candidate_hash = CandidateHash(Hash::repeat_byte(42)); + let group_index = GroupIndex(0); + let group_size = 3; + let local_knowledge = StatementFilter::blank(group_size); + + let groups = dummy_groups(group_size); + + // Confirm the candidate. + let receivers = tracker.add_backed_candidate( + &session_topology, + candidate_hash, + group_index, + local_knowledge.clone(), + ); + assert_eq!(receivers, vec![(send_to, ManifestKind::Full)]); + + // Learn a statement from a different validator. + tracker.learned_fresh_statement( + &groups, + &session_topology, + ValidatorIndex(2), + &CompactStatement::Seconded(candidate_hash), + ); + + // Test receiving followed by sending an ack. + { + // Should start with no pending statements. + assert_eq!(tracker.pending_statements_for(receive_from, candidate_hash), None); + assert_eq!(tracker.all_pending_statements_for(receive_from), vec![]); + let ack = tracker.import_manifest( + &session_topology, + &groups, + candidate_hash, + 3, + ManifestSummary { + claimed_parent_hash: Hash::repeat_byte(0), + claimed_group_index: group_index, + statement_knowledge: StatementFilter { + seconded_in_group: bitvec::bitvec![u8, Lsb0; 0, 1, 0], + validated_in_group: bitvec::bitvec![u8, Lsb0; 1, 0, 1], + }, + }, + ManifestKind::Full, + receive_from, + ); + assert_matches!(ack, Ok(true)); + + // Send ack now. + tracker.manifest_sent_to( + &groups, + receive_from, + candidate_hash, + local_knowledge.clone(), + ); + + // There should be pending statements now. + assert_eq!( + tracker.pending_statements_for(receive_from, candidate_hash), + Some(StatementFilter { + seconded_in_group: bitvec::bitvec![u8, Lsb0; 0, 0, 1], + validated_in_group: bitvec::bitvec![u8, Lsb0; 0, 0, 0], + }) + ); + assert_eq!( + tracker.all_pending_statements_for(receive_from), + vec![(ValidatorIndex(2), CompactStatement::Seconded(candidate_hash))] + ); + } + + // Test sending followed by receiving an ack. + { + // Should start with no pending statements. + assert_eq!(tracker.pending_statements_for(send_to, candidate_hash), None); + assert_eq!(tracker.all_pending_statements_for(send_to), vec![]); + + tracker.manifest_sent_to(&groups, send_to, candidate_hash, local_knowledge.clone()); + let ack = tracker.import_manifest( + &session_topology, + &groups, + candidate_hash, + 3, + ManifestSummary { + claimed_parent_hash: Hash::repeat_byte(0), + claimed_group_index: group_index, + statement_knowledge: StatementFilter { + seconded_in_group: bitvec::bitvec![u8, Lsb0; 0, 1, 0], + validated_in_group: bitvec::bitvec![u8, Lsb0; 0, 0, 1], + }, + }, + ManifestKind::Acknowledgement, + send_to, + ); + assert_matches!(ack, Ok(false)); + + // There should be pending statements now. + assert_eq!( + tracker.pending_statements_for(send_to, candidate_hash), + Some(StatementFilter { + seconded_in_group: bitvec::bitvec![u8, Lsb0; 0, 0, 1], + validated_in_group: bitvec::bitvec![u8, Lsb0; 0, 0, 0], + }) + ); + assert_eq!( + tracker.all_pending_statements_for(send_to), + vec![(ValidatorIndex(2), CompactStatement::Seconded(candidate_hash))] + ); + } + } + + #[test] + fn invalid_fresh_statement_import() { + let validator_index = ValidatorIndex(0); + + let mut tracker = GridTracker::default(); + let session_topology = SessionTopologyView { + group_views: vec![( + GroupIndex(0), + GroupSubView { + sending: HashSet::new(), + receiving: HashSet::from([validator_index]), + }, + )] + .into_iter() + .collect(), + }; + + let candidate_hash = CandidateHash(Hash::repeat_byte(42)); + let group_index = GroupIndex(0); + let group_size = 3; + let local_knowledge = StatementFilter::blank(group_size); + + let groups = dummy_groups(group_size); + + // Should start with no pending statements. + assert_eq!(tracker.pending_statements_for(validator_index, candidate_hash), None); + assert_eq!(tracker.all_pending_statements_for(validator_index), vec![]); + + // Try to import fresh statement. Candidate not backed. + let statement = CompactStatement::Seconded(candidate_hash); + tracker.learned_fresh_statement(&groups, &session_topology, validator_index, &statement); + + assert_eq!(tracker.pending_statements_for(validator_index, candidate_hash), None); + assert_eq!(tracker.all_pending_statements_for(validator_index), vec![]); + + // Add the candidate as backed. + tracker.add_backed_candidate( + &session_topology, + candidate_hash, + group_index, + local_knowledge.clone(), + ); + + // Try to import fresh statement. Unknown group for validator index. + let statement = CompactStatement::Seconded(candidate_hash); + tracker.learned_fresh_statement(&groups, &session_topology, ValidatorIndex(1), &statement); + + assert_eq!(tracker.pending_statements_for(validator_index, candidate_hash), None); + assert_eq!(tracker.all_pending_statements_for(validator_index), vec![]); + } + + #[test] + fn pending_statements_updated_when_importing_fresh_statement() { + let validator_index = ValidatorIndex(0); + + let mut tracker = GridTracker::default(); + let session_topology = SessionTopologyView { + group_views: vec![( + GroupIndex(0), + GroupSubView { + sending: HashSet::new(), + receiving: HashSet::from([validator_index]), + }, + )] + .into_iter() + .collect(), + }; + + let candidate_hash = CandidateHash(Hash::repeat_byte(42)); + let group_index = GroupIndex(0); + let group_size = 3; + let local_knowledge = StatementFilter::blank(group_size); + + let groups = dummy_groups(group_size); + + // Should start with no pending statements. + assert_eq!(tracker.pending_statements_for(validator_index, candidate_hash), None); + assert_eq!(tracker.all_pending_statements_for(validator_index), vec![]); + + // Add the candidate as backed. + tracker.add_backed_candidate( + &session_topology, + candidate_hash, + group_index, + local_knowledge.clone(), + ); + + // Import fresh statement. + + let ack = tracker.import_manifest( + &session_topology, + &groups, + candidate_hash, + 3, + ManifestSummary { + claimed_parent_hash: Hash::repeat_byte(0), + claimed_group_index: group_index, + statement_knowledge: StatementFilter { + seconded_in_group: bitvec::bitvec![u8, Lsb0; 0, 1, 0], + validated_in_group: bitvec::bitvec![u8, Lsb0; 1, 0, 1], + }, + }, + ManifestKind::Full, + validator_index, + ); + assert_matches!(ack, Ok(true)); + tracker.manifest_sent_to(&groups, validator_index, candidate_hash, local_knowledge); + let statement = CompactStatement::Seconded(candidate_hash); + tracker.learned_fresh_statement(&groups, &session_topology, validator_index, &statement); + + // There should be pending statements now. + let statements = StatementFilter { + seconded_in_group: bitvec::bitvec![u8, Lsb0; 1, 0, 0], + validated_in_group: bitvec::bitvec![u8, Lsb0; 0, 0, 0], + }; + assert_eq!( + tracker.pending_statements_for(validator_index, candidate_hash), + Some(statements.clone()) + ); + assert_eq!( + tracker.all_pending_statements_for(validator_index), + vec![(ValidatorIndex(0), CompactStatement::Seconded(candidate_hash))] + ); + + // After successful import, try importing again. Nothing should change. + + tracker.learned_fresh_statement(&groups, &session_topology, validator_index, &statement); + assert_eq!( + tracker.pending_statements_for(validator_index, candidate_hash), + Some(statements) + ); + assert_eq!( + tracker.all_pending_statements_for(validator_index), + vec![(ValidatorIndex(0), CompactStatement::Seconded(candidate_hash))] + ); + } + + // After learning fresh statements, we should not generate pending statements for knowledge that + // the validator already has. + #[test] + fn pending_statements_respect_remote_knowledge() { + let validator_index = ValidatorIndex(0); + + let mut tracker = GridTracker::default(); + let session_topology = SessionTopologyView { + group_views: vec![( + GroupIndex(0), + GroupSubView { + sending: HashSet::new(), + receiving: HashSet::from([validator_index]), + }, + )] + .into_iter() + .collect(), + }; + + let candidate_hash = CandidateHash(Hash::repeat_byte(42)); + let group_index = GroupIndex(0); + let group_size = 3; + let local_knowledge = StatementFilter::blank(group_size); + + let groups = dummy_groups(group_size); + + // Should start with no pending statements. + assert_eq!(tracker.pending_statements_for(validator_index, candidate_hash), None); + assert_eq!(tracker.all_pending_statements_for(validator_index), vec![]); + + // Add the candidate as backed. + tracker.add_backed_candidate( + &session_topology, + candidate_hash, + group_index, + local_knowledge.clone(), + ); + + // Import fresh statement. + let ack = tracker.import_manifest( + &session_topology, + &groups, + candidate_hash, + 3, + ManifestSummary { + claimed_parent_hash: Hash::repeat_byte(0), + claimed_group_index: group_index, + statement_knowledge: StatementFilter { + seconded_in_group: bitvec::bitvec![u8, Lsb0; 1, 0, 1], + validated_in_group: bitvec::bitvec![u8, Lsb0; 0, 0, 0], + }, + }, + ManifestKind::Full, + validator_index, + ); + assert_matches!(ack, Ok(true)); + tracker.manifest_sent_to(&groups, validator_index, candidate_hash, local_knowledge); + tracker.learned_fresh_statement( + &groups, + &session_topology, + validator_index, + &CompactStatement::Seconded(candidate_hash), + ); + tracker.learned_fresh_statement( + &groups, + &session_topology, + validator_index, + &CompactStatement::Valid(candidate_hash), + ); + + // The pending statements should respect the remote knowledge (meaning the Seconded + // statement is ignored, but not the Valid statement). + let statements = StatementFilter { + seconded_in_group: bitvec::bitvec![u8, Lsb0; 0, 0, 0], + validated_in_group: bitvec::bitvec![u8, Lsb0; 1, 0, 0], + }; + assert_eq!( + tracker.pending_statements_for(validator_index, candidate_hash), + Some(statements.clone()) + ); + assert_eq!( + tracker.all_pending_statements_for(validator_index), + vec![(ValidatorIndex(0), CompactStatement::Valid(candidate_hash))] + ); + } + + #[test] + fn pending_statements_cleared_when_sending() { + let validator_index = ValidatorIndex(0); + let counterparty = ValidatorIndex(1); + + let mut tracker = GridTracker::default(); + let session_topology = SessionTopologyView { + group_views: vec![( + GroupIndex(0), + GroupSubView { + sending: HashSet::new(), + receiving: HashSet::from([validator_index, counterparty]), + }, + )] + .into_iter() + .collect(), + }; + + let candidate_hash = CandidateHash(Hash::repeat_byte(42)); + let group_index = GroupIndex(0); + let group_size = 3; + let local_knowledge = StatementFilter::blank(group_size); + + let groups = dummy_groups(group_size); + + // Should start with no pending statements. + assert_eq!(tracker.pending_statements_for(validator_index, candidate_hash), None); + assert_eq!(tracker.all_pending_statements_for(validator_index), vec![]); + + // Add the candidate as backed. + tracker.add_backed_candidate( + &session_topology, + candidate_hash, + group_index, + local_knowledge.clone(), + ); + + // Import statement for originator. + tracker + .import_manifest( + &session_topology, + &groups, + candidate_hash, + 3, + ManifestSummary { + claimed_parent_hash: Hash::repeat_byte(0), + claimed_group_index: group_index, + statement_knowledge: StatementFilter { + seconded_in_group: bitvec::bitvec![u8, Lsb0; 0, 1, 0], + validated_in_group: bitvec::bitvec![u8, Lsb0; 1, 0, 1], + }, + }, + ManifestKind::Full, + validator_index, + ) + .ok() + .unwrap(); + tracker.manifest_sent_to(&groups, validator_index, candidate_hash, local_knowledge.clone()); + let statement = CompactStatement::Seconded(candidate_hash); + tracker.learned_fresh_statement(&groups, &session_topology, validator_index, &statement); + + // Import statement for counterparty. + tracker + .import_manifest( + &session_topology, + &groups, + candidate_hash, + 3, + ManifestSummary { + claimed_parent_hash: Hash::repeat_byte(0), + claimed_group_index: group_index, + statement_knowledge: StatementFilter { + seconded_in_group: bitvec::bitvec![u8, Lsb0; 0, 1, 0], + validated_in_group: bitvec::bitvec![u8, Lsb0; 1, 0, 1], + }, + }, + ManifestKind::Full, + counterparty, + ) + .ok() + .unwrap(); + tracker.manifest_sent_to(&groups, counterparty, candidate_hash, local_knowledge); + let statement = CompactStatement::Seconded(candidate_hash); + tracker.learned_fresh_statement(&groups, &session_topology, counterparty, &statement); + + // There should be pending statements now. + let statements = StatementFilter { + seconded_in_group: bitvec::bitvec![u8, Lsb0; 1, 0, 0], + validated_in_group: bitvec::bitvec![u8, Lsb0; 0, 0, 0], + }; + assert_eq!( + tracker.pending_statements_for(validator_index, candidate_hash), + Some(statements.clone()) + ); + assert_eq!( + tracker.all_pending_statements_for(validator_index), + vec![(ValidatorIndex(0), CompactStatement::Seconded(candidate_hash))] + ); + assert_eq!( + tracker.pending_statements_for(counterparty, candidate_hash), + Some(statements.clone()) + ); + assert_eq!( + tracker.all_pending_statements_for(counterparty), + vec![(ValidatorIndex(0), CompactStatement::Seconded(candidate_hash))] + ); + + tracker.learned_fresh_statement(&groups, &session_topology, validator_index, &statement); + tracker.sent_or_received_direct_statement( + &groups, + validator_index, + counterparty, + &statement, + ); + + // There should be no pending statements now (for the counterparty). + assert_eq!( + tracker.pending_statements_for(counterparty, candidate_hash), + Some(StatementFilter::blank(group_size)) + ); + assert_eq!(tracker.all_pending_statements_for(counterparty), vec![]); + } +} diff --git a/node/network/statement-distribution/src/vstaging/groups.rs b/node/network/statement-distribution/src/vstaging/groups.rs new file mode 100644 index 000000000000..86321b30f220 --- /dev/null +++ b/node/network/statement-distribution/src/vstaging/groups.rs @@ -0,0 +1,70 @@ +// Copyright 2022 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! A utility for tracking groups and their members within a session. + +use polkadot_node_primitives::minimum_votes; +use polkadot_primitives::vstaging::{GroupIndex, IndexedVec, ValidatorIndex}; + +use std::collections::HashMap; + +/// Validator groups within a session, plus some helpful indexing for +/// looking up groups by validator indices or authority discovery ID. +#[derive(Debug, Clone)] +pub struct Groups { + groups: IndexedVec>, + by_validator_index: HashMap, +} + +impl Groups { + /// Create a new [`Groups`] tracker with the groups and discovery keys + /// from the session. + pub fn new(groups: IndexedVec>) -> Self { + let mut by_validator_index = HashMap::new(); + + for (i, group) in groups.iter().enumerate() { + let index = GroupIndex(i as _); + for v in group { + by_validator_index.insert(*v, index); + } + } + + Groups { groups, by_validator_index } + } + + /// Access all the underlying groups. + pub fn all(&self) -> &IndexedVec> { + &self.groups + } + + /// Get the underlying group validators by group index. + pub fn get(&self, group_index: GroupIndex) -> Option<&[ValidatorIndex]> { + self.groups.get(group_index).map(|x| &x[..]) + } + + /// Get the backing group size and backing threshold. + pub fn get_size_and_backing_threshold( + &self, + group_index: GroupIndex, + ) -> Option<(usize, usize)> { + self.get(group_index).map(|g| (g.len(), minimum_votes(g.len()))) + } + + /// Get the group index for a validator by index. + pub fn by_validator_index(&self, validator_index: ValidatorIndex) -> Option { + self.by_validator_index.get(&validator_index).map(|x| *x) + } +} diff --git a/node/network/statement-distribution/src/vstaging/mod.rs b/node/network/statement-distribution/src/vstaging/mod.rs new file mode 100644 index 000000000000..a562668627a1 --- /dev/null +++ b/node/network/statement-distribution/src/vstaging/mod.rs @@ -0,0 +1,2658 @@ +// Copyright 2022 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! Implementation of the v2 statement distribution protocol, +//! designed for asynchronous backing. + +use polkadot_node_network_protocol::{ + self as net_protocol, + grid_topology::SessionGridTopology, + peer_set::ValidationVersion, + request_response::{ + incoming::OutgoingResponse, + vstaging::{AttestedCandidateRequest, AttestedCandidateResponse}, + IncomingRequest, IncomingRequestReceiver, Requests, + MAX_PARALLEL_ATTESTED_CANDIDATE_REQUESTS, + }, + vstaging::{self as protocol_vstaging, StatementFilter}, + IfDisconnected, PeerId, UnifiedReputationChange as Rep, Versioned, View, +}; +use polkadot_node_primitives::{ + SignedFullStatementWithPVD, StatementWithPVD as FullStatementWithPVD, +}; +use polkadot_node_subsystem::{ + messages::{ + CandidateBackingMessage, HypotheticalCandidate, HypotheticalFrontierRequest, + NetworkBridgeEvent, NetworkBridgeTxMessage, ProspectiveParachainsMessage, + }, + overseer, ActivatedLeaf, +}; +use polkadot_node_subsystem_util::{ + backing_implicit_view::View as ImplicitView, runtime::ProspectiveParachainsMode, +}; +use polkadot_primitives::vstaging::{ + AuthorityDiscoveryId, CandidateHash, CompactStatement, CoreIndex, CoreState, GroupIndex, + GroupRotationInfo, Hash, Id as ParaId, IndexedVec, SessionIndex, SessionInfo, SignedStatement, + SigningContext, UncheckedSignedStatement, ValidatorId, ValidatorIndex, +}; + +use sp_keystore::SyncCryptoStorePtr; + +use fatality::Nested; +use futures::{ + channel::{mpsc, oneshot}, + stream::FuturesUnordered, + SinkExt, StreamExt, +}; + +use std::collections::{ + hash_map::{Entry, HashMap}, + HashSet, +}; + +use crate::{ + error::{JfyiError, JfyiErrorResult}, + LOG_TARGET, +}; +use candidates::{BadAdvertisement, Candidates, PostConfirmation}; +use cluster::{Accept as ClusterAccept, ClusterTracker, RejectIncoming as ClusterRejectIncoming}; +use grid::GridTracker; +use groups::Groups; +use requests::{CandidateIdentifier, RequestProperties}; +use statement_store::{StatementOrigin, StatementStore}; + +pub use requests::{RequestManager, UnhandledResponse}; + +mod candidates; +mod cluster; +mod grid; +mod groups; +mod requests; +mod statement_store; + +#[cfg(test)] +mod tests; + +const COST_UNEXPECTED_STATEMENT: Rep = Rep::CostMinor("Unexpected Statement"); +const COST_UNEXPECTED_STATEMENT_MISSING_KNOWLEDGE: Rep = + Rep::CostMinor("Unexpected Statement, missing knowledge for relay parent"); +const COST_EXCESSIVE_SECONDED: Rep = Rep::CostMinor("Sent Excessive `Seconded` Statements"); + +const COST_UNEXPECTED_MANIFEST_MISSING_KNOWLEDGE: Rep = + Rep::CostMinor("Unexpected Manifest, missing knowlege for relay parent"); +const COST_UNEXPECTED_MANIFEST_DISALLOWED: Rep = + Rep::CostMinor("Unexpected Manifest, Peer Disallowed"); +const COST_CONFLICTING_MANIFEST: Rep = Rep::CostMajor("Manifest conflicts with previous"); +const COST_INSUFFICIENT_MANIFEST: Rep = + Rep::CostMajor("Manifest statements insufficient to back candidate"); +const COST_MALFORMED_MANIFEST: Rep = Rep::CostMajor("Manifest is malformed"); +const COST_UNEXPECTED_ACKNOWLEDGEMENT_UNKNOWN_CANDIDATE: Rep = + Rep::CostMinor("Unexpected acknowledgement, unknown candidate"); + +const COST_INVALID_SIGNATURE: Rep = Rep::CostMajor("Invalid Statement Signature"); +const COST_IMPROPERLY_DECODED_RESPONSE: Rep = + Rep::CostMajor("Improperly Encoded Candidate Response"); +const COST_INVALID_RESPONSE: Rep = Rep::CostMajor("Invalid Candidate Response"); +const COST_UNREQUESTED_RESPONSE_STATEMENT: Rep = + Rep::CostMajor("Un-requested Statement In Response"); +const COST_INACCURATE_ADVERTISEMENT: Rep = + Rep::CostMajor("Peer advertised a candidate inaccurately"); + +const COST_INVALID_REQUEST: Rep = Rep::CostMajor("Peer sent unparsable request"); +const COST_INVALID_REQUEST_BITFIELD_SIZE: Rep = + Rep::CostMajor("Attested candidate request bitfields have wrong size"); +const COST_UNEXPECTED_REQUEST: Rep = Rep::CostMajor("Unexpected attested candidate request"); + +const BENEFIT_VALID_RESPONSE: Rep = Rep::BenefitMajor("Peer Answered Candidate Request"); +const BENEFIT_VALID_STATEMENT: Rep = Rep::BenefitMajor("Peer provided a valid statement"); +const BENEFIT_VALID_STATEMENT_FIRST: Rep = + Rep::BenefitMajorFirst("Peer was the first to provide a valid statement"); + +struct PerRelayParentState { + local_validator: Option, + statement_store: StatementStore, + availability_cores: Vec, + group_rotation_info: GroupRotationInfo, + seconding_limit: usize, + session: SessionIndex, +} + +// per-relay-parent local validator state. +struct LocalValidatorState { + // The index of the validator. + index: ValidatorIndex, + // our validator group + group: GroupIndex, + // the assignment of our validator group, if any. + assignment: Option, + // the 'direct-in-group' communication at this relay-parent. + cluster_tracker: ClusterTracker, + // the grid-level communication at this relay-parent. + grid_tracker: GridTracker, +} + +struct PerSessionState { + session_info: SessionInfo, + groups: Groups, + authority_lookup: HashMap, + // is only `None` in the time between seeing a session and + // getting the topology from the gossip-support subsystem + grid_view: Option, + local_validator: Option, +} + +impl PerSessionState { + async fn new(session_info: SessionInfo, keystore: &SyncCryptoStorePtr) -> Self { + let groups = Groups::new(session_info.validator_groups.clone()); + let mut authority_lookup = HashMap::new(); + for (i, ad) in session_info.discovery_keys.iter().cloned().enumerate() { + authority_lookup.insert(ad, ValidatorIndex(i as _)); + } + + let local_validator = polkadot_node_subsystem_util::signing_key_and_index( + session_info.validators.iter(), + keystore, + ) + .await; + + PerSessionState { + session_info, + groups, + authority_lookup, + grid_view: None, + local_validator: local_validator.map(|(_key, index)| index), + } + } + + fn supply_topology(&mut self, topology: &SessionGridTopology) { + let grid_view = grid::build_session_topology( + self.session_info.validator_groups.iter(), + topology, + self.local_validator, + ); + + self.grid_view = Some(grid_view); + } +} + +pub(crate) struct State { + /// The utility for managing the implicit and explicit views in a consistent way. + /// + /// We only feed leaves which have prospective parachains enabled to this view. + implicit_view: ImplicitView, + candidates: Candidates, + per_relay_parent: HashMap, + per_session: HashMap, + peers: HashMap, + keystore: SyncCryptoStorePtr, + authorities: HashMap, + request_manager: RequestManager, +} + +impl State { + /// Create a new state. + pub(crate) fn new(keystore: SyncCryptoStorePtr) -> Self { + State { + implicit_view: Default::default(), + candidates: Default::default(), + per_relay_parent: HashMap::new(), + per_session: HashMap::new(), + peers: HashMap::new(), + keystore, + authorities: HashMap::new(), + request_manager: RequestManager::new(), + } + } +} + +// For the provided validator index, if there is a connected peer controlling the given authority +// ID, then return that peer's `PeerId`. +fn connected_validator_peer( + authorities: &HashMap, + per_session: &PerSessionState, + validator_index: ValidatorIndex, +) -> Option { + per_session + .session_info + .discovery_keys + .get(validator_index.0 as usize) + .and_then(|k| authorities.get(k)) + .map(|p| *p) +} + +struct PeerState { + view: View, + implicit_view: HashSet, + discovery_ids: Option>, +} + +impl PeerState { + // Update the view, returning a vector of implicit relay-parents which weren't previously + // part of the view. + fn update_view(&mut self, new_view: View, local_implicit: &ImplicitView) -> Vec { + let next_implicit = new_view + .iter() + .flat_map(|x| local_implicit.known_allowed_relay_parents_under(x, None)) + .flatten() + .cloned() + .collect::>(); + + let fresh_implicit = next_implicit + .iter() + .filter(|x| !self.implicit_view.contains(x)) + .cloned() + .collect(); + + self.view = new_view; + self.implicit_view = next_implicit; + + fresh_implicit + } + + // Attempt to reconcile the view with new information about the implicit relay parents + // under an active leaf. + fn reconcile_active_leaf(&mut self, leaf_hash: Hash, implicit: &[Hash]) -> Vec { + if !self.view.contains(&leaf_hash) { + return Vec::new() + } + + let mut v = Vec::with_capacity(implicit.len()); + for i in implicit { + if self.implicit_view.insert(*i) { + v.push(*i); + } + } + v + } + + // Whether we know that a peer knows a relay-parent. + // The peer knows the relay-parent if it is either implicit or explicit + // in their view. However, if it is implicit via an active-leaf we don't + // recognize, we will not accurately be able to recognize them as 'knowing' + // the relay-parent. + fn knows_relay_parent(&self, relay_parent: &Hash) -> bool { + self.implicit_view.contains(relay_parent) || self.view.contains(relay_parent) + } + + fn is_authority(&self, authority_id: &AuthorityDiscoveryId) -> bool { + self.discovery_ids.as_ref().map_or(false, |x| x.contains(authority_id)) + } + + fn iter_known_discovery_ids(&self) -> impl Iterator { + self.discovery_ids.as_ref().into_iter().flatten() + } +} + +#[overseer::contextbounds(StatementDistribution, prefix=self::overseer)] +pub(crate) async fn handle_network_update( + ctx: &mut Context, + state: &mut State, + update: NetworkBridgeEvent, +) { + match update { + NetworkBridgeEvent::PeerConnected(peer_id, role, protocol_version, mut authority_ids) => { + gum::trace!(target: LOG_TARGET, ?peer_id, ?role, ?protocol_version, "Peer connected"); + + if protocol_version != ValidationVersion::VStaging.into() { + return + } + + if let Some(ref mut authority_ids) = authority_ids { + authority_ids.retain(|a| match state.authorities.entry(a.clone()) { + Entry::Vacant(e) => { + e.insert(peer_id); + true + }, + Entry::Occupied(e) => { + gum::trace!( + target: LOG_TARGET, + authority_id = ?a, + existing_peer = ?e.get(), + new_peer = ?peer_id, + "Ignoring new peer with duplicate authority ID as a bearer of that identity" + ); + + false + }, + }); + } + + state.peers.insert( + peer_id, + PeerState { + view: View::default(), + implicit_view: HashSet::new(), + discovery_ids: authority_ids, + }, + ); + }, + NetworkBridgeEvent::PeerDisconnected(peer_id) => { + if let Some(p) = state.peers.remove(&peer_id) { + for discovery_key in p.discovery_ids.into_iter().flatten() { + state.authorities.remove(&discovery_key); + } + } + }, + NetworkBridgeEvent::NewGossipTopology(topology) => { + let new_session_index = topology.session; + let new_topology = topology.topology; + + if let Some(per_session) = state.per_session.get_mut(&new_session_index) { + per_session.supply_topology(&new_topology); + } + + // TODO [https://github.com/paritytech/polkadot/issues/6194] + // technically, we should account for the fact that the session topology might + // come late, and for all relay-parents with this session, send all grid peers + // any `BackedCandidateInv` messages they might need. + // + // in practice, this is a small issue & the API of receiving topologies could + // be altered to fix it altogether. + }, + NetworkBridgeEvent::PeerMessage(peer_id, message) => match message { + net_protocol::StatementDistributionMessage::V1(_) => return, + net_protocol::StatementDistributionMessage::VStaging( + protocol_vstaging::StatementDistributionMessage::V1Compatibility(_), + ) => return, + net_protocol::StatementDistributionMessage::VStaging( + protocol_vstaging::StatementDistributionMessage::Statement(relay_parent, statement), + ) => handle_incoming_statement(ctx, state, peer_id, relay_parent, statement).await, + net_protocol::StatementDistributionMessage::VStaging( + protocol_vstaging::StatementDistributionMessage::BackedCandidateManifest(inner), + ) => handle_incoming_manifest(ctx, state, peer_id, inner).await, + net_protocol::StatementDistributionMessage::VStaging( + protocol_vstaging::StatementDistributionMessage::BackedCandidateKnown(inner), + ) => handle_incoming_acknowledgement(ctx, state, peer_id, inner).await, + }, + NetworkBridgeEvent::PeerViewChange(peer_id, view) => + handle_peer_view_update(ctx, state, peer_id, view).await, + NetworkBridgeEvent::OurViewChange(_view) => { + // handled by `handle_activated_leaf` + }, + } +} + +/// If there is a new leaf, this should only be called for leaves which support +/// prospective parachains. +#[overseer::contextbounds(StatementDistribution, prefix=self::overseer)] +pub(crate) async fn handle_active_leaves_update( + ctx: &mut Context, + state: &mut State, + activated: &ActivatedLeaf, + leaf_mode: ProspectiveParachainsMode, +) -> JfyiErrorResult<()> { + let seconding_limit = match leaf_mode { + ProspectiveParachainsMode::Disabled => return Ok(()), + ProspectiveParachainsMode::Enabled { max_candidate_depth, .. } => max_candidate_depth + 1, + }; + + state + .implicit_view + .activate_leaf(ctx.sender(), activated.hash) + .await + .map_err(JfyiError::ActivateLeafFailure)?; + + let new_relay_parents = + state.implicit_view.all_allowed_relay_parents().cloned().collect::>(); + for new_relay_parent in new_relay_parents.iter().cloned() { + if state.per_relay_parent.contains_key(&new_relay_parent) { + continue + } + + // New leaf: fetch info from runtime API and initialize + // `per_relay_parent`. + + let session_index = polkadot_node_subsystem_util::request_session_index_for_child( + new_relay_parent, + ctx.sender(), + ) + .await + .await + .map_err(JfyiError::RuntimeApiUnavailable)? + .map_err(JfyiError::FetchSessionIndex)?; + + let availability_cores = polkadot_node_subsystem_util::request_availability_cores( + new_relay_parent, + ctx.sender(), + ) + .await + .await + .map_err(JfyiError::RuntimeApiUnavailable)? + .map_err(JfyiError::FetchAvailabilityCores)?; + + let group_rotation_info = + polkadot_node_subsystem_util::request_validator_groups(new_relay_parent, ctx.sender()) + .await + .await + .map_err(JfyiError::RuntimeApiUnavailable)? + .map_err(JfyiError::FetchValidatorGroups)? + .1; + + if !state.per_session.contains_key(&session_index) { + let session_info = polkadot_node_subsystem_util::request_session_info( + new_relay_parent, + session_index, + ctx.sender(), + ) + .await + .await + .map_err(JfyiError::RuntimeApiUnavailable)? + .map_err(JfyiError::FetchSessionInfo)?; + + let session_info = match session_info { + None => { + gum::warn!( + target: LOG_TARGET, + relay_parent = ?new_relay_parent, + "No session info available for current session" + ); + + continue + }, + Some(s) => s, + }; + + state + .per_session + .insert(session_index, PerSessionState::new(session_info, &state.keystore).await); + } + + let per_session = state + .per_session + .get(&session_index) + .expect("either existed or just inserted; qed"); + + let local_validator = per_session.local_validator.and_then(|v| { + find_local_validator_state( + v, + &per_session.groups, + &availability_cores, + &group_rotation_info, + seconding_limit, + ) + }); + + state.per_relay_parent.insert( + new_relay_parent, + PerRelayParentState { + local_validator, + statement_store: StatementStore::new(&per_session.groups), + availability_cores, + group_rotation_info, + seconding_limit, + session: session_index, + }, + ); + } + + // Reconcile all peers' views with the active leaf and any relay parents + // it implies. If they learned about the block before we did, this reconciliation will give non-empty + // results and we should send them messages concerning all activated relay-parents. + { + let mut update_peers = Vec::new(); + for (peer, peer_state) in state.peers.iter_mut() { + let fresh = peer_state.reconcile_active_leaf(activated.hash, &new_relay_parents); + if !fresh.is_empty() { + update_peers.push((*peer, fresh)); + } + } + + for (peer, fresh) in update_peers { + for fresh_relay_parent in fresh { + send_peer_messages_for_relay_parent(ctx, state, peer, fresh_relay_parent).await; + } + } + } + + new_leaf_fragment_tree_updates(ctx, state, activated.hash).await; + + Ok(()) +} + +fn find_local_validator_state( + validator_index: ValidatorIndex, + groups: &Groups, + availability_cores: &[CoreState], + group_rotation_info: &GroupRotationInfo, + seconding_limit: usize, +) -> Option { + if groups.all().is_empty() { + return None + } + + let our_group = groups.by_validator_index(validator_index)?; + + // note: this won't work well for parathreads because it only works + // when core assignments to paras are static throughout the session. + + let core = group_rotation_info.core_for_group(our_group, availability_cores.len()); + let para = availability_cores.get(core.0 as usize).and_then(|c| c.para_id()); + let group_validators = groups.get(our_group)?.to_owned(); + + Some(LocalValidatorState { + index: validator_index, + group: our_group, + assignment: para, + cluster_tracker: ClusterTracker::new(group_validators, seconding_limit) + .expect("group is non-empty because we are in it; qed"), + grid_tracker: GridTracker::default(), + }) +} + +pub(crate) fn handle_deactivate_leaves(state: &mut State, leaves: &[Hash]) { + // deactivate the leaf in the implicit view. + for leaf in leaves { + state.implicit_view.deactivate_leaf(*leaf); + } + + let relay_parents = state.implicit_view.all_allowed_relay_parents().collect::>(); + + // fast exit for no-op. + if relay_parents.len() == state.per_relay_parent.len() { + return + } + + // clean up per-relay-parent data based on everything removed. + state.per_relay_parent.retain(|r, _| relay_parents.contains(r)); + + // Clean up all requests + for leaf in leaves { + state.request_manager.remove_by_relay_parent(*leaf); + } + + state.candidates.on_deactivate_leaves(&leaves, |h| relay_parents.contains(h)); + + // clean up sessions based on everything remaining. + let sessions: HashSet<_> = state.per_relay_parent.values().map(|r| r.session).collect(); + state.per_session.retain(|s, _| sessions.contains(s)); +} + +#[overseer::contextbounds(StatementDistribution, prefix=self::overseer)] +async fn handle_peer_view_update( + ctx: &mut Context, + state: &mut State, + peer: PeerId, + new_view: View, +) { + let fresh_implicit = { + let peer_data = match state.peers.get_mut(&peer) { + None => return, + Some(p) => p, + }; + + peer_data.update_view(new_view, &state.implicit_view) + }; + + for new_relay_parent in fresh_implicit { + send_peer_messages_for_relay_parent(ctx, state, peer, new_relay_parent).await; + } +} + +// Returns an iterator over known validator indices, given an iterator over discovery IDs +// and a mapping from discovery IDs to validator indices. +fn find_validator_ids<'a>( + known_discovery_ids: impl IntoIterator, + discovery_mapping: impl Fn(&AuthorityDiscoveryId) -> Option<&'a ValidatorIndex>, +) -> impl Iterator { + known_discovery_ids.into_iter().filter_map(discovery_mapping).cloned() +} + +/// Send a peer, apparently just becoming aware of a relay-parent, all messages +/// concerning that relay-parent. +/// +/// In particular, we send all statements pertaining to our common cluster, +/// as well as all manifests, acknowledgements, or other grid statements. +/// +/// Note that due to the way we handle views, our knowledge of peers' relay parents +/// may "oscillate" with relay parents repeatedly leaving and entering the +/// view of a peer based on the implicit view of active leaves. +/// +/// This function is designed to be cheap and not to send duplicate messages in repeated +/// cases. +#[overseer::contextbounds(StatementDistribution, prefix=self::overseer)] +async fn send_peer_messages_for_relay_parent( + ctx: &mut Context, + state: &mut State, + peer: PeerId, + relay_parent: Hash, +) { + let peer_data = match state.peers.get_mut(&peer) { + None => return, + Some(p) => p, + }; + + let relay_parent_state = match state.per_relay_parent.get_mut(&relay_parent) { + None => return, + Some(s) => s, + }; + + let per_session_state = match state.per_session.get(&relay_parent_state.session) { + None => return, + Some(s) => s, + }; + + for validator_id in find_validator_ids(peer_data.iter_known_discovery_ids(), |a| { + per_session_state.authority_lookup.get(a) + }) { + if let Some(local_validator_state) = relay_parent_state.local_validator.as_mut() { + send_pending_cluster_statements( + ctx, + relay_parent, + &peer, + validator_id, + &mut local_validator_state.cluster_tracker, + &state.candidates, + &relay_parent_state.statement_store, + ) + .await; + } + + send_pending_grid_messages( + ctx, + relay_parent, + &peer, + validator_id, + &per_session_state.groups, + relay_parent_state, + &state.candidates, + ) + .await; + } +} + +fn pending_statement_network_message( + statement_store: &StatementStore, + relay_parent: Hash, + peer: &PeerId, + originator: ValidatorIndex, + compact: CompactStatement, +) -> Option<(Vec, net_protocol::VersionedValidationProtocol)> { + statement_store + .validator_statement(originator, compact) + .map(|s| s.as_unchecked().clone()) + .map(|signed| { + protocol_vstaging::StatementDistributionMessage::Statement(relay_parent, signed) + }) + .map(|msg| (vec![*peer], Versioned::VStaging(msg).into())) +} + +/// Send a peer all pending cluster statements for a relay parent. +#[overseer::contextbounds(StatementDistribution, prefix=self::overseer)] +async fn send_pending_cluster_statements( + ctx: &mut Context, + relay_parent: Hash, + peer_id: &PeerId, + peer_validator_id: ValidatorIndex, + cluster_tracker: &mut ClusterTracker, + candidates: &Candidates, + statement_store: &StatementStore, +) { + let pending_statements = cluster_tracker.pending_statements_for(peer_validator_id); + let network_messages = pending_statements + .into_iter() + .filter_map(|(originator, compact)| { + if !candidates.is_confirmed(compact.candidate_hash()) { + return None + } + + let res = pending_statement_network_message( + &statement_store, + relay_parent, + peer_id, + originator, + compact.clone(), + ); + + if res.is_some() { + cluster_tracker.note_sent(peer_validator_id, originator, compact); + } + + res + }) + .collect::>(); + + if network_messages.is_empty() { + return + } + + ctx.send_message(NetworkBridgeTxMessage::SendValidationMessages(network_messages)) + .await; +} + +/// Send a peer all pending grid messages / acknowledgements / follow up statements +/// upon learning about a new relay parent. +#[overseer::contextbounds(StatementDistribution, prefix=self::overseer)] +async fn send_pending_grid_messages( + ctx: &mut Context, + relay_parent: Hash, + peer_id: &PeerId, + peer_validator_id: ValidatorIndex, + groups: &Groups, + relay_parent_state: &mut PerRelayParentState, + candidates: &Candidates, +) { + let pending_manifests = { + let local_validator = match relay_parent_state.local_validator.as_mut() { + None => return, + Some(l) => l, + }; + + let grid_tracker = &mut local_validator.grid_tracker; + grid_tracker.pending_manifests_for(peer_validator_id) + }; + + let mut messages: Vec<(Vec, net_protocol::VersionedValidationProtocol)> = Vec::new(); + for (candidate_hash, kind) in pending_manifests { + let confirmed_candidate = match candidates.get_confirmed(&candidate_hash) { + None => continue, // sanity + Some(c) => c, + }; + + let group_index = confirmed_candidate.group_index(); + + let local_knowledge = { + let group_size = match groups.get(group_index) { + None => return, // sanity + Some(x) => x.len(), + }; + + local_knowledge_filter( + group_size, + group_index, + candidate_hash, + &relay_parent_state.statement_store, + ) + }; + + match kind { + grid::ManifestKind::Full => { + let manifest = protocol_vstaging::BackedCandidateManifest { + relay_parent, + candidate_hash, + group_index, + para_id: confirmed_candidate.para_id(), + parent_head_data_hash: confirmed_candidate.parent_head_data_hash(), + statement_knowledge: local_knowledge.clone(), + }; + + let grid = &mut relay_parent_state + .local_validator + .as_mut() + .expect("determined to be some earlier in this function; qed") + .grid_tracker; + + grid.manifest_sent_to( + groups, + peer_validator_id, + candidate_hash, + local_knowledge.clone(), + ); + + messages.push(( + vec![*peer_id], + Versioned::VStaging( + protocol_vstaging::StatementDistributionMessage::BackedCandidateManifest( + manifest, + ), + ) + .into(), + )); + }, + grid::ManifestKind::Acknowledgement => { + messages.extend(acknowledgement_and_statement_messages( + *peer_id, + peer_validator_id, + groups, + relay_parent_state, + relay_parent, + group_index, + candidate_hash, + local_knowledge, + )); + }, + } + } + + // Send all remaining pending grid statements for a validator, not just + // those for the acknowledgements we've sent. + // + // otherwise, we might receive statements while the grid peer is "out of view" and then + // not send them when they get back "in view". problem! + { + let grid_tracker = &mut relay_parent_state + .local_validator + .as_mut() + .expect("checked earlier; qed") + .grid_tracker; + + let pending_statements = grid_tracker.all_pending_statements_for(peer_validator_id); + + let extra_statements = + pending_statements.into_iter().filter_map(|(originator, compact)| { + let res = pending_statement_network_message( + &relay_parent_state.statement_store, + relay_parent, + peer_id, + originator, + compact.clone(), + ); + + if res.is_some() { + grid_tracker.sent_or_received_direct_statement( + groups, + originator, + peer_validator_id, + &compact, + ); + } + + res + }); + + messages.extend(extra_statements); + } + + if messages.is_empty() { + return + } + ctx.send_message(NetworkBridgeTxMessage::SendValidationMessages(messages)).await; +} + +// Imports a locally originating statement and distributes it to peers. +#[overseer::contextbounds(StatementDistribution, prefix=self::overseer)] +pub(crate) async fn share_local_statement( + ctx: &mut Context, + state: &mut State, + relay_parent: Hash, + statement: SignedFullStatementWithPVD, +) -> JfyiErrorResult<()> { + let per_relay_parent = match state.per_relay_parent.get_mut(&relay_parent) { + None => return Err(JfyiError::InvalidShare), + Some(x) => x, + }; + + let per_session = match state.per_session.get(&per_relay_parent.session) { + Some(s) => s, + None => return Ok(()), + }; + + let (local_index, local_assignment, local_group) = + match per_relay_parent.local_validator.as_ref() { + None => return Err(JfyiError::InvalidShare), + Some(l) => (l.index, l.assignment, l.group), + }; + + // Two possibilities: either the statement is `Seconded` or we already + // have the candidate. Sanity: check the para-id is valid. + let expected = match statement.payload() { + FullStatementWithPVD::Seconded(ref c, _) => + Some((c.descriptor().para_id, c.descriptor().relay_parent)), + FullStatementWithPVD::Valid(hash) => + state.candidates.get_confirmed(&hash).map(|c| (c.para_id(), c.relay_parent())), + }; + + let is_seconded = match statement.payload() { + FullStatementWithPVD::Seconded(_, _) => true, + FullStatementWithPVD::Valid(_) => false, + }; + + let (expected_para, expected_relay_parent) = match expected { + None => return Err(JfyiError::InvalidShare), + Some(x) => x, + }; + + if local_index != statement.validator_index() { + return Err(JfyiError::InvalidShare) + } + + if is_seconded && + per_relay_parent.statement_store.seconded_count(&local_index) == + per_relay_parent.seconding_limit + { + gum::warn!( + target: LOG_TARGET, + limit = ?per_relay_parent.seconding_limit, + "Local node has issued too many `Seconded` statements", + ); + return Err(JfyiError::InvalidShare) + } + + if local_assignment != Some(expected_para) || relay_parent != expected_relay_parent { + return Err(JfyiError::InvalidShare) + } + + let mut post_confirmation = None; + + // Insert candidate if unknown + more sanity checks. + let compact_statement = { + let compact_statement = FullStatementWithPVD::signed_to_compact(statement.clone()); + let candidate_hash = CandidateHash(*statement.payload().candidate_hash()); + + if let FullStatementWithPVD::Seconded(ref c, ref pvd) = statement.payload() { + post_confirmation = state.candidates.confirm_candidate( + candidate_hash, + c.clone(), + pvd.clone(), + local_group, + ); + }; + + match per_relay_parent.statement_store.insert( + &per_session.groups, + compact_statement.clone(), + StatementOrigin::Local, + ) { + Ok(false) | Err(_) => { + gum::warn!( + target: LOG_TARGET, + statement = ?compact_statement.payload(), + "Candidate backing issued redundant statement?", + ); + return Err(JfyiError::InvalidShare) + }, + Ok(true) => {}, + } + + { + let l = per_relay_parent.local_validator.as_mut().expect("checked above; qed"); + l.cluster_tracker.note_issued(local_index, compact_statement.payload().clone()); + } + + if let Some(ref session_topology) = per_session.grid_view { + let l = per_relay_parent.local_validator.as_mut().expect("checked above; qed"); + l.grid_tracker.learned_fresh_statement( + &per_session.groups, + session_topology, + local_index, + &compact_statement.payload(), + ); + } + + compact_statement + }; + + // send the compact version of the statement to any peers which need it. + circulate_statement( + ctx, + relay_parent, + per_relay_parent, + per_session, + &state.candidates, + &state.authorities, + &state.peers, + compact_statement, + ) + .await; + + if let Some(post_confirmation) = post_confirmation { + apply_post_confirmation(ctx, state, post_confirmation).await; + } + + Ok(()) +} + +// two kinds of targets: those in our 'cluster' (currently just those in the same group), +// and those we are propagating to through the grid. +enum DirectTargetKind { + Cluster, + Grid, +} + +// Circulates a compact statement to all peers who need it: those in the current group of the +// local validator and grid peers which have already indicated that they know the candidate as backed. +// +// We only circulate statements for which we have the confirmed candidate, even to the local group. +// +// The group index which is _canonically assigned_ to this parachain must be +// specified already. This function should not be used when the candidate receipt and +// therefore the canonical group for the parachain is unknown. +// +// preconditions: the candidate entry exists in the state under the relay parent +// and the statement has already been imported into the entry. If this is a `Valid` +// statement, then there must be at least one `Seconded` statement. +#[overseer::contextbounds(StatementDistribution, prefix=self::overseer)] +async fn circulate_statement( + ctx: &mut Context, + relay_parent: Hash, + relay_parent_state: &mut PerRelayParentState, + per_session: &PerSessionState, + candidates: &Candidates, + authorities: &HashMap, + peers: &HashMap, + statement: SignedStatement, +) { + let session_info = &per_session.session_info; + + let candidate_hash = *statement.payload().candidate_hash(); + + let compact_statement = statement.payload().clone(); + let is_confirmed = candidates.is_confirmed(&candidate_hash); + + let originator = statement.validator_index(); + let (local_validator, targets) = { + let local_validator = match relay_parent_state.local_validator.as_mut() { + Some(v) => v, + None => return, // sanity: nothing to propagate if not a validator. + }; + + let statement_group = per_session.groups.by_validator_index(originator); + + // We're not meant to circulate statements in the cluster until we have the confirmed candidate. + let cluster_relevant = Some(local_validator.group) == statement_group; + let cluster_targets = if is_confirmed && cluster_relevant { + Some( + local_validator + .cluster_tracker + .targets() + .iter() + .filter(|&&v| { + local_validator + .cluster_tracker + .can_send(v, originator, compact_statement.clone()) + .is_ok() + }) + .filter(|&v| v != &local_validator.index) + .map(|v| (*v, DirectTargetKind::Cluster)), + ) + } else { + None + }; + + let grid_targets = local_validator + .grid_tracker + .direct_statement_targets(&per_session.groups, originator, &compact_statement) + .into_iter() + .filter(|v| !cluster_relevant || !local_validator.cluster_tracker.targets().contains(v)) + .map(|v| (v, DirectTargetKind::Grid)); + + let targets = cluster_targets + .into_iter() + .flatten() + .chain(grid_targets) + .filter_map(|(v, k)| { + session_info.discovery_keys.get(v.0 as usize).map(|a| (v, a.clone(), k)) + }) + .collect::>(); + + (local_validator, targets) + }; + + let mut statement_to = Vec::new(); + for (target, authority_id, kind) in targets { + // Find peer ID based on authority ID, and also filter to connected. + let peer_id: PeerId = match authorities.get(&authority_id) { + Some(p) if peers.get(p).map_or(false, |p| p.knows_relay_parent(&relay_parent)) => *p, + None | Some(_) => continue, + }; + + match kind { + DirectTargetKind::Cluster => { + // At this point, all peers in the cluster should 'know' + // the candidate, so we don't expect for this to fail. + if let Ok(()) = local_validator.cluster_tracker.can_send( + target, + originator, + compact_statement.clone(), + ) { + local_validator.cluster_tracker.note_sent( + target, + originator, + compact_statement.clone(), + ); + statement_to.push(peer_id); + } + }, + DirectTargetKind::Grid => { + statement_to.push(peer_id); + local_validator.grid_tracker.sent_or_received_direct_statement( + &per_session.groups, + originator, + target, + &compact_statement, + ); + }, + } + } + + // ship off the network messages to the network bridge. + + if !statement_to.is_empty() { + ctx.send_message(NetworkBridgeTxMessage::SendValidationMessage( + statement_to, + Versioned::VStaging(protocol_vstaging::StatementDistributionMessage::Statement( + relay_parent, + statement.as_unchecked().clone(), + )) + .into(), + )) + .await; + } +} +/// Check a statement signature under this parent hash. +fn check_statement_signature( + session_index: SessionIndex, + validators: &IndexedVec, + relay_parent: Hash, + statement: UncheckedSignedStatement, +) -> std::result::Result { + let signing_context = SigningContext { session_index, parent_hash: relay_parent }; + + validators + .get(statement.unchecked_validator_index()) + .ok_or_else(|| statement.clone()) + .and_then(|v| statement.try_into_checked(&signing_context, v)) +} + +async fn report_peer( + sender: &mut impl overseer::StatementDistributionSenderTrait, + peer: PeerId, + rep: Rep, +) { + sender.send_message(NetworkBridgeTxMessage::ReportPeer(peer, rep)).await +} + +/// Handle an incoming statement. +/// +/// This checks whether the sender is allowed to send the statement, +/// either via the cluster or the grid. +/// +/// This also checks the signature of the statement. +/// If the statement is fresh, this function guarantees that after completion +/// - The statement is re-circulated to all relevant peers in both the cluster +/// and the grid +/// - If the candidate is out-of-cluster and is backable and importable, +/// all statements about the candidate have been sent to backing +/// - If the candidate is in-cluster and is importable, +/// the statement has been sent to backing +#[overseer::contextbounds(StatementDistribution, prefix=self::overseer)] +async fn handle_incoming_statement( + ctx: &mut Context, + state: &mut State, + peer: PeerId, + relay_parent: Hash, + statement: UncheckedSignedStatement, +) { + let peer_state = match state.peers.get(&peer) { + None => { + // sanity: should be impossible. + return + }, + Some(p) => p, + }; + + // Ensure we know the relay parent. + let per_relay_parent = match state.per_relay_parent.get_mut(&relay_parent) { + None => { + report_peer(ctx.sender(), peer, COST_UNEXPECTED_STATEMENT_MISSING_KNOWLEDGE).await; + return + }, + Some(p) => p, + }; + + let per_session = match state.per_session.get(&per_relay_parent.session) { + None => { + gum::warn!( + target: LOG_TARGET, + session = ?per_relay_parent.session, + "Missing expected session info.", + ); + + return + }, + Some(s) => s, + }; + let session_info = &per_session.session_info; + + let local_validator = match per_relay_parent.local_validator.as_mut() { + None => { + // we shouldn't be receiving statements unless we're a validator + // this session. + report_peer(ctx.sender(), peer, COST_UNEXPECTED_STATEMENT).await; + return + }, + Some(l) => l, + }; + + let originator_group = + match per_session.groups.by_validator_index(statement.unchecked_validator_index()) { + Some(g) => g, + None => { + report_peer(ctx.sender(), peer, COST_UNEXPECTED_STATEMENT).await; + return + }, + }; + + let cluster_sender_index = { + // This block of code only returns `Some` when both the originator and + // the sending peer are in the cluster. + + let allowed_senders = local_validator + .cluster_tracker + .senders_for_originator(statement.unchecked_validator_index()); + + allowed_senders + .iter() + .filter_map(|i| session_info.discovery_keys.get(i.0 as usize).map(|ad| (*i, ad))) + .filter(|(_, ad)| peer_state.is_authority(ad)) + .map(|(i, _)| i) + .next() + }; + + let checked_statement = if let Some(cluster_sender_index) = cluster_sender_index { + match handle_cluster_statement( + relay_parent, + &mut local_validator.cluster_tracker, + per_relay_parent.session, + &per_session.session_info, + statement, + cluster_sender_index, + ) { + Ok(Some(s)) => s, + Ok(None) => return, + Err(rep) => { + report_peer(ctx.sender(), peer, rep).await; + return + }, + } + } else { + let grid_sender_index = local_validator + .grid_tracker + .direct_statement_providers( + &per_session.groups, + statement.unchecked_validator_index(), + statement.unchecked_payload(), + ) + .into_iter() + .filter_map(|i| session_info.discovery_keys.get(i.0 as usize).map(|ad| (i, ad))) + .filter(|(_, ad)| peer_state.is_authority(ad)) + .map(|(i, _)| i) + .next(); + + if let Some(grid_sender_index) = grid_sender_index { + match handle_grid_statement( + relay_parent, + &mut local_validator.grid_tracker, + per_relay_parent.session, + &per_session, + statement, + grid_sender_index, + ) { + Ok(s) => s, + Err(rep) => { + report_peer(ctx.sender(), peer, rep).await; + return + }, + } + } else { + // Not a cluster or grid peer. + report_peer(ctx.sender(), peer, COST_UNEXPECTED_STATEMENT).await; + return + } + }; + + let statement = checked_statement.payload().clone(); + let originator_index = checked_statement.validator_index(); + let candidate_hash = *checked_statement.payload().candidate_hash(); + + // Insert an unconfirmed candidate entry if needed. Note that if the candidate is already confirmed, + // this ensures that the assigned group of the originator matches the expected group of the + // parachain. + { + let res = state.candidates.insert_unconfirmed( + peer, + candidate_hash, + relay_parent, + originator_group, + None, + ); + + if let Err(BadAdvertisement) = res { + report_peer(ctx.sender(), peer, COST_UNEXPECTED_STATEMENT).await; + return + } + } + + let confirmed = state.candidates.get_confirmed(&candidate_hash); + let is_confirmed = state.candidates.is_confirmed(&candidate_hash); + if !is_confirmed { + // If the candidate is not confirmed, note that we should attempt + // to request it from the given peer. + let mut request_entry = + state + .request_manager + .get_or_insert(relay_parent, candidate_hash, originator_group); + + request_entry.add_peer(peer); + + // We only successfully accept statements from the grid on confirmed + // candidates, therefore this check only passes if the statement is from the cluster + request_entry.set_cluster_priority(); + } + + let was_fresh = match per_relay_parent.statement_store.insert( + &per_session.groups, + checked_statement.clone(), + StatementOrigin::Remote, + ) { + Err(statement_store::ValidatorUnknown) => { + // sanity: should never happen. + gum::warn!( + target: LOG_TARGET, + ?relay_parent, + validator_index = ?originator_index, + "Error - accepted message from unknown validator." + ); + + return + }, + Ok(known) => known, + }; + + if was_fresh { + report_peer(ctx.sender(), peer, BENEFIT_VALID_STATEMENT_FIRST).await; + let is_importable = state.candidates.is_importable(&candidate_hash); + + if let Some(ref session_topology) = per_session.grid_view { + local_validator.grid_tracker.learned_fresh_statement( + &per_session.groups, + session_topology, + local_validator.index, + &statement, + ); + } + + if let (true, &Some(confirmed)) = (is_importable, &confirmed) { + send_backing_fresh_statements( + ctx, + candidate_hash, + originator_group, + &relay_parent, + &mut *per_relay_parent, + confirmed, + per_session, + ) + .await; + } + + // We always circulate statements at this point. + circulate_statement( + ctx, + relay_parent, + per_relay_parent, + per_session, + &state.candidates, + &state.authorities, + &state.peers, + checked_statement, + ) + .await; + } else { + report_peer(ctx.sender(), peer, BENEFIT_VALID_STATEMENT).await; + } +} + +/// Checks whether a statement is allowed, whether the signature is accurate, +/// and importing into the cluster tracker if successful. +/// +/// if successful, this returns a checked signed statement if it should be imported +/// or otherwise an error indicating a reputational fault. +fn handle_cluster_statement( + relay_parent: Hash, + cluster_tracker: &mut ClusterTracker, + session: SessionIndex, + session_info: &SessionInfo, + statement: UncheckedSignedStatement, + cluster_sender_index: ValidatorIndex, +) -> Result, Rep> { + // additional cluster checks. + let should_import = { + match cluster_tracker.can_receive( + cluster_sender_index, + statement.unchecked_validator_index(), + statement.unchecked_payload().clone(), + ) { + Ok(ClusterAccept::Ok) => true, + Ok(ClusterAccept::WithPrejudice) => false, + Err(ClusterRejectIncoming::ExcessiveSeconded) => return Err(COST_EXCESSIVE_SECONDED), + Err(ClusterRejectIncoming::CandidateUnknown | ClusterRejectIncoming::Duplicate) => + return Err(COST_UNEXPECTED_STATEMENT), + Err(ClusterRejectIncoming::NotInGroup) => { + // sanity: shouldn't be possible; we already filtered this + // out above. + return Err(COST_UNEXPECTED_STATEMENT) + }, + } + }; + + // Ensure the statement is correctly signed. + let checked_statement = + match check_statement_signature(session, &session_info.validators, relay_parent, statement) + { + Ok(s) => s, + Err(_) => return Err(COST_INVALID_SIGNATURE), + }; + + cluster_tracker.note_received( + cluster_sender_index, + checked_statement.validator_index(), + checked_statement.payload().clone(), + ); + + Ok(if should_import { Some(checked_statement) } else { None }) +} + +/// Checks whether the signature is accurate, +/// importing into the grid tracker if successful. +/// +/// if successful, this returns a checked signed statement if it should be imported +/// or otherwise an error indicating a reputational fault. +fn handle_grid_statement( + relay_parent: Hash, + grid_tracker: &mut GridTracker, + session: SessionIndex, + per_session: &PerSessionState, + statement: UncheckedSignedStatement, + grid_sender_index: ValidatorIndex, +) -> Result { + // Ensure the statement is correctly signed. + let checked_statement = match check_statement_signature( + session, + &per_session.session_info.validators, + relay_parent, + statement, + ) { + Ok(s) => s, + Err(_) => return Err(COST_INVALID_SIGNATURE), + }; + + grid_tracker.sent_or_received_direct_statement( + &per_session.groups, + checked_statement.validator_index(), + grid_sender_index, + &checked_statement.payload(), + ); + + Ok(checked_statement) +} + +/// Send backing fresh statements. This should only be performed on importable & confirmed +/// candidates. +#[overseer::contextbounds(StatementDistribution, prefix=self::overseer)] +async fn send_backing_fresh_statements( + ctx: &mut Context, + candidate_hash: CandidateHash, + group_index: GroupIndex, + relay_parent: &Hash, + relay_parent_state: &mut PerRelayParentState, + confirmed: &candidates::ConfirmedCandidate, + per_session: &PerSessionState, +) { + let group_validators = per_session.groups.get(group_index).unwrap_or(&[]); + let mut imported = Vec::new(); + + for statement in relay_parent_state + .statement_store + .fresh_statements_for_backing(group_validators, candidate_hash) + { + let v = statement.validator_index(); + let compact = statement.payload().clone(); + imported.push((v, compact)); + let carrying_pvd = statement + .clone() + .convert_to_superpayload_with(|statement| match statement { + CompactStatement::Seconded(_) => FullStatementWithPVD::Seconded( + (&**confirmed.candidate_receipt()).clone(), + confirmed.persisted_validation_data().clone(), + ), + CompactStatement::Valid(c_hash) => FullStatementWithPVD::Valid(c_hash), + }) + .expect("statements refer to same candidate; qed"); + + ctx.send_message(CandidateBackingMessage::Statement(*relay_parent, carrying_pvd)) + .await; + } + + for (v, s) in imported { + relay_parent_state.statement_store.note_known_by_backing(v, s); + } +} + +fn local_knowledge_filter( + group_size: usize, + group_index: GroupIndex, + candidate_hash: CandidateHash, + statement_store: &StatementStore, +) -> StatementFilter { + let mut f = StatementFilter::blank(group_size); + statement_store.fill_statement_filter(group_index, candidate_hash, &mut f); + f +} + +// This provides a backable candidate to the grid and dispatches backable candidate announcements +// and acknowledgements via the grid topology. If the session topology is not yet +// available, this will be a no-op. +#[overseer::contextbounds(StatementDistribution, prefix=self::overseer)] +async fn provide_candidate_to_grid( + ctx: &mut Context, + candidate_hash: CandidateHash, + relay_parent_state: &mut PerRelayParentState, + confirmed_candidate: &candidates::ConfirmedCandidate, + per_session: &PerSessionState, + authorities: &HashMap, + peers: &HashMap, +) { + let local_validator = match relay_parent_state.local_validator { + Some(ref mut v) => v, + None => return, + }; + + let relay_parent = confirmed_candidate.relay_parent(); + let group_index = confirmed_candidate.group_index(); + + let grid_view = match per_session.grid_view { + Some(ref t) => t, + None => { + gum::trace!( + target: LOG_TARGET, + session = relay_parent_state.session, + "Cannot handle backable candidate due to lack of topology", + ); + + return + }, + }; + + let group_size = match per_session.groups.get(group_index) { + None => { + gum::warn!( + target: LOG_TARGET, + ?candidate_hash, + ?relay_parent, + ?group_index, + session = relay_parent_state.session, + "Handled backed candidate with unknown group?", + ); + + return + }, + Some(g) => g.len(), + }; + + let filter = local_knowledge_filter( + group_size, + group_index, + candidate_hash, + &relay_parent_state.statement_store, + ); + + let actions = local_validator.grid_tracker.add_backed_candidate( + grid_view, + candidate_hash, + group_index, + filter.clone(), + ); + + let manifest = protocol_vstaging::BackedCandidateManifest { + relay_parent, + candidate_hash, + group_index, + para_id: confirmed_candidate.para_id(), + parent_head_data_hash: confirmed_candidate.parent_head_data_hash(), + statement_knowledge: filter.clone(), + }; + let acknowledgement = protocol_vstaging::BackedCandidateAcknowledgement { + candidate_hash, + statement_knowledge: filter.clone(), + }; + + let manifest_message = Versioned::VStaging( + protocol_vstaging::StatementDistributionMessage::BackedCandidateManifest(manifest), + ); + let ack_message = Versioned::VStaging( + protocol_vstaging::StatementDistributionMessage::BackedCandidateKnown(acknowledgement), + ); + + let mut manifest_peers = Vec::new(); + let mut ack_peers = Vec::new(); + + let mut post_statements = Vec::new(); + for (v, action) in actions { + let p = match connected_validator_peer(authorities, per_session, v) { + None => continue, + Some(p) => + if peers.get(&p).map_or(false, |d| d.knows_relay_parent(&relay_parent)) { + p + } else { + continue + }, + }; + + match action { + grid::ManifestKind::Full => manifest_peers.push(p), + grid::ManifestKind::Acknowledgement => ack_peers.push(p), + } + + local_validator.grid_tracker.manifest_sent_to( + &per_session.groups, + v, + candidate_hash, + filter.clone(), + ); + post_statements.extend( + post_acknowledgement_statement_messages( + v, + relay_parent, + &mut local_validator.grid_tracker, + &relay_parent_state.statement_store, + &per_session.groups, + group_index, + candidate_hash, + ) + .into_iter() + .map(|m| (vec![p], m)), + ); + } + + if !manifest_peers.is_empty() { + ctx.send_message(NetworkBridgeTxMessage::SendValidationMessage( + manifest_peers, + manifest_message.into(), + )) + .await; + } + + if !ack_peers.is_empty() { + ctx.send_message(NetworkBridgeTxMessage::SendValidationMessage( + ack_peers, + ack_message.into(), + )) + .await; + } + + if !post_statements.is_empty() { + ctx.send_message(NetworkBridgeTxMessage::SendValidationMessages(post_statements)) + .await; + } +} + +fn group_for_para( + availability_cores: &[CoreState], + group_rotation_info: &GroupRotationInfo, + para_id: ParaId, +) -> Option { + // Note: this won't work well for parathreads as it assumes that core assignments are fixed + // across blocks. + let core_index = availability_cores.iter().position(|c| c.para_id() == Some(para_id)); + + core_index + .map(|c| group_rotation_info.group_for_core(CoreIndex(c as _), availability_cores.len())) +} + +#[overseer::contextbounds(StatementDistribution, prefix=self::overseer)] +async fn fragment_tree_update_inner( + ctx: &mut Context, + state: &mut State, + active_leaf_hash: Option, + required_parent_info: Option<(Hash, ParaId)>, + known_hypotheticals: Option>, +) { + // 1. get hypothetical candidates + let hypotheticals = match known_hypotheticals { + None => state.candidates.frontier_hypotheticals(required_parent_info), + Some(h) => h, + }; + + // 2. find out which are in the frontier + let frontier = { + let (tx, rx) = oneshot::channel(); + ctx.send_message(ProspectiveParachainsMessage::GetHypotheticalFrontier( + HypotheticalFrontierRequest { + candidates: hypotheticals, + fragment_tree_relay_parent: active_leaf_hash, + backed_in_path_only: false, + }, + tx, + )) + .await; + + match rx.await { + Ok(frontier) => frontier, + Err(oneshot::Canceled) => return, + } + }; + // 3. note that they are importable under a given leaf hash. + for (hypo, membership) in frontier { + // skip parablocks outside of the frontier + if membership.is_empty() { + continue + } + + for (leaf_hash, _) in membership { + state.candidates.note_importable_under(&hypo, leaf_hash); + } + + // 4. for confirmed candidates, send all statements which are new to backing. + if let HypotheticalCandidate::Complete { + candidate_hash, + receipt, + persisted_validation_data: _, + } = hypo + { + let confirmed_candidate = state.candidates.get_confirmed(&candidate_hash); + let prs = state.per_relay_parent.get_mut(&receipt.descriptor().relay_parent); + if let (Some(confirmed), Some(prs)) = (confirmed_candidate, prs) { + let group_index = group_for_para( + &prs.availability_cores, + &prs.group_rotation_info, + receipt.descriptor().para_id, + ); + + let per_session = state.per_session.get(&prs.session); + if let (Some(per_session), Some(group_index)) = (per_session, group_index) { + send_backing_fresh_statements( + ctx, + candidate_hash, + group_index, + &receipt.descriptor().relay_parent, + prs, + confirmed, + per_session, + ) + .await; + } + } + } + } +} + +#[overseer::contextbounds(StatementDistribution, prefix=self::overseer)] +async fn new_leaf_fragment_tree_updates( + ctx: &mut Context, + state: &mut State, + leaf_hash: Hash, +) { + fragment_tree_update_inner(ctx, state, Some(leaf_hash), None, None).await +} + +#[overseer::contextbounds(StatementDistribution, prefix=self::overseer)] +async fn prospective_backed_notification_fragment_tree_updates( + ctx: &mut Context, + state: &mut State, + para_id: ParaId, + para_head: Hash, +) { + fragment_tree_update_inner(ctx, state, None, Some((para_head, para_id)), None).await +} + +#[overseer::contextbounds(StatementDistribution, prefix=self::overseer)] +async fn new_confirmed_candidate_fragment_tree_updates( + ctx: &mut Context, + state: &mut State, + candidate: HypotheticalCandidate, +) { + fragment_tree_update_inner(ctx, state, None, None, Some(vec![candidate])).await +} + +struct ManifestImportSuccess<'a> { + relay_parent_state: &'a mut PerRelayParentState, + per_session: &'a PerSessionState, + acknowledge: bool, + sender_index: ValidatorIndex, +} + +/// Handles the common part of incoming manifests of both types (full & acknowledgement) +/// +/// Basic sanity checks around data, importing the manifest into the grid tracker, finding the +/// sending peer's validator index, reporting the peer for any misbehavior, etc. +#[overseer::contextbounds(StatementDistribution, prefix=self::overseer)] +async fn handle_incoming_manifest_common<'a, Context>( + ctx: &mut Context, + peer: PeerId, + peers: &HashMap, + per_relay_parent: &'a mut HashMap, + per_session: &'a HashMap, + candidates: &mut Candidates, + candidate_hash: CandidateHash, + relay_parent: Hash, + para_id: ParaId, + manifest_summary: grid::ManifestSummary, + manifest_kind: grid::ManifestKind, +) -> Option> { + // 1. sanity checks: peer is connected, relay-parent in state, para ID matches group index. + let peer_state = match peers.get(&peer) { + None => return None, + Some(p) => p, + }; + + let relay_parent_state = match per_relay_parent.get_mut(&relay_parent) { + None => { + report_peer(ctx.sender(), peer, COST_UNEXPECTED_MANIFEST_MISSING_KNOWLEDGE).await; + return None + }, + Some(s) => s, + }; + + let per_session = match per_session.get(&relay_parent_state.session) { + None => return None, + Some(s) => s, + }; + + let local_validator = match relay_parent_state.local_validator.as_mut() { + None => { + report_peer(ctx.sender(), peer, COST_UNEXPECTED_MANIFEST_MISSING_KNOWLEDGE).await; + return None + }, + Some(x) => x, + }; + + let expected_group = group_for_para( + &relay_parent_state.availability_cores, + &relay_parent_state.group_rotation_info, + para_id, + ); + + if expected_group != Some(manifest_summary.claimed_group_index) { + report_peer(ctx.sender(), peer, COST_MALFORMED_MANIFEST).await; + return None + } + + let grid_topology = match per_session.grid_view.as_ref() { + None => return None, + Some(x) => x, + }; + + let sender_index = grid_topology + .iter_sending_for_group(manifest_summary.claimed_group_index, manifest_kind) + .filter_map(|i| per_session.session_info.discovery_keys.get(i.0 as usize).map(|ad| (i, ad))) + .filter(|(_, ad)| peer_state.is_authority(ad)) + .map(|(i, _)| i) + .next(); + + let sender_index = match sender_index { + None => { + report_peer(ctx.sender(), peer, COST_UNEXPECTED_MANIFEST_DISALLOWED).await; + return None + }, + Some(s) => s, + }; + + // 2. sanity checks: peer is validator, bitvec size, import into grid tracker + let group_index = manifest_summary.claimed_group_index; + let claimed_parent_hash = manifest_summary.claimed_parent_hash; + let acknowledge = match local_validator.grid_tracker.import_manifest( + grid_topology, + &per_session.groups, + candidate_hash, + relay_parent_state.seconding_limit, + manifest_summary, + manifest_kind, + sender_index, + ) { + Ok(x) => x, + Err(grid::ManifestImportError::Conflicting) => { + report_peer(ctx.sender(), peer, COST_CONFLICTING_MANIFEST).await; + return None + }, + Err(grid::ManifestImportError::Overflow) => { + report_peer(ctx.sender(), peer, COST_EXCESSIVE_SECONDED).await; + return None + }, + Err(grid::ManifestImportError::Insufficient) => { + report_peer(ctx.sender(), peer, COST_INSUFFICIENT_MANIFEST).await; + return None + }, + Err(grid::ManifestImportError::Malformed) => { + report_peer(ctx.sender(), peer, COST_MALFORMED_MANIFEST).await; + return None + }, + Err(grid::ManifestImportError::Disallowed) => { + report_peer(ctx.sender(), peer, COST_UNEXPECTED_MANIFEST_DISALLOWED).await; + return None + }, + }; + + // 3. if accepted by grid, insert as unconfirmed. + if let Err(BadAdvertisement) = candidates.insert_unconfirmed( + peer, + candidate_hash, + relay_parent, + group_index, + Some((claimed_parent_hash, para_id)), + ) { + report_peer(ctx.sender(), peer, COST_INACCURATE_ADVERTISEMENT).await; + return None + } + + Some(ManifestImportSuccess { relay_parent_state, per_session, acknowledge, sender_index }) +} + +/// Produce a list of network messages to send to a peer, following acknowledgement of a manifest. +/// This notes the messages as sent within the grid state. +fn post_acknowledgement_statement_messages( + recipient: ValidatorIndex, + relay_parent: Hash, + grid_tracker: &mut GridTracker, + statement_store: &StatementStore, + groups: &Groups, + group_index: GroupIndex, + candidate_hash: CandidateHash, +) -> Vec { + let sending_filter = match grid_tracker.pending_statements_for(recipient, candidate_hash) { + None => return Vec::new(), + Some(f) => f, + }; + + let mut messages = Vec::new(); + for statement in + statement_store.group_statements(groups, group_index, candidate_hash, &sending_filter) + { + grid_tracker.sent_or_received_direct_statement( + groups, + statement.validator_index(), + recipient, + statement.payload(), + ); + + messages.push(Versioned::VStaging( + protocol_vstaging::StatementDistributionMessage::Statement( + relay_parent, + statement.as_unchecked().clone(), + ) + .into(), + )); + } + + messages +} + +#[overseer::contextbounds(StatementDistribution, prefix=self::overseer)] +async fn handle_incoming_manifest( + ctx: &mut Context, + state: &mut State, + peer: PeerId, + manifest: net_protocol::vstaging::BackedCandidateManifest, +) { + let x = match handle_incoming_manifest_common( + ctx, + peer, + &state.peers, + &mut state.per_relay_parent, + &state.per_session, + &mut state.candidates, + manifest.candidate_hash, + manifest.relay_parent, + manifest.para_id, + grid::ManifestSummary { + claimed_parent_hash: manifest.parent_head_data_hash, + claimed_group_index: manifest.group_index, + statement_knowledge: manifest.statement_knowledge, + }, + grid::ManifestKind::Full, + ) + .await + { + Some(x) => x, + None => return, + }; + + let ManifestImportSuccess { relay_parent_state, per_session, acknowledge, sender_index } = x; + + if acknowledge { + // 4. if already known within grid (confirmed & backed), acknowledge candidate + + let local_knowledge = { + let group_size = match per_session.groups.get(manifest.group_index) { + None => return, // sanity + Some(x) => x.len(), + }; + + local_knowledge_filter( + group_size, + manifest.group_index, + manifest.candidate_hash, + &relay_parent_state.statement_store, + ) + }; + + let messages = acknowledgement_and_statement_messages( + peer, + sender_index, + &per_session.groups, + relay_parent_state, + manifest.relay_parent, + manifest.group_index, + manifest.candidate_hash, + local_knowledge, + ); + + if !messages.is_empty() { + ctx.send_message(NetworkBridgeTxMessage::SendValidationMessages(messages)).await; + } + } else if !state.candidates.is_confirmed(&manifest.candidate_hash) { + // 5. if unconfirmed, add request entry + state + .request_manager + .get_or_insert(manifest.relay_parent, manifest.candidate_hash, manifest.group_index) + .add_peer(peer); + } +} + +/// Produces acknowledgement and statement messages to be sent over the network, +/// noting that they have been sent within the grid topology tracker as well. +fn acknowledgement_and_statement_messages( + peer: PeerId, + validator_index: ValidatorIndex, + groups: &Groups, + relay_parent_state: &mut PerRelayParentState, + relay_parent: Hash, + group_index: GroupIndex, + candidate_hash: CandidateHash, + local_knowledge: StatementFilter, +) -> Vec<(Vec, net_protocol::VersionedValidationProtocol)> { + let local_validator = match relay_parent_state.local_validator.as_mut() { + None => return Vec::new(), + Some(l) => l, + }; + + let acknowledgement = protocol_vstaging::BackedCandidateAcknowledgement { + candidate_hash, + statement_knowledge: local_knowledge.clone(), + }; + + let msg = Versioned::VStaging( + protocol_vstaging::StatementDistributionMessage::BackedCandidateKnown(acknowledgement), + ); + + let mut messages = vec![(vec![peer], msg.into())]; + + local_validator.grid_tracker.manifest_sent_to( + groups, + validator_index, + candidate_hash, + local_knowledge.clone(), + ); + + let statement_messages = post_acknowledgement_statement_messages( + validator_index, + relay_parent, + &mut local_validator.grid_tracker, + &relay_parent_state.statement_store, + &groups, + group_index, + candidate_hash, + ); + + messages.extend(statement_messages.into_iter().map(|m| (vec![peer], m))); + + messages +} + +#[overseer::contextbounds(StatementDistribution, prefix=self::overseer)] +async fn handle_incoming_acknowledgement( + ctx: &mut Context, + state: &mut State, + peer: PeerId, + acknowledgement: net_protocol::vstaging::BackedCandidateAcknowledgement, +) { + // The key difference between acknowledgments and full manifests is that only + // the candidate hash is included alongside the bitfields, so the candidate + // must be confirmed for us to even process it. + + let candidate_hash = acknowledgement.candidate_hash; + let (relay_parent, parent_head_data_hash, group_index, para_id) = { + match state.candidates.get_confirmed(&candidate_hash) { + Some(c) => (c.relay_parent(), c.parent_head_data_hash(), c.group_index(), c.para_id()), + None => { + report_peer(ctx.sender(), peer, COST_UNEXPECTED_ACKNOWLEDGEMENT_UNKNOWN_CANDIDATE) + .await; + return + }, + } + }; + + let x = match handle_incoming_manifest_common( + ctx, + peer, + &state.peers, + &mut state.per_relay_parent, + &state.per_session, + &mut state.candidates, + candidate_hash, + relay_parent, + para_id, + grid::ManifestSummary { + claimed_parent_hash: parent_head_data_hash, + claimed_group_index: group_index, + statement_knowledge: acknowledgement.statement_knowledge, + }, + grid::ManifestKind::Acknowledgement, + ) + .await + { + Some(x) => x, + None => return, + }; + + let ManifestImportSuccess { relay_parent_state, per_session, sender_index, .. } = x; + + let local_validator = match relay_parent_state.local_validator.as_mut() { + None => return, + Some(l) => l, + }; + + let messages = post_acknowledgement_statement_messages( + sender_index, + relay_parent, + &mut local_validator.grid_tracker, + &relay_parent_state.statement_store, + &per_session.groups, + group_index, + candidate_hash, + ); + + if !messages.is_empty() { + ctx.send_message(NetworkBridgeTxMessage::SendValidationMessages( + messages.into_iter().map(|m| (vec![peer], m)).collect(), + )) + .await; + } +} + +/// Handle a notification of a candidate being backed. +#[overseer::contextbounds(StatementDistribution, prefix=self::overseer)] +pub(crate) async fn handle_backed_candidate_message( + ctx: &mut Context, + state: &mut State, + candidate_hash: CandidateHash, +) { + // If the candidate is unknown or unconfirmed, it's a race (pruned before receiving message) + // or a bug. Ignore if so + let confirmed = match state.candidates.get_confirmed(&candidate_hash) { + None => { + gum::debug!( + target: LOG_TARGET, + ?candidate_hash, + "Received backed candidate notification for unknown or unconfirmed", + ); + + return + }, + Some(c) => c, + }; + + let relay_parent_state = match state.per_relay_parent.get_mut(&confirmed.relay_parent()) { + None => return, + Some(s) => s, + }; + + let per_session = match state.per_session.get(&relay_parent_state.session) { + None => return, + Some(s) => s, + }; + + provide_candidate_to_grid( + ctx, + candidate_hash, + relay_parent_state, + confirmed, + per_session, + &state.authorities, + &state.peers, + ) + .await; + + // Search for children of the backed candidate to request. + prospective_backed_notification_fragment_tree_updates( + ctx, + state, + confirmed.para_id(), + confirmed.candidate_receipt().descriptor().para_head, + ) + .await; +} + +/// Sends all messages about a candidate to all peers in the cluster, +/// with `Seconded` statements first. +#[overseer::contextbounds(StatementDistribution, prefix=self::overseer)] +async fn send_cluster_candidate_statements( + ctx: &mut Context, + state: &mut State, + candidate_hash: CandidateHash, + relay_parent: Hash, +) { + let relay_parent_state = match state.per_relay_parent.get_mut(&relay_parent) { + None => return, + Some(s) => s, + }; + + let per_session = match state.per_session.get(&relay_parent_state.session) { + None => return, + Some(s) => s, + }; + + let local_group = match relay_parent_state.local_validator.as_mut() { + None => return, + Some(v) => v.group, + }; + + let group_size = match per_session.groups.get(local_group) { + None => return, + Some(g) => g.len(), + }; + + let statements: Vec<_> = relay_parent_state + .statement_store + .group_statements( + &per_session.groups, + local_group, + candidate_hash, + &StatementFilter::full(group_size), + ) + .map(|x| x.clone()) + .collect(); + + for statement in statements { + circulate_statement( + ctx, + relay_parent, + relay_parent_state, + per_session, + &state.candidates, + &state.authorities, + &state.peers, + statement, + ) + .await; + } +} + +/// Applies state & p2p updates as a result of a newly confirmed candidate. +/// +/// This punishes peers which advertised the candidate incorrectly, as well as +/// doing an importability analysis of the confirmed candidate and providing +/// statements to the backing subsystem if importable. It also cleans up +/// any pending requests for the candidate. +#[overseer::contextbounds(StatementDistribution, prefix=self::overseer)] +async fn apply_post_confirmation( + ctx: &mut Context, + state: &mut State, + post_confirmation: PostConfirmation, +) { + for peer in post_confirmation.reckoning.incorrect { + report_peer(ctx.sender(), peer, COST_INACCURATE_ADVERTISEMENT).await; + } + + let candidate_hash = post_confirmation.hypothetical.candidate_hash(); + state.request_manager.remove_for(candidate_hash); + + send_cluster_candidate_statements( + ctx, + state, + candidate_hash, + post_confirmation.hypothetical.relay_parent(), + ) + .await; + new_confirmed_candidate_fragment_tree_updates(ctx, state, post_confirmation.hypothetical).await; +} + +/// Dispatch pending requests for candidate data & statements. +#[overseer::contextbounds(StatementDistribution, prefix=self::overseer)] +pub(crate) async fn dispatch_requests(ctx: &mut Context, state: &mut State) { + let peers = &state.peers; + let peer_advertised = |identifier: &CandidateIdentifier, peer: &_| { + let peer_data = peers.get(peer)?; + + let relay_parent_state = state.per_relay_parent.get(&identifier.relay_parent)?; + let per_session = state.per_session.get(&relay_parent_state.session)?; + + let local_validator = relay_parent_state.local_validator.as_ref()?; + + for validator_id in find_validator_ids(peer_data.iter_known_discovery_ids(), |a| { + per_session.authority_lookup.get(a) + }) { + // For cluster members, they haven't advertised any statements in particular, + // but have surely sent us some. + if local_validator + .cluster_tracker + .knows_candidate(validator_id, identifier.candidate_hash) + { + return Some(StatementFilter::blank(local_validator.cluster_tracker.targets().len())) + } + + let filter = local_validator + .grid_tracker + .advertised_statements(validator_id, &identifier.candidate_hash); + + if let Some(f) = filter { + return Some(f) + } + } + + None + }; + let request_props = |identifier: &CandidateIdentifier| { + let &CandidateIdentifier { relay_parent, group_index, .. } = identifier; + + let relay_parent_state = state.per_relay_parent.get(&relay_parent)?; + let per_session = state.per_session.get(&relay_parent_state.session)?; + let group = per_session.groups.get(group_index)?; + let seconding_limit = relay_parent_state.seconding_limit; + + // Request nothing which would be an 'over-seconded' statement. + let mut unwanted_mask = StatementFilter::blank(group.len()); + for (i, v) in group.iter().enumerate() { + if relay_parent_state.statement_store.seconded_count(v) >= seconding_limit { + unwanted_mask.seconded_in_group.set(i, true); + } + } + + // don't require a backing threshold for cluster candidates. + let require_backing = relay_parent_state.local_validator.as_ref()?.group != group_index; + + Some(RequestProperties { + unwanted_mask, + backing_threshold: if require_backing { + Some(polkadot_node_primitives::minimum_votes(group.len())) + } else { + None + }, + }) + }; + + while let Some(request) = state.request_manager.next_request(request_props, peer_advertised) { + // Peer is supposedly connected. + ctx.send_message(NetworkBridgeTxMessage::SendRequests( + vec![Requests::AttestedCandidateVStaging(request)], + IfDisconnected::ImmediateError, + )) + .await; + } +} + +/// Wait on the next incoming response. If there are no requests pending, this +/// future never resolves. It is the responsibility of the user of this API +/// to interrupt the future. +pub(crate) async fn receive_response(state: &mut State) -> UnhandledResponse { + match state.request_manager.await_incoming().await { + Some(r) => r, + None => futures::future::pending().await, + } +} + +/// Handles an incoming response. This does the actual work of validating the response, +/// importing statements, sending acknowledgements, etc. +#[overseer::contextbounds(StatementDistribution, prefix=self::overseer)] +pub(crate) async fn handle_response( + ctx: &mut Context, + state: &mut State, + response: UnhandledResponse, +) { + let &requests::CandidateIdentifier { relay_parent, candidate_hash, group_index } = + response.candidate_identifier(); + + let post_confirmation = { + let relay_parent_state = match state.per_relay_parent.get_mut(&relay_parent) { + None => return, + Some(s) => s, + }; + + let per_session = match state.per_session.get(&relay_parent_state.session) { + None => return, + Some(s) => s, + }; + + let group = match per_session.groups.get(group_index) { + None => return, + Some(g) => g, + }; + + let res = response.validate_response( + &mut state.request_manager, + group, + relay_parent_state.session, + |v| per_session.session_info.validators.get(v).map(|x| x.clone()), + |para, g_index| { + let expected_group = group_for_para( + &relay_parent_state.availability_cores, + &relay_parent_state.group_rotation_info, + para, + ); + + Some(g_index) == expected_group + }, + ); + + for (peer, rep) in res.reputation_changes { + report_peer(ctx.sender(), peer, rep).await; + } + + let (candidate, pvd, statements) = match res.request_status { + requests::CandidateRequestStatus::Outdated => return, + requests::CandidateRequestStatus::Incomplete => return, + requests::CandidateRequestStatus::Complete { + candidate, + persisted_validation_data, + statements, + } => (candidate, persisted_validation_data, statements), + }; + + for statement in statements { + let _ = relay_parent_state.statement_store.insert( + &per_session.groups, + statement, + StatementOrigin::Remote, + ); + } + + if let Some(post_confirmation) = + state.candidates.confirm_candidate(candidate_hash, candidate, pvd, group_index) + { + post_confirmation + } else { + gum::warn!( + target: LOG_TARGET, + ?candidate_hash, + "Candidate re-confirmed by request/response: logic error", + ); + + return + } + }; + + // Note that this implicitly circulates all statements via the cluster. + apply_post_confirmation(ctx, state, post_confirmation).await; + + let confirmed = state.candidates.get_confirmed(&candidate_hash).expect("just confirmed; qed"); + + // Although the candidate is confirmed, it isn't yet on the + // hypothetical frontier of the fragment tree. Later, when it is, + // we will import statements. + if !confirmed.is_importable(None) { + return + } + + let relay_parent_state = match state.per_relay_parent.get_mut(&relay_parent) { + None => return, + Some(s) => s, + }; + + let per_session = match state.per_session.get(&relay_parent_state.session) { + None => return, + Some(s) => s, + }; + + send_backing_fresh_statements( + ctx, + candidate_hash, + group_index, + &relay_parent, + relay_parent_state, + confirmed, + per_session, + ) + .await; + + // we don't need to send acknowledgement yet because + // 1. the candidate is not known yet, so cannot be backed. + // any previous confirmation is a bug, because `apply_post_confirmation` is meant to + // clear requests. + // 2. providing the statements to backing will lead to 'Backed' message. + // 3. on 'Backed' we will send acknowledgements/follow up statements when this becomes + // includable. +} + +/// Answer an incoming request for a candidate. +pub(crate) fn answer_request(state: &mut State, message: ResponderMessage) { + let ResponderMessage { request, sent_feedback } = message; + let AttestedCandidateRequest { candidate_hash, ref mask } = &request.payload; + + // Signal to the responder that we started processing this request. + let _ = sent_feedback.send(()); + + let confirmed = match state.candidates.get_confirmed(&candidate_hash) { + None => return, // drop request, candidate not known. + Some(c) => c, + }; + + let relay_parent_state = match state.per_relay_parent.get(&confirmed.relay_parent()) { + None => return, + Some(s) => s, + }; + + let local_validator = match relay_parent_state.local_validator.as_ref() { + None => return, + Some(s) => s, + }; + + let per_session = match state.per_session.get(&relay_parent_state.session) { + None => return, + Some(s) => s, + }; + + let peer_data = match state.peers.get(&request.peer) { + None => return, + Some(d) => d, + }; + + let group_size = per_session + .groups + .get(confirmed.group_index()) + .expect("group from session's candidate always known; qed") + .len(); + + // check request bitfields are right size. + if mask.seconded_in_group.len() != group_size || mask.validated_in_group.len() != group_size { + let _ = request.send_outgoing_response(OutgoingResponse { + result: Err(()), + reputation_changes: vec![COST_INVALID_REQUEST_BITFIELD_SIZE], + sent_feedback: None, + }); + + return + } + + // check peer is allowed to request the candidate (i.e. we've sent them a manifest) + { + let mut can_request = false; + for validator_id in find_validator_ids(peer_data.iter_known_discovery_ids(), |a| { + per_session.authority_lookup.get(a) + }) { + if local_validator.grid_tracker.can_request(validator_id, *candidate_hash) { + can_request = true; + break + } + } + + if !can_request { + let _ = request.send_outgoing_response(OutgoingResponse { + result: Err(()), + reputation_changes: vec![COST_UNEXPECTED_REQUEST], + sent_feedback: None, + }); + + return + } + } + + // Transform mask with 'OR' semantics into one with 'AND' semantics for the API used + // below. + let and_mask = StatementFilter { + seconded_in_group: !mask.seconded_in_group.clone(), + validated_in_group: !mask.validated_in_group.clone(), + }; + + let response = AttestedCandidateResponse { + candidate_receipt: (&**confirmed.candidate_receipt()).clone(), + persisted_validation_data: confirmed.persisted_validation_data().clone(), + statements: relay_parent_state + .statement_store + .group_statements( + &per_session.groups, + confirmed.group_index(), + *candidate_hash, + &and_mask, + ) + .map(|s| s.as_unchecked().clone()) + .collect(), + }; + + let _ = request.send_response(response); +} + +/// Messages coming from the background respond task. +pub struct ResponderMessage { + request: IncomingRequest, + sent_feedback: oneshot::Sender<()>, +} + +/// A fetching task, taking care of fetching candidates via request/response. +/// +/// Runs in a background task and feeds request to [`answer_request`] through [`MuxedMessage`]. +pub async fn respond_task( + mut receiver: IncomingRequestReceiver, + mut sender: mpsc::Sender, +) { + let mut pending_out = FuturesUnordered::new(); + loop { + // Ensure we are not handling too many requests in parallel. + if pending_out.len() >= MAX_PARALLEL_ATTESTED_CANDIDATE_REQUESTS as usize { + // Wait for one to finish: + pending_out.next().await; + } + + let req = match receiver.recv(|| vec![COST_INVALID_REQUEST]).await.into_nested() { + Ok(Ok(v)) => v, + Err(fatal) => { + gum::debug!(target: LOG_TARGET, error = ?fatal, "Shutting down request responder"); + return + }, + Ok(Err(jfyi)) => { + gum::debug!(target: LOG_TARGET, error = ?jfyi, "Decoding request failed"); + continue + }, + }; + + let (pending_sent_tx, pending_sent_rx) = oneshot::channel(); + if let Err(err) = sender + .feed(ResponderMessage { request: req, sent_feedback: pending_sent_tx }) + .await + { + gum::debug!(target: LOG_TARGET, ?err, "Shutting down responder"); + return + } + pending_out.push(pending_sent_rx); + } +} diff --git a/node/network/statement-distribution/src/vstaging/requests.rs b/node/network/statement-distribution/src/vstaging/requests.rs new file mode 100644 index 000000000000..507bbbb0ef18 --- /dev/null +++ b/node/network/statement-distribution/src/vstaging/requests.rs @@ -0,0 +1,1165 @@ +// Copyright 2022 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +//! A requester for full information on candidates. +//! +//! 1. We use `RequestManager::get_or_insert().get_mut()` to add and mutate [`RequestedCandidate`]s, either setting the +//! priority or adding a peer we know has the candidate. We currently prioritize "cluster" candidates (those from our +//! own group, although the cluster mechanism could be made to include multiple groups in the future) over "grid" +//! candidates (those from other groups). +//! +//! 2. The main loop of the module will invoke [`RequestManager::next_request`] in a loop until it returns `None`, +//! dispatching all requests with the `NetworkBridgeTxMessage`. The receiving half of the channel is owned by the +//! [`RequestManager`]. +//! +//! 3. The main loop of the module will also select over [`RequestManager::await_incoming`] to receive +//! [`UnhandledResponse`]s, which it then validates using [`UnhandledResponse::validate_response`] (which requires state +//! not owned by the request manager). + +use super::{ + BENEFIT_VALID_RESPONSE, BENEFIT_VALID_STATEMENT, COST_IMPROPERLY_DECODED_RESPONSE, + COST_INVALID_RESPONSE, COST_INVALID_SIGNATURE, COST_UNREQUESTED_RESPONSE_STATEMENT, +}; +use crate::LOG_TARGET; + +use polkadot_node_network_protocol::{ + request_response::{ + outgoing::{Recipient as RequestRecipient, RequestError}, + vstaging::{AttestedCandidateRequest, AttestedCandidateResponse}, + OutgoingRequest, OutgoingResult, MAX_PARALLEL_ATTESTED_CANDIDATE_REQUESTS, + }, + vstaging::StatementFilter, + PeerId, UnifiedReputationChange as Rep, +}; +use polkadot_primitives::vstaging::{ + CandidateHash, CommittedCandidateReceipt, CompactStatement, GroupIndex, Hash, ParaId, + PersistedValidationData, SessionIndex, SignedStatement, SigningContext, ValidatorId, + ValidatorIndex, +}; + +use futures::{future::BoxFuture, prelude::*, stream::FuturesUnordered}; + +use std::collections::{ + hash_map::{Entry as HEntry, HashMap}, + HashSet, VecDeque, +}; + +/// An identifier for a candidate. +/// +/// In this module, we are requesting candidates +/// for which we have no information other than the candidate hash and statements signed +/// by validators. It is possible for validators for multiple groups to abuse this lack of +/// information: until we actually get the preimage of this candidate we cannot confirm +/// anything other than the candidate hash. +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct CandidateIdentifier { + /// The relay-parent this candidate is ostensibly under. + pub relay_parent: Hash, + /// The hash of the candidate. + pub candidate_hash: CandidateHash, + /// The index of the group claiming to be assigned to the candidate's + /// para. + pub group_index: GroupIndex, +} + +struct TaggedResponse { + identifier: CandidateIdentifier, + requested_peer: PeerId, + props: RequestProperties, + response: OutgoingResult, +} + +/// A pending request. +#[derive(Debug)] +pub struct RequestedCandidate { + priority: Priority, + known_by: VecDeque, + in_flight: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +enum Origin { + Cluster = 0, + Unspecified = 1, +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +struct Priority { + origin: Origin, + attempts: usize, +} + +/// An entry for manipulating a requested candidate. +pub struct Entry<'a> { + prev_index: usize, + identifier: CandidateIdentifier, + by_priority: &'a mut Vec<(Priority, CandidateIdentifier)>, + requested: &'a mut RequestedCandidate, +} + +impl<'a> Entry<'a> { + /// Add a peer to the set of known peers. + pub fn add_peer(&mut self, peer: PeerId) { + if !self.requested.known_by.contains(&peer) { + self.requested.known_by.push_back(peer); + } + } + + /// Note that the candidate is required for the cluster. + pub fn set_cluster_priority(&mut self) { + self.requested.priority.origin = Origin::Cluster; + + insert_or_update_priority( + &mut *self.by_priority, + Some(self.prev_index), + self.identifier.clone(), + self.requested.priority.clone(), + ); + } +} + +/// A manager for outgoing requests. +pub struct RequestManager { + pending_responses: FuturesUnordered>, + requests: HashMap, + // sorted by priority. + by_priority: Vec<(Priority, CandidateIdentifier)>, + // all unique identifiers for the candidate. + unique_identifiers: HashMap>, +} + +impl RequestManager { + /// Create a new [`RequestManager`]. + pub fn new() -> Self { + RequestManager { + pending_responses: FuturesUnordered::new(), + requests: HashMap::new(), + by_priority: Vec::new(), + unique_identifiers: HashMap::new(), + } + } + + /// Gets an [`Entry`] for mutating a request and inserts it if the + /// manager doesn't store this request already. + pub fn get_or_insert( + &mut self, + relay_parent: Hash, + candidate_hash: CandidateHash, + group_index: GroupIndex, + ) -> Entry { + let identifier = CandidateIdentifier { relay_parent, candidate_hash, group_index }; + + let (candidate, fresh) = match self.requests.entry(identifier.clone()) { + HEntry::Occupied(e) => (e.into_mut(), false), + HEntry::Vacant(e) => ( + e.insert(RequestedCandidate { + priority: Priority { attempts: 0, origin: Origin::Unspecified }, + known_by: VecDeque::new(), + in_flight: false, + }), + true, + ), + }; + + let priority_index = if fresh { + self.unique_identifiers + .entry(candidate_hash) + .or_default() + .insert(identifier.clone()); + + insert_or_update_priority( + &mut self.by_priority, + None, + identifier.clone(), + candidate.priority.clone(), + ) + } else { + match self + .by_priority + .binary_search(&(candidate.priority.clone(), identifier.clone())) + { + Ok(i) => i, + Err(_) => unreachable!("requested candidates always have a priority entry; qed"), + } + }; + + Entry { + prev_index: priority_index, + identifier, + by_priority: &mut self.by_priority, + requested: candidate, + } + } + + /// Remove all pending requests for the given candidate. + pub fn remove_for(&mut self, candidate: CandidateHash) { + if let Some(identifiers) = self.unique_identifiers.remove(&candidate) { + self.by_priority.retain(|(_priority, id)| !identifiers.contains(&id)); + for id in identifiers { + self.requests.remove(&id); + } + } + } + + /// Remove based on relay-parent. + pub fn remove_by_relay_parent(&mut self, relay_parent: Hash) { + let mut candidate_hashes = HashSet::new(); + + // Remove from `by_priority` and `requests`. + self.by_priority.retain(|(_priority, id)| { + let retain = relay_parent != id.relay_parent; + if !retain { + self.requests.remove(id); + candidate_hashes.insert(id.candidate_hash); + } + retain + }); + + // Remove from `unique_identifiers`. + for candidate_hash in candidate_hashes { + match self.unique_identifiers.entry(candidate_hash) { + HEntry::Occupied(mut entry) => { + entry.get_mut().retain(|id| relay_parent != id.relay_parent); + if entry.get().is_empty() { + entry.remove(); + } + }, + // We can expect to encounter vacant entries, but only if nodes are misbehaving and + // we don't use a deduplicating collection; there are no issues from ignoring it. + HEntry::Vacant(_) => (), + } + } + } + + /// Yields the next request to dispatch, if there is any. + /// + /// This function accepts two closures as an argument. + /// + /// The first closure is used to gather information about the desired + /// properties of a response, which is used to select targets and validate + /// the response later on. + /// + /// The second closure is used to determine the specific advertised + /// statements by a peer, to be compared against the mask and backing + /// threshold and returns `None` if the peer is no longer connected. + pub fn next_request( + &mut self, + request_props: impl Fn(&CandidateIdentifier) -> Option, + peer_advertised: impl Fn(&CandidateIdentifier, &PeerId) -> Option, + ) -> Option> { + if self.pending_responses.len() >= MAX_PARALLEL_ATTESTED_CANDIDATE_REQUESTS as usize { + return None + } + + let mut res = None; + + // loop over all requests, in order of priority. + // do some active maintenance of the connected peers. + // dispatch the first request which is not in-flight already. + + let mut cleanup_outdated = Vec::new(); + for (i, (_priority, id)) in self.by_priority.iter().enumerate() { + let entry = match self.requests.get_mut(&id) { + None => { + gum::error!( + target: LOG_TARGET, + identifier = ?id, + "Missing entry for priority queue member", + ); + + continue + }, + Some(e) => e, + }; + + if entry.in_flight { + continue + } + + let props = match request_props(&id) { + None => { + cleanup_outdated.push((i, id.clone())); + continue + }, + Some(s) => s, + }; + + let target = match find_request_target_with_update( + &mut entry.known_by, + id, + &props, + &peer_advertised, + ) { + None => continue, + Some(t) => t, + }; + + let (request, response_fut) = OutgoingRequest::new( + RequestRecipient::Peer(target), + AttestedCandidateRequest { + candidate_hash: id.candidate_hash, + mask: props.unwanted_mask.clone(), + }, + ); + + let stored_id = id.clone(); + self.pending_responses.push(Box::pin(async move { + TaggedResponse { + identifier: stored_id, + requested_peer: target, + props, + response: response_fut.await, + } + })); + + entry.in_flight = true; + + res = Some(request); + break + } + + for (priority_index, identifier) in cleanup_outdated.into_iter().rev() { + self.by_priority.remove(priority_index); + self.requests.remove(&identifier); + if let HEntry::Occupied(mut e) = + self.unique_identifiers.entry(identifier.candidate_hash) + { + e.get_mut().remove(&identifier); + if e.get().is_empty() { + e.remove(); + } + } + } + + res + } + + /// Await the next incoming response to a sent request, or immediately + /// return `None` if there are no pending responses. + pub async fn await_incoming(&mut self) -> Option { + self.pending_responses + .next() + .await + .map(|response| UnhandledResponse { response }) + } +} + +/// Properties used in target selection and validation of a request. +#[derive(Clone)] +pub struct RequestProperties { + /// A mask for limiting the statements the response is allowed to contain. + /// The mask has `OR` semantics: statements by validators corresponding to bits + /// in the mask are not desired. It also returns the required backing threshold + /// for the candidate. + pub unwanted_mask: StatementFilter, + /// The required backing threshold, if any. If this is `Some`, then requests will only + /// be made to peers which can provide enough statements to back the candidate, when + /// taking into account the `unwanted_mask`, and a response will only be validated + /// in the case of those statements. + /// + /// If this is `None`, it is assumed that only the candidate itself is needed. + pub backing_threshold: Option, +} + +/// Finds a valid request target, returning `None` if none exists. +/// Cleans up disconnected peers and places the returned peer at the back of the queue. +fn find_request_target_with_update( + known_by: &mut VecDeque, + candidate_identifier: &CandidateIdentifier, + props: &RequestProperties, + peer_advertised: impl Fn(&CandidateIdentifier, &PeerId) -> Option, +) -> Option { + let mut prune = Vec::new(); + let mut target = None; + for (i, p) in known_by.iter().enumerate() { + let mut filter = match peer_advertised(candidate_identifier, p) { + None => { + prune.push(i); + continue + }, + Some(f) => f, + }; + + filter.mask_seconded(&props.unwanted_mask.seconded_in_group); + filter.mask_valid(&props.unwanted_mask.validated_in_group); + if seconded_and_sufficient(&filter, props.backing_threshold) { + target = Some((i, *p)); + break + } + } + + let prune_count = prune.len(); + for i in prune { + known_by.remove(i); + } + + if let Some((i, p)) = target { + known_by.remove(i - prune_count); + known_by.push_back(p); + Some(p) + } else { + None + } +} + +fn seconded_and_sufficient(filter: &StatementFilter, backing_threshold: Option) -> bool { + backing_threshold.map_or(true, |t| filter.has_seconded() && filter.backing_validators() >= t) +} + +/// A response to a request, which has not yet been handled. +pub struct UnhandledResponse { + response: TaggedResponse, +} + +impl UnhandledResponse { + /// Get the candidate identifier which the corresponding request + /// was classified under. + pub fn candidate_identifier(&self) -> &CandidateIdentifier { + &self.response.identifier + } + + /// Validate the response. If the response is valid, this will yield the + /// candidate, the [`PersistedValidationData`] of the candidate, and requested + /// checked statements. + /// + /// Valid responses are defined as those which provide a valid candidate + /// and signatures which match the identifier, and provide enough statements to back the candidate. + /// + /// This will also produce a record of misbehaviors by peers: + /// * If the response is partially valid, misbehavior by the responding peer. + /// * If there are other peers which have advertised the same candidate for different + /// relay-parents or para-ids, misbehavior reports for those peers will also + /// be generated. + /// + /// Finally, in the case that the response is either valid or partially valid, + /// this will clean up all remaining requests for the candidate in the manager. + /// + /// As parameters, the user should supply the canonical group array as well + /// as a mapping from validator index to validator ID. The validator pubkey mapping + /// will not be queried except for validator indices in the group. + pub fn validate_response( + self, + manager: &mut RequestManager, + group: &[ValidatorIndex], + session: SessionIndex, + validator_key_lookup: impl Fn(ValidatorIndex) -> Option, + allowed_para_lookup: impl Fn(ParaId, GroupIndex) -> bool, + ) -> ResponseValidationOutput { + let UnhandledResponse { + response: TaggedResponse { identifier, requested_peer, props, response }, + } = self; + + // handle races if the candidate is no longer known. + // this could happen if we requested the candidate under two + // different identifiers at the same time, and received a valid + // response on the other. + // + // it could also happen in the case that we had a request in-flight + // and the request entry was garbage-collected on outdated relay parent. + let entry = match manager.requests.get_mut(&identifier) { + None => + return ResponseValidationOutput { + requested_peer, + reputation_changes: Vec::new(), + request_status: CandidateRequestStatus::Outdated, + }, + Some(e) => e, + }; + + let priority_index = match manager + .by_priority + .binary_search(&(entry.priority.clone(), identifier.clone())) + { + Ok(i) => i, + Err(_) => unreachable!("requested candidates always have a priority entry; qed"), + }; + + entry.in_flight = false; + entry.priority.attempts += 1; + + // update the location in the priority queue. + insert_or_update_priority( + &mut manager.by_priority, + Some(priority_index), + identifier.clone(), + entry.priority.clone(), + ); + + let complete_response = match response { + Err(RequestError::InvalidResponse(e)) => { + gum::trace!( + target: LOG_TARGET, + err = ?e, + peer = ?requested_peer, + "Improperly encoded response" + ); + + return ResponseValidationOutput { + requested_peer, + reputation_changes: vec![(requested_peer, COST_IMPROPERLY_DECODED_RESPONSE)], + request_status: CandidateRequestStatus::Incomplete, + } + }, + Err(RequestError::NetworkError(_) | RequestError::Canceled(_)) => + return ResponseValidationOutput { + requested_peer, + reputation_changes: vec![], + request_status: CandidateRequestStatus::Incomplete, + }, + Ok(response) => response, + }; + + let output = validate_complete_response( + &identifier, + props, + complete_response, + requested_peer, + group, + session, + validator_key_lookup, + allowed_para_lookup, + ); + + if let CandidateRequestStatus::Complete { .. } = output.request_status { + manager.remove_for(identifier.candidate_hash); + } + + output + } +} + +fn validate_complete_response( + identifier: &CandidateIdentifier, + props: RequestProperties, + response: AttestedCandidateResponse, + requested_peer: PeerId, + group: &[ValidatorIndex], + session: SessionIndex, + validator_key_lookup: impl Fn(ValidatorIndex) -> Option, + allowed_para_lookup: impl Fn(ParaId, GroupIndex) -> bool, +) -> ResponseValidationOutput { + let RequestProperties { backing_threshold, mut unwanted_mask } = props; + + // sanity check bitmask size. this is based entirely on + // local logic here. + if !unwanted_mask.has_len(group.len()) { + gum::error!( + target: LOG_TARGET, + group_len = group.len(), + "Logic bug: group size != sent bitmask len" + ); + + // resize and attempt to continue. + unwanted_mask.seconded_in_group.resize(group.len(), true); + unwanted_mask.validated_in_group.resize(group.len(), true); + } + + let invalid_candidate_output = || ResponseValidationOutput { + request_status: CandidateRequestStatus::Incomplete, + reputation_changes: vec![(requested_peer, COST_INVALID_RESPONSE)], + requested_peer, + }; + + // sanity-check candidate response. + // note: roughly ascending cost of operations + { + if response.candidate_receipt.descriptor.relay_parent != identifier.relay_parent { + return invalid_candidate_output() + } + + if response.candidate_receipt.descriptor.persisted_validation_data_hash != + response.persisted_validation_data.hash() + { + return invalid_candidate_output() + } + + if !allowed_para_lookup( + response.candidate_receipt.descriptor.para_id, + identifier.group_index, + ) { + return invalid_candidate_output() + } + + if response.candidate_receipt.hash() != identifier.candidate_hash { + return invalid_candidate_output() + } + } + + // statement checks. + let mut rep_changes = Vec::new(); + let statements = { + let mut statements = + Vec::with_capacity(std::cmp::min(response.statements.len(), group.len() * 2)); + + let mut received_filter = StatementFilter::blank(group.len()); + + let index_in_group = |v: ValidatorIndex| group.iter().position(|x| &v == x); + + let signing_context = + SigningContext { parent_hash: identifier.relay_parent, session_index: session }; + + for unchecked_statement in response.statements.into_iter().take(group.len() * 2) { + // ensure statement is from a validator in the group. + let i = match index_in_group(unchecked_statement.unchecked_validator_index()) { + Some(i) => i, + None => { + rep_changes.push((requested_peer, COST_UNREQUESTED_RESPONSE_STATEMENT)); + continue + }, + }; + + // ensure statement is on the correct candidate hash. + if unchecked_statement.unchecked_payload().candidate_hash() != + &identifier.candidate_hash + { + rep_changes.push((requested_peer, COST_UNREQUESTED_RESPONSE_STATEMENT)); + continue + } + + // filter out duplicates or statements outside the mask. + // note on indexing: we have ensured that the bitmask and the + // duplicate trackers have the correct size for the group. + match unchecked_statement.unchecked_payload() { + CompactStatement::Seconded(_) => { + if unwanted_mask.seconded_in_group[i] { + rep_changes.push((requested_peer, COST_UNREQUESTED_RESPONSE_STATEMENT)); + continue + } + + if received_filter.seconded_in_group[i] { + rep_changes.push((requested_peer, COST_UNREQUESTED_RESPONSE_STATEMENT)); + continue + } + }, + CompactStatement::Valid(_) => { + if unwanted_mask.validated_in_group[i] { + rep_changes.push((requested_peer, COST_UNREQUESTED_RESPONSE_STATEMENT)); + continue + } + + if received_filter.validated_in_group[i] { + rep_changes.push((requested_peer, COST_UNREQUESTED_RESPONSE_STATEMENT)); + continue + } + }, + } + + let validator_public = + match validator_key_lookup(unchecked_statement.unchecked_validator_index()) { + None => { + rep_changes.push((requested_peer, COST_INVALID_SIGNATURE)); + continue + }, + Some(p) => p, + }; + + let checked_statement = + match unchecked_statement.try_into_checked(&signing_context, &validator_public) { + Err(_) => { + rep_changes.push((requested_peer, COST_INVALID_SIGNATURE)); + continue + }, + Ok(checked) => checked, + }; + + match checked_statement.payload() { + CompactStatement::Seconded(_) => { + received_filter.seconded_in_group.set(i, true); + }, + CompactStatement::Valid(_) => { + received_filter.validated_in_group.set(i, true); + }, + } + + statements.push(checked_statement); + rep_changes.push((requested_peer, BENEFIT_VALID_STATEMENT)); + } + + // Only accept responses which are sufficient, according to our + // required backing threshold. + if !seconded_and_sufficient(&received_filter, backing_threshold) { + return invalid_candidate_output() + } + + statements + }; + + rep_changes.push((requested_peer, BENEFIT_VALID_RESPONSE)); + + ResponseValidationOutput { + requested_peer, + request_status: CandidateRequestStatus::Complete { + candidate: response.candidate_receipt, + persisted_validation_data: response.persisted_validation_data, + statements, + }, + reputation_changes: rep_changes, + } +} + +/// The status of the candidate request after the handling of a response. +#[derive(Debug, PartialEq)] +pub enum CandidateRequestStatus { + /// The request was outdated at the point of receiving the response. + Outdated, + /// The response either did not arrive or was invalid. + Incomplete, + /// The response completed the request. Statements sent beyond the + /// mask have been ignored. + Complete { + candidate: CommittedCandidateReceipt, + persisted_validation_data: PersistedValidationData, + statements: Vec, + }, +} + +/// Output of the response validation. +#[derive(Debug, PartialEq)] +pub struct ResponseValidationOutput { + /// The peer we requested from. + pub requested_peer: PeerId, + /// The status of the request. + pub request_status: CandidateRequestStatus, + /// Any reputation changes as a result of validating the response. + pub reputation_changes: Vec<(PeerId, Rep)>, +} + +fn insert_or_update_priority( + priority_sorted: &mut Vec<(Priority, CandidateIdentifier)>, + prev_index: Option, + candidate_identifier: CandidateIdentifier, + new_priority: Priority, +) -> usize { + if let Some(prev_index) = prev_index { + // GIGO: this behaves strangely if prev-index is not for the + // expected identifier. + if priority_sorted[prev_index].0 == new_priority { + // unchanged. + return prev_index + } else { + priority_sorted.remove(prev_index); + } + } + + let item = (new_priority, candidate_identifier); + match priority_sorted.binary_search(&item) { + Ok(i) => i, // ignore if already present. + Err(i) => { + priority_sorted.insert(i, item); + i + }, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use polkadot_primitives::HeadData; + use polkadot_primitives_test_helpers as test_helpers; + + fn dummy_pvd() -> PersistedValidationData { + PersistedValidationData { + parent_head: HeadData(vec![7, 8, 9]), + relay_parent_number: 5, + max_pov_size: 1024, + relay_parent_storage_root: Default::default(), + } + } + + #[test] + fn test_remove_by_relay_parent() { + let parent_a = Hash::from_low_u64_le(1); + let parent_b = Hash::from_low_u64_le(2); + let parent_c = Hash::from_low_u64_le(3); + + let candidate_a1 = CandidateHash(Hash::from_low_u64_le(11)); + let candidate_a2 = CandidateHash(Hash::from_low_u64_le(12)); + let candidate_b1 = CandidateHash(Hash::from_low_u64_le(21)); + let candidate_b2 = CandidateHash(Hash::from_low_u64_le(22)); + let candidate_c1 = CandidateHash(Hash::from_low_u64_le(31)); + let duplicate_hash = CandidateHash(Hash::from_low_u64_le(31)); + + let mut request_manager = RequestManager::new(); + request_manager.get_or_insert(parent_a, candidate_a1, 1.into()); + request_manager.get_or_insert(parent_a, candidate_a2, 1.into()); + request_manager.get_or_insert(parent_b, candidate_b1, 1.into()); + request_manager.get_or_insert(parent_b, candidate_b2, 2.into()); + request_manager.get_or_insert(parent_c, candidate_c1, 2.into()); + request_manager.get_or_insert(parent_a, duplicate_hash, 1.into()); + + assert_eq!(request_manager.requests.len(), 6); + assert_eq!(request_manager.by_priority.len(), 6); + assert_eq!(request_manager.unique_identifiers.len(), 5); + + request_manager.remove_by_relay_parent(parent_a); + + assert_eq!(request_manager.requests.len(), 3); + assert_eq!(request_manager.by_priority.len(), 3); + assert_eq!(request_manager.unique_identifiers.len(), 3); + + assert!(!request_manager.unique_identifiers.contains_key(&candidate_a1)); + assert!(!request_manager.unique_identifiers.contains_key(&candidate_a2)); + // Duplicate hash should still be there (under a different parent). + assert!(request_manager.unique_identifiers.contains_key(&duplicate_hash)); + + request_manager.remove_by_relay_parent(parent_b); + + assert_eq!(request_manager.requests.len(), 1); + assert_eq!(request_manager.by_priority.len(), 1); + assert_eq!(request_manager.unique_identifiers.len(), 1); + + assert!(!request_manager.unique_identifiers.contains_key(&candidate_b1)); + assert!(!request_manager.unique_identifiers.contains_key(&candidate_b2)); + + request_manager.remove_by_relay_parent(parent_c); + + assert!(request_manager.requests.is_empty()); + assert!(request_manager.by_priority.is_empty()); + assert!(request_manager.unique_identifiers.is_empty()); + } + + #[test] + fn test_priority_ordering() { + let parent_a = Hash::from_low_u64_le(1); + let parent_b = Hash::from_low_u64_le(2); + let parent_c = Hash::from_low_u64_le(3); + + let candidate_a1 = CandidateHash(Hash::from_low_u64_le(11)); + let candidate_a2 = CandidateHash(Hash::from_low_u64_le(12)); + let candidate_b1 = CandidateHash(Hash::from_low_u64_le(21)); + let candidate_b2 = CandidateHash(Hash::from_low_u64_le(22)); + let candidate_c1 = CandidateHash(Hash::from_low_u64_le(31)); + + let mut request_manager = RequestManager::new(); + + // Add some entries, set a couple of them to cluster (high) priority. + let identifier_a1 = request_manager + .get_or_insert(parent_a, candidate_a1, 1.into()) + .identifier + .clone(); + let identifier_a2 = { + let mut entry = request_manager.get_or_insert(parent_a, candidate_a2, 1.into()); + entry.set_cluster_priority(); + entry.identifier.clone() + }; + let identifier_b1 = request_manager + .get_or_insert(parent_b, candidate_b1, 1.into()) + .identifier + .clone(); + let identifier_b2 = request_manager + .get_or_insert(parent_b, candidate_b2, 2.into()) + .identifier + .clone(); + let identifier_c1 = { + let mut entry = request_manager.get_or_insert(parent_c, candidate_c1, 2.into()); + entry.set_cluster_priority(); + entry.identifier.clone() + }; + + let attempts = 0; + assert_eq!( + request_manager.by_priority, + vec![ + (Priority { origin: Origin::Cluster, attempts }, identifier_a2), + (Priority { origin: Origin::Cluster, attempts }, identifier_c1), + (Priority { origin: Origin::Unspecified, attempts }, identifier_a1), + (Priority { origin: Origin::Unspecified, attempts }, identifier_b1), + (Priority { origin: Origin::Unspecified, attempts }, identifier_b2), + ] + ); + } + + // Test case where candidate is requested under two different identifiers at the same time. + // Should result in `Outdated` error. + #[test] + fn handle_outdated_response_due_to_requests_for_different_identifiers() { + let mut request_manager = RequestManager::new(); + + let relay_parent = Hash::from_low_u64_le(1); + let mut candidate_receipt = test_helpers::dummy_committed_candidate_receipt(relay_parent); + let persisted_validation_data = dummy_pvd(); + candidate_receipt.descriptor.persisted_validation_data_hash = + persisted_validation_data.hash(); + let candidate = candidate_receipt.hash(); + let requested_peer = PeerId::random(); + + let identifier1 = request_manager + .get_or_insert(relay_parent, candidate, 1.into()) + .identifier + .clone(); + request_manager + .get_or_insert(relay_parent, candidate, 1.into()) + .add_peer(requested_peer); + let identifier2 = request_manager + .get_or_insert(relay_parent, candidate, 2.into()) + .identifier + .clone(); + request_manager + .get_or_insert(relay_parent, candidate, 2.into()) + .add_peer(requested_peer); + + assert_ne!(identifier1, identifier2); + assert_eq!(request_manager.requests.len(), 2); + + let group_size = 3; + let group = &[ValidatorIndex(0), ValidatorIndex(1), ValidatorIndex(2)]; + + let unwanted_mask = StatementFilter::blank(group_size); + let request_properties = RequestProperties { unwanted_mask, backing_threshold: None }; + + // Get requests. + { + let request_props = + |_identifier: &CandidateIdentifier| Some((&request_properties).clone()); + let peer_advertised = |_identifier: &CandidateIdentifier, _peer: &_| { + Some(StatementFilter::full(group_size)) + }; + let outgoing = request_manager.next_request(request_props, peer_advertised).unwrap(); + assert_eq!(outgoing.payload.candidate_hash, candidate); + let outgoing = request_manager.next_request(request_props, peer_advertised).unwrap(); + assert_eq!(outgoing.payload.candidate_hash, candidate); + } + + // Validate first response. + { + let statements = vec![]; + let response = UnhandledResponse { + response: TaggedResponse { + identifier: identifier1, + requested_peer, + props: request_properties.clone(), + response: Ok(AttestedCandidateResponse { + candidate_receipt: candidate_receipt.clone(), + persisted_validation_data: persisted_validation_data.clone(), + statements, + }), + }, + }; + let validator_key_lookup = |_v| None; + let allowed_para_lookup = |_para, _g_index| true; + let statements = vec![]; + let output = response.validate_response( + &mut request_manager, + group, + 0, + validator_key_lookup, + allowed_para_lookup, + ); + assert_eq!( + output, + ResponseValidationOutput { + requested_peer, + request_status: CandidateRequestStatus::Complete { + candidate: candidate_receipt.clone(), + persisted_validation_data: persisted_validation_data.clone(), + statements, + }, + reputation_changes: vec![(requested_peer, BENEFIT_VALID_RESPONSE)], + } + ); + } + + // Try to validate second response. + { + let statements = vec![]; + let response = UnhandledResponse { + response: TaggedResponse { + identifier: identifier2, + requested_peer, + props: request_properties, + response: Ok(AttestedCandidateResponse { + candidate_receipt: candidate_receipt.clone(), + persisted_validation_data: persisted_validation_data.clone(), + statements, + }), + }, + }; + let validator_key_lookup = |_v| None; + let allowed_para_lookup = |_para, _g_index| true; + let output = response.validate_response( + &mut request_manager, + group, + 0, + validator_key_lookup, + allowed_para_lookup, + ); + assert_eq!( + output, + ResponseValidationOutput { + requested_peer, + request_status: CandidateRequestStatus::Outdated, + reputation_changes: vec![], + } + ); + } + } + + // Test case where we had a request in-flight and the request entry was garbage-collected on + // outdated relay parent. + #[test] + fn handle_outdated_response_due_to_garbage_collection() { + let mut request_manager = RequestManager::new(); + + let relay_parent = Hash::from_low_u64_le(1); + let mut candidate_receipt = test_helpers::dummy_committed_candidate_receipt(relay_parent); + let persisted_validation_data = dummy_pvd(); + candidate_receipt.descriptor.persisted_validation_data_hash = + persisted_validation_data.hash(); + let candidate = candidate_receipt.hash(); + let requested_peer = PeerId::random(); + + let identifier = request_manager + .get_or_insert(relay_parent, candidate, 1.into()) + .identifier + .clone(); + request_manager + .get_or_insert(relay_parent, candidate, 1.into()) + .add_peer(requested_peer); + + let group_size = 3; + let group = &[ValidatorIndex(0), ValidatorIndex(1), ValidatorIndex(2)]; + + let unwanted_mask = StatementFilter::blank(group_size); + let request_properties = RequestProperties { unwanted_mask, backing_threshold: None }; + let peer_advertised = + |_identifier: &CandidateIdentifier, _peer: &_| Some(StatementFilter::full(group_size)); + + // Get request once successfully. + { + let request_props = + |_identifier: &CandidateIdentifier| Some((&request_properties).clone()); + let outgoing = request_manager.next_request(request_props, peer_advertised).unwrap(); + assert_eq!(outgoing.payload.candidate_hash, candidate); + } + + // Garbage collect based on relay parent. + request_manager.remove_by_relay_parent(relay_parent); + + // Try to validate response. + { + let statements = vec![]; + let response = UnhandledResponse { + response: TaggedResponse { + identifier, + requested_peer, + props: request_properties, + response: Ok(AttestedCandidateResponse { + candidate_receipt: candidate_receipt.clone(), + persisted_validation_data: persisted_validation_data.clone(), + statements, + }), + }, + }; + let validator_key_lookup = |_v| None; + let allowed_para_lookup = |_para, _g_index| true; + let output = response.validate_response( + &mut request_manager, + group, + 0, + validator_key_lookup, + allowed_para_lookup, + ); + assert_eq!( + output, + ResponseValidationOutput { + requested_peer, + request_status: CandidateRequestStatus::Outdated, + reputation_changes: vec![], + } + ); + } + } + + #[test] + fn should_clean_up_after_successful_requests() { + let mut request_manager = RequestManager::new(); + + let relay_parent = Hash::from_low_u64_le(1); + let mut candidate_receipt = test_helpers::dummy_committed_candidate_receipt(relay_parent); + let persisted_validation_data = dummy_pvd(); + candidate_receipt.descriptor.persisted_validation_data_hash = + persisted_validation_data.hash(); + let candidate = candidate_receipt.hash(); + let requested_peer = PeerId::random(); + + let identifier = request_manager + .get_or_insert(relay_parent, candidate, 1.into()) + .identifier + .clone(); + request_manager + .get_or_insert(relay_parent, candidate, 1.into()) + .add_peer(requested_peer); + + assert_eq!(request_manager.requests.len(), 1); + assert_eq!(request_manager.by_priority.len(), 1); + + let group_size = 3; + let group = &[ValidatorIndex(0), ValidatorIndex(1), ValidatorIndex(2)]; + + let unwanted_mask = StatementFilter::blank(group_size); + let request_properties = RequestProperties { unwanted_mask, backing_threshold: None }; + let peer_advertised = + |_identifier: &CandidateIdentifier, _peer: &_| Some(StatementFilter::full(group_size)); + + // Get request once successfully. + { + let request_props = + |_identifier: &CandidateIdentifier| Some((&request_properties).clone()); + let outgoing = request_manager.next_request(request_props, peer_advertised).unwrap(); + assert_eq!(outgoing.payload.candidate_hash, candidate); + } + + // Validate response. + { + let statements = vec![]; + let response = UnhandledResponse { + response: TaggedResponse { + identifier, + requested_peer, + props: request_properties.clone(), + response: Ok(AttestedCandidateResponse { + candidate_receipt: candidate_receipt.clone(), + persisted_validation_data: persisted_validation_data.clone(), + statements, + }), + }, + }; + let validator_key_lookup = |_v| None; + let allowed_para_lookup = |_para, _g_index| true; + let statements = vec![]; + let output = response.validate_response( + &mut request_manager, + group, + 0, + validator_key_lookup, + allowed_para_lookup, + ); + assert_eq!( + output, + ResponseValidationOutput { + requested_peer, + request_status: CandidateRequestStatus::Complete { + candidate: candidate_receipt.clone(), + persisted_validation_data: persisted_validation_data.clone(), + statements, + }, + reputation_changes: vec![(requested_peer, BENEFIT_VALID_RESPONSE)], + } + ); + } + + // Ensure that cleanup occurred. + assert_eq!(request_manager.requests.len(), 0); + assert_eq!(request_manager.by_priority.len(), 0); + } +} diff --git a/node/network/statement-distribution/src/vstaging/statement_store.rs b/node/network/statement-distribution/src/vstaging/statement_store.rs new file mode 100644 index 000000000000..50ac99d0a813 --- /dev/null +++ b/node/network/statement-distribution/src/vstaging/statement_store.rs @@ -0,0 +1,283 @@ +// Copyright 2022 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! A store of all statements under a given relay-parent. +//! +//! This structure doesn't attempt to do any spam protection, which must +//! be provided at a higher level. +//! +//! This keeps track of statements submitted with a number of different of +//! views into this data: views based on the candidate, views based on the validator +//! groups, and views based on the validators themselves. + +use bitvec::{order::Lsb0 as BitOrderLsb0, vec::BitVec}; +use polkadot_node_network_protocol::vstaging::StatementFilter; +use polkadot_primitives::vstaging::{ + CandidateHash, CompactStatement, GroupIndex, SignedStatement, ValidatorIndex, +}; +use std::collections::hash_map::{Entry as HEntry, HashMap}; + +use super::groups::Groups; + +/// Possible origins of a statement. +pub enum StatementOrigin { + /// The statement originated locally. + Local, + /// The statement originated from a remote peer. + Remote, +} + +impl StatementOrigin { + fn is_local(&self) -> bool { + match *self { + StatementOrigin::Local => true, + StatementOrigin::Remote => false, + } + } +} + +struct StoredStatement { + statement: SignedStatement, + known_by_backing: bool, +} + +/// Storage for statements. Intended to be used for statements signed under +/// the same relay-parent. See module docs for more details. +pub struct StatementStore { + validator_meta: HashMap, + + // we keep statements per-group because even though only one group _should_ be + // producing statements about a candidate, until we have the candidate receipt + // itself, we can't tell which group that is. + group_statements: HashMap<(GroupIndex, CandidateHash), GroupStatements>, + known_statements: HashMap, +} + +impl StatementStore { + /// Create a new [`StatementStore`] + pub fn new(groups: &Groups) -> Self { + let mut validator_meta = HashMap::new(); + for (g, group) in groups.all().iter().enumerate() { + for (i, v) in group.iter().enumerate() { + validator_meta.insert( + *v, + ValidatorMeta { + seconded_count: 0, + within_group_index: i, + group: GroupIndex(g as _), + }, + ); + } + } + + StatementStore { + validator_meta, + group_statements: HashMap::new(), + known_statements: HashMap::new(), + } + } + + /// Insert a statement. Returns `true` if was not known already, `false` if it was. + /// Ignores statements by unknown validators and returns an error. + pub fn insert( + &mut self, + groups: &Groups, + statement: SignedStatement, + origin: StatementOrigin, + ) -> Result { + let validator_index = statement.validator_index(); + let validator_meta = match self.validator_meta.get_mut(&validator_index) { + None => return Err(ValidatorUnknown), + Some(m) => m, + }; + + let compact = statement.payload().clone(); + let fingerprint = (validator_index, compact.clone()); + match self.known_statements.entry(fingerprint) { + HEntry::Occupied(mut e) => { + if let StatementOrigin::Local = origin { + e.get_mut().known_by_backing = true; + } + + return Ok(false) + }, + HEntry::Vacant(e) => { + e.insert(StoredStatement { statement, known_by_backing: origin.is_local() }); + }, + } + + let candidate_hash = *compact.candidate_hash(); + let seconded = if let CompactStatement::Seconded(_) = compact { true } else { false }; + + // cross-reference updates. + { + let group_index = validator_meta.group; + let group = match groups.get(group_index) { + Some(g) => g, + None => { + gum::error!( + target: crate::LOG_TARGET, + ?group_index, + "groups passed into `insert` differ from those used at store creation" + ); + + return Err(ValidatorUnknown) + }, + }; + + let group_statements = self + .group_statements + .entry((group_index, candidate_hash)) + .or_insert_with(|| GroupStatements::with_group_size(group.len())); + + if seconded { + validator_meta.seconded_count += 1; + group_statements.note_seconded(validator_meta.within_group_index); + } else { + group_statements.note_validated(validator_meta.within_group_index); + } + } + + Ok(true) + } + + /// Fill a `StatementFilter` to be used in the grid topology with all statements + /// we are already aware of. + pub fn fill_statement_filter( + &self, + group_index: GroupIndex, + candidate_hash: CandidateHash, + statement_filter: &mut StatementFilter, + ) { + if let Some(statements) = self.group_statements.get(&(group_index, candidate_hash)) { + statement_filter.seconded_in_group |= statements.seconded.as_bitslice(); + statement_filter.validated_in_group |= statements.valid.as_bitslice(); + } + } + + /// Get an iterator over stored signed statements by the group conforming to the + /// given filter. + /// + /// Seconded statements are provided first. + pub fn group_statements<'a>( + &'a self, + groups: &'a Groups, + group_index: GroupIndex, + candidate_hash: CandidateHash, + filter: &'a StatementFilter, + ) -> impl Iterator + 'a { + let group_validators = groups.get(group_index); + + let seconded_statements = filter + .seconded_in_group + .iter_ones() + .filter_map(move |i| group_validators.as_ref().and_then(|g| g.get(i))) + .filter_map(move |v| { + self.known_statements.get(&(*v, CompactStatement::Seconded(candidate_hash))) + }) + .map(|s| &s.statement); + + let valid_statements = filter + .validated_in_group + .iter_ones() + .filter_map(move |i| group_validators.as_ref().and_then(|g| g.get(i))) + .filter_map(move |v| { + self.known_statements.get(&(*v, CompactStatement::Valid(candidate_hash))) + }) + .map(|s| &s.statement); + + seconded_statements.chain(valid_statements) + } + + /// Get the full statement of this kind issued by this validator, if it is known. + pub fn validator_statement( + &self, + validator_index: ValidatorIndex, + statement: CompactStatement, + ) -> Option<&SignedStatement> { + self.known_statements.get(&(validator_index, statement)).map(|s| &s.statement) + } + + /// Get an iterator over all statements marked as being unknown by the backing subsystem. + pub fn fresh_statements_for_backing<'a>( + &'a self, + validators: &'a [ValidatorIndex], + candidate_hash: CandidateHash, + ) -> impl Iterator + 'a { + let s_st = CompactStatement::Seconded(candidate_hash); + let v_st = CompactStatement::Valid(candidate_hash); + + validators + .iter() + .flat_map(move |v| { + let a = self.known_statements.get(&(*v, s_st.clone())); + let b = self.known_statements.get(&(*v, v_st.clone())); + + a.into_iter().chain(b) + }) + .filter(|stored| !stored.known_by_backing) + .map(|stored| &stored.statement) + } + + /// Get the amount of known `Seconded` statements by the given validator index. + pub fn seconded_count(&self, validator_index: &ValidatorIndex) -> usize { + self.validator_meta.get(validator_index).map_or(0, |m| m.seconded_count) + } + + /// Note that a statement is known by the backing subsystem. + pub fn note_known_by_backing( + &mut self, + validator_index: ValidatorIndex, + statement: CompactStatement, + ) { + if let Some(stored) = self.known_statements.get_mut(&(validator_index, statement)) { + stored.known_by_backing = true; + } + } +} + +/// Error indicating that the validator was unknown. +pub struct ValidatorUnknown; + +type Fingerprint = (ValidatorIndex, CompactStatement); + +struct ValidatorMeta { + group: GroupIndex, + within_group_index: usize, + seconded_count: usize, +} + +struct GroupStatements { + seconded: BitVec, + valid: BitVec, +} + +impl GroupStatements { + fn with_group_size(group_size: usize) -> Self { + GroupStatements { + seconded: BitVec::repeat(false, group_size), + valid: BitVec::repeat(false, group_size), + } + } + + fn note_seconded(&mut self, within_group_index: usize) { + self.seconded.set(within_group_index, true); + } + + fn note_validated(&mut self, within_group_index: usize) { + self.valid.set(within_group_index, true); + } +} diff --git a/node/network/statement-distribution/src/vstaging/tests/cluster.rs b/node/network/statement-distribution/src/vstaging/tests/cluster.rs new file mode 100644 index 000000000000..ca849dbd39a6 --- /dev/null +++ b/node/network/statement-distribution/src/vstaging/tests/cluster.rs @@ -0,0 +1,1216 @@ +// Copyright 2023 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +use super::*; + +use polkadot_primitives_test_helpers::make_candidate; + +#[test] +fn share_seconded_circulated_to_cluster() { + let config = TestConfig { validator_count: 20, group_size: 3, local_validator: true }; + + let relay_parent = Hash::repeat_byte(1); + let peer_a = PeerId::random(); + let peer_b = PeerId::random(); + let peer_c = PeerId::random(); + + test_harness(config, |state, mut overseer| async move { + let local_validator = state.local.clone().unwrap(); + let local_para = ParaId::from(local_validator.group_index.0); + + let (candidate, pvd) = make_candidate( + relay_parent, + 1, + local_para, + vec![1, 2, 3].into(), + vec![4, 5, 6].into(), + Hash::repeat_byte(42).into(), + ); + let candidate_hash = candidate.hash(); + + let test_leaf = TestLeaf { + number: 1, + hash: relay_parent, + parent_hash: Hash::repeat_byte(0), + session: 1, + availability_cores: state.make_availability_cores(|i| { + CoreState::Scheduled(ScheduledCore { + para_id: ParaId::from(i as u32), + collator: None, + }) + }), + para_data: (0..state.session_info.validator_groups.len()) + .map(|i| { + ( + ParaId::from(i as u32), + PerParaData { min_relay_parent: 1, head_data: vec![1, 2, 3].into() }, + ) + }) + .collect(), + }; + + // peer A is in group, has relay parent in view. + // peer B is in group, has no relay parent in view. + // peer C is not in group, has relay parent in view. + { + let other_group_validators = state.group_validators(local_validator.group_index, true); + + connect_peer( + &mut overseer, + peer_a.clone(), + Some(vec![state.discovery_id(other_group_validators[0])].into_iter().collect()), + ) + .await; + + connect_peer( + &mut overseer, + peer_b.clone(), + Some(vec![state.discovery_id(other_group_validators[1])].into_iter().collect()), + ) + .await; + + connect_peer(&mut overseer, peer_c.clone(), None).await; + + send_peer_view_change(&mut overseer, peer_a.clone(), view![relay_parent]).await; + send_peer_view_change(&mut overseer, peer_c.clone(), view![relay_parent]).await; + } + + activate_leaf(&mut overseer, local_para, &test_leaf, &state, true).await; + + answer_expected_hypothetical_depth_request( + &mut overseer, + vec![], + Some(relay_parent), + false, + ) + .await; + + let full_signed = state + .sign_statement( + local_validator.validator_index, + CompactStatement::Seconded(candidate_hash), + &SigningContext { session_index: 1, parent_hash: relay_parent }, + ) + .convert_to_superpayload(StatementWithPVD::Seconded(candidate.clone(), pvd.clone())) + .unwrap(); + + overseer + .send(FromOrchestra::Communication { + msg: StatementDistributionMessage::Share(relay_parent, full_signed), + }) + .await; + + assert_matches!( + overseer.recv().await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( + peers, + Versioned::VStaging(protocol_vstaging::ValidationProtocol::StatementDistribution( + protocol_vstaging::StatementDistributionMessage::Statement( + r, + s, + ) + )) + )) => { + assert_eq!(peers, vec![peer_a.clone()]); + assert_eq!(r, relay_parent); + assert_eq!(s.unchecked_payload(), &CompactStatement::Seconded(candidate_hash)); + assert_eq!(s.unchecked_validator_index(), local_validator.validator_index); + } + ); + + // sharing a `Seconded` message confirms a candidate, which leads to new + // fragment tree updates. + answer_expected_hypothetical_depth_request(&mut overseer, vec![], None, false).await; + + overseer + }); +} + +#[test] +fn cluster_valid_statement_before_seconded_ignored() { + let config = TestConfig { validator_count: 20, group_size: 3, local_validator: true }; + + let relay_parent = Hash::repeat_byte(1); + let peer_a = PeerId::random(); + + test_harness(config, |state, mut overseer| async move { + let local_validator = state.local.clone().unwrap(); + let local_para = ParaId::from(local_validator.group_index.0); + let candidate_hash = CandidateHash(Hash::repeat_byte(42)); + + let test_leaf = TestLeaf { + number: 1, + hash: relay_parent, + parent_hash: Hash::repeat_byte(0), + session: 1, + availability_cores: state.make_availability_cores(|i| { + CoreState::Scheduled(ScheduledCore { + para_id: ParaId::from(i as u32), + collator: None, + }) + }), + para_data: (0..state.session_info.validator_groups.len()) + .map(|i| { + ( + ParaId::from(i as u32), + PerParaData { min_relay_parent: 1, head_data: vec![1, 2, 3].into() }, + ) + }) + .collect(), + }; + + // peer A is in group, has relay parent in view. + let other_group_validators = state.group_validators(local_validator.group_index, true); + let v_a = other_group_validators[0]; + connect_peer( + &mut overseer, + peer_a.clone(), + Some(vec![state.discovery_id(v_a)].into_iter().collect()), + ) + .await; + + send_peer_view_change(&mut overseer, peer_a.clone(), view![relay_parent]).await; + activate_leaf(&mut overseer, local_para, &test_leaf, &state, true).await; + + answer_expected_hypothetical_depth_request( + &mut overseer, + vec![], + Some(relay_parent), + false, + ) + .await; + + let signed_valid = state.sign_statement( + v_a, + CompactStatement::Valid(candidate_hash), + &SigningContext { parent_hash: relay_parent, session_index: 1 }, + ); + + send_peer_message( + &mut overseer, + peer_a.clone(), + protocol_vstaging::StatementDistributionMessage::Statement( + relay_parent, + signed_valid.as_unchecked().clone(), + ), + ) + .await; + + assert_matches!( + overseer.recv().await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::ReportPeer(p, r)) => { + assert_eq!(p, peer_a); + assert_eq!(r, COST_UNEXPECTED_STATEMENT); + } + ); + + overseer + }); +} + +#[test] +fn cluster_statement_bad_signature() { + let config = TestConfig { validator_count: 20, group_size: 3, local_validator: true }; + + let relay_parent = Hash::repeat_byte(1); + let peer_a = PeerId::random(); + + test_harness(config, |state, mut overseer| async move { + let local_validator = state.local.clone().unwrap(); + let local_para = ParaId::from(local_validator.group_index.0); + let candidate_hash = CandidateHash(Hash::repeat_byte(42)); + + let test_leaf = TestLeaf { + number: 1, + hash: relay_parent, + parent_hash: Hash::repeat_byte(0), + session: 1, + availability_cores: state.make_availability_cores(|i| { + CoreState::Scheduled(ScheduledCore { + para_id: ParaId::from(i as u32), + collator: None, + }) + }), + para_data: (0..state.session_info.validator_groups.len()) + .map(|i| { + ( + ParaId::from(i as u32), + PerParaData { min_relay_parent: 1, head_data: vec![1, 2, 3].into() }, + ) + }) + .collect(), + }; + + // peer A is in group, has relay parent in view. + let other_group_validators = state.group_validators(local_validator.group_index, true); + let v_a = other_group_validators[0]; + let v_b = other_group_validators[1]; + + connect_peer( + &mut overseer, + peer_a.clone(), + Some(vec![state.discovery_id(v_a)].into_iter().collect()), + ) + .await; + + send_peer_view_change(&mut overseer, peer_a.clone(), view![relay_parent]).await; + activate_leaf(&mut overseer, local_para, &test_leaf, &state, true).await; + + answer_expected_hypothetical_depth_request( + &mut overseer, + vec![], + Some(relay_parent), + false, + ) + .await; + + // sign statements with wrong signing context, leading to bad signature. + let statements = vec![ + (v_a, CompactStatement::Seconded(candidate_hash)), + (v_b, CompactStatement::Seconded(candidate_hash)), + ] + .into_iter() + .map(|(v, s)| { + state.sign_statement( + v, + s, + &SigningContext { parent_hash: Hash::repeat_byte(69), session_index: 1 }, + ) + }) + .map(|s| s.as_unchecked().clone()); + + for statement in statements { + send_peer_message( + &mut overseer, + peer_a.clone(), + protocol_vstaging::StatementDistributionMessage::Statement( + relay_parent, + statement.clone(), + ), + ) + .await; + + assert_matches!( + overseer.recv().await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::ReportPeer(p, r)) + if p == peer_a && r == COST_INVALID_SIGNATURE => { }, + "{:?}", + statement + ); + } + + overseer + }); +} + +#[test] +fn useful_cluster_statement_from_non_cluster_peer_rejected() { + let config = TestConfig { validator_count: 20, group_size: 3, local_validator: true }; + + let relay_parent = Hash::repeat_byte(1); + let peer_a = PeerId::random(); + + test_harness(config, |state, mut overseer| async move { + let local_validator = state.local.clone().unwrap(); + let local_para = ParaId::from(local_validator.group_index.0); + let candidate_hash = CandidateHash(Hash::repeat_byte(42)); + + let test_leaf = TestLeaf { + number: 1, + hash: relay_parent, + parent_hash: Hash::repeat_byte(0), + session: 1, + availability_cores: state.make_availability_cores(|i| { + CoreState::Scheduled(ScheduledCore { + para_id: ParaId::from(i as u32), + collator: None, + }) + }), + para_data: (0..state.session_info.validator_groups.len()) + .map(|i| { + ( + ParaId::from(i as u32), + PerParaData { min_relay_parent: 1, head_data: vec![1, 2, 3].into() }, + ) + }) + .collect(), + }; + + // peer A is not in group, has relay parent in view. + let not_our_group = + if local_validator.group_index.0 == 0 { GroupIndex(1) } else { GroupIndex(0) }; + + let that_group_validators = state.group_validators(not_our_group, false); + let v_non = that_group_validators[0]; + + connect_peer( + &mut overseer, + peer_a.clone(), + Some(vec![state.discovery_id(v_non)].into_iter().collect()), + ) + .await; + + send_peer_view_change(&mut overseer, peer_a.clone(), view![relay_parent]).await; + activate_leaf(&mut overseer, local_para, &test_leaf, &state, true).await; + + answer_expected_hypothetical_depth_request( + &mut overseer, + vec![], + Some(relay_parent), + false, + ) + .await; + + let statement = state + .sign_statement( + v_non, + CompactStatement::Seconded(candidate_hash), + &SigningContext { parent_hash: relay_parent, session_index: 1 }, + ) + .as_unchecked() + .clone(); + + send_peer_message( + &mut overseer, + peer_a.clone(), + protocol_vstaging::StatementDistributionMessage::Statement(relay_parent, statement), + ) + .await; + + assert_matches!( + overseer.recv().await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::ReportPeer(p, r)) + if p == peer_a && r == COST_UNEXPECTED_STATEMENT => { } + ); + + overseer + }); +} + +#[test] +fn statement_from_non_cluster_originator_unexpected() { + let config = TestConfig { validator_count: 20, group_size: 3, local_validator: true }; + + let relay_parent = Hash::repeat_byte(1); + let peer_a = PeerId::random(); + + test_harness(config, |state, mut overseer| async move { + let local_validator = state.local.clone().unwrap(); + let local_para = ParaId::from(local_validator.group_index.0); + let candidate_hash = CandidateHash(Hash::repeat_byte(42)); + + let test_leaf = TestLeaf { + number: 1, + hash: relay_parent, + parent_hash: Hash::repeat_byte(0), + session: 1, + availability_cores: state.make_availability_cores(|i| { + CoreState::Scheduled(ScheduledCore { + para_id: ParaId::from(i as u32), + collator: None, + }) + }), + para_data: (0..state.session_info.validator_groups.len()) + .map(|i| { + ( + ParaId::from(i as u32), + PerParaData { min_relay_parent: 1, head_data: vec![1, 2, 3].into() }, + ) + }) + .collect(), + }; + + // peer A is not in group, has relay parent in view. + let other_group_validators = state.group_validators(local_validator.group_index, true); + let v_a = other_group_validators[0]; + + connect_peer(&mut overseer, peer_a.clone(), None).await; + + send_peer_view_change(&mut overseer, peer_a.clone(), view![relay_parent]).await; + activate_leaf(&mut overseer, local_para, &test_leaf, &state, true).await; + + answer_expected_hypothetical_depth_request( + &mut overseer, + vec![], + Some(relay_parent), + false, + ) + .await; + + let statement = state + .sign_statement( + v_a, + CompactStatement::Seconded(candidate_hash), + &SigningContext { parent_hash: relay_parent, session_index: 1 }, + ) + .as_unchecked() + .clone(); + + send_peer_message( + &mut overseer, + peer_a.clone(), + protocol_vstaging::StatementDistributionMessage::Statement(relay_parent, statement), + ) + .await; + + assert_matches!( + overseer.recv().await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::ReportPeer(p, r)) + if p == peer_a && r == COST_UNEXPECTED_STATEMENT => { } + ); + + overseer + }); +} + +#[test] +fn seconded_statement_leads_to_request() { + let config = TestConfig { validator_count: 20, group_size: 3, local_validator: true }; + + let relay_parent = Hash::repeat_byte(1); + let peer_a = PeerId::random(); + + test_harness(config, |state, mut overseer| async move { + let local_validator = state.local.clone().unwrap(); + let local_para = ParaId::from(local_validator.group_index.0); + let candidate_hash = CandidateHash(Hash::repeat_byte(42)); + + let test_leaf = TestLeaf { + number: 1, + hash: relay_parent, + parent_hash: Hash::repeat_byte(0), + session: 1, + availability_cores: state.make_availability_cores(|i| { + CoreState::Scheduled(ScheduledCore { + para_id: ParaId::from(i as u32), + collator: None, + }) + }), + para_data: (0..state.session_info.validator_groups.len()) + .map(|i| { + ( + ParaId::from(i as u32), + PerParaData { min_relay_parent: 1, head_data: vec![1, 2, 3].into() }, + ) + }) + .collect(), + }; + + // peer A is in group, has relay parent in view. + let other_group_validators = state.group_validators(local_validator.group_index, true); + let v_a = other_group_validators[0]; + + connect_peer( + &mut overseer, + peer_a.clone(), + Some(vec![state.discovery_id(v_a)].into_iter().collect()), + ) + .await; + + send_peer_view_change(&mut overseer, peer_a.clone(), view![relay_parent]).await; + activate_leaf(&mut overseer, local_para, &test_leaf, &state, true).await; + + answer_expected_hypothetical_depth_request( + &mut overseer, + vec![], + Some(relay_parent), + false, + ) + .await; + + let statement = state + .sign_statement( + v_a, + CompactStatement::Seconded(candidate_hash), + &SigningContext { parent_hash: relay_parent, session_index: 1 }, + ) + .as_unchecked() + .clone(); + + send_peer_message( + &mut overseer, + peer_a.clone(), + protocol_vstaging::StatementDistributionMessage::Statement(relay_parent, statement), + ) + .await; + + assert_matches!( + overseer.recv().await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::ReportPeer(p, r)) + if p == peer_a && r == BENEFIT_VALID_STATEMENT_FIRST => { } + ); + + assert_matches!( + overseer.recv().await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendRequests(requests, IfDisconnected::ImmediateError)) => { + assert_eq!(requests.len(), 1); + assert_matches!( + &requests[0], + Requests::AttestedCandidateVStaging(outgoing) => { + assert_eq!(outgoing.peer, Recipient::Peer(peer_a.clone())); + assert_eq!(outgoing.payload.candidate_hash, candidate_hash); + } + ); + } + ); + + overseer + }); +} + +#[test] +fn cluster_statements_shared_seconded_first() { + let config = TestConfig { validator_count: 20, group_size: 3, local_validator: true }; + + let relay_parent = Hash::repeat_byte(1); + let peer_a = PeerId::random(); + + test_harness(config, |state, mut overseer| async move { + let local_validator = state.local.clone().unwrap(); + let local_para = ParaId::from(local_validator.group_index.0); + + let (candidate, pvd) = make_candidate( + relay_parent, + 1, + local_para, + vec![1, 2, 3].into(), + vec![4, 5, 6].into(), + Hash::repeat_byte(42).into(), + ); + let candidate_hash = candidate.hash(); + + let test_leaf = TestLeaf { + number: 1, + hash: relay_parent, + parent_hash: Hash::repeat_byte(0), + session: 1, + availability_cores: state.make_availability_cores(|i| { + CoreState::Scheduled(ScheduledCore { + para_id: ParaId::from(i as u32), + collator: None, + }) + }), + para_data: (0..state.session_info.validator_groups.len()) + .map(|i| { + ( + ParaId::from(i as u32), + PerParaData { min_relay_parent: 1, head_data: vec![1, 2, 3].into() }, + ) + }) + .collect(), + }; + + // peer A is in group, no relay parent in view. + { + let other_group_validators = state.group_validators(local_validator.group_index, true); + + connect_peer( + &mut overseer, + peer_a.clone(), + Some(vec![state.discovery_id(other_group_validators[0])].into_iter().collect()), + ) + .await; + } + + activate_leaf(&mut overseer, local_para, &test_leaf, &state, true).await; + + answer_expected_hypothetical_depth_request( + &mut overseer, + vec![], + Some(relay_parent), + false, + ) + .await; + + let full_signed = state + .sign_statement( + local_validator.validator_index, + CompactStatement::Seconded(candidate_hash), + &SigningContext { session_index: 1, parent_hash: relay_parent }, + ) + .convert_to_superpayload(StatementWithPVD::Seconded(candidate.clone(), pvd.clone())) + .unwrap(); + + let valid_signed = state + .sign_statement( + local_validator.validator_index, + CompactStatement::Valid(candidate_hash), + &SigningContext { session_index: 1, parent_hash: relay_parent }, + ) + .convert_to_superpayload(StatementWithPVD::Valid(candidate_hash)) + .unwrap(); + + overseer + .send(FromOrchestra::Communication { + msg: StatementDistributionMessage::Share(relay_parent, full_signed), + }) + .await; + + // result of new confirmed candidate. + answer_expected_hypothetical_depth_request(&mut overseer, vec![], None, false).await; + + overseer + .send(FromOrchestra::Communication { + msg: StatementDistributionMessage::Share(relay_parent, valid_signed), + }) + .await; + + send_peer_view_change(&mut overseer, peer_a.clone(), view![relay_parent]).await; + + assert_matches!( + overseer.recv().await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessages(messages)) => { + assert_eq!(messages.len(), 2); + + assert_eq!(messages[0].0, vec![peer_a]); + assert_eq!(messages[1].0, vec![peer_a]); + + assert_matches!( + &messages[0].1, + Versioned::VStaging(protocol_vstaging::ValidationProtocol::StatementDistribution( + protocol_vstaging::StatementDistributionMessage::Statement( + r, + s, + ) + )) if r == &relay_parent + && s.unchecked_payload() == &CompactStatement::Seconded(candidate_hash) => {} + ); + + assert_matches!( + &messages[1].1, + Versioned::VStaging(protocol_vstaging::ValidationProtocol::StatementDistribution( + protocol_vstaging::StatementDistributionMessage::Statement( + r, + s, + ) + )) if r == &relay_parent + && s.unchecked_payload() == &CompactStatement::Valid(candidate_hash) => {} + ); + } + ); + + overseer + }); +} + +#[test] +fn cluster_accounts_for_implicit_view() { + let config = TestConfig { validator_count: 20, group_size: 3, local_validator: true }; + + let relay_parent = Hash::repeat_byte(1); + let peer_a = PeerId::random(); + let peer_b = PeerId::random(); + + test_harness(config, |state, mut overseer| async move { + let local_validator = state.local.clone().unwrap(); + let local_para = ParaId::from(local_validator.group_index.0); + + let (candidate, pvd) = make_candidate( + relay_parent, + 1, + local_para, + vec![1, 2, 3].into(), + vec![4, 5, 6].into(), + Hash::repeat_byte(42).into(), + ); + let candidate_hash = candidate.hash(); + + let test_leaf = TestLeaf { + number: 1, + hash: relay_parent, + parent_hash: Hash::repeat_byte(0), + session: 1, + availability_cores: state.make_availability_cores(|i| { + CoreState::Scheduled(ScheduledCore { + para_id: ParaId::from(i as u32), + collator: None, + }) + }), + para_data: (0..state.session_info.validator_groups.len()) + .map(|i| { + ( + ParaId::from(i as u32), + PerParaData { min_relay_parent: 1, head_data: vec![1, 2, 3].into() }, + ) + }) + .collect(), + }; + + // peer A is in group, has relay parent in view. + // peer B is in group, has no relay parent in view. + { + let other_group_validators = state.group_validators(local_validator.group_index, true); + + connect_peer( + &mut overseer, + peer_a.clone(), + Some(vec![state.discovery_id(other_group_validators[0])].into_iter().collect()), + ) + .await; + + connect_peer( + &mut overseer, + peer_b.clone(), + Some(vec![state.discovery_id(other_group_validators[1])].into_iter().collect()), + ) + .await; + + send_peer_view_change(&mut overseer, peer_a.clone(), view![relay_parent]).await; + } + + activate_leaf(&mut overseer, local_para, &test_leaf, &state, true).await; + + answer_expected_hypothetical_depth_request( + &mut overseer, + vec![], + Some(relay_parent), + false, + ) + .await; + + let full_signed = state + .sign_statement( + local_validator.validator_index, + CompactStatement::Seconded(candidate_hash), + &SigningContext { session_index: 1, parent_hash: relay_parent }, + ) + .convert_to_superpayload(StatementWithPVD::Seconded(candidate.clone(), pvd.clone())) + .unwrap(); + + overseer + .send(FromOrchestra::Communication { + msg: StatementDistributionMessage::Share(relay_parent, full_signed), + }) + .await; + + assert_matches!( + overseer.recv().await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( + peers, + Versioned::VStaging(protocol_vstaging::ValidationProtocol::StatementDistribution( + protocol_vstaging::StatementDistributionMessage::Statement( + r, + s, + ) + )) + )) => { + assert_eq!(peers, vec![peer_a.clone()]); + assert_eq!(r, relay_parent); + assert_eq!(s.unchecked_payload(), &CompactStatement::Seconded(candidate_hash)); + assert_eq!(s.unchecked_validator_index(), local_validator.validator_index); + } + ); + + // sharing a `Seconded` message confirms a candidate, which leads to new + // fragment tree updates. + answer_expected_hypothetical_depth_request(&mut overseer, vec![], None, false).await; + + // activate new leaf, which has relay-parent in implicit view. + let next_relay_parent = Hash::repeat_byte(2); + let next_test_leaf = TestLeaf { + number: 2, + hash: next_relay_parent, + parent_hash: relay_parent, + session: 1, + availability_cores: state.make_availability_cores(|i| { + CoreState::Scheduled(ScheduledCore { + para_id: ParaId::from(i as u32), + collator: None, + }) + }), + para_data: (0..state.session_info.validator_groups.len()) + .map(|i| { + ( + ParaId::from(i as u32), + PerParaData { min_relay_parent: 1, head_data: vec![1, 2, 3].into() }, + ) + }) + .collect(), + }; + + activate_leaf(&mut overseer, local_para, &next_test_leaf, &state, false).await; + + answer_expected_hypothetical_depth_request( + &mut overseer, + vec![], + Some(next_relay_parent), + false, + ) + .await; + + send_peer_view_change(&mut overseer, peer_a.clone(), view![next_relay_parent]).await; + send_peer_view_change(&mut overseer, peer_b.clone(), view![next_relay_parent]).await; + + // peer B never had the relay parent in its view, so this tests that + // the implicit view is working correctly for B. + // + // the fact that the statement isn't sent again to A also indicates that it works + // it's working. + assert_matches!( + overseer.recv().await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessages(messages)) => { + assert_eq!(messages.len(), 1); + assert_matches!( + &messages[0], + ( + peers, + Versioned::VStaging(protocol_vstaging::ValidationProtocol::StatementDistribution( + protocol_vstaging::StatementDistributionMessage::Statement( + r, + s, + ) + )) + ) => { + assert_eq!(peers, &vec![peer_b.clone()]); + assert_eq!(r, &relay_parent); + assert_eq!(s.unchecked_payload(), &CompactStatement::Seconded(candidate_hash)); + assert_eq!(s.unchecked_validator_index(), local_validator.validator_index); + } + ) + } + ); + + overseer + }); +} + +#[test] +fn cluster_messages_imported_after_confirmed_candidate_importable_check() { + let config = TestConfig { validator_count: 20, group_size: 3, local_validator: true }; + + let relay_parent = Hash::repeat_byte(1); + let peer_a = PeerId::random(); + let peer_b = PeerId::random(); + + test_harness(config, |state, mut overseer| async move { + let local_validator = state.local.clone().unwrap(); + let local_para = ParaId::from(local_validator.group_index.0); + + let (candidate, pvd) = make_candidate( + relay_parent, + 1, + local_para, + vec![1, 2, 3].into(), + vec![4, 5, 6].into(), + Hash::repeat_byte(42).into(), + ); + let candidate_hash = candidate.hash(); + + let test_leaf = TestLeaf { + number: 1, + hash: relay_parent, + parent_hash: Hash::repeat_byte(0), + session: 1, + availability_cores: state.make_availability_cores(|i| { + CoreState::Scheduled(ScheduledCore { + para_id: ParaId::from(i as u32), + collator: None, + }) + }), + para_data: (0..state.session_info.validator_groups.len()) + .map(|i| { + ( + ParaId::from(i as u32), + PerParaData { min_relay_parent: 1, head_data: vec![1, 2, 3].into() }, + ) + }) + .collect(), + }; + + // peer A is in group, has relay parent in view. + let other_group_validators = state.group_validators(local_validator.group_index, true); + let v_a = other_group_validators[0]; + { + connect_peer( + &mut overseer, + peer_a.clone(), + Some(vec![state.discovery_id(v_a)].into_iter().collect()), + ) + .await; + + send_peer_view_change(&mut overseer, peer_a.clone(), view![relay_parent]).await; + } + + activate_leaf(&mut overseer, local_para, &test_leaf, &state, true).await; + + answer_expected_hypothetical_depth_request( + &mut overseer, + vec![], + Some(relay_parent), + false, + ) + .await; + + let a_seconded = state + .sign_statement( + v_a, + CompactStatement::Seconded(candidate_hash), + &SigningContext { parent_hash: relay_parent, session_index: 1 }, + ) + .as_unchecked() + .clone(); + + send_peer_message( + &mut overseer, + peer_a.clone(), + protocol_vstaging::StatementDistributionMessage::Statement(relay_parent, a_seconded), + ) + .await; + + assert_matches!( + overseer.recv().await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::ReportPeer(p, r)) + if p == peer_a && r == BENEFIT_VALID_STATEMENT_FIRST => { } + ); + + let req = assert_matches!( + overseer.recv().await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendRequests(mut requests, IfDisconnected::ImmediateError)) => { + assert_eq!(requests.len(), 1); + assert_matches!( + requests.pop().unwrap(), + Requests::AttestedCandidateVStaging(mut outgoing) => { + assert_eq!(outgoing.peer, Recipient::Peer(peer_a.clone())); + assert_eq!(outgoing.payload.candidate_hash, candidate_hash); + + let res = AttestedCandidateResponse { + candidate_receipt: candidate.clone(), + persisted_validation_data: pvd.clone(), + statements: vec![], + }; + outgoing.pending_response.send(Ok(res.encode())); + } + ); + } + ); + + assert_matches!( + overseer.recv().await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::ReportPeer(p, r)) + if p == peer_a && r == BENEFIT_VALID_RESPONSE => { } + ); + + answer_expected_hypothetical_depth_request( + &mut overseer, + vec![( + HypotheticalCandidate::Complete { + candidate_hash, + receipt: Arc::new(candidate.clone()), + persisted_validation_data: pvd.clone(), + }, + vec![(relay_parent, vec![0])], + )], + None, + false, + ) + .await; + + assert_matches!( + overseer.recv().await, + AllMessages::CandidateBacking(CandidateBackingMessage::Statement( + r, + s, + )) if r == relay_parent => { + assert_matches!( + s.payload(), + FullStatementWithPVD::Seconded(c, p) + if c == &candidate && p == &pvd => {} + ); + assert_eq!(s.validator_index(), v_a); + } + ); + + overseer + }); +} + +#[test] +fn cluster_messages_imported_after_new_leaf_importable_check() { + let config = TestConfig { validator_count: 20, group_size: 3, local_validator: true }; + + let relay_parent = Hash::repeat_byte(1); + let peer_a = PeerId::random(); + let peer_b = PeerId::random(); + + test_harness(config, |state, mut overseer| async move { + let local_validator = state.local.clone().unwrap(); + let local_para = ParaId::from(local_validator.group_index.0); + + let (candidate, pvd) = make_candidate( + relay_parent, + 1, + local_para, + vec![1, 2, 3].into(), + vec![4, 5, 6].into(), + Hash::repeat_byte(42).into(), + ); + let candidate_hash = candidate.hash(); + + let test_leaf = TestLeaf { + number: 1, + hash: relay_parent, + parent_hash: Hash::repeat_byte(0), + session: 1, + availability_cores: state.make_availability_cores(|i| { + CoreState::Scheduled(ScheduledCore { + para_id: ParaId::from(i as u32), + collator: None, + }) + }), + para_data: (0..state.session_info.validator_groups.len()) + .map(|i| { + ( + ParaId::from(i as u32), + PerParaData { min_relay_parent: 1, head_data: vec![1, 2, 3].into() }, + ) + }) + .collect(), + }; + + // peer A is in group, has relay parent in view. + let other_group_validators = state.group_validators(local_validator.group_index, true); + let v_a = other_group_validators[0]; + { + connect_peer( + &mut overseer, + peer_a.clone(), + Some(vec![state.discovery_id(v_a)].into_iter().collect()), + ) + .await; + + send_peer_view_change(&mut overseer, peer_a.clone(), view![relay_parent]).await; + } + + activate_leaf(&mut overseer, local_para, &test_leaf, &state, true).await; + + answer_expected_hypothetical_depth_request( + &mut overseer, + vec![], + Some(relay_parent), + false, + ) + .await; + + let a_seconded = state + .sign_statement( + v_a, + CompactStatement::Seconded(candidate_hash), + &SigningContext { parent_hash: relay_parent, session_index: 1 }, + ) + .as_unchecked() + .clone(); + + send_peer_message( + &mut overseer, + peer_a.clone(), + protocol_vstaging::StatementDistributionMessage::Statement(relay_parent, a_seconded), + ) + .await; + + assert_matches!( + overseer.recv().await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::ReportPeer(p, r)) + if p == peer_a && r == BENEFIT_VALID_STATEMENT_FIRST => { } + ); + + let req = assert_matches!( + overseer.recv().await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendRequests(mut requests, IfDisconnected::ImmediateError)) => { + assert_eq!(requests.len(), 1); + assert_matches!( + requests.pop().unwrap(), + Requests::AttestedCandidateVStaging(mut outgoing) => { + assert_eq!(outgoing.peer, Recipient::Peer(peer_a.clone())); + assert_eq!(outgoing.payload.candidate_hash, candidate_hash); + + let res = AttestedCandidateResponse { + candidate_receipt: candidate.clone(), + persisted_validation_data: pvd.clone(), + statements: vec![], + }; + outgoing.pending_response.send(Ok(res.encode())); + } + ); + } + ); + + assert_matches!( + overseer.recv().await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::ReportPeer(p, r)) + if p == peer_a && r == BENEFIT_VALID_RESPONSE => { } + ); + + answer_expected_hypothetical_depth_request(&mut overseer, vec![], None, false).await; + + let next_relay_parent = Hash::repeat_byte(2); + let next_test_leaf = TestLeaf { + number: 2, + hash: next_relay_parent, + parent_hash: relay_parent, + session: 1, + availability_cores: state.make_availability_cores(|i| { + CoreState::Scheduled(ScheduledCore { + para_id: ParaId::from(i as u32), + collator: None, + }) + }), + para_data: (0..state.session_info.validator_groups.len()) + .map(|i| { + ( + ParaId::from(i as u32), + PerParaData { min_relay_parent: 1, head_data: vec![1, 2, 3].into() }, + ) + }) + .collect(), + }; + + activate_leaf(&mut overseer, local_para, &next_test_leaf, &state, false).await; + + answer_expected_hypothetical_depth_request( + &mut overseer, + vec![( + HypotheticalCandidate::Complete { + candidate_hash, + receipt: Arc::new(candidate.clone()), + persisted_validation_data: pvd.clone(), + }, + vec![(relay_parent, vec![0])], + )], + Some(next_relay_parent), + false, + ) + .await; + + assert_matches!( + overseer.recv().await, + AllMessages::CandidateBacking(CandidateBackingMessage::Statement( + r, + s, + )) if r == relay_parent => { + assert_matches!( + s.payload(), + FullStatementWithPVD::Seconded(c, p) + if c == &candidate && p == &pvd => {} + ); + assert_eq!(s.validator_index(), v_a); + } + ); + + overseer + }); +} + +// TODO [now]: ensure seconding limit is respected diff --git a/node/network/statement-distribution/src/vstaging/tests/grid.rs b/node/network/statement-distribution/src/vstaging/tests/grid.rs new file mode 100644 index 000000000000..c5eb3826846e --- /dev/null +++ b/node/network/statement-distribution/src/vstaging/tests/grid.rs @@ -0,0 +1,41 @@ +// Copyright 2023 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +// TODO [now]: backed candidate leads to advertisement to relevant validators with relay-parent + +// TODO [now]: received advertisement before confirmation leads to request + +// TODO [now]: received advertisement after backing leads to acknowledgement + +// TODO [now]: received advertisement after confirmation but before backing leads to nothing + +// TODO [now]: additional statements are shared after manifest exchange + +// TODO [now]: grid-sending validator view entering relay-parent leads to advertisement + +// TODO [now]: advertisement not re-sent after re-entering relay parent (view oscillation) + +// TODO [now]: acknowledgements sent only when candidate backed + +// TODO [now]: grid statements imported to backing once candidate enters hypothetical frontier + +// TODO [now]: advertisements rejected from incorrect peers + +// TODO [now]: manifests rejected with unknown relay parent or when not a validator + +// TODO [now]: advertisements rejected when candidate group doers not match para + +// TODO [now]: peer reported when advertisement conflicting with confirmed candidate. diff --git a/node/network/statement-distribution/src/vstaging/tests/mod.rs b/node/network/statement-distribution/src/vstaging/tests/mod.rs new file mode 100644 index 000000000000..583d17616629 --- /dev/null +++ b/node/network/statement-distribution/src/vstaging/tests/mod.rs @@ -0,0 +1,503 @@ +// Copyright 2023 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +// TODO [now]: Remove once some tests are written. +#![allow(unused)] + +use super::*; +use crate::*; +use polkadot_node_network_protocol::{ + request_response::{outgoing::Recipient, ReqProtocolNames}, + view, ObservedRole, +}; +use polkadot_node_subsystem::messages::{ + network_bridge_event::NewGossipTopology, AllMessages, ChainApiMessage, FragmentTreeMembership, + HypotheticalCandidate, NetworkBridgeEvent, ProspectiveParachainsMessage, RuntimeApiMessage, + RuntimeApiRequest, +}; +use polkadot_node_subsystem_test_helpers as test_helpers; +use polkadot_node_subsystem_types::{jaeger, ActivatedLeaf, LeafStatus}; +use polkadot_primitives::vstaging::{ + AssignmentId, AssignmentPair, AsyncBackingParameters, BlockNumber, CandidateCommitments, + CandidateDescriptor, CommittedCandidateReceipt, CoreState, GroupRotationInfo, HeadData, Header, + IndexedVec, PersistedValidationData, ScheduledCore, SessionIndex, SessionInfo, + ValidationCodeHash, ValidatorPair, +}; +use sc_keystore::LocalKeystore; +use sp_application_crypto::Pair as PairT; +use sp_authority_discovery::AuthorityPair as AuthorityDiscoveryPair; +use sp_keyring::Sr25519Keyring; + +use assert_matches::assert_matches; +use futures::Future; +use parity_scale_codec::Encode; +use rand::{Rng, SeedableRng}; + +use std::sync::Arc; + +mod cluster; +mod grid; +mod requests; + +type VirtualOverseer = test_helpers::TestSubsystemContextHandle; + +const ASYNC_BACKING_PARAMETERS: AsyncBackingParameters = + AsyncBackingParameters { max_candidate_depth: 4, allowed_ancestry_len: 3 }; + +// Some deterministic genesis hash for req/res protocol names +const GENESIS_HASH: Hash = Hash::repeat_byte(0xff); + +struct TestConfig { + validator_count: usize, + // how many validators to place in each group. + group_size: usize, + // whether the local node should be a validator + local_validator: bool, +} + +#[derive(Clone)] +struct TestLocalValidator { + validator_id: ValidatorId, + validator_index: ValidatorIndex, + group_index: GroupIndex, +} + +struct TestState { + config: TestConfig, + local: Option, + validators: Vec, + session_info: SessionInfo, + req_sender: futures::channel::mpsc::Sender, +} + +impl TestState { + fn from_config( + config: TestConfig, + req_sender: futures::channel::mpsc::Sender, + rng: &mut impl Rng, + ) -> Self { + if config.group_size == 0 { + panic!("group size cannot be 0"); + } + + let mut validators = Vec::new(); + let mut discovery_keys = Vec::new(); + let mut assignment_keys = Vec::new(); + let mut validator_groups = Vec::new(); + + let local_validator_pos = if config.local_validator { + // ensure local validator is always in a full group. + Some(rng.gen_range(0..config.validator_count).saturating_sub(config.group_size - 1)) + } else { + None + }; + + for i in 0..config.validator_count { + let validator_pair = if Some(i) == local_validator_pos { + // Note: the specific key is used to ensure the keystore holds + // this key and the subsystem can detect that it is a validator. + Sr25519Keyring::Ferdie.pair().into() + } else { + ValidatorPair::generate().0 + }; + let assignment_id = AssignmentPair::generate().0.public(); + let discovery_id = AuthorityDiscoveryPair::generate().0.public(); + + let group_index = i / config.group_size; + validators.push(validator_pair); + discovery_keys.push(discovery_id); + assignment_keys.push(assignment_id); + if validator_groups.len() == group_index { + validator_groups.push(vec![ValidatorIndex(i as _)]); + } else { + validator_groups.last_mut().unwrap().push(ValidatorIndex(i as _)); + } + } + + let local = if let Some(local_pos) = local_validator_pos { + Some(TestLocalValidator { + validator_id: validators[local_pos].public().clone(), + validator_index: ValidatorIndex(local_pos as _), + group_index: GroupIndex((local_pos / config.group_size) as _), + }) + } else { + None + }; + + let validator_public = validator_pubkeys(&validators); + let session_info = SessionInfo { + validators: validator_public, + discovery_keys, + validator_groups: IndexedVec::from(validator_groups), + assignment_keys, + n_cores: 0, + zeroth_delay_tranche_width: 0, + relay_vrf_modulo_samples: 0, + n_delay_tranches: 0, + no_show_slots: 0, + needed_approvals: 0, + active_validator_indices: vec![], + dispute_period: 6, + random_seed: [0u8; 32], + }; + + TestState { config, local, validators, session_info, req_sender } + } + + fn make_availability_cores(&self, f: impl Fn(usize) -> CoreState) -> Vec { + (0..self.session_info.validator_groups.len()).map(f).collect() + } + + fn group_validators( + &self, + group_index: GroupIndex, + exclude_local: bool, + ) -> Vec { + self.session_info + .validator_groups + .get(group_index) + .unwrap() + .iter() + .cloned() + .filter(|&i| { + self.local.as_ref().map_or(true, |l| !exclude_local || l.validator_index != i) + }) + .collect() + } + + fn validator_id(&self, validator_index: ValidatorIndex) -> ValidatorId { + self.session_info.validators.get(validator_index).unwrap().clone() + } + + fn discovery_id(&self, validator_index: ValidatorIndex) -> AuthorityDiscoveryId { + self.session_info.discovery_keys[validator_index.0 as usize].clone() + } + + fn sign_statement( + &self, + validator_index: ValidatorIndex, + statement: CompactStatement, + context: &SigningContext, + ) -> SignedStatement { + let payload = statement.signing_payload(context); + let pair = &self.validators[validator_index.0 as usize]; + let signature = pair.sign(&payload[..]); + + SignedStatement::new(statement, validator_index, signature, context, &pair.public()) + .unwrap() + } + + // send a request out, returning a future which expects a response. + async fn send_request( + &mut self, + peer: PeerId, + request: AttestedCandidateRequest, + ) -> impl Future { + let (tx, rx) = futures::channel::oneshot::channel(); + let req = sc_network::config::IncomingRequest { + peer, + payload: request.encode(), + pending_response: tx, + }; + self.req_sender.send(req).await.unwrap(); + + rx.map(|r| r.unwrap()) + } +} + +fn test_harness>( + config: TestConfig, + test: impl FnOnce(TestState, VirtualOverseer) -> T, +) { + let pool = sp_core::testing::TaskExecutor::new(); + let keystore = if config.local_validator { + test_helpers::mock::make_ferdie_keystore() + } else { + Arc::new(LocalKeystore::in_memory()) as SyncCryptoStorePtr + }; + let req_protocol_names = ReqProtocolNames::new(&GENESIS_HASH, None); + let (statement_req_receiver, _) = IncomingRequest::get_config_receiver(&req_protocol_names); + let (candidate_req_receiver, req_cfg) = + IncomingRequest::get_config_receiver(&req_protocol_names); + let mut rng = rand_chacha::ChaCha8Rng::seed_from_u64(0); + + let test_state = TestState::from_config(config, req_cfg.inbound_queue.unwrap(), &mut rng); + + let (context, virtual_overseer) = test_helpers::make_subsystem_context(pool.clone()); + let subsystem = async move { + let subsystem = crate::StatementDistributionSubsystem::new( + keystore, + statement_req_receiver, + candidate_req_receiver, + Metrics::default(), + rng, + ); + + if let Err(e) = subsystem.run(context).await { + panic!("Fatal error: {:?}", e); + } + }; + + let test_fut = test(test_state, virtual_overseer); + + futures::pin_mut!(test_fut); + futures::pin_mut!(subsystem); + futures::executor::block_on(future::join( + async move { + let mut virtual_overseer = test_fut.await; + virtual_overseer.send(FromOrchestra::Signal(OverseerSignal::Conclude)).await; + }, + subsystem, + )); +} + +struct PerParaData { + min_relay_parent: BlockNumber, + head_data: HeadData, +} + +impl PerParaData { + pub fn new(min_relay_parent: BlockNumber, head_data: HeadData) -> Self { + Self { min_relay_parent, head_data } + } +} + +struct TestLeaf { + number: BlockNumber, + hash: Hash, + parent_hash: Hash, + session: SessionIndex, + availability_cores: Vec, + para_data: Vec<(ParaId, PerParaData)>, +} + +impl TestLeaf { + pub fn para_data(&self, para_id: ParaId) -> &PerParaData { + self.para_data + .iter() + .find_map(|(p_id, data)| if *p_id == para_id { Some(data) } else { None }) + .unwrap() + } +} + +async fn activate_leaf( + virtual_overseer: &mut VirtualOverseer, + para_id: ParaId, + leaf: &TestLeaf, + test_state: &TestState, + expect_session_info_request: bool, +) { + let activated = ActivatedLeaf { + hash: leaf.hash, + number: leaf.number, + status: LeafStatus::Fresh, + span: Arc::new(jaeger::Span::Disabled), + }; + + virtual_overseer + .send(FromOrchestra::Signal(OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work( + activated, + )))) + .await; + + handle_leaf_activation( + virtual_overseer, + para_id, + leaf, + test_state, + expect_session_info_request, + ) + .await; +} + +async fn handle_leaf_activation( + virtual_overseer: &mut VirtualOverseer, + para_id: ParaId, + leaf: &TestLeaf, + test_state: &TestState, + expect_session_info_request: bool, +) { + let TestLeaf { number, hash, parent_hash, para_data, session, availability_cores } = leaf; + let PerParaData { min_relay_parent, head_data } = leaf.para_data(para_id); + + assert_matches!( + virtual_overseer.recv().await, + AllMessages::RuntimeApi( + RuntimeApiMessage::Request(parent, RuntimeApiRequest::StagingAsyncBackingParameters(tx)) + ) if parent == *hash => { + tx.send(Ok(ASYNC_BACKING_PARAMETERS)).unwrap(); + } + ); + + let mrp_response: Vec<(ParaId, BlockNumber)> = para_data + .iter() + .map(|(para_id, data)| (*para_id, data.min_relay_parent)) + .collect(); + assert_matches!( + virtual_overseer.recv().await, + AllMessages::ProspectiveParachains( + ProspectiveParachainsMessage::GetMinimumRelayParents(parent, tx) + ) if parent == *hash => { + tx.send(mrp_response).unwrap(); + } + ); + + let header = Header { + parent_hash: *parent_hash, + number: *number, + state_root: Hash::zero(), + extrinsics_root: Hash::zero(), + digest: Default::default(), + }; + assert_matches!( + virtual_overseer.recv().await, + AllMessages::ChainApi( + ChainApiMessage::BlockHeader(parent, tx) + ) if parent == *hash => { + tx.send(Ok(Some(header))).unwrap(); + } + ); + + assert_matches!( + virtual_overseer.recv().await, + AllMessages::RuntimeApi( + RuntimeApiMessage::Request(parent, RuntimeApiRequest::SessionIndexForChild(tx))) if parent == *hash => { + tx.send(Ok(*session)).unwrap(); + } + ); + + assert_matches!( + virtual_overseer.recv().await, + AllMessages::RuntimeApi( + RuntimeApiMessage::Request(parent, RuntimeApiRequest::AvailabilityCores(tx))) if parent == *hash => { + tx.send(Ok(availability_cores.clone())).unwrap(); + } + ); + + let validator_groups = test_state.session_info.validator_groups.to_vec(); + let group_rotation_info = + GroupRotationInfo { session_start_block: 1, group_rotation_frequency: 12, now: 1 }; + assert_matches!( + virtual_overseer.recv().await, + AllMessages::RuntimeApi( + RuntimeApiMessage::Request(parent, RuntimeApiRequest::ValidatorGroups(tx))) if parent == *hash => { + tx.send(Ok((validator_groups, group_rotation_info))).unwrap(); + } + ); + + if expect_session_info_request { + assert_matches!( + virtual_overseer.recv().await, + AllMessages::RuntimeApi( + RuntimeApiMessage::Request(parent, RuntimeApiRequest::SessionInfo(s, tx))) if s == *session => { + tx.send(Ok(Some(test_state.session_info.clone()))).unwrap(); + } + ); + } +} + +async fn answer_expected_hypothetical_depth_request( + virtual_overseer: &mut VirtualOverseer, + responses: Vec<(HypotheticalCandidate, FragmentTreeMembership)>, + expected_leaf_hash: Option, + expected_backed_in_path_only: bool, +) { + assert_matches!( + virtual_overseer.recv().await, + AllMessages::ProspectiveParachains( + ProspectiveParachainsMessage::GetHypotheticalFrontier(req, tx) + ) => { + assert_eq!(req.fragment_tree_relay_parent, expected_leaf_hash); + assert_eq!(req.backed_in_path_only, expected_backed_in_path_only); + for (i, (candidate, _)) in responses.iter().enumerate() { + assert!( + req.candidates.iter().find(|c| c == &candidate).is_some(), + "did not receive request for hypothetical candidate {}", + i, + ); + } + + tx.send(responses); + } + ) +} + +fn validator_pubkeys(val_ids: &[ValidatorPair]) -> IndexedVec { + val_ids.iter().map(|v| v.public().into()).collect() +} + +async fn connect_peer( + virtual_overseer: &mut VirtualOverseer, + peer: PeerId, + authority_ids: Option>, +) { + virtual_overseer + .send(FromOrchestra::Communication { + msg: StatementDistributionMessage::NetworkBridgeUpdate( + NetworkBridgeEvent::PeerConnected( + peer, + ObservedRole::Authority, + ValidationVersion::VStaging.into(), + authority_ids, + ), + ), + }) + .await; +} + +async fn disconnect_peer(virtual_overseer: &mut VirtualOverseer, peer: PeerId) { + virtual_overseer + .send(FromOrchestra::Communication { + msg: StatementDistributionMessage::NetworkBridgeUpdate( + NetworkBridgeEvent::PeerDisconnected(peer), + ), + }) + .await; +} + +async fn send_peer_view_change(virtual_overseer: &mut VirtualOverseer, peer: PeerId, view: View) { + virtual_overseer + .send(FromOrchestra::Communication { + msg: StatementDistributionMessage::NetworkBridgeUpdate( + NetworkBridgeEvent::PeerViewChange(peer, view), + ), + }) + .await; +} + +async fn send_peer_message( + virtual_overseer: &mut VirtualOverseer, + peer: PeerId, + message: protocol_vstaging::StatementDistributionMessage, +) { + virtual_overseer + .send(FromOrchestra::Communication { + msg: StatementDistributionMessage::NetworkBridgeUpdate( + NetworkBridgeEvent::PeerMessage(peer, Versioned::VStaging(message)), + ), + }) + .await; +} + +async fn send_new_topology(virtual_overseer: &mut VirtualOverseer, topology: NewGossipTopology) { + virtual_overseer + .send(FromOrchestra::Communication { + msg: StatementDistributionMessage::NetworkBridgeUpdate( + NetworkBridgeEvent::NewGossipTopology(topology), + ), + }) + .await; +} diff --git a/node/network/statement-distribution/src/vstaging/tests/requests.rs b/node/network/statement-distribution/src/vstaging/tests/requests.rs new file mode 100644 index 000000000000..5e624bd622eb --- /dev/null +++ b/node/network/statement-distribution/src/vstaging/tests/requests.rs @@ -0,0 +1,27 @@ +// Copyright 2023 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +// TODO [now]: peer reported for providing statements meant to be masked out + +// TODO [now]: peer reported for not providing enough statements, request retried + +// TODO [now]: peer reported for providing duplicate statements + +// TODO [now]: peer reported for providing statements with invalid signatures or wrong validator IDs + +// TODO [now]: local node sanity checks incoming requests + +// TODO [now]: local node respects statement mask diff --git a/node/overseer/src/lib.rs b/node/overseer/src/lib.rs index 56b656ae9e3f..25252e6bf201 100644 --- a/node/overseer/src/lib.rs +++ b/node/overseer/src/lib.rs @@ -482,6 +482,8 @@ pub struct Overseer { NetworkBridgeTxMessage, CandidateBackingMessage, RuntimeApiMessage, + ProspectiveParachainsMessage, + ChainApiMessage, ])] statement_distribution: StatementDistribution, diff --git a/node/primitives/src/lib.rs b/node/primitives/src/lib.rs index 5080c34b7fca..33fd7c733f4a 100644 --- a/node/primitives/src/lib.rs +++ b/node/primitives/src/lib.rs @@ -275,7 +275,15 @@ impl StatementWithPVD { pub fn drop_pvd_from_signed(signed: SignedFullStatementWithPVD) -> SignedFullStatement { signed .convert_to_superpayload_with(|s| s.drop_pvd()) - .expect("persisted_validation_data doesn't affect encoded_as; qed") + .expect("persisted_validation_data doesn't affect encode_as; qed") + } + + /// Converts the statement to a compact signed statement by dropping the [`CommittedCandidateReceipt`] + /// and the [`PersistedValidationData`]. + pub fn signed_to_compact(signed: SignedFullStatementWithPVD) -> Signed { + signed + .convert_to_superpayload_with(|s| s.to_compact()) + .expect("doesn't affect encode_as; qed") } } @@ -613,3 +621,10 @@ pub fn maybe_compress_pov(pov: PoV) -> PoV { let pov = PoV { block_data: BlockData(raw) }; pov } + +/// How many votes we need to consider a candidate backed. +/// +/// WARNING: This has to be kept in sync with the runtime check in the inclusion module. +pub fn minimum_votes(n_validators: usize) -> usize { + std::cmp::min(2, n_validators) +} diff --git a/node/service/src/lib.rs b/node/service/src/lib.rs index d6b3bcefbbd4..ad3615d6a662 100644 --- a/node/service/src/lib.rs +++ b/node/service/src/lib.rs @@ -901,6 +901,9 @@ where config.network.request_response_protocols.push(cfg); let (statement_req_receiver, cfg) = IncomingRequest::get_config_receiver(&req_protocol_names); config.network.request_response_protocols.push(cfg); + let (candidate_req_vstaging_receiver, cfg) = + IncomingRequest::get_config_receiver(&req_protocol_names); + config.network.request_response_protocols.push(cfg); let (dispute_req_receiver, cfg) = IncomingRequest::get_config_receiver(&req_protocol_names); config.network.request_response_protocols.push(cfg); @@ -1086,6 +1089,7 @@ where collation_req_vstaging_receiver, available_data_req_receiver, statement_req_receiver, + candidate_req_vstaging_receiver, dispute_req_receiver, registry: prometheus_registry.as_ref(), spawner, diff --git a/node/service/src/overseer.rs b/node/service/src/overseer.rs index 3dd1142482dd..404067af0da3 100644 --- a/node/service/src/overseer.rs +++ b/node/service/src/overseer.rs @@ -110,6 +110,9 @@ where IncomingRequestReceiver, /// Receiver for incoming large statement requests. pub statement_req_receiver: IncomingRequestReceiver, + /// Receiver for incoming candidate requests. + pub candidate_req_vstaging_receiver: + IncomingRequestReceiver, /// Receiver for incoming disputes. pub dispute_req_receiver: IncomingRequestReceiver, /// Prometheus registry, commonly used for production systems, less so for test. @@ -154,6 +157,7 @@ pub fn prepared_overseer_builder( collation_req_vstaging_receiver, available_data_req_receiver, statement_req_receiver, + candidate_req_vstaging_receiver, dispute_req_receiver, registry, spawner, @@ -294,6 +298,7 @@ where .statement_distribution(StatementDistributionSubsystem::new( keystore.clone(), statement_req_receiver, + candidate_req_vstaging_receiver, Metrics::register(registry)?, rand::rngs::StdRng::from_entropy(), )) diff --git a/node/subsystem-types/src/messages.rs b/node/subsystem-types/src/messages.rs index 10330757104e..56a19fb5cee0 100644 --- a/node/subsystem-types/src/messages.rs +++ b/node/subsystem-types/src/messages.rs @@ -887,6 +887,17 @@ pub enum GossipSupportMessage { NetworkBridgeUpdate(NetworkBridgeEvent), } +/// Request introduction of a candidate into the prospective parachains subsystem. +#[derive(Debug, PartialEq, Eq, Clone)] +pub struct IntroduceCandidateRequest { + /// The para-id of the candidate. + pub candidate_para: ParaId, + /// The candidate receipt itself. + pub candidate_receipt: CommittedCandidateReceipt, + /// The persisted validation data of the candidate. + pub persisted_validation_data: PersistedValidationData, +} + /// A hypothetical candidate to be evaluated for frontier membership /// in the prospective parachains subsystem. /// @@ -994,17 +1005,16 @@ pub type FragmentTreeMembership = Vec<(Hash, Vec)>; pub enum ProspectiveParachainsMessage { /// Inform the Prospective Parachains Subsystem of a new candidate. /// - /// The response sender accepts the candidate membership, which is empty - /// if the candidate was already known. - CandidateSeconded( - ParaId, - CommittedCandidateReceipt, - PersistedValidationData, - oneshot::Sender, - ), - /// Inform the Prospective Parachains Subsystem that a previously seconded candidate - /// has been backed. This requires that `CandidateSeconded` was sent for the candidate - /// some time in the past. + /// The response sender accepts the candidate membership, which is the existing + /// membership of the candidate if it was already known. + IntroduceCandidate(IntroduceCandidateRequest, oneshot::Sender), + /// Inform the Prospective Parachains Subsystem that a previously introduced candidate + /// has been seconded. This requires that the candidate was successfully introduced in + /// the past. + CandidateSeconded(ParaId, CandidateHash), + /// Inform the Prospective Parachains Subsystem that a previously introduced candidate + /// has been backed. This requires that the candidate was successfully introduced in + /// the past. CandidateBacked(ParaId, CandidateHash), /// Get a backable candidate hash for the given parachain, under the given relay-parent hash, /// which is a descendant of the given candidate hashes. Returns `None` on the channel diff --git a/node/subsystem-util/src/lib.rs b/node/subsystem-util/src/lib.rs index 723aaa4b8f2b..56e1593b0229 100644 --- a/node/subsystem-util/src/lib.rs +++ b/node/subsystem-util/src/lib.rs @@ -275,7 +275,7 @@ pub async fn executor_params_at_relay_parent( /// From the given set of validators, find the first key we can sign with, if any. pub async fn signing_key( - validators: &[ValidatorId], + validators: impl IntoIterator, keystore: &SyncCryptoStorePtr, ) -> Option { signing_key_and_index(validators, keystore).await.map(|(k, _)| k) @@ -284,10 +284,10 @@ pub async fn signing_key( /// From the given set of validators, find the first key we can sign with, if any, and return it /// along with the validator index. pub async fn signing_key_and_index( - validators: &[ValidatorId], + validators: impl IntoIterator, keystore: &SyncCryptoStorePtr, ) -> Option<(ValidatorId, ValidatorIndex)> { - for (i, v) in validators.iter().enumerate() { + for (i, v) in validators.into_iter().enumerate() { if CryptoStore::has_keys(&**keystore, &[(v.to_raw_vec(), ValidatorId::ID)]).await { return Some((v.clone(), ValidatorIndex(i as _))) } diff --git a/primitives/src/v2/mod.rs b/primitives/src/v2/mod.rs index ea4ae8470cdf..f0e128a18c18 100644 --- a/primitives/src/v2/mod.rs +++ b/primitives/src/v2/mod.rs @@ -750,7 +750,7 @@ impl TypeIndex for CoreIndex { } /// The unique (during session) index of a validator group. -#[derive(Encode, Decode, Default, Clone, Copy, Debug, PartialEq, Eq, TypeInfo)] +#[derive(Encode, Decode, Default, Clone, Copy, Debug, PartialEq, Eq, TypeInfo, PartialOrd, Ord)] #[cfg_attr(feature = "std", derive(Hash))] pub struct GroupIndex(pub u32); @@ -1451,7 +1451,7 @@ const BACKING_STATEMENT_MAGIC: [u8; 4] = *b"BKNG"; /// Statements that can be made about parachain candidates. These are the /// actual values that are signed. -#[derive(Clone, PartialEq, Eq, RuntimeDebug)] +#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, RuntimeDebug)] #[cfg_attr(feature = "std", derive(Hash))] pub enum CompactStatement { /// Proposal of a parachain candidate. @@ -1466,6 +1466,13 @@ impl CompactStatement { pub fn signing_payload(&self, context: &SigningContext) -> Vec { (self, context).encode() } + + /// Get the underlying candidate hash this references. + pub fn candidate_hash(&self) -> &CandidateHash { + match *self { + CompactStatement::Seconded(ref h) | CompactStatement::Valid(ref h) => h, + } + } } // Inner helper for codec on `CompactStatement`. @@ -1514,15 +1521,6 @@ impl parity_scale_codec::Decode for CompactStatement { } } -impl CompactStatement { - /// Get the underlying candidate hash this references. - pub fn candidate_hash(&self) -> &CandidateHash { - match *self { - CompactStatement::Seconded(ref h) | CompactStatement::Valid(ref h) => h, - } - } -} - /// `IndexedVec` struct indexed by type specific indices. #[derive(Clone, Encode, Decode, RuntimeDebug, TypeInfo)] #[cfg_attr(feature = "std", derive(PartialEq))] diff --git a/primitives/test-helpers/src/lib.rs b/primitives/test-helpers/src/lib.rs index f1603a53bf2a..ecec7d455ad9 100644 --- a/primitives/test-helpers/src/lib.rs +++ b/primitives/test-helpers/src/lib.rs @@ -23,14 +23,16 @@ //! contain randomness based data. use polkadot_primitives::{ CandidateCommitments, CandidateDescriptor, CandidateReceipt, CollatorId, CollatorSignature, - CommittedCandidateReceipt, Hash, HeadData, Id as ParaId, ValidationCode, ValidationCodeHash, - ValidatorId, + CommittedCandidateReceipt, Hash, HeadData, Id as ParaId, PersistedValidationData, + ValidationCode, ValidationCodeHash, ValidatorId, }; pub use rand; use sp_application_crypto::sr25519; use sp_keyring::Sr25519Keyring; use sp_runtime::generic::Digest; +const MAX_POV_SIZE: u32 = 1_000_000; + /// Creates a candidate receipt with filler data. pub fn dummy_candidate_receipt>(relay_parent: H) -> CandidateReceipt { CandidateReceipt:: { @@ -146,6 +148,46 @@ pub fn dummy_collator_signature() -> CollatorSignature { CollatorSignature::from(sr25519::Signature([0u8; 64])) } +/// Create a meaningless persisted validation data. +pub fn dummy_pvd(parent_head: HeadData, relay_parent_number: u32) -> PersistedValidationData { + PersistedValidationData { + parent_head, + relay_parent_number, + max_pov_size: MAX_POV_SIZE, + relay_parent_storage_root: dummy_hash(), + } +} + +/// Create a meaningless candidate, returning its receipt and PVD. +pub fn make_candidate( + relay_parent_hash: Hash, + relay_parent_number: u32, + para_id: ParaId, + parent_head: HeadData, + head_data: HeadData, + validation_code_hash: ValidationCodeHash, +) -> (CommittedCandidateReceipt, PersistedValidationData) { + let pvd = dummy_pvd(parent_head, relay_parent_number); + let commitments = CandidateCommitments { + head_data, + horizontal_messages: Default::default(), + upward_messages: Default::default(), + new_validation_code: None, + processed_downward_messages: 0, + hrmp_watermark: relay_parent_number, + }; + + let mut candidate = + dummy_candidate_receipt_bad_sig(relay_parent_hash, Some(Default::default())); + candidate.commitments_hash = commitments.hash(); + candidate.descriptor.para_id = para_id; + candidate.descriptor.persisted_validation_data_hash = pvd.hash(); + candidate.descriptor.validation_code_hash = validation_code_hash; + let candidate = CommittedCandidateReceipt { descriptor: candidate.descriptor, commitments }; + + (candidate, pvd) +} + /// Create a new candidate descriptor, and apply a valid signature /// using the provided `collator` key. pub fn make_valid_candidate_descriptor>( diff --git a/runtime/parachains/src/inclusion/mod.rs b/runtime/parachains/src/inclusion/mod.rs index 4f5a7d24635b..14f654a9ce70 100644 --- a/runtime/parachains/src/inclusion/mod.rs +++ b/runtime/parachains/src/inclusion/mod.rs @@ -172,8 +172,7 @@ impl Default for ProcessedCandidates { /// Number of backing votes we need for a valid backing. /// -/// WARNING: This check has to be kept in sync with the node side check in the backing -/// subsystem. +/// WARNING: This check has to be kept in sync with the node side checks. pub fn minimum_backing_votes(n_validators: usize) -> usize { // For considerations on this value see: // https://github.com/paritytech/polkadot/pull/1656#issuecomment-999734650