Store a linked list of nominators in order of stake to avoid OOM when getting top N voters

frame/staking/Cargo.toml

@@ -28,6 +28,7 @@ sp-application-crypto = { version = "3.0.0", default-features = false, path = ".
 frame-election-provider-support = { version = "3.0.0", default-features = false, path = "../election-provider-support" }
 log = { version = "0.4.14", default-features = false }
 paste = "1.0"
+thiserror = "1.0.25"
 
 # Optional imports for benchmarking
 frame-benchmarking = { version = "3.1.0", default-features = false, path = "../benchmarking", optional = true }

frame/staking/src/benchmarking.rs

@@ -561,14 +561,14 @@ benchmarks! {
 	}
 
 	get_npos_voters {
-		// number of validator intention.
-		let v in 200 .. 400;
-		// number of nominator intention.
-		let n in 200 .. 400;
+		// total number of voters (validators + nominators)
+		let v in 400 .. 800;
 		// total number of slashing spans. Assigned to validators randomly.
 		let s in 1 .. 20;
 
-		let validators = create_validators_with_nominators_for_era::<T>(v, n, T::MAX_NOMINATIONS as usize, false, None)?
+		// this isn't ideal, but as we don't store the numbers of nominators and validators
+		// distinctly, we can't really parametrize this function with different quantities of each
+		let validators = create_validators_with_nominators_for_era::<T>(v / 2, n / 2, T::MAX_NOMINATIONS as usize, false, None)?
 			.into_iter()
 			.map(|v| T::Lookup::lookup(v).unwrap())
 			.collect::<Vec<_>>();

frame/staking/src/lib.rs

@@ -276,6 +276,7 @@ pub mod testing_utils;
 #[cfg(any(feature = "runtime-benchmarks", test))]
 pub mod benchmarking;
 
+pub mod voter_list;
 pub mod slashing;
 pub mod inflation;
 pub mod weights;
@@ -353,6 +354,9 @@ type NegativeImbalanceOf<T> = <<T as Config>::Currency as Currency<
 	<T as frame_system::Config>::AccountId,
 >>::NegativeImbalance;
 
+type AccountIdOf<T> = <T as frame_system::Config>::AccountId;
+type VotingDataOf<T> = (AccountIdOf<T>, VoteWeight, Vec<AccountIdOf<T>>);
+
 /// Information regarding the active era (era in used in session).
 #[derive(Encode, Decode, RuntimeDebug)]
 pub struct ActiveEraInfo {
@@ -994,6 +998,19 @@ decl_storage! {
 		///
 		/// This is set to v6.0.0 for new networks.
 		StorageVersion build(|_: &GenesisConfig<T>| Releases::V6_0_0): Releases;
+
+		// The next three storage items collectively comprise the `voter_list::VoterList`
+		// data structure. They should not be accessed individually, but only through the interface
+		// provided by that type.
+
+		/// Count of nominators. Size of `NominatorList`.
+		NominatorCount: u32;
+
+		/// Linked list of nominators, sorted in decreasing order of stake.
+		NominatorList: voter_list::VoterList<T>;
+
+		/// Map of nodes in the nominator list.
+		NominatorNodes: map hasher(twox_64_concat) T::AccountId => Option<voter_list::Node<T>>;
 	}
 	add_extra_genesis {
 		config(stakers):
@@ -2495,37 +2512,22 @@ impl<T: Config> Module<T> {
 	/// auto-chilled.
 	///
 	/// Note that this is VERY expensive. Use with care.
-	pub fn get_npos_voters() -> Vec<(T::AccountId, VoteWeight, Vec<T::AccountId>)> {
-		let weight_of = Self::slashable_balance_of_fn();
-		let mut all_voters = Vec::new();
-
-		for (validator, _) in <Validators<T>>::iter() {
-			// append self vote
-			let self_vote = (validator.clone(), weight_of(&validator), vec![validator.clone()]);
-			all_voters.push(self_vote);
-		}
+	pub fn get_npos_voters(
+		maybe_max_len: Option<usize>,
+	) -> Vec<(T::AccountId, VoteWeight, Vec<T::AccountId>)> {
+		// nominator list contains all nominators and validators
+		let voter_count = voter_list::VoterList::<T>::decode_len().unwrap_or_default();
+		let wanted_voter_count = maybe_max_len.unwrap_or(voter_count).min(voter_count);
 
+		let weight_of = Self::slashable_balance_of_fn();
 		// collect all slashing spans into a BTreeMap for further queries.
-		let slashing_spans = <SlashingSpans<T>>::iter().collect::<BTreeMap<_, _>>();
+		let slashing_spans = <SlashingSpans<T>>::iter().collect();
 
-		for (nominator, nominations) in <Nominators<T>>::iter() {
-			let Nominations { submitted_in, mut targets, suppressed: _ } = nominations;
-
-			// Filter out nomination targets which were nominated before the most recent
-			// slashing span.
-			targets.retain(|stash| {
-				slashing_spans
-					.get(stash)
-					.map_or(true, |spans| submitted_in >= spans.last_nonzero_slash())
-			});
-
-			if !targets.is_empty() {
-				let vote_weight = weight_of(&nominator);
-				all_voters.push((nominator, vote_weight, targets))
-			}
-		}
-
-		all_voters
+		Self::voter_list()
+			.iter()
+			.filter_map(|node| node.voting_data(&weight_of, &slashing_spans))
+			.take(wanted_voter_count)
+			.collect()
 	}
 
 	pub fn get_npos_targets() -> Vec<T::AccountId> {
@@ -2543,25 +2545,11 @@ impl<T: Config> frame_election_provider_support::ElectionDataProvider<T::Account
 
 	fn voters(
 		maybe_max_len: Option<usize>,
-	) -> data_provider::Result<(Vec<(T::AccountId, VoteWeight, Vec<T::AccountId>)>, Weight)> {
-		// NOTE: reading these counts already needs to iterate a lot of storage keys, but they get
-		// cached. This is okay for the case of `Ok(_)`, but bad for `Err(_)`, as the trait does not
-		// report weight in failures.
-		let nominator_count = <Nominators<T>>::iter().count();
-		let validator_count = <Validators<T>>::iter().count();
-		let voter_count = nominator_count.saturating_add(validator_count);
-
-		if maybe_max_len.map_or(false, |max_len| voter_count > max_len) {
-			return Err("Voter snapshot too big");
-		}
-
+	) -> data_provider::Result<(Vec<VotingDataOf<T>>, Weight)> {
+		let voter_count = voter_list::VoterList::<T>::decode_len().unwrap_or_default();
 		let slashing_span_count = <SlashingSpans<T>>::iter().count();
-		let weight = T::WeightInfo::get_npos_voters(
-			nominator_count as u32,
-			validator_count as u32,
-			slashing_span_count as u32,
-		);
-		Ok((Self::get_npos_voters(), weight))
+		let weight = T::WeightInfo::get_npos_voters(voter_count as u32, slashing_span_count as u32);
+		Ok((Self::get_npos_voters(maybe_max_len), weight))
 	}
 
 	fn targets(maybe_max_len: Option<usize>) -> data_provider::Result<(Vec<T::AccountId>, Weight)> {