client/beefy: persist voter state

client/beefy/src/aux_schema.rs

@@ -0,0 +1,91 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+//! Schema for BEEFY state persisted in the aux-db.
+
+use crate::worker::PersistedState;
+use beefy_primitives::{crypto::AuthorityId, ValidatorSet};
+use codec::{Decode, Encode};
+use log::info;
+use sc_client_api::{backend::AuxStore, Backend};
+use sp_blockchain::{Error as ClientError, Result as ClientResult};
+use sp_runtime::traits::{Block as BlockT, NumberFor, One};
+
+const VERSION_KEY: &[u8] = b"beefy_auxschema_version";
+const WORKER_STATE: &[u8] = b"beefy_voter_state";
+// const BEST_JUSTIFICATION: &[u8] = b"beefy_best_justification";
+
+const CURRENT_VERSION: u32 = 1;
+
+/// Write BEEFY DB aux schema version.
+pub(crate) fn set_db_version<BE: AuxStore>(backend: &BE) -> ClientResult<()> {
+	backend.insert_aux(&[(VERSION_KEY, CURRENT_VERSION.encode().as_slice())], &[])
+}
+
+/// Write voter state.
+pub(crate) fn write_voter_state<Block: BlockT, B: AuxStore>(
+	backend: &B,
+	state: &PersistedState<Block>,
+) -> ClientResult<()> {
+	backend.insert_aux(&[(WORKER_STATE, state.encode().as_slice())], &[])
+}
+
+fn load_decode<B: AuxStore, T: Decode>(backend: &B, key: &[u8]) -> ClientResult<Option<T>> {
+	match backend.get_aux(key)? {
+		None => Ok(None),
+		Some(t) => T::decode(&mut &t[..])
+			.map_err(|e| ClientError::Backend(format!("BEEFY DB is corrupted: {}", e)))
+			.map(Some),
+	}
+}
+
+/// Load or initialize persistent data from backend.
+pub(crate) fn load_persistent<Block: BlockT, BE, G>(
+	backend: &BE,
+	best_grandpa_header: <Block as BlockT>::Header,
+	genesis_number: NumberFor<Block>,
+	genesis_validator_set: G,
+	min_block_delta: u32,
+) -> ClientResult<PersistedState<Block>>
+where
+	BE: Backend<Block>,
+	G: FnOnce() -> ClientResult<ValidatorSet<AuthorityId>>,
+{
+	let version: Option<u32> = load_decode(backend, VERSION_KEY)?;
+
+	match version {
+		None => set_db_version(backend)?,
+		Some(1) => {
+			if let Some(state) = load_decode::<_, PersistedState<Block>>(backend, WORKER_STATE)? {
+				return Ok(state)
+			}
+		},
+		other =>
+			return Err(ClientError::Backend(format!("Unsupported BEEFY DB version: {:?}", other))),
+	}
+
+	// genesis.
+	info!(target: "beefy", "🥩 Loading BEEFY voter state \
+		from genesis on what appears to be first startup.");
+
+	// TODO: use these to initialize voter rounds
+	let _genesis_session_start = genesis_number.max(One::one());
+	let _genesis_set = genesis_validator_set()?;
+
+	Ok(PersistedState::new(best_grandpa_header, genesis_number, min_block_delta))
+}

client/beefy/src/lib.rs

@@ -32,6 +32,7 @@ use sp_mmr_primitives::MmrApi;
 use sp_runtime::traits::Block;
 use std::{marker::PhantomData, sync::Arc};
 
+mod aux_schema;
 mod error;
 mod keystore;
 mod metrics;

client/beefy/src/round.rs

@@ -16,27 +16,23 @@
 // You should have received a copy of the GNU General Public License
 // along with this program. If not, see <https://www.gnu.org/licenses/>.
 
-use std::{
-	collections::{BTreeMap, HashMap},
-	hash::Hash,
-};
-
-use log::{debug, trace};
-
 use beefy_primitives::{
 	crypto::{Public, Signature},
 	ValidatorSet, ValidatorSetId,
 };
+use codec::{Decode, Encode};
+use log::{debug, trace};
 use sp_runtime::traits::{Block, NumberFor};
+use std::{collections::BTreeMap, hash::Hash};
 
 /// Tracks for each round which validators have voted/signed and
 /// whether the local `self` validator has voted/signed.
 ///
 /// Does not do any validation on votes or signatures, layers above need to handle that (gossip).
-#[derive(Debug, Default)]
+#[derive(Debug, Decode, Default, Encode)]
 struct RoundTracker {
 	self_vote: bool,
-	votes: HashMap<Public, Signature>,
+	votes: BTreeMap<Public, Signature>,
 }
 
 impl RoundTracker {
@@ -69,7 +65,7 @@ pub fn threshold(authorities: usize) -> usize {
 /// Only round numbers > `best_done` are of interest, all others are considered stale.
 ///
 /// Does not do any validation on votes or signatures, layers above need to handle that (gossip).
-#[derive(Debug)]
+#[derive(Debug, Decode, Encode)]
 pub(crate) struct Rounds<Payload, B: Block> {
 	rounds: BTreeMap<(Payload, NumberFor<B>), RoundTracker>,
 	session_start: NumberFor<B>,

client/beefy/src/worker.rs

@@ -42,7 +42,7 @@ use sp_consensus::SyncOracle;
 use sp_mmr_primitives::MmrApi;
 use sp_runtime::{
 	generic::OpaqueDigestItemId,
-	traits::{Block, Header, NumberFor},
+	traits::{Block, Header, NumberFor, Zero},
 	SaturatedConversion,
 };
 
@@ -75,6 +75,7 @@ enum RoundAction {
 /// Responsible for the voting strategy.
 /// It chooses which incoming votes to accept and which votes to generate.
 /// Keeps track of voting seen for current and future rounds.
+#[derive(Debug, Decode, Encode)]
 struct VoterOracle<B: Block> {
 	/// Queue of known sessions. Keeps track of voting rounds (block numbers) within each session.
 	///
@@ -226,7 +227,8 @@ pub(crate) struct WorkerParams<B: Block, BE, C, P, R, N> {
 	pub min_block_delta: u32,
 }
 
-struct PersistedState<B: Block> {
+#[derive(Debug, Decode, Encode)]
+pub(crate) struct PersistedState<B: Block> {
 	/// Best block we received a GRANDPA finality for.
 	best_grandpa_block_header: <B as Block>::Header,
 	/// Best block a BEEFY voting round has been concluded for.
@@ -236,6 +238,20 @@ struct PersistedState<B: Block> {
 	voting_oracle: VoterOracle<B>,
 }
 
+impl<B: Block> PersistedState<B> {
+	pub fn new(
+		best_grandpa_header: <B as Block>::Header,
+		best_beefy: NumberFor<B>,
+		min_block_delta: u32,
+	) -> Self {
+		PersistedState {
+			best_grandpa_block_header: best_grandpa_header,
+			best_beefy_block: best_beefy,
+			voting_oracle: VoterOracle::new(min_block_delta),
+		}
+	}
+}
+
 /// A BEEFY worker plays the BEEFY protocol
 pub(crate) struct BeefyWorker<B: Block, BE, C, P, R, N> {
 	// utilities
@@ -305,12 +321,22 @@ where
 			.expect_header(BlockId::number(backend.blockchain().info().finalized_number))
 			.expect("latest block always has header available; qed.");
 
-		// TODO: load this from aux db
-		let persisted_state = PersistedState {
-			best_grandpa_block_header: last_finalized_header,
-			best_beefy_block: 0u32.into(),
-			voting_oracle: VoterOracle::new(min_block_delta),
-		};
+		// TODO: get these from runtime? how do we set the initial BEEFY authorities and block?
+		let genesis_number = Zero::zero();
+		let genesis_validator_set =
+			|| Err(sp_blockchain::Error::Backend("FIXME: get from pallet?".into()));
+
+		let mut persisted_state = crate::aux_schema::load_persistent(
+			&*backend,
+			last_finalized_header,
+			genesis_number,
+			genesis_validator_set,
+			min_block_delta,
+		)
+		.expect("FIXME: bubble up this error.");
+
+		// Overwrite persisted data with new client min_block_delta.
+		persisted_state.voting_oracle.min_block_delta = min_block_delta;
 
 		BeefyWorker {
 			client: client.clone(),
@@ -527,9 +553,10 @@ where
 	}
 
 	/// Provide BEEFY finality for block based on `finality_proof`:
-	/// 1. Prune irrelevant past sessions from the oracle,
+	/// 1. Prune now-irrelevant past sessions from the oracle,
 	/// 2. Set BEEFY best block,
-	/// 3. Send best block hash and `finality_proof` to RPC worker.
+	/// 3. Persist voter state,
+	/// 4. Send best block hash and `finality_proof` to RPC worker.
 	///
 	/// Expects `finality proof` to be valid.
 	fn finalize(&mut self, finality_proof: BeefyVersionedFinalityProof<B>) -> Result<(), Error> {
@@ -543,6 +570,9 @@ where
 		if block_num > self.best_beefy_block() {
 			// Set new best BEEFY block number.
 			self.persisted_state.best_beefy_block = block_num;
+			crate::aux_schema::write_voter_state(&*self.backend, &self.persisted_state)
+				.map_err(|e| Error::Backend(e.to_string()))?;
+
 			metric_set!(self, beefy_best_block, block_num);
 
 			self.on_demand_justifications.cancel_requests_older_than(block_num);