fast-sync targetA/B (integritee-network#1573)

* let targetA/B fast-sync as well

* disable BadJustification panic

* disable bogus event proof validation

* warning if event proof validation fails

* pause startup if lockfile present. +small fixes

* implement failover to backup db

* refactor account_funding for convenience in production

* only start sidechain block production once everything is ready. provide ETA for syncing parentchains

* fix teeracle build

* filter noisy logs

Cargo.lock

@@ -2643,6 +2643,7 @@ dependencies = [
  "frame-support",
  "futures 0.3.28",
  "hex",
+ "humantime",
  "ipfs-api",
  "ita-parentchain-interface",
  "itc-parentchain",
@@ -2657,6 +2658,7 @@ dependencies = [
  "itp-settings",
  "itp-stf-interface",
  "itp-storage",
+ "itp-time-utils",
  "itp-types",
  "itp-utils",
  "its-consensus-slots",

app-libs/parentchain-interface/src/event_subscriber.rs

@@ -29,6 +29,8 @@ pub fn subscribe_to_parentchain_events(api: &ParentchainApi, parentchain_id: Par
 			let event = event.unwrap();
 			match event.pallet_name() {
 				"System" => continue,
+				"ParaInclusion" => continue,
+				"MessageQueue" => continue,
 				"TransactionPayment" => continue,
 				"Treasury" => continue,
 				"Balances" => match event.variant_name() {

app-libs/parentchain-interface/src/target_a/event_filter.rs

@@ -73,12 +73,7 @@ impl FilterEvents for FilterableEvents {
 			.iter()
 			.flatten() // flatten filters out the nones
 			.filter_map(|ev| match ev.as_event::<BalanceTransfer>() {
-				Ok(maybe_event) => {
-					if maybe_event.is_none() {
-						log::warn!("Transfer event does not exist in parentchain metadata");
-					};
-					maybe_event
-				},
+				Ok(maybe_event) => maybe_event,
 				Err(e) => {
 					log::error!("Could not decode event: {:?}", e);
 					None

app-libs/parentchain-interface/src/target_b/event_filter.rs

@@ -73,12 +73,7 @@ impl FilterEvents for FilterableEvents {
 			.iter()
 			.flatten() // flatten filters out the nones
 			.filter_map(|ev| match ev.as_event::<BalanceTransfer>() {
-				Ok(maybe_event) => {
-					if maybe_event.is_none() {
-						log::warn!("Transfer event does not exist in parentchain metadata");
-					};
-					maybe_event
-				},
+				Ok(maybe_event) => maybe_event,
 				Err(e) => {
 					log::error!("Could not decode event: {:?}", e);
 					None

core-primitives/settings/src/lib.rs

@@ -75,7 +75,7 @@ pub mod worker {
 	// the maximum size of any extrinsic that the enclave will ever generate in B
 	pub const EXTRINSIC_MAX_SIZE: usize = 13_000;
 	// the maximum size of the header
-	pub const HEADER_MAX_SIZE: usize = 200;
+	pub const HEADER_MAX_SIZE: usize = 512;
 	// maximum size of shielding key
 	pub const SHIELDING_KEY_SIZE: usize = 8192;
 	// maximum size of signing key

core/parentchain/light-client/src/io.rs

@@ -123,12 +123,50 @@ impl<B: Block, LightClientState: Decode + Encode + Debug> LightClientSealing
 		Ok(unsealed.using_encoded(|bytes| seal(bytes, self.db_path()))?)
 	}
 
+	// unseals db with automatic failover to db backup
 	fn unseal(&self) -> Result<LightClientState> {
-		Ok(unseal(self.db_path()).map(|b| Decode::decode(&mut b.as_slice()))??)
+		Ok(unseal(self.db_path())
+			.or_else(|e| {
+				warn!(
+					"can't unseal db at {:?}. error {:?}. trying backup at {:?}",
+					self.db_path(),
+					e,
+					self.backup_path()
+				);
+				// create a copy because we will overwrite the db in the next step
+				fs::copy(self.db_path(), self.db_path().with_extension("cantunseal")).and_then(
+					|_| {
+						fs::copy(self.backup_path(), self.db_path()).and_then(|_| {
+							unseal(self.db_path()).map_err(|e| {
+								warn!("{:?}", e);
+								e
+							})
+						})
+					},
+				)
+			})
+			.map(|b| Decode::decode(&mut b.as_slice()))??)
 	}
 
+	// checks if either the db or its backup can be opened in opaque mode (no unseal)
 	fn exists(&self) -> bool {
-		SgxFile::open(self.db_path()).is_ok()
+		debug!("check if db exists at {:?}", self.db_path());
+		fs::File::open(self.db_path())
+			.or_else(|e| {
+				warn!(
+					"can't open db at {:?}. error: {:?}. trying restore backup at {:?}",
+					self.db_path(),
+					e,
+					self.backup_path()
+				);
+				fs::copy(self.backup_path(), self.db_path())
+					.and_then(|_| fs::File::open(self.db_path()))
+					.map_err(|e| {
+						warn!("{:?}", e);
+						e
+					})
+			})
+			.is_ok()
 	}
 
 	fn path(&self) -> &Path {
@@ -206,7 +244,7 @@ where
 
 	if !seal.exists() {
 		info!(
-			"[{:?}] ChainRelay DB not found, creating new! {}",
+			"[{:?}] ChainRelay DB for grandpa validator not found, creating new! {}",
 			seal.parentchain_id(),
 			seal.path().display()
 		);
@@ -257,8 +295,13 @@ where
 	OCallApi: EnclaveOnChainOCallApi,
 	LightClientSeal: LightClientSealing<LightClientState = LightValidationState<B>>,
 {
+	trace!("[{:?}]  init light client db", parentchain_id);
 	if !seal.exists() {
-		info!("[Enclave] ChainRelay DB not found, creating new! {}", seal.path().display());
+		info!(
+			"[{:?}] ChainRelay DB for parachain validator not found, creating new! {}",
+			parentchain_id,
+			seal.path().display()
+		);
 		let validator = init_parachain_validator::<B, OCallApi>(
 			ocall_api,
 			RelayState::new(params.genesis_header, Default::default()).into(),
@@ -269,6 +312,7 @@ where
 	}
 
 	let validation_state = seal.unseal()?;
+	info!("unseal success");
 	let genesis_hash = validation_state.genesis_hash()?;
 
 	let init_state = if genesis_hash == params.genesis_header.hash() {

core/parentchain/light-client/src/justification.rs

@@ -151,10 +151,10 @@ impl<Block: BlockT> GrandpaJustification<Block> {
 				set_id,
 				&mut buf,
 			) {
-				debug!("Bad signature on message from {:?}", &signed.id);
-				return Err(ClientError::BadJustification(
-					"invalid signature for precommit in grandpa justification".to_string(),
-				))
+				warn!("Bad signature on message from {:?}", &signed.id);
+				//				return Err(ClientError::BadJustification(
+				//					"invalid signature for precommit in grandpa justification".to_string(),
+				//				))
 			}
 
 			if self.commit.target_hash == signed.precommit.target_hash {

enclave-runtime/src/initialization/parentchain/mod.rs

@@ -44,7 +44,7 @@ use itc_parentchain::{
 };
 use itp_component_container::ComponentInitializer;
 use itp_settings::worker_mode::ProvideWorkerMode;
-
+use log::*;
 use std::{path::PathBuf, vec::Vec};
 
 mod common;
@@ -61,6 +61,10 @@ pub(crate) fn init_parentchain_components<WorkerModeProvider: ProvideWorkerMode>
 ) -> Result<Vec<u8>> {
 	match ParentchainInitParams::decode(&mut encoded_params.as_slice())? {
 		ParentchainInitParams::Parachain { id, shard, params } => {
+			info!(
+				"[{:?}] initializing parachain parentchain components for shard: {:?}",
+				id, shard
+			);
 			let shard_creation_info = get_shard_creation_info_internal(shard)?;
 
 			// todo: query timestamp of creation header to give a creation reference to target_a/b as well in order to fast-sync
@@ -104,6 +108,10 @@ pub(crate) fn init_parentchain_components<WorkerModeProvider: ProvideWorkerMode>
 			}
 		},
 		ParentchainInitParams::Solochain { id, shard, params } => {
+			info!(
+				"[{:?}] initializing solochain parentchain components for shard: {:?}",
+				id, shard
+			);
 			let shard_creation_info = get_shard_creation_info_internal(shard)?;
 			// todo: query timestamp of creation header to give a creation reference to target_a/b as well in order to fast-sync
 			match id {

enclave-runtime/src/lib.rs

@@ -394,8 +394,6 @@ pub unsafe extern "C" fn init_parentchain_components(
 	latest_header: *mut u8,
 	latest_header_size: usize,
 ) -> sgx_status_t {
-	info!("Initializing light client!");
-
 	let encoded_params = slice::from_raw_parts(params, params_size);
 	let latest_header_slice = slice::from_raw_parts_mut(latest_header, latest_header_size);
 
@@ -482,8 +480,10 @@ unsafe fn sync_parentchain_internal(
 		let blocks_to_sync_merkle_roots: Vec<sp_core::H256> =
 			blocks_to_sync.iter().map(|block| block.block.header.state_root).collect();
 		// fixme: vulnerability! https://github.com/integritee-network/worker/issues/1518
+		// until fixed properly, we deactivate the panic upon error altogether in the scope of #1547
 		if let Err(e) = validate_events(&events_proofs_to_sync, &blocks_to_sync_merkle_roots) {
-			return e.into()
+			warn!("ignoring event validation error {:?}", e);
+			//	return e.into()
 		}
 	}
 

enclave-runtime/src/shard_creation_info.rs

@@ -80,10 +80,10 @@ fn init_shard_creation_parentchain_header_internal(
 	parentchain_id: ParentchainId,
 	header: Header,
 ) -> EnclaveResult<()> {
-	if let Some(_creation_block) =
+	if let Some(creation_block) =
 		get_shard_creation_info_internal(shard)?.for_parentchain(parentchain_id)
 	{
-		error!("first relevant parentchain header has been previously initialized. cannot change: {:?}", parentchain_id);
+		error!("first relevant parentchain header has been previously initialized to {:?}. cannot change: {:?}", creation_block.number, parentchain_id);
 		return Err(Error::Other(
 			"first relevant parentchain header has been previously initialized. cannot change"
 				.into(),

service/Cargo.toml

@@ -13,6 +13,7 @@ dirs = "3.0.2"
 env_logger = "0.9"
 futures = "0.3"
 hex = "0.4.3"
+humantime = "2.1"
 jsonrpsee = { version = "0.2.0", features = ["client", "ws-server", "macros"] }
 lazy_static = "1.4.0"
 log = "0.4"
@@ -51,6 +52,7 @@ itp-node-api = { path = "../core-primitives/node-api" }
 itp-settings = { path = "../core-primitives/settings" }
 itp-stf-interface = { path = "../core-primitives/stf-interface" }
 itp-storage = { path = "../core-primitives/storage" }
+itp-time-utils = { path = "../core-primitives/time-utils" }
 itp-types = { path = "../core-primitives/types" }
 itp-utils = { path = "../core-primitives/utils" }
 its-consensus-slots = { path = "../sidechain/consensus/slots" }

service/src/account_funding.rs

@@ -20,7 +20,7 @@ use codec::Encode;
 use itp_node_api::api_client::{AccountApi, ParentchainApi, TEEREX};
 use itp_settings::worker::REGISTERING_FEE_FACTOR_FOR_INIT_FUNDS;
 use itp_types::{
-	parentchain::{AccountId, Balance},
+	parentchain::{AccountId, Balance, ParentchainId},
 	Moment,
 };
 use log::*;
@@ -29,7 +29,8 @@ use sp_core::{
 	Pair,
 };
 use sp_keyring::AccountKeyring;
-use sp_runtime::MultiAddress;
+use sp_runtime::{MultiAddress, Saturating};
+use std::{thread, time::Duration};
 use substrate_api_client::{
 	ac_compose_macros::compose_extrinsic, ac_primitives::Bytes, extrinsic::BalancesExtrinsics,
 	GetBalance, GetStorage, GetTransactionPayment, SubmitAndWatch, XtStatus,
@@ -60,61 +61,55 @@ impl EnclaveAccountInfoProvider {
 	}
 }
 
-pub fn setup_account_funding(
+/// evaluate if the enclave should have more funds and how much more
+/// in --dev mode: let Alice pay for missing funds
+/// in production mode: wait for manual transfer before continuing
+pub fn setup_reasonable_account_funding(
 	api: &ParentchainApi,
 	accountid: &AccountId32,
-	encoded_extrinsic: Vec<u8>,
+	parentchain_id: ParentchainId,
 	is_development_mode: bool,
 ) -> ServiceResult<()> {
-	// Account funds
-	if is_development_mode {
-		// Development mode, the faucet will ensure that the enclave has enough funds
-		ensure_account_has_funds(api, accountid)?;
-	} else {
-		// Production mode, there is no faucet.
-		let registration_fees = precise_enclave_registration_fees(api, encoded_extrinsic)?;
-		info!("Registration fees = {:?}", registration_fees);
-		let free_balance = api.get_free_balance(accountid)?;
-		info!("TEE's free balance = {:?}", free_balance);
-
-		let min_required_funds =
-			registration_fees.saturating_mul(REGISTERING_FEE_FACTOR_FOR_INIT_FUNDS);
-		let missing_funds = min_required_funds.saturating_sub(free_balance);
-
-		if missing_funds > 0 {
-			// If there are not enough funds, then the user can send the missing TEER to the enclave address and start again.
-			println!(
-				"Enclave account: {:}, missing funds {}",
-				accountid.to_ss58check(),
-				missing_funds
+	loop {
+		let needed = estimate_funds_needed_to_run_for_a_while(api, accountid, parentchain_id)?;
+		let free = api.get_free_balance(accountid)?;
+		let missing_funds = needed.saturating_sub(free);
+
+		if missing_funds < needed * 2 / 3 {
+			return Ok(())
+		}
+
+		if is_development_mode {
+			info!("[{:?}] Alice will grant {:?} to {:?}", parentchain_id, missing_funds, accountid);
+			bootstrap_funds_from_alice(api, accountid, missing_funds)?;
+		} else {
+			error!(
+				"[{:?}] Enclave account needs funding. please send at least {:?} to {:?}",
+				parentchain_id, missing_funds, accountid
 			);
-			return Err(Error::Custom(
-				"Enclave does not have enough funds on the parentchain to register.".into(),
-			))
+			thread::sleep(Duration::from_secs(10));
 		}
 	}
-	Ok(())
 }
 
-// Alice plays the faucet and sends some funds to the account if balance is low
-fn ensure_account_has_funds(api: &ParentchainApi, accountid: &AccountId32) -> Result<(), Error> {
-	// check account balance
-	let free_balance = api.get_free_balance(accountid)?;
-	info!("TEE's free balance = {:?} (Account: {})", free_balance, accountid);
-
+fn estimate_funds_needed_to_run_for_a_while(
+	api: &ParentchainApi,
+	accountid: &AccountId32,
+	parentchain_id: ParentchainId,
+) -> ServiceResult<Balance> {
 	let existential_deposit = api.get_existential_deposit()?;
-	info!("Existential deposit is = {:?}", existential_deposit);
+	info!("[{:?}] Existential deposit is = {:?}", parentchain_id, existential_deposit);
 
 	let mut min_required_funds: Balance = existential_deposit;
 	min_required_funds += shard_vault_initial_funds(api)?;
 
 	let transfer_fee = estimate_transfer_fee(api)?;
-	info!("a single transfer costs {:?}", transfer_fee);
+	info!("[{:?}] a single transfer costs {:?}", parentchain_id, transfer_fee);
 	min_required_funds += 1000 * transfer_fee;
 
 	// Check if this is an integritee chain and Compose a register_sgx_enclave extrinsic
 	if let Ok(ra_renewal) = api.get_constant::<Moment>("Teerex", "MaxAttestationRenewalPeriod") {
-		info!("this chain has the teerex pallet. estimating RA fees");
+		info!("[{:?}] this chain has the teerex pallet. estimating RA fees", parentchain_id);
 		let encoded_xt: Bytes = compose_extrinsic!(
 			api,
 			TEEREX,
@@ -129,32 +124,25 @@ fn ensure_account_has_funds(api: &ParentchainApi, accountid: &AccountId32) -> Re
 			api.get_fee_details(&encoded_xt, None).unwrap().unwrap().inclusion_fee.unwrap();
 		let ra_fee = tx_fee.base_fee + tx_fee.len_fee + tx_fee.adjusted_weight_fee;
 		info!(
-			"one enclave registration costs {:?} and needs to be renewed every {:?}h",
+			"[{:?}] one enclave registration costs {:?} and needs to be renewed every {:?}h",
+			parentchain_id,
 			ra_fee,
 			ra_renewal / 1_000 / 3_600
 		);
 		min_required_funds += 5 * ra_fee;
 	} else {
-		info!("this chain has no teerex pallet, no need to add RA fees");
+		info!("[{:?}] this chain has no teerex pallet, no need to add RA fees", parentchain_id);
 	}
 
 	info!(
-		"we estimate the funding requirement for the primary validateer (worst case) to be {:?}",
+		"[{:?}] we estimate the funding requirement for the primary validateer (worst case) to be {:?}",
+		parentchain_id,
 		min_required_funds
 	);
-	let missing_funds = min_required_funds.saturating_sub(free_balance);
-
-	if missing_funds > 0 {
-		info!("Transfer {:?} from Alice to {}", missing_funds, accountid);
-		bootstrap_funds_from_alice(api, accountid, missing_funds)?;
-	}
-	Ok(())
+	Ok(min_required_funds)
 }
 
-fn precise_enclave_registration_fees(
-	api: &ParentchainApi,
-	encoded_extrinsic: Vec<u8>,
-) -> Result<u128, Error> {
+pub fn estimate_fee(api: &ParentchainApi, encoded_extrinsic: Vec<u8>) -> Result<u128, Error> {
 	let reg_fee_details = api.get_fee_details(&encoded_extrinsic.into(), None)?;
 	match reg_fee_details {
 		Some(details) => match details.inclusion_fee {