Merge remote-tracking branch 'origin/master' into bko-validate-xcm-nesting

Author: bkontur

.gitlab/pipeline/publish.yml

@@ -74,6 +74,8 @@ publish-subsystem-benchmarks:
       artifacts: true
     - job: subsystem-benchmark-availability-distribution
       artifacts: true
+    - job: subsystem-benchmark-approval-voting
+      artifacts: true
     - job: publish-rustdoc
       artifacts: false
   script:
@@ -115,6 +117,8 @@ trigger_workflow:
       artifacts: true
     - job: subsystem-benchmark-availability-distribution
       artifacts: true
+    - job: subsystem-benchmark-approval-voting
+      artifacts: true
   script:
     - echo "Triggering workflow"
     - >

.gitlab/pipeline/test.yml

@@ -511,7 +511,7 @@ test-syscalls:
       fi
   allow_failure: false # this rarely triggers in practice
 
-subsystem-benchmark-availability-recovery:
+.subsystem-benchmark-template:
   stage: test
   artifacts:
     name: "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}"
@@ -523,26 +523,26 @@ subsystem-benchmark-availability-recovery:
     - .docker-env
     - .common-refs
     - .run-immediately
-  script:
-    - cargo bench -p polkadot-availability-recovery --bench availability-recovery-regression-bench --features subsystem-benchmarks
   tags:
     - benchmark
+
+subsystem-benchmark-availability-recovery:
+  extends:
+    - .subsystem-benchmark-template
+  script:
+    - cargo bench -p polkadot-availability-recovery --bench availability-recovery-regression-bench --features subsystem-benchmarks
   allow_failure: true
 
 subsystem-benchmark-availability-distribution:
-  stage: test
-  artifacts:
-    name: "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}"
-    when: always
-    expire_in: 1 hour
-    paths:
-      - charts/
   extends:
-    - .docker-env
-    - .common-refs
-    - .run-immediately
+    - .subsystem-benchmark-template
   script:
     - cargo bench -p polkadot-availability-distribution --bench availability-distribution-regression-bench --features subsystem-benchmarks
-  tags:
-    - benchmark
+  allow_failure: true
+
+subsystem-benchmark-approval-voting:
+  extends:
+    - .subsystem-benchmark-template
+  script:
+    - cargo bench -p polkadot-node-core-approval-voting --bench approval-voting-regression-bench --features subsystem-benchmarks
   allow_failure: true

Cargo.lock

@@ -51,9 +51,9 @@ use xcm_executor::XcmExecutor;
 
 parameter_types! {
 	pub const RelayLocation: Location = Location::parent();
-	pub const RelayNetwork: Option<NetworkId> = None;
+	pub const RelayNetwork: NetworkId = NetworkId::Rococo;
 	pub RelayChainOrigin: RuntimeOrigin = cumulus_pallet_xcm::Origin::Relay.into();
-	pub UniversalLocation: InteriorLocation = Parachain(ParachainInfo::parachain_id().into()).into();
+	pub UniversalLocation: InteriorLocation = [GlobalConsensus(RelayNetwork::get()), Parachain(ParachainInfo::parachain_id().into())].into();
 	pub const ExecutiveBody: BodyId = BodyId::Executive;
 	pub TreasuryAccount: AccountId = TREASURY_PALLET_ID.into_account_truncating();
 	pub RelayTreasuryLocation: Location = (Parent, PalletInstance(rococo_runtime_constants::TREASURY_PALLET_ID)).into();

cumulus/parachains/runtimes/contracts/contracts-rococo/src/xcm_config.rs

@@ -30,8 +30,8 @@ use xcm_builder::{
 
 parameter_types! {
 	pub const WestendLocation: Location = Location::parent();
-	pub const WestendNetwork: Option<NetworkId> = Some(NetworkId::Westend);
-	pub UniversalLocation: InteriorLocation = [Parachain(ParachainInfo::parachain_id().into())].into();
+	pub const WestendNetwork: NetworkId = NetworkId::Westend;
+	pub UniversalLocation: InteriorLocation = [GlobalConsensus(WestendNetwork::get()), Parachain(ParachainInfo::parachain_id().into())].into();
 }
 
 /// This is the type we use to convert an (incoming) XCM origin into a local `Origin` instance,

cumulus/parachains/runtimes/glutton/glutton-westend/src/xcm_config.rs

@@ -30,8 +30,8 @@ use xcm_builder::{
 
 parameter_types! {
 	pub const RococoLocation: Location = Location::parent();
-	pub const RococoNetwork: Option<NetworkId> = Some(NetworkId::Rococo);
-	pub UniversalLocation: InteriorLocation = [Parachain(ParachainInfo::parachain_id().into())].into();
+	pub const RococoNetwork: NetworkId = NetworkId::Rococo;
+	pub UniversalLocation: InteriorLocation = [GlobalConsensus(RococoNetwork::get()), Parachain(ParachainInfo::parachain_id().into())].into();
 }
 
 /// This is the type we use to convert an (incoming) XCM origin into a local `Origin` instance,

cumulus/parachains/runtimes/starters/shell/src/xcm_config.rs

@@ -327,9 +327,9 @@ impl cumulus_pallet_aura_ext::Config for Runtime {}
 
 parameter_types! {
 	pub const RocLocation: Location = Location::parent();
-	pub const RococoNetwork: Option<NetworkId> = Some(NetworkId::Rococo);
+	pub const RococoNetwork: NetworkId = NetworkId::Rococo;
 	pub RelayChainOrigin: RuntimeOrigin = cumulus_pallet_xcm::Origin::Relay.into();
-	pub UniversalLocation: InteriorLocation = [Parachain(ParachainInfo::parachain_id().into())].into();
+	pub UniversalLocation: InteriorLocation = [GlobalConsensus(RococoNetwork::get()), Parachain(ParachainInfo::parachain_id().into())].into();
 	pub CheckingAccount: AccountId = PolkadotXcm::check_account();
 }
 

cumulus/parachains/runtimes/testing/rococo-parachain/src/lib.rs

@@ -53,3 +53,14 @@ kvdb-memorydb = "0.13.0"
 test-helpers = { package = "polkadot-primitives-test-helpers", path = "../../../primitives/test-helpers" }
 log = { workspace = true, default-features = true }
 env_logger = "0.11"
+
+polkadot-subsystem-bench = { path = "../../subsystem-bench" }
+
+[[bench]]
+name = "approval-voting-regression-bench"
+path = "benches/approval-voting-regression-bench.rs"
+harness = false
+required-features = ["subsystem-benchmarks"]
+
+[features]
+subsystem-benchmarks = []

polkadot/node/core/approval-voting/Cargo.toml

@@ -0,0 +1,94 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+//! approval-voting throughput test
+//!
+//! Approval Voting benchmark based on Kusama parameters and scale.
+//!
+//! Subsystems involved:
+//! - approval-distribution
+//! - approval-voting
+
+use polkadot_subsystem_bench::{
+	self,
+	approval::{bench_approvals, prepare_test, ApprovalsOptions},
+	configuration::TestConfiguration,
+	usage::BenchmarkUsage,
+	utils::save_to_file,
+};
+use std::io::Write;
+
+const BENCH_COUNT: usize = 10;
+
+fn main() -> Result<(), String> {
+	let mut messages = vec![];
+	let mut config = TestConfiguration::default();
+	config.n_cores = 100;
+	config.n_validators = 500;
+	config.num_blocks = 10;
+	config.peer_bandwidth = 524288000000;
+	config.bandwidth = 524288000000;
+	config.latency = None;
+	config.connectivity = 100;
+	config.generate_pov_sizes();
+	let options = ApprovalsOptions {
+		last_considered_tranche: 89,
+		coalesce_mean: 3.0,
+		coalesce_std_dev: 1.0,
+		coalesce_tranche_diff: 12,
+		enable_assignments_v2: true,
+		stop_when_approved: false,
+		workdir_prefix: "/tmp".to_string(),
+		num_no_shows_per_candidate: 0,
+	};
+
+	println!("Benchmarking...");
+	let usages: Vec<BenchmarkUsage> = (0..BENCH_COUNT)
+		.map(|n| {
+			print!("\r[{}{}]", "#".repeat(n), "_".repeat(BENCH_COUNT - n));
+			std::io::stdout().flush().unwrap();
+			let (mut env, state) = prepare_test(config.clone(), options.clone(), false);
+			env.runtime().block_on(bench_approvals("approvals_throughput", &mut env, state))
+		})
+		.collect();
+	println!("\rDone!{}", " ".repeat(BENCH_COUNT));
+
+	let average_usage = BenchmarkUsage::average(&usages);
+	save_to_file(
+		"charts/availability-distribution-regression-bench.json",
+		average_usage.to_chart_json().map_err(|e| e.to_string())?,
+	)
+	.map_err(|e| e.to_string())?;
+	println!("{}", average_usage);
+
+	// We expect no variance for received and sent
+	// but use 0.001 because we operate with floats
+	messages.extend(average_usage.check_network_usage(&[
+		("Received from peers", 52944.7000, 0.001),
+		("Sent to peers", 63532.2000, 0.001),
+	]));
+	messages.extend(average_usage.check_cpu_usage(&[
+		("approval-distribution", 7.7883, 0.1),
+		("approval-voting", 10.4655, 0.1),
+	]));
+
+	if messages.is_empty() {
+		Ok(())
+	} else {
+		eprintln!("{}", messages.join("\n"));
+		Err("Regressions found".to_string())
+	}
+}

polkadot/node/core/approval-voting/benches/approval-voting-regression-bench.rs

@@ -29,7 +29,6 @@ log = { workspace = true, default-features = true }
 [dev-dependencies]
 pallet-balances = { path = "../../../substrate/frame/balances" }
 pallet-assets = { path = "../../../substrate/frame/assets" }
-sp-core = { path = "../../../substrate/primitives/core" }
 sp-tracing = { path = "../../../substrate/primitives/tracing" }
 xcm = { package = "staging-xcm", path = ".." }
 # temp