Merge branch 'master' into rococo-experimental

* master:
  double the allocator limit (#9102)
  Make it possible to override maximum payload of RPC (#9019)
  Aura: Skip initialize block & remove cache (#9132)
  Avoid running some test 2 times when unneeded (#9124)

Author: Andronik Ordian

.gitlab-ci.yml

@@ -319,7 +319,7 @@ test-linux-stable:                 &test-linux
   script:
     # this job runs all tests in former runtime-benchmarks, frame-staking and wasmtime tests
     - time cargo test --workspace --locked --release --verbose --features runtime-benchmarks --manifest-path bin/node/cli/Cargo.toml
-    - time cargo test -p frame-support-test --features=conditional-storage --manifest-path frame/support/test/Cargo.toml # does not reuse cache 1 min 44 sec
+    - time cargo test -p frame-support-test --features=conditional-storage --manifest-path frame/support/test/Cargo.toml --test pallet # does not reuse cache 1 min 44 sec
     - SUBSTRATE_TEST_TIMEOUT=1 time cargo test -p substrate-test-utils --release --verbose --locked -- --ignored timeout
     - sccache -s
 

client/cli/src/commands/run_cmd.rs

@@ -42,12 +42,11 @@ pub struct RunCmd {
 	/// The node will be started with the authority role and actively
 	/// participate in any consensus task that it can (e.g. depending on
 	/// availability of local keys).
-	#[structopt(
-		long = "validator"
-	)]
+	#[structopt(long)]
 	pub validator: bool,
 
-	/// Disable GRANDPA voter when running in validator mode, otherwise disable the GRANDPA observer.
+	/// Disable GRANDPA voter when running in validator mode, otherwise disable the GRANDPA
+	/// observer.
 	#[structopt(long)]
 	pub no_grandpa: bool,
 
@@ -57,8 +56,8 @@ pub struct RunCmd {
 
 	/// Listen to all RPC interfaces.
 	///
-	/// Default is local. Note: not all RPC methods are safe to be exposed publicly. Use an RPC proxy
-	/// server to filter out dangerous methods. More details:
+	/// Default is local. Note: not all RPC methods are safe to be exposed publicly. Use an RPC
+	/// proxy server to filter out dangerous methods. More details:
 	/// <https://github.com/paritytech/substrate/wiki/Public-RPC>.
 	/// Use `--unsafe-rpc-external` to suppress the warning if you understand the risks.
 	#[structopt(long = "rpc-external")]
@@ -74,8 +73,8 @@ pub struct RunCmd {
 	///
 	/// - `Unsafe`: Exposes every RPC method.
 	/// - `Safe`: Exposes only a safe subset of RPC methods, denying unsafe RPC methods.
-	/// - `Auto`: Acts as `Safe` if RPC is served externally, e.g. when `--{rpc,ws}-external` is passed,
-	///   otherwise acts as `Unsafe`.
+	/// - `Auto`: Acts as `Safe` if RPC is served externally, e.g. when `--{rpc,ws}-external` is
+	///   passed, otherwise acts as `Unsafe`.
 	#[structopt(
 		long,
 		value_name = "METHOD SET",
@@ -88,8 +87,9 @@ pub struct RunCmd {
 
 	/// Listen to all Websocket interfaces.
 	///
-	/// Default is local. Note: not all RPC methods are safe to be exposed publicly. Use an RPC proxy
-	/// server to filter out dangerous methods. More details: <https://github.com/paritytech/substrate/wiki/Public-RPC>.
+	/// Default is local. Note: not all RPC methods are safe to be exposed publicly. Use an RPC
+	/// proxy server to filter out dangerous methods. More details:
+	/// <https://github.com/paritytech/substrate/wiki/Public-RPC>.
 	/// Use `--unsafe-ws-external` to suppress the warning if you understand the risks.
 	#[structopt(long = "ws-external")]
 	pub ws_external: bool,
@@ -100,6 +100,11 @@ pub struct RunCmd {
 	#[structopt(long = "unsafe-ws-external")]
 	pub unsafe_ws_external: bool,
 
+	/// Set the the maximum RPC payload size for both requests and responses (both http and ws), in
+	/// megabytes. Default is 15MiB.
+	#[structopt(long = "rpc-max-payload")]
+	pub rpc_max_payload: Option<usize>,
+
 	/// Listen to all Prometheus data source interfaces.
 	///
 	/// Default is local.
@@ -194,7 +199,8 @@ pub struct RunCmd {
 	#[structopt(long, conflicts_with_all = &["alice", "charlie", "dave", "eve", "ferdie", "one", "two"])]
 	pub bob: bool,
 
-	/// Shortcut for `--name Charlie --validator` with session keys for `Charlie` added to keystore.
+	/// Shortcut for `--name Charlie --validator` with session keys for `Charlie` added to
+	/// keystore.
 	#[structopt(long, conflicts_with_all = &["alice", "bob", "dave", "eve", "ferdie", "one", "two"])]
 	pub charlie: bool,
 
@@ -435,6 +441,10 @@ impl CliConfiguration for RunCmd {
 		Ok(self.rpc_methods.into())
 	}
 
+	fn rpc_max_payload(&self) -> Result<Option<usize>> {
+		Ok(self.rpc_max_payload)
+	}
+
 	fn transaction_pool(&self) -> Result<TransactionPoolOptions> {
 		Ok(self.pool_config.transaction_pool())
 	}

client/cli/src/config.rs

@@ -372,6 +372,11 @@ pub trait CliConfiguration<DCV: DefaultConfigurationValues = ()>: Sized {
 		Ok(Some(Vec::new()))
 	}
 
+	/// Get maximum RPC payload.
+	fn rpc_max_payload(&self) -> Result<Option<usize>> {
+		Ok(None)
+	}
+
 	/// Get the prometheus configuration (`None` if disabled)
 	///
 	/// By default this is `None`.
@@ -535,6 +540,7 @@ pub trait CliConfiguration<DCV: DefaultConfigurationValues = ()>: Sized {
 			rpc_ws_max_connections: self.rpc_ws_max_connections()?,
 			rpc_http_threads: self.rpc_http_threads()?,
 			rpc_cors: self.rpc_cors(is_dev)?,
+			rpc_max_payload: self.rpc_max_payload()?,
 			prometheus_config: self.prometheus_config(DCV::prometheus_listen_port())?,
 			telemetry_endpoints,
 			telemetry_external_transport: self.telemetry_external_transport()?,

client/consensus/aura/src/import_queue.rs

@@ -317,43 +317,6 @@ impl<B: BlockT, C, P, CAW, CIDP> Verifier<B> for AuraVerifier<C, P, CAW, CIDP> w
 	}
 }
 
-fn initialize_authorities_cache<A, B, C>(client: &C) -> Result<(), ConsensusError> where
-	A: Codec + Debug,
-	B: BlockT,
-	C: ProvideRuntimeApi<B> + BlockOf + ProvideCache<B> + UsageProvider<B>,
-	C::Api: AuraApi<B, A>,
-{
-	// no cache => no initialization
-	let cache = match client.cache() {
-		Some(cache) => cache,
-		None => return Ok(()),
-	};
-
-	let best_hash = client.usage_info().chain.best_hash;
-
-	// check if we already have initialized the cache
-	let map_err = |error| sp_consensus::Error::from(sp_consensus::Error::ClientImport(
-		format!(
-			"Error initializing authorities cache: {}",
-			error,
-		)));
-
-	let block_id = BlockId::hash(best_hash);
-	let authorities: Option<Vec<A>> = cache
-		.get_at(&well_known_cache_keys::AUTHORITIES, &block_id)
-		.unwrap_or(None)
-		.and_then(|(_, _, v)| Decode::decode(&mut &v[..]).ok());
-	if authorities.is_some() {
-		return Ok(());
-	}
-
-	let authorities = crate::authorities(client, &block_id)?;
-	cache.initialize(&well_known_cache_keys::AUTHORITIES, authorities.encode())
-		.map_err(map_err)?;
-
-	Ok(())
-}
-
 /// Should we check for equivocation of a block author?
 #[derive(Debug, Clone, Copy)]
 pub enum CheckForEquivocation {
@@ -438,8 +401,6 @@ pub fn import_queue<'a, P, Block, I, C, S, CAW, CIDP>(
 	CIDP: CreateInherentDataProviders<Block, ()> + Sync + Send + 'static,
 	CIDP::InherentDataProviders: InherentDataProviderExt + Send + Sync,
 {
-	initialize_authorities_cache(&*client)?;
-
 	let verifier = build_verifier::<P, _, _, _>(
 		BuildVerifierParams {
 			client,

client/consensus/aura/src/lib.rs

@@ -45,7 +45,7 @@ use sp_consensus::{
 	BlockOrigin, Error as ConsensusError, SelectChain,
 };
 use sc_client_api::{backend::AuxStore, BlockOf, UsageProvider};
-use sp_blockchain::{Result as CResult, well_known_cache_keys, ProvideCache, HeaderBackend};
+use sp_blockchain::{Result as CResult, ProvideCache, HeaderBackend};
 use sp_core::crypto::Public;
 use sp_application_crypto::{AppKey, AppPublic};
 use sp_runtime::{generic::BlockId, traits::NumberFor};
@@ -546,14 +546,9 @@ fn authorities<A, B, C>(client: &C, at: &BlockId<B>) -> Result<Vec<A>, Consensus
 	C: ProvideRuntimeApi<B> + BlockOf + ProvideCache<B>,
 	C::Api: AuraApi<B, A>,
 {
-	client
-		.cache()
-		.and_then(|cache| cache
-			.get_at(&well_known_cache_keys::AUTHORITIES, at)
-			.unwrap_or(None)
-			.and_then(|(_, _, v)| Decode::decode(&mut &v[..]).ok())
-		)
-		.or_else(|| AuraApi::authorities(&*client.runtime_api(), at).ok())
+	client.runtime_api()
+		.authorities(at)
+		.ok()
 		.ok_or_else(|| sp_consensus::Error::InvalidAuthoritiesSet.into())
 }
 

client/executor/src/native_executor.rs

@@ -291,7 +291,7 @@ impl<D: NativeExecutionDispatch> NativeExecutor<D> {
 		default_heap_pages: Option<u64>,
 		max_runtime_instances: usize,
 	) -> Self {
-		let extended =  D::ExtendHostFunctions::host_functions();
+		let extended = D::ExtendHostFunctions::host_functions();
 		let mut host_functions = sp_io::SubstrateHostFunctions::host_functions()
 			.into_iter()
 			// filter out any host function overrides provided.

client/rpc-servers/src/lib.rs

@@ -27,8 +27,10 @@ use jsonrpc_core::{IoHandlerExtension, MetaIoHandler};
 use log::error;
 use pubsub::PubSubMetadata;
 
+const MEGABYTE: usize = 1024 * 1024;
+
 /// Maximal payload accepted by RPC servers.
-pub const MAX_PAYLOAD: usize = 15 * 1024 * 1024;
+pub const RPC_MAX_PAYLOAD_DEFAULT: usize = 15 * MEGABYTE;
 
 /// Default maximum number of connections for WS RPC servers.
 const WS_MAX_CONNECTIONS: usize = 100;
@@ -85,7 +87,10 @@ mod inner {
 		thread_pool_size: Option<usize>,
 		cors: Option<&Vec<String>>,
 		io: RpcHandler<M>,
+		maybe_max_payload_mb: Option<usize>,
 	) -> io::Result<http::Server> {
+		let max_request_body_size = maybe_max_payload_mb.map(|mb| mb.saturating_mul(MEGABYTE))
+			.unwrap_or(RPC_MAX_PAYLOAD_DEFAULT);
 		http::ServerBuilder::new(io)
 			.threads(thread_pool_size.unwrap_or(HTTP_THREADS))
 			.health_api(("/health", "system_health"))
@@ -96,7 +101,7 @@ mod inner {
 				http::RestApi::Unsecure
 			})
 			.cors(map_cors::<http::AccessControlAllowOrigin>(cors))
-			.max_request_body_size(MAX_PAYLOAD)
+			.max_request_body_size(max_request_body_size)
 			.start_http(addr)
 	}
 
@@ -120,14 +125,19 @@ mod inner {
 	/// Start WS server listening on given address.
 	///
 	/// **Note**: Only available if `not(target_os = "unknown")`.
-	pub fn start_ws<M: pubsub::PubSubMetadata + From<jsonrpc_core::futures::sync::mpsc::Sender<String>>> (
+	pub fn start_ws<
+		M: pubsub::PubSubMetadata + From<jsonrpc_core::futures::sync::mpsc::Sender<String>>,
+	>(
 		addr: &std::net::SocketAddr,
 		max_connections: Option<usize>,
 		cors: Option<&Vec<String>>,
 		io: RpcHandler<M>,
+		maybe_max_payload_mb: Option<usize>,
 	) -> io::Result<ws::Server> {
+		let rpc_max_payload = maybe_max_payload_mb.map(|mb| mb.saturating_mul(MEGABYTE))
+			.unwrap_or(RPC_MAX_PAYLOAD_DEFAULT);
 		ws::ServerBuilder::with_meta_extractor(io, |context: &ws::RequestContext| context.sender().into())
-			.max_payload(MAX_PAYLOAD)
+			.max_payload(rpc_max_payload)
 			.max_connections(max_connections.unwrap_or(WS_MAX_CONNECTIONS))
 			.allowed_origins(map_cors(cors))
 			.allowed_hosts(hosts_filtering(cors.is_some()))

client/rpc/src/state/mod.rs

@@ -182,6 +182,7 @@ pub fn new_full<BE, Block: BlockT, Client>(
 	client: Arc<Client>,
 	subscriptions: SubscriptionManager,
 	deny_unsafe: DenyUnsafe,
+	rpc_max_payload: Option<usize>,
 ) -> (State<Block, Client>, ChildState<Block, Client>)
 	where
 		Block: BlockT + 'static,
@@ -193,9 +194,11 @@ pub fn new_full<BE, Block: BlockT, Client>(
 		Client::Api: Metadata<Block>,
 {
 	let child_backend = Box::new(
-		self::state_full::FullState::new(client.clone(), subscriptions.clone())
+		self::state_full::FullState::new(
+			client.clone(), subscriptions.clone(), rpc_max_payload
+		)
 	);
-	let backend = Box::new(self::state_full::FullState::new(client, subscriptions));
+	let backend = Box::new(self::state_full::FullState::new(client, subscriptions, rpc_max_payload));
 	(State { backend, deny_unsafe }, ChildState { backend: child_backend })
 }
 

client/rpc/src/state/state_full.rs

@@ -67,7 +67,8 @@ struct QueryStorageRange<Block: BlockT> {
 pub struct FullState<BE, Block: BlockT, Client> {
 	client: Arc<Client>,
 	subscriptions: SubscriptionManager,
-	_phantom: PhantomData<(BE, Block)>
+	_phantom: PhantomData<(BE, Block)>,
+	rpc_max_payload: Option<usize>,
 }
 
 impl<BE, Block: BlockT, Client> FullState<BE, Block, Client>
@@ -78,8 +79,12 @@ impl<BE, Block: BlockT, Client> FullState<BE, Block, Client>
 		Block: BlockT + 'static,
 {
 	/// Create new state API backend for full nodes.
-	pub fn new(client: Arc<Client>, subscriptions: SubscriptionManager) -> Self {
-		Self { client, subscriptions, _phantom: PhantomData }
+	pub fn new(
+		client: Arc<Client>,
+		subscriptions: SubscriptionManager,
+		rpc_max_payload: Option<usize>,
+	) -> Self {
+		Self { client, subscriptions, _phantom: PhantomData, rpc_max_payload }
 	}
 
 	/// Returns given block hash or best block hash if None is passed.
@@ -540,9 +545,15 @@ impl<BE, Block, Client> StateBackend<Block, Client> for FullState<BE, Block, Cli
 		targets: Option<String>,
 		storage_keys: Option<String>,
 	) -> FutureResult<sp_rpc::tracing::TraceBlockResponse> {
+		let block_executor = sc_tracing::block::BlockExecutor::new(
+			self.client.clone(),
+			block,
+			targets,
+			storage_keys,
+			self.rpc_max_payload,
+		);
 		Box::new(result(
-			sc_tracing::block::BlockExecutor::new(self.client.clone(), block, targets, storage_keys)
-				.trace_block()
+			block_executor.trace_block()
 				.map_err(|e| invalid_block::<Block>(block, None, e.to_string()))
 		))
 	}

client/rpc/src/state/tests.rs

@@ -63,6 +63,7 @@ fn should_return_storage() {
 		Arc::new(client),
 		SubscriptionManager::new(Arc::new(TaskExecutor)),
 		DenyUnsafe::No,
+		None,
 	);
 	let key = StorageKey(KEY.to_vec());
 
@@ -105,6 +106,7 @@ fn should_return_child_storage() {
 		client,
 		SubscriptionManager::new(Arc::new(TaskExecutor)),
 		DenyUnsafe::No,
+		None,
 	);
 	let child_key = prefixed_storage_key();
 	let key = StorageKey(b"key".to_vec());
@@ -144,6 +146,7 @@ fn should_call_contract() {
 		client,
 		SubscriptionManager::new(Arc::new(TaskExecutor)),
 		DenyUnsafe::No,
+		None,
 	);
 
 	assert_matches!(
@@ -162,6 +165,7 @@ fn should_notify_about_storage_changes() {
 			client.clone(),
 			SubscriptionManager::new(Arc::new(TaskExecutor)),
 			DenyUnsafe::No,
+			None,
 		);
 
 		api.subscribe_storage(Default::default(), subscriber, None.into());
@@ -200,6 +204,7 @@ fn should_send_initial_storage_changes_and_notifications() {
 			client.clone(),
 			SubscriptionManager::new(Arc::new(TaskExecutor)),
 			DenyUnsafe::No,
+			None,
 		);
 
 		let alice_balance_key = blake2_256(&runtime::system::balance_of_key(AccountKeyring::Alice.into()));
@@ -242,6 +247,7 @@ fn should_query_storage() {
 			client.clone(),
 			SubscriptionManager::new(Arc::new(TaskExecutor)),
 			DenyUnsafe::No,
+			None,
 		);
 
 		let mut add_block = |nonce| {
@@ -463,6 +469,7 @@ fn should_return_runtime_version() {
 		client.clone(),
 		SubscriptionManager::new(Arc::new(TaskExecutor)),
 		DenyUnsafe::No,
+		None,
 	);
 
 	let result = "{\"specName\":\"test\",\"implName\":\"parity-test\",\"authoringVersion\":1,\
@@ -490,6 +497,7 @@ fn should_notify_on_runtime_version_initially() {
 			client.clone(),
 			SubscriptionManager::new(Arc::new(TaskExecutor)),
 			DenyUnsafe::No,
+			None,
 		);
 
 		api.subscribe_runtime_version(Default::default(), subscriber);