Make basic collation working

collator/Cargo.toml

@@ -7,6 +7,9 @@ edition = "2018"
 [dependencies]
 # Substrate dependencies
 sr-primitives = { package = "sr-primitives", git = "https://github.com/paritytech/substrate", branch = "bkchr-cumulus-branch" }
+substrate-primitives = { git = "https://github.com/paritytech/substrate", branch = "bkchr-cumulus-branch" }
+substrate-client = { git = "https://github.com/paritytech/substrate", branch = "bkchr-cumulus-branch" }
+substrate-service = { git = "https://github.com/paritytech/substrate", branch = "bkchr-cumulus-branch" }
 consensus-common = { package = "substrate-consensus-common", git = "https://github.com/paritytech/substrate", branch = "bkchr-cumulus-branch" }
 inherents = { package = "substrate-inherents", git = "https://github.com/paritytech/substrate", branch = "bkchr-cumulus-branch" }
 cli = { package = "substrate-cli", git = "https://github.com/paritytech/substrate", branch = "bkchr-cumulus-branch" }
@@ -15,6 +18,9 @@ cli = { package = "substrate-cli", git = "https://github.com/paritytech/substrat
 polkadot-collator = { git = "https://github.com/paritytech/polkadot", branch = "bkchr-cumulus-branch" }
 polkadot-primitives = { git = "https://github.com/paritytech/polkadot", branch = "bkchr-cumulus-branch" }
 
+# Cumulus dependencies
+cumulus-consensus = { path = "../consensus" }
+
 # other deps
 log = "0.4.8"
 codec = { package = "parity-scale-codec", version = "1.0.6", features = [ "derive" ] }

collator/src/lib.rs

@@ -19,12 +19,14 @@
 use sr_primitives::traits::Block as BlockT;
 use consensus_common::{Environment, Proposer};
 use inherents::InherentDataProviders;
+use substrate_primitives::Blake2Hasher;
 
 use polkadot_collator::{
 	InvalidHead, ParachainContext, BuildParachainContext, Network as CollatorNetwork, VersionInfo,
+	TaskExecutor, PolkadotClient,
 };
 use polkadot_primitives::{
-	Hash,
+	Hash as PHash, Block as PBlock,
 	parachain::{
 		self, BlockData, Message, Id as ParaId, OutgoingMessages, Status as ParachainStatus,
 		CollatorPair,
@@ -33,12 +35,12 @@ use polkadot_primitives::{
 
 use codec::{Decode, Encode};
 
-use log::error;
+use log::{error, trace};
 
 use futures03::TryFutureExt;
 use futures::{Future, future::IntoFuture};
 
-use std::{sync::Arc, marker::PhantomData, time::Duration};
+use std::{sync::Arc, marker::PhantomData, time::Duration, fmt::Debug};
 
 use parking_lot::Mutex;
 
@@ -85,121 +87,153 @@ impl<Block, PF> Clone for Collator<Block, PF> {
 
 impl<Block, PF> ParachainContext for Collator<Block, PF> where
 	Block: BlockT,
-	PF: Environment<Block> + 'static + Send + Sync,
-	PF::Error: std::fmt::Debug,
-	PF::Proposer: Send + Sync,
-	<PF::Proposer as Proposer<Block>>::Create: Unpin + Send + Sync,
+	PF: Environment<Block> + 'static + Send,
+	PF::Error: Debug,
+	<PF::Proposer as Proposer<Block>>::Create: Unpin + Send,
 {
 	type ProduceCandidate = Box<
 		dyn Future<Item=(BlockData, parachain::HeadData, OutgoingMessages), Error=InvalidHead>
-			+ Send + Sync
+			+ Send
 	>;
 
 	fn produce_candidate<I: IntoIterator<Item=(ParaId, Message)>>(
 		&mut self,
-		_relay_chain_parent: Hash,
+		_relay_chain_parent: PHash,
 		status: ParachainStatus,
 		_: I,
 	) -> Self::ProduceCandidate {
+		trace!(target: "cumulus-collator", "Producing candidate");
+
 		let factory = self.proposer_factory.clone();
 		let inherent_providers = self.inherent_data_providers.clone();
 
 		let res = HeadData::<Block>::decode(&mut &status.head_data.0[..])
 			.map_err(|_| InvalidHead)
 			.into_future()
-			.and_then(move |last_head|
+			.and_then(move |last_head| {
 				factory.lock()
 					.init(&last_head.header)
 					.map_err(|e| {
 						//TODO: Do we want to return the real error?
 						error!("Could not create proposer: {:?}", e);
 						InvalidHead
 					})
-			)
-			.and_then(move |proposer|
-				inherent_providers.create_inherent_data()
-					.map(|id| (proposer, id))
-					.map_err(|e| {
-						error!("Failed to create inherent data: {:?}", e);
-						InvalidHead
+					.and_then(|mut proposer| {
+						error!("PROPOSING");
+						let inherent_data = inherent_providers.create_inherent_data()
+							.map_err(|e| {
+								error!("Failed to create inherent data: {:?}", e);
+								InvalidHead
+							})?;
+
+						let future = proposer.propose(
+							inherent_data,
+							Default::default(),
+							//TODO: Fix this.
+							Duration::from_secs(6),
+						)
+						.map_err(|e| {
+							error!("Proposing failed: {:?}", e);
+							InvalidHead
+						})
+						.compat();
+
+						Ok(future)
 					})
-			)
-			.and_then(|(mut proposer, inherent_data)| {
-				proposer.propose(
-					inherent_data,
-					Default::default(),
-					//TODO: Fix this.
-					Duration::from_secs(6),
-				)
-				.map_err(|e| {
-					error!("Proposing failed: {:?}", e);
-					InvalidHead
-				})
-				.compat()
 			})
+			.flatten()
 			.map(|b| {
+				error!("BUILDING BLOCKDATA");
 				let block_data = BlockData(b.encode());
 				let head_data = HeadData::<Block> { header: b.deconstruct().0 };
 				let messages = OutgoingMessages { outgoing_messages: Vec::new() };
 
 				(block_data, parachain::HeadData(head_data.encode()), messages)
+			})
+			.then(|r| {
+				trace!(target: "cumulus-collator", "Produced candidate: {:?}", r);
+				r
 			});
 
 		Box::new(res)
 	}
 }
 
 /// Implements `BuildParachainContext` to build a collator instance.
-struct CollatorBuilder<Block, PF> {
-	inherent_data_providers: InherentDataProviders,
-	proposer_factory: PF,
-	_phantom: PhantomData<Block>,
+struct CollatorBuilder<Block, SP> {
+	setup_parachain: SP,
+	_marker: PhantomData<Block>,
 }
 
-impl<Block, PF> CollatorBuilder<Block, PF> {
+impl<Block, SP> CollatorBuilder<Block, SP> {
 	/// Create a new instance of self.
-	fn new(proposer_factory: PF, inherent_data_providers: InherentDataProviders) -> Self {
+	fn new(setup_parachain: SP) -> Self {
 		Self {
-			inherent_data_providers,
-			proposer_factory,
-			_phantom: Default::default(),
+			setup_parachain,
+			_marker: PhantomData,
 		}
 	}
 }
 
-impl<Block, PF> BuildParachainContext for CollatorBuilder<Block, PF> where
-	Block: BlockT,
-	PF: Environment<Block> + 'static + Send + Sync,
-	PF::Error: std::fmt::Debug,
-	PF::Proposer: Send + Sync,
-	<PF::Proposer as Proposer<Block>>::Create: Unpin + Send + Sync,
+impl<Block: BlockT, SP: SetupParachain<Block>> BuildParachainContext for CollatorBuilder<Block, SP>
+	where
+		<<SP::ProposerFactory as Environment<Block>>::Proposer as Proposer<Block>>::Create: Send + Unpin,
+		<SP::ProposerFactory as Environment<Block>>::Error: Debug,
 {
-	type ParachainContext = Collator<Block, PF>;
-
-	fn build(self, network: Arc<dyn CollatorNetwork>) -> Result<Self::ParachainContext, ()> {
-		Ok(Collator::new(self.proposer_factory, self.inherent_data_providers, network))
+	type ParachainContext = Collator<Block, SP::ProposerFactory>;
+
+	fn build<B, E>(
+		self,
+		client: Arc<PolkadotClient<B, E>>,
+		task_executor: TaskExecutor,
+		network: Arc<dyn CollatorNetwork>,
+	) -> Result<Self::ParachainContext, ()>
+		where
+			B: substrate_client::backend::Backend<PBlock, Blake2Hasher> + 'static,
+			E: substrate_client::CallExecutor<PBlock, Blake2Hasher> + Clone + Send + Sync + 'static
+	{
+		let (proposer_factory, inherent_data_providers) = self.setup_parachain
+			.setup_parachain(client, task_executor)
+			.map_err(|e| error!("Error setting up the parachain: {}", e))?;
+
+		Ok(Collator::new(proposer_factory, inherent_data_providers, network))
 	}
 }
 
+/// Something that can setup a parachain.
+pub trait SetupParachain<Block: BlockT>: Send
+	where
+		<<Self::ProposerFactory as Environment<Block>>::Proposer as Proposer<Block>>::Create: Send + Unpin,
+		<Self::ProposerFactory as Environment<Block>>::Error: Debug,
+{
+	/// The proposer factory of the parachain to build blocks.
+	type ProposerFactory: Environment<Block> + Send + 'static;
+
+	/// Setup the parachain.
+	fn setup_parachain<P: cumulus_consensus::PolkadotClient>(
+		self,
+		polkadot_client: P,
+		task_executor: TaskExecutor,
+	) -> Result<(Self::ProposerFactory, InherentDataProviders), String>;
+}
+
 /// Run a collator with the given proposer factory.
-pub fn run_collator<Block, PF, E, I>(
-	proposer_factory: PF,
-	inherent_data_providers: InherentDataProviders,
+pub fn run_collator<Block, SP, E>(
+	setup_parachain: SP,
 	para_id: ParaId,
 	exit: E,
 	key: Arc<CollatorPair>,
 	version: VersionInfo,
 ) -> Result<(), cli::error::Error>
 where
 	Block: BlockT,
-	PF: Environment<Block> + 'static + Send + Sync,
-	PF::Error: std::fmt::Debug,
-	PF::Proposer: Send + Sync,
-	<PF::Proposer as Proposer<Block>>::Create: Unpin + Send + Sync,
+	SP: SetupParachain<Block> + Send + 'static,
+	<<SP::ProposerFactory as Environment<Block>>::Proposer as Proposer<Block>>::Create: Send + Unpin,
+	<SP::ProposerFactory as Environment<Block>>::Error: Debug,
 	E: IntoFuture<Item=(), Error=()>,
 	E::Future: Send + Clone + Sync + 'static,
 {
-	let builder = CollatorBuilder::new(proposer_factory, inherent_data_providers);
+	let builder = CollatorBuilder::new(setup_parachain);
 	polkadot_collator::run_collator(builder, para_id, exit, key, version)
 }
 
@@ -273,7 +307,7 @@ mod tests {
 			unimplemented!("Not required in tests")
 		}
 
-		fn checked_statements(&self, _: Hash) ->
+		fn checked_statements(&self, _: PHash) ->
 			Box<dyn Stream<Item=SignedStatement, Error=()>>
 		{
 			unimplemented!("Not required in tests")

consensus/src/lib.rs

@@ -66,12 +66,12 @@ pub struct HeadUpdate {
 
 /// Helper for the Polkadot client. This is expected to be a lightweight handle
 /// like an `Arc`.
-pub trait PolkadotClient: Clone {
+pub trait PolkadotClient: Clone + 'static {
 	/// The error type for interacting with the Polkadot client.
 	type Error: std::fmt::Debug + Send;
 
 	/// A stream that yields finalized head-data for a certain parachain.
-	type Finalized: Stream<Item = Vec<u8>> + Send;
+	type Finalized: Stream<Item = Vec<u8>> + Send + Unpin;
 
 	/// Get a stream of finalized heads.
 	fn finalized_heads(&self, para_id: ParaId) -> ClientResult<Self::Finalized>;
@@ -85,11 +85,11 @@ pub trait PolkadotClient: Clone {
 }
 
 /// Spawns a future that follows the Polkadot relay chain for the given parachain.
-pub fn follow_polkadot<'a, L: 'a, P: 'a>(para_id: ParaId, local: Arc<L>, polkadot: P)
-	-> ClientResult<impl Future<Output = ()> + Send + 'a>
+pub fn follow_polkadot<L, P>(para_id: ParaId, local: Arc<L>, polkadot: P)
+	-> ClientResult<impl Future<Output = ()> + Send + Unpin>
 	where
 		L: LocalClient + Send + Sync,
-		P: PolkadotClient + Send + Sync,
+		P: PolkadotClient,
 {
 	let finalized_heads = polkadot.finalized_heads(para_id)?;
 

test/parachain/Cargo.toml

@@ -13,6 +13,7 @@ path = 'src/main.rs'
 derive_more = '0.15.0'
 exit-future = '0.1.4'
 futures = '0.1.29'
+futures03 = { package = "futures-preview", version = "0.3.0-alpha.19", features = ["compat"] }
 log = '0.4.8'
 parking_lot = '0.9.0'
 tokio = '0.1.22'
@@ -26,19 +27,25 @@ ctrlc = { version = "3.1.3", features = ["termination"] }
 sr-primitives = { git = "https://github.com/paritytech/substrate", default_features = false, branch = "bkchr-cumulus-branch" }
 sr-io = { git = "https://github.com/paritytech/substrate", branch = "bkchr-cumulus-branch" }
 substrate-cli = { git = "https://github.com/paritytech/substrate", branch = "bkchr-cumulus-branch" }
-primitives = { package = "substrate-primitives", git = "https://github.com/paritytech/substrate", branch = "bkchr-cumulus-branch" }
+substrate-primitives = { package = "substrate-primitives", git = "https://github.com/paritytech/substrate", branch = "bkchr-cumulus-branch" }
 substrate-executor = { git = "https://github.com/paritytech/substrate", branch = "bkchr-cumulus-branch" }
 substrate-service = { git = "https://github.com/paritytech/substrate", branch = "bkchr-cumulus-branch" }
 inherents = { package = "substrate-inherents", git = "https://github.com/paritytech/substrate", branch = "bkchr-cumulus-branch" }
 transaction-pool = { package = "substrate-transaction-pool", git = "https://github.com/paritytech/substrate", branch = "bkchr-cumulus-branch" }
 network = { package = "substrate-network", git = "https://github.com/paritytech/substrate", branch = "bkchr-cumulus-branch" }
 substrate-client = { git = "https://github.com/paritytech/substrate", branch = "bkchr-cumulus-branch" }
+srml-timestamp = { git = "https://github.com/paritytech/substrate", branch = "bkchr-cumulus-branch" }
 basic-authorship = { package = "substrate-basic-authorship", git = "https://github.com/paritytech/substrate", branch = "bkchr-cumulus-branch" }
+consensus-common = { package = "substrate-consensus-common", git = "https://github.com/paritytech/substrate", branch = "bkchr-cumulus-branch" }
 
 # Cumulus dependencies
 cumulus-consensus = { path = "../../consensus" }
 cumulus-collator = { path = "../../collator" }
 
+# Polkadot dependencies
+polkadot-primitives = { git = "https://github.com/paritytech/polkadot", branch = "bkchr-cumulus-branch" }
+polkadot-collator = { git = "https://github.com/paritytech/polkadot", branch = "bkchr-cumulus-branch" }
+
 [build-dependencies]
 vergen = '3.0.4'
 

test/parachain/build.rs

@@ -1,3 +1,19 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Cumulus.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Cumulus.  If not, see <http://www.gnu.org/licenses/>.
+
 use std::{env, path::PathBuf};
 
 use vergen::{ConstantsFlags, generate_cargo_keys};

test/parachain/runtime/build.rs

@@ -1,5 +1,5 @@
 // Copyright 2019 Parity Technologies (UK) Ltd.
-// This file is part of Substrate.
+// This file is part of Cumulus.
 
 // Substrate is free software: you can redistribute it and/or modify
 // it under the terms of the GNU General Public License as published by
@@ -12,7 +12,7 @@
 // GNU General Public License for more details.
 
 // You should have received a copy of the GNU General Public License
-// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+// along with Cumulus.  If not, see <http://www.gnu.org/licenses/>.
 
 use wasm_builder_runner::{build_current_project_with_rustflags, WasmBuilderSource};
 

test/parachain/runtime/src/lib.rs

@@ -1,4 +1,18 @@
-//! The Substrate Node Template runtime. This can be compiled with `#[no_std]`, ready for Wasm.
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Cumulus.
+
+// Cumulus is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Cumulus is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Cumulus.  If not, see <http://www.gnu.org/licenses/>.
 
 #![cfg_attr(not(feature = "std"), no_std)]
 // `construct_runtime!` does a lot of recursion and requires us to increase the limit to 256.
@@ -9,7 +23,7 @@
 include!(concat!(env!("OUT_DIR"), "/wasm_binary.rs"));
 
 use rstd::prelude::*;
-use primitives::{OpaqueMetadata, crypto::key_types};
+use primitives::OpaqueMetadata;
 use sr_primitives::{
 	ApplyResult, transaction_validity::TransactionValidity, generic, create_runtime_str,
 	impl_opaque_keys, AnySignature