Master.into()

README.md

@@ -15,7 +15,7 @@ also try out one of the [tutorials](https://substrate.dev/en/tutorials).
 
 ## Contributions & Code of Conduct
 
-Please follow the contributions guidelines as outlined in [`docs/CONTRIBUTING.adoc`](docs/CONTRIBUTING.adoc). In all communications and contributions, this project follows the [Contributor Covenant Code of Conduct](docs/CODE_OF_CONDUCT.adoc).
+Please follow the contributions guidelines as outlined in [`docs/CONTRIBUTING.adoc`](docs/CONTRIBUTING.adoc). In all communications and contributions, this project follows the [Contributor Covenant Code of Conduct](docs/CODE_OF_CONDUCT.md).
 
 ## Security
 

client/executor/wasmtime/Cargo.toml

@@ -21,12 +21,8 @@ sp-wasm-interface = { version = "2.0.0-rc5", path = "../../../primitives/wasm-in
 sp-runtime-interface = { version = "2.0.0-rc5", path = "../../../primitives/runtime-interface" }
 sp-core = { version = "2.0.0-rc5", path = "../../../primitives/core" }
 sp-allocator = { version = "2.0.0-rc5", path = "../../../primitives/allocator" }
-wasmtime = { package = "substrate-wasmtime", version = "0.19.0" }
-wasmtime-runtime = { version = "0.19.0" }
-wasmtime-environ = { version = "0.19.0" }
-cranelift-wasm = { version = "0.66.0" }
-cranelift-codegen = { version = "0.66.0" }
-
+wasmtime = "0.19"
+pwasm-utils = "0.14.0"
 
 [dev-dependencies]
 assert_matches = "1.3.0"

client/executor/wasmtime/src/imports.rs

@@ -294,7 +294,7 @@ fn into_wasmtime_val_type(val_ty: ValueType) -> wasmtime::ValType {
 /// Converts a `Val` into a substrate runtime interface `Value`.
 ///
 /// Panics if the given value doesn't have a corresponding variant in `Value`.
-fn into_value(val: Val) -> Value {
+pub fn into_value(val: Val) -> Value {
 	match val {
 		Val::I32(v) => Value::I32(v),
 		Val::I64(v) => Value::I64(v),
@@ -304,7 +304,7 @@ fn into_value(val: Val) -> Value {
 	}
 }
 
-fn into_wasmtime_val(value: Value) -> wasmtime::Val {
+pub fn into_wasmtime_val(value: Value) -> wasmtime::Val {
 	match value {
 		Value::I32(v) => Val::I32(v),
 		Value::I64(v) => Val::I64(v),

client/executor/wasmtime/src/instance_wrapper.rs

@@ -29,36 +29,36 @@ use sc_executor_common::{
 };
 use sp_wasm_interface::{Pointer, WordSize, Value};
 use wasmtime::{Engine, Instance, Module, Memory, Table, Val, Func, Extern, Global, Store};
+use parity_wasm::elements;
 
 mod globals_snapshot;
 
 pub use globals_snapshot::GlobalsSnapshot;
 
 pub struct ModuleWrapper {
-	imported_globals_count: u32,
-	globals_count: u32,
 	module: Module,
 	data_segments_snapshot: DataSegmentsSnapshot,
 }
 
 impl ModuleWrapper {
 	pub fn new(engine: &Engine, code: &[u8]) -> Result<Self> {
-		let module = Module::new(engine, code)
+		let mut raw_module: elements::Module = elements::deserialize_buffer(code)
+			.map_err(|e| Error::from(format!("cannot decode module: {}", e)))?;
+		pwasm_utils::export_mutable_globals(&mut raw_module, "exported_internal_global");
+		let instrumented_code = elements::serialize(raw_module)
+			.map_err(|e| Error::from(format!("cannot encode module: {}", e)))?;
+
+		let module = Module::new(engine, &instrumented_code)
 			.map_err(|e| Error::from(format!("cannot create module: {}", e)))?;
 
 		let module_info = WasmModuleInfo::new(code)
 			.ok_or_else(|| Error::from("cannot deserialize module".to_string()))?;
-		let declared_globals_count = module_info.declared_globals_count();
-		let imported_globals_count = module_info.imported_globals_count();
-		let globals_count = imported_globals_count + declared_globals_count;
 
 		let data_segments_snapshot = DataSegmentsSnapshot::take(&module_info)
 			.map_err(|e| Error::from(format!("cannot take data segments snapshot: {}", e)))?;
 
 		Ok(Self {
 			module,
-			imported_globals_count,
-			globals_count,
 			data_segments_snapshot,
 		})
 	}
@@ -78,8 +78,6 @@ impl ModuleWrapper {
 /// routines.
 pub struct InstanceWrapper {
 	instance: Instance,
-	globals_count: u32,
-	imported_globals_count: u32,
 	// The memory instance of the `instance`.
 	//
 	// It is important to make sure that we don't make any copies of this to make it easier to proof
@@ -143,8 +141,6 @@ impl InstanceWrapper {
 		Ok(Self {
 			table: get_table(&instance),
 			instance,
-			globals_count: module_wrapper.globals_count,
-			imported_globals_count: module_wrapper.imported_globals_count,
 			memory,
 			_not_send_nor_sync: marker::PhantomData,
 		})

client/executor/wasmtime/src/instance_wrapper/globals_snapshot.rs

@@ -17,115 +17,68 @@
 // along with this program. If not, see <https://www.gnu.org/licenses/>.
 
 use super::InstanceWrapper;
-use sc_executor_common::{
-	error::{Error, Result},
-};
+use sc_executor_common::error::{Result, Error};
 use sp_wasm_interface::Value;
-use cranelift_codegen::ir;
-use cranelift_wasm::GlobalIndex;
-use wasmtime_runtime::{ExportGlobal, Export};
+use crate::imports::{into_value, into_wasmtime_val};
+
+/// Saved value of particular exported global.
+struct SavedValue {
+	/// Index of the export.
+	index: usize,
+	/// Global value.
+	value: Value,
+}
 
 /// A snapshot of a global variables values. This snapshot can be used later for restoring the
 /// values to the preserved state.
 ///
 /// Technically, a snapshot stores only values of mutable global variables. This is because
 /// immutable global variables always have the same values.
-pub struct GlobalsSnapshot {
-	handle: wasmtime_runtime::InstanceHandle,
-	preserved_mut_globals: Vec<(*mut wasmtime_runtime::VMGlobalDefinition, Value)>,
-}
+pub struct GlobalsSnapshot(Vec<SavedValue>);
 
 impl GlobalsSnapshot {
 	/// Take a snapshot of global variables for a given instance.
 	pub fn take(instance_wrapper: &InstanceWrapper) -> Result<Self> {
-		// EVIL:
-		// Usage of an undocumented function.
-		let handle = instance_wrapper.instance.handle().clone().handle;
-
-		let mut preserved_mut_globals = vec![];
-
-		for global_idx in instance_wrapper.imported_globals_count..instance_wrapper.globals_count {
-			let (def, global) = match handle.lookup_by_declaration(
-				&wasmtime_environ::EntityIndex::Global(GlobalIndex::from_u32(global_idx)),
-			) {
-				Export::Global(ExportGlobal { definition, global, .. }) => (definition, global),
-				_ => unreachable!("only globals can be returned for a global request"),
-			};
-
-			// skip immutable globals.
-			if !global.mutability {
-				continue;
-			}
-
-			let value = unsafe {
-				// Safety of this function solely depends on the correctness of the reference and
-				// the type information of the global.
-				read_global(def, global.ty)?
-			};
-			preserved_mut_globals.push((def, value));
-		}
-
-		Ok(Self {
-			preserved_mut_globals,
-			handle,
-		})
+		let data = instance_wrapper.instance
+			.exports()
+			.enumerate()
+			.filter_map(|(index, export)| {
+				if export.name().starts_with("exported_internal_global") {
+					export.into_global().map(
+						|g| SavedValue { index, value: into_value(g.get()) }
+					)
+				} else { None }
+			})
+			.collect::<Vec<_>>();
+
+		Ok(Self(data))
 	}
 
 	/// Apply the snapshot to the given instance.
 	///
 	/// This instance must be the same that was used for creation of this snapshot.
 	pub fn apply(&self, instance_wrapper: &InstanceWrapper) -> Result<()> {
-		if instance_wrapper.instance.handle().handle != self.handle {
-			return Err(Error::from("unexpected instance handle".to_string()));
-		}
-
-		for (def, value) in &self.preserved_mut_globals {
-			unsafe {
-				// The following writes are safe if the precondition that this is the same instance
-				// this snapshot was created with:
-				//
-				// 1. These pointers must be still not-NULL and allocated.
-				// 2. The set of global variables is fixed for the lifetime of the same instance.
-				// 3. We obviously assume that the wasmtime references are correct in the first place.
-				// 4. We write the data with the same type it was read in the first place.
-				write_global(*def, *value)?;
+		// This is a pointer over saved items, it moves forward when the loop value below takes over it's current value.
+		// Since both pointers (`current` and `index` below) are over ordered lists, they eventually hit all
+		// equal referenced values.
+		let mut current = 0;
+		for (index, export) in instance_wrapper.instance.exports().enumerate() {
+			if current >= self.0.len() { break; }
+			let current_saved = &self.0[current];
+			if index < current_saved.index { continue; }
+			else if index > current_saved.index { current += 1; continue; }
+			else {
+				export.into_global()
+					.ok_or_else(|| Error::Other(
+						"Wrong instance in GlobalsSnapshot::apply: what should be global is not global.".to_string()
+					))?
+					.set(into_wasmtime_val(current_saved.value))
+					.map_err(|_e| Error::Other(
+						"Wrong instance in GlobalsSnapshot::apply: global saved type does not matched applied.".to_string()
+					))?;
 			}
 		}
-		Ok(())
-	}
-}
 
-unsafe fn read_global(
-	def: *const wasmtime_runtime::VMGlobalDefinition,
-	ty: ir::Type,
-) -> Result<Value> {
-	let def = def
-		.as_ref()
-		.ok_or_else(|| Error::from("wasmtime global reference is null during read".to_string()))?;
-	let val = match ty {
-		ir::types::I32 => Value::I32(*def.as_i32()),
-		ir::types::I64 => Value::I64(*def.as_i64()),
-		ir::types::F32 => Value::F32(*def.as_u32()),
-		ir::types::F64 => Value::F64(*def.as_u64()),
-		_ => {
-			return Err(Error::from(format!(
-				"unsupported global variable type: {}",
-				ty
-			)))
-		}
-	};
-	Ok(val)
-}
-
-unsafe fn write_global(def: *mut wasmtime_runtime::VMGlobalDefinition, value: Value) -> Result<()> {
-	let def = def
-		.as_mut()
-		.ok_or_else(|| Error::from("wasmtime global reference is null during write".to_string()))?;
-	match value {
-		Value::I32(v) => *def.as_i32_mut() = v,
-		Value::I64(v) => *def.as_i64_mut() = v,
-		Value::F32(v) => *def.as_u32_mut() = v,
-		Value::F64(v) => *def.as_u64_mut() = v,
+		Ok(())
 	}
-	Ok(())
 }

client/finality-grandpa/src/communication/gossip.rs

@@ -922,7 +922,7 @@ impl<Block: BlockT> Inner<Block> {
 			PendingCatchUp::Processing { .. } => {
 				self.pending_catch_up = PendingCatchUp::None;
 			},
-			state => trace!(target: "afg",
+			state => debug!(target: "afg",
 				"Noted processed catch up message when state was: {:?}",
 				state,
 			),
@@ -1043,7 +1043,7 @@ impl<Block: BlockT> Inner<Block> {
 				let (catch_up_allowed, catch_up_report) = self.note_catch_up_request(who, &request);
 
 				if catch_up_allowed {
-					trace!(target: "afg", "Sending catch-up request for round {} to {}",
+					debug!(target: "afg", "Sending catch-up request for round {} to {}",
 						   round,
 						   who,
 					);

client/finality-grandpa/src/import.rs

@@ -18,7 +18,7 @@
 
 use std::{sync::Arc, collections::HashMap};
 
-use log::{debug, trace};
+use log::debug;
 use parity_scale_codec::Encode;
 use parking_lot::RwLockWriteGuard;
 
@@ -527,7 +527,7 @@ impl<BE, Block: BlockT, Client, SC> BlockImport<Block>
 			},
 			None => {
 				if needs_justification {
-					trace!(
+					debug!(
 						target: "afg",
 						"Imported unjustified block #{} that enacts authority set change, waiting for finality for enactment.",
 						number,

client/network/src/block_requests.rs

@@ -119,6 +119,7 @@ pub enum Event<B: Block> {
 #[derive(Debug, Clone)]
 pub struct Config {
 	max_block_data_response: u32,
+	max_block_body_bytes: usize,
 	max_request_len: usize,
 	max_response_len: usize,
 	inactivity_timeout: Duration,
@@ -137,6 +138,7 @@ impl Config {
 	pub fn new(id: &ProtocolId) -> Self {
 		let mut c = Config {
 			max_block_data_response: 128,
+			max_block_body_bytes: 8 * 1024 * 1024,
 			max_request_len: 1024 * 1024,
 			max_response_len: 16 * 1024 * 1024,
 			inactivity_timeout: Duration::from_secs(15),
@@ -171,6 +173,15 @@ impl Config {
 		self
 	}
 
+	/// Set the maximum total bytes of block bodies that are send in the response.
+	/// Note that at least one block is always sent regardless of the limit.
+	/// This should be lower than the value specified in `set_max_response_len`
+	/// accounting for headers, justifications and encoding overhead.
+	pub fn set_max_block_body_bytes(&mut self, v: usize) -> &mut Self {
+		self.max_block_body_bytes = v;
+		self
+	}
+
 	/// Set protocol to use for upgrade negotiation.
 	pub fn set_protocol(&mut self, id: &ProtocolId) -> &mut Self {
 		let mut v = Vec::new();
@@ -385,8 +396,11 @@ where
 
 		let mut blocks = Vec::new();
 		let mut block_id = from_block_id;
+		let mut total_size = 0;
 		while let Some(header) = self.chain.header(block_id).unwrap_or(None) {
-			if blocks.len() >= max_blocks as usize {
+			if blocks.len() >= max_blocks as usize
+				|| (blocks.len() >= 1 && total_size > self.config.max_block_body_bytes)
+			{
 				break
 			}
 
@@ -400,28 +414,35 @@ where
 			};
 			let is_empty_justification = justification.as_ref().map(|j| j.is_empty()).unwrap_or(false);
 
+			let body = if get_body {
+				match self.chain.block_body(&BlockId::Hash(hash))? {
+					Some(mut extrinsics) => extrinsics.iter_mut()
+						.map(|extrinsic| extrinsic.encode())
+						.collect(),
+					None => {
+						log::trace!(target: "sync", "Missing data for block request.");
+						break;
+					}
+				}
+			} else {
+				Vec::new()
+			};
+
 			let block_data = schema::v1::BlockData {
 				hash: hash.encode(),
 				header: if get_header {
 					header.encode()
 				} else {
 					Vec::new()
 				},
-				body: if get_body {
-					self.chain.block_body(&BlockId::Hash(hash))?
-						.unwrap_or_default()
-						.iter_mut()
-						.map(|extrinsic| extrinsic.encode())
-						.collect()
-				} else {
-					Vec::new()
-				},
+				body,
 				receipt: Vec::new(),
 				message_queue: Vec::new(),
 				justification: justification.unwrap_or_default(),
 				is_empty_justification,
 			};
 
+			total_size += block_data.body.len();
 			blocks.push(block_data);
 
 			match direction {