state_db tests passing.

core/state-db/src/branch.rs

@@ -160,6 +160,10 @@ impl RangeSet {
 			if let Some(Some(StatesBranch{ state, parent_branch_index, .. })) = self.storage.get(&branch_index) {
 				// TODO EMCH consider vecdeque ??
 				let state = if state.end > previous_start {
+					if state.start >= previous_start {
+						// empty branch stop here
+						break;
+					}
 					BranchStateRef {
 						start: state.start,
 						end: previous_start,
@@ -295,22 +299,19 @@ impl RangeSet {
 	pub fn update_finalize_treshold(
 		&mut self,
 		branch_index: u64,
-		linear_index: Option<u64>,
+		linear_index: u64,
 		full: bool,
 	) {
 
 		if branch_index < self.treshold {
 			return;
 		}
-		if branch_index == self.treshold && linear_index.is_none() {
-			return;
-		}
 		// range set update
 		let old_treshold = self.treshold;
 		// we do not finalize current branch cause it
 		// can contains other blocks
 		self.treshold = branch_index;
-		if (branch_index == 0 && linear_index.is_none()) || !full {
+		if !full {
 			// remove cached value under treshold only
 			let new_storage = self.storage.split_off(&(self.treshold));
 			self.storage = new_storage;
@@ -330,17 +331,19 @@ impl RangeSet {
 
 	/// Apply a post finalize without considering treshold.
 	/// TODO EMCH rename as it is also use to prepare a gc
-	/// without moving the treshould first.
+	/// without moving the treshold first.
+	///
 	pub fn finalize_full(
 		&mut self,
 		branch_index: u64,
-		linear_index: Option<u64>,
+		linear_index: u64,
 	) {
 		if let Some(Some(state)) = self.storage.remove(&branch_index) {
 			let mut child_anchor: LinearStates = state.clone();
-			child_anchor.state.start = linear_index.unwrap_or(child_anchor.state.start);
+			child_anchor.state.start = linear_index;
 			let old_storage = std::mem::replace(&mut self.storage, BTreeMap::new());
-			self.storage.insert(branch_index, Some(child_anchor));
+			// insert anchor
+			self.storage.insert(branch_index, Some(child_anchor.clone()));
 			for (index, state) in old_storage.into_iter().filter_map(|(k, v)| v.map(|v| (k, v))) {
 				// ordered property of branch index allows to skip in depth branch search
 				if let Some(Some(parent_state)) = self.storage.get(&state.parent_branch_index) {
@@ -355,18 +358,17 @@ impl RangeSet {
 					}
 				}
 			}
+			// remove anchor block
+			child_anchor.state.start += 1;
+			if child_anchor.state.start < child_anchor.state.end {
+				self.storage.insert(branch_index, Some(child_anchor.clone()));
+			} else {
+				self.storage.remove(&branch_index);
+			}
 		}
 
 	}
 
-	#[cfg(test)]
-	pub fn test_finalize_full(
-		&mut self,
-		branch_index: u64,
-	) {
-		self.finalize_full(branch_index, None);
-	}
-
 	/// Revert some ranges, without any way to revert.
 	/// Returning ranges for the parent index.
 	/// TODO EMCH can remove ??
@@ -488,7 +490,7 @@ mod test {
 	// 0
 	// |> 2: [10,2]
 	//       |> 3:  [11,2]
-	fn build_finalize_set() ->  (RangeSet, u64) {
+	fn build_finalize_set() ->  (RangeSet, (u64, u64)) {
 		let mut set = RangeSet::default();
 		set.import(10, 0, None);
 		let (b1, anchor_1) = set.import(10, 0, None);
@@ -499,7 +501,7 @@ mod test {
 		set.import(12, finalize, Some(bf.clone()));
 		set.import(12, finalize, Some(bf));
 
-		(set, finalize)
+		(set, (finalize, 11))
 	}
 
 	#[test]
@@ -511,7 +513,7 @@ mod test {
 		| {
 			let (mut ranges, finalize) = build_finalize_set();
 
-			let _ = ranges.update_finalize_treshold(finalize, None, full);
+			let _ = ranges.update_finalize_treshold(finalize.0, finalize.1, full);
 
 			assert!(ranges.range_treshold() == 3);
 
@@ -529,7 +531,8 @@ mod test {
 			&[(1, 1), (2, 2)][..],
 		);
 		with_full_finalize(true,
-			&[(3, 2), (5, 1)][..],
+			// (3, 1) because finalized block is removed
+			&[(3, 1), (5, 1)][..],
 			&[(1, 1), (2, 2), (4, 2)][..],
 		);
 	}

core/state-db/src/lib.rs

@@ -654,7 +654,11 @@ mod tests {
 			max_mem: None,
 		}));
 		assert!(db.data_eq(&make_db(&[21, 3, 922, 94])));
-		assert!(db.offstate_eq(&[21, 3, 822, 84]));
+		assert!(db.offstate_eq_at(&[822, 84], Some(0)));
+		assert!(db.offstate_eq_at(&[822, 84], Some(1)));
+		assert!(db.offstate_eq_at(&[21, 822, 84], Some(2)));
+		assert!(db.offstate_eq_at(&[3, 21, 822, 84], Some(3)));
+		assert!(db.offstate_eq(&[3, 21, 822, 84]));
 	}
 
 	#[test]
@@ -668,7 +672,11 @@ mod tests {
 		assert!(sdb.is_pruned(&H256::from_low_u64_be(21), 2));
 		assert!(sdb.is_pruned(&H256::from_low_u64_be(22), 2));
 		assert!(db.data_eq(&make_db(&[21, 3, 922, 93, 94])));
-		assert!(db.offstate_eq(&[21, 3, 822, 83, 84]));
+		assert!(db.offstate_eq_at(&[822, 83, 84], Some(0)));
+		assert!(db.offstate_eq_at(&[822, 83, 84], Some(1)));
+		assert!(db.offstate_eq_at(&[21, 822, 83, 84], Some(2)));
+		assert!(db.offstate_eq_at(&[3, 21, 822, 84], Some(3)));
+		assert!(db.offstate_eq(&[3, 21, 822, 84]));
 	}
 
 	#[test]
@@ -682,6 +690,10 @@ mod tests {
 		assert!(!sdb.is_pruned(&H256::from_low_u64_be(21), 2));
 		assert!(sdb.is_pruned(&H256::from_low_u64_be(22), 2));
 		assert!(db.data_eq(&make_db(&[1, 21, 3, 921, 922, 93, 94])));
-		assert!(db.offstate_eq(&[1, 21, 3, 821, 822, 83, 84]));
+		assert!(db.offstate_eq_at(&[821, 822, 83, 84], Some(0)));
+		assert!(db.offstate_eq_at(&[1, 821, 822, 83, 84], Some(1)));
+		assert!(db.offstate_eq_at(&[21, 822, 83, 84], Some(2)));
+		assert!(db.offstate_eq_at(&[3, 21, 822, 84], Some(3)));
+		assert!(db.offstate_eq(&[3, 21, 822, 84]));
 	}
 }

core/state-db/src/noncanonical.rs

@@ -98,7 +98,9 @@ impl OffstatePendingGC {
 					match offstate_values.get_mut(&key).map(|historied_value| {
 						historied_value.gc(branches.reverse_iter_ranges())
 					}) {
-						Some(historied_data::PruneResult::Cleared) => { let _ = offstate_values.remove(&key); },
+						Some(historied_data::PruneResult::Cleared) => {
+							let _ = offstate_values.remove(&key);
+						},
 						_ => (),
 					}
 				}
@@ -647,7 +649,7 @@ impl<BlockHash: Hash, Key: Hash> NonCanonicalOverlay<BlockHash, Key> {
 			if let Some(branch_index_cannonicalize) = last_index {
 				// this needs to be call after parents update
 				// TODO EMCH may be needed in 'canonicalize', and restore or commit here
-				self.branches.update_finalize_treshold(branch_index_cannonicalize, Some(block_number + 1), true);
+				self.branches.update_finalize_treshold(branch_index_cannonicalize, block_number, true);
 				// gc is at the right place
 				self.offstate_gc.set_pending_gc(branch_index_cannonicalize);
 				// try to run the garbage collection (can run later if there is
@@ -673,9 +675,7 @@ impl<BlockHash: Hash, Key: Hash> NonCanonicalOverlay<BlockHash, Key> {
 
 	/// Get a value from the node overlay. This searches in every existing changeset.
 	pub fn get_offstate(&self, key: &[u8], state: &BranchRanges) -> Option<&Option<DBValue>> {
-			println!("b: {:?}", state);
 		if let Some(value) = self.offstate_values.get(key) {
-			println!("v: {:?}", value);
 			return value.get(state);
 		}
 		None
@@ -1147,6 +1147,7 @@ mod tests {
 		assert!(!contains_any(&overlay, 22, &h_2_2, 2));
 		assert!(!contains_any(&overlay, 211, &h_2_1_1, 3));
 		assert!(contains_both(&overlay, 111, &h_1_1_1, 3));
+		assert!(contains_both(&overlay, 12, &h_1_2, 2));
 		// check that journals are deleted
 		assert!(db.get_meta(&to_journal_key(1, 0)).unwrap().is_none());
 		assert!(db.get_meta(&to_journal_key(1, 1)).unwrap().is_none());
@@ -1270,6 +1271,7 @@ mod tests {
 
 		overlay.pin(&h_1);
 		let h1_context = overlay.get_branch_range(&h_1, 1).unwrap();
+		let h2_context = overlay.get_branch_range(&h_2, 1).unwrap();
 
 		let mut commit = CommitSet::default();
 		overlay.canonicalize::<io::Error>(&h_2, &mut commit).unwrap();

core/state-db/src/test.rs

@@ -99,13 +99,8 @@ impl TestDb {
 		if let Some((block_prune, offstate_prune_key)) = commit.offstate_prune.as_ref() {
 			for k in offstate_prune_key.iter() {
 				match self.offstate.get_mut(k).map(|v| {
-					println!("t{}", *block_prune);
-					println!("t{:?}", k);
-					println!("ser{:?}", v);
 					let mut ser = Ser::from_mut(v);
-					let r = ser.prune(*block_prune);
-					println!("ser{:?}", ser.into_vec());
-					r
+					ser.prune(*block_prune)
 				}) {
 					Some(PruneResult::Cleared) => { let _ = self.offstate.remove(k); },
 					Some(PruneResult::Changed) // changed applyied on mutable buffer without copy.
@@ -127,7 +122,7 @@ impl TestDb {
 	pub fn offstate_eq_at(&self, values: &[u64], block: Option<u64>) -> bool {
 		let data = make_offstate_changeset(values, &[]);
 		let self_offstate: BTreeMap<_, _> = self.get_offstate_pairs(&block).into_iter().collect();
-		println!("---{:?}", self_offstate);
+println!("of_eq {:?}", self_offstate);
 		self_offstate == data.into_iter().filter_map(|(k, v)| v.map(|v| (k,v))).collect()
 	}
 

core/utils/historied-data/src/linear.rs

@@ -251,14 +251,18 @@ impl<'a, F: SerializedConfig> Serialized<'a, F> {
 		};
 		let delete_size = elt_end - elt_start;
 		for _ in elt_start..elt_end {
-			let _ = self.0.to_mut().remove(elt_start);
+			let _ = self.0.to_mut().remove(elt_start); // TODO EMCH slice copy instead of that horror
 		}
 		let start_ix = start_ix - delete_size;
-		for i in end..end + (end - index) - 1 {
-			let old_value = self.read_le_usize(start_ix + i * SIZE_BYTE_LEN);
-			self.write_le_usize(start_ix + (i - (end - index)) * SIZE_BYTE_LEN, old_value - delete_size);
-		}
+
 		let len = len - (end - index);
+		for i in index..end {
+			let pos = i + (end - index);
+			if pos < len {
+				let old_value = self.read_le_usize(start_ix + pos * SIZE_BYTE_LEN);
+				self.write_le_usize(start_ix + i * SIZE_BYTE_LEN, old_value - delete_size);
+			}
+		}
 		let end_index = start_ix + len * SIZE_BYTE_LEN;
 		self.write_le_usize(end_index - SIZE_BYTE_LEN, len);
 		self.0.to_mut().truncate(end_index);

core/utils/historied-data/src/tree.rs

@@ -640,6 +640,7 @@ impl<V> History<V> {
 
 	/// Gc an historied value other its possible values.
 	/// Iterator need to be reversed ordered by branch index.
+	/// TODO EMCH this needs rewrite and proper testing!!
 	pub fn gc<IT, S, I>(&mut self, mut states: IT) -> PruneResult
 		where
 			IT: Iterator<Item = (S, I)>,
@@ -686,11 +687,10 @@ impl<V> History<V> {
 							break;
 						}
 					}
+				} else {
 					self.0.remove(branch_index);
 					changed = true;
 					break;
-				} else {
-					break;
 				}
 			}
 		}
@@ -821,29 +821,23 @@ impl<'a, F: SerializedConfig> Serialized<'a, F> {
 		let len = self.0.len();
 		let mut last_index_with_value = None;
 		let mut index = 0;
-						println!("from {}", from);
 		while index < len {
 			let history = self.0.get_state(index);
 			if history.index == from + 1 {
 				// new first content
 				if history.value.len() == 0 {
-				println!("yoop");
 					// delete so first content is after (if any)
 					last_index_with_value = None;
 				} else {
-					println!("start val: {}", index);
 					// start value over a value drop until here
 					last_index_with_value = Some(index);
 					break;
 				}
 			} else if history.index > from {
-				println!("IN {}", history.index);
 				if history.value.len() == 0 
 				  && last_index_with_value.is_none() {
-				println!("doo");
 						// delete on delete, continue
 				} else {
-				println!("daa");
 					if last_index_with_value.is_none() {
 						// first value, use this index
 						last_index_with_value = Some(index);
@@ -861,7 +855,6 @@ impl<'a, F: SerializedConfig> Serialized<'a, F> {
 
 		if let Some(last_index_with_value) = last_index_with_value {
 			if last_index_with_value > 0 {
-				println!("IN {}", last_index_with_value);
 				self.0.truncate_until(last_index_with_value);
 				return PruneResult::Changed;
 			}