Skip to content
This repository was archived by the owner on Nov 15, 2023. It is now read-only.
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
80 commits
Select commit Hold shift + click to select a range
611f481
DigestItem::ChangesTrieSignal
svyatonik Jun 25, 2019
8b65b04
introduce changes_trie::State
svyatonik Jun 26, 2019
0cd05df
introduce config activation block
svyatonik Jun 26, 2019
40ed2c2
ChangesTrieSignal::as_new_configuration
svyatonik Jun 26, 2019
c01ecd6
moved well_known_cache_keys to client
svyatonik Jun 27, 2019
73f1933
extracted DbChangesTrieStorage to separate file
svyatonik Jun 27, 2019
315303a
change meaning of none in blockchain cache
svyatonik Jun 28, 2019
67642d1
changes trie config (FULL) cache draft
svyatonik Jul 1, 2019
a7e240d
eliminating const ChangesTrieConfiguration
svyatonik Jul 1, 2019
5256d03
delay pruning
svyatonik Jul 2, 2019
698e2ff
continue elimination
svyatonik Jul 3, 2019
04a8fa0
do not prune CT config from cache
svyatonik Jul 3, 2019
afda822
removed redundant code
svyatonik Jul 3, 2019
e08183d
fix some TODOs
svyatonik Jul 4, 2019
368563f
Merge branch 'master' into changes_tries_update_configuration
svyatonik Jul 24, 2019
01e3792
introduce ConfigurationRange
svyatonik Jul 29, 2019
c1913fc
use Configuration range in build
svyatonik Jul 29, 2019
fb6ae9c
build skewed digest
svyatonik Jul 29, 2019
e429061
remove debug print
svyatonik Jul 29, 2019
e6acae8
extracted surface iterator
svyatonik Jul 30, 2019
68527fb
key_changes works with skewed digests
svyatonik Jul 30, 2019
5a839e5
fix client build
svyatonik Jul 30, 2019
9729912
add test for NeverPrune
svyatonik Jul 30, 2019
bca439a
fix TODO
svyatonik Jul 30, 2019
383795a
fixed some TODOs
svyatonik Jul 30, 2019
f6993f6
more tests
svyatonik Jul 31, 2019
8b70450
fixing TODOs
svyatonik Aug 1, 2019
1b3b565
fixed compilation
svyatonik Aug 1, 2019
fa7d631
Merge branch 'master' into changes_tries_update_configuration
svyatonik Aug 1, 2019
1de7f77
update runtime version
svyatonik Aug 1, 2019
ce78f66
git rid of large tuple
svyatonik Aug 1, 2019
82e74d5
too long lines
svyatonik Aug 1, 2019
cfb1cf1
config_activation_block -> zero
svyatonik Aug 1, 2019
94979f8
obsolete TODO
svyatonik Aug 1, 2019
dd5b602
removed unjustified expect
svyatonik Aug 1, 2019
7260719
update TODOs with issue number
svyatonik Aug 1, 2019
c563d30
Merge branch 'master' into changes_tries_update_configuration
svyatonik Aug 6, 2019
4552586
new CT pruning algorithm
svyatonik Aug 7, 2019
62a5aa8
Merge branch 'master' into changes_tries_update_configuration
svyatonik Aug 13, 2019
27eecae
BlockIdOrHeader isn't really required
svyatonik Aug 14, 2019
a158250
Merge branch 'master' into changes_tries_update_configuration
svyatonik Nov 8, 2019
ce92672
Merge branch 'master' into changes_tries_update_configuration
svyatonik Nov 11, 2019
9da96a5
removed debug leftovers + some docs
svyatonik Nov 12, 2019
f8a14f5
more docs
svyatonik Nov 12, 2019
68c4db0
more post-merge fixes
svyatonik Nov 12, 2019
d63d65e
more post-merge fixes
svyatonik Nov 12, 2019
9baac30
Merge branch 'master' into changes_tries_update_configuration
svyatonik Nov 12, 2019
6da8b23
revertes some unnecessary changes
svyatonik Nov 12, 2019
d302d1e
reverted unnecessary changes
svyatonik Nov 12, 2019
f4a82de
fix compilation + unnecessary changes
svyatonik Nov 12, 2019
801169c
(restart CI)
svyatonik Nov 12, 2019
eae183b
fix cache update when finalizing multiple blocks
svyatonik Nov 12, 2019
6d2b4db
fixed tests
svyatonik Nov 12, 2019
d7abf44
Merge branch 'master' into changes_tries_update_configuration
svyatonik Nov 25, 2019
6b3d12d
Merge branch 'master' into changes_tries_update_configuration
svyatonik Nov 29, 2019
d89c979
Merge remote-tracking branch 'origin/master' into changes_tries_updat…
svyatonik Dec 11, 2019
ed31869
Merge remote-tracking branch 'origin/master' into changes_tries_updat…
svyatonik Dec 17, 2019
af3c925
collect_extrinsics -> set_collect_extrinsics
svyatonik Jan 5, 2020
935fb7b
restore lost test
svyatonik Jan 5, 2020
9da28c8
do not calculate block number twice
svyatonik Jan 5, 2020
bf7fea2
Update primitives/blockchain/src/error.rs
svyatonik Jan 5, 2020
3d570f1
map_err -> unwrap_or
svyatonik Jan 5, 2020
b898191
document get_at Result
svyatonik Jan 5, 2020
7db5596
delete abandoned file
svyatonik Jan 5, 2020
9481f59
added weight for set_changes_trie_config
svyatonik Jan 5, 2020
05630d8
Merge branch 'changes_tries_update_configuration' of https://github.c…
svyatonik Jan 5, 2020
06755b0
prefer_configs -> fail_if_disabled
svyatonik Jan 5, 2020
d0b2de2
Update client/api/src/backend.rs
svyatonik Jan 5, 2020
2f795e6
Merge branch 'changes_tries_update_configuration' of https://github.c…
svyatonik Jan 5, 2020
c4c8871
Update client/db/src/changes_tries_storage.rs
svyatonik Jan 6, 2020
51fa091
CommitOperation+merge -> CommitOperations
svyatonik Jan 6, 2020
98d1958
Merge branch 'changes_tries_update_configuration' of https://github.c…
svyatonik Jan 7, 2020
46d4721
fixed test compilation
svyatonik Jan 7, 2020
39c5f64
merged two different CTRange structs
svyatonik Jan 9, 2020
ee16f37
Merge remote-tracking branch 'origin/master' into changes_tries_updat…
svyatonik Jan 9, 2020
25452a5
lost file
svyatonik Jan 9, 2020
54f3d47
Merge remote-tracking branch 'origin/master' into changes_tries_updat…
svyatonik Jan 13, 2020
233afad
uggrade db from v0 to v1 (init CT cache + add column)
svyatonik Jan 14, 2020
0997bd6
Merge branch 'master' into changes_tries_update_configuration
gavofyork Jan 16, 2020
7658040
fix after merge
svyatonik Jan 16, 2020
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
introduce ConfigurationRange
  • Loading branch information
svyatonik committed Jul 29, 2019
commit 01e37924ebe565a982e29ad1eb01ad849dfdc04f
85 changes: 41 additions & 44 deletions core/state-machine/src/changes_trie/changes_iterator.rs
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ use parity_codec::{Decode, Encode};
use hash_db::{HashDB, Hasher};
use num_traits::One;
use trie::{Recorder, MemoryDB};
use crate::changes_trie::{AnchorBlockId, Configuration, RootsStorage, Storage, BlockNumber};
use crate::changes_trie::{AnchorBlockId, ConfigurationRange, RootsStorage, Storage, BlockNumber};
use crate::changes_trie::input::{DigestIndex, ExtrinsicIndex, DigestIndexValue, ExtrinsicIndexValue};
use crate::changes_trie::storage::{TrieBackendAdapter, InMemoryStorage};
use crate::proving_backend::ProvingBackendEssence;
Expand All @@ -33,9 +33,8 @@ use crate::trie_backend_essence::{TrieBackendEssence};
/// `max` is the number of best known block.
/// Changes are returned in descending order (i.e. last block comes first).
pub fn key_changes<'a, H: Hasher, Number: BlockNumber>(
config: &'a Configuration,
config: ConfigurationRange<'a, Number>,
storage: &'a dyn Storage<H, Number>,
zero: Number,
begin: Number,
end: &'a AnchorBlockId<H::Out, Number>,
max: Number,
Expand All @@ -53,7 +52,6 @@ pub fn key_changes<'a, H: Hasher, Number: BlockNumber>(
end,
surface: surface_iterator(
config,
zero,
max,
begin,
end.number.clone(),
Expand All @@ -69,10 +67,9 @@ pub fn key_changes<'a, H: Hasher, Number: BlockNumber>(

/// Returns proof of changes of given key at given blocks range.
/// `max` is the number of best known block.
pub fn key_changes_proof<H: Hasher, Number: BlockNumber>(
config: &Configuration,
pub fn key_changes_proof<'a, H: Hasher, Number: BlockNumber>(
config: ConfigurationRange<'a, Number>,
storage: &dyn Storage<H, Number>,
zero: Number,
begin: Number,
end: &AnchorBlockId<H::Out, Number>,
max: Number,
Expand All @@ -90,7 +87,6 @@ pub fn key_changes_proof<H: Hasher, Number: BlockNumber>(
end,
surface: surface_iterator(
config,
zero,
max,
begin,
end.number.clone(),
Expand All @@ -115,11 +111,10 @@ pub fn key_changes_proof<H: Hasher, Number: BlockNumber>(
/// Check key changes proof and return changes of the key at given blocks range.
/// `max` is the number of best known block.
/// Changes are returned in descending order (i.e. last block comes first).
pub fn key_changes_proof_check<H: Hasher, Number: BlockNumber>(
config: &Configuration,
pub fn key_changes_proof_check<'a, H: Hasher, Number: BlockNumber>(
config: ConfigurationRange<'a, Number>,
roots_storage: &dyn RootsStorage<H, Number>,
proof: Vec<Vec<u8>>,
zero: Number,
begin: Number,
end: &AnchorBlockId<H::Out, Number>,
max: Number,
Expand All @@ -143,7 +138,6 @@ pub fn key_changes_proof_check<H: Hasher, Number: BlockNumber>(
end,
surface: surface_iterator(
config,
zero,
max,
begin,
end.number.clone(),
Expand All @@ -160,8 +154,7 @@ pub fn key_changes_proof_check<H: Hasher, Number: BlockNumber>(
/// Surface iterator - only traverses top-level digests from given range and tries to find
/// all digest changes for the key.
pub struct SurfaceIterator<'a, Number: BlockNumber> {
config: &'a Configuration,
zero: Number,
config: ConfigurationRange<'a, Number>,
begin: Number,
max: Number,
current: Option<Number>,
Expand Down Expand Up @@ -189,8 +182,7 @@ impl<'a, Number: BlockNumber> Iterator for SurfaceIterator<'a, Number> {
self.current = Some(next);
} else {
let max_digest_interval = lower_bound_max_digest(
self.config,
self.zero.clone(),
self.config.clone(),
self.max.clone(),
self.begin.clone(),
next,
Expand Down Expand Up @@ -378,22 +370,19 @@ impl<'a, H, Number> Iterator for ProvingDrilldownIterator<'a, H, Number>

/// Returns surface iterator for given range of blocks.
fn surface_iterator<'a, Number: BlockNumber>(
config: &'a Configuration,
zero: Number,
config: ConfigurationRange<'a, Number>,
max: Number,
begin: Number,
end: Number,
) -> Result<SurfaceIterator<'a, Number>, String> {
let (current, current_begin, digest_step, digest_level) = lower_bound_max_digest(
config,
zero.clone(),
config.clone(),
max.clone(),
begin.clone(),
end,
)?;
Ok(SurfaceIterator {
config,
zero,
begin,
max,
current: Some(current),
Expand All @@ -405,9 +394,8 @@ fn surface_iterator<'a, Number: BlockNumber>(

/// Returns parameters of highest level digest block that includes the end of given range
/// and tends to include the whole range.
fn lower_bound_max_digest<Number: BlockNumber>(
config: &Configuration,
zero: Number,
fn lower_bound_max_digest<'a, Number: BlockNumber>(
config: ConfigurationRange<'a, Number>,
max: Number,
begin: Number,
end: Number,
Expand All @@ -422,13 +410,13 @@ fn lower_bound_max_digest<Number: BlockNumber>(
let mut current = end.clone();
let mut current_begin = begin.clone();
if current_begin != current {
while digest_level != config.digest_levels {
while digest_level != config.config.digest_levels {
let new_digest_level = digest_level + 1;
let new_digest_step = digest_step * config.digest_interval;
let new_digest_interval = config.digest_interval * {
let new_digest_step = digest_step * config.config.digest_interval;
let new_digest_interval = config.config.digest_interval * {
if digest_interval == 0 { 1 } else { digest_interval }
};
let new_digest_begin = zero.clone() + ((current.clone() - One::one() - zero.clone())
let new_digest_begin = config.zero.clone() + ((current.clone() - One::one() - config.zero.clone())
/ new_digest_interval.into()) * new_digest_interval.into();
let new_digest_end = new_digest_begin.clone() + new_digest_interval.into();
let new_current = new_digest_begin.clone() + new_digest_interval.into();
Expand Down Expand Up @@ -464,6 +452,7 @@ fn lower_bound_max_digest<Number: BlockNumber>(
mod tests {
use std::iter::FromIterator;
use primitives::Blake2Hasher;
use crate::changes_trie::Configuration;
use crate::changes_trie::input::InputPair;
use crate::changes_trie::storage::InMemoryStorage;
use super::*;
Expand Down Expand Up @@ -507,19 +496,27 @@ mod tests {
(config, backend)
}

fn configuration_range<'a>(config: &'a Configuration, zero: u64) -> ConfigurationRange<'a, u64> {
ConfigurationRange {
config,
zero,
end: None,
}
}

#[test]
fn lower_bound_max_digest_works() {
let config = Configuration { digest_interval: 4, digest_levels: 2 };

// when config activates at 0
assert_eq!(
lower_bound_max_digest(&config, 0u64, 100_000u64, 20u64, 180u64).unwrap(),
lower_bound_max_digest(configuration_range(&config, 0u64), 100_000u64, 20u64, 180u64).unwrap(),
(192, 176, 16, 2),
);

// when config activates at 30
assert_eq!(
lower_bound_max_digest(&config, 30u64, 100_000u64, 20u64, 180u64).unwrap(),
lower_bound_max_digest(configuration_range(&config, 30u64), 100_000u64, 20u64, 180u64).unwrap(),
(190, 174, 16, 2),
);
}
Expand All @@ -530,7 +527,7 @@ mod tests {

// when config activates at 0
assert_eq!(
surface_iterator(&config, 0u64, 100_000u64, 40u64, 180u64).unwrap().collect::<Vec<_>>(),
surface_iterator(configuration_range(&config, 0u64), 100_000u64, 40u64, 180u64).unwrap().collect::<Vec<_>>(),
vec![
Ok((192, 2)), Ok((176, 2)), Ok((160, 2)), Ok((144, 2)), Ok((128, 2)), Ok((112, 2)),
Ok((96, 2)), Ok((80, 2)), Ok((64, 2)), Ok((48, 2)),
Expand All @@ -539,7 +536,7 @@ mod tests {

// when config activates at 30
assert_eq!(
surface_iterator(&config, 30u64, 100_000u64, 40u64, 180u64).unwrap().collect::<Vec<_>>(),
surface_iterator(configuration_range(&config, 30u64), 100_000u64, 40u64, 180u64).unwrap().collect::<Vec<_>>(),
vec![
Ok((190, 2)), Ok((174, 2)), Ok((158, 2)), Ok((142, 2)), Ok((126, 2)), Ok((110, 2)),
Ok((94, 2)), Ok((78, 2)), Ok((62, 2)), Ok((46, 2)),
Expand All @@ -551,27 +548,27 @@ mod tests {
fn drilldown_iterator_works() {
let (config, storage) = prepare_for_drilldown();
let drilldown_result = key_changes::<Blake2Hasher, u64>(
&config, &storage, 0, 0, &AnchorBlockId { hash: Default::default(), number: 16 }, 16, &[42])
configuration_range(&config, 0), &storage, 0, &AnchorBlockId { hash: Default::default(), number: 16 }, 16, &[42])
.and_then(Result::from_iter);
assert_eq!(drilldown_result, Ok(vec![(8, 2), (8, 1), (6, 3), (3, 0)]));

let drilldown_result = key_changes::<Blake2Hasher, u64>(
&config, &storage, 0, 0, &AnchorBlockId { hash: Default::default(), number: 2 }, 4, &[42])
configuration_range(&config, 0), &storage, 0, &AnchorBlockId { hash: Default::default(), number: 2 }, 4, &[42])
.and_then(Result::from_iter);
assert_eq!(drilldown_result, Ok(vec![]));

let drilldown_result = key_changes::<Blake2Hasher, u64>(
&config, &storage, 0, 0, &AnchorBlockId { hash: Default::default(), number: 3 }, 4, &[42])
configuration_range(&config, 0), &storage, 0, &AnchorBlockId { hash: Default::default(), number: 3 }, 4, &[42])
.and_then(Result::from_iter);
assert_eq!(drilldown_result, Ok(vec![(3, 0)]));

let drilldown_result = key_changes::<Blake2Hasher, u64>(
&config, &storage, 0, 7, &AnchorBlockId { hash: Default::default(), number: 8 }, 8, &[42])
configuration_range(&config, 0), &storage, 7, &AnchorBlockId { hash: Default::default(), number: 8 }, 8, &[42])
.and_then(Result::from_iter);
assert_eq!(drilldown_result, Ok(vec![(8, 2), (8, 1)]));

let drilldown_result = key_changes::<Blake2Hasher, u64>(
&config, &storage, 0, 5, &AnchorBlockId { hash: Default::default(), number: 7 }, 8, &[42])
configuration_range(&config, 0), &storage, 5, &AnchorBlockId { hash: Default::default(), number: 7 }, 8, &[42])
.and_then(Result::from_iter);
assert_eq!(drilldown_result, Ok(vec![(6, 3)]));
}
Expand All @@ -582,17 +579,17 @@ mod tests {
storage.clear_storage();

assert!(key_changes::<Blake2Hasher, u64>(
&config, &storage, 0, 0, &AnchorBlockId { hash: Default::default(), number: 100 }, 1000, &[42])
configuration_range(&config, 0), &storage, 0, &AnchorBlockId { hash: Default::default(), number: 100 }, 1000, &[42])
.and_then(|i| i.collect::<Result<Vec<_>, _>>()).is_err());
}

#[test]
fn drilldown_iterator_fails_when_range_is_invalid() {
let (config, storage) = prepare_for_drilldown();
assert!(key_changes::<Blake2Hasher, u64>(
&config, &storage, 0, 0, &AnchorBlockId { hash: Default::default(), number: 100 }, 50, &[42]).is_err());
configuration_range(&config, 0), &storage, 0, &AnchorBlockId { hash: Default::default(), number: 100 }, 50, &[42]).is_err());
assert!(key_changes::<Blake2Hasher, u64>(
&config, &storage, 0, 20, &AnchorBlockId { hash: Default::default(), number: 10 }, 100, &[42]).is_err());
configuration_range(&config, 0), &storage, 20, &AnchorBlockId { hash: Default::default(), number: 10 }, 100, &[42]).is_err());
}


Expand All @@ -603,17 +600,17 @@ mod tests {
// create drilldown iterator that records all trie nodes during drilldown
let (remote_config, remote_storage) = prepare_for_drilldown();
let remote_proof = key_changes_proof::<Blake2Hasher, u64>(
&remote_config, &remote_storage,
0, 0, &AnchorBlockId { hash: Default::default(), number: 16 }, 16, &[42]).unwrap();
configuration_range(&remote_config, 0), &remote_storage,
0, &AnchorBlockId { hash: Default::default(), number: 16 }, 16, &[42]).unwrap();

// happens on local light node:

// create drilldown iterator that works the same, but only depends on trie
let (local_config, local_storage) = prepare_for_drilldown();
local_storage.clear_storage();
let local_result = key_changes_proof_check::<Blake2Hasher, u64>(
&local_config, &local_storage, remote_proof,
0, 0, &AnchorBlockId { hash: Default::default(), number: 16 }, 16, &[42]);
configuration_range(&local_config, 0), &local_storage, remote_proof,
0, &AnchorBlockId { hash: Default::default(), number: 16 }, 16, &[42]);

// check that drilldown result is the same as if it was happening at the full node
assert_eq!(local_result, Ok(vec![(8, 2), (8, 1), (6, 3), (3, 0)]));
Expand Down
11 changes: 11 additions & 0 deletions core/state-machine/src/changes_trie/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,17 @@ impl<'a, H: Hasher, N: BlockNumber> crate::TrieBackendStorage<H> for TrieBackend
/// Changes trie configuration.
pub type Configuration = primitives::ChangesTrieConfiguration;

/// Blocks range where configuration has been constant.
#[derive(Clone)]
pub struct ConfigurationRange<'a, N> {
/// Active configuration.
pub config: &'a Configuration,
/// Zero block of this configuration. The configuration is active starting from the next block.
pub zero: N,
/// End block of this configuration. It is the last block where configuration has been active.
pub end: Option<N>,
}

impl<'a, H, Number> State<'a, H, Number> {
/// Create state with given config and storage.
pub fn new(
Expand Down