diff --git a/Cargo.lock b/Cargo.lock index 7754e0ae6b62f..37773a4ae3a69 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2314,6 +2314,21 @@ version = "0.3.55" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f5f3913fa0bfe7ee1fd8248b6b9f42a5af4b9d65ec2dd2c3c26132b950ecfc2" +[[package]] +name = "generate-bags" +version = "3.0.0" +dependencies = [ + "chrono", + "frame-election-provider-support", + "frame-support", + "frame-system", + "git2", + "num-format", + "pallet-staking", + "sp-io", + "structopt", +] + [[package]] name = "generic-array" version = "0.12.4" @@ -2395,6 +2410,19 @@ dependencies = [ "stable_deref_trait", ] +[[package]] +name = "git2" +version = "0.13.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "659cd14835e75b64d9dba5b660463506763cf0aa6cb640aeeb0e98d841093490" +dependencies = [ + "bitflags", + "libc", + "libgit2-sys", + "log 0.4.14", + "url 2.2.1", +] + [[package]] name = "glob" version = "0.3.0" @@ -3203,6 +3231,18 @@ version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "789da6d93f1b866ffe175afc5322a4d76c038605a1c3319bb57b06967ca98a36" +[[package]] +name = "libgit2-sys" +version = "0.12.22+1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89c53ac117c44f7042ad8d8f5681378dfbc6010e49ec2c0d1f11dfedc7a4a1c3" +dependencies = [ + "cc", + "libc", + "libz-sys", + "pkg-config", +] + [[package]] name = "libloading" version = "0.5.2" @@ -3775,6 +3815,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "602113192b08db8f38796c4e85c39e960c145965140e918018bcde1952429655" dependencies = [ "cc", + "libc", "pkg-config", "vcpkg", ] @@ -4530,6 +4571,7 @@ dependencies = [ "pallet-authority-discovery", "pallet-authorship", "pallet-babe", + "pallet-bags-list", "pallet-balances", "pallet-bounties", "pallet-collective", @@ -4591,6 +4633,15 @@ dependencies = [ "substrate-wasm-builder", ] +[[package]] +name = "node-runtime-generate-bags" +version = "3.0.0" +dependencies = [ + "generate-bags", + "node-runtime", + "structopt", +] + [[package]] name = "node-template" version = "3.0.0" @@ -4750,6 +4801,16 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-format" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bafe4179722c2894288ee77a9f044f02811c86af699344c498b0840c698a2465" +dependencies = [ + "arrayvec 0.4.12", + "itoa", +] + [[package]] name = "num-integer" version = "0.1.44" @@ -5017,6 +5078,25 @@ dependencies = [ "sp-std", ] +[[package]] +name = "pallet-bags-list" +version = "4.0.0-dev" +dependencies = [ + "frame-benchmarking", + "frame-election-provider-support", + "frame-support", + "frame-system", + "log 0.4.14", + "pallet-balances", + "parity-scale-codec", + "scale-info", + "sp-core", + "sp-io", + "sp-runtime", + "sp-std", + "sp-tracing", +] + [[package]] name = "pallet-balances" version = "4.0.0-dev" @@ -5707,6 +5787,7 @@ dependencies = [ "frame-system", "log 0.4.14", "pallet-authorship", + "pallet-bags-list", "pallet-balances", "pallet-session", "pallet-staking-reward-curve", @@ -5718,6 +5799,7 @@ dependencies = [ "sp-application-crypto", "sp-core", "sp-io", + "sp-npos-elections", "sp-runtime", "sp-staking", "sp-std", diff --git a/Cargo.toml b/Cargo.toml index bca0c816217ee..e110c27b20d77 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -128,6 +128,7 @@ members = [ "frame/uniques", "frame/utility", "frame/vesting", + "frame/bags-list", "primitives/api", "primitives/api/proc-macro", "primitives/api/test", @@ -198,6 +199,8 @@ members = [ "utils/frame/try-runtime/cli", "utils/frame/rpc/support", "utils/frame/rpc/system", + "utils/frame/generate-bags", + "utils/frame/generate-bags/node-runtime", "utils/prometheus", "utils/wasm-builder", ] diff --git a/bin/node/runtime/Cargo.toml b/bin/node/runtime/Cargo.toml index dafd9db8bab96..d434be8f3c609 100644 --- a/bin/node/runtime/Cargo.toml +++ b/bin/node/runtime/Cargo.toml @@ -55,6 +55,7 @@ pallet-assets = { version = "4.0.0-dev", default-features = false, path = "../.. pallet-authority-discovery = { version = "4.0.0-dev", default-features = false, path = "../../../frame/authority-discovery" } pallet-authorship = { version = "4.0.0-dev", default-features = false, path = "../../../frame/authorship" } pallet-babe = { version = "4.0.0-dev", default-features = false, path = "../../../frame/babe" } +pallet-bags-list = { version = "4.0.0-dev", default-features = false, path = "../../../frame/bags-list" } pallet-balances = { version = "4.0.0-dev", default-features = false, path = "../../../frame/balances" } pallet-bounties = { version = "4.0.0-dev", default-features = false, path = "../../../frame/bounties" } pallet-collective = { version = "4.0.0-dev", default-features = false, path = "../../../frame/collective" } @@ -110,6 +111,7 @@ std = [ "pallet-authorship/std", "sp-consensus-babe/std", "pallet-babe/std", + "pallet-bags-list/std", "pallet-balances/std", "pallet-bounties/std", "sp-block-builder/std", @@ -179,6 +181,7 @@ runtime-benchmarks = [ "sp-runtime/runtime-benchmarks", "pallet-assets/runtime-benchmarks", "pallet-babe/runtime-benchmarks", + "pallet-bags-list/runtime-benchmarks", "pallet-balances/runtime-benchmarks", "pallet-bounties/runtime-benchmarks", "pallet-collective/runtime-benchmarks", diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index 7c6475bd18d6a..9e8adfcd0910e 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -92,6 +92,9 @@ pub mod constants; use constants::{currency::*, time::*}; use sp_runtime::generic::Era; +/// Generated voter bag information. +mod voter_bags; + // Make the WASM binary available. #[cfg(feature = "std")] include!(concat!(env!("OUT_DIR"), "/wasm_binary.rs")); @@ -525,6 +528,9 @@ impl pallet_staking::Config for Runtime { type MaxNominatorRewardedPerValidator = MaxNominatorRewardedPerValidator; type ElectionProvider = ElectionProviderMultiPhase; type GenesisElectionProvider = onchain::OnChainSequentialPhragmen; + // Alternatively, use pallet_staking::UseNominatorsMap to just use the nominators map. + // Note that the aforementioned does not scale to a very large number of nominators. + type SortedListProvider = BagsList; type WeightInfo = pallet_staking::weights::SubstrateWeight; } @@ -552,6 +558,11 @@ parameter_types! { *RuntimeBlockLength::get() .max .get(DispatchClass::Normal); + + // BagsList allows a practically unbounded count of nominators to participate in NPoS elections. + // To ensure we respect memory limits when using the BagsList this must be set to a number of + // voters we know can fit into a single vec allocation. + pub const VoterSnapshotPerBlock: u32 = 10_000; } sp_npos_elections::generate_solution_type!( @@ -634,6 +645,18 @@ impl pallet_election_provider_multi_phase::Config for Runtime { type WeightInfo = pallet_election_provider_multi_phase::weights::SubstrateWeight; type ForceOrigin = EnsureRootOrHalfCouncil; type BenchmarkingConfig = BenchmarkConfig; + type VoterSnapshotPerBlock = VoterSnapshotPerBlock; +} + +parameter_types! { + pub const BagThresholds: &'static [u64] = &voter_bags::THRESHOLDS; +} + +impl pallet_bags_list::Config for Runtime { + type Event = Event; + type VoteWeightProvider = Staking; + type WeightInfo = pallet_bags_list::weights::SubstrateWeight; + type BagThresholds = BagThresholds; } parameter_types! { @@ -1254,6 +1277,7 @@ construct_runtime!( Gilt: pallet_gilt::{Pallet, Call, Storage, Event, Config}, Uniques: pallet_uniques::{Pallet, Call, Storage, Event}, TransactionStorage: pallet_transaction_storage::{Pallet, Call, Storage, Inherent, Config, Event}, + BagsList: pallet_bags_list::{Pallet, Call, Storage, Event}, } ); @@ -1581,6 +1605,7 @@ impl_runtime_apis! { list_benchmark!(list, extra, pallet_assets, Assets); list_benchmark!(list, extra, pallet_babe, Babe); + list_benchmark!(list, extra, pallet_bags_list, BagsList); list_benchmark!(list, extra, pallet_balances, Balances); list_benchmark!(list, extra, pallet_bounties, Bounties); list_benchmark!(list, extra, pallet_collective, Council); @@ -1655,6 +1680,7 @@ impl_runtime_apis! { add_benchmark!(params, batches, pallet_assets, Assets); add_benchmark!(params, batches, pallet_babe, Babe); add_benchmark!(params, batches, pallet_balances, Balances); + add_benchmark!(params, batches, pallet_bags_list, BagsList); add_benchmark!(params, batches, pallet_bounties, Bounties); add_benchmark!(params, batches, pallet_collective, Council); add_benchmark!(params, batches, pallet_contracts, Contracts); diff --git a/bin/node/runtime/src/voter_bags.rs b/bin/node/runtime/src/voter_bags.rs new file mode 100644 index 0000000000000..c4c731a58badc --- /dev/null +++ b/bin/node/runtime/src/voter_bags.rs @@ -0,0 +1,235 @@ +// This file is part of Substrate. + +// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Autogenerated voter bag thresholds. +//! +//! Generated on 2021-07-05T09:17:40.469754927+00:00 +//! for the node runtime. + +/// Existential weight for this runtime. +#[cfg(any(test, feature = "std"))] +#[allow(unused)] +pub const EXISTENTIAL_WEIGHT: u64 = 100_000_000_000_000; + +/// Constant ratio between bags for this runtime. +#[cfg(any(test, feature = "std"))] +#[allow(unused)] +pub const CONSTANT_RATIO: f64 = 1.0628253590743408; + +/// Upper thresholds delimiting the bag list. +pub const THRESHOLDS: [u64; 200] = [ + 100_000_000_000_000, + 106_282_535_907_434, + 112_959_774_389_150, + 120_056_512_776_105, + 127_599_106_300_477, + 135_615_565_971_369, + 144_135_662_599_590, + 153_191_037_357_827, + 162_815_319_286_803, + 173_044_250_183_800, + 183_915_817_337_347, + 195_470_394_601_017, + 207_750_892_330_229, + 220_802_916_738_890, + 234_674_939_267_673, + 249_418_476_592_914, + 265_088_281_944_639, + 281_742_548_444_211, + 299_443_125_216_738, + 318_255_747_080_822, + 338_250_278_668_647, + 359_500_973_883_001, + 382_086_751_654_776, + 406_091_489_025_036, + 431_604_332_640_068, + 458_720_029_816_222, + 487_539_280_404_019, + 518_169_110_758_247, + 550_723_271_202_866, + 585_322_658_466_782, + 622_095_764_659_305, + 661_179_154_452_653, + 702_717_972_243_610, + 746_866_481_177_808, + 793_788_636_038_393, + 843_658_692_126_636, + 896_661_852_395_681, + 952_994_955_240_703, + 1_012_867_205_499_736, + 1_076_500_951_379_881, + 1_144_132_510_194_192, + 1_216_013_045_975_769, + 1_292_409_502_228_280, + 1_373_605_593_276_862, + 1_459_902_857_901_004, + 1_551_621_779_162_291, + 1_649_102_974_585_730, + 1_752_708_461_114_642, + 1_862_822_999_536_805, + 1_979_855_523_374_646, + 2_104_240_657_545_975, + 2_236_440_332_435_128, + 2_376_945_499_368_703, + 2_526_277_953_866_680, + 2_684_992_273_439_945, + 2_853_677_877_130_641, + 3_032_961_214_443_876, + 3_223_508_091_799_862, + 3_426_026_145_146_232, + 3_641_267_467_913_124, + 3_870_031_404_070_482, + 4_113_167_516_660_186, + 4_371_578_742_827_277, + 4_646_224_747_067_156, + 4_938_125_485_141_739, + 5_248_364_991_899_922, + 5_578_095_407_069_235, + 5_928_541_253_969_291, + 6_301_003_987_036_955, + 6_696_866_825_051_405, + 7_117_599_888_008_300, + 7_564_765_656_719_910, + 8_040_024_775_416_580, + 8_545_142_218_898_723, + 9_081_993_847_142_344, + 9_652_573_371_700_016, + 10_258_999_759_768_490, + 10_903_525_103_419_522, + 11_588_542_983_217_942, + 12_316_597_357_287_042, + 13_090_392_008_832_678, + 13_912_800_587_211_472, + 14_786_877_279_832_732, + 15_715_868_154_526_436, + 16_703_223_214_499_558, + 17_752_609_210_649_358, + 18_867_923_258_814_856, + 20_053_307_312_537_008, + 21_313_163_545_075_252, + 22_652_170_697_804_756, + 24_075_301_455_707_600, + 25_587_840_914_485_432, + 27_195_406_207_875_088, + 28_903_967_368_057_400, + 30_719_869_496_628_636, + 32_649_856_328_471_220, + 34_701_095_276_033_064, + 36_881_204_047_022_752, + 39_198_278_934_370_992, + 41_660_924_883_519_016, + 44_278_287_448_695_240, + 47_060_086_756_856_400, + 50_016_653_605_425_536, + 53_158_967_827_883_320, + 56_498_699_069_691_424, + 60_048_250_125_977_912, + 63_820_803_001_928_304, + 67_830_367_866_937_216, + 72_091_835_084_322_176, + 76_621_030_509_822_880, + 81_434_774_264_248_528, + 86_550_943_198_537_824, + 91_988_537_283_208_848, + 97_767_750_168_749_840, + 103_910_044_178_992_000, + 110_438_230_015_967_792, + 117_376_551_472_255_616, + 124_750_775_465_407_920, + 132_588_287_728_824_640, + 140_918_194_514_440_064, + 149_771_430_684_917_568, + 159_180_874_596_775_264, + 169_181_470_201_085_280, + 179_810_356_815_193_344, + 191_107_007_047_393_216, + 203_113_373_386_768_288, + 215_874_044_002_592_672, + 229_436_408_331_885_600, + 243_850_833_070_063_392, + 259_170_849_218_267_264, + 275_453_350_882_006_752, + 292_758_806_559_399_232, + 311_151_483_703_668_992, + 330_699_687_393_865_920, + 351_476_014_000_157_824, + 373_557_620_785_735_808, + 397_026_512_446_556_096, + 421_969_845_653_044_224, + 448_480_252_724_740_928, + 476_656_185_639_923_904, + 506_602_281_657_757_760, + 538_429_751_910_786_752, + 572_256_794_410_890_176, + 608_209_033_002_485_632, + 646_419_983_893_124_352, + 687_031_551_494_039_552, + 730_194_555_412_054_016, + 776_069_290_549_944_960, + 824_826_122_395_314_176, + 876_646_119_708_695_936, + 931_721_726_960_522_368, + 990_257_479_014_182_144, + 1_052_470_760_709_299_712, + 1_118_592_614_166_106_112, + 1_188_868_596_808_997_376, + 1_263_559_693_295_730_432, + 1_342_943_284_738_898_688, + 1_427_314_178_819_094_784, + 1_516_985_704_615_302_400, + 1_612_290_876_218_400_768, + 1_713_583_629_449_105_408, + 1_821_240_136_273_157_632, + 1_935_660_201_795_120_128, + 2_057_268_749_018_809_600, + 2_186_517_396_888_336_384, + 2_323_886_137_470_138_880, + 2_469_885_118_504_583_168, + 2_625_056_537_947_004_416, + 2_789_976_657_533_970_944, + 2_965_257_942_852_572_160, + 3_151_551_337_860_326_400, + 3_349_548_682_302_620_672, + 3_559_985_281_005_267_968, + 3_783_642_634_583_792_128, + 4_021_351_341_710_503_936, + 4_273_994_183_717_548_544, + 4_542_509_402_991_247_872, + 4_827_894_187_332_742_144, + 5_131_208_373_224_844_288, + 5_453_578_381_757_959_168, + 5_796_201_401_831_965_696, + 6_160_349_836_169_256_960, + 6_547_376_026_650_146_816, + 6_958_717_276_519_173_120, + 7_395_901_188_113_309_696, + 7_860_551_335_934_872_576, + 8_354_393_296_137_270_272, + 8_879_261_054_815_360_000, + 9_437_103_818_898_946_048, + 10_029_993_254_943_105_024, + 10_660_131_182_698_121_216, + 11_329_857_752_030_707_712, + 12_041_660_133_563_240_448, + 12_798_181_755_305_525_248, + 13_602_232_119_581_272_064, + 14_456_797_236_706_498_560, + 15_365_050_714_167_523_328, + 16_330_365_542_480_556_032, + 17_356_326_621_502_140_416, + 18_446_744_073_709_551_615, +]; diff --git a/frame/babe/src/mock.rs b/frame/babe/src/mock.rs index bc0be32624cba..833a68fbddb6c 100644 --- a/frame/babe/src/mock.rs +++ b/frame/babe/src/mock.rs @@ -216,6 +216,7 @@ impl pallet_staking::Config for Test { type ElectionProvider = onchain::OnChainSequentialPhragmen; type GenesisElectionProvider = Self::ElectionProvider; type WeightInfo = (); + type SortedListProvider = pallet_staking::UseNominatorsMap; } impl pallet_offences::Config for Test { diff --git a/frame/bags-list/Cargo.toml b/frame/bags-list/Cargo.toml new file mode 100644 index 0000000000000..860a6edc42143 --- /dev/null +++ b/frame/bags-list/Cargo.toml @@ -0,0 +1,66 @@ +[package] +name = "pallet-bags-list" +version = "4.0.0-dev" +authors = ["Parity Technologies "] +edition = "2018" +license = "Apache-2.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "FRAME pallet bags list" +readme = "README.md" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + +[dependencies] +# parity +codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false, features = ["derive"] } +scale-info = { version = "1.0", default-features = false, features = ["derive"] } + +# primitives +sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../primitives/runtime" } +sp-std = { version = "4.0.0-dev", default-features = false, path = "../../primitives/std" } + +# FRAME +frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } +frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" } +frame-election-provider-support = { version = "4.0.0-dev", default-features = false, path = "../election-provider-support" } + +# third party +log = { version = "0.4.14", default-features = false } + +# Optional imports for benchmarking +frame-benchmarking = { version = "4.0.0-dev", path = "../benchmarking", optional = true, default-features = false } +pallet-balances = { version = "4.0.0-dev", path = "../balances", optional = true, default-features = false } +sp-core = { version = "4.0.0-dev", path = "../../primitives/core", optional = true, default-features = false } +sp-io = { version = "4.0.0-dev", path = "../../primitives/io", optional = true, default-features = false } +sp-tracing = { version = "4.0.0-dev", path = "../../primitives/tracing", optional = true, default-features = false } + +[dev-dependencies] +sp-core = { version = "4.0.0-dev", path = "../../primitives/core"} +sp-io = { version = "4.0.0-dev", path = "../../primitives/io"} +sp-tracing = { version = "4.0.0-dev", path = "../../primitives/tracing" } +pallet-balances = { version = "4.0.0-dev", path = "../balances" } +frame-election-provider-support = { version = "4.0.0-dev", path = "../election-provider-support", features = ["runtime-benchmarks"] } +frame-benchmarking = { version = "4.0.0-dev", path = "../benchmarking" } + +[features] +default = ["std"] +std = [ + "codec/std", + "sp-runtime/std", + "sp-std/std", + "frame-support/std", + "frame-system/std", + "frame-election-provider-support/std", + "log/std", +] +runtime-benchmarks = [ + "frame-benchmarking", + "sp-core", + "sp-io", + "pallet-balances", + "sp-tracing", + "frame-election-provider-support/runtime-benchmarks", +] + diff --git a/frame/bags-list/src/benchmarks.rs b/frame/bags-list/src/benchmarks.rs new file mode 100644 index 0000000000000..a820eeba13b12 --- /dev/null +++ b/frame/bags-list/src/benchmarks.rs @@ -0,0 +1,144 @@ +// This file is part of Substrate. + +// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Benchmarks for the bags list pallet. + +use super::*; +use crate::list::List; +use frame_benchmarking::{account, whitelisted_caller}; +use frame_election_provider_support::VoteWeightProvider; +use frame_support::{assert_ok, traits::Get}; +use frame_system::RawOrigin as SystemOrigin; + +frame_benchmarking::benchmarks! { + rebag_non_terminal { + // An expensive case for rebag-ing (rebag a non-terminal node): + // + // - The node to be rebagged, _R_, should exist as a non-terminal node in a bag with at + // least 2 other nodes. Thus _R_ will have both its `prev` and `next` nodes updated when + // it is removed. (3 W/R) + // - The destination bag is not empty, thus we need to update the `next` pointer of the last + // node in the destination in addition to the work we do otherwise. (2 W/R) + + // clear any pre-existing storage. + List::::clear(None); + + // define our origin and destination thresholds. + let origin_bag_thresh = T::BagThresholds::get()[0]; + let dest_bag_thresh = T::BagThresholds::get()[1]; + + // seed items in the origin bag. + let origin_head: T::AccountId = account("origin_head", 0, 0); + assert_ok!(List::::insert(origin_head.clone(), origin_bag_thresh)); + + let origin_middle: T::AccountId = account("origin_middle", 0, 0); // the node we rebag (_R_) + assert_ok!(List::::insert(origin_middle.clone(), origin_bag_thresh)); + + let origin_tail: T::AccountId = account("origin_tail", 0, 0); + assert_ok!(List::::insert(origin_tail.clone(), origin_bag_thresh)); + + // seed items in the destination bag. + let dest_head: T::AccountId = account("dest_head", 0, 0); + assert_ok!(List::::insert(dest_head.clone(), dest_bag_thresh)); + + // the bags are in the expected state after initial setup. + assert_eq!( + List::::get_bags(), + vec![ + (origin_bag_thresh, vec![origin_head.clone(), origin_middle.clone(), origin_tail.clone()]), + (dest_bag_thresh, vec![dest_head.clone()]) + ] + ); + + let caller = whitelisted_caller(); + // update the weight of `origin_middle` to guarantee it will be rebagged into the destination. + T::VoteWeightProvider::set_vote_weight_of(&origin_middle, dest_bag_thresh); + }: rebag(SystemOrigin::Signed(caller), origin_middle.clone()) + verify { + // check the bags have updated as expected. + assert_eq!( + List::::get_bags(), + vec![ + ( + origin_bag_thresh, + vec![origin_head, origin_tail], + ), + ( + dest_bag_thresh, + vec![dest_head, origin_middle], + ) + ] + ); + } + + rebag_terminal { + // An expensive case for rebag-ing (rebag a terminal node): + // + // - The node to be rebagged, _R_, is a terminal node; so _R_, the node pointing to _R_ and + // the origin bag itself will need to be updated. (3 W/R) + // - The destination bag is not empty, thus we need to update the `next` pointer of the last + // node in the destination in addition to the work we do otherwise. (2 W/R) + + // clear any pre-existing storage. + List::::clear(None); + + // define our origin and destination thresholds. + let origin_bag_thresh = T::BagThresholds::get()[0]; + let dest_bag_thresh = T::BagThresholds::get()[1]; + + // seed items in the origin bag. + let origin_head: T::AccountId = account("origin_head", 0, 0); + assert_ok!(List::::insert(origin_head.clone(), origin_bag_thresh)); + + let origin_tail: T::AccountId = account("origin_tail", 0, 0); // the node we rebag (_R_) + assert_ok!(List::::insert(origin_tail.clone(), origin_bag_thresh)); + + // seed items in the destination bag. + let dest_head: T::AccountId = account("dest_head", 0, 0); + assert_ok!(List::::insert(dest_head.clone(), dest_bag_thresh)); + + // the bags are in the expected state after initial setup. + assert_eq!( + List::::get_bags(), + vec![ + (origin_bag_thresh, vec![origin_head.clone(), origin_tail.clone()]), + (dest_bag_thresh, vec![dest_head.clone()]) + ] + ); + + let caller = whitelisted_caller(); + // update the weight of `origin_tail` to guarantee it will be rebagged into the destination. + T::VoteWeightProvider::set_vote_weight_of(&origin_tail, dest_bag_thresh); + }: rebag(SystemOrigin::Signed(caller), origin_tail.clone()) + verify { + // check the bags have updated as expected. + assert_eq!( + List::::get_bags(), + vec![ + (origin_bag_thresh, vec![origin_head.clone()]), + (dest_bag_thresh, vec![dest_head.clone(), origin_tail.clone()]) + ] + ); + } +} + +use frame_benchmarking::impl_benchmark_test_suite; +impl_benchmark_test_suite!( + Pallet, + crate::mock::ExtBuilder::default().build(), + crate::mock::Runtime, +); diff --git a/frame/bags-list/src/lib.rs b/frame/bags-list/src/lib.rs new file mode 100644 index 0000000000000..4202a4d499895 --- /dev/null +++ b/frame/bags-list/src/lib.rs @@ -0,0 +1,306 @@ +// This file is part of Substrate. + +// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! # Bags-List Pallet +//! +//! A semi-sorted list, where items hold an `AccountId` based on some `VoteWeight`. The `AccountId` +//! (`id` for short) might be synonym to a `voter` or `nominator` in some context, and `VoteWeight` +//! signifies the chance of each id being included in the final [`VoteWeightProvider::iter`]. +//! +//! It implements [`sp_election_provider_support::SortedListProvider`] to provide a semi-sorted list +//! of accounts to another pallet. It needs some other pallet to give it some information about the +//! weights of accounts via [`sp_election_provider_support::VoteWeightProvider`]. +//! +//! This pallet is not configurable at genesis. Whoever uses it should call appropriate functions of +//! the `SortedListProvider` (e.g. `on_insert`, or `regenerate`) at their genesis. +//! +//! # Goals +//! +//! The data structure exposed by this pallet aims to be optimized for: +//! +//! - insertions and removals. +//! - iteration over the top* N items by weight, where the precise ordering of items doesn't +//! particularly matter. +//! +//! # Details +//! +//! - items are kept in bags, which are delineated by their range of weight (See [`BagThresholds`]). +//! - for iteration, bags are chained together from highest to lowest and elements within the bag +//! are iterated from head to tail. +//! - items within a bag are iterated in order of insertion. Thus removing an item and re-inserting +//! it will worsen its position in list iteration; this reduces incentives for some types of spam +//! that involve consistently removing and inserting for better position. Further, ordering +//! granularity is thus dictated by range between each bag threshold. +//! - if an item's weight changes to a value no longer within the range of its current bag the +//! item's position will need to be updated by an external actor with rebag (update), or removal +//! and insertion. + +#![cfg_attr(not(feature = "std"), no_std)] + +use frame_election_provider_support::{SortedListProvider, VoteWeight, VoteWeightProvider}; +use frame_system::ensure_signed; +use sp_std::prelude::*; + +#[cfg(any(feature = "runtime-benchmarks", test))] +mod benchmarks; + +mod list; +#[cfg(test)] +mod mock; +#[cfg(test)] +mod tests; +pub mod weights; + +pub use pallet::*; +pub use weights::WeightInfo; + +pub use list::Error; +use list::List; + +pub(crate) const LOG_TARGET: &'static str = "runtime::bags_list"; + +// syntactic sugar for logging. +#[macro_export] +macro_rules! log { + ($level:tt, $patter:expr $(, $values:expr)* $(,)?) => { + log::$level!( + target: crate::LOG_TARGET, + concat!("[{:?}] 👜", $patter), >::block_number() $(, $values)* + ) + }; +} + +#[frame_support::pallet] +pub mod pallet { + use super::*; + use frame_support::pallet_prelude::*; + use frame_system::pallet_prelude::*; + + #[pallet::pallet] + #[pallet::generate_store(pub(crate) trait Store)] + #[pallet::generate_storage_info] + pub struct Pallet(_); + + #[pallet::config] + pub trait Config: frame_system::Config { + /// The overarching event type. + type Event: From> + IsType<::Event>; + + /// Weight information for extrinsics in this pallet. + type WeightInfo: weights::WeightInfo; + + /// Something that provides the weights of ids. + type VoteWeightProvider: VoteWeightProvider; + + /// The list of thresholds separating the various bags. + /// + /// Ids are separated into unsorted bags according to their vote weight. This specifies the + /// thresholds separating the bags. An id's bag is the largest bag for which the id's weight + /// is less than or equal to its upper threshold. + /// + /// When ids are iterated, higher bags are iterated completely before lower bags. This means + /// that iteration is _semi-sorted_: ids of higher weight tend to come before ids of lower + /// weight, but peer ids within a particular bag are sorted in insertion order. + /// + /// # Expressing the constant + /// + /// This constant must be sorted in strictly increasing order. Duplicate items are not + /// permitted. + /// + /// There is an implied upper limit of `VoteWeight::MAX`; that value does not need to be + /// specified within the bag. For any two threshold lists, if one ends with + /// `VoteWeight::MAX`, the other one does not, and they are otherwise equal, the two lists + /// will behave identically. + /// + /// # Calculation + /// + /// It is recommended to generate the set of thresholds in a geometric series, such that + /// there exists some constant ratio such that `threshold[k + 1] == (threshold[k] * + /// constant_ratio).max(threshold[k] + 1)` for all `k`. + /// + /// The helpers in the `/utils/frame/generate-bags` module can simplify this calculation. + /// + /// # Examples + /// + /// - If `BagThresholds::get().is_empty()`, then all ids are put into the same bag, and + /// iteration is strictly in insertion order. + /// - If `BagThresholds::get().len() == 64`, and the thresholds are determined according to + /// the procedure given above, then the constant ratio is equal to 2. + /// - If `BagThresholds::get().len() == 200`, and the thresholds are determined according to + /// the procedure given above, then the constant ratio is approximately equal to 1.248. + /// - If the threshold list begins `[1, 2, 3, ...]`, then an id with weight 0 or 1 will fall + /// into bag 0, an id with weight 2 will fall into bag 1, etc. + /// + /// # Migration + /// + /// In the event that this list ever changes, a copy of the old bags list must be retained. + /// With that `List::migrate` can be called, which will perform the appropriate migration. + #[pallet::constant] + type BagThresholds: Get<&'static [VoteWeight]>; + } + + /// How many ids are registered. + // NOTE: This is merely a counter for `ListNodes`. It should someday be replaced by the + // `CountedMaop` storage. + #[pallet::storage] + pub(crate) type CounterForListNodes = StorageValue<_, u32, ValueQuery>; + + /// A single node, within some bag. + /// + /// Nodes store links forward and back within their respective bags. + #[pallet::storage] + pub(crate) type ListNodes = StorageMap<_, Twox64Concat, T::AccountId, list::Node>; + + /// A bag stored in storage. + /// + /// Stores a `Bag` struct, which stores head and tail pointers to itself. + #[pallet::storage] + pub(crate) type ListBags = StorageMap<_, Twox64Concat, VoteWeight, list::Bag>; + + #[pallet::event] + #[pallet::generate_deposit(pub(crate) fn deposit_event)] + pub enum Event { + /// Moved an account from one bag to another. \[who, from, to\]. + Rebagged(T::AccountId, VoteWeight, VoteWeight), + } + + #[pallet::call] + impl Pallet { + /// Declare that some `dislocated` account has, through rewards or penalties, sufficiently + /// changed its weight that it should properly fall into a different bag than its current + /// one. + /// + /// Anyone can call this function about any potentially dislocated account. + /// + /// Will never return an error; if `dislocated` does not exist or doesn't need a rebag, then + /// it is a noop and fees are still collected from `origin`. + #[pallet::weight(T::WeightInfo::rebag_non_terminal().max(T::WeightInfo::rebag_terminal()))] + pub fn rebag(origin: OriginFor, dislocated: T::AccountId) -> DispatchResult { + ensure_signed(origin)?; + let current_weight = T::VoteWeightProvider::vote_weight(&dislocated); + let _ = Pallet::::do_rebag(&dislocated, current_weight); + Ok(()) + } + } + + #[pallet::hooks] + impl Hooks> for Pallet { + fn integrity_test() { + // ensure they are strictly increasing, this also implies that duplicates are detected. + assert!( + T::BagThresholds::get().windows(2).all(|window| window[1] > window[0]), + "thresholds must strictly increase, and have no duplicates", + ); + } + } +} + +impl Pallet { + /// Move an account from one bag to another, depositing an event on success. + /// + /// If the account changed bags, returns `Some((from, to))`. + pub fn do_rebag( + account: &T::AccountId, + new_weight: VoteWeight, + ) -> Option<(VoteWeight, VoteWeight)> { + // if no voter at that node, don't do anything. + // the caller just wasted the fee to call this. + let maybe_movement = list::Node::::get(&account) + .and_then(|node| List::update_position_for(node, new_weight)); + if let Some((from, to)) = maybe_movement { + Self::deposit_event(Event::::Rebagged(account.clone(), from, to)); + }; + maybe_movement + } + + /// Equivalent to `ListBags::get`, but public. Useful for tests in outside of this crate. + #[cfg(feature = "std")] + pub fn list_bags_get(weight: VoteWeight) -> Option> { + ListBags::get(weight) + } +} + +impl SortedListProvider for Pallet { + type Error = Error; + + fn iter() -> Box> { + Box::new(List::::iter().map(|n| n.id().clone())) + } + + fn count() -> u32 { + CounterForListNodes::::get() + } + + fn contains(id: &T::AccountId) -> bool { + List::::contains(id) + } + + fn on_insert(id: T::AccountId, weight: VoteWeight) -> Result<(), Error> { + List::::insert(id, weight) + } + + fn on_update(id: &T::AccountId, new_weight: VoteWeight) { + Pallet::::do_rebag(id, new_weight); + } + + fn on_remove(id: &T::AccountId) { + List::::remove(id) + } + + fn regenerate( + all: impl IntoIterator, + weight_of: Box VoteWeight>, + ) -> u32 { + List::::regenerate(all, weight_of) + } + + #[cfg(feature = "std")] + fn sanity_check() -> Result<(), &'static str> { + List::::sanity_check() + } + + #[cfg(not(feature = "std"))] + fn sanity_check() -> Result<(), &'static str> { + Ok(()) + } + + fn clear(maybe_count: Option) -> u32 { + List::::clear(maybe_count) + } + + #[cfg(feature = "runtime-benchmarks")] + fn weight_update_worst_case(who: &T::AccountId, is_increase: bool) -> VoteWeight { + use frame_support::traits::Get as _; + let thresholds = T::BagThresholds::get(); + let node = list::Node::::get(who).unwrap(); + let current_bag_idx = thresholds + .iter() + .chain(sp_std::iter::once(&VoteWeight::MAX)) + .position(|w| w == &node.bag_upper()) + .unwrap(); + + if is_increase { + let next_threshold_idx = current_bag_idx + 1; + assert!(thresholds.len() > next_threshold_idx); + thresholds[next_threshold_idx] + } else { + assert!(current_bag_idx != 0); + let prev_threshold_idx = current_bag_idx - 1; + thresholds[prev_threshold_idx] + } + } +} diff --git a/frame/bags-list/src/list/mod.rs b/frame/bags-list/src/list/mod.rs new file mode 100644 index 0000000000000..3f55f22271910 --- /dev/null +++ b/frame/bags-list/src/list/mod.rs @@ -0,0 +1,786 @@ +// This file is part of Substrate. + +// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Implementation of a "bags list": a semi-sorted list where ordering granularity is dictated by +//! configurable thresholds that delineate the boundaries of bags. It uses a pattern of composite +//! data structures, where multiple storage items are masked by one outer API. See [`ListNodes`], +//! [`CounterForListNodes`] and [`ListBags`] for more information. +//! +//! The outer API of this module is the [`List`] struct. It wraps all acceptable operations on top +//! of the aggregate linked list. All operations with the bags list should happen through this +//! interface. + +use crate::Config; +use codec::{Decode, Encode, MaxEncodedLen}; +use frame_election_provider_support::{VoteWeight, VoteWeightProvider}; +use frame_support::{traits::Get, DefaultNoBound}; +use scale_info::TypeInfo; +use sp_std::{ + boxed::Box, + collections::{btree_map::BTreeMap, btree_set::BTreeSet}, + iter, + marker::PhantomData, + vec::Vec, +}; + +#[derive(Debug, PartialEq, Eq)] +pub enum Error { + /// A duplicate id has been detected. + Duplicate, +} + +#[cfg(test)] +mod tests; + +/// Given a certain vote weight, to which bag does it belong to? +/// +/// Bags are identified by their upper threshold; the value returned by this function is guaranteed +/// to be a member of `T::BagThresholds`. +/// +/// Note that even if the thresholds list does not have `VoteWeight::MAX` as its final member, this +/// function behaves as if it does. +pub(crate) fn notional_bag_for(weight: VoteWeight) -> VoteWeight { + let thresholds = T::BagThresholds::get(); + let idx = thresholds.partition_point(|&threshold| weight > threshold); + thresholds.get(idx).copied().unwrap_or(VoteWeight::MAX) +} + +/// The **ONLY** entry point of this module. All operations to the bags-list should happen through +/// this interface. It is forbidden to access other module members directly. +// +// Data structure providing efficient mostly-accurate selection of the top N id by `VoteWeight`. +// +// It's implemented as a set of linked lists. Each linked list comprises a bag of ids of +// arbitrary and unbounded length, all having a vote weight within a particular constant range. +// This structure means that ids can be added and removed in `O(1)` time. +// +// Iteration is accomplished by chaining the iteration of each bag, from greatest to least. While +// the users within any particular bag are sorted in an entirely arbitrary order, the overall vote +// weight decreases as successive bags are reached. This means that it is valid to truncate +// iteration at any desired point; only those ids in the lowest bag can be excluded. This +// satisfies both the desire for fairness and the requirement for efficiency. +pub struct List(PhantomData); + +impl List { + /// Remove all data associated with the list from storage. Parameter `items` is the number of + /// items to clear from the list. WARNING: `None` will clear all items and should generally not + /// be used in production as it could lead to an infinite number of storage accesses. + pub(crate) fn clear(maybe_count: Option) -> u32 { + crate::ListBags::::remove_all(maybe_count); + crate::ListNodes::::remove_all(maybe_count); + if let Some(count) = maybe_count { + crate::CounterForListNodes::::mutate(|items| *items - count); + count + } else { + crate::CounterForListNodes::::take() + } + } + + /// Regenerate all of the data from the given ids. + /// + /// WARNING: this is expensive and should only ever be performed when the list needs to be + /// generated from scratch. Care needs to be taken to ensure + /// + /// This may or may not need to be called at genesis as well, based on the configuration of the + /// pallet using this `List`. + /// + /// Returns the number of ids migrated. + pub fn regenerate( + all: impl IntoIterator, + weight_of: Box VoteWeight>, + ) -> u32 { + Self::clear(None); + Self::insert_many(all, weight_of) + } + + /// Migrate the list from one set of thresholds to another. + /// + /// This should only be called as part of an intentional migration; it's fairly expensive. + /// + /// Returns the number of accounts affected. + /// + /// Preconditions: + /// + /// - `old_thresholds` is the previous list of thresholds. + /// - All `bag_upper` currently in storage are members of `old_thresholds`. + /// - `T::BagThresholds` has already been updated and is the new set of thresholds. + /// + /// Postconditions: + /// + /// - All `bag_upper` currently in storage are members of `T::BagThresholds`. + /// - No id is changed unless required to by the difference between the old threshold list and + /// the new. + /// - ids whose bags change at all are implicitly rebagged into the appropriate bag in the new + /// threshold set. + #[allow(dead_code)] + pub fn migrate(old_thresholds: &[VoteWeight]) -> u32 { + let new_thresholds = T::BagThresholds::get(); + if new_thresholds == old_thresholds { + return 0 + } + + // we can't check all preconditions, but we can check one + debug_assert!( + crate::ListBags::::iter().all(|(threshold, _)| old_thresholds.contains(&threshold)), + "not all `bag_upper` currently in storage are members of `old_thresholds`", + ); + debug_assert!( + crate::ListNodes::::iter().all(|(_, node)| old_thresholds.contains(&node.bag_upper)), + "not all `node.bag_upper` currently in storage are members of `old_thresholds`", + ); + + let old_set: BTreeSet<_> = old_thresholds.iter().copied().collect(); + let new_set: BTreeSet<_> = new_thresholds.iter().copied().collect(); + + // accounts that need to be rebagged + let mut affected_accounts = BTreeSet::new(); + // track affected old bags to make sure we only iterate them once + let mut affected_old_bags = BTreeSet::new(); + + let new_bags = new_set.difference(&old_set).copied(); + // a new bag means that all accounts previously using the old bag's threshold must now + // be rebagged + for inserted_bag in new_bags { + let affected_bag = { + // this recreates `notional_bag_for` logic, but with the old thresholds. + let idx = old_thresholds.partition_point(|&threshold| inserted_bag > threshold); + old_thresholds.get(idx).copied().unwrap_or(VoteWeight::MAX) + }; + if !affected_old_bags.insert(affected_bag) { + // If the previous threshold list was [10, 20], and we insert [3, 5], then there's + // no point iterating through bag 10 twice. + continue + } + + if let Some(bag) = Bag::::get(affected_bag) { + affected_accounts.extend(bag.iter().map(|node| node.id)); + } + } + + let removed_bags = old_set.difference(&new_set).copied(); + // a removed bag means that all members of that bag must be rebagged + for removed_bag in removed_bags.clone() { + if !affected_old_bags.insert(removed_bag) { + continue + } + + if let Some(bag) = Bag::::get(removed_bag) { + affected_accounts.extend(bag.iter().map(|node| node.id)); + } + } + + // migrate the voters whose bag has changed + let num_affected = affected_accounts.len() as u32; + let weight_of = T::VoteWeightProvider::vote_weight; + let _removed = Self::remove_many(&affected_accounts); + debug_assert_eq!(_removed, num_affected); + let _inserted = Self::insert_many(affected_accounts.into_iter(), weight_of); + debug_assert_eq!(_inserted, num_affected); + + // we couldn't previously remove the old bags because both insertion and removal assume that + // it's always safe to add a bag if it's not present. Now that that's sorted, we can get rid + // of them. + // + // it's pretty cheap to iterate this again, because both sets are in-memory and require no + // lookups. + for removed_bag in removed_bags { + debug_assert!( + !crate::ListNodes::::iter().any(|(_, node)| node.bag_upper == removed_bag), + "no id should be present in a removed bag", + ); + crate::ListBags::::remove(removed_bag); + } + + debug_assert_eq!(Self::sanity_check(), Ok(())); + + num_affected + } + + /// Returns `true` if the list contains `id`, otherwise returns `false`. + pub(crate) fn contains(id: &T::AccountId) -> bool { + crate::ListNodes::::contains_key(id) + } + + /// Iterate over all nodes in all bags in the list. + /// + /// Full iteration can be expensive; it's recommended to limit the number of items with + /// `.take(n)`. + pub(crate) fn iter() -> impl Iterator> { + // We need a touch of special handling here: because we permit `T::BagThresholds` to + // omit the final bound, we need to ensure that we explicitly include that threshold in the + // list. + // + // It's important to retain the ability to omit the final bound because it makes tests much + // easier; they can just configure `type BagThresholds = ()`. + let thresholds = T::BagThresholds::get(); + let iter = thresholds.iter().copied(); + let iter: Box> = if thresholds.last() == Some(&VoteWeight::MAX) { + // in the event that they included it, we can just pass the iterator through unchanged. + Box::new(iter.rev()) + } else { + // otherwise, insert it here. + Box::new(iter.chain(iter::once(VoteWeight::MAX)).rev()) + }; + + iter.filter_map(Bag::get).flat_map(|bag| bag.iter()) + } + + /// Insert several ids into the appropriate bags in the list. Continues with insertions + /// if duplicates are detected. + /// + /// Returns the final count of number of ids inserted. + fn insert_many( + ids: impl IntoIterator, + weight_of: impl Fn(&T::AccountId) -> VoteWeight, + ) -> u32 { + let mut count = 0; + ids.into_iter().for_each(|v| { + let weight = weight_of(&v); + if Self::insert(v, weight).is_ok() { + count += 1; + } + }); + + count + } + + /// Insert a new id into the appropriate bag in the list. + /// + /// Returns an error if the list already contains `id`. + pub(crate) fn insert(id: T::AccountId, weight: VoteWeight) -> Result<(), Error> { + if Self::contains(&id) { + return Err(Error::Duplicate) + } + + let bag_weight = notional_bag_for::(weight); + let mut bag = Bag::::get_or_make(bag_weight); + // unchecked insertion is okay; we just got the correct `notional_bag_for`. + bag.insert_unchecked(id.clone()); + + // new inserts are always the tail, so we must write the bag. + bag.put(); + + crate::CounterForListNodes::::mutate(|prev_count| { + *prev_count = prev_count.saturating_add(1) + }); + + crate::log!( + debug, + "inserted {:?} with weight {} into bag {:?}, new count is {}", + id, + weight, + bag_weight, + crate::CounterForListNodes::::get(), + ); + + Ok(()) + } + + /// Remove an id from the list. + pub(crate) fn remove(id: &T::AccountId) { + Self::remove_many(sp_std::iter::once(id)); + } + + /// Remove many ids from the list. + /// + /// This is more efficient than repeated calls to `Self::remove`. + /// + /// Returns the final count of number of ids removed. + fn remove_many<'a>(ids: impl IntoIterator) -> u32 { + let mut bags = BTreeMap::new(); + let mut count = 0; + + for id in ids.into_iter() { + let node = match Node::::get(id) { + Some(node) => node, + None => continue, + }; + count += 1; + + if !node.is_terminal() { + // this node is not a head or a tail and thus the bag does not need to be updated + node.excise() + } else { + // this node is a head or tail, so the bag needs to be updated + let bag = bags + .entry(node.bag_upper) + .or_insert_with(|| Bag::::get_or_make(node.bag_upper)); + // node.bag_upper must be correct, therefore this bag will contain this node. + bag.remove_node_unchecked(&node); + } + + // now get rid of the node itself + node.remove_from_storage_unchecked() + } + + for (_, bag) in bags { + bag.put(); + } + + crate::CounterForListNodes::::mutate(|prev_count| { + *prev_count = prev_count.saturating_sub(count) + }); + + count + } + + /// Update a node's position in the list. + /// + /// If the node was in the correct bag, no effect. If the node was in the incorrect bag, they + /// are moved into the correct bag. + /// + /// Returns `Some((old_idx, new_idx))` if the node moved, otherwise `None`. + /// + /// This operation is somewhat more efficient than simply calling [`self.remove`] followed by + /// [`self.insert`]. However, given large quantities of nodes to move, it may be more efficient + /// to call [`self.remove_many`] followed by [`self.insert_many`]. + pub(crate) fn update_position_for( + node: Node, + new_weight: VoteWeight, + ) -> Option<(VoteWeight, VoteWeight)> { + node.is_misplaced(new_weight).then(move || { + let old_bag_upper = node.bag_upper; + + if !node.is_terminal() { + // this node is not a head or a tail, so we can just cut it out of the list. update + // and put the prev and next of this node, we do `node.put` inside `insert_note`. + node.excise(); + } else if let Some(mut bag) = Bag::::get(node.bag_upper) { + // this is a head or tail, so the bag must be updated. + bag.remove_node_unchecked(&node); + bag.put(); + } else { + crate::log!( + error, + "Node {:?} did not have a bag; ListBags is in an inconsistent state", + node.id, + ); + debug_assert!(false, "every node must have an extant bag associated with it"); + } + + // put the node into the appropriate new bag. + let new_bag_upper = notional_bag_for::(new_weight); + let mut bag = Bag::::get_or_make(new_bag_upper); + // prev, next, and bag_upper of the node are updated inside `insert_node`, also + // `node.put` is in there. + bag.insert_node_unchecked(node); + bag.put(); + + (old_bag_upper, new_bag_upper) + }) + } + + /// Sanity check the list. + /// + /// This should be called from the call-site, whenever one of the mutating apis (e.g. `insert`) + /// is being used, after all other staking data (such as counter) has been updated. It checks: + /// + /// * there are no duplicate ids, + /// * length of this list is in sync with `CounterForListNodes`, + /// * and sanity-checks all bags. This will cascade down all the checks and makes sure all bags + /// are checked per *any* update to `List`. + #[cfg(feature = "std")] + pub(crate) fn sanity_check() -> Result<(), &'static str> { + use frame_support::ensure; + let mut seen_in_list = BTreeSet::new(); + ensure!( + Self::iter().map(|node| node.id).all(|id| seen_in_list.insert(id)), + "duplicate identified", + ); + + let iter_count = Self::iter().count() as u32; + let stored_count = crate::CounterForListNodes::::get(); + let nodes_count = crate::ListNodes::::iter().count() as u32; + ensure!(iter_count == stored_count, "iter_count != stored_count"); + ensure!(stored_count == nodes_count, "stored_count != nodes_count"); + + crate::log!(debug, "count of nodes: {}", stored_count); + + let active_bags = { + let thresholds = T::BagThresholds::get().iter().copied(); + let thresholds: Vec = if thresholds.clone().last() == Some(VoteWeight::MAX) { + // in the event that they included it, we don't need to make any changes + // Box::new(thresholds.collect() + thresholds.collect() + } else { + // otherwise, insert it here. + thresholds.chain(iter::once(VoteWeight::MAX)).collect() + }; + thresholds.into_iter().filter_map(|t| Bag::::get(t)) + }; + + let _ = active_bags.clone().map(|b| b.sanity_check()).collect::>()?; + + let nodes_in_bags_count = + active_bags.clone().fold(0u32, |acc, cur| acc + cur.iter().count() as u32); + ensure!(nodes_count == nodes_in_bags_count, "stored_count != nodes_in_bags_count"); + + crate::log!(debug, "count of active bags {}", active_bags.count()); + + // check that all nodes are sane. We check the `ListNodes` storage item directly in case we + // have some "stale" nodes that are not in a bag. + for (_id, node) in crate::ListNodes::::iter() { + node.sanity_check()? + } + + Ok(()) + } + + #[cfg(not(feature = "std"))] + pub(crate) fn sanity_check() -> Result<(), &'static str> { + Ok(()) + } + + /// Returns the nodes of all non-empty bags. For testing and benchmarks. + #[cfg(any(feature = "std", feature = "runtime-benchmarks"))] + #[allow(dead_code)] + pub(crate) fn get_bags() -> Vec<(VoteWeight, Vec)> { + use frame_support::traits::Get as _; + + let thresholds = T::BagThresholds::get(); + let iter = thresholds.iter().copied(); + let iter: Box> = if thresholds.last() == Some(&VoteWeight::MAX) { + // in the event that they included it, we can just pass the iterator through unchanged. + Box::new(iter) + } else { + // otherwise, insert it here. + Box::new(iter.chain(sp_std::iter::once(VoteWeight::MAX))) + }; + + iter.filter_map(|t| { + Bag::::get(t).map(|bag| (t, bag.iter().map(|n| n.id().clone()).collect::>())) + }) + .collect::>() + } +} + +/// A Bag is a doubly-linked list of ids, where each id is mapped to a [`ListNode`]. +/// +/// Note that we maintain both head and tail pointers. While it would be possible to get away with +/// maintaining only a head pointer and cons-ing elements onto the front of the list, it's more +/// desirable to ensure that there is some element of first-come, first-serve to the list's +/// iteration so that there's no incentive to churn ids positioning to improve the chances of +/// appearing within the ids set. +#[derive(DefaultNoBound, Encode, Decode, MaxEncodedLen, TypeInfo)] +#[codec(mel_bound(T: Config))] +#[scale_info(skip_type_params(T))] +#[cfg_attr(feature = "std", derive(frame_support::DebugNoBound, Clone, PartialEq))] +pub struct Bag { + head: Option, + tail: Option, + + #[codec(skip)] + bag_upper: VoteWeight, +} + +impl Bag { + #[cfg(test)] + pub(crate) fn new( + head: Option, + tail: Option, + bag_upper: VoteWeight, + ) -> Self { + Self { head, tail, bag_upper } + } + + /// Get a bag by its upper vote weight. + pub(crate) fn get(bag_upper: VoteWeight) -> Option> { + crate::ListBags::::try_get(bag_upper).ok().map(|mut bag| { + bag.bag_upper = bag_upper; + bag + }) + } + + /// Get a bag by its upper vote weight or make it, appropriately initialized. Does not check if + /// if `bag_upper` is a valid threshold. + fn get_or_make(bag_upper: VoteWeight) -> Bag { + Self::get(bag_upper).unwrap_or(Bag { bag_upper, ..Default::default() }) + } + + /// `True` if self is empty. + fn is_empty(&self) -> bool { + self.head.is_none() && self.tail.is_none() + } + + /// Put the bag back into storage. + fn put(self) { + if self.is_empty() { + crate::ListBags::::remove(self.bag_upper); + } else { + crate::ListBags::::insert(self.bag_upper, self); + } + } + + /// Get the head node in this bag. + fn head(&self) -> Option> { + self.head.as_ref().and_then(|id| Node::get(id)) + } + + /// Get the tail node in this bag. + fn tail(&self) -> Option> { + self.tail.as_ref().and_then(|id| Node::get(id)) + } + + /// Iterate over the nodes in this bag. + pub(crate) fn iter(&self) -> impl Iterator> { + sp_std::iter::successors(self.head(), |prev| prev.next()) + } + + /// Insert a new id into this bag. + /// + /// This is private on purpose because it's naive: it doesn't check whether this is the + /// appropriate bag for this id at all. Generally, use [`List::insert`] instead. + /// + /// Storage note: this modifies storage, but only for the nodes. You still need to call + /// `self.put()` after use. + fn insert_unchecked(&mut self, id: T::AccountId) { + // insert_node will overwrite `prev`, `next` and `bag_upper` to the proper values. As long + // as this bag is the correct one, we're good. All calls to this must come after getting the + // correct [`notional_bag_for`]. + self.insert_node_unchecked(Node:: { id, prev: None, next: None, bag_upper: 0 }); + } + + /// Insert a node into this bag. + /// + /// This is private on purpose because it's naive; it doesn't check whether this is the + /// appropriate bag for this node at all. Generally, use [`List::insert`] instead. + /// + /// Storage note: this modifies storage, but only for the node. You still need to call + /// `self.put()` after use. + fn insert_node_unchecked(&mut self, mut node: Node) { + if let Some(tail) = &self.tail { + if *tail == node.id { + // this should never happen, but this check prevents one path to a worst case + // infinite loop. + debug_assert!(false, "system logic error: inserting a node who has the id of tail"); + crate::log!(warn, "system logic error: inserting a node who has the id of tail"); + return + }; + } + + // re-set the `bag_upper`. Regardless of whatever the node had previously, now it is going + // to be `self.bag_upper`. + node.bag_upper = self.bag_upper; + + let id = node.id.clone(); + // update this node now, treating it as the new tail. + node.prev = self.tail.clone(); + node.next = None; + node.put(); + + // update the previous tail. + if let Some(mut old_tail) = self.tail() { + old_tail.next = Some(id.clone()); + old_tail.put(); + } + self.tail = Some(id.clone()); + + // ensure head exist. This is only set when the length of the bag is just 1, i.e. if this is + // the first insertion into the bag. In this case, both head and tail should point to the + // same node. + if self.head.is_none() { + self.head = Some(id.clone()); + debug_assert!(self.iter().count() == 1); + } + } + + /// Remove a node from this bag. + /// + /// This is private on purpose because it doesn't check whether this bag contains the node in + /// the first place. Generally, use [`List::remove`] instead, similar to `insert_unchecked`. + /// + /// Storage note: this modifies storage, but only for adjacent nodes. You still need to call + /// `self.put()` and `ListNodes::remove(id)` to update storage for the bag and `node`. + fn remove_node_unchecked(&mut self, node: &Node) { + // reassign neighboring nodes. + node.excise(); + + // clear the bag head/tail pointers as necessary. + if self.tail.as_ref() == Some(&node.id) { + self.tail = node.prev.clone(); + } + if self.head.as_ref() == Some(&node.id) { + self.head = node.next.clone(); + } + } + + /// Sanity check this bag. + /// + /// Should be called by the call-site, after any mutating operation on a bag. The call site of + /// this struct is always `List`. + /// + /// * Ensures head has no prev. + /// * Ensures tail has no next. + /// * Ensures there are no loops, traversal from head to tail is correct. + #[cfg(feature = "std")] + fn sanity_check(&self) -> Result<(), &'static str> { + frame_support::ensure!( + self.head() + .map(|head| head.prev().is_none()) + // if there is no head, then there must not be a tail, meaning that the bag is + // empty. + .unwrap_or_else(|| self.tail.is_none()), + "head has a prev" + ); + + frame_support::ensure!( + self.tail() + .map(|tail| tail.next().is_none()) + // if there is no tail, then there must not be a head, meaning that the bag is + // empty. + .unwrap_or_else(|| self.head.is_none()), + "tail has a next" + ); + + let mut seen_in_bag = BTreeSet::new(); + frame_support::ensure!( + self.iter() + .map(|node| node.id) + // each voter is only seen once, thus there is no cycle within a bag + .all(|voter| seen_in_bag.insert(voter)), + "duplicate found in bag" + ); + + Ok(()) + } + + #[cfg(not(feature = "std"))] + fn sanity_check(&self) -> Result<(), &'static str> { + Ok(()) + } + + /// Iterate over the nodes in this bag (public for tests). + #[cfg(feature = "std")] + #[allow(dead_code)] + pub fn std_iter(&self) -> impl Iterator> { + sp_std::iter::successors(self.head(), |prev| prev.next()) + } + + /// Check if the bag contains a node with `id`. + #[cfg(feature = "std")] + fn contains(&self, id: &T::AccountId) -> bool { + self.iter().find(|n| n.id() == id).is_some() + } +} + +/// A Node is the fundamental element comprising the doubly-linked list described by `Bag`. +#[derive(Encode, Decode, MaxEncodedLen, TypeInfo)] +#[codec(mel_bound(T: Config))] +#[scale_info(skip_type_params(T))] +#[cfg_attr(feature = "std", derive(frame_support::DebugNoBound, Clone, PartialEq))] +pub struct Node { + id: T::AccountId, + prev: Option, + next: Option, + bag_upper: VoteWeight, +} + +impl Node { + /// Get a node by id. + pub(crate) fn get(id: &T::AccountId) -> Option> { + crate::ListNodes::::try_get(id).ok() + } + + /// Put the node back into storage. + fn put(self) { + crate::ListNodes::::insert(self.id.clone(), self); + } + + /// Update neighboring nodes to point to reach other. + /// + /// Only updates storage for adjacent nodes, but not `self`; so the user may need to call + /// `self.put`. + fn excise(&self) { + // Update previous node. + if let Some(mut prev) = self.prev() { + prev.next = self.next.clone(); + prev.put(); + } + // Update next self. + if let Some(mut next) = self.next() { + next.prev = self.prev.clone(); + next.put(); + } + } + + /// This is a naive function that removes a node from the `ListNodes` storage item. + /// + /// It is naive because it does not check if the node has first been removed from its bag. + fn remove_from_storage_unchecked(&self) { + crate::ListNodes::::remove(&self.id) + } + + /// Get the previous node in the bag. + fn prev(&self) -> Option> { + self.prev.as_ref().and_then(|id| Node::get(id)) + } + + /// Get the next node in the bag. + fn next(&self) -> Option> { + self.next.as_ref().and_then(|id| Node::get(id)) + } + + /// `true` when this voter is in the wrong bag. + pub(crate) fn is_misplaced(&self, current_weight: VoteWeight) -> bool { + notional_bag_for::(current_weight) != self.bag_upper + } + + /// `true` when this voter is a bag head or tail. + fn is_terminal(&self) -> bool { + self.prev.is_none() || self.next.is_none() + } + + /// Get the underlying voter. + pub(crate) fn id(&self) -> &T::AccountId { + &self.id + } + + /// Get the underlying voter (public fo tests). + #[cfg(feature = "std")] + #[allow(dead_code)] + pub fn std_id(&self) -> &T::AccountId { + &self.id + } + + /// The bag this nodes belongs to (public for benchmarks). + #[cfg(feature = "runtime-benchmarks")] + #[allow(dead_code)] + pub fn bag_upper(&self) -> VoteWeight { + self.bag_upper + } + + #[cfg(feature = "std")] + fn sanity_check(&self) -> Result<(), &'static str> { + let expected_bag = Bag::::get(self.bag_upper).ok_or("bag not found for node")?; + + let id = self.id(); + + frame_support::ensure!( + expected_bag.contains(id), + "node does not exist in the expected bag" + ); + + frame_support::ensure!( + !self.is_terminal() || + expected_bag.head.as_ref() == Some(id) || + expected_bag.tail.as_ref() == Some(id), + "a terminal node is neither its bag head or tail" + ); + + Ok(()) + } +} diff --git a/frame/bags-list/src/list/tests.rs b/frame/bags-list/src/list/tests.rs new file mode 100644 index 0000000000000..e2730cbf4e33d --- /dev/null +++ b/frame/bags-list/src/list/tests.rs @@ -0,0 +1,735 @@ +// This file is part of Substrate. + +// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::*; +use crate::{ + mock::{test_utils::*, *}, + CounterForListNodes, ListBags, ListNodes, +}; +use frame_election_provider_support::SortedListProvider; +use frame_support::{assert_ok, assert_storage_noop}; + +#[test] +fn basic_setup_works() { + ExtBuilder::default().build_and_execute(|| { + // syntactic sugar to create a raw node + let node = |id, prev, next, bag_upper| Node:: { id, prev, next, bag_upper }; + + assert_eq!(CounterForListNodes::::get(), 4); + assert_eq!(ListNodes::::iter().count(), 4); + assert_eq!(ListBags::::iter().count(), 2); + + assert_eq!(List::::get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4])]); + + // the state of the bags is as expected + assert_eq!( + ListBags::::get(10).unwrap(), + Bag:: { head: Some(1), tail: Some(1), bag_upper: 0 } + ); + assert_eq!( + ListBags::::get(1_000).unwrap(), + Bag:: { head: Some(2), tail: Some(4), bag_upper: 0 } + ); + + assert_eq!(ListNodes::::get(2).unwrap(), node(2, None, Some(3), 1_000)); + assert_eq!(ListNodes::::get(3).unwrap(), node(3, Some(2), Some(4), 1_000)); + assert_eq!(ListNodes::::get(4).unwrap(), node(4, Some(3), None, 1_000)); + assert_eq!(ListNodes::::get(1).unwrap(), node(1, None, None, 10)); + + // non-existent id does not have a storage footprint + assert_eq!(ListNodes::::get(42), None); + + // iteration of the bags would yield: + assert_eq!( + List::::iter().map(|n| *n.id()).collect::>(), + vec![2, 3, 4, 1], + // ^^ note the order of insertion in genesis! + ); + }); +} + +#[test] +fn notional_bag_for_works() { + // under a threshold gives the next threshold. + assert_eq!(notional_bag_for::(0), 10); + assert_eq!(notional_bag_for::(9), 10); + + // at a threshold gives that threshold. + assert_eq!(notional_bag_for::(10), 10); + + // above the threshold, gives the next threshold. + assert_eq!(notional_bag_for::(11), 20); + + let max_explicit_threshold = *::BagThresholds::get().last().unwrap(); + assert_eq!(max_explicit_threshold, 10_000); + + // if the max explicit threshold is less than VoteWeight::MAX, + assert!(VoteWeight::MAX > max_explicit_threshold); + + // then anything above it will belong to the VoteWeight::MAX bag. + assert_eq!(notional_bag_for::(max_explicit_threshold), max_explicit_threshold); + assert_eq!(notional_bag_for::(max_explicit_threshold + 1), VoteWeight::MAX); +} + +#[test] +fn remove_last_node_in_bags_cleans_bag() { + ExtBuilder::default().build_and_execute(|| { + // given + assert_eq!(List::::get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4])]); + + // bump 1 to a bigger bag + List::::remove(&1); + assert_ok!(List::::insert(1, 10_000)); + + // then the bag with bound 10 is wiped from storage. + assert_eq!(List::::get_bags(), vec![(1_000, vec![2, 3, 4]), (10_000, vec![1])]); + + // and can be recreated again as needed. + assert_ok!(List::::insert(77, 10)); + assert_eq!( + List::::get_bags(), + vec![(10, vec![77]), (1_000, vec![2, 3, 4]), (10_000, vec![1])] + ); + }); +} + +#[test] +fn migrate_works() { + ExtBuilder::default() + .add_ids(vec![(710, 15), (711, 16), (712, 2_000)]) + .build_and_execute(|| { + // given + assert_eq!( + List::::get_bags(), + vec![ + (10, vec![1]), + (20, vec![710, 711]), + (1_000, vec![2, 3, 4]), + (2_000, vec![712]) + ] + ); + let old_thresholds = ::BagThresholds::get(); + assert_eq!(old_thresholds, vec![10, 20, 30, 40, 50, 60, 1_000, 2_000, 10_000]); + + // when the new thresholds adds `15` and removes `2_000` + const NEW_THRESHOLDS: &'static [VoteWeight] = + &[10, 15, 20, 30, 40, 50, 60, 1_000, 10_000]; + BagThresholds::set(NEW_THRESHOLDS); + // and we call + List::::migrate(old_thresholds); + + // then + assert_eq!( + List::::get_bags(), + vec![ + (10, vec![1]), + (15, vec![710]), // nodes in range 11 ..= 15 move from bag 20 to bag 15 + (20, vec![711]), + (1_000, vec![2, 3, 4]), + // nodes in range 1_001 ..= 2_000 move from bag 2_000 to bag 10_000 + (10_000, vec![712]), + ] + ); + }); +} + +mod list { + use super::*; + + #[test] + fn iteration_is_semi_sorted() { + ExtBuilder::default() + .add_ids(vec![(5, 2_000), (6, 2_000)]) + .build_and_execute(|| { + // given + assert_eq!( + List::::get_bags(), + vec![(10, vec![1]), (1_000, vec![2, 3, 4]), (2_000, vec![5, 6])] + ); + assert_eq!( + get_list_as_ids(), + vec![ + 5, 6, // best bag + 2, 3, 4, // middle bag + 1, // last bag. + ] + ); + + // when adding an id that has a higher weight than pre-existing ids in the bag + assert_ok!(List::::insert(7, 10)); + + // then + assert_eq!( + get_list_as_ids(), + vec![ + 5, 6, // best bag + 2, 3, 4, // middle bag + 1, 7, // last bag; new id is last. + ] + ); + }) + } + + /// we can `take` x ids, even if that quantity ends midway through a list. + #[test] + fn take_works() { + ExtBuilder::default() + .add_ids(vec![(5, 2_000), (6, 2_000)]) + .build_and_execute(|| { + // given + assert_eq!( + List::::get_bags(), + vec![(10, vec![1]), (1_000, vec![2, 3, 4]), (2_000, vec![5, 6])] + ); + + // when + let iteration = + List::::iter().map(|node| *node.id()).take(4).collect::>(); + + // then + assert_eq!( + iteration, + vec![ + 5, 6, // best bag, fully iterated + 2, 3, // middle bag, partially iterated + ] + ); + }) + } + + #[test] + fn insert_works() { + ExtBuilder::default().build_and_execute(|| { + // when inserting into an existing bag + assert_ok!(List::::insert(5, 1_000)); + + // then + assert_eq!(List::::get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4, 5])]); + assert_eq!(get_list_as_ids(), vec![2, 3, 4, 5, 1]); + + // when inserting into a non-existent bag + assert_ok!(List::::insert(6, 1_001)); + + // then + assert_eq!( + List::::get_bags(), + vec![(10, vec![1]), (1_000, vec![2, 3, 4, 5]), (2_000, vec![6])] + ); + assert_eq!(get_list_as_ids(), vec![6, 2, 3, 4, 5, 1]); + }); + } + + #[test] + fn insert_errors_with_duplicate_id() { + ExtBuilder::default().build_and_execute(|| { + // given + assert!(get_list_as_ids().contains(&3)); + + // then + assert_storage_noop!(assert_eq!( + List::::insert(3, 20).unwrap_err(), + Error::Duplicate + )); + }); + } + + #[test] + fn remove_works() { + use crate::{CounterForListNodes, ListBags, ListNodes}; + let ensure_left = |id, counter| { + assert!(!ListNodes::::contains_key(id)); + assert_eq!(CounterForListNodes::::get(), counter); + assert_eq!(ListNodes::::iter().count() as u32, counter); + }; + + ExtBuilder::default().build_and_execute(|| { + // removing a non-existent id is a noop + assert!(!ListNodes::::contains_key(42)); + assert_storage_noop!(List::::remove(&42)); + + // when removing a node from a bag with multiple nodes: + List::::remove(&2); + + // then + assert_eq!(get_list_as_ids(), vec![3, 4, 1]); + assert_eq!(List::::get_bags(), vec![(10, vec![1]), (1_000, vec![3, 4])]); + ensure_left(2, 3); + + // when removing a node from a bag with only one node: + List::::remove(&1); + + // then + assert_eq!(get_list_as_ids(), vec![3, 4]); + assert_eq!(List::::get_bags(), vec![(1_000, vec![3, 4])]); + ensure_left(1, 2); + // bag 10 is removed + assert!(!ListBags::::contains_key(10)); + + // remove remaining ids to make sure storage cleans up as expected + List::::remove(&3); + ensure_left(3, 1); + assert_eq!(get_list_as_ids(), vec![4]); + + List::::remove(&4); + ensure_left(4, 0); + assert_eq!(get_list_as_ids(), Vec::::new()); + + // bags are deleted via removals + assert_eq!(ListBags::::iter().count(), 0); + }); + } + + #[test] + fn remove_many_is_noop_with_non_existent_ids() { + ExtBuilder::default().build_and_execute(|| { + let non_existent_ids = vec![&42, &666, &13]; + + // when account ids don' exist in the list + assert!(non_existent_ids.iter().all(|id| !BagsList::contains(id))); + + // then removing them is a noop + assert_storage_noop!(List::::remove_many(non_existent_ids)); + }); + } + + #[test] + fn update_position_for_works() { + ExtBuilder::default().build_and_execute(|| { + // given a correctly placed account 1 at bag 10. + let node = Node::::get(&1).unwrap(); + assert!(!node.is_misplaced(10)); + + // .. it is invalid with weight 20 + assert!(node.is_misplaced(20)); + + // move it to bag 20. + assert_eq!(List::::update_position_for(node, 20), Some((10, 20))); + + assert_eq!(List::::get_bags(), vec![(20, vec![1]), (1_000, vec![2, 3, 4])]); + + // get the new updated node; try and update the position with no change in weight. + let node = Node::::get(&1).unwrap(); + assert_storage_noop!(assert_eq!( + List::::update_position_for(node.clone(), 20), + None + )); + + // then move it to bag 1_000 by giving it weight 500. + assert_eq!(List::::update_position_for(node.clone(), 500), Some((20, 1_000))); + assert_eq!(List::::get_bags(), vec![(1_000, vec![2, 3, 4, 1])]); + + // moving within that bag again is a noop + let node = Node::::get(&1).unwrap(); + assert_storage_noop!(assert_eq!( + List::::update_position_for(node.clone(), 750), + None, + )); + assert_storage_noop!(assert_eq!( + List::::update_position_for(node, 1_000), + None, + )); + }); + } + + #[test] + fn sanity_check_works() { + ExtBuilder::default().build_and_execute_no_post_check(|| { + assert_ok!(List::::sanity_check()); + }); + + // make sure there are no duplicates. + ExtBuilder::default().build_and_execute_no_post_check(|| { + Bag::::get(10).unwrap().insert_unchecked(2); + assert_eq!(List::::sanity_check(), Err("duplicate identified")); + }); + + // ensure count is in sync with `CounterForListNodes`. + ExtBuilder::default().build_and_execute_no_post_check(|| { + crate::CounterForListNodes::::mutate(|counter| *counter += 1); + assert_eq!(crate::CounterForListNodes::::get(), 5); + assert_eq!(List::::sanity_check(), Err("iter_count != stored_count")); + }); + } + + #[test] + fn contains_works() { + ExtBuilder::default().build_and_execute(|| { + assert!(GENESIS_IDS.iter().all(|(id, _)| List::::contains(id))); + + let non_existent_ids = vec![&42, &666, &13]; + assert!(non_existent_ids.iter().all(|id| !List::::contains(id))); + }) + } +} + +mod bags { + use super::*; + + #[test] + fn get_works() { + ExtBuilder::default().build_and_execute(|| { + let check_bag = |bag_upper, head, tail, ids| { + let bag = Bag::::get(bag_upper).unwrap(); + let bag_ids = bag.iter().map(|n| *n.id()).collect::>(); + + assert_eq!(bag, Bag:: { head, tail, bag_upper }); + assert_eq!(bag_ids, ids); + }; + + assert_eq!(List::::get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4])]); + + // we can fetch them + check_bag(10, Some(1), Some(1), vec![1]); + check_bag(1_000, Some(2), Some(4), vec![2, 3, 4]); + + // and all other bag thresholds don't get bags. + ::BagThresholds::get() + .iter() + .chain(iter::once(&VoteWeight::MAX)) + .filter(|bag_upper| !vec![10, 1_000].contains(bag_upper)) + .for_each(|bag_upper| { + assert_storage_noop!(assert_eq!(Bag::::get(*bag_upper), None)); + assert!(!ListBags::::contains_key(*bag_upper)); + }); + + // when we make a pre-existing bag empty + List::::remove(&1); + + // then + assert_eq!(Bag::::get(10), None) + }); + } + + #[test] + fn insert_node_sets_proper_bag() { + ExtBuilder::default().build_and_execute_no_post_check(|| { + let node = |id, bag_upper| Node:: { id, prev: None, next: None, bag_upper }; + + assert_eq!(List::::get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4])]); + + let mut bag_10 = Bag::::get(10).unwrap(); + bag_10.insert_node_unchecked(node(42, 5)); + + assert_eq!( + ListNodes::::get(&42).unwrap(), + Node { bag_upper: 10, prev: Some(1), next: None, id: 42 } + ); + }); + } + + #[test] + fn insert_node_happy_paths_works() { + ExtBuilder::default().build_and_execute_no_post_check(|| { + let node = |id, bag_upper| Node:: { id, prev: None, next: None, bag_upper }; + + // when inserting into a bag with 1 node + let mut bag_10 = Bag::::get(10).unwrap(); + bag_10.insert_node_unchecked(node(42, bag_10.bag_upper)); + // then + assert_eq!(bag_as_ids(&bag_10), vec![1, 42]); + + // when inserting into a bag with 3 nodes + let mut bag_1000 = Bag::::get(1_000).unwrap(); + bag_1000.insert_node_unchecked(node(52, bag_1000.bag_upper)); + // then + assert_eq!(bag_as_ids(&bag_1000), vec![2, 3, 4, 52]); + + // when inserting into a new bag + let mut bag_20 = Bag::::get_or_make(20); + bag_20.insert_node_unchecked(node(62, bag_20.bag_upper)); + // then + assert_eq!(bag_as_ids(&bag_20), vec![62]); + + // when inserting a node pointing to the accounts not in the bag + let node_61 = + Node:: { id: 61, prev: Some(21), next: Some(101), bag_upper: 20 }; + bag_20.insert_node_unchecked(node_61); + // then ids are in order + assert_eq!(bag_as_ids(&bag_20), vec![62, 61]); + // and when the node is re-fetched all the info is correct + assert_eq!( + Node::::get(&61).unwrap(), + Node:: { id: 61, prev: Some(62), next: None, bag_upper: 20 } + ); + + // state of all bags is as expected + bag_20.put(); // need to put this newly created bag so its in the storage map + assert_eq!( + List::::get_bags(), + vec![(10, vec![1, 42]), (20, vec![62, 61]), (1_000, vec![2, 3, 4, 52])] + ); + }); + } + + // Document improper ways `insert_node` may be getting used. + #[test] + fn insert_node_bad_paths_documented() { + let node = |id, prev, next, bag_upper| Node:: { id, prev, next, bag_upper }; + ExtBuilder::default().build_and_execute_no_post_check(|| { + // when inserting a node with both prev & next pointing at an account in an incorrect + // bag. + let mut bag_1000 = Bag::::get(1_000).unwrap(); + bag_1000.insert_node_unchecked(node(42, Some(1), Some(1), 500)); + + // then the proper prev and next is set. + assert_eq!(bag_as_ids(&bag_1000), vec![2, 3, 4, 42]); + + // and when the node is re-fetched all the info is correct + assert_eq!( + Node::::get(&42).unwrap(), + node(42, Some(4), None, bag_1000.bag_upper) + ); + }); + + ExtBuilder::default().build_and_execute_no_post_check(|| { + // given 3 is in bag_1000 (and not a tail node) + let mut bag_1000 = Bag::::get(1_000).unwrap(); + assert_eq!(bag_as_ids(&bag_1000), vec![2, 3, 4]); + + // when inserting a node with duplicate id 3 + bag_1000.insert_node_unchecked(node(3, None, None, bag_1000.bag_upper)); + + // then all the nodes after the duplicate are lost (because it is set as the tail) + assert_eq!(bag_as_ids(&bag_1000), vec![2, 3]); + // also in the full iteration, 2 and 3 are from bag_1000 and 1 is from bag_10. + assert_eq!(get_list_as_ids(), vec![2, 3, 1]); + + // and the last accessible node has an **incorrect** prev pointer. + assert_eq!( + Node::::get(&3).unwrap(), + node(3, Some(4), None, bag_1000.bag_upper) + ); + }); + + ExtBuilder::default().build_and_execute_no_post_check(|| { + // when inserting a duplicate id of the head + let mut bag_1000 = Bag::::get(1_000).unwrap(); + assert_eq!(bag_as_ids(&bag_1000), vec![2, 3, 4]); + bag_1000.insert_node_unchecked(node(2, None, None, 0)); + + // then all nodes after the head are lost + assert_eq!(bag_as_ids(&bag_1000), vec![2]); + + // and the re-fetched node has bad pointers + assert_eq!( + Node::::get(&2).unwrap(), + node(2, Some(4), None, bag_1000.bag_upper) + ); + // ^^^ despite being the bags head, it has a prev + + assert_eq!(bag_1000, Bag { head: Some(2), tail: Some(2), bag_upper: 1_000 }) + }); + } + + // Panics in case of duplicate tail insert (which would result in an infinite loop). + #[test] + #[should_panic = "system logic error: inserting a node who has the id of tail"] + fn insert_node_duplicate_tail_panics_with_debug_assert() { + ExtBuilder::default().build_and_execute(|| { + let node = |id, prev, next, bag_upper| Node:: { id, prev, next, bag_upper }; + + // given + assert_eq!(List::::get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4])],); + let mut bag_1000 = Bag::::get(1_000).unwrap(); + + // when inserting a duplicate id that is already the tail + assert_eq!(bag_1000.tail, Some(4)); + bag_1000.insert_node_unchecked(node(4, None, None, bag_1000.bag_upper)); // panics + }); + } + + #[test] + fn remove_node_happy_paths_works() { + ExtBuilder::default() + .add_ids(vec![ + (11, 10), + (12, 10), + (13, 1_000), + (14, 1_000), + (15, 2_000), + (16, 2_000), + (17, 2_000), + (18, 2_000), + (19, 2_000), + ]) + .build_and_execute_no_post_check(|| { + let mut bag_10 = Bag::::get(10).unwrap(); + let mut bag_1000 = Bag::::get(1_000).unwrap(); + let mut bag_2000 = Bag::::get(2_000).unwrap(); + + // given + assert_eq!(bag_as_ids(&bag_10), vec![1, 11, 12]); + assert_eq!(bag_as_ids(&bag_1000), vec![2, 3, 4, 13, 14]); + assert_eq!(bag_as_ids(&bag_2000), vec![15, 16, 17, 18, 19]); + + // when removing a node that is not pointing at the head or tail + let node_4 = Node::::get(&4).unwrap(); + let node_4_pre_remove = node_4.clone(); + bag_1000.remove_node_unchecked(&node_4); + + // then + assert_eq!(bag_as_ids(&bag_1000), vec![2, 3, 13, 14]); + assert_ok!(bag_1000.sanity_check()); + // and the node isn't mutated when its removed + assert_eq!(node_4, node_4_pre_remove); + + // when removing a head that is not pointing at the tail + let node_2 = Node::::get(&2).unwrap(); + bag_1000.remove_node_unchecked(&node_2); + + // then + assert_eq!(bag_as_ids(&bag_1000), vec![3, 13, 14]); + assert_ok!(bag_1000.sanity_check()); + + // when removing a tail that is not pointing at the head + let node_14 = Node::::get(&14).unwrap(); + bag_1000.remove_node_unchecked(&node_14); + + // then + assert_eq!(bag_as_ids(&bag_1000), vec![3, 13]); + assert_ok!(bag_1000.sanity_check()); + + // when removing a tail that is pointing at the head + let node_13 = Node::::get(&13).unwrap(); + bag_1000.remove_node_unchecked(&node_13); + + // then + assert_eq!(bag_as_ids(&bag_1000), vec![3]); + assert_ok!(bag_1000.sanity_check()); + + // when removing a node that is both the head & tail + let node_3 = Node::::get(&3).unwrap(); + bag_1000.remove_node_unchecked(&node_3); + bag_1000.put(); // put into storage so `get` returns the updated bag + + // then + assert_eq!(Bag::::get(1_000), None); + + // when removing a node that is pointing at both the head & tail + let node_11 = Node::::get(&11).unwrap(); + bag_10.remove_node_unchecked(&node_11); + + // then + assert_eq!(bag_as_ids(&bag_10), vec![1, 12]); + assert_ok!(bag_10.sanity_check()); + + // when removing a head that is pointing at the tail + let node_1 = Node::::get(&1).unwrap(); + bag_10.remove_node_unchecked(&node_1); + + // then + assert_eq!(bag_as_ids(&bag_10), vec![12]); + assert_ok!(bag_10.sanity_check()); + // and since we updated the bag's head/tail, we need to write this storage so we + // can correctly `get` it again in later checks + bag_10.put(); + + // when removing a node that is pointing at the head but not the tail + let node_16 = Node::::get(&16).unwrap(); + bag_2000.remove_node_unchecked(&node_16); + + // then + assert_eq!(bag_as_ids(&bag_2000), vec![15, 17, 18, 19]); + assert_ok!(bag_2000.sanity_check()); + + // when removing a node that is pointing at tail, but not head + let node_18 = Node::::get(&18).unwrap(); + bag_2000.remove_node_unchecked(&node_18); + + // then + assert_eq!(bag_as_ids(&bag_2000), vec![15, 17, 19]); + assert_ok!(bag_2000.sanity_check()); + + // finally, when reading from storage, the state of all bags is as expected + assert_eq!( + List::::get_bags(), + vec![(10, vec![12]), (2_000, vec![15, 17, 19])] + ); + }); + } + + #[test] + fn remove_node_bad_paths_documented() { + ExtBuilder::default().build_and_execute_no_post_check(|| { + let bad_upper_node_2 = Node:: { + id: 2, + prev: None, + next: Some(3), + bag_upper: 10, // should be 1_000 + }; + let mut bag_1000 = Bag::::get(1_000).unwrap(); + + // when removing a node that is in the bag but has the wrong upper + bag_1000.remove_node_unchecked(&bad_upper_node_2); + bag_1000.put(); + + // then the node is no longer in any bags + assert_eq!(List::::get_bags(), vec![(10, vec![1]), (1_000, vec![3, 4])]); + // .. and the bag it was removed from + let bag_1000 = Bag::::get(1_000).unwrap(); + // is sane + assert_ok!(bag_1000.sanity_check()); + // and has the correct head and tail. + assert_eq!(bag_1000.head, Some(3)); + assert_eq!(bag_1000.tail, Some(4)); + }); + + // Removing a node that is in another bag, will mess up that other bag. + ExtBuilder::default().build_and_execute_no_post_check(|| { + // given a tail node is in bag 1_000 + let node_4 = Node::::get(&4).unwrap(); + + // when we remove it from bag 10 + let mut bag_10 = Bag::::get(10).unwrap(); + bag_10.remove_node_unchecked(&node_4); + bag_10.put(); + + // then bag remove was called on is ok, + let bag_10 = Bag::::get(10).unwrap(); + assert_eq!(bag_10.tail, Some(1)); + assert_eq!(bag_10.head, Some(1)); + + // but the bag that the node belonged to is in an invalid state + let bag_1000 = Bag::::get(1_000).unwrap(); + // because it still has the removed node as its tail. + assert_eq!(bag_1000.tail, Some(4)); + assert_eq!(bag_1000.head, Some(2)); + }); + } +} + +mod node { + use super::*; + + #[test] + fn is_misplaced_works() { + ExtBuilder::default().build_and_execute(|| { + let node = Node::::get(&1).unwrap(); + + // given + assert_eq!(node.bag_upper, 10); + + // then within bag 10 its not misplaced, + assert!(!node.is_misplaced(0)); + assert!(!node.is_misplaced(9)); + assert!(!node.is_misplaced(10)); + + // and out of bag 10 it is misplaced + assert!(node.is_misplaced(11)); + }); + } +} diff --git a/frame/bags-list/src/mock.rs b/frame/bags-list/src/mock.rs new file mode 100644 index 0000000000000..a6ab35896b1e7 --- /dev/null +++ b/frame/bags-list/src/mock.rs @@ -0,0 +1,154 @@ +// This file is part of Substrate. + +// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Mock runtime for pallet-bags-lists tests. + +use super::*; +use crate::{self as bags_list}; +use frame_election_provider_support::VoteWeight; +use frame_support::parameter_types; + +pub type AccountId = u32; +pub type Balance = u32; + +parameter_types! { + pub static NextVoteWeight: VoteWeight = 0; +} + +pub struct StakingMock; +impl frame_election_provider_support::VoteWeightProvider for StakingMock { + fn vote_weight(id: &AccountId) -> VoteWeight { + match id { + 710 => 15, + 711 => 16, + 712 => 2_000, // special cases used for migrate test + _ => NextVoteWeight::get(), + } + } + #[cfg(any(feature = "runtime-benchmarks", test))] + fn set_vote_weight_of(_: &AccountId, weight: VoteWeight) { + // we don't really keep a mapping, just set weight for everyone. + NextVoteWeight::set(weight) + } +} + +impl frame_system::Config for Runtime { + type SS58Prefix = (); + type BaseCallFilter = frame_support::traits::Everything; + type Origin = Origin; + type Index = u64; + type BlockNumber = u64; + type Call = Call; + type Hash = sp_core::H256; + type Hashing = sp_runtime::traits::BlakeTwo256; + type AccountId = AccountId; + type Lookup = sp_runtime::traits::IdentityLookup; + type Header = sp_runtime::testing::Header; + type Event = Event; + type BlockHashCount = (); + type DbWeight = (); + type BlockLength = (); + type BlockWeights = (); + type Version = (); + type PalletInfo = PalletInfo; + type AccountData = pallet_balances::AccountData; + type OnNewAccount = (); + type OnKilledAccount = (); + type SystemWeightInfo = (); + type OnSetCode = (); +} + +parameter_types! { + pub static BagThresholds: &'static [VoteWeight] = &[10, 20, 30, 40, 50, 60, 1_000, 2_000, 10_000]; +} + +impl bags_list::Config for Runtime { + type Event = Event; + type WeightInfo = (); + type BagThresholds = BagThresholds; + type VoteWeightProvider = StakingMock; +} + +type UncheckedExtrinsic = frame_system::mocking::MockUncheckedExtrinsic; +type Block = frame_system::mocking::MockBlock; +frame_support::construct_runtime!( + pub enum Runtime where + Block = Block, + NodeBlock = Block, + UncheckedExtrinsic = UncheckedExtrinsic, + { + System: frame_system::{Pallet, Call, Storage, Event, Config}, + BagsList: bags_list::{Pallet, Call, Storage, Event}, + } +); + +/// Default AccountIds and their weights. +pub(crate) const GENESIS_IDS: [(AccountId, VoteWeight); 4] = + [(1, 10), (2, 1_000), (3, 1_000), (4, 1_000)]; + +#[derive(Default)] +pub(crate) struct ExtBuilder { + ids: Vec<(AccountId, VoteWeight)>, +} + +impl ExtBuilder { + /// Add some AccountIds to insert into `List`. + pub(crate) fn add_ids(mut self, ids: Vec<(AccountId, VoteWeight)>) -> Self { + self.ids = ids; + self + } + + pub(crate) fn build(self) -> sp_io::TestExternalities { + sp_tracing::try_init_simple(); + let storage = frame_system::GenesisConfig::default().build_storage::().unwrap(); + + let mut ext = sp_io::TestExternalities::from(storage); + ext.execute_with(|| { + for (id, weight) in GENESIS_IDS.iter().chain(self.ids.iter()) { + frame_support::assert_ok!(List::::insert(*id, *weight)); + } + }); + + ext + } + + pub(crate) fn build_and_execute(self, test: impl FnOnce() -> ()) { + self.build().execute_with(|| { + test(); + List::::sanity_check().expect("Sanity check post condition failed") + }) + } + + pub(crate) fn build_and_execute_no_post_check(self, test: impl FnOnce() -> ()) { + self.build().execute_with(test) + } +} + +pub(crate) mod test_utils { + use super::*; + use list::Bag; + + /// Returns the ordered ids within the given bag. + pub(crate) fn bag_as_ids(bag: &Bag) -> Vec { + bag.iter().map(|n| *n.id()).collect::>() + } + + /// Returns the ordered ids from the list. + pub(crate) fn get_list_as_ids() -> Vec { + List::::iter().map(|n| *n.id()).collect::>() + } +} diff --git a/frame/bags-list/src/tests.rs b/frame/bags-list/src/tests.rs new file mode 100644 index 0000000000000..e94017730668b --- /dev/null +++ b/frame/bags-list/src/tests.rs @@ -0,0 +1,389 @@ +// This file is part of Substrate. + +// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use frame_support::{assert_ok, assert_storage_noop, traits::IntegrityTest}; + +use super::*; +use frame_election_provider_support::SortedListProvider; +use list::Bag; +use mock::{test_utils::*, *}; + +mod pallet { + use super::*; + + #[test] + fn rebag_works() { + ExtBuilder::default().add_ids(vec![(42, 20)]).build_and_execute(|| { + // given + assert_eq!( + List::::get_bags(), + vec![(10, vec![1]), (20, vec![42]), (1_000, vec![2, 3, 4])] + ); + + // when increasing vote weight to the level of non-existent bag + NextVoteWeight::set(2_000); + assert_ok!(BagsList::rebag(Origin::signed(0), 42)); + + // then a new bag is created and the id moves into it + assert_eq!( + List::::get_bags(), + vec![(10, vec![1]), (1_000, vec![2, 3, 4]), (2_000, vec![42])] + ); + + // when decreasing weight within the range of the current bag + NextVoteWeight::set(1001); + assert_ok!(BagsList::rebag(Origin::signed(0), 42)); + + // then the id does not move + assert_eq!( + List::::get_bags(), + vec![(10, vec![1]), (1_000, vec![2, 3, 4]), (2_000, vec![42])] + ); + + // when reducing weight to the level of a non-existent bag + NextVoteWeight::set(30); + assert_ok!(BagsList::rebag(Origin::signed(0), 42)); + + // then a new bag is created and the id moves into it + assert_eq!( + List::::get_bags(), + vec![(10, vec![1]), (30, vec![42]), (1_000, vec![2, 3, 4])] + ); + + // when increasing weight to the level of a pre-existing bag + NextVoteWeight::set(500); + assert_ok!(BagsList::rebag(Origin::signed(0), 42)); + + // then the id moves into that bag + assert_eq!( + List::::get_bags(), + vec![(10, vec![1]), (1_000, vec![2, 3, 4, 42])] + ); + }); + } + + // Rebagging the tail of a bag results in the old bag having a new tail and an overall correct + // state. + #[test] + fn rebag_tail_works() { + ExtBuilder::default().build_and_execute(|| { + // given + assert_eq!(List::::get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4])]); + + // when + NextVoteWeight::set(10); + assert_ok!(BagsList::rebag(Origin::signed(0), 4)); + + // then + assert_eq!(List::::get_bags(), vec![(10, vec![1, 4]), (1_000, vec![2, 3])]); + assert_eq!(Bag::::get(1_000).unwrap(), Bag::new(Some(2), Some(3), 1_000)); + + // when + assert_ok!(BagsList::rebag(Origin::signed(0), 3)); + + // then + assert_eq!(List::::get_bags(), vec![(10, vec![1, 4, 3]), (1_000, vec![2])]); + + assert_eq!(Bag::::get(10).unwrap(), Bag::new(Some(1), Some(3), 10)); + assert_eq!(Bag::::get(1_000).unwrap(), Bag::new(Some(2), Some(2), 1_000)); + assert_eq!(get_list_as_ids(), vec![2u32, 1, 4, 3]); + + // when + assert_ok!(BagsList::rebag(Origin::signed(0), 2)); + + // then + assert_eq!(List::::get_bags(), vec![(10, vec![1, 4, 3, 2])]); + assert_eq!(Bag::::get(1_000), None); + }); + } + + // Rebagging the head of a bag results in the old bag having a new head and an overall correct + // state. + #[test] + fn rebag_head_works() { + ExtBuilder::default().build_and_execute(|| { + // when + NextVoteWeight::set(10); + assert_ok!(BagsList::rebag(Origin::signed(0), 2)); + + // then + assert_eq!(List::::get_bags(), vec![(10, vec![1, 2]), (1_000, vec![3, 4])]); + assert_eq!(Bag::::get(1_000).unwrap(), Bag::new(Some(3), Some(4), 1_000)); + + // when + assert_ok!(BagsList::rebag(Origin::signed(0), 3)); + + // then + assert_eq!(List::::get_bags(), vec![(10, vec![1, 2, 3]), (1_000, vec![4])]); + assert_eq!(Bag::::get(1_000).unwrap(), Bag::new(Some(4), Some(4), 1_000)); + + // when + assert_ok!(BagsList::rebag(Origin::signed(0), 4)); + + // then + assert_eq!(List::::get_bags(), vec![(10, vec![1, 2, 3, 4])]); + assert_eq!(Bag::::get(1_000), None); + }); + } + + #[test] + fn wrong_rebag_is_noop() { + ExtBuilder::default().build_and_execute(|| { + let node_3 = list::Node::::get(&3).unwrap(); + // when account 3 is _not_ misplaced with weight 500 + NextVoteWeight::set(500); + assert!(!node_3.is_misplaced(500)); + + // then calling rebag on account 3 with weight 500 is a noop + assert_storage_noop!(assert_eq!(BagsList::rebag(Origin::signed(0), 3), Ok(()))); + + // when account 42 is not in the list + assert!(!BagsList::contains(&42)); + + // then rebag-ing account 42 is a noop + assert_storage_noop!(assert_eq!(BagsList::rebag(Origin::signed(0), 42), Ok(()))); + }); + } + + #[test] + #[should_panic = "thresholds must strictly increase, and have no duplicates"] + fn duplicate_in_bags_threshold_panics() { + const DUPE_THRESH: &[VoteWeight; 4] = &[10, 20, 30, 30]; + BagThresholds::set(DUPE_THRESH); + BagsList::integrity_test(); + } + + #[test] + #[should_panic = "thresholds must strictly increase, and have no duplicates"] + fn decreasing_in_bags_threshold_panics() { + const DECREASING_THRESH: &[VoteWeight; 4] = &[10, 30, 20, 40]; + BagThresholds::set(DECREASING_THRESH); + BagsList::integrity_test(); + } + + #[test] + fn empty_threshold_works() { + BagThresholds::set(Default::default()); // which is the same as passing `()` to `Get<_>`. + + ExtBuilder::default().build_and_execute(|| { + // everyone in the same bag. + assert_eq!(List::::get_bags(), vec![(VoteWeight::MAX, vec![1, 2, 3, 4])]); + + // any insertion goes there as well. + assert_ok!(List::::insert(5, 999)); + assert_ok!(List::::insert(6, 0)); + assert_eq!( + List::::get_bags(), + vec![(VoteWeight::MAX, vec![1, 2, 3, 4, 5, 6])] + ); + + // any rebag is noop. + assert_storage_noop!(assert!(BagsList::rebag(Origin::signed(0), 1).is_ok())); + assert_storage_noop!(assert!(BagsList::rebag(Origin::signed(0), 10).is_ok())); + }) + } +} + +mod sorted_list_provider { + use super::*; + + #[test] + fn iter_works() { + ExtBuilder::default().build_and_execute(|| { + let expected = vec![2, 3, 4, 1]; + for (i, id) in BagsList::iter().enumerate() { + assert_eq!(id, expected[i]) + } + }); + } + + #[test] + fn count_works() { + ExtBuilder::default().build_and_execute(|| { + // given + assert_eq!(BagsList::count(), 4); + + // when inserting + assert_ok!(BagsList::on_insert(201, 0)); + // then the count goes up + assert_eq!(BagsList::count(), 5); + + // when removing + BagsList::on_remove(&201); + // then the count goes down + assert_eq!(BagsList::count(), 4); + + // when updating + BagsList::on_update(&201, VoteWeight::MAX); + // then the count stays the same + assert_eq!(BagsList::count(), 4); + }); + } + + #[test] + fn on_insert_works() { + ExtBuilder::default().build_and_execute(|| { + // when + assert_ok!(BagsList::on_insert(6, 1_000)); + + // then the bags + assert_eq!(List::::get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4, 6])]); + // and list correctly include the new id, + assert_eq!(BagsList::iter().collect::>(), vec![2, 3, 4, 6, 1]); + // and the count is incremented. + assert_eq!(BagsList::count(), 5); + + // when + assert_ok!(BagsList::on_insert(7, 1_001)); + + // then the bags + assert_eq!( + List::::get_bags(), + vec![(10, vec![1]), (1_000, vec![2, 3, 4, 6]), (2_000, vec![7])] + ); + // and list correctly include the new id, + assert_eq!(BagsList::iter().collect::>(), vec![7, 2, 3, 4, 6, 1]); + // and the count is incremented. + assert_eq!(BagsList::count(), 6); + }) + } + + #[test] + fn on_insert_errors_with_duplicate_id() { + ExtBuilder::default().build_and_execute(|| { + // given + assert!(get_list_as_ids().contains(&3)); + + // then + assert_storage_noop!(assert_eq!( + BagsList::on_insert(3, 20).unwrap_err(), + Error::Duplicate + )); + }); + } + + #[test] + fn on_update_works() { + ExtBuilder::default().add_ids(vec![(42, 20)]).build_and_execute(|| { + // given + assert_eq!( + List::::get_bags(), + vec![(10, vec![1]), (20, vec![42]), (1_000, vec![2, 3, 4])] + ); + assert_eq!(BagsList::count(), 5); + + // when increasing weight to the level of non-existent bag + BagsList::on_update(&42, 2_000); + + // then the bag is created with the id in it, + assert_eq!( + List::::get_bags(), + vec![(10, vec![1]), (1_000, vec![2, 3, 4]), (2000, vec![42])] + ); + // and the id position is updated in the list. + assert_eq!(BagsList::iter().collect::>(), vec![42, 2, 3, 4, 1]); + + // when decreasing weight within the range of the current bag + BagsList::on_update(&42, 1_001); + + // then the id does not change bags, + assert_eq!( + List::::get_bags(), + vec![(10, vec![1]), (1_000, vec![2, 3, 4]), (2000, vec![42])] + ); + // or change position in the list. + assert_eq!(BagsList::iter().collect::>(), vec![42, 2, 3, 4, 1]); + + // when increasing weight to the level of a non-existent bag with the max threshold + BagsList::on_update(&42, VoteWeight::MAX); + + // the the new bag is created with the id in it, + assert_eq!( + List::::get_bags(), + vec![(10, vec![1]), (1_000, vec![2, 3, 4]), (VoteWeight::MAX, vec![42])] + ); + // and the id position is updated in the list. + assert_eq!(BagsList::iter().collect::>(), vec![42, 2, 3, 4, 1]); + + // when decreasing the weight to a pre-existing bag + BagsList::on_update(&42, 1_000); + + // then id is moved to the correct bag (as the last member), + assert_eq!( + List::::get_bags(), + vec![(10, vec![1]), (1_000, vec![2, 3, 4, 42])] + ); + // and the id position is updated in the list. + assert_eq!(BagsList::iter().collect::>(), vec![2, 3, 4, 42, 1]); + + // since we have only called on_update, the `count` has not changed. + assert_eq!(BagsList::count(), 5); + }); + } + + #[test] + fn on_remove_works() { + let ensure_left = |id, counter| { + assert!(!ListNodes::::contains_key(id)); + assert_eq!(BagsList::count(), counter); + assert_eq!(CounterForListNodes::::get(), counter); + assert_eq!(ListNodes::::iter().count() as u32, counter); + }; + + ExtBuilder::default().build_and_execute(|| { + // it is a noop removing a non-existent id + assert!(!ListNodes::::contains_key(42)); + assert_storage_noop!(BagsList::on_remove(&42)); + + // when removing a node from a bag with multiple nodes + BagsList::on_remove(&2); + + // then + assert_eq!(get_list_as_ids(), vec![3, 4, 1]); + assert_eq!(List::::get_bags(), vec![(10, vec![1]), (1_000, vec![3, 4])]); + ensure_left(2, 3); + + // when removing a node from a bag with only one node + BagsList::on_remove(&1); + + // then + assert_eq!(get_list_as_ids(), vec![3, 4]); + assert_eq!(List::::get_bags(), vec![(1_000, vec![3, 4])]); + ensure_left(1, 2); + + // when removing all remaining ids + BagsList::on_remove(&4); + assert_eq!(get_list_as_ids(), vec![3]); + ensure_left(4, 1); + BagsList::on_remove(&3); + + // then the storage is completely cleaned up + assert_eq!(get_list_as_ids(), Vec::::new()); + ensure_left(3, 0); + }); + } + + #[test] + fn contains_works() { + ExtBuilder::default().build_and_execute(|| { + assert!(GENESIS_IDS.iter().all(|(id, _)| BagsList::contains(id))); + + let non_existent_ids = vec![&42, &666, &13]; + assert!(non_existent_ids.iter().all(|id| !BagsList::contains(id))); + }) + } +} diff --git a/frame/bags-list/src/weights.rs b/frame/bags-list/src/weights.rs new file mode 100644 index 0000000000000..95d3dfa6eb989 --- /dev/null +++ b/frame/bags-list/src/weights.rs @@ -0,0 +1,95 @@ +// This file is part of Substrate. + +// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Autogenerated weights for pallet_bags_list +//! +//! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev +//! DATE: 2021-09-15, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 128 + +// Executed Command: +// target/release/substrate +// benchmark +// --chain=dev +// --steps=50 +// --repeat=20 +// --pallet=pallet_bags_list +// --extrinsic=* +// --execution=wasm +// --wasm-execution=compiled +// --heap-pages=4096 +// --output=./frame/bags-list/src/weights.rs +// --template=./.maintain/frame-weight-template.hbs + + +#![cfg_attr(rustfmt, rustfmt_skip)] +#![allow(unused_parens)] +#![allow(unused_imports)] + +use frame_support::{traits::Get, weights::{Weight, constants::RocksDbWeight}}; +use sp_std::marker::PhantomData; + +/// Weight functions needed for pallet_bags_list. +pub trait WeightInfo { + fn rebag_non_terminal() -> Weight; + fn rebag_terminal() -> Weight; +} + +/// Weights for pallet_bags_list using the Substrate node and recommended hardware. +pub struct SubstrateWeight(PhantomData); +impl WeightInfo for SubstrateWeight { + // Storage: Staking Bonded (r:1 w:0) + // Storage: Staking Ledger (r:1 w:0) + // Storage: BagsList ListNodes (r:4 w:4) + // Storage: BagsList ListBags (r:1 w:1) + fn rebag_non_terminal() -> Weight { + (74_175_000 as Weight) + .saturating_add(T::DbWeight::get().reads(7 as Weight)) + .saturating_add(T::DbWeight::get().writes(5 as Weight)) + } + // Storage: Staking Bonded (r:1 w:0) + // Storage: Staking Ledger (r:1 w:0) + // Storage: BagsList ListNodes (r:3 w:3) + // Storage: BagsList ListBags (r:2 w:2) + fn rebag_terminal() -> Weight { + (73_305_000 as Weight) + .saturating_add(T::DbWeight::get().reads(7 as Weight)) + .saturating_add(T::DbWeight::get().writes(5 as Weight)) + } +} + +// For backwards compatibility and tests +impl WeightInfo for () { + // Storage: Staking Bonded (r:1 w:0) + // Storage: Staking Ledger (r:1 w:0) + // Storage: BagsList ListNodes (r:4 w:4) + // Storage: BagsList ListBags (r:1 w:1) + fn rebag_non_terminal() -> Weight { + (74_175_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(7 as Weight)) + .saturating_add(RocksDbWeight::get().writes(5 as Weight)) + } + // Storage: Staking Bonded (r:1 w:0) + // Storage: Staking Ledger (r:1 w:0) + // Storage: BagsList ListNodes (r:3 w:3) + // Storage: BagsList ListBags (r:2 w:2) + fn rebag_terminal() -> Weight { + (73_305_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(7 as Weight)) + .saturating_add(RocksDbWeight::get().writes(5 as Weight)) + } +} diff --git a/frame/election-provider-multi-phase/src/lib.rs b/frame/election-provider-multi-phase/src/lib.rs index 269057b55b094..e83c49433e2bb 100644 --- a/frame/election-provider-multi-phase/src/lib.rs +++ b/frame/election-provider-multi-phase/src/lib.rs @@ -620,6 +620,15 @@ pub mod pallet { #[pallet::constant] type SignedDepositWeight: Get>; + /// The maximum number of voters to put in the snapshot. At the moment, snapshots are only + /// over a single block, but once multi-block elections are introduced they will take place + /// over multiple blocks. + /// + /// Also, note the data type: If the voters are represented by a `u32` in `type + /// CompactSolution`, the same `u32` is used here to ensure bounds are respected. + #[pallet::constant] + type VoterSnapshotPerBlock: Get>; + /// Handler for the slashed deposits. type SlashHandler: OnUnbalanced>; @@ -1274,7 +1283,8 @@ impl Pallet { fn create_snapshot_external( ) -> Result<(Vec, Vec>, u32), ElectionError> { let target_limit = >::max_value().saturated_into::(); - let voter_limit = >::max_value().saturated_into::(); + // for now we have just a single block snapshot. + let voter_limit = T::VoterSnapshotPerBlock::get().saturated_into::(); let targets = T::DataProvider::targets(Some(target_limit)).map_err(ElectionError::DataProvider)?; @@ -1933,7 +1943,8 @@ mod tests { } #[test] - fn snapshot_creation_fails_if_too_big() { + fn snapshot_too_big_failure_onchain_fallback() { + // the `MockStaking` is designed such that if it has too many targets, it simply fails. ExtBuilder::default().build_and_execute(|| { Targets::set((0..(TargetIndex::max_value() as AccountId) + 1).collect::>()); @@ -1949,6 +1960,49 @@ mod tests { roll_to(29); let supports = MultiPhase::elect().unwrap(); assert!(supports.len() > 0); + }); + } + + #[test] + fn snapshot_too_big_failure_no_fallback() { + // and if the backup mode is nothing, we go into the emergency mode.. + ExtBuilder::default().onchain_fallback(false).build_and_execute(|| { + crate::mock::Targets::set( + (0..(TargetIndex::max_value() as AccountId) + 1).collect::>(), + ); + + // Signed phase failed to open. + roll_to(15); + assert_eq!(MultiPhase::current_phase(), Phase::Off); + + // Unsigned phase failed to open. + roll_to(25); + assert_eq!(MultiPhase::current_phase(), Phase::Off); + + roll_to(29); + let err = MultiPhase::elect().unwrap_err(); + assert_eq!(err, ElectionError::Fallback("NoFallback.")); + assert_eq!(MultiPhase::current_phase(), Phase::Emergency); + }); + } + + #[test] + fn snapshot_too_big_truncate() { + // but if there are too many voters, we simply truncate them. + ExtBuilder::default().build_and_execute(|| { + // we have 8 voters in total. + assert_eq!(crate::mock::Voters::get().len(), 8); + // but we want to take 2. + crate::mock::VoterSnapshotPerBlock::set(2); + + // Signed phase opens just fine. + roll_to(15); + assert_eq!(MultiPhase::current_phase(), Phase::Signed); + + assert_eq!( + MultiPhase::snapshot_metadata().unwrap(), + SolutionOrSnapshotSize { voters: 2, targets: 4 } + ); }) } diff --git a/frame/election-provider-multi-phase/src/mock.rs b/frame/election-provider-multi-phase/src/mock.rs index 28a15291e6520..0d563955595a8 100644 --- a/frame/election-provider-multi-phase/src/mock.rs +++ b/frame/election-provider-multi-phase/src/mock.rs @@ -268,6 +268,7 @@ parameter_types! { pub static MinerMaxWeight: Weight = BlockWeights::get().max_block; pub static MinerMaxLength: u32 = 256; pub static MockWeightInfo: bool = false; + pub static VoterSnapshotPerBlock: VoterIndex = u32::max_value(); pub static EpochLength: u64 = 30; pub static OnChianFallback: bool = true; @@ -401,6 +402,7 @@ impl crate::Config for Runtime { type Fallback = MockFallback; type ForceOrigin = frame_system::EnsureRoot; type Solution = TestNposSolution; + type VoterSnapshotPerBlock = VoterSnapshotPerBlock; type Solver = SequentialPhragmen, Balancing>; } @@ -433,9 +435,9 @@ impl ElectionDataProvider for StakingMock { fn voters( maybe_max_len: Option, ) -> data_provider::Result)>> { - let voters = Voters::get(); - if maybe_max_len.map_or(false, |max_len| voters.len() > max_len) { - return Err("Voters too big") + let mut voters = Voters::get(); + if let Some(max_len) = maybe_max_len { + voters.truncate(max_len) } Ok(voters) diff --git a/frame/election-provider-support/src/lib.rs b/frame/election-provider-support/src/lib.rs index d2c4b1053cc6d..cb36e025c3bee 100644 --- a/frame/election-provider-support/src/lib.rs +++ b/frame/election-provider-support/src/lib.rs @@ -297,6 +297,75 @@ impl ElectionProvider for () { } } +/// A utility trait for something to implement `ElectionDataProvider` in a sensible way. +/// +/// This is generic over `AccountId` and it can represent a validator, a nominator, or any other +/// entity. +/// +/// To simplify the trait, the `VoteWeight` is hardcoded as the weight of each entity. The weights +/// are ascending, the higher, the better. In the long term, if this trait ends up having use cases +/// outside of the election context, it is easy enough to make it generic over the `VoteWeight`. +/// +/// Something that implements this trait will do a best-effort sort over ids, and thus can be +/// used on the implementing side of [`ElectionDataProvider`]. +pub trait SortedListProvider { + /// The list's error type. + type Error; + + /// An iterator over the list, which can have `take` called on it. + fn iter() -> Box>; + + /// The current count of ids in the list. + fn count() -> u32; + + /// Return true if the list already contains `id`. + fn contains(id: &AccountId) -> bool; + + /// Hook for inserting a new id. + fn on_insert(id: AccountId, weight: VoteWeight) -> Result<(), Self::Error>; + + /// Hook for updating a single id. + fn on_update(id: &AccountId, weight: VoteWeight); + + /// Hook for removing am id from the list. + fn on_remove(id: &AccountId); + + /// Regenerate this list from scratch. Returns the count of items inserted. + /// + /// This should typically only be used at a runtime upgrade. + fn regenerate( + all: impl IntoIterator, + weight_of: Box VoteWeight>, + ) -> u32; + + /// Remove `maybe_count` number of items from the list. Returns the number of items actually + /// removed. WARNING: removes all items if `maybe_count` is `None`, which should never be done + /// in production settings because it can lead to an unbounded amount of storage accesses. + fn clear(maybe_count: Option) -> u32; + + /// Sanity check internal state of list. Only meant for debug compilation. + fn sanity_check() -> Result<(), &'static str>; + + /// If `who` changes by the returned amount they are guaranteed to have a worst case change + /// in their list position. + #[cfg(feature = "runtime-benchmarks")] + fn weight_update_worst_case(_who: &AccountId, _is_increase: bool) -> VoteWeight { + VoteWeight::MAX + } +} + +/// Something that can provide the `VoteWeight` of an account. Similar to [`ElectionProvider`] and +/// [`ElectionDataProvider`], this should typically be implementing by whoever is supposed to *use* +/// `SortedListProvider`. +pub trait VoteWeightProvider { + /// Get the current `VoteWeight` of `who`. + fn vote_weight(who: &AccountId) -> VoteWeight; + + /// For tests and benchmarks, set the `VoteWeight`. + #[cfg(any(feature = "runtime-benchmarks", test))] + fn set_vote_weight_of(_: &AccountId, _: VoteWeight) {} +} + /// Something that can compute the result to an NPoS solution. pub trait NposSolver { /// The account identifier type of this solver. diff --git a/frame/executive/src/lib.rs b/frame/executive/src/lib.rs index 655a38fe1b540..9a0fce4d6b5b4 100644 --- a/frame/executive/src/lib.rs +++ b/frame/executive/src/lib.rs @@ -229,7 +229,7 @@ where (frame_system::Pallet::, COnRuntimeUpgrade, AllPallets) as OnRuntimeUpgrade - >::pre_upgrade()?; + >::pre_upgrade().unwrap(); let weight = Self::execute_on_runtime_upgrade(); @@ -237,7 +237,7 @@ where (frame_system::Pallet::, COnRuntimeUpgrade, AllPallets) as OnRuntimeUpgrade - >::post_upgrade()?; + >::post_upgrade().unwrap(); Ok(weight) } diff --git a/frame/grandpa/src/mock.rs b/frame/grandpa/src/mock.rs index 26dda514516a3..2f1b2630b2241 100644 --- a/frame/grandpa/src/mock.rs +++ b/frame/grandpa/src/mock.rs @@ -217,6 +217,7 @@ impl pallet_staking::Config for Test { type NextNewSession = Session; type ElectionProvider = onchain::OnChainSequentialPhragmen; type GenesisElectionProvider = Self::ElectionProvider; + type SortedListProvider = pallet_staking::UseNominatorsMap; type WeightInfo = (); } diff --git a/frame/offences/benchmarking/src/mock.rs b/frame/offences/benchmarking/src/mock.rs index c4fd88def0e33..82662295dea84 100644 --- a/frame/offences/benchmarking/src/mock.rs +++ b/frame/offences/benchmarking/src/mock.rs @@ -174,6 +174,7 @@ impl pallet_staking::Config for Test { type MaxNominatorRewardedPerValidator = MaxNominatorRewardedPerValidator; type ElectionProvider = onchain::OnChainSequentialPhragmen; type GenesisElectionProvider = Self::ElectionProvider; + type SortedListProvider = pallet_staking::UseNominatorsMap; type WeightInfo = (); } diff --git a/frame/session/benchmarking/src/mock.rs b/frame/session/benchmarking/src/mock.rs index c685db2bb2524..4d3a1a2d8689d 100644 --- a/frame/session/benchmarking/src/mock.rs +++ b/frame/session/benchmarking/src/mock.rs @@ -182,6 +182,7 @@ impl pallet_staking::Config for Test { type MaxNominatorRewardedPerValidator = MaxNominatorRewardedPerValidator; type ElectionProvider = onchain::OnChainSequentialPhragmen; type GenesisElectionProvider = Self::ElectionProvider; + type SortedListProvider = pallet_staking::UseNominatorsMap; type WeightInfo = (); } diff --git a/frame/staking/Cargo.toml b/frame/staking/Cargo.toml index aba19ba56357a..70637bcd7726f 100644 --- a/frame/staking/Cargo.toml +++ b/frame/staking/Cargo.toml @@ -39,9 +39,11 @@ rand_chacha = { version = "0.2", default-features = false, optional = true } [dev-dependencies] sp-tracing = { version = "4.0.0-dev", path = "../../primitives/tracing" } sp-core = { version = "4.0.0-dev", path = "../../primitives/core" } +sp-npos-elections = { version = "4.0.0-dev", path = "../../primitives/npos-elections" } pallet-balances = { version = "4.0.0-dev", path = "../balances" } pallet-timestamp = { version = "4.0.0-dev", path = "../timestamp" } pallet-staking-reward-curve = { version = "4.0.0-dev", path = "../staking/reward-curve" } +pallet-bags-list = { version = "4.0.0-dev", features = ["runtime-benchmarks"], path = "../bags-list" } substrate-test-utils = { version = "4.0.0-dev", path = "../../test-utils" } frame-benchmarking = { version = "4.0.0-dev", path = "../benchmarking" } frame-election-provider-support = { version = "4.0.0-dev", path = "../election-provider-support" } diff --git a/frame/staking/src/benchmarking.rs b/frame/staking/src/benchmarking.rs index bdc3d81f3c29b..f3def7206320c 100644 --- a/frame/staking/src/benchmarking.rs +++ b/frame/staking/src/benchmarking.rs @@ -21,9 +21,10 @@ use super::*; use crate::Pallet as Staking; use testing_utils::*; +use frame_election_provider_support::SortedListProvider; use frame_support::{ pallet_prelude::*, - traits::{Currency, Get, Imbalance}, + traits::{Currency, CurrencyToVote, Get, Imbalance}, }; use sp_runtime::{ traits::{StaticLookup, Zero}, @@ -110,6 +111,8 @@ pub fn create_validator_with_nominators( assert_eq!(new_validators.len(), 1); assert_eq!(new_validators[0], v_stash, "Our validator was not selected!"); + assert_ne!(CounterForValidators::::get(), 0); + assert_ne!(CounterForNominators::::get(), 0); // Give Era Points let reward = EraRewardPoints:: { @@ -129,13 +132,91 @@ pub fn create_validator_with_nominators( Ok((v_stash, nominators)) } +struct ListScenario { + /// Stash that is expected to be moved. + origin_stash1: T::AccountId, + /// Controller of the Stash that is expected to be moved. + origin_controller1: T::AccountId, + dest_weight: BalanceOf, +} + +impl ListScenario { + /// An expensive scenario for bags-list implementation: + /// + /// - the node to be updated (r) is the head of a bag that has at least one other node. The bag + /// itself will need to be read and written to update its head. The node pointed to by r.next + /// will need to be read and written as it will need to have its prev pointer updated. Note + /// that there are two other worst case scenarios for bag removal: 1) the node is a tail and + /// 2) the node is a middle node with prev and next; all scenarios end up with the same number + /// of storage reads and writes. + /// + /// - the destination bag has at least one node, which will need its next pointer updated. + /// + /// NOTE: while this scenario specifically targets a worst case for the bags-list, it should + /// also elicit a worst case for other known `SortedListProvider` implementations; although + /// this may not be true against unknown `SortedListProvider` implementations. + fn new(origin_weight: BalanceOf, is_increase: bool) -> Result { + ensure!(!origin_weight.is_zero(), "origin weight must be greater than 0"); + + // burn the entire issuance. + let i = T::Currency::burn(T::Currency::total_issuance()); + sp_std::mem::forget(i); + + // create accounts with the origin weight + + let (origin_stash1, origin_controller1) = create_stash_controller_with_balance::( + USER_SEED + 2, + origin_weight, + Default::default(), + )?; + Staking::::nominate( + RawOrigin::Signed(origin_controller1.clone()).into(), + // NOTE: these don't really need to be validators. + vec![T::Lookup::unlookup(account("random_validator", 0, SEED))], + )?; + + let (_origin_stash2, origin_controller2) = create_stash_controller_with_balance::( + USER_SEED + 3, + origin_weight, + Default::default(), + )?; + Staking::::nominate( + RawOrigin::Signed(origin_controller2.clone()).into(), + vec![T::Lookup::unlookup(account("random_validator", 0, SEED))].clone(), + )?; + + // find a destination weight that will trigger the worst case scenario + let dest_weight_as_vote = + T::SortedListProvider::weight_update_worst_case(&origin_stash1, is_increase); + + let total_issuance = T::Currency::total_issuance(); + + let dest_weight = + T::CurrencyToVote::to_currency(dest_weight_as_vote as u128, total_issuance); + + // create an account with the worst case destination weight + let (_dest_stash1, dest_controller1) = create_stash_controller_with_balance::( + USER_SEED + 1, + dest_weight, + Default::default(), + )?; + Staking::::nominate( + RawOrigin::Signed(dest_controller1).into(), + vec![T::Lookup::unlookup(account("random_validator", 0, SEED))], + )?; + + Ok(ListScenario { origin_stash1, origin_controller1, dest_weight }) + } +} + const USER_SEED: u32 = 999666; benchmarks! { bond { let stash = create_funded_user::("stash", USER_SEED, 100); let controller = create_funded_user::("controller", USER_SEED, 100); - let controller_lookup: ::Source = T::Lookup::unlookup(controller.clone()); + let controller_lookup: ::Source + = T::Lookup::unlookup(controller.clone()); let reward_destination = RewardDestination::Staked; let amount = T::Currency::minimum_balance() * 10u32.into(); whitelist_account!(stash); @@ -146,10 +227,25 @@ benchmarks! { } bond_extra { - let (stash, controller) = create_stash_controller::(USER_SEED, 100, Default::default())?; - let max_additional = T::Currency::minimum_balance() * 10u32.into(); - let ledger = Ledger::::get(&controller).ok_or("ledger not created before")?; - let original_bonded: BalanceOf = ledger.active; + // clean up any existing state. + clear_validators_and_nominators::(); + + let origin_weight = MinNominatorBond::::get().max(T::Currency::minimum_balance()); + + // setup the worst case list scenario. + + // the weight the nominator will start at. + let scenario = ListScenario::::new(origin_weight, true)?; + + let max_additional = scenario.dest_weight.clone() - origin_weight; + + let stash = scenario.origin_stash1.clone(); + let controller = scenario.origin_controller1.clone(); + let original_bonded: BalanceOf + = Ledger::::get(&controller).map(|l| l.active).ok_or("ledger not created after")?; + + T::Currency::deposit_into_existing(&stash, max_additional).unwrap(); + whitelist_account!(stash); }: _(RawOrigin::Signed(stash), max_additional) verify { @@ -159,10 +255,25 @@ benchmarks! { } unbond { - let (_, controller) = create_stash_controller::(USER_SEED, 100, Default::default())?; - let amount = T::Currency::minimum_balance() * 10u32.into(); + use sp_std::convert::TryFrom; + // clean up any existing state. + clear_validators_and_nominators::(); + + // setup the worst case list scenario. + let total_issuance = T::Currency::total_issuance(); + // the weight the nominator will start at. The value used here is expected to be + // significantly higher than the first position in a list (e.g. the first bag threshold). + let origin_weight = BalanceOf::::try_from(952_994_955_240_703u128) + .map_err(|_| "balance expected to be a u128") + .unwrap(); + let scenario = ListScenario::::new(origin_weight, false)?; + + let stash = scenario.origin_stash1.clone(); + let controller = scenario.origin_controller1.clone(); + let amount = origin_weight - scenario.dest_weight.clone(); let ledger = Ledger::::get(&controller).ok_or("ledger not created before")?; let original_bonded: BalanceOf = ledger.active; + whitelist_account!(controller); }: _(RawOrigin::Signed(controller.clone()), amount) verify { @@ -194,26 +305,50 @@ benchmarks! { withdraw_unbonded_kill { // Slashing Spans let s in 0 .. MAX_SPANS; - let (stash, controller) = create_stash_controller::(0, 100, Default::default())?; - add_slashing_spans::(&stash, s); - let amount = T::Currency::minimum_balance() * 10u32.into(); - Staking::::unbond(RawOrigin::Signed(controller.clone()).into(), amount)?; + // clean up any existing state. + clear_validators_and_nominators::(); + + let origin_weight = MinNominatorBond::::get().max(T::Currency::minimum_balance()); + + // setup a worst case list scenario. Note that we don't care about the setup of the + // destination position because we are doing a removal from the list but no insert. + let scenario = ListScenario::::new(origin_weight, true)?; + let controller = scenario.origin_controller1.clone(); + let stash = scenario.origin_stash1.clone(); + assert!(T::SortedListProvider::contains(&stash)); + + let ed = T::Currency::minimum_balance(); + let mut ledger = Ledger::::get(&controller).unwrap(); + ledger.active = ed - One::one(); + Ledger::::insert(&controller, ledger); CurrentEra::::put(EraIndex::max_value()); - let ledger = Ledger::::get(&controller).ok_or("ledger not created before")?; - let original_total: BalanceOf = ledger.total; + whitelist_account!(controller); }: withdraw_unbonded(RawOrigin::Signed(controller.clone()), s) verify { assert!(!Ledger::::contains_key(controller)); + assert!(!T::SortedListProvider::contains(&stash)); } validate { - let (stash, controller) = create_stash_controller::(USER_SEED, 100, Default::default())?; + // clean up any existing state. + clear_validators_and_nominators::(); + + let origin_weight = MinNominatorBond::::get().max(T::Currency::minimum_balance()); + + // setup a worst case scenario where the user calling validate was formerly a nominator so + // they must be removed from the list. + let scenario = ListScenario::::new(origin_weight, true)?; + let controller = scenario.origin_controller1.clone(); + let stash = scenario.origin_stash1.clone(); + assert!(T::SortedListProvider::contains(&stash)); + let prefs = ValidatorPrefs::default(); whitelist_account!(controller); }: _(RawOrigin::Signed(controller), prefs) verify { - assert!(Validators::::contains_key(stash)); + assert!(Validators::::contains_key(&stash)); + assert!(!T::SortedListProvider::contains(&stash)); } kick { @@ -225,7 +360,7 @@ benchmarks! { // these are the other validators; there are `T::MAX_NOMINATIONS - 1` of them, so // there are a total of `T::MAX_NOMINATIONS` validators in the system. - let rest_of_validators = create_validators::(T::MAX_NOMINATIONS - 1, 100)?; + let rest_of_validators = create_validators_with_seed::(T::MAX_NOMINATIONS - 1, 100, 415)?; // this is the validator that will be kicking. let (stash, controller) = create_stash_controller::( @@ -282,18 +417,50 @@ benchmarks! { // Worst case scenario, T::MAX_NOMINATIONS nominate { let n in 1 .. T::MAX_NOMINATIONS; - let (stash, controller) = create_stash_controller::(n + 1, 100, Default::default())?; - let validators = create_validators::(n, 100)?; + + // clean up any existing state. + clear_validators_and_nominators::(); + + let origin_weight = MinNominatorBond::::get().max(T::Currency::minimum_balance()); + + // setup a worst case list scenario. Note we don't care about the destination position, because + // we are just doing an insert into the origin position. + let scenario = ListScenario::::new(origin_weight, true)?; + let (stash, controller) = create_stash_controller_with_balance::( + SEED + T::MAX_NOMINATIONS + 1, // make sure the account does not conflict with others + origin_weight, + Default::default(), + ).unwrap(); + + assert!(!Nominators::::contains_key(&stash)); + assert!(!T::SortedListProvider::contains(&stash)); + + let validators = create_validators::(n, 100).unwrap(); whitelist_account!(controller); }: _(RawOrigin::Signed(controller), validators) verify { - assert!(Nominators::::contains_key(stash)); + assert!(Nominators::::contains_key(&stash)); + assert!(T::SortedListProvider::contains(&stash)) } chill { - let (_, controller) = create_stash_controller::(USER_SEED, 100, Default::default())?; + // clean up any existing state. + clear_validators_and_nominators::(); + + let origin_weight = MinNominatorBond::::get().max(T::Currency::minimum_balance()); + + // setup a worst case list scenario. Note that we don't care about the setup of the + // destination position because we are doing a removal from the list but no insert. + let scenario = ListScenario::::new(origin_weight, true)?; + let controller = scenario.origin_controller1.clone(); + let stash = scenario.origin_stash1.clone(); + assert!(T::SortedListProvider::contains(&stash)); + whitelist_account!(controller); }: _(RawOrigin::Signed(controller)) + verify { + assert!(!T::SortedListProvider::contains(&stash)); + } set_payee { let (stash, controller) = create_stash_controller::(USER_SEED, 100, Default::default())?; @@ -345,11 +512,23 @@ benchmarks! { force_unstake { // Slashing Spans let s in 0 .. MAX_SPANS; - let (stash, controller) = create_stash_controller::(0, 100, Default::default())?; + // Clean up any existing state. + clear_validators_and_nominators::(); + + let origin_weight = MinNominatorBond::::get().max(T::Currency::minimum_balance()); + + // setup a worst case list scenario. Note that we don't care about the setup of the + // destination position because we are doing a removal from the list but no insert. + let scenario = ListScenario::::new(origin_weight, true)?; + let controller = scenario.origin_controller1.clone(); + let stash = scenario.origin_stash1.clone(); + assert!(T::SortedListProvider::contains(&stash)); add_slashing_spans::(&stash, s); - }: _(RawOrigin::Root, stash, s) + + }: _(RawOrigin::Root, stash.clone(), s) verify { assert!(!Ledger::::contains_key(&controller)); + assert!(!T::SortedListProvider::contains(&stash)); } cancel_deferred_slash { @@ -438,19 +617,46 @@ benchmarks! { rebond { let l in 1 .. MAX_UNLOCKING_CHUNKS as u32; - let (_, controller) = create_stash_controller::(USER_SEED, 100, Default::default())?; - let mut staking_ledger = Ledger::::get(controller.clone()).unwrap(); + + // clean up any existing state. + clear_validators_and_nominators::(); + + let origin_weight = MinNominatorBond::::get() + .max(T::Currency::minimum_balance()) + // we use 100 to play friendly with the list threshold values in the mock + .max(100u32.into()); + + // setup a worst case list scenario. + let scenario = ListScenario::::new(origin_weight, true)?; + let dest_weight = scenario.dest_weight.clone(); + + // rebond an amount that will give the user dest_weight + let rebond_amount = dest_weight - origin_weight; + + // spread that amount to rebond across `l` unlocking chunks, + let value = rebond_amount / l.into(); + // if `value` is zero, we need a greater delta between dest <=> origin weight + assert_ne!(value, Zero::zero()); + // so the sum of unlocking chunks puts voter into the dest bag. + assert!(value * l.into() + origin_weight > origin_weight); + assert!(value * l.into() + origin_weight <= dest_weight); let unlock_chunk = UnlockChunk::> { - value: 1u32.into(), + value, era: EraIndex::zero(), }; + + let stash = scenario.origin_stash1.clone(); + let controller = scenario.origin_controller1.clone(); + let mut staking_ledger = Ledger::::get(controller.clone()).unwrap(); + for _ in 0 .. l { staking_ledger.unlocking.push(unlock_chunk.clone()) } Ledger::::insert(controller.clone(), staking_ledger.clone()); let original_bonded: BalanceOf = staking_ledger.active; + whitelist_account!(controller); - }: _(RawOrigin::Signed(controller.clone()), (l + 100).into()) + }: _(RawOrigin::Signed(controller.clone()), rebond_amount) verify { let ledger = Ledger::::get(&controller).ok_or("ledger not created after")?; let new_bonded: BalanceOf = ledger.active; @@ -477,19 +683,28 @@ benchmarks! { reap_stash { let s in 1 .. MAX_SPANS; - let (stash, controller) = create_stash_controller::(0, 100, Default::default())?; - Staking::::validate(RawOrigin::Signed(controller.clone()).into(), ValidatorPrefs::default())?; + // clean up any existing state. + clear_validators_and_nominators::(); + + let origin_weight = MinNominatorBond::::get().max(T::Currency::minimum_balance()); + + // setup a worst case list scenario. Note that we don't care about the setup of the + // destination position because we are doing a removal from the list but no insert. + let scenario = ListScenario::::new(origin_weight, true)?; + let controller = scenario.origin_controller1.clone(); + let stash = scenario.origin_stash1.clone(); + add_slashing_spans::(&stash, s); T::Currency::make_free_balance_be(&stash, T::Currency::minimum_balance()); - whitelist_account!(controller); assert!(Bonded::::contains_key(&stash)); - assert!(Validators::::contains_key(&stash)); + assert!(T::SortedListProvider::contains(&stash)); + whitelist_account!(controller); }: _(RawOrigin::Signed(controller), stash.clone(), s) verify { assert!(!Bonded::::contains_key(&stash)); - assert!(!Validators::::contains_key(&stash)); + assert!(!T::SortedListProvider::contains(&stash)); } new_era { @@ -590,17 +805,21 @@ benchmarks! { // total number of slashing spans. Assigned to validators randomly. let s in 1 .. 20; - let validators = create_validators_with_nominators_for_era::(v, n, T::MAX_NOMINATIONS as usize, false, None)? - .into_iter() - .map(|v| T::Lookup::lookup(v).unwrap()) - .collect::>(); + let validators = create_validators_with_nominators_for_era::( + v, n, T::MAX_NOMINATIONS as usize, false, None + )? + .into_iter() + .map(|v| T::Lookup::lookup(v).unwrap()) + .collect::>(); (0..s).for_each(|index| { add_slashing_spans::(&validators[index as usize], 10); }); + + let num_voters = (v + n) as usize; }: { - let voters = >::get_npos_voters(); - assert_eq!(voters.len() as u32, v + n); + let voters = >::get_npos_voters(None); + assert_eq!(voters.len(), num_voters); } get_npos_targets { @@ -609,7 +828,9 @@ benchmarks! { // number of nominator intention. let n = MAX_NOMINATORS; - let _ = create_validators_with_nominators_for_era::(v, n, T::MAX_NOMINATIONS as usize, false, None)?; + let _ = create_validators_with_nominators_for_era::( + v, n, T::MAX_NOMINATIONS as usize, false, None + )?; }: { let targets = >::get_npos_targets(); assert_eq!(targets.len() as u32, v); @@ -633,8 +854,18 @@ benchmarks! { } chill_other { - let (_, controller) = create_stash_controller::(USER_SEED, 100, Default::default())?; - Staking::::validate(RawOrigin::Signed(controller.clone()).into(), ValidatorPrefs::default())?; + // clean up any existing state. + clear_validators_and_nominators::(); + + let origin_weight = MinNominatorBond::::get().max(T::Currency::minimum_balance()); + + // setup a worst case list scenario. Note that we don't care about the setup of the + // destination position because we are doing a removal from the list but no insert. + let scenario = ListScenario::::new(origin_weight, true)?; + let controller = scenario.origin_controller1.clone(); + let stash = scenario.origin_stash1.clone(); + assert!(T::SortedListProvider::contains(&stash)); + Staking::::set_staking_limits( RawOrigin::Root.into(), BalanceOf::::max_value(), @@ -643,10 +874,11 @@ benchmarks! { Some(0), Some(Percent::from_percent(0)) )?; + let caller = whitelisted_caller(); }: _(RawOrigin::Signed(caller), controller.clone()) verify { - assert!(!Validators::::contains_key(controller)); + assert!(!T::SortedListProvider::contains(&stash)); } } @@ -658,7 +890,7 @@ mod tests { #[test] fn create_validators_with_nominators_for_era_works() { - ExtBuilder::default().has_stakers(true).build_and_execute(|| { + ExtBuilder::default().build_and_execute(|| { let v = 10; let n = 100; @@ -674,6 +906,9 @@ mod tests { let count_validators = Validators::::iter().count(); let count_nominators = Nominators::::iter().count(); + assert_eq!(count_validators, CounterForValidators::::get() as usize); + assert_eq!(count_nominators, CounterForNominators::::get() as usize); + assert_eq!(count_validators, v as usize); assert_eq!(count_nominators, n as usize); }); @@ -681,7 +916,7 @@ mod tests { #[test] fn create_validator_with_nominators_works() { - ExtBuilder::default().has_stakers(true).build_and_execute(|| { + ExtBuilder::default().build_and_execute(|| { let n = 10; let (validator_stash, nominators) = create_validator_with_nominators::( @@ -706,7 +941,7 @@ mod tests { #[test] fn add_slashing_spans_works() { - ExtBuilder::default().has_stakers(true).build_and_execute(|| { + ExtBuilder::default().build_and_execute(|| { let n = 10; let (validator_stash, _nominators) = create_validator_with_nominators::( @@ -738,7 +973,7 @@ mod tests { #[test] fn test_payout_all() { - ExtBuilder::default().has_stakers(true).build_and_execute(|| { + ExtBuilder::default().build_and_execute(|| { let v = 10; let n = 100; diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index 31b35acdd99aa..136515a5d6168 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -100,6 +100,13 @@ //! //! An account can become a nominator via the [`nominate`](Call::nominate) call. //! +//! #### Voting +//! +//! Staking is closely related to elections; actual validators are chosen from among all potential +//! validators via election by the potential validators and nominators. To reduce use of the phrase +//! "potential validators and nominators", we often use the term **voters**, who are simply +//! the union of potential validators and nominators. +//! //! #### Rewards and Slash //! //! The **reward and slashing** procedure is the core of the Staking pallet, attempting to _embrace @@ -264,15 +271,15 @@ //! - [Session](../pallet_session/index.html): Used to manage sessions. Also, a list of new //! validators is stored in the Session pallet's `Validators` at the end of each era. -#![recursion_limit = "128"] #![cfg_attr(not(feature = "std"), no_std)] #[cfg(feature = "runtime-benchmarks")] pub mod benchmarking; -#[cfg(test)] -mod mock; #[cfg(any(feature = "runtime-benchmarks", test))] pub mod testing_utils; + +#[cfg(test)] +pub(crate) mod mock; #[cfg(test)] mod tests; @@ -420,6 +427,7 @@ pub struct UnlockChunk { } /// The ledger of a (bonded) stash. +#[cfg_attr(feature = "runtime-benchmarks", derive(Default))] #[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug, TypeInfo)] pub struct StakingLedger { /// The stash account whose balance is actually locked and at stake. @@ -727,11 +735,12 @@ enum Releases { V5_0_0, // blockable validators. V6_0_0, // removal of all storage associated with offchain phragmen. V7_0_0, // keep track of number of nominators / validators in map + V8_0_0, // populate `SortedListProvider`. } impl Default for Releases { fn default() -> Self { - Releases::V7_0_0 + Releases::V8_0_0 } } diff --git a/frame/staking/src/migrations.rs b/frame/staking/src/migrations.rs index d7fa2afc63082..7064f06dd12c7 100644 --- a/frame/staking/src/migrations.rs +++ b/frame/staking/src/migrations.rs @@ -18,6 +18,56 @@ use super::*; +pub mod v8 { + use frame_election_provider_support::SortedListProvider; + use frame_support::traits::Get; + + use crate::{Config, Nominators, Pallet, StorageVersion, Weight}; + + #[cfg(feature = "try-runtime")] + pub fn pre_migrate() -> Result<(), &'static str> { + frame_support::ensure!( + StorageVersion::::get() == crate::Releases::V7_0_0, + "must upgrade linearly" + ); + + crate::log!(info, "👜 staking bags-list migration passes PRE migrate checks ✅",); + Ok(()) + } + + /// Migration to sorted [`SortedListProvider`]. + pub fn migrate() -> Weight { + if StorageVersion::::get() == crate::Releases::V7_0_0 { + crate::log!(info, "migrating staking to Releases::V8_0_0"); + + let migrated = T::SortedListProvider::regenerate( + Nominators::::iter().map(|(id, _)| id), + Pallet::::weight_of_fn(), + ); + debug_assert_eq!(T::SortedListProvider::sanity_check(), Ok(())); + + StorageVersion::::put(crate::Releases::V8_0_0); + crate::log!( + info, + "👜 completed staking migration to Releases::V8_0_0 with {} voters migrated", + migrated, + ); + + T::BlockWeights::get().max_block + } else { + T::DbWeight::get().reads(1) + } + } + + #[cfg(feature = "try-runtime")] + pub fn post_migrate() -> Result<(), &'static str> { + T::SortedListProvider::sanity_check() + .map_err(|_| "SortedListProvider is not in a sane state.")?; + crate::log!(info, "👜 staking bags-list migration passes POST migrate checks ✅",); + Ok(()) + } +} + pub mod v7 { use super::*; diff --git a/frame/staking/src/mock.rs b/frame/staking/src/mock.rs index 0357fa05cb1dd..b3ce8e063cb61 100644 --- a/frame/staking/src/mock.rs +++ b/frame/staking/src/mock.rs @@ -17,9 +17,8 @@ //! Test utilities -use crate as staking; -use crate::*; -use frame_election_provider_support::onchain; +use crate::{self as pallet_staking, *}; +use frame_election_provider_support::{onchain, SortedListProvider}; use frame_support::{ assert_ok, parameter_types, traits::{ @@ -104,8 +103,9 @@ frame_support::construct_runtime!( Authorship: pallet_authorship::{Pallet, Call, Storage, Inherent}, Timestamp: pallet_timestamp::{Pallet, Call, Storage, Inherent}, Balances: pallet_balances::{Pallet, Call, Storage, Config, Event}, - Staking: staking::{Pallet, Call, Config, Storage, Event}, + Staking: pallet_staking::{Pallet, Call, Config, Storage, Event}, Session: pallet_session::{Pallet, Call, Storage, Event, Config}, + BagsList: pallet_bags_list::{Pallet, Call, Storage, Event}, } ); @@ -242,12 +242,26 @@ impl OnUnbalanced> for RewardRemainderMock { } } +const THRESHOLDS: [sp_npos_elections::VoteWeight; 9] = + [10, 20, 30, 40, 50, 60, 1_000, 2_000, 10_000]; + +parameter_types! { + pub static BagThresholds: &'static [sp_npos_elections::VoteWeight] = &THRESHOLDS; +} + +impl pallet_bags_list::Config for Test { + type Event = Event; + type WeightInfo = (); + type VoteWeightProvider = Staking; + type BagThresholds = BagThresholds; +} + impl onchain::Config for Test { type Accuracy = Perbill; type DataProvider = Staking; } -impl Config for Test { +impl crate::pallet::pallet::Config for Test { const MAX_NOMINATIONS: u32 = 16; type Currency = Balances; type UnixTime = Timestamp; @@ -267,6 +281,8 @@ impl Config for Test { type ElectionProvider = onchain::OnChainSequentialPhragmen; type GenesisElectionProvider = Self::ElectionProvider; type WeightInfo = (); + // NOTE: consider a macro and use `UseNominatorsMap` as well. + type SortedListProvider = BagsList; } impl frame_system::offchain::SendTransactionTypes for Test @@ -469,7 +485,7 @@ impl ExtBuilder { stakers.extend(self.stakers) } - let _ = staking::GenesisConfig:: { + let _ = pallet_staking::GenesisConfig:: { stakers, validator_count: self.validator_count, minimum_validator_count: self.minimum_validator_count, @@ -533,6 +549,10 @@ fn check_count() { let validator_count = Validators::::iter().count() as u32; assert_eq!(nominator_count, CounterForNominators::::get()); assert_eq!(validator_count, CounterForValidators::::get()); + + // the voters that the `SortedListProvider` list is storing for us. + let external_voters = ::SortedListProvider::count(); + assert_eq!(external_voters, nominator_count); } fn check_ledgers() { @@ -625,10 +645,14 @@ pub(crate) fn current_era() -> EraIndex { Staking::current_era().unwrap() } -pub(crate) fn bond_validator(stash: AccountId, ctrl: AccountId, val: Balance) { +pub(crate) fn bond(stash: AccountId, ctrl: AccountId, val: Balance) { let _ = Balances::make_free_balance_be(&stash, val); let _ = Balances::make_free_balance_be(&ctrl, val); assert_ok!(Staking::bond(Origin::signed(stash), ctrl, val, RewardDestination::Controller)); +} + +pub(crate) fn bond_validator(stash: AccountId, ctrl: AccountId, val: Balance) { + bond(stash, ctrl, val); assert_ok!(Staking::validate(Origin::signed(ctrl), ValidatorPrefs::default())); } @@ -638,9 +662,7 @@ pub(crate) fn bond_nominator( val: Balance, target: Vec, ) { - let _ = Balances::make_free_balance_be(&stash, val); - let _ = Balances::make_free_balance_be(&ctrl, val); - assert_ok!(Staking::bond(Origin::signed(stash), ctrl, val, RewardDestination::Controller)); + bond(stash, ctrl, val); assert_ok!(Staking::nominate(Origin::signed(ctrl), target)); } @@ -833,7 +855,7 @@ macro_rules! assert_session_era { }; } -pub(crate) fn staking_events() -> Vec> { +pub(crate) fn staking_events() -> Vec> { System::events() .into_iter() .map(|r| r.event) diff --git a/frame/staking/src/pallet/impls.rs b/frame/staking/src/pallet/impls.rs index fecd493eea022..3ae520872f278 100644 --- a/frame/staking/src/pallet/impls.rs +++ b/frame/staking/src/pallet/impls.rs @@ -17,7 +17,10 @@ //! Implementations for the Staking FRAME Pallet. -use frame_election_provider_support::{data_provider, ElectionProvider, Supports, VoteWeight}; +use frame_election_provider_support::{ + data_provider, ElectionDataProvider, ElectionProvider, SortedListProvider, Supports, + VoteWeight, VoteWeightProvider, +}; use frame_support::{ pallet_prelude::*, traits::{ @@ -26,6 +29,7 @@ use frame_support::{ }, weights::{Weight, WithPostDispatchInfo}, }; +use frame_system::pallet_prelude::BlockNumberFor; use pallet_session::historical; use sp_runtime::{ traits::{Bounded, Convert, SaturatedConversion, Saturating, Zero}, @@ -64,7 +68,7 @@ impl Pallet { /// /// This prevents call sites from repeatedly requesting `total_issuance` from backend. But it is /// important to be only used while the total issuance is not changing. - pub fn slashable_balance_of_fn() -> Box VoteWeight> { + pub fn weight_of_fn() -> Box VoteWeight> { // NOTE: changing this to unboxed `impl Fn(..)` return type and the pallet will still // compile, while some types in mock fail to resolve. let issuance = T::Currency::total_issuance(); @@ -73,6 +77,12 @@ impl Pallet { }) } + /// Same as `weight_of_fn`, but made for one time use. + pub fn weight_of(who: &T::AccountId) -> VoteWeight { + let issuance = T::Currency::total_issuance(); + Self::slashable_balance_of_vote_weight(who, issuance) + } + pub(super) fn do_payout_stakers( validator_stash: T::AccountId, era: EraIndex, @@ -629,54 +639,92 @@ impl Pallet { /// Get all of the voters that are eligible for the npos election. /// - /// This will use all on-chain nominators, and all the validators will inject a self vote. + /// `maybe_max_len` can imposes a cap on the number of voters returned; First all the validator + /// are included in no particular order, then remainder is taken from the nominators, as + /// returned by [`Config::SortedListProvider`]. + /// + /// This will use nominators, and all the validators will inject a self vote. /// /// This function is self-weighing as [`DispatchClass::Mandatory`]. /// /// ### Slashing /// /// All nominations that have been submitted before the last non-zero slash of the validator are - /// auto-chilled. - pub fn get_npos_voters() -> Vec<(T::AccountId, VoteWeight, Vec)> { - let weight_of = Self::slashable_balance_of_fn(); - let mut all_voters = Vec::new(); + /// auto-chilled, but still count towards the limit imposed by `maybe_max_len`. + pub fn get_npos_voters( + maybe_max_len: Option, + ) -> Vec<(T::AccountId, VoteWeight, Vec)> { + let max_allowed_len = { + let nominator_count = CounterForNominators::::get() as usize; + let validator_count = CounterForValidators::::get() as usize; + let all_voter_count = validator_count.saturating_add(nominator_count); + maybe_max_len.unwrap_or(all_voter_count).min(all_voter_count) + }; - let mut validator_count = 0u32; - for (validator, _) in >::iter() { + let mut all_voters = Vec::<_>::with_capacity(max_allowed_len); + + // first, grab all validators in no particular order, capped by the maximum allowed length. + let mut validators_taken = 0u32; + for (validator, _) in >::iter().take(max_allowed_len) { // Append self vote. - let self_vote = (validator.clone(), weight_of(&validator), vec![validator.clone()]); + let self_vote = + (validator.clone(), Self::weight_of(&validator), vec![validator.clone()]); all_voters.push(self_vote); - validator_count.saturating_inc(); + validators_taken.saturating_inc(); } - // Collect all slashing spans into a BTreeMap for further queries. + // .. and grab whatever we have left from nominators. + let nominators_quota = (max_allowed_len as u32).saturating_sub(validators_taken); let slashing_spans = >::iter().collect::>(); - let mut nominator_count = 0u32; - for (nominator, nominations) in Nominators::::iter() { - let Nominations { submitted_in, mut targets, suppressed: _ } = nominations; - - // Filter out nomination targets which were nominated before the most recent - // slashing span. - targets.retain(|stash| { - slashing_spans - .get(stash) - .map_or(true, |spans| submitted_in >= spans.last_nonzero_slash()) - }); + // track the count of nominators added to `all_voters + let mut nominators_taken = 0u32; + // track every nominator iterated over, but not necessarily added to `all_voters` + let mut nominators_seen = 0u32; + + let mut nominators_iter = T::SortedListProvider::iter(); + while nominators_taken < nominators_quota && nominators_seen < nominators_quota * 2 { + let nominator = match nominators_iter.next() { + Some(nominator) => { + nominators_seen.saturating_inc(); + nominator + }, + None => break, + }; - if !targets.is_empty() { - let vote_weight = weight_of(&nominator); - all_voters.push((nominator, vote_weight, targets)); - nominator_count.saturating_inc(); + if let Some(Nominations { submitted_in, mut targets, suppressed: _ }) = + >::get(&nominator) + { + targets.retain(|stash| { + slashing_spans + .get(stash) + .map_or(true, |spans| submitted_in >= spans.last_nonzero_slash()) + }); + if !targets.len().is_zero() { + all_voters.push((nominator.clone(), Self::weight_of(&nominator), targets)); + nominators_taken.saturating_inc(); + } + } else { + log!(error, "invalid item in `SortedListProvider`: {:?}", nominator) } } + // all_voters should have not re-allocated. + debug_assert!(all_voters.capacity() == max_allowed_len); + Self::register_weight(T::WeightInfo::get_npos_voters( - validator_count, - nominator_count, + validators_taken, + nominators_taken, slashing_spans.len() as u32, )); + log!( + info, + "generated {} npos voters, {} from validators and {} nominators", + all_voters.len(), + validators_taken, + nominators_taken + ); all_voters } @@ -698,34 +746,59 @@ impl Pallet { } /// This function will add a nominator to the `Nominators` storage map, - /// and keep track of the `CounterForNominators`. + /// [`SortedListProvider`] and keep track of the `CounterForNominators`. /// /// If the nominator already exists, their nominations will be updated. + /// + /// NOTE: you must ALWAYS use this function to add nominator or update their targets. Any access + /// to `Nominators`, its counter, or `VoterList` outside of this function is almost certainly + /// wrong. pub fn do_add_nominator(who: &T::AccountId, nominations: Nominations) { if !Nominators::::contains_key(who) { - CounterForNominators::::mutate(|x| x.saturating_inc()) + // maybe update the counter. + CounterForNominators::::mutate(|x| x.saturating_inc()); + + // maybe update sorted list. Error checking is defensive-only - this should never fail. + if T::SortedListProvider::on_insert(who.clone(), Self::weight_of(who)).is_err() { + log!(warn, "attempt to insert duplicate nominator ({:#?})", who); + debug_assert!(false, "attempt to insert duplicate nominator"); + }; + + debug_assert_eq!(T::SortedListProvider::sanity_check(), Ok(())); } + Nominators::::insert(who, nominations); } /// This function will remove a nominator from the `Nominators` storage map, - /// and keep track of the `CounterForNominators`. + /// [`SortedListProvider`] and keep track of the `CounterForNominators`. /// /// Returns true if `who` was removed from `Nominators`, otherwise false. + /// + /// NOTE: you must ALWAYS use this function to remove a nominator from the system. Any access to + /// `Nominators`, its counter, or `VoterList` outside of this function is almost certainly + /// wrong. pub fn do_remove_nominator(who: &T::AccountId) -> bool { if Nominators::::contains_key(who) { Nominators::::remove(who); CounterForNominators::::mutate(|x| x.saturating_dec()); + T::SortedListProvider::on_remove(who); + debug_assert_eq!(T::SortedListProvider::sanity_check(), Ok(())); + debug_assert_eq!(CounterForNominators::::get(), T::SortedListProvider::count()); true } else { false } } - /// This function will add a validator to the `Validators` storage map, - /// and keep track of the `CounterForValidators`. + /// This function will add a validator to the `Validators` storage map, and keep track of the + /// `CounterForValidators`. /// /// If the validator already exists, their preferences will be updated. + /// + /// NOTE: you must ALWAYS use this function to add a validator to the system. Any access to + /// `Validators`, its counter, or `VoterList` outside of this function is almost certainly + /// wrong. pub fn do_add_validator(who: &T::AccountId, prefs: ValidatorPrefs) { if !Validators::::contains_key(who) { CounterForValidators::::mutate(|x| x.saturating_inc()) @@ -737,6 +810,10 @@ impl Pallet { /// and keep track of the `CounterForValidators`. /// /// Returns true if `who` was removed from `Validators`, otherwise false. + /// + /// NOTE: you must ALWAYS use this function to remove a validator from the system. Any access to + /// `Validators`, its counter, or `VoterList` outside of this function is almost certainly + /// wrong. pub fn do_remove_validator(who: &T::AccountId) -> bool { if Validators::::contains_key(who) { Validators::::remove(who); @@ -758,10 +835,9 @@ impl Pallet { } } -impl frame_election_provider_support::ElectionDataProvider - for Pallet -{ +impl ElectionDataProvider> for Pallet { const MAXIMUM_VOTES_PER_VOTER: u32 = T::MAX_NOMINATIONS; + fn desired_targets() -> data_provider::Result { Self::register_weight(T::DbWeight::get().reads(1)); Ok(Self::validator_count()) @@ -770,30 +846,26 @@ impl frame_election_provider_support::ElectionDataProvider, ) -> data_provider::Result)>> { - let nominator_count = CounterForNominators::::get(); - let validator_count = CounterForValidators::::get(); - - let voter_count = nominator_count.saturating_add(validator_count) as usize; debug_assert!(>::iter().count() as u32 == CounterForNominators::::get()); debug_assert!(>::iter().count() as u32 == CounterForValidators::::get()); + debug_assert_eq!( + CounterForNominators::::get(), + T::SortedListProvider::count(), + "voter_count must be accurate", + ); - // register the extra 2 reads - Self::register_weight(T::DbWeight::get().reads(2)); - - if maybe_max_len.map_or(false, |max_len| voter_count > max_len) { - return Err("Voter snapshot too big") - } + // This can never fail -- if `maybe_max_len` is `Some(_)` we handle it. + let voters = Self::get_npos_voters(maybe_max_len); + debug_assert!(maybe_max_len.map_or(true, |max| voters.len() <= max)); - Ok(Self::get_npos_voters()) + Ok(voters) } fn targets(maybe_max_len: Option) -> data_provider::Result> { - let target_count = CounterForValidators::::get() as usize; + let target_count = CounterForValidators::::get(); - // register the extra 1 read - Self::register_weight(T::DbWeight::get().reads(1)); - - if maybe_max_len.map_or(false, |max_len| target_count > max_len) { + // We can't handle this case yet -- return an error. + if maybe_max_len.map_or(false, |max_len| target_count > max_len as u32) { return Err("Target snapshot too big") } @@ -879,6 +951,9 @@ impl frame_election_provider_support::ElectionDataProvider>::remove_all(None); >::remove_all(None); >::remove_all(None); + >::kill(); + >::kill(); + let _ = T::SortedListProvider::clear(None); } #[cfg(feature = "runtime-benchmarks")] @@ -1152,3 +1227,77 @@ where consumed_weight } } + +impl VoteWeightProvider for Pallet { + fn vote_weight(who: &T::AccountId) -> VoteWeight { + Self::weight_of(who) + } + + #[cfg(feature = "runtime-benchmarks")] + fn set_vote_weight_of(who: &T::AccountId, weight: VoteWeight) { + // this will clearly results in an inconsistent state, but it should not matter for a + // benchmark. + use sp_std::convert::TryInto; + let active: BalanceOf = weight.try_into().map_err(|_| ()).unwrap(); + let mut ledger = Self::ledger(who).unwrap_or_default(); + ledger.active = active; + >::insert(who, ledger); + >::insert(who, who); + + // also, we play a trick to make sure that a issuance based-`CurrencyToVote` behaves well: + // This will make sure that total issuance is zero, thus the currency to vote will be a 1-1 + // conversion. + let imbalance = T::Currency::burn(T::Currency::total_issuance()); + // kinda ugly, but gets the job done. The fact that this works here is a HUGE exception. + // Don't try this pattern in other places. + sp_std::mem::forget(imbalance); + } +} + +/// A simple voter list implementation that does not require any additional pallets. Note, this +/// does not provided nominators in sorted ordered. If you desire nominators in a sorted order take +/// a look at [`pallet-bags-list]. +pub struct UseNominatorsMap(sp_std::marker::PhantomData); +impl SortedListProvider for UseNominatorsMap { + type Error = (); + + /// Returns iterator over voter list, which can have `take` called on it. + fn iter() -> Box> { + Box::new(Nominators::::iter().map(|(n, _)| n)) + } + fn count() -> u32 { + CounterForNominators::::get() + } + fn contains(id: &T::AccountId) -> bool { + Nominators::::contains_key(id) + } + fn on_insert(_: T::AccountId, _weight: VoteWeight) -> Result<(), Self::Error> { + // nothing to do on insert. + Ok(()) + } + fn on_update(_: &T::AccountId, _weight: VoteWeight) { + // nothing to do on update. + } + fn on_remove(_: &T::AccountId) { + // nothing to do on remove. + } + fn regenerate( + _: impl IntoIterator, + _: Box VoteWeight>, + ) -> u32 { + // nothing to do upon regenerate. + 0 + } + fn sanity_check() -> Result<(), &'static str> { + Ok(()) + } + fn clear(maybe_count: Option) -> u32 { + Nominators::::remove_all(maybe_count); + if let Some(count) = maybe_count { + CounterForNominators::::mutate(|noms| *noms - count); + count + } else { + CounterForNominators::::take() + } + } +} diff --git a/frame/staking/src/pallet/mod.rs b/frame/staking/src/pallet/mod.rs index d99cd89f3b06c..c71130a3492b1 100644 --- a/frame/staking/src/pallet/mod.rs +++ b/frame/staking/src/pallet/mod.rs @@ -17,6 +17,7 @@ //! Staking FRAME Pallet. +use frame_election_provider_support::SortedListProvider; use frame_support::{ pallet_prelude::*, traits::{ @@ -38,7 +39,7 @@ mod impls; pub use impls::*; use crate::{ - migrations, slashing, weights::WeightInfo, ActiveEraInfo, BalanceOf, EraIndex, EraPayout, + log, migrations, slashing, weights::WeightInfo, ActiveEraInfo, BalanceOf, EraIndex, EraPayout, EraRewardPoints, Exposure, Forcing, NegativeImbalanceOf, Nominations, PositiveImbalanceOf, Releases, RewardDestination, SessionInterface, StakingLedger, UnappliedSlash, UnlockChunk, ValidatorPrefs, @@ -140,6 +141,11 @@ pub mod pallet { #[pallet::constant] type MaxNominatorRewardedPerValidator: Get; + /// Something that can provide a sorted list of voters in a somewhat sorted way. The + /// original use case for this was designed with [`pallet_bags_list::Pallet`] in mind. If + /// the bags-list is not desired, [`impls::UseNominatorsMap`] is likely the desired option. + type SortedListProvider: SortedListProvider; + /// Weight information for extrinsics in this pallet. type WeightInfo: WeightInfo; } @@ -492,6 +498,13 @@ pub mod pallet { MinValidatorBond::::put(self.min_validator_bond); for &(ref stash, ref controller, balance, ref status) in &self.stakers { + log!( + trace, + "inserting genesis staker: {:?} => {:?} => {:?}", + stash, + balance, + status + ); assert!( T::Currency::free_balance(&stash) >= balance, "Stash does not have enough balance to bond." @@ -514,6 +527,13 @@ pub mod pallet { _ => Ok(()), }); } + + // all voters are reported to the `SortedListProvider`. + assert_eq!( + T::SortedListProvider::count(), + CounterForNominators::::get(), + "not all genesis stakers were inserted into sorted list provider, something is wrong." + ); } } @@ -763,8 +783,15 @@ pub mod pallet { Error::::InsufficientBond ); - Self::deposit_event(Event::::Bonded(stash, extra)); + // NOTE: ledger must be updated prior to calling `Self::weight_of`. Self::update_ledger(&controller, &ledger); + // update this staker in the sorted list, if they exist in it. + if T::SortedListProvider::contains(&stash) { + T::SortedListProvider::on_update(&stash, Self::weight_of(&ledger.stash)); + debug_assert_eq!(T::SortedListProvider::sanity_check(), Ok(())); + } + + Self::deposit_event(Event::::Bonded(stash.clone(), extra)); } Ok(()) } @@ -823,7 +850,14 @@ pub mod pallet { // Note: in case there is no current era it is fine to bond one era more. let era = Self::current_era().unwrap_or(0) + T::BondingDuration::get(); ledger.unlocking.push(UnlockChunk { value, era }); + // NOTE: ledger must be updated prior to calling `Self::weight_of`. Self::update_ledger(&controller, &ledger); + + // update this staker in the sorted list, if they exist in it. + if T::SortedListProvider::contains(&ledger.stash) { + T::SortedListProvider::on_update(&ledger.stash, Self::weight_of(&ledger.stash)); + } + Self::deposit_event(Event::::Unbonded(ledger.stash, value)); } Ok(()) @@ -1319,7 +1353,12 @@ pub mod pallet { ensure!(ledger.active >= T::Currency::minimum_balance(), Error::::InsufficientBond); Self::deposit_event(Event::::Bonded(ledger.stash.clone(), value)); + + // NOTE: ledger must be updated prior to calling `Self::weight_of`. Self::update_ledger(&controller, &ledger); + if T::SortedListProvider::contains(&ledger.stash) { + T::SortedListProvider::on_update(&ledger.stash, Self::weight_of(&ledger.stash)); + } let removed_chunks = 1u32 // for the case where the last iterated chunk is not removed .saturating_add(initial_unlocking) @@ -1492,8 +1531,6 @@ pub mod pallet { /// /// This can be helpful if bond requirements are updated, and we need to remove old users /// who do not satisfy these requirements. - // TODO: Maybe we can deprecate `chill` in the future. - // https://github.com/paritytech/substrate/issues/9111 #[pallet::weight(T::WeightInfo::chill_other())] pub fn chill_other(origin: OriginFor, controller: T::AccountId) -> DispatchResult { // Anyone can call this function. diff --git a/frame/staking/src/testing_utils.rs b/frame/staking/src/testing_utils.rs index 795c066d09bb3..13762cf5886db 100644 --- a/frame/staking/src/testing_utils.rs +++ b/frame/staking/src/testing_utils.rs @@ -27,6 +27,7 @@ use rand_chacha::{ }; use sp_io::hashing::blake2_256; +use frame_election_provider_support::SortedListProvider; use frame_support::{pallet_prelude::*, traits::Currency}; use sp_runtime::{traits::StaticLookup, Perbill}; use sp_std::prelude::*; @@ -37,8 +38,11 @@ const SEED: u32 = 0; pub fn clear_validators_and_nominators() { Validators::::remove_all(None); CounterForValidators::::kill(); + + // whenever we touch nominators counter we should update `T::SortedListProvider` as well. Nominators::::remove_all(None); CounterForNominators::::kill(); + let _ = T::SortedListProvider::clear(None); } /// Grab a funded user. @@ -49,9 +53,18 @@ pub fn create_funded_user( ) -> T::AccountId { let user = account(string, n, SEED); let balance = T::Currency::minimum_balance() * balance_factor.into(); - T::Currency::make_free_balance_be(&user, balance); - // ensure T::CurrencyToVote will work correctly. - T::Currency::issue(balance); + let _ = T::Currency::make_free_balance_be(&user, balance); + user +} + +/// Grab a funded user with max Balance. +pub fn create_funded_user_with_balance( + string: &'static str, + n: u32, + balance: BalanceOf, +) -> T::AccountId { + let user = account(string, n, SEED); + let _ = T::Currency::make_free_balance_be(&user, balance); user } @@ -75,6 +88,26 @@ pub fn create_stash_controller( return Ok((stash, controller)) } +/// Create a stash and controller pair with fixed balance. +pub fn create_stash_controller_with_balance( + n: u32, + balance: crate::BalanceOf, + destination: RewardDestination, +) -> Result<(T::AccountId, T::AccountId), &'static str> { + let stash = create_funded_user_with_balance::("stash", n, balance); + let controller = create_funded_user_with_balance::("controller", n, balance); + let controller_lookup: ::Source = + T::Lookup::unlookup(controller.clone()); + + Staking::::bond( + RawOrigin::Signed(stash.clone()).into(), + controller_lookup, + balance, + destination, + )?; + Ok((stash, controller)) +} + /// Create a stash and controller pair, where the controller is dead, and payouts go to controller. /// This is used to test worst case payout scenarios. pub fn create_stash_and_dead_controller( @@ -101,11 +134,20 @@ pub fn create_stash_and_dead_controller( pub fn create_validators( max: u32, balance_factor: u32, +) -> Result::Source>, &'static str> { + create_validators_with_seed::(max, balance_factor, 0) +} + +/// create `max` validators, with a seed to help unintentional prevent account collisions. +pub fn create_validators_with_seed( + max: u32, + balance_factor: u32, + seed: u32, ) -> Result::Source>, &'static str> { let mut validators: Vec<::Source> = Vec::with_capacity(max as usize); for i in 0..max { let (stash, controller) = - create_stash_controller::(i, balance_factor, RewardDestination::Staked)?; + create_stash_controller::(i + seed, balance_factor, RewardDestination::Staked)?; let validator_prefs = ValidatorPrefs { commission: Perbill::from_percent(50), ..Default::default() }; Staking::::validate(RawOrigin::Signed(controller).into(), validator_prefs)?; diff --git a/frame/staking/src/tests.rs b/frame/staking/src/tests.rs index 97dfaa39c84a9..5e7fe3d6266aa 100644 --- a/frame/staking/src/tests.rs +++ b/frame/staking/src/tests.rs @@ -18,7 +18,7 @@ //! Tests for the module. use super::{Event, *}; -use frame_election_provider_support::{ElectionProvider, Support}; +use frame_election_provider_support::{ElectionProvider, SortedListProvider, Support}; use frame_support::{ assert_noop, assert_ok, dispatch::WithPostDispatchInfo, @@ -542,8 +542,8 @@ fn nominating_and_rewards_should_work() { total: 1000 + 800, own: 1000, others: vec![ - IndividualExposure { who: 3, value: 400 }, IndividualExposure { who: 1, value: 400 }, + IndividualExposure { who: 3, value: 400 }, ] }, ); @@ -553,8 +553,8 @@ fn nominating_and_rewards_should_work() { total: 1000 + 1200, own: 1000, others: vec![ - IndividualExposure { who: 3, value: 600 }, IndividualExposure { who: 1, value: 600 }, + IndividualExposure { who: 3, value: 600 }, ] }, ); @@ -1907,8 +1907,8 @@ fn bond_with_duplicate_vote_should_be_ignored_by_election_provider() { assert_eq!( supports, vec![ - (21, Support { total: 1800, voters: vec![(21, 1000), (3, 400), (1, 400)] }), - (31, Support { total: 2200, voters: vec![(31, 1000), (3, 600), (1, 600)] }) + (21, Support { total: 1800, voters: vec![(21, 1000), (1, 400), (3, 400)] }), + (31, Support { total: 2200, voters: vec![(31, 1000), (1, 600), (3, 600)] }) ], ); }); @@ -1952,7 +1952,7 @@ fn bond_with_duplicate_vote_should_be_ignored_by_election_provider_elected() { supports, vec![ (11, Support { total: 1500, voters: vec![(11, 1000), (1, 500)] }), - (21, Support { total: 2500, voters: vec![(21, 1000), (3, 1000), (1, 500)] }) + (21, Support { total: 2500, voters: vec![(21, 1000), (1, 500), (3, 1000)] }) ], ); }); @@ -3881,11 +3881,137 @@ mod election_data_provider { } #[test] - fn respects_len_limits() { - ExtBuilder::default().build_and_execute(|| { - assert_eq!(Staking::voters(Some(1)).unwrap_err(), "Voter snapshot too big"); - assert_eq!(Staking::targets(Some(1)).unwrap_err(), "Target snapshot too big"); - }); + fn respects_snapshot_len_limits() { + ExtBuilder::default() + .set_status(41, StakerStatus::Validator) + .build_and_execute(|| { + // sum of all nominators who'd be voters (1), plus the self-votes (4). + assert_eq!( + ::SortedListProvider::count() + + >::iter().count() as u32, + 5 + ); + + // if limits is less.. + assert_eq!(Staking::voters(Some(1)).unwrap().len(), 1); + + // if limit is equal.. + assert_eq!(Staking::voters(Some(5)).unwrap().len(), 5); + + // if limit is more. + assert_eq!(Staking::voters(Some(55)).unwrap().len(), 5); + + // if target limit is more.. + assert_eq!(Staking::targets(Some(6)).unwrap().len(), 4); + assert_eq!(Staking::targets(Some(4)).unwrap().len(), 4); + + // if target limit is less, then we return an error. + assert_eq!(Staking::targets(Some(1)).unwrap_err(), "Target snapshot too big"); + }); + } + + #[test] + fn only_iterates_max_2_times_nominators_quota() { + ExtBuilder::default() + .nominate(true) // add nominator 101, who nominates [11, 21] + // the other nominators only nominate 21 + .add_staker(61, 60, 2_000, StakerStatus::::Nominator(vec![21])) + .add_staker(71, 70, 2_000, StakerStatus::::Nominator(vec![21])) + .add_staker(81, 80, 2_000, StakerStatus::::Nominator(vec![21])) + .build_and_execute(|| { + // given our nominators ordered by stake, + assert_eq!( + ::SortedListProvider::iter().collect::>(), + vec![61, 71, 81, 101] + ); + + // and total voters + assert_eq!( + ::SortedListProvider::count() + + >::iter().count() as u32, + 7 + ); + + // roll to session 5 + run_to_block(25); + + // slash 21, the only validator nominated by our first 3 nominators + add_slash(&21); + + // we take 4 voters: 2 validators and 2 nominators (so nominators quota = 2) + assert_eq!( + Staking::voters(Some(3)) + .unwrap() + .iter() + .map(|(stash, _, _)| stash) + .copied() + .collect::>(), + vec![31, 11], // 2 validators, but no nominators because we hit the quota + ); + }); + } + + // Even if some of the higher staked nominators are slashed, we still get up to max len voters + // by adding more lower staked nominators. In other words, we assert that we keep on adding + // valid nominators until we reach max len voters; which is opposed to simply stopping after we + // have iterated max len voters, but not adding all of them to voters due to some nominators not + // having valid targets. + #[test] + fn get_max_len_voters_even_if_some_nominators_are_slashed() { + ExtBuilder::default() + .nominate(true) // add nominator 101, who nominates [11, 21] + .add_staker(61, 60, 20, StakerStatus::::Nominator(vec![21])) + // 61 only nominates validator 21 ^^ + .add_staker(71, 70, 10, StakerStatus::::Nominator(vec![11, 21])) + .build_and_execute(|| { + // given our nominators ordered by stake, + assert_eq!( + ::SortedListProvider::iter().collect::>(), + vec![101, 61, 71] + ); + + // and total voters + assert_eq!( + ::SortedListProvider::count() + + >::iter().count() as u32, + 6 + ); + + // we take 5 voters + assert_eq!( + Staking::voters(Some(5)) + .unwrap() + .iter() + .map(|(stash, _, _)| stash) + .copied() + .collect::>(), + // then + vec![ + 31, 21, 11, // 3 nominators + 101, 61 // 2 validators, and 71 is excluded + ], + ); + + // roll to session 5 + run_to_block(25); + + // slash 21, the only validator nominated by 61 + add_slash(&21); + + // we take 4 voters + assert_eq!( + Staking::voters(Some(4)) + .unwrap() + .iter() + .map(|(stash, _, _)| stash) + .copied() + .collect::>(), + vec![ + 31, 11, // 2 validators (21 was slashed) + 101, 71 // 2 nominators, excluding 61 + ], + ); + }); } #[test] @@ -4232,3 +4358,28 @@ fn capped_stakers_works() { assert_ok!(Staking::validate(Origin::signed(last_validator), ValidatorPrefs::default())); }) } + +mod sorted_list_provider { + use super::*; + use frame_election_provider_support::SortedListProvider; + + #[test] + fn re_nominate_does_not_change_counters_or_list() { + ExtBuilder::default().nominate(true).build_and_execute(|| { + // given + let pre_insert_nominator_count = Nominators::::iter().count() as u32; + assert_eq!(::SortedListProvider::count(), pre_insert_nominator_count); + assert!(Nominators::::contains_key(101)); + assert_eq!(::SortedListProvider::iter().collect::>(), vec![101]); + + // when account 101 renominates + assert_ok!(Staking::nominate(Origin::signed(100), vec![41])); + + // then counts don't change + assert_eq!(::SortedListProvider::count(), pre_insert_nominator_count); + assert_eq!(Nominators::::iter().count() as u32, pre_insert_nominator_count); + // and the list is the same + assert_eq!(::SortedListProvider::iter().collect::>(), vec![101]); + }); + } +} diff --git a/frame/staking/src/weights.rs b/frame/staking/src/weights.rs index 0bcf179e29339..32c8dc80da158 100644 --- a/frame/staking/src/weights.rs +++ b/frame/staking/src/weights.rs @@ -18,7 +18,7 @@ //! Autogenerated weights for pallet_staking //! //! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev -//! DATE: 2021-08-18, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! DATE: 2021-09-04, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` //! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 128 // Executed Command: @@ -85,37 +85,42 @@ impl WeightInfo for SubstrateWeight { // Storage: Balances Locks (r:1 w:1) // Storage: Staking Payee (r:0 w:1) fn bond() -> Weight { - (73_523_000 as Weight) + (73_865_000 as Weight) .saturating_add(T::DbWeight::get().reads(5 as Weight)) .saturating_add(T::DbWeight::get().writes(4 as Weight)) } // Storage: Staking Bonded (r:1 w:0) // Storage: Staking Ledger (r:1 w:1) // Storage: Balances Locks (r:1 w:1) + // Storage: BagsList ListNodes (r:3 w:3) + // Storage: BagsList ListBags (r:2 w:2) fn bond_extra() -> Weight { - (58_129_000 as Weight) - .saturating_add(T::DbWeight::get().reads(3 as Weight)) - .saturating_add(T::DbWeight::get().writes(2 as Weight)) + (114_296_000 as Weight) + .saturating_add(T::DbWeight::get().reads(8 as Weight)) + .saturating_add(T::DbWeight::get().writes(7 as Weight)) } // Storage: Staking Ledger (r:1 w:1) // Storage: Staking Nominators (r:1 w:0) - // Storage: Staking Validators (r:1 w:0) + // Storage: Staking MinNominatorBond (r:1 w:0) // Storage: Staking CurrentEra (r:1 w:0) // Storage: Balances Locks (r:1 w:1) // Storage: System Account (r:1 w:1) + // Storage: BagsList ListNodes (r:3 w:3) + // Storage: Staking Bonded (r:1 w:0) + // Storage: BagsList ListBags (r:2 w:2) fn unbond() -> Weight { - (61_542_000 as Weight) - .saturating_add(T::DbWeight::get().reads(6 as Weight)) - .saturating_add(T::DbWeight::get().writes(3 as Weight)) + (121_737_000 as Weight) + .saturating_add(T::DbWeight::get().reads(12 as Weight)) + .saturating_add(T::DbWeight::get().writes(8 as Weight)) } // Storage: Staking Ledger (r:1 w:1) // Storage: Staking CurrentEra (r:1 w:0) // Storage: Balances Locks (r:1 w:1) // Storage: System Account (r:1 w:1) fn withdraw_unbonded_update(s: u32, ) -> Weight { - (53_160_000 as Weight) + (51_631_000 as Weight) // Standard Error: 0 - .saturating_add((53_000 as Weight).saturating_mul(s as Weight)) + .saturating_add((55_000 as Weight).saturating_mul(s as Weight)) .saturating_add(T::DbWeight::get().reads(4 as Weight)) .saturating_add(T::DbWeight::get().writes(3 as Weight)) } @@ -124,36 +129,40 @@ impl WeightInfo for SubstrateWeight { // Storage: Staking Bonded (r:1 w:1) // Storage: Staking SlashingSpans (r:1 w:0) // Storage: Staking Validators (r:1 w:0) - // Storage: Staking Nominators (r:1 w:0) + // Storage: Staking Nominators (r:1 w:1) + // Storage: Staking CounterForNominators (r:1 w:1) + // Storage: BagsList ListNodes (r:2 w:2) + // Storage: BagsList ListBags (r:1 w:1) + // Storage: BagsList CounterForListNodes (r:1 w:1) // Storage: System Account (r:1 w:1) // Storage: Balances Locks (r:1 w:1) // Storage: Staking Payee (r:0 w:1) - // Storage: Staking SpanSlash (r:0 w:2) - fn withdraw_unbonded_kill(s: u32, ) -> Weight { - (85_826_000 as Weight) - // Standard Error: 2_000 - .saturating_add((2_453_000 as Weight).saturating_mul(s as Weight)) - .saturating_add(T::DbWeight::get().reads(8 as Weight)) - .saturating_add(T::DbWeight::get().writes(6 as Weight)) - .saturating_add(T::DbWeight::get().writes((1 as Weight).saturating_mul(s as Weight))) + fn withdraw_unbonded_kill(_s: u32, ) -> Weight { + (101_870_000 as Weight) + .saturating_add(T::DbWeight::get().reads(13 as Weight)) + .saturating_add(T::DbWeight::get().writes(11 as Weight)) } // Storage: Staking Ledger (r:1 w:0) // Storage: Staking MinValidatorBond (r:1 w:0) // Storage: Staking Validators (r:1 w:1) // Storage: Staking MaxValidatorsCount (r:1 w:0) - // Storage: Staking Nominators (r:1 w:0) + // Storage: Staking Nominators (r:1 w:1) + // Storage: Staking CounterForNominators (r:1 w:1) + // Storage: BagsList ListNodes (r:2 w:2) + // Storage: BagsList ListBags (r:1 w:1) + // Storage: BagsList CounterForListNodes (r:1 w:1) // Storage: Staking CounterForValidators (r:1 w:1) fn validate() -> Weight { - (34_936_000 as Weight) - .saturating_add(T::DbWeight::get().reads(6 as Weight)) - .saturating_add(T::DbWeight::get().writes(2 as Weight)) + (69_092_000 as Weight) + .saturating_add(T::DbWeight::get().reads(11 as Weight)) + .saturating_add(T::DbWeight::get().writes(8 as Weight)) } // Storage: Staking Ledger (r:1 w:0) // Storage: Staking Nominators (r:1 w:1) fn kick(k: u32, ) -> Weight { - (23_493_000 as Weight) - // Standard Error: 17_000 - .saturating_add((16_632_000 as Weight).saturating_mul(k as Weight)) + (21_468_000 as Weight) + // Standard Error: 19_000 + .saturating_add((16_415_000 as Weight).saturating_mul(k as Weight)) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().reads((1 as Weight).saturating_mul(k as Weight))) .saturating_add(T::DbWeight::get().writes((1 as Weight).saturating_mul(k as Weight))) @@ -165,84 +174,97 @@ impl WeightInfo for SubstrateWeight { // Storage: Staking Validators (r:2 w:0) // Storage: Staking CurrentEra (r:1 w:0) // Storage: Staking CounterForNominators (r:1 w:1) + // Storage: Staking Bonded (r:1 w:0) + // Storage: BagsList ListNodes (r:2 w:2) + // Storage: BagsList ListBags (r:1 w:1) + // Storage: BagsList CounterForListNodes (r:1 w:1) fn nominate(n: u32, ) -> Weight { - (41_733_000 as Weight) - // Standard Error: 11_000 - .saturating_add((5_840_000 as Weight).saturating_mul(n as Weight)) - .saturating_add(T::DbWeight::get().reads(7 as Weight)) + (82_389_000 as Weight) + // Standard Error: 14_000 + .saturating_add((5_597_000 as Weight).saturating_mul(n as Weight)) + .saturating_add(T::DbWeight::get().reads(12 as Weight)) .saturating_add(T::DbWeight::get().reads((1 as Weight).saturating_mul(n as Weight))) - .saturating_add(T::DbWeight::get().writes(2 as Weight)) + .saturating_add(T::DbWeight::get().writes(6 as Weight)) } // Storage: Staking Ledger (r:1 w:0) // Storage: Staking Validators (r:1 w:0) - // Storage: Staking Nominators (r:1 w:0) + // Storage: Staking Nominators (r:1 w:1) + // Storage: Staking CounterForNominators (r:1 w:1) + // Storage: BagsList ListNodes (r:2 w:2) + // Storage: BagsList ListBags (r:1 w:1) + // Storage: BagsList CounterForListNodes (r:1 w:1) fn chill() -> Weight { - (17_901_000 as Weight) - .saturating_add(T::DbWeight::get().reads(3 as Weight)) + (69_655_000 as Weight) + .saturating_add(T::DbWeight::get().reads(8 as Weight)) + .saturating_add(T::DbWeight::get().writes(6 as Weight)) } // Storage: Staking Ledger (r:1 w:0) // Storage: Staking Payee (r:0 w:1) fn set_payee() -> Weight { - (13_760_000 as Weight) + (12_770_000 as Weight) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Staking Bonded (r:1 w:1) // Storage: Staking Ledger (r:2 w:2) fn set_controller() -> Weight { - (28_388_000 as Weight) + (27_756_000 as Weight) .saturating_add(T::DbWeight::get().reads(3 as Weight)) .saturating_add(T::DbWeight::get().writes(3 as Weight)) } // Storage: Staking ValidatorCount (r:0 w:1) fn set_validator_count() -> Weight { - (2_537_000 as Weight) + (2_446_000 as Weight) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Staking ForceEra (r:0 w:1) fn force_no_eras() -> Weight { - (2_749_000 as Weight) + (2_720_000 as Weight) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Staking ForceEra (r:0 w:1) fn force_new_era() -> Weight { - (2_834_000 as Weight) + (2_711_000 as Weight) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Staking ForceEra (r:0 w:1) fn force_new_era_always() -> Weight { - (2_800_000 as Weight) + (2_796_000 as Weight) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Staking Invulnerables (r:0 w:1) fn set_invulnerables(v: u32, ) -> Weight { - (3_429_000 as Weight) + (3_141_000 as Weight) // Standard Error: 0 - .saturating_add((56_000 as Weight).saturating_mul(v as Weight)) + .saturating_add((53_000 as Weight).saturating_mul(v as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Staking Bonded (r:1 w:1) // Storage: Staking SlashingSpans (r:1 w:0) // Storage: Staking Validators (r:1 w:0) - // Storage: Staking Nominators (r:1 w:0) + // Storage: Staking Nominators (r:1 w:1) + // Storage: Staking CounterForNominators (r:1 w:1) + // Storage: BagsList ListNodes (r:2 w:2) + // Storage: BagsList ListBags (r:1 w:1) + // Storage: BagsList CounterForListNodes (r:1 w:1) // Storage: System Account (r:1 w:1) // Storage: Balances Locks (r:1 w:1) // Storage: Staking Ledger (r:0 w:1) // Storage: Staking Payee (r:0 w:1) // Storage: Staking SpanSlash (r:0 w:2) fn force_unstake(s: u32, ) -> Weight { - (61_799_000 as Weight) + (97_394_000 as Weight) // Standard Error: 2_000 - .saturating_add((2_451_000 as Weight).saturating_mul(s as Weight)) - .saturating_add(T::DbWeight::get().reads(6 as Weight)) - .saturating_add(T::DbWeight::get().writes(6 as Weight)) + .saturating_add((2_370_000 as Weight).saturating_mul(s as Weight)) + .saturating_add(T::DbWeight::get().reads(11 as Weight)) + .saturating_add(T::DbWeight::get().writes(12 as Weight)) .saturating_add(T::DbWeight::get().writes((1 as Weight).saturating_mul(s as Weight))) } // Storage: Staking UnappliedSlashes (r:1 w:1) fn cancel_deferred_slash(s: u32, ) -> Weight { - (3_383_988_000 as Weight) - // Standard Error: 223_000 - .saturating_add((19_981_000 as Weight).saturating_mul(s as Weight)) + (2_783_746_000 as Weight) + // Standard Error: 182_000 + .saturating_add((16_223_000 as Weight).saturating_mul(s as Weight)) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } @@ -257,9 +279,9 @@ impl WeightInfo for SubstrateWeight { // Storage: Staking Payee (r:2 w:0) // Storage: System Account (r:2 w:2) fn payout_stakers_dead_controller(n: u32, ) -> Weight { - (124_714_000 as Weight) - // Standard Error: 23_000 - .saturating_add((47_575_000 as Weight).saturating_mul(n as Weight)) + (109_233_000 as Weight) + // Standard Error: 17_000 + .saturating_add((47_612_000 as Weight).saturating_mul(n as Weight)) .saturating_add(T::DbWeight::get().reads(10 as Weight)) .saturating_add(T::DbWeight::get().reads((3 as Weight).saturating_mul(n as Weight))) .saturating_add(T::DbWeight::get().writes(2 as Weight)) @@ -277,9 +299,9 @@ impl WeightInfo for SubstrateWeight { // Storage: System Account (r:2 w:2) // Storage: Balances Locks (r:2 w:2) fn payout_stakers_alive_staked(n: u32, ) -> Weight { - (160_203_000 as Weight) - // Standard Error: 24_000 - .saturating_add((61_321_000 as Weight).saturating_mul(n as Weight)) + (177_392_000 as Weight) + // Standard Error: 20_000 + .saturating_add((60_771_000 as Weight).saturating_mul(n as Weight)) .saturating_add(T::DbWeight::get().reads(11 as Weight)) .saturating_add(T::DbWeight::get().reads((5 as Weight).saturating_mul(n as Weight))) .saturating_add(T::DbWeight::get().writes(3 as Weight)) @@ -288,12 +310,15 @@ impl WeightInfo for SubstrateWeight { // Storage: Staking Ledger (r:1 w:1) // Storage: Balances Locks (r:1 w:1) // Storage: System Account (r:1 w:1) + // Storage: BagsList ListNodes (r:3 w:3) + // Storage: Staking Bonded (r:1 w:0) + // Storage: BagsList ListBags (r:2 w:2) fn rebond(l: u32, ) -> Weight { - (49_593_000 as Weight) - // Standard Error: 3_000 - .saturating_add((78_000 as Weight).saturating_mul(l as Weight)) - .saturating_add(T::DbWeight::get().reads(3 as Weight)) - .saturating_add(T::DbWeight::get().writes(3 as Weight)) + (111_858_000 as Weight) + // Standard Error: 4_000 + .saturating_add((36_000 as Weight).saturating_mul(l as Weight)) + .saturating_add(T::DbWeight::get().reads(9 as Weight)) + .saturating_add(T::DbWeight::get().writes(8 as Weight)) } // Storage: Staking CurrentEra (r:1 w:0) // Storage: Staking HistoryDepth (r:1 w:1) @@ -306,8 +331,8 @@ impl WeightInfo for SubstrateWeight { // Storage: Staking ErasStartSessionIndex (r:0 w:1) fn set_history_depth(e: u32, ) -> Weight { (0 as Weight) - // Standard Error: 71_000 - .saturating_add((35_237_000 as Weight).saturating_mul(e as Weight)) + // Standard Error: 68_000 + .saturating_add((33_495_000 as Weight).saturating_mul(e as Weight)) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(4 as Weight)) .saturating_add(T::DbWeight::get().writes((7 as Weight).saturating_mul(e as Weight))) @@ -315,19 +340,22 @@ impl WeightInfo for SubstrateWeight { // Storage: System Account (r:1 w:1) // Storage: Staking Bonded (r:1 w:1) // Storage: Staking SlashingSpans (r:1 w:1) - // Storage: Staking Validators (r:1 w:1) - // Storage: Staking CounterForValidators (r:1 w:1) - // Storage: Staking Nominators (r:1 w:0) + // Storage: Staking Validators (r:1 w:0) + // Storage: Staking Nominators (r:1 w:1) + // Storage: Staking CounterForNominators (r:1 w:1) + // Storage: BagsList ListNodes (r:2 w:2) + // Storage: BagsList ListBags (r:1 w:1) + // Storage: BagsList CounterForListNodes (r:1 w:1) // Storage: Balances Locks (r:1 w:1) // Storage: Staking Ledger (r:0 w:1) // Storage: Staking Payee (r:0 w:1) // Storage: Staking SpanSlash (r:0 w:1) fn reap_stash(s: u32, ) -> Weight { - (72_484_000 as Weight) - // Standard Error: 2_000 - .saturating_add((2_452_000 as Weight).saturating_mul(s as Weight)) - .saturating_add(T::DbWeight::get().reads(7 as Weight)) - .saturating_add(T::DbWeight::get().writes(8 as Weight)) + (100_178_000 as Weight) + // Standard Error: 1_000 + .saturating_add((2_358_000 as Weight).saturating_mul(s as Weight)) + .saturating_add(T::DbWeight::get().reads(11 as Weight)) + .saturating_add(T::DbWeight::get().writes(12 as Weight)) .saturating_add(T::DbWeight::get().writes((1 as Weight).saturating_mul(s as Weight))) } // Storage: Staking CounterForNominators (r:1 w:0) @@ -336,7 +364,9 @@ impl WeightInfo for SubstrateWeight { // Storage: Staking Bonded (r:101 w:0) // Storage: Staking Ledger (r:101 w:0) // Storage: Staking SlashingSpans (r:1 w:0) - // Storage: Staking Nominators (r:101 w:0) + // Storage: BagsList ListBags (r:200 w:0) + // Storage: BagsList ListNodes (r:100 w:0) + // Storage: Staking Nominators (r:100 w:0) // Storage: Staking ValidatorCount (r:1 w:0) // Storage: Staking MinimumValidatorCount (r:1 w:0) // Storage: Staking CurrentEra (r:1 w:1) @@ -348,39 +378,43 @@ impl WeightInfo for SubstrateWeight { // Storage: Staking ErasStartSessionIndex (r:0 w:1) fn new_era(v: u32, n: u32, ) -> Weight { (0 as Weight) - // Standard Error: 856_000 - .saturating_add((305_057_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 860_000 + .saturating_add((298_721_000 as Weight).saturating_mul(v as Weight)) // Standard Error: 43_000 - .saturating_add((47_890_000 as Weight).saturating_mul(n as Weight)) - .saturating_add(T::DbWeight::get().reads(9 as Weight)) + .saturating_add((49_427_000 as Weight).saturating_mul(n as Weight)) + .saturating_add(T::DbWeight::get().reads(208 as Weight)) .saturating_add(T::DbWeight::get().reads((3 as Weight).saturating_mul(v as Weight))) - .saturating_add(T::DbWeight::get().reads((3 as Weight).saturating_mul(n as Weight))) + .saturating_add(T::DbWeight::get().reads((4 as Weight).saturating_mul(n as Weight))) .saturating_add(T::DbWeight::get().writes(3 as Weight)) .saturating_add(T::DbWeight::get().writes((3 as Weight).saturating_mul(v as Weight))) } + // Storage: Staking CounterForNominators (r:1 w:0) + // Storage: Staking CounterForValidators (r:1 w:0) // Storage: Staking Validators (r:501 w:0) // Storage: Staking Bonded (r:1500 w:0) // Storage: Staking Ledger (r:1500 w:0) // Storage: Staking SlashingSpans (r:21 w:0) - // Storage: Staking Nominators (r:1001 w:0) + // Storage: BagsList ListBags (r:200 w:0) + // Storage: BagsList ListNodes (r:1000 w:0) + // Storage: Staking Nominators (r:1000 w:0) fn get_npos_voters(v: u32, n: u32, s: u32, ) -> Weight { (0 as Weight) - // Standard Error: 98_000 - .saturating_add((25_610_000 as Weight).saturating_mul(v as Weight)) - // Standard Error: 98_000 - .saturating_add((28_064_000 as Weight).saturating_mul(n as Weight)) - // Standard Error: 3_346_000 - .saturating_add((18_123_000 as Weight).saturating_mul(s as Weight)) - .saturating_add(T::DbWeight::get().reads(3 as Weight)) + // Standard Error: 91_000 + .saturating_add((26_605_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 91_000 + .saturating_add((31_481_000 as Weight).saturating_mul(n as Weight)) + // Standard Error: 3_122_000 + .saturating_add((16_672_000 as Weight).saturating_mul(s as Weight)) + .saturating_add(T::DbWeight::get().reads(204 as Weight)) .saturating_add(T::DbWeight::get().reads((3 as Weight).saturating_mul(v as Weight))) - .saturating_add(T::DbWeight::get().reads((3 as Weight).saturating_mul(n as Weight))) + .saturating_add(T::DbWeight::get().reads((4 as Weight).saturating_mul(n as Weight))) .saturating_add(T::DbWeight::get().reads((1 as Weight).saturating_mul(s as Weight))) } // Storage: Staking Validators (r:501 w:0) fn get_npos_targets(v: u32, ) -> Weight { - (30_422_000 as Weight) - // Standard Error: 33_000 - .saturating_add((11_252_000 as Weight).saturating_mul(v as Weight)) + (0 as Weight) + // Standard Error: 34_000 + .saturating_add((10_558_000 as Weight).saturating_mul(v as Weight)) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().reads((1 as Weight).saturating_mul(v as Weight))) } @@ -390,20 +424,23 @@ impl WeightInfo for SubstrateWeight { // Storage: Staking MaxNominatorsCount (r:0 w:1) // Storage: Staking MinNominatorBond (r:0 w:1) fn set_staking_limits() -> Weight { - (6_486_000 as Weight) + (6_353_000 as Weight) .saturating_add(T::DbWeight::get().writes(5 as Weight)) } // Storage: Staking Ledger (r:1 w:0) // Storage: Staking ChillThreshold (r:1 w:0) - // Storage: Staking Nominators (r:1 w:0) - // Storage: Staking Validators (r:1 w:1) - // Storage: Staking MaxValidatorsCount (r:1 w:0) - // Storage: Staking CounterForValidators (r:1 w:1) - // Storage: Staking MinValidatorBond (r:1 w:0) + // Storage: Staking Nominators (r:1 w:1) + // Storage: Staking MaxNominatorsCount (r:1 w:0) + // Storage: Staking CounterForNominators (r:1 w:1) + // Storage: Staking MinNominatorBond (r:1 w:0) + // Storage: Staking Validators (r:1 w:0) + // Storage: BagsList ListNodes (r:2 w:2) + // Storage: BagsList ListBags (r:1 w:1) + // Storage: BagsList CounterForListNodes (r:1 w:1) fn chill_other() -> Weight { - (58_222_000 as Weight) - .saturating_add(T::DbWeight::get().reads(7 as Weight)) - .saturating_add(T::DbWeight::get().writes(2 as Weight)) + (83_389_000 as Weight) + .saturating_add(T::DbWeight::get().reads(11 as Weight)) + .saturating_add(T::DbWeight::get().writes(6 as Weight)) } } @@ -416,37 +453,42 @@ impl WeightInfo for () { // Storage: Balances Locks (r:1 w:1) // Storage: Staking Payee (r:0 w:1) fn bond() -> Weight { - (73_523_000 as Weight) + (73_865_000 as Weight) .saturating_add(RocksDbWeight::get().reads(5 as Weight)) .saturating_add(RocksDbWeight::get().writes(4 as Weight)) } // Storage: Staking Bonded (r:1 w:0) // Storage: Staking Ledger (r:1 w:1) // Storage: Balances Locks (r:1 w:1) + // Storage: BagsList ListNodes (r:3 w:3) + // Storage: BagsList ListBags (r:2 w:2) fn bond_extra() -> Weight { - (58_129_000 as Weight) - .saturating_add(RocksDbWeight::get().reads(3 as Weight)) - .saturating_add(RocksDbWeight::get().writes(2 as Weight)) + (114_296_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(8 as Weight)) + .saturating_add(RocksDbWeight::get().writes(7 as Weight)) } // Storage: Staking Ledger (r:1 w:1) // Storage: Staking Nominators (r:1 w:0) - // Storage: Staking Validators (r:1 w:0) + // Storage: Staking MinNominatorBond (r:1 w:0) // Storage: Staking CurrentEra (r:1 w:0) // Storage: Balances Locks (r:1 w:1) // Storage: System Account (r:1 w:1) + // Storage: BagsList ListNodes (r:3 w:3) + // Storage: Staking Bonded (r:1 w:0) + // Storage: BagsList ListBags (r:2 w:2) fn unbond() -> Weight { - (61_542_000 as Weight) - .saturating_add(RocksDbWeight::get().reads(6 as Weight)) - .saturating_add(RocksDbWeight::get().writes(3 as Weight)) + (121_737_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(12 as Weight)) + .saturating_add(RocksDbWeight::get().writes(8 as Weight)) } // Storage: Staking Ledger (r:1 w:1) // Storage: Staking CurrentEra (r:1 w:0) // Storage: Balances Locks (r:1 w:1) // Storage: System Account (r:1 w:1) fn withdraw_unbonded_update(s: u32, ) -> Weight { - (53_160_000 as Weight) + (51_631_000 as Weight) // Standard Error: 0 - .saturating_add((53_000 as Weight).saturating_mul(s as Weight)) + .saturating_add((55_000 as Weight).saturating_mul(s as Weight)) .saturating_add(RocksDbWeight::get().reads(4 as Weight)) .saturating_add(RocksDbWeight::get().writes(3 as Weight)) } @@ -455,36 +497,40 @@ impl WeightInfo for () { // Storage: Staking Bonded (r:1 w:1) // Storage: Staking SlashingSpans (r:1 w:0) // Storage: Staking Validators (r:1 w:0) - // Storage: Staking Nominators (r:1 w:0) + // Storage: Staking Nominators (r:1 w:1) + // Storage: Staking CounterForNominators (r:1 w:1) + // Storage: BagsList ListNodes (r:2 w:2) + // Storage: BagsList ListBags (r:1 w:1) + // Storage: BagsList CounterForListNodes (r:1 w:1) // Storage: System Account (r:1 w:1) // Storage: Balances Locks (r:1 w:1) // Storage: Staking Payee (r:0 w:1) - // Storage: Staking SpanSlash (r:0 w:2) - fn withdraw_unbonded_kill(s: u32, ) -> Weight { - (85_826_000 as Weight) - // Standard Error: 2_000 - .saturating_add((2_453_000 as Weight).saturating_mul(s as Weight)) - .saturating_add(RocksDbWeight::get().reads(8 as Weight)) - .saturating_add(RocksDbWeight::get().writes(6 as Weight)) - .saturating_add(RocksDbWeight::get().writes((1 as Weight).saturating_mul(s as Weight))) + fn withdraw_unbonded_kill(_s: u32, ) -> Weight { + (101_870_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(13 as Weight)) + .saturating_add(RocksDbWeight::get().writes(11 as Weight)) } // Storage: Staking Ledger (r:1 w:0) // Storage: Staking MinValidatorBond (r:1 w:0) // Storage: Staking Validators (r:1 w:1) // Storage: Staking MaxValidatorsCount (r:1 w:0) - // Storage: Staking Nominators (r:1 w:0) + // Storage: Staking Nominators (r:1 w:1) + // Storage: Staking CounterForNominators (r:1 w:1) + // Storage: BagsList ListNodes (r:2 w:2) + // Storage: BagsList ListBags (r:1 w:1) + // Storage: BagsList CounterForListNodes (r:1 w:1) // Storage: Staking CounterForValidators (r:1 w:1) fn validate() -> Weight { - (34_936_000 as Weight) - .saturating_add(RocksDbWeight::get().reads(6 as Weight)) - .saturating_add(RocksDbWeight::get().writes(2 as Weight)) + (69_092_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(11 as Weight)) + .saturating_add(RocksDbWeight::get().writes(8 as Weight)) } // Storage: Staking Ledger (r:1 w:0) // Storage: Staking Nominators (r:1 w:1) fn kick(k: u32, ) -> Weight { - (23_493_000 as Weight) - // Standard Error: 17_000 - .saturating_add((16_632_000 as Weight).saturating_mul(k as Weight)) + (21_468_000 as Weight) + // Standard Error: 19_000 + .saturating_add((16_415_000 as Weight).saturating_mul(k as Weight)) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().reads((1 as Weight).saturating_mul(k as Weight))) .saturating_add(RocksDbWeight::get().writes((1 as Weight).saturating_mul(k as Weight))) @@ -496,84 +542,97 @@ impl WeightInfo for () { // Storage: Staking Validators (r:2 w:0) // Storage: Staking CurrentEra (r:1 w:0) // Storage: Staking CounterForNominators (r:1 w:1) + // Storage: Staking Bonded (r:1 w:0) + // Storage: BagsList ListNodes (r:2 w:2) + // Storage: BagsList ListBags (r:1 w:1) + // Storage: BagsList CounterForListNodes (r:1 w:1) fn nominate(n: u32, ) -> Weight { - (41_733_000 as Weight) - // Standard Error: 11_000 - .saturating_add((5_840_000 as Weight).saturating_mul(n as Weight)) - .saturating_add(RocksDbWeight::get().reads(7 as Weight)) + (82_389_000 as Weight) + // Standard Error: 14_000 + .saturating_add((5_597_000 as Weight).saturating_mul(n as Weight)) + .saturating_add(RocksDbWeight::get().reads(12 as Weight)) .saturating_add(RocksDbWeight::get().reads((1 as Weight).saturating_mul(n as Weight))) - .saturating_add(RocksDbWeight::get().writes(2 as Weight)) + .saturating_add(RocksDbWeight::get().writes(6 as Weight)) } // Storage: Staking Ledger (r:1 w:0) // Storage: Staking Validators (r:1 w:0) - // Storage: Staking Nominators (r:1 w:0) + // Storage: Staking Nominators (r:1 w:1) + // Storage: Staking CounterForNominators (r:1 w:1) + // Storage: BagsList ListNodes (r:2 w:2) + // Storage: BagsList ListBags (r:1 w:1) + // Storage: BagsList CounterForListNodes (r:1 w:1) fn chill() -> Weight { - (17_901_000 as Weight) - .saturating_add(RocksDbWeight::get().reads(3 as Weight)) + (69_655_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(8 as Weight)) + .saturating_add(RocksDbWeight::get().writes(6 as Weight)) } // Storage: Staking Ledger (r:1 w:0) // Storage: Staking Payee (r:0 w:1) fn set_payee() -> Weight { - (13_760_000 as Weight) + (12_770_000 as Weight) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Staking Bonded (r:1 w:1) // Storage: Staking Ledger (r:2 w:2) fn set_controller() -> Weight { - (28_388_000 as Weight) + (27_756_000 as Weight) .saturating_add(RocksDbWeight::get().reads(3 as Weight)) .saturating_add(RocksDbWeight::get().writes(3 as Weight)) } // Storage: Staking ValidatorCount (r:0 w:1) fn set_validator_count() -> Weight { - (2_537_000 as Weight) + (2_446_000 as Weight) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Staking ForceEra (r:0 w:1) fn force_no_eras() -> Weight { - (2_749_000 as Weight) + (2_720_000 as Weight) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Staking ForceEra (r:0 w:1) fn force_new_era() -> Weight { - (2_834_000 as Weight) + (2_711_000 as Weight) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Staking ForceEra (r:0 w:1) fn force_new_era_always() -> Weight { - (2_800_000 as Weight) + (2_796_000 as Weight) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Staking Invulnerables (r:0 w:1) fn set_invulnerables(v: u32, ) -> Weight { - (3_429_000 as Weight) + (3_141_000 as Weight) // Standard Error: 0 - .saturating_add((56_000 as Weight).saturating_mul(v as Weight)) + .saturating_add((53_000 as Weight).saturating_mul(v as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Staking Bonded (r:1 w:1) // Storage: Staking SlashingSpans (r:1 w:0) // Storage: Staking Validators (r:1 w:0) - // Storage: Staking Nominators (r:1 w:0) + // Storage: Staking Nominators (r:1 w:1) + // Storage: Staking CounterForNominators (r:1 w:1) + // Storage: BagsList ListNodes (r:2 w:2) + // Storage: BagsList ListBags (r:1 w:1) + // Storage: BagsList CounterForListNodes (r:1 w:1) // Storage: System Account (r:1 w:1) // Storage: Balances Locks (r:1 w:1) // Storage: Staking Ledger (r:0 w:1) // Storage: Staking Payee (r:0 w:1) // Storage: Staking SpanSlash (r:0 w:2) fn force_unstake(s: u32, ) -> Weight { - (61_799_000 as Weight) + (97_394_000 as Weight) // Standard Error: 2_000 - .saturating_add((2_451_000 as Weight).saturating_mul(s as Weight)) - .saturating_add(RocksDbWeight::get().reads(6 as Weight)) - .saturating_add(RocksDbWeight::get().writes(6 as Weight)) + .saturating_add((2_370_000 as Weight).saturating_mul(s as Weight)) + .saturating_add(RocksDbWeight::get().reads(11 as Weight)) + .saturating_add(RocksDbWeight::get().writes(12 as Weight)) .saturating_add(RocksDbWeight::get().writes((1 as Weight).saturating_mul(s as Weight))) } // Storage: Staking UnappliedSlashes (r:1 w:1) fn cancel_deferred_slash(s: u32, ) -> Weight { - (3_383_988_000 as Weight) - // Standard Error: 223_000 - .saturating_add((19_981_000 as Weight).saturating_mul(s as Weight)) + (2_783_746_000 as Weight) + // Standard Error: 182_000 + .saturating_add((16_223_000 as Weight).saturating_mul(s as Weight)) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } @@ -588,9 +647,9 @@ impl WeightInfo for () { // Storage: Staking Payee (r:2 w:0) // Storage: System Account (r:2 w:2) fn payout_stakers_dead_controller(n: u32, ) -> Weight { - (124_714_000 as Weight) - // Standard Error: 23_000 - .saturating_add((47_575_000 as Weight).saturating_mul(n as Weight)) + (109_233_000 as Weight) + // Standard Error: 17_000 + .saturating_add((47_612_000 as Weight).saturating_mul(n as Weight)) .saturating_add(RocksDbWeight::get().reads(10 as Weight)) .saturating_add(RocksDbWeight::get().reads((3 as Weight).saturating_mul(n as Weight))) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) @@ -608,9 +667,9 @@ impl WeightInfo for () { // Storage: System Account (r:2 w:2) // Storage: Balances Locks (r:2 w:2) fn payout_stakers_alive_staked(n: u32, ) -> Weight { - (160_203_000 as Weight) - // Standard Error: 24_000 - .saturating_add((61_321_000 as Weight).saturating_mul(n as Weight)) + (177_392_000 as Weight) + // Standard Error: 20_000 + .saturating_add((60_771_000 as Weight).saturating_mul(n as Weight)) .saturating_add(RocksDbWeight::get().reads(11 as Weight)) .saturating_add(RocksDbWeight::get().reads((5 as Weight).saturating_mul(n as Weight))) .saturating_add(RocksDbWeight::get().writes(3 as Weight)) @@ -619,12 +678,15 @@ impl WeightInfo for () { // Storage: Staking Ledger (r:1 w:1) // Storage: Balances Locks (r:1 w:1) // Storage: System Account (r:1 w:1) + // Storage: BagsList ListNodes (r:3 w:3) + // Storage: Staking Bonded (r:1 w:0) + // Storage: BagsList ListBags (r:2 w:2) fn rebond(l: u32, ) -> Weight { - (49_593_000 as Weight) - // Standard Error: 3_000 - .saturating_add((78_000 as Weight).saturating_mul(l as Weight)) - .saturating_add(RocksDbWeight::get().reads(3 as Weight)) - .saturating_add(RocksDbWeight::get().writes(3 as Weight)) + (111_858_000 as Weight) + // Standard Error: 4_000 + .saturating_add((36_000 as Weight).saturating_mul(l as Weight)) + .saturating_add(RocksDbWeight::get().reads(9 as Weight)) + .saturating_add(RocksDbWeight::get().writes(8 as Weight)) } // Storage: Staking CurrentEra (r:1 w:0) // Storage: Staking HistoryDepth (r:1 w:1) @@ -637,8 +699,8 @@ impl WeightInfo for () { // Storage: Staking ErasStartSessionIndex (r:0 w:1) fn set_history_depth(e: u32, ) -> Weight { (0 as Weight) - // Standard Error: 71_000 - .saturating_add((35_237_000 as Weight).saturating_mul(e as Weight)) + // Standard Error: 68_000 + .saturating_add((33_495_000 as Weight).saturating_mul(e as Weight)) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(4 as Weight)) .saturating_add(RocksDbWeight::get().writes((7 as Weight).saturating_mul(e as Weight))) @@ -646,19 +708,22 @@ impl WeightInfo for () { // Storage: System Account (r:1 w:1) // Storage: Staking Bonded (r:1 w:1) // Storage: Staking SlashingSpans (r:1 w:1) - // Storage: Staking Validators (r:1 w:1) - // Storage: Staking CounterForValidators (r:1 w:1) - // Storage: Staking Nominators (r:1 w:0) + // Storage: Staking Validators (r:1 w:0) + // Storage: Staking Nominators (r:1 w:1) + // Storage: Staking CounterForNominators (r:1 w:1) + // Storage: BagsList ListNodes (r:2 w:2) + // Storage: BagsList ListBags (r:1 w:1) + // Storage: BagsList CounterForListNodes (r:1 w:1) // Storage: Balances Locks (r:1 w:1) // Storage: Staking Ledger (r:0 w:1) // Storage: Staking Payee (r:0 w:1) // Storage: Staking SpanSlash (r:0 w:1) fn reap_stash(s: u32, ) -> Weight { - (72_484_000 as Weight) - // Standard Error: 2_000 - .saturating_add((2_452_000 as Weight).saturating_mul(s as Weight)) - .saturating_add(RocksDbWeight::get().reads(7 as Weight)) - .saturating_add(RocksDbWeight::get().writes(8 as Weight)) + (100_178_000 as Weight) + // Standard Error: 1_000 + .saturating_add((2_358_000 as Weight).saturating_mul(s as Weight)) + .saturating_add(RocksDbWeight::get().reads(11 as Weight)) + .saturating_add(RocksDbWeight::get().writes(12 as Weight)) .saturating_add(RocksDbWeight::get().writes((1 as Weight).saturating_mul(s as Weight))) } // Storage: Staking CounterForNominators (r:1 w:0) @@ -667,7 +732,9 @@ impl WeightInfo for () { // Storage: Staking Bonded (r:101 w:0) // Storage: Staking Ledger (r:101 w:0) // Storage: Staking SlashingSpans (r:1 w:0) - // Storage: Staking Nominators (r:101 w:0) + // Storage: BagsList ListBags (r:200 w:0) + // Storage: BagsList ListNodes (r:100 w:0) + // Storage: Staking Nominators (r:100 w:0) // Storage: Staking ValidatorCount (r:1 w:0) // Storage: Staking MinimumValidatorCount (r:1 w:0) // Storage: Staking CurrentEra (r:1 w:1) @@ -679,39 +746,43 @@ impl WeightInfo for () { // Storage: Staking ErasStartSessionIndex (r:0 w:1) fn new_era(v: u32, n: u32, ) -> Weight { (0 as Weight) - // Standard Error: 856_000 - .saturating_add((305_057_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 860_000 + .saturating_add((298_721_000 as Weight).saturating_mul(v as Weight)) // Standard Error: 43_000 - .saturating_add((47_890_000 as Weight).saturating_mul(n as Weight)) - .saturating_add(RocksDbWeight::get().reads(9 as Weight)) + .saturating_add((49_427_000 as Weight).saturating_mul(n as Weight)) + .saturating_add(RocksDbWeight::get().reads(208 as Weight)) .saturating_add(RocksDbWeight::get().reads((3 as Weight).saturating_mul(v as Weight))) - .saturating_add(RocksDbWeight::get().reads((3 as Weight).saturating_mul(n as Weight))) + .saturating_add(RocksDbWeight::get().reads((4 as Weight).saturating_mul(n as Weight))) .saturating_add(RocksDbWeight::get().writes(3 as Weight)) .saturating_add(RocksDbWeight::get().writes((3 as Weight).saturating_mul(v as Weight))) } + // Storage: Staking CounterForNominators (r:1 w:0) + // Storage: Staking CounterForValidators (r:1 w:0) // Storage: Staking Validators (r:501 w:0) // Storage: Staking Bonded (r:1500 w:0) // Storage: Staking Ledger (r:1500 w:0) // Storage: Staking SlashingSpans (r:21 w:0) - // Storage: Staking Nominators (r:1001 w:0) + // Storage: BagsList ListBags (r:200 w:0) + // Storage: BagsList ListNodes (r:1000 w:0) + // Storage: Staking Nominators (r:1000 w:0) fn get_npos_voters(v: u32, n: u32, s: u32, ) -> Weight { (0 as Weight) - // Standard Error: 98_000 - .saturating_add((25_610_000 as Weight).saturating_mul(v as Weight)) - // Standard Error: 98_000 - .saturating_add((28_064_000 as Weight).saturating_mul(n as Weight)) - // Standard Error: 3_346_000 - .saturating_add((18_123_000 as Weight).saturating_mul(s as Weight)) - .saturating_add(RocksDbWeight::get().reads(3 as Weight)) + // Standard Error: 91_000 + .saturating_add((26_605_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 91_000 + .saturating_add((31_481_000 as Weight).saturating_mul(n as Weight)) + // Standard Error: 3_122_000 + .saturating_add((16_672_000 as Weight).saturating_mul(s as Weight)) + .saturating_add(RocksDbWeight::get().reads(204 as Weight)) .saturating_add(RocksDbWeight::get().reads((3 as Weight).saturating_mul(v as Weight))) - .saturating_add(RocksDbWeight::get().reads((3 as Weight).saturating_mul(n as Weight))) + .saturating_add(RocksDbWeight::get().reads((4 as Weight).saturating_mul(n as Weight))) .saturating_add(RocksDbWeight::get().reads((1 as Weight).saturating_mul(s as Weight))) } // Storage: Staking Validators (r:501 w:0) fn get_npos_targets(v: u32, ) -> Weight { - (30_422_000 as Weight) - // Standard Error: 33_000 - .saturating_add((11_252_000 as Weight).saturating_mul(v as Weight)) + (0 as Weight) + // Standard Error: 34_000 + .saturating_add((10_558_000 as Weight).saturating_mul(v as Weight)) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().reads((1 as Weight).saturating_mul(v as Weight))) } @@ -721,19 +792,22 @@ impl WeightInfo for () { // Storage: Staking MaxNominatorsCount (r:0 w:1) // Storage: Staking MinNominatorBond (r:0 w:1) fn set_staking_limits() -> Weight { - (6_486_000 as Weight) + (6_353_000 as Weight) .saturating_add(RocksDbWeight::get().writes(5 as Weight)) } // Storage: Staking Ledger (r:1 w:0) // Storage: Staking ChillThreshold (r:1 w:0) - // Storage: Staking Nominators (r:1 w:0) - // Storage: Staking Validators (r:1 w:1) - // Storage: Staking MaxValidatorsCount (r:1 w:0) - // Storage: Staking CounterForValidators (r:1 w:1) - // Storage: Staking MinValidatorBond (r:1 w:0) + // Storage: Staking Nominators (r:1 w:1) + // Storage: Staking MaxNominatorsCount (r:1 w:0) + // Storage: Staking CounterForNominators (r:1 w:1) + // Storage: Staking MinNominatorBond (r:1 w:0) + // Storage: Staking Validators (r:1 w:0) + // Storage: BagsList ListNodes (r:2 w:2) + // Storage: BagsList ListBags (r:1 w:1) + // Storage: BagsList CounterForListNodes (r:1 w:1) fn chill_other() -> Weight { - (58_222_000 as Weight) - .saturating_add(RocksDbWeight::get().reads(7 as Weight)) - .saturating_add(RocksDbWeight::get().writes(2 as Weight)) + (83_389_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(11 as Weight)) + .saturating_add(RocksDbWeight::get().writes(6 as Weight)) } } diff --git a/primitives/npos-elections/src/traits.rs b/primitives/npos-elections/src/traits.rs index 45b6fa368ae2a..597d7e648fd9b 100644 --- a/primitives/npos-elections/src/traits.rs +++ b/primitives/npos-elections/src/traits.rs @@ -10,8 +10,8 @@ // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// distributed under the License is distributed on an "AS IS" BASIS, // See the License for the specific language governing permissions and // limitations under the License. @@ -22,6 +22,7 @@ use crate::{ VoteWeight, }; use codec::Encode; +use scale_info::TypeInfo; use sp_arithmetic::{ traits::{Bounded, UniqueSaturatedInto}, PerThing, @@ -72,7 +73,8 @@ where + Copy + Clone + Bounded - + Encode; + + Encode + + TypeInfo; /// The target type. Needs to be an index (convert to usize). type TargetIndex: UniqueSaturatedInto @@ -82,7 +84,8 @@ where + Copy + Clone + Bounded - + Encode; + + Encode + + TypeInfo; /// The weight/accuracy type of each vote. type Accuracy: PerThing128; diff --git a/utils/frame/generate-bags/Cargo.toml b/utils/frame/generate-bags/Cargo.toml new file mode 100644 index 0000000000000..384307fbec9e5 --- /dev/null +++ b/utils/frame/generate-bags/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "generate-bags" +version = "3.0.0" +authors = ["Parity Technologies "] +edition = "2018" +license = "Apache-2.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "Bag threshold generation script for pallet-bag-list" +readme = "README.md" + +[dependencies] +# FRAME +frame-support = { version = "4.0.0-dev", default-features = false, path = "../../../frame/support" } +frame-election-provider-support = { version = "4.0.0-dev", path = "../../../frame/election-provider-support", features = ["runtime-benchmarks"] } +frame-system = { version = "4.0.0-dev", default-features = false, path = "../../../frame/system" } +pallet-staking = { version = "4.0.0-dev", default-features = false, path = "../../../frame/staking" } + +# primitives +sp-io = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/io" } + +# third party +chrono = { version = "0.4.19" } +git2 = { version = "0.13.20", default-features = false } +num-format = { version = "0.4.0" } +structopt = "0.3.21" diff --git a/utils/frame/generate-bags/node-runtime/Cargo.toml b/utils/frame/generate-bags/node-runtime/Cargo.toml new file mode 100644 index 0000000000000..7fcd981a6bbd6 --- /dev/null +++ b/utils/frame/generate-bags/node-runtime/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "node-runtime-generate-bags" +version = "3.0.0" +authors = ["Parity Technologies "] +edition = "2018" +license = "Apache-2.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "Bag threshold generation script for pallet-bag-list and node-runtime." +readme = "README.md" + +[dependencies] +node-runtime = { version = "3.0.0-dev", path = "../../../../bin/node/runtime" } +generate-bags = { version = "3.0.0", path = "../" } + +# third-party +structopt = "0.3.21" diff --git a/utils/frame/generate-bags/node-runtime/src/main.rs b/utils/frame/generate-bags/node-runtime/src/main.rs new file mode 100644 index 0000000000000..5d36b381a7d0c --- /dev/null +++ b/utils/frame/generate-bags/node-runtime/src/main.rs @@ -0,0 +1,46 @@ +// This file is part of Substrate. + +// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Make the set of bag thresholds to be used with pallet-bags-list. + +use generate_bags::generate_thresholds; +use std::path::PathBuf; +use structopt::StructOpt; + +#[derive(Debug, StructOpt)] +struct Opt { + /// How many bags to generate. + #[structopt(long, default_value = "200")] + n_bags: usize, + + /// Where to write the output. + output: PathBuf, + + /// The total issuance of the currency used to create `VoteWeight`. + #[structopt(short, long)] + total_issuance: u128, + + /// The minimum account balance (i.e. existential deposit) for the currency used to create + /// `VoteWeight`. + #[structopt(short, long)] + minimum_balance: u128, +} + +fn main() -> Result<(), std::io::Error> { + let Opt { n_bags, output, total_issuance, minimum_balance } = Opt::from_args(); + generate_thresholds::(n_bags, &output, total_issuance, minimum_balance) +} diff --git a/utils/frame/generate-bags/src/lib.rs b/utils/frame/generate-bags/src/lib.rs new file mode 100644 index 0000000000000..af9df4435bcab --- /dev/null +++ b/utils/frame/generate-bags/src/lib.rs @@ -0,0 +1,246 @@ +// This file is part of Substrate. + +// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Support code to ease the process of generating bag thresholds. +//! +//! NOTE: this assume the runtime implements [`pallet_staking::Config`], as it requires an +//! implementation of the traits [`frame_support::traits::Currency`] and +//! [`frame_support::traits::CurrencyToVote`]. +//! +//! The process of adding bags to a runtime requires only four steps. +//! +//! 1. Update the runtime definition. +//! +//! ```ignore +//! parameter_types!{ +//! pub const BagThresholds: &'static [u64] = &[]; +//! } +//! +//! impl pallet_bags_list::Config for Runtime { +//! // +//! type BagThresholds = BagThresholds; +//! } +//! ``` +//! +//! 2. Write a little program to generate the definitions. This program exists only to hook together +//! the runtime definitions with the various calculations here. Take a look at +//! _utils/frame/generate_bags/node-runtime_ for an example. +//! +//! 3. Run that program: +//! +//! ```sh,notrust +//! $ cargo run -p node-runtime-generate-bags -- --total-issuance 1234 --minimum-balance 1 +//! output.rs ``` +//! +//! 4. Update the runtime definition. +//! +//! ```diff,notrust +//! + mod output; +//! - pub const BagThresholds: &'static [u64] = &[]; +//! + pub const BagThresholds: &'static [u64] = &output::THRESHOLDS; +//! ``` + +use frame_election_provider_support::VoteWeight; +use frame_support::traits::Get; +use std::{ + io::Write, + path::{Path, PathBuf}, +}; + +/// Compute the existential weight for the specified configuration. +/// +/// Note that this value depends on the current issuance, a quantity known to change over time. +/// This makes the project of computing a static value suitable for inclusion in a static, +/// generated file _excitingly unstable_. +fn existential_weight( + total_issuance: u128, + minimum_balance: u128, +) -> VoteWeight { + use frame_support::traits::CurrencyToVote; + use std::convert::TryInto; + + T::CurrencyToVote::to_vote( + minimum_balance + .try_into() + .map_err(|_| "failed to convert minimum_balance to type Balance") + .unwrap(), + total_issuance + .try_into() + .map_err(|_| "failed to convert total_issuance to type Balance") + .unwrap(), + ) +} + +/// Return the path to a header file used in this repository if is exists. +/// +/// Just searches the git working directory root for files matching certain patterns; it's +/// pretty naive. +fn path_to_header_file() -> Option { + let repo = git2::Repository::open_from_env().ok()?; + let workdir = repo.workdir()?; + for file_name in &["HEADER-APACHE2", "HEADER-GPL3", "HEADER", "file_header.txt"] { + let path = workdir.join(file_name); + if path.exists() { + return Some(path) + } + } + None +} + +/// Create an underscore formatter: a formatter which inserts `_` every 3 digits of a number. +fn underscore_formatter() -> num_format::CustomFormat { + num_format::CustomFormat::builder() + .grouping(num_format::Grouping::Standard) + .separator("_") + .build() + .expect("format described here meets all constraints") +} + +/// Compute the constant ratio for the thresholds. +/// +/// This ratio ensures that each bag, with the possible exceptions of certain small ones and the +/// final one, is a constant multiple of the previous, while fully occupying the `VoteWeight` +/// space. +pub fn constant_ratio(existential_weight: VoteWeight, n_bags: usize) -> f64 { + ((VoteWeight::MAX as f64 / existential_weight as f64).ln() / ((n_bags - 1) as f64)).exp() +} + +/// Compute the list of bag thresholds. +/// +/// Returns a list of exactly `n_bags` elements, except in the case of overflow. +/// The first element is always `existential_weight`. +/// The last element is always `VoteWeight::MAX`. +/// +/// All other elements are computed from the previous according to the formula +/// `threshold[k + 1] = (threshold[k] * ratio).max(threshold[k] + 1); +pub fn thresholds( + existential_weight: VoteWeight, + constant_ratio: f64, + n_bags: usize, +) -> Vec { + const WEIGHT_LIMIT: f64 = VoteWeight::MAX as f64; + + let mut thresholds = Vec::with_capacity(n_bags); + + if n_bags > 1 { + thresholds.push(existential_weight); + } + + while n_bags > 0 && thresholds.len() < n_bags - 1 { + let last = thresholds.last().copied().unwrap_or(existential_weight); + let successor = (last as f64 * constant_ratio).round().max(last as f64 + 1.0); + if successor < WEIGHT_LIMIT { + thresholds.push(successor as VoteWeight); + } else { + eprintln!("unexpectedly exceeded weight limit; breaking threshold generation loop"); + break + } + } + + thresholds.push(VoteWeight::MAX); + + debug_assert_eq!(thresholds.len(), n_bags); + debug_assert!(n_bags == 0 || thresholds[0] == existential_weight); + debug_assert!(n_bags == 0 || thresholds[thresholds.len() - 1] == VoteWeight::MAX); + + thresholds +} + +/// Write a thresholds module to the path specified. +/// +/// Parameters: +/// - `n_bags` the number of bags to generate. +/// - `output` the path to write to; should terminate with a Rust module name, i.e. +/// `foo/bar/thresholds.rs`. +/// - `total_issuance` the total amount of the currency in the network. +/// - `minimum_balance` the minimum balance of the currency required for an account to exist (i.e. +/// existential deposit). +/// +/// This generated module contains, in order: +/// +/// - The contents of the header file in this repository's root, if found. +/// - Module documentation noting that this is autogenerated and when. +/// - Some associated constants. +/// - The constant array of thresholds. +pub fn generate_thresholds( + n_bags: usize, + output: &Path, + total_issuance: u128, + minimum_balance: u128, +) -> Result<(), std::io::Error> { + // ensure the file is accessable + if let Some(parent) = output.parent() { + if !parent.exists() { + std::fs::create_dir_all(parent)?; + } + } + + // copy the header file + if let Some(header_path) = path_to_header_file() { + std::fs::copy(header_path, output)?; + } + + // open an append buffer + let file = std::fs::OpenOptions::new().create(true).append(true).open(output)?; + let mut buf = std::io::BufWriter::new(file); + + // create underscore formatter and format buffer + let mut num_buf = num_format::Buffer::new(); + let format = underscore_formatter(); + + // module docs + let now = chrono::Utc::now(); + writeln!(buf)?; + writeln!(buf, "//! Autogenerated bag thresholds.")?; + writeln!(buf, "//!")?; + writeln!(buf, "//! Generated on {}", now.to_rfc3339())?; + writeln!( + buf, + "//! for the {} runtime.", + ::Version::get().spec_name, + )?; + + let existential_weight = existential_weight::(total_issuance, minimum_balance); + num_buf.write_formatted(&existential_weight, &format); + writeln!(buf)?; + writeln!(buf, "/// Existential weight for this runtime.")?; + writeln!(buf, "#[cfg(any(test, feature = \"std\"))]")?; + writeln!(buf, "#[allow(unused)]")?; + writeln!(buf, "pub const EXISTENTIAL_WEIGHT: u64 = {};", num_buf.as_str())?; + + // constant ratio + let constant_ratio = constant_ratio(existential_weight, n_bags); + writeln!(buf)?; + writeln!(buf, "/// Constant ratio between bags for this runtime.")?; + writeln!(buf, "#[cfg(any(test, feature = \"std\"))]")?; + writeln!(buf, "#[allow(unused)]")?; + writeln!(buf, "pub const CONSTANT_RATIO: f64 = {:.16};", constant_ratio)?; + + // thresholds + let thresholds = thresholds(existential_weight, constant_ratio, n_bags); + writeln!(buf)?; + writeln!(buf, "/// Upper thresholds delimiting the bag list.")?; + writeln!(buf, "pub const THRESHOLDS: [u64; {}] = [", thresholds.len())?; + for threshold in thresholds { + num_buf.write_formatted(&threshold, &format); + // u64::MAX, with spacers every 3 digits, is 26 characters wide + writeln!(buf, " {:>26},", num_buf.as_str())?; + } + writeln!(buf, "];")?; + + Ok(()) +}