Skip to content
This repository was archived by the owner on Nov 15, 2023. It is now read-only.

Make it possible to disable RocksDB completely #11537

Merged
paritytech-processbot merged 5 commits into from
Jun 14, 2022
Merged

Make it possible to disable RocksDB completely #11537

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
c304c45
Make it possible to disable RocksDB completely
nazar-pc May 28, 2022
61ad9b8
Make ParityDB non-optional
nazar-pc May 30, 2022
e2e6371
Address review comments
nazar-pc Jun 1, 2022
b4974e4
Merge remote-tracking branch 'upstream/master' into optional-rocksdb
nazar-pc Jun 1, 2022
9317c64
Merge remote-tracking branch 'upstream/master' into optional-rocksdb
nazar-pc Jun 13, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion client/cli/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ thiserror = "1.0.30"
tiny-bip39 = "0.8.2"
tokio = { version = "1.17.0", features = ["signal", "rt-multi-thread", "parking_lot"] }
sc-client-api = { version = "4.0.0-dev", path = "../api" }
sc-client-db = { version = "0.10.0-dev", path = "../db" }
sc-client-db = { version = "0.10.0-dev", default-features = false, path = "../db" }
sc-keystore = { version = "4.0.0-dev", path = "../keystore" }
sc-network = { version = "0.10.0-dev", path = "../network" }
sc-service = { version = "0.10.0-dev", default-features = false, path = "../service" }
Expand All @@ -50,4 +50,7 @@ sp-version = { version = "5.0.0", path = "../../primitives/version" }
tempfile = "3.1.0"

[features]
default = ["with-parity-db", "with-rocks-db"]
with-parity-db = ["sc-client-db/with-parity-db"]
Copy link
Contributor

@koute koute May 30, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Considering how the endgame is to make the ParityDB the default and eventually remove RocksDB I think it makes sense to only make the rocksdb support optional. Are there any reasonable use cases where someone would want to compile a binary with only RocksDB?

Copy link
Member

@arkpar arkpar May 30, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These features were originally introduced for substrate-in-the-browser build, which had both backends disabled and used custom in-memory storage. As far as I know it is no longer maintained and has been deprecated in favor of smoldot, so we can remove extra features.

Copy link
Contributor Author

@nazar-pc nazar-pc May 30, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Made ParityDB non-optional and removed extra features

with-rocks-db = ["sc-client-db/with-kvdb-rocksdb"]
wasmtime = ["sc-service/wasmtime"]
29 changes: 22 additions & 7 deletions client/cli/src/arg_enums.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -237,13 +237,16 @@ impl Into<sc_service::config::RpcMethods> for RpcMethods {
/// Database backend
#[derive(Debug, Clone, PartialEq, Copy)]
pub enum Database {
#[cfg(feature = "with-rocks-db")]
/// Facebooks RocksDB
RocksDb,
#[cfg(feature = "with-parity-db")]
/// ParityDb. <https://github.com/paritytech/parity-db/>
ParityDb,
/// Detect whether there is an existing database. Use it, if there is, if not, create new
/// instance of ParityDb
Auto,
#[cfg(feature = "with-parity-db")]
/// ParityDb. <https://github.com/paritytech/parity-db/>
ParityDbDeprecated,
}
Expand All @@ -252,13 +255,17 @@ impl std::str::FromStr for Database {
type Err = String;

fn from_str(s: &str) -> Result<Self, String> {
#[cfg(feature = "with-rocks-db")]
if s.eq_ignore_ascii_case("rocksdb") {
Ok(Self::RocksDb)
} else if s.eq_ignore_ascii_case("paritydb-experimental") {
Ok(Self::ParityDbDeprecated)
return Ok(Self::RocksDb)
}
#[cfg(feature = "with-parity-db")]
if s.eq_ignore_ascii_case("paritydb-experimental") {
return Ok(Self::ParityDbDeprecated)
} else if s.eq_ignore_ascii_case("paritydb") {
Ok(Self::ParityDb)
} else if s.eq_ignore_ascii_case("auto") {
return Ok(Self::ParityDb)
}
if s.eq_ignore_ascii_case("auto") {
Ok(Self::Auto)
} else {
Err(format!("Unknown variant `{}`, known variants: {:?}", s, Self::variants()))
Expand All @@ -268,8 +275,16 @@ impl std::str::FromStr for Database {

impl Database {
/// Returns all the variants of this enum to be shown in the cli.
pub fn variants() -> &'static [&'static str] {
&["rocksdb", "paritydb", "paritydb-experimental", "auto"]
pub const fn variants() -> &'static [&'static str] {
&[
#[cfg(feature = "with-rocks-db")]
"rocksdb",
#[cfg(feature = "with-parity-db")]
"paritydb",
#[cfg(feature = "with-parity-db")]
"paritydb-experimental",
"auto",
]
}
}

Expand Down
14 changes: 13 additions & 1 deletion client/cli/src/config.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -220,8 +220,11 @@ pub trait CliConfiguration<DCV: DefaultConfigurationValues = ()>: Sized {
let rocksdb_path = base_path.join("db").join(role_dir);
let paritydb_path = base_path.join("paritydb").join(role_dir);
Ok(match database {
#[cfg(feature = "with-rocks-db")]
Database::RocksDb => DatabaseSource::RocksDb { path: rocksdb_path, cache_size },
#[cfg(feature = "with-parity-db")]
Database::ParityDb => DatabaseSource::ParityDb { path: paritydb_path },
#[cfg(feature = "with-parity-db")]
Database::ParityDbDeprecated => {
eprintln!(
"WARNING: \"paritydb-experimental\" database setting is deprecated and will be removed in future releases. \
Expand Down Expand Up @@ -500,7 +503,16 @@ pub trait CliConfiguration<DCV: DefaultConfigurationValues = ()>: Sized {
let net_config_dir = config_dir.join(DEFAULT_NETWORK_CONFIG_PATH);
let client_id = C::client_id();
let database_cache_size = self.database_cache_size()?.unwrap_or(1024);
let database = self.database()?.unwrap_or(Database::RocksDb);
let database = self.database()?.unwrap_or(
#[cfg(feature = "with-rocks-db")]
{
Database::RocksDb
},
#[cfg(all(feature = "with-parity-db", not(feature = "with-rocks-db")))]
{
Database::ParityDb
},
);
let node_key = self.node_key(&net_config_dir)?;
let role = self.role(is_dev)?;
let max_runtime_instances = self.max_runtime_instances()?.unwrap_or(8);
Expand Down
20 changes: 16 additions & 4 deletions client/db/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@

pub mod offchain;

#[cfg(any(feature = "with-kvdb-rocksdb", test))]
#[cfg(any(feature = "with-parity-db", feature = "with-kvdb-rocksdb", test))]
pub mod bench;

mod children;
Expand Down Expand Up @@ -94,7 +94,7 @@ use sp_trie::{prefixed_key, MemoryDB, PrefixedMemoryDB};
pub use sc_state_db::PruningMode;
pub use sp_database::Database;

#[cfg(any(feature = "with-kvdb-rocksdb", test))]
#[cfg(any(feature = "with-parity-db", feature = "with-kvdb-rocksdb", test))]
pub use bench::BenchmarkingState;

const CACHE_HEADERS: usize = 8;
Expand Down Expand Up @@ -327,6 +327,7 @@ pub enum DatabaseSource {
cache_size: usize,
},
/// Load a RocksDB database from a given path. Recommended for most uses.
#[cfg(feature = "with-kvdb-rocksdb")]
RocksDb {
/// Path to the database.
path: PathBuf,
Expand All @@ -335,6 +336,7 @@ pub enum DatabaseSource {
},

/// Load a ParityDb database from a given path.
#[cfg(feature = "with-parity-db")]
ParityDb {
/// Path to the database.
path: PathBuf,
Expand All @@ -359,7 +361,10 @@ impl DatabaseSource {
// IIUC this is needed for polkadot to create its own dbs, so until it can use parity db
// I would think rocksdb, but later parity-db.
DatabaseSource::Auto { paritydb_path, .. } => Some(paritydb_path),
DatabaseSource::RocksDb { path, .. } | DatabaseSource::ParityDb { path } => Some(path),
#[cfg(feature = "with-kvdb-rocksdb")]
DatabaseSource::RocksDb { path, .. } => Some(path),
#[cfg(feature = "with-parity-db")]
DatabaseSource::ParityDb { path } => Some(path),
DatabaseSource::Custom { .. } => None,
}
}
Expand All @@ -371,7 +376,12 @@ impl DatabaseSource {
*paritydb_path = p.into();
true
},
DatabaseSource::RocksDb { ref mut path, .. } |
#[cfg(feature = "with-kvdb-rocksdb")]
DatabaseSource::RocksDb { ref mut path, .. } => {
*path = p.into();
true
},
#[cfg(feature = "with-parity-db")]
DatabaseSource::ParityDb { ref mut path } => {
*path = p.into();
true
Expand All @@ -385,7 +395,9 @@ impl std::fmt::Display for DatabaseSource {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let name = match self {
DatabaseSource::Auto { .. } => "Auto",
#[cfg(feature = "with-kvdb-rocksdb")]
DatabaseSource::RocksDb { .. } => "RocksDb",
#[cfg(feature = "with-parity-db")]
DatabaseSource::ParityDb { .. } => "ParityDb",
DatabaseSource::Custom { .. } => "Custom",
};
Expand Down
2 changes: 2 additions & 0 deletions client/db/src/utils.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -197,7 +197,9 @@ fn open_database_at<Block: BlockT>(
create: bool,
) -> OpenDbResult {
let db: Arc<dyn Database<DbHash>> = match &db_source {
#[cfg(feature = "with-parity-db")]
DatabaseSource::ParityDb { path } => open_parity_db::<Block>(path, db_type, create)?,
#[cfg(feature = "with-kvdb-rocksdb")]
DatabaseSource::RocksDb { path, cache_size } =>
open_kvdb_rocksdb::<Block>(path, db_type, create, *cache_size)?,
DatabaseSource::Custom { db, require_create_flag } => {
Expand Down
4 changes: 3 additions & 1 deletion client/service/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,9 @@ targets = ["x86_64-unknown-linux-gnu"]
default = ["db"]
# The RocksDB feature activates the RocksDB database backend. If it is not activated, and you pass
# a path to a database, an error will be produced at runtime.
db = ["sc-client-db/with-kvdb-rocksdb", "sc-client-db/with-parity-db"]
with-parity-db = ["sc-client-db/with-parity-db"]
with-rocks-db = ["sc-client-db/with-kvdb-rocksdb"]
db = ["with-parity-db", "with-rocks-db"]
wasmtime = ["sc-executor/wasmtime"]
# exposes the client type
test-helpers = []
Expand Down
2 changes: 1 addition & 1 deletion test-utils/client/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ hex = "0.4"
serde = "1.0.136"
serde_json = "1.0.79"
sc-client-api = { version = "4.0.0-dev", path = "../../client/api" }
sc-client-db = { version = "0.10.0-dev", features = [
sc-client-db = { version = "0.10.0-dev", default-features = false, features = [
"test-helpers",
], path = "../../client/db" }
sc-consensus = { version = "0.10.0-dev", path = "../../client/consensus/common" }
Expand Down
11 changes: 7 additions & 4 deletions utils/frame/benchmarking-cli/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,9 +39,9 @@ frame-benchmarking = { version = "4.0.0-dev", path = "../../../frame/benchmarkin
frame-support = { version = "4.0.0-dev", path = "../../../frame/support" }
frame-system = { version = "4.0.0-dev", path = "../../../frame/system" }
sc-block-builder = { version = "0.10.0-dev", path = "../../../client/block-builder" }
sc-cli = { version = "0.10.0-dev", path = "../../../client/cli" }
sc-cli = { version = "0.10.0-dev", default-features = false, path = "../../../client/cli" }
sc-client-api = { version = "4.0.0-dev", path = "../../../client/api" }
sc-client-db = { version = "0.10.0-dev", path = "../../../client/db" }
sc-client-db = { version = "0.10.0-dev", default-features = false, path = "../../../client/db" }
sc-executor = { version = "0.10.0-dev", path = "../../../client/executor" }
sc-service = { version = "0.10.0-dev", default-features = false, path = "../../../client/service" }
sc-sysinfo = { version = "6.0.0-dev", path = "../../../client/sysinfo" }
Expand All @@ -58,5 +58,8 @@ sp-storage = { version = "6.0.0", path = "../../../primitives/storage" }
sp-trie = { version = "6.0.0", path = "../../../primitives/trie" }

[features]
default = ["db", "sc-client-db/runtime-benchmarks"]
db = ["sc-client-db/with-kvdb-rocksdb", "sc-client-db/with-parity-db"]
default = ["db", "runtime-benchmarks"]
runtime-benchmarks = ["sc-client-db/runtime-benchmarks"]
with-parity-db = ["sc-cli/with-parity-db", "sc-client-db/with-parity-db"]
with-rocks-db = ["sc-cli/with-rocks-db", "sc-client-db/with-kvdb-rocksdb"]
db = ["with-parity-db", "with-rocks-db"]