diff --git a/Cargo.lock b/Cargo.lock index 92268bafd..2adca97e4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1256,6 +1256,7 @@ dependencies = [ "clap", "fs-err", "kit", + "rayon", "serde", "serde_json", "tokio", @@ -3517,7 +3518,7 @@ dependencies = [ [[package]] name = "hyperdrive" -version = "1.8.1" +version = "1.8.3" dependencies = [ "aes-gcm", "alloy", @@ -3577,7 +3578,7 @@ dependencies = [ [[package]] name = "hyperdrive_lib" -version = "1.8.1" +version = "1.8.3" dependencies = [ "lib", ] @@ -4265,7 +4266,7 @@ dependencies = [ [[package]] name = "kit" version = "3.1.1" -source = "git+https://github.com/hyperware-ai/kit?rev=275f02c#275f02c839e239083ba656871e10be845f4d85f5" +source = "git+https://github.com/hyperware-ai/kit?rev=0d0469e#0d0469ec89ce5ff8f25223e3d968cd5760554251" dependencies = [ "alloy", "alloy-sol-macro", @@ -4334,7 +4335,7 @@ checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" [[package]] name = "lib" -version = "1.8.1" +version = "1.8.3" dependencies = [ "alloy", "anyhow", diff --git a/Cargo.toml b/Cargo.toml index 124104a0a..3b8ab5cd2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "hyperdrive_lib" authors = ["Sybil Technologies AG"] -version = "1.8.2" +version = "1.9.0" edition = "2021" description = "A general-purpose sovereign cloud computing platform" homepage = "https://hyperware.ai" @@ -22,8 +22,8 @@ members = [ "hyperdrive/packages/homepage/homepage", "hyperdrive/packages/hns-indexer/hns-indexer", "hyperdrive/packages/hns-indexer/get-block", "hyperdrive/packages/settings/settings", "hyperdrive/packages/hns-indexer/reset", "hyperdrive/packages/hns-indexer/node-info", "hyperdrive/packages/hns-indexer/state", - "hyperdrive/packages/hypermap-cacher/hypermap-cacher", "hyperdrive/packages/hypermap-cacher/reset-cache", "hyperdrive/packages/hypermap-cacher/set-nodes", - "hyperdrive/packages/hypermap-cacher/start-providing", "hyperdrive/packages/hypermap-cacher/stop-providing", + "hyperdrive/packages/hypermap-cacher/binding-cacher", "hyperdrive/packages/hypermap-cacher/hypermap-cacher", "hyperdrive/packages/hypermap-cacher/reset-cache", + "hyperdrive/packages/hypermap-cacher/set-nodes", "hyperdrive/packages/hypermap-cacher/start-providing", "hyperdrive/packages/hypermap-cacher/stop-providing", "hyperdrive/packages/sign/sign", "hyperdrive/packages/spider/spider", "hyperdrive/packages/terminal/terminal", "hyperdrive/packages/terminal/add-node-provider", "hyperdrive/packages/terminal/add-rpcurl-provider", diff --git a/hyperdrive/Cargo.toml b/hyperdrive/Cargo.toml index 908563d76..956d93981 100644 --- a/hyperdrive/Cargo.toml +++ b/hyperdrive/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "hyperdrive" authors = ["Sybil Technologies AG"] -version = "1.8.2" +version = "1.9.0" edition = "2021" description = "A general-purpose sovereign cloud computing platform" homepage = "https://hyperware.ai" diff --git a/hyperdrive/dependencies/anthropic-api-key-manager b/hyperdrive/dependencies/anthropic-api-key-manager index aeaa740d0..e17d63dc2 160000 --- a/hyperdrive/dependencies/anthropic-api-key-manager +++ b/hyperdrive/dependencies/anthropic-api-key-manager @@ -1 +1 @@ -Subproject commit aeaa740d03fbaa29de82f7983aa8c06137d57bfd +Subproject commit e17d63dc2d3e4381013e4533fa24e8894045b22e diff --git a/hyperdrive/packages/app-store/Cargo.lock b/hyperdrive/packages/app-store/Cargo.lock index 881b278f8..a5db5f05d 100644 --- a/hyperdrive/packages/app-store/Cargo.lock +++ b/hyperdrive/packages/app-store/Cargo.lock @@ -585,7 +585,7 @@ dependencies = [ "alloy-sol-types", "anyhow", "bincode", - "hyperware_process_lib", + "hyperware_process_lib 2.1.0", "process_macros", "rand", "serde", @@ -951,7 +951,8 @@ dependencies = [ "alloy-sol-types", "anyhow", "bincode", - "hyperware_process_lib", + "hex", + "hyperware_process_lib 2.2.1", "process_macros", "rand", "serde", @@ -1155,7 +1156,7 @@ name = "download" version = "0.1.0" dependencies = [ "anyhow", - "hyperware_process_lib", + "hyperware_process_lib 2.1.0", "process_macros", "serde", "serde_json", @@ -1167,7 +1168,7 @@ name = "downloads" version = "0.5.0" dependencies = [ "anyhow", - "hyperware_process_lib", + "hyperware_process_lib 2.1.0", "process_macros", "rand", "serde", @@ -1343,7 +1344,7 @@ version = "0.2.0" dependencies = [ "anyhow", "bincode", - "hyperware_process_lib", + "hyperware_process_lib 2.1.0", "process_macros", "rand", "serde", @@ -1680,6 +1681,30 @@ dependencies = [ "wit-bindgen", ] +[[package]] +name = "hyperware_process_lib" +version = "2.2.1" +source = "git+https://github.com/hyperware-ai/process_lib?rev=4e91521#4e915218b382e4b3f7f93cddeec30415f02a504b" +dependencies = [ + "alloy", + "alloy-primitives", + "alloy-sol-macro", + "alloy-sol-types", + "anyhow", + "base64", + "bincode", + "http", + "mime_guess", + "rand", + "regex", + "rmp-serde", + "serde", + "serde_json", + "thiserror 1.0.69", + "url", + "wit-bindgen", +] + [[package]] name = "icu_collections" version = "1.5.0" @@ -1861,7 +1886,7 @@ name = "install" version = "0.1.0" dependencies = [ "anyhow", - "hyperware_process_lib", + "hyperware_process_lib 2.1.0", "process_macros", "serde", "serde_json", @@ -2555,7 +2580,7 @@ name = "reset-store" version = "0.1.0" dependencies = [ "anyhow", - "hyperware_process_lib", + "hyperware_process_lib 2.1.0", "process_macros", "serde", "serde_json", @@ -3316,7 +3341,7 @@ name = "uninstall" version = "0.1.0" dependencies = [ "anyhow", - "hyperware_process_lib", + "hyperware_process_lib 2.1.0", "process_macros", "serde", "serde_json", diff --git a/hyperdrive/packages/app-store/api/app-store:sys-v1.wit b/hyperdrive/packages/app-store/api/app-store:sys-v2.wit similarity index 97% rename from hyperdrive/packages/app-store/api/app-store:sys-v1.wit rename to hyperdrive/packages/app-store/api/app-store:sys-v2.wit index caceea515..e5b72308e 100644 --- a/hyperdrive/packages/app-store/api/app-store:sys-v1.wit +++ b/hyperdrive/packages/app-store/api/app-store:sys-v2.wit @@ -172,6 +172,9 @@ interface chain { metadata-hash: string, metadata: option, auto-update: bool, + /// Total binding power for this app (sum of all user binds) + /// Computed dynamically based on remaining lock/bind duration + binding-power: option, } /// Metadata associated with an on-chain app @@ -405,8 +408,8 @@ interface downloads { } } -/// The app-store-sys-v1 world, which includes the main, downloads, and chain interfaces -world app-store-sys-v1 { +/// The app-store-sys-v2 world, which includes the main, downloads, and chain interfaces +world app-store-sys-v2 { import main; import downloads; import chain; diff --git a/hyperdrive/packages/app-store/app-store/src/lib.rs b/hyperdrive/packages/app-store/app-store/src/lib.rs index 4f71fba8d..b465f2e05 100644 --- a/hyperdrive/packages/app-store/app-store/src/lib.rs +++ b/hyperdrive/packages/app-store/app-store/src/lib.rs @@ -45,7 +45,7 @@ use state::{State, UpdateInfo, Updates}; wit_bindgen::generate!({ path: "../target/wit", generate_unused_types: true, - world: "app-store-sys-v1", + world: "app-store-sys-v2", additional_derives: [serde::Deserialize, serde::Serialize, process_macros::SerdeJsonInto], }); diff --git a/hyperdrive/packages/app-store/chain/Cargo.toml b/hyperdrive/packages/app-store/chain/Cargo.toml index 66e73fc05..fcd1b2904 100644 --- a/hyperdrive/packages/app-store/chain/Cargo.toml +++ b/hyperdrive/packages/app-store/chain/Cargo.toml @@ -10,8 +10,9 @@ simulation-mode = [] alloy-primitives = "0.8.15" alloy-sol-types = "0.8.15" anyhow = "1.0" +hex = "0.4" bincode = "1.3.3" -hyperware_process_lib = "2.1.0" +hyperware_process_lib = { git = "https://github.com/hyperware-ai/process_lib", rev = "4e91521" } process_macros = "0.1" rand = "0.8" serde = { version = "1.0", features = ["derive"] } diff --git a/hyperdrive/packages/app-store/chain/src/lib.rs b/hyperdrive/packages/app-store/chain/src/lib.rs index 0de1de78f..b296a3ee0 100644 --- a/hyperdrive/packages/app-store/chain/src/lib.rs +++ b/hyperdrive/packages/app-store/chain/src/lib.rs @@ -28,12 +28,23 @@ use crate::hyperware::process::chain::{ ChainError, ChainRequest, OnchainApp, OnchainMetadata, OnchainProperties, }; use crate::hyperware::process::downloads::{AutoUpdateRequest, DownloadRequest}; -use alloy_primitives::keccak256; +use alloy_primitives::{keccak256, U256}; use alloy_sol_types::SolEvent; +use hex; use hyperware::process::chain::ChainResponse; use hyperware_process_lib::{ - await_message, call_init, eth, get_blob, http, hypermap, kernel_types as kt, print_to_terminal, - println, + await_message, + bindings::{ + contract::{ + BindAmountIncreased, BindCreated, BindDurationExtended, ExpiredBindReclaimed, + LockExtended, TokensLocked, TokensWithdrawn, + }, + decode_bind_amount_increased_log, decode_bind_created_log, + decode_bind_duration_extended_log, decode_expired_bind_reclaimed_log, + decode_lock_extended_log, decode_tokens_locked_log, decode_tokens_withdrawn_log, Bindings, + BINDINGS_FIRST_BLOCK, + }, + call_init, eth, get_blob, http, hypermap, kernel_types as kt, print_to_terminal, println, sqlite::{self, Sqlite}, timer, Address, Message, PackageId, Request, Response, }; @@ -44,7 +55,7 @@ use std::str::FromStr; wit_bindgen::generate!({ path: "../target/wit", generate_unused_types: true, - world: "app-store-sys-v1", + world: "app-store-sys-v2", additional_derives: [serde::Deserialize, serde::Serialize, process_macros::SerdeJsonInto], }); @@ -60,14 +71,19 @@ const HYPERMAP_ADDRESS: &'static str = hypermap::HYPERMAP_ADDRESS; const DELAY_MS: u64 = 1_000; // 1s const SUBSCRIPTION_NUMBER: u64 = 1; +const BINDINGS_SUBSCRIPTION: u64 = 2; pub struct State { /// the hypermap helper we are using pub hypermap: hypermap::Hypermap, + /// the bindings helper for token registry + pub bindings: Bindings, /// the last block at which we saved the state of the listings to disk. /// when we boot, we can read logs starting from this block and /// rebuild latest state. pub last_saved_block: u64, + /// the last block for binding events + pub last_bindings_block: u64, /// tables: listings: , published: vec pub db: DB, } @@ -83,11 +99,30 @@ pub struct PackageListing { pub block: u64, } +/// User lock state from TokenRegistry +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct UserLock { + pub user_address: String, + pub amount: String, // U256 as string + pub end_time: u64, + pub block: u64, +} + +/// User bind to an app namehash +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct UserBind { + pub namehash: String, + pub user_address: String, + pub amount: String, // U256 as string + pub end_time: u64, + pub block: u64, +} + pub struct DB { inner: Sqlite, } -const DB_VERSION: u64 = 1; +const DB_VERSION: u64 = 2; impl DB { pub fn connect(our: &Address) -> anyhow::Result { @@ -96,15 +131,25 @@ impl DB { inner.write(CREATE_META_TABLE.into(), vec![], None)?; inner.write(CREATE_LISTINGS_TABLE.into(), vec![], None)?; inner.write(CREATE_PUBLISHED_TABLE.into(), vec![], None)?; + // binding-related tables + inner.write(CREATE_APP_NAMEHASHES_TABLE.into(), vec![], None)?; + inner.write(CREATE_USER_LOCKS_TABLE.into(), vec![], None)?; + inner.write(CREATE_USER_BINDS_TABLE.into(), vec![], None)?; + inner.write(CREATE_USER_BINDS_INDEX.into(), vec![], None)?; let db = Self { inner }; // versions and migrations let version = db.get_version()?; - if let None = version { + if version.is_none() { // clean up inconsistent state by re-indexing from block 0 db.set_last_saved_block(0)?; + db.set_last_bindings_block(0)?; + db.set_version(DB_VERSION)?; + } else if version == Some(1) { + // Migrate from version 1 to 2: binding tables already created above + db.set_last_bindings_block(0)?; db.set_version(DB_VERSION)?; } @@ -116,6 +161,13 @@ impl DB { self.inner.write(DROP_LISTINGS_TABLE.into(), vec![], None)?; self.inner .write(DROP_PUBLISHED_TABLE.into(), vec![], None)?; + // binding-related tables + self.inner + .write(DROP_APP_NAMEHASHES_TABLE.into(), vec![], None)?; + self.inner + .write(DROP_USER_LOCKS_TABLE.into(), vec![], None)?; + self.inner + .write(DROP_USER_BINDS_TABLE.into(), vec![], None)?; Ok(()) } @@ -361,6 +413,164 @@ impl DB { )?; Ok(()) } + + // Binding-related methods + + pub fn get_last_bindings_block(&self) -> anyhow::Result { + let query = "SELECT value FROM meta WHERE key = 'last_bindings_block'"; + let rows = self.inner.read(query.into(), vec![])?; + if let Some(row) = rows.first() { + if let Some(val_str) = row.get("value").and_then(|v| v.as_str()) { + if let Ok(block) = val_str.parse::() { + return Ok(block); + } + } + } + Ok(0) + } + + pub fn set_last_bindings_block(&self, block: u64) -> anyhow::Result<()> { + let query = "INSERT INTO meta (key, value) VALUES ('last_bindings_block', ?) + ON CONFLICT(key) DO UPDATE SET value=excluded.value"; + let params = vec![block.to_string().into()]; + self.inner.write(query.into(), params, None)?; + Ok(()) + } + + pub fn insert_app_namehash( + &self, + namehash: &str, + package_id: &PackageId, + block: u64, + ) -> anyhow::Result<()> { + let query = "INSERT INTO app_namehashes (namehash, package_name, publisher_node, block) + VALUES (?, ?, ?, ?) + ON CONFLICT(namehash) DO UPDATE SET + package_name=excluded.package_name, + publisher_node=excluded.publisher_node, + block=excluded.block"; + let params = vec![ + namehash.into(), + package_id.package_name.clone().into(), + package_id.publisher_node.clone().into(), + block.into(), + ]; + self.inner.write(query.into(), params, None)?; + Ok(()) + } + + pub fn get_app_namehash(&self, namehash: &str) -> anyhow::Result> { + let query = "SELECT package_name, publisher_node FROM app_namehashes WHERE namehash = ?"; + let params = vec![namehash.into()]; + let rows = self.inner.read(query.into(), params)?; + if let Some(row) = rows.first() { + let package_name = row["package_name"].as_str().unwrap_or("").to_string(); + let publisher_node = row["publisher_node"].as_str().unwrap_or("").to_string(); + return Ok(Some(PackageId { + package_name, + publisher_node, + })); + } + Ok(None) + } + + pub fn upsert_user_lock( + &self, + user_address: &str, + amount: &str, + end_time: u64, + block: u64, + ) -> anyhow::Result<()> { + let query = "INSERT INTO user_locks (user_address, amount, end_time, block) + VALUES (?, ?, ?, ?) + ON CONFLICT(user_address) DO UPDATE SET + amount=excluded.amount, + end_time=excluded.end_time, + block=excluded.block"; + let params = vec![ + user_address.into(), + amount.into(), + end_time.into(), + block.into(), + ]; + self.inner.write(query.into(), params, None)?; + Ok(()) + } + + pub fn get_user_lock(&self, user_address: &str) -> anyhow::Result> { + let query = "SELECT amount, end_time, block FROM user_locks WHERE user_address = ?"; + let params = vec![user_address.into()]; + let rows = self.inner.read(query.into(), params)?; + if let Some(row) = rows.first() { + return Ok(Some(UserLock { + user_address: user_address.to_string(), + amount: row["amount"].as_str().unwrap_or("0").to_string(), + end_time: row["end_time"].as_u64().unwrap_or(0), + block: row["block"].as_u64().unwrap_or(0), + })); + } + Ok(None) + } + + pub fn delete_user_lock(&self, user_address: &str) -> anyhow::Result<()> { + let query = "DELETE FROM user_locks WHERE user_address = ?"; + let params = vec![user_address.into()]; + self.inner.write(query.into(), params, None)?; + Ok(()) + } + + pub fn upsert_user_bind( + &self, + namehash: &str, + user_address: &str, + amount: &str, + end_time: u64, + block: u64, + ) -> anyhow::Result<()> { + let query = "INSERT INTO user_binds (namehash, user_address, amount, end_time, block) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(namehash, user_address) DO UPDATE SET + amount=excluded.amount, + end_time=excluded.end_time, + block=excluded.block"; + let params = vec![ + namehash.into(), + user_address.into(), + amount.into(), + end_time.into(), + block.into(), + ]; + self.inner.write(query.into(), params, None)?; + Ok(()) + } + + pub fn delete_user_bind(&self, namehash: &str, user_address: &str) -> anyhow::Result<()> { + let query = "DELETE FROM user_binds WHERE namehash = ? AND user_address = ?"; + let params = vec![namehash.into(), user_address.into()]; + self.inner.write(query.into(), params, None)?; + Ok(()) + } + + pub fn get_all_binds_for_app(&self, namehash: &str) -> anyhow::Result> { + // Only return binds that correspond to known apps + let query = "SELECT ub.namehash, ub.user_address, ub.amount, ub.end_time, ub.block + FROM user_binds ub + INNER JOIN app_namehashes an ON ub.namehash = an.namehash + WHERE ub.namehash = ?"; + let params = vec![namehash.into()]; + let rows = self.inner.read(query.into(), params)?; + let mut binds = Vec::new(); + for row in rows { + binds.push(UserBind { + namehash: row["namehash"].as_str().unwrap_or("").to_string(), + user_address: row["user_address"].as_str().unwrap_or("").to_string(), + amount: row["amount"].as_str().unwrap_or("0").to_string(), + end_time: row["end_time"].as_u64().unwrap_or(0), + block: row["block"].as_u64().unwrap_or(0), + }); + } + Ok(binds) + } } const CREATE_META_TABLE: &str = " @@ -401,6 +611,41 @@ const DROP_PUBLISHED_TABLE: &str = " DROP TABLE IF EXISTS published; "; +// Binding-related tables +const CREATE_APP_NAMEHASHES_TABLE: &str = " +CREATE TABLE IF NOT EXISTS app_namehashes ( + namehash TEXT PRIMARY KEY, + package_name TEXT NOT NULL, + publisher_node TEXT NOT NULL, + block INTEGER NOT NULL DEFAULT 0 +);"; + +const CREATE_USER_LOCKS_TABLE: &str = " +CREATE TABLE IF NOT EXISTS user_locks ( + user_address TEXT PRIMARY KEY, + amount TEXT NOT NULL, + end_time INTEGER NOT NULL, + block INTEGER NOT NULL DEFAULT 0 +);"; + +const CREATE_USER_BINDS_TABLE: &str = " +CREATE TABLE IF NOT EXISTS user_binds ( + namehash TEXT NOT NULL, + user_address TEXT NOT NULL, + amount TEXT NOT NULL, + end_time INTEGER NOT NULL, + block INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (namehash, user_address) +);"; + +const CREATE_USER_BINDS_INDEX: &str = " +CREATE INDEX IF NOT EXISTS idx_user_binds_namehash ON user_binds(namehash); +"; + +const DROP_APP_NAMEHASHES_TABLE: &str = "DROP TABLE IF EXISTS app_namehashes;"; +const DROP_USER_LOCKS_TABLE: &str = "DROP TABLE IF EXISTS user_locks;"; +const DROP_USER_BINDS_TABLE: &str = "DROP TABLE IF EXISTS user_binds;"; + call_init!(init); fn init(our: Address) { loop { @@ -410,16 +655,20 @@ fn init(our: Address) { let db = DB::connect(&our).expect("failed to open DB"); let hypermap_helper = hypermap::Hypermap::new( - eth_provider, + eth_provider.clone(), eth::Address::from_str(HYPERMAP_ADDRESS).unwrap(), ); + let bindings_helper = Bindings::default(CHAIN_TIMEOUT); let last_saved_block = db .get_last_saved_block() .unwrap_or(hypermap::HYPERMAP_FIRST_BLOCK); + let last_bindings_block = db.get_last_bindings_block().unwrap_or(BINDINGS_FIRST_BLOCK); let mut state = State { hypermap: hypermap_helper, + bindings: bindings_helper, last_saved_block, + last_bindings_block, db, }; @@ -447,6 +696,13 @@ fn init(our: Address) { } /// returns true if we should re-index +/// Context wrapper to distinguish log types in timer callbacks +#[derive(Debug, Deserialize, Serialize)] +enum LogContext { + Hypermap(eth::Log), + Bindings(eth::Log), +} + fn handle_message(our: &Address, state: &mut State, message: &Message) -> anyhow::Result { if !message.is_local() { // networking is off: we will never get non-local messages @@ -458,26 +714,48 @@ fn handle_message(our: &Address, state: &mut State, message: &Message) -> anyhow let Some(context) = message.context() else { return Err(anyhow::anyhow!("No context in timer message")); }; - let log = serde_json::from_slice(context)?; - handle_eth_log(our, state, log, false)?; + let log_context: LogContext = serde_json::from_slice(context)?; + match log_context { + LogContext::Hypermap(log) => { + handle_eth_log(our, state, log, false)?; + } + LogContext::Bindings(log) => { + handle_binding_log(state, log)?; + // Save bindings block after processing + if let Err(e) = state.db.set_last_bindings_block(state.last_bindings_block) { + print_to_terminal(0, &format!("error saving bindings block: {e}")); + } + } + } return Ok(false); } } else { if message.source().process == "eth:distro:sys" { let eth_result = serde_json::from_slice::(message.body())?; - if let Ok(eth::EthSub { result, .. }) = eth_result { + if let Ok(eth::EthSub { id, result }) = eth_result { if let Ok(eth::SubscriptionResult::Log(ref log)) = serde_json::from_value::(result) { + // Determine which subscription this is from + // Note: log is Box, we need to dereference it + let log_ref: ð::Log = &**log; + let context = if id == SUBSCRIPTION_NUMBER { + LogContext::Hypermap(log_ref.clone()) + } else if id == BINDINGS_SUBSCRIPTION { + LogContext::Bindings(log_ref.clone()) + } else { + return Ok(false); // Unknown subscription + }; // delay handling of ETH RPC subscriptions by DELAY_MS // to allow hns to have a chance to process block - timer::set_timer(DELAY_MS, Some(serde_json::to_vec(log)?)); + timer::set_timer(DELAY_MS, Some(serde_json::to_vec(&context)?)); } } else { // unsubscribe to make sure we have cleaned up after ourselves; // drop Result since we don't care if no subscription exists, // just being diligent in case it does! let _ = state.hypermap.provider.unsubscribe(SUBSCRIPTION_NUMBER); + let _ = state.bindings.provider.unsubscribe(BINDINGS_SUBSCRIPTION); // re-subscribe if error state.hypermap.provider.subscribe_loop( SUBSCRIPTION_NUMBER, @@ -485,6 +763,12 @@ fn handle_message(our: &Address, state: &mut State, message: &Message) -> anyhow 1, 0, ); + state.bindings.provider.subscribe_loop( + BINDINGS_SUBSCRIPTION, + bindings_filter(&state.bindings), + 1, + 0, + ); } } else { let req = serde_json::from_slice::(message.body())?; @@ -495,31 +779,78 @@ fn handle_message(our: &Address, state: &mut State, message: &Message) -> anyhow } fn handle_local_request(state: &mut State, req: ChainRequest) -> anyhow::Result { + // Get current timestamp for binding power calculations + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0); + match req { ChainRequest::GetApp(package_id) => { let pid = package_id.clone().to_process_lib(); let listing = state.db.get_listing(&pid)?; - let onchain_app = listing.map(|app| app.to_onchain_app(&pid)); + let onchain_app = listing.map(|app| { + // Compute binding power for this app + // hypermap::namehash returns a hex String, use it directly + let namehash = + hypermap::namehash(&format!("{}.{}", pid.package_name, pid.publisher_node)); + let power = compute_app_total_binding_power(&state.db, &namehash, now) + .ok() + .map(|p| p.to_string()); + app.to_onchain_app(&pid, power) + }); let response = ChainResponse::GetApp(onchain_app); Response::new().body(&response).send()?; } ChainRequest::GetApps => { let listings = state.db.get_all_listings()?; - let apps: Vec = listings + // Compute binding power for each app and collect with power for sorting + let mut apps_with_power: Vec<(OnchainApp, U256)> = listings .into_iter() - .map(|(pid, listing)| listing.to_onchain_app(&pid)) + .map(|(pid, listing)| { + // hypermap::namehash returns a hex String, use it directly + let namehash = hypermap::namehash(&format!( + "{}.{}", + pid.package_name, pid.publisher_node + )); + let power = compute_app_total_binding_power(&state.db, &namehash, now) + .unwrap_or(U256::ZERO); + print_to_terminal( + 2, + &format!( + "[DEBUG BINDING] GetApps: {}.{}\n computed namehash: {}\n binding_power: {}", + pid.package_name, pid.publisher_node, namehash, power + ), + ); + let app = listing.to_onchain_app(&pid, Some(power.to_string())); + (app, power) + }) .collect(); + + // Sort by binding power descending + apps_with_power.sort_by(|a, b| b.1.cmp(&a.1)); + + let apps: Vec = apps_with_power.into_iter().map(|(app, _)| app).collect(); let response = ChainResponse::GetApps(apps); Response::new().body(&response).send()?; } ChainRequest::GetOurApps => { let published_list = state.db.get_all_published()?; - let mut apps = Vec::new(); + let mut apps_with_power: Vec<(OnchainApp, U256)> = Vec::new(); for pid in published_list { if let Some(listing) = state.db.get_listing(&pid)? { - apps.push(listing.to_onchain_app(&pid)); + // hypermap::namehash returns a hex String, use it directly + let namehash = + hypermap::namehash(&format!("{}.{}", pid.package_name, pid.publisher_node)); + let power = compute_app_total_binding_power(&state.db, &namehash, now) + .unwrap_or(U256::ZERO); + apps_with_power + .push((listing.to_onchain_app(&pid, Some(power.to_string())), power)); } } + // Sort by binding power descending + apps_with_power.sort_by(|a, b| b.1.cmp(&a.1)); + let apps: Vec = apps_with_power.into_iter().map(|(app, _)| app).collect(); let response = ChainResponse::GetOurApps(apps); Response::new().body(&response).send()?; } @@ -552,7 +883,9 @@ fn handle_local_request(state: &mut State, req: ChainRequest) -> anyhow::Result< println!("re-indexing state!"); // set last_saved_block to 0 & drop tables to force re-index state.last_saved_block = 0; + state.last_bindings_block = 0; state.db.set_last_saved_block(0)?; + state.db.set_last_bindings_block(0)?; state.db.drop_all()?; return Ok(true); } @@ -674,6 +1007,28 @@ fn handle_eth_log( state.db.insert_or_update_listing(&package_id, &listing)?; + // Store the app's namehash for binding power lookups. + // The parenthash is topics[1] from the Note event. + if log.topics().len() >= 2 { + let parenthash = log.topics()[1]; + let namehash_str = format!("0x{}", hex::encode(parenthash)); + // hypermap::namehash returns a hex String, use it directly + let computed_namehash = hypermap::namehash(&format!( + "{}.{}", + package_id.package_name, package_id.publisher_node + )); + print_to_terminal( + 2, + &format!( + "[DEBUG BINDING] handle_eth_log: storing app namehash\n package_id: {}\n stored (topics[1]): {}\n computed_namehash: {}\n match: {}", + package_id, namehash_str, computed_namehash, namehash_str == computed_namehash + ), + ); + state + .db + .insert_app_namehash(&namehash_str, &package_id, block_number)?; + } + if !startup && listing.auto_update { println!("kicking off auto-update for {package_id}"); Request::to(("our", "downloads", "app-store", "sys")) @@ -824,6 +1179,184 @@ pub fn app_store_filter(state: &State) -> eth::Filter { .topic3(notes) } +/// create a filter for binding events from TokenRegistry +fn bindings_filter(bindings: &Bindings) -> eth::Filter { + eth::Filter::new().address(*bindings.address()).events([ + TokensLocked::SIGNATURE, + LockExtended::SIGNATURE, + TokensWithdrawn::SIGNATURE, + BindCreated::SIGNATURE, + BindAmountIncreased::SIGNATURE, + BindDurationExtended::SIGNATURE, + ExpiredBindReclaimed::SIGNATURE, + ]) +} + +/// handle a binding log from the TokenRegistry contract +fn handle_binding_log(state: &mut State, log: eth::Log) -> anyhow::Result<()> { + let block_number = log + .block_number + .ok_or(anyhow::anyhow!("log missing block number"))?; + let topic0 = *log + .topics() + .first() + .ok_or(anyhow::anyhow!("log missing topic"))?; + + // Handle lock events + if topic0 == TokensLocked::SIGNATURE_HASH { + let parsed = decode_tokens_locked_log(&log) + .map_err(|e| anyhow::anyhow!("failed to decode TokensLocked: {:?}", e))?; + state.db.upsert_user_lock( + &parsed.account.to_string(), + &parsed.balance.to_string(), + parsed.end_time.to::(), + block_number, + )?; + } else if topic0 == LockExtended::SIGNATURE_HASH { + let parsed = decode_lock_extended_log(&log) + .map_err(|e| anyhow::anyhow!("failed to decode LockExtended: {:?}", e))?; + state.db.upsert_user_lock( + &parsed.account.to_string(), + &parsed.balance.to_string(), + parsed.end_time.to::(), + block_number, + )?; + } else if topic0 == TokensWithdrawn::SIGNATURE_HASH { + let parsed = decode_tokens_withdrawn_log(&log) + .map_err(|e| anyhow::anyhow!("failed to decode TokensWithdrawn: {:?}", e))?; + if parsed.remaining_amount.is_zero() { + state.db.delete_user_lock(&parsed.user.to_string())?; + } else { + state.db.upsert_user_lock( + &parsed.user.to_string(), + &parsed.remaining_amount.to_string(), + parsed.end_time.to::(), + block_number, + )?; + } + } + // Handle bind events - store all, filter on query + else if topic0 == BindCreated::SIGNATURE_HASH { + let parsed = decode_bind_created_log(&log) + .map_err(|e| anyhow::anyhow!("failed to decode BindCreated: {:?}", e))?; + let namehash_str = format!("0x{}", hex::encode(parsed.namehash)); + print_to_terminal( + 2, + &format!( + "[DEBUG BINDING] handle_binding_log BindCreated:\n user: {}\n namehash: {}\n amount: {}\n end_time: {}", + parsed.user, namehash_str, parsed.amount, parsed.end_time + ), + ); + state.db.upsert_user_bind( + &namehash_str, + &parsed.user.to_string(), + &parsed.amount.to_string(), + parsed.end_time.to::(), + block_number, + )?; + } else if topic0 == BindAmountIncreased::SIGNATURE_HASH { + let parsed = decode_bind_amount_increased_log(&log) + .map_err(|e| anyhow::anyhow!("failed to decode BindAmountIncreased: {:?}", e))?; + let namehash_str = format!("0x{}", hex::encode(parsed.namehash)); + state.db.upsert_user_bind( + &namehash_str, + &parsed.user.to_string(), + &parsed.amount.to_string(), + parsed.end_time.to::(), + block_number, + )?; + } else if topic0 == BindDurationExtended::SIGNATURE_HASH { + let parsed = decode_bind_duration_extended_log(&log) + .map_err(|e| anyhow::anyhow!("failed to decode BindDurationExtended: {:?}", e))?; + let namehash_str = format!("0x{}", hex::encode(parsed.namehash)); + state.db.upsert_user_bind( + &namehash_str, + &parsed.user.to_string(), + &parsed.amount.to_string(), + parsed.end_time.to::(), + block_number, + )?; + } else if topic0 == ExpiredBindReclaimed::SIGNATURE_HASH { + let parsed = decode_expired_bind_reclaimed_log(&log) + .map_err(|e| anyhow::anyhow!("failed to decode ExpiredBindReclaimed: {:?}", e))?; + let namehash_str = format!("0x{}", hex::encode(parsed.namehash)); + state + .db + .delete_user_bind(&namehash_str, &parsed.user.to_string())?; + } + + if block_number > state.last_bindings_block { + state.last_bindings_block = block_number; + } + Ok(()) +} + +// Binding power formula constants (matching Solidity contract) +const BINDING_POWER_A: u128 = 1; +const BINDING_POWER_B: u128 = 2_000_000_000_000_000_000_000_000_000; // 2 * 1 * 1_000_000_000e18 +const MIN_LOCK_DURATION: u64 = 4 * 7 * 24 * 60 * 60; // 4 weeks in seconds +const MAX_LOCK_DURATION: u64 = 4 * 52 * 7 * 24 * 60 * 60; // ~4 years in seconds +const BINDING_POWER_C: u64 = MIN_LOCK_DURATION / 100; // 24192 +const BINDING_POWER_D: u128 = 2 * (BINDING_POWER_C as u128) * (MAX_LOCK_DURATION as u128); + +/// Compute binding power for a single bind using the sublinear formula +/// from the Solidity contract +fn compute_binding_power(value: U256, remaining_duration: u64) -> U256 { + if remaining_duration == 0 || value.is_zero() { + return U256::ZERO; + } + + let value_u128: u128 = value.try_into().unwrap_or(u128::MAX); + // Round up duration to minimum if below + let duration = remaining_duration.max(MIN_LOCK_DURATION) as u128; + + // value_term = (value / A - value * value / B) + let value_term = (value_u128 / BINDING_POWER_A) + .saturating_sub(value_u128.saturating_mul(value_u128) / BINDING_POWER_B); + + // duration_term = (duration / C - duration * duration / D) + let duration_term = (duration / BINDING_POWER_C as u128) + .saturating_sub(duration.saturating_mul(duration) / BINDING_POWER_D); + + U256::from(value_term.saturating_mul(duration_term)) +} + +/// Compute total binding power for an app by summing across all user binds +fn compute_app_total_binding_power(db: &DB, namehash: &str, now: u64) -> anyhow::Result { + let binds = db.get_all_binds_for_app(namehash)?; + print_to_terminal( + 2, + &format!( + "[DEBUG BINDING] compute_app_total_binding_power:\n query namehash: {}\n binds found: {}\n now: {}", + namehash, binds.len(), now + ), + ); + for bind in &binds { + print_to_terminal( + 2, + &format!( + " bind: user={}, amount={}, end_time={}", + bind.user_address, bind.amount, bind.end_time + ), + ); + } + let mut total = U256::ZERO; + + for bind in binds { + if let Some(lock) = db.get_user_lock(&bind.user_address)? { + // effective_duration = min(lock_end_time, bind_end_time) - now + let effective_end = std::cmp::min(lock.end_time, bind.end_time); + if effective_end > now { + let remaining = effective_end - now; + let amount = U256::from_str(&bind.amount).unwrap_or(U256::ZERO); + total += compute_binding_power(amount, remaining); + } + } + } + + Ok(total) +} + /// create a filter to fetch app store event logs from chain and subscribe to new events pub fn fetch_and_subscribe_logs(our: &Address, state: &mut State, last_saved_block: u64) { let filter = app_store_filter(state); @@ -887,6 +1420,53 @@ pub fn fetch_and_subscribe_logs(our: &Address, state: &mut State, last_saved_blo // update metadata for any noncached elements update_all_metadata(state, block_from_cache); + + // Now handle bindings subscription and bootstrap + let bindings_fltr = bindings_filter(&state.bindings); + let _ = state.bindings.provider.unsubscribe(BINDINGS_SUBSCRIPTION); + state + .bindings + .provider + .subscribe_loop(BINDINGS_SUBSCRIPTION, bindings_fltr.clone(), 1, 0); + + // Bootstrap bindings from cacher + println!( + "bootstrapping bindings from block {}", + state.last_bindings_block + ); + match state + .bindings + .get_bootstrap(Some(state.last_bindings_block), Some((5, None)), None) + { + Err(e) => println!("bindings bootstrap from cache failed: {e:?}"), + Ok((block, logs)) => { + for log in logs { + if let Err(e) = handle_binding_log(state, log) { + print_to_terminal(1, &format!("error ingesting binding log: {e}")); + } + } + if block > state.last_bindings_block { + state.last_bindings_block = block; + if let Err(e) = state.db.set_last_bindings_block(block) { + print_to_terminal(0, &format!("error saving bindings block: {e}")); + } + } + } + } + + // Fetch remaining binding logs via RPC + for log in fetch_logs( + &state.bindings.provider, + &bindings_fltr.from_block(state.last_bindings_block), + ) { + if let Err(e) = handle_binding_log(state, log) { + print_to_terminal(1, &format!("error ingesting binding log: {e}")); + } + } + println!( + "bindings bootstrap complete, last block: {}", + state.last_bindings_block + ); } /// fetch logs from the chain with a given filter @@ -954,7 +1534,11 @@ impl crate::hyperware::process::main::PackageId { } impl PackageListing { - pub fn to_onchain_app(&self, package_id: &PackageId) -> OnchainApp { + pub fn to_onchain_app( + &self, + package_id: &PackageId, + binding_power: Option, + ) -> OnchainApp { OnchainApp { package_id: crate::hyperware::process::main::PackageId::from_process_lib( package_id.clone(), @@ -964,6 +1548,7 @@ impl PackageListing { metadata_hash: self.metadata_hash.clone(), metadata: self.metadata.as_ref().map(|m| m.clone().into()), auto_update: self.auto_update, + binding_power, } } } diff --git a/hyperdrive/packages/app-store/download/src/lib.rs b/hyperdrive/packages/app-store/download/src/lib.rs index 011658491..d7d15403a 100644 --- a/hyperdrive/packages/app-store/download/src/lib.rs +++ b/hyperdrive/packages/app-store/download/src/lib.rs @@ -20,7 +20,7 @@ use hyperware_process_lib::{ wit_bindgen::generate!({ path: "../target/wit", generate_unused_types: true, - world: "app-store-sys-v1", + world: "app-store-sys-v2", additional_derives: [PartialEq, serde::Deserialize, serde::Serialize, process_macros::SerdeJsonInto], }); diff --git a/hyperdrive/packages/app-store/downloads/src/lib.rs b/hyperdrive/packages/app-store/downloads/src/lib.rs index f42e9e98a..044def476 100644 --- a/hyperdrive/packages/app-store/downloads/src/lib.rs +++ b/hyperdrive/packages/app-store/downloads/src/lib.rs @@ -68,7 +68,7 @@ use std::{ wit_bindgen::generate!({ path: "../target/wit", generate_unused_types: true, - world: "app-store-sys-v1", + world: "app-store-sys-v2", additional_derives: [serde::Deserialize, serde::Serialize, process_macros::SerdeJsonInto], }); diff --git a/hyperdrive/packages/app-store/ft-worker/src/lib.rs b/hyperdrive/packages/app-store/ft-worker/src/lib.rs index a049d0bbd..712cdb20d 100644 --- a/hyperdrive/packages/app-store/ft-worker/src/lib.rs +++ b/hyperdrive/packages/app-store/ft-worker/src/lib.rs @@ -57,7 +57,7 @@ pub mod ft_worker_lib; wit_bindgen::generate!({ path: "../target/wit", generate_unused_types: true, - world: "app-store-sys-v1", + world: "app-store-sys-v2", additional_derives: [serde::Deserialize, serde::Serialize, process_macros::SerdeJsonInto], }); diff --git a/hyperdrive/packages/app-store/install/src/lib.rs b/hyperdrive/packages/app-store/install/src/lib.rs index 3ea18f723..508946a92 100644 --- a/hyperdrive/packages/app-store/install/src/lib.rs +++ b/hyperdrive/packages/app-store/install/src/lib.rs @@ -18,7 +18,7 @@ use hyperware_process_lib::{ wit_bindgen::generate!({ path: "../target/wit", generate_unused_types: true, - world: "app-store-sys-v1", + world: "app-store-sys-v2", additional_derives: [PartialEq, serde::Deserialize, serde::Serialize, process_macros::SerdeJsonInto], }); diff --git a/hyperdrive/packages/app-store/pkg/manifest.json b/hyperdrive/packages/app-store/pkg/manifest.json index a2577f509..3242d94db 100644 --- a/hyperdrive/packages/app-store/pkg/manifest.json +++ b/hyperdrive/packages/app-store/pkg/manifest.json @@ -30,6 +30,7 @@ "on_exit": "Restart", "request_networking": false, "request_capabilities": [ + "binding-cacher:hypermap-cacher:sys", "downloads:app-store:sys", "eth:distro:sys", "hns-indexer:hns-indexer:sys", @@ -47,6 +48,7 @@ } ], "grant_capabilities": [ + "binding-cacher:hypermap-cacher:sys", "eth:distro:sys", "hns-indexer:hns-indexer:sys", "http-client:distro:sys", diff --git a/hyperdrive/packages/app-store/public-ui/src/components/AppDetail.tsx b/hyperdrive/packages/app-store/public-ui/src/components/AppDetail.tsx index cf58792e4..449e36e74 100644 --- a/hyperdrive/packages/app-store/public-ui/src/components/AppDetail.tsx +++ b/hyperdrive/packages/app-store/public-ui/src/components/AppDetail.tsx @@ -185,7 +185,7 @@ export default function AppDetail() {
To use this app: Get a node diff --git a/hyperdrive/packages/app-store/reset-store/src/lib.rs b/hyperdrive/packages/app-store/reset-store/src/lib.rs index 9889c18ea..03b59c666 100644 --- a/hyperdrive/packages/app-store/reset-store/src/lib.rs +++ b/hyperdrive/packages/app-store/reset-store/src/lib.rs @@ -13,7 +13,7 @@ use hyperware_process_lib::{call_init, println, Address, Message, Request}; wit_bindgen::generate!({ path: "../target/wit", generate_unused_types: true, - world: "app-store-sys-v1", + world: "app-store-sys-v2", additional_derives: [PartialEq, serde::Deserialize, serde::Serialize, process_macros::SerdeJsonInto], }); diff --git a/hyperdrive/packages/app-store/ui/src/types/Apps.ts b/hyperdrive/packages/app-store/ui/src/types/Apps.ts index f360c51b5..6fbd15831 100644 --- a/hyperdrive/packages/app-store/ui/src/types/Apps.ts +++ b/hyperdrive/packages/app-store/ui/src/types/Apps.ts @@ -13,6 +13,7 @@ export interface AppListing { metadata_hash: string metadata?: OnchainPackageMetadata auto_update: boolean + binding_power?: string // Total binding power (U256 as string) } export type DownloadItem = { diff --git a/hyperdrive/packages/app-store/uninstall/src/lib.rs b/hyperdrive/packages/app-store/uninstall/src/lib.rs index 3d75a85a0..cc49ff5e4 100644 --- a/hyperdrive/packages/app-store/uninstall/src/lib.rs +++ b/hyperdrive/packages/app-store/uninstall/src/lib.rs @@ -15,7 +15,7 @@ use hyperware_process_lib::{ wit_bindgen::generate!({ path: "../target/wit", generate_unused_types: true, - world: "app-store-sys-v1", + world: "app-store-sys-v2", additional_derives: [PartialEq, serde::Deserialize, serde::Serialize, process_macros::SerdeJsonInto], }); diff --git a/hyperdrive/packages/file-explorer/explorer/Cargo.toml b/hyperdrive/packages/file-explorer/explorer/Cargo.toml index 884c9deeb..33fefce8f 100644 --- a/hyperdrive/packages/file-explorer/explorer/Cargo.toml +++ b/hyperdrive/packages/file-explorer/explorer/Cargo.toml @@ -1,20 +1,17 @@ [dependencies] anyhow = "1.0" md5 = "0.7" +hyperapp_macro = "0.1.1" process_macros = "0.1" serde_json = "1.0" serde_urlencoded = "0.7" tracing = "0.1.37" wit-bindgen = "0.42.1" -[dependencies.hyperprocess_macro] -git = "https://github.com/hyperware-ai/hyperprocess-macro" -rev = "66884c0" - [dependencies.hyperware_process_lib] features = ["hyperapp"] git = "https://github.com/hyperware-ai/process_lib" -rev = "4beff93" +rev = "1a6ad9d" [dependencies.serde] features = ["derive"] diff --git a/hyperdrive/packages/file-explorer/explorer/src/lib.rs b/hyperdrive/packages/file-explorer/explorer/src/lib.rs index 482cd9432..cbcfb5411 100644 --- a/hyperdrive/packages/file-explorer/explorer/src/lib.rs +++ b/hyperdrive/packages/file-explorer/explorer/src/lib.rs @@ -1,4 +1,3 @@ -use hyperprocess_macro::hyperprocess; use hyperware_process_lib::hyperapp::{add_response_header, get_path, send, SaveOptions}; use hyperware_process_lib::logging::{debug, error, info, init_logging, Level}; use hyperware_process_lib::our; @@ -35,7 +34,7 @@ struct FileExplorerState { cwd: String, } -#[hyperprocess( +#[hyperapp_macro::hyperapp( name = "file-explorer", ui = Some(HttpBindingConfig::default().secure_subdomain(true)), endpoints = vec![ diff --git a/hyperdrive/packages/hns-indexer/Cargo.lock b/hyperdrive/packages/hns-indexer/Cargo.lock index 5b898a0cf..111a29fbf 100644 --- a/hyperdrive/packages/hns-indexer/Cargo.lock +++ b/hyperdrive/packages/hns-indexer/Cargo.lock @@ -1379,7 +1379,7 @@ dependencies = [ name = "get-block" version = "0.1.0" dependencies = [ - "hyperware_process_lib", + "hyperware_process_lib 2.1.0", "serde", "serde_json", "wit-bindgen", @@ -1487,7 +1487,7 @@ dependencies = [ "alloy-sol-types", "anyhow", "hex", - "hyperware_process_lib", + "hyperware_process_lib 2.2.1", "process_macros", "rmp-serde", "serde", @@ -1603,7 +1603,6 @@ dependencies = [ "anyhow", "base64", "bincode", - "color-eyre", "hex", "http", "mime_guess", @@ -1614,6 +1613,31 @@ dependencies = [ "serde_json", "sha3", "thiserror 1.0.69", + "url", + "wit-bindgen", +] + +[[package]] +name = "hyperware_process_lib" +version = "2.2.1" +source = "git+https://github.com/hyperware-ai/process_lib?rev=4e91521#4e915218b382e4b3f7f93cddeec30415f02a504b" +dependencies = [ + "alloy", + "alloy-primitives", + "alloy-sol-macro", + "alloy-sol-types", + "anyhow", + "base64", + "bincode", + "color-eyre", + "http", + "mime_guess", + "rand", + "regex", + "rmp-serde", + "serde", + "serde_json", + "thiserror 1.0.69", "tracing", "tracing-error", "tracing-subscriber", @@ -2008,7 +2032,7 @@ dependencies = [ name = "node-info" version = "0.1.0" dependencies = [ - "hyperware_process_lib", + "hyperware_process_lib 2.1.0", "process_macros", "serde", "serde_json", @@ -2545,7 +2569,7 @@ dependencies = [ name = "reset" version = "0.1.0" dependencies = [ - "hyperware_process_lib", + "hyperware_process_lib 2.1.0", "process_macros", "serde", "serde_json", @@ -2949,7 +2973,7 @@ checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" name = "state" version = "0.1.0" dependencies = [ - "hyperware_process_lib", + "hyperware_process_lib 2.1.0", "process_macros", "serde", "serde_json", diff --git a/hyperdrive/packages/hns-indexer/hns-indexer/Cargo.toml b/hyperdrive/packages/hns-indexer/hns-indexer/Cargo.toml index 105d04b79..1d4c67035 100644 --- a/hyperdrive/packages/hns-indexer/hns-indexer/Cargo.toml +++ b/hyperdrive/packages/hns-indexer/hns-indexer/Cargo.toml @@ -11,7 +11,7 @@ anyhow = "1.0" alloy-primitives = "0.8.15" alloy-sol-types = "0.8.15" hex = "0.4.3" -hyperware_process_lib = { version = "2.1.0", features = ["logging"] } +hyperware_process_lib = { git = "https://github.com/hyperware-ai/process_lib", rev = "4e91521", features = ["logging"] } process_macros = "0.1" rmp-serde = "1.1.2" serde = { version = "1.0", features = ["derive"] } diff --git a/hyperdrive/packages/homepage/ui/src/components/Home/index.tsx b/hyperdrive/packages/homepage/ui/src/components/Home/index.tsx index 1e94a16de..e4dca47b8 100644 --- a/hyperdrive/packages/homepage/ui/src/components/Home/index.tsx +++ b/hyperdrive/packages/homepage/ui/src/components/Home/index.tsx @@ -37,14 +37,16 @@ export default function Home() { // try to automatically open it useEffect(() => { if (window?.location?.hash?.startsWith('#app-')) { - // could be a path on the hash, or a query. delete them - let appNameToOpen = window.location.hash.replace('#app-', '') - .replace(/\?.*/, '') - .replace(/\/.*/, ''); + const hashWithoutPrefix = window.location.hash.replace('#app-', ''); + // Extract app name (everything before first / or ?) + const appNameMatch = hashWithoutPrefix.match(/^([^/?]+)/); + const appNameToOpen = appNameMatch ? appNameMatch[1] : ''; + // Capture path/query (everything after the app name) + const remainder = hashWithoutPrefix.slice(appNameToOpen.length); const appToOpen = apps?.find(app => app?.id === appNameToOpen); - console.log('found window hash. attempting open', { hash: window.location.hash, appNameToOpen, appToOpen }); + console.log('found window hash. attempting open', { hash: window.location.hash, appNameToOpen, remainder, appToOpen }); if (appToOpen) { - openApp(appToOpen) + openApp(appToOpen, remainder || undefined) } } }, [apps, window.location]); diff --git a/hyperdrive/packages/homepage/ui/src/stores/navigationStore.ts b/hyperdrive/packages/homepage/ui/src/stores/navigationStore.ts index e0af3bb7e..aed28a198 100644 --- a/hyperdrive/packages/homepage/ui/src/stores/navigationStore.ts +++ b/hyperdrive/packages/homepage/ui/src/stores/navigationStore.ts @@ -76,12 +76,18 @@ export const useNavigationStore = create((set, get) => ({ return; } + // Normalize query to avoid double slashes + let normalizedQuery = query || ''; + if (normalizedQuery.startsWith('/') && app.path.endsWith('/')) { + normalizedQuery = normalizedQuery.slice(1); + } + // Check if we need to open in a new tab (localhost) const isLocalhost = window.location.host.includes("localhost"); if (isLocalhost) { console.log('[homepage] opening app in new tab:', { app }); // don't use secure subdomain for localhost - const path = app.path.replace(/^(https?:\/\/)(.*)localhost/, '$1localhost') + (query || ''); + const path = app.path.replace(/^(https?:\/\/)(.*)localhost/, '$1localhost') + normalizedQuery; console.log({ path }) window.open(path, '_blank'); set({ isAppDrawerOpen: false, isRecentAppsOpen: false }); @@ -100,7 +106,7 @@ export const useNavigationStore = create((set, get) => ({ let maybeSlash = ''; - if (query && query[0] && query[0] !== '?' && query[0] !== '/') { + if (normalizedQuery && normalizedQuery[0] !== '?' && normalizedQuery[0] !== '/') { // autoprepend a slash for the window history when the query type is unknown console.log('autoprepended / to unknown query format'); maybeSlash = '/' @@ -110,13 +116,13 @@ export const useNavigationStore = create((set, get) => ({ window?.history?.pushState( { type: 'app', appId: app.id, previousAppId: currentAppId }, '', - `#app-${app.id}${maybeSlash}${query || ''}` + `#app-${app.id}${maybeSlash}${normalizedQuery}` ); if (existingApp) { set({ runningApps: runningApps.map(rApp => { - const path = `${app.path}${query || ''}`; + const path = `${app.path}${normalizedQuery}`; console.log(path, rApp.id, app.id); if (rApp.id === app.id) { console.log('found rApp') @@ -135,7 +141,7 @@ export const useNavigationStore = create((set, get) => ({ set({ runningApps: [...runningApps, { ...app, - path: `${app.path}${query || ''}`, + path: `${app.path}${normalizedQuery}`, openedAt: Date.now() }], currentAppId: app.id, diff --git a/hyperdrive/packages/hypermap-cacher/Cargo.lock b/hyperdrive/packages/hypermap-cacher/Cargo.lock index 4806c43a8..04499b149 100644 --- a/hyperdrive/packages/hypermap-cacher/Cargo.lock +++ b/hyperdrive/packages/hypermap-cacher/Cargo.lock @@ -817,6 +817,25 @@ dependencies = [ "serde", ] +[[package]] +name = "binding-cacher" +version = "0.1.0" +dependencies = [ + "alloy", + "alloy-primitives", + "alloy-sol-types", + "anyhow", + "chrono", + "hex", + "hyperware_process_lib", + "process_macros", + "rand 0.8.5", + "rmp-serde", + "serde", + "serde_json", + "wit-bindgen", +] + [[package]] name = "bit-set" version = "0.8.0" @@ -1661,9 +1680,9 @@ dependencies = [ [[package]] name = "hyperware_process_lib" -version = "2.1.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f3abd008d22c3b96ee43300c4c8dffbf1d072a680a13635b5f9da11a0ce9395" +checksum = "abe2b6795488c3f7d3410808cf7e29404725f862ad42a3822886464b92451a21" dependencies = [ "alloy", "alloy-primitives", @@ -1673,7 +1692,6 @@ dependencies = [ "base64", "bincode", "color-eyre", - "hex", "http", "mime_guess", "rand 0.8.5", @@ -1681,7 +1699,6 @@ dependencies = [ "rmp-serde", "serde", "serde_json", - "sha3", "thiserror 1.0.69", "tracing", "tracing-error", diff --git a/hyperdrive/packages/hypermap-cacher/Cargo.toml b/hyperdrive/packages/hypermap-cacher/Cargo.toml index ff6b64639..ff4a19f1f 100644 --- a/hyperdrive/packages/hypermap-cacher/Cargo.toml +++ b/hyperdrive/packages/hypermap-cacher/Cargo.toml @@ -1,6 +1,7 @@ [workspace] resolver = "2" members = [ + "binding-cacher", "hypermap-cacher", "reset-cache", "set-nodes", diff --git a/hyperdrive/packages/hypermap-cacher/api/hypermap-cacher:sys-v0.wit b/hyperdrive/packages/hypermap-cacher/api/hypermap-cacher:sys-v0.wit deleted file mode 100644 index 21a3652ae..000000000 --- a/hyperdrive/packages/hypermap-cacher/api/hypermap-cacher:sys-v0.wit +++ /dev/null @@ -1,83 +0,0 @@ -interface hypermap-cacher { - // Metadata associated with a batch of Ethereum logs. - record logs-metadata { - chain-id: string, - from-block: string, - to-block: string, - time-created: string, - created-by: string, - signature: string, - } - - // Represents an item in the manifest, detailing a single log cache file. - record manifest-item { - metadata: logs-metadata, - is-empty: bool, - file-hash: string, - file-name: string, - } - - // The main manifest structure, listing all available log cache files. - // WIT does not support direct map types, so a list of key-value tuples is used. - record manifest { - // The key is the filename of the log cache. - items: list>, - manifest-filename: string, - chain-id: string, - protocol-version: string, - } - - record get-logs-by-range-request { - from-block: u64, - to-block: option, // If None, signifies to the latest available/relevant cached block. - } - - variant get-logs-by-range-ok-response { - logs(tuple), - latest(u64), - } - - // Defines the types of requests that can be sent to the Hypermap Cacher process. - variant cacher-request { - get-manifest, - get-log-cache-content(string), - get-status, - get-logs-by-range(get-logs-by-range-request), - start-providing, - stop-providing, - set-nodes(list), - reset(option>), - } - - // Represents the operational status of the cacher. - record cacher-status { - last-cached-block: u64, - chain-id: string, - protocol-version: string, - next-cache-attempt-in-seconds: option, - manifest-filename: string, - log-files-count: u32, - our-address: string, - is-providing: bool, - } - - // Defines the types of responses the Hypermap Cacher process can send. - variant cacher-response { - get-manifest(option), - get-log-cache-content(result, string>), - get-status(cacher-status), - get-logs-by-range(result), - start-providing(result), - stop-providing(result), - set-nodes(result), - reset(result), - rejected, - is-starting, - } -} - -world hypermap-cacher-sys-v0 { - import sign; - import hypermap-cacher; - include process-v1; -} diff --git a/hyperdrive/packages/hypermap-cacher/api/hypermap-cacher:sys-v1.wit b/hyperdrive/packages/hypermap-cacher/api/hypermap-cacher:sys-v1.wit new file mode 100644 index 000000000..472e9b6e2 --- /dev/null +++ b/hyperdrive/packages/hypermap-cacher/api/hypermap-cacher:sys-v1.wit @@ -0,0 +1,162 @@ +interface binding-cacher { + // Metadata associated with a batch of Ethereum logs. + record binding-logs-metadata { + chain-id: string, + from-block: string, + to-block: string, + time-created: string, + created-by: string, + signature: string, + } + + // Represents an item in the manifest, detailing a single log cache file. + record binding-manifest-item { + metadata: binding-logs-metadata, + is-empty: bool, + file-hash: string, + file-name: string, + } + + // The main manifest structure, listing all available log cache files. + // WIT does not support direct map types, so a list of key-value tuples is used. + record binding-manifest { + // The key is the filename of the log cache. + items: list>, + manifest-filename: string, + chain-id: string, + protocol-version: string, + } + + record binding-get-logs-by-range-request { + from-block: u64, + to-block: option, // If None, signifies to the latest available/relevant cached block. + } + + variant binding-get-logs-by-range-ok-response { + logs(tuple), + latest(u64), + } + + // Defines the types of requests that can be sent to the Hypermap Cacher process. + variant binding-cacher-request { + get-manifest, + get-log-cache-content(string), + get-status, + get-logs-by-range(binding-get-logs-by-range-request), + start-providing, + stop-providing, + set-nodes(list), + reset(option>), + } + + // Represents the operational status of the cacher. + record binding-cacher-status { + last-cached-block: u64, + chain-id: string, + protocol-version: string, + next-cache-attempt-in-seconds: option, + manifest-filename: string, + log-files-count: u32, + our-address: string, + is-providing: bool, + } + + // Defines the types of responses the Hypermap Cacher process can send. + variant binding-cacher-response { + get-manifest(option), + get-log-cache-content(result, string>), + get-status(binding-cacher-status), + get-logs-by-range(result), + start-providing(result), + stop-providing(result), + set-nodes(result), + reset(result), + rejected, + is-starting, + } +} + +interface hypermap-cacher { + // Metadata associated with a batch of Ethereum logs. + record logs-metadata { + chain-id: string, + from-block: string, + to-block: string, + time-created: string, + created-by: string, + signature: string, + } + + // Represents an item in the manifest, detailing a single log cache file. + record manifest-item { + metadata: logs-metadata, + is-empty: bool, + file-hash: string, + file-name: string, + } + + // The main manifest structure, listing all available log cache files. + // WIT does not support direct map types, so a list of key-value tuples is used. + record manifest { + // The key is the filename of the log cache. + items: list>, + manifest-filename: string, + chain-id: string, + protocol-version: string, + } + + record get-logs-by-range-request { + from-block: u64, + to-block: option, // If None, signifies to the latest available/relevant cached block. + } + + variant get-logs-by-range-ok-response { + logs(tuple), + latest(u64), + } + + // Defines the types of requests that can be sent to the Hypermap Cacher process. + variant cacher-request { + get-manifest, + get-log-cache-content(string), + get-status, + get-logs-by-range(get-logs-by-range-request), + start-providing, + stop-providing, + set-nodes(list), + reset(option>), + } + + // Represents the operational status of the cacher. + record cacher-status { + last-cached-block: u64, + chain-id: string, + protocol-version: string, + next-cache-attempt-in-seconds: option, + manifest-filename: string, + log-files-count: u32, + our-address: string, + is-providing: bool, + } + + // Defines the types of responses the Hypermap Cacher process can send. + variant cacher-response { + get-manifest(option), + get-log-cache-content(result, string>), + get-status(cacher-status), + get-logs-by-range(result), + start-providing(result), + stop-providing(result), + set-nodes(result), + reset(result), + rejected, + is-starting, + } +} + +world hypermap-cacher-sys-v1 { + import sign; + import binding-cacher; + import hypermap-cacher; + include process-v1; +} diff --git a/hyperdrive/packages/hypermap-cacher/binding-cacher/Cargo.toml b/hyperdrive/packages/hypermap-cacher/binding-cacher/Cargo.toml new file mode 100644 index 000000000..e0504d478 --- /dev/null +++ b/hyperdrive/packages/hypermap-cacher/binding-cacher/Cargo.toml @@ -0,0 +1,33 @@ +[package] +name = "binding-cacher" +version = "0.1.0" +edition = "2021" +publish = false + +[features] +simulation-mode = ["hyperware_process_lib/simulation-mode"] + +[dependencies] +anyhow = "1.0" +alloy-primitives = "0.8.15" +alloy-sol-types = "0.8.15" +alloy = { version = "0.8.1", features = [ + "json-rpc", + "rpc-client", + "rpc-types", +] } +chrono = "0.4.41" +hex = "0.4.3" +hyperware_process_lib = { version = "2.3.0", features = ["logging"] } +process_macros = "0.1.0" +rand = "0.8" +rmp-serde = "1.1.2" +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +wit-bindgen = "0.42.1" + +[lib] +crate-type = ["cdylib"] + +[package.metadata.component] +package = "hyperware:process" diff --git a/hyperdrive/packages/hypermap-cacher/binding-cacher/src/lib.rs b/hyperdrive/packages/hypermap-cacher/binding-cacher/src/lib.rs new file mode 100644 index 000000000..497dc3306 --- /dev/null +++ b/hyperdrive/packages/hypermap-cacher/binding-cacher/src/lib.rs @@ -0,0 +1,1514 @@ +use std::{ + cmp::{max, min}, + collections::HashMap, + str::FromStr, +}; + +use alloy::hex; +use alloy_primitives::keccak256; +use rand::seq::SliceRandom; +use rand::thread_rng; +use serde::{Deserialize, Serialize}; + +use crate::hyperware::process::binding_cacher::{ + BindingCacherRequest as CacherRequest, BindingCacherResponse as CacherResponse, + BindingCacherStatus as CacherStatus, + BindingGetLogsByRangeOkResponse as GetLogsByRangeOkResponse, + BindingGetLogsByRangeRequest as GetLogsByRangeRequest, BindingLogsMetadata as WitLogsMetadata, + BindingManifest as WitManifest, BindingManifestItem as WitManifestItem, +}; +use hyperware_process_lib::{ + await_message, bindings, call_init, eth, get_state, http, hypermap, + logging::{debug, error, info, init_logging, warn, Level}, + net::{NetAction, NetResponse}, + our, set_state, sign, timer, vfs, Address, ProcessId, Request, Response, +}; +use hyperware_process_lib::{wait_for_process_ready, WaitClassification}; + +wit_bindgen::generate!({ + path: "../target/wit", + world: "hypermap-cacher-sys-v1", + generate_unused_types: true, + additional_derives: [serde::Deserialize, serde::Serialize, process_macros::SerdeJsonInto], +}); + +const PROTOCOL_VERSION: &str = "0"; +const DEFAULT_BLOCK_BATCH_SIZE: u64 = 10; +const DEFAULT_CACHE_INTERVAL_S: u64 = 3_600; // 2s / block -> 1hr ~ 1800 blocks +const MAX_LOG_RETRIES: u8 = 3; +const RETRY_DELAY_S: u64 = 10; +const LOG_ITERATION_DELAY_MS: u64 = 200; + +#[cfg(not(feature = "simulation-mode"))] +const DEFAULT_NODES: &[&str] = &[ + "us-cacher-1.hypr", + "eu-cacher-1.hypr", + "nick.hypr", + "nick1udwig.os", +]; +#[cfg(feature = "simulation-mode")] +const DEFAULT_NODES: &[&str] = &["fake.os"]; + +// Internal representation of LogsMetadata, similar to WIT but for Rust logic. +#[derive(Serialize, Deserialize, Debug, Clone)] +struct LogsMetadataInternal { + #[serde(rename = "chainId")] + chain_id: String, + #[serde(rename = "fromBlock")] + from_block: String, + #[serde(rename = "toBlock")] + to_block: String, + #[serde(rename = "timeCreated")] + time_created: String, + #[serde(rename = "createdBy")] + created_by: String, + signature: String, // Keccak256 hash of the log file content. +} + +// Internal representation of a LogCache, containing metadata and actual logs. +#[derive(Serialize, Deserialize, Debug, Clone)] +struct LogCacheInternal { + metadata: LogsMetadataInternal, + logs: Vec, // The actual Ethereum logs. +} + +// Internal representation of a ManifestItem. +#[derive(Serialize, Deserialize, Debug, Clone)] +struct ManifestItemInternal { + metadata: LogsMetadataInternal, + #[serde(rename = "isEmpty")] + is_empty: bool, + #[serde(rename = "fileHash")] + file_hash: String, + #[serde(rename = "fileName")] + file_name: String, +} + +// Internal representation of the Manifest. +#[derive(Serialize, Deserialize, Debug, Clone, Default)] +struct ManifestInternal { + items: HashMap, + manifest_filename: String, + chain_id: String, + protocol_version: String, +} + +// The main state structure for the Hypermap Binding Cacher process. +#[derive(Serialize, Deserialize, Debug)] +struct State { + hypermap_binding_address: eth::Address, + manifest: ManifestInternal, + last_cached_block: u64, + chain_id: String, + protocol_version: String, + cache_interval_s: u64, + block_batch_size: u64, + is_cache_timer_live: bool, + drive_path: String, + is_providing: bool, + nodes: Vec, + #[serde(skip)] + is_starting: bool, +} + +// Generates a timestamp string. +fn get_current_timestamp_str() -> String { + let datetime = chrono::Utc::now(); + datetime.format("%Y%m%dT%H%M%SZ").to_string() +} + +fn is_local_request(our: &Address, source: &Address) -> bool { + our.node == source.node +} + +impl State { + fn new(drive_path: &str) -> Self { + let chain_id = bindings::BINDINGS_CHAIN_ID.to_string(); + let hypermap_binding_address = eth::Address::from_str(bindings::BINDINGS_ADDRESS) + .expect("Failed to parse BINDINGS_ADDRESS"); + + let manifest_filename = format!( + "manifest-chain{}-protocol{}.json", + chain_id, PROTOCOL_VERSION + ); + let initial_manifest = ManifestInternal { + items: HashMap::new(), + manifest_filename: manifest_filename.clone(), + chain_id: chain_id.clone(), + protocol_version: PROTOCOL_VERSION.to_string(), + }; + + State { + hypermap_binding_address, + manifest: initial_manifest, + last_cached_block: bindings::BINDINGS_FIRST_BLOCK, + chain_id, + protocol_version: PROTOCOL_VERSION.to_string(), + cache_interval_s: DEFAULT_CACHE_INTERVAL_S, + block_batch_size: 0, // Will be determined dynamically + is_cache_timer_live: false, + drive_path: drive_path.to_string(), + is_providing: false, + nodes: DEFAULT_NODES.iter().map(|s| s.to_string()).collect(), + is_starting: true, + } + } + + fn load(drive_path: &str) -> Self { + match get_state() { + Some(state_bytes) => match serde_json::from_slice::(&state_bytes) { + Ok(mut loaded_state) => { + info!("Successfully loaded state from checkpoint."); + // Always start in starting mode to bootstrap from other nodes + // is_starting is not serialized, so it defaults to false and we set it to true + loaded_state.is_starting = true; + loaded_state.drive_path = drive_path.to_string(); + + // Validate state against manifest file on disk + if let Err(e) = loaded_state.validate_state_against_manifest() { + warn!("State validation failed: {:?}. Clearing drive and creating fresh state.", e); + if let Err(clear_err) = loaded_state.clear_drive() { + error!("Failed to clear drive: {:?}", clear_err); + } + return Self::new(drive_path); + } + + loaded_state + } + Err(e) => { + warn!( + "Failed to deserialize saved state: {:?}. Creating new state.", + e + ); + Self::new(drive_path) + } + }, + None => { + info!("No saved state found. Creating new state."); + Self::new(drive_path) + } + } + } + + fn save(&self) { + match serde_json::to_vec(self) { + Ok(state_bytes) => set_state(&state_bytes), + Err(e) => error!("Fatal: Failed to serialize state for saving: {:?}", e), + } + info!( + "State checkpoint saved. Last cached block: {}", + self.last_cached_block + ); + } + + // Core logic for fetching logs, creating cache files, and updating the manifest. + fn cache_logs_and_update_manifest(&mut self, provider: ð::Provider) -> anyhow::Result<()> { + // Ensure batch size is determined + if self.block_batch_size == 0 { + self.determine_batch_size(provider)?; + } + + let current_chain_head = match provider.get_block_number() { + Ok(block_num) => block_num, + Err(e) => { + error!( + "Failed to get current block number: {:?}. Skipping cycle.", + e + ); + return Err(anyhow::anyhow!("Failed to get block number: {:?}", e)); + } + }; + + if self.last_cached_block >= current_chain_head { + info!( + "Already caught up to chain head ({}). Nothing to cache.", + current_chain_head + ); + return Ok(()); + } + + while self.last_cached_block != current_chain_head { + self.cache_logs_and_update_manifest_step(provider, Some(current_chain_head))?; + + std::thread::sleep(std::time::Duration::from_millis(LOG_ITERATION_DELAY_MS)); + } + + Ok(()) + } + + fn cache_logs_and_update_manifest_step( + &mut self, + provider: ð::Provider, + to_block: Option, + ) -> anyhow::Result<()> { + info!( + "Starting caching cycle. From block: {}", + self.last_cached_block + 1 + ); + + let current_chain_head = match to_block { + Some(b) => b, + None => match provider.get_block_number() { + Ok(block_num) => block_num, + Err(e) => { + error!( + "Failed to get current block number: {:?}. Skipping cycle.", + e + ); + return Err(anyhow::anyhow!("Failed to get block number: {:?}", e)); + } + }, + }; + + if self.last_cached_block >= current_chain_head { + info!( + "Already caught up to chain head ({}). Nothing to cache.", + current_chain_head + ); + return Ok(()); + } + + let from_block = self.last_cached_block + 1; + let mut to_block = from_block + self.block_batch_size - 1; + if to_block > current_chain_head { + to_block = current_chain_head; + } + + if from_block > to_block { + info!("From_block {} is greater than to_block {}. Chain might not have advanced enough. Skipping.", from_block, to_block); + return Ok(()); + } + + let filter = eth::Filter::new() + .address(self.hypermap_binding_address) + .from_block(from_block) + .to_block(eth::BlockNumberOrTag::Number(to_block)); + + let logs = { + let mut attempt = 0; + loop { + match provider.get_logs(&filter) { + Ok(logs) => break logs, + Err(e) => { + attempt += 1; + if attempt >= MAX_LOG_RETRIES { + error!( + "Failed to get logs after {} retries: {:?}", + MAX_LOG_RETRIES, e + ); + return Err(anyhow::anyhow!("Failed to get logs: {:?}", e)); + } + warn!( + "Error getting logs (attempt {}/{}): {:?}. Retrying in {}s...", + attempt, MAX_LOG_RETRIES, e, RETRY_DELAY_S + ); + std::thread::sleep(std::time::Duration::from_secs(RETRY_DELAY_S)); + } + } + } + }; + + info!( + "Fetched {} logs from block {} to {}.", + logs.len(), + from_block, + to_block + ); + + let our = our(); + + let metadata = LogsMetadataInternal { + chain_id: self.chain_id.clone(), + from_block: from_block.to_string(), + to_block: to_block.to_string(), + time_created: get_current_timestamp_str(), + created_by: our.to_string(), + signature: "".to_string(), + }; + + let mut log_cache = LogCacheInternal { + metadata, + logs: logs.clone(), + }; + + let mut logs_bytes_for_sig = serde_json::to_vec(&log_cache.logs).unwrap_or_default(); + logs_bytes_for_sig.extend_from_slice(&from_block.to_be_bytes()); + logs_bytes_for_sig.extend_from_slice(&to_block.to_be_bytes()); + let logs_hash_for_sig = keccak256(&logs_bytes_for_sig); + + let signature = sign::net_key_sign(logs_hash_for_sig.to_vec())?; + + log_cache.metadata.signature = format!("0x{}", hex::encode(signature)); + + // Final serialization of LogCacheInternal with the signature. + let final_log_cache_bytes = match serde_json::to_vec(&log_cache) { + Ok(bytes) => bytes, + Err(e) => { + error!( + "Failed to re-serialize LogCacheInternal with signature: {:?}", + e + ); + return Err(e.into()); + } + }; + + let file_hash_for_manifest = + format!("0x{}", hex::encode(keccak256(&final_log_cache_bytes))); + + let log_cache_filename = format!( + "{}-chain{}-from{}-to{}-protocol{}.json", + log_cache + .metadata + .time_created + .replace(":", "") + .replace("-", ""), // Make timestamp filename-safe + self.chain_id, + from_block, + to_block, + self.protocol_version + ); + + if !logs.is_empty() { + let log_cache_path = format!("{}/{}", self.drive_path, log_cache_filename); + let mut log_cache_file = vfs::open_file(&log_cache_path, true, None)?; + + if let Err(e) = log_cache_file.write_all(&final_log_cache_bytes) { + error!("Failed to write log cache file {}: {:?}", log_cache_path, e); + return Err(e.into()); + } + info!("Successfully wrote log cache file: {}", log_cache_path); + } + + let manifest_item = ManifestItemInternal { + metadata: log_cache.metadata.clone(), + is_empty: logs.is_empty(), + file_hash: file_hash_for_manifest, + file_name: if logs.is_empty() { + "".to_string() + } else { + log_cache_filename.clone() + }, + }; + self.manifest + .items + .insert(log_cache_filename.clone(), manifest_item); + self.manifest.chain_id = self.chain_id.clone(); + self.manifest.protocol_version = self.protocol_version.clone(); + + let manifest_bytes = match serde_json::to_vec(&self.manifest) { + Ok(bytes) => bytes, + Err(e) => { + error!("Failed to serialize manifest: {:?}", e); + return Err(e.into()); + } + }; + + let manifest_path = format!("{}/{}", self.drive_path, self.manifest.manifest_filename); + let manifest_file = vfs::open_file(&manifest_path, true, None)?; + + if let Err(e) = manifest_file.write(&manifest_bytes) { + error!("Failed to write manifest file {}: {:?}", manifest_path, e); + return Err(e.into()); + } + info!( + "Successfully updated and wrote manifest file: {}", + manifest_path + ); + + self.last_cached_block = to_block; + self.save(); + + Ok(()) + } + + // Validate that the in-memory state matches the manifest file on disk + fn validate_state_against_manifest(&self) -> anyhow::Result<()> { + let manifest_path = format!("{}/{}", self.drive_path, self.manifest.manifest_filename); + + // Check if manifest file exists + match vfs::open_file(&manifest_path, false, None) { + Ok(manifest_file) => { + match manifest_file.read() { + Ok(disk_manifest_bytes) => { + match serde_json::from_slice::(&disk_manifest_bytes) { + Ok(disk_manifest) => { + // Compare key aspects of the manifests + if self.manifest.chain_id != disk_manifest.chain_id { + return Err(anyhow::anyhow!( + "Chain ID mismatch: state has {}, disk has {}", + self.manifest.chain_id, + disk_manifest.chain_id + )); + } + + if self.manifest.protocol_version != disk_manifest.protocol_version + { + return Err(anyhow::anyhow!( + "Protocol version mismatch: state has {}, disk has {}", + self.manifest.protocol_version, + disk_manifest.protocol_version + )); + } + + // Check if all files mentioned in state manifest exist on disk + for (_filename, item) in &self.manifest.items { + if !item.file_name.is_empty() { + let file_path = + format!("{}/{}", self.drive_path, item.file_name); + if vfs::metadata(&file_path, None).is_err() { + return Err(anyhow::anyhow!( + "File {} mentioned in state manifest does not exist on disk", + item.file_name + )); + } + } + } + + // Check if disk manifest has more recent data than our state + let disk_max_block = disk_manifest + .items + .values() + .filter_map(|item| item.metadata.to_block.parse::().ok()) + .max() + .unwrap_or(0); + + let state_max_block = self + .manifest + .items + .values() + .filter_map(|item| item.metadata.to_block.parse::().ok()) + .max() + .unwrap_or(0); + + if disk_max_block > state_max_block { + return Err(anyhow::anyhow!( + "Disk manifest has more recent data (block {}) than state (block {})", + disk_max_block, state_max_block + )); + } + + info!("State validation passed - state matches manifest file"); + Ok(()) + } + Err(e) => { + Err(anyhow::anyhow!("Failed to parse manifest file: {:?}", e)) + } + } + } + Err(e) => Err(anyhow::anyhow!("Failed to read manifest file: {:?}", e)), + } + } + Err(_) => { + // Manifest file doesn't exist - this is okay for new installs + if self.manifest.items.is_empty() { + info!("No manifest file found, but state is also empty - validation passed"); + Ok(()) + } else { + Err(anyhow::anyhow!( + "State has manifest items but no manifest file exists on disk" + )) + } + } + } + } + + // Clear all files from the drive + fn clear_drive(&self) -> anyhow::Result<()> { + info!("Clearing all files from drive: {}", self.drive_path); + + // Remove the manifest file + let manifest_path = format!("{}/{}", self.drive_path, self.manifest.manifest_filename); + match vfs::remove_file(&manifest_path, None) { + Ok(_) => info!("Removed manifest file: {}", manifest_path), + Err(e) => warn!("Failed to remove manifest file {}: {:?}", manifest_path, e), + } + + // Remove all files mentioned in the manifest + for (_, item) in &self.manifest.items { + if !item.file_name.is_empty() { + let file_path = format!("{}/{}", self.drive_path, item.file_name); + match vfs::remove_file(&file_path, None) { + Ok(_) => info!("Removed cache file: {}", file_path), + Err(e) => warn!("Failed to remove cache file {}: {:?}", file_path, e), + } + } + } + + info!("Drive clearing completed"); + Ok(()) + } + + // Bootstrap state from other nodes, then fallback to RPC + fn bootstrap_state(&mut self, provider: ð::Provider) -> anyhow::Result<()> { + info!("Starting state bootstrap process..."); + + // Try to bootstrap from other nodes first + if let Ok(()) = self.try_bootstrap_from_nodes() { + info!("Successfully bootstrapped from other nodes"); + } + + self.try_bootstrap_from_rpc(provider)?; + + // Mark as no longer starting + self.is_starting = false; + self.save(); + info!("Bootstrap process completed, cacher is now ready"); + Ok(()) + } + + // Try to bootstrap from other binding-cacher nodes + fn try_bootstrap_from_nodes(&mut self) -> anyhow::Result<()> { + // Create alternate drive for initfiles and read the test data + let alt_drive_path = vfs::create_drive(our().package_id(), "initfiles", None).unwrap(); + + // Try to read the cache_sources file from the initfiles drive + match vfs::open_file(&format!("{}/cache_sources", alt_drive_path), false, None) { + Ok(file) => { + match file.read() { + Ok(contents) => { + let content_str = String::from_utf8_lossy(&contents); + info!("Contents of cache_sources: {}", content_str); + + // Parse the JSON to get the vector of node names + match serde_json::from_str::>(&content_str) { + Ok(custom_cache_nodes) => { + if !custom_cache_nodes.is_empty() { + info!( + "Loading custom cache source nodes: {:?}", + custom_cache_nodes + ); + // Clear existing nodes and add custom ones + self.nodes.clear(); + for node_name in custom_cache_nodes { + self.nodes.push(node_name.clone()); + } + } else { + info!("Custom cache nodes list is empty, keeping existing node configuration"); + } + } + Err(e) => { + info!("Failed to parse cache_sources as JSON: {}, keeping existing node configuration", e); + } + } + } + Err(e) => { + info!( + "Failed to read cache_sources: {}, keeping existing node configuration", + e + ); + } + } + } + Err(e) => { + info!( + "Failed to open cache_sources: {}, keeping existing node configuration", + e + ); + } + } + + if self.nodes.is_empty() { + info!("No nodes configured for bootstrap, will fallback to RPC"); + return Err(anyhow::anyhow!("No nodes configured for bootstrap")); + } + + info!("Attempting to bootstrap from {} nodes", self.nodes.len()); + + let mut nodes = self.nodes.clone(); + + // If using default nodes, shuffle them for random order + let default_nodes: Vec = DEFAULT_NODES.iter().map(|s| s.to_string()).collect(); + if nodes == default_nodes { + nodes.shuffle(&mut thread_rng()); + } + + let mut nodes_not_yet_in_net = nodes.clone(); + let num_retries = 10; + for _ in 0..num_retries { + nodes_not_yet_in_net.retain(|node| { + let Ok(Ok(response)) = Request::new() + .target(("our", "net", "distro", "sys")) + .body(rmp_serde::to_vec(&NetAction::GetPeer(node.clone())).unwrap()) + .send_and_await_response(1) + else { + return true; // keep the node + }; + + !matches!( + rmp_serde::from_slice::(response.body()), + Ok(NetResponse::Peer(Some(_))), + ) + }); + if nodes_not_yet_in_net.is_empty() { + break; + } + std::thread::sleep(std::time::Duration::from_secs(1)); + } + if !nodes_not_yet_in_net.is_empty() { + error!("failed to get peering info for {nodes_not_yet_in_net:?}"); + } + + for node in nodes { + info!("Requesting logs from node: {}", node); + + let cacher_process_address = + Address::new(&node, ("binding-cacher", "hypermap-cacher", "sys")); + + if cacher_process_address == our() { + continue; + } + + // ping node for quicker failure if not online/providing/... + let Ok(Ok(response)) = Request::to(cacher_process_address.clone()) + .body(CacherRequest::GetStatus) + .send_and_await_response(3) + else { + warn!("Node {node} failed to respond to ping; trying next one..."); + continue; + }; + let Ok(CacherResponse::GetStatus(_)) = response.body().try_into() else { + warn!("Node {node} failed to respond to ping with expected GetStatus; trying next one..."); + continue; + }; + + // get the logs + let get_logs_request = GetLogsByRangeRequest { + from_block: self.last_cached_block + 1, + to_block: None, // Get all available logs + }; + + match Request::to(cacher_process_address.clone()) + .body(CacherRequest::GetLogsByRange(get_logs_request)) + .send_and_await_response(15) + { + Ok(Ok(response_msg)) => match response_msg.body().try_into() { + Ok(CacherResponse::GetLogsByRange(Ok(get_logs))) => { + match get_logs { + GetLogsByRangeOkResponse::Logs((block, json_string)) => { + if let Ok(log_caches) = + serde_json::from_str::>(&json_string) + { + self.process_received_log_caches(log_caches)?; + } + if block > self.last_cached_block { + self.last_cached_block = block; + } + } + GetLogsByRangeOkResponse::Latest(block) => { + if block > self.last_cached_block { + self.last_cached_block = block; + } + } + } + return Ok(()); + } + Ok(CacherResponse::GetLogsByRange(Err(e))) => { + warn!("Node {} returned error: {}", cacher_process_address, e); + } + Ok(CacherResponse::IsStarting) => { + info!( + "Node {} is still starting, trying next node", + cacher_process_address + ); + } + Ok(CacherResponse::Rejected) => { + warn!("Node {} rejected our request", cacher_process_address); + } + Ok(_) => { + warn!( + "Node {} returned unexpected response type", + cacher_process_address + ); + } + Err(e) => { + warn!( + "Failed to parse response from {}: {:?}", + cacher_process_address, e + ); + } + }, + Ok(Err(e)) => { + warn!("Error response from {}: {:?}", cacher_process_address, e); + } + Err(e) => { + warn!( + "Failed to send request to {}: {:?}", + cacher_process_address, e + ); + } + } + } + + Err(anyhow::anyhow!("Failed to bootstrap from any node")) + } + + // Helper function to write nodes to cache_sources file + fn write_nodes_to_file(&self) -> anyhow::Result<()> { + info!("Beginning of subroutine"); + let alt_drive_path = vfs::create_drive(our().package_id(), "initfiles", None)?; + info!("drive path defined"); + let nodes_json = serde_json::to_string(&self.nodes)?; + info!("nodes_json defined"); + let file_path = format!("{}/cache_sources", alt_drive_path); + info!("file_path defined"); + + // Open file in write mode which should truncate, but to be safe we'll write exact bytes + let mut file = vfs::open_file(&file_path, true, None)?; + + // Get the bytes to write + let bytes = nodes_json.as_bytes(); + + // Write all bytes + file.write_all(bytes)?; + + // Explicitly set the file length to the exact size of what we wrote + // This ensures any old content beyond this point is truncated + file.set_len(bytes.len() as u64)?; + + info!("Updated cache_sources with {} nodes", self.nodes.len()); + Ok(()) + } + + // Process received log caches and write them to VFS + fn process_received_log_caches( + &mut self, + log_caches: Vec, + ) -> anyhow::Result<()> { + info!("Processing {} received log caches", log_caches.len()); + + for log_cache in log_caches { + // Validate the log cache signature + // TODO Remove or find other method + // Temporarily skip + /* + if !self.validate_log_cache(&log_cache)? { + warn!("Invalid log cache signature, skipping"); + continue; + } + */ + + // Generate filename from metadata + let filename = format!( + "{}-chain{}-from{}-to{}-protocol{}.json", + log_cache + .metadata + .time_created + .replace(":", "") + .replace("-", ""), + log_cache.metadata.chain_id, + log_cache.metadata.from_block, + log_cache.metadata.to_block, + PROTOCOL_VERSION + ); + + // Write log cache to VFS + let file_path = format!("{}/{}", self.drive_path, filename); + let log_cache_bytes = serde_json::to_vec(&log_cache)?; + + let mut file = vfs::open_file(&file_path, true, None)?; + file.write_all(&log_cache_bytes)?; + + info!("Wrote log cache file: {}", file_path); + + // Update manifest + let file_hash = format!("0x{}", hex::encode(keccak256(&log_cache_bytes))); + let manifest_item = ManifestItemInternal { + metadata: log_cache.metadata.clone(), + is_empty: log_cache.logs.is_empty(), + file_hash, + file_name: filename.clone(), + }; + + self.manifest.items.insert(filename, manifest_item); + + // Update last cached block if this cache goes beyond it + if let Ok(to_block) = log_cache.metadata.to_block.parse::() { + if to_block > self.last_cached_block { + self.last_cached_block = to_block; + } + } + } + + // Write updated manifest + self.write_manifest()?; + + Ok(()) + } + + // Validate a log cache signature + fn validate_log_cache(&self, log_cache: &LogCacheInternal) -> anyhow::Result { + let from_block = log_cache.metadata.from_block.parse::()?; + let to_block = log_cache.metadata.to_block.parse::()?; + + let mut bytes_to_verify = serde_json::to_vec(&log_cache.logs)?; + bytes_to_verify.extend_from_slice(&from_block.to_be_bytes()); + bytes_to_verify.extend_from_slice(&to_block.to_be_bytes()); + let hashed_data = keccak256(&bytes_to_verify); + + let signature_hex = log_cache.metadata.signature.trim_start_matches("0x"); + let signature_bytes = hex::decode(signature_hex)?; + + let created_by_address = log_cache.metadata.created_by.parse::
()?; + + Ok(sign::net_key_verify( + hashed_data.to_vec(), + &created_by_address, + signature_bytes, + )?) + } + + // Write manifest to VFS + fn write_manifest(&self) -> anyhow::Result<()> { + let manifest_bytes = serde_json::to_vec(&self.manifest)?; + let manifest_path = format!("{}/{}", self.drive_path, self.manifest.manifest_filename); + let manifest_file = vfs::open_file(&manifest_path, true, None)?; + manifest_file.write(&manifest_bytes)?; + info!("Updated manifest file: {}", manifest_path); + Ok(()) + } + + // Determine optimal batch size dynamically + fn determine_batch_size(&mut self, provider: ð::Provider) -> anyhow::Result<()> { + if self.block_batch_size > 0 { + // Already determined + return Ok(()); + } + + let current_block = match provider.get_block_number() { + Ok(block_num) => block_num, + Err(e) => { + error!("Failed to get current block number: {:?}", e); + // Fall back to default if we can't get the current block + self.block_batch_size = DEFAULT_BLOCK_BATCH_SIZE; + return Ok(()); + } + }; + + // Start with the difference between current block and HYPERMAP_FIRST_BLOCK + let mut batch_size = current_block.saturating_sub(hypermap::HYPERMAP_FIRST_BLOCK); + + // Ensure we have at least a minimum batch size + if batch_size < 1 { + batch_size = DEFAULT_BLOCK_BATCH_SIZE; + self.block_batch_size = batch_size; + info!("Using default batch size: {batch_size}"); + return Ok(()); + } + + info!("Determining optimal batch size starting from {batch_size}"); + + // Try progressively smaller batch sizes until we find one that works + loop { + let from_block = hypermap::HYPERMAP_FIRST_BLOCK; + let to_block = from_block + batch_size; + + let filter = eth::Filter::new() + .address(self.hypermap_binding_address) + .from_block(from_block) + .to_block(eth::BlockNumberOrTag::Number(to_block)); + + match provider.get_logs(&filter) { + Ok(_) => { + // Success! This batch size works + self.block_batch_size = batch_size; + info!("Successfully determined batch size: {}", batch_size); + return Ok(()); + } + Err(e) => { + // Request failed or timed out, try smaller batch + warn!("Batch size {} failed: {:?}, halving...", batch_size, e); + batch_size = batch_size / 2; + + // Don't go below a minimum threshold + if batch_size < 10 { + warn!("Could not determine optimal batch size, using minimum: {DEFAULT_BLOCK_BATCH_SIZE}"); + self.block_batch_size = DEFAULT_BLOCK_BATCH_SIZE; + return Ok(()); + } + } + } + } + } + + // Fallback to RPC bootstrap - catch up from where we left off + fn try_bootstrap_from_rpc(&mut self, provider: ð::Provider) -> anyhow::Result<()> { + info!( + "Bootstrapping from RPC, starting from block {}", + self.last_cached_block + 1 + ); + + // Catch up remainder (or as fallback) using RPC + self.cache_logs_and_update_manifest(provider)?; + + // run it twice for fresh boot case: + // - initial bootstrap takes much time + // - in that time, the block you are updating to is no longer the head of the chain + // - so run again to get to the head of the chain + self.cache_logs_and_update_manifest(provider)?; + + Ok(()) + } + + fn to_wit_manifest(&self) -> WitManifest { + let items = self + .manifest + .items + .iter() + .map(|(k, v)| { + let wit_meta = WitLogsMetadata { + chain_id: v.metadata.chain_id.clone(), + from_block: v.metadata.from_block.clone(), + to_block: v.metadata.to_block.clone(), + time_created: v.metadata.time_created.clone(), + created_by: v.metadata.created_by.clone(), + signature: v.metadata.signature.clone(), + }; + let wit_item = WitManifestItem { + metadata: wit_meta, + is_empty: v.is_empty, + file_hash: v.file_hash.clone(), + file_name: v.file_name.clone(), + }; + (k.clone(), wit_item) + }) + .collect::>(); + + WitManifest { + items, + manifest_filename: self.manifest.manifest_filename.clone(), + chain_id: self.manifest.chain_id.clone(), + protocol_version: self.manifest.protocol_version.clone(), + } + } +} + +#[derive(Debug, Serialize, Deserialize)] +enum HttpApi { + GetManifest, + GetLogCacheFile(String), + GetStatus, +} + +fn http_handler( + state: &mut State, + path: &str, +) -> anyhow::Result<(http::server::HttpResponse, Vec)> { + let response = http::server::HttpResponse::new(http::StatusCode::OK) + .header("Content-Type", "application/json"); + + // Basic routing based on path + Ok(if path == "/manifest" || path == "/manifest.json" { + let manifest_path = format!("{}/{}", state.drive_path, state.manifest.manifest_filename); + let manifest_file = vfs::open_file(&manifest_path, true, None)?; + match manifest_file.read() { + Ok(content) => (response, content), + Err(e) => { + error!( + "HTTP: Failed to read manifest file {}: {:?}", + manifest_path, e + ); + ( + http::server::HttpResponse::new(http::StatusCode::NOT_FOUND), + b"Manifest not found".to_vec(), + ) + } + } + } else if path.starts_with("/log-cache/") { + let filename = path.trim_start_matches("/log-cache/"); + if filename.is_empty() || filename.contains("..") { + // Basic security check + return Ok(( + http::server::HttpResponse::new(http::StatusCode::BAD_REQUEST), + b"Invalid filename".to_vec(), + )); + } + let log_cache_path = format!("{}/{}", state.drive_path, filename); + let log_cache_file = vfs::open_file(&log_cache_path, true, None)?; + match log_cache_file.read() { + Ok(content) => (response, content), + Err(e) => { + error!( + "HTTP: Failed to read log cache file {}: {:?}", + log_cache_path, e + ); + ( + http::server::HttpResponse::new(http::StatusCode::NOT_FOUND), + b"Log cache file not found".to_vec(), + ) + } + } + } else if path == "/status" { + let status_info = CacherStatus { + last_cached_block: state.last_cached_block, + chain_id: state.chain_id.clone(), + protocol_version: state.protocol_version.clone(), + next_cache_attempt_in_seconds: if state.is_cache_timer_live { + Some(state.cache_interval_s) + } else { + None + }, + manifest_filename: state.manifest.manifest_filename.clone(), + log_files_count: state.manifest.items.len() as u32, + our_address: our().to_string(), + is_providing: state.is_providing, + }; + match serde_json::to_vec(&status_info) { + Ok(body) => (response, body), + Err(e) => { + error!("HTTP: Failed to serialize status: {:?}", e); + ( + http::server::HttpResponse::new(http::StatusCode::INTERNAL_SERVER_ERROR), + b"Error serializing status".to_vec(), + ) + } + } + } else { + ( + http::server::HttpResponse::new(http::StatusCode::NOT_FOUND), + b"Not Found".to_vec(), + ) + }) +} + +fn handle_request( + our: &Address, + source: &Address, + state: &mut State, + request: CacherRequest, +) -> anyhow::Result<()> { + let is_local = is_local_request(our, source); + + // If we're still starting, respond with IsStarting to all requests + if state.is_starting { + Response::new().body(CacherResponse::IsStarting).send()?; + return Ok(()); + } + + if !is_local && source.process.to_string() != "binding-cacher:hypermap-cacher:sys" { + warn!("Rejecting remote request from non-binding-cacher: {source}"); + Response::new().body(CacherResponse::Rejected).send()?; + return Ok(()); + } + + if !is_local + && !state.is_providing + && source.process.to_string() == "binding-cacher:hypermap-cacher:sys" + { + warn!("Rejecting remote request from {source} - not in provider mode"); + Response::new().body(CacherResponse::Rejected).send()?; + return Ok(()); + } + let response_body = match request { + CacherRequest::GetManifest => { + let manifest_path = + format!("{}/{}", state.drive_path, state.manifest.manifest_filename); + if state.manifest.items.is_empty() && vfs::metadata(&manifest_path, None).is_err() { + CacherResponse::GetManifest(None) + } else { + // Ensure manifest is loaded from VFS if state is fresh and manifest file exists + // This is usually handled by State::load, but as a fallback: + if state.manifest.items.is_empty() { + // If manifest in memory is empty, try to load it + let manifest_file = vfs::open_file(&manifest_path, true, None)?; + if let Ok(bytes) = manifest_file.read() { + if let Ok(disk_manifest) = + serde_json::from_slice::(&bytes) + { + state.manifest = disk_manifest; + } + } + } + CacherResponse::GetManifest(Some(state.to_wit_manifest())) + } + } + CacherRequest::GetLogCacheContent(filename) => { + let log_cache_path = format!("{}/{}", state.drive_path, filename); + let log_cache_file = vfs::open_file(&log_cache_path, true, None)?; + match log_cache_file.read() { + Ok(content_bytes) => { + // Content is raw JSON bytes of LogCacheInternal. + // The WIT expects a string. + match String::from_utf8(content_bytes) { + Ok(content_str) => { + CacherResponse::GetLogCacheContent(Ok(Some(content_str))) + } + Err(e) => { + error!("Failed to convert log cache content to UTF-8 string: {}", e); + CacherResponse::GetLogCacheContent(Err(format!( + "File content not valid UTF-8: {}", + e + ))) + } + } + } + Err(_) => CacherResponse::GetLogCacheContent(Ok(None)), + } + } + CacherRequest::GetStatus => { + let status = CacherStatus { + last_cached_block: state.last_cached_block, + chain_id: state.chain_id.clone(), + protocol_version: state.protocol_version.clone(), + next_cache_attempt_in_seconds: if state.is_cache_timer_live { + Some(state.cache_interval_s) + } else { + None + }, + manifest_filename: state.manifest.manifest_filename.clone(), + log_files_count: state.manifest.items.len() as u32, + our_address: our.to_string(), + is_providing: state.is_providing, + }; + CacherResponse::GetStatus(status) + } + CacherRequest::GetLogsByRange(req_params) => { + let mut relevant_caches: Vec = Vec::new(); + let req_from_block = req_params.from_block; + // If req_params.to_block is None, we effectively want to go up to the highest block available in caches. + // For simplicity in overlap calculation, we can treat None as u64::MAX here. + let effective_req_to_block = req_params.to_block.unwrap_or(u64::MAX); + + for item in state.manifest.items.values() { + // Skip items that don't have an actual file (e.g., empty log ranges not written to disk). + if item.file_name.is_empty() { + continue; + } + + let cache_from = match item.metadata.from_block.parse::() { + Ok(b) => b, + Err(_) => { + warn!( + "Could not parse from_block for cache item {}: {}", + item.file_name, item.metadata.from_block + ); + continue; + } + }; + let cache_to = match item.metadata.to_block.parse::() { + Ok(b) => b, + Err(_) => { + warn!( + "Could not parse to_block for cache item {}: {}", + item.file_name, item.metadata.to_block + ); + continue; + } + }; + + // Check for overlap: max(start1, start2) <= min(end1, end2) + if max(req_from_block, cache_from) <= min(effective_req_to_block, cache_to) { + // This cache file overlaps with the requested range. + let file_vfs_path = format!("{}/{}", state.drive_path, item.file_name); + match vfs::open_file(&file_vfs_path, false, None) { + Ok(file) => match file.read() { + Ok(content_bytes) => { + match serde_json::from_slice::(&content_bytes) { + Ok(log_cache) => relevant_caches.push(log_cache), + Err(e) => { + error!( + "Failed to deserialize LogCacheInternal from {}: {:?}", + item.file_name, e + ); + // Decide: return error or skip this cache? For now, skip. + } + } + } + Err(e) => error!("Failed to read VFS file {}: {:?}", item.file_name, e), + }, + Err(e) => error!("Failed to open VFS file {}: {e:?}", item.file_name), + } + } + } + + // Sort caches by their from_block. + relevant_caches + .sort_by_key(|cache| cache.metadata.from_block.parse::().unwrap_or(0)); + + if relevant_caches.is_empty() { + CacherResponse::GetLogsByRange(Ok(GetLogsByRangeOkResponse::Latest( + state.last_cached_block, + ))) + } else { + match serde_json::to_string(&relevant_caches) { + Ok(json_string) => CacherResponse::GetLogsByRange(Ok( + GetLogsByRangeOkResponse::Logs((state.last_cached_block, json_string)), + )), + Err(e) => CacherResponse::GetLogsByRange(Err(format!( + "Failed to serialize relevant caches: {e}" + ))), + } + } + } + CacherRequest::StartProviding => { + if !is_local { + // should never happen: should be caught in check above + Response::new().body(CacherResponse::Rejected).send()?; + return Ok(()); + } + state.is_providing = true; + state.save(); + info!("Provider mode enabled"); + CacherResponse::StartProviding(Ok("Provider mode enabled".to_string())) + } + CacherRequest::StopProviding => { + if !is_local { + Response::new().body(CacherResponse::Rejected).send()?; + warn!("Rejecting remote request from {source} to alter provider mode"); + return Ok(()); + } + state.is_providing = false; + state.save(); + info!("Provider mode disabled"); + CacherResponse::StopProviding(Ok("Provider mode disabled".to_string())) + } + CacherRequest::SetNodes(new_nodes) => { + if !is_local { + Response::new().body(CacherResponse::Rejected).send()?; + warn!("Rejecting remote request from {source} to set nodes"); + return Ok(()); + } + state.nodes = new_nodes; + state.save(); + if let Err(e) = state.write_nodes_to_file() { + error!("Failed to write nodes to cache_sources: {:?}", e); + } + info!("Nodes updated to: {:?}", state.nodes); + CacherResponse::SetNodes(Ok("Nodes updated successfully".to_string())) + } + CacherRequest::Reset(custom_nodes) => { + if !is_local { + Response::new().body(CacherResponse::Rejected).send()?; + warn!("Rejecting remote request from {source} to reset"); + return Ok(()); + } + + info!("Resetting binding-cacher state and clearing VFS..."); + + // Clear all files from the drive + if let Err(e) = state.clear_drive() { + error!("Failed to clear drive during reset: {:?}", e); + CacherResponse::Reset(Err(format!("Failed to clear drive: {:?}", e))) + } else { + // Create new state with custom nodes if provided, otherwise use defaults + let nodes = match custom_nodes { + Some(nodes) => nodes, + None => DEFAULT_NODES.iter().map(|s| s.to_string()).collect(), + }; + + *state = State::new(&state.drive_path); + state.nodes = nodes; + state.save(); + if let Err(e) = state.write_nodes_to_file() { + error!("Failed to write nodes to cache_sources: {:?}", e); + } + info!( + "binding-cacher reset complete. New nodes: {:?}", + state.nodes + ); + CacherResponse::Reset(Ok( + "Reset completed successfully. Binding Cacher will restart with new settings." + .to_string(), + )) + } + } + }; + + Response::new().body(response_body).send()?; + Ok(()) +} + +fn main_loop( + our: &Address, + state: &mut State, + provider: ð::Provider, + server: &http::server::HttpServer, +) -> anyhow::Result<()> { + info!( + "Hypermap Binding Cacher main_loop started. Our address: {}", + our + ); + info!( + "Monitoring Binding contract: {}", + state.hypermap_binding_address.to_string() + ); + info!( + "Chain ID: {}, Protocol Version: {}", + state.chain_id, state.protocol_version + ); + info!("Last cached block: {}", state.last_cached_block); + + // Always bootstrap on start to get latest state from other nodes or RPC + while state.is_starting { + match state.bootstrap_state(provider) { + Ok(_) => info!("Bootstrap process completed successfully."), + Err(e) => { + error!("Error during bootstrap process: {:?}", e); + std::thread::sleep(std::time::Duration::from_secs(RETRY_DELAY_S)); + } + } + } + + // Set up the main caching timer. + info!( + "Setting cache timer for {} seconds.", + state.cache_interval_s + ); + timer::set_timer(state.cache_interval_s * 1000, Some(b"cache_cycle".to_vec())); + state.is_cache_timer_live = true; + state.save(); + + loop { + let Ok(message) = await_message() else { + warn!("Failed to get message, continuing loop."); + continue; + }; + let source = message.source(); + + if message.is_request() { + if source.process == ProcessId::from_str("http-server:distro:sys").unwrap() { + // HTTP request from the system's HTTP server process + let Ok(http::server::HttpServerRequest::Http(http_request)) = + server.parse_request(message.body()) + else { + error!("Failed to parse HTTP request from http-server:distro:sys"); + // Potentially send an error response back if possible/expected + continue; + }; + let (http_response, body) = http_handler(state, &http_request.path()?)?; + Response::new() + .body(serde_json::to_vec(&http_response).unwrap()) + .blob_bytes(body) + .send()?; + } else { + // Standard process-to-process request + match serde_json::from_slice::(message.body()) { + Ok(request) => { + if let Err(e) = handle_request(our, &source, state, request) { + error!("Error handling request from {:?}: {:?}", source, e); + } + } + Err(e) => { + error!( + "Failed to deserialize CacherRequest from {:?}: {:?}", + source, e + ); + } + } + } + } else { + // It's a Response or other kind of message + if source.process == ProcessId::from_str("timer:distro:sys").unwrap() { + if message.context() == Some(b"cache_cycle") { + info!("Cache timer triggered."); + state.is_cache_timer_live = false; + match state.cache_logs_and_update_manifest(provider) { + Ok(_) => info!("Periodic cache cycle complete."), + Err(e) => error!("Error during periodic cache cycle: {:?}", e), + } + // Reset the timer for the next cycle + if !state.is_cache_timer_live { + timer::set_timer( + state.cache_interval_s * 1000, + Some(b"cache_cycle".to_vec()), + ); + state.is_cache_timer_live = true; + state.save(); + } + } + } else { + debug!( + "Received unhandled response or other message from {:?}.", + source + ); + } + } + } +} + +call_init!(init); +fn init(our: Address) { + init_logging(Level::INFO, Level::DEBUG, None, None, None).unwrap(); + info!("Hypermap Binding Cacher process starting..."); + + let drive_path = vfs::create_drive(our.package_id(), "binding-cache", None).unwrap(); + // Create alternate drive for initfiles and read the test data + let alt_drive_path = vfs::create_drive(our.package_id(), "initfiles", None).unwrap(); + + // Try to read the cache_sources file from the initfiles drive + match vfs::open_file(&format!("{}/cache_sources", alt_drive_path), false, None) { + Ok(file) => match file.read() { + Ok(contents) => { + let content_str = String::from_utf8_lossy(&contents); + info!("Contents of cache_sources: {}", content_str); + } + Err(e) => { + info!("Failed to read cache_sources: {}", e); + } + }, + Err(e) => { + info!("Failed to open cache_sources: {}", e); + } + } + + let bind_config = http::server::HttpBindingConfig::default().authenticated(false); + let mut server = http::server::HttpServer::new(5); + + let provider = eth::Provider::new(hypermap::HYPERMAP_CHAIN_ID, 60); + + server + .bind_http_path("/manifest", bind_config.clone()) + .expect("Failed to bind /manifest"); + server + .bind_http_path("/manifest.json", bind_config.clone()) + .expect("Failed to bind /manifest.json"); + server + .bind_http_path("/log-cache/*", bind_config.clone()) + .expect("Failed to bind /log-cache/*"); + server + .bind_http_path("/status", bind_config.clone()) + .expect("Failed to bind /status"); + info!("Bound HTTP paths: /manifest, /log-cache/*, /status"); + + let mut state = State::load(&drive_path); + + // Wait for hypermap-cacher to be ready before entering main loop + let cacher_addr = Address::new("our", ("hypermap-cacher", "hypermap-cacher", "sys")); + info!( + "Waiting for hypermap-cacher at {} to report ready before starting binding-cacher...", + cacher_addr + ); + wait_for_process_ready( + cacher_addr, + b"\"GetStatus\"".to_vec(), + 15, + 2, + |body| { + let body_str = String::from_utf8_lossy(body); + if body_str.contains("IsStarting") || body_str.contains(r#""IsStarting""#) { + WaitClassification::Starting + } else if body_str.contains("GetStatus") || body_str.contains("last_cached_block") { + WaitClassification::Ready + } else { + WaitClassification::Unknown + } + }, + true, + None, + ); + info!("hypermap-cacher is ready; continuing binding-cacher startup."); + + loop { + match main_loop(&our, &mut state, &provider, &server) { + Ok(()) => { + // main_loop should not exit with Ok in normal operation as it's an infinite loop. + error!("main_loop exited unexpectedly with Ok. Restarting."); + } + Err(e) => { + error!("main_loop exited with error: {:?}. Restarting.", e); + std::thread::sleep(std::time::Duration::from_secs(5)); + } + } + // Reload state in case of restart, or re-initialize if necessary. + state = State::load(&drive_path); + } +} diff --git a/hyperdrive/packages/hypermap-cacher/hypermap-cacher/Cargo.toml b/hyperdrive/packages/hypermap-cacher/hypermap-cacher/Cargo.toml index bb9b33d5d..1b316cdfb 100644 --- a/hyperdrive/packages/hypermap-cacher/hypermap-cacher/Cargo.toml +++ b/hyperdrive/packages/hypermap-cacher/hypermap-cacher/Cargo.toml @@ -18,7 +18,7 @@ alloy = { version = "0.8.1", features = [ ] } chrono = "0.4.41" hex = "0.4.3" -hyperware_process_lib = { version = "2.1.0", features = ["logging"] } +hyperware_process_lib = { version = "2.3.0", features = ["logging"] } process_macros = "0.1.0" rand = "0.8" rmp-serde = "1.1.2" diff --git a/hyperdrive/packages/hypermap-cacher/hypermap-cacher/src/lib.rs b/hyperdrive/packages/hypermap-cacher/hypermap-cacher/src/lib.rs index e64affc86..08483465f 100644 --- a/hyperdrive/packages/hypermap-cacher/hypermap-cacher/src/lib.rs +++ b/hyperdrive/packages/hypermap-cacher/hypermap-cacher/src/lib.rs @@ -24,7 +24,7 @@ use hyperware_process_lib::{ wit_bindgen::generate!({ path: "../target/wit", - world: "hypermap-cacher-sys-v0", + world: "hypermap-cacher-sys-v1", generate_unused_types: true, additional_derives: [serde::Deserialize, serde::Serialize, process_macros::SerdeJsonInto], }); @@ -199,16 +199,13 @@ impl State { } // Core logic for fetching logs, creating cache files, and updating the manifest. - fn cache_logs_and_update_manifest( - &mut self, - hypermap: &hypermap::Hypermap, - ) -> anyhow::Result<()> { + fn cache_logs_and_update_manifest(&mut self, provider: ð::Provider) -> anyhow::Result<()> { // Ensure batch size is determined if self.block_batch_size == 0 { - self.determine_batch_size(hypermap)?; + self.determine_batch_size(provider)?; } - let current_chain_head = match hypermap.provider.get_block_number() { + let current_chain_head = match provider.get_block_number() { Ok(block_num) => block_num, Err(e) => { error!( @@ -228,7 +225,7 @@ impl State { } while self.last_cached_block != current_chain_head { - self.cache_logs_and_update_manifest_step(hypermap, Some(current_chain_head))?; + self.cache_logs_and_update_manifest_step(provider, Some(current_chain_head))?; std::thread::sleep(std::time::Duration::from_millis(LOG_ITERATION_DELAY_MS)); } @@ -238,7 +235,7 @@ impl State { fn cache_logs_and_update_manifest_step( &mut self, - hypermap: &hypermap::Hypermap, + provider: ð::Provider, to_block: Option, ) -> anyhow::Result<()> { info!( @@ -248,7 +245,7 @@ impl State { let current_chain_head = match to_block { Some(b) => b, - None => match hypermap.provider.get_block_number() { + None => match provider.get_block_number() { Ok(block_num) => block_num, Err(e) => { error!( @@ -287,7 +284,7 @@ impl State { let logs = { let mut attempt = 0; loop { - match hypermap.provider.get_logs(&filter) { + match provider.get_logs(&filter) { Ok(logs) => break logs, Err(e) => { attempt += 1; @@ -539,7 +536,7 @@ impl State { } // Bootstrap state from other nodes, then fallback to RPC - fn bootstrap_state(&mut self, hypermap: &hypermap::Hypermap) -> anyhow::Result<()> { + fn bootstrap_state(&mut self, provider: ð::Provider) -> anyhow::Result<()> { info!("Starting state bootstrap process..."); // Try to bootstrap from other nodes first @@ -547,7 +544,7 @@ impl State { info!("Successfully bootstrapped from other nodes"); } - self.try_bootstrap_from_rpc(hypermap)?; + self.try_bootstrap_from_rpc(provider)?; // Mark as no longer starting self.is_starting = false; @@ -778,10 +775,14 @@ impl State { for log_cache in log_caches { // Validate the log cache signature + // TODO Remove or find other method + // Temporarily skip + /* if !self.validate_log_cache(&log_cache)? { warn!("Invalid log cache signature, skipping"); continue; } + */ // Generate filename from metadata let filename = format!( @@ -864,13 +865,13 @@ impl State { } // Determine optimal batch size dynamically - fn determine_batch_size(&mut self, hypermap: &hypermap::Hypermap) -> anyhow::Result<()> { + fn determine_batch_size(&mut self, provider: ð::Provider) -> anyhow::Result<()> { if self.block_batch_size > 0 { // Already determined return Ok(()); } - let current_block = match hypermap.provider.get_block_number() { + let current_block = match provider.get_block_number() { Ok(block_num) => block_num, Err(e) => { error!("Failed to get current block number: {:?}", e); @@ -903,7 +904,7 @@ impl State { .from_block(from_block) .to_block(eth::BlockNumberOrTag::Number(to_block)); - match hypermap.provider.get_logs(&filter) { + match provider.get_logs(&filter) { Ok(_) => { // Success! This batch size works self.block_batch_size = batch_size; @@ -927,20 +928,20 @@ impl State { } // Fallback to RPC bootstrap - catch up from where we left off - fn try_bootstrap_from_rpc(&mut self, hypermap: &hypermap::Hypermap) -> anyhow::Result<()> { + fn try_bootstrap_from_rpc(&mut self, provider: ð::Provider) -> anyhow::Result<()> { info!( "Bootstrapping from RPC, starting from block {}", self.last_cached_block + 1 ); // Catch up remainder (or as fallback) using RPC - self.cache_logs_and_update_manifest(hypermap)?; + self.cache_logs_and_update_manifest(provider)?; // run it twice for fresh boot case: // - initial bootstrap takes much time // - in that time, the block you are updating to is no longer the head of the chain // - so run again to get to the head of the chain - self.cache_logs_and_update_manifest(hypermap)?; + self.cache_logs_and_update_manifest(provider)?; Ok(()) } @@ -1298,11 +1299,11 @@ fn handle_request( error!("Failed to write nodes to cache_sources: {:?}", e); } info!( - "Hypermap-cacher reset complete. New nodes: {:?}", + "hypermap-cacher reset complete. New nodes: {:?}", state.nodes ); CacherResponse::Reset(Ok( - "Reset completed successfully. Cacher will restart with new settings." + "Reset completed successfully. Hypermap Cacher will restart with new settings." .to_string(), )) } @@ -1316,7 +1317,7 @@ fn handle_request( fn main_loop( our: &Address, state: &mut State, - hypermap: &hypermap::Hypermap, + provider: ð::Provider, server: &http::server::HttpServer, ) -> anyhow::Result<()> { info!("Hypermap Cacher main_loop started. Our address: {}", our); @@ -1332,7 +1333,7 @@ fn main_loop( // Always bootstrap on start to get latest state from other nodes or RPC while state.is_starting { - match state.bootstrap_state(hypermap) { + match state.bootstrap_state(provider) { Ok(_) => info!("Bootstrap process completed successfully."), Err(e) => { error!("Error during bootstrap process: {:?}", e); @@ -1394,7 +1395,7 @@ fn main_loop( if message.context() == Some(b"cache_cycle") { info!("Cache timer triggered."); state.is_cache_timer_live = false; - match state.cache_logs_and_update_manifest(hypermap) { + match state.cache_logs_and_update_manifest(provider) { Ok(_) => info!("Periodic cache cycle complete."), Err(e) => error!("Error during periodic cache cycle: {:?}", e), } @@ -1446,7 +1447,7 @@ fn init(our: Address) { let bind_config = http::server::HttpBindingConfig::default().authenticated(false); let mut server = http::server::HttpServer::new(5); - let hypermap_provider = hypermap::Hypermap::default(60); + let provider = eth::Provider::new(hypermap::HYPERMAP_CHAIN_ID, 60); server .bind_http_path("/manifest", bind_config.clone()) @@ -1465,7 +1466,7 @@ fn init(our: Address) { let mut state = State::load(&drive_path); loop { - match main_loop(&our, &mut state, &hypermap_provider, &server) { + match main_loop(&our, &mut state, &provider, &server) { Ok(()) => { // main_loop should not exit with Ok in normal operation as it's an infinite loop. error!("main_loop exited unexpectedly with Ok. Restarting."); diff --git a/hyperdrive/packages/hypermap-cacher/pkg/manifest.json b/hyperdrive/packages/hypermap-cacher/pkg/manifest.json index ec498453e..e4afacf23 100644 --- a/hyperdrive/packages/hypermap-cacher/pkg/manifest.json +++ b/hyperdrive/packages/hypermap-cacher/pkg/manifest.json @@ -1,10 +1,39 @@ [ + { + "process_name": "binding-cacher", + "process_wasm_path": "/binding-cacher.wasm", + "on_exit": "Restart", + "request_networking": true, + "request_capabilities": [ + "eth:distro:sys", + "http-server:distro:sys", + "hypermap-cacher:hypermap-cacher:sys", + "net:distro:sys", + "sign:sign:sys", + "terminal:terminal:sys", + "timer:distro:sys", + "vfs:distro:sys" + ], + "grant_capabilities": [ + "eth:distro:sys", + "http-server:distro:sys", + "hypermap-cacher:hypermap-cacher:sys", + "net:distro:sys", + "sign:sign:sys", + "terminal:terminal:sys", + "timer:distro:sys", + "vfs:distro:sys" + ], + "public": false, + "wit_version": 1 + }, { "process_name": "hypermap-cacher", "process_wasm_path": "/hypermap-cacher.wasm", "on_exit": "Restart", "request_networking": true, "request_capabilities": [ + "binding-cacher:hypermap-cacher:sys", "eth:distro:sys", "http-server:distro:sys", "net:distro:sys", @@ -14,6 +43,7 @@ "vfs:distro:sys" ], "grant_capabilities": [ + "binding-cacher:hypermap-cacher:sys", "eth:distro:sys", "http-server:distro:sys", "net:distro:sys", diff --git a/hyperdrive/packages/hypermap-cacher/pkg/scripts.json b/hyperdrive/packages/hypermap-cacher/pkg/scripts.json index aab9de842..5b1bfaefe 100644 --- a/hyperdrive/packages/hypermap-cacher/pkg/scripts.json +++ b/hyperdrive/packages/hypermap-cacher/pkg/scripts.json @@ -4,9 +4,11 @@ "public": false, "request_networking": false, "request_capabilities": [ + "binding-cacher:hypermap-cacher:sys", "hypermap-cacher:hypermap-cacher:sys" ], "grant_capabilities": [ + "binding-cacher:hypermap-cacher:sys", "hypermap-cacher:hypermap-cacher:sys" ], "wit_version": 1 @@ -16,9 +18,11 @@ "public": false, "request_networking": false, "request_capabilities": [ + "binding-cacher:hypermap-cacher:sys", "hypermap-cacher:hypermap-cacher:sys" ], "grant_capabilities": [ + "binding-cacher:hypermap-cacher:sys", "hypermap-cacher:hypermap-cacher:sys" ], "wit_version": 1 @@ -28,9 +32,11 @@ "public": false, "request_networking": false, "request_capabilities": [ + "binding-cacher:hypermap-cacher:sys", "hypermap-cacher:hypermap-cacher:sys" ], "grant_capabilities": [ + "binding-cacher:hypermap-cacher:sys", "hypermap-cacher:hypermap-cacher:sys" ], "wit_version": 1 @@ -40,9 +46,11 @@ "public": false, "request_networking": false, "request_capabilities": [ + "binding-cacher:hypermap-cacher:sys", "hypermap-cacher:hypermap-cacher:sys" ], "grant_capabilities": [ + "binding-cacher:hypermap-cacher:sys", "hypermap-cacher:hypermap-cacher:sys" ], "wit_version": 1 diff --git a/hyperdrive/packages/hypermap-cacher/reset-cache/Cargo.toml b/hyperdrive/packages/hypermap-cacher/reset-cache/Cargo.toml index 64c780831..a796f9b13 100644 --- a/hyperdrive/packages/hypermap-cacher/reset-cache/Cargo.toml +++ b/hyperdrive/packages/hypermap-cacher/reset-cache/Cargo.toml @@ -5,7 +5,7 @@ edition = "2021" publish = false [dependencies] -hyperware_process_lib = "2.1.0" +hyperware_process_lib = { version = "2.3.0" } process_macros = "0.1.0" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" diff --git a/hyperdrive/packages/hypermap-cacher/reset-cache/src/lib.rs b/hyperdrive/packages/hypermap-cacher/reset-cache/src/lib.rs index 45adadd23..495847667 100644 --- a/hyperdrive/packages/hypermap-cacher/reset-cache/src/lib.rs +++ b/hyperdrive/packages/hypermap-cacher/reset-cache/src/lib.rs @@ -12,12 +12,13 @@ //! reset:hypermap-cacher:sys # Reset with default nodes //! reset:hypermap-cacher:sys alice.os bob.os # Reset with custom nodes +use crate::hyperware::process::binding_cacher::{BindingCacherRequest, BindingCacherResponse}; use crate::hyperware::process::hypermap_cacher::{CacherRequest, CacherResponse}; use hyperware_process_lib::{await_next_message_body, call_init, println, Address, Request}; wit_bindgen::generate!({ path: "../target/wit", - world: "hypermap-cacher-sys-v0", + world: "hypermap-cacher-sys-v1", generate_unused_types: true, additional_derives: [serde::Deserialize, serde::Serialize, process_macros::SerdeJsonInto], }); @@ -33,13 +34,14 @@ fn init(_our: Address) { let parts: Vec<&str> = args.split_whitespace().collect(); let custom_nodes = if parts.is_empty() { - println!("Resetting hypermap-cacher with default nodes..."); + println!("Resetting cachers with default nodes..."); None } else { let nodes: Vec = parts.iter().map(|s| s.to_string()).collect(); - println!("Resetting hypermap-cacher with custom nodes: {:?}", nodes); + println!("Resetting cachers with custom nodes: {:?}", nodes); Some(nodes) }; + let binding_custom_nodes = custom_nodes.clone(); let response = Request::to(("our", "hypermap-cacher", "hypermap-cacher", "sys")) .body(CacherRequest::Reset(custom_nodes)) @@ -51,7 +53,7 @@ fn init(_our: Address) { println!("✓ {}", msg); } Ok(CacherResponse::Reset(Err(err))) => { - println!("✗ Failed to reset: {}", err); + println!("✗ Failed to reset hypermap-cacher: {}", err); } _ => { println!("✗ Unexpected response from hypermap-cacher"); @@ -64,4 +66,28 @@ fn init(_our: Address) { println!("✗ Communication error: {:?}", err); } } + + let response = Request::to(("our", "binding-cacher", "hypermap-cacher", "sys")) + .body(BindingCacherRequest::Reset(binding_custom_nodes)) + .send_and_await_response(10); // Give it more time for reset operations + + match response { + Ok(Ok(message)) => match message.body().try_into() { + Ok(BindingCacherResponse::Reset(Ok(msg))) => { + println!("✓ {}", msg); + } + Ok(BindingCacherResponse::Reset(Err(err))) => { + println!("✗ Failed to reset binding-cacher: {}", err); + } + _ => { + println!("✗ Unexpected response from binding-cacher"); + } + }, + Ok(Err(err)) => { + println!("✗ Request failed: {:?}", err); + } + Err(err) => { + println!("✗ Communication error: {:?}", err); + } + } } diff --git a/hyperdrive/packages/hypermap-cacher/set-nodes/Cargo.toml b/hyperdrive/packages/hypermap-cacher/set-nodes/Cargo.toml index 6a00070f2..1b3538908 100644 --- a/hyperdrive/packages/hypermap-cacher/set-nodes/Cargo.toml +++ b/hyperdrive/packages/hypermap-cacher/set-nodes/Cargo.toml @@ -5,7 +5,7 @@ edition = "2021" publish = false [dependencies] -hyperware_process_lib = "2.1.0" +hyperware_process_lib = { version = "2.3.0" } process_macros = "0.1.0" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" diff --git a/hyperdrive/packages/hypermap-cacher/set-nodes/src/lib.rs b/hyperdrive/packages/hypermap-cacher/set-nodes/src/lib.rs index 9236ca28e..c8479ef9e 100644 --- a/hyperdrive/packages/hypermap-cacher/set-nodes/src/lib.rs +++ b/hyperdrive/packages/hypermap-cacher/set-nodes/src/lib.rs @@ -10,12 +10,13 @@ //! Example: //! set-nodes:hypermap-cacher:sys alice.os bob.os charlie.os +use crate::hyperware::process::binding_cacher::{BindingCacherRequest, BindingCacherResponse}; use crate::hyperware::process::hypermap_cacher::{CacherRequest, CacherResponse}; use hyperware_process_lib::{await_next_message_body, call_init, println, Address, Request}; wit_bindgen::generate!({ path: "../target/wit", - world: "hypermap-cacher-sys-v0", + world: "hypermap-cacher-sys-v1", generate_unused_types: true, additional_derives: [serde::Deserialize, serde::Serialize, process_macros::SerdeJsonInto], }); @@ -37,6 +38,7 @@ fn init(_our: Address) { } let nodes: Vec = parts.iter().map(|s| s.to_string()).collect(); + let binding_nodes = nodes.clone(); println!("Setting hypermap-cacher nodes to: {:?}", nodes); @@ -50,7 +52,7 @@ fn init(_our: Address) { println!("✓ {}", msg); } Ok(CacherResponse::SetNodes(Err(err))) => { - println!("✗ Failed to set nodes: {}", err); + println!("✗ Failed to set nodes for hypermap-cacher: {}", err); } _ => { println!("✗ Unexpected response from hypermap-cacher"); @@ -63,4 +65,30 @@ fn init(_our: Address) { println!("✗ Communication error: {:?}", err); } } + + println!("Setting binding-cacher nodes to: {:?}", binding_nodes); + + let response = Request::to(("our", "binding-cacher", "hypermap-cacher", "sys")) + .body(BindingCacherRequest::SetNodes(binding_nodes)) + .send_and_await_response(5); + + match response { + Ok(Ok(message)) => match message.body().try_into() { + Ok(BindingCacherResponse::SetNodes(Ok(msg))) => { + println!("✓ {}", msg); + } + Ok(BindingCacherResponse::SetNodes(Err(err))) => { + println!("✗ Failed to set nodes for binding-cacher: {}", err); + } + _ => { + println!("✗ Unexpected response from binding-cacher"); + } + }, + Ok(Err(err)) => { + println!("✗ Request failed: {:?}", err); + } + Err(err) => { + println!("✗ Communication error: {:?}", err); + } + } } diff --git a/hyperdrive/packages/hypermap-cacher/start-providing/Cargo.toml b/hyperdrive/packages/hypermap-cacher/start-providing/Cargo.toml index fd508eefc..3797fb50a 100644 --- a/hyperdrive/packages/hypermap-cacher/start-providing/Cargo.toml +++ b/hyperdrive/packages/hypermap-cacher/start-providing/Cargo.toml @@ -5,7 +5,7 @@ edition = "2021" publish = false [dependencies] -hyperware_process_lib = "2.1.0" +hyperware_process_lib = { version = "2.3.0" } process_macros = "0.1.0" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" diff --git a/hyperdrive/packages/hypermap-cacher/start-providing/src/lib.rs b/hyperdrive/packages/hypermap-cacher/start-providing/src/lib.rs index 8162c77ab..f74f7abe7 100644 --- a/hyperdrive/packages/hypermap-cacher/start-providing/src/lib.rs +++ b/hyperdrive/packages/hypermap-cacher/start-providing/src/lib.rs @@ -1,9 +1,10 @@ +use crate::hyperware::process::binding_cacher::{BindingCacherRequest, BindingCacherResponse}; use crate::hyperware::process::hypermap_cacher::{CacherRequest, CacherResponse}; use hyperware_process_lib::{call_init, println, Address, Request}; wit_bindgen::generate!({ path: "../target/wit", - world: "hypermap-cacher-sys-v0", + world: "hypermap-cacher-sys-v1", generate_unused_types: true, additional_derives: [serde::Deserialize, serde::Serialize, process_macros::SerdeJsonInto], }); @@ -22,7 +23,7 @@ fn init(_our: Address) { println!("✓ {}", msg); } Ok(CacherResponse::StartProviding(Err(err))) => { - println!("✗ Failed to enable provider mode: {}", err); + println!("✗ Failed to enable hypermap-cacher provider mode: {}", err); } _ => { println!("✗ Unexpected response from hypermap-cacher"); @@ -35,4 +36,30 @@ fn init(_our: Address) { println!("✗ Communication error: {:?}", err); } } + + println!("Enabling binding-cacher provider mode..."); + + let response = Request::to(("our", "binding-cacher", "hypermap-cacher", "sys")) + .body(BindingCacherRequest::StartProviding) + .send_and_await_response(5); + + match response { + Ok(Ok(message)) => match message.body().try_into() { + Ok(BindingCacherResponse::StartProviding(Ok(msg))) => { + println!("✓ {}", msg); + } + Ok(BindingCacherResponse::StartProviding(Err(err))) => { + println!("✗ Failed to enable binding-cacher provider mode: {}", err); + } + _ => { + println!("✗ Unexpected response from binding-cacher"); + } + }, + Ok(Err(err)) => { + println!("✗ Request failed: {:?}", err); + } + Err(err) => { + println!("✗ Communication error: {:?}", err); + } + } } diff --git a/hyperdrive/packages/hypermap-cacher/stop-providing/Cargo.toml b/hyperdrive/packages/hypermap-cacher/stop-providing/Cargo.toml index dfbc1efcf..4f75e6add 100644 --- a/hyperdrive/packages/hypermap-cacher/stop-providing/Cargo.toml +++ b/hyperdrive/packages/hypermap-cacher/stop-providing/Cargo.toml @@ -5,7 +5,7 @@ edition = "2021" publish = false [dependencies] -hyperware_process_lib = "2.1.0" +hyperware_process_lib = { version = "2.3.0" } process_macros = "0.1.0" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" diff --git a/hyperdrive/packages/hypermap-cacher/stop-providing/src/lib.rs b/hyperdrive/packages/hypermap-cacher/stop-providing/src/lib.rs index bf4491ca7..e6d55b831 100644 --- a/hyperdrive/packages/hypermap-cacher/stop-providing/src/lib.rs +++ b/hyperdrive/packages/hypermap-cacher/stop-providing/src/lib.rs @@ -1,9 +1,10 @@ +use crate::hyperware::process::binding_cacher::{BindingCacherRequest, BindingCacherResponse}; use crate::hyperware::process::hypermap_cacher::{CacherRequest, CacherResponse}; use hyperware_process_lib::{call_init, println, Address, Request}; wit_bindgen::generate!({ path: "../target/wit", - world: "hypermap-cacher-sys-v0", + world: "hypermap-cacher-sys-v1", generate_unused_types: true, additional_derives: [serde::Deserialize, serde::Serialize, process_macros::SerdeJsonInto], }); @@ -22,7 +23,7 @@ fn init(_our: Address) { println!("✓ {}", msg); } Ok(CacherResponse::StopProviding(Err(err))) => { - println!("✗ Failed to disable provider mode: {}", err); + println!("✗ Failed to disable hypermap-cacher provider mode: {}", err); } _ => { println!("✗ Unexpected response from hypermap-cacher"); @@ -35,4 +36,30 @@ fn init(_our: Address) { println!("✗ Communication error: {:?}", err); } } + + println!("Disabling binding-cacher provider mode..."); + + let response = Request::to(("our", "binding-cacher", "hypermap-cacher", "sys")) + .body(BindingCacherRequest::StopProviding) + .send_and_await_response(5); + + match response { + Ok(Ok(message)) => match message.body().try_into() { + Ok(BindingCacherResponse::StopProviding(Ok(msg))) => { + println!("✓ {}", msg); + } + Ok(BindingCacherResponse::StopProviding(Err(err))) => { + println!("✗ Failed to disable binding-cacher provider mode: {}", err); + } + _ => { + println!("✗ Unexpected response from binding-cacher"); + } + }, + Ok(Err(err)) => { + println!("✗ Request failed: {:?}", err); + } + Err(err) => { + println!("✗ Communication error: {:?}", err); + } + } } diff --git a/hyperdrive/packages/spider/spider/Cargo.toml b/hyperdrive/packages/spider/spider/Cargo.toml index 16fb2dc27..6f5b2539a 100644 --- a/hyperdrive/packages/spider/spider/Cargo.toml +++ b/hyperdrive/packages/spider/spider/Cargo.toml @@ -13,13 +13,13 @@ wit-bindgen = "0.42.1" features = ["serde"] version = "0.4" -[dependencies.hyperprocess_macro] -git = "https://github.com/hyperware-ai/hyperprocess-macro" -rev = "98fac21" +[dependencies.hyperapp_macro] +git = "https://github.com/hyperware-ai/hyperapp-macro" +rev = "5c7cc7a" [dependencies.hyperware-anthropic-sdk] git = "https://github.com/hyperware-ai/hyperware-anthropic-sdk" -rev = "607bbc6" +rev = "1fc7b0c" [dependencies.hyperware-parse-wit] path = "../crates/hyperware-parse-wit" @@ -27,7 +27,7 @@ path = "../crates/hyperware-parse-wit" [dependencies.hyperware_process_lib] features = ["hyperapp"] git = "https://github.com/hyperware-ai/process_lib" -rev = "232fe25" +rev = "41f25ce" [dependencies.serde] features = ["derive"] @@ -45,6 +45,7 @@ features = ["serde"] version = "0.220.0" [features] +public-mode = [] simulation-mode = [] [lib] diff --git a/hyperdrive/packages/spider/spider/src/lib.rs b/hyperdrive/packages/spider/spider/src/lib.rs index 19123e8ed..7f4b0dceb 100644 --- a/hyperdrive/packages/spider/spider/src/lib.rs +++ b/hyperdrive/packages/spider/spider/src/lib.rs @@ -6,7 +6,8 @@ use chrono::Utc; use serde_json::{json, Value}; use uuid::Uuid; -use hyperprocess_macro::hyperprocess; +#[cfg(feature = "public-mode")] +use hyperware_process_lib::hyperapp::{get_http_request, get_request_header, get_ws_channel_addr}; use hyperware_process_lib::{ homepage::add_to_homepage, http::{ @@ -14,7 +15,8 @@ use hyperware_process_lib::{ server::{send_ws_push, WsMessageType}, }, hyperapp::{add_response_header, source}, - our, println, Address, LazyLoadBlob, ProcessId, Request, + logging::{debug, error, info, warn}, + our, Address, LazyLoadBlob, ProcessId, Request, }; #[cfg(not(feature = "simulation-mode"))] use spider_caller_utils::anthropic_api_key_manager::request_api_key_remote_rpc; @@ -23,6 +25,8 @@ mod provider; use provider::create_llm_provider; mod types; +#[cfg(feature = "public-mode")] +use types::RateLimitError; use types::{ AddMcpServerReq, ApiKey, ApiKeyInfo, ChatClient, ChatReq, ChatRes, ConfigRes, ConnectMcpServerReq, Conversation, ConversationMetadata, CreateSpiderKeyReq, @@ -67,7 +71,7 @@ const HYPERGRID: &str = "operator:hypergrid:ware.hypr"; const TODO: &str = "todo:todo:ware.hypr"; const TTSTT: (&str, &str, &str) = ("ttstt", "spider", "sys"); -#[hyperprocess( +#[hyperapp_macro::hyperapp( name = "Spider", ui = Some(HttpBindingConfig::default()), endpoints = vec![ @@ -96,7 +100,7 @@ impl SpiderState { const RETRY_DELAY_S: u64 = 2; const TIMEOUT_S: u64 = 15; - println!("Spider: Waiting for hypermap-cacher to be ready..."); + info!("Waiting for hypermap-cacher to be ready..."); loop { // Create GetStatus request JSON @@ -113,8 +117,8 @@ impl SpiderState { if response_str.contains("IsStarting") || response_str.contains(r#""IsStarting""#) { - println!( - "Spider: hypermap-cacher is still starting (attempt {}). Retrying in {}s...", + debug!( + "hypermap-cacher is still starting (attempt {}). Retrying in {}s...", attempt, RETRY_DELAY_S ); std::thread::sleep(std::time::Duration::from_secs(RETRY_DELAY_S)); @@ -125,25 +129,25 @@ impl SpiderState { if response_str.contains("GetStatus") || response_str.contains("last_cached_block") { - println!("Spider: hypermap-cacher is ready!"); + info!("hypermap-cacher is ready!"); break; } } // If we get here, we got some response we don't understand, but cacher is at least responding - println!("Spider: hypermap-cacher responded, proceeding with initialization"); + info!("hypermap-cacher responded, proceeding with initialization"); break; } Ok(Err(e)) => { - println!( - "Spider: Error response from hypermap-cacher (attempt {}): {:?}", + warn!( + "Error response from hypermap-cacher (attempt {}): {:?}", attempt, e ); std::thread::sleep(std::time::Duration::from_secs(RETRY_DELAY_S)); attempt += 1; } Err(e) => { - println!( - "Spider: Failed to contact hypermap-cacher (attempt {}): {:?}", + warn!( + "Failed to contact hypermap-cacher (attempt {}): {:?}", attempt, e ); std::thread::sleep(std::time::Duration::from_secs(RETRY_DELAY_S)); @@ -164,7 +168,7 @@ impl SpiderState { self.next_channel_id = 1000; // Start channel IDs at 1000 let our_node = our().node.clone(); - println!("Spider MCP client initialized on node: {}", our_node); + info!("MCP client initialized on node: {}", our_node); // Register Build Container tool provider let build_container_provider = BuildContainerToolProvider::new(); @@ -201,10 +205,10 @@ impl SpiderState { }; self.mcp_servers.push(build_container_server); - println!("Spider: Build Container MCP server initialized"); + info!("Build Container MCP server initialized"); } else { // Server exists, refresh its tools from the provider - println!("Spider: Refreshing Build Container tools on startup"); + debug!("Refreshing Build Container tools on startup"); // Get fresh tools from provider let build_container_provider = BuildContainerToolProvider::new(); @@ -217,8 +221,8 @@ impl SpiderState { .find(|s| s.id == "build_container") { server.tools = fresh_tools; - println!( - "Spider: Build Container tools refreshed with {} tools", + debug!( + "Build Container tools refreshed with {} tools", server.tools.len() ); } @@ -261,18 +265,18 @@ impl SpiderState { connected: true, // Always mark as connected }; self.mcp_servers.push(hyperware_server); - println!("Spider: Hyperware MCP server initialized"); + info!("Hyperware MCP server initialized"); } else { // Server exists, refresh its tools from the provider - println!("Spider: Refreshing Hyperware tools on startup"); + debug!("Refreshing Hyperware tools on startup"); // Get fresh tools from provider let hyperware_provider = HyperwareToolProvider::new(); let fresh_tools = hyperware_provider.get_tools(self); // Update the existing server's tools if let Some(server) = self.mcp_servers.iter_mut().find(|s| s.id == "hyperware") { server.tools = fresh_tools; - println!( - "Spider: Hyperware tools refreshed with {} tools", + debug!( + "Hyperware tools refreshed with {} tools", server.tools.len() ); } @@ -306,9 +310,9 @@ impl SpiderState { }; self.mcp_servers.push(hypergrid_server); - println!("Spider: Hypergrid MCP server initialized (unconfigured)"); + info!("Hypergrid MCP server initialized (unconfigured)"); } else { - println!("Spider: Refreshing Hypergrid tools on startup"); + debug!("Refreshing Hypergrid tools on startup"); // Get fresh tools from provider let hypergrid_provider = HypergridToolProvider::new("hypergrid_default".to_string()); @@ -321,8 +325,8 @@ impl SpiderState { .find(|s| s.id == "hypergrid_default") { server.tools = fresh_tools; - println!( - "Spider: Hypergrid tools refreshed with {} tools", + debug!( + "Hypergrid tools refreshed with {} tools", server.tools.len() ); } @@ -330,12 +334,12 @@ impl SpiderState { // Restore hypergrid connections for configured servers for server in self.mcp_servers.iter() { if server.transport.transport_type == "hypergrid" { - println!( - "Spider: Found hypergrid server '{}' (id: {})", + debug!( + "Found hypergrid server '{}' (id: {})", server.name, server.id ); - println!(" - URL: {:?}", server.transport.url); - println!( + debug!(" - URL: {:?}", server.transport.url); + debug!( " - Token: {}", server .transport @@ -348,9 +352,9 @@ impl SpiderState { }) .unwrap_or_else(|| "None".to_string()) ); - println!(" - Client ID: {:?}", server.transport.hypergrid_client_id); - println!(" - Node: {:?}", server.transport.hypergrid_node); - println!(" - Tools: {} available", server.tools.len()); + debug!(" - Client ID: {:?}", server.transport.hypergrid_client_id); + debug!(" - Node: {:?}", server.transport.hypergrid_node); + debug!(" - Tools: {} available", server.tools.len()); if let (Some(url), Some(token), Some(client_id), Some(node)) = ( &server.transport.url, @@ -371,15 +375,12 @@ impl SpiderState { }; self.hypergrid_connections .insert(server.id.clone(), hypergrid_conn); - println!( - "Spider: ✅ Restored hypergrid connection for {} ({})", + info!( + "Restored hypergrid connection for {} ({})", server.name, node ); } else { - println!( - "Spider: ⚠️ Hypergrid server '{}' is not fully configured", - server.name - ); + warn!("Hypergrid server '{}' is not fully configured", server.name); } } } @@ -410,9 +411,9 @@ impl SpiderState { }; self.spider_api_keys.push(admin_key.clone()); - println!("Spider: Created admin GUI key: {}", admin_key.key); + info!("Created admin GUI key: {}", admin_key.key); } else { - println!("Spider: Admin GUI key already exists"); + debug!("Admin GUI key already exists"); } // VFS directory creation will be handled when actually saving files @@ -423,7 +424,7 @@ impl SpiderState { self.mcp_servers.iter().map(|s| s.id.clone()).collect(); for server_id in servers_to_reconnect { - println!("Auto-reconnecting to MCP server: {}", server_id); + debug!("Auto-reconnecting to MCP server: {}", server_id); // Retry logic with exponential backoff let max_retries = 10; @@ -440,7 +441,7 @@ impl SpiderState { }) .map(|k| k.key.clone()) .unwrap_or_else(|| { - println!("Warning: No admin key found for auto-reconnect"); + warn!("No admin key found for auto-reconnect"); String::new() }); @@ -450,18 +451,18 @@ impl SpiderState { }; match self.connect_mcp_server(connect_request).await { Ok(msg) => { - println!("Auto-reconnect successful: {}", msg); + debug!("Auto-reconnect successful: {}", msg); success = true; break; } Err(e) => { - println!( + warn!( "Failed to auto-reconnect to MCP server {} (attempt {}/{}): {}", server_id, attempt, max_retries, e ); if attempt < max_retries { - println!("Retrying in {} ms...", retry_delay_ms); + debug!("Retrying in {} ms...", retry_delay_ms); let _ = hyperware_process_lib::hyperapp::sleep(retry_delay_ms).await; // Exponential backoff with max delay of 10 seconds @@ -472,7 +473,7 @@ impl SpiderState { } if !success { - println!( + error!( "Failed to reconnect to MCP server {} after {} attempts", server_id, max_retries ); @@ -482,7 +483,7 @@ impl SpiderState { // Check if we need to request a free API key #[cfg(not(feature = "simulation-mode"))] if self.api_keys.is_empty() { - println!("Spider: No API keys configured, requesting free trial key..."); + info!("No API keys configured, requesting free trial key..."); let api_key_dispenser = Address::new(API_KEY_DISPENSER_NODE, API_KEY_DISPENSER_PROCESS_ID); @@ -490,7 +491,7 @@ impl SpiderState { // Call the RPC function to request an API key match request_api_key_remote_rpc(&api_key_dispenser).await { Ok(Ok(api_key)) => { - println!("Spider: Successfully obtained free trial API key"); + info!("Successfully obtained free trial API key"); // Add the key to our API keys let encrypted_key = encrypt_key(&api_key); self.api_keys.push(( @@ -509,15 +510,15 @@ impl SpiderState { self.show_trial_key_notification = true; } Ok(Err(e)) => { - println!("Spider: API key dispenser returned error: {}", e); + error!("API key dispenser returned error: {}", e); } Err(e) => { - println!("Spider: API key dispenser send error: {}", e); + error!("API key dispenser send error: {}", e); } } } - println!("Spider initialization complete"); + info!("Initialization complete"); } #[ws] @@ -527,13 +528,13 @@ impl SpiderState { message_type: WsMessageType, blob: LazyLoadBlob, ) { - println!("handle_websocket {channel_id}"); + debug!("handle_websocket {channel_id}"); match message_type { WsMessageType::Text | WsMessageType::Binary => { let message_bytes = blob.bytes.clone(); let message_str = String::from_utf8(message_bytes).unwrap_or_default(); - println!("handle_websocket: got {message_str}"); + debug!("handle_websocket: got {message_str}"); // Parse the incoming message using typed enum match serde_json::from_str::(&message_str) { @@ -544,6 +545,16 @@ impl SpiderState { if self.validate_spider_key(&api_key) && self.validate_permission(&api_key, "write") { + // Capture IP address for rate limiting in public-mode + #[cfg(feature = "public-mode")] + let ip_address = Self::get_client_ip(Some(channel_id)); + #[cfg(feature = "public-mode")] + debug!("[RATE-LIMIT] WebSocket Auth - captured IP for channel {}: {:?}", channel_id, ip_address); + #[cfg(not(feature = "public-mode"))] + let ip_address: Option< + String, + > = None; + self.chat_clients.insert( channel_id, ChatClient { @@ -551,6 +562,7 @@ impl SpiderState { api_key: api_key.clone(), conversation_id: None, connected_at: Utc::now().timestamp() as u64, + ip_address, }, ); @@ -608,6 +620,48 @@ impl SpiderState { return; } + // Rate limiting for public-mode + #[cfg(feature = "public-mode")] + { + debug!("[RATE-LIMIT] Checking rate limit for chat message on channel {}", channel_id); + debug!( + "[RATE-LIMIT] Client IP address: {:?}", + client.ip_address + ); + if let Some(ref ip) = client.ip_address { + if let Err(e) = self.check_rate_limit(ip) { + debug!( + "[RATE-LIMIT] Rate limit BLOCKED for {}: {}", + ip, e + ); + // Send structured error for frontend + let error = RateLimitError { + error_type: "OutOfRequests".to_string(), + message: e, + retry_after_seconds: self + .get_retry_after_seconds(ip), + }; + let response = WsServerMessage::Error { + error: serde_json::to_string(&error).unwrap(), + }; + let json = + serde_json::to_string(&response).unwrap(); + send_ws_push( + channel_id, + WsMessageType::Text, + LazyLoadBlob::new( + Some("application/json"), + json, + ), + ); + return; + } + debug!("[RATE-LIMIT] Rate limit OK for {}", ip); + } else { + warn!("[RATE-LIMIT] No IP address available - rate limiting SKIPPED!"); + } + } + // Convert WsChatPayload to ChatReq let chat_request = ChatReq { api_key: client.api_key, @@ -669,10 +723,7 @@ impl SpiderState { self.active_chat_cancellation.get(&channel_id) { cancel_flag.store(true, Ordering::Relaxed); - println!( - "Spider: Cancelling chat request for channel {}", - channel_id - ); + debug!("Cancelling chat request for channel {}", channel_id); // Send cancellation confirmation let response = WsServerMessage::Status { @@ -700,8 +751,8 @@ impl SpiderState { } } Err(e) => { - println!( - "Spider: Failed to parse WebSocket message from channel {}: {}", + warn!( + "Failed to parse WebSocket message from channel {}: {}", channel_id, e ); let error_response = WsServerMessage::Error { @@ -719,7 +770,7 @@ impl SpiderState { WsMessageType::Close => { // Clean up client connection self.chat_clients.remove(&channel_id); - println!("Chat client {} disconnected", channel_id); + debug!("Chat client {} disconnected", channel_id); } WsMessageType::Ping | WsMessageType::Pong => { // Handle ping/pong for keepalive @@ -736,31 +787,28 @@ impl SpiderState { ) { match message_type { WsMessageType::Text | WsMessageType::Binary => { - println!("Got WS Text"); + debug!("Got WS Text"); // Handle incoming message from the WebSocket server let message_bytes = blob.bytes; // Parse the message as JSON let message_str = String::from_utf8(message_bytes).unwrap_or_default(); - println!( - "Spider: Received WebSocket message on channel {}: {}", + debug!( + "Received WebSocket message on channel {}: {}", channel_id, message_str ); if let Ok(json_msg) = serde_json::from_str::(&message_str) { self.handle_mcp_message(channel_id, json_msg); } else { - println!( - "Spider: Failed to parse MCP message from channel {}: {}", + warn!( + "Failed to parse MCP message from channel {}: {}", channel_id, message_str ); } } WsMessageType::Close => { // Handle connection close - println!( - "Spider: WebSocket connection closed for channel {}", - channel_id - ); + debug!("WebSocket connection closed for channel {}", channel_id); // Find and disconnect the server if let Some(conn) = self.ws_connections.remove(&channel_id) { @@ -769,7 +817,7 @@ impl SpiderState { self.mcp_servers.iter_mut().find(|s| s.id == conn.server_id) { server.connected = false; - println!("Spider: MCP server {} disconnected", server.name); + info!("MCP server {} disconnected", server.name); } // Also remove any ws_mcp server that was created for this connection @@ -1360,6 +1408,32 @@ impl SpiderState { #[local] #[http] async fn chat(&mut self, request: ChatReq) -> Result { + // Rate limiting for public-mode + #[cfg(feature = "public-mode")] + { + debug!("[RATE-LIMIT] HTTP /chat endpoint - checking rate limit"); + let ip = Self::get_client_ip(None); + debug!("[RATE-LIMIT] HTTP /chat - client IP: {:?}", ip); + if let Some(ref ip_addr) = ip { + if let Err(e) = self.check_rate_limit(ip_addr) { + warn!( + "[RATE-LIMIT] HTTP /chat - rate limit BLOCKED for {}: {}", + ip_addr, e + ); + // Return structured error for frontend + let error = RateLimitError { + error_type: "OutOfRequests".to_string(), + message: e, + retry_after_seconds: self.get_retry_after_seconds(ip_addr), + }; + return Err(serde_json::to_string(&error).unwrap()); + } + debug!("[RATE-LIMIT] HTTP /chat - rate limit OK for {}", ip_addr); + } else { + warn!("[RATE-LIMIT] HTTP /chat - WARNING: No IP address available - rate limiting SKIPPED!"); + } + } + // Use the shared internal chat processing logic (without WebSocket streaming) let source = source(); if source.publisher() == "sys" @@ -1536,6 +1610,152 @@ impl SpiderState { .any(|k| k.key == key && k.permissions.contains(&permission.to_string())) } + /// Get client IP address from request headers (proxy-aware) or socket address + /// For WebSocket contexts, pass the channel_id to look up the stored socket address. + #[cfg(feature = "public-mode")] + fn get_client_ip(channel_id: Option) -> Option { + debug!("=== get_client_ip DEBUG ==="); + + // Check if we have HTTP context at all + let http_req = get_http_request(); + debug!( + "[RATE-LIMIT] HTTP request context exists: {}", + http_req.is_some() + ); + + // First try X-Forwarded-For header (proxy scenario) + let xff = get_request_header("X-Forwarded-For"); + debug!("[RATE-LIMIT] X-Forwarded-For header: {:?}", xff); + if let Some(ref xff_val) = xff { + // X-Forwarded-For can be comma-separated; take the first (original client) + if let Some(first_ip) = xff_val.split(',').next() { + let ip = first_ip.trim().to_string(); + debug!("[RATE-LIMIT] Extracted IP from X-Forwarded-For: {}", ip); + debug!("=== END get_client_ip DEBUG ==="); + return Some(ip); + } + } + + // Fallback to X-Real-IP header + let real_ip = get_request_header("X-Real-IP"); + debug!("[RATE-LIMIT] X-Real-IP header: {:?}", real_ip); + if let Some(ref real_ip_val) = real_ip { + let ip = real_ip_val.trim().to_string(); + debug!("[RATE-LIMIT] Using X-Real-IP: {}", ip); + debug!("=== END get_client_ip DEBUG ==="); + return Some(ip); + } + + // Try Cf-Connecting-Ip (Cloudflare) - just for debug logging + let cf_ip = get_request_header("Cf-Connecting-Ip"); + debug!("[RATE-LIMIT] Cf-Connecting-Ip header: {:?}", cf_ip); + + // Try HTTP socket address + let http_socket_result = http_req.and_then(|req| req.source_socket_addr().ok()); + debug!("[RATE-LIMIT] HTTP socket address: {:?}", http_socket_result); + if let Some(addr) = http_socket_result { + let ip = addr.ip().to_string(); + debug!("[RATE-LIMIT] Using HTTP socket address: {}", ip); + debug!("=== END get_client_ip DEBUG ==="); + return Some(ip); + } + + // Fallback to WebSocket channel address (for WS contexts) + if let Some(ch_id) = channel_id { + let ws_addr = get_ws_channel_addr(ch_id); + debug!( + "[RATE-LIMIT] WebSocket channel {} address: {:?}", + ch_id, ws_addr + ); + if let Some(addr_str) = ws_addr { + // Parse socket address string to extract IP (format: "ip:port") + if let Some(ip) = addr_str.split(':').next() { + let ip = ip.to_string(); + debug!("[RATE-LIMIT] Using WebSocket channel address: {}", ip); + debug!("=== END get_client_ip DEBUG ==="); + return Some(ip); + } + } + } + + debug!("[RATE-LIMIT] Final get_client_ip result: None"); + debug!("=== END get_client_ip DEBUG ==="); + None + } + + /// Check rate limit for an IP address. Returns Ok(()) if allowed, Err with message if rate limited. + #[cfg(feature = "public-mode")] + fn check_rate_limit(&mut self, ip: &str) -> Result<(), String> { + const MAX_CHATS_PER_DAY: usize = 3; + const WINDOW_SECONDS: u64 = 24 * 60 * 60; // 24 hours + + debug!("[RATE-LIMIT] check_rate_limit called for IP: {}", ip); + debug!( + "[RATE-LIMIT] Current ip_chat_counts keys: {:?}", + self.ip_chat_counts.keys().collect::>() + ); + + let now = Utc::now().timestamp() as u64; + let cutoff = now - WINDOW_SECONDS; + + // Get or create entry for this IP + let timestamps = self + .ip_chat_counts + .entry(ip.to_string()) + .or_insert_with(Vec::new); + + // Remove expired timestamps + let before_cleanup = timestamps.len(); + timestamps.retain(|&t| t > cutoff); + debug!( + "[RATE-LIMIT] Timestamps for {} after cleanup: {} (was {})", + ip, + timestamps.len(), + before_cleanup + ); + + // Check if limit exceeded + if timestamps.len() >= MAX_CHATS_PER_DAY { + warn!( + "[RATE-LIMIT] RATE LIMIT EXCEEDED for {}: {} >= {}", + ip, + timestamps.len(), + MAX_CHATS_PER_DAY + ); + return Err(format!( + "Rate limit exceeded: {} chats allowed per 24 hours. Try again later.", + MAX_CHATS_PER_DAY + )); + } + + // Record this chat + timestamps.push(now); + debug!( + "[RATE-LIMIT] Recorded chat for {}. New count: {}/{}", + ip, + timestamps.len(), + MAX_CHATS_PER_DAY + ); + Ok(()) + } + + /// Get seconds until the oldest chat request expires (for retry_after_seconds) + #[cfg(feature = "public-mode")] + fn get_retry_after_seconds(&self, ip: &str) -> Option { + const WINDOW_SECONDS: u64 = 24 * 60 * 60; + + if let Some(timestamps) = self.ip_chat_counts.get(ip) { + if let Some(&oldest) = timestamps.iter().min() { + let now = Utc::now().timestamp() as u64; + let expires_at = oldest + WINDOW_SECONDS; + if expires_at > now { + return Some(expires_at - now); + } + } + } + None + } + fn cleanup_disconnected_build_containers(&mut self) { // Find all ws_mcp_* servers that are disconnected let disconnected_server_ids: Vec = self @@ -1549,7 +1769,7 @@ impl SpiderState { .collect(); if !disconnected_server_ids.is_empty() { - println!( + info!( "Spider: Cleaning up {} disconnected Build Container MCP connections", disconnected_server_ids.len() ); @@ -1560,7 +1780,7 @@ impl SpiderState { if let Ok(old_channel_id) = channel_str.parse::() { // Remove from ws_connections if it exists if self.ws_connections.remove(&old_channel_id).is_some() { - println!( + debug!( "Spider: Removed ws_connection for channel {}", old_channel_id ); @@ -1583,12 +1803,12 @@ impl SpiderState { // Remove the server from mcp_servers list self.mcp_servers.retain(|s| s.id != server_id); - println!("Spider: Removed Build Container MCP server {}", server_id); + info!("Spider: Removed Build Container MCP server {}", server_id); } - println!("Spider: Build Container cleanup complete"); + info!("Spider: Build Container cleanup complete"); } else { - println!("Spider: No disconnected Build Container MCP connections to clean up"); + debug!("Spider: No disconnected Build Container MCP connections to clean up"); } } @@ -1792,7 +2012,7 @@ impl SpiderState { .unwrap_or("Unknown Key".to_string()) }; - println!( + info!( "Spider: Starting new conversation {} with provider {} (key: {})", conversation_id, llm_provider, key_name ); @@ -1857,7 +2077,7 @@ impl SpiderState { // Convert audio to text using ttstt match self.convert_audio_to_text(&message.content).await { Ok(text) => { - println!("Spider: Converted audio to text: {}", text); + debug!("Spider: Converted audio to text: {}", text); message.content = MessageContent::Text(text); } Err(e) => { @@ -1981,7 +2201,7 @@ impl SpiderState { if let Some(cancel_flag) = self.active_chat_cancellation.get(&ch_id) { let is_cancelled = cancel_flag.load(Ordering::Relaxed); if is_cancelled { - println!( + info!( "Spider: Chat request cancelled at iteration {}", iteration_count ); @@ -2018,7 +2238,7 @@ impl SpiderState { Ok(response) => response, Err(e) => { // Log the error for debugging - println!("Spider: Error calling LLM provider {}: {}", llm_provider, e); + error!("Spider: Error calling LLM provider {}: {}", llm_provider, e); // Check if it's an API key error if e.contains("401") || e.contains("unauthorized") || e.contains("api key") { @@ -2045,26 +2265,26 @@ impl SpiderState { }; // Check if the response contains tool calls - println!("[DEBUG] LLM response received:"); - println!( + debug!("[DEBUG] LLM response received:"); + debug!( "[DEBUG] - content: {}", match &llm_response.content { MessageContent::Text(t) => t.as_str(), MessageContent::Audio(_) | MessageContent::BaseSixFourAudio(_) => "