diff --git a/Cargo.lock b/Cargo.lock index df88d28c3b..25190c61aa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -865,28 +865,6 @@ dependencies = [ "pin-project-lite", ] -[[package]] -name = "async-stream" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" -dependencies = [ - "async-stream-impl", - "futures-core", - "pin-project-lite", -] - -[[package]] -name = "async-stream-impl" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.111", -] - [[package]] name = "async-trait" version = "0.1.89" @@ -2218,12 +2196,6 @@ dependencies = [ "pin-project-lite", ] -[[package]] -name = "fallible-iterator" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" - [[package]] name = "fastbloom" version = "0.14.0" @@ -2395,6 +2367,7 @@ dependencies = [ "serial_test", "solana-account-decoder", "solana-client", + "solana-commitment-config", "solana-program", "solana-pubkey 2.4.0", "solana-rpc-client-api", @@ -2414,12 +2387,8 @@ version = "2.0.0" dependencies = [ "account-compression", "anchor-lang", - "anyhow", - "async-stream", "async-trait", "bb8", - "bs58", - "futures", "governor 0.8.1", "light-account-checks", "light-batched-merkle-tree", @@ -2428,25 +2397,19 @@ dependencies = [ "light-concurrent-merkle-tree", "light-hash-set", "light-hasher", - "light-indexed-array", "light-indexed-merkle-tree", - "light-merkle-tree-metadata", "light-merkle-tree-reference", "light-prover-client", "light-registry", "light-sdk", "light-sparse-merkle-tree", "light-token-interface", - "num-bigint 0.4.6", "num-traits", - "serde", - "serde_json", "solana-instruction", "solana-pubkey 2.4.0", "solana-sdk", "thiserror 2.0.17", "tokio", - "tokio-postgres", "tracing", ] @@ -2989,6 +2952,7 @@ dependencies = [ "http 1.4.0", "http-body 1.0.1", "httparse", + "httpdate", "itoa", "pin-project-lite", "pin-utils", @@ -3450,7 +3414,6 @@ checksum = "df15f6eac291ed1cf25865b1ee60399f57e7c227e7f51bdbd4c5270396a9ed50" dependencies = [ "bitflags 2.10.0", "libc", - "redox_syscall 0.6.0", ] [[package]] @@ -4444,16 +4407,6 @@ dependencies = [ "regex-automata", ] -[[package]] -name = "md-5" -version = "0.10.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" -dependencies = [ - "cfg-if", - "digest 0.10.7", -] - [[package]] name = "memchr" version = "2.7.6" @@ -4902,7 +4855,7 @@ checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.5.18", + "redox_syscall", "smallvec", "windows-link", ] @@ -4955,25 +4908,6 @@ dependencies = [ "num", ] -[[package]] -name = "phf" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf" -dependencies = [ - "phf_shared", - "serde", -] - -[[package]] -name = "phf_shared" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e57fef6bc5981e38c2ce2d63bfa546861309f875b8a75f092d1d54ae2d64f266" -dependencies = [ - "siphasher 1.0.1", -] - [[package]] name = "photon-api" version = "0.53.0" @@ -5121,35 +5055,6 @@ dependencies = [ "portable-atomic", ] -[[package]] -name = "postgres-protocol" -version = "0.6.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbef655056b916eb868048276cfd5d6a7dea4f81560dfd047f97c8c6fe3fcfd4" -dependencies = [ - "base64 0.22.1", - "byteorder", - "bytes", - "fallible-iterator", - "hmac 0.12.1", - "md-5", - "memchr", - "rand 0.9.2", - "sha2 0.10.9", - "stringprep", -] - -[[package]] -name = "postgres-types" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef4605b7c057056dd35baeb6ac0c0338e4975b1f2bef0f65da953285eb007095" -dependencies = [ - "bytes", - "fallible-iterator", - "postgres-protocol", -] - [[package]] name = "potential_utf" version = "0.1.4" @@ -5525,15 +5430,6 @@ dependencies = [ "bitflags 2.10.0", ] -[[package]] -name = "redox_syscall" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec96166dafa0886eb81fe1c0a388bece180fbef2135f97c1e2cf8302e74b43b5" -dependencies = [ - "bitflags 2.10.0", -] - [[package]] name = "redox_users" version = "0.4.6" @@ -10055,17 +9951,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" -[[package]] -name = "stringprep" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" -dependencies = [ - "unicode-bidi", - "unicode-normalization", - "unicode-properties", -] - [[package]] name = "strsim" version = "0.8.0" @@ -10549,32 +10434,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "tokio-postgres" -version = "0.7.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b40d66d9b2cfe04b628173409368e58247e8eddbbd3b0e6c6ba1d09f20f6c9e" -dependencies = [ - "async-trait", - "byteorder", - "bytes", - "fallible-iterator", - "futures-channel", - "futures-util", - "log", - "parking_lot", - "percent-encoding", - "phf", - "pin-project-lite", - "postgres-protocol", - "postgres-types", - "rand 0.9.2", - "socket2 0.6.1", - "tokio", - "tokio-util 0.7.17", - "whoami", -] - [[package]] name = "tokio-rustls" version = "0.24.1" @@ -10970,12 +10829,6 @@ version = "2.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75b844d17643ee918803943289730bec8aac480150456169e647ed0b576ba539" -[[package]] -name = "unicode-bidi" -version = "0.3.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" - [[package]] name = "unicode-ident" version = "1.0.22" @@ -10991,12 +10844,6 @@ dependencies = [ "tinyvec", ] -[[package]] -name = "unicode-properties" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" - [[package]] name = "unicode-segmentation" version = "1.12.0" @@ -11165,6 +11012,8 @@ dependencies = [ "http 1.4.0", "http-body 1.0.1", "http-body-util", + "hyper 1.8.1", + "hyper-util", "log", "mime", "mime_guess", @@ -11201,12 +11050,6 @@ dependencies = [ "wit-bindgen", ] -[[package]] -name = "wasite" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" - [[package]] name = "wasm-bindgen" version = "0.2.106" @@ -11318,17 +11161,6 @@ dependencies = [ "rustls-pki-types", ] -[[package]] -name = "whoami" -version = "1.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d4a4db5077702ca3015d3d02d74974948aba2ad9e12ab7df718ee64ccd7e97d" -dependencies = [ - "libredox", - "wasite", - "web-sys", -] - [[package]] name = "winapi" version = "0.3.9" diff --git a/forester-utils/Cargo.toml b/forester-utils/Cargo.toml index 9bbb143256..72c92b7e64 100644 --- a/forester-utils/Cargo.toml +++ b/forester-utils/Cargo.toml @@ -13,20 +13,16 @@ devenv = ["v2", "light-client/devenv", "light-prover-client/devenv"] v2 = ["light-client/v2"] [dependencies] - light-hash-set = { workspace = true } light-hasher = { workspace = true, features = ["poseidon"] } light-concurrent-merkle-tree = { workspace = true } light-indexed-merkle-tree = { workspace = true } -light-indexed-array = { workspace = true } light-compressed-account = { workspace = true, features = ["std"] } light-batched-merkle-tree = { workspace = true } -light-merkle-tree-metadata = { workspace = true } light-merkle-tree-reference = { workspace = true } light-sparse-merkle-tree = { workspace = true } light-account-checks = { workspace = true } light-sdk = { workspace = true } - light-client = { workspace = true } light-prover-client = { workspace = true } light-registry = { workspace = true, features = ["cpi"] } @@ -35,29 +31,15 @@ light-token-interface = { workspace = true } solana-instruction = { workspace = true } solana-pubkey = { workspace = true } - -tokio = { workspace = true } -futures = { workspace = true } -async-stream = "0.3" - -anchor-lang = { workspace = true } - solana-sdk = { workspace = true } +anchor-lang = { workspace = true } -thiserror = { workspace = true } -anyhow = { workspace = true } - -tracing = { workspace = true } - -num-traits = { workspace = true } -num-bigint = { workspace = true } - +tokio = { workspace = true } bb8 = { workspace = true } async-trait = { workspace = true } governor = { workspace = true } +num-traits = { workspace = true } + +thiserror = { workspace = true } +tracing = { workspace = true } -[dev-dependencies] -tokio-postgres = "0.7" -bs58 = { workspace = true } -serde = { workspace = true } -serde_json = { workspace = true } diff --git a/forester-utils/src/account_zero_copy.rs b/forester-utils/src/account_zero_copy.rs index b1cedc2b3c..bd72ec245f 100644 --- a/forester-utils/src/account_zero_copy.rs +++ b/forester-utils/src/account_zero_copy.rs @@ -1,13 +1,23 @@ use std::{fmt, marker::PhantomData, mem, pin::Pin}; -use account_compression::processor::initialize_address_merkle_tree::Pubkey; use light_client::rpc::Rpc; -use light_concurrent_merkle_tree::copy::ConcurrentMerkleTreeCopy; +use light_concurrent_merkle_tree::{ + copy::ConcurrentMerkleTreeCopy, errors::ConcurrentMerkleTreeError, +}; use light_hash_set::HashSet; use light_hasher::Hasher; -use light_indexed_merkle_tree::copy::IndexedMerkleTreeCopy; +use light_indexed_merkle_tree::{copy::IndexedMerkleTreeCopy, errors::IndexedMerkleTreeError}; use num_traits::{CheckedAdd, CheckedSub, ToBytes, Unsigned}; -use solana_sdk::account::Account; +use solana_sdk::{account::Account, pubkey::Pubkey}; +use thiserror::Error; + +#[derive(Error, Debug)] +pub enum AccountZeroCopyError { + #[error("RPC error: {0}")] + RpcError(String), + #[error("Account not found: {0}")] + AccountNotFound(Pubkey), +} #[derive(Debug, Clone)] pub struct AccountZeroCopy<'a, T> { @@ -17,15 +27,23 @@ pub struct AccountZeroCopy<'a, T> { } impl<'a, T> AccountZeroCopy<'a, T> { - pub async fn new(rpc: &mut R, address: Pubkey) -> AccountZeroCopy<'a, T> { - let account = Box::pin(rpc.get_account(address).await.unwrap().unwrap()); + pub async fn new( + rpc: &mut R, + address: Pubkey, + ) -> Result, AccountZeroCopyError> { + let account = rpc + .get_account(address) + .await + .map_err(|e| AccountZeroCopyError::RpcError(e.to_string()))? + .ok_or(AccountZeroCopyError::AccountNotFound(address))?; + let account = Box::pin(account); let deserialized = account.data[8..].as_ptr() as *const T; - Self { + Ok(Self { account, deserialized, _phantom_data: PhantomData, - } + }) } // Safe method to access `deserialized` ensuring the lifetime is respected @@ -46,10 +64,19 @@ impl<'a, T> AccountZeroCopy<'a, T> { /// * The account data is aligned. /// /// Is the caller's responsibility. -pub async unsafe fn get_hash_set(rpc: &mut R, pubkey: Pubkey) -> HashSet { - let mut data = rpc.get_account(pubkey).await.unwrap().unwrap().data.clone(); +pub async unsafe fn get_hash_set( + rpc: &mut R, + pubkey: Pubkey, +) -> Result { + let account = rpc + .get_account(pubkey) + .await + .map_err(|e| AccountZeroCopyError::RpcError(e.to_string()))? + .ok_or(AccountZeroCopyError::AccountNotFound(pubkey))?; + let mut data = account.data.clone(); - HashSet::from_bytes_copy(&mut data[8 + mem::size_of::()..]).unwrap() + HashSet::from_bytes_copy(&mut data[8 + mem::size_of::()..]) + .map_err(|e| AccountZeroCopyError::RpcError(format!("HashSet parse error: {:?}", e))) } /// Fetches the given account, then copies and serializes it as a @@ -57,14 +84,20 @@ pub async unsafe fn get_hash_set(rpc: &mut R, pubkey: Pubkey) -> Hash pub async fn get_concurrent_merkle_tree( rpc: &mut R, pubkey: Pubkey, -) -> ConcurrentMerkleTreeCopy +) -> Result, AccountZeroCopyError> where R: Rpc, H: Hasher, { - let account = rpc.get_account(pubkey).await.unwrap().unwrap(); + let account = rpc + .get_account(pubkey) + .await + .map_err(|e| AccountZeroCopyError::RpcError(e.to_string()))? + .ok_or(AccountZeroCopyError::AccountNotFound(pubkey))?; - ConcurrentMerkleTreeCopy::from_bytes_copy(&account.data[8 + mem::size_of::()..]).unwrap() + ConcurrentMerkleTreeCopy::from_bytes_copy(&account.data[8 + mem::size_of::()..]).map_err( + |e| AccountZeroCopyError::RpcError(format!("ConcurrentMerkleTree parse error: {:?}", e)), + ) } // TODO: do discriminator check /// Fetches the given account, then copies and serializes it as an @@ -72,7 +105,7 @@ where pub async fn get_indexed_merkle_tree( rpc: &mut R, pubkey: Pubkey, -) -> IndexedMerkleTreeCopy +) -> Result, AccountZeroCopyError> where R: Rpc, H: Hasher, @@ -87,7 +120,68 @@ where + Unsigned, usize: From, { - let account = rpc.get_account(pubkey).await.unwrap().unwrap(); + let account = rpc + .get_account(pubkey) + .await + .map_err(|e| AccountZeroCopyError::RpcError(e.to_string()))? + .ok_or(AccountZeroCopyError::AccountNotFound(pubkey))?; + + IndexedMerkleTreeCopy::from_bytes_copy(&account.data[8 + mem::size_of::()..]).map_err(|e| { + AccountZeroCopyError::RpcError(format!("IndexedMerkleTree parse error: {:?}", e)) + }) +} + +/// Parse ConcurrentMerkleTree from raw account data bytes. +pub fn parse_concurrent_merkle_tree_from_bytes( + data: &[u8], +) -> Result, ConcurrentMerkleTreeError> +where + H: Hasher, +{ + let offset = 8 + mem::size_of::(); + if data.len() <= offset { + return Err(ConcurrentMerkleTreeError::BufferSize(offset, data.len())); + } + ConcurrentMerkleTreeCopy::from_bytes_copy(&data[offset..]) +} - IndexedMerkleTreeCopy::from_bytes_copy(&account.data[8 + mem::size_of::()..]).unwrap() +/// Parse IndexedMerkleTree from raw account data byte +pub fn parse_indexed_merkle_tree_from_bytes( + data: &[u8], +) -> Result, IndexedMerkleTreeError> +where + H: Hasher, + I: CheckedAdd + + CheckedSub + + Copy + + Clone + + fmt::Debug + + PartialOrd + + ToBytes + + TryFrom + + Unsigned, + usize: From, +{ + let offset = 8 + mem::size_of::(); + if data.len() <= offset { + return Err(IndexedMerkleTreeError::ConcurrentMerkleTree( + ConcurrentMerkleTreeError::BufferSize(offset, data.len()), + )); + } + IndexedMerkleTreeCopy::from_bytes_copy(&data[offset..]) +} + +/// Parse HashSet from raw queue account data bytes +/// +/// # Safety +/// Same safety requirements as `get_hash_set`. +pub unsafe fn parse_hash_set_from_bytes( + data: &[u8], +) -> Result { + let offset = 8 + mem::size_of::(); + if data.len() <= offset { + return Err(light_hash_set::HashSetError::BufferSize(offset, data.len())); + } + let mut data_copy = data[offset..].to_vec(); + HashSet::from_bytes_copy(&mut data_copy) } diff --git a/forester-utils/src/address_merkle_tree_config.rs b/forester-utils/src/address_merkle_tree_config.rs index aa30f6282b..8d8018c92e 100644 --- a/forester-utils/src/address_merkle_tree_config.rs +++ b/forester-utils/src/address_merkle_tree_config.rs @@ -15,34 +15,39 @@ use solana_sdk::pubkey::Pubkey; use crate::account_zero_copy::{ get_concurrent_merkle_tree, get_hash_set, get_indexed_merkle_tree, AccountZeroCopy, + AccountZeroCopyError, }; pub async fn get_address_bundle_config( rpc: &mut R, address_bundle: AddressMerkleTreeAccounts, -) -> (AddressMerkleTreeConfig, AddressQueueConfig) { - let address_queue_meta_data = AccountZeroCopy::::new(rpc, address_bundle.queue) - .await - .deserialized() - .metadata; - let address_queue = unsafe { get_hash_set::(rpc, address_bundle.queue).await }; +) -> Result<(AddressMerkleTreeConfig, AddressQueueConfig), AccountZeroCopyError> { + // Get queue metadata - don't hold AccountZeroCopy across await points + let address_queue_meta_data = { + let account = AccountZeroCopy::::new(rpc, address_bundle.queue).await?; + account.deserialized().metadata + }; + let address_queue = + unsafe { get_hash_set::(rpc, address_bundle.queue).await? }; let queue_config = AddressQueueConfig { network_fee: Some(address_queue_meta_data.rollover_metadata.network_fee), // rollover_threshold: address_queue_meta_data.rollover_threshold, capacity: address_queue.get_capacity() as u16, sequence_threshold: address_queue.sequence_threshold as u64, }; - let address_tree_meta_data = - AccountZeroCopy::::new(rpc, address_bundle.merkle_tree) - .await - .deserialized() - .metadata; + // Get tree metadata - don't hold AccountZeroCopy across await points + let address_tree_meta_data = { + let account = + AccountZeroCopy::::new(rpc, address_bundle.merkle_tree) + .await?; + account.deserialized().metadata + }; let address_tree = get_indexed_merkle_tree::( rpc, address_bundle.merkle_tree, ) - .await; + .await?; let address_merkle_tree_config = AddressMerkleTreeConfig { height: address_tree.height as u32, changelog_size: address_tree.merkle_tree.changelog.capacity() as u64, @@ -61,35 +66,38 @@ pub async fn get_address_bundle_config( network_fee: Some(address_tree_meta_data.rollover_metadata.network_fee), close_threshold: None, }; - (address_merkle_tree_config, queue_config) + Ok((address_merkle_tree_config, queue_config)) } pub async fn get_state_bundle_config( rpc: &mut R, state_tree_bundle: StateMerkleTreeAccounts, -) -> (StateMerkleTreeConfig, NullifierQueueConfig) { - let address_queue_meta_data = - AccountZeroCopy::::new(rpc, state_tree_bundle.nullifier_queue) - .await - .deserialized() - .metadata; +) -> Result<(StateMerkleTreeConfig, NullifierQueueConfig), AccountZeroCopyError> { + // Get queue metadata - don't hold AccountZeroCopy across await points + let address_queue_meta_data = { + let account = + AccountZeroCopy::::new(rpc, state_tree_bundle.nullifier_queue).await?; + account.deserialized().metadata + }; let address_queue = - unsafe { get_hash_set::(rpc, state_tree_bundle.nullifier_queue).await }; + unsafe { get_hash_set::(rpc, state_tree_bundle.nullifier_queue).await? }; let queue_config = NullifierQueueConfig { network_fee: Some(address_queue_meta_data.rollover_metadata.network_fee), capacity: address_queue.get_capacity() as u16, sequence_threshold: address_queue.sequence_threshold as u64, }; - let address_tree_meta_data = - AccountZeroCopy::::new(rpc, state_tree_bundle.merkle_tree) - .await - .deserialized() - .metadata; + // Get tree metadata - don't hold AccountZeroCopy across await points + let address_tree_meta_data = { + let account = + AccountZeroCopy::::new(rpc, state_tree_bundle.merkle_tree) + .await?; + account.deserialized().metadata + }; let address_tree = get_concurrent_merkle_tree::( rpc, state_tree_bundle.merkle_tree, ) - .await; + .await?; let address_merkle_tree_config = StateMerkleTreeConfig { height: address_tree.height as u32, changelog_size: address_tree.changelog.capacity() as u64, @@ -107,59 +115,81 @@ pub async fn get_state_bundle_config( network_fee: Some(address_tree_meta_data.rollover_metadata.network_fee), close_threshold: None, }; - (address_merkle_tree_config, queue_config) + Ok((address_merkle_tree_config, queue_config)) } -pub async fn address_tree_ready_for_rollover(rpc: &mut R, merkle_tree: Pubkey) -> bool { - let account = AccountZeroCopy::::new(rpc, merkle_tree).await; +pub async fn address_tree_ready_for_rollover( + rpc: &mut R, + merkle_tree: Pubkey, +) -> Result { + // Get account data - don't hold AccountZeroCopy across await points + let (address_tree_meta_data, account_data_len, account_lamports) = { + let account = AccountZeroCopy::::new(rpc, merkle_tree).await?; + ( + account.deserialized().metadata, + account.account.data.len(), + account.account.lamports, + ) + }; let rent_exemption = rpc - .get_minimum_balance_for_rent_exemption(account.account.data.len()) + .get_minimum_balance_for_rent_exemption(account_data_len) .await - .unwrap(); - let address_tree_meta_data = account.deserialized().metadata; + .map_err(|e| AccountZeroCopyError::RpcError(e.to_string()))?; let address_tree = get_indexed_merkle_tree::( rpc, merkle_tree, ) - .await; + .await?; // rollover threshold is reached - address_tree.next_index() + Ok(address_tree.next_index() >= ((1 << address_tree.merkle_tree.height) * address_tree_meta_data.rollover_metadata.rollover_threshold / 100) as usize - // hash sufficient funds for rollover -&& account.account.lamports >= rent_exemption * 2 + // has sufficient funds for rollover + && account_lamports >= rent_exemption * 2 // has not been rolled over - && address_tree_meta_data.rollover_metadata.rolledover_slot == u64::MAX + && address_tree_meta_data.rollover_metadata.rolledover_slot == u64::MAX) } -pub async fn state_tree_ready_for_rollover(rpc: &mut R, merkle_tree: Pubkey) -> bool { - let mut account = rpc.get_account(merkle_tree).await.unwrap().unwrap(); +pub async fn state_tree_ready_for_rollover( + rpc: &mut R, + merkle_tree: Pubkey, +) -> Result { + let mut account = rpc + .get_account(merkle_tree) + .await + .map_err(|e| AccountZeroCopyError::RpcError(e.to_string()))? + .ok_or(AccountZeroCopyError::AccountNotFound(merkle_tree))?; let rent_exemption = rpc .get_minimum_balance_for_rent_exemption(account.data.len()) .await - .unwrap(); - let discriminator = account.data[0..8].try_into().unwrap(); + .map_err(|e| AccountZeroCopyError::RpcError(e.to_string()))?; + let discriminator = &account.data[0..8]; let (next_index, tree_meta_data, height) = match discriminator { - StateMerkleTreeAccount::DISCRIMINATOR => { - let account = AccountZeroCopy::::new(rpc, merkle_tree).await; - - let tree_meta_data = account.deserialized().metadata; + d if d == StateMerkleTreeAccount::DISCRIMINATOR => { + // Get tree metadata - don't hold AccountZeroCopy across await points + let tree_meta_data = { + let account = + AccountZeroCopy::::new(rpc, merkle_tree).await?; + account.deserialized().metadata + }; let tree = get_concurrent_merkle_tree::( rpc, merkle_tree, ) - .await; + .await?; (tree.next_index(), tree_meta_data, 26) } - BatchedMerkleTreeAccount::LIGHT_DISCRIMINATOR_SLICE => { + d if d == BatchedMerkleTreeAccount::LIGHT_DISCRIMINATOR_SLICE => { let tree_meta_data = BatchedMerkleTreeAccount::state_from_bytes( account.data.as_mut_slice(), &merkle_tree.into(), ) - .unwrap(); + .map_err(|e| { + AccountZeroCopyError::RpcError(format!("Failed to parse batched tree: {:?}", e)) + })?; ( tree_meta_data.next_index as usize, @@ -167,13 +197,18 @@ pub async fn state_tree_ready_for_rollover(rpc: &mut R, merkle_tree: Pub tree_meta_data.height, ) } - _ => panic!("Invalid discriminator"), + _ => { + return Err(AccountZeroCopyError::RpcError( + "Invalid discriminator".to_string(), + )) + } }; // rollover threshold is reached - - next_index>= ((1 << height) * tree_meta_data.rollover_metadata.rollover_threshold / 100) as usize - // hash sufficient funds for rollover + Ok( + next_index >= ((1 << height) * tree_meta_data.rollover_metadata.rollover_threshold / 100) as usize + // has sufficient funds for rollover && account.lamports >= rent_exemption * 2 // has not been rolled over - && tree_meta_data.rollover_metadata.rolledover_slot == u64::MAX + && tree_meta_data.rollover_metadata.rolledover_slot == u64::MAX, + ) } diff --git a/forester-utils/src/forester_epoch.rs b/forester-utils/src/forester_epoch.rs index 38c006037b..d92bac8fdb 100644 --- a/forester-utils/src/forester_epoch.rs +++ b/forester-utils/src/forester_epoch.rs @@ -62,6 +62,7 @@ pub struct TreeAccounts { // TODO: evaluate whether we need pub is_rolledover: bool, pub tree_type: TreeType, + pub owner: Pubkey, } impl TreeAccounts { @@ -70,12 +71,14 @@ impl TreeAccounts { queue: Pubkey, tree_type: TreeType, is_rolledover: bool, + owner: Pubkey, ) -> Self { Self { merkle_tree, queue, tree_type, is_rolledover, + owner, } } } @@ -170,9 +173,15 @@ impl TreeForesterSchedule { tree_accounts: *tree_accounts, slots: Vec::new(), }; + // V2 trees use merkle_tree pubkey for eligibility check on-chain, + // V1 trees use queue pubkey. Must match on-chain check_forester logic. + let eligibility_pubkey = match tree_accounts.tree_type { + TreeType::StateV2 | TreeType::AddressV2 => &tree_accounts.merkle_tree, + _ => &tree_accounts.queue, + }; _self.slots = get_schedule_for_forester_in_queue( solana_slot, - &_self.tree_accounts.queue, + eligibility_pubkey, epoch_pda.registered_weight, forester_epoch_pda, )?; diff --git a/forester-utils/src/registry.rs b/forester-utils/src/registry.rs index 17f9e1e485..b7b49308bc 100644 --- a/forester-utils/src/registry.rs +++ b/forester-utils/src/registry.rs @@ -23,6 +23,7 @@ use solana_sdk::{ }; use crate::{ + account_zero_copy::AccountZeroCopyError, address_merkle_tree_config::{get_address_bundle_config, get_state_bundle_config}, instructions::create_account::create_account_instruction, }; @@ -87,13 +88,14 @@ pub async fn get_rent_exemption_for_address_merkle_tree_and_queue( rpc: &mut R, address_merkle_tree_config: &AddressMerkleTreeConfig, address_queue_config: &AddressQueueConfig, -) -> (RentExemption, RentExemption) { - let queue_size = QueueAccount::size(address_queue_config.capacity as usize).unwrap(); +) -> Result<(RentExemption, RentExemption), AccountZeroCopyError> { + let queue_size = QueueAccount::size(address_queue_config.capacity as usize) + .map_err(|e| AccountZeroCopyError::RpcError(format!("Queue size error: {:?}", e)))?; let queue_rent_exempt_lamports = rpc .get_minimum_balance_for_rent_exemption(queue_size) .await - .unwrap(); + .map_err(|e| AccountZeroCopyError::RpcError(e.to_string()))?; let tree_size = account_compression::state::AddressMerkleTreeAccount::size( address_merkle_tree_config.height as usize, address_merkle_tree_config.changelog_size as usize, @@ -104,8 +106,8 @@ pub async fn get_rent_exemption_for_address_merkle_tree_and_queue( let merkle_tree_rent_exempt_lamports = rpc .get_minimum_balance_for_rent_exemption(tree_size) .await - .unwrap(); - ( + .map_err(|e| AccountZeroCopyError::RpcError(e.to_string()))?; + Ok(( RentExemption { lamports: merkle_tree_rent_exempt_lamports, size: tree_size, @@ -114,20 +116,21 @@ pub async fn get_rent_exemption_for_address_merkle_tree_and_queue( lamports: queue_rent_exempt_lamports, size: queue_size, }, - ) + )) } pub async fn get_rent_exemption_for_state_merkle_tree_and_queue( rpc: &mut R, merkle_tree_config: &StateMerkleTreeConfig, queue_config: &NullifierQueueConfig, -) -> (RentExemption, RentExemption) { - let queue_size = QueueAccount::size(queue_config.capacity as usize).unwrap(); +) -> Result<(RentExemption, RentExemption), AccountZeroCopyError> { + let queue_size = QueueAccount::size(queue_config.capacity as usize) + .map_err(|e| AccountZeroCopyError::RpcError(format!("Queue size error: {:?}", e)))?; let queue_rent_exempt_lamports = rpc .get_minimum_balance_for_rent_exemption(queue_size) .await - .unwrap(); + .map_err(|e| AccountZeroCopyError::RpcError(e.to_string()))?; let tree_size = account_compression::state::StateMerkleTreeAccount::size( merkle_tree_config.height as usize, merkle_tree_config.changelog_size as usize, @@ -137,8 +140,8 @@ pub async fn get_rent_exemption_for_state_merkle_tree_and_queue( let merkle_tree_rent_exempt_lamports = rpc .get_minimum_balance_for_rent_exemption(tree_size) .await - .unwrap(); - ( + .map_err(|e| AccountZeroCopyError::RpcError(e.to_string()))?; + Ok(( RentExemption { lamports: merkle_tree_rent_exempt_lamports, size: tree_size, @@ -147,7 +150,7 @@ pub async fn get_rent_exemption_for_state_merkle_tree_and_queue( lamports: queue_rent_exempt_lamports, size: queue_size, }, - ) + )) } #[allow(clippy::too_many_arguments)] @@ -161,7 +164,7 @@ pub async fn create_rollover_address_merkle_tree_instructions( nullifier_queue_pubkey: &Pubkey, epoch: u64, is_metadata_forester: bool, -) -> Vec { +) -> Result, AccountZeroCopyError> { let (merkle_tree_config, queue_config) = get_address_bundle_config( rpc, AddressMerkleTreeAccounts { @@ -169,14 +172,14 @@ pub async fn create_rollover_address_merkle_tree_instructions( queue: *nullifier_queue_pubkey, }, ) - .await; + .await?; let (merkle_tree_rent_exemption, queue_rent_exemption) = get_rent_exemption_for_address_merkle_tree_and_queue( rpc, &merkle_tree_config, &queue_config, ) - .await; + .await?; let create_nullifier_queue_instruction = create_account_instruction( authority, queue_rent_exemption.size, @@ -203,11 +206,11 @@ pub async fn create_rollover_address_merkle_tree_instructions( is_metadata_forester, },epoch ); - vec![ + Ok(vec![ create_nullifier_queue_instruction, create_state_merkle_tree_instruction, instruction, - ] + ]) } #[allow(clippy::too_many_arguments)] @@ -221,7 +224,7 @@ pub async fn perform_state_merkle_tree_roll_over( nullifier_queue_pubkey: &Pubkey, epoch: u64, is_metadata_forester: bool, -) -> Result<(), RpcError> { +) -> Result<(), AccountZeroCopyError> { let instructions = create_rollover_address_merkle_tree_instructions( rpc, &authority.pubkey(), @@ -233,7 +236,7 @@ pub async fn perform_state_merkle_tree_roll_over( epoch, is_metadata_forester, ) - .await; + .await?; rpc.create_and_send_transaction( &instructions, &authority.pubkey(), @@ -243,7 +246,8 @@ pub async fn perform_state_merkle_tree_roll_over( new_state_merkle_tree_keypair, ], ) - .await?; + .await + .map_err(|e| AccountZeroCopyError::RpcError(e.to_string()))?; Ok(()) } #[allow(clippy::too_many_arguments)] @@ -258,7 +262,7 @@ pub async fn create_rollover_state_merkle_tree_instructions( nullifier_queue_pubkey: &Pubkey, epoch: u64, is_metadata_forester: bool, -) -> Vec { +) -> Result, AccountZeroCopyError> { let (merkle_tree_config, queue_config) = get_state_bundle_config( rpc, StateMerkleTreeAccounts { @@ -268,10 +272,10 @@ pub async fn create_rollover_state_merkle_tree_instructions( tree_type: light_compressed_account::TreeType::StateV1, // not used }, ) - .await; + .await?; let (state_merkle_tree_rent_exemption, queue_rent_exemption) = get_rent_exemption_for_state_merkle_tree_and_queue(rpc, &merkle_tree_config, &queue_config) - .await; + .await?; let create_nullifier_queue_instruction = create_account_instruction( authority, queue_rent_exemption.size, @@ -287,12 +291,14 @@ pub async fn create_rollover_state_merkle_tree_instructions( Some(new_state_merkle_tree_keypair), ); let account_size: usize = ProtocolConfig::default().cpi_context_size as usize; + let cpi_rent = rpc + .get_minimum_balance_for_rent_exemption(account_size) + .await + .map_err(|e| AccountZeroCopyError::RpcError(e.to_string()))?; let create_cpi_context_account_instruction = create_account_instruction( authority, account_size, - rpc.get_minimum_balance_for_rent_exemption(account_size) - .await - .unwrap(), + cpi_rent, &Pubkey::from(light_sdk::constants::LIGHT_SYSTEM_PROGRAM_ID), Some(new_cpi_context_keypair), ); @@ -309,10 +315,10 @@ pub async fn create_rollover_state_merkle_tree_instructions( }, epoch, ); - vec![ + Ok(vec![ create_nullifier_queue_instruction, create_state_merkle_tree_instruction, create_cpi_context_account_instruction, instruction, - ] + ]) } diff --git a/forester/.env.example b/forester/.env.example index 72dbc56d1b..cf77ec3ee8 100644 --- a/forester/.env.example +++ b/forester/.env.example @@ -1,56 +1,56 @@ # Core RPC Configuration -export FORESTER_RPC_URL="https://api.devnet.solana.com" -export FORESTER_WS_RPC_URL="wss://api.devnet.solana.com" +export RPC_URL="https://api.devnet.solana.com" +export WS_RPC_URL="wss://api.devnet.solana.com" # Photon -export FORESTER_INDEXER_URL="http://localhost:8784" -export FORESTER_PHOTON_API_KEY="00000000-0000-0000-0000-000000000000" +export INDEXER_URL="http://localhost:8784" +export PHOTON_API_KEY="00000000-0000-0000-0000-000000000000" # Prover Configuration -export FORESTER_PROVER_URL="http://localhost:3001" +export PROVER_URL="http://localhost:3001" # Prover Configuration (V2) -export FORESTER_PROVER_APPEND_URL="http://localhost:3001/append" -export FORESTER_PROVER_UPDATE_URL="http://localhost:3001/update" -export FORESTER_PROVER_ADDRESS_APPEND_URL="http://localhost:3001/address-append" -export FORESTER_PROVER_API_KEY="your-prover-api-key-here" +export PROVER_APPEND_URL="http://localhost:3001/append" +export PROVER_UPDATE_URL="http://localhost:3001/update" +export PROVER_ADDRESS_APPEND_URL="http://localhost:3001/address-append" +export PROVER_API_KEY="your-prover-api-key-here" # Monitoring & Alerts -export FORESTER_PUSH_GATEWAY_URL="http://localhost:9092/metrics/job/forester" -export FORESTER_PAGERDUTY_ROUTING_KEY="your-pagerduty-key-here" +export PUSH_GATEWAY_URL="http://localhost:9092/metrics/job/forester" +export PAGERDUTY_ROUTING_KEY="your-pagerduty-key-here" # Auth -export FORESTER_PAYER='[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64]' -export FORESTER_DERIVATION_PUBKEY='[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32]' +export PAYER='[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64]' +export DERIVATION_PUBKEY='[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32]' # Performance Tuning - RPC Pool -export FORESTER_RPC_POOL_SIZE=100 -export FORESTER_RPC_POOL_CONNECTION_TIMEOUT_SECS=15 # Connection timeout (default: 15) -export FORESTER_RPC_POOL_IDLE_TIMEOUT_SECS=300 # Idle connection timeout (default: 300) +export RPC_POOL_SIZE=100 +export RPC_POOL_CONNECTION_TIMEOUT_SECS=15 # Connection timeout (default: 15) +export RPC_POOL_IDLE_TIMEOUT_SECS=300 # Idle connection timeout (default: 300) # Performance Tuning - Transaction Processing -export FORESTER_MAX_CONCURRENT_SENDS=200 # Max concurrent transaction sends (default: 50) -export FORESTER_LEGACY_IXS_PER_TX=1 # Instructions per v1 transaction -export FORESTER_TRANSACTION_MAX_CONCURRENT_BATCHES=20 # Max concurrent transaction batches (default: 20) -export FORESTER_CU_LIMIT=400000 # Compute unit limit per transaction (default: 1000000) -export FORESTER_ENABLE_PRIORITY_FEES=true # Enable dynamic priority fees (default: false) +export MAX_CONCURRENT_SENDS=200 # Max concurrent transaction sends (default: 50) +export LEGACY_IXS_PER_TX=1 # Instructions per v1 transaction +export TRANSACTION_MAX_CONCURRENT_BATCHES=20 # Max concurrent transaction batches (default: 20) +export CU_LIMIT=400000 # Compute unit limit per transaction (default: 1000000) +export ENABLE_PRIORITY_FEES=true # Enable dynamic priority fees (default: false) # Performance Tuning V1 - Indexer -export FORESTER_INDEXER_BATCH_SIZE=50 # Batch size for indexer requests (default: 50) -export FORESTER_INDEXER_MAX_CONCURRENT_BATCHES=10 # Max concurrent indexer batches (default: 10) +export INDEXER_BATCH_SIZE=50 # Batch size for indexer requests (default: 50) +export INDEXER_MAX_CONCURRENT_BATCHES=10 # Max concurrent indexer batches (default: 10) # Cache Configuration -export FORESTER_TX_CACHE_TTL_SECONDS=180 # Transaction cache TTL (default: 180) -export FORESTER_OPS_CACHE_TTL_SECONDS=180 # Operations cache TTL (default: 180) +export TX_CACHE_TTL_SECONDS=180 # Transaction cache TTL (default: 180) +export OPS_CACHE_TTL_SECONDS=180 # Operations cache TTL (default: 180) # Retry Configuration -export FORESTER_RETRY_DELAY=1000 # Retry delay in ms (default: 1000) -export FORESTER_RETRY_TIMEOUT=30000 # Retry timeout in ms (default: 30000) -export FORESTER_RPC_POOL_MAX_RETRIES=100 # Max RPC retries (default: 100) +export RETRY_DELAY=1000 # Retry delay in ms (default: 1000) +export RETRY_TIMEOUT=30000 # Retry timeout in ms (default: 30000) +export RPC_POOL_MAX_RETRIES=100 # Max RPC retries (default: 100) # Slot Tracker Configuration -export FORESTER_SLOT_UPDATE_INTERVAL_SECONDS=10 # Slot update interval (default: 10) -export FORESTER_TREE_DISCOVERY_INTERVAL_SECONDS=1 # Tree discovery interval (default: 1) +export SLOT_UPDATE_INTERVAL_SECONDS=10 # Slot update interval (default: 10) +export TREE_DISCOVERY_INTERVAL_SECONDS=1 # Tree discovery interval (default: 1) # Processor Mode -export FORESTER_PROCESSOR_MODE=v1 # Options: v1, v2, mixed (default: mixed) +export PROCESSOR_MODE=v1 # Options: v1, v2, mixed (default: mixed) diff --git a/forester/Cargo.toml b/forester/Cargo.toml index 5d829727e4..b0ae60bab6 100644 --- a/forester/Cargo.toml +++ b/forester/Cargo.toml @@ -8,6 +8,7 @@ publish = false anchor-lang = { workspace = true } clap = { version = "4.5.53", features = ["derive", "env"] } solana-sdk = { workspace = true } +solana-commitment-config = { workspace = true } solana-client = { workspace = true } solana-account-decoder = { workspace = true } solana-program = { workspace = true } @@ -50,12 +51,13 @@ anyhow = { workspace = true } prometheus = "0.14" lazy_static = { workspace = true } -warp = "0.4" +warp = { version = "0.4", features = ["server"] } dashmap = { workspace = true } scopeguard = "1.2" itertools = "0.14" async-channel = "2.5" solana-pubkey = { workspace = true } +dotenvy = "0.15" [dev-dependencies] serial_test = { workspace = true } @@ -63,7 +65,6 @@ light-prover-client = { workspace = true, features = ["devenv"] } light-test-utils = { workspace = true } light-program-test = { workspace = true, features = ["devenv"] } light-token-client = { workspace = true } -dotenvy = "0.15" light-compressed-token = { workspace = true } rand = { workspace = true } create-address-test-program = { workspace = true } diff --git a/forester/Dockerfile b/forester/Dockerfile index 462de1ecb4..79b3f06f36 100644 --- a/forester/Dockerfile +++ b/forester/Dockerfile @@ -14,8 +14,9 @@ RUN cargo build --release --package forester FROM debian:bookworm-slim RUN apt-get update && apt-get install -y ca-certificates libssl3 && rm -rf /var/lib/apt/lists/* -RUN mkdir -p /app/config +RUN mkdir -p /app/config /app/static COPY --from=builder /app/target/release/forester /usr/local/bin/forester +COPY --from=builder /app/forester/static /app/static WORKDIR /app ENTRYPOINT ["/usr/local/bin/forester"] diff --git a/forester/src/api_server.rs b/forester/src/api_server.rs new file mode 100644 index 0000000000..22aeeaceb6 --- /dev/null +++ b/forester/src/api_server.rs @@ -0,0 +1,262 @@ +use std::{collections::HashMap, net::SocketAddr, thread::JoinHandle, time::Duration}; + +use serde::{Deserialize, Serialize}; +use tokio::sync::oneshot; +use tracing::{error, info, warn}; +use warp::{http::StatusCode, Filter}; + +use crate::{forester_status::get_forester_status, metrics::REGISTRY}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct HealthResponse { + pub status: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ErrorResponse { + pub error: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MetricsResponse { + pub transactions_processed_total: HashMap, + pub transaction_rate: HashMap, + pub last_run_timestamp: i64, + pub forester_balances: HashMap, + pub queue_lengths: HashMap, +} + +const DASHBOARD_HTML: &str = include_str!("../static/dashboard.html"); + +/// Default timeout for status endpoint in seconds +const STATUS_TIMEOUT_SECS: u64 = 30; + +/// Handle returned by spawn_api_server for graceful shutdown +pub struct ApiServerHandle { + /// Thread handle for the API server + pub thread_handle: JoinHandle<()>, + /// Sender to trigger graceful shutdown + pub shutdown_tx: oneshot::Sender<()>, +} + +impl ApiServerHandle { + /// Trigger graceful shutdown and wait for the server to stop + pub fn shutdown(self) { + // Send shutdown signal (ignore error if receiver already dropped) + let _ = self.shutdown_tx.send(()); + // Wait for the thread to finish + if let Err(e) = self.thread_handle.join() { + error!("API server thread panicked: {:?}", e); + } + } +} + +/// Spawn the HTTP API server with graceful shutdown support. +/// +/// # Arguments +/// * `rpc_url` - RPC URL for forester status endpoint +/// * `port` - Port to bind to +/// * `allow_public_bind` - If true, binds to 0.0.0.0; if false, binds to 127.0.0.1 +/// +/// # Returns +/// An `ApiServerHandle` that can be used to trigger graceful shutdown +pub fn spawn_api_server(rpc_url: String, port: u16, allow_public_bind: bool) -> ApiServerHandle { + let (shutdown_tx, shutdown_rx) = oneshot::channel::<()>(); + + let thread_handle = std::thread::spawn(move || { + let rt = match tokio::runtime::Runtime::new() { + Ok(rt) => rt, + Err(e) => { + error!("Failed to create tokio runtime for API server: {}", e); + return; + } + }; + rt.block_on(async move { + let addr = if allow_public_bind { + warn!( + "API server binding to 0.0.0.0:{} - endpoints /status and /metrics/json will be publicly accessible", + port + ); + SocketAddr::from(([0, 0, 0, 0], port)) + } else { + SocketAddr::from(([127, 0, 0, 1], port)) + }; + info!("Starting HTTP API server on {}", addr); + + let dashboard_route = warp::path::end() + .and(warp::get()) + .map(|| warp::reply::html(DASHBOARD_HTML)); + + let health_route = warp::path("health").and(warp::get()).map(|| { + warp::reply::json(&HealthResponse { + status: "ok".to_string(), + }) + }); + + let status_route = warp::path("status").and(warp::get()).and_then(move || { + let rpc_url = rpc_url.clone(); + async move { + let timeout_duration = Duration::from_secs(STATUS_TIMEOUT_SECS); + match tokio::time::timeout(timeout_duration, get_forester_status(&rpc_url)) + .await + { + Ok(Ok(status)) => Ok::<_, warp::Rejection>(warp::reply::with_status( + warp::reply::json(&status), + StatusCode::OK, + )), + Ok(Err(e)) => { + error!("Failed to get forester status: {:?}", e); + let error_response = ErrorResponse { + error: format!("Failed to get forester status: {}", e), + }; + Ok(warp::reply::with_status( + warp::reply::json(&error_response), + StatusCode::INTERNAL_SERVER_ERROR, + )) + } + Err(_elapsed) => { + error!( + "Forester status request timed out after {}s", + STATUS_TIMEOUT_SECS + ); + let error_response = ErrorResponse { + error: format!( + "Request timed out after {} seconds", + STATUS_TIMEOUT_SECS + ), + }; + Ok(warp::reply::with_status( + warp::reply::json(&error_response), + StatusCode::GATEWAY_TIMEOUT, + )) + } + } + } + }); + + let metrics_route = + warp::path!("metrics" / "json") + .and(warp::get()) + .and_then(|| async move { + match get_metrics_json() { + Ok(metrics) => Ok::<_, warp::Rejection>(warp::reply::with_status( + warp::reply::json(&metrics), + StatusCode::OK, + )), + Err(e) => { + error!("Failed to encode metrics: {}", e); + let error_response = ErrorResponse { + error: format!("Failed to encode metrics: {}", e), + }; + Ok(warp::reply::with_status( + warp::reply::json(&error_response), + StatusCode::INTERNAL_SERVER_ERROR, + )) + } + } + }); + + let routes = dashboard_route + .or(health_route) + .or(status_route) + .or(metrics_route); + + warp::serve(routes) + .bind(addr) + .await + .graceful(async { + let _ = shutdown_rx.await; + info!("API server received shutdown signal"); + }) + .run() + .await; + info!("API server shut down gracefully"); + }); + }); + + ApiServerHandle { + thread_handle, + shutdown_tx, + } +} + +fn get_metrics_json() -> Result { + use prometheus::proto::MetricType; + + let metric_families = REGISTRY.gather(); + + let mut transactions_processed: HashMap = HashMap::new(); + let mut transaction_rate: HashMap = HashMap::new(); + let mut last_run_timestamp: i64 = 0; + let mut forester_balances: HashMap = HashMap::new(); + let mut queue_lengths: HashMap = HashMap::new(); + + for mf in metric_families { + let name = mf.name(); + let metric_type = mf.get_field_type(); + + for metric in mf.get_metric() { + // Extract labels into a map for easy lookup + let labels: HashMap<&str, &str> = metric + .get_label() + .iter() + .map(|lp| (lp.name(), lp.value())) + .collect(); + + // Get the metric value based on type + let value = match metric_type { + MetricType::COUNTER => metric.get_counter().value(), + MetricType::GAUGE => metric.get_gauge().value(), + MetricType::HISTOGRAM => { + // For histogram, use sample_sum as a representative value + metric.get_histogram().get_sample_sum() + } + MetricType::SUMMARY => { + // For summary, use sample_sum as a representative value + metric.get_summary().sample_sum() + } + _ => continue, + }; + + // Skip NaN and Inf values + if value.is_nan() || value.is_infinite() { + continue; + } + + match name { + "forester_transactions_processed_total" => { + if let Some(epoch) = labels.get("epoch") { + transactions_processed.insert((*epoch).to_string(), value as u64); + } + } + "forester_transaction_rate" => { + if let Some(epoch) = labels.get("epoch") { + transaction_rate.insert((*epoch).to_string(), value); + } + } + "forester_last_run_timestamp" => { + last_run_timestamp = value as i64; + } + "forester_sol_balance" => { + if let Some(pubkey) = labels.get("pubkey") { + forester_balances.insert((*pubkey).to_string(), value); + } + } + "queue_length" => { + if let Some(tree_pubkey) = labels.get("tree_pubkey") { + queue_lengths.insert((*tree_pubkey).to_string(), value as i64); + } + } + _ => {} + } + } + } + + Ok(MetricsResponse { + transactions_processed_total: transactions_processed, + transaction_rate, + last_run_timestamp, + forester_balances, + queue_lengths, + }) +} diff --git a/forester/src/cli.rs b/forester/src/cli.rs index f64c03322f..ff8fe633a2 100644 --- a/forester/src/cli.rs +++ b/forester/src/cli.rs @@ -17,96 +17,88 @@ pub enum Commands { #[derive(Parser, Clone, Debug)] pub struct StartArgs { - #[arg(long, env = "FORESTER_RPC_URL")] + #[arg(long, env = "RPC_URL")] pub rpc_url: Option, - #[arg(long, env = "FORESTER_PUSH_GATEWAY_URL")] + #[arg(long, env = "PUSH_GATEWAY_URL")] pub push_gateway_url: Option, - #[arg(long, env = "FORESTER_PAGERDUTY_ROUTING_KEY")] + #[arg(long, env = "PAGERDUTY_ROUTING_KEY")] pub pagerduty_routing_key: Option, - #[arg(long, env = "FORESTER_WS_RPC_URL")] + #[arg(long, env = "WS_RPC_URL")] pub ws_rpc_url: Option, - #[arg(long, env = "FORESTER_INDEXER_URL")] + #[arg(long, env = "INDEXER_URL")] pub indexer_url: Option, - #[arg(long, env = "FORESTER_PROVER_URL")] + #[arg(long, env = "PROVER_URL")] pub prover_url: Option, #[arg( long, - env = "FORESTER_PROVER_APPEND_URL", + env = "PROVER_APPEND_URL", help = "Prover URL for append operations. If not specified, uses prover_url" )] pub prover_append_url: Option, #[arg( long, - env = "FORESTER_PROVER_UPDATE_URL", + env = "PROVER_UPDATE_URL", help = "Prover URL for update operations. If not specified, uses prover_url" )] pub prover_update_url: Option, #[arg( long, - env = "FORESTER_PROVER_ADDRESS_APPEND_URL", + env = "PROVER_ADDRESS_APPEND_URL", help = "Prover URL for address-append operations. If not specified, uses prover_url" )] pub prover_address_append_url: Option, - #[arg(long, env = "FORESTER_PROVER_API_KEY")] + #[arg(long, env = "PROVER_API_KEY")] pub prover_api_key: Option, #[arg( long, - env = "FORESTER_PROVER_POLLING_INTERVAL_MS", + env = "PROVER_POLLING_INTERVAL_MS", help = "Prover polling interval in milliseconds (default: 1000)" )] pub prover_polling_interval_ms: Option, #[arg( long, - env = "FORESTER_PROVER_MAX_WAIT_TIME_SECS", + env = "PROVER_MAX_WAIT_TIME_SECS", help = "Maximum time to wait for prover response in seconds (default: 600)" )] pub prover_max_wait_time_secs: Option, - #[arg(long, env = "FORESTER_PAYER")] + #[arg(long, env = "PAYER")] pub payer: Option, - #[arg(long, env = "FORESTER_DERIVATION_PUBKEY")] + #[arg(long, env = "DERIVATION_PUBKEY")] pub derivation: Option, - #[arg(long, env = "FORESTER_PHOTON_API_KEY")] + #[arg(long, env = "PHOTON_API_KEY")] pub photon_api_key: Option, - #[arg(long, env = "FORESTER_PHOTON_GRPC_URL")] + #[arg(long, env = "PHOTON_GRPC_URL")] pub photon_grpc_url: Option, - #[arg(long, env = "FORESTER_INDEXER_BATCH_SIZE", default_value = "50")] + #[arg(long, env = "INDEXER_BATCH_SIZE", default_value = "50")] pub indexer_batch_size: usize, - #[arg( - long, - env = "FORESTER_INDEXER_MAX_CONCURRENT_BATCHES", - default_value = "10" - )] + #[arg(long, env = "INDEXER_MAX_CONCURRENT_BATCHES", default_value = "10")] pub indexer_max_concurrent_batches: usize, - #[arg(long, env = "FORESTER_LEGACY_XS_PER_TX", default_value = "1")] + #[arg(long, env = "LEGACY_IXS_PER_TX", default_value = "1")] pub legacy_ixs_per_tx: usize, - #[arg( - long, - env = "FORESTER_TRANSACTION_MAX_CONCURRENT_BATCHES", - default_value = "20" - )] + #[arg(long, env = "TRANSACTION_MAX_CONCURRENT_BATCHES", default_value = "20")] pub transaction_max_concurrent_batches: usize, #[arg( long, - env = "FORESTER_MAX_CONCURRENT_SENDS", + env = "MAX_CONCURRENT_SENDS", default_value = "50", help = "Maximum number of concurrent transaction sends per batch" )] @@ -114,7 +106,15 @@ pub struct StartArgs { #[arg( long, - env = "FORESTER_TX_CACHE_TTL_SECONDS", + env = "MAX_BATCHES_PER_TREE", + default_value = "4", + help = "Maximum batches to process per tree per iteration (1-20, default: 4)" + )] + pub max_batches_per_tree: usize, + + #[arg( + long, + env = "TX_CACHE_TTL_SECONDS", default_value = "180", help = "TTL in seconds to prevent duplicate transaction processing" )] @@ -122,7 +122,7 @@ pub struct StartArgs { #[arg( long, - env = "FORESTER_OPS_CACHE_TTL_SECONDS", + env = "OPS_CACHE_TTL_SECONDS", default_value = "180", help = "TTL in seconds to prevent duplicate batch operations processing" )] @@ -130,7 +130,7 @@ pub struct StartArgs { #[arg( long, - env = "FORESTER_CONFIRMATION_MAX_ATTEMPTS", + env = "CONFIRMATION_MAX_ATTEMPTS", default_value = "60", help = "Maximum attempts to confirm a transaction before timing out" )] @@ -138,107 +138,75 @@ pub struct StartArgs { #[arg( long, - env = "FORESTER_CONFIRMATION_POLL_INTERVAL_MS", + env = "CONFIRMATION_POLL_INTERVAL_MS", default_value = "500", help = "Interval between confirmation polling attempts in milliseconds" )] pub confirmation_poll_interval_ms: u64, - #[arg(long, env = "FORESTER_CU_LIMIT", default_value = "1000000")] + #[arg(long, env = "CU_LIMIT", default_value = "1000000")] pub cu_limit: u32, - #[arg(long, env = "FORESTER_ENABLE_PRIORITY_FEES", default_value = "false")] + #[arg(long, env = "ENABLE_PRIORITY_FEES", default_value = "false")] pub enable_priority_fees: bool, - #[arg(long, env = "FORESTER_RPC_POOL_SIZE", default_value = "100")] + #[arg(long, env = "RPC_POOL_SIZE", default_value = "100")] pub rpc_pool_size: u32, - #[arg( - long, - env = "FORESTER_RPC_POOL_CONNECTION_TIMEOUT_SECS", - default_value = "15" - )] + #[arg(long, env = "RPC_POOL_CONNECTION_TIMEOUT_SECS", default_value = "15")] pub rpc_pool_connection_timeout_secs: u64, - #[arg( - long, - env = "FORESTER_RPC_POOL_IDLE_TIMEOUT_SECS", - default_value = "300" - )] + #[arg(long, env = "RPC_POOL_IDLE_TIMEOUT_SECS", default_value = "300")] pub rpc_pool_idle_timeout_secs: u64, - #[arg(long, env = "FORESTER_RPC_POOL_MAX_RETRIES", default_value = "100")] + #[arg(long, env = "RPC_POOL_MAX_RETRIES", default_value = "100")] pub rpc_pool_max_retries: u32, - #[arg( - long, - env = "FORESTER_RPC_POOL_INITIAL_RETRY_DELAY_MS", - default_value = "1000" - )] + #[arg(long, env = "RPC_POOL_INITIAL_RETRY_DELAY_MS", default_value = "1000")] pub rpc_pool_initial_retry_delay_ms: u64, - #[arg( - long, - env = "FORESTER_RPC_POOL_MAX_RETRY_DELAY_MS", - default_value = "16000" - )] + #[arg(long, env = "RPC_POOL_MAX_RETRY_DELAY_MS", default_value = "16000")] pub rpc_pool_max_retry_delay_ms: u64, - #[arg( - long, - env = "FORESTER_SLOT_UPDATE_INTERVAL_SECONDS", - default_value = "10" - )] + #[arg(long, env = "SLOT_UPDATE_INTERVAL_SECONDS", default_value = "10")] pub slot_update_interval_seconds: u64, - #[arg( - long, - env = "FORESTER_TREE_DISCOVERY_INTERVAL_SECONDS", - default_value = "5" - )] + #[arg(long, env = "TREE_DISCOVERY_INTERVAL_SECONDS", default_value = "5")] pub tree_discovery_interval_seconds: u64, - #[arg(long, env = "FORESTER_MAX_RETRIES", default_value = "3")] + #[arg(long, env = "MAX_RETRIES", default_value = "3")] pub max_retries: u32, - #[arg(long, env = "FORESTER_RETRY_DELAY", default_value = "1000")] + #[arg(long, env = "RETRY_DELAY", default_value = "1000")] pub retry_delay: u64, - #[arg(long, env = "FORESTER_RETRY_TIMEOUT", default_value = "30000")] + #[arg(long, env = "RETRY_TIMEOUT", default_value = "30000")] pub retry_timeout: u64, - #[arg(long, env = "FORESTER_STATE_QUEUE_START_INDEX", default_value = "0")] + #[arg(long, env = "STATE_QUEUE_START_INDEX", default_value = "0")] pub state_queue_start_index: u16, - #[arg( - long, - env = "FORESTER_STATE_PROCESSING_LENGTH", - default_value = "28807" - )] + #[arg(long, env = "STATE_PROCESSING_LENGTH", default_value = "28807")] pub state_queue_processing_length: u16, - #[arg(long, env = "FORESTER_ADDRESS_QUEUE_START_INDEX", default_value = "0")] + #[arg(long, env = "ADDRESS_QUEUE_START_INDEX", default_value = "0")] pub address_queue_start_index: u16, - #[arg( - long, - env = "FORESTER_ADDRESS_PROCESSING_LENGTH", - default_value = "28807" - )] + #[arg(long, env = "ADDRESS_PROCESSING_LENGTH", default_value = "28807")] pub address_queue_processing_length: u16, - #[arg(long, env = "FORESTER_RPC_RATE_LIMIT")] + #[arg(long, env = "RPC_RATE_LIMIT")] pub rpc_rate_limit: Option, - #[arg(long, env = "FORESTER_PHOTON_RATE_LIMIT")] + #[arg(long, env = "PHOTON_RATE_LIMIT")] pub photon_rate_limit: Option, - #[arg(long, env = "FORESTER_SEND_TRANSACTION_RATE_LIMIT")] + #[arg(long, env = "SEND_TRANSACTION_RATE_LIMIT")] pub send_tx_rate_limit: Option, #[arg( long, - env = "FORESTER_PROCESSOR_MODE", + env = "PROCESSOR_MODE", default_value_t = ProcessorMode::All, help = "Processor mode: v1 (process only v1 trees), v2 (process only v2 trees), all (process all trees)" )] @@ -246,7 +214,7 @@ pub struct StartArgs { #[arg( long, - env = "FORESTER_QUEUE_POLLING_MODE", + env = "QUEUE_POLLING_MODE", default_value_t = QueuePollingMode::Indexer, help = "Queue polling mode: indexer (poll indexer API, requires indexer_url), onchain (read queue status directly from RPC)" )] @@ -254,7 +222,7 @@ pub struct StartArgs { #[arg( long = "tree-id", - env = "FORESTER_TREE_IDS", + env = "TREE_IDS", help = "Process only the specified trees (Pubkeys). Can be specified multiple times. If specified, forester will process only these trees and ignore all others", value_delimiter = ',' )] @@ -262,7 +230,7 @@ pub struct StartArgs { #[arg( long, - env = "FORESTER_ENABLE_COMPRESSIBLE", + env = "ENABLE_COMPRESSIBLE", help = "Enable compressible account tracking and compression using ws_rpc_url (requires --ws-rpc-url)", default_value = "false" )] @@ -270,20 +238,47 @@ pub struct StartArgs { #[arg( long, - env = "FORESTER_LOOKUP_TABLE_ADDRESS", + env = "LOOKUP_TABLE_ADDRESS", help = "Address lookup table pubkey for versioned transactions. If not provided, legacy transactions will be used." )] pub lookup_table_address: Option, + + #[arg( + long, + env = "API_SERVER_PORT", + help = "HTTP API server port (default: 8080)", + default_value = "8080" + )] + pub api_server_port: u16, + + #[arg( + long, + env = "API_SERVER_PUBLIC_BIND", + help = "Bind API server to 0.0.0.0 instead of 127.0.0.1", + default_value = "false" + )] + pub api_server_public_bind: bool, + + #[arg( + long, + env = "GROUP_AUTHORITY", + help = "Filter trees by group authority pubkey. Only process trees owned by this authority." + )] + pub group_authority: Option, } #[derive(Parser, Clone, Debug)] pub struct StatusArgs { - #[arg(long, env = "FORESTER_RPC_URL")] + #[arg(long, env = "RPC_URL", value_name = "RPC_URL", alias = "RPC_URL")] pub rpc_url: String, - #[arg(long, env = "FORESTER_PUSH_GATEWAY_URL")] + #[arg(long, env = "PUSH_GATEWAY_URL", value_name = "PUSH_GATEWAY_URL")] pub push_gateway_url: Option, - #[arg(long, env = "FORESTER_PAGERDUTY_ROUTING_KEY")] + #[arg( + long, + env = "PAGERDUTY_ROUTING_KEY", + value_name = "PAGERDUTY_ROUTING_KEY" + )] pub pagerduty_routing_key: Option, /// Select to run compressed token program tests. #[clap(long)] @@ -314,13 +309,13 @@ pub struct HealthArgs { #[arg(long, help = "Check forester registration for current epoch")] pub check_registration: bool, - #[arg(long, env = "FORESTER_RPC_URL")] + #[arg(long, env = "RPC_URL")] pub rpc_url: Option, - #[arg(long, env = "FORESTER_PAYER")] + #[arg(long, env = "PAYER")] pub payer: Option, - #[arg(long, env = "FORESTER_DERIVATION_PUBKEY")] + #[arg(long, env = "DERIVATION_PUBKEY")] pub derivation: Option, #[arg( diff --git a/forester/src/config.rs b/forester/src/config.rs index dd7c96121c..2a345589f5 100644 --- a/forester/src/config.rs +++ b/forester/src/config.rs @@ -80,6 +80,8 @@ pub struct TransactionConfig { pub confirmation_max_attempts: u32, /// Interval between confirmation polling attempts in milliseconds. pub confirmation_poll_interval_ms: u64, + /// Maximum zkp batches to fetch from indexer per v2 tree per iteration + pub max_batches_per_tree: usize, } #[derive(Debug, Clone)] @@ -95,6 +97,7 @@ pub struct GeneralConfig { pub sleep_after_processing_ms: u64, pub sleep_when_idle_ms: u64, pub queue_polling_mode: QueuePollingMode, + pub group_authority: Option, } impl Default for GeneralConfig { @@ -111,6 +114,7 @@ impl Default for GeneralConfig { sleep_after_processing_ms: 10_000, sleep_when_idle_ms: 45_000, queue_polling_mode: QueuePollingMode::Indexer, + group_authority: None, } } } @@ -129,6 +133,7 @@ impl GeneralConfig { sleep_after_processing_ms: 50, sleep_when_idle_ms: 100, queue_polling_mode: QueuePollingMode::Indexer, + group_authority: None, } } @@ -145,6 +150,7 @@ impl GeneralConfig { sleep_after_processing_ms: 50, sleep_when_idle_ms: 100, queue_polling_mode: QueuePollingMode::Indexer, + group_authority: None, } } } @@ -191,6 +197,7 @@ impl Default for TransactionConfig { ops_cache_ttl_seconds: 180, confirmation_max_attempts: 60, confirmation_poll_interval_ms: 500, + max_batches_per_tree: 4, } } } @@ -294,6 +301,7 @@ impl ForesterConfig { ops_cache_ttl_seconds: args.ops_cache_ttl_seconds, confirmation_max_attempts: args.confirmation_max_attempts, confirmation_poll_interval_ms: args.confirmation_poll_interval_ms, + max_batches_per_tree: args.max_batches_per_tree, }, general_config: GeneralConfig { slot_update_interval_seconds: args.slot_update_interval_seconds, @@ -325,6 +333,16 @@ impl ForesterConfig { sleep_after_processing_ms: 10_000, sleep_when_idle_ms: 45_000, queue_polling_mode: args.queue_polling_mode, + group_authority: args + .group_authority + .as_ref() + .map(|s| { + Pubkey::from_str(s).map_err(|e| ConfigError::InvalidArguments { + field: "group_authority", + invalid_values: vec![e.to_string()], + }) + }) + .transpose()?, }, rpc_pool_config: RpcPoolConfig { max_size: args.rpc_pool_size, @@ -415,6 +433,7 @@ impl ForesterConfig { sleep_after_processing_ms: 10_000, sleep_when_idle_ms: 45_000, queue_polling_mode: QueuePollingMode::OnChain, // Status uses on-chain reads + group_authority: None, }, rpc_pool_config: RpcPoolConfig { max_size: 10, diff --git a/forester/src/epoch_manager.rs b/forester/src/epoch_manager.rs index 9c1ad6a059..b02ad2d74b 100644 --- a/forester/src/epoch_manager.rs +++ b/forester/src/epoch_manager.rs @@ -72,7 +72,7 @@ use crate::{ perform_state_merkle_tree_rollover_forester, }, slot_tracker::{slot_duration, wait_until_slot_reached, SlotTracker}, - tree_data_sync::fetch_trees, + tree_data_sync::{fetch_protocol_group_authority, fetch_trees}, ForesterConfig, ForesterEpochInfo, Result, }; @@ -282,7 +282,7 @@ impl EpochManager { let (tx, mut rx) = mpsc::channel(100); let tx = Arc::new(tx); - let monitor_handle = tokio::spawn({ + let mut monitor_handle = tokio::spawn({ let self_clone = Arc::clone(&self); let tx_clone = Arc::clone(&tx); async move { self_clone.monitor_epochs(tx_clone).await } @@ -311,32 +311,67 @@ impl EpochManager { let _guard = scopeguard::guard( ( - monitor_handle, current_previous_handle, new_tree_handle, balance_check_handle, ), - |(h1, h2, h3, h4)| { + |(h2, h3, h4)| { info!("Aborting EpochManager background tasks"); - h1.abort(); h2.abort(); h3.abort(); h4.abort(); }, ); - while let Some(epoch) = rx.recv().await { - debug!("Received new epoch: {}", epoch); - - let self_clone = Arc::clone(&self); - tokio::spawn(async move { - if let Err(e) = self_clone.process_epoch(epoch).await { - error!("Error processing epoch {}: {:?}", epoch, e); + let result = loop { + tokio::select! { + epoch_opt = rx.recv() => { + match epoch_opt { + Some(epoch) => { + debug!("Received new epoch: {}", epoch); + let self_clone = Arc::clone(&self); + tokio::spawn(async move { + if let Err(e) = self_clone.process_epoch(epoch).await { + error!("Error processing epoch {}: {:?}", epoch, e); + } + }); + } + None => { + error!("Epoch monitor channel closed unexpectedly!"); + break Err(anyhow!( + "Epoch monitor channel closed - forester cannot function without it" + )); + } + } } - }); - } + result = &mut monitor_handle => { + match result { + Ok(Ok(())) => { + error!("Epoch monitor exited unexpectedly with Ok(())"); + } + Ok(Err(e)) => { + error!("Epoch monitor exited with error: {:?}", e); + } + Err(e) => { + error!("Epoch monitor task panicked or was cancelled: {:?}", e); + } + } + if let Some(pagerduty_key) = &self.config.external_services.pagerduty_routing_key { + let _ = send_pagerduty_alert( + pagerduty_key, + &format!("Forester epoch monitor died unexpectedly on {}", self.config.payer_keypair.pubkey()), + "critical", + "epoch_monitor_dead", + ).await; + } + break Err(anyhow!("Epoch monitor exited unexpectedly - forester cannot function without it")); + } + } + }; - Ok(()) + // Abort monitor_handle on exit + monitor_handle.abort(); + result } async fn check_sol_balance_periodically(self: Arc) -> Result<()> { @@ -461,10 +496,45 @@ impl EpochManager { #[instrument(level = "debug", skip(self, tx))] async fn monitor_epochs(&self, tx: Arc>) -> Result<()> { let mut last_epoch: Option = None; - debug!("Starting epoch monitor"); + let mut consecutive_failures = 0u32; + const MAX_BACKOFF_SECS: u64 = 60; + + info!("Starting epoch monitor"); loop { - let (slot, current_epoch) = self.get_current_slot_and_epoch().await?; + let (slot, current_epoch) = match self.get_current_slot_and_epoch().await { + Ok(result) => { + if consecutive_failures > 0 { + info!( + "Epoch monitor recovered after {} consecutive failures", + consecutive_failures + ); + } + consecutive_failures = 0; + result + } + Err(e) => { + consecutive_failures += 1; + let backoff_secs = 2u64.pow(consecutive_failures.min(6)).min(MAX_BACKOFF_SECS); + let backoff = Duration::from_secs(backoff_secs); + + if consecutive_failures == 1 { + warn!( + "Epoch monitor: failed to get slot/epoch: {:?}. Retrying in {:?}", + e, backoff + ); + } else if consecutive_failures.is_multiple_of(10) { + error!( + "Epoch monitor: {} consecutive failures, last error: {:?}. Still retrying every {:?}", + consecutive_failures, e, backoff + ); + } + + tokio::time::sleep(backoff).await; + continue; + } + }; + debug!( "last_epoch: {:?}, current_epoch: {:?}, slot: {:?}", last_epoch, current_epoch, slot @@ -477,10 +547,10 @@ impl EpochManager { debug!("Sending current epoch {} for processing", current_epoch); if let Err(e) = tx.send(current_epoch).await { error!( - "Failed to send current epoch {} for processing: {:?}", + "Failed to send current epoch {} for processing: {:?}. Channel closed, exiting.", current_epoch, e ); - return Ok(()); + return Err(anyhow!("Epoch channel closed: {}", e)); } last_epoch = Some(current_epoch); } @@ -612,10 +682,12 @@ impl EpochManager { let forester_epoch_pda_pubkey = get_forester_epoch_pda_from_authority(&self.config.derivation_pubkey, epoch).0; - let rpc = self.rpc_pool.get_connection().await?; - let existing_pda = rpc - .get_anchor_account::(&forester_epoch_pda_pubkey) - .await?; + + let existing_pda = { + let rpc = self.rpc_pool.get_connection().await?; + rpc.get_anchor_account::(&forester_epoch_pda_pubkey) + .await? + }; existing_pda .map(|pda| async move { @@ -695,15 +767,16 @@ impl EpochManager { Ok(()) } - #[instrument(level = "debug", skip(self), fields(forester = %self.config.payer_keypair.pubkey(), epoch = epoch - ))] + #[instrument(level = "debug", skip(self), fields(forester = %self.config.payer_keypair.pubkey(), epoch = epoch))] async fn process_epoch(&self, epoch: u64) -> Result<()> { - info!("Entering process_epoch"); - + // Clone the Arc immediately to release the DashMap shard lock. + // Without .clone(), the RefMut guard would be held across async operations, + // blocking other epochs from accessing the DashMap if they hash to the same shard. let processing_flag = self .processing_epochs .entry(epoch) - .or_insert_with(|| Arc::new(AtomicBool::new(false))); + .or_insert_with(|| Arc::new(AtomicBool::new(false))) + .clone(); if processing_flag .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst) @@ -713,6 +786,7 @@ impl EpochManager { debug!("Epoch {} is already being processed, skipping", epoch); return Ok(()); } + let phases = get_epoch_phases(&self.protocol_config, epoch); // Attempt to recover registration info @@ -1144,6 +1218,7 @@ impl EpochManager { queue: solana_sdk::pubkey::Pubkey::default(), tree_type: TreeType::Unknown, is_rolledover: false, + owner: solana_sdk::pubkey::Pubkey::default(), }; let tree_schedule = TreeForesterSchedule::new_with_schedule( &compression_tree_accounts, @@ -1213,7 +1288,9 @@ impl EpochManager { handles.push(handle); } - for result in join_all(handles).await { + info!("Waiting for {} tree processing tasks", handles.len()); + let results = join_all(handles).await; + for result in results { match result { Ok(Ok(())) => { debug!("Queue processed successfully"); @@ -1422,7 +1499,9 @@ impl EpochManager { ) .await; - push_metrics(&self.config.external_services.pushgateway_url).await?; + if let Err(e) = push_metrics(&self.config.external_services.pushgateway_url).await { + warn!("Failed to push metrics: {:?}", e); + } estimated_slot = self.slot_tracker.estimated_current_slot(); let sleep_duration_ms = if items_processed_this_iteration > 0 { @@ -1509,7 +1588,7 @@ impl EpochManager { .check_forester_eligibility( epoch_pda, current_light_slot, - &tree_accounts.queue, + &tree_accounts.merkle_tree, epoch_info.epoch, epoch_info, ) @@ -1551,7 +1630,9 @@ impl EpochManager { } } - push_metrics(&self.config.external_services.pushgateway_url).await?; + if let Err(e) = push_metrics(&self.config.external_services.pushgateway_url).await { + warn!("Failed to push metrics: {:?}", e); + } estimated_slot = self.slot_tracker.estimated_current_slot(); } @@ -1950,6 +2031,7 @@ impl EpochManager { confirmation_poll_interval: Duration::from_millis( self.config.transaction_config.confirmation_poll_interval_ms, ), + max_batches_per_tree: self.config.transaction_config.max_batches_per_tree, } } @@ -2816,6 +2898,36 @@ pub async fn run_service( fetch_result = fetch_trees(&*rpc) => { match fetch_result { Ok(mut fetched_trees) => { + let group_authority = match config.general_config.group_authority { + Some(ga) => Some(ga), + None => { + match fetch_protocol_group_authority(&*rpc).await { + Ok(ga) => { + info!("Using protocol default group authority: {}", ga); + Some(ga) + } + Err(e) => { + warn!( + "Failed to fetch protocol group authority, processing all trees: {:?}", + e + ); + None + } + } + } + }; + + if let Some(group_authority) = group_authority { + let before_count = fetched_trees.len(); + fetched_trees.retain(|tree| tree.owner == group_authority); + info!( + "Filtered trees by group authority {}: {} -> {} trees", + group_authority, + before_count, + fetched_trees.len() + ); + } + if !config.general_config.tree_ids.is_empty() { let tree_ids = &config.general_config.tree_ids; fetched_trees.retain(|tree| tree_ids.contains(&tree.merkle_tree)); @@ -3058,6 +3170,7 @@ mod tests { sleep_after_processing_ms: 50, sleep_when_idle_ms: 100, queue_polling_mode: crate::cli::QueuePollingMode::Indexer, + group_authority: None, }, rpc_pool_config: Default::default(), registry_pubkey: Pubkey::default(), @@ -3158,6 +3271,7 @@ mod tests { queue: Pubkey::new_unique(), is_rolledover: false, tree_type: TreeType::AddressV1, + owner: Default::default(), }; let work_item = WorkItem { @@ -3179,6 +3293,7 @@ mod tests { queue: Pubkey::new_unique(), is_rolledover: false, tree_type: TreeType::StateV1, + owner: Default::default(), }; let work_item = WorkItem { diff --git a/forester/src/forester_status.rs b/forester/src/forester_status.rs index 9c6b4c74eb..5149307391 100644 --- a/forester/src/forester_status.rs +++ b/forester/src/forester_status.rs @@ -1,25 +1,658 @@ -use std::sync::Arc; +use std::{collections::HashMap, sync::Arc}; +use account_compression::{ + utils::constants::{ADDRESS_MERKLE_TREE_HEIGHT, STATE_MERKLE_TREE_HEIGHT}, + AddressMerkleTreeAccount, QueueAccount, StateMerkleTreeAccount, +}; use anchor_lang::{AccountDeserialize, Discriminator}; use anyhow::Context; -use forester_utils::forester_epoch::{get_epoch_phases, TreeAccounts}; +use borsh::BorshDeserialize; +use forester_utils::{ + account_zero_copy::{ + parse_concurrent_merkle_tree_from_bytes, parse_hash_set_from_bytes, + parse_indexed_merkle_tree_from_bytes, + }, + forester_epoch::{get_epoch_phases, TreeAccounts}, +}; use itertools::Itertools; +use light_batched_merkle_tree::merkle_tree::BatchedMerkleTreeAccount; use light_client::rpc::{LightClient, LightClientConfig, Rpc}; use light_compressed_account::TreeType; +use light_hasher::Poseidon; use light_registry::{protocol_config::state::ProtocolConfigPda, EpochPda, ForesterEpochPda}; +use serde::{Deserialize, Serialize}; use solana_program::{clock::Slot, pubkey::Pubkey}; -use solana_sdk::{account::ReadableAccount, commitment_config::CommitmentConfig}; +use solana_sdk::{ + account::{Account, ReadableAccount}, + commitment_config::CommitmentConfig, +}; use tracing::{debug, warn}; use crate::{ cli::StatusArgs, metrics::{push_metrics, register_metrics, update_registered_foresters}, + queue_helpers::{parse_address_v2_queue_info, parse_state_v2_queue_info, V2QueueInfo}, rollover::get_tree_fullness, run_queue_info, - tree_data_sync::fetch_trees, + tree_data_sync::{fetch_protocol_group_authority, fetch_trees}, ForesterConfig, }; +const INDEXED_MERKLE_TREE_V1_INITIAL_LEAVES: usize = 3; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ForesterInfo { + pub authority: String, + pub balance_sol: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ForesterStatus { + pub slot: u64, + pub current_active_epoch: u64, + pub current_registration_epoch: u64, + pub active_epoch_progress: u64, + pub active_phase_length: u64, + pub active_epoch_progress_percentage: f64, + pub hours_until_next_epoch: u64, + pub slots_until_next_registration: u64, + pub hours_until_next_registration: u64, + pub active_epoch_foresters: Vec, + pub registration_epoch_foresters: Vec, + pub trees: Vec, + /// Current light slot index (None if not in active phase) + pub current_light_slot: Option, + /// Solana slots per light slot (forester rotation interval) + pub light_slot_length: u64, + /// Slots remaining until next light slot (forester rotation) + pub slots_until_next_light_slot: Option, + /// Total number of light slots in the active phase + pub total_light_slots: u64, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TreeStatus { + pub tree_type: String, + pub merkle_tree: String, + pub queue: String, + pub fullness_percentage: f64, + pub next_index: u64, + pub threshold: u64, + pub is_rolledover: bool, + pub queue_length: Option, + pub v2_queue_info: Option, + /// Currently assigned forester for this tree (in current light slot) + pub assigned_forester: Option, + /// Schedule: forester index (into active_epoch_foresters) for each light slot + /// None means no forester assigned for that slot + pub schedule: Vec>, + /// Owner (group authority) of the tree + pub owner: String, +} + +/// Get forester status with optional group authority filtering. +/// +/// # Arguments +/// * `rpc_url` - RPC URL to connect to +/// * `filter_by_group_authority` - If true, filter trees by protocol group authority. +/// If false, show all trees without filtering. +pub async fn get_forester_status(rpc_url: &str) -> crate::Result { + get_forester_status_with_options(rpc_url, true).await +} + +/// Get forester status with explicit control over group authority filtering. +pub async fn get_forester_status_with_options( + rpc_url: &str, + filter_by_group_authority: bool, +) -> crate::Result { + let rpc = LightClient::new(LightClientConfig { + url: rpc_url.to_string(), + photon_url: None, + api_key: None, + commitment_config: None, + fetch_active_tree: false, + }) + .await + .context("Failed to create LightClient")?; + + // Phase 1: Fetch registry accounts and slot in parallel + let (registry_result, slot_result) = + tokio::join!(fetch_registry_accounts_filtered(&rpc), rpc.get_slot(),); + + let (forester_epoch_pdas, _epoch_pdas, protocol_config_pdas) = registry_result?; + let slot = slot_result.context("Failed to get slot")?; + + let protocol_config_pda = protocol_config_pdas + .first() + .cloned() + .context("No ProtocolConfigPda found in registry program accounts")?; + + let current_active_epoch = protocol_config_pda.config.get_current_active_epoch(slot)?; + let current_registration_epoch = protocol_config_pda.config.get_latest_register_epoch(slot)?; + + let active_epoch_progress = protocol_config_pda + .config + .get_current_active_epoch_progress(slot); + let active_phase_length = protocol_config_pda.config.active_phase_length; + let active_epoch_progress_percentage = + active_epoch_progress as f64 / active_phase_length as f64 * 100f64; + + let hours_until_next_epoch = + active_phase_length.saturating_sub(active_epoch_progress) * 460 / 1000 / 3600; + + let slots_until_next_registration = protocol_config_pda + .config + .registration_phase_length + .saturating_sub(active_epoch_progress); + let hours_until_next_registration = slots_until_next_registration * 460 / 1000 / 3600; + + // Collect forester authorities for both epochs + let active_forester_authorities: Vec = forester_epoch_pdas + .iter() + .filter(|pda| pda.epoch == current_active_epoch) + .map(|pda| pda.authority) + .collect(); + + let registration_forester_authorities: Vec = forester_epoch_pdas + .iter() + .filter(|pda| pda.epoch == current_registration_epoch) + .map(|pda| pda.authority) + .collect(); + + // Fetch all forester balances in one batch call using Rpc trait + let all_forester_pubkeys: Vec = active_forester_authorities + .iter() + .chain(registration_forester_authorities.iter()) + .cloned() + .collect(); + + let forester_balances = fetch_forester_balances(&rpc, &all_forester_pubkeys).await; + + // Build ForesterInfo with balances + let active_epoch_foresters: Vec = active_forester_authorities + .iter() + .map(|authority| { + let balance = forester_balances.get(authority).copied().flatten(); + ForesterInfo { + authority: authority.to_string(), + balance_sol: balance, + } + }) + .collect(); + + let registration_epoch_foresters: Vec = registration_forester_authorities + .iter() + .map(|authority| { + let balance = forester_balances.get(authority).copied().flatten(); + ForesterInfo { + authority: authority.to_string(), + balance_sol: balance, + } + }) + .collect(); + + // Phase 2: Fetch trees using existing optimized method + let mut trees = match fetch_trees(&rpc).await { + Ok(trees) => trees, + Err(e) => { + warn!("Failed to fetch trees: {:?}", e); + return Ok(ForesterStatus { + slot, + current_active_epoch, + current_registration_epoch, + active_epoch_progress, + active_phase_length, + active_epoch_progress_percentage, + hours_until_next_epoch, + slots_until_next_registration, + hours_until_next_registration, + active_epoch_foresters, + registration_epoch_foresters, + trees: vec![], + current_light_slot: None, + light_slot_length: protocol_config_pda.config.slot_length, + slots_until_next_light_slot: None, + total_light_slots: 0, + }); + } + }; + + // Filter trees by protocol group authority if enabled + if filter_by_group_authority { + match fetch_protocol_group_authority(&rpc).await { + Ok(group_authority) => { + let before_count = trees.len(); + trees.retain(|tree| tree.owner == group_authority); + debug!( + "Group authority filtering enabled: filtered by {} ({} -> {} trees)", + group_authority, + before_count, + trees.len() + ); + } + Err(e) => { + warn!( + "Group authority filtering enabled but fetch failed: {:?}. Showing all trees.", + e + ); + } + } + } else { + debug!( + "Group authority filtering disabled, showing all {} trees", + trees.len() + ); + } + + // Phase 3: Batch fetch all tree and queue accounts using Rpc trait + let mut tree_statuses = fetch_tree_statuses_batched(&rpc, &trees).await; + + // Phase 4: Compute light slot info and forester assignments + let light_slot_length = protocol_config_pda.config.slot_length; + let mut current_light_slot: Option = None; + let mut slots_until_next_light_slot: Option = None; + let mut total_light_slots: u64 = 0; + + let active_epoch_forester_pdas: Vec<&ForesterEpochPda> = forester_epoch_pdas + .iter() + .filter(|pda| pda.epoch == current_active_epoch) + .collect(); + + // Build authority -> index map for schedule + let authority_to_index: HashMap = active_epoch_foresters + .iter() + .enumerate() + .map(|(i, f)| (f.authority.clone(), i)) + .collect(); + + if !active_epoch_forester_pdas.is_empty() { + if let Some(total_epoch_weight) = active_epoch_forester_pdas + .first() + .and_then(|pda| pda.total_epoch_weight) + .filter(|&w| w > 0) + { + let epoch_phases = get_epoch_phases(&protocol_config_pda.config, current_active_epoch); + + if light_slot_length > 0 { + total_light_slots = epoch_phases.active.length() / light_slot_length; + + // Compute current light slot if in active phase + if slot >= epoch_phases.active.start && slot < epoch_phases.active.end { + let current_light_slot_index = + (slot - epoch_phases.active.start) / light_slot_length; + current_light_slot = Some(current_light_slot_index); + + // Calculate slots until next light slot + let next_light_slot_start = epoch_phases.active.start + + (current_light_slot_index + 1) * light_slot_length; + slots_until_next_light_slot = Some(next_light_slot_start.saturating_sub(slot)); + } + + // Build full schedule for each tree + for status in &mut tree_statuses { + // V2 trees use the merkle tree pubkey as seed for eligibility, + // while V1 trees use the queue pubkey + let is_v2_tree = + status.tree_type == "StateV2" || status.tree_type == "AddressV2"; + let seed_pubkey_str = if is_v2_tree { + &status.merkle_tree + } else { + &status.queue + }; + let seed_pubkey: Pubkey = match seed_pubkey_str.parse() { + Ok(pk) => pk, + Err(e) => { + warn!( + "Failed to parse {} pubkey '{}': {}, skipping schedule computation", + if is_v2_tree { "merkle tree" } else { "queue" }, + seed_pubkey_str, + e + ); + continue; + } + }; + let mut schedule: Vec> = + Vec::with_capacity(total_light_slots as usize); + + for light_slot_idx in 0..total_light_slots { + let forester_idx = ForesterEpochPda::get_eligible_forester_index( + light_slot_idx, + &seed_pubkey, + total_epoch_weight, + current_active_epoch, + ) + .ok() + .and_then(|eligible_idx| { + active_epoch_forester_pdas + .iter() + .find(|pda| pda.is_eligible(eligible_idx)) + .and_then(|pda| authority_to_index.get(&pda.authority.to_string())) + .copied() + }); + schedule.push(forester_idx); + } + + // Set current assigned forester + if let Some(current_idx) = current_light_slot { + if let Some(Some(forester_idx)) = schedule.get(current_idx as usize) { + status.assigned_forester = + Some(active_epoch_foresters[*forester_idx].authority.clone()); + } + } + + status.schedule = schedule; + } + } + } + } + + Ok(ForesterStatus { + slot, + current_active_epoch, + current_registration_epoch, + active_epoch_progress, + active_phase_length, + active_epoch_progress_percentage, + hours_until_next_epoch, + slots_until_next_registration, + hours_until_next_registration, + active_epoch_foresters, + registration_epoch_foresters, + trees: tree_statuses, + current_light_slot, + light_slot_length, + slots_until_next_light_slot, + total_light_slots, + }) +} + +async fn fetch_registry_accounts_filtered( + rpc: &R, +) -> crate::Result<(Vec, Vec, Vec)> { + let program_id = light_registry::ID; + + // Use try_join! to propagate RPC errors instead of silently returning empty data + let (forester_accounts, epoch_accounts, config_accounts) = tokio::try_join!( + rpc.get_program_accounts_with_discriminator(&program_id, ForesterEpochPda::DISCRIMINATOR), + rpc.get_program_accounts_with_discriminator(&program_id, EpochPda::DISCRIMINATOR), + rpc.get_program_accounts_with_discriminator(&program_id, ProtocolConfigPda::DISCRIMINATOR), + ) + .context("Failed to fetch registry program accounts")?; + + let mut forester_epoch_pdas = Vec::new(); + let mut epoch_pdas = Vec::new(); + let mut protocol_config_pdas = Vec::new(); + + for (_, account) in forester_accounts { + let mut data: &[u8] = &account.data; + if let Ok(pda) = ForesterEpochPda::try_deserialize_unchecked(&mut data) { + forester_epoch_pdas.push(pda); + } + } + + for (_, account) in epoch_accounts { + let mut data: &[u8] = &account.data; + if let Ok(pda) = EpochPda::try_deserialize_unchecked(&mut data) { + epoch_pdas.push(pda); + } + } + + for (_, account) in config_accounts { + let mut data: &[u8] = &account.data; + if let Ok(pda) = ProtocolConfigPda::try_deserialize_unchecked(&mut data) { + protocol_config_pdas.push(pda); + } + } + + forester_epoch_pdas.sort_by(|a, b| a.epoch.cmp(&b.epoch)); + epoch_pdas.sort_by(|a, b| a.epoch.cmp(&b.epoch)); + + Ok((forester_epoch_pdas, epoch_pdas, protocol_config_pdas)) +} + +async fn fetch_tree_statuses_batched(rpc: &R, trees: &[TreeAccounts]) -> Vec { + if trees.is_empty() { + return vec![]; + } + + let mut pubkeys: Vec = Vec::with_capacity(trees.len() * 2); + let mut pubkey_map: Vec<(usize, &str)> = Vec::with_capacity(trees.len() * 2); + + for (i, tree) in trees.iter().enumerate() { + pubkeys.push(tree.merkle_tree); + pubkey_map.push((i, "merkle_tree")); + + if tree.tree_type != TreeType::AddressV2 { + pubkeys.push(tree.queue); + pubkey_map.push((i, "queue")); + } + } + + let accounts = match rpc.get_multiple_accounts(&pubkeys).await { + Ok(accounts) => accounts, + Err(e) => { + tracing::warn!("Failed to batch fetch accounts: {:?}", e); + return vec![]; + } + }; + + let mut tree_accounts: Vec<(Option, Option)> = + vec![(None, None); trees.len()]; + + for (idx, (tree_idx, account_type)) in pubkey_map.iter().enumerate() { + if let Some(Some(account)) = accounts.get(idx) { + match *account_type { + "merkle_tree" => tree_accounts[*tree_idx].0 = Some(account.clone()), + "queue" => tree_accounts[*tree_idx].1 = Some(account.clone()), + _ => {} + } + } + } + + let mut tree_statuses = Vec::with_capacity(trees.len()); + + for (i, tree) in trees.iter().enumerate() { + let (merkle_account, queue_account) = &tree_accounts[i]; + + match parse_tree_status(tree, merkle_account.clone(), queue_account.clone()) { + Ok(status) => tree_statuses.push(status), + Err(e) => { + tracing::warn!( + "Failed to parse tree status for {}: {:?}", + tree.merkle_tree, + e + ); + } + } + } + + tree_statuses +} + +async fn fetch_forester_balances( + rpc: &R, + pubkeys: &[Pubkey], +) -> HashMap> { + let mut balances = HashMap::new(); + + if pubkeys.is_empty() { + return balances; + } + + match rpc.get_multiple_accounts(pubkeys).await { + Ok(accounts) => { + for (i, account_opt) in accounts.iter().enumerate() { + if let Some(pubkey) = pubkeys.get(i) { + let balance = account_opt + .as_ref() + .map(|acc| acc.lamports as f64 / 1_000_000_000.0); + balances.insert(*pubkey, balance); + } + } + } + Err(e) => { + tracing::warn!("Failed to fetch forester balances: {:?}", e); + for pubkey in pubkeys { + balances.insert(*pubkey, None); + } + } + } + + balances +} + +fn parse_tree_status( + tree: &TreeAccounts, + merkle_account: Option, + queue_account: Option, +) -> crate::Result { + let mut merkle_account = + merkle_account.ok_or_else(|| anyhow::anyhow!("Merkle tree account not found"))?; + + let (fullness_percentage, next_index, threshold, queue_length, v2_queue_info) = match tree + .tree_type + { + TreeType::StateV1 => { + let tree_account = StateMerkleTreeAccount::deserialize(&mut &merkle_account.data[8..]) + .map_err(|e| anyhow::anyhow!("Failed to deserialize StateV1 metadata: {}", e))?; + + let height = STATE_MERKLE_TREE_HEIGHT; + let capacity = 1u64 << height; + let threshold_val = capacity + .saturating_mul(tree_account.metadata.rollover_metadata.rollover_threshold) + / 100; + + let merkle_tree = + parse_concurrent_merkle_tree_from_bytes::( + &merkle_account.data, + ) + .map_err(|e| anyhow::anyhow!("Failed to parse StateV1 tree: {:?}", e))?; + + let next_index = merkle_tree.next_index() as u64; + let fullness = next_index as f64 / capacity as f64 * 100.0; + + let queue_len = queue_account.and_then(|acc| { + unsafe { parse_hash_set_from_bytes::(&acc.data) } + .ok() + .map(|hs| { + hs.iter() + .filter(|(_, cell)| cell.sequence_number.is_none()) + .count() as u64 + }) + }); + + (fullness, next_index, threshold_val, queue_len, None) + } + TreeType::AddressV1 => { + let height = ADDRESS_MERKLE_TREE_HEIGHT; + let capacity = 1u64 << height; + + let threshold_val = queue_account + .as_ref() + .and_then(|acc| QueueAccount::deserialize(&mut &acc.data[8..]).ok()) + .map(|q| { + capacity.saturating_mul(q.metadata.rollover_metadata.rollover_threshold) / 100 + }) + .unwrap_or(0); + + let merkle_tree = parse_indexed_merkle_tree_from_bytes::< + AddressMerkleTreeAccount, + Poseidon, + usize, + 26, + 16, + >(&merkle_account.data) + .map_err(|e| anyhow::anyhow!("Failed to parse AddressV1 tree: {:?}", e))?; + + let next_index = merkle_tree + .next_index() + .saturating_sub(INDEXED_MERKLE_TREE_V1_INITIAL_LEAVES) + as u64; + let fullness = next_index as f64 / capacity as f64 * 100.0; + + let queue_len = queue_account.and_then(|acc| { + unsafe { parse_hash_set_from_bytes::(&acc.data) } + .ok() + .map(|hs| { + hs.iter() + .filter(|(_, cell)| cell.sequence_number.is_none()) + .count() as u64 + }) + }); + + (fullness, next_index, threshold_val, queue_len, None) + } + TreeType::StateV2 => { + let merkle_tree = BatchedMerkleTreeAccount::state_from_bytes( + &mut merkle_account.data, + &tree.merkle_tree.into(), + ) + .map_err(|e| anyhow::anyhow!("Failed to parse StateV2 tree: {:?}", e))?; + + let height = merkle_tree.height as u64; + let capacity = 1u64 << height; + let threshold_val = + (1u64 << height) * merkle_tree.metadata.rollover_metadata.rollover_threshold / 100; + let next_index = merkle_tree.next_index; + let fullness = next_index as f64 / capacity as f64 * 100.0; + + let v2_info = queue_account + .and_then(|mut acc| parse_state_v2_queue_info(&merkle_tree, &mut acc.data).ok()); + let queue_len = v2_info + .as_ref() + .map(|i| (i.input_pending_batches + i.output_pending_batches) * i.zkp_batch_size); + + (fullness, next_index, threshold_val, queue_len, v2_info) + } + TreeType::AddressV2 => { + let merkle_tree = BatchedMerkleTreeAccount::address_from_bytes( + &mut merkle_account.data, + &tree.merkle_tree.into(), + ) + .map_err(|e| anyhow::anyhow!("Failed to parse AddressV2 tree: {:?}", e))?; + + let height = merkle_tree.height as u64; + let capacity = 1u64 << height; + let threshold_val = + capacity * merkle_tree.metadata.rollover_metadata.rollover_threshold / 100; + let fullness = merkle_tree.next_index as f64 / capacity as f64 * 100.0; + + let v2_info = parse_address_v2_queue_info(&merkle_tree); + let queue_len = Some(v2_info.input_pending_batches * v2_info.zkp_batch_size); + + ( + fullness, + merkle_tree.next_index, + threshold_val, + queue_len, + Some(v2_info), + ) + } + TreeType::Unknown => { + warn!( + "Encountered unknown tree type for merkle_tree={}, queue={}", + tree.merkle_tree, tree.queue + ); + (0.0, 0, 0, None, None) + } + }; + + Ok(TreeStatus { + tree_type: tree.tree_type.to_string(), + merkle_tree: tree.merkle_tree.to_string(), + queue: tree.queue.to_string(), + fullness_percentage, + next_index, + threshold, + is_rolledover: tree.is_rolledover, + queue_length, + v2_queue_info, + assigned_forester: None, + schedule: Vec::new(), + owner: tree.owner.to_string(), + }) +} + pub async fn fetch_forester_status(args: &StatusArgs) -> crate::Result<()> { let commitment_config = CommitmentConfig::confirmed(); diff --git a/forester/src/lib.rs b/forester/src/lib.rs index e44e07f36c..85f136db0c 100644 --- a/forester/src/lib.rs +++ b/forester/src/lib.rs @@ -1,5 +1,6 @@ pub type Result = anyhow::Result; +pub mod api_server; pub mod cli; pub mod compressible; pub mod config; @@ -22,7 +23,6 @@ pub mod utils; use std::{sync::Arc, time::Duration}; -use account_compression::utils::constants::{ADDRESS_QUEUE_VALUES, STATE_NULLIFIER_QUEUE_VALUES}; pub use config::{ForesterConfig, ForesterEpochInfo}; use forester_utils::{ forester_epoch::TreeAccounts, rate_limiter::RateLimiter, rpc_pool::SolanaRpcPoolBuilder, @@ -43,7 +43,7 @@ use crate::{ print_state_v2_output_queue_info, }, slot_tracker::SlotTracker, - utils::get_protocol_config, + utils::{get_protocol_config_with_retry, get_slot_with_retry}, }; pub async fn run_queue_info( @@ -69,15 +69,9 @@ pub async fn run_queue_info( for tree_data in trees { match tree_data.tree_type { TreeType::StateV1 => { - let queue_length = fetch_queue_item_data( - &mut rpc, - &tree_data.queue, - 0, - STATE_NULLIFIER_QUEUE_VALUES, - STATE_NULLIFIER_QUEUE_VALUES, - ) - .await? - .len(); + let queue_length = fetch_queue_item_data(&mut rpc, &tree_data.queue, 0) + .await? + .len(); QUEUE_LENGTH .with_label_values(&[ &*queue_type.to_string(), @@ -91,15 +85,9 @@ pub async fn run_queue_info( ); } TreeType::AddressV1 => { - let queue_length = fetch_queue_item_data( - &mut rpc, - &tree_data.queue, - 0, - ADDRESS_QUEUE_VALUES, - ADDRESS_QUEUE_VALUES, - ) - .await? - .len(); + let queue_length = fetch_queue_item_data(&mut rpc, &tree_data.queue, 0) + .await? + .len(); QUEUE_LENGTH .with_label_values(&[ &*queue_type.to_string(), @@ -184,8 +172,8 @@ pub async fn run_pipeline( let (protocol_config, slot) = { let mut rpc = arc_pool.get_connection().await?; - let protocol_config = get_protocol_config(&mut *rpc).await?; - let slot = rpc.get_slot().await?; + let protocol_config = get_protocol_config_with_retry(&mut *rpc).await; + let slot = get_slot_with_retry(&mut *rpc).await; (protocol_config, slot) }; let slot_tracker = SlotTracker::new( diff --git a/forester/src/main.rs b/forester/src/main.rs index 48bbb49300..19d52bc4b1 100644 --- a/forester/src/main.rs +++ b/forester/src/main.rs @@ -2,6 +2,7 @@ use std::sync::Arc; use clap::Parser; use forester::{ + api_server::spawn_api_server, cli::{Cli, Commands}, errors::ForesterError, forester_status, @@ -54,6 +55,7 @@ where #[tokio::main] #[allow(clippy::result_large_err)] async fn main() -> Result<(), ForesterError> { + dotenvy::dotenv().ok(); setup_telemetry(); let cli = Cli::parse(); @@ -69,6 +71,28 @@ async fn main() -> Result<(), ForesterError> { let (shutdown_sender_service, shutdown_receiver_service) = oneshot::channel(); let (work_report_sender, mut work_report_receiver) = mpsc::channel(100); + tokio::spawn(async move { + while let Some(report) = work_report_receiver.recv().await { + debug!("Work Report: {:?}", report); + } + }); + + let rpc_rate_limiter = config + .external_services + .rpc_rate_limit + .map(RateLimiter::new); + let send_tx_limiter = config + .external_services + .send_tx_rate_limit + .map(RateLimiter::new); + + let rpc_url_for_api: String = config.external_services.rpc_url.to_string(); + let api_server_handle = spawn_api_server( + rpc_url_for_api, + args.api_server_port, + args.api_server_public_bind, + ); + // Create compressible shutdown channels if compressible is enabled let (shutdown_receiver_compressible, shutdown_receiver_bootstrap) = if config.compressible_config.is_some() { @@ -81,6 +105,7 @@ async fn main() -> Result<(), ForesterError> { Some(move || { let _ = shutdown_sender_compressible.send(()); let _ = shutdown_sender_bootstrap.send(()); + api_server_handle.shutdown(); }), ); ( @@ -88,26 +113,15 @@ async fn main() -> Result<(), ForesterError> { Some(shutdown_receiver_bootstrap), ) } else { - spawn_shutdown_handler::(shutdown_sender_service, None); + spawn_shutdown_handler( + shutdown_sender_service, + Some(move || { + api_server_handle.shutdown(); + }), + ); (None, None) }; - tokio::spawn(async move { - while let Some(report) = work_report_receiver.recv().await { - debug!("Work Report: {:?}", report); - } - }); - - let mut rpc_rate_limiter = None; - if let Some(rate_limit) = config.external_services.rpc_rate_limit { - rpc_rate_limiter = Some(RateLimiter::new(rate_limit)); - } - - let mut send_tx_limiter = None; - if let Some(rate_limit) = config.external_services.send_tx_rate_limit { - send_tx_limiter = Some(RateLimiter::new(rate_limit)); - } - run_pipeline::( config, rpc_rate_limiter, diff --git a/forester/src/metrics.rs b/forester/src/metrics.rs index 9b01c33cee..e5a4c9aa8c 100644 --- a/forester/src/metrics.rs +++ b/forester/src/metrics.rs @@ -4,7 +4,9 @@ use std::{ }; use lazy_static::lazy_static; -use prometheus::{Encoder, GaugeVec, IntCounterVec, IntGauge, IntGaugeVec, Registry, TextEncoder}; +use prometheus::{ + Encoder, GaugeVec, HistogramVec, IntCounterVec, IntGauge, IntGaugeVec, Registry, TextEncoder, +}; use reqwest::Client; use tokio::sync::Mutex; use tracing::{debug, error, log::trace}; @@ -81,6 +83,29 @@ lazy_static! { error!("Failed to create metric REGISTERED_FORESTERS: {:?}", e); std::process::exit(1); }); + pub static ref INDEXER_RESPONSE_TIME: HistogramVec = HistogramVec::new( + prometheus::HistogramOpts::new( + "forester_indexer_response_time_seconds", + "Response time for indexer proof requests in seconds" + ) + .buckets(vec![0.01, 0.05, 0.1, 0.5, 1.0, 5.0, 10.0]), + &["operation", "tree_type"] + ) + .unwrap_or_else(|e| { + error!("Failed to create metric INDEXER_RESPONSE_TIME: {:?}", e); + std::process::exit(1); + }); + pub static ref INDEXER_PROOF_COUNT: IntCounterVec = IntCounterVec::new( + prometheus::opts!( + "forester_indexer_proof_count", + "Number of proofs requested vs received from indexer" + ), + &["tree_type", "metric"] + ) + .unwrap_or_else(|e| { + error!("Failed to create metric INDEXER_PROOF_COUNT: {:?}", e); + std::process::exit(1); + }); static ref METRIC_UPDATES: Mutex> = Mutex::new(Vec::new()); } @@ -109,6 +134,12 @@ pub fn register_metrics() { if let Err(e) = REGISTRY.register(Box::new(REGISTERED_FORESTERS.clone())) { error!("Failed to register metric REGISTERED_FORESTERS: {:?}", e); } + if let Err(e) = REGISTRY.register(Box::new(INDEXER_RESPONSE_TIME.clone())) { + error!("Failed to register metric INDEXER_RESPONSE_TIME: {:?}", e); + } + if let Err(e) = REGISTRY.register(Box::new(INDEXER_PROOF_COUNT.clone())) { + error!("Failed to register metric INDEXER_PROOF_COUNT: {:?}", e); + } }); } @@ -180,6 +211,29 @@ pub fn update_registered_foresters(epoch: u64, authority: &str) { .set(1.0); } +pub fn update_indexer_response_time(operation: &str, tree_type: &str, duration_secs: f64) { + // Ensure metrics are registered before updating (idempotent via Once) + register_metrics(); + INDEXER_RESPONSE_TIME + .with_label_values(&[operation, tree_type]) + .observe(duration_secs); + debug!( + "Indexer {} for {} took {:.3}s", + operation, tree_type, duration_secs + ); +} + +pub fn update_indexer_proof_count(tree_type: &str, requested: u64, received: u64) { + // Ensure metrics are registered before updating (idempotent via Once) + register_metrics(); + INDEXER_PROOF_COUNT + .with_label_values(&[tree_type, "requested"]) + .inc_by(requested); + INDEXER_PROOF_COUNT + .with_label_values(&[tree_type, "received"]) + .inc_by(received); +} + pub async fn push_metrics(url: &Option) -> Result<()> { let url = match url { Some(url) => url, diff --git a/forester/src/processor/v1/helpers.rs b/forester/src/processor/v1/helpers.rs index 1b4e4609f6..606f3c1cbd 100644 --- a/forester/src/processor/v1/helpers.rs +++ b/forester/src/processor/v1/helpers.rs @@ -8,10 +8,7 @@ use account_compression::{ }, }; use forester_utils::{rpc_pool::SolanaRpcPool, utils::wait_for_indexer}; -use light_client::{ - indexer::{Indexer, Items, MerkleProof, NewAddressProofWithContext}, - rpc::Rpc, -}; +use light_client::{indexer::Indexer, rpc::Rpc}; use light_compressed_account::TreeType; use light_registry::account_compression_cpi::sdk::{ create_nullify_instruction, create_update_address_merkle_tree_instruction, @@ -19,8 +16,14 @@ use light_registry::account_compression_cpi::sdk::{ }; use reqwest::Url; use solana_program::instruction::Instruction; -use tokio::join; -use tracing::warn; +use tokio::time::Instant; +use tracing::{info, warn}; + +use crate::metrics::{update_indexer_proof_count, update_indexer_response_time}; + +const ADDRESS_PROOF_BATCH_SIZE: usize = 100; +const ADDRESS_PROOF_MAX_RETRIES: u32 = 3; +const ADDRESS_PROOF_RETRY_BASE_DELAY_MS: u64 = 500; use crate::{ epoch_manager::{MerkleProofType, WorkItem}, @@ -93,48 +96,207 @@ pub async fn fetch_proofs_and_create_instructions( warn!("Indexer not fully caught up, but proceeding anyway: {}", e); } - let (address_proofs_result, state_proofs_result) = { - let address_future = async { - if let Some((merkle_tree, addresses)) = address_data { - rpc.indexer()? - .get_multiple_new_address_proofs(merkle_tree, addresses, None) + let address_proofs = if let Some((merkle_tree, addresses)) = address_data { + let total_addresses = addresses.len(); + info!( + "Fetching {} address proofs in batches of {}", + total_addresses, ADDRESS_PROOF_BATCH_SIZE + ); + + let start_time = Instant::now(); + let mut all_proofs = Vec::with_capacity(total_addresses); + + for (batch_idx, batch) in addresses.chunks(ADDRESS_PROOF_BATCH_SIZE).enumerate() { + let batch_start = Instant::now(); + // Pass slice directly if indexer accepts it, otherwise clone + let batch_addresses: Vec<[u8; 32]> = batch.to_vec(); + let batch_size = batch_addresses.len(); + + // Retry loop for transient network errors + let mut last_error = None; + for attempt in 0..=ADDRESS_PROOF_MAX_RETRIES { + if attempt > 0 { + // Exponential backoff: 500ms, 1000ms, 2000ms + let delay_ms = ADDRESS_PROOF_RETRY_BASE_DELAY_MS * (1 << (attempt - 1)); + warn!( + "Retrying address proof batch {} (attempt {}/{}), waiting {}ms", + batch_idx, + attempt + 1, + ADDRESS_PROOF_MAX_RETRIES + 1, + delay_ms + ); + tokio::time::sleep(std::time::Duration::from_millis(delay_ms)).await; + } + + match rpc + .indexer()? + .get_multiple_new_address_proofs(merkle_tree, batch_addresses.clone(), None) .await - } else { - Ok(light_client::indexer::Response { - context: light_client::indexer::Context::default(), - value: Items::::default(), - }) + { + Ok(response) => { + let batch_duration = batch_start.elapsed(); + let proofs_received = response.value.items.len(); + + info!( + "Address proof batch {}: requested={}, received={}, duration={:.3}s{}", + batch_idx, + batch_size, + proofs_received, + batch_duration.as_secs_f64(), + if attempt > 0 { + format!(" (after {} retries)", attempt) + } else { + String::new() + } + ); + + if proofs_received != batch_size { + warn!( + "Address proof count mismatch in batch {}: requested={}, received={}", + batch_idx, batch_size, proofs_received + ); + } + + all_proofs.extend(response.value.items); + last_error = None; + break; + } + Err(e) => { + last_error = Some(e); + } + } } - }; - let state_future = async { - if let Some(states) = state_data { - rpc.indexer()? - .get_multiple_compressed_account_proofs(states, None) - .await - } else { - Ok(light_client::indexer::Response { - context: light_client::indexer::Context::default(), - value: Items::::default(), - }) + // If we exhausted all retries, return the last error + if let Some(e) = last_error { + let batch_duration = batch_start.elapsed(); + warn!( + "Failed to get address proofs for batch {} after {} attempts ({:.3}s): {}", + batch_idx, + ADDRESS_PROOF_MAX_RETRIES + 1, + batch_duration.as_secs_f64(), + e + ); + return Err(anyhow::anyhow!( + "Failed to get address proofs for batch {} after {} retries: {}", + batch_idx, + ADDRESS_PROOF_MAX_RETRIES, + e + )); } - }; + } + + let total_duration = start_time.elapsed(); + info!( + "Address proofs complete: requested={}, received={}, total_duration={:.3}s", + total_addresses, + all_proofs.len(), + total_duration.as_secs_f64() + ); + + update_indexer_response_time( + "get_multiple_new_address_proofs", + "AddressV1", + total_duration.as_secs_f64(), + ); + update_indexer_proof_count("AddressV1", total_addresses as u64, all_proofs.len() as u64); - join!(address_future, state_future) + all_proofs + } else { + Vec::new() }; - let address_proofs = match address_proofs_result { - Ok(response) => response.value.items, - Err(e) => { - return Err(anyhow::anyhow!("Failed to get address proofs: {}", e)); + let state_proofs = if let Some(states) = state_data { + let total_states = states.len(); + info!("Fetching {} state proofs", total_states); + + let start_time = Instant::now(); + + // Retry loop for transient network errors + let mut last_error = None; + let mut proofs = None; + + for attempt in 0..=ADDRESS_PROOF_MAX_RETRIES { + if attempt > 0 { + // Exponential backoff: 500ms, 1000ms, 2000ms + let delay_ms = ADDRESS_PROOF_RETRY_BASE_DELAY_MS * (1 << (attempt - 1)); + warn!( + "Retrying state proofs (attempt {}/{}), waiting {}ms", + attempt + 1, + ADDRESS_PROOF_MAX_RETRIES + 1, + delay_ms + ); + tokio::time::sleep(std::time::Duration::from_millis(delay_ms)).await; + } + + match rpc + .indexer()? + .get_multiple_compressed_account_proofs(states.clone(), None) + .await + { + Ok(response) => { + let duration = start_time.elapsed(); + let proofs_received = response.value.items.len(); + + info!( + "State proofs complete: requested={}, received={}, duration={:.3}s{}", + total_states, + proofs_received, + duration.as_secs_f64(), + if attempt > 0 { + format!(" (after {} retries)", attempt) + } else { + String::new() + } + ); + + if proofs_received != total_states { + warn!( + "State proof count mismatch: requested={}, received={}", + total_states, proofs_received + ); + } + + update_indexer_response_time( + "get_multiple_compressed_account_proofs", + "StateV1", + duration.as_secs_f64(), + ); + update_indexer_proof_count( + "StateV1", + total_states as u64, + proofs_received as u64, + ); + + proofs = Some(response.value.items); + last_error = None; + break; + } + Err(e) => { + last_error = Some(e); + } + } } - }; - let state_proofs = match state_proofs_result { - Ok(response) => response.value.items, - Err(e) => { - return Err(anyhow::anyhow!("Failed to get state proofs: {}", e)); + // If we exhausted all retries, return the last error + if let Some(e) = last_error { + let duration = start_time.elapsed(); + warn!( + "Failed to get state proofs after {} attempts ({:.3}s): {}", + ADDRESS_PROOF_MAX_RETRIES + 1, + duration.as_secs_f64(), + e + ); + return Err(anyhow::anyhow!( + "Failed to get state proofs after {} retries: {}", + ADDRESS_PROOF_MAX_RETRIES, + e + )); } + + proofs.unwrap_or_default() + } else { + Vec::new() }; for (item, proof) in address_items.iter().zip(address_proofs.into_iter()) { diff --git a/forester/src/processor/v1/send_transaction.rs b/forester/src/processor/v1/send_transaction.rs index 555710c8d9..a91a94c3a9 100644 --- a/forester/src/processor/v1/send_transaction.rs +++ b/forester/src/processor/v1/send_transaction.rs @@ -6,7 +6,6 @@ use std::{ vec, }; -use account_compression::utils::constants::{ADDRESS_QUEUE_VALUES, STATE_NULLIFIER_QUEUE_VALUES}; use forester_utils::{forester_epoch::TreeAccounts, rpc_pool::SolanaRpcPool}; use futures::StreamExt; use light_client::rpc::Rpc; @@ -14,15 +13,17 @@ use light_compressed_account::TreeType; use light_registry::utils::get_forester_epoch_pda_from_authority; use reqwest::Url; use solana_client::rpc_config::RpcSendTransactionConfig; +use solana_commitment_config::CommitmentLevel; use solana_sdk::{ - commitment_config::CommitmentLevel, hash::Hash, pubkey::Pubkey, signature::{Keypair, Signature, Signer}, transaction::Transaction, }; use tokio::time::Instant; -use tracing::{error, trace, warn}; +use tracing::{error, info, trace, warn}; + +const WORK_ITEM_BATCH_SIZE: usize = 100; use crate::{ epoch_manager::WorkItem, @@ -91,12 +92,16 @@ pub async fn send_batched_transactions( ) -> Result> { let tree_id_str = tree_accounts.merkle_tree.to_string(); - let (queue_total_capacity, queue_fetch_start_index, queue_fetch_length) = - match tree_accounts.tree_type { - TreeType::StateV1 => ( - STATE_NULLIFIER_QUEUE_VALUES, - config.queue_config.state_queue_start_index, - config.queue_config.state_queue_length, - ), - TreeType::AddressV1 => ( - ADDRESS_QUEUE_VALUES, - config.queue_config.address_queue_start_index, - config.queue_config.address_queue_length, - ), - _ => { - error!( - tree = %tree_id_str, - "prepare_batch_prerequisites called with unsupported tree type: {:?}", - tree_accounts.tree_type - ); - return Err(ForesterError::InvalidTreeType(tree_accounts.tree_type).into()); - } - }; + let queue_fetch_start_index = match tree_accounts.tree_type { + TreeType::StateV1 => config.queue_config.state_queue_start_index, + TreeType::AddressV1 => config.queue_config.address_queue_start_index, + _ => { + error!( + tree = %tree_id_str, + "prepare_batch_prerequisites called with unsupported tree type: {:?}", + tree_accounts.tree_type + ); + return Err(ForesterError::InvalidTreeType(tree_accounts.tree_type).into()); + } + }; let queue_item_data = { let mut rpc = pool.get_connection().await.map_err(|e| { error!(tree = %tree_id_str, "Failed to get RPC for queue data: {:?}", e); ForesterError::RpcPool(e) })?; - fetch_queue_item_data( - &mut *rpc, - &tree_accounts.queue, - queue_fetch_start_index, - queue_fetch_length, - queue_total_capacity, - ) - .await - .map_err(|e| { - error!(tree = %tree_id_str, "Failed to fetch queue item data: {:?}", e); - ForesterError::General { - error: format!("Fetch queue data failed for {}: {}", tree_id_str, e), - } - })? + fetch_queue_item_data(&mut *rpc, &tree_accounts.queue, queue_fetch_start_index) + .await + .map_err(|e| { + warn!(tree = %tree_id_str, "Failed to fetch queue item data: {:?}", e); + ForesterError::General { + error: format!("Fetch queue data failed for {}: {}", tree_id_str, e), + } + })? }; if queue_item_data.is_empty() { diff --git a/forester/src/processor/v2/common.rs b/forester/src/processor/v2/common.rs index 9f0616c9d4..21703db081 100644 --- a/forester/src/processor/v2/common.rs +++ b/forester/src/processor/v2/common.rs @@ -95,6 +95,8 @@ pub struct BatchContext { pub confirmation_max_attempts: u32, /// Interval between confirmation polling attempts. pub confirmation_poll_interval: Duration, + /// Maximum batches to process per tree per iteration + pub max_batches_per_tree: usize, } impl Clone for BatchContext { @@ -117,6 +119,7 @@ impl Clone for BatchContext { address_lookup_tables: self.address_lookup_tables.clone(), confirmation_max_attempts: self.confirmation_max_attempts, confirmation_poll_interval: self.confirmation_poll_interval, + max_batches_per_tree: self.max_batches_per_tree, } } } diff --git a/forester/src/processor/v2/processor.rs b/forester/src/processor/v2/processor.rs index 91a606910d..4df6ed4eae 100644 --- a/forester/src/processor/v2/processor.rs +++ b/forester/src/processor/v2/processor.rs @@ -26,8 +26,6 @@ use crate::{ }, }; -const MAX_BATCHES_PER_TREE: usize = 20; - #[derive(Debug, Default, Clone)] struct BatchTimings { append_circuit_inputs: Duration, @@ -114,7 +112,7 @@ where } pub async fn process(&mut self) -> crate::Result { - let queue_size = self.zkp_batch_size * MAX_BATCHES_PER_TREE as u64; + let queue_size = self.zkp_batch_size * self.context.max_batches_per_tree as u64; self.process_queue_update(queue_size).await } @@ -149,7 +147,7 @@ where if actual_available == usize::MAX { "max".to_string() } else { actual_available.to_string() } ); - let batches_to_process = remaining.min(MAX_BATCHES_PER_TREE); + let batches_to_process = remaining.min(self.context.max_batches_per_tree); let queue_data = QueueData { staging_tree: cached.staging_tree, initial_root: self.current_root, @@ -168,9 +166,9 @@ where } let available_batches = (queue_size / self.zkp_batch_size) as usize; - let fetch_batches = available_batches.min(MAX_BATCHES_PER_TREE); + let fetch_batches = available_batches.min(self.context.max_batches_per_tree); - if available_batches > MAX_BATCHES_PER_TREE { + if available_batches > self.context.max_batches_per_tree { debug!( "Queue has {} batches available, fetching {} for {} iterations", available_batches, @@ -197,7 +195,7 @@ where if self.current_root == [0u8; 32] || queue_data.initial_root == self.current_root { let total_batches = queue_data.num_batches; - let process_now = total_batches.min(MAX_BATCHES_PER_TREE); + let process_now = total_batches.min(self.context.max_batches_per_tree); return self .process_batches(queue_data, 0, process_now, total_batches) .await; @@ -207,7 +205,7 @@ where match reconcile_roots(self.current_root, queue_data.initial_root, onchain_root) { RootReconcileDecision::Proceed => { let total_batches = queue_data.num_batches; - let process_now = total_batches.min(MAX_BATCHES_PER_TREE); + let process_now = total_batches.min(self.context.max_batches_per_tree); self.process_batches(queue_data, 0, process_now, total_batches) .await } @@ -228,7 +226,7 @@ where self.current_root = root; self.cached_state = None; let total_batches = queue_data.num_batches; - let process_now = total_batches.min(MAX_BATCHES_PER_TREE); + let process_now = total_batches.min(self.context.max_batches_per_tree); self.process_batches(queue_data, 0, process_now, total_batches) .await } @@ -503,7 +501,8 @@ where return Ok(ProcessingResult::default()); } - let max_batches = ((queue_size / self.zkp_batch_size) as usize).min(MAX_BATCHES_PER_TREE); + let max_batches = + ((queue_size / self.zkp_batch_size) as usize).min(self.context.max_batches_per_tree); if self.worker_pool.is_none() { let job_tx = spawn_proof_workers(&self.context.prover_config); @@ -532,7 +531,7 @@ where return Ok(ProcessingResult::default()); } - let max_batches = max_batches.min(MAX_BATCHES_PER_TREE); + let max_batches = max_batches.min(self.context.max_batches_per_tree); if self.worker_pool.is_none() { let job_tx = spawn_proof_workers(&self.context.prover_config); diff --git a/forester/src/processor/v2/tx_sender.rs b/forester/src/processor/v2/tx_sender.rs index d154686b6e..0fd42fe687 100644 --- a/forester/src/processor/v2/tx_sender.rs +++ b/forester/src/processor/v2/tx_sender.rs @@ -388,9 +388,41 @@ impl TxSender { break; } - let result = match proof_rx.recv().await { - Some(r) => r, - None => break, + let current_slot = self.context.slot_tracker.estimated_current_slot(); + if !self.is_still_eligible_at(current_slot) { + let proofs_saved = self.save_proofs_to_cache(&mut proof_rx, None).await; + info!( + "Active phase ended for epoch {}, stopping tx sender before recv (saved {} proofs to cache)", + self.context.epoch, proofs_saved + ); + drop(batch_tx); + let (items_processed, tx_sending_duration) = sender_handle + .await + .map_err(|e| anyhow::anyhow!("Sender panic: {}", e))??; + return Ok(TxSenderResult { + items_processed, + proof_timings: self.proof_timings, + proofs_saved_to_cache: proofs_saved, + tx_sending_duration, + }); + } + + // Use biased select to ensure the 1-second sleep branch is checked first. + // This guarantees periodic re-evaluation of sender eligibility even when + // proof_rx.recv() would otherwise block indefinitely waiting for proofs. + // Near epoch boundaries, this prevents getting stuck waiting on proofs + // when the forester's eligibility window is about to end. + let result = tokio::select! { + biased; + _ = tokio::time::sleep(Duration::from_secs(1)) => { + continue; + } + result = proof_rx.recv() => { + match result { + Some(r) => r, + None => break, + } + } }; let current_slot = self.context.slot_tracker.estimated_current_slot(); diff --git a/forester/src/queue_helpers.rs b/forester/src/queue_helpers.rs index aad6b5984a..8a1b008677 100644 --- a/forester/src/queue_helpers.rs +++ b/forester/src/queue_helpers.rs @@ -1,16 +1,125 @@ use account_compression::QueueAccount; use light_batched_merkle_tree::{ - constants::{DEFAULT_ADDRESS_ZKP_BATCH_SIZE, DEFAULT_ZKP_BATCH_SIZE}, - merkle_tree::BatchedMerkleTreeAccount, - queue::BatchedQueueAccount, + batch::BatchState, merkle_tree::BatchedMerkleTreeAccount, queue::BatchedQueueAccount, }; use light_client::rpc::Rpc; use light_hash_set::HashSet; +use serde::{Deserialize, Serialize}; use solana_sdk::pubkey::Pubkey; use tracing::trace; use crate::Result; +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct V2QueueInfo { + pub next_index: u64, + pub pending_batch_index: u64, + pub zkp_batch_size: u64, + pub batches: Vec, + pub input_pending_batches: u64, + pub output_pending_batches: u64, + pub input_items_in_current_zkp_batch: u64, + pub output_items_in_current_zkp_batch: u64, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BatchInfo { + pub batch_index: usize, + pub batch_state: u64, + pub num_inserted: u64, + pub current_index: u64, + pub pending: u64, + pub items_in_current_zkp_batch: u64, +} + +#[derive(Debug, Clone)] +pub struct ParsedBatchData { + pub batch_infos: Vec, + pub total_pending_batches: u64, + pub zkp_batch_size: u64, + pub items_in_current_zkp_batch: u64, +} + +pub fn parse_batch_metadata( + batches: &[light_batched_merkle_tree::batch::Batch], +) -> ParsedBatchData { + use light_batched_merkle_tree::constants::DEFAULT_ZKP_BATCH_SIZE; + + let mut zkp_batch_size = DEFAULT_ZKP_BATCH_SIZE; + let mut total_pending_batches = 0u64; + let mut batch_infos = Vec::with_capacity(batches.len()); + let mut items_in_current_zkp_batch = 0u64; + + for (batch_idx, batch) in batches.iter().enumerate() { + zkp_batch_size = batch.zkp_batch_size; + let num_inserted = batch.get_num_inserted_zkps(); + let current_index = batch.get_current_zkp_batch_index(); + let pending_in_batch = current_index.saturating_sub(num_inserted); + + if batch.get_state() == BatchState::Fill { + items_in_current_zkp_batch = batch.get_num_inserted_zkp_batch(); + } + + batch_infos.push(BatchInfo { + batch_index: batch_idx, + batch_state: batch.get_state().into(), + num_inserted, + current_index, + pending: pending_in_batch, + items_in_current_zkp_batch: batch.get_num_inserted_zkp_batch(), + }); + + total_pending_batches += pending_in_batch; + } + + ParsedBatchData { + batch_infos, + total_pending_batches, + zkp_batch_size, + items_in_current_zkp_batch, + } +} + +pub fn parse_state_v2_queue_info( + merkle_tree: &BatchedMerkleTreeAccount, + output_queue_data: &mut [u8], +) -> Result { + let output_queue = BatchedQueueAccount::output_from_bytes(output_queue_data) + .map_err(|e| anyhow::anyhow!("Failed to parse StateV2 output queue: {:?}", e))?; + + let next_index = output_queue.batch_metadata.next_index; + + let output_parsed = parse_batch_metadata(&output_queue.batch_metadata.batches); + let input_parsed = parse_batch_metadata(&merkle_tree.queue_batches.batches); + + Ok(V2QueueInfo { + next_index, + pending_batch_index: output_queue.batch_metadata.pending_batch_index, + zkp_batch_size: output_parsed.zkp_batch_size, + batches: output_parsed.batch_infos, + input_pending_batches: input_parsed.total_pending_batches, + output_pending_batches: output_parsed.total_pending_batches, + input_items_in_current_zkp_batch: input_parsed.items_in_current_zkp_batch, + output_items_in_current_zkp_batch: output_parsed.items_in_current_zkp_batch, + }) +} + +pub fn parse_address_v2_queue_info(merkle_tree: &BatchedMerkleTreeAccount) -> V2QueueInfo { + let next_index = merkle_tree.queue_batches.next_index; + let parsed = parse_batch_metadata(&merkle_tree.queue_batches.batches); + + V2QueueInfo { + next_index, + pending_batch_index: merkle_tree.queue_batches.pending_batch_index, + zkp_batch_size: parsed.zkp_batch_size, + batches: parsed.batch_infos, + input_pending_batches: parsed.total_pending_batches, + output_pending_batches: 0, + input_items_in_current_zkp_batch: parsed.items_in_current_zkp_batch, + output_items_in_current_zkp_batch: 0, + } +} + #[derive(Debug, Clone)] pub struct QueueItemData { pub hash: [u8; 32], @@ -21,8 +130,6 @@ pub async fn fetch_queue_item_data( rpc: &mut R, queue_pubkey: &Pubkey, start_index: u16, - processing_length: u16, - queue_length: u16, ) -> Result> { trace!("Fetching queue data for {:?}", queue_pubkey); let account = rpc.get_account(*queue_pubkey).await?; @@ -36,7 +143,7 @@ pub async fn fetch_queue_item_data( return Ok(Vec::new()); } }; - let offset = 8 + std::mem::size_of::(); + let offset = 8 + size_of::(); if account.data.len() < offset { tracing::warn!( "Queue account {} data too short ({} < {})", @@ -47,20 +154,38 @@ pub async fn fetch_queue_item_data( return Ok(Vec::new()); } let queue: HashSet = unsafe { HashSet::from_bytes_copy(&mut account.data[offset..])? }; - let end_index = (start_index + processing_length).min(queue_length); - let filtered_queue = queue + let end_index = queue.get_capacity(); + + let all_items: Vec<(usize, [u8; 32], bool)> = queue .iter() - .filter(|(index, cell)| { - *index >= start_index as usize - && *index < end_index as usize - && cell.sequence_number.is_none() - }) - .map(|(index, cell)| QueueItemData { - hash: cell.value_bytes(), - index, + .map(|(index, cell)| (index, cell.value_bytes(), cell.sequence_number.is_none())) + .collect(); + + let total_items = all_items.len(); + let total_pending = all_items + .iter() + .filter(|(_, _, is_pending)| *is_pending) + .count(); + + let filtered_queue: Vec = all_items + .into_iter() + .filter(|(index, _, is_pending)| { + *index >= start_index as usize && *index < end_index && *is_pending }) + .map(|(index, hash, _)| QueueItemData { hash, index }) .collect(); + + tracing::debug!( + "Queue {}: total_items={}, total_pending={}, range={}..{}, filtered_result={}", + queue_pubkey, + total_items, + total_pending, + start_index, + end_index, + filtered_queue.len() + ); + Ok(filtered_queue) } @@ -72,37 +197,32 @@ pub async fn print_state_v2_output_queue_info( let output_queue = BatchedQueueAccount::output_from_bytes(account.data.as_mut_slice())?; let metadata = output_queue.get_metadata(); let next_index = metadata.batch_metadata.next_index; + let zkp_batch_size = metadata.batch_metadata.zkp_batch_size; - let mut zkp_batch_size = DEFAULT_ZKP_BATCH_SIZE; - let mut total_unprocessed = 0; - let mut batch_details = Vec::new(); - let mut total_completed_operations = 0; + let parsed = parse_batch_metadata(&metadata.batch_metadata.batches); - for (batch_idx, batch) in metadata.batch_metadata.batches.iter().enumerate() { - zkp_batch_size = batch.zkp_batch_size; - let num_inserted = batch.get_num_inserted_zkps(); - let current_index = batch.get_current_zkp_batch_index(); - let pending_in_batch = current_index.saturating_sub(num_inserted); + // Calculate completed and pending operations (in items, not batches) + let mut total_completed_operations = 0u64; + let mut total_unprocessed = 0u64; + let mut batch_details = Vec::new(); - let completed_operations_in_batch = - num_inserted * metadata.batch_metadata.zkp_batch_size; + for batch_info in &parsed.batch_infos { + let completed_operations_in_batch = batch_info.num_inserted * zkp_batch_size; total_completed_operations += completed_operations_in_batch; - let pending_operations_in_batch = - pending_in_batch * metadata.batch_metadata.zkp_batch_size; + let pending_operations_in_batch = batch_info.pending * zkp_batch_size; + total_unprocessed += pending_operations_in_batch; batch_details.push(format!( "batch_{}: state={:?}, zkp_inserted={}, zkp_current={}, zkp_pending={}, items_completed={}, items_pending={}", - batch_idx, - batch.get_state(), - num_inserted, - current_index, - pending_in_batch, + batch_info.batch_index, + BatchState::from(batch_info.batch_state), + batch_info.num_inserted, + batch_info.current_index, + batch_info.pending, completed_operations_in_batch, pending_operations_in_batch )); - - total_unprocessed += pending_operations_in_batch; } println!("StateV2 {} APPEND:", output_queue_pubkey); @@ -116,10 +236,7 @@ pub async fn print_state_v2_output_queue_info( " pending_batch_index: {}", metadata.batch_metadata.pending_batch_index ); - println!( - " zkp_batch_size: {}", - metadata.batch_metadata.zkp_batch_size - ); + println!(" zkp_batch_size: {}", zkp_batch_size); println!( " SUMMARY: {} items added, {} items processed, {} items pending", next_index, total_completed_operations, total_unprocessed @@ -129,7 +246,7 @@ pub async fn print_state_v2_output_queue_info( } println!( " Total pending APPEND operations: {}", - total_unprocessed / zkp_batch_size + parsed.total_pending_batches ); Ok(total_unprocessed as usize) @@ -149,30 +266,25 @@ pub async fn print_state_v2_input_queue_info( )?; let next_index = merkle_tree.queue_batches.next_index; + let parsed = parse_batch_metadata(&merkle_tree.queue_batches.batches); + let mut total_unprocessed = 0; let mut batch_details = Vec::new(); let mut total_completed_operations = 0; - let mut zkp_batch_size = DEFAULT_ZKP_BATCH_SIZE; - - for (batch_idx, batch) in merkle_tree.queue_batches.batches.iter().enumerate() { - zkp_batch_size = batch.zkp_batch_size; - let num_inserted = batch.get_num_inserted_zkps(); - let current_index = batch.get_current_zkp_batch_index(); - let pending_in_batch = current_index.saturating_sub(num_inserted); - - let completed_operations_in_batch = num_inserted * batch.zkp_batch_size; + for batch_info in &parsed.batch_infos { + let completed_operations_in_batch = batch_info.num_inserted * parsed.zkp_batch_size; total_completed_operations += completed_operations_in_batch; - let pending_operations_in_batch = pending_in_batch * batch.zkp_batch_size; + let pending_operations_in_batch = batch_info.pending * parsed.zkp_batch_size; batch_details.push(format!( "batch_{}: state={:?}, zkp_inserted={}, zkp_current={}, zkp_pending={}, items_completed={}, items_pending={}", - batch_idx, - batch.get_state(), - num_inserted, - current_index, - pending_in_batch, + batch_info.batch_index, + BatchState::from(batch_info.batch_state), + batch_info.num_inserted, + batch_info.current_index, + batch_info.pending, completed_operations_in_batch, pending_operations_in_batch )); @@ -191,7 +303,7 @@ pub async fn print_state_v2_input_queue_info( " pending_batch_index: {}", merkle_tree.queue_batches.pending_batch_index ); - println!(" zkp_batch_size: {}", zkp_batch_size); + println!(" zkp_batch_size: {}", parsed.zkp_batch_size); println!( " SUMMARY: {} items added, {} items processed, {} items pending", next_index, total_completed_operations, total_unprocessed @@ -201,7 +313,7 @@ pub async fn print_state_v2_input_queue_info( } println!( " Total pending NULLIFY operations: {}", - total_unprocessed / zkp_batch_size + total_unprocessed / parsed.zkp_batch_size ); Ok(total_unprocessed as usize) @@ -221,26 +333,22 @@ pub async fn print_address_v2_queue_info( )?; let next_index = merkle_tree.queue_batches.next_index; - let mut zkp_batch_size = DEFAULT_ADDRESS_ZKP_BATCH_SIZE; + let parsed = parse_batch_metadata(&merkle_tree.queue_batches.batches); + let mut total_unprocessed = 0; let mut batch_details = Vec::new(); - for (batch_idx, batch) in merkle_tree.queue_batches.batches.iter().enumerate() { - zkp_batch_size = batch.zkp_batch_size; - let num_inserted = batch.get_num_inserted_zkps(); - let current_index = batch.get_current_zkp_batch_index(); - let pending_in_batch = current_index.saturating_sub(num_inserted); - + for batch_info in &parsed.batch_infos { batch_details.push(format!( "batch_{}: state={:?}, inserted={}, current={}, pending={}", - batch_idx, - batch.get_state(), - num_inserted, - current_index, - pending_in_batch + batch_info.batch_index, + BatchState::from(batch_info.batch_state), + batch_info.num_inserted, + batch_info.current_index, + batch_info.pending )); - total_unprocessed += pending_in_batch; + total_unprocessed += batch_info.pending; } println!("AddressV2 {}:", merkle_tree_pubkey); @@ -250,7 +358,7 @@ pub async fn print_address_v2_queue_info( " pending_batch_index: {}", merkle_tree.queue_batches.pending_batch_index ); - println!(" zkp_batch_size: {}", zkp_batch_size); + println!(" zkp_batch_size: {}", parsed.zkp_batch_size); for detail in batch_details { println!(" {}", detail); } @@ -268,3 +376,43 @@ pub struct QueueUpdate { pub pubkey: Pubkey, pub slot: u64, } + +pub async fn get_address_v2_queue_info( + rpc: &mut R, + merkle_tree_pubkey: &Pubkey, +) -> Result { + if let Some(mut account) = rpc.get_account(*merkle_tree_pubkey).await? { + let merkle_tree = BatchedMerkleTreeAccount::address_from_bytes( + account.data.as_mut_slice(), + &(*merkle_tree_pubkey).into(), + )?; + Ok(parse_address_v2_queue_info(&merkle_tree)) + } else { + Err(anyhow::anyhow!("account not found")) + } +} + +pub async fn get_state_v2_output_queue_info( + rpc: &mut R, + queue_pubkey: &Pubkey, +) -> Result { + if let Some(mut account) = rpc.get_account(*queue_pubkey).await? { + let queue = BatchedQueueAccount::output_from_bytes(account.data.as_mut_slice())?; + let next_index = queue.batch_metadata.next_index; + + let parsed = parse_batch_metadata(&queue.batch_metadata.batches); + + Ok(V2QueueInfo { + next_index, + pending_batch_index: queue.batch_metadata.pending_batch_index, + zkp_batch_size: parsed.zkp_batch_size, + batches: parsed.batch_infos, + input_pending_batches: 0, + output_pending_batches: parsed.total_pending_batches, + input_items_in_current_zkp_batch: 0, + output_items_in_current_zkp_batch: parsed.items_in_current_zkp_batch, + }) + } else { + Err(anyhow::anyhow!("account not found")) + } +} diff --git a/forester/src/rollover/operations.rs b/forester/src/rollover/operations.rs index cfab6c1210..c6a11a078e 100644 --- a/forester/src/rollover/operations.rs +++ b/forester/src/rollover/operations.rs @@ -65,7 +65,8 @@ pub async fn get_tree_fullness( rpc, tree_pubkey, ) - .await; + .await + .map_err(|e| ForesterError::Other(anyhow::anyhow!("{}", e)))?; let height = 26; let capacity = 1 << height; @@ -104,7 +105,8 @@ pub async fn get_tree_fullness( rpc, tree_pubkey, ) - .await; + .await + .map_err(|e| ForesterError::Other(anyhow::anyhow!("{}", e)))?; let height = 26; let capacity = 1 << height; @@ -334,7 +336,8 @@ pub async fn create_rollover_address_merkle_tree_instructions( queue: *nullifier_queue_pubkey, }, ) - .await; + .await + .map_err(|e| ForesterError::Other(anyhow::anyhow!("{}", e)))?; let (merkle_tree_rent_exemption, queue_rent_exemption) = get_rent_exemption_for_address_merkle_tree_and_queue( rpc, @@ -399,7 +402,8 @@ pub async fn create_rollover_state_merkle_tree_instructions( tree_type: TreeType::StateV1, // not used. }, ) - .await; + .await + .map_err(|e| ForesterError::Other(e.into()))?; let (state_merkle_tree_rent_exemption, queue_rent_exemption) = get_rent_exemption_for_state_merkle_tree_and_queue(rpc, &merkle_tree_config, &queue_config) .await?; diff --git a/forester/src/tree_data_sync.rs b/forester/src/tree_data_sync.rs index 3f0f2bd3ce..7d452e4da9 100644 --- a/forester/src/tree_data_sync.rs +++ b/forester/src/tree_data_sync.rs @@ -1,5 +1,5 @@ use account_compression::{ - utils::check_discriminator::check_discriminator, AddressMerkleTreeAccount, + utils::check_discriminator::check_discriminator, AddressMerkleTreeAccount, RegisteredProgram, StateMerkleTreeAccount, }; use borsh::BorshDeserialize; @@ -10,7 +10,7 @@ use light_compressed_account::TreeType; use light_merkle_tree_metadata::merkle_tree::MerkleTreeMetadata; use serde_json::json; use solana_sdk::{account::Account, pubkey::Pubkey}; -use tracing::{debug, trace, warn}; +use tracing::{debug, info, trace, warn}; use crate::{errors::AccountDeserializationError, Result}; @@ -302,14 +302,43 @@ fn create_tree_accounts( metadata.associated_queue.into(), tree_type, metadata.rollover_metadata.rolledover_slot != u64::MAX, + metadata.access_metadata.owner.into(), ); trace!( - "{:?} Merkle Tree account found. Pubkey: {}. Queue pubkey: {}. Rolledover: {}", + "{:?} Merkle Tree account found. Pubkey: {}. Queue pubkey: {}. Rolledover: {}. Owner: {}", tree_type, pubkey, tree_accounts.queue, - tree_accounts.is_rolledover + tree_accounts.is_rolledover, + tree_accounts.owner ); tree_accounts } + +pub async fn fetch_protocol_group_authority(rpc: &R) -> Result { + let registered_program_pda = + light_registry::account_compression_cpi::sdk::get_registered_program_pda( + &light_registry::ID, + ); + + let account = rpc + .get_account(registered_program_pda) + .await? + .ok_or_else(|| { + anyhow::anyhow!( + "RegisteredProgram PDA not found for light_registry at {}", + registered_program_pda + ) + })?; + + let registered_program = RegisteredProgram::deserialize(&mut &account.data[8..]) + .map_err(|e| anyhow::anyhow!("Failed to deserialize RegisteredProgram: {}", e))?; + + info!( + "Fetched protocol group authority: {}", + registered_program.group_authority_pda + ); + + Ok(registered_program.group_authority_pda) +} diff --git a/forester/src/utils.rs b/forester/src/utils.rs index 9590c63eb3..58b6c4b657 100644 --- a/forester/src/utils.rs +++ b/forester/src/utils.rs @@ -1,23 +1,97 @@ -use std::time::{SystemTime, UNIX_EPOCH}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; -use light_client::rpc::Rpc; +use light_client::rpc::{errors::RpcError, Rpc}; use light_registry::{ protocol_config::state::{ProtocolConfig, ProtocolConfigPda}, utils::get_protocol_config_pda_address, }; -use tracing::{debug, warn}; +use tracing::{debug, info, warn}; -pub async fn get_protocol_config(rpc: &mut R) -> crate::Result { +pub async fn get_protocol_config(rpc: &mut R) -> Result { let authority_pda = get_protocol_config_pda_address(); let protocol_config_account = rpc .get_anchor_account::(&authority_pda.0) - .await - .map_err(|e| anyhow::anyhow!("Failed to fetch protocol config account: {}", e))? - .ok_or_else(|| anyhow::anyhow!("Protocol config account not found"))?; + .await? + .ok_or_else(|| RpcError::AccountDoesNotExist(authority_pda.0.to_string()))?; debug!("Protocol config account: {:?}", protocol_config_account); Ok(protocol_config_account.config) } +/// Fetches protocol config with infinite retry on rate limit errors. +/// Uses exponential backoff starting at 5 seconds, maxing at 60 seconds. +pub async fn get_protocol_config_with_retry(rpc: &mut R) -> ProtocolConfig { + let mut retry_delay = Duration::from_secs(5); + let max_delay = Duration::from_secs(60); + let mut attempt = 0u64; + + loop { + attempt += 1; + match get_protocol_config(rpc).await { + Ok(config) => { + if attempt > 1 { + info!( + "Successfully fetched protocol config after {} attempts", + attempt + ); + } + return config; + } + Err(RpcError::RateLimited) => { + warn!( + "Rate limited fetching protocol config (attempt {}), retrying in {:?}...", + attempt, retry_delay + ); + tokio::time::sleep(retry_delay).await; + retry_delay = std::cmp::min(retry_delay * 2, max_delay); + } + Err(e) => { + warn!( + "Failed to fetch protocol config (attempt {}): {:?}, retrying in {:?}...", + attempt, e, retry_delay + ); + tokio::time::sleep(retry_delay).await; + retry_delay = std::cmp::min(retry_delay * 2, max_delay); + } + } + } +} + +/// Fetches current slot with infinite retry on rate limit errors. +/// Uses exponential backoff starting at 5 seconds, maxing at 60 seconds. +pub async fn get_slot_with_retry(rpc: &mut R) -> u64 { + let mut retry_delay = Duration::from_secs(5); + let max_delay = Duration::from_secs(60); + let mut attempt = 0u64; + + loop { + attempt += 1; + match rpc.get_slot().await { + Ok(slot) => { + if attempt > 1 { + info!("Successfully fetched slot after {} attempts", attempt); + } + return slot; + } + Err(RpcError::RateLimited) => { + warn!( + "Rate limited fetching slot (attempt {}), retrying in {:?}...", + attempt, retry_delay + ); + tokio::time::sleep(retry_delay).await; + retry_delay = std::cmp::min(retry_delay * 2, max_delay); + } + Err(e) => { + warn!( + "Failed to fetch slot (attempt {}): {:?}, retrying in {:?}...", + attempt, e, retry_delay + ); + tokio::time::sleep(retry_delay).await; + retry_delay = std::cmp::min(retry_delay * 2, max_delay); + } + } + } +} + pub fn get_current_system_time_ms() -> u128 { match SystemTime::now().duration_since(UNIX_EPOCH) { Ok(d) => d.as_millis(), diff --git a/forester/static/dashboard.html b/forester/static/dashboard.html new file mode 100644 index 0000000000..12f8ef6c25 --- /dev/null +++ b/forester/static/dashboard.html @@ -0,0 +1,1236 @@ + + + + + + Forester Dashboard + + + +
+
+

Forester Dashboard

+
+ +
+ + Loading +
+
+
+ +
+
+
+ + Loading status... +
+
+
+ + + + diff --git a/forester/test.sh b/forester/test.sh deleted file mode 100644 index fed0b252e5..0000000000 --- a/forester/test.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -../cli/test_bin/run start-prover --run-mode forester -../cli/test_bin/run test-validator --skip-prover --skip-indexer -sleep 10 -(cd ../../photon && cargo run 2>&1 > photon.log) - -sleep 60 * 5 - -RUST_LOG=forester=debug,forester_utils=debug cargo test --package forester test_state_indexer_async_batched -- --nocapture diff --git a/forester/tests/e2e_test.rs b/forester/tests/e2e_test.rs index 31c950c022..b44db52977 100644 --- a/forester/tests/e2e_test.rs +++ b/forester/tests/e2e_test.rs @@ -241,6 +241,7 @@ async fn e2e_test() { sleep_after_processing_ms: 50, sleep_when_idle_ms: 100, queue_polling_mode: Default::default(), + group_authority: None, }, rpc_pool_config: RpcPoolConfig { max_size: 50, @@ -609,7 +610,8 @@ async fn get_initial_merkle_tree_state( rpc, *merkle_tree_pubkey, ) - .await; + .await + .unwrap(); let next_index = merkle_tree.next_index() as u64; let sequence_number = account.metadata.rollover_metadata.rolledover_slot; @@ -641,7 +643,8 @@ async fn get_initial_merkle_tree_state( 26, 16, >(rpc, *merkle_tree_pubkey) - .await; + .await + .unwrap(); let next_index = merkle_tree.next_index() as u64; let sequence_number = account.metadata.rollover_metadata.rolledover_slot; @@ -702,7 +705,8 @@ async fn verify_root_changed( rpc, *merkle_tree_pubkey, ) - .await; + .await + .unwrap(); println!( "Final V1 state tree next_index: {}", @@ -719,7 +723,8 @@ async fn verify_root_changed( 26, 16, >(rpc, *merkle_tree_pubkey) - .await; + .await + .unwrap(); println!( "Final V1 address tree next_index: {}", diff --git a/forester/tests/priority_fee_test.rs b/forester/tests/priority_fee_test.rs index 96475c429e..b8212a5fc1 100644 --- a/forester/tests/priority_fee_test.rs +++ b/forester/tests/priority_fee_test.rs @@ -21,22 +21,17 @@ async fn test_priority_fee_request() { init(None).await; let args = StartArgs { - rpc_url: Some( - std::env::var("FORESTER_RPC_URL").expect("FORESTER_RPC_URL must be set in environment"), - ), + rpc_url: Some(std::env::var("RPC_URL").expect("RPC_URL must be set in environment")), push_gateway_url: None, pagerduty_routing_key: None, ws_rpc_url: Some( - std::env::var("FORESTER_WS_RPC_URL") - .expect("FORESTER_WS_RPC_URL must be set in environment"), + std::env::var("WS_RPC_URL").expect("WS_RPC_URL must be set in environment"), ), indexer_url: Some( - std::env::var("FORESTER_INDEXER_URL") - .expect("FORESTER_INDEXER_URL must be set in environment"), + std::env::var("INDEXER_URL").expect("INDEXER_URL must be set in environment"), ), prover_url: Some( - std::env::var("FORESTER_PROVER_URL") - .expect("FORESTER_PROVER_URL must be set in environment"), + std::env::var("PROVER_URL").expect("PROVER_URL must be set in environment"), ), prover_append_url: None, prover_update_url: None, @@ -44,22 +39,22 @@ async fn test_priority_fee_request() { prover_api_key: None, prover_polling_interval_ms: None, prover_max_wait_time_secs: None, - payer: Some( - std::env::var("FORESTER_PAYER").expect("FORESTER_PAYER must be set in environment"), - ), + payer: Some(std::env::var("PAYER").expect("PAYER must be set in environment")), derivation: Some( - std::env::var("FORESTER_DERIVATION_PUBKEY") - .expect("FORESTER_DERIVATION_PUBKEY must be set in environment"), + std::env::var("DERIVATION_PUBKEY") + .expect("DERIVATION_PUBKEY must be set in environment"), ), photon_api_key: Some( std::env::var("PHOTON_API_KEY").expect("PHOTON_API_KEY must be set in environment"), ), + api_server_public_bind: false, photon_grpc_url: None, indexer_batch_size: 50, indexer_max_concurrent_batches: 10, legacy_ixs_per_tx: 1, transaction_max_concurrent_batches: 20, max_concurrent_sends: 50, + max_batches_per_tree: 4, tx_cache_ttl_seconds: 15, ops_cache_ttl_seconds: 180, confirmation_max_attempts: 30, @@ -89,11 +84,12 @@ async fn test_priority_fee_request() { tree_ids: vec![], enable_compressible: true, lookup_table_address: None, + api_server_port: 8080, + group_authority: None, }; let config = ForesterConfig::new_for_start(&args).expect("Failed to create config"); - // Setup RPC connection using config let mut rpc = LightClient::new(LightClientConfig::local()).await.unwrap(); rpc.payer = config.payer_keypair.insecure_clone(); diff --git a/forester/tests/test_utils.rs b/forester/tests/test_utils.rs index 5c1ce7a4b7..1ae2f159d7 100644 --- a/forester/tests/test_utils.rs +++ b/forester/tests/test_utils.rs @@ -115,6 +115,7 @@ pub fn forester_config() -> ForesterConfig { sleep_after_processing_ms: 50, sleep_when_idle_ms: 100, queue_polling_mode: QueuePollingMode::OnChain, + group_authority: None, }, rpc_pool_config: RpcPoolConfig { max_size: 50, diff --git a/program-tests/account-compression-test/tests/address_merkle_tree_tests.rs b/program-tests/account-compression-test/tests/address_merkle_tree_tests.rs index 9a306a485c..2964ba9053 100644 --- a/program-tests/account-compression-test/tests/address_merkle_tree_tests.rs +++ b/program-tests/account-compression-test/tests/address_merkle_tree_tests.rs @@ -77,7 +77,8 @@ async fn address_queue_and_tree_functional( .unwrap(); let address_queue = unsafe { get_hash_set::(&mut context, address_queue_pubkey).await - }; + } + .unwrap(); assert!(address_queue.contains(&address1, None).unwrap()); assert!(address_queue.contains(&address2, None).unwrap()); @@ -106,7 +107,8 @@ async fn address_queue_and_tree_functional( .unwrap(); let address_queue = unsafe { get_hash_set::(&mut context, address_queue_pubkey).await - }; + } + .unwrap(); address_queue .find_element(&address3, None) .unwrap() @@ -596,7 +598,8 @@ async fn update_address_merkle_tree_failing_tests( .unwrap(); let address_queue = unsafe { get_hash_set::(&mut context, address_queue_pubkey).await - }; + } + .unwrap(); // CHECK: 2.1 cannot insert an address with an invalid low address test_with_invalid_low_element( &mut context, @@ -817,7 +820,8 @@ async fn update_address_merkle_tree_failing_tests( 26, 16, >(&mut context, address_merkle_tree_pubkey) - .await; + .await + .unwrap(); let changelog_index = address_merkle_tree.changelog_index(); @@ -1085,7 +1089,8 @@ async fn update_address_merkle_tree_wrap_around( let address_queue = unsafe { get_hash_set::(&mut context, address_queue_pubkey).await - }; + } + .unwrap(); let value_index = address_queue .find_element_index(&address1, None) .unwrap() diff --git a/program-tests/account-compression-test/tests/batched_merkle_tree_test.rs b/program-tests/account-compression-test/tests/batched_merkle_tree_test.rs index 38bae5f372..018e1192f5 100644 --- a/program-tests/account-compression-test/tests/batched_merkle_tree_test.rs +++ b/program-tests/account-compression-test/tests/batched_merkle_tree_test.rs @@ -281,10 +281,13 @@ async fn test_batch_state_merkle_tree() { .unwrap(); let mut merkle_tree = AccountZeroCopy::::new(&mut context, merkle_tree_pubkey) - .await; + .await + .unwrap(); let mut queue = - AccountZeroCopy::::new(&mut context, output_queue_pubkey).await; + AccountZeroCopy::::new(&mut context, output_queue_pubkey) + .await + .unwrap(); let owner = context.get_payer().pubkey(); let mt_params = @@ -1027,10 +1030,13 @@ async fn test_init_batch_state_merkle_trees() { .unwrap(); let merkle_tree = AccountZeroCopy::::new(&mut context, merkle_tree_pubkey) - .await; + .await + .unwrap(); let mut queue = - AccountZeroCopy::::new(&mut context, output_queue_pubkey).await; + AccountZeroCopy::::new(&mut context, output_queue_pubkey) + .await + .unwrap(); let owner = context.get_payer().pubkey(); let mt_params = CreateTreeParams::from_state_ix_params( *params, @@ -1598,7 +1604,8 @@ async fn test_init_batch_address_merkle_trees() { .unwrap(); let merkle_tree = AccountZeroCopy::::new(&mut context, merkle_tree_pubkey) - .await; + .await + .unwrap(); let mt_params = CreateTreeParams::from_address_ix_params( *params, owner.into(), diff --git a/program-tests/account-compression-test/tests/merkle_tree_tests.rs b/program-tests/account-compression-test/tests/merkle_tree_tests.rs index 13c6b09a5b..3c6724b311 100644 --- a/program-tests/account-compression-test/tests/merkle_tree_tests.rs +++ b/program-tests/account-compression-test/tests/merkle_tree_tests.rs @@ -308,12 +308,14 @@ async fn test_full_nullifier_queue( &mut rpc, merkle_tree_pubkey, ) - .await; + .await + .unwrap(); assert_eq!(merkle_tree.root(), reference_merkle_tree.root()); let leaf_index = reference_merkle_tree.get_leaf_index(&leaf).unwrap() as u64; let element_index = unsafe { get_hash_set::(&mut rpc, nullifier_queue_pubkey) .await + .unwrap() .find_element_index(&BigUint::from_bytes_be(&leaf), None) .unwrap() }; @@ -1223,7 +1225,8 @@ async fn functional_2_test_insert_into_nullifier_queues( ) .await .unwrap(); - let array = unsafe { get_hash_set::(rpc, *nullifier_queue_pubkey).await }; + let array = + unsafe { get_hash_set::(rpc, *nullifier_queue_pubkey).await }.unwrap(); let element_0 = BigUint::from_bytes_be(&elements[0]); let (array_element_0, _) = array.find_element(&element_0, None).unwrap().unwrap(); assert_eq!(array_element_0.value_bytes(), [1u8; 32]); @@ -1304,7 +1307,8 @@ async fn functional_5_test_insert_into_nullifier_queue( ) .await .unwrap(); - let array = unsafe { get_hash_set::(rpc, *nullifier_queue_pubkey).await }; + let array = + unsafe { get_hash_set::(rpc, *nullifier_queue_pubkey).await }.unwrap(); let (array_element, _) = array.find_element(&element, None).unwrap().unwrap(); assert_eq!(array_element.value_biguint(), element); @@ -1862,7 +1866,8 @@ pub async fn functional_3_append_leaves_to_merkle_tree( context, merkle_tree_pubkeys[(*i) as usize], ) - .await; + .await + .unwrap(); hash_map .entry(merkle_tree_pubkeys[(*i) as usize]) .or_insert_with(|| { @@ -1893,7 +1898,8 @@ pub async fn functional_3_append_leaves_to_merkle_tree( let merkle_tree = get_concurrent_merkle_tree::(context, *pubkey) - .await; + .await + .unwrap(); assert_eq!(merkle_tree.next_index(), next_index + num_leaves); let leaves: Vec<&[u8; 32]> = leaves.iter().collect(); @@ -1999,7 +2005,8 @@ pub async fn nullify( rpc, *merkle_tree_pubkey, ) - .await; + .await + .unwrap(); reference_merkle_tree .update(&ZERO_BYTES[0], element_index as usize) .unwrap(); @@ -2159,7 +2166,8 @@ pub async fn assert_element_inserted_in_nullifier_queue( ) { let array = unsafe { get_hash_set::(rpc, *nullifier_queue_pubkey).await - }; + } + .unwrap(); let nullifier_bn = BigUint::from_bytes_be(&nullifier); let (array_element, _) = array.find_element(&nullifier_bn, None).unwrap().unwrap(); assert_eq!(array_element.value_bytes(), nullifier); diff --git a/program-tests/registry-test/tests/tests.rs b/program-tests/registry-test/tests/tests.rs index fd659a5302..399435372c 100644 --- a/program-tests/registry-test/tests/tests.rs +++ b/program-tests/registry-test/tests/tests.rs @@ -876,12 +876,14 @@ async fn test_register_and_update_forester_pda() { merkle_tree: env.v1_state_trees[0].merkle_tree, queue: env.v1_state_trees[0].nullifier_queue, is_rolledover: false, + owner: Default::default(), }, TreeAccounts { tree_type: TreeType::AddressV1, merkle_tree: env.v1_address_trees[0].merkle_tree, queue: env.v1_address_trees[0].queue, is_rolledover: false, + owner: Default::default(), }, ]; @@ -1257,7 +1259,8 @@ async fn failing_test_forester() { 0, // TODO: adapt epoch false, ) - .await; + .await + .unwrap(); // Swap the derived forester pda with an initialized but invalid one. instructions[2].accounts[0].pubkey = get_forester_epoch_pda_from_authority(&env.protocol.forester.pubkey(), 0).0; @@ -1290,7 +1293,8 @@ async fn failing_test_forester() { 0, // TODO: adapt epoch false, ) - .await; + .await + .unwrap(); // Swap the derived forester pda with an initialized but invalid one. instructions[3].accounts[0].pubkey = get_forester_epoch_pda_from_authority(&env.protocol.forester.pubkey(), 0).0; @@ -1476,7 +1480,8 @@ async fn test_migrate_state() { &mut rpc, test_accounts.v1_state_trees[0].merkle_tree, ) - .await; + .await + .unwrap(); let compressed_account = &test_indexer.get_compressed_accounts_with_merkle_context_by_owner(&payer.pubkey())[0]; let hash = compressed_account.hash().unwrap(); @@ -1531,7 +1536,8 @@ async fn test_migrate_state() { Poseidon, 26, >(&mut rpc, test_accounts.v1_state_trees[0].merkle_tree) - .await; + .await + .unwrap(); let bundle = test_indexer .get_state_merkle_trees_mut() .iter_mut() @@ -1567,7 +1573,8 @@ async fn test_migrate_state() { &mut rpc, test_accounts.v1_state_trees[0].merkle_tree, ) - .await; + .await + .unwrap(); let compressed_account = &test_indexer.get_compressed_accounts_with_merkle_context_by_owner(&payer.pubkey())[1]; let hash = compressed_account.hash().unwrap(); diff --git a/program-tests/system-cpi-test/tests/test_program_owned_trees.rs b/program-tests/system-cpi-test/tests/test_program_owned_trees.rs index 9451028c41..9897a8ea27 100644 --- a/program-tests/system-cpi-test/tests/test_program_owned_trees.rs +++ b/program-tests/system-cpi-test/tests/test_program_owned_trees.rs @@ -99,7 +99,8 @@ async fn test_program_owned_merkle_tree() { &mut rpc, program_owned_merkle_tree_pubkey, ) - .await; + .await + .unwrap(); let event = TestRpc::create_and_send_transaction_with_public_event( &mut rpc, &[instruction], @@ -122,7 +123,8 @@ async fn test_program_owned_merkle_tree() { &mut rpc, program_owned_merkle_tree_pubkey, ) - .await; + .await + .unwrap(); let slot: u64 = rpc.get_slot().await.unwrap(); test_indexer.add_compressed_accounts_with_token_data(slot, &event.0); assert_ne!(post_merkle_tree.root(), pre_merkle_tree.root()); diff --git a/program-tests/utils/src/address_tree_rollover.rs b/program-tests/utils/src/address_tree_rollover.rs index d15b24e34a..efc52387d5 100644 --- a/program-tests/utils/src/address_tree_rollover.rs +++ b/program-tests/utils/src/address_tree_rollover.rs @@ -194,13 +194,15 @@ pub async fn assert_rolled_over_address_merkle_tree_and_queue( rpc, old_mt_account.key(), ) - .await; + .await + .unwrap(); let struct_new = get_indexed_merkle_tree::( rpc, new_mt_account.key(), ) - .await; + .await + .unwrap(); assert_rolledover_merkle_trees(&struct_old.merkle_tree, &struct_new.merkle_tree); assert_eq!( struct_old.merkle_tree.changelog.capacity(), @@ -257,9 +259,9 @@ pub async fn assert_rolled_over_address_merkle_tree_and_queue( assert_eq!(*fee_payer_prior_balance, fee_payer_post_balance + 15000); { let old_address_queue = - unsafe { get_hash_set::(rpc, *old_queue_pubkey).await }; + unsafe { get_hash_set::(rpc, *old_queue_pubkey).await }.unwrap(); let new_address_queue = - unsafe { get_hash_set::(rpc, *new_queue_pubkey).await }; + unsafe { get_hash_set::(rpc, *new_queue_pubkey).await }.unwrap(); assert_eq!( old_address_queue.get_capacity(), @@ -295,7 +297,8 @@ pub async fn perform_address_merkle_tree_roll_over_forester( epoch, is_metadata_forester, ) - .await; + .await + .unwrap(); let blockhash = context.get_latest_blockhash().await?; let transaction = Transaction::new_signed_with_payer( &instructions, @@ -330,7 +333,8 @@ pub async fn perform_state_merkle_tree_roll_over_forester( epoch, is_metadata_forester, ) - .await; + .await + .unwrap(); let blockhash = context.get_latest_blockhash().await?; let transaction = Transaction::new_signed_with_payer( &instructions, diff --git a/program-tests/utils/src/assert_compressed_tx.rs b/program-tests/utils/src/assert_compressed_tx.rs index b983f4c233..8d6c2ced9d 100644 --- a/program-tests/utils/src/assert_compressed_tx.rs +++ b/program-tests/utils/src/assert_compressed_tx.rs @@ -136,7 +136,8 @@ pub async fn assert_nullifiers_exist_in_hash_sets( let nullifier_queue = unsafe { get_hash_set::(rpc, snapshots[i].accounts.nullifier_queue) .await - }; + } + .unwrap(); assert!(nullifier_queue .contains(&BigUint::from_be_bytes(hash.as_slice()), None) .unwrap()); @@ -182,7 +183,8 @@ pub async fn assert_addresses_exist_in_hash_sets( let discriminator = &account.data[0..8]; match discriminator { QueueAccount::DISCRIMINATOR => { - let address_queue = unsafe { get_hash_set::(rpc, *pubkey).await }; + let address_queue = + unsafe { get_hash_set::(rpc, *pubkey).await }.unwrap(); assert!(address_queue .contains(&BigUint::from_be_bytes(address), None) .unwrap()); @@ -343,7 +345,8 @@ pub async fn assert_merkle_tree_after_tx( rpc, account_bundle.merkle_tree, ) - .await; + .await + .unwrap(); let merkle_tree_account = AccountZeroCopy::::new(rpc, account_bundle.merkle_tree) - .await; + .await + .unwrap(); let queue_account_lamports = match rpc .get_account(account_bundle.nullifier_queue) @@ -479,7 +484,8 @@ pub async fn get_merkle_tree_snapshots( rpc, account_bundle.nullifier_queue, ) - .await; + .await + .unwrap(); snapshots.push(MerkleTreeTestSnapShot { accounts: *account_bundle, diff --git a/program-tests/utils/src/assert_merkle_tree.rs b/program-tests/utils/src/assert_merkle_tree.rs index 4355655dcf..aa5efbf44b 100644 --- a/program-tests/utils/src/assert_merkle_tree.rs +++ b/program-tests/utils/src/assert_merkle_tree.rs @@ -27,7 +27,8 @@ pub async fn assert_merkle_tree_initialized( rpc, *merkle_tree_pubkey, ) - .await; + .await + .unwrap(); let merkle_tree_account = merkle_tree_account.deserialized(); let balance_merkle_tree = rpc @@ -112,7 +113,8 @@ pub async fn assert_merkle_tree_initialized( rpc, *merkle_tree_pubkey, ) - .await; + .await + .unwrap(); assert_eq!(merkle_tree.height, height); assert_eq!(merkle_tree.changelog.capacity(), changelog_capacity); diff --git a/program-tests/utils/src/assert_queue.rs b/program-tests/utils/src/assert_queue.rs index 3865e40e66..042a7bd523 100644 --- a/program-tests/utils/src/assert_queue.rs +++ b/program-tests/utils/src/assert_queue.rs @@ -154,7 +154,9 @@ pub async fn assert_queue( expected_next_queue: Option, payer_pubkey: &Pubkey, ) { - let queue = AccountZeroCopy::::new(rpc, *queue_pubkey).await; + let queue = AccountZeroCopy::::new(rpc, *queue_pubkey) + .await + .unwrap(); let queue_account = queue.deserialized(); let expected_rollover_meta_data = RolloverMetadata { @@ -182,7 +184,7 @@ pub async fn assert_queue( }; assert_eq!(queue_account.metadata, expected_queue_meta_data); - let queue = unsafe { get_hash_set::(rpc, *queue_pubkey).await }; + let queue = unsafe { get_hash_set::(rpc, *queue_pubkey).await }.unwrap(); assert_eq!(queue.get_capacity(), queue_config.capacity as usize); assert_eq!( queue.sequence_threshold, diff --git a/program-tests/utils/src/batched_address_tree.rs b/program-tests/utils/src/batched_address_tree.rs index 7a4960639e..979b5b9d37 100644 --- a/program-tests/utils/src/batched_address_tree.rs +++ b/program-tests/utils/src/batched_address_tree.rs @@ -139,7 +139,8 @@ pub async fn assert_address_merkle_tree_initialized( rpc, *merkle_tree_pubkey, ) - .await; + .await + .unwrap(); let merkle_tree_account = merkle_tree.deserialized(); assert_eq!( @@ -205,7 +206,8 @@ pub async fn assert_address_merkle_tree_initialized( 26, 16, >(rpc, *merkle_tree_pubkey) - .await; + .await + .unwrap(); assert_eq!(merkle_tree.height, merkle_tree_config.height as usize); assert_eq!( @@ -349,7 +351,9 @@ pub async fn assert_queue( expected_next_queue: Option, payer_pubkey: &Pubkey, ) { - let queue = AccountZeroCopy::::new(rpc, *queue_pubkey).await; + let queue = AccountZeroCopy::::new(rpc, *queue_pubkey) + .await + .unwrap(); let queue_account = queue.deserialized(); let expected_rollover_meta_data = RolloverMetadata { @@ -378,7 +382,8 @@ pub async fn assert_queue( assert_eq!(queue_account.metadata, expected_queue_meta_data); let queue = - unsafe { get_hash_set::(rpc, *queue_pubkey).await }; + unsafe { get_hash_set::(rpc, *queue_pubkey).await } + .unwrap(); assert_eq!(queue.get_capacity(), queue_config.capacity as usize); assert_eq!( queue.sequence_threshold, diff --git a/program-tests/utils/src/e2e_test_env.rs b/program-tests/utils/src/e2e_test_env.rs index 6aefb38f94..2b96d1b4eb 100644 --- a/program-tests/utils/src/e2e_test_env.rs +++ b/program-tests/utils/src/e2e_test_env.rs @@ -896,7 +896,8 @@ where .accounts .merkle_tree, ) - .await; + .await + .unwrap(); if self .rng .gen_bool(self.general_action_config.rollover.unwrap_or_default()) @@ -937,7 +938,8 @@ where .accounts .merkle_tree, ) - .await; + .await + .unwrap(); if self .rng .gen_bool(self.general_action_config.rollover.unwrap_or_default()) @@ -1129,6 +1131,7 @@ where merkle_tree: state_merkle_tree_bundle.accounts.merkle_tree, queue: state_merkle_tree_bundle.accounts.nullifier_queue, is_rolledover: false, + owner: Default::default(), } }) .collect::>(); @@ -1139,6 +1142,7 @@ where merkle_tree: address_merkle_tree_bundle.accounts.merkle_tree, queue: address_merkle_tree_bundle.accounts.queue, is_rolledover: false, + owner: Default::default(), }); }, ); @@ -1267,7 +1271,8 @@ where &mut self.rpc, nullifier_queue_keypair.pubkey(), ) - .await; + .await + .unwrap(); self.indexer .get_state_merkle_trees_mut() .push(StateMerkleTreeBundle { @@ -1362,7 +1367,8 @@ where &mut self.rpc, nullifier_queue_keypair.pubkey(), ) - .await; + .await + .unwrap(); let mut bundle = AddressMerkleTreeBundle::new_v1(AddressMerkleTreeAccounts { merkle_tree: merkle_tree_keypair.pubkey(), queue: nullifier_queue_keypair.pubkey(), diff --git a/program-tests/utils/src/setup_forester.rs b/program-tests/utils/src/setup_forester.rs index b82698212d..78d9e9e939 100644 --- a/program-tests/utils/src/setup_forester.rs +++ b/program-tests/utils/src/setup_forester.rs @@ -56,12 +56,14 @@ pub async fn setup_forester_and_advance_to_epoch( test_keypairs.nullifier_queue.pubkey(), TreeType::StateV1, false, + Default::default(), ), TreeAccounts::new( test_keypairs.address_merkle_tree.pubkey(), test_keypairs.address_merkle_tree_queue.pubkey(), TreeType::AddressV1, false, + Default::default(), ), ]; diff --git a/program-tests/utils/src/state_tree_rollover.rs b/program-tests/utils/src/state_tree_rollover.rs index a77bf01878..1402c2f749 100644 --- a/program-tests/utils/src/state_tree_rollover.rs +++ b/program-tests/utils/src/state_tree_rollover.rs @@ -284,9 +284,9 @@ pub async fn assert_rolled_over_pair( fee_payer_post_balance + 5000 * num_signatures + additional_rent ); let old_address_queue = - unsafe { get_hash_set::(rpc, *old_nullifier_queue_pubkey).await }; + unsafe { get_hash_set::(rpc, *old_nullifier_queue_pubkey).await }.unwrap(); let new_address_queue = - unsafe { get_hash_set::(rpc, *new_nullifier_queue_pubkey).await }; + unsafe { get_hash_set::(rpc, *new_nullifier_queue_pubkey).await }.unwrap(); assert_eq!( old_address_queue.get_capacity(), diff --git a/program-tests/utils/src/test_batch_forester.rs b/program-tests/utils/src/test_batch_forester.rs index 416bb3b80c..576b887447 100644 --- a/program-tests/utils/src/test_batch_forester.rs +++ b/program-tests/utils/src/test_batch_forester.rs @@ -335,9 +335,13 @@ pub async fn assert_registry_created_batched_state_merkle_tree( params: InitStateTreeAccountsInstructionData, ) -> Result<(), RpcError> { let mut merkle_tree = - AccountZeroCopy::::new(rpc, merkle_tree_pubkey).await; + AccountZeroCopy::::new(rpc, merkle_tree_pubkey) + .await + .unwrap(); - let mut queue = AccountZeroCopy::::new(rpc, output_queue_pubkey).await; + let mut queue = AccountZeroCopy::::new(rpc, output_queue_pubkey) + .await + .unwrap(); let mt_params = CreateTreeParams::from_state_ix_params( params, payer_pubkey.into(), @@ -605,7 +609,9 @@ pub async fn assert_registry_created_batched_address_merkle_tree( params: InitAddressTreeAccountsInstructionData, ) -> Result<(), RpcError> { let mut merkle_tree = - AccountZeroCopy::::new(rpc, merkle_tree_pubkey).await; + AccountZeroCopy::::new(rpc, merkle_tree_pubkey) + .await + .unwrap(); let mt_account_size = get_merkle_tree_account_size( params.input_queue_batch_size, diff --git a/program-tests/utils/src/test_forester.rs b/program-tests/utils/src/test_forester.rs index cbc244ed7b..224468d096 100644 --- a/program-tests/utils/src/test_forester.rs +++ b/program-tests/utils/src/test_forester.rs @@ -58,7 +58,8 @@ pub async fn nullify_compressed_accounts( ) -> Result<(), RpcError> { let nullifier_queue = unsafe { get_hash_set::(rpc, state_tree_bundle.accounts.nullifier_queue).await - }; + } + .unwrap(); let pre_forester_counter = if is_metadata_forester { 0 } else { @@ -75,7 +76,8 @@ pub async fn nullify_compressed_accounts( rpc, state_tree_bundle.accounts.merkle_tree, ) - .await; + .await + .unwrap(); assert_eq!( onchain_merkle_tree.root(), state_tree_bundle.merkle_tree.root() @@ -193,7 +195,8 @@ pub async fn nullify_compressed_accounts( rpc, state_tree_bundle.accounts.merkle_tree, ) - .await; + .await + .unwrap(); assert_eq!( onchain_merkle_tree.root(), state_tree_bundle.merkle_tree.root() @@ -223,7 +226,8 @@ async fn assert_value_is_marked_in_queue( ) { let nullifier_queue = unsafe { get_hash_set::(rpc, state_tree_bundle.accounts.nullifier_queue).await - }; + } + .unwrap(); let array_element = nullifier_queue .get_bucket(*index_in_nullifier_queue) .unwrap() @@ -234,7 +238,8 @@ async fn assert_value_is_marked_in_queue( rpc, state_tree_bundle.accounts.merkle_tree, ) - .await; + .await + .unwrap(); assert_eq!( array_element.sequence_number(), Some( @@ -301,7 +306,8 @@ pub async fn empty_address_queue_test( rpc, address_merkle_tree_pubkey, ) - .await; + .await + .unwrap(); let indexed_changelog_index = address_merkle_tree.indexed_changelog_index() as u16; let changelog_index = address_merkle_tree.changelog_index() as u16; let mut counter = 0; @@ -322,10 +328,11 @@ pub async fn empty_address_queue_test( rpc, address_merkle_tree_pubkey, ) - .await; + .await + .unwrap(); assert_eq!(address_tree_bundle.root(), address_merkle_tree.root()); let address_queue = - unsafe { get_hash_set::(rpc, address_queue_pubkey).await }; + unsafe { get_hash_set::(rpc, address_queue_pubkey).await }.unwrap(); let address = address_queue.first_no_seq().unwrap(); @@ -472,7 +479,8 @@ pub async fn empty_address_queue_test( rpc, address_merkle_tree_pubkey, ) - .await; + .await + .unwrap(); let (old_low_address, _) = address_tree_bundle .find_low_element_for_nonexistent(&address.value_biguint()) @@ -481,7 +489,8 @@ pub async fn empty_address_queue_test( .new_element_with_low_element_index(old_low_address.index, &address.value_biguint()) .unwrap(); let address_queue = - unsafe { get_hash_set::(rpc, address_queue_pubkey).await }; + unsafe { get_hash_set::(rpc, address_queue_pubkey).await } + .unwrap(); assert_eq!( address_queue @@ -578,7 +587,8 @@ pub async fn update_merkle_tree( rpc, address_merkle_tree_pubkey, ) - .await; + .await + .unwrap(); address_merkle_tree.changelog_index() as u16 } @@ -591,7 +601,8 @@ pub async fn update_merkle_tree( rpc, address_merkle_tree_pubkey, ) - .await; + .await + .unwrap(); address_merkle_tree.indexed_changelog_index() as u16 } diff --git a/sdk-libs/client/src/rpc/client.rs b/sdk-libs/client/src/rpc/client.rs index 8a3226c72e..09dabfa7cb 100644 --- a/sdk-libs/client/src/rpc/client.rs +++ b/sdk-libs/client/src/rpc/client.rs @@ -466,6 +466,37 @@ impl Rpc for LightClient { .await } + async fn get_program_accounts_with_discriminator( + &self, + program_id: &Pubkey, + discriminator: &[u8], + ) -> Result, RpcError> { + use solana_rpc_client_api::{ + config::{RpcAccountInfoConfig, RpcProgramAccountsConfig}, + filter::{Memcmp, RpcFilterType}, + }; + + let discriminator = discriminator.to_vec(); + self.retry(|| async { + let config = RpcProgramAccountsConfig { + filters: Some(vec![RpcFilterType::Memcmp(Memcmp::new_base58_encoded( + 0, + &discriminator, + ))]), + account_config: RpcAccountInfoConfig { + encoding: Some(solana_account_decoder_client_types::UiAccountEncoding::Base64), + commitment: Some(self.client.commitment()), + ..Default::default() + }, + ..Default::default() + }; + self.client + .get_program_accounts_with_config(program_id, config) + .map_err(RpcError::from) + }) + .await + } + async fn process_transaction( &mut self, transaction: Transaction, diff --git a/sdk-libs/client/src/rpc/errors.rs b/sdk-libs/client/src/rpc/errors.rs index fdff022a73..6453fcc4b0 100644 --- a/sdk-libs/client/src/rpc/errors.rs +++ b/sdk-libs/client/src/rpc/errors.rs @@ -13,6 +13,9 @@ pub enum RpcError { #[error("BanksError: {0}")] BanksError(#[from] solana_banks_client::BanksClientError), + #[error("Rate limited")] + RateLimited, + #[error("State tree lookup table not found")] StateTreeLookupTableNotFound, @@ -26,7 +29,7 @@ pub enum RpcError { TransactionError(#[from] TransactionError), #[error("ClientError: {0}")] - ClientError(#[from] ClientError), + ClientError(ClientError), #[error("IoError: {0}")] IoError(#[from] io::Error), @@ -75,11 +78,27 @@ impl From for RpcError { } } +impl From for RpcError { + fn from(e: ClientError) -> Self { + let error_str = e.to_string(); + if error_str.contains("429") + || error_str.contains("Too Many Requests") + || error_str.contains("max usage") + || error_str.contains("rate limit") + { + RpcError::RateLimited + } else { + RpcError::ClientError(e) + } + } +} + impl Clone for RpcError { fn clone(&self) -> Self { match self { #[cfg(feature = "program-test")] RpcError::BanksError(_) => RpcError::CustomError("BanksError".to_string()), + RpcError::RateLimited => RpcError::RateLimited, RpcError::TransactionError(e) => RpcError::TransactionError(e.clone()), RpcError::ClientError(_) => RpcError::CustomError("ClientError".to_string()), RpcError::IoError(e) => RpcError::IoError(e.kind().into()), diff --git a/sdk-libs/client/src/rpc/rpc_trait.rs b/sdk-libs/client/src/rpc/rpc_trait.rs index 2ece7386fd..104c32d51e 100644 --- a/sdk-libs/client/src/rpc/rpc_trait.rs +++ b/sdk-libs/client/src/rpc/rpc_trait.rs @@ -97,6 +97,13 @@ pub trait Rpc: Send + Sync + Debug + 'static { &self, program_id: &Pubkey, ) -> Result, RpcError>; + + async fn get_program_accounts_with_discriminator( + &self, + program_id: &Pubkey, + discriminator: &[u8], + ) -> Result, RpcError>; + // TODO: add send transaction with config async fn confirm_transaction(&self, signature: Signature) -> Result; diff --git a/sdk-libs/program-test/src/program_test/rpc.rs b/sdk-libs/program-test/src/program_test/rpc.rs index 3172d7f829..f87e89703f 100644 --- a/sdk-libs/program-test/src/program_test/rpc.rs +++ b/sdk-libs/program-test/src/program_test/rpc.rs @@ -63,6 +63,21 @@ impl Rpc for LightProgramTest { Ok(self.context.get_program_accounts(program_id)) } + async fn get_program_accounts_with_discriminator( + &self, + program_id: &Pubkey, + discriminator: &[u8], + ) -> Result, RpcError> { + let all_accounts = self.context.get_program_accounts(program_id); + Ok(all_accounts + .into_iter() + .filter(|(_, account)| { + account.data.len() >= discriminator.len() + && &account.data[..discriminator.len()] == discriminator + }) + .collect()) + } + async fn confirm_transaction(&self, _transaction: Signature) -> Result { Ok(true) }