From deb9cf5ecca10aea1c532070a9facc2fab0898f8 Mon Sep 17 00:00:00 2001 From: "Paul H. Liu" Date: Sat, 27 Dec 2025 17:19:32 +0800 Subject: [PATCH 1/4] feat(rosetta-api): show DB index optimization progress --- .../Cargo.toml | 2 +- .../src/blocks.rs | 164 ++++++++++++++++-- 2 files changed, 152 insertions(+), 14 deletions(-) diff --git a/rs/rosetta-api/icp/ledger_canister_blocks_synchronizer/Cargo.toml b/rs/rosetta-api/icp/ledger_canister_blocks_synchronizer/Cargo.toml index 07385bdf74df..4fa20580f27e 100644 --- a/rs/rosetta-api/icp/ledger_canister_blocks_synchronizer/Cargo.toml +++ b/rs/rosetta-api/icp/ledger_canister_blocks_synchronizer/Cargo.toml @@ -22,7 +22,7 @@ icp-ledger = { path = "../../../ledger_suite/icp" } on_wire = { path = "../../../rust_canisters/on_wire" } reqwest = { workspace = true } rosetta-core = { path = "../../common/rosetta_core" } -rusqlite = { version = "~0.28.0", features = ["bundled"] } +rusqlite = { version = "~0.28.0", features = ["bundled", "hooks"] } serde = { workspace = true } tokio = { workspace = true } tracing = { workspace = true } diff --git a/rs/rosetta-api/icp/ledger_canister_blocks_synchronizer/src/blocks.rs b/rs/rosetta-api/icp/ledger_canister_blocks_synchronizer/src/blocks.rs index a8feb827820a..5969f5949f8c 100644 --- a/rs/rosetta-api/icp/ledger_canister_blocks_synchronizer/src/blocks.rs +++ b/rs/rosetta-api/icp/ledger_canister_blocks_synchronizer/src/blocks.rs @@ -904,46 +904,122 @@ impl Blocks { [], )?; + tx.commit()?; + // Add account and operation type indexes if optimization is enabled if config.index_optimization == IndexOptimization::Enabled { + let num_blocks: u64 = connection + .query_row("SELECT count(*) FROM blocks", [], |row| row.get(0)) + .unwrap_or(0); + + if num_blocks > 0 { + info!( + "Checking and creating additional search indexes. This may take a while as there are {} blocks in the store.", + num_blocks + ); + } + + fn create_index_if_not_exists( + conn: &mut rusqlite::Connection, + name: &str, + sql: &str, + num_blocks: u64, + ) -> Result<(), rusqlite::Error> { + let exists: bool = conn + .query_row( + "SELECT 1 FROM sqlite_master WHERE type='index' AND name=?", + [name], + |_| Ok(()), + ) + .optional()? + .is_some(); + + if !exists { + if num_blocks > 0 { + info!("Creating index '{}'...", name); + } + + // Set progress handler to log every ~5 seconds + // SQLite instructions are fast, so we set N to something reasonable like 10000 + // and check time. + let start = std::time::Instant::now(); + let mut last_log = start; + let index_name = name.to_string(); // Clone for closure + + conn.progress_handler( + 10000, + Some(move || { + let now = std::time::Instant::now(); + if now.duration_since(last_log).as_secs() >= 5 { + info!( + "Still creating index '{}'... ({:?} elapsed)", + index_name, + now.duration_since(start) + ); + last_log = now; + } + true // continue + }), + ); + + let result = conn.execute(sql, []); + + conn.progress_handler(0, None:: bool>); + result?; + + if num_blocks > 0 { + info!("Finished creating index '{}'", name); + } + } + Ok(()) + } + // From account index - tx.execute( + create_index_if_not_exists( + &mut connection, + "from_account_index", r#" - CREATE INDEX IF NOT EXISTS from_account_index + CREATE INDEX from_account_index ON blocks(from_account) "#, - [], + num_blocks, )?; // To account index - tx.execute( + create_index_if_not_exists( + &mut connection, + "to_account_index", r#" - CREATE INDEX IF NOT EXISTS to_account_index + CREATE INDEX to_account_index ON blocks(to_account) "#, - [], + num_blocks, )?; // Spender account index - tx.execute( + create_index_if_not_exists( + &mut connection, + "spender_account_index", r#" - CREATE INDEX IF NOT EXISTS spender_account_index + CREATE INDEX spender_account_index ON blocks(spender_account) "#, - [], + num_blocks, )?; // Operation type index for searching by transaction type - tx.execute( + create_index_if_not_exists( + &mut connection, + "operation_type_index", r#" - CREATE INDEX IF NOT EXISTS operation_type_index + CREATE INDEX operation_type_index ON blocks(operation_type) "#, - [], + num_blocks, )?; } - tx.commit() + Ok(()) } fn cache_rosetta_blocks_mode(&mut self) -> Result<(), rusqlite::Error> { @@ -2057,4 +2133,66 @@ VALUES (:idx, :block_idx)"#, assert!(!index_names.contains(&"spender_account_index".to_string())); assert!(!index_names.contains(&"operation_type_index".to_string())); } + + #[test] + fn test_search_indexes_creation_existing_db() { + let tmp_dir = ic_ledger_canister_blocks_synchronizer_test_utils::create_tmp_dir(); + let config_disabled = RosettaDbConfig::default_disabled(); + + { + let mut blocks = Blocks::new_persistent(tmp_dir.path(), config_disabled).unwrap(); + + let to = AccountIdentifier::new(ic_types::PrincipalId(Principal::anonymous()), None); + let transaction0 = Transaction { + operation: Operation::Mint { + to, + amount: Tokens::from_e8s(1), + }, + memo: Memo(0), + created_at_time: None, + icrc1_memo: None, + }; + let block0 = Block { + parent_hash: None, + transaction: transaction0.clone(), + timestamp: TimeStamp::from_nanos_since_unix_epoch(1), + }; + let encoded_block0 = block0.clone().encode(); + let hashed_block0 = crate::blocks::HashedBlock::hash_block( + encoded_block0.clone(), + block0.parent_hash, + 0, + block0.timestamp, + ); + blocks.push(&hashed_block0).unwrap(); + } + + // Re-open with optimization enabled + let config_enabled = RosettaDbConfig::default_enabled(); + let blocks = Blocks::new_persistent(tmp_dir.path(), config_enabled).unwrap(); + let connection = blocks.connection.lock().unwrap(); + + // Check indexes + let index_query = + "SELECT name FROM sqlite_master WHERE type='index' AND name IN (?, ?, ?, ?)"; + let mut stmt = connection.prepare(index_query).unwrap(); + let index_names: Vec = stmt + .query_map( + [ + "from_account_index", + "to_account_index", + "spender_account_index", + "operation_type_index", + ], + |row| row.get::<_, String>(0), + ) + .unwrap() + .collect::, _>>() + .unwrap(); + + assert!(index_names.contains(&"from_account_index".to_string())); + assert!(index_names.contains(&"to_account_index".to_string())); + assert!(index_names.contains(&"spender_account_index".to_string())); + assert!(index_names.contains(&"operation_type_index".to_string())); + } } From 76044e1561b1b3a08c65d060d20a4aa3e460668e Mon Sep 17 00:00:00 2001 From: "Paul H. Liu" Date: Sat, 27 Dec 2025 21:52:36 -0800 Subject: [PATCH 2/4] Fix bazel --- bazel/rust.MODULE.bazel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bazel/rust.MODULE.bazel b/bazel/rust.MODULE.bazel index 95c5c62c27ca..925e86cd4cb1 100644 --- a/bazel/rust.MODULE.bazel +++ b/bazel/rust.MODULE.bazel @@ -1322,7 +1322,7 @@ crate.spec( version = "0.9", ) crate.spec( - features = ["bundled"], + features = ["bundled", "hooks"], package = "rusqlite", version = "^0.28.0", ) From 3f34e4b90cbfb3100277b3bf0feb8b1638b2d713 Mon Sep 17 00:00:00 2001 From: "Paul H. Liu" Date: Sun, 28 Dec 2025 05:55:50 +0000 Subject: [PATCH 3/4] Fix bazel lock --- Cargo.Bazel.json.lock | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Cargo.Bazel.json.lock b/Cargo.Bazel.json.lock index 2712f3adbaab..a11ee28dcbf8 100644 --- a/Cargo.Bazel.json.lock +++ b/Cargo.Bazel.json.lock @@ -1,5 +1,5 @@ { - "checksum": "e9d5e9b93b2036cff0104e43b708a1a72d5337db5d7a3c6e13f62f25cb2ed4a9", + "checksum": "ade345bd9fd0b9f6f8934b96b84c4635e41c6cf75efe2c56dac104d970f4751c", "crates": { "abnf 0.12.0": { "name": "abnf", @@ -68368,6 +68368,7 @@ "crate_features": { "common": [ "bundled", + "hooks", "modern_sqlite" ], "selects": {} From e8a09772c85d93f6b665e4b9b8bb0864d462d178 Mon Sep 17 00:00:00 2001 From: IDX GitHub Automation Date: Sun, 28 Dec 2025 06:34:00 +0000 Subject: [PATCH 4/4] Automatically fixing code for linting and formatting issues --- bazel/rust.MODULE.bazel | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bazel/rust.MODULE.bazel b/bazel/rust.MODULE.bazel index 925e86cd4cb1..9be16d5c1fcd 100644 --- a/bazel/rust.MODULE.bazel +++ b/bazel/rust.MODULE.bazel @@ -1322,7 +1322,10 @@ crate.spec( version = "0.9", ) crate.spec( - features = ["bundled", "hooks"], + features = [ + "bundled", + "hooks", + ], package = "rusqlite", version = "^0.28.0", )