feat(rosetta-api): display DB index optimizations status

Cargo.Bazel.json.lock

@@ -1,5 +1,5 @@
 {
-  "checksum": "e9d5e9b93b2036cff0104e43b708a1a72d5337db5d7a3c6e13f62f25cb2ed4a9",
+  "checksum": "ade345bd9fd0b9f6f8934b96b84c4635e41c6cf75efe2c56dac104d970f4751c",
   "crates": {
     "abnf 0.12.0": {
       "name": "abnf",
@@ -68368,6 +68368,7 @@
         "crate_features": {
           "common": [
             "bundled",
+            "hooks",
             "modern_sqlite"
           ],
           "selects": {}

bazel/rust.MODULE.bazel

@@ -1322,7 +1322,10 @@ crate.spec(
     version = "0.9",
 )
 crate.spec(
-    features = ["bundled"],
+    features = [
+        "bundled",
+        "hooks",
+    ],
     package = "rusqlite",
     version = "^0.28.0",
 )

rs/rosetta-api/icp/ledger_canister_blocks_synchronizer/Cargo.toml

@@ -22,7 +22,7 @@ icp-ledger = { path = "../../../ledger_suite/icp" }
 on_wire = { path = "../../../rust_canisters/on_wire" }
 reqwest = { workspace = true }
 rosetta-core = { path = "../../common/rosetta_core" }
-rusqlite = { version = "~0.28.0", features = ["bundled"] }
+rusqlite = { version = "~0.28.0", features = ["bundled", "hooks"] }
 serde = { workspace = true }
 tokio = { workspace = true }
 tracing = { workspace = true }

rs/rosetta-api/icp/ledger_canister_blocks_synchronizer/src/blocks.rs

@@ -904,46 +904,122 @@ impl Blocks {
             [],
         )?;
 
+        tx.commit()?;
+
         // Add account and operation type indexes if optimization is enabled
         if config.index_optimization == IndexOptimization::Enabled {
+            let num_blocks: u64 = connection
+                .query_row("SELECT count(*) FROM blocks", [], |row| row.get(0))
+                .unwrap_or(0);
+
+            if num_blocks > 0 {
+                info!(
+                    "Checking and creating additional search indexes. This may take a while as there are {} blocks in the store.",
+                    num_blocks
+                );
+            }
+
+            fn create_index_if_not_exists(
+                conn: &mut rusqlite::Connection,
+                name: &str,
+                sql: &str,
+                num_blocks: u64,
+            ) -> Result<(), rusqlite::Error> {
+                let exists: bool = conn
+                    .query_row(
+                        "SELECT 1 FROM sqlite_master WHERE type='index' AND name=?",
+                        [name],
+                        |_| Ok(()),
+                    )
+                    .optional()?
+                    .is_some();
+
+                if !exists {
+                    if num_blocks > 0 {
+                        info!("Creating index '{}'...", name);
+                    }
+
+                    // Set progress handler to log every ~5 seconds
+                    // SQLite instructions are fast, so we set N to something reasonable like 10000
+                    // and check time.
+                    let start = std::time::Instant::now();
+                    let mut last_log = start;
+                    let index_name = name.to_string(); // Clone for closure
+
+                    conn.progress_handler(
+                        10000,
+                        Some(move || {
+                            let now = std::time::Instant::now();
+                            if now.duration_since(last_log).as_secs() >= 5 {
+                                info!(
+                                    "Still creating index '{}'... ({:?} elapsed)",
+                                    index_name,
+                                    now.duration_since(start)
+                                );
+                                last_log = now;
+                            }
+                            true // continue
+                        }),
+                    );
+
+                    let result = conn.execute(sql, []);
+
+                    conn.progress_handler(0, None::<fn() -> bool>);
+                    result?;
+
+                    if num_blocks > 0 {
+                        info!("Finished creating index '{}'", name);
+                    }
+                }
+                Ok(())
+            }
+
             // From account index
-            tx.execute(
+            create_index_if_not_exists(
+                &mut connection,
+                "from_account_index",
                 r#"
-            CREATE INDEX IF NOT EXISTS from_account_index 
+            CREATE INDEX from_account_index 
             ON blocks(from_account)
             "#,
-                [],
+                num_blocks,
             )?;
 
             // To account index
-            tx.execute(
+            create_index_if_not_exists(
+                &mut connection,
+                "to_account_index",
                 r#"
-            CREATE INDEX IF NOT EXISTS to_account_index 
+            CREATE INDEX to_account_index 
             ON blocks(to_account)
             "#,
-                [],
+                num_blocks,
             )?;
 
             // Spender account index
-            tx.execute(
+            create_index_if_not_exists(
+                &mut connection,
+                "spender_account_index",
                 r#"
-            CREATE INDEX IF NOT EXISTS spender_account_index 
+            CREATE INDEX spender_account_index 
             ON blocks(spender_account)
             "#,
-                [],
+                num_blocks,
             )?;
 
             // Operation type index for searching by transaction type
-            tx.execute(
+            create_index_if_not_exists(
+                &mut connection,
+                "operation_type_index",
                 r#"
-            CREATE INDEX IF NOT EXISTS operation_type_index 
+            CREATE INDEX operation_type_index 
             ON blocks(operation_type)
             "#,
-                [],
+                num_blocks,
             )?;
         }
 
-        tx.commit()
+        Ok(())
     }
 
     fn cache_rosetta_blocks_mode(&mut self) -> Result<(), rusqlite::Error> {
@@ -2057,4 +2133,66 @@ VALUES (:idx, :block_idx)"#,
         assert!(!index_names.contains(&"spender_account_index".to_string()));
         assert!(!index_names.contains(&"operation_type_index".to_string()));
     }
+
+    #[test]
+    fn test_search_indexes_creation_existing_db() {
+        let tmp_dir = ic_ledger_canister_blocks_synchronizer_test_utils::create_tmp_dir();
+        let config_disabled = RosettaDbConfig::default_disabled();
+
+        {
+            let mut blocks = Blocks::new_persistent(tmp_dir.path(), config_disabled).unwrap();
+
+            let to = AccountIdentifier::new(ic_types::PrincipalId(Principal::anonymous()), None);
+            let transaction0 = Transaction {
+                operation: Operation::Mint {
+                    to,
+                    amount: Tokens::from_e8s(1),
+                },
+                memo: Memo(0),
+                created_at_time: None,
+                icrc1_memo: None,
+            };
+            let block0 = Block {
+                parent_hash: None,
+                transaction: transaction0.clone(),
+                timestamp: TimeStamp::from_nanos_since_unix_epoch(1),
+            };
+            let encoded_block0 = block0.clone().encode();
+            let hashed_block0 = crate::blocks::HashedBlock::hash_block(
+                encoded_block0.clone(),
+                block0.parent_hash,
+                0,
+                block0.timestamp,
+            );
+            blocks.push(&hashed_block0).unwrap();
+        }
+
+        // Re-open with optimization enabled
+        let config_enabled = RosettaDbConfig::default_enabled();
+        let blocks = Blocks::new_persistent(tmp_dir.path(), config_enabled).unwrap();
+        let connection = blocks.connection.lock().unwrap();
+
+        // Check indexes
+        let index_query =
+            "SELECT name FROM sqlite_master WHERE type='index' AND name IN (?, ?, ?, ?)";
+        let mut stmt = connection.prepare(index_query).unwrap();
+        let index_names: Vec<String> = stmt
+            .query_map(
+                [
+                    "from_account_index",
+                    "to_account_index",
+                    "spender_account_index",
+                    "operation_type_index",
+                ],
+                |row| row.get::<_, String>(0),
+            )
+            .unwrap()
+            .collect::<Result<Vec<_>, _>>()
+            .unwrap();
+
+        assert!(index_names.contains(&"from_account_index".to_string()));
+        assert!(index_names.contains(&"to_account_index".to_string()));
+        assert!(index_names.contains(&"spender_account_index".to_string()));
+        assert!(index_names.contains(&"operation_type_index".to_string()));
+    }
 }