feat: automatically deploy a Prometheus VM in testnets

rs/ethereum/cketh/mainnet/README.md

@@ -186,9 +186,6 @@ To test the proposals with a testnet that uses the same canister IDs as in the p
 The simplest is to tweak the setup from [small](https://sourcegraph.com/github.com/dfinity/ic@7313a15e21d8fb06fa119ef3ab9371da47c2cddc/-/blob/rs/tests/idx/testnets/small.rs?L62)
 ```rust
 pub fn setup(env: TestEnv) {
-    PrometheusVm::default()
-        .start(&env)
-        .expect("Failed to start prometheus VM");
     let mut ic = InternetComputer::new().add_subnet(Subnet::new(SubnetType::System).add_nodes(1));
     for _ in 0..36 {
         ic = ic.add_subnet(Subnet::new(SubnetType::Application).add_nodes(1));
@@ -200,7 +197,6 @@ pub fn setup(env: TestEnv) {
         env.topology_snapshot(),
         NnsCustomizations::default(),
     );
-    env.sync_with_prometheus();
 }
 ```
 

rs/tests/boundary_nodes/BUILD.bazel

@@ -27,6 +27,8 @@ system_test_nns(
         "vcpus": 16,
     },
     enable_head_nns_variant = False,
+    enable_metrics = True,
+    prometheus_vm_required_host_features = ["performance"],
     tags = [
         "colocate",
     ],
@@ -135,6 +137,8 @@ system_test_nns(
         "memory_kibibytes": 512142680,
         "boot_image_minimal_size_gibibytes": 500,
     },
+    enable_metrics = True,
+    prometheus_vm_required_host_features = ["performance"],
     tags = ["manual"],
     runtime_deps = COUNTER_CANISTER_RUNTIME_DEPS,
     deps = [
@@ -154,7 +158,9 @@ system_test(
         "memory_kibibytes": 512142680,
         "boot_image_minimal_size_gibibytes": 500,
     },
+    enable_metrics = True,
     env_inherit = ["BOUNDARY_NODE_IPV6"],
+    prometheus_vm_required_host_features = ["performance"],
     tags = [
         "manual",
     ],
@@ -175,7 +181,9 @@ system_test(
         "memory_kibibytes": 512142680,
         "boot_image_minimal_size_gibibytes": 500,
     },
+    enable_metrics = True,
     env_inherit = ["BOUNDARY_NODE_IPV6"],
+    prometheus_vm_required_host_features = ["performance"],
     tags = [
         "manual",
     ],

rs/tests/boundary_nodes/performance_test_common/src/lib.rs

@@ -10,7 +10,6 @@ use ic_system_test_driver::{
     driver::{
         farm::HostFeature,
         ic::{AmountOfMemoryKiB, ImageSizeGiB, InternetComputer, NrOfVCPUs, Subnet, VmResources},
-        prometheus_vm::{HasPrometheus, PrometheusVm},
         test_env::TestEnv,
         test_env_api::{
             HasPublicApiUrl, HasTopologySnapshot, IcNodeContainer, NnsInstallationBuilder,
@@ -40,10 +39,6 @@ const MAX_RUNTIME_THREADS: usize = 64;
 const MAX_RUNTIME_BLOCKING_THREADS: usize = MAX_RUNTIME_THREADS;
 
 pub fn setup(env: TestEnv) {
-    PrometheusVm::default()
-        .with_required_host_features(vec![HostFeature::Performance])
-        .start(&env)
-        .expect("failed to start prometheus VM");
     InternetComputer::new()
         .with_required_host_features(vec![HostFeature::Performance])
         .add_subnet(Subnet::new(SubnetType::System).add_nodes(1))
@@ -68,8 +63,6 @@ pub fn setup(env: TestEnv) {
     NnsInstallationBuilder::new()
         .install(&nns_node, &env)
         .expect("Could not install NNS canisters");
-
-    env.sync_with_prometheus();
 }
 
 // Execute update calls (without polling) with an increasing req/s rate, against a counter canister via the boundary node agent.

rs/tests/consensus/BUILD.bazel

@@ -45,6 +45,7 @@ rust_library(
 system_test(
     name = "catch_up_loop_prevention_test",
     guestos = "malicious",
+    prometheus_vm_scrape_interval_secs = 5,
     # TODO(NET-1683): Adjust test for faster p2p
     tags = [
         "long_test",
@@ -63,6 +64,7 @@ system_test(
 system_test(
     name = "catch_up_possible_test",
     guestos = "malicious",
+    prometheus_vm_scrape_interval_secs = 5,
     # TODO(NET-1683): Adjust test for faster p2p
     tags = [
         "long_test",
@@ -352,7 +354,9 @@ system_test_nns(
         "//conditions:default": [],
     }),
     enable_head_nns_variant = False,
+    enable_metrics = True,
     env = {"ENV_DEPS__IC_VERSION_FILE": "$(rootpath //bazel:version.txt)"},
+    prometheus_vm_required_host_features = ["performance"],
     tags = [
         "colocate",
         "manual",

rs/tests/consensus/catch_up_loop_prevention_test.rs

@@ -25,7 +25,6 @@ use ic_consensus_system_test_catch_up_test_common::test_catch_up_possible;
 use ic_registry_subnet_type::SubnetType;
 use ic_system_test_driver::driver::group::SystemTestGroup;
 use ic_system_test_driver::driver::ic::{InternetComputer, Subnet};
-use ic_system_test_driver::driver::prometheus_vm::{HasPrometheus, PrometheusVm};
 use ic_system_test_driver::driver::test_env::TestEnv;
 use ic_system_test_driver::systest;
 
@@ -34,7 +33,6 @@ use ic_types::Height;
 use ic_types::malicious_behavior::MaliciousBehavior;
 use std::time::Duration;
 
-const PROMETHEUS_SCRAPE_INTERVAL: Duration = Duration::from_secs(5);
 const TIMEOUT: Duration = Duration::from_secs(30 * 60);
 
 const EXECUTION_DELAY_FACTOR: f64 = 1.2;
@@ -47,11 +45,6 @@ fn setup(env: TestEnv) {
     let execution_delay_ms = (EXECUTION_DELAY_FACTOR * TARGET_FR_MS as f64) as u64;
     let state_sync_delay_ms = (STATE_SYNC_DELAY_FACTOR * DKG_INTERVAL_TIME_MS as f64) as u64;
 
-    PrometheusVm::default()
-        .with_scrape_interval(PROMETHEUS_SCRAPE_INTERVAL)
-        .start(&env)
-        .expect("failed to start prometheus VM");
-
     InternetComputer::new()
         .add_subnet(
             Subnet::new(SubnetType::System)
@@ -70,8 +63,6 @@ fn setup(env: TestEnv) {
         )
         .setup_and_start(&env)
         .expect("failed to setup IC under test");
-
-    env.sync_with_prometheus();
 }
 
 fn main() -> Result<()> {

rs/tests/consensus/catch_up_possible_test.rs

@@ -25,7 +25,6 @@ use ic_consensus_system_test_catch_up_test_common::test_catch_up_possible;
 use ic_registry_subnet_type::SubnetType;
 use ic_system_test_driver::driver::group::SystemTestGroup;
 use ic_system_test_driver::driver::ic::{InternetComputer, Subnet};
-use ic_system_test_driver::driver::prometheus_vm::{HasPrometheus, PrometheusVm};
 use ic_system_test_driver::driver::test_env::TestEnv;
 use ic_system_test_driver::systest;
 
@@ -34,7 +33,6 @@ use ic_types::Height;
 use ic_types::malicious_behavior::MaliciousBehavior;
 use std::time::Duration;
 
-const PROMETHEUS_SCRAPE_INTERVAL: Duration = Duration::from_secs(5);
 const TIMEOUT: Duration = Duration::from_secs(30 * 60);
 
 const EXECUTION_DELAY_FACTOR: f64 = 0.8;
@@ -47,11 +45,6 @@ fn setup(env: TestEnv) {
     let execution_delay_ms = (EXECUTION_DELAY_FACTOR * TARGET_FR_MS as f64) as u64;
     let state_sync_delay_ms = (STATE_SYNC_DELAY_FACTOR * DKG_INTERVAL_TIME_MS as f64) as u64;
 
-    PrometheusVm::default()
-        .with_scrape_interval(PROMETHEUS_SCRAPE_INTERVAL)
-        .start(&env)
-        .expect("failed to start prometheus VM");
-
     InternetComputer::new()
         .add_subnet(
             Subnet::new(SubnetType::System)
@@ -70,8 +63,6 @@ fn setup(env: TestEnv) {
         )
         .setup_and_start(&env)
         .expect("failed to setup IC under test");
-
-    env.sync_with_prometheus();
 }
 
 fn main() -> Result<()> {

rs/tests/consensus/consensus_performance.rs

@@ -67,7 +67,7 @@ use ic_system_test_driver::driver::test_env_api::get_current_branch_version;
 use ic_system_test_driver::driver::{
     farm::HostFeature,
     ic::{AmountOfMemoryKiB, ImageSizeGiB, InternetComputer, NrOfVCPUs, Subnet, VmResources},
-    prometheus_vm::{HasPrometheus, PrometheusVm},
+    prometheus_vm::HasPrometheus,
     simulate_network::{FixedNetworkSimulation, SimulateNetwork},
     test_env::TestEnv,
     test_env_api::{HasTopologySnapshot, NnsCustomizations},
@@ -99,11 +99,6 @@ const NETWORK_SIMULATION: FixedNetworkSimulation = FixedNetworkSimulation::new()
 const SHOULD_SPAWN_JAEGER_VM: bool = false;
 
 fn setup(env: TestEnv) {
-    PrometheusVm::default()
-        .with_required_host_features(vec![HostFeature::Performance])
-        .start(&env)
-        .expect("Failed to start prometheus VM");
-
     let mut ic_builder = InternetComputer::new();
 
     if SHOULD_SPAWN_JAEGER_VM {
@@ -141,7 +136,6 @@ fn setup(env: TestEnv) {
         env.topology_snapshot(),
         NnsCustomizations::default(),
     );
-    env.sync_with_prometheus();
 
     let topology_snapshot = env.topology_snapshot();
     let (app_subnet, _) = get_app_subnet_and_node(&topology_snapshot);

rs/tests/consensus/tecdsa/BUILD.bazel

@@ -300,6 +300,11 @@ tecdsa_performance_test_template = system_test_nns(
         "//bazel:upload_perf_systest_results_enabled": ["upload_perf_systest_results"],
         "//conditions:default": [],
     }),
+    enable_metrics = True,
+    prometheus_vm_required_host_features = [
+        "performance",
+        "spm",
+    ],
     tags = [
         "colocate",
         "manual",
@@ -342,11 +347,16 @@ system_test_nns(
         "boot_image_minimal_size_gibibytes": 500,
     },
     enable_head_nns_variant = False,
+    enable_metrics = True,
     env = MESSAGE_CANISTER_ENV | SIGNER_CANISTER_ENV | {
         "TECDSA_PERFORMANCE_TEST_KEY_IDS": "ecdsa_secp256k1",
         "BENCHMARK_NAME": "tecdsa_performance_test",
         "DEFAULT_NODES_COUNT": DEFAULT_NODES_COUNT,
     },
+    prometheus_vm_required_host_features = [
+        "performance",
+        "spm",
+    ],
     tags = [
         "colocate",
         "manual",
@@ -371,11 +381,16 @@ system_test_nns(
         "boot_image_minimal_size_gibibytes": 500,
     },
     enable_head_nns_variant = False,
+    enable_metrics = True,
     env = MESSAGE_CANISTER_ENV | SIGNER_CANISTER_ENV | {
         "TECDSA_PERFORMANCE_TEST_KEY_IDS": "schnorr_ed25519",
         "BENCHMARK_NAME": "tschnorr_ed25519_performance_test",
         "DEFAULT_NODES_COUNT": DEFAULT_NODES_COUNT,
     },
+    prometheus_vm_required_host_features = [
+        "performance",
+        "spm",
+    ],
     tags = [
         "colocate",
         "manual",
@@ -400,11 +415,16 @@ system_test_nns(
         "boot_image_minimal_size_gibibytes": 500,
     },
     enable_head_nns_variant = False,
+    enable_metrics = True,
     env = MESSAGE_CANISTER_ENV | SIGNER_CANISTER_ENV | {
         "TECDSA_PERFORMANCE_TEST_KEY_IDS": "schnorr_bip340",
         "BENCHMARK_NAME": "tschnorr_bip340_performance_test",
         "DEFAULT_NODES_COUNT": DEFAULT_NODES_COUNT,
     },
+    prometheus_vm_required_host_features = [
+        "performance",
+        "spm",
+    ],
     tags = [
         "colocate",
         "manual",
@@ -429,11 +449,16 @@ system_test_nns(
         "boot_image_minimal_size_gibibytes": 500,
     },
     enable_head_nns_variant = False,
+    enable_metrics = True,
     env = MESSAGE_CANISTER_ENV | SIGNER_CANISTER_ENV | {
         "TECDSA_PERFORMANCE_TEST_KEY_IDS": "vetkd_bls12_381_g2",
         "BENCHMARK_NAME": "vetkd_performance_test",
         "DEFAULT_NODES_COUNT": DEFAULT_NODES_COUNT,
     },
+    prometheus_vm_required_host_features = [
+        "performance",
+        "spm",
+    ],
     tags = [
         "colocate",
         "manual",

rs/tests/consensus/tecdsa/tecdsa_performance_test_template.rs

@@ -70,7 +70,7 @@ use ic_system_test_driver::driver::test_env_api::HasPublicApiUrl;
 use ic_system_test_driver::driver::{
     farm::HostFeature,
     ic::{AmountOfMemoryKiB, ImageSizeGiB, InternetComputer, NrOfVCPUs, Subnet, VmResources},
-    prometheus_vm::{HasPrometheus, PrometheusVm},
+    prometheus_vm::HasPrometheus,
     simulate_network::{FixedNetworkSimulation, SimulateNetwork},
     test_env::TestEnv,
     test_env_api::{HasTopologySnapshot, IcNodeContainer, NnsCustomizations},
@@ -181,11 +181,6 @@ pub fn setup(env: TestEnv) {
     let key_ids = make_key_ids();
     info!(env.logger(), "Running the test with key ids: {:?}", key_ids);
 
-    PrometheusVm::default()
-        .with_required_host_features(vec![HostFeature::Performance, HostFeature::Supermicro])
-        .start(&env)
-        .expect("Failed to start prometheus VM");
-
     let vm_resources = VmResources {
         vcpus: Some(NrOfVCPUs::new(64)),
         memory_kibibytes: Some(AmountOfMemoryKiB::new(512_142_680)),
@@ -229,7 +224,6 @@ pub fn setup(env: TestEnv) {
         env.topology_snapshot(),
         NnsCustomizations::default(),
     );
-    env.sync_with_prometheus();
 }
 
 pub fn test(env: TestEnv) {

rs/tests/dre/utils/lib.rs

@@ -5,7 +5,6 @@ use ic_registry_subnet_type::SubnetType;
 use ic_system_test_driver::driver::{
     ic::{InternetComputer, Node, Subnet},
     node_software_version::NodeSoftwareVersion,
-    prometheus_vm::{HasPrometheus, PrometheusVm},
     test_env::TestEnv,
     test_env_api::{HasTopologySnapshot, NnsCustomizations, READY_WAIT_TIMEOUT, RETRY_BACKOFF},
 };
@@ -71,19 +70,13 @@ pub fn setup(env: TestEnv, config: IcConfig) {
                 .for_each(|un| ic = ic.clone().with_api_boundary_node(un)),
         }
     }
-
-    PrometheusVm::default()
-        .start(&env)
-        .expect("Failed to start prometheus VM");
     ic.setup_and_start(&env)
         .expect("Failed to setup IC under test");
 
     install_nns_with_customizations_and_check_progress(
         env.topology_snapshot(),
         NnsCustomizations::default(),
     );
-
-    env.sync_with_prometheus();
 }
 
 #[derive(Deserialize, Debug)]

rs/tests/driver/BUILD.bazel

@@ -109,7 +109,6 @@ rust_library(
         "@crate_index//:json5",
         "@crate_index//:lazy_static",
         "@crate_index//:macaddr",
-        "@crate_index//:maplit",
         "@crate_index//:nix",
         "@crate_index//:num_cpus",
         "@crate_index//:once_cell",

rs/tests/driver/Cargo.toml

@@ -84,7 +84,6 @@ itertools = { workspace = true }
 json5 = "0.4.1"
 lazy_static = { workspace = true }
 macaddr = { workspace = true }
-maplit = "1.0.2"
 nix = { workspace = true }
 num_cpus = "1.13.1"
 on_wire = { path = "../../rust_canisters/on_wire" }

rs/tests/driver/src/driver/context.rs

@@ -24,6 +24,7 @@ pub struct GroupContext {
     pub keepalive: bool,
     pub no_farm_keepalive: bool,
     pub group_base_name: String,
+    pub enable_metrics: bool,
     pub logs_enabled: bool,
     pub exclude_logs: Vec<Regex>,
     pub quiet: bool,
@@ -42,6 +43,7 @@ impl GroupContext {
         keepalive: bool,
         no_farm_keepalive: bool,
         group_base_name: String,
+        enable_metrics: bool,
         logs_enabled: bool,
         exclude_logs: Vec<Regex>,
         quiet: bool,
@@ -68,6 +70,7 @@ impl GroupContext {
             keepalive,
             no_farm_keepalive,
             group_base_name,
+            enable_metrics,
             logs_enabled,
             exclude_logs,
             quiet,