From 3fc94cb99e218d46b951d0088334c01f3c1c82f8 Mon Sep 17 00:00:00 2001 From: hamdykhader Date: Wed, 26 Nov 2025 18:32:27 +0300 Subject: [PATCH 01/11] Implements sfam-2459 Adds the LVolScheduler service definition to docker and k8s deployment scripts Adds the service creation on cluster update to support update from older cluster Adds missing service to k8s deployment script (simplyblock-tasks-runner-sync-lvol-del) --- simplyblock_core/cluster_ops.py | 78 +++++++++----- simplyblock_core/constants.py | 1 + simplyblock_core/env_var | 2 +- .../scripts/charts/templates/app_k8s.yaml | 102 ++++++++++++++++++ .../scripts/docker-compose-swarm.yml | 14 +++ simplyblock_core/services/lvol_scheduler.py | 37 +++++++ simplyblock_core/utils/__init__.py | 23 +++- 7 files changed, 229 insertions(+), 28 deletions(-) create mode 100644 simplyblock_core/services/lvol_scheduler.py diff --git a/simplyblock_core/cluster_ops.py b/simplyblock_core/cluster_ops.py index dc429b8f9..2b169ac7a 100644 --- a/simplyblock_core/cluster_ops.py +++ b/simplyblock_core/cluster_ops.py @@ -1184,44 +1184,43 @@ def update_cluster(cluster_id, mgmt_only=False, restart=False, spdk_image=None, service_names.append(service.attrs['Spec']['Name']) if "app_SnapshotMonitor" not in service_names: - logger.info("Creating snapshot monitor service") - cluster_docker.services.create( - image=service_image, - command="python simplyblock_core/services/snapshot_monitor.py", - name="app_SnapshotMonitor", - mounts=["/etc/foundationdb:/etc/foundationdb"], - env=["SIMPLYBLOCK_LOG_LEVEL=DEBUG"], - networks=["host"], - constraints=["node.role == manager"] - ) + utils.create_docker_service( + cluster_docker=cluster_docker, + service_name="app_SnapshotMonitor", + service_file="python simplyblock_core/services/snapshot_monitor.py", + service_image=service_image) if "app_TasksRunnerLVolSyncDelete" not in service_names: - logger.info("Creating lvol sync delete service") - cluster_docker.services.create( - image=service_image, - command="python simplyblock_core/services/tasks_runner_sync_lvol_del.py", - name="app_TasksRunnerLVolSyncDelete", - mounts=["/etc/foundationdb:/etc/foundationdb"], - env=["SIMPLYBLOCK_LOG_LEVEL=DEBUG"], - networks=["host"], - constraints=["node.role == manager"] - ) + utils.create_docker_service( + cluster_docker=cluster_docker, + service_name="app_TasksRunnerLVolSyncDelete", + service_file="python simplyblock_core/services/tasks_runner_sync_lvol_del.py", + service_image=service_image) + + if "app_LVolScheduler" not in service_names: + utils.create_docker_service( + cluster_docker=cluster_docker, + service_name="app_LVolScheduler", + service_file="python simplyblock_core/services/lvol_scheduler.py", + service_image=service_image) + logger.info("Done updating mgmt cluster") elif cluster.mode == "kubernetes": utils.load_kube_config_with_fallback() apps_v1 = k8s_client.AppsV1Api() - + namespace = constants.K8S_NAMESPACE image_without_tag = constants.SIMPLY_BLOCK_DOCKER_IMAGE.split(":")[0] image_parts = "/".join(image_without_tag.split("/")[-2:]) service_image = mgmt_image or constants.SIMPLY_BLOCK_DOCKER_IMAGE - + deployment_names = [] # Update Deployments - deployments = apps_v1.list_namespaced_deployment(namespace=constants.K8S_NAMESPACE) + deployments = apps_v1.list_namespaced_deployment(namespace=namespace) for deploy in deployments.items: if deploy.metadata.name == constants.ADMIN_DEPLOY_NAME: logger.info(f"Skipping deployment {deploy.metadata.name}") continue + deployment_names.append(deploy.metadata.name) for c in deploy.spec.template.spec.containers: if image_parts in c.image: logger.info(f"Updating deployment {deploy.metadata.name} image to {service_image}") @@ -1231,12 +1230,39 @@ def update_cluster(cluster_id, mgmt_only=False, restart=False, spdk_image=None, deploy.spec.template.metadata.annotations = annotations apps_v1.patch_namespaced_deployment( name=deploy.metadata.name, - namespace=constants.K8S_NAMESPACE, + namespace=namespace, body={"spec": {"template": deploy.spec.template}} ) + if f"{namespace}-tasks-runner-sync-lvol-del" not in deployment_names: + utils.create_k8s_service( + k8s_apps_client=apps_v1, + namespace=namespace, + deployment_name=f"{namespace}-tasks-runner-sync-lvol-del", + container_name="tasks-runner-sync-lvol-del", + service_file="simplyblock_core/services/tasks_runner_sync_lvol_del.py", + container_image=service_image) + + if f"{namespace}-snapshot-monitor" not in deployment_names: + utils.create_k8s_service( + k8s_apps_client=apps_v1, + namespace=namespace, + deployment_name=f"{namespace}-snapshot-monitor", + container_name="snapshot-monitor", + service_file="simplyblock_core/services/snapshot_monitor.py", + container_image=service_image) + + if f"{namespace}-lvol-scheduler" not in deployment_names: + utils.create_k8s_service( + k8s_apps_client=apps_v1, + namespace=namespace, + deployment_name=f"{namespace}-lvol-scheduler", + container_name="lvol-scheduler", + service_file="simplyblock_core/services/lvol_scheduler.py", + container_image=service_image) + # Update DaemonSets - daemonsets = apps_v1.list_namespaced_daemon_set(namespace=constants.K8S_NAMESPACE) + daemonsets = apps_v1.list_namespaced_daemon_set(namespace=namespace) for ds in daemonsets.items: for c in ds.spec.template.spec.containers: if image_parts in c.image: @@ -1247,7 +1273,7 @@ def update_cluster(cluster_id, mgmt_only=False, restart=False, spdk_image=None, ds.spec.template.metadata.annotations = annotations apps_v1.patch_namespaced_daemon_set( name=ds.metadata.name, - namespace=constants.K8S_NAMESPACE, + namespace=namespace, body={"spec": {"template": ds.spec.template}} ) diff --git a/simplyblock_core/constants.py b/simplyblock_core/constants.py index 36ba14a9e..8d33597c4 100644 --- a/simplyblock_core/constants.py +++ b/simplyblock_core/constants.py @@ -49,6 +49,7 @@ def get_config_var(name, default=None): SSD_VENDOR_WHITE_LIST = ["1d0f:cd01", "1d0f:cd00"] CACHED_LVOL_STAT_COLLECTOR_INTERVAL_SEC = 5 DEV_DISCOVERY_INTERVAL_SEC = 60 +LVOL_SCHEDULER_INTERVAL_SEC = 60*15 PMEM_DIR = '/tmp/pmem' diff --git a/simplyblock_core/env_var b/simplyblock_core/env_var index f34a430a9..7be56c52b 100644 --- a/simplyblock_core/env_var +++ b/simplyblock_core/env_var @@ -1,6 +1,6 @@ SIMPLY_BLOCK_COMMAND_NAME=sbcli-dev SIMPLY_BLOCK_VERSION=19.2.27 -SIMPLY_BLOCK_DOCKER_IMAGE=public.ecr.aws/simply-block/simplyblock:main +SIMPLY_BLOCK_DOCKER_IMAGE=public.ecr.aws/simply-block/simplyblock:main-lvol-scheduler SIMPLY_BLOCK_SPDK_ULTRA_IMAGE=public.ecr.aws/simply-block/ultra:main-latest diff --git a/simplyblock_core/scripts/charts/templates/app_k8s.yaml b/simplyblock_core/scripts/charts/templates/app_k8s.yaml index d17ea092a..2a3a434e6 100644 --- a/simplyblock_core/scripts/charts/templates/app_k8s.yaml +++ b/simplyblock_core/scripts/charts/templates/app_k8s.yaml @@ -1100,6 +1100,108 @@ spec: - key: cluster-file path: fdb.cluster --- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: simplyblock-lvol-scheduler + namespace: {{ .Release.Namespace }} +spec: + replicas: 1 + selector: + matchLabels: + app: simplyblock-lvol-scheduler + template: + metadata: + annotations: + log-collector/enabled: "true" + reloader.stakater.com/auto: "true" + reloader.stakater.com/configmap: "simplyblock-fdb-cluster-config" + labels: + app: simplyblock-lvol-scheduler + spec: + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + containers: + - name: lvol-scheduler + image: "{{ .Values.image.simplyblock.repository }}:{{ .Values.image.simplyblock.tag }}" + imagePullPolicy: "{{ .Values.image.simplyblock.pullPolicy }}" + command: ["python", "simplyblock_core/services/lvol_scheduler.py"] + env: + - name: SIMPLYBLOCK_LOG_LEVEL + valueFrom: + configMapKeyRef: + name: simplyblock-config + key: LOG_LEVEL + volumeMounts: + - name: fdb-cluster-file + mountPath: /etc/foundationdb/fdb.cluster + subPath: fdb.cluster + resources: + requests: + cpu: "100m" + memory: "256Mi" + limits: + cpu: "400m" + memory: "1Gi" + volumes: + - name: fdb-cluster-file + configMap: + name: simplyblock-fdb-cluster-config + items: + - key: cluster-file + path: fdb.cluster +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: simplyblock-tasks-runner-sync-lvol-del + namespace: {{ .Release.Namespace }} +spec: + replicas: 1 + selector: + matchLabels: + app: simplyblock-tasks-runner-sync-lvol-del + template: + metadata: + annotations: + log-collector/enabled: "true" + reloader.stakater.com/auto: "true" + reloader.stakater.com/configmap: "simplyblock-fdb-cluster-config" + labels: + app: simplyblock-tasks-runner-sync-lvol-del + spec: + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + containers: + - name: tasks-runner-sync-lvol-del + image: "{{ .Values.image.simplyblock.repository }}:{{ .Values.image.simplyblock.tag }}" + imagePullPolicy: "{{ .Values.image.simplyblock.pullPolicy }}" + command: ["python", "simplyblock_core/services/tasks_runner_sync_lvol_del.py"] + env: + - name: SIMPLYBLOCK_LOG_LEVEL + valueFrom: + configMapKeyRef: + name: simplyblock-config + key: LOG_LEVEL + volumeMounts: + - name: fdb-cluster-file + mountPath: /etc/foundationdb/fdb.cluster + subPath: fdb.cluster + resources: + requests: + cpu: "200m" + memory: "256Mi" + limits: + cpu: "400m" + memory: "1Gi" + volumes: + - name: fdb-cluster-file + configMap: + name: simplyblock-fdb-cluster-config + items: + - key: cluster-file + path: fdb.cluster +--- apiVersion: apps/v1 kind: DaemonSet diff --git a/simplyblock_core/scripts/docker-compose-swarm.yml b/simplyblock_core/scripts/docker-compose-swarm.yml index fd79f43c1..2fdda5059 100644 --- a/simplyblock_core/scripts/docker-compose-swarm.yml +++ b/simplyblock_core/scripts/docker-compose-swarm.yml @@ -363,6 +363,20 @@ services: environment: SIMPLYBLOCK_LOG_LEVEL: "$LOG_LEVEL" + LVolScheduler: + <<: *service-base + image: $SIMPLYBLOCK_DOCKER_IMAGE + command: "python simplyblock_core/services/lvol_scheduler.py" + deploy: + placement: + constraints: [node.role == manager] + volumes: + - "/etc/foundationdb:/etc/foundationdb" + networks: + - hostnet + environment: + SIMPLYBLOCK_LOG_LEVEL: "$LOG_LEVEL" + networks: monitoring-net: external: true diff --git a/simplyblock_core/services/lvol_scheduler.py b/simplyblock_core/services/lvol_scheduler.py new file mode 100644 index 000000000..ad350622e --- /dev/null +++ b/simplyblock_core/services/lvol_scheduler.py @@ -0,0 +1,37 @@ +# coding=utf-8 +import time + +from simplyblock_core import constants, db_controller, utils +from simplyblock_core.models.cluster import Cluster +from simplyblock_core.models.storage_node import StorageNode +from simplyblock_core.rpc_client import RPCClient + +logger = utils.get_logger(__name__) + + +# get DB controller +db = db_controller.DBController() + +logger.info("Starting stats collector...") +while True: + + for cluster in db.get_clusters(): + + if cluster.status in [Cluster.STATUS_INACTIVE, Cluster.STATUS_UNREADY, Cluster.STATUS_IN_ACTIVATION]: + logger.warning(f"Cluster {cluster.get_id()} is in {cluster.status} state, skipping") + continue + + for snode in db.get_storage_nodes_by_cluster_id(cluster.get_id()): + + lvol_list = db.get_lvols_by_node_id(snode.get_id()) + + if not lvol_list: + continue + + if snode.status in [StorageNode.STATUS_ONLINE, StorageNode.STATUS_SUSPENDED, StorageNode.STATUS_DOWN]: + + rpc_client = RPCClient( + snode.mgmt_ip, snode.rpc_port, + snode.rpc_username, snode.rpc_password, timeout=3, retry=2) + + time.sleep(constants.LVOL_SCHEDULER_INTERVAL_SEC) diff --git a/simplyblock_core/utils/__init__.py b/simplyblock_core/utils/__init__.py index 96a00ecac..c1bac3c4f 100644 --- a/simplyblock_core/utils/__init__.py +++ b/simplyblock_core/utils/__init__.py @@ -12,8 +12,9 @@ import time import socket from typing import Union, Any, Optional, Tuple +from docker import DockerClient from kubernetes import client, config -from kubernetes.client import ApiException +from kubernetes.client import ApiException, AppsV1Api import docker from prettytable import PrettyTable from docker.errors import APIError, DockerException, ImageNotFound, NotFound @@ -2081,3 +2082,23 @@ def patch_prometheus_configmap(username: str, password: str): except Exception as e: logger.error(f"Unexpected error while patching ConfigMap: {e}") return False + + +def create_docker_service(cluster_docker: DockerClient, service_name: str, service_file: str, service_image: str): + logger.info(f"Creating service: {service_name}") + cluster_docker.services.create( + image=service_image, + command=service_file, + name=service_name, + mounts=["/etc/foundationdb:/etc/foundationdb"], + env=["SIMPLYBLOCK_LOG_LEVEL=DEBUG"], + networks=["host"], + constraints=["node.role == manager"] + ) + + +def create_k8s_service(k8s_apps_client: AppsV1Api, namespace: str, deployment_name: str, + container_name: str, service_file: str, container_image: str): + # TODO(Geoffrey): Add implementation to create a service on k8s to support cluster update from older version + logger.info(f"Creating deployment: {deployment_name}") + pass From b2cd483958fbe0a1e139efe2d045580e7a8859cd Mon Sep 17 00:00:00 2001 From: hamdykhader Date: Wed, 26 Nov 2025 20:06:07 +0300 Subject: [PATCH 02/11] adds service python header --- simplyblock_core/services/lvol_scheduler.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/simplyblock_core/services/lvol_scheduler.py b/simplyblock_core/services/lvol_scheduler.py index ad350622e..a512d93e7 100644 --- a/simplyblock_core/services/lvol_scheduler.py +++ b/simplyblock_core/services/lvol_scheduler.py @@ -1,4 +1,22 @@ # coding=utf-8 +""" +Filename: lvol_scheduler.py +Author: Hamdy Khader +Email: hamdy@simplyblock.io +Description: +LVol scheduler service will collect, calculate and store the required metric parameters for lvol scheduler algorithm +to make the following decisions: + - Which node should be used for hosting the next lvol? + - Is lvol transfer required because of high RAM consumption ? if so then which lvol/s to which node? + +Here we have the metric parameters used (per node): + ram_utilization + a timely reading of random memory utilization from file /etc/meminfo of the node + lvol_utilization + sum of the consumed data size (bytes) for all lvol on the node + lvol_count + count of lvols on the node +""" import time from simplyblock_core import constants, db_controller, utils From 4b21d27b9e14436365106ae5429ca4c0b20c740e Mon Sep 17 00:00:00 2001 From: hamdykhader Date: Wed, 26 Nov 2025 21:13:45 +0300 Subject: [PATCH 03/11] WIP --- simplyblock_core/models/storage_node.py | 5 ++++ simplyblock_core/services/lvol_scheduler.py | 27 +++++++++++++++------ 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/simplyblock_core/models/storage_node.py b/simplyblock_core/models/storage_node.py index 81639c556..54251ee31 100644 --- a/simplyblock_core/models/storage_node.py +++ b/simplyblock_core/models/storage_node.py @@ -9,6 +9,7 @@ from simplyblock_core.models.iface import IFace from simplyblock_core.models.nvme_device import NVMeDevice, JMDevice from simplyblock_core.rpc_client import RPCClient, RPCException +from simplyblock_core.snode_client import SNodeClient logger = utils.get_logger(__name__) @@ -110,6 +111,10 @@ def rpc_client(self, **kwargs): self.mgmt_ip, self.rpc_port, self.rpc_username, self.rpc_password, **kwargs) + def snode_api(self, **kwargs) -> SNodeClient: + """Return storage node API client to this node""" + return SNodeClient(f"{self.mgmt_ip}:5000", timeout=10, retry=2) + def expose_bdev(self, nqn, bdev_name, model_number, uuid, nguid, port): rpc_client = self.rpc_client() diff --git a/simplyblock_core/services/lvol_scheduler.py b/simplyblock_core/services/lvol_scheduler.py index a512d93e7..38ee7f2e4 100644 --- a/simplyblock_core/services/lvol_scheduler.py +++ b/simplyblock_core/services/lvol_scheduler.py @@ -22,7 +22,6 @@ from simplyblock_core import constants, db_controller, utils from simplyblock_core.models.cluster import Cluster from simplyblock_core.models.storage_node import StorageNode -from simplyblock_core.rpc_client import RPCClient logger = utils.get_logger(__name__) @@ -41,15 +40,27 @@ for snode in db.get_storage_nodes_by_cluster_id(cluster.get_id()): - lvol_list = db.get_lvols_by_node_id(snode.get_id()) - - if not lvol_list: + if snode.status not in [StorageNode.STATUS_ONLINE, StorageNode.STATUS_SUSPENDED, StorageNode.STATUS_DOWN]: continue - if snode.status in [StorageNode.STATUS_ONLINE, StorageNode.STATUS_SUSPENDED, StorageNode.STATUS_DOWN]: + node_info = snode.snode_api().info() + ram_utilization= 0 + lvol_utilization = 0 + lvol_count = 0 - rpc_client = RPCClient( - snode.mgmt_ip, snode.rpc_port, - snode.rpc_username, snode.rpc_password, timeout=3, retry=2) + # check for memory + if "memory_details" in node_info and node_info['memory_details']: + memory_details = node_info['memory_details'] + logger.info("Node Memory info") + logger.info(f"Total: {utils.humanbytes(memory_details['total'])}") + logger.info(f"Free: {utils.humanbytes(memory_details['free'])}") + ram_utilization = int(memory_details['free']/memory_details['total']*100) + + lvol_list = db.get_lvols_by_node_id(snode.get_id()) + lvol_count = len(lvol_list) + for lvol in lvol_list: + records = db.get_lvol_stats(lvol, 1) + if records: + lvol_utilization += records[0].size_used time.sleep(constants.LVOL_SCHEDULER_INTERVAL_SEC) From 74d34f71bce37c2b649422efa7ea39babc3f3c4b Mon Sep 17 00:00:00 2001 From: hamdykhader Date: Thu, 27 Nov 2025 14:58:37 +0300 Subject: [PATCH 04/11] WIP 2 --- simplyblock_core/cluster_ops.py | 16 ----- .../controllers/lvol_controller.py | 66 +++++++++++-------- simplyblock_core/models/storage_node.py | 2 + .../scripts/charts/templates/app_k8s.yaml | 51 -------------- .../scripts/docker-compose-swarm.yml | 13 ---- .../services/capacity_and_stats_collector.py | 39 +++++++++-- simplyblock_core/services/lvol_scheduler.py | 66 ------------------- 7 files changed, 75 insertions(+), 178 deletions(-) delete mode 100644 simplyblock_core/services/lvol_scheduler.py diff --git a/simplyblock_core/cluster_ops.py b/simplyblock_core/cluster_ops.py index 2b169ac7a..09fde6911 100644 --- a/simplyblock_core/cluster_ops.py +++ b/simplyblock_core/cluster_ops.py @@ -1197,13 +1197,6 @@ def update_cluster(cluster_id, mgmt_only=False, restart=False, spdk_image=None, service_file="python simplyblock_core/services/tasks_runner_sync_lvol_del.py", service_image=service_image) - if "app_LVolScheduler" not in service_names: - utils.create_docker_service( - cluster_docker=cluster_docker, - service_name="app_LVolScheduler", - service_file="python simplyblock_core/services/lvol_scheduler.py", - service_image=service_image) - logger.info("Done updating mgmt cluster") elif cluster.mode == "kubernetes": @@ -1252,15 +1245,6 @@ def update_cluster(cluster_id, mgmt_only=False, restart=False, spdk_image=None, service_file="simplyblock_core/services/snapshot_monitor.py", container_image=service_image) - if f"{namespace}-lvol-scheduler" not in deployment_names: - utils.create_k8s_service( - k8s_apps_client=apps_v1, - namespace=namespace, - deployment_name=f"{namespace}-lvol-scheduler", - container_name="lvol-scheduler", - service_file="simplyblock_core/services/lvol_scheduler.py", - container_image=service_image) - # Update DaemonSets daemonsets = apps_v1.list_namespaced_daemon_set(namespace=namespace) for ds in daemonsets.items: diff --git a/simplyblock_core/controllers/lvol_controller.py b/simplyblock_core/controllers/lvol_controller.py index 4d7a5aad3..37e62d771 100644 --- a/simplyblock_core/controllers/lvol_controller.py +++ b/simplyblock_core/controllers/lvol_controller.py @@ -133,46 +133,56 @@ def validate_add_lvol_func(name, size, host_id_or_name, pool_id_or_name, def _get_next_3_nodes(cluster_id, lvol_size=0): db_controller = DBController() - snodes = db_controller.get_storage_nodes_by_cluster_id(cluster_id) - online_nodes = [] node_stats = {} - for node in snodes: + all_online_nodes = [] + nodes_between_25_75 = [] + nodes_above_75 = [] + + for node in db_controller.get_storage_nodes_by_cluster_id(cluster_id): if node.is_secondary_node: # pass continue if node.status == node.STATUS_ONLINE: - lvol_count = len(db_controller.get_lvols_by_node_id(node.get_id())) - if lvol_count >= node.max_lvol: + """ + if sum of lvols (+snapshots, including namespace lvols) per node is utilized < [0.25 * max-size] AND + number of lvols (snapshots dont count extra and namspaces on same subsystem count only once) < [0.25 * max-lvol] + + --> simply round-robin schedule lvols (no randomization, no weights) + BUT: if storage utilization > 25% on one or more nodes, those nodes are excluded from round robin + + + + + + Once all nodes have > 25% of storage utilization, we weight + (relative-number-of-lvol-compared-to-total-number + relative-utilization-compared-to-total-utilzation) + --> and based on the weight just a random location + + Once a node has > 75% uof storage utilization, it is excluded to add new lvols + (unless all nodes exceed this limit, than it is weighted again) + + + """ + all_online_nodes.append(node) + + if node.node_size_util < 25: continue + elif node.node_size_util >= 75: + nodes_above_75.append(node) + else: + nodes_between_25_75.append(node) - # Validate Eligible nodes for adding lvol - # snode_api = SNodeClient(node.api_endpoint) - # result, _ = snode_api.info() - # memory_free = result["memory_details"]["free"] - # huge_free = result["memory_details"]["huge_free"] - # total_node_capacity = db_controller.get_snode_size(node.get_id()) - # error = utils.validate_add_lvol_or_snap_on_node(memory_free, huge_free, node.max_lvol, lvol_size, total_node_capacity, len(node.lvols)) - # if error: - # logger.warning(error) - # continue - # - online_nodes.append(node) - # node_stat_list = db_controller.get_node_stats(node, limit=1000) - # combined_record = utils.sum_records(node_stat_list) node_st = { - "lvol": lvol_count+1, - # "cpu": 1 + (node.cpu * node.cpu_hz), - # "r_io": combined_record.read_io_ps, - # "w_io": combined_record.write_io_ps, - # "r_b": combined_record.read_bytes_ps, - # "w_b": combined_record.write_bytes_ps + "node_size_util": node.node_size_util , + "lvol_count_util": node.lvol_count_util , } node_stats[node.get_id()] = node_st - if len(online_nodes) <= 1: - return online_nodes + if len(node_stats) <= 1: + return all_online_nodes + cluster_stats = utils.dict_agg([node_stats[k] for k in node_stats]) nodes_weight = utils.get_weights(node_stats, cluster_stats) @@ -230,7 +240,7 @@ def _get_next_3_nodes(cluster_id, lvol_size=0): ret.append(node) return ret else: - return online_nodes + return all_online_nodes def is_hex(s: str) -> bool: """ diff --git a/simplyblock_core/models/storage_node.py b/simplyblock_core/models/storage_node.py index 54251ee31..490d4c61f 100644 --- a/simplyblock_core/models/storage_node.py +++ b/simplyblock_core/models/storage_node.py @@ -103,6 +103,8 @@ class StorageNode(BaseNodeObject): hublvol: HubLVol = None # type: ignore[assignment] active_tcp: bool = True active_rdma: bool = False + lvol_count_util = 0 + node_size_util = 0 def rpc_client(self, **kwargs): """Return rpc client to this node diff --git a/simplyblock_core/scripts/charts/templates/app_k8s.yaml b/simplyblock_core/scripts/charts/templates/app_k8s.yaml index 2a3a434e6..49c7490b7 100644 --- a/simplyblock_core/scripts/charts/templates/app_k8s.yaml +++ b/simplyblock_core/scripts/charts/templates/app_k8s.yaml @@ -1102,57 +1102,6 @@ spec: --- apiVersion: apps/v1 kind: Deployment -metadata: - name: simplyblock-lvol-scheduler - namespace: {{ .Release.Namespace }} -spec: - replicas: 1 - selector: - matchLabels: - app: simplyblock-lvol-scheduler - template: - metadata: - annotations: - log-collector/enabled: "true" - reloader.stakater.com/auto: "true" - reloader.stakater.com/configmap: "simplyblock-fdb-cluster-config" - labels: - app: simplyblock-lvol-scheduler - spec: - hostNetwork: true - dnsPolicy: ClusterFirstWithHostNet - containers: - - name: lvol-scheduler - image: "{{ .Values.image.simplyblock.repository }}:{{ .Values.image.simplyblock.tag }}" - imagePullPolicy: "{{ .Values.image.simplyblock.pullPolicy }}" - command: ["python", "simplyblock_core/services/lvol_scheduler.py"] - env: - - name: SIMPLYBLOCK_LOG_LEVEL - valueFrom: - configMapKeyRef: - name: simplyblock-config - key: LOG_LEVEL - volumeMounts: - - name: fdb-cluster-file - mountPath: /etc/foundationdb/fdb.cluster - subPath: fdb.cluster - resources: - requests: - cpu: "100m" - memory: "256Mi" - limits: - cpu: "400m" - memory: "1Gi" - volumes: - - name: fdb-cluster-file - configMap: - name: simplyblock-fdb-cluster-config - items: - - key: cluster-file - path: fdb.cluster ---- -apiVersion: apps/v1 -kind: Deployment metadata: name: simplyblock-tasks-runner-sync-lvol-del namespace: {{ .Release.Namespace }} diff --git a/simplyblock_core/scripts/docker-compose-swarm.yml b/simplyblock_core/scripts/docker-compose-swarm.yml index 2fdda5059..97dd5ec8a 100644 --- a/simplyblock_core/scripts/docker-compose-swarm.yml +++ b/simplyblock_core/scripts/docker-compose-swarm.yml @@ -363,19 +363,6 @@ services: environment: SIMPLYBLOCK_LOG_LEVEL: "$LOG_LEVEL" - LVolScheduler: - <<: *service-base - image: $SIMPLYBLOCK_DOCKER_IMAGE - command: "python simplyblock_core/services/lvol_scheduler.py" - deploy: - placement: - constraints: [node.role == manager] - volumes: - - "/etc/foundationdb:/etc/foundationdb" - networks: - - hostnet - environment: - SIMPLYBLOCK_LOG_LEVEL: "$LOG_LEVEL" networks: monitoring-net: diff --git a/simplyblock_core/services/capacity_and_stats_collector.py b/simplyblock_core/services/capacity_and_stats_collector.py index 6f702d051..fdbabf9f6 100644 --- a/simplyblock_core/services/capacity_and_stats_collector.py +++ b/simplyblock_core/services/capacity_and_stats_collector.py @@ -7,9 +7,9 @@ from simplyblock_core.rpc_client import RPCClient from simplyblock_core.models.stats import DeviceStatObject, NodeStatObject, ClusterStatObject -logger = utils.get_logger(__name__) - +logger = utils.get_logger(__name__) +db = db_controller.DBController() last_object_record: dict[str, DeviceStatObject] = {} @@ -149,9 +149,36 @@ def add_cluster_stats(cl, records): return stat_obj +def add_lvol_scheduler_values(node: StorageNode): + node_used_size = 0 + lvols_subsystems = [] + for lvol in db.get_lvols_by_node_id(node.get_id()): + records = db.get_lvol_stats(lvol, 1) + if records: + node_used_size += records[0].size_used + if lvol.nqn not in lvols_subsystems: + lvols_subsystems.append(lvol.nqn) + for snap in db.get_snapshots_by_node_id(node.get_id()): + node_used_size += snap.used_size + + lvol_count_util = int(len(lvols_subsystems) / node.max_lvol * 100) + node_size_util = int(node_used_size / node.max_prov * 100) + + if lvol_count_util <= 0 or node_size_util <= 0: + return False + db_node = db.get_storage_node_by_id(node.get_id()) + db_node.lvol_count_util = lvol_count_util + db_node.node_size_util = node_size_util + db_node.write_to_db() + + # Once a node has > 90% of storage utilization, the largest lvol will be live migrated to another node + # (the one with small storage utilization) + if db_node.node_size_util > 90: + pass # todo: migration lvols to free space + + return True + -# get DB controller -db = db_controller.DBController() logger.info("Starting capacity and stats collector...") while True: @@ -199,6 +226,10 @@ def add_cluster_stats(cl, records): node_record = add_node_stats(node, devices_records) node_records.append(node_record) + ret = add_lvol_scheduler_values(node) + if not ret: + logger.warning("Failed to add lvol scheduler values") + add_cluster_stats(cl, node_records) time.sleep(constants.DEV_STAT_COLLECTOR_INTERVAL_SEC) diff --git a/simplyblock_core/services/lvol_scheduler.py b/simplyblock_core/services/lvol_scheduler.py deleted file mode 100644 index 38ee7f2e4..000000000 --- a/simplyblock_core/services/lvol_scheduler.py +++ /dev/null @@ -1,66 +0,0 @@ -# coding=utf-8 -""" -Filename: lvol_scheduler.py -Author: Hamdy Khader -Email: hamdy@simplyblock.io -Description: -LVol scheduler service will collect, calculate and store the required metric parameters for lvol scheduler algorithm -to make the following decisions: - - Which node should be used for hosting the next lvol? - - Is lvol transfer required because of high RAM consumption ? if so then which lvol/s to which node? - -Here we have the metric parameters used (per node): - ram_utilization - a timely reading of random memory utilization from file /etc/meminfo of the node - lvol_utilization - sum of the consumed data size (bytes) for all lvol on the node - lvol_count - count of lvols on the node -""" -import time - -from simplyblock_core import constants, db_controller, utils -from simplyblock_core.models.cluster import Cluster -from simplyblock_core.models.storage_node import StorageNode - -logger = utils.get_logger(__name__) - - -# get DB controller -db = db_controller.DBController() - -logger.info("Starting stats collector...") -while True: - - for cluster in db.get_clusters(): - - if cluster.status in [Cluster.STATUS_INACTIVE, Cluster.STATUS_UNREADY, Cluster.STATUS_IN_ACTIVATION]: - logger.warning(f"Cluster {cluster.get_id()} is in {cluster.status} state, skipping") - continue - - for snode in db.get_storage_nodes_by_cluster_id(cluster.get_id()): - - if snode.status not in [StorageNode.STATUS_ONLINE, StorageNode.STATUS_SUSPENDED, StorageNode.STATUS_DOWN]: - continue - - node_info = snode.snode_api().info() - ram_utilization= 0 - lvol_utilization = 0 - lvol_count = 0 - - # check for memory - if "memory_details" in node_info and node_info['memory_details']: - memory_details = node_info['memory_details'] - logger.info("Node Memory info") - logger.info(f"Total: {utils.humanbytes(memory_details['total'])}") - logger.info(f"Free: {utils.humanbytes(memory_details['free'])}") - ram_utilization = int(memory_details['free']/memory_details['total']*100) - - lvol_list = db.get_lvols_by_node_id(snode.get_id()) - lvol_count = len(lvol_list) - for lvol in lvol_list: - records = db.get_lvol_stats(lvol, 1) - if records: - lvol_utilization += records[0].size_used - - time.sleep(constants.LVOL_SCHEDULER_INTERVAL_SEC) From 6c363b909b5ccdc85dd1071bb1304fc309c4ddda Mon Sep 17 00:00:00 2001 From: hamdykhader Date: Tue, 2 Dec 2025 03:20:50 +0300 Subject: [PATCH 05/11] wip --- simplyblock_core/cluster_ops.py | 62 ++++++++----------- .../scripts/charts/templates/app_k8s.yaml | 51 --------------- .../scripts/docker-compose-swarm.yml | 1 - simplyblock_core/utils/__init__.py | 23 +------ 4 files changed, 27 insertions(+), 110 deletions(-) diff --git a/simplyblock_core/cluster_ops.py b/simplyblock_core/cluster_ops.py index f177f0327..24be657d7 100644 --- a/simplyblock_core/cluster_ops.py +++ b/simplyblock_core/cluster_ops.py @@ -1180,36 +1180,44 @@ def update_cluster(cluster_id, mgmt_only=False, restart=False, spdk_image=None, service_names.append(service.attrs['Spec']['Name']) if "app_SnapshotMonitor" not in service_names: - utils.create_docker_service( - cluster_docker=cluster_docker, - service_name="app_SnapshotMonitor", - service_file="python simplyblock_core/services/snapshot_monitor.py", - service_image=service_image) + logger.info("Creating snapshot monitor service") + cluster_docker.services.create( + image=service_image, + command="python simplyblock_core/services/snapshot_monitor.py", + name="app_SnapshotMonitor", + mounts=["/etc/foundationdb:/etc/foundationdb"], + env=["SIMPLYBLOCK_LOG_LEVEL=DEBUG"], + networks=["host"], + constraints=["node.role == manager"] + ) if "app_TasksRunnerLVolSyncDelete" not in service_names: - utils.create_docker_service( - cluster_docker=cluster_docker, - service_name="app_TasksRunnerLVolSyncDelete", - service_file="python simplyblock_core/services/tasks_runner_sync_lvol_del.py", - service_image=service_image) - + logger.info("Creating lvol sync delete service") + cluster_docker.services.create( + image=service_image, + command="python simplyblock_core/services/tasks_runner_sync_lvol_del.py", + name="app_TasksRunnerLVolSyncDelete", + mounts=["/etc/foundationdb:/etc/foundationdb"], + env=["SIMPLYBLOCK_LOG_LEVEL=DEBUG"], + networks=["host"], + constraints=["node.role == manager"] + ) logger.info("Done updating mgmt cluster") elif cluster.mode == "kubernetes": utils.load_kube_config_with_fallback() apps_v1 = k8s_client.AppsV1Api() - namespace = constants.K8S_NAMESPACE + image_without_tag = constants.SIMPLY_BLOCK_DOCKER_IMAGE.split(":")[0] image_parts = "/".join(image_without_tag.split("/")[-2:]) service_image = mgmt_image or constants.SIMPLY_BLOCK_DOCKER_IMAGE - deployment_names = [] + # Update Deployments - deployments = apps_v1.list_namespaced_deployment(namespace=namespace) + deployments = apps_v1.list_namespaced_deployment(namespace=constants.K8S_NAMESPACE) for deploy in deployments.items: if deploy.metadata.name == constants.ADMIN_DEPLOY_NAME: logger.info(f"Skipping deployment {deploy.metadata.name}") continue - deployment_names.append(deploy.metadata.name) for c in deploy.spec.template.spec.containers: if image_parts in c.image: logger.info(f"Updating deployment {deploy.metadata.name} image to {service_image}") @@ -1219,30 +1227,12 @@ def update_cluster(cluster_id, mgmt_only=False, restart=False, spdk_image=None, deploy.spec.template.metadata.annotations = annotations apps_v1.patch_namespaced_deployment( name=deploy.metadata.name, - namespace=namespace, + namespace=constants.K8S_NAMESPACE, body={"spec": {"template": deploy.spec.template}} ) - if f"{namespace}-tasks-runner-sync-lvol-del" not in deployment_names: - utils.create_k8s_service( - k8s_apps_client=apps_v1, - namespace=namespace, - deployment_name=f"{namespace}-tasks-runner-sync-lvol-del", - container_name="tasks-runner-sync-lvol-del", - service_file="simplyblock_core/services/tasks_runner_sync_lvol_del.py", - container_image=service_image) - - if f"{namespace}-snapshot-monitor" not in deployment_names: - utils.create_k8s_service( - k8s_apps_client=apps_v1, - namespace=namespace, - deployment_name=f"{namespace}-snapshot-monitor", - container_name="snapshot-monitor", - service_file="simplyblock_core/services/snapshot_monitor.py", - container_image=service_image) - # Update DaemonSets - daemonsets = apps_v1.list_namespaced_daemon_set(namespace=namespace) + daemonsets = apps_v1.list_namespaced_daemon_set(namespace=constants.K8S_NAMESPACE) for ds in daemonsets.items: for c in ds.spec.template.spec.containers: if image_parts in c.image: @@ -1253,7 +1243,7 @@ def update_cluster(cluster_id, mgmt_only=False, restart=False, spdk_image=None, ds.spec.template.metadata.annotations = annotations apps_v1.patch_namespaced_daemon_set( name=ds.metadata.name, - namespace=namespace, + namespace=constants.K8S_NAMESPACE, body={"spec": {"template": ds.spec.template}} ) diff --git a/simplyblock_core/scripts/charts/templates/app_k8s.yaml b/simplyblock_core/scripts/charts/templates/app_k8s.yaml index 49c7490b7..d17ea092a 100644 --- a/simplyblock_core/scripts/charts/templates/app_k8s.yaml +++ b/simplyblock_core/scripts/charts/templates/app_k8s.yaml @@ -1100,57 +1100,6 @@ spec: - key: cluster-file path: fdb.cluster --- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: simplyblock-tasks-runner-sync-lvol-del - namespace: {{ .Release.Namespace }} -spec: - replicas: 1 - selector: - matchLabels: - app: simplyblock-tasks-runner-sync-lvol-del - template: - metadata: - annotations: - log-collector/enabled: "true" - reloader.stakater.com/auto: "true" - reloader.stakater.com/configmap: "simplyblock-fdb-cluster-config" - labels: - app: simplyblock-tasks-runner-sync-lvol-del - spec: - hostNetwork: true - dnsPolicy: ClusterFirstWithHostNet - containers: - - name: tasks-runner-sync-lvol-del - image: "{{ .Values.image.simplyblock.repository }}:{{ .Values.image.simplyblock.tag }}" - imagePullPolicy: "{{ .Values.image.simplyblock.pullPolicy }}" - command: ["python", "simplyblock_core/services/tasks_runner_sync_lvol_del.py"] - env: - - name: SIMPLYBLOCK_LOG_LEVEL - valueFrom: - configMapKeyRef: - name: simplyblock-config - key: LOG_LEVEL - volumeMounts: - - name: fdb-cluster-file - mountPath: /etc/foundationdb/fdb.cluster - subPath: fdb.cluster - resources: - requests: - cpu: "200m" - memory: "256Mi" - limits: - cpu: "400m" - memory: "1Gi" - volumes: - - name: fdb-cluster-file - configMap: - name: simplyblock-fdb-cluster-config - items: - - key: cluster-file - path: fdb.cluster ---- apiVersion: apps/v1 kind: DaemonSet diff --git a/simplyblock_core/scripts/docker-compose-swarm.yml b/simplyblock_core/scripts/docker-compose-swarm.yml index 97dd5ec8a..fd79f43c1 100644 --- a/simplyblock_core/scripts/docker-compose-swarm.yml +++ b/simplyblock_core/scripts/docker-compose-swarm.yml @@ -363,7 +363,6 @@ services: environment: SIMPLYBLOCK_LOG_LEVEL: "$LOG_LEVEL" - networks: monitoring-net: external: true diff --git a/simplyblock_core/utils/__init__.py b/simplyblock_core/utils/__init__.py index c1bac3c4f..96a00ecac 100644 --- a/simplyblock_core/utils/__init__.py +++ b/simplyblock_core/utils/__init__.py @@ -12,9 +12,8 @@ import time import socket from typing import Union, Any, Optional, Tuple -from docker import DockerClient from kubernetes import client, config -from kubernetes.client import ApiException, AppsV1Api +from kubernetes.client import ApiException import docker from prettytable import PrettyTable from docker.errors import APIError, DockerException, ImageNotFound, NotFound @@ -2082,23 +2081,3 @@ def patch_prometheus_configmap(username: str, password: str): except Exception as e: logger.error(f"Unexpected error while patching ConfigMap: {e}") return False - - -def create_docker_service(cluster_docker: DockerClient, service_name: str, service_file: str, service_image: str): - logger.info(f"Creating service: {service_name}") - cluster_docker.services.create( - image=service_image, - command=service_file, - name=service_name, - mounts=["/etc/foundationdb:/etc/foundationdb"], - env=["SIMPLYBLOCK_LOG_LEVEL=DEBUG"], - networks=["host"], - constraints=["node.role == manager"] - ) - - -def create_k8s_service(k8s_apps_client: AppsV1Api, namespace: str, deployment_name: str, - container_name: str, service_file: str, container_image: str): - # TODO(Geoffrey): Add implementation to create a service on k8s to support cluster update from older version - logger.info(f"Creating deployment: {deployment_name}") - pass From c621a9af068d1c40f03e5acdcc0257c318bff951 Mon Sep 17 00:00:00 2001 From: hamdykhader Date: Tue, 2 Dec 2025 17:35:31 +0300 Subject: [PATCH 06/11] continue lvol new sch impl --- simplyblock_core/constants.py | 9 -- .../controllers/lvol_controller.py | 90 +++++++++++-------- simplyblock_core/utils/__init__.py | 6 +- 3 files changed, 56 insertions(+), 49 deletions(-) diff --git a/simplyblock_core/constants.py b/simplyblock_core/constants.py index 8d33597c4..ea970cec5 100644 --- a/simplyblock_core/constants.py +++ b/simplyblock_core/constants.py @@ -61,15 +61,6 @@ def get_config_var(name, default=None): CLUSTER_NQN = "nqn.2023-02.io.simplyblock" -weights = { - "lvol": 100, - # "cpu": 10, - # "r_io": 10, - # "w_io": 10, - # "r_b": 10, - # "w_b": 10 -} - HEALTH_CHECK_INTERVAL_SEC = 30 diff --git a/simplyblock_core/controllers/lvol_controller.py b/simplyblock_core/controllers/lvol_controller.py index 26ab3970b..b0d5e0031 100644 --- a/simplyblock_core/controllers/lvol_controller.py +++ b/simplyblock_core/controllers/lvol_controller.py @@ -135,58 +135,74 @@ def validate_add_lvol_func(name, size, host_id_or_name, pool_id_or_name, def _get_next_3_nodes(cluster_id, lvol_size=0): db_controller = DBController() node_stats = {} - all_online_nodes = [] + nodes_below_25 = [] nodes_between_25_75 = [] nodes_above_75 = [] for node in db_controller.get_storage_nodes_by_cluster_id(cluster_id): if node.is_secondary_node: # pass continue - if node.status == node.STATUS_ONLINE: - - """ - if sum of lvols (+snapshots, including namespace lvols) per node is utilized < [0.25 * max-size] AND - number of lvols (snapshots dont count extra and namspaces on same subsystem count only once) < [0.25 * max-lvol] - - --> simply round-robin schedule lvols (no randomization, no weights) - BUT: if storage utilization > 25% on one or more nodes, those nodes are excluded from round robin - - - - - - Once all nodes have > 25% of storage utilization, we weight - (relative-number-of-lvol-compared-to-total-number + relative-utilization-compared-to-total-utilzation) - --> and based on the weight just a random location - - Once a node has > 75% uof storage utilization, it is excluded to add new lvols - (unless all nodes exceed this limit, than it is weighted again) - - - """ - all_online_nodes.append(node) - if node.node_size_util < 25: - continue + nodes_below_25.append(node) elif node.node_size_util >= 75: nodes_above_75.append(node) else: nodes_between_25_75.append(node) - node_st = { - "node_size_util": node.node_size_util , - "lvol_count_util": node.lvol_count_util , - } - - node_stats[node.get_id()] = node_st - if len(node_stats) <= 1: - return all_online_nodes - + return nodes_below_25+nodes_between_25_75+nodes_above_75 + + if len(nodes_below_25) > len(nodes_between_25_75): + """ + if sum of lvols (+snapshots, including namespace lvols) per node is utilized < [0.25 * max-size] AND + number of lvols (snapshots dont count extra and namspaces on same subsystem count only once) < [0.25 * max-lvol] + + --> simply round-robin schedule lvols (no randomization, no weights) + BUT: if storage utilization > 25% on one or more nodes, those nodes are excluded from round robin + + """ + + for node in nodes_below_25: + node_stats[node.get_id()] = node.lvol_count_util + + sorted_keys = list(node_stats.keys()) + sorted_keys.sort() + sorted_nodes = [] + for k in sorted_keys: + for node in nodes_below_25: + if node.lvol_count_util == k: + sorted_nodes.append(node) + + return sorted_nodes + + elif len(nodes_between_25_75) > len(nodes_above_75): + """ + Once all nodes have > 25% of storage utilization, we weight + (relative-number-of-lvol-compared-to-total-number + relative-utilization-compared-to-total-utilzation) + --> and based on the weight just a random location + """ + for node in nodes_between_25_75: + node_stats[node.get_id()] = { + "lvol_count_util": node.lvol_count_util, + "node_size_util": node.node_size_util} + + elif len(nodes_below_25) == 0 and len(nodes_between_25_75) == 0 and len(nodes_above_75) > 0 : + """ + Once a node has > 75% uof storage utilization, it is excluded to add new lvols + (unless all nodes exceed this limit, than it is weighted again) + """ + for node in nodes_above_75: + node_stats[node.get_id()] = { + "lvol_count_util": node.lvol_count_util, + "node_size_util": node.node_size_util} + + + keys_weights = { + "lvol_count_util": 50, + "node_size_util": 50} cluster_stats = utils.dict_agg([node_stats[k] for k in node_stats]) - - nodes_weight = utils.get_weights(node_stats, cluster_stats) + nodes_weight = utils.get_weights(node_stats, cluster_stats, keys_weights) node_start_end = {} n_start = 0 diff --git a/simplyblock_core/utils/__init__.py b/simplyblock_core/utils/__init__.py index 7bc2fa112..5a81fd37f 100644 --- a/simplyblock_core/utils/__init__.py +++ b/simplyblock_core/utils/__init__.py @@ -229,7 +229,7 @@ def dict_agg(data, mean=False, keys=None): return out -def get_weights(node_stats, cluster_stats): +def get_weights(node_stats, cluster_stats, keys_weights): """" node_st = { "lvol": len(node.lvols), @@ -241,8 +241,8 @@ def get_weights(node_stats, cluster_stats): """ def _normalize_w(key, v): - if key in constants.weights: - return round(((v * constants.weights[key]) / 100), 2) + if key in keys_weights: + return round(((v * keys_weights[key]) / 100), 2) else: return v From a0005ffcaac862f704613b5238c200073ce4700a Mon Sep 17 00:00:00 2001 From: hamdykhader Date: Wed, 10 Dec 2025 18:58:22 +0300 Subject: [PATCH 07/11] wip --- .../controllers/lvol_controller.py | 34 ++++++++----------- simplyblock_core/utils/__init__.py | 2 +- 2 files changed, 16 insertions(+), 20 deletions(-) diff --git a/simplyblock_core/controllers/lvol_controller.py b/simplyblock_core/controllers/lvol_controller.py index b0d5e0031..deecc57bb 100644 --- a/simplyblock_core/controllers/lvol_controller.py +++ b/simplyblock_core/controllers/lvol_controller.py @@ -134,7 +134,7 @@ def validate_add_lvol_func(name, size, host_id_or_name, pool_id_or_name, def _get_next_3_nodes(cluster_id, lvol_size=0): db_controller = DBController() - node_stats = {} + node_stats: dict[str: dict] = {} nodes_below_25 = [] nodes_between_25_75 = [] nodes_above_75 = [] @@ -150,7 +150,11 @@ def _get_next_3_nodes(cluster_id, lvol_size=0): else: nodes_between_25_75.append(node) - if len(node_stats) <= 1: + logger.info(f"nodes_below_25: {nodes_below_25}") + logger.info(f"nodes_between_25_75: {nodes_between_25_75}") + logger.info(f"nodes_above_75: {nodes_above_75}") + + if len(nodes_below_25+nodes_between_25_75+nodes_above_75) <= 1: return nodes_below_25+nodes_between_25_75+nodes_above_75 if len(nodes_below_25) > len(nodes_between_25_75): @@ -217,19 +221,14 @@ def _get_next_3_nodes(cluster_id, lvol_size=0): for node_id in node_start_end: node_start_end[node_id]['%'] = int(node_start_end[node_id]['weight'] * 100 / n_start) - ############# log - print("Node stats") - utils.print_table_dict({**node_stats, "Cluster": cluster_stats}) - print("Node weights") - utils.print_table_dict({**nodes_weight, "weights": {"lvol": n_start, "total": n_start}}) - print("Node selection range") - utils.print_table_dict(node_start_end) - ############# + logger.info(f"Node stats: \n {utils.print_table_dict({**node_stats, 'Cluster': cluster_stats})}") + logger.info(f"Node weights: \n {utils.print_table_dict({**nodes_weight, 'weights': {'lvol': n_start, 'total': n_start}})}") + logger.info(f"Node selection range: \n {utils.print_table_dict(node_start_end)}") selected_node_ids: List[str] = [] while len(selected_node_ids) < min(len(node_stats), 3): r_index = random.randint(0, n_start) - print(f"Random is {r_index}/{n_start}") + logger.info(f"Random is {r_index}/{n_start}") for node_id in node_start_end: if node_start_end[node_id]['start'] <= r_index <= node_start_end[node_id]['end']: if node_id not in selected_node_ids: @@ -250,14 +249,11 @@ def _get_next_3_nodes(cluster_id, lvol_size=0): break ret = [] - if selected_node_ids: - for node_id in selected_node_ids: - node = db_controller.get_storage_node_by_id(node_id) - print(f"Selected node: {node_id}, {node.hostname}") - ret.append(node) - return ret - else: - return all_online_nodes + for node_id in selected_node_ids: + node = db_controller.get_storage_node_by_id(node_id) + logger.info(f"Selected node: {node_id}, {node.hostname}") + ret.append(node) + return ret def is_hex(s: str) -> bool: """ diff --git a/simplyblock_core/utils/__init__.py b/simplyblock_core/utils/__init__.py index 5a81fd37f..fcab968bc 100644 --- a/simplyblock_core/utils/__init__.py +++ b/simplyblock_core/utils/__init__.py @@ -282,7 +282,7 @@ def print_table_dict(node_stats): data = {"node_id": node_id} data.update(node_stats[node_id]) d.append(data) - print(print_table(d)) + return str(print_table(d)) def generate_rpc_user_and_pass(): From f69ae9518db28b5cdfafe36873fcdd78e416b6a0 Mon Sep 17 00:00:00 2001 From: hamdykhader Date: Wed, 10 Dec 2025 23:24:28 +0300 Subject: [PATCH 08/11] Adds unit tests for lvol scheduler --- .github/workflows/python-checks.yml | 2 +- requirements.txt | 1 + .../controllers/lvol_controller.py | 18 +++--- simplyblock_core/models/storage_node.py | 4 +- simplyblock_core/test/test_lvol_scheduler.py | 64 +++++++++++++++++++ simplyblock_core/test/test_utils.py | 38 +++++++++++ simplyblock_core/utils/__init__.py | 8 --- 7 files changed, 115 insertions(+), 20 deletions(-) create mode 100644 simplyblock_core/test/test_lvol_scheduler.py diff --git a/.github/workflows/python-checks.yml b/.github/workflows/python-checks.yml index 82e3c09cb..ffa5a3657 100644 --- a/.github/workflows/python-checks.yml +++ b/.github/workflows/python-checks.yml @@ -55,7 +55,7 @@ jobs: run: pip install . - name: Run tests - run: pytest --import-mode=importlib -v + run: pytest --import-mode=importlib --tb=short --capture=no lint: diff --git a/requirements.txt b/requirements.txt index 9ee458f00..0bfcf7035 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,6 +5,7 @@ docker psutil py-cpuinfo pytest +pytest-mock mock flask kubernetes diff --git a/simplyblock_core/controllers/lvol_controller.py b/simplyblock_core/controllers/lvol_controller.py index deecc57bb..291da7f73 100644 --- a/simplyblock_core/controllers/lvol_controller.py +++ b/simplyblock_core/controllers/lvol_controller.py @@ -150,14 +150,14 @@ def _get_next_3_nodes(cluster_id, lvol_size=0): else: nodes_between_25_75.append(node) - logger.info(f"nodes_below_25: {nodes_below_25}") - logger.info(f"nodes_between_25_75: {nodes_between_25_75}") - logger.info(f"nodes_above_75: {nodes_above_75}") + logger.info(f"nodes_below_25: {len(nodes_below_25)}") + logger.info(f"nodes_between_25_75: {len(nodes_between_25_75)}") + logger.info(f"nodes_above_75: {len(nodes_above_75)}") if len(nodes_below_25+nodes_between_25_75+nodes_above_75) <= 1: return nodes_below_25+nodes_between_25_75+nodes_above_75 - if len(nodes_below_25) > len(nodes_between_25_75): + if len(nodes_below_25) > len(nodes_between_25_75) and len(nodes_below_25) > len(nodes_above_75): """ if sum of lvols (+snapshots, including namespace lvols) per node is utilized < [0.25 * max-size] AND number of lvols (snapshots dont count extra and namspaces on same subsystem count only once) < [0.25 * max-lvol] @@ -170,14 +170,14 @@ def _get_next_3_nodes(cluster_id, lvol_size=0): for node in nodes_below_25: node_stats[node.get_id()] = node.lvol_count_util - sorted_keys = list(node_stats.keys()) + sorted_keys = list(node_stats.values()) sorted_keys.sort() sorted_nodes = [] for k in sorted_keys: for node in nodes_below_25: if node.lvol_count_util == k: - sorted_nodes.append(node) - + if node not in sorted_nodes: + sorted_nodes.append(node) return sorted_nodes elif len(nodes_between_25_75) > len(nodes_above_75): @@ -191,7 +191,7 @@ def _get_next_3_nodes(cluster_id, lvol_size=0): "lvol_count_util": node.lvol_count_util, "node_size_util": node.node_size_util} - elif len(nodes_below_25) == 0 and len(nodes_between_25_75) == 0 and len(nodes_above_75) > 0 : + elif len(nodes_below_25) < len(nodes_above_75) and len(nodes_between_25_75) < len(nodes_above_75) : """ Once a node has > 75% uof storage utilization, it is excluded to add new lvols (unless all nodes exceed this limit, than it is weighted again) @@ -222,7 +222,7 @@ def _get_next_3_nodes(cluster_id, lvol_size=0): node_start_end[node_id]['%'] = int(node_start_end[node_id]['weight'] * 100 / n_start) logger.info(f"Node stats: \n {utils.print_table_dict({**node_stats, 'Cluster': cluster_stats})}") - logger.info(f"Node weights: \n {utils.print_table_dict({**nodes_weight, 'weights': {'lvol': n_start, 'total': n_start}})}") + logger.info(f"Node weights: \n {utils.print_table_dict({**nodes_weight})}") logger.info(f"Node selection range: \n {utils.print_table_dict(node_start_end)}") selected_node_ids: List[str] = [] diff --git a/simplyblock_core/models/storage_node.py b/simplyblock_core/models/storage_node.py index 832d37048..99a30502c 100644 --- a/simplyblock_core/models/storage_node.py +++ b/simplyblock_core/models/storage_node.py @@ -103,8 +103,8 @@ class StorageNode(BaseNodeObject): hublvol: HubLVol = None # type: ignore[assignment] active_tcp: bool = True active_rdma: bool = False - lvol_count_util = 0 - node_size_util = 0 + lvol_count_util: int = 0 + node_size_util: int = 0 def rpc_client(self, **kwargs): """Return rpc client to this node diff --git a/simplyblock_core/test/test_lvol_scheduler.py b/simplyblock_core/test/test_lvol_scheduler.py new file mode 100644 index 000000000..4779de6cc --- /dev/null +++ b/simplyblock_core/test/test_lvol_scheduler.py @@ -0,0 +1,64 @@ +from simplyblock_core.test import test_utils + + +def test_scheduler_below_25(): + print("\ntest_scheduler_below_25") + testing_nodes_25 = [ + [ + {"uuid": "123", "node_size_util": 20, "lvol_count_util": 10}, + {"uuid": "456", "node_size_util": 20, "lvol_count_util": 10}, + {"uuid": "789", "node_size_util": 20, "lvol_count_util": 10} + ], + [ + {"uuid": "123", "node_size_util": 10, "lvol_count_util": 10}, + {"uuid": "456", "node_size_util": 15, "lvol_count_util": 50}, + {"uuid": "789", "node_size_util": 26, "lvol_count_util": 100} + ], + ] + test_utils.run_lvol_scheduler_test(testing_nodes_25) + + +def test_scheduler_25_75(): + print("\ntest_scheduler_25_75") + testing_nodes_25_75 = [ + [ + {"uuid": "123", "node_size_util": 25, "lvol_count_util": 10}, + {"uuid": "456", "node_size_util": 25, "lvol_count_util": 10}, + {"uuid": "789", "node_size_util": 25, "lvol_count_util": 10} + ], + [ + {"uuid": "123", "node_size_util": 75, "lvol_count_util": 10}, + {"uuid": "456", "node_size_util": 75, "lvol_count_util": 50}, + {"uuid": "789", "node_size_util": 75, "lvol_count_util": 20} + ], + [ + {"uuid": "123", "node_size_util": 20, "lvol_count_util": 10}, + {"uuid": "456", "node_size_util": 50, "lvol_count_util": 50}, + {"uuid": "789", "node_size_util": 50, "lvol_count_util": 70} + ], + ] + test_utils.run_lvol_scheduler_test(testing_nodes_25_75) + + + +def test_scheduler_75(): + print("\ntest_scheduler_75") + testing_nodes_25_75 = [ + [ + {"uuid": "123", "node_size_util": 75, "lvol_count_util": 10}, + {"uuid": "456", "node_size_util": 80, "lvol_count_util": 10}, + {"uuid": "789", "node_size_util": 85, "lvol_count_util": 10} + ], + [ + {"uuid": "123", "node_size_util": 60, "lvol_count_util": 10}, + {"uuid": "456", "node_size_util": 80, "lvol_count_util": 10}, + {"uuid": "789", "node_size_util": 90, "lvol_count_util": 5} + ], + [ + {"uuid": "123", "node_size_util": 20, "lvol_count_util": 10}, + {"uuid": "456", "node_size_util": 80, "lvol_count_util": 20}, + {"uuid": "789", "node_size_util": 85, "lvol_count_util": 5} + ], + ] + test_utils.run_lvol_scheduler_test(testing_nodes_25_75) + diff --git a/simplyblock_core/test/test_utils.py b/simplyblock_core/test/test_utils.py index da22a73ba..90d3a70ef 100644 --- a/simplyblock_core/test/test_utils.py +++ b/simplyblock_core/test/test_utils.py @@ -1,9 +1,13 @@ from typing import ContextManager import pytest +from unittest.mock import patch from simplyblock_core import utils from simplyblock_core.utils import helpers, parse_thread_siblings_list +from simplyblock_core.controllers import lvol_controller +from simplyblock_core.db_controller import DBController +from simplyblock_core.models.storage_node import StorageNode @pytest.mark.parametrize('args,expected', [ @@ -146,3 +150,37 @@ def test_parse_thread_siblings_list(input, expected): parse_thread_siblings_list(input) else: assert parse_thread_siblings_list(input) == expected + + +@patch.object(DBController, 'get_storage_nodes_by_cluster_id') +@patch.object(DBController, 'get_storage_node_by_id') +def run_lvol_scheduler_test(testing_nodes, db_controller_get_storage_node_by_id, db_controller_get_storage_nodes_by_cluster_id): + RUN_PER_TEST = 1000 + print("-" * 100) + for testing_map in testing_nodes: + nodes = {n['uuid']: n for n in testing_map} + def get_node_by_id(node_id): + for node in testing_map: + if node['uuid'] == node_id: + return StorageNode({"status": StorageNode.STATUS_ONLINE, **node}) + db_controller_get_storage_node_by_id.side_effect = get_node_by_id + db_controller_get_storage_nodes_by_cluster_id.return_value = [ + StorageNode({"status": StorageNode.STATUS_ONLINE, **node_params}) for node_params in testing_map] + cluster_id = "cluster_id" + out = {} + total = RUN_PER_TEST + for i in range(total): + selected_nodes = lvol_controller._get_next_3_nodes(cluster_id) + for index, node in enumerate(selected_nodes): + if node.get_id() not in out: + out[node.get_id()] = {f"{index}": 1} + else: + out[node.get_id()][f"{index}"] = out[node.get_id()].get(f"{index}", 0) + 1 + # assert len(nodes) == 3 + + for k, v in out.items(): + print(f"node {k}: size_util={nodes[k]['node_size_util']} lvols_util={nodes[k]['lvol_count_util']} stats: {[f'{sk}: {int(v[sk]/total*100)}%' for sk in sorted(v.keys())]}") + for v in testing_map: + if v['uuid'] not in out: + print(f"node {v['uuid']}: size_util={v['node_size_util']} lvols_util={v['lvol_count_util']} stats: excluded") + print("-"*100) diff --git a/simplyblock_core/utils/__init__.py b/simplyblock_core/utils/__init__.py index fcab968bc..58daaa19e 100644 --- a/simplyblock_core/utils/__init__.py +++ b/simplyblock_core/utils/__init__.py @@ -255,8 +255,6 @@ def _get_key_w(node_id, key): return w out: dict = {} - heavy_node_w = 0 - heavy_node_id = None for node_id in node_stats: out[node_id] = {} total = 0 @@ -266,12 +264,6 @@ def _get_key_w(node_id, key): out[node_id][key] = w total += w out[node_id]['total'] = int(total) - if total > heavy_node_w: - heavy_node_w = total - heavy_node_id = node_id - - if heavy_node_id: - out[heavy_node_id]['total'] *= 5 return out From c94282b97952194678a2f451c57bf0dcdf3013d8 Mon Sep 17 00:00:00 2001 From: hamdykhader Date: Wed, 10 Dec 2025 23:26:07 +0300 Subject: [PATCH 09/11] use 1m test run --- simplyblock_core/test/test_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/simplyblock_core/test/test_utils.py b/simplyblock_core/test/test_utils.py index 90d3a70ef..02a1b15e5 100644 --- a/simplyblock_core/test/test_utils.py +++ b/simplyblock_core/test/test_utils.py @@ -155,7 +155,7 @@ def test_parse_thread_siblings_list(input, expected): @patch.object(DBController, 'get_storage_nodes_by_cluster_id') @patch.object(DBController, 'get_storage_node_by_id') def run_lvol_scheduler_test(testing_nodes, db_controller_get_storage_node_by_id, db_controller_get_storage_nodes_by_cluster_id): - RUN_PER_TEST = 1000 + RUN_PER_TEST = 1000000 print("-" * 100) for testing_map in testing_nodes: nodes = {n['uuid']: n for n in testing_map} From 497f415a875bea4097e3437d8ca6ead1e90a0d80 Mon Sep 17 00:00:00 2001 From: hamdykhader Date: Wed, 10 Dec 2025 23:32:31 +0300 Subject: [PATCH 10/11] use 100k test run --- simplyblock_core/controllers/lvol_controller.py | 2 +- simplyblock_core/test/test_utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/simplyblock_core/controllers/lvol_controller.py b/simplyblock_core/controllers/lvol_controller.py index 291da7f73..0b674ae23 100644 --- a/simplyblock_core/controllers/lvol_controller.py +++ b/simplyblock_core/controllers/lvol_controller.py @@ -134,7 +134,7 @@ def validate_add_lvol_func(name, size, host_id_or_name, pool_id_or_name, def _get_next_3_nodes(cluster_id, lvol_size=0): db_controller = DBController() - node_stats: dict[str: dict] = {} + node_stats: dict = {} nodes_below_25 = [] nodes_between_25_75 = [] nodes_above_75 = [] diff --git a/simplyblock_core/test/test_utils.py b/simplyblock_core/test/test_utils.py index 02a1b15e5..8d67ea5a0 100644 --- a/simplyblock_core/test/test_utils.py +++ b/simplyblock_core/test/test_utils.py @@ -155,7 +155,7 @@ def test_parse_thread_siblings_list(input, expected): @patch.object(DBController, 'get_storage_nodes_by_cluster_id') @patch.object(DBController, 'get_storage_node_by_id') def run_lvol_scheduler_test(testing_nodes, db_controller_get_storage_node_by_id, db_controller_get_storage_nodes_by_cluster_id): - RUN_PER_TEST = 1000000 + RUN_PER_TEST = 10000 print("-" * 100) for testing_map in testing_nodes: nodes = {n['uuid']: n for n in testing_map} From f6b697d96d2e42c50701ea0b4c1f0d837c3f8390 Mon Sep 17 00:00:00 2001 From: hamdykhader Date: Wed, 10 Dec 2025 23:39:59 +0300 Subject: [PATCH 11/11] fix linter --- e2e/continuous_log_collector.py | 1 - e2e/e2e_tests/cluster_test_base.py | 2 +- e2e/stress_test/continuous_failover_ha_multi_client.py | 4 ++-- .../continuous_failover_ha_multi_client_quick_outage.py | 3 +-- e2e/utils/ssh_utils.py | 3 ++- 5 files changed, 6 insertions(+), 7 deletions(-) diff --git a/e2e/continuous_log_collector.py b/e2e/continuous_log_collector.py index 96b157760..d1ea68c38 100644 --- a/e2e/continuous_log_collector.py +++ b/e2e/continuous_log_collector.py @@ -1,6 +1,5 @@ import os from datetime import datetime -from pathlib import Path from utils.ssh_utils import SshUtils, RunnerK8sLog from logger_config import setup_logger diff --git a/e2e/e2e_tests/cluster_test_base.py b/e2e/e2e_tests/cluster_test_base.py index 15743725b..d37222c88 100644 --- a/e2e/e2e_tests/cluster_test_base.py +++ b/e2e/e2e_tests/cluster_test_base.py @@ -405,7 +405,7 @@ def collect_management_details(self, post_teardown=False): self.ssh_obj.exec_command(self.mgmt_nodes[0], cmd) node+=1 - all_nodes = self.storage_nodes + self.mgmt_nodes + self.client_machines: + all_nodes = self.storage_nodes + self.mgmt_nodes + self.client_machines for node in all_nodes: base_path = os.path.join(self.docker_logs_path, node) cmd = f"journalctl -k --no-tail >& {base_path}/jounalctl_{node}-final.txt" diff --git a/e2e/stress_test/continuous_failover_ha_multi_client.py b/e2e/stress_test/continuous_failover_ha_multi_client.py index a97c42676..0f0c9f94e 100644 --- a/e2e/stress_test/continuous_failover_ha_multi_client.py +++ b/e2e/stress_test/continuous_failover_ha_multi_client.py @@ -329,7 +329,7 @@ def perform_random_outage(self): for node in self.sn_nodes_with_sec: # self.ssh_obj.dump_lvstore(node_ip=self.mgmt_nodes[0], # storage_node_id=node) - self.logger.info(f"Skipping lvstore dump!!") + self.logger.info("Skipping lvstore dump!!") for node in self.sn_nodes_with_sec: cur_node_details = self.sbcli_utils.get_storage_node_details(node) cur_node_ip = cur_node_details[0]["mgmt_ip"] @@ -663,7 +663,7 @@ def restart_nodes_after_failover(self, outage_type, restart=False): for node in self.sn_nodes_with_sec: # self.ssh_obj.dump_lvstore(node_ip=self.mgmt_nodes[0], # storage_node_id=node) - self.logger.info(f"Skipping lvstore dump!!") + self.logger.info("Skipping lvstore dump!!") def create_snapshots_and_clones(self): """Create snapshots and clones during an outage.""" diff --git a/e2e/stress_test/continuous_failover_ha_multi_client_quick_outage.py b/e2e/stress_test/continuous_failover_ha_multi_client_quick_outage.py index afa98b055..c2c1051a2 100644 --- a/e2e/stress_test/continuous_failover_ha_multi_client_quick_outage.py +++ b/e2e/stress_test/continuous_failover_ha_multi_client_quick_outage.py @@ -306,7 +306,7 @@ def _seed_snapshots_and_clones(self): if err: nqn = self.sbcli_utils.get_lvol_details(lvol_id=self.clone_mount_details[clone_name]["ID"])[0]["nqn"] self.ssh_obj.disconnect_nvme(node=client, nqn_grep=nqn) - self.logger.info(f"[LFNG] connect clone error → cleanup") + self.logger.info("[LFNG] connect clone error → cleanup") self.sbcli_utils.delete_lvol(lvol_name=clone_name, max_attempt=20, skip_error=True) sleep_n_sec(3) del self.clone_mount_details[clone_name] @@ -431,7 +431,6 @@ def _perform_outage(self): return outage_type def restart_nodes_after_failover(self, outage_type): - node_details = self.sbcli_utils.get_storage_node_details(self.current_outage_node) self.logger.info(f"[LFNG] Recover outage={outage_type} node={self.current_outage_node}") diff --git a/e2e/utils/ssh_utils.py b/e2e/utils/ssh_utils.py index ee265d507..a50a61726 100644 --- a/e2e/utils/ssh_utils.py +++ b/e2e/utils/ssh_utils.py @@ -2891,7 +2891,8 @@ def stop_log_monitor(self): print("K8s log monitor thread stopped.") def _rid(n=6): - import string, random + import string + import random letters = string.ascii_uppercase digits = string.digits return random.choice(letters) + ''.join(random.choices(letters + digits, k=n-1))