From 92d5f0b4db0e036cde79b6619623730583376b25 Mon Sep 17 00:00:00 2001 From: "Christoph Engelbert (noctarius)" Date: Fri, 25 Jul 2025 10:51:09 +0200 Subject: [PATCH] Added control center deployment on k8s control planes --- simplyblock_cli/cli-reference.yaml | 7 +++ simplyblock_cli/cli.py | 3 + simplyblock_cli/clibase.py | 4 +- simplyblock_core/cluster_ops.py | 13 ++++- simplyblock_core/scripts/__init__.py | 6 +- .../charts/templates/app_configmap.yaml | 21 ++++++- .../scripts/charts/templates/app_k8s.yaml | 55 +++++++++++++++++++ .../scripts/charts/templates/app_secret.yaml | 8 +++ .../scripts/charts/values-template.yaml | 8 +++ simplyblock_core/scripts/deploy_k8s_stack.sh | 4 ++ 10 files changed, 121 insertions(+), 8 deletions(-) create mode 100644 simplyblock_core/scripts/charts/templates/app_secret.yaml diff --git a/simplyblock_cli/cli-reference.yaml b/simplyblock_cli/cli-reference.yaml index 3fa81c857..ba6915c48 100644 --- a/simplyblock_cli/cli-reference.yaml +++ b/simplyblock_cli/cli-reference.yaml @@ -818,6 +818,13 @@ commands: aliases: - "-n" type: str + - name: "--cc-refresh-token-secret" + help: > + Sets a user-chosen secret used to sign and validate refresh tokens in the control center. A valid secret + consists of 128 characters. + dest: refresh_token_secret + type: str + private: true - name: add help: "Adds a new cluster" arguments: diff --git a/simplyblock_cli/cli.py b/simplyblock_cli/cli.py index b46612c5c..beffc0ba1 100644 --- a/simplyblock_cli/cli.py +++ b/simplyblock_cli/cli.py @@ -366,6 +366,8 @@ def init_cluster__create(self, subparser): argument = subcommand.add_argument('--disable-monitoring', help='Disable monitoring stack, false by default', dest='disable_monitoring', action='store_true') argument = subcommand.add_argument('--strict-node-anti-affinity', help='Enable strict node anti affinity for storage nodes. Never more than one chunk is placed on a node. This requires a minimum of _data-chunks-in-stripe + parity-chunks-in-stripe + 1_ nodes in the cluster.', dest='strict_node_anti_affinity', action='store_true') argument = subcommand.add_argument('--name', '-n', help='Assigns a name to the newly created cluster.', type=str, dest='name') + if self.developer_mode: + argument = subcommand.add_argument('--cc-refresh-token-secret', help='Sets a user-chosen secret used to sign and validate refresh tokens in the control center. A valid secret consists of 128 characters.', type=str, dest='refresh_token_secret') def init_cluster__add(self, subparser): subcommand = self.add_sub_command(subparser, 'add', 'Adds a new cluster') @@ -894,6 +896,7 @@ def run(self): args.inflight_io_threshold = 4 args.enable_qos = False args.disable_monitoring = False + args.refresh_token_secret = None ret = self.cluster__create(sub_command, args) elif sub_command in ['add']: if not self.developer_mode: diff --git a/simplyblock_cli/clibase.py b/simplyblock_cli/clibase.py index 7bb73acff..2162e00aa 100644 --- a/simplyblock_cli/clibase.py +++ b/simplyblock_cli/clibase.py @@ -684,13 +684,15 @@ def cluster_create(self, args): disable_monitoring = args.disable_monitoring strict_node_anti_affinity = args.strict_node_anti_affinity name = args.name + refresh_token_secret = args.refresh_token_secret return cluster_ops.create_cluster( blk_size, page_size_in_blocks, CLI_PASS, cap_warn, cap_crit, prov_cap_warn, prov_cap_crit, ifname, mgmt_ip, log_del_interval, metrics_retention_period, contact_point, grafana_endpoint, distr_ndcs, distr_npcs, distr_bs, distr_chunk_bs, ha_type, mode, enable_node_affinity, - qpair_count, max_queue_size, inflight_io_threshold, enable_qos, disable_monitoring, strict_node_anti_affinity, name) + qpair_count, max_queue_size, inflight_io_threshold, enable_qos, disable_monitoring, + strict_node_anti_affinity, name, refresh_token_secret) def query_yes_no(self, question, default="yes"): """Ask a yes/no question via raw_input() and return their answer. diff --git a/simplyblock_core/cluster_ops.py b/simplyblock_core/cluster_ops.py index 4c7377b30..ddaa3f420 100644 --- a/simplyblock_core/cluster_ops.py +++ b/simplyblock_core/cluster_ops.py @@ -206,7 +206,8 @@ def _set_max_result_window(cluster_ip, max_window=100000): def create_cluster(blk_size, page_size_in_blocks, cli_pass, cap_warn, cap_crit, prov_cap_warn, prov_cap_crit, ifname, mgmt_ip, log_del_interval, metrics_retention_period, contact_point, grafana_endpoint, distr_ndcs, distr_npcs, distr_bs, distr_chunk_bs, ha_type, mode, - enable_node_affinity, qpair_count, max_queue_size, inflight_io_threshold, enable_qos, disable_monitoring, strict_node_anti_affinity, name) -> str: + enable_node_affinity, qpair_count, max_queue_size, inflight_io_threshold, enable_qos, disable_monitoring, + strict_node_anti_affinity, name, refresh_token_secret) -> str: if distr_ndcs == 0 and distr_npcs == 0: raise ValueError("both distr_ndcs and distr_npcs cannot be 0") @@ -307,6 +308,12 @@ def create_cluster(blk_size, page_size_in_blocks, cli_pass, cluster.disable_monitoring = disable_monitoring cluster.mode = mode + if not refresh_token_secret: + refresh_token_secret = utils.generate_string(128) + + if refresh_token_secret is not None and len(refresh_token_secret) != 128: + raise ValueError(f"Illegal refresh token secret given, required length=128, given {len(refresh_token_secret)}") + if mode == "docker": if not disable_monitoring: utils.render_and_deploy_alerting_configs(contact_point, cluster.grafana_endpoint, cluster.uuid, cluster.secret) @@ -324,7 +331,9 @@ def create_cluster(blk_size, page_size_in_blocks, cli_pass, logger.info("Deploying helm stack ...") log_level = "DEBUG" if constants.LOG_WEB_DEBUG else "INFO" scripts.deploy_k8s_stack(cli_pass, dev_ip, constants.SIMPLY_BLOCK_DOCKER_IMAGE, cluster.secret, cluster.uuid, - log_del_interval, metrics_retention_period, log_level, cluster.grafana_endpoint, contact_point, db_connection, constants.K8S_NAMESPACE, str(disable_monitoring)) + log_del_interval, metrics_retention_period, log_level, cluster.grafana_endpoint, + contact_point, db_connection, constants.K8S_NAMESPACE, str(disable_monitoring), + refresh_token_secret) logger.info("Deploying helm stack > Done") logger.info("Configuring DB...") diff --git a/simplyblock_core/scripts/__init__.py b/simplyblock_core/scripts/__init__.py index bed25334c..d34599674 100644 --- a/simplyblock_core/scripts/__init__.py +++ b/simplyblock_core/scripts/__init__.py @@ -31,11 +31,13 @@ def deploy_stack(cli_pass, dev_ip, image_name, graylog_password, cluster_id, graylog_password, cluster_id, log_del_interval, metrics_retention_period, log_level, grafana_endpoint, disable_monitoring]) def deploy_k8s_stack(cli_pass, dev_ip, image_name, graylog_password, cluster_id, - log_del_interval, metrics_retention_period, log_level, grafana_endpoint, contact_point, db_connection, k8s_namespace, disable_monitoring): + log_del_interval, metrics_retention_period, log_level, grafana_endpoint, contact_point, + db_connection, k8s_namespace, disable_monitoring, cc_refresh_token_secret): pass_hash = hashlib.sha256(graylog_password.encode('utf-8')).hexdigest() __run_script( ['sudo', 'bash', '-x', os.path.join(DIR_PATH, 'deploy_k8s_stack.sh'), cli_pass, dev_ip, image_name, pass_hash, - graylog_password, cluster_id, log_del_interval, metrics_retention_period, log_level, grafana_endpoint, contact_point, db_connection, k8s_namespace, disable_monitoring]) + graylog_password, cluster_id, log_del_interval, metrics_retention_period, log_level, grafana_endpoint, + contact_point, db_connection, k8s_namespace, disable_monitoring, cc_refresh_token_secret]) def deploy_cleaner(): __run_script(['sudo', 'bash', '-x', os.path.join(DIR_PATH, 'clean_local_storage_deploy.sh')]) diff --git a/simplyblock_core/scripts/charts/templates/app_configmap.yaml b/simplyblock_core/scripts/charts/templates/app_configmap.yaml index 5738202fa..929febc8f 100644 --- a/simplyblock_core/scripts/charts/templates/app_configmap.yaml +++ b/simplyblock_core/scripts/charts/templates/app_configmap.yaml @@ -55,8 +55,8 @@ data: [OUTPUT] Name gelf Match * - Host simplyblock-graylog - Port 12201 + Host simplyblock-haproxy + Port 12202 Mode tcp Gelf_Short_Message_Key log @@ -76,7 +76,22 @@ data: end return -1, record end - +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: simplyblock-controlcenter-config + namespace: {{ .Release.Namespace }} +data: + CONTROL_PLANE_ADDR: "https://simplyblock-haproxy/" + CONTROL_PLANE_CLUSTER_ID: "{{ .Values.cluster.id }}" + CONTROL_PLANE_CLUSTER_SECRET: "{{ .Values.cluster.secret }}" + CONTROL_CENTER_LOGGER_LEVEL: "debug" + CONTROL_CENTER_SRV_TLS: "false" + CONTROL_CENTER_SRV_PORT: "80" + #DATABASE_URI: "mongodb://simplyblock-mongodb-headless:27017/controlcenter" + FDB_CLUSTER_FILE_CONTENT: {{ .Values.foundationdb.fdbClusterFileContent }} + FDB_CLUSTER_FILE_LOCATION: "{{ .Values.controlcenter.fdbClusterFileLocation }}/fdb.cluster" --- apiVersion: v1 kind: ConfigMap diff --git a/simplyblock_core/scripts/charts/templates/app_k8s.yaml b/simplyblock_core/scripts/charts/templates/app_k8s.yaml index 9b0236a5d..467a0d3de 100644 --- a/simplyblock_core/scripts/charts/templates/app_k8s.yaml +++ b/simplyblock_core/scripts/charts/templates/app_k8s.yaml @@ -842,3 +842,58 @@ spec: - name: config configMap: name: simplyblock-fluent-bit-config + +--- + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: simplyblock-controlcenter + namespace: {{ .Release.Namespace }} + labels: + app: simplyblock-controlcenter +spec: + replicas: 1 + selector: + matchLabels: + app: simplyblock-controlcenter + template: + metadata: + labels: + app: simplyblock-controlcenter + spec: + nodeSelector: + simplyblock.io/role: mgmt-plane + containers: + - name: simplyblock-controlcenter + image: "{{ .Values.image.controlcenter.repository }}:{{ .Values.image.controlcenter.tag }}" + imagePullPolicy: "{{ .Values.image.controlcenter.pullPolicy }}" + resources: + limits: + memory: 400Mi + requests: + cpu: 250m + memory: 200Mi + envFrom: + - configMapRef: + name: simplyblock-controlcenter-config + env: + - name: REFRESH_TOKEN_KEY + valueFrom: + secretKeyRef: + name: simplyblock-controlcenter-secret + key: REFRESH_TOKEN_SECRET + volumeMounts: + - name: simplyblock-controlcenter-fdb-cluster-file + mountPath: {{ .Values.controlcenter.fdbClusterFileLocation }} + readOnly: true + ports: + - containerPort: 443 + name: controlcenter-https + volumes: + - name: simplyblock-controlcenter-fdb-cluster-file + configMap: + name: simplyblock-controlcenter-config + items: + - key: FDB_CLUSTER_FILE_CONTENT + path: fdb.cluster diff --git a/simplyblock_core/scripts/charts/templates/app_secret.yaml b/simplyblock_core/scripts/charts/templates/app_secret.yaml new file mode 100644 index 000000000..827af5bdb --- /dev/null +++ b/simplyblock_core/scripts/charts/templates/app_secret.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +metadata: + name: simplyblock-controlcenter-secrets + namespace: {{ .Release.Namespace }} +type: Opaque +stringData: + REFRESH_TOKEN_KEY: "{{ .Values.controlcenter.refreshTokenSecret }}" diff --git a/simplyblock_core/scripts/charts/values-template.yaml b/simplyblock_core/scripts/charts/values-template.yaml index 3623185bf..abec30456 100644 --- a/simplyblock_core/scripts/charts/values-template.yaml +++ b/simplyblock_core/scripts/charts/values-template.yaml @@ -7,6 +7,10 @@ cluster: id: "${CLUSTER_ID}" ip: "${CLUSTER_IP}" +controlcenter: + refreshTokenSecret: "${CONTROLCENTER_REFRESH_TOKEN_SECRET}" + fdbClusterFileLocation: "/etc/foundationdb" + monitoring: enabled: ${ENABLE_MONITORING} @@ -28,6 +32,10 @@ image: repository: "${SIMPLYBLOCK_REPOSITORY}" tag: "${SIMPLYBLOCK_TAG}" pullPolicy: "Always" + controlcenter: + repository: "${CONTROLCENTER_REPOSITORY}" + tag: "${CONTROLCENTER_TAG}" + pullPolicy: "Always" openebs: enabled: true diff --git a/simplyblock_core/scripts/deploy_k8s_stack.sh b/simplyblock_core/scripts/deploy_k8s_stack.sh index b831db327..bc121afb3 100644 --- a/simplyblock_core/scripts/deploy_k8s_stack.sh +++ b/simplyblock_core/scripts/deploy_k8s_stack.sh @@ -32,6 +32,9 @@ export SIMPLYBLOCK_DOCKER_IMAGE=$3 export SIMPLYBLOCK_REPOSITORY="${SIMPLYBLOCK_DOCKER_IMAGE%%:*}" export SIMPLYBLOCK_TAG="${SIMPLYBLOCK_DOCKER_IMAGE##*:}" +export CONTROLCENTER_REPOSITORY="https://hub.docker.com/r/simplyblock/controlcenter" +export CONTROLCENTER_TAG="latest" + export GRAYLOG_ROOT_PASSWORD_SHA2=$4 export GRAYLOG_PASSWORD_SECRET="is6SP2EdWg0NdmVGv6CEp5hRHNL7BKVMFem4t9pouMqDQnHwXMSomas1qcbKSt5yISr8eBHv4Y7Dbswhyz84Ut0TW6kqsiPs" @@ -45,6 +48,7 @@ export CONTACT_POINT=${11} export DB_CONNECTION=${12} export K8S_NAMESPACE=${13} export DISABLE_MONITORING=${14} +export CONTROLCENTER_REFRESH_TOKEN_SECRET=${15} export DIR="$(dirname "$(realpath "$0")")" export FDB_CLUSTER_FILE_CONTENTS=${DB_CONNECTION}