Add support for external cluster providers

.dockerignore

@@ -9,3 +9,8 @@ tests/data
 .checkpoints
 **/.terraform
 data
+*.kube*
+Dockerfile*
+README.md
+docs
+.cco

.gitignore

@@ -11,4 +11,6 @@ __pycache__
 .coverage
 .pytest_cache/
 .vscode/
-build/
+build/
+*.kube*
+.cco*

.travis.sh

@@ -43,6 +43,19 @@ elif [ "${1}" == "install-tools" ]; then
       echo AWS Dependencies Installed Successfully!
     fi
 
+    if [[ $K8_PROVIDER == custom-* ]]; then
+      echo Installing ${K8_PROVIDER} from ${K8_PROVIDER_CUSTOM_DOWNLOAD_URL} &&\
+      mkdir -p /usr/local/lib/cco/${K8_PROVIDER} &&\
+      cd /usr/local/lib/cco/${K8_PROVIDER} &&\
+      curl -Lo custom.tar.gz ${K8_PROVIDER_CUSTOM_DOWNLOAD_URL} &&\
+      tar -xzvf custom.tar.gz && rm custom.tar.gz &&\
+      mv `ls`/* ./ &&\
+      source "/usr/local/lib/cco/${K8_PROVIDER}/install_tools_constants.sh"
+      [ "$?" != "0" ] && exit 1
+      ! bash "/usr/local/lib/cco/${K8_PROVIDER}/install_tools.sh" && exit 2
+      echo ${K8_PROVIDER} Dependencies Installed Successfully!
+    fi
+
     curl -LO https://storage.googleapis.com/kubernetes-release/release/`curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt`/bin/linux/amd64/kubectl
     chmod +x ./kubectl && sudo mv ./kubectl /usr/local/bin/kubectl
     echo Kubectl Installed Successfully!
@@ -58,8 +71,18 @@ elif [ "${1}" == "install-tools" ]; then
     echo Instalation Complete && exit 0
 
 elif [ "${1}" == "script" ]; then
-    ! docker build --build-arg "CKAN_CLOUD_OPERATOR_IMAGE_TAG=${TAG}" --cache-from viderum/ckan-cloud-operator:latest -t ckan-cloud-operator . && echo Failed to build image && exit 1
-    ! docker build --build-arg "CKAN_CLOUD_OPERATOR_IMAGE_TAG=${TAG}" --cache-from viderum/ckan-cloud-operator:jnlp-latest -t ckan-cloud-operator-jnlp -f Dockerfile.jenkins-jnlp . && echo Failed to build jnlp image && exit 1
+    ! docker build --cache-from viderum/ckan-cloud-operator:latest \
+                   -t ckan-cloud-operator \
+                   . && echo Failed to build image && exit 1
+    ! docker build --build-arg "CKAN_CLOUD_OPERATOR_IMAGE_TAG=${TAG}" \
+                   --cache-from viderum/ckan-cloud-operator:jnlp-latest \
+                   -t ckan-cloud-operator-jnlp \
+                   -f Dockerfile.jenkins-jnlp \
+                   . && echo Failed to build jnlp image && exit 1
+    ! docker build --build-arg "K8_PROVIDER=minikube" \
+                   --cache-from viderum/ckan-cloud-operator:minikube-latest \
+                   -t ckan-cloud-operator-minikube \
+                   . && echo Failed to build minikube image && exit 1
     echo Great Success! && exit 0
 
 elif [ "${1}" == "test" ]; then
@@ -79,27 +102,42 @@ elif [ "${1}" == "deploy" ]; then
     echo && echo "viderum/ckan-cloud-operator:${TAG}" && echo &&\
     docker push "viderum/ckan-cloud-operator:${TAG}"
     [ "$?" != "0" ] && echo Failed to tag and push && exit 1
+
     docker tag ckan-cloud-operator-jnlp "viderum/ckan-cloud-operator:jnlp-${TAG}" &&\
     echo && echo "viderum/ckan-cloud-operator:jnlp-${TAG}" && echo &&\
     docker push "viderum/ckan-cloud-operator:jnlp-${TAG}"
     [ "$?" != "0" ] && echo Failed to tag and push jnlp image && exit 1
+
+    docker tag ckan-cloud-operator-minikube "viderum/ckan-cloud-operator:minikube-${TAG}" &&\
+    echo && echo "viderum/ckan-cloud-operator:minikube-${TAG}" && echo &&\
+    docker push "viderum/ckan-cloud-operator:minikube-${TAG}"
+    [ "$?" != "0" ] && echo Failed to tag and push minikube image && exit 1
+
     if [ "${TRAVIS_BRANCH}" == "master" ]; then
         docker tag ckan-cloud-operator viderum/ckan-cloud-operator:latest &&\
         echo && echo viderum/ckan-cloud-operator:latest && echo &&\
         docker push viderum/ckan-cloud-operator:latest
         [ "$?" != "0" ] && echo Failed to tag and push latest image && exit 1
+
         docker tag ckan-cloud-operator-jnlp viderum/ckan-cloud-operator:jnlp-latest &&\
         echo && echo viderum/ckan-cloud-operator:jnlp-latest && echo &&\
         docker push viderum/ckan-cloud-operator:jnlp-latest
         [ "$?" != "0" ] && echo Failed to tag and push jnlp latest image && exit 1
+
+        docker tag ckan-cloud-operator-minikube viderum/ckan-cloud-operator:minikube-latest &&\
+        echo && echo viderum/ckan-cloud-operator:minikube-latest && echo &&\
+        docker push viderum/ckan-cloud-operator:minikube-latest
+        [ "$?" != "0" ] && echo Failed to tag and push minikube latest image && exit 1
     fi
+
     if [ "${TRAVIS_TAG}" != "" ]; then
         export DEPLOY_JNLP_IMAGE="viderum/ckan-cloud-operator:jnlp-${TAG}"
         echo "Running Jenkins deploy jnlp job (JNLP_IMAGE=${DEPLOY_JNLP_IMAGE})"
         STATUS_CODE=$(curl -X POST "${JENKINS_JNLP_DEPLOY_URL}" --user "${JENKINS_USER}:${JENKINS_TOKEN}" --data "JNLP_IMAGE=${DEPLOY_JNLP_IMAGE}" -s -o /dev/stderr -w "%{http_code}")
         echo "jenkins jnlp deploy job status code: ${STATUS_CODE}"
         [ "${STATUS_CODE}" != "200" ] && [ "${STATUS_CODE}" != "201" ] && echo Deploy failed && exit 1
     fi
+
     echo Great Success! && exit 0
 
 else

.travis.yml

@@ -25,7 +25,7 @@ jobs:
         - minikube config set cpus 4 && minikube config set memory 8192 && minikube config set vm-driver none && minikube config set kubernetes-version "${K8_VERSION}" 
         - sudo minikube start
       script:
-        - ckan-cloud-operator cluster initialize --interactive --cluster-provider=minikube
+        - ckan-cloud-operator cluster initialize --cluster-provider=minikube
         - sleep 60 && kubectl get ns && kubectl get pods -n ckan-cloud 
         - ckan-cloud-operator ckan instance create helm --instance-id a-ckan-instance --instance-name a-ckan-instance --update sample-values/values.datagov.yaml
       after_success:

Dockerfile

@@ -7,7 +7,10 @@ RUN apt-get install -y nano curl unzip sudo bash libpq-dev build-essential
 ENV EDITOR nano
 
 COPY . .
-RUN K8_PROVIDER=aws TERRAFORM_VERSION=0.12.18 /cco/.travis.sh install-tools
+ARG K8_PROVIDER=aws
+ENV K8_PROVIDER=$K8_PROVIDER
+ARG K8_PROVIDER_CUSTOM_DOWNLOAD_URL=""
+RUN /cco/.travis.sh install-tools
 RUN pip install .
 
 ENTRYPOINT [ "/bin/bash" ]

README.md

@@ -8,6 +8,7 @@ CKAN Cloud Operator manages, provisions and configures CKAN Cloud instances and
   - AWS
   - GCP
   - Azure
+  - Custom / Self-hosted Provider
   - Minikube for local development
 - `ckan-cloud-operator` CLI will manage the cluster and any other services necessary for day-to-day operations
 - Management server, comes preinstalled with ckan-cloud-operator, required tools (terraform, kubectl, helm, awscli etc.) and [a Jenkins Server](/docs/JENKINS.md).
@@ -24,6 +25,8 @@ In order to start using ckan-cloud-operator, you need to
    - Run the TBD (on Azure)
 
    Note: While technically possible, we recommend not to run ckan-cloud-operator directly on you machine to avoid version incompatibilities between the various tools involved in the process. You should use one of our pre-built images or our Docker image instead.
+
+   If you do want to run locally for development, refer to the [local development doc](development/LOCAL-DEVELOPMENT.md)  
 
 2. Create a Kubernetes cluster and provision it.
     - [Instructions for AWS](docs/PRODUCTION-AWS-CLUSTER.md):
@@ -38,6 +41,10 @@ In order to start using ckan-cloud-operator, you need to
         - Create a cluster using terraform
         - Initialize the cluster using ckan-cloud-operator
 
+    - [Instructions for Custom / Self-hosted providers](docs/PRODUCTION-CUSTOM-CLUSTER.md):
+        - Create servers using custom provisioning tools
+        - Initialize the cluster using ckan-cloud-operator
+
     - Instructions for Minikube:
         - Initialize the cluster using ckan-cloud-operator
 

ckan_cloud_operator/config/manager.py

@@ -6,7 +6,6 @@
 from ckan_cloud_operator import kubectl
 from ckan_cloud_operator import logs
 
-from ckan_cloud_operator.providers.cluster import manager as cluster_manager
 from ckan_cloud_operator.labels import manager as labels_manager
 
 
@@ -88,7 +87,9 @@ def delete_by_extra_operator_labels(extra_operator_labels):
 
 def list_configs(namespace=None, full=False, show_secrets=False):
     label_prefix = labels_manager.get_label_prefix()
-    if not namespace: namespace = cluster_manager.get_operator_namespace_name()
+    if not namespace:
+        from ckan_cloud_operator.providers.cluster import manager as cluster_manager
+        namespace = cluster_manager.get_operator_namespace_name()
     what = 'configmaps'
     if show_secrets:
         what += ',secrets'
@@ -111,6 +112,7 @@ def list_configs(namespace=None, full=False, show_secrets=False):
                     data['values'] = None
             yield data
 
+
 def get_preset_answer(namespace, configmap_name, secret_name, key):
     interactive_file = os.environ.get('CCO_INTERACTIVE_CI')
     if not interactive_file:
@@ -254,6 +256,7 @@ def _get_labels(cache_key=None, secret_name=None, configmap_name=None, namespace
 def _get_cache_key(secret_name, configmap_name, namespace):
     if secret_name:
         assert not configmap_name, f'Invalid arguments: cannot specify both secret_name and configmap_name: {secret_name}, {configmap_name}'
+    from ckan_cloud_operator.providers.cluster import manager as cluster_manager
     configmap_name, namespace = cluster_manager.get_operator_configmap_namespace_defaults(configmap_name, namespace)
     return f'secret:{namespace}:{secret_name}' if secret_name else f'configmap:{namespace}:{configmap_name}'
 

ckan_cloud_operator/providers/cluster/manager.py

@@ -7,6 +7,7 @@
 from ckan_cloud_operator import kubectl
 from ckan_cloud_operator import logs
 from ckan_cloud_operator.labels import manager as labels_manager
+from ckan_cloud_operator.config.manager import get_preset_answer
 
 from .constants import OPERATOR_NAMESPACE, OPERATOR_CONFIGMAP
 
@@ -48,7 +49,22 @@ def print_info(debug=False, minimal=False):
             return True
 
 
+def _initialize_cluster(default_cluster_provider, interactive):
+    from ckan_cloud_operator.providers import manager as providers_manager
+
+    default_provider_id = default_cluster_provider
+    if default_cluster_provider.startswith('custom-'):
+        default_provider_id = default_cluster_provider.replace('custom-', '')
+    providers_manager.get_provider(
+        'cluster',
+        default=default_provider_id
+    ).initialize(interactive=interactive)
+
+
 def initialize(log_kwargs=None, interactive=False, default_cluster_provider=None, skip_to=None):
+    if os.environ.get('CCO_INTERACTIVE_CI'):
+        interactive = True
+
     if interactive:
         logs.info('Starting interactive initialization of the operator on the following cluster:')
         print_info(minimal=True)
@@ -57,12 +73,27 @@ def initialize(log_kwargs=None, interactive=False, default_cluster_provider=None
 
     if not skip_to:
         logs.info(f'Creating operator namespace: {OPERATOR_NAMESPACE}', **(log_kwargs or {}))
-        subprocess.call(f'kubectl create ns {OPERATOR_NAMESPACE}', shell=True)
-        assert default_cluster_provider in ['gcloud', 'aws', 'minikube'], f'invalid cluster provider: {default_cluster_provider}'
-        subprocess.call(f'kubectl -n {OPERATOR_NAMESPACE} create secret generic ckan-cloud-provider-cluster-{default_cluster_provider}', shell=True)
-        subprocess.call(f'kubectl -n {OPERATOR_NAMESPACE} create configmap operator-conf --from-literal=ckan-cloud-operator-image=viderum/ckan-cloud-operator:latest --from-literal=label-prefix={OPERATOR_NAMESPACE}', shell=True)
+        if subprocess.call(f'kubectl get ns {OPERATOR_NAMESPACE}'.split(' ')) != 0:
+            subprocess.check_call(f'kubectl create ns {OPERATOR_NAMESPACE}'.split(' '))
+        assert (default_cluster_provider in ['gcloud', 'aws', 'minikube']
+                or default_cluster_provider.startswith('custom-')), \
+            f'invalid cluster provider: {default_cluster_provider}'
+        if subprocess.call(f'kubectl -n {OPERATOR_NAMESPACE} '
+                           f'get secret ckan-cloud-provider-cluster-{default_cluster_provider}'.split(' ')) != 0:
+            subprocess.check_call(f'kubectl -n {OPERATOR_NAMESPACE} '
+                                  f'create secret generic '
+                                  f'ckan-cloud-provider-cluster-{default_cluster_provider}'.split(' '))
+        if subprocess.call(f'kubectl -n {OPERATOR_NAMESPACE} get configmap operator-conf'.split(' ')) != 0:
+            try:
+                ckan_cloud_operator_image = get_preset_answer('default', 'cluster-init', None,
+                                                              'ckan-cloud-operator-image')
+            except Exception:
+                ckan_cloud_operator_image = 'viderum/ckan-cloud-operator:latest'
+            subprocess.call(f'kubectl -n {OPERATOR_NAMESPACE} '
+                            f'create configmap operator-conf '
+                            f'--from-literal=ckan-cloud-operator-image={ckan_cloud_operator_image} '
+                            f'--from-literal=label-prefix={OPERATOR_NAMESPACE}'.split(' '))
 
-    from ckan_cloud_operator.providers import manager as providers_manager
     from ckan_cloud_operator.labels import manager as labels_manager
     from ckan_cloud_operator.crds import manager as crds_manager
     from ckan_cloud_operator.providers.db import manager as db_manager
@@ -72,22 +103,31 @@ def initialize(log_kwargs=None, interactive=False, default_cluster_provider=None
     from ckan_cloud_operator.providers.storage import manager as storage_manager
     from ckan_cloud_operator.providers.apps import manager as apps_manager
 
-    for component, func in (
-            ('labels', lambda lk: labels_manager.initialize(log_kwargs=lk)),
-            ('cluster', lambda lk: providers_manager.get_provider('cluster', default=default_cluster_provider).initialize(interactive=interactive)),
-            ('crds', lambda lk: crds_manager.initialize(log_kwargs=lk)),
-            ('db', lambda lk: db_manager.initialize(log_kwargs=lk, interactive=interactive, default_cluster_provider=default_cluster_provider)),
-            ('routers', lambda lk: routers_manager.initialize(interactive=interactive)),
-            ('solr', lambda lk: solr_manager.initialize(interactive=interactive)),
-            ('storage', lambda lk: storage_manager.initialize(interactive=interactive)),
-            ('ckan', lambda lk: ckan_manager.initialize(interactive=interactive)),
-            ('apps', lambda lk: apps_manager.initialize(interactive=interactive)),
-    ):
+    components = dict((
+        ('labels', lambda lk: labels_manager.initialize(log_kwargs=lk)),
+        ('cluster', lambda lk: _initialize_cluster(default_cluster_provider, interactive)),
+        ('crds', lambda lk: crds_manager.initialize(log_kwargs=lk)),
+        ('db', lambda lk: db_manager.initialize(log_kwargs=lk, interactive=interactive,
+                                                default_cluster_provider=default_cluster_provider)),
+        ('routers', lambda lk: routers_manager.initialize(interactive=interactive)),
+        ('solr', lambda lk: solr_manager.initialize(interactive=interactive)),
+        ('storage', lambda lk: storage_manager.initialize(interactive=interactive)),
+        ('ckan', lambda lk: ckan_manager.initialize(interactive=interactive)),
+        ('apps', lambda lk: apps_manager.initialize(interactive=interactive)),
+    ))
+
+    try:
+        selected_components = list(get_preset_answer('default', 'cluster-init', None, 'components'))
+    except Exception:
+        selected_components = list(components.keys())
+
+    for component, func in components.items():
         if not skip_to or skip_to == component:
             skip_to = None
-            log_kwargs = {'cluster-init': component}
-            logs.info(f'Initializing', **log_kwargs)
-            func(log_kwargs)
+            if component in selected_components:
+                log_kwargs = {'cluster-init': component}
+                logs.info(f'Initializing', **log_kwargs)
+                func(log_kwargs)
 
 
 def get_kubeconfig_info():

ckan_cloud_operator/providers/helpers.py

@@ -0,0 +1,36 @@
+def get_provider_functions(PROVIDER_SUBMODULE, PROVIDER_ID):
+    from ckan_cloud_operator.providers import manager as providers_manager
+
+    def _get_resource_name(suffix=None):
+        return providers_manager.get_resource_name(PROVIDER_SUBMODULE, PROVIDER_ID,
+                                                   suffix=suffix)
+
+    def _get_resource_labels(for_deployment=False):
+        return providers_manager.get_resource_labels(PROVIDER_SUBMODULE, PROVIDER_ID,
+                                                     for_deployment=for_deployment)
+
+    def _get_resource_annotations(suffix=None):
+        return providers_manager.get_resource_annotations(PROVIDER_SUBMODULE, PROVIDER_ID,
+                                                          suffix=suffix)
+
+    def _set_provider():
+        providers_manager.set_provider(PROVIDER_SUBMODULE, PROVIDER_ID)
+
+    def _config_set(key=None, value=None, values=None, namespace=None, is_secret=False, suffix=None):
+        providers_manager.config_set(PROVIDER_SUBMODULE, PROVIDER_ID,
+                                     key=key, value=value,
+                                     values=values, namespace=namespace, is_secret=is_secret,
+                                     suffix=suffix)
+
+    def _config_get(key=None, default=None, required=False, namespace=None, is_secret=False, suffix=None):
+        return providers_manager.config_get(PROVIDER_SUBMODULE, PROVIDER_ID,
+                                            key=key, default=default, required=required,
+                                            namespace=namespace, is_secret=is_secret,
+                                            suffix=suffix)
+
+    def _config_interactive_set(default_values, namespace=None, is_secret=False, suffix=None, from_file=False):
+        providers_manager.config_interactive_set(PROVIDER_SUBMODULE, PROVIDER_ID,
+                                                 default_values, namespace, is_secret, suffix, from_file)
+
+    return (_config_interactive_set, _config_get, _config_set, _set_provider, _get_resource_annotations,
+            _get_resource_labels, _get_resource_name)

ckan_cloud_operator/providers/manager.py

@@ -2,6 +2,7 @@
 from ckan_cloud_operator.config import manager as config_manager
 from ckan_cloud_operator.labels import manager as labels_manager
 from ckan_cloud_operator.annotations import manager as annotations_manager
+import importlib
 
 
 def get_provider(submodule, required=True, supported_provider_ids=None, default=None, verbose=False, provider_id=None):
@@ -350,6 +351,11 @@ def _get_submodule_ids_provider_or_provider_ids(submodule=None, provider_id=None
 
             return solr_solrcloud_manager
 
+    try:
+        return importlib.import_module(f'cco_provider_{provider_id}.manager')
+    except ModuleNotFoundError as e:
+        logs.warning(str(e))
+
     if not provider_id:
         return []
     else:

docs/PRODUCTION-AWS-CLUSTER.md

@@ -57,7 +57,7 @@ For an unattended initialization, using the outputs of `terraform apply`, run:
 
 ```bash
 export CCO_INTERACTIVE_CI=interactive.yaml
-ckan-cloud-operator cluster initialize --interactive --cluster-provider=aws
+ckan-cloud-operator cluster initialize --cluster-provider=aws
 ```
 
 You can also run the same without setting the `CCO_INTERACTIVE_CI` environment variable for an interactive initialization session.

docs/PRODUCTION-CUSTOM-CLUSTER.md

@@ -0,0 +1,19 @@
+# Creating a Production CKAN Cloud cluster using a custom / self-hosted provider
+
+## Prerequisites
+
+* Custom Provider values, check the custom provider docs for details
+    * e.g. `https://github.com/OriHoch/cco-provider-kamatera/blob/master/README.md`
+* An external domain
+* A CKAN Cloud Operator [working environment](./WORKING-ENVIRONMENT.md)
+
+## Provision the cluster
+
+Check the custom provider docs for creating the cluster.
+
+Once the cluster was created on the custom provider, you can continue initializing ckan-cloud-operator: 
+
+```
+CCO_INTERACTIVE_CI=/path/to/interactive.yaml \
+    ckan-cloud-operator cluster initialize --cluster-provider=PROVIDER_NAME
+```