diff --git a/.dockerignore b/.dockerignore index 5e2091de..c971820f 100644 --- a/.dockerignore +++ b/.dockerignore @@ -9,3 +9,8 @@ tests/data .checkpoints **/.terraform data +*.kube* +Dockerfile* +README.md +docs +.cco diff --git a/.gitignore b/.gitignore index 6b3aeaf0..925be409 100644 --- a/.gitignore +++ b/.gitignore @@ -11,4 +11,6 @@ __pycache__ .coverage .pytest_cache/ .vscode/ -build/ \ No newline at end of file +build/ +*.kube* +.cco* diff --git a/.travis.sh b/.travis.sh index f86a27fe..331d66fe 100755 --- a/.travis.sh +++ b/.travis.sh @@ -43,6 +43,19 @@ elif [ "${1}" == "install-tools" ]; then echo AWS Dependencies Installed Successfully! fi + if [[ $K8_PROVIDER == custom-* ]]; then + echo Installing ${K8_PROVIDER} from ${K8_PROVIDER_CUSTOM_DOWNLOAD_URL} &&\ + mkdir -p /usr/local/lib/cco/${K8_PROVIDER} &&\ + cd /usr/local/lib/cco/${K8_PROVIDER} &&\ + curl -Lo custom.tar.gz ${K8_PROVIDER_CUSTOM_DOWNLOAD_URL} &&\ + tar -xzvf custom.tar.gz && rm custom.tar.gz &&\ + mv `ls`/* ./ &&\ + source "/usr/local/lib/cco/${K8_PROVIDER}/install_tools_constants.sh" + [ "$?" != "0" ] && exit 1 + ! bash "/usr/local/lib/cco/${K8_PROVIDER}/install_tools.sh" && exit 2 + echo ${K8_PROVIDER} Dependencies Installed Successfully! + fi + curl -LO https://storage.googleapis.com/kubernetes-release/release/`curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt`/bin/linux/amd64/kubectl chmod +x ./kubectl && sudo mv ./kubectl /usr/local/bin/kubectl echo Kubectl Installed Successfully! @@ -58,8 +71,18 @@ elif [ "${1}" == "install-tools" ]; then echo Instalation Complete && exit 0 elif [ "${1}" == "script" ]; then - ! docker build --build-arg "CKAN_CLOUD_OPERATOR_IMAGE_TAG=${TAG}" --cache-from viderum/ckan-cloud-operator:latest -t ckan-cloud-operator . && echo Failed to build image && exit 1 - ! docker build --build-arg "CKAN_CLOUD_OPERATOR_IMAGE_TAG=${TAG}" --cache-from viderum/ckan-cloud-operator:jnlp-latest -t ckan-cloud-operator-jnlp -f Dockerfile.jenkins-jnlp . && echo Failed to build jnlp image && exit 1 + ! docker build --cache-from viderum/ckan-cloud-operator:latest \ + -t ckan-cloud-operator \ + . && echo Failed to build image && exit 1 + ! docker build --build-arg "CKAN_CLOUD_OPERATOR_IMAGE_TAG=${TAG}" \ + --cache-from viderum/ckan-cloud-operator:jnlp-latest \ + -t ckan-cloud-operator-jnlp \ + -f Dockerfile.jenkins-jnlp \ + . && echo Failed to build jnlp image && exit 1 + ! docker build --build-arg "K8_PROVIDER=minikube" \ + --cache-from viderum/ckan-cloud-operator:minikube-latest \ + -t ckan-cloud-operator-minikube \ + . && echo Failed to build minikube image && exit 1 echo Great Success! && exit 0 elif [ "${1}" == "test" ]; then @@ -79,20 +102,34 @@ elif [ "${1}" == "deploy" ]; then echo && echo "viderum/ckan-cloud-operator:${TAG}" && echo &&\ docker push "viderum/ckan-cloud-operator:${TAG}" [ "$?" != "0" ] && echo Failed to tag and push && exit 1 + docker tag ckan-cloud-operator-jnlp "viderum/ckan-cloud-operator:jnlp-${TAG}" &&\ echo && echo "viderum/ckan-cloud-operator:jnlp-${TAG}" && echo &&\ docker push "viderum/ckan-cloud-operator:jnlp-${TAG}" [ "$?" != "0" ] && echo Failed to tag and push jnlp image && exit 1 + + docker tag ckan-cloud-operator-minikube "viderum/ckan-cloud-operator:minikube-${TAG}" &&\ + echo && echo "viderum/ckan-cloud-operator:minikube-${TAG}" && echo &&\ + docker push "viderum/ckan-cloud-operator:minikube-${TAG}" + [ "$?" != "0" ] && echo Failed to tag and push minikube image && exit 1 + if [ "${TRAVIS_BRANCH}" == "master" ]; then docker tag ckan-cloud-operator viderum/ckan-cloud-operator:latest &&\ echo && echo viderum/ckan-cloud-operator:latest && echo &&\ docker push viderum/ckan-cloud-operator:latest [ "$?" != "0" ] && echo Failed to tag and push latest image && exit 1 + docker tag ckan-cloud-operator-jnlp viderum/ckan-cloud-operator:jnlp-latest &&\ echo && echo viderum/ckan-cloud-operator:jnlp-latest && echo &&\ docker push viderum/ckan-cloud-operator:jnlp-latest [ "$?" != "0" ] && echo Failed to tag and push jnlp latest image && exit 1 + + docker tag ckan-cloud-operator-minikube viderum/ckan-cloud-operator:minikube-latest &&\ + echo && echo viderum/ckan-cloud-operator:minikube-latest && echo &&\ + docker push viderum/ckan-cloud-operator:minikube-latest + [ "$?" != "0" ] && echo Failed to tag and push minikube latest image && exit 1 fi + if [ "${TRAVIS_TAG}" != "" ]; then export DEPLOY_JNLP_IMAGE="viderum/ckan-cloud-operator:jnlp-${TAG}" echo "Running Jenkins deploy jnlp job (JNLP_IMAGE=${DEPLOY_JNLP_IMAGE})" @@ -100,6 +137,7 @@ elif [ "${1}" == "deploy" ]; then echo "jenkins jnlp deploy job status code: ${STATUS_CODE}" [ "${STATUS_CODE}" != "200" ] && [ "${STATUS_CODE}" != "201" ] && echo Deploy failed && exit 1 fi + echo Great Success! && exit 0 else diff --git a/.travis.yml b/.travis.yml index d6b6f309..8d6f653f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,7 +25,7 @@ jobs: - minikube config set cpus 4 && minikube config set memory 8192 && minikube config set vm-driver none && minikube config set kubernetes-version "${K8_VERSION}" - sudo minikube start script: - - ckan-cloud-operator cluster initialize --interactive --cluster-provider=minikube + - ckan-cloud-operator cluster initialize --cluster-provider=minikube - sleep 60 && kubectl get ns && kubectl get pods -n ckan-cloud - ckan-cloud-operator ckan instance create helm --instance-id a-ckan-instance --instance-name a-ckan-instance --update sample-values/values.datagov.yaml after_success: diff --git a/Dockerfile b/Dockerfile index 7cc52bbd..28e8e19d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,7 +7,10 @@ RUN apt-get install -y nano curl unzip sudo bash libpq-dev build-essential ENV EDITOR nano COPY . . -RUN K8_PROVIDER=aws TERRAFORM_VERSION=0.12.18 /cco/.travis.sh install-tools +ARG K8_PROVIDER=aws +ENV K8_PROVIDER=$K8_PROVIDER +ARG K8_PROVIDER_CUSTOM_DOWNLOAD_URL="" +RUN /cco/.travis.sh install-tools RUN pip install . ENTRYPOINT [ "/bin/bash" ] \ No newline at end of file diff --git a/README.md b/README.md index 23db4118..5af34178 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,7 @@ CKAN Cloud Operator manages, provisions and configures CKAN Cloud instances and - AWS - GCP - Azure + - Custom / Self-hosted Provider - Minikube for local development - `ckan-cloud-operator` CLI will manage the cluster and any other services necessary for day-to-day operations - Management server, comes preinstalled with ckan-cloud-operator, required tools (terraform, kubectl, helm, awscli etc.) and [a Jenkins Server](/docs/JENKINS.md). @@ -24,6 +25,8 @@ In order to start using ckan-cloud-operator, you need to - Run the TBD (on Azure) Note: While technically possible, we recommend not to run ckan-cloud-operator directly on you machine to avoid version incompatibilities between the various tools involved in the process. You should use one of our pre-built images or our Docker image instead. + + If you do want to run locally for development, refer to the [local development doc](development/LOCAL-DEVELOPMENT.md) 2. Create a Kubernetes cluster and provision it. - [Instructions for AWS](docs/PRODUCTION-AWS-CLUSTER.md): @@ -38,6 +41,10 @@ In order to start using ckan-cloud-operator, you need to - Create a cluster using terraform - Initialize the cluster using ckan-cloud-operator + - [Instructions for Custom / Self-hosted providers](docs/PRODUCTION-CUSTOM-CLUSTER.md): + - Create servers using custom provisioning tools + - Initialize the cluster using ckan-cloud-operator + - Instructions for Minikube: - Initialize the cluster using ckan-cloud-operator diff --git a/ckan_cloud_operator/config/manager.py b/ckan_cloud_operator/config/manager.py index a3c9614d..0326ab05 100644 --- a/ckan_cloud_operator/config/manager.py +++ b/ckan_cloud_operator/config/manager.py @@ -6,7 +6,6 @@ from ckan_cloud_operator import kubectl from ckan_cloud_operator import logs -from ckan_cloud_operator.providers.cluster import manager as cluster_manager from ckan_cloud_operator.labels import manager as labels_manager @@ -88,7 +87,9 @@ def delete_by_extra_operator_labels(extra_operator_labels): def list_configs(namespace=None, full=False, show_secrets=False): label_prefix = labels_manager.get_label_prefix() - if not namespace: namespace = cluster_manager.get_operator_namespace_name() + if not namespace: + from ckan_cloud_operator.providers.cluster import manager as cluster_manager + namespace = cluster_manager.get_operator_namespace_name() what = 'configmaps' if show_secrets: what += ',secrets' @@ -111,6 +112,7 @@ def list_configs(namespace=None, full=False, show_secrets=False): data['values'] = None yield data + def get_preset_answer(namespace, configmap_name, secret_name, key): interactive_file = os.environ.get('CCO_INTERACTIVE_CI') if not interactive_file: @@ -254,6 +256,7 @@ def _get_labels(cache_key=None, secret_name=None, configmap_name=None, namespace def _get_cache_key(secret_name, configmap_name, namespace): if secret_name: assert not configmap_name, f'Invalid arguments: cannot specify both secret_name and configmap_name: {secret_name}, {configmap_name}' + from ckan_cloud_operator.providers.cluster import manager as cluster_manager configmap_name, namespace = cluster_manager.get_operator_configmap_namespace_defaults(configmap_name, namespace) return f'secret:{namespace}:{secret_name}' if secret_name else f'configmap:{namespace}:{configmap_name}' diff --git a/ckan_cloud_operator/providers/cluster/manager.py b/ckan_cloud_operator/providers/cluster/manager.py index f78cb575..ff912813 100644 --- a/ckan_cloud_operator/providers/cluster/manager.py +++ b/ckan_cloud_operator/providers/cluster/manager.py @@ -7,6 +7,7 @@ from ckan_cloud_operator import kubectl from ckan_cloud_operator import logs from ckan_cloud_operator.labels import manager as labels_manager +from ckan_cloud_operator.config.manager import get_preset_answer from .constants import OPERATOR_NAMESPACE, OPERATOR_CONFIGMAP @@ -48,7 +49,22 @@ def print_info(debug=False, minimal=False): return True +def _initialize_cluster(default_cluster_provider, interactive): + from ckan_cloud_operator.providers import manager as providers_manager + + default_provider_id = default_cluster_provider + if default_cluster_provider.startswith('custom-'): + default_provider_id = default_cluster_provider.replace('custom-', '') + providers_manager.get_provider( + 'cluster', + default=default_provider_id + ).initialize(interactive=interactive) + + def initialize(log_kwargs=None, interactive=False, default_cluster_provider=None, skip_to=None): + if os.environ.get('CCO_INTERACTIVE_CI'): + interactive = True + if interactive: logs.info('Starting interactive initialization of the operator on the following cluster:') print_info(minimal=True) @@ -57,12 +73,27 @@ def initialize(log_kwargs=None, interactive=False, default_cluster_provider=None if not skip_to: logs.info(f'Creating operator namespace: {OPERATOR_NAMESPACE}', **(log_kwargs or {})) - subprocess.call(f'kubectl create ns {OPERATOR_NAMESPACE}', shell=True) - assert default_cluster_provider in ['gcloud', 'aws', 'minikube'], f'invalid cluster provider: {default_cluster_provider}' - subprocess.call(f'kubectl -n {OPERATOR_NAMESPACE} create secret generic ckan-cloud-provider-cluster-{default_cluster_provider}', shell=True) - subprocess.call(f'kubectl -n {OPERATOR_NAMESPACE} create configmap operator-conf --from-literal=ckan-cloud-operator-image=viderum/ckan-cloud-operator:latest --from-literal=label-prefix={OPERATOR_NAMESPACE}', shell=True) + if subprocess.call(f'kubectl get ns {OPERATOR_NAMESPACE}'.split(' ')) != 0: + subprocess.check_call(f'kubectl create ns {OPERATOR_NAMESPACE}'.split(' ')) + assert (default_cluster_provider in ['gcloud', 'aws', 'minikube'] + or default_cluster_provider.startswith('custom-')), \ + f'invalid cluster provider: {default_cluster_provider}' + if subprocess.call(f'kubectl -n {OPERATOR_NAMESPACE} ' + f'get secret ckan-cloud-provider-cluster-{default_cluster_provider}'.split(' ')) != 0: + subprocess.check_call(f'kubectl -n {OPERATOR_NAMESPACE} ' + f'create secret generic ' + f'ckan-cloud-provider-cluster-{default_cluster_provider}'.split(' ')) + if subprocess.call(f'kubectl -n {OPERATOR_NAMESPACE} get configmap operator-conf'.split(' ')) != 0: + try: + ckan_cloud_operator_image = get_preset_answer('default', 'cluster-init', None, + 'ckan-cloud-operator-image') + except Exception: + ckan_cloud_operator_image = 'viderum/ckan-cloud-operator:latest' + subprocess.call(f'kubectl -n {OPERATOR_NAMESPACE} ' + f'create configmap operator-conf ' + f'--from-literal=ckan-cloud-operator-image={ckan_cloud_operator_image} ' + f'--from-literal=label-prefix={OPERATOR_NAMESPACE}'.split(' ')) - from ckan_cloud_operator.providers import manager as providers_manager from ckan_cloud_operator.labels import manager as labels_manager from ckan_cloud_operator.crds import manager as crds_manager from ckan_cloud_operator.providers.db import manager as db_manager @@ -72,22 +103,31 @@ def initialize(log_kwargs=None, interactive=False, default_cluster_provider=None from ckan_cloud_operator.providers.storage import manager as storage_manager from ckan_cloud_operator.providers.apps import manager as apps_manager - for component, func in ( - ('labels', lambda lk: labels_manager.initialize(log_kwargs=lk)), - ('cluster', lambda lk: providers_manager.get_provider('cluster', default=default_cluster_provider).initialize(interactive=interactive)), - ('crds', lambda lk: crds_manager.initialize(log_kwargs=lk)), - ('db', lambda lk: db_manager.initialize(log_kwargs=lk, interactive=interactive, default_cluster_provider=default_cluster_provider)), - ('routers', lambda lk: routers_manager.initialize(interactive=interactive)), - ('solr', lambda lk: solr_manager.initialize(interactive=interactive)), - ('storage', lambda lk: storage_manager.initialize(interactive=interactive)), - ('ckan', lambda lk: ckan_manager.initialize(interactive=interactive)), - ('apps', lambda lk: apps_manager.initialize(interactive=interactive)), - ): + components = dict(( + ('labels', lambda lk: labels_manager.initialize(log_kwargs=lk)), + ('cluster', lambda lk: _initialize_cluster(default_cluster_provider, interactive)), + ('crds', lambda lk: crds_manager.initialize(log_kwargs=lk)), + ('db', lambda lk: db_manager.initialize(log_kwargs=lk, interactive=interactive, + default_cluster_provider=default_cluster_provider)), + ('routers', lambda lk: routers_manager.initialize(interactive=interactive)), + ('solr', lambda lk: solr_manager.initialize(interactive=interactive)), + ('storage', lambda lk: storage_manager.initialize(interactive=interactive)), + ('ckan', lambda lk: ckan_manager.initialize(interactive=interactive)), + ('apps', lambda lk: apps_manager.initialize(interactive=interactive)), + )) + + try: + selected_components = list(get_preset_answer('default', 'cluster-init', None, 'components')) + except Exception: + selected_components = list(components.keys()) + + for component, func in components.items(): if not skip_to or skip_to == component: skip_to = None - log_kwargs = {'cluster-init': component} - logs.info(f'Initializing', **log_kwargs) - func(log_kwargs) + if component in selected_components: + log_kwargs = {'cluster-init': component} + logs.info(f'Initializing', **log_kwargs) + func(log_kwargs) def get_kubeconfig_info(): diff --git a/ckan_cloud_operator/providers/helpers.py b/ckan_cloud_operator/providers/helpers.py new file mode 100644 index 00000000..2868cc5b --- /dev/null +++ b/ckan_cloud_operator/providers/helpers.py @@ -0,0 +1,36 @@ +def get_provider_functions(PROVIDER_SUBMODULE, PROVIDER_ID): + from ckan_cloud_operator.providers import manager as providers_manager + + def _get_resource_name(suffix=None): + return providers_manager.get_resource_name(PROVIDER_SUBMODULE, PROVIDER_ID, + suffix=suffix) + + def _get_resource_labels(for_deployment=False): + return providers_manager.get_resource_labels(PROVIDER_SUBMODULE, PROVIDER_ID, + for_deployment=for_deployment) + + def _get_resource_annotations(suffix=None): + return providers_manager.get_resource_annotations(PROVIDER_SUBMODULE, PROVIDER_ID, + suffix=suffix) + + def _set_provider(): + providers_manager.set_provider(PROVIDER_SUBMODULE, PROVIDER_ID) + + def _config_set(key=None, value=None, values=None, namespace=None, is_secret=False, suffix=None): + providers_manager.config_set(PROVIDER_SUBMODULE, PROVIDER_ID, + key=key, value=value, + values=values, namespace=namespace, is_secret=is_secret, + suffix=suffix) + + def _config_get(key=None, default=None, required=False, namespace=None, is_secret=False, suffix=None): + return providers_manager.config_get(PROVIDER_SUBMODULE, PROVIDER_ID, + key=key, default=default, required=required, + namespace=namespace, is_secret=is_secret, + suffix=suffix) + + def _config_interactive_set(default_values, namespace=None, is_secret=False, suffix=None, from_file=False): + providers_manager.config_interactive_set(PROVIDER_SUBMODULE, PROVIDER_ID, + default_values, namespace, is_secret, suffix, from_file) + + return (_config_interactive_set, _config_get, _config_set, _set_provider, _get_resource_annotations, + _get_resource_labels, _get_resource_name) diff --git a/ckan_cloud_operator/providers/manager.py b/ckan_cloud_operator/providers/manager.py index fa31f8d7..b657cc95 100644 --- a/ckan_cloud_operator/providers/manager.py +++ b/ckan_cloud_operator/providers/manager.py @@ -2,6 +2,7 @@ from ckan_cloud_operator.config import manager as config_manager from ckan_cloud_operator.labels import manager as labels_manager from ckan_cloud_operator.annotations import manager as annotations_manager +import importlib def get_provider(submodule, required=True, supported_provider_ids=None, default=None, verbose=False, provider_id=None): @@ -350,6 +351,11 @@ def _get_submodule_ids_provider_or_provider_ids(submodule=None, provider_id=None return solr_solrcloud_manager + try: + return importlib.import_module(f'cco_provider_{provider_id}.manager') + except ModuleNotFoundError as e: + logs.warning(str(e)) + if not provider_id: return [] else: diff --git a/docs/PRODUCTION-AWS-CLUSTER.md b/docs/PRODUCTION-AWS-CLUSTER.md index 964ae05f..5f5684bf 100644 --- a/docs/PRODUCTION-AWS-CLUSTER.md +++ b/docs/PRODUCTION-AWS-CLUSTER.md @@ -57,7 +57,7 @@ For an unattended initialization, using the outputs of `terraform apply`, run: ```bash export CCO_INTERACTIVE_CI=interactive.yaml -ckan-cloud-operator cluster initialize --interactive --cluster-provider=aws +ckan-cloud-operator cluster initialize --cluster-provider=aws ``` You can also run the same without setting the `CCO_INTERACTIVE_CI` environment variable for an interactive initialization session. diff --git a/docs/PRODUCTION-CUSTOM-CLUSTER.md b/docs/PRODUCTION-CUSTOM-CLUSTER.md new file mode 100644 index 00000000..54082d94 --- /dev/null +++ b/docs/PRODUCTION-CUSTOM-CLUSTER.md @@ -0,0 +1,19 @@ +# Creating a Production CKAN Cloud cluster using a custom / self-hosted provider + +## Prerequisites + +* Custom Provider values, check the custom provider docs for details + * e.g. `https://github.com/OriHoch/cco-provider-kamatera/blob/master/README.md` +* An external domain +* A CKAN Cloud Operator [working environment](./WORKING-ENVIRONMENT.md) + +## Provision the cluster + +Check the custom provider docs for creating the cluster. + +Once the cluster was created on the custom provider, you can continue initializing ckan-cloud-operator: + +``` +CCO_INTERACTIVE_CI=/path/to/interactive.yaml \ + ckan-cloud-operator cluster initialize --cluster-provider=PROVIDER_NAME +``` diff --git a/docs/WORKING-ENVIRONMENT.md b/docs/WORKING-ENVIRONMENT.md index 5d0a16e1..cd776d68 100644 --- a/docs/WORKING-ENVIRONMENT.md +++ b/docs/WORKING-ENVIRONMENT.md @@ -6,10 +6,17 @@ In order to ensure that there are no version incompatibilities among the differe ## Using the Docker images -To use the docker image run this from the command line: +Choose a Docker image for your target environment: + +* AWS: `viderum/ckan-cloud-operator` +* Minikube: `viderum/ckan-cloud-operator:minikube-latest` +* Custom providers: + * Kamatera: `orihoch/ckan-cloud-operator:kamatera-latest` + +Run the selected docker image: ```bash -$ docker run -it -v $PWD/.cco:/root/ viderum/ckan-cloud-operator:latest +$ docker run -it -v $PWD/.cco:/root/ viderum/ckan-cloud-operator ``` We are mapping a directory named `.cco` in the current working directory to store important state (e.g. kubectl config, terraform state), but you can choose to map a different directory on you machine. diff --git a/docs/development/LOCAL-DEVELOPMENT-WIP.md b/docs/development/LOCAL-DEVELOPMENT-WIP.md deleted file mode 100644 index 4fa58547..00000000 --- a/docs/development/LOCAL-DEVELOPMENT-WIP.md +++ /dev/null @@ -1,120 +0,0 @@ - -## Install - -`ckan-cloud-operator-env` is used to install and manage CKAN Cloud operator environments on your local PC - -**Prerequisites:** - -* `.kube-config` file with permissions to the relevant cluster - -Install latest ckan-cloud-operator-env - -``` -curl -s https://raw.githubusercontent.com/datopian/ckan-cloud-operator/master/ckan-cloud-operator-env.sh \ -| sudo tee /usr/local/bin/ckan-cloud-operator-env >/dev/null && sudo chmod +x /usr/local/bin/ckan-cloud-operator-env -``` - -Add an environment (sudo is required to install the executable) - -``` -sudo ckan-cloud-operator-env add -``` - -Verify conection to the cluster and installed operator version (may take a while on first run or after upgrade of operator version) - -``` -ckan-cloud-operator cluster info -``` - -**Important** Re-run the add command and cluster info to verify compatible version is installed. - -## Usage - -Use the CLI help messages for the reference documentation and usage examples. - -``` -ckan-cloud-operator --help -ckan-cloud-operator deis-instance --help -. -. -``` - -You can use bash completion inside this shell - -``` -ckan-cloud-operator -``` - - -## Managing multiple environments - -ckan-cloud-operator-env supports managing multiple environments - -Add environments using `ckan-cloud-operator-env add ` - -Each environment is accessible using executable `ckan-cloud-operator-` - -Activating an environment sets the `ckan-cloud-operator` executable to use to the relevant environment executable - -``` -ckan-cloud-operator-env activate -``` - -## Run ckan-cloud-operator locally - -Ensure you have `kubectl` and `gcloud` binaries, authenticated to the relevant gcloud account / kubernetes cluster. - -See the required system dependencies: [environment.yaml](environment.yaml) - -You can [Install miniconda3](https://conda.io/miniconda.html), then create the environment using: `conda env create -f environment.yaml` - -Activate the conda environment using `conda activate ckan-cloud-operator` - -Install the Python package: - -``` -python3 -m pip install -e . -``` - -Authenticate the gcloud CLI to the relevant account: - -``` -ckan-cloud-operator activate-gcloud-auth -``` - -Run ckan-cloud-operator without arguments to get a help message: - -``` -ckan-cloud-operator -``` - -Enable Bash completion - -``` -eval "$(ckan-cloud-operator bash-completion)" -``` - -## Using Jupyter Lab - -Jupyter Lab can be used to run bulk operations or aggregate data from ckan-cloud-operator - -You should run ckan-cloud-operator locally and run the following from the activated ckan-cloud-operator environment - -Install jupyterlab - -``` -python3 -m pip install jupyterlab -``` - -Run jupyterlab - -``` -jupyter lab -``` - -Open notebooks from the `notebooks` directory - -## Run tests -If you already have `ckan-cloud-operator` executable in your PATH, you could run test suite with `ckan-cloud-operator test` command. - -The other way to run test suite is `coverage run -m unittest discover`. diff --git a/docs/development/LOCAL-DEVELOPMENT.md b/docs/development/LOCAL-DEVELOPMENT.md new file mode 100644 index 00000000..bd76809a --- /dev/null +++ b/docs/development/LOCAL-DEVELOPMENT.md @@ -0,0 +1,114 @@ +## Local Development + +### Using the Docker image + +This is the preferred solution as it ensures you have a consistent environment. + +Build the image using the command for your target environment: + +* AWS: + +``` +docker build -t ckan-cloud-operator . +``` + +* Minikube: + +``` +docker build -t ckan-cloud-operator \ + --build-arg K8_PROVIDER=minikube \ + . +``` + +Custom (e.g. using the `kamatera` custom provider): + +``` +docker build -t ckan-cloud-operator \ + --build-arg K8_PROVIDER=custom-kamatera \ + --build-arg K8_PROVIDER_CUSTOM_DOWNLOAD_URL=https://github.com/OriHoch/cco-provider-kamatera/archive/v0.0.1.tar.gz \ + . +``` + +Start a bash shell: + +``` +docker run -it -v $PWD/.cco:/root/ -v $PWD:/cco ckan-cloud-operator +``` + +For development of custom providers, add the following argument: + +``` +-v /path/to/custom-provider-repo:/usr/local/lib/cco/$K8_PROVIDER +``` + +Install the Python package for development: + +``` +pip install -e . +``` + +For custom providers, install the custom provider package: + +``` +pip install -e /usr/local/lib/cco/$K8_PROVIDER +``` + +Run ckan-cloud-operator commands, any changes in .py files will be reflected inside the container: + +``` +ckan-cloud-operator --help +``` + +Enable Bash completion + +``` +eval "$(ckan-cloud-operator bash-completion)" +``` + +### Without Docker + +Ensure you have all required dependencies according to the Dockerfile, related scripts and build-args for the target environment. + +Install the Python package: + +``` +pip install -e . +``` + +Run ckan-cloud-operator commands + +``` +ckan-cloud-operator --help +``` + +Enable Bash completion + +``` +eval "$(ckan-cloud-operator bash-completion)" +``` + +### Using Jupyter Lab + +Jupyter Lab can be used to run bulk operations or aggregate data from ckan-cloud-operator + +You should run ckan-cloud-operator locally and run the following from the activated ckan-cloud-operator environment + +Install jupyterlab + +``` +pip install jupyterlab +``` + +Run jupyterlab + +``` +jupyter lab +``` + +Open notebooks from the `notebooks` directory + +### Run tests + +If you already have `ckan-cloud-operator` executable in your PATH, you could run test suite with `ckan-cloud-operator test` command. + +The other way to run test suite is `coverage run -m unittest discover`. diff --git a/interactive.yaml b/interactive.yaml index 43b8e99e..724065d8 100644 --- a/interactive.yaml +++ b/interactive.yaml @@ -1,5 +1,17 @@ default: config: + cluster-init: + ckan-cloud-operator-image: viderum/ckan-cloud-operator + components: + - labels + - cluster + - crds + - db + - routers + - solr + - storage + - ckan + - apps routers-config: env-id: p default-root-domain: localhost diff --git a/terraform/aws/ami/cco-test.json b/terraform/aws/ami/cco-test.json index ee9084eb..8230d2d3 100644 --- a/terraform/aws/ami/cco-test.json +++ b/terraform/aws/ami/cco-test.json @@ -56,7 +56,7 @@ "pyenv global 3.7.6", "python --version", "pip --version", - "K8_PROVIDER=aws TERRAFORM_VERSION=0.12.18 /home/ubuntu/.travis.sh install-tools", + "K8_PROVIDER=aws /home/ubuntu/.travis.sh install-tools", "python3 -m pip install /home/ubuntu/" ] } diff --git a/terraform/aws/ami/cco.json b/terraform/aws/ami/cco.json index 1b3cb555..351d51ba 100644 --- a/terraform/aws/ami/cco.json +++ b/terraform/aws/ami/cco.json @@ -54,7 +54,7 @@ "pyenv global 3.7.6", "python --version", "pip --version", - "K8_PROVIDER=aws TERRAFORM_VERSION=0.12.18 /home/ubuntu/.travis.sh install-tools", + "K8_PROVIDER=aws /home/ubuntu/.travis.sh install-tools", "python3 -m pip install -e /home/ubuntu/", "sudo sh -c \"echo 'jenkins ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers.d/91-jenkins\"" ] diff --git a/terraform/aws/init_cluster.sh b/terraform/aws/init_cluster.sh index 6a3bead8..2a64dd9e 100755 --- a/terraform/aws/init_cluster.sh +++ b/terraform/aws/init_cluster.sh @@ -38,4 +38,4 @@ CREDENTIALS export CCO_INTERACTIVE_CI=interactive.yaml cp ./kubeconfig_terraform-cco ~/.kube/config cp terraform.tfstate ~/ -ckan-cloud-operator cluster initialize --interactive --cluster-provider=aws +ckan-cloud-operator cluster initialize --cluster-provider=aws