Initial attempts at packaging for OperatorHub / OLM

.github/workflows/release.yml

@@ -24,6 +24,11 @@ jobs:
           AWS_ACCESS_KEY_ID: ${{ secrets.ECR_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.ECR_SECRET }}
         run: aws ecr get-login-password --region us-east-1 | skopeo login -u AWS --password-stdin ${ECR_REPO}
+      - name: Login Skopeo Red Hat
+        env:
+          REDHAT_REPO: ${{ secrets.REDHAT_REPO }}
+          REDHAT_SECRET: ${{ secrets.REDHAT_SECRET }}
+        run: docker login -u unused --password "$REDHAT_SECRET" $REDHAT_REPO
       - name: Publish to Dockerhub
         run: |
           ./scripts/push-release.sh

docs/developer/build-redhat-bundles.fish

@@ -0,0 +1,22 @@
+set VERSIONS 1.0.0 1.1.0 1.2.0 1.3.0 1.4.0 1.4.1 1.5.0 1.5.1
+for VERSION in $VERSIONS
+  echo "Version: $VERSION"
+  echo "docker build -t bradfordcp/cass-operator-bundle:$VERSION -f bundle-$VERSION.Dockerfile ."
+  docker build -t harbor.sjc.dsinternal.org/cass-operator/cass-operator-bundle:$VERSION -f bundle-$VERSION.Dockerfile .
+
+  echo "docker push harbor.sjc.dsinternal.org/cass-operator/cass-operator-bundle:$VERSION"
+  docker push harbor.sjc.dsinternal.org/cass-operator/cass-operator-bundle:$VERSION
+end
+
+
+set BUNDLELIST ""
+
+for VERSION in $VERSIONS
+  set BUNDLELIST $BUNDLELIST,harbor.sjc.dsinternal.org/cass-operator/cass-operator-bundle:$VERSION
+end
+# Remove ',' from start of bundlelist
+set BUNDLELIST (string sub -s 2 $BUNDLELIST)
+
+echo "opm index add --bundles $BUNDLELIST --tag harbor.sjc.dsinternal.org/cass-operator/cass-operator-index:latest -u docker"
+opm index add --bundles $BUNDLELIST --tag harbor.sjc.dsinternal.org/cass-operator/cass-operator-index:latest -u docker
+docker push harbor.sjc.dsinternal.org/cass-operator/cass-operator-index:latest

docs/developer/csv-description.md

@@ -0,0 +1,106 @@
+## Apache Cassandra
+
+The Apache Cassandra database is the right choice when you need scalability and
+high availability without compromising performance. Linear scalability and
+proven fault-tolerance on commodity hardware or cloud infrastructure make it the
+perfect platform for mission-critical data. Cassandra's support for replicating
+across multiple datacenters is best-in-class, providing lower latency for your
+users and the peace of mind of knowing that you can survive regional outages.
+
+## DataStax Enterprise
+
+The most advanced distribution of Apache Cassandra™ on the market, with the
+enterprise functionality needed for serious production systems and backed up and
+supported by the best distributed-experts in the world. It's one platform for
+all types of applications anywhere, any cloud, any model: key-value, graph,
+tabular, JSON.
+
+DataStax Enterprise is a fully integrated and optimized database, with graph,
+analytics, and search included, all with a unified security model. Simply put,
+it's the only database capable of meeting today's demanding requirements
+
+## Operator Details
+
+`cass-operator` is designed as a modular operator for Apache Cassandra and
+derived  distributions. Apache Cassandra is a distributed database consisting of
+multiple nodes working in concert to store data and process queries along a
+number of fault domains. `cass-operator` has the deployment of a Cassandra
+cluster around the logical domain of a datacenter with the `CassandraDatacenter`
+custom resource. Upon submission of one of these resources it handles
+provisioning the underlying stateful sets (analogous to C\* logical racks),
+services, and configuration. Additionally through monitoring pod state via
+Kubernetes callbacks it handles day to day operations such as restarting failed
+processes, scaling clusters up, and deploying configuration changes in a
+rolling, non-disruptive, fashion. This operator is designed to be `Namespace`
+scoped. A single Kubernetes cluster may be running multiple instances of this
+operator, in separate namespaces, to support a number of C\* clusters and
+environments. Configuration is simple with the usage of YAML based overrides in
+the Custom Resource paired with an `init` container. In C\* clusters ordering and
+timing of certain operations are important to keep the system evenly
+distributed. `cass-operator` takes advantage of a sidecar process within the
+main container to handle the orchestration of starting our main C* process.
+
+## Pre-requisites
+
+### WebHook Secret
+
+This secret is intentionally left blank and will be filled in as the operator is
+brought is started and certificates are generated.
+
+```yaml
+apiVersion: v1
+data:
+  tls.crt: ""
+  tls.key: ""
+kind: Secret
+metadata:
+  name: cass-operator-webhook-config
+```
+
+### WebHook Configuration
+
+Note the namespace _must_ be updated in the configuration below.
+
+```yaml
+apiVersion: admissionregistration.k8s.io/v1beta1
+kind: ValidatingWebhookConfiguration
+metadata:
+  name: "cassandradatacenter-webhook-registration"
+webhooks:
+- name: "cassandradatacenter-webhook.cassandra.datastax.com"
+  rules:
+  - apiGroups:   ["cassandra.datastax.com"]
+    apiVersions: ["v1beta1"]
+    operations:  ["CREATE", "UPDATE", "DELETE"]
+    resources:   ["cassandradatacenters"]
+    scope:       "*"
+  clientConfig:
+    service:
+      name: "cassandradatacenter-webhook-service"
+      namespace: "cass-operator"
+      path: /validate-cassandra-datastax-com-v1beta1-cassandradatacenter
+  admissionReviewVersions: ["v1beta1"]
+  failurePolicy: "Ignore"
+  matchPolicy: "Equivalent"
+  sideEffects: None
+  timeoutSeconds: 10
+```
+
+### WebHook Service
+
+Note the namespace _must_ be updated in the configuration below.
+
+```yaml
+apiVersion: v1
+kind: Service
+metadata:
+  name: cassandradatacenter-webhook-service
+  labels:
+    name: cass-operator-webhook
+spec:
+  ports:
+  - port: 443
+    targetPort: 8443
+  selector:
+    name: cass-operator
+```

docs/developer/olm/catalogsource.yaml

@@ -0,0 +1,10 @@
+apiVersion: operators.coreos.com/v1alpha1
+kind: CatalogSource
+metadata:
+  name: local
+  namespace: openshift-marketplace
+spec:
+  displayName: Local Catalog
+  publisher: Developer
+  sourceType: grpc
+  image: harbor.sjc.dsinternal.org/cass-operator/cass-operator-index:latest

docs/developer/olm/operatorgroup.yaml

@@ -0,0 +1,8 @@
+apiVersion: operators.coreos.com/v1alpha2
+kind: OperatorGroup
+metadata:
+  name: group
+  namespace: test-operator
+spec:
+  targetNamespaces:
+  - test-operator

docs/developer/olm/persistentvolume.yaml

@@ -0,0 +1,17 @@
+kind: PersistentVolume
+apiVersion: v1
+metadata:
+  name: cass-operator-test-pv
+spec:
+  capacity:
+    storage: 100Gi
+  hostPath:
+    path: /mnt/pv-data/cass-op-pv
+    type: ''
+  accessModes:
+    - ReadWriteOnce
+    - ReadWriteMany
+    - ReadOnlyMany
+  persistentVolumeReclaimPolicy: Recycle
+  volumeMode: Filesystem
+  storageClassName: no-provisioner

docs/developer/olm/prereqs.yaml

@@ -0,0 +1,52 @@
+---
+  apiVersion: v1
+  data:
+    tls.crt: ""
+    tls.key: ""
+  kind: Secret
+  metadata:
+    name: cass-operator-webhook-config
+    namespace: test-operator
+---
+apiVersion: admissionregistration.k8s.io/v1beta1
+kind: ValidatingWebhookConfiguration
+metadata:
+  name: cassandradatacenter-webhook-registration
+webhooks:
+- admissionReviewVersions:
+  - v1beta1
+  clientConfig:
+    service:
+      name: cassandradatacenter-webhook-service
+      namespace: test-operator
+      path: /validate-cassandra-datastax-com-v1beta1-cassandradatacenter
+  failurePolicy: Ignore
+  matchPolicy: Equivalent
+  name: cassandradatacenter-webhook.cassandra.datastax.com
+  rules:
+  - apiGroups:
+    - cassandra.datastax.com
+    apiVersions:
+    - v1beta1
+    operations:
+    - CREATE
+    - UPDATE
+    resources:
+    - cassandradatacenters
+    scope: '*'
+  sideEffects: None
+  timeoutSeconds: 10
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    name: cass-operator-webhook
+  name: cassandradatacenter-webhook-service
+  namespace: test-operator
+spec:
+  ports:
+  - port: 443
+    targetPort: 8443
+  selector:
+    name: cass-operator

docs/developer/olm/storageclass.yaml

@@ -0,0 +1,7 @@
+kind: StorageClass
+apiVersion: storage.k8s.io/v1
+metadata:
+  name: no-provisioner
+provisioner: kubernetes.io/no-provisioner
+reclaimPolicy: Delete
+volumeBindingMode: WaitForFirstConsumer

docs/developer/olm/subscription.yaml

@@ -0,0 +1,10 @@
+apiVersion: operators.coreos.com/v1alpha1
+kind: Subscription
+metadata:
+  name: cass-operator-subscription
+  namespace: test-operator 
+spec:
+  channel: stable
+  name: cass-operator
+  source: local
+  sourceNamespace: openshift-marketplace

docs/developer/red-hat-packaging.md

@@ -0,0 +1,141 @@
+# Packaging for Operator Hub / Red Hat
+
+## Tools
+* `opm`
+  1. Checkout a copy of [operator-framework/operator-registry](https://github.com/operator-framework/operator-registry)
+  2. Build with `make build`
+  3. Binary resides at `bin/opm`
+* `k3d`
+
+### Setup Test Cluster
+
+Spin up local cluster for testing and install OLM
+
+```console
+k3d cluster create
+kubectl create namespace test-operator
+operator-sdk olm install
+
+# Optional
+kubectl delete catalogsource operatorhubio-catalog -n olm
+```
+
+### Unsupported Resource Types
+The following types may not be included in the operator bundle
+
+1. Namespace
+2. Secret
+3. ValidatingWebhookConfiguration
+4. Service
+
+## Common Issues
+* `runAsUser` set to `999` in the deployment - OpenShift prefers to set a randomly assigned user at container start time. If this field is not left empty the user field must be set extremely high. 999 is too low.
+* Prerequisite custom resources have not been included in the appropriate section of the operator metadata testing page
+* The package has not been marked as published in the Red Hat repo.
+
+# Upgrade Workflow
+
+_Note see Red Hat's [Gitbook](https://redhat-connect.gitbook.io/partner-guide-for-red-hat-openshift-and-container/certify-your-operator/upgrading-your-operator)_
+
+1. Copy an existing version as the base
+
+    ```console
+    OLD_VERSION=1.5.1
+    NEW_VERSION=1.6.0
+    cd operator/bundle
+    cp -r $OLD_VERSION $NEW_VERSION
+    cp bundle-$OLD_VERSION.Dockerfile bundle-$NEW_VERSION.Dockerfile
+    ```
+2. Rename files with version numbers
+
+    ```console
+    mv $NEW_VERSION/manifests/cass-operator.v$OLD_VERSION.clusterserviceversion.yaml $NEW_VERSION/manifests/cass-operator.v$NEW_VERSION.clusterserviceversion.yaml
+    ```
+
+3. Update version numbers in ClusterServiceVersion files
+
+    ```console
+    sed -i s/"$OLD_VERSION"/$NEW_VERSION/g $NEW_VERSION/manifests/cass-operator.v$NEW_VERSION.clusterserviceversion.yaml
+    sed -i s/"$OLD_VERSION"/$NEW_VERSION/g bundle-$NEW_VERSION.Dockerfile
+    ```
+
+4. Copy in updated CRD and patch
+
+    ```console
+    cp ../deploy/crds/* $NEW_VERSION/manifests/
+    patch $NEW_VERSION/manifests/cassandra.datastax.com_cassandradatacenters_crd.yaml ../../mage/operator/crd.patch
+    rm $NEW_VERSION/manifests/cassandra.datastax.com_cassandradatacenters_crd.yaml.orig
+    ```
+
+5. Compare the old and new CSVs for differences and update `$NEW_VERSION/manifests/cass-operator.v$NEW_VERSION.clusterserviceversion.yaml`
+   1. Update `metadata.annotations.containerImage` version
+   2. Update `metadata.annotations.createdAt` datestamp
+   3. Update `metadata.annotations.name` field
+   4. Update `spec.install.spec.deployments[0].template.spec.containers[0].image` version
+   5. Update `spec.customresourcedefinitions.owned[0].specDescriptors` to include any new spec fields. See [Descriptor](https://github.com/openshift/console/blob/master/frontend/packages/operator-lifecycle-manager/src/components/descriptors/reference/reference.md) [Documentation](https://github.com/openshift/console/tree/release-4.3/frontend/packages/operator-lifecycle-manager/src/components/descriptors).
+   6. Update `spec.customresourcedefinitions.owned[0].statusDescriptors` to include any new status fields
+   7. Update `spec.replaces` to replace previous version
+   8. Update `spec.version` with new version value
+6. Build bundle container and push to staging repo
+
+    ```console
+    docker build -t harbor.sjc.dsinternal.org/marketplace/cass-operator-bundle:$NEW_VERSION -f bundle-$NEW_VERSION.Dockerfile .    
+    docker push harbor.sjc.dsinternal.org/marketplace/cass-operator-bundle:$NEW_VERSION
+    ```
+7. Build local catalog index for testing
+
+    ```console
+    opm index add --bundles harbor.sjc.dsinternal.org/marketplace/cass-operator-bundle:1.0.0,harbor.sjc.dsinternal.org/marketplace/cass-operator-bundle:1.2.0,harbor.sjc.dsinternal.org/marketplace/cass-operator-bundle:1.3.0,harbor.sjc.dsinternal.org/marketplace/cass-operator-bundle:1.4.0,harbor.sjc.dsinternal.org/marketplace/cass-operator-bundle:1.4.1,harbor.sjc.dsinternal.org/marketplace/cass-operator-bundle:1.5.0,harbor.sjc.dsinternal.org/marketplace/cass-operator-bundle:1.5.1,harbor.sjc.dsinternal.org/marketplace/cass-operator-bundle:1.6.0 --tag harbor.sjc.dsinternal.org/marketplace/catalog-index:1.6.0 -u docker
+    docker tag harbor.sjc.dsinternal.org/marketplace/catalog-index:1.6.0 harbor.sjc.dsinternal.org/marketplace/catalog-index:latest
+
+    docker push harbor.sjc.dsinternal.org/marketplace/catalog-index:1.6.0
+    docker push harbor.sjc.dsinternal.org/marketplace/catalog-index:latest
+    ```
+8. Add index as a [`CatalogSource`](olm/catalogsource.yaml) in k8s
+
+    ```console
+    kubectl apply -f docs/developer/olm/catalogsource.yaml
+    ```
+9. Verify packagemanifests are being pulled from index
+
+    ```console
+    kubectl describe packagemanifests cass-operator -n olm
+    ```
+10. Create an [`OperatorGroup`](olm/operatorgroup.yaml) to tell cass-operator where to watch for CassDC instances
+
+    ```console
+    kubectl apply -f docs/developer/olm/operatorgroup.yaml
+    ```
+11. Install prereqs that OLM doesn't handle
+
+    ```console
+    kubectl apply -f docs/developer/olm/prereqs.yaml
+    ```
+12. Install cass-operator with OLM via a [`Subscription`](olm/subscription.yaml)
+
+    ```console
+    kubectl apply -f docs/developer/olm/subscription.yaml
+    ```
+13. Check the `InstallPlan` to see the operator successfully installed
+
+    ```console
+    kubectl describe installplan -n test-operator | less
+    ```
+
+    If there is a failure, fix the issue locally, remove the `Subscription` and `CatalogSource`. Goto step 6 and repackage everything.
+14. Check the operator is running
+
+    ```console
+    kubectl get deployments -n test-operator
+    kubectl get pods -n test-operator
+    ```
+14. Provision a sample cassdc in the default namespace
+15. Push to Red Hat
+
+    ```console
+    docker tag bradfordcp/cass-operator-bundle:$NEW_VERSION $REDHAT_REGISTRY/cass-operator-bundle:$NEW_VERSION
+    docker push $REDHAT_REGISTRY/cass-operator-bundle:$NEW_VERSION
+    ```
+16. Red Hat automatically runs certification tests on push. These take 1-2 hours
+17. Login to the Red Hat project and verify certification results. Repeat steps 5-7 until certification passes
+18. Publish certified bundle