CLOUDP-358677: Add Flex e2e2 test

Makefile

@@ -491,10 +491,13 @@ post-install-hook:
 x509-cert: ## Create X.509 cert at path tmp/x509/ (see docs/x509-user.md)
 	go run scripts/create_x509.go
 
-clean: ## Clean built binaries
+.PHONY: clean-gen-crds
+clean-gen-crds: ## Clean only generated CRD files
+	rm -f config/generated/crd/bases/crds.yaml
+
+clean: clean-gen-crds ## Clean built binaries
 	rm -rf bin/*
 	rm -rf config/manifests/bases/
-	rm -f config/generated/crd/bases/crds.yaml
 	rm -f config/crd/bases/*.yaml
 	rm -f helm-charts/atlas-operator-crds/templates/*.yaml
 	rm -f config/rbac/clusterwide/role.yaml
@@ -607,7 +610,7 @@ clear-e2e-leftovers: ## Clear the e2e test leftovers quickly
 install-crds: manifests ## Install CRDs in Kubernetes
 	kubectl apply -k config/crd
 ifdef EXPERIMENTAL
-	$(MAKE) clean gen-crds
+	$(MAKE) regen-crds
 	kubectl apply -f config/generated/crd/bases/crds.yaml
 endif
 
@@ -881,6 +884,9 @@ gen-crds: tools/openapi2crd/bin/openapi2crd
 	--output $(realpath .)/config/generated/crd/bases/crds.yaml
 	cp $(realpath .)/config/generated/crd/bases/crds.yaml $(realpath .)/internal/generated/crds/crds.yaml
 
+.PHONY: regen-crds
+regen-crds: clean-gen-crds gen-crds ## Clean and regenerate CRDs
+
 gen-go-types:
 	@echo "==> Generating Go models from CRDs..."
 	$(CRD2GO) --input $(realpath .)/config/generated/crd/bases/crds.yaml \

config/samples/atlas_generated_v1_flexcluster.yaml

@@ -0,0 +1,16 @@
+apiVersion: atlas.generated.mongodb.com/v1
+kind: FlexCluster
+metadata:
+  name: flexy
+spec:
+  connectionSecretRef:
+    name: mongodb-atlas-operator-api-key
+  v20250312:
+    groupId: "60f1b3c4e4b0e8b8c8b8c8b"
+    entry:
+      name: flexy
+      terminationProtectionEnabled: true
+      providerSettings:
+        backingProviderName: GCP
+        regionName: CENTRAL_US
+

config/samples/atlas_generated_v1_flexcluster_with_groupref.yaml

@@ -0,0 +1,29 @@
+apiVersion: atlas.generated.mongodb.com/v1
+kind: Group
+metadata:
+  name: my-group-for-flexcluster
+spec:
+  connectionSecretRef:
+    name: mongodb-atlas-operator-api-key
+  v20250312:
+    entry:
+      orgId: "60f1b3c4e4b0e8b8c8b8c8b"
+      name: my-group-for-flexcluster
+---
+apiVersion: atlas.generated.mongodb.com/v1
+kind: FlexCluster
+metadata:
+  name: flexy-with-groupref
+  annotations:
+    some-tag: tag
+spec:
+  v20250312:
+    groupRef:
+      name: my-group-for-flexcluster
+    entry:
+      name: flexy-with-groupref
+      terminationProtectionEnabled: true
+      providerSettings:
+        backingProviderName: GCP
+        regionName: CENTRAL_US
+

config/samples/atlas_generated_v1_group.yaml

@@ -0,0 +1,12 @@
+apiVersion: atlas.generated.mongodb.com/v1
+kind: Group
+metadata:
+  name: my-group
+spec:
+  connectionSecretRef:
+    name: mongodb-atlas-operator-api-key
+  v20250312:
+    entry:
+      orgId: "60f1b3c4e4b0e8b8c8b8c8b"
+      name: my-group
+

config/samples/kustomization.yaml

@@ -20,4 +20,7 @@ resources:
   - atlas_v1_atlasbackupcompliancepolicy.yaml
   - atlas_v1_atlascustomrole.yaml
   - atlas_v1_atlasthirdpartyintegration.yaml
+  - atlas_generated_v1_group.yaml
+  - atlas_generated_v1_flexcluster.yaml
+  - atlas_generated_v1_flexcluster_with_groupref.yaml
 # +kubebuilder:scaffold:manifestskustomizesamples

internal/generated/controller/flexcluster/handler_v20250312.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 
 	v20250312sdk "go.mongodb.org/atlas-sdk/v20250312009/admin"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	controllerruntime "sigs.k8s.io/controller-runtime"
 	builder "sigs.k8s.io/controller-runtime/pkg/builder"
 	client "sigs.k8s.io/controller-runtime/pkg/client"
@@ -71,6 +72,26 @@ func (h *Handlerv20250312) getDependencies(ctx context.Context, flexcluster *ako
 	return result, nil
 }
 
+// getMinimalGroupFromStatusOrSpec creates a minimal Group object with group ID from status (preferred) or spec (fallback).
+// Returns nil if no group ID is available. This allows deletion to proceed even if the Group CR is gone from Kubernetes.
+func (h *Handlerv20250312) getMinimalGroupFromStatusOrSpec(flexcluster *akov2generated.FlexCluster) *akov2generated.Group {
+	var groupID *string
+	if flexcluster.Status.V20250312 != nil {
+		groupID = flexcluster.Status.V20250312.GroupId
+	}
+	if groupID == nil && flexcluster.Spec.V20250312 != nil {
+		groupID = flexcluster.Spec.V20250312.GroupId
+	}
+	if groupID == nil || *groupID == "" {
+		return nil
+	}
+	return &akov2generated.Group{
+		Status: akov2generated.GroupStatus{
+			V20250312: &akov2generated.GroupStatusV20250312{Id: groupID},
+		},
+	}
+}
+
 // HandleInitial handles the initial state for version v20250312
 func (h *Handlerv20250312) HandleInitial(ctx context.Context, flexcluster *akov2generated.FlexCluster) (ctrlstate.Result, error) {
 	deps, err := h.getDependencies(ctx, flexcluster)
@@ -165,7 +186,18 @@ func (h *Handlerv20250312) HandleDeletionRequested(ctx context.Context, flexclus
 
 	deps, err := h.getDependencies(ctx, flexcluster)
 	if err != nil {
-		return result.Error(state.StateDeletionRequested, fmt.Errorf("failed to get dependencies: %w", err))
+		// Race condition: Group CR may be deleted from K8s before FlexCluster finishes deletion.
+		// If Group is not found but we have group ID in status, use it to proceed with deletion.
+		var statusErr *apierrors.StatusError
+		if errors.As(err, &statusErr) && apierrors.IsNotFound(statusErr) {
+			if group := h.getMinimalGroupFromStatusOrSpec(flexcluster); group != nil {
+				deps = []client.Object{group}
+			} else {
+				return result.Error(state.StateDeletionRequested, fmt.Errorf("failed to get dependencies: %w", err))
+			}
+		} else {
+			return result.Error(state.StateDeletionRequested, fmt.Errorf("failed to get dependencies: %w", err))
+		}
 	}
 
 	params := &v20250312sdk.DeleteFlexClusterApiParams{}
@@ -189,7 +221,18 @@ func (h *Handlerv20250312) HandleDeletionRequested(ctx context.Context, flexclus
 func (h *Handlerv20250312) HandleDeleting(ctx context.Context, flexcluster *akov2generated.FlexCluster) (ctrlstate.Result, error) {
 	deps, err := h.getDependencies(ctx, flexcluster)
 	if err != nil {
-		return result.Error(state.StateDeleting, fmt.Errorf("failed to get dependencies: %w", err))
+		// Race condition: Group CR may be deleted from K8s before FlexCluster finishes deletion.
+		// If Group is not found but we have group ID in status, use it to proceed with deletion.
+		var statusErr *apierrors.StatusError
+		if errors.As(err, &statusErr) && apierrors.IsNotFound(statusErr) {
+			if group := h.getMinimalGroupFromStatusOrSpec(flexcluster); group != nil {
+				deps = []client.Object{group}
+			} else {
+				return result.Error(state.StateDeleting, fmt.Errorf("failed to get dependencies: %w", err))
+			}
+		} else {
+			return result.Error(state.StateDeleting, fmt.Errorf("failed to get dependencies: %w", err))
+		}
 	}
 
 	params := &v20250312sdk.GetFlexClusterApiParams{}

internal/generated/controller/flexcluster/handler_v20250312_test.go

@@ -1114,18 +1114,20 @@ func withGeneration(flexCluster *akov2generated.FlexCluster, generation int64) *
 // withObservedGeneration sets the observed generation in status conditions
 func withObservedGeneration(flexCluster *akov2generated.FlexCluster, observedGen int64) *akov2generated.FlexCluster {
 	if flexCluster.Status.Conditions == nil {
-		flexCluster.Status.Conditions = &[]metav1.Condition{}
+		// Allocate a new empty slice pointer to avoid storing a pointer to a temporary value
+		flexCluster.Status.Conditions = new([]metav1.Condition)
 	}
 	conditions := *flexCluster.Status.Conditions
 	conditions = append(conditions, metav1.Condition{
 		Type:               state.StateCondition,
 		ObservedGeneration: observedGen,
 		Status:             metav1.ConditionTrue,
 	})
-	// Allocate a new slice to avoid storing a pointer to a local variable
+	// Allocate a new slice pointer that persists beyond function scope
 	newConditions := make([]metav1.Condition, len(conditions))
 	copy(newConditions, conditions)
-	flexCluster.Status.Conditions = &newConditions
+	flexCluster.Status.Conditions = new([]metav1.Condition)
+	*flexCluster.Status.Conditions = newConditions
 	return flexCluster
 }
 

test/e2e2/ako_test.go

@@ -48,6 +48,14 @@ var _ = Describe("Atlas Operator Start and Stop test", Ordered, Label("ako-start
 		ako = runTestAKO(DefaultGlobalCredentials, control.MustEnvVar("OPERATOR_NAMESPACE"), deletionProtectionOff)
 		ako.Start(GinkgoT())
 
+		// Register cleanup - this should even when the process is interrupted with Ctrl+C
+		// AfterAll is not reliable in such cases.
+		DeferCleanup(func() {
+			if ako != nil {
+				ako.Stop(GinkgoT())
+			}
+		})
+
 		ctx = context.Background()
 		client, err := kube.NewTestClient()
 		Expect(err).To(Succeed())
@@ -57,10 +65,6 @@ var _ = Describe("Atlas Operator Start and Stop test", Ordered, Label("ako-start
 		})).To(Succeed())
 	})
 
-	_ = AfterAll(func() {
-		ako.Stop(GinkgoT())
-	})
-
 	_ = BeforeEach(func() {
 		testNamespace = &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{
 			Name: utils.RandomName("ako-ns"),

test/e2e2/e2e2_suite_test.go

@@ -75,6 +75,7 @@ func initTestLogging(t *testing.T) {
 	ctrllog.SetLogger(logrLogger.WithName("test"))
 }
 
+// nolint:unparam
 func runTestAKO(globalCreds, ns string, deletionprotection bool) operator.Operator {
 	args := []string{
 		"--log-level=-9",

test/e2e2/flex_to_dedicated_test.go

@@ -51,18 +51,20 @@ var _ = Describe("Flex to Dedicated Upgrade", Ordered, Label("flex-to-dedicated"
 		ako = runTestAKO(DefaultGlobalCredentials, control.MustEnvVar("OPERATOR_NAMESPACE"), false)
 		ako.Start(GinkgoT())
 
+		// Register cleanup - this should even when the process is interrupted with Ctrl+C
+		// AfterAll is not reliable in such cases.
+		DeferCleanup(func() {
+			if ako != nil {
+				ako.Stop(GinkgoT())
+			}
+		})
+
 		ctx = context.Background()
 		client, err := kube.NewTestClient()
 		Expect(err).ToNot(HaveOccurred())
 		kubeClient = client
 	})
 
-	_ = AfterAll(func() {
-		if ako != nil {
-			ako.Stop(GinkgoT())
-		}
-	})
-
 	_ = BeforeEach(func() {
 		resourcePrefix = utils.RandomName("flex-to-dedicated")
 		testNamespace = &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{