diff --git a/internal/controller/reconcile-capapplication.go b/internal/controller/reconcile-capapplication.go index 47f27876..a185f8b4 100644 --- a/internal/controller/reconcile-capapplication.go +++ b/internal/controller/reconcile-capapplication.go @@ -16,6 +16,7 @@ import ( "github.com/sap/cap-operator/internal/util" "github.com/sap/cap-operator/pkg/apis/sme.sap.com/v1alpha1" + "golang.org/x/sync/errgroup" corev1 "k8s.io/api/core/v1" k8sErrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -134,7 +135,7 @@ func (c *Controller) handleCAPApplicationDependentResources(ctx context.Context, return } - // step 6 - check and set consistent status + // step 6 - check and set consistent status; check for newer versions and trigger tenant networking updates return c.verifyApplicationConsistent(ctx, ca) } @@ -151,10 +152,11 @@ func (c *Controller) verifyApplicationConsistent(ctx context.Context, ca *v1alph } // Check for newer CAPApplicationVersion - return nil, c.checkNewCAPApplicationVersion(ctx, ca) + return nil, c.checkNewCavAndTenantNetworking(ctx, ca) } -func (c *Controller) checkNewCAPApplicationVersion(ctx context.Context, ca *v1alpha1.CAPApplication) error { +func (c *Controller) checkNewCavAndTenantNetworking(ctx context.Context, ca *v1alpha1.CAPApplication) error { + // Get the latest CAV for the tenant cav, err := c.getLatestReadyCAPApplicationVersion(ca, false) if err != nil { return err @@ -165,31 +167,28 @@ func (c *Controller) checkNewCAPApplicationVersion(ctx context.Context, ca *v1al if err != nil || len(tenants) == 0 { return err } + + netUpdGrp := errgroup.Group{} updated := false for _, tenant := range tenants { - if tenant.Spec.VersionUpgradeStrategy == v1alpha1.VersionUpgradeStrategyTypeNever { - // Skip non relevant tenants - continue - } - if tenant.Status.State == v1alpha1.CAPTenantStateProvisioning || tenant.Status.State == v1alpha1.CAPTenantStateUpgrading || tenant.Status.State == v1alpha1.CAPTenantStateDeleting { - // Skip tenants that are not ready or not in processing or not in error - continue + if tenant.Status.CurrentCAPApplicationVersionInstance != "" { + t := tenant + netUpdGrp.Go(func() error { + return c.reconcileTenantNetworking(ctx, t, t.Status.CurrentCAPApplicationVersionInstance, ca) + }) } - // Assume we may have to update the tenant and prepare a copy - cat := tenant.DeepCopy() - - // Check version of tenant - if cat.Spec.Version != cav.Spec.Version { - // update CAPTenant Spec to point to the latest version - cat.Spec.Version = cav.Spec.Version - // Trigger update on CAPTenant (modifies Generation) --> which would reconcile the tenant - if _, err = c.crdClient.SmeV1alpha1().CAPTenants(ca.Namespace).Update(ctx, cat, metav1.UpdateOptions{}); err != nil { - return fmt.Errorf("could not update %s %s.%s: %w", v1alpha1.CAPTenantKind, cat.Namespace, cat.Name, err) - } - c.Event(tenant, ca, corev1.EventTypeNormal, CAPTenantEventAutoVersionUpdate, EventActionUpgrade, fmt.Sprintf("version updated to %s for initiating tenant upgrade", cav.Spec.Version)) + + if upd, err := c.checkForTenantVersionUpgrade(ctx, ca, cav, tenant); err != nil { + return err + } else if upd { updated = true } } + + if err = netUpdGrp.Wait(); err != nil { + return fmt.Errorf("failed to reconcile tenant networking: %w", err) + } + if updated { msg := fmt.Sprintf("new version %s.%s was used to trigger tenant upgrades", cav.Namespace, cav.Name) ca.SetStatusWithReadyCondition(v1alpha1.CAPApplicationStateProcessing, metav1.ConditionFalse, CAPApplicationEventNewCAVTriggeredTenantUpgrade, msg) @@ -200,6 +199,33 @@ func (c *Controller) checkNewCAPApplicationVersion(ctx context.Context, ca *v1al return nil } +func (c *Controller) checkForTenantVersionUpgrade(ctx context.Context, ca *v1alpha1.CAPApplication, cav *v1alpha1.CAPApplicationVersion, tenant *v1alpha1.CAPTenant) (bool, error) { + if tenant.Spec.VersionUpgradeStrategy == v1alpha1.VersionUpgradeStrategyTypeNever { + // Skip non relevant tenants + return false, nil + } + if tenant.Status.State == v1alpha1.CAPTenantStateProvisioning || tenant.Status.State == v1alpha1.CAPTenantStateUpgrading || tenant.Status.State == v1alpha1.CAPTenantStateDeleting { + // Skip tenants that are not ready or not in processing or not in error + return false, nil + } + + // Assume we may have to update the tenant and prepare a copy + cat := tenant.DeepCopy() + + // Check version of tenant + if cat.Spec.Version != cav.Spec.Version { + // update CAPTenant Spec to point to the latest version + cat.Spec.Version = cav.Spec.Version + // Trigger update on CAPTenant (modifies Generation) --> which would reconcile the tenant + if _, err := c.crdClient.SmeV1alpha1().CAPTenants(ca.Namespace).Update(ctx, cat, metav1.UpdateOptions{}); err != nil { + return false, fmt.Errorf("could not update %s %s.%s: %w", v1alpha1.CAPTenantKind, cat.Namespace, cat.Name, err) + } + c.Event(tenant, ca, corev1.EventTypeNormal, CAPTenantEventAutoVersionUpdate, EventActionUpgrade, fmt.Sprintf("version updated to %s for initiating tenant upgrade", cav.Spec.Version)) + return true, nil + } + return false, nil +} + func (c *Controller) checkAdditionalConditions(ca *v1alpha1.CAPApplication, result *ReconcileResult, err error) (*ReconcileResult, error) { // In case of explicit Reconcile or errors return back with the original result if result != nil || err != nil { diff --git a/internal/controller/reconcile-capapplication_test.go b/internal/controller/reconcile-capapplication_test.go index f859e77f..ff7c65ec 100644 --- a/internal/controller/reconcile-capapplication_test.go +++ b/internal/controller/reconcile-capapplication_test.go @@ -375,7 +375,7 @@ func TestController_handleCAPApplicationConsistent_Case2(t *testing.T) { } func TestController_handleCAPApplicationConsistent_Case3(t *testing.T) { - reconcileTestItem( + err := reconcileTestItem( context.TODO(), t, QueueItem{Key: ResourceCAPApplication, ResourceKey: NamespacedResourceKey{Namespace: "default", Name: "test-cap-01"}}, TestData{ @@ -389,35 +389,13 @@ func TestController_handleCAPApplicationConsistent_Case3(t *testing.T) { "testdata/capapplication/cat-consumer-upg-never-ready.yaml", "testdata/common/credential-secrets.yaml", }, - expectedResources: "testdata/capapplication/ca-31.expected.yaml", - backlogItems: []string{ - "ERP4SMEPREPWORKAPPPLAT-2881", - }, + expectError: true, }, ) -} -func TestController_handleCAPApplicationConsistent_Case4(t *testing.T) { - reconcileTestItem( - context.TODO(), t, - QueueItem{Key: ResourceCAPApplication, ResourceKey: NamespacedResourceKey{Namespace: "default", Name: "test-cap-01"}}, - TestData{ - description: "Consistent state with a CAV name update", - initialResources: []string{ - "testdata/common/domain-ready.yaml", - "testdata/common/cluster-domain-ready.yaml", - "testdata/capapplication/ca-32.initial.yaml", - "testdata/capapplication/cav-name-modified-ready.yaml", - "testdata/capapplication/cat-provider-no-finalizers-ready.yaml", - "testdata/capapplication/cat-consumer-no-finalizers-ready.yaml", - "testdata/common/credential-secrets.yaml", - }, - expectedResources: "testdata/capapplication/ca-32.expected.yaml", - backlogItems: []string{ - "ERP4SMEPREPWORKAPPPLAT-2881", - }, - }, - ) + if err.Error() != "failed to reconcile tenant networking: capapplicationversion.sme.sap.com \"test-cap-01-cav-v1\" not found" { + t.Error("Wrong error message") + } } func TestController_handleCAPApplicationConsistent_Case5(t *testing.T) { @@ -481,12 +459,11 @@ func TestCAPApplicationConsistentWithNewCAPApplicationVersionTenantUpdateError(t "testdata/capapplication/cat-provider-no-finalizers-ready.yaml", "testdata/common/credential-secrets.yaml", }, - expectError: true, - mockErrorForResources: []ResourceAction{{Verb: "update", Group: "sme.sap.com", Version: "v1alpha1", Resource: "captenants", Namespace: "*", Name: "*"}}, + expectError: true, }, ) - if err.Error() != "could not update CAPTenant default.test-cap-01-provider: mocked api error (captenants.sme.sap.com/v1alpha1)" { - t.Error("error message is different from expected") + if err.Error() != "failed to reconcile tenant networking: capapplicationversion.sme.sap.com \"test-cap-01-cav-v1\" not found" { + t.Error("Wrong error message") } } @@ -535,7 +512,7 @@ func TestAdditionalConditionsWithTenantDeletingUpgradeStrategyNever(t *testing.T } func TestController_handleCAPApplicationConsistent_versionUpgrade(t *testing.T) { - reconcileTestItem( + err := reconcileTestItem( context.TODO(), t, QueueItem{Key: ResourceCAPApplication, ResourceKey: NamespacedResourceKey{Namespace: "default", Name: "test-cap-01"}}, TestData{ @@ -549,9 +526,13 @@ func TestController_handleCAPApplicationConsistent_versionUpgrade(t *testing.T) "testdata/capapplication/cat-consumer-upgrading.yaml", "testdata/common/credential-secrets.yaml", }, - expectedResources: "testdata/capapplication/ca-31.expected.yaml", + expectError: true, }, ) + + if err.Error() != "failed to reconcile tenant networking: capapplicationversion.sme.sap.com \"test-cap-01-cav-v1\" not found" { + t.Error("Wrong error message") + } } func TestCA_ServicesOnly_Consistent(t *testing.T) { diff --git a/internal/controller/reconcile-captenant.go b/internal/controller/reconcile-captenant.go index eda8e1f3..906c80f4 100644 --- a/internal/controller/reconcile-captenant.go +++ b/internal/controller/reconcile-captenant.go @@ -163,6 +163,10 @@ var handleCompletedProvisioningUpgradeOperation = func(ctx context.Context, c *C return NewReconcileResultWithResource(ResourceCAPTenant, cat.Name, cat.Namespace, 10*time.Second), nil } + // set current CAPApplicationVersionInstance and update previous versions + // required for tenant networking reconciliation as it relies on the current and previous version in the status of the tenant + cat.SetStatusCAPApplicationVersion(ctop.Spec.CAPApplicationVersionInstance) + // check and reconcile tenant virtual service // adjust virtual service only when tenant is finalizing (after provisioning or upgrade) err = c.reconcileTenantNetworking(ctx, cat, ctop.Spec.CAPApplicationVersionInstance, ca) @@ -172,7 +176,6 @@ var handleCompletedProvisioningUpgradeOperation = func(ctx context.Context, c *C // the ObservedGeneration of the tenant should be updated here (when Ready) cat.SetStatusWithReadyCondition(target.state, target.conditionStatus, target.conditionReason, message) - cat.SetStatusCAPApplicationVersion(ctop.Spec.CAPApplicationVersionInstance) return getTenantReconcileResultConsideringDeletion(cat, nil), nil } @@ -235,13 +238,6 @@ func (c *Controller) reconcileCAPTenant(ctx context.Context, item QueueItem, _ i return requeue, nil } - // if cat.DeletionTimestamp == nil { - // // Create relevant DNSEntries for this tenant. DNS entries are checked before setting the tenant as ready - // if err = c.reconcileTenantDNSEntries(ctx, cat); err != nil { - // return - // } - // } - // create and track CAPTenantOperations based on state, deletion timestamp, version change etc. requeue, err = c.handleTenantOperationsForCAPTenant(ctx, cat) if err != nil || requeue != nil { @@ -249,7 +245,11 @@ func (c *Controller) reconcileCAPTenant(ctx context.Context, item QueueItem, _ i } if cat.DeletionTimestamp == nil && cat.Status.CurrentCAPApplicationVersionInstance != "" { - err = c.reconcileTenantNetworking(ctx, cat, cat.Status.CurrentCAPApplicationVersionInstance, nil) + ca, caGetErr := c.getCachedCAPApplication(cat.Namespace, cat.Spec.CAPApplicationInstance) + if caGetErr != nil { + return nil, caGetErr + } + err = c.reconcileTenantNetworking(ctx, cat, cat.Status.CurrentCAPApplicationVersionInstance, ca) if err == nil { util.LogInfo("Tenant processing completed", string(Ready), cat, nil, "tenantId", cat.Spec.TenantId, "version", cat.Spec.Version) } diff --git a/internal/controller/reconcile-captenant_test.go b/internal/controller/reconcile-captenant_test.go index 3dc5b774..90ecaa8f 100644 --- a/internal/controller/reconcile-captenant_test.go +++ b/internal/controller/reconcile-captenant_test.go @@ -558,3 +558,158 @@ func TestCAPTenantVSHeadersErrorRes(t *testing.T) { t.Error("error message is different from expected, actual:", err.Error()) } } + +func TestCAPTenantProvisioningCompletedWithSessionAffinityEnabled(t *testing.T) { + reconcileTestItem( + context.TODO(), t, + QueueItem{Key: ResourceCAPTenant, ResourceKey: NamespacedResourceKey{Namespace: "default", Name: "test-cap-01-provider"}}, + TestData{ + description: "captenant provisioning operation completed (creates virtual service and destination rule) with session affinity enabled", + initialResources: []string{ + "testdata/common/domain-ready.yaml", + "testdata/common/cluster-domain-ready.yaml", + "testdata/common/capapplication-session-affinity.yaml", + "testdata/common/capapplicationversion-v1.yaml", + "testdata/captenant/cat-04.initial.yaml", + }, + expectedResources: "testdata/captenant/cat-with-session-affinity-dr-vs.yaml", + }, + ) +} + +func TestCAPTenantProvisioningCompletedWithSessionAffinityEnabledAndVsheaders(t *testing.T) { + reconcileTestItem( + context.TODO(), t, + QueueItem{Key: ResourceCAPTenant, ResourceKey: NamespacedResourceKey{Namespace: "default", Name: "test-cap-01-provider"}}, + TestData{ + description: "captenant provisioning operation completed (creates virtual service and destination rule) with session affinity enabled and virtual service headers", + initialResources: []string{ + "testdata/common/domain-ready.yaml", + "testdata/common/cluster-domain-ready.yaml", + "testdata/common/capapplication-session-affinity-vs-headers.yaml", + "testdata/common/capapplicationversion-v1.yaml", + "testdata/captenant/cat-04.initial.yaml", + }, + expectedResources: "testdata/captenant/cat-with-session-affinity-dr-vs-headers.yaml", + }, + ) +} + +func TestCAPTenantProvisioningCompletedWithSessionAffinityEnabledCustomLogout(t *testing.T) { + reconcileTestItem( + context.TODO(), t, + QueueItem{Key: ResourceCAPTenant, ResourceKey: NamespacedResourceKey{Namespace: "default", Name: "test-cap-01-provider"}}, + TestData{ + description: "captenant provisioning operation completed (creates virtual service and destination rule) with session affinity enabled using custom logout routes", + initialResources: []string{ + "testdata/common/domain-ready.yaml", + "testdata/common/cluster-domain-ready.yaml", + "testdata/common/capapplication-session-affinity.yaml", + "testdata/common/capapplicationversion-v1-custom-logout-endpoint.yaml", + "testdata/captenant/cat-04.initial.yaml", + }, + expectedResources: "testdata/captenant/cat-with-session-affinity-dr-vs-logout-endpoint.yaml", + }, + ) +} + +func TestCAPTenantUpgradeOperationCompletedWithSessionAffinityEnabled(t *testing.T) { + reconcileTestItem( + context.TODO(), t, + QueueItem{Key: ResourceCAPTenant, ResourceKey: NamespacedResourceKey{Namespace: "default", Name: "test-cap-01-provider"}}, + TestData{ + description: "captenant upgrade operation completed expecting virtual service, destination rule adjustments with session affinity enabled", + initialResources: []string{ + "testdata/common/domain-ready.yaml", + "testdata/common/cluster-domain-ready.yaml", + "testdata/common/capapplication-session-affinity.yaml", + "testdata/common/capapplicationversion-v1.yaml", + "testdata/common/capapplicationversion-v2.yaml", + "testdata/captenant/provider-tenant-vs-v1.yaml", + "testdata/captenant/provider-tenant-dr-v1.yaml", + "testdata/captenant/cat-13.initial.yaml", + }, + expectedResources: "testdata/captenant/cat-with-session-affinity-dr-vs-upgrade.yaml", + }, + ) +} + +func TestCAPTenantUpgradedThirdTimeWithSessionAffinityEnabled(t *testing.T) { + reconcileTestItem( + context.TODO(), t, + QueueItem{Key: ResourceCAPTenant, ResourceKey: NamespacedResourceKey{Namespace: "default", Name: "test-cap-01-provider"}}, + TestData{ + description: "captenant upgraded third time - expecting virtual service, destination rule adjustments by removing config corresponding to v1 and by adding config for v3", + initialResources: []string{ + "testdata/common/domain-ready.yaml", + "testdata/common/cluster-domain-ready.yaml", + "testdata/common/capapplication-session-affinity.yaml", + "testdata/common/capapplicationversion-v1.yaml", + "testdata/common/capapplicationversion-v2.yaml", + "testdata/common/capapplicationversion-v3.yaml", + "testdata/captenant/cat-with-session-affinity-dr-vs-upgrade-to-cav-v3.yaml", + }, + expectedResources: "testdata/captenant/cat-with-session-affinity-dr-vs-upgrade-to-cav-v3.expected.yaml", + }, + ) +} + +func TestCAPTenantUpgradeOperationCompletedWithSessionAffinityEnabledAndPreviousCAVRemoved(t *testing.T) { + reconcileTestItem( + context.TODO(), t, + QueueItem{Key: ResourceCAPTenant, ResourceKey: NamespacedResourceKey{Namespace: "default", Name: "test-cap-01-provider"}}, + TestData{ + description: "captenant upgraded - expecting virtual service, destination rule adjustments after removing previous cav v1", + initialResources: []string{ + "testdata/common/domain-ready.yaml", + "testdata/common/cluster-domain-ready.yaml", + "testdata/common/capapplication-session-affinity.yaml", + "testdata/common/capapplicationversion-v2.yaml", + "testdata/captenant/cat-with-session-affinity-dr-vs-upgrade.yaml", + }, + expectedResources: "testdata/captenant/cat-with-session-affinity-dr-vs-prev-cav-removed.yaml", + }, + ) +} + +func TestCAPTenantUpgradeOperationCompletedWithSessionAffinitySwitchedFromEnabledToDisabled(t *testing.T) { + reconcileTestItem( + context.TODO(), t, + QueueItem{Key: ResourceCAPTenant, ResourceKey: NamespacedResourceKey{Namespace: "default", Name: "test-cap-01-provider"}}, + TestData{ + description: "captenant upgraded - expecting virtual service, destination rule adjustments after switching session affinity from enabled to disabled", + initialResources: []string{ + "testdata/common/domain-ready.yaml", + "testdata/common/cluster-domain-ready.yaml", + "testdata/common/capapplication.yaml", + "testdata/common/capapplicationversion-v1.yaml", + "testdata/common/capapplicationversion-v2.yaml", + "testdata/captenant/cat-with-session-affinity-dr-vs-upgrade.yaml", + }, + expectedResources: "testdata/captenant/cat-with-session-affinity-disabled-dr-vs.yaml", + }, + ) +} + +func TestCAPTenantUpgradeOperationCompletedWithSessionAffinityEnabledAndPreviousCAVRemovedButDRDeletionFailed(t *testing.T) { + err := reconcileTestItem( + context.TODO(), t, + QueueItem{Key: ResourceCAPTenant, ResourceKey: NamespacedResourceKey{Namespace: "default", Name: "test-cap-01-provider"}}, + TestData{ + description: "captenant upgraded - expecting virtual service, destination rule adjustments after removing previous cav v1 but DR deletion fails for some reason", + initialResources: []string{ + "testdata/common/domain-ready.yaml", + "testdata/common/cluster-domain-ready.yaml", + "testdata/common/capapplication-session-affinity.yaml", + "testdata/common/capapplicationversion-v2.yaml", + "testdata/captenant/cat-with-session-affinity-dr-vs-upgrade.yaml", + }, + mockErrorForResources: []ResourceAction{{Verb: "delete", Group: "networking.istio.io", Version: "v1", Resource: "destinationrules", Namespace: "*", Name: "test-cap-01-provider-test-cap-01-cav-v1"}}, + expectError: true, + }, + ) + + if err.Error() != "mocked api error (destinationrules.networking.istio.io/v1)" { + t.Error("error message is different from expected, actual:", err.Error()) + } +} diff --git a/internal/controller/reconcile-networking.go b/internal/controller/reconcile-networking.go index 33aa6b12..655bc827 100644 --- a/internal/controller/reconcile-networking.go +++ b/internal/controller/reconcile-networking.go @@ -30,13 +30,18 @@ const ( EventServiceVirtualServiceModificationFailed = "ServiceVirtualServiceModificationFailed" ) -const serviceDNSSuffix = ".svc.cluster.local" +const ( + serviceDNSSuffix = ".svc.cluster.local" + setCookie = "Set-Cookie" + AnnotationLogoutEndpoint = "sme.sap.com/logout-endpoint" + AnnotationEnableSessionAffinity = "sme.sap.com/enable-session-affinity" +) func (c *Controller) reconcileTenantNetworking(ctx context.Context, cat *v1alpha1.CAPTenant, cavName string, ca *v1alpha1.CAPApplication) (err error) { var ( - reason, message string - drModified, vsModified bool - eventType string = corev1.EventTypeNormal + reason, message string + drModified, vsModified, prevCavDrModified bool + eventType string = corev1.EventTypeNormal ) defer func() { @@ -49,12 +54,19 @@ func (c *Controller) reconcileTenantNetworking(ctx context.Context, cat *v1alpha } }() - if drModified, err = c.reconcileTenantDestinationRule(ctx, cat, cavName); err != nil { + if drModified, err = c.reconcileTenantDestinationRule(ctx, cat, cat.Name, cavName); err != nil { util.LogError(err, "Destination rule reconciliation failed", string(Processing), cat, nil, "tenantId", cat.Spec.TenantId, "version", cat.Spec.Version) reason = CAPTenantEventDestinationRuleModificationFailed return } + // Enable session affinity + if prevCavDrModified, err = c.reconcileTenantDestinationRuleForPrevCav(ctx, ca, cat); err != nil { + util.LogError(err, "Destination rule reconciliation failed for previous cav", string(Processing), cat, nil, "tenantId", cat.Spec.TenantId, "version", cat.Spec.Version) + reason = CAPTenantEventDestinationRuleModificationFailed + return + } + if vsModified, err = c.reconcileTenantVirtualService(ctx, cat, cavName, ca); err != nil { util.LogError(err, "Virtual service reconciliation failed", string(Processing), cat, nil, "tenantId", cat.Spec.TenantId, "version", cat.Spec.Version) reason = CAPTenantEventVirtualServiceModificationFailed @@ -62,7 +74,7 @@ func (c *Controller) reconcileTenantNetworking(ctx context.Context, cat *v1alpha } // update tenant status - if drModified || vsModified { + if drModified || vsModified || prevCavDrModified { message = fmt.Sprintf("VirtualService (and DestinationRule) %s.%s was reconciled", cat.Namespace, cat.Name) reason = CAPTenantEventTenantNetworkingModified conditionStatus := metav1.ConditionFalse @@ -75,16 +87,16 @@ func (c *Controller) reconcileTenantNetworking(ctx context.Context, cat *v1alpha return } -func (c *Controller) reconcileTenantDestinationRule(ctx context.Context, cat *v1alpha1.CAPTenant, cavName string) (modified bool, err error) { +func (c *Controller) reconcileTenantDestinationRule(ctx context.Context, cat *v1alpha1.CAPTenant, drName string, cavName string) (modified bool, err error) { var ( create, update bool dr *istionwv1.DestinationRule ) - dr, err = c.istioClient.NetworkingV1().DestinationRules(cat.Namespace).Get(ctx, cat.Name, metav1.GetOptions{}) + dr, err = c.istioClient.NetworkingV1().DestinationRules(cat.Namespace).Get(ctx, drName, metav1.GetOptions{}) if errors.IsNotFound(err) { dr = &istionwv1.DestinationRule{ ObjectMeta: metav1.ObjectMeta{ - Name: cat.Name, // keep the same name as CAPTenant to avoid duplicates + Name: drName, Namespace: cat.Namespace, Labels: map[string]string{}, OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(cat, v1alpha1.SchemeGroupVersion.WithKind(v1alpha1.CAPTenantKind))}, @@ -111,6 +123,91 @@ func (c *Controller) reconcileTenantDestinationRule(ctx context.Context, cat *v1 return create || update, err } +func (c *Controller) reconcileTenantDestinationRuleForPrevCav(ctx context.Context, ca *v1alpha1.CAPApplication, cat *v1alpha1.CAPTenant) (modified bool, err error) { + if len(cat.Status.PreviousCAPApplicationVersions) == 0 { + return false, nil + } + + if ca.Annotations[AnnotationEnableSessionAffinity] == "true" { + return c.handleSessionAffinityEnabled(ctx, cat) + } + + return c.cleanupAllPreviousCavDRs(ctx, cat) +} + +func (c *Controller) handleSessionAffinityEnabled(ctx context.Context, cat *v1alpha1.CAPTenant) (bool, error) { + var modified bool + var err error + prevCav := cat.Status.PreviousCAPApplicationVersions[len(cat.Status.PreviousCAPApplicationVersions)-1] + + // Check if previous CAV exists + _, cavGetErr := c.crdInformerFactory.Sme().V1alpha1().CAPApplicationVersions().Lister().CAPApplicationVersions(cat.Namespace).Get(prevCav) + switch { + case errors.IsNotFound(cavGetErr): + // CAV doesn't exist, cleanup its DR + modified, err = c.deleteDRIfExists(ctx, cat.Namespace, cat.Name+"-"+prevCav) + if err != nil { + return false, err + } + case cavGetErr != nil: + // Some other error occurred while fetching CAV + return false, cavGetErr + default: + // CAV exists, reconcile its DR + modified, err = c.reconcileTenantDestinationRule(ctx, cat, cat.Name+"-"+prevCav, prevCav) + if err != nil { + return false, err + } + } + + // Clean up second-to-last CAV DR if it exists + if len(cat.Status.PreviousCAPApplicationVersions) > 1 { + secondLastCav := cat.Status.PreviousCAPApplicationVersions[len(cat.Status.PreviousCAPApplicationVersions)-2] + drDeleted, err := c.deleteDRIfExists(ctx, cat.Namespace, cat.Name+"-"+secondLastCav) + if err != nil { + return false, err + } + modified = modified || drDeleted + } + + return modified, nil +} + +func (c *Controller) cleanupAllPreviousCavDRs(ctx context.Context, cat *v1alpha1.CAPTenant) (bool, error) { + drNames := make(map[string]struct{}) + for _, cav := range cat.Status.PreviousCAPApplicationVersions { + drNames[cat.Name+"-"+cav] = struct{}{} + } + + drList, err := c.istioClient.NetworkingV1().DestinationRules(cat.Namespace).List(ctx, metav1.ListOptions{}) + if err != nil { + return false, err + } + + var modified bool + for _, dr := range drList.Items { + if _, exists := drNames[dr.Name]; exists { + if err := c.istioClient.NetworkingV1().DestinationRules(cat.Namespace).Delete(ctx, dr.Name, metav1.DeleteOptions{}); err != nil { + return false, err + } + modified = true + } + } + return modified, nil +} + +func (c *Controller) deleteDRIfExists(ctx context.Context, namespace, drName string) (bool, error) { + err := c.istioClient.NetworkingV1().DestinationRules(namespace).Delete(ctx, drName, metav1.DeleteOptions{}) + switch { + case errors.IsNotFound(err): + return false, nil // Nothing to delete + case err != nil: + return false, err // Unexpected error + default: + return true, nil // Deleted successfully + } +} + func (c *Controller) getUpdatedTenantDestinationRuleObject(cat *v1alpha1.CAPTenant, dr *istionwv1.DestinationRule, cavName string) (modified bool, err error) { // verify owner reference modified, err = c.enforceTenantResourceOwnership(&dr.ObjectMeta, &dr.TypeMeta, cat) @@ -195,30 +292,30 @@ func (c *Controller) reconcileTenantVirtualService(ctx context.Context, cat *v1a } func (c *Controller) getUpdatedTenantVirtualServiceObject(cat *v1alpha1.CAPTenant, vs *istionwv1.VirtualService, cavName string, ca *v1alpha1.CAPApplication) (modified bool, err error) { - if ca == nil { - ca, err = c.getCachedCAPApplication(cat.Namespace, cat.Spec.CAPApplicationInstance) - if err != nil { - return modified, err - } - } - // verify owner reference modified, err = c.enforceTenantResourceOwnership(&vs.ObjectMeta, &vs.TypeMeta, cat) if err != nil { return modified, err } - routerPortInfo, err := c.getRouterServicePortInfo(cavName, ca.Namespace) - if err != nil { - return modified, err - } - headers, err := getNetworkingHeaders(ca) if err != nil { return modified, fmt.Errorf("error getting headers via CA annotations for %s %s.%s, error: %v", vs.Kind, vs.Namespace, vs.Name, err) } - spec := &networkingv1.VirtualService{ - Http: []*networkingv1.HTTPRoute{{ + + spec := &networkingv1.VirtualService{} + // check if session affinity is enabled + if ca.Annotations[AnnotationEnableSessionAffinity] == "true" { + spec.Http, err = c.getVirtualServiceHttpRoutes(cat, cavName, headers) + if err != nil { + return modified, err + } + } else { + routerPortInfo, err := c.getRouterServicePortInfo(cavName, ca.Namespace) + if err != nil { + return modified, err + } + spec.Http = []*networkingv1.HTTPRoute{{ Match: []*networkingv1.HTTPMatchRequest{ {Uri: &networkingv1.StringMatch{MatchType: &networkingv1.StringMatch_Prefix{Prefix: "/"}}}, }, @@ -230,8 +327,9 @@ func (c *Controller) getUpdatedTenantVirtualServiceObject(cat *v1alpha1.CAPTenan Weight: 100, Headers: headers, }}, - }}, + }} } + err = c.updateVirtualServiceSpecFromDomainReferences(spec, cat.Spec.SubDomain, ca) if err != nil { return modified, err @@ -252,6 +350,159 @@ func (c *Controller) getUpdatedTenantVirtualServiceObject(cat *v1alpha1.CAPTenan return modified, err } +func (c *Controller) getVirtualServiceHttpRoutes(cat *v1alpha1.CAPTenant, currentCavName string, headers *networkingv1.Headers) ([]*networkingv1.HTTPRoute, error) { + var ( + httpRoutes []*networkingv1.HTTPRoute + prevCav *v1alpha1.CAPApplicationVersion + prevDest *networkingv1.Destination + err error + ) + + // Lookup previous CAV (if any) + if len(cat.Status.PreviousCAPApplicationVersions) > 0 { + prevCavName := cat.Status.PreviousCAPApplicationVersions[len(cat.Status.PreviousCAPApplicationVersions)-1] + prevCav, err = c.crdInformerFactory.Sme().V1alpha1().CAPApplicationVersions().Lister().CAPApplicationVersions(cat.Namespace).Get(prevCavName) + + if err == nil { // only if found + if prevDest, err = c.getVirtualServiceHttpRouteDestination(prevCavName, cat.Namespace); err != nil { + return nil, err + } + } else if !errors.IsNotFound(err) { + return nil, err + } + } + + // Lookup current CAV destination + currentDest, err := c.getVirtualServiceHttpRouteDestination(currentCavName, cat.Namespace) + if err != nil { + return nil, err + } + + // Retrieve current CAV for logout endpointannotations + currentCav, err := c.crdInformerFactory.Sme().V1alpha1().CAPApplicationVersions().Lister().CAPApplicationVersions(cat.Namespace).Get(currentCavName) + if err != nil { + return nil, err + } + + // --- Add routes --- + // Logoff/logout routes + if prevDest != nil { + httpRoutes = append(httpRoutes, buildVirtualServiceLogOffHttpRoute(prevCav.Name, prevCav.Annotations[AnnotationLogoutEndpoint], prevDest, headers)) + } + httpRoutes = append(httpRoutes, buildVirtualServiceLogOffHttpRoute(currentCavName, currentCav.Annotations[AnnotationLogoutEndpoint], currentDest, headers)) + + // Cookie routes + if prevDest != nil { + httpRoutes = append(httpRoutes, buildVirtualServiceCookieHttpRoute(prevCav.Name, prevDest)) + } + httpRoutes = append(httpRoutes, buildVirtualServiceCookieHttpRoute(currentCavName, currentDest)) + + // Default route to current CAV + httpRoutes = append(httpRoutes, buildVirtualServiceDefaultHttpRoute(currentCavName, currentDest, headers)) + + return httpRoutes, nil +} + +func (c *Controller) getVirtualServiceHttpRouteDestination(cavName string, namespace string) (*networkingv1.Destination, error) { + CAVRouterPortInfo, err := c.getRouterServicePortInfo(cavName, namespace) + if err != nil { + return nil, err + } + + return &networkingv1.Destination{ + Host: CAVRouterPortInfo.WorkloadName + ServiceSuffix + "." + namespace + ".svc.cluster.local", + Port: &networkingv1.PortSelector{Number: uint32(CAVRouterPortInfo.Ports[0].Port)}, + }, nil +} + +func buildVirtualServiceDefaultHttpRoute(cavName string, dest *networkingv1.Destination, headers *networkingv1.Headers) *networkingv1.HTTPRoute { + return &networkingv1.HTTPRoute{ + Route: []*networkingv1.HTTPRouteDestination{{ + Destination: dest, + Weight: 100, + }}, + Headers: enhanceHeadersWithCookie(headers, sessionCookie(cavName), "add"), + } +} + +func buildVirtualServiceLogOffHttpRoute(cavName, logoutEndpoint string, dest *networkingv1.Destination, headers *networkingv1.Headers) *networkingv1.HTTPRoute { + // Default logout/logoff regex + uriRegex := "^|.*(logout|logoff).*" + if logoutEndpoint != "" { + uriRegex = "^|.*(" + logoutEndpoint + ").*" + } + + return &networkingv1.HTTPRoute{ + Match: []*networkingv1.HTTPMatchRequest{{ + Headers: map[string]*networkingv1.StringMatch{ + "Cookie": {MatchType: &networkingv1.StringMatch_Regex{Regex: cookieRegex(cavName)}}, + }, + Uri: &networkingv1.StringMatch{ + MatchType: &networkingv1.StringMatch_Regex{Regex: uriRegex}, + }, + }}, + Route: []*networkingv1.HTTPRouteDestination{{ + Destination: dest, + Weight: 100, + }}, + Headers: enhanceHeadersWithCookie(headers, expiredCookie(cavName), "set"), + } +} + +func buildVirtualServiceCookieHttpRoute(cavName string, dest *networkingv1.Destination) *networkingv1.HTTPRoute { + return &networkingv1.HTTPRoute{ + Match: []*networkingv1.HTTPMatchRequest{{ + Headers: map[string]*networkingv1.StringMatch{ + "Cookie": {MatchType: &networkingv1.StringMatch_Regex{Regex: cookieRegex(cavName)}}, + }, + }}, + Route: []*networkingv1.HTTPRouteDestination{{ + Destination: dest, + Weight: 100, + }}, + } +} + +func enhanceHeadersWithCookie(headers *networkingv1.Headers, cookie string, op string) *networkingv1.Headers { + if headers != nil && headers.Response != nil { + h := headers.DeepCopy() + if h.Response.Add == nil { + h.Response.Add = map[string]string{} + } + if h.Response.Set == nil { + h.Response.Set = map[string]string{} + } + switch op { + case "add": + h.Response.Add[setCookie] = cookie + case "set": + h.Response.Set[setCookie] = cookie + } + return h + } + + if op == "add" { + return &networkingv1.Headers{Response: &networkingv1.Headers_HeaderOperations{ + Add: map[string]string{setCookie: cookie}, + }} + } + return &networkingv1.Headers{Response: &networkingv1.Headers_HeaderOperations{ + Set: map[string]string{setCookie: cookie}, + }} +} + +func cookieRegex(cavName string) string { + return "(^|.*; )COP_CAV=" + cavName + "($|; .*)" +} + +func sessionCookie(cavName string) string { + return "COP_CAV=" + cavName + ";Path=/;HttpOnly;Secure" +} + +func expiredCookie(cavName string) string { + return "COP_CAV=" + cavName + ";Path=/;HttpOnly;Secure;Max-Age=0" +} + func (c *Controller) updateVirtualServiceSpecFromDomainReferences(spec *networkingv1.VirtualService, subdomain string, ca *v1alpha1.CAPApplication) error { doms, cdoms, err := fetchDomainResourcesFromCache(c, ca.Spec.DomainRefs, ca.Namespace) if err != nil { diff --git a/internal/controller/testdata/capapplication/ca-31.expected.yaml b/internal/controller/testdata/capapplication/ca-31.expected.yaml deleted file mode 100644 index 94c23c77..00000000 --- a/internal/controller/testdata/capapplication/ca-31.expected.yaml +++ /dev/null @@ -1,58 +0,0 @@ -apiVersion: sme.sap.com/v1alpha1 -kind: CAPApplication -metadata: - finalizers: - - sme.sap.com/capapplication - generation: 0 - name: test-cap-01 - namespace: default - resourceVersion: "11373799" - uid: 3c7ba7cb-dc04-4fd1-be86-3eb3a5c64a98 - annotations: - sme.sap.com/btp-app-identifier: btp-glo-acc-id.test-cap-01 - labels: - sme.sap.com/btp-app-identifier-hash: f20cc8aeb2003b3abc33f749a16bd53544b6bab2 -spec: - btp: - services: - - class: xsuaa - name: cap-uaa - secret: cap-cap-01-uaa-bind-cf - - class: saas-registry - name: cap-saas-registry - secret: cap-cap-01-saas-bind-cf - - class: service-manager - name: cap-service-manager - secret: cap-cap-01-svc-man-bind-cf - btpAppName: test-cap-01 - domainRefs: - - kind: Domain - name: test-cap-01-primary - - kind: ClusterDomain - name: test-cap-01-secondary - globalAccountId: btp-glo-acc-id - provider: - subDomain: my-provider - tenantId: tenant-id-for-provider -status: - conditions: - - reason: NewCAVTriggeredTenantUpgrade - message: "new version default.test-cap-01-cav-v1-modified was used to trigger tenant upgrades" - observedGeneration: 0 - status: "False" - type: Ready - - type: LatestVersionReady - reason: LatestVersionReady - observedGeneration: 0 - status: "True" - - type: AllTenantsReady - reason: NotAllTenantsReady - observedGeneration: 0 - status: "False" - observedGeneration: 0 - state: Processing - servicesOnly: false - domainSpecHash: '{"ClusterDomain.test-cap-01-secondary":"foo.bar.local","Domain.default.test-cap-01-primary":"app-domain.test.local"}' - observedSubdomains: - - my-consumer - - my-provider diff --git a/internal/controller/testdata/capapplication/ca-32.expected.yaml b/internal/controller/testdata/capapplication/ca-32.expected.yaml deleted file mode 100644 index 94c23c77..00000000 --- a/internal/controller/testdata/capapplication/ca-32.expected.yaml +++ /dev/null @@ -1,58 +0,0 @@ -apiVersion: sme.sap.com/v1alpha1 -kind: CAPApplication -metadata: - finalizers: - - sme.sap.com/capapplication - generation: 0 - name: test-cap-01 - namespace: default - resourceVersion: "11373799" - uid: 3c7ba7cb-dc04-4fd1-be86-3eb3a5c64a98 - annotations: - sme.sap.com/btp-app-identifier: btp-glo-acc-id.test-cap-01 - labels: - sme.sap.com/btp-app-identifier-hash: f20cc8aeb2003b3abc33f749a16bd53544b6bab2 -spec: - btp: - services: - - class: xsuaa - name: cap-uaa - secret: cap-cap-01-uaa-bind-cf - - class: saas-registry - name: cap-saas-registry - secret: cap-cap-01-saas-bind-cf - - class: service-manager - name: cap-service-manager - secret: cap-cap-01-svc-man-bind-cf - btpAppName: test-cap-01 - domainRefs: - - kind: Domain - name: test-cap-01-primary - - kind: ClusterDomain - name: test-cap-01-secondary - globalAccountId: btp-glo-acc-id - provider: - subDomain: my-provider - tenantId: tenant-id-for-provider -status: - conditions: - - reason: NewCAVTriggeredTenantUpgrade - message: "new version default.test-cap-01-cav-v1-modified was used to trigger tenant upgrades" - observedGeneration: 0 - status: "False" - type: Ready - - type: LatestVersionReady - reason: LatestVersionReady - observedGeneration: 0 - status: "True" - - type: AllTenantsReady - reason: NotAllTenantsReady - observedGeneration: 0 - status: "False" - observedGeneration: 0 - state: Processing - servicesOnly: false - domainSpecHash: '{"ClusterDomain.test-cap-01-secondary":"foo.bar.local","Domain.default.test-cap-01-primary":"app-domain.test.local"}' - observedSubdomains: - - my-consumer - - my-provider diff --git a/internal/controller/testdata/captenant/cat-with-session-affinity-disabled-dr-vs.yaml b/internal/controller/testdata/captenant/cat-with-session-affinity-disabled-dr-vs.yaml new file mode 100644 index 00000000..3cddfea9 --- /dev/null +++ b/internal/controller/testdata/captenant/cat-with-session-affinity-disabled-dr-vs.yaml @@ -0,0 +1,105 @@ +apiVersion: sme.sap.com/v1alpha1 +kind: CAPTenant +metadata: + generation: 2 + finalizers: + - sme.sap.com/captenant + annotations: + sme.sap.com/btp-app-identifier: btp-glo-acc-id.test-cap-01 + sme.sap.com/owner-identifier: default.test-cap-01 + labels: + sme.sap.com/btp-app-identifier-hash: f20cc8aeb2003b3abc33f749a16bd53544b6bab2 + sme.sap.com/btp-tenant-id: tenant-id-for-provider + sme.sap.com/owner-generation: "2" + sme.sap.com/owner-identifier-hash: 1f74ae2fbff71a708786a4df4bb2ca87ec603581 + sme.sap.com/tenant-type: provider + name: test-cap-01-provider + namespace: default + ownerReferences: + - apiVersion: sme.sap.com/v1alpha1 + blockOwnerDeletion: true + controller: true + kind: CAPApplication + name: test-cap-01 + uid: 3c7ba7cb-dc04-4fd1-be86-3eb3a5c64a98 +spec: + capApplicationInstance: test-cap-01 + subDomain: my-provider + tenantId: tenant-id-for-provider + version: 8.9.10 + versionUpgradeStrategy: always +status: + conditions: + - message: "VirtualService (and DestinationRule) default.test-cap-01-provider was reconciled" + reason: TenantNetworkingModified + observedGeneration: 2 + status: "True" + type: Ready + state: Ready + currentCAPApplicationVersionInstance: test-cap-01-cav-v2 + previousCAPApplicationVersions: + - test-cap-01-cav-v1 + observedGeneration: 2 +--- +apiVersion: networking.istio.io/v1 +kind: VirtualService +metadata: + annotations: + sme.sap.com/resource-hash: ac57fa31ed22c65186a46b7110cb1504999edc415eb29193765e7f170322baa7 + sme.sap.com/owner-identifier: default.test-cap-01-provider + labels: + sme.sap.com/owner-generation: "2" + sme.sap.com/owner-identifier-hash: db1f1fd7eaeb0e6407c741b7e4b2540044bcc4ec + name: test-cap-01-provider + namespace: default + ownerReferences: + - apiVersion: sme.sap.com/v1alpha1 + blockOwnerDeletion: true + controller: true + kind: CAPTenant + name: test-cap-01-provider +spec: + gateways: + - test-cap-01-primary-gen + - default/test-cap-01-secondary-gen + hosts: + - my-provider.app-domain.test.local + - my-provider.foo.bar.local + http: + - match: + - uri: + prefix: / + route: + - destination: + host: test-cap-01-cav-v2-app-router-svc.default.svc.cluster.local + port: + number: 5000 + weight: 100 +--- +apiVersion: networking.istio.io/v1 +kind: DestinationRule +metadata: + annotations: + sme.sap.com/resource-hash: 97ecccd1443b2219a85159a5fdb2f8444543746a5ae890114462e6666e556696 + sme.sap.com/owner-identifier: default.test-cap-01-provider + generation: 1 + labels: + sme.sap.com/owner-generation: "2" + sme.sap.com/owner-identifier-hash: db1f1fd7eaeb0e6407c741b7e4b2540044bcc4ec + name: test-cap-01-provider + namespace: default + ownerReferences: + - apiVersion: sme.sap.com/v1alpha1 + blockOwnerDeletion: true + controller: true + kind: CAPTenant + name: test-cap-01-provider +spec: + host: test-cap-01-cav-v2-app-router-svc.default.svc.cluster.local + trafficPolicy: + loadBalancer: + consistentHash: + httpCookie: + name: CAPOP_ROUTER_STICKY + path: / + ttl: 0s diff --git a/internal/controller/testdata/captenant/cat-with-session-affinity-dr-vs-headers.yaml b/internal/controller/testdata/captenant/cat-with-session-affinity-dr-vs-headers.yaml new file mode 100644 index 00000000..40af3d79 --- /dev/null +++ b/internal/controller/testdata/captenant/cat-with-session-affinity-dr-vs-headers.yaml @@ -0,0 +1,137 @@ +apiVersion: sme.sap.com/v1alpha1 +kind: CAPTenant +metadata: + finalizers: + - sme.sap.com/captenant + annotations: + sme.sap.com/btp-app-identifier: btp-glo-acc-id.test-cap-01 + sme.sap.com/owner-identifier: default.test-cap-01 + labels: + sme.sap.com/btp-app-identifier-hash: f20cc8aeb2003b3abc33f749a16bd53544b6bab2 + sme.sap.com/btp-tenant-id: tenant-id-for-provider + sme.sap.com/owner-generation: "2" + sme.sap.com/owner-identifier-hash: 1f74ae2fbff71a708786a4df4bb2ca87ec603581 + sme.sap.com/tenant-type: provider + name: test-cap-01-provider + namespace: default + ownerReferences: + - apiVersion: sme.sap.com/v1alpha1 + blockOwnerDeletion: true + controller: true + kind: CAPApplication + name: test-cap-01 + uid: 3c7ba7cb-dc04-4fd1-be86-3eb3a5c64a98 +spec: + capApplicationInstance: test-cap-01 + subDomain: my-provider + tenantId: tenant-id-for-provider + version: 5.6.7 + versionUpgradeStrategy: always +status: + conditions: + - message: "CAPTenantOperation default.test-cap-01-provider-s6f4l successfully completed" + reason: ProvisioningCompleted + status: "True" + type: Ready + state: Ready + currentCAPApplicationVersionInstance: test-cap-01-cav-v1 +--- +apiVersion: networking.istio.io/v1 +kind: VirtualService +metadata: + annotations: + sme.sap.com/resource-hash: 833b6e4bf8841e273d08cba96c21e269c540d97ed202be29230fe594dc60f0c9 + sme.sap.com/owner-identifier: default.test-cap-01-provider + labels: + sme.sap.com/owner-generation: "0" + sme.sap.com/owner-identifier-hash: db1f1fd7eaeb0e6407c741b7e4b2540044bcc4ec + name: test-cap-01-provider + namespace: default + ownerReferences: + - apiVersion: sme.sap.com/v1alpha1 + blockOwnerDeletion: true + controller: true + kind: CAPTenant + name: test-cap-01-provider +spec: + gateways: + - test-cap-01-primary-gen + - default/test-cap-01-secondary-gen + hosts: + - my-provider.app-domain.test.local + - my-provider.foo.bar.local + http: + - headers: + request: + set: + x-downstream-peer-subject: "%DOWNSTREAM_PEER_SUBJECT%" + x-tenant-id: invalid-tenant-id + response: + set: + Set-Cookie: COP_CAV=test-cap-01-cav-v1;Path=/;HttpOnly;Secure;Max-Age=0 + foo: bar + match: + - headers: + Cookie: + regex: (^|.*; )COP_CAV=test-cap-01-cav-v1($|; .*) + uri: + regex: ^|.*(logout|logoff).* + route: + - destination: + host: test-cap-01-cav-v1-app-router-svc.default.svc.cluster.local + port: + number: 5000 + weight: 100 + - match: + - headers: + Cookie: + regex: (^|.*; )COP_CAV=test-cap-01-cav-v1($|; .*) + route: + - destination: + host: test-cap-01-cav-v1-app-router-svc.default.svc.cluster.local + port: + number: 5000 + weight: 100 + - headers: + request: + set: + x-downstream-peer-subject: "%DOWNSTREAM_PEER_SUBJECT%" + x-tenant-id: invalid-tenant-id + response: + set: + foo: bar + add: + Set-Cookie: COP_CAV=test-cap-01-cav-v1;Path=/;HttpOnly;Secure + route: + - destination: + host: test-cap-01-cav-v1-app-router-svc.default.svc.cluster.local + port: + number: 5000 + weight: 100 +--- +apiVersion: networking.istio.io/v1 +kind: DestinationRule +metadata: + annotations: + sme.sap.com/resource-hash: 76ac6b80ce55711ae052011d8d29727030c897d4869ba6c403ac6842f08b93d6 + sme.sap.com/owner-identifier: default.test-cap-01-provider + labels: + sme.sap.com/owner-generation: "0" + sme.sap.com/owner-identifier-hash: db1f1fd7eaeb0e6407c741b7e4b2540044bcc4ec + name: test-cap-01-provider + namespace: default + ownerReferences: + - apiVersion: sme.sap.com/v1alpha1 + blockOwnerDeletion: true + controller: true + kind: CAPTenant + name: test-cap-01-provider +spec: + host: test-cap-01-cav-v1-app-router-svc.default.svc.cluster.local + trafficPolicy: + loadBalancer: + consistentHash: + httpCookie: + name: CAPOP_ROUTER_STICKY + path: / + ttl: 0s diff --git a/internal/controller/testdata/captenant/cat-with-session-affinity-dr-vs-logout-endpoint.yaml b/internal/controller/testdata/captenant/cat-with-session-affinity-dr-vs-logout-endpoint.yaml new file mode 100644 index 00000000..8c46d88b --- /dev/null +++ b/internal/controller/testdata/captenant/cat-with-session-affinity-dr-vs-logout-endpoint.yaml @@ -0,0 +1,126 @@ +apiVersion: sme.sap.com/v1alpha1 +kind: CAPTenant +metadata: + finalizers: + - sme.sap.com/captenant + annotations: + sme.sap.com/btp-app-identifier: btp-glo-acc-id.test-cap-01 + sme.sap.com/owner-identifier: default.test-cap-01 + labels: + sme.sap.com/btp-app-identifier-hash: f20cc8aeb2003b3abc33f749a16bd53544b6bab2 + sme.sap.com/btp-tenant-id: tenant-id-for-provider + sme.sap.com/owner-generation: "2" + sme.sap.com/owner-identifier-hash: 1f74ae2fbff71a708786a4df4bb2ca87ec603581 + sme.sap.com/tenant-type: provider + name: test-cap-01-provider + namespace: default + ownerReferences: + - apiVersion: sme.sap.com/v1alpha1 + blockOwnerDeletion: true + controller: true + kind: CAPApplication + name: test-cap-01 + uid: 3c7ba7cb-dc04-4fd1-be86-3eb3a5c64a98 +spec: + capApplicationInstance: test-cap-01 + subDomain: my-provider + tenantId: tenant-id-for-provider + version: 5.6.7 + versionUpgradeStrategy: always +status: + conditions: + - message: "CAPTenantOperation default.test-cap-01-provider-s6f4l successfully completed" + reason: ProvisioningCompleted + status: "True" + type: Ready + state: Ready + currentCAPApplicationVersionInstance: test-cap-01-cav-v1 +--- +apiVersion: networking.istio.io/v1 +kind: VirtualService +metadata: + annotations: + sme.sap.com/resource-hash: af0113f26d508fb0a84671cbaa263fdc4a9dc666143ed205dc507f08740a0691 + sme.sap.com/owner-identifier: default.test-cap-01-provider + labels: + sme.sap.com/owner-generation: "0" + sme.sap.com/owner-identifier-hash: db1f1fd7eaeb0e6407c741b7e4b2540044bcc4ec + name: test-cap-01-provider + namespace: default + ownerReferences: + - apiVersion: sme.sap.com/v1alpha1 + blockOwnerDeletion: true + controller: true + kind: CAPTenant + name: test-cap-01-provider +spec: + gateways: + - test-cap-01-primary-gen + - default/test-cap-01-secondary-gen + hosts: + - my-provider.app-domain.test.local + - my-provider.foo.bar.local + http: + - headers: + response: + set: + Set-Cookie: COP_CAV=test-cap-01-cav-v1;Path=/;HttpOnly;Secure;Max-Age=0 + match: + - headers: + Cookie: + regex: (^|.*; )COP_CAV=test-cap-01-cav-v1($|; .*) + uri: + regex: ^|.*(signOff).* + route: + - destination: + host: test-cap-01-cav-v1-app-router-svc.default.svc.cluster.local + port: + number: 5000 + weight: 100 + - match: + - headers: + Cookie: + regex: (^|.*; )COP_CAV=test-cap-01-cav-v1($|; .*) + route: + - destination: + host: test-cap-01-cav-v1-app-router-svc.default.svc.cluster.local + port: + number: 5000 + weight: 100 + - headers: + response: + add: + Set-Cookie: COP_CAV=test-cap-01-cav-v1;Path=/;HttpOnly;Secure + route: + - destination: + host: test-cap-01-cav-v1-app-router-svc.default.svc.cluster.local + port: + number: 5000 + weight: 100 +--- +apiVersion: networking.istio.io/v1 +kind: DestinationRule +metadata: + annotations: + sme.sap.com/resource-hash: 76ac6b80ce55711ae052011d8d29727030c897d4869ba6c403ac6842f08b93d6 + sme.sap.com/owner-identifier: default.test-cap-01-provider + labels: + sme.sap.com/owner-generation: "0" + sme.sap.com/owner-identifier-hash: db1f1fd7eaeb0e6407c741b7e4b2540044bcc4ec + name: test-cap-01-provider + namespace: default + ownerReferences: + - apiVersion: sme.sap.com/v1alpha1 + blockOwnerDeletion: true + controller: true + kind: CAPTenant + name: test-cap-01-provider +spec: + host: test-cap-01-cav-v1-app-router-svc.default.svc.cluster.local + trafficPolicy: + loadBalancer: + consistentHash: + httpCookie: + name: CAPOP_ROUTER_STICKY + path: / + ttl: 0s diff --git a/internal/controller/testdata/captenant/cat-with-session-affinity-dr-vs-prev-cav-removed.yaml b/internal/controller/testdata/captenant/cat-with-session-affinity-dr-vs-prev-cav-removed.yaml new file mode 100644 index 00000000..ef470b6b --- /dev/null +++ b/internal/controller/testdata/captenant/cat-with-session-affinity-dr-vs-prev-cav-removed.yaml @@ -0,0 +1,132 @@ +apiVersion: sme.sap.com/v1alpha1 +kind: CAPTenant +metadata: + finalizers: + - sme.sap.com/captenant + generation: 2 + annotations: + sme.sap.com/btp-app-identifier: btp-glo-acc-id.test-cap-01 + sme.sap.com/owner-identifier: default.test-cap-01 + labels: + sme.sap.com/btp-app-identifier-hash: f20cc8aeb2003b3abc33f749a16bd53544b6bab2 + sme.sap.com/btp-tenant-id: tenant-id-for-provider + sme.sap.com/owner-generation: "2" + sme.sap.com/owner-identifier-hash: 1f74ae2fbff71a708786a4df4bb2ca87ec603581 + sme.sap.com/tenant-type: provider + name: test-cap-01-provider + namespace: default + ownerReferences: + - apiVersion: sme.sap.com/v1alpha1 + blockOwnerDeletion: true + controller: true + kind: CAPApplication + name: test-cap-01 + uid: 3c7ba7cb-dc04-4fd1-be86-3eb3a5c64a98 +spec: + capApplicationInstance: test-cap-01 + subDomain: my-provider + tenantId: tenant-id-for-provider + version: 8.9.10 + versionUpgradeStrategy: always +status: + observedGeneration: 2 + conditions: + - message: "VirtualService (and DestinationRule) default.test-cap-01-provider was reconciled" + observedGeneration: 2 + reason: TenantNetworkingModified + status: "True" + type: Ready + state: Ready + currentCAPApplicationVersionInstance: test-cap-01-cav-v2 + previousCAPApplicationVersions: + - test-cap-01-cav-v1 +--- +apiVersion: networking.istio.io/v1 +kind: VirtualService +metadata: + annotations: + sme.sap.com/resource-hash: 3c12498460e81e03c83478a8e3698cd1111b610a871093abf74f37a14f7af186 + sme.sap.com/owner-identifier: default.test-cap-01-provider + labels: + sme.sap.com/owner-generation: "2" + sme.sap.com/owner-identifier-hash: db1f1fd7eaeb0e6407c741b7e4b2540044bcc4ec + name: test-cap-01-provider + namespace: default + ownerReferences: + - apiVersion: sme.sap.com/v1alpha1 + blockOwnerDeletion: true + controller: true + kind: CAPTenant + name: test-cap-01-provider +spec: + gateways: + - test-cap-01-primary-gen + - default/test-cap-01-secondary-gen + hosts: + - my-provider.app-domain.test.local + - my-provider.foo.bar.local + http: + - headers: + response: + set: + Set-Cookie: COP_CAV=test-cap-01-cav-v2;Path=/;HttpOnly;Secure;Max-Age=0 + match: + - headers: + Cookie: + regex: (^|.*; )COP_CAV=test-cap-01-cav-v2($|; .*) + uri: + regex: ^|.*(logout|logoff).* + route: + - destination: + host: test-cap-01-cav-v2-app-router-svc.default.svc.cluster.local + port: + number: 5000 + weight: 100 + - match: + - headers: + Cookie: + regex: (^|.*; )COP_CAV=test-cap-01-cav-v2($|; .*) + route: + - destination: + host: test-cap-01-cav-v2-app-router-svc.default.svc.cluster.local + port: + number: 5000 + weight: 100 + - headers: + response: + add: + Set-Cookie: COP_CAV=test-cap-01-cav-v2;Path=/;HttpOnly;Secure + route: + - destination: + host: test-cap-01-cav-v2-app-router-svc.default.svc.cluster.local + port: + number: 5000 + weight: 100 +--- +apiVersion: networking.istio.io/v1 +kind: DestinationRule +metadata: + annotations: + sme.sap.com/resource-hash: 97ecccd1443b2219a85159a5fdb2f8444543746a5ae890114462e6666e556696 + sme.sap.com/owner-identifier: default.test-cap-01-provider + labels: + sme.sap.com/owner-generation: "2" + sme.sap.com/owner-identifier-hash: db1f1fd7eaeb0e6407c741b7e4b2540044bcc4ec + generation: 1 + name: test-cap-01-provider + namespace: default + ownerReferences: + - apiVersion: sme.sap.com/v1alpha1 + blockOwnerDeletion: true + controller: true + kind: CAPTenant + name: test-cap-01-provider +spec: + host: test-cap-01-cav-v2-app-router-svc.default.svc.cluster.local + trafficPolicy: + loadBalancer: + consistentHash: + httpCookie: + name: CAPOP_ROUTER_STICKY + path: / + ttl: 0s diff --git a/internal/controller/testdata/captenant/cat-with-session-affinity-dr-vs-upgrade-to-cav-v3.expected.yaml b/internal/controller/testdata/captenant/cat-with-session-affinity-dr-vs-upgrade-to-cav-v3.expected.yaml new file mode 100644 index 00000000..6a5f5973 --- /dev/null +++ b/internal/controller/testdata/captenant/cat-with-session-affinity-dr-vs-upgrade-to-cav-v3.expected.yaml @@ -0,0 +1,186 @@ +apiVersion: sme.sap.com/v1alpha1 +kind: CAPTenant +metadata: + finalizers: + - sme.sap.com/captenant + generation: 2 + annotations: + sme.sap.com/btp-app-identifier: btp-glo-acc-id.test-cap-01 + sme.sap.com/owner-identifier: default.test-cap-01 + labels: + sme.sap.com/btp-app-identifier-hash: f20cc8aeb2003b3abc33f749a16bd53544b6bab2 + sme.sap.com/btp-tenant-id: tenant-id-for-provider + sme.sap.com/owner-generation: "2" + sme.sap.com/owner-identifier-hash: 1f74ae2fbff71a708786a4df4bb2ca87ec603581 + sme.sap.com/tenant-type: provider + name: test-cap-01-provider + namespace: default + ownerReferences: + - apiVersion: sme.sap.com/v1alpha1 + blockOwnerDeletion: true + controller: true + kind: CAPApplication + name: test-cap-01 + uid: 3c7ba7cb-dc04-4fd1-be86-3eb3a5c64a98 +spec: + capApplicationInstance: test-cap-01 + subDomain: my-provider + tenantId: tenant-id-for-provider + version: 11.12.13 + versionUpgradeStrategy: always +status: + observedGeneration: 2 + conditions: + - message: "VirtualService (and DestinationRule) default.test-cap-01-provider was reconciled" + observedGeneration: 2 + reason: TenantNetworkingModified + status: "True" + type: Ready + state: Ready + currentCAPApplicationVersionInstance: test-cap-01-cav-v3 + previousCAPApplicationVersions: + - test-cap-01-cav-v1 + - test-cap-01-cav-v2 +--- +apiVersion: networking.istio.io/v1 +kind: VirtualService +metadata: + annotations: + sme.sap.com/resource-hash: cea263f1fc91c820d166e30cab1dcf242966a7919090a3eb1a9907c72d596998 + sme.sap.com/owner-identifier: default.test-cap-01-provider + labels: + sme.sap.com/owner-generation: "2" + sme.sap.com/owner-identifier-hash: db1f1fd7eaeb0e6407c741b7e4b2540044bcc4ec + name: test-cap-01-provider + namespace: default + ownerReferences: + - apiVersion: sme.sap.com/v1alpha1 + blockOwnerDeletion: true + controller: true + kind: CAPTenant + name: test-cap-01-provider +spec: + gateways: + - test-cap-01-primary-gen + - default/test-cap-01-secondary-gen + hosts: + - my-provider.app-domain.test.local + - my-provider.foo.bar.local + http: + - headers: + response: + set: + Set-Cookie: COP_CAV=test-cap-01-cav-v2;Path=/;HttpOnly;Secure;Max-Age=0 + match: + - headers: + Cookie: + regex: (^|.*; )COP_CAV=test-cap-01-cav-v2($|; .*) + uri: + regex: ^|.*(logout|logoff).* + route: + - destination: + host: test-cap-01-cav-v2-app-router-svc.default.svc.cluster.local + port: + number: 5000 + weight: 100 + - headers: + response: + set: + Set-Cookie: COP_CAV=test-cap-01-cav-v3;Path=/;HttpOnly;Secure;Max-Age=0 + match: + - headers: + Cookie: + regex: (^|.*; )COP_CAV=test-cap-01-cav-v3($|; .*) + uri: + regex: ^|.*(logout|logoff).* + route: + - destination: + host: test-cap-01-cav-v3-app-router-svc.default.svc.cluster.local + port: + number: 5000 + weight: 100 + - match: + - headers: + Cookie: + regex: (^|.*; )COP_CAV=test-cap-01-cav-v2($|; .*) + route: + - destination: + host: test-cap-01-cav-v2-app-router-svc.default.svc.cluster.local + port: + number: 5000 + weight: 100 + - match: + - headers: + Cookie: + regex: (^|.*; )COP_CAV=test-cap-01-cav-v3($|; .*) + route: + - destination: + host: test-cap-01-cav-v3-app-router-svc.default.svc.cluster.local + port: + number: 5000 + weight: 100 + - headers: + response: + add: + Set-Cookie: COP_CAV=test-cap-01-cav-v3;Path=/;HttpOnly;Secure + route: + - destination: + host: test-cap-01-cav-v3-app-router-svc.default.svc.cluster.local + port: + number: 5000 + weight: 100 +--- +apiVersion: networking.istio.io/v1 +kind: DestinationRule +metadata: + annotations: + sme.sap.com/resource-hash: 921a053e719ece9bc922497f98452ffd3ad4d057039fd831e3e46da92c4d2df4 + sme.sap.com/owner-identifier: default.test-cap-01-provider + labels: + sme.sap.com/owner-generation: "2" + sme.sap.com/owner-identifier-hash: db1f1fd7eaeb0e6407c741b7e4b2540044bcc4ec + generation: 1 + name: test-cap-01-provider + namespace: default + ownerReferences: + - apiVersion: sme.sap.com/v1alpha1 + blockOwnerDeletion: true + controller: true + kind: CAPTenant + name: test-cap-01-provider +spec: + host: test-cap-01-cav-v3-app-router-svc.default.svc.cluster.local + trafficPolicy: + loadBalancer: + consistentHash: + httpCookie: + name: CAPOP_ROUTER_STICKY + path: / + ttl: 0s +--- +apiVersion: networking.istio.io/v1 +kind: DestinationRule +metadata: + annotations: + sme.sap.com/resource-hash: 97ecccd1443b2219a85159a5fdb2f8444543746a5ae890114462e6666e556696 + sme.sap.com/owner-identifier: default.test-cap-01-provider + labels: + sme.sap.com/owner-generation: "2" + sme.sap.com/owner-identifier-hash: db1f1fd7eaeb0e6407c741b7e4b2540044bcc4ec + name: test-cap-01-provider-test-cap-01-cav-v2 + namespace: default + ownerReferences: + - apiVersion: sme.sap.com/v1alpha1 + blockOwnerDeletion: true + controller: true + kind: CAPTenant + name: test-cap-01-provider +spec: + host: test-cap-01-cav-v2-app-router-svc.default.svc.cluster.local + trafficPolicy: + loadBalancer: + consistentHash: + httpCookie: + name: CAPOP_ROUTER_STICKY + path: / + ttl: 0s diff --git a/internal/controller/testdata/captenant/cat-with-session-affinity-dr-vs-upgrade-to-cav-v3.yaml b/internal/controller/testdata/captenant/cat-with-session-affinity-dr-vs-upgrade-to-cav-v3.yaml new file mode 100644 index 00000000..8f19bd68 --- /dev/null +++ b/internal/controller/testdata/captenant/cat-with-session-affinity-dr-vs-upgrade-to-cav-v3.yaml @@ -0,0 +1,223 @@ +apiVersion: sme.sap.com/v1alpha1 +kind: CAPTenant +metadata: + finalizers: + - sme.sap.com/captenant + generation: 2 + annotations: + sme.sap.com/btp-app-identifier: btp-glo-acc-id.test-cap-01 + sme.sap.com/owner-identifier: default.test-cap-01 + labels: + sme.sap.com/btp-app-identifier-hash: f20cc8aeb2003b3abc33f749a16bd53544b6bab2 + sme.sap.com/btp-tenant-id: tenant-id-for-provider + sme.sap.com/owner-generation: "2" + sme.sap.com/owner-identifier-hash: 1f74ae2fbff71a708786a4df4bb2ca87ec603581 + sme.sap.com/tenant-type: provider + name: test-cap-01-provider + namespace: default + ownerReferences: + - apiVersion: sme.sap.com/v1alpha1 + blockOwnerDeletion: true + controller: true + kind: CAPApplication + name: test-cap-01 + uid: 3c7ba7cb-dc04-4fd1-be86-3eb3a5c64a98 +spec: + capApplicationInstance: test-cap-01 + subDomain: my-provider + tenantId: tenant-id-for-provider + version: 11.12.13 + versionUpgradeStrategy: always +status: + observedGeneration: 2 + conditions: + - message: "VirtualService (and DestinationRule) default.test-cap-01-provider was reconciled" + observedGeneration: 2 + reason: TenantNetworkingModified + status: "True" + type: Ready + state: Ready + currentCAPApplicationVersionInstance: test-cap-01-cav-v3 + previousCAPApplicationVersions: + - test-cap-01-cav-v1 + - test-cap-01-cav-v2 +--- +apiVersion: sme.sap.com/v1alpha1 +kind: CAPTenantOperation +metadata: + generateName: test-cap-01-provider- + annotations: + sme.sap.com/owner-identifier: default.test-cap-01-provider + labels: + sme.sap.com/cav-version: "11.12.13" + sme.sap.com/tenant-operation-type: upgrade + sme.sap.com/owner-generation: "2" + sme.sap.com/owner-identifier-hash: db1f1fd7eaeb0e6407c741b7e4b2540044bcc4ec + name: test-cap-01-provider-upg + namespace: default + finalizers: + - sme.sap.com/captenantoperation + ownerReferences: + - apiVersion: sme.sap.com/v1alpha1 + blockOwnerDeletion: true + controller: true + kind: CAPTenant + name: test-cap-01-provider +spec: + capApplicationVersionInstance: test-cap-01-cav-v3 + subDomain: my-provider + tenantId: tenant-id-for-provider + operation: upgrade + steps: + - name: mtx + type: TenantOperation +status: + conditions: + - message: job default.test-cap-01-provider-upg-abcd completed + reason: StepCompleted + status: "True" + type: Ready + state: Completed +--- +apiVersion: networking.istio.io/v1 +kind: VirtualService +metadata: + annotations: + sme.sap.com/resource-hash: 0ff80461a6fee2f65b5e7ccd5243bec76c8db2b2bf330524fe7ab0911a7339ce + sme.sap.com/owner-identifier: default.test-cap-01-provider + labels: + sme.sap.com/owner-generation: "2" + sme.sap.com/owner-identifier-hash: db1f1fd7eaeb0e6407c741b7e4b2540044bcc4ec + name: test-cap-01-provider + namespace: default + ownerReferences: + - apiVersion: sme.sap.com/v1alpha1 + blockOwnerDeletion: true + controller: true + kind: CAPTenant + name: test-cap-01-provider +spec: + gateways: + - test-cap-01-primary-gen + - default/test-cap-01-secondary-gen + hosts: + - my-provider.app-domain.test.local + - my-provider.foo.bar.local + http: + - headers: + response: + set: + Set-Cookie: COP_CAV=test-cap-01-cav-v1;Path=/;HttpOnly;Secure;Max-Age=0 + match: + - headers: + Cookie: + regex: (^|.*; )COP_CAV=test-cap-01-cav-v1($|; .*) + uri: + regex: ^|.*(logout|logoff).* + route: + - destination: + host: test-cap-01-cav-v1-app-router-svc.default.svc.cluster.local + port: + number: 5000 + weight: 100 + - headers: + response: + set: + Set-Cookie: COP_CAV=test-cap-01-cav-v2;Path=/;HttpOnly;Secure;Max-Age=0 + match: + - headers: + Cookie: + regex: (^|.*; )COP_CAV=test-cap-01-cav-v2($|; .*) + uri: + regex: ^|.*(logout|logoff).* + route: + - destination: + host: test-cap-01-cav-v2-app-router-svc.default.svc.cluster.local + port: + number: 5000 + weight: 100 + - match: + - headers: + Cookie: + regex: (^|.*; )COP_CAV=test-cap-01-cav-v1($|; .*) + route: + - destination: + host: test-cap-01-cav-v1-app-router-svc.default.svc.cluster.local + port: + number: 5000 + weight: 100 + - match: + - headers: + Cookie: + regex: (^|.*; )COP_CAV=test-cap-01-cav-v2($|; .*) + route: + - destination: + host: test-cap-01-cav-v2-app-router-svc.default.svc.cluster.local + port: + number: 5000 + weight: 100 + - headers: + response: + add: + Set-Cookie: COP_CAV=test-cap-01-cav-v2;Path=/;HttpOnly;Secure + route: + - destination: + host: test-cap-01-cav-v2-app-router-svc.default.svc.cluster.local + port: + number: 5000 + weight: 100 +--- +apiVersion: networking.istio.io/v1 +kind: DestinationRule +metadata: + annotations: + sme.sap.com/resource-hash: 97ecccd1443b2219a85159a5fdb2f8444543746a5ae890114462e6666e556696 + sme.sap.com/owner-identifier: default.test-cap-01-provider + labels: + sme.sap.com/owner-generation: "2" + sme.sap.com/owner-identifier-hash: db1f1fd7eaeb0e6407c741b7e4b2540044bcc4ec + generation: 1 + name: test-cap-01-provider + namespace: default + ownerReferences: + - apiVersion: sme.sap.com/v1alpha1 + blockOwnerDeletion: true + controller: true + kind: CAPTenant + name: test-cap-01-provider +spec: + host: test-cap-01-cav-v2-app-router-svc.default.svc.cluster.local + trafficPolicy: + loadBalancer: + consistentHash: + httpCookie: + name: CAPOP_ROUTER_STICKY + path: / + ttl: 0s +--- +apiVersion: networking.istio.io/v1 +kind: DestinationRule +metadata: + annotations: + sme.sap.com/resource-hash: 76ac6b80ce55711ae052011d8d29727030c897d4869ba6c403ac6842f08b93d6 + sme.sap.com/owner-identifier: default.test-cap-01-provider + labels: + sme.sap.com/owner-generation: "2" + sme.sap.com/owner-identifier-hash: db1f1fd7eaeb0e6407c741b7e4b2540044bcc4ec + name: test-cap-01-provider-test-cap-01-cav-v1 + namespace: default + ownerReferences: + - apiVersion: sme.sap.com/v1alpha1 + blockOwnerDeletion: true + controller: true + kind: CAPTenant + name: test-cap-01-provider +spec: + host: test-cap-01-cav-v1-app-router-svc.default.svc.cluster.local + trafficPolicy: + loadBalancer: + consistentHash: + httpCookie: + name: CAPOP_ROUTER_STICKY + path: / + ttl: 0s diff --git a/internal/controller/testdata/captenant/cat-with-session-affinity-dr-vs-upgrade.yaml b/internal/controller/testdata/captenant/cat-with-session-affinity-dr-vs-upgrade.yaml new file mode 100644 index 00000000..5b10be2b --- /dev/null +++ b/internal/controller/testdata/captenant/cat-with-session-affinity-dr-vs-upgrade.yaml @@ -0,0 +1,185 @@ +apiVersion: sme.sap.com/v1alpha1 +kind: CAPTenant +metadata: + finalizers: + - sme.sap.com/captenant + generation: 2 + annotations: + sme.sap.com/btp-app-identifier: btp-glo-acc-id.test-cap-01 + sme.sap.com/owner-identifier: default.test-cap-01 + labels: + sme.sap.com/btp-app-identifier-hash: f20cc8aeb2003b3abc33f749a16bd53544b6bab2 + sme.sap.com/btp-tenant-id: tenant-id-for-provider + sme.sap.com/owner-generation: "2" + sme.sap.com/owner-identifier-hash: 1f74ae2fbff71a708786a4df4bb2ca87ec603581 + sme.sap.com/tenant-type: provider + name: test-cap-01-provider + namespace: default + ownerReferences: + - apiVersion: sme.sap.com/v1alpha1 + blockOwnerDeletion: true + controller: true + kind: CAPApplication + name: test-cap-01 + uid: 3c7ba7cb-dc04-4fd1-be86-3eb3a5c64a98 +spec: + capApplicationInstance: test-cap-01 + subDomain: my-provider + tenantId: tenant-id-for-provider + version: 8.9.10 + versionUpgradeStrategy: always +status: + observedGeneration: 2 + conditions: + - message: "CAPTenantOperation default.test-cap-01-provider-upg successfully completed" + observedGeneration: 2 + reason: UpgradeCompleted + status: "True" + type: Ready + state: Ready + currentCAPApplicationVersionInstance: test-cap-01-cav-v2 + previousCAPApplicationVersions: + - test-cap-01-cav-v1 +--- +apiVersion: networking.istio.io/v1 +kind: VirtualService +metadata: + annotations: + sme.sap.com/resource-hash: 0ff80461a6fee2f65b5e7ccd5243bec76c8db2b2bf330524fe7ab0911a7339ce + sme.sap.com/owner-identifier: default.test-cap-01-provider + labels: + sme.sap.com/owner-generation: "2" + sme.sap.com/owner-identifier-hash: db1f1fd7eaeb0e6407c741b7e4b2540044bcc4ec + name: test-cap-01-provider + namespace: default + ownerReferences: + - apiVersion: sme.sap.com/v1alpha1 + blockOwnerDeletion: true + controller: true + kind: CAPTenant + name: test-cap-01-provider +spec: + gateways: + - test-cap-01-primary-gen + - default/test-cap-01-secondary-gen + hosts: + - my-provider.app-domain.test.local + - my-provider.foo.bar.local + http: + - headers: + response: + set: + Set-Cookie: COP_CAV=test-cap-01-cav-v1;Path=/;HttpOnly;Secure;Max-Age=0 + match: + - headers: + Cookie: + regex: (^|.*; )COP_CAV=test-cap-01-cav-v1($|; .*) + uri: + regex: ^|.*(logout|logoff).* + route: + - destination: + host: test-cap-01-cav-v1-app-router-svc.default.svc.cluster.local + port: + number: 5000 + weight: 100 + - headers: + response: + set: + Set-Cookie: COP_CAV=test-cap-01-cav-v2;Path=/;HttpOnly;Secure;Max-Age=0 + match: + - headers: + Cookie: + regex: (^|.*; )COP_CAV=test-cap-01-cav-v2($|; .*) + uri: + regex: ^|.*(logout|logoff).* + route: + - destination: + host: test-cap-01-cav-v2-app-router-svc.default.svc.cluster.local + port: + number: 5000 + weight: 100 + - match: + - headers: + Cookie: + regex: (^|.*; )COP_CAV=test-cap-01-cav-v1($|; .*) + route: + - destination: + host: test-cap-01-cav-v1-app-router-svc.default.svc.cluster.local + port: + number: 5000 + weight: 100 + - match: + - headers: + Cookie: + regex: (^|.*; )COP_CAV=test-cap-01-cav-v2($|; .*) + route: + - destination: + host: test-cap-01-cav-v2-app-router-svc.default.svc.cluster.local + port: + number: 5000 + weight: 100 + - headers: + response: + add: + Set-Cookie: COP_CAV=test-cap-01-cav-v2;Path=/;HttpOnly;Secure + route: + - destination: + host: test-cap-01-cav-v2-app-router-svc.default.svc.cluster.local + port: + number: 5000 + weight: 100 +--- +apiVersion: networking.istio.io/v1 +kind: DestinationRule +metadata: + annotations: + sme.sap.com/resource-hash: 97ecccd1443b2219a85159a5fdb2f8444543746a5ae890114462e6666e556696 + sme.sap.com/owner-identifier: default.test-cap-01-provider + labels: + sme.sap.com/owner-generation: "2" + sme.sap.com/owner-identifier-hash: db1f1fd7eaeb0e6407c741b7e4b2540044bcc4ec + generation: 1 + name: test-cap-01-provider + namespace: default + ownerReferences: + - apiVersion: sme.sap.com/v1alpha1 + blockOwnerDeletion: true + controller: true + kind: CAPTenant + name: test-cap-01-provider +spec: + host: test-cap-01-cav-v2-app-router-svc.default.svc.cluster.local + trafficPolicy: + loadBalancer: + consistentHash: + httpCookie: + name: CAPOP_ROUTER_STICKY + path: / + ttl: 0s +--- +apiVersion: networking.istio.io/v1 +kind: DestinationRule +metadata: + annotations: + sme.sap.com/resource-hash: 76ac6b80ce55711ae052011d8d29727030c897d4869ba6c403ac6842f08b93d6 + sme.sap.com/owner-identifier: default.test-cap-01-provider + labels: + sme.sap.com/owner-generation: "2" + sme.sap.com/owner-identifier-hash: db1f1fd7eaeb0e6407c741b7e4b2540044bcc4ec + name: test-cap-01-provider-test-cap-01-cav-v1 + namespace: default + ownerReferences: + - apiVersion: sme.sap.com/v1alpha1 + blockOwnerDeletion: true + controller: true + kind: CAPTenant + name: test-cap-01-provider +spec: + host: test-cap-01-cav-v1-app-router-svc.default.svc.cluster.local + trafficPolicy: + loadBalancer: + consistentHash: + httpCookie: + name: CAPOP_ROUTER_STICKY + path: / + ttl: 0s diff --git a/internal/controller/testdata/captenant/cat-with-session-affinity-dr-vs.yaml b/internal/controller/testdata/captenant/cat-with-session-affinity-dr-vs.yaml new file mode 100644 index 00000000..a78f1dcb --- /dev/null +++ b/internal/controller/testdata/captenant/cat-with-session-affinity-dr-vs.yaml @@ -0,0 +1,126 @@ +apiVersion: sme.sap.com/v1alpha1 +kind: CAPTenant +metadata: + finalizers: + - sme.sap.com/captenant + annotations: + sme.sap.com/btp-app-identifier: btp-glo-acc-id.test-cap-01 + sme.sap.com/owner-identifier: default.test-cap-01 + labels: + sme.sap.com/btp-app-identifier-hash: f20cc8aeb2003b3abc33f749a16bd53544b6bab2 + sme.sap.com/btp-tenant-id: tenant-id-for-provider + sme.sap.com/owner-generation: "2" + sme.sap.com/owner-identifier-hash: 1f74ae2fbff71a708786a4df4bb2ca87ec603581 + sme.sap.com/tenant-type: provider + name: test-cap-01-provider + namespace: default + ownerReferences: + - apiVersion: sme.sap.com/v1alpha1 + blockOwnerDeletion: true + controller: true + kind: CAPApplication + name: test-cap-01 + uid: 3c7ba7cb-dc04-4fd1-be86-3eb3a5c64a98 +spec: + capApplicationInstance: test-cap-01 + subDomain: my-provider + tenantId: tenant-id-for-provider + version: 5.6.7 + versionUpgradeStrategy: always +status: + conditions: + - message: "CAPTenantOperation default.test-cap-01-provider-s6f4l successfully completed" + reason: ProvisioningCompleted + status: "True" + type: Ready + state: Ready + currentCAPApplicationVersionInstance: test-cap-01-cav-v1 +--- +apiVersion: networking.istio.io/v1 +kind: VirtualService +metadata: + annotations: + sme.sap.com/resource-hash: c2d2f45edacdf3d50dcef3149f4874391120b8df3a91b2fe13ae321aee606784 + sme.sap.com/owner-identifier: default.test-cap-01-provider + labels: + sme.sap.com/owner-generation: "0" + sme.sap.com/owner-identifier-hash: db1f1fd7eaeb0e6407c741b7e4b2540044bcc4ec + name: test-cap-01-provider + namespace: default + ownerReferences: + - apiVersion: sme.sap.com/v1alpha1 + blockOwnerDeletion: true + controller: true + kind: CAPTenant + name: test-cap-01-provider +spec: + gateways: + - test-cap-01-primary-gen + - default/test-cap-01-secondary-gen + hosts: + - my-provider.app-domain.test.local + - my-provider.foo.bar.local + http: + - headers: + response: + set: + Set-Cookie: COP_CAV=test-cap-01-cav-v1;Path=/;HttpOnly;Secure;Max-Age=0 + match: + - headers: + Cookie: + regex: (^|.*; )COP_CAV=test-cap-01-cav-v1($|; .*) + uri: + regex: ^|.*(logout|logoff).* + route: + - destination: + host: test-cap-01-cav-v1-app-router-svc.default.svc.cluster.local + port: + number: 5000 + weight: 100 + - match: + - headers: + Cookie: + regex: (^|.*; )COP_CAV=test-cap-01-cav-v1($|; .*) + route: + - destination: + host: test-cap-01-cav-v1-app-router-svc.default.svc.cluster.local + port: + number: 5000 + weight: 100 + - headers: + response: + add: + Set-Cookie: COP_CAV=test-cap-01-cav-v1;Path=/;HttpOnly;Secure + route: + - destination: + host: test-cap-01-cav-v1-app-router-svc.default.svc.cluster.local + port: + number: 5000 + weight: 100 +--- +apiVersion: networking.istio.io/v1 +kind: DestinationRule +metadata: + annotations: + sme.sap.com/resource-hash: 76ac6b80ce55711ae052011d8d29727030c897d4869ba6c403ac6842f08b93d6 + sme.sap.com/owner-identifier: default.test-cap-01-provider + labels: + sme.sap.com/owner-generation: "0" + sme.sap.com/owner-identifier-hash: db1f1fd7eaeb0e6407c741b7e4b2540044bcc4ec + name: test-cap-01-provider + namespace: default + ownerReferences: + - apiVersion: sme.sap.com/v1alpha1 + blockOwnerDeletion: true + controller: true + kind: CAPTenant + name: test-cap-01-provider +spec: + host: test-cap-01-cav-v1-app-router-svc.default.svc.cluster.local + trafficPolicy: + loadBalancer: + consistentHash: + httpCookie: + name: CAPOP_ROUTER_STICKY + path: / + ttl: 0s diff --git a/internal/controller/testdata/capapplication/ca-32.initial.yaml b/internal/controller/testdata/common/capapplication-session-affinity-vs-headers.yaml similarity index 64% rename from internal/controller/testdata/capapplication/ca-32.initial.yaml rename to internal/controller/testdata/common/capapplication-session-affinity-vs-headers.yaml index 8911e4d3..69b0730c 100644 --- a/internal/controller/testdata/capapplication/ca-32.initial.yaml +++ b/internal/controller/testdata/common/capapplication-session-affinity-vs-headers.yaml @@ -1,23 +1,33 @@ apiVersion: sme.sap.com/v1alpha1 kind: CAPApplication metadata: + annotations: + sme.sap.com/enable-session-affinity: "true" + sme.sap.com/vs-route-request-header-set: | + { + "x-downstream-peer-subject": "%DOWNSTREAM_PEER_SUBJECT%", + "x-tenant-id": "invalid-tenant-id" + } + sme.sap.com/vs-route-response-header-set: | + { + "foo": "bar" + } finalizers: - sme.sap.com/capapplication - generation: 0 + generation: 2 name: test-cap-01 namespace: default resourceVersion: "11373799" uid: 3c7ba7cb-dc04-4fd1-be86-3eb3a5c64a98 - annotations: - sme.sap.com/btp-app-identifier: btp-glo-acc-id.test-cap-01 - labels: - sme.sap.com/btp-app-identifier-hash: f20cc8aeb2003b3abc33f749a16bd53544b6bab2 spec: btp: services: - class: xsuaa name: cap-uaa secret: cap-cap-01-uaa-bind-cf + - class: xsuaa + name: cap-uaa2 + secret: cap-cap-01-uaa2-bind-cf - class: saas-registry name: cap-saas-registry secret: cap-cap-01-saas-bind-cf @@ -34,11 +44,3 @@ spec: provider: subDomain: my-provider tenantId: tenant-id-for-provider -status: - observedGeneration: 0 - state: "Consistent" - servicesOnly: false - domainSpecHash: '{"ClusterDomain.test-cap-01-secondary":"foo.bar.local","Domain.default.test-cap-01-primary":"app-domain.test.local"}' - observedSubdomains: - - my-consumer - - my-provider diff --git a/internal/controller/testdata/common/capapplication-session-affinity.yaml b/internal/controller/testdata/common/capapplication-session-affinity.yaml new file mode 100644 index 00000000..88e2d79a --- /dev/null +++ b/internal/controller/testdata/common/capapplication-session-affinity.yaml @@ -0,0 +1,37 @@ +apiVersion: sme.sap.com/v1alpha1 +kind: CAPApplication +metadata: + annotations: + sme.sap.com/enable-session-affinity: "true" + finalizers: + - sme.sap.com/capapplication + generation: 2 + name: test-cap-01 + namespace: default + resourceVersion: "11373799" + uid: 3c7ba7cb-dc04-4fd1-be86-3eb3a5c64a98 +spec: + btp: + services: + - class: xsuaa + name: cap-uaa + secret: cap-cap-01-uaa-bind-cf + - class: xsuaa + name: cap-uaa2 + secret: cap-cap-01-uaa2-bind-cf + - class: saas-registry + name: cap-saas-registry + secret: cap-cap-01-saas-bind-cf + - class: service-manager + name: cap-service-manager + secret: cap-cap-01-svc-man-bind-cf + btpAppName: test-cap-01 + domainRefs: + - kind: Domain + name: test-cap-01-primary + - kind: ClusterDomain + name: test-cap-01-secondary + globalAccountId: btp-glo-acc-id + provider: + subDomain: my-provider + tenantId: tenant-id-for-provider diff --git a/internal/controller/testdata/common/capapplicationversion-v1-custom-logout-endpoint.yaml b/internal/controller/testdata/common/capapplicationversion-v1-custom-logout-endpoint.yaml new file mode 100644 index 00000000..fa3b1c1d --- /dev/null +++ b/internal/controller/testdata/common/capapplicationversion-v1-custom-logout-endpoint.yaml @@ -0,0 +1,68 @@ +apiVersion: sme.sap.com/v1alpha1 +kind: CAPApplicationVersion +metadata: + creationTimestamp: "2022-03-18T22:14:33Z" + generation: 1 + annotations: + sme.sap.com/btp-app-identifier: btp-glo-acc-id.test-cap-01 + sme.sap.com/owner-identifier: default.test-cap-01 + sme.sap.com/logout-endpoint: "signOff" + labels: + sme.sap.com/btp-app-identifier-hash: f20cc8aeb2003b3abc33f749a16bd53544b6bab2 + sme.sap.com/owner-generation: "2" + sme.sap.com/owner-identifier-hash: 1f74ae2fbff71a708786a4df4bb2ca87ec603581 + name: test-cap-01-cav-v1 + namespace: default + ownerReferences: + - apiVersion: sme.sap.com/v1alpha1 + blockOwnerDeletion: true + controller: true + kind: CAPApplication + name: test-cap-01 + uid: 3c7ba7cb-dc04-4fd1-be86-3eb3a5c64a98 + resourceVersion: "11371108" + uid: 5e64489b-7346-4984-8617-e8c37338b3d8 +spec: + capApplicationInstance: test-cap-01 + registrySecrets: + - regcred + version: 5.6.7 + workloads: + - name: cap-backend + consumedBTPServices: + - cap-uaa + - cap-service-manager + - cap-saas-registry + deploymentDefinition: + type: CAP + image: docker.image.repo/srv/server:latest + - name: content-job + consumedBTPServices: + - cap-uaa + jobDefinition: + type: Content + image: docker.image.repo/content/cap-content:latest + - name: mtx + consumedBTPServices: + - cap-uaa + - cap-service-manager + - cap-saas-registry + jobDefinition: + type: "TenantOperation" + image: docker.image.repo/srv/server:latest + - name: app-router + consumedBTPServices: + - cap-uaa + - cap-saas-registry + deploymentDefinition: + type: Router + image: docker.image.repo/approuter/approuter:latest +status: + conditions: + - lastTransitionTime: "2022-03-18T23:07:47Z" + lastUpdateTime: "2022-03-18T23:07:47Z" + reason: CreatedDeployments + status: "True" + type: Ready + observedGeneration: 1 + state: Ready diff --git a/website/content/en/docs/usage/version-upgrade.md b/website/content/en/docs/usage/version-upgrade.md index ad9e4b86..8eb6df8e 100644 --- a/website/content/en/docs/usage/version-upgrade.md +++ b/website/content/en/docs/usage/version-upgrade.md @@ -118,3 +118,42 @@ spec: The `CAPTenantOperation` creates jobs for each of the steps involved and executes them sequentially until all the jobs are finished or one of them fails. The `CAPTenant` is notified about the result and updates its state accordingly. A successful completion of the `CAPTenantOperation` will cause the `VirtualService` managed by the `CAPTenant` to be modified to route HTTP traffic to the deployments of the newer `CAPApplicationVersion`. Once all tenants have been upgraded, the outdated `CAPApplicationVersion` can be deleted. + +## Session Afinity during Upgrade + +Normally once the upgrade is done, the incoming requests get routed to the new instance of the Approuter. If [External Session Management](https://www.npmjs.com/package/@sap/approuter#external-session-management) is not enabled on the Approuter, the exisiting user sessions will be lost and users will be logged out. To avoid this, you can enable Session Affinity by adding the following annotation `sme.sap.com/enable-session-affinity: "true"` to the `CAPApplication` resource. + +```yaml +apiVersion: sme.sap.com/v1alpha1 +kind: CAPApplicationVersion +metadata: + name: cav-cap-app-01-2 + namespace: cap-app-01 + annotations: + sme.sap.com/enable-session-affinity: "true" # <-- enable session affinity +spec: + capApplicationInstance: cap-cap-app-01 + version: "2.0.1" + registrySecrets: + - regcred + .... +``` + +Once this annotation is set, CAP Operator will set session cookies to ensure that all existing requests are routed to the existing Approuter instance until the session expires. New requests without session cookies will be routed to the new Approuter instance. This ensures that existing user sessions are not interrupted during the upgrade. Once the sessions expire or the logout endpoint is called, the requests will be routed to the new Approuter instance. + +CAP Operator defaults the logout endpoint to `logout` or `logoff`. If the Approuter is configured with a different endpoint, it must be specified using the annotation `sme.sap.com/logout-endpoint` in your `CAPApplicationVersion` resource. + +```yaml +apiVersion: sme.sap.com/v1alpha1 +kind: CAPApplicationVersion +metadata: + name: cav-cap-app-01-2 + namespace: cap-app-01 + annotations: + sme.sap.com/logout-endpoint: "custom-logout" # <-- specify custom logout endpoint (Don't include leading slash) +spec: + capApplicationInstance: cap-cap-app-01 + version: "2.0.1" + registrySecrets: + - regcred +```