feat: Read component descriptor from oci registry

.github/actions/deploy-lifecycle-manager-e2e/action.yml

@@ -21,6 +21,45 @@ runs:
         else
           echo "E2E_KUSTOMIZE_DIR=config/watcher_local_test" >> $GITHUB_ENV
         fi
+    - name: Patch local OCI registry host
+      if: ${{ matrix.e2e-test != 'oci-reg-cred-secret' && matrix.e2e-test != 'module-transferred-to-another-oci-registry' }}
+      working-directory: lifecycle-manager
+      shell: bash
+      run: |
+        pushd ${E2E_KUSTOMIZE_DIR}
+        echo \
+        "- op: add
+          path: /spec/template/spec/containers/0/args/-
+          value: --oci-registry-host=http://k3d-kcp-registry.localhost:5000" >> oci_registry_host.yaml
+        cat oci_registry_host.yaml
+        kustomize edit add patch --path oci_registry_host.yaml --kind Deployment
+        popd
+    - name: Patch remote OCI registry host
+      if: ${{ matrix.e2e-test == 'module-transferred-to-another-oci-registry' }}
+      working-directory: lifecycle-manager
+      shell: bash
+      run: |
+        pushd ${E2E_KUSTOMIZE_DIR}
+        echo \
+        "- op: add
+          path: /spec/template/spec/containers/0/args/-
+          value: --oci-registry-host=https://europe-west3-docker.pkg.dev/sap-kyma-jellyfish-dev/restricted-market" >> oci_registry_host.yaml
+        cat oci_registry_host.yaml
+        kustomize edit add patch --path oci_registry_host.yaml --kind Deployment
+        popd
+    - name: Patch private OCI registry secret
+      if: ${{ matrix.e2e-test == 'oci-reg-cred-secret' }}
+      working-directory: lifecycle-manager
+      shell: bash
+      run: |
+        pushd ${E2E_KUSTOMIZE_DIR}
+        echo \
+        "- op: add
+          path: /spec/template/spec/containers/0/args/-
+          value: --oci-registry-cred-secret=private-oci-reg-creds" >> oci_registry_host.yaml
+        cat oci_registry_host.yaml
+        kustomize edit add patch --path oci_registry_host.yaml --kind Deployment
+        popd
     - name: Patch purge finalizer flags
       if: ${{ matrix.e2e-test == 'purge-controller' ||  matrix.e2e-test == 'purge-metrics'}}
       working-directory: lifecycle-manager
@@ -148,14 +187,6 @@ runs:
         cat legacy-secret-rotation.yaml
         kustomize edit add patch --path legacy-secret-rotation.yaml --kind Deployment
         popd
-    - name: Use private OCI registry credentials
-      if: ${{matrix.e2e-test == 'oci-reg-cred-secret'}}
-      working-directory: lifecycle-manager
-      shell: bash
-      run: |
-        pushd ${E2E_KUSTOMIZE_DIR}
-        sed -i 's|value: --oci-registry-host=europe-docker.pkg.dev/kyma-project/kyma-modules|value: --oci-registry-cred-secret=private-oci-reg-creds|' kustomization.yaml
-        popd
     - name: Create and use maintenance window policy
       if: ${{matrix.e2e-test == 'maintenance-windows' ||
         matrix.e2e-test == 'maintenance-windows-initial-installation' ||

.github/actions/deploy-template-operator-with-modulereleasemeta/action.yml

@@ -79,10 +79,9 @@ runs:
         sed -i 's/k3d-private-oci-reg.localhost:5001/private-oci-reg.localhost:5000/g' ./template.yaml
         kubectl get crds
         kubectl apply -f template.yaml
-    - name: Create and apply ModuleReleaseMeta from the latest release
+    - name: Create and apply ModuleReleaseMeta from the template-operator repo
       working-directory: template-operator
       if: ${{ matrix.e2e-test == 'kyma-metrics' ||
-        matrix.e2e-test == 'non-blocking-deletion' ||
         matrix.e2e-test == 'purge-controller' ||
         matrix.e2e-test == 'purge-metrics' ||
         matrix.e2e-test == 'kyma-deprovision-with-foreground-propagation' ||

.github/scripts/debug/teardown.sh

@@ -1,6 +1,23 @@
 #!/usr/bin/env bash
+kubectl config use-context k3d-kcp
 
 k3d cluster list
+echo "--- KCP ModuleTemplate ---"
+kubectl get moduletemplate -n kcp-system -o wide
+kubectl get moduletemplate -n kcp-system -o yaml
+
+echo "--- KCP ModuleReleaseMeta ---"
+kubectl get modulereleasemeta -n kcp-system -o wide
+kubectl get modulereleasemeta -n kcp-system -o yaml
+
+echo "--- KCP Kyma ---"
+kubectl get kyma -n kcp-system -o wide
+kubectl get kyma -n kcp-system -o yaml
+
+echo "--- KCP Manifest ---"
+kubectl get manifest -n kcp-system -o wide
+kubectl get manifest -n kcp-system -o yaml
+
 echo "--- KLM DEPLOYMENT ---"
 kubectl get deploy klm-controller-manager -n kcp-system -o yaml
 kubectl describe deploy klm-controller-manager  -n kcp-system
@@ -13,7 +30,14 @@ set -e
 
 kubectl config use-context k3d-skr
 
+
+echo "--- SKR DEPLOYMENT OVERVIEW ---"
+kubectl get deploy -A -o wide
+
 echo "--- SKR-WEBHOOK POD ---"
+kubectl describe deploy/skr-webhook  -n kyma-system
 kubectl get pods -l app=skr-webhook -n kyma-system -o wide
+
 echo "--- SKR-WEBHOOK LOGS ---"
 kubectl logs deploy/skr-webhook -n kyma-system --container server
+

.golangci.yaml

@@ -132,6 +132,8 @@ linters:
           alias: secretrepository
         - pkg: k8s.io/api/networking/v1
           alias: apinetworkv1
+        - pkg: github.com/kyma-project/lifecycle-manager/internal/descriptor/cache
+          alias: descriptorcache
         - pkg: github.com/kyma-project/lifecycle-manager/internal/service/watcher/resources
           alias: skrwebhookresources
         - pkg: github.com/kyma-project/lifecycle-manager/internal/service/watcher/certificate/renewal/certmanager

cmd/composition/provider/componentdescriptorcache/provider.go

@@ -0,0 +1,13 @@
+package componentdescriptorcache
+
+import (
+	"github.com/kyma-project/lifecycle-manager/internal/descriptor/provider"
+)
+
+// ComposeComponentDescriptorService manges creation of a new instance of the Cached ComponentDescriptor Provider.
+func ComposeCachedDescriptorProvider(
+	service provider.DescriptorService,
+	cache provider.DescriptorCache,
+) *provider.CachedDescriptorProvider {
+	return provider.NewCachedDescriptorProvider(service, cache)
+}

cmd/composition/repository/oci/ocirepository.go

@@ -0,0 +1,31 @@
+package oci
+
+import (
+	"os"
+
+	"github.com/go-logr/logr"
+
+	"github.com/kyma-project/lifecycle-manager/internal/manifest/spec"
+	"github.com/kyma-project/lifecycle-manager/internal/repository/ocm"
+	"github.com/kyma-project/lifecycle-manager/internal/repository/ocm/oci"
+)
+
+func ComposeOCIRepository(
+	kcl spec.KeyChainLookup,
+	hostWithPort string,
+	insecure bool,
+	logger logr.Logger,
+	bootstrapFailedExitCode int,
+) *ocm.RepositoryReader {
+	ociRepository, err := oci.NewRepository(kcl, insecure)
+	if err != nil {
+		logger.Error(err, "failed to create OCI repository")
+		os.Exit(bootstrapFailedExitCode)
+	}
+	ocmRepository, err := ocm.NewRepository(hostWithPort, ociRepository)
+	if err != nil {
+		logger.Error(err, "failed to create OCI repository")
+		os.Exit(bootstrapFailedExitCode)
+	}
+	return ocmRepository
+}

cmd/composition/service/componentdescriptor/componentdescriptor.go

@@ -0,0 +1,27 @@
+package componentdescriptor
+
+import (
+	"os"
+
+	"github.com/go-logr/logr"
+
+	"github.com/kyma-project/lifecycle-manager/internal/service/componentdescriptor"
+)
+
+// ComposeComponentDescriptorService manges creation of a new instance of the ComponentDescriptor Service.
+func ComposeComponentDescriptorService(
+	repository componentdescriptor.OCIRepository,
+	logger logr.Logger,
+	bootstrapFailedExitCode int,
+) *componentdescriptor.Service {
+	tarExtractor := componentdescriptor.NewTarExtractor()
+	fileExtractor := componentdescriptor.NewFileExtractor(tarExtractor)
+
+	service, err := componentdescriptor.NewService(repository, fileExtractor)
+	if err != nil {
+		logger.Error(err, "failed to create OCM descriptor service")
+		os.Exit(bootstrapFailedExitCode)
+	}
+
+	return service
+}

cmd/main.go

@@ -24,6 +24,7 @@ import (
 	"net/http"
 	"net/http/pprof"
 	"os"
+	"strings"
 	"time"
 
 	certmanagerv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1"
@@ -50,6 +51,9 @@ import (
 	"github.com/kyma-project/lifecycle-manager/api"
 	"github.com/kyma-project/lifecycle-manager/api/shared"
 	"github.com/kyma-project/lifecycle-manager/api/v1beta2"
+	"github.com/kyma-project/lifecycle-manager/cmd/composition/provider/componentdescriptorcache"
+	"github.com/kyma-project/lifecycle-manager/cmd/composition/repository/oci"
+	"github.com/kyma-project/lifecycle-manager/cmd/composition/service/componentdescriptor"
 	"github.com/kyma-project/lifecycle-manager/cmd/composition/service/skrwebhook"
 	"github.com/kyma-project/lifecycle-manager/internal"
 	"github.com/kyma-project/lifecycle-manager/internal/controller/istiogatewaysecret"
@@ -60,6 +64,7 @@ import (
 	watcherctrl "github.com/kyma-project/lifecycle-manager/internal/controller/watcher"
 	"github.com/kyma-project/lifecycle-manager/internal/crd"
 	declarativev2 "github.com/kyma-project/lifecycle-manager/internal/declarative/v2"
+	descriptorcache "github.com/kyma-project/lifecycle-manager/internal/descriptor/cache"
 	"github.com/kyma-project/lifecycle-manager/internal/descriptor/provider"
 	"github.com/kyma-project/lifecycle-manager/internal/event"
 	gatewaysecretclient "github.com/kyma-project/lifecycle-manager/internal/gatewaysecret/client"
@@ -221,21 +226,46 @@ func setupManager(flagVar *flags.FlagVar, cacheOptions cache.Options, scheme *ma
 	}
 
 	sharedMetrics := metrics.NewSharedMetrics()
-	descriptorProvider := provider.NewCachedDescriptorProvider()
+
+	ociRegistryHost := getOciRegistryHost(mgr.GetConfig(), flagVar, logger)
+	var insecure bool
+
+	if noSchemeRef, found := strings.CutPrefix(ociRegistryHost, "http://"); found {
+		insecure = true
+		ociRegistryHost = noSchemeRef
+	} else if noSchemeRef, found := strings.CutPrefix(ociRegistryHost, "https://"); found {
+		ociRegistryHost = noSchemeRef
+	}
+
+	ocmDescriptorRepository := oci.ComposeOCIRepository(
+		keychainLookupFromFlag(mgr.GetClient(), flagVar),
+		ociRegistryHost,
+		insecure,
+		logger,
+		bootstrapFailedExitCode,
+	)
+	ocmDescriptorService := componentdescriptor.ComposeComponentDescriptorService(
+		ocmDescriptorRepository,
+		logger,
+		bootstrapFailedExitCode,
+	)
+
+	descriptorProvider := componentdescriptorcache.ComposeCachedDescriptorProvider(
+		ocmDescriptorService,
+		descriptorcache.NewDescriptorCache(),
+	)
+
 	kymaMetrics := metrics.NewKymaMetrics(sharedMetrics)
 	mandatoryModulesMetrics := metrics.NewMandatoryModulesMetrics()
 	maintenanceWindow := initMaintenanceWindow(flagVar.MinMaintenanceWindowSize, logger)
 	metrics.NewFipsMetrics().Update()
 
-	//nolint:godox // this will be used in the future
-	// TODO: use the oci registry host //nolint:godox // this will be used in the future
-	_ = getOciRegistryHost(mgr.GetConfig(), flagVar, logger)
-
-	setupKymaReconciler(mgr, descriptorProvider, skrContextProvider, eventRecorder, flagVar, options, skrWebhookManager,
-		kymaMetrics, logger, maintenanceWindow)
-	setupManifestReconciler(mgr, flagVar, options, sharedMetrics, mandatoryModulesMetrics, accessManagerService, logger,
-		eventRecorder)
-	setupMandatoryModuleReconciler(mgr, descriptorProvider, flagVar, options, mandatoryModulesMetrics, logger)
+	setupKymaReconciler(mgr, descriptorProvider, skrContextProvider, eventRecorder,
+		flagVar, options, skrWebhookManager, kymaMetrics, logger, maintenanceWindow, ociRegistryHost)
+	setupManifestReconciler(mgr, flagVar, options, sharedMetrics, mandatoryModulesMetrics,
+		accessManagerService, logger, eventRecorder)
+	setupMandatoryModuleReconciler(mgr, descriptorProvider, flagVar, options,
+		mandatoryModulesMetrics, logger, ociRegistryHost)
 	setupMandatoryModuleDeletionReconciler(mgr, descriptorProvider, eventRecorder, flagVar, options, logger)
 	if flagVar.EnablePurgeFinalizer {
 		setupPurgeReconciler(mgr, skrContextProvider, eventRecorder, flagVar, options, logger)
@@ -379,7 +409,7 @@ func scheduleMetricsCleanup(kymaMetrics *metrics.KymaMetrics, cleanupIntervalInM
 func setupKymaReconciler(mgr ctrl.Manager, descriptorProvider *provider.CachedDescriptorProvider,
 	skrContextFactory remote.SkrContextProvider, event event.Event, flagVar *flags.FlagVar, options ctrlruntime.Options,
 	skrWebhookManager *watcher.SkrWebhookManifestManager, kymaMetrics *metrics.KymaMetrics,
-	setupLog logr.Logger, maintenanceWindow maintenancewindows.MaintenanceWindow,
+	setupLog logr.Logger, maintenanceWindow maintenancewindows.MaintenanceWindow, ociRegistryHost string,
 ) {
 	options.RateLimiter = internal.RateLimiter(flagVar.FailureBaseDelay,
 		flagVar.FailureMaxDelay, flagVar.RateLimiterFrequency, flagVar.RateLimiterBurst)
@@ -420,6 +450,7 @@ func setupKymaReconciler(mgr ctrl.Manager, descriptorProvider *provider.CachedDe
 			flagVar.RemoteSyncNamespace),
 		TemplateLookup: templatelookup.NewTemplateLookup(kcpClient, descriptorProvider,
 			moduleTemplateInfoLookupStrategies),
+		OCIRegistryHost: ociRegistryHost,
 	}).SetupWithManager(
 		mgr, options, kyma.SetupOptions{
 			ListenerAddr:                 flagVar.KymaListenerAddr,
@@ -476,7 +507,7 @@ func setupManifestReconciler(mgr ctrl.Manager,
 	manifestClient := manifestclient.NewManifestClient(event, mgr.GetClient())
 	orphanDetectionClient := kymarepository.NewClient(mgr.GetClient())
 	orphanDetectionService := orphan.NewDetectionService(orphanDetectionClient)
-	specResolver := spec.NewResolver(keychainLookupFromFlag(mgr, flagVar), img.NewPathExtractor())
+	specResolver := spec.NewResolver(keychainLookupFromFlag(mgr.GetClient(), flagVar), img.NewPathExtractor())
 	clientCache := skrclientcache.NewService()
 	skrClient := skrclient.NewService(mgr.GetConfig().QPS, mgr.GetConfig().Burst, accessManagerService)
 
@@ -504,9 +535,9 @@ func setupManifestReconciler(mgr ctrl.Manager,
 }
 
 //nolint:ireturn // constructor functions can return interfaces
-func keychainLookupFromFlag(mgr ctrl.Manager, flagVar *flags.FlagVar) spec.KeyChainLookup {
+func keychainLookupFromFlag(clnt client.Client, flagVar *flags.FlagVar) spec.KeyChainLookup {
 	if flagVar.OciRegistryCredSecretName != "" {
-		return keychainprovider.NewFromSecretKeyChainProvider(mgr.GetClient(),
+		return keychainprovider.NewFromSecretKeyChainProvider(clnt,
 			types.NamespacedName{
 				Namespace: shared.DefaultControlPlaneNamespace,
 				Name:      flagVar.OciRegistryCredSecretName,
@@ -547,6 +578,7 @@ func setupMandatoryModuleReconciler(mgr ctrl.Manager,
 	options ctrlruntime.Options,
 	metrics *metrics.MandatoryModulesMetrics,
 	setupLog logr.Logger,
+	ociRegistryHost string,
 ) {
 	options.RateLimiter = internal.RateLimiter(flagVar.FailureBaseDelay,
 		flagVar.FailureMaxDelay, flagVar.RateLimiterFrequency, flagVar.RateLimiterBurst)
@@ -564,6 +596,7 @@ func setupMandatoryModuleReconciler(mgr ctrl.Manager,
 		RemoteSyncNamespace: flagVar.RemoteSyncNamespace,
 		DescriptorProvider:  descriptorProvider,
 		Metrics:             metrics,
+		OCIRegistryHost:     ociRegistryHost,
 	}).SetupWithManager(mgr, options); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "MandatoryModule")
 		os.Exit(bootstrapFailedExitCode)

config/watcher_local_test/kustomization.yaml

@@ -81,9 +81,6 @@ patches:
       - op: add
         path: /spec/template/spec/containers/0/args/-
         value: --leader-election-retry-period=3s
-      - op: add
-        path: /spec/template/spec/containers/0/args/-
-        value: --oci-registry-host=europe-docker.pkg.dev/kyma-project/kyma-modules
       - op: replace
         path: /spec/template/spec/containers/0/imagePullPolicy
         value: Always

config/watcher_local_test_gcm/kustomization.yaml

@@ -77,9 +77,6 @@ patches:
       - op: add
         path: /spec/template/spec/containers/0/args/-
         value: --leader-election-retry-period=3s
-      - op: add
-        path: /spec/template/spec/containers/0/args/-
-        value: --oci-registry-host=europe-docker.pkg.dev/kyma-project/kyma-modules
       - op: replace
         path: /spec/template/spec/containers/0/imagePullPolicy
         value: Always

internal/controller/kyma/controller.go

@@ -86,6 +86,7 @@ type Reconciler struct {
 	Metrics              *metrics.KymaMetrics
 	RemoteCatalog        *remote.RemoteCatalog
 	TemplateLookup       *templatelookup.TemplateLookup
+	OCIRegistryHost      string
 }
 
 func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
@@ -584,7 +585,7 @@ func (r *Reconciler) updateKyma(ctx context.Context, kyma *v1beta2.Kyma) error {
 
 func (r *Reconciler) reconcileManifests(ctx context.Context, kyma *v1beta2.Kyma) error {
 	templates := r.TemplateLookup.GetRegularTemplates(ctx, kyma)
-	prsr := parser.NewParser(r.Client, r.DescriptorProvider, r.RemoteSyncNamespace)
+	prsr := parser.NewParser(r.Client, r.DescriptorProvider, r.RemoteSyncNamespace, r.OCIRegistryHost)
 	modules := prsr.GenerateModulesFromTemplates(kyma, templates)
 
 	runner := sync.New(r)