diff --git a/agent/Containerfile.operator b/agent/Containerfile.operator index 8111979a5..b37de840a 100644 --- a/agent/Containerfile.operator +++ b/agent/Containerfile.operator @@ -10,7 +10,7 @@ COPY go.mod go.sum ./ COPY ./agent/ ./agent/ COPY ./pkg/ ./pkg/ -RUN CGO_ENABLED=1 GOFLAGS="-p=4" go build -tags strictfipsruntime -a -v -o bin/agent ./agent/cmd/agent/main.go +RUN CGO_ENABLED=1 GOFLAGS="-p=4" go build -tags strictfipsruntime -a -v -o bin/agent ./agent/cmd/main.go # Stage 2: Copy the binaries from the image builder to the base image FROM registry.access.redhat.com/ubi9/ubi-minimal:latest @@ -29,22 +29,22 @@ ARG IMAGE_SUMMARY ARG IMAGE_OPENSHIFT_TAGS LABEL org.label-schema.vendor="Red Hat" \ - org.label-schema.name="$IMAGE_NAME_ARCH" \ - org.label-schema.description="$IMAGE_DESCRIPTION" \ - org.label-schema.vcs-ref=$VCS_REF \ - org.label-schema.vcs-url=$VCS_URL \ - org.label-schema.license="Red Hat Advanced Cluster Management for Kubernetes EULA" \ - org.label-schema.schema-version="1.0" \ - name="$IMAGE_NAME" \ - maintainer="$IMAGE_MAINTAINER" \ - vendor="$IMAGE_VENDOR" \ - version="$IMAGE_VERSION" \ - release="$IMAGE_RELEASE" \ - description="$IMAGE_DESCRIPTION" \ - summary="$IMAGE_SUMMARY" \ - io.k8s.display-name="$IMAGE_DISPLAY_NAME" \ - io.k8s.description="$IMAGE_DESCRIPTION" \ - io.openshift.tags="$IMAGE_OPENSHIFT_TAGS" + org.label-schema.name="$IMAGE_NAME_ARCH" \ + org.label-schema.description="$IMAGE_DESCRIPTION" \ + org.label-schema.vcs-ref=$VCS_REF \ + org.label-schema.vcs-url=$VCS_URL \ + org.label-schema.license="Red Hat Advanced Cluster Management for Kubernetes EULA" \ + org.label-schema.schema-version="1.0" \ + name="$IMAGE_NAME" \ + maintainer="$IMAGE_MAINTAINER" \ + vendor="$IMAGE_VENDOR" \ + version="$IMAGE_VERSION" \ + release="$IMAGE_RELEASE" \ + description="$IMAGE_DESCRIPTION" \ + summary="$IMAGE_SUMMARY" \ + io.k8s.display-name="$IMAGE_DISPLAY_NAME" \ + io.k8s.description="$IMAGE_DESCRIPTION" \ + io.openshift.tags="$IMAGE_OPENSHIFT_TAGS" ENV USER_UID=1001 ENV USER_NAME=agent @@ -53,10 +53,9 @@ ENV USER_NAME=agent COPY --from=builder /workspace/bin/agent /usr/local/bin/agent COPY ./agent/scripts/user_setup /usr/local/scripts/user_setup -RUN /usr/local/scripts/user_setup +RUN /usr/local/scripts/user_setup -RUN microdnf update -y && \ - microdnf clean all +RUN microdnf update -y && microdnf clean all USER ${USER_UID} -ENTRYPOINT ["/usr/local/bin/agent"] \ No newline at end of file +ENTRYPOINT ["/usr/local/bin/agent"] diff --git a/agent/Dockerfile b/agent/Dockerfile index 1e5c6fd32..b4e8d7092 100644 --- a/agent/Dockerfile +++ b/agent/Dockerfile @@ -9,7 +9,7 @@ COPY go.mod go.sum ./ COPY ./agent/ ./agent/ COPY ./pkg/ ./pkg/ -RUN go build -o bin/agent ./agent/cmd/agent/main.go +RUN go build -o bin/agent ./agent/cmd/main.go # Stage 2: Copy the binaries from the image builder to the base image FROM registry.access.redhat.com/ubi9/ubi-minimal:latest @@ -20,10 +20,9 @@ ENV USER_NAME=agent COPY --from=builder /workspace/bin/agent /usr/local/bin/agent COPY ./agent/scripts/user_setup /usr/local/scripts/user_setup -RUN /usr/local/scripts/user_setup +RUN /usr/local/scripts/user_setup -RUN microdnf update -y && \ - microdnf clean all +RUN microdnf update -y && microdnf clean all USER ${USER_UID} ENTRYPOINT ["/usr/local/bin/agent"] diff --git a/agent/Makefile b/agent/Makefile index 67da5d96a..a8f10354b 100644 --- a/agent/Makefile +++ b/agent/Makefile @@ -36,7 +36,7 @@ manifests: controller-gen .PHONY: build ##builds the binary build: - @go build -o bin/${COMPONENT} cmd/agent/main.go + @go build -o bin/${COMPONENT} cmd/main.go .PHONY: clean ##cleans the build directories clean: diff --git a/agent/README.md b/agent/README.md new file mode 100644 index 000000000..3e26c8abb --- /dev/null +++ b/agent/README.md @@ -0,0 +1,26 @@ +# Multicluster Global Hub Agent + +The **Global Hub Agent** component is responsible for applying resources to the hub cluster (spec path) and reporting the resource status to the Global Hub Manager via Kafka. It also synchronizes events with the Inventory API (status path). Additionally, the agent can be run in [standalone mode](./../doc/event-exporter/README.md), which only enables the status path feature. + +## Structure + +- **agent** + - **cmd**: Command-line utilities for the agent. + - **pkg**: Contains the core logic and functionalities. + - **configs**: Holds configurations, schemas, and related assets. + - **controllers**: Common controllers, including initialization, cluster claim, and lease controllers. + - **inventory**: The controllers that report resources via the inventory API. + - **spec**: + - **rbac**: Manages role-based access control. + - **syncers**: Syncs resources and signals from the Global Hub Manager. + - **workers**: Backend goroutines that execute tasks received from the spec syncers. + - **status**: + - **filter**: Deduplicates events when reporting resource statuses. + - **generic**: Common implementations for the status syncer. + - **controller**: Specifies the types of resources to be synced. + - **handler**: Updates the bundle synced to the manager by the watched resources. + - **emitter**: Sends the bundle created/updated by the handler to the transport layer (e.g., via CloudEvents). + - **multi-event syncer**: A template for sending multiple events related to a single object, such as the policy syncer. + - **multi-object syncer**: A template for sending one event related to multiple objects, such as the managedhub info syncer. + - **interfaces**: Defines the behaviors for the Controller, Handler, and Emitter. + - **syncers**: Specifies the resources to be synced, following templates provided by the generic syncers. \ No newline at end of file diff --git a/agent/cmd/agent/main.go b/agent/cmd/main.go similarity index 89% rename from agent/cmd/agent/main.go rename to agent/cmd/main.go index 5a48ee712..3151c108a 100644 --- a/agent/cmd/agent/main.go +++ b/agent/cmd/main.go @@ -30,9 +30,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log/zap" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" - "github.com/stolostron/multicluster-global-hub/agent/pkg/config" + "github.com/stolostron/multicluster-global-hub/agent/pkg/configs" "github.com/stolostron/multicluster-global-hub/agent/pkg/controllers" - statusconfig "github.com/stolostron/multicluster-global-hub/agent/pkg/status/controller/config" "github.com/stolostron/multicluster-global-hub/pkg/constants" "github.com/stolostron/multicluster-global-hub/pkg/jobs" commonobjects "github.com/stolostron/multicluster-global-hub/pkg/objects" @@ -60,7 +59,7 @@ func main() { restConfig.QPS = agentConfig.QPS restConfig.Burst = agentConfig.Burst - c, err := client.New(restConfig, client.Options{Scheme: config.GetRuntimeScheme()}) + c, err := client.New(restConfig, client.Options{Scheme: configs.GetRuntimeScheme()}) if err != nil { setupLog.Error(err, "failed to int controller runtime client") os.Exit(1) @@ -82,7 +81,7 @@ func doTermination(ctx context.Context, c client.Client) int { } // function to handle defers with exit, see https://stackoverflow.com/a/27629493/553720. -func doMain(ctx context.Context, restConfig *rest.Config, agentConfig *config.AgentConfig, c client.Client) int { +func doMain(ctx context.Context, restConfig *rest.Config, agentConfig *configs.AgentConfig, c client.Client) int { if err := completeConfig(ctx, c, agentConfig); err != nil { setupLog.Error(err, "failed to get managed hub configuration from command line flags") return 1 @@ -106,8 +105,8 @@ func doMain(ctx context.Context, restConfig *rest.Config, agentConfig *config.Ag return 0 } -func parseFlags() *config.AgentConfig { - agentConfig := &config.AgentConfig{ +func parseFlags() *configs.AgentConfig { + agentConfig := &configs.AgentConfig{ ElectionConfig: &commonobjects.LeaderElectionConfig{}, TransportConfig: &transport.TransportInternalConfig{ // IsManager specifies the send/receive topics from specTopic and statusTopic @@ -162,7 +161,7 @@ func parseFlags() *config.AgentConfig { return agentConfig } -func completeConfig(ctx context.Context, c client.Client, agentConfig *config.AgentConfig) error { +func completeConfig(ctx context.Context, c client.Client, agentConfig *configs.AgentConfig) error { if !agentConfig.Standalone && agentConfig.LeafHubName == "" { return fmt.Errorf("the leaf-hub-name must not be empty") } @@ -181,7 +180,6 @@ func completeConfig(ctx context.Context, c client.Client, agentConfig *config.Ag } agentConfig.LeafHubName = clusterID } - statusconfig.SetLeafHubName(agentConfig.LeafHubName) if agentConfig.MetricsAddress == "" { agentConfig.MetricsAddress = fmt.Sprintf("%s:%d", metricsHost, metricsPort) @@ -196,11 +194,11 @@ func completeConfig(ctx context.Context, c client.Client, agentConfig *config.Ag return fmt.Errorf("flag consumer-worker-pool-size should be in the scope [1, 100]") } } - config.SetAgentConfig(agentConfig) + configs.SetAgentConfig(agentConfig) return nil } -func createManager(restConfig *rest.Config, agentConfig *config.AgentConfig) ( +func createManager(restConfig *rest.Config, agentConfig *configs.AgentConfig) ( ctrl.Manager, error, ) { leaseDuration := time.Duration(agentConfig.ElectionConfig.LeaseDuration) * time.Second @@ -223,7 +221,7 @@ func createManager(restConfig *rest.Config, agentConfig *config.AgentConfig) ( BindAddress: agentConfig.MetricsAddress, }, LeaderElection: true, - Scheme: config.GetRuntimeScheme(), + Scheme: configs.GetRuntimeScheme(), LeaderElectionConfig: leaderElectionConfig, LeaderElectionID: leaderElectionLockID, LeaderElectionNamespace: agentConfig.PodNamespace, @@ -253,16 +251,10 @@ func createManager(restConfig *rest.Config, agentConfig *config.AgentConfig) ( } // if the transport consumer and producer is ready then the func will be invoked by the transport controller -func transportCallback(mgr ctrl.Manager, agentConfig *config.AgentConfig, +func transportCallback(mgr ctrl.Manager, agentConfig *configs.AgentConfig, ) controller.TransportCallback { return func(transportClient transport.TransportClient) error { - // Need this controller to update the value of clusterclaim hub.open-cluster-management.io - // we use the value to decide whether install the ACM or not - if err := controllers.AddHubClusterClaimController(mgr); err != nil { - return fmt.Errorf("failed to add hub.open-cluster-management.io clusterclaim controller: %w", err) - } - - if err := controllers.AddCRDController(mgr, mgr.GetConfig(), agentConfig, transportClient); err != nil { + if err := controllers.AddInitController(mgr, mgr.GetConfig(), agentConfig, transportClient); err != nil { return fmt.Errorf("failed to add crd controller: %w", err) } @@ -284,9 +276,9 @@ func initCache(restConfig *rest.Config, cacheOpts cache.Options) (cache.Cache, e &clusterv1beta1.PlacementDecision{}: {}, &appsv1alpha1.SubscriptionReport{}: {}, &coordinationv1.Lease{}: { - Field: fields.OneTermEqualSelector("metadata.namespace", config.GetAgentConfig().PodNamespace), + Field: fields.OneTermEqualSelector("metadata.namespace", configs.GetAgentConfig().PodNamespace), }, - &corev1.Event{}: {}, // TODO: need a filter for the target events + &corev1.Event{}: {}, } return cache.New(restConfig, cacheOpts) } diff --git a/agent/cmd/agent/main_test.go b/agent/cmd/main_test.go similarity index 88% rename from agent/cmd/agent/main_test.go rename to agent/cmd/main_test.go index 0c1fc4e64..228a590fd 100644 --- a/agent/cmd/agent/main_test.go +++ b/agent/cmd/main_test.go @@ -11,7 +11,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" - "github.com/stolostron/multicluster-global-hub/agent/pkg/config" + "github.com/stolostron/multicluster-global-hub/agent/pkg/configs" + config "github.com/stolostron/multicluster-global-hub/agent/pkg/configs" "github.com/stolostron/multicluster-global-hub/pkg/transport" ) @@ -38,14 +39,14 @@ func TestParseFlags(t *testing.T) { func TestCompleteConfig(t *testing.T) { testCases := []struct { name string - agentConfig *config.AgentConfig + agentConfig *configs.AgentConfig fakeClient client.Client - expectConfig *config.AgentConfig + expectConfig *configs.AgentConfig expectErrorMsg string }{ { name: "Invalid leaf-hub-name without standalone mode", - agentConfig: &config.AgentConfig{ + agentConfig: &configs.AgentConfig{ LeafHubName: "", Standalone: false, }, @@ -57,7 +58,7 @@ func TestCompleteConfig(t *testing.T) { }, { name: "Empty leaf-hub-name(clusterId) with standalone mode", - agentConfig: &config.AgentConfig{ + agentConfig: &configs.AgentConfig{ LeafHubName: "", Standalone: true, }, @@ -69,7 +70,7 @@ func TestCompleteConfig(t *testing.T) { }, { name: "Invalid leaf-hub-name(clusterId) under standalone mode", - agentConfig: &config.AgentConfig{ + agentConfig: &configs.AgentConfig{ LeafHubName: "", Standalone: true, }, @@ -78,7 +79,7 @@ func TestCompleteConfig(t *testing.T) { }, { name: "Valid configuration under standalone mode", - agentConfig: &config.AgentConfig{ + agentConfig: &configs.AgentConfig{ LeafHubName: "", Standalone: true, TransportConfig: &transport.TransportInternalConfig{ @@ -90,7 +91,7 @@ func TestCompleteConfig(t *testing.T) { ObjectMeta: metav1.ObjectMeta{Name: "version"}, Spec: configv1.ClusterVersionSpec{ClusterID: configv1.ClusterID("123")}, }).Build(), - expectConfig: &config.AgentConfig{ + expectConfig: &configs.AgentConfig{ LeafHubName: "123", Standalone: true, SpecWorkPoolSize: 0, @@ -103,19 +104,19 @@ func TestCompleteConfig(t *testing.T) { }, { name: "Invalid work pool size", - agentConfig: &config.AgentConfig{ + agentConfig: &configs.AgentConfig{ LeafHubName: "hub1", Standalone: false, TransportConfig: &transport.TransportInternalConfig{ TransportType: string(transport.Kafka), }, }, - fakeClient: fake.NewClientBuilder().WithScheme(config.GetRuntimeScheme()).WithObjects().Build(), + fakeClient: fake.NewClientBuilder().WithScheme(configs.GetRuntimeScheme()).WithObjects().Build(), expectErrorMsg: "flag consumer-worker-pool-size should be in the scope [1, 100]", }, { name: "Valid configuration without standalone mode", - agentConfig: &config.AgentConfig{ + agentConfig: &configs.AgentConfig{ LeafHubName: "hub1", Standalone: false, SpecWorkPoolSize: 5, @@ -124,7 +125,7 @@ func TestCompleteConfig(t *testing.T) { }, }, fakeClient: fake.NewClientBuilder().WithScheme(config.GetRuntimeScheme()).WithObjects().Build(), - expectConfig: &config.AgentConfig{ + expectConfig: &configs.AgentConfig{ LeafHubName: "hub1", Standalone: false, SpecWorkPoolSize: 5, @@ -152,7 +153,7 @@ func TestCompleteConfig(t *testing.T) { } func TestDoMain(t *testing.T) { - agentConfig := &config.AgentConfig{ + agentConfig := &configs.AgentConfig{ LeafHubName: "hub1", Standalone: false, SpecWorkPoolSize: 0, diff --git a/agent/pkg/config/agent_config.go b/agent/pkg/configs/agent_config.go similarity index 92% rename from agent/pkg/config/agent_config.go rename to agent/pkg/configs/agent_config.go index be60ebc35..a83ece3c0 100644 --- a/agent/pkg/config/agent_config.go +++ b/agent/pkg/configs/agent_config.go @@ -1,4 +1,4 @@ -package config +package configs import ( "time" @@ -36,6 +36,10 @@ func GetAgentConfig() *AgentConfig { return agentConfigData } +func GetLeafHubName() string { + return agentConfigData.LeafHubName +} + var mchVersion string func GetMCHVersion() string { diff --git a/agent/pkg/config/scheme.go b/agent/pkg/configs/scheme.go similarity index 99% rename from agent/pkg/config/scheme.go rename to agent/pkg/configs/scheme.go index cd9316841..752b3d08c 100644 --- a/agent/pkg/config/scheme.go +++ b/agent/pkg/configs/scheme.go @@ -1,7 +1,7 @@ // Copyright (c) 2023 Red Hat, Inc. // Copyright Contributors to the Open Cluster Management project -package config +package configs import ( configv1 "github.com/openshift/api/config/v1" diff --git a/agent/pkg/controllers/hub_clusterclaim_controller.go b/agent/pkg/controllers/clusterclaim_hub_controller.go similarity index 100% rename from agent/pkg/controllers/hub_clusterclaim_controller.go rename to agent/pkg/controllers/clusterclaim_hub_controller.go diff --git a/agent/pkg/controllers/version_clusterclaim_controller.go b/agent/pkg/controllers/clusterclaim_version_controller.go similarity index 95% rename from agent/pkg/controllers/version_clusterclaim_controller.go rename to agent/pkg/controllers/clusterclaim_version_controller.go index 50ef5abf5..4d2652609 100644 --- a/agent/pkg/controllers/version_clusterclaim_controller.go +++ b/agent/pkg/controllers/clusterclaim_version_controller.go @@ -17,7 +17,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/predicate" - "github.com/stolostron/multicluster-global-hub/agent/pkg/config" + "github.com/stolostron/multicluster-global-hub/agent/pkg/configs" "github.com/stolostron/multicluster-global-hub/pkg/constants" ) @@ -28,7 +28,7 @@ type versionClusterClaimController struct { log logr.Logger } -// TODO: consider to unify the hub and version claim in one controller +// consider to unify the hub and version claim in one controller func (c *versionClusterClaimController) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.Result, error) { reqLogger := c.log.WithValues("Request.Namespace", request.Namespace, "Request.Name", request.Name) reqLogger.V(2).Info("cluster claim controller", "NamespacedName:", request.NamespacedName) @@ -40,7 +40,7 @@ func (c *versionClusterClaimController) Reconcile(ctx context.Context, request c } if mch != nil && mch.Status.CurrentVersion != "" { - config.SetMCHVersion(mch.Status.CurrentVersion) + configs.SetMCHVersion(mch.Status.CurrentVersion) return ctrl.Result{}, updateClusterClaim(ctx, c.client, constants.VersionClusterClaimName, mch.Status.CurrentVersion) } @@ -58,6 +58,7 @@ func AddVersionClusterClaimController(mgr ctrl.Manager) error { constants.GlobalHubOwnerLabelKey: constants.GHAgentOwnerLabelValue, }, }) + err := ctrl.NewControllerManagedBy(mgr).Named("clusterclaim-controller"). For(&clustersv1alpha1.ClusterClaim{}, builder.WithPredicates(clusterClaimPredicate)). Watches(&mchv1.MultiClusterHub{}, &handler.EnqueueRequestForObject{}). diff --git a/agent/pkg/controllers/crd_controller.go b/agent/pkg/controllers/init_controller.go similarity index 65% rename from agent/pkg/controllers/crd_controller.go rename to agent/pkg/controllers/init_controller.go index b432b7e56..bb0d93390 100644 --- a/agent/pkg/controllers/crd_controller.go +++ b/agent/pkg/controllers/init_controller.go @@ -15,10 +15,11 @@ import ( "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/predicate" - "github.com/stolostron/multicluster-global-hub/agent/pkg/config" - specController "github.com/stolostron/multicluster-global-hub/agent/pkg/spec/controller" - statusController "github.com/stolostron/multicluster-global-hub/agent/pkg/status/controller" - "github.com/stolostron/multicluster-global-hub/agent/pkg/status/controller/security" + "github.com/stolostron/multicluster-global-hub/agent/pkg/configs" + "github.com/stolostron/multicluster-global-hub/agent/pkg/controllers/inventory" + agentspec "github.com/stolostron/multicluster-global-hub/agent/pkg/spec" + "github.com/stolostron/multicluster-global-hub/agent/pkg/status" + "github.com/stolostron/multicluster-global-hub/agent/pkg/status/syncers/security" "github.com/stolostron/multicluster-global-hub/pkg/transport" ) @@ -27,17 +28,17 @@ const ( stackRoxCentralCRDName = "centrals.platform.stackrox.io" ) -var crdCtrlStarted = false +var initCtrlStarted = false -type crdController struct { +type initController struct { mgr ctrl.Manager log logr.Logger restConfig *rest.Config - agentConfig *config.AgentConfig + agentConfig *configs.AgentConfig transportClient transport.TransportClient } -func (c *crdController) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.Result, error) { +func (c *initController) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.Result, error) { switch { case request.Name == clusterManagersCRDName: return c.addACMController(ctx, request) @@ -48,12 +49,20 @@ func (c *crdController) Reconcile(ctx context.Context, request ctrl.Request) (ct } } -func (c *crdController) addACMController(ctx context.Context, request ctrl.Request) (ctrl.Result, error) { +func (c *initController) addACMController(ctx context.Context, request ctrl.Request) (ctrl.Result, error) { reqLogger := c.log.WithValues("Request.Namespace", request.Namespace, "Request.Name", request.Name) - reqLogger.V(2).Info("crd controller", "NamespacedName:", request.NamespacedName) - - if err := statusController.AddControllers(ctx, c.mgr, c.transportClient, c.agentConfig); err != nil { - return ctrl.Result{}, fmt.Errorf("failed to add status syncer: %w", err) + reqLogger.V(2).Info("init controller", "NamespacedName:", request.NamespacedName) + + // status syncers or inventory + var err error + switch c.agentConfig.TransportConfig.TransportType { + case string(transport.Kafka): + err = status.AddToManager(ctx, c.mgr, c.transportClient, c.agentConfig) + case string(transport.Rest): + err = inventory.AddToManager(ctx, c.mgr, c.transportClient, c.agentConfig) + } + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed to add the syncer: %w", err) } // only enable the status controller in the standalone mode @@ -61,26 +70,31 @@ func (c *crdController) addACMController(ctx context.Context, request ctrl.Reque return ctrl.Result{}, nil } - // add spec controllers - if err := specController.AddToManager(ctx, c.mgr, c.transportClient.GetConsumer(), c.agentConfig); err != nil { - return ctrl.Result{}, fmt.Errorf("failed to add spec syncer: %w", err) + // Need this controller to update the value of clusterclaim hub.open-cluster-management.io + // we use the value to decide whether install the ACM or not + if err := AddHubClusterClaimController(c.mgr); err != nil { + return ctrl.Result{}, fmt.Errorf("failed to add hub.open-cluster-management.io clusterclaim controller: %w", err) } - reqLogger.V(2).Info("add spec controllers to manager") - // Need this controller to update the value of clusterclaim version.open-cluster-management.io if err := AddVersionClusterClaimController(c.mgr); err != nil { return ctrl.Result{}, fmt.Errorf("failed to add controllers: %w", err) } - if err := config.AddHoHLeaseUpdater(c.mgr, c.agentConfig.PodNamespace, - "multicluster-global-hub-controller"); err != nil { + // add spec controllers + if err := agentspec.AddToManager(ctx, c.mgr, c.transportClient.GetConsumer(), c.agentConfig); err != nil { + return ctrl.Result{}, fmt.Errorf("failed to add spec syncer: %w", err) + } + reqLogger.V(2).Info("add spec controllers to manager") + + // all the controller started, then add the lease controller + if err := AddLeaseController(c.mgr, c.agentConfig.PodNamespace, "multicluster-global-hub-agent"); err != nil { return ctrl.Result{}, fmt.Errorf("failed to add lease updater: %w", err) } return ctrl.Result{}, nil } -func (c *crdController) addStackRoxCentrals() (result ctrl.Result, err error) { +func (c *initController) addStackRoxCentrals() (result ctrl.Result, err error) { c.log.Info("Detected the presence of the StackRox central CRD") // Create the object that polls the StackRox API and publishes the message, then add it to the controller @@ -113,10 +127,10 @@ func (c *crdController) addStackRoxCentrals() (result ctrl.Result, err error) { // this controller is used to watch the multiclusterhub crd or clustermanager crd // if the crd exists, then add controllers to the manager dynamically -func AddCRDController(mgr ctrl.Manager, restConfig *rest.Config, agentConfig *config.AgentConfig, +func AddInitController(mgr ctrl.Manager, restConfig *rest.Config, agentConfig *configs.AgentConfig, transportClient transport.TransportClient, ) error { - if crdCtrlStarted { + if initCtrlStarted { return nil } if err := ctrl.NewControllerManagedBy(mgr). @@ -139,15 +153,15 @@ func AddCRDController(mgr ctrl.Manager, restConfig *rest.Config, agentConfig *co }, }), ). - Complete(&crdController{ + Complete(&initController{ mgr: mgr, restConfig: restConfig, agentConfig: agentConfig, transportClient: transportClient, - log: ctrl.Log.WithName("crd-controller"), + log: ctrl.Log.WithName("init-controller"), }); err != nil { return err } - crdCtrlStarted = true + initCtrlStarted = true return nil } diff --git a/agent/pkg/controllers/crd_controller_test.go b/agent/pkg/controllers/init_controller_test.go similarity index 90% rename from agent/pkg/controllers/crd_controller_test.go rename to agent/pkg/controllers/init_controller_test.go index e78276f4d..f428927df 100644 --- a/agent/pkg/controllers/crd_controller_test.go +++ b/agent/pkg/controllers/init_controller_test.go @@ -10,7 +10,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "github.com/stolostron/multicluster-global-hub/agent/pkg/config" + "github.com/stolostron/multicluster-global-hub/agent/pkg/configs" ) func TestAddCRDController(t *testing.T) { @@ -22,9 +22,9 @@ func TestAddCRDController(t *testing.T) { } mgr, err := manager.New(cfg, manager.Options{}) require.NoError(t, err) - crdCtrl := &crdController{ + crdCtrl := &initController{ mgr: mgr, - agentConfig: &config.AgentConfig{LeafHubName: "hub1"}, + agentConfig: &configs.AgentConfig{LeafHubName: "hub1"}, } _, err = crdCtrl.Reconcile(context.TODO(), reconcile.Request{NamespacedName: types.NamespacedName{ Namespace: "hello", diff --git a/agent/pkg/controllers/inventory/inventory.go b/agent/pkg/controllers/inventory/inventory.go new file mode 100644 index 000000000..fd657d1ed --- /dev/null +++ b/agent/pkg/controllers/inventory/inventory.go @@ -0,0 +1,43 @@ +// Copyright (c) 2024 Red Hat, Inc. +// Copyright Contributors to the Open Cluster Management project + +package inventory + +import ( + "context" + + ctrl "sigs.k8s.io/controller-runtime" + + "github.com/stolostron/multicluster-global-hub/agent/pkg/configs" + "github.com/stolostron/multicluster-global-hub/agent/pkg/controllers/inventory/managedclusterinfo" + "github.com/stolostron/multicluster-global-hub/agent/pkg/controllers/inventory/policy" + "github.com/stolostron/multicluster-global-hub/pkg/transport" + "github.com/stolostron/multicluster-global-hub/pkg/utils" +) + +var inventoryCtrlStarted = false + +// AddToManager adds all the syncers to the Manager. +func AddToManager(ctx context.Context, mgr ctrl.Manager, transportClient transport.TransportClient, + agentConfig *configs.AgentConfig, +) error { + if inventoryCtrlStarted { + return nil + } + + mch, err := utils.ListMCH(ctx, mgr.GetClient()) + if err != nil { + return err + } + configs.SetMCHVersion(mch.Status.CurrentVersion) + + if err := managedclusterinfo.AddManagedClusterInfoInventorySyncer(mgr, transportClient.GetRequester()); err != nil { + return err + } + if err := policy.AddPolicyInventorySyncer(mgr, transportClient.GetRequester()); err != nil { + return err + } + + inventoryCtrlStarted = true + return nil +} diff --git a/agent/pkg/status/controller/managedclusters/managed_cluster_info_syncer.go b/agent/pkg/controllers/inventory/managedclusterinfo/managed_cluster_info_controller.go similarity index 93% rename from agent/pkg/status/controller/managedclusters/managed_cluster_info_syncer.go rename to agent/pkg/controllers/inventory/managedclusterinfo/managed_cluster_info_controller.go index 549ad44be..71c74bfbe 100644 --- a/agent/pkg/status/controller/managedclusters/managed_cluster_info_syncer.go +++ b/agent/pkg/controllers/inventory/managedclusterinfo/managed_cluster_info_controller.go @@ -1,4 +1,4 @@ -package managedclusters +package managedclusterinfo import ( "context" @@ -17,20 +17,19 @@ import ( "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/predicate" - "github.com/stolostron/multicluster-global-hub/agent/pkg/config" - statusconfig "github.com/stolostron/multicluster-global-hub/agent/pkg/status/controller/config" + "github.com/stolostron/multicluster-global-hub/agent/pkg/configs" "github.com/stolostron/multicluster-global-hub/pkg/constants" "github.com/stolostron/multicluster-global-hub/pkg/transport" "github.com/stolostron/multicluster-global-hub/pkg/transport/requester" ) -type ManagedClusterInfoCtrl struct { +type ManagedClusterInfoInventorySyncer struct { runtimeClient client.Client requester transport.Requester clientCN string } -func (r *ManagedClusterInfoCtrl) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { +func (r *ManagedClusterInfoInventorySyncer) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { clusterInfo := &clusterinfov1beta1.ManagedClusterInfo{} err := r.runtimeClient.Get(ctx, types.NamespacedName{Namespace: req.Namespace, Name: req.Name}, clusterInfo) if errors.IsNotFound(err) { @@ -83,7 +82,7 @@ func (r *ManagedClusterInfoCtrl) Reconcile(ctx context.Context, req ctrl.Request return ctrl.Result{}, nil } -func AddManagedClusterInfoCtrl(mgr ctrl.Manager, inventoryRequester transport.Requester) error { +func AddManagedClusterInfoInventorySyncer(mgr ctrl.Manager, inventoryRequester transport.Requester) error { clusterInfoPredicate := predicate.Funcs{ UpdateFunc: func(e event.UpdateEvent) bool { return e.ObjectOld.GetResourceVersion() < e.ObjectNew.GetResourceVersion() @@ -107,10 +106,10 @@ func AddManagedClusterInfoCtrl(mgr ctrl.Manager, inventoryRequester transport.Re return ctrl.NewControllerManagedBy(mgr).Named("inventory-managedclusterinfo-controller"). For(&clusterinfov1beta1.ManagedClusterInfo{}). WithEventFilter(clusterInfoPredicate). - Complete(&ManagedClusterInfoCtrl{ + Complete(&ManagedClusterInfoInventorySyncer{ runtimeClient: mgr.GetClient(), requester: inventoryRequester, - clientCN: requester.GetInventoryClientName(statusconfig.GetLeafHubName()), + clientCN: requester.GetInventoryClientName(configs.GetLeafHubName()), }) } @@ -132,7 +131,7 @@ func GetK8SCluster(clusterInfo *clusterinfov1beta1.ManagedClusterInfo, ReporterData: &kessel.ReporterData{ ReporterType: kessel.ReporterData_ACM, ReporterInstanceId: clientCN, - ReporterVersion: config.GetMCHVersion(), + ReporterVersion: configs.GetMCHVersion(), LocalResourceId: clusterInfo.Name, ApiHref: clusterInfo.Spec.MasterEndpoint, ConsoleHref: clusterInfo.Status.ConsoleURL, diff --git a/agent/pkg/status/controller/managedclusters/k8s_cluster_test.go b/agent/pkg/controllers/inventory/managedclusterinfo/managed_cluster_info_controller_test.go similarity index 57% rename from agent/pkg/status/controller/managedclusters/k8s_cluster_test.go rename to agent/pkg/controllers/inventory/managedclusterinfo/managed_cluster_info_controller_test.go index 06c28aa91..3f0a5204c 100644 --- a/agent/pkg/status/controller/managedclusters/k8s_cluster_test.go +++ b/agent/pkg/controllers/inventory/managedclusterinfo/managed_cluster_info_controller_test.go @@ -1,16 +1,144 @@ -package managedclusters +package managedclusterinfo import ( + "context" "testing" + http "github.com/go-kratos/kratos/v2/transport/http" kessel "github.com/project-kessel/inventory-api/api/kessel/inventory/v1beta1/resources" + "github.com/project-kessel/inventory-client-go/v1beta1" clusterinfov1beta1 "github.com/stolostron/cluster-lifecycle-api/clusterinfo/v1beta1" "github.com/stretchr/testify/assert" + "helm.sh/helm/v3/pkg/time" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" clusterv1 "open-cluster-management.io/api/cluster/v1" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/stolostron/multicluster-global-hub/pkg/constants" + "github.com/stolostron/multicluster-global-hub/pkg/transport" ) +func TestManagedClusterInfoCtrlReconcile(t *testing.T) { + // Setup scheme and mock requester + scheme := runtime.NewScheme() + _ = clusterinfov1beta1.AddToScheme(scheme) + mockRequester := &MockRequest{} + + // Define test cases + creatingTime := metav1.Now() + deletintTime := metav1.NewTime(time.Now().Time) + tests := []struct { + name string + clusterInfo *clusterinfov1beta1.ManagedClusterInfo + expectedResult reconcile.Result + expectedError bool + }{ + { + name: "Creating new cluster", + clusterInfo: &clusterinfov1beta1.ManagedClusterInfo{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + Namespace: "default", + Annotations: map[string]string{ + constants.InventoryResourceCreatingAnnotationlKey: "", + }, + }, + }, + expectedResult: reconcile.Result{}, + expectedError: false, + }, + { + name: "Updating existing cluster", + clusterInfo: &clusterinfov1beta1.ManagedClusterInfo{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + Namespace: "default", + CreationTimestamp: creatingTime, + }, + }, + expectedResult: reconcile.Result{}, + expectedError: false, + }, + { + name: "Deleting cluster", + clusterInfo: &clusterinfov1beta1.ManagedClusterInfo{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + Namespace: "default", + CreationTimestamp: creatingTime, + DeletionTimestamp: &deletintTime, + Finalizers: []string{constants.InventoryResourceFinalizer}, + }, + }, + expectedResult: reconcile.Result{}, + expectedError: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Setup the fake Kubernetes client with test objects + fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(tt.clusterInfo).Build() + + // Create the controller with the mock requester and fake client + r := &ManagedClusterInfoInventorySyncer{ + runtimeClient: fakeClient, + requester: mockRequester, + clientCN: "test-clientCN", + } + + // Call the Reconcile method + result, err := r.Reconcile(context.TODO(), reconcile.Request{ + NamespacedName: types.NamespacedName{Name: "test-cluster", Namespace: "default"}, + }) + + // Assert results + if tt.expectedError { + assert.Error(t, err) + } else { + assert.NoError(t, err) + } + assert.Equal(t, tt.expectedResult, result) + }) + } +} + +type MockRequest struct{} + +func (c *MockRequest) RefreshClient(ctx context.Context, restfulConn *transport.RestfulConfig) error { + return nil +} + +func (c *MockRequest) GetHttpClient() *v1beta1.InventoryHttpClient { + return &v1beta1.InventoryHttpClient{ + K8sClusterService: &ClusterServiceClient{}, + } +} + +type ClusterServiceClient struct{} + +func (c *ClusterServiceClient) CreateK8SCluster(ctx context.Context, in *kessel.CreateK8SClusterRequest, + opts ...http.CallOption, +) (*kessel.CreateK8SClusterResponse, error) { + return nil, nil +} + +func (c *ClusterServiceClient) UpdateK8SCluster(ctx context.Context, in *kessel.UpdateK8SClusterRequest, + opts ...http.CallOption, +) (*kessel.UpdateK8SClusterResponse, error) { + return nil, nil +} + +func (c *ClusterServiceClient) DeleteK8SCluster(ctx context.Context, in *kessel.DeleteK8SClusterRequest, + opts ...http.CallOption, +) (*kessel.DeleteK8SClusterResponse, error) { + return nil, nil +} + func TestGetK8SClusterInfo(t *testing.T) { clusterInfo := createMockClusterInfo("test-cluster", clusterinfov1beta1.KubeVendorOpenShift, "4.10.0", clusterinfov1beta1.CloudVendorAWS) diff --git a/agent/pkg/status/controller/policies/policy_controller.go b/agent/pkg/controllers/inventory/policy/policy_controller.go similarity index 93% rename from agent/pkg/status/controller/policies/policy_controller.go rename to agent/pkg/controllers/inventory/policy/policy_controller.go index a1034311d..177c88603 100644 --- a/agent/pkg/status/controller/policies/policy_controller.go +++ b/agent/pkg/controllers/inventory/policy/policy_controller.go @@ -2,7 +2,7 @@ // Copyright Contributors to the Open Cluster Management project // Policy Controller is used to send the policy create/update/delete to restful API -package policies +package policy import ( "context" @@ -19,20 +19,19 @@ import ( "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/predicate" - "github.com/stolostron/multicluster-global-hub/agent/pkg/config" - statusconfig "github.com/stolostron/multicluster-global-hub/agent/pkg/status/controller/config" + "github.com/stolostron/multicluster-global-hub/agent/pkg/configs" "github.com/stolostron/multicluster-global-hub/pkg/constants" "github.com/stolostron/multicluster-global-hub/pkg/transport" "github.com/stolostron/multicluster-global-hub/pkg/transport/requester" ) -type PolicyController struct { +type PolicyInventorySyncer struct { runtimeClient client.Client reporterInstanceId string requester transport.Requester } -func AddPolicyController(mgr ctrl.Manager, inventoryRequester transport.Requester) error { +func AddPolicyInventorySyncer(mgr ctrl.Manager, inventoryRequester transport.Requester) error { policyPredicate := predicate.Funcs{ UpdateFunc: func(e event.UpdateEvent) bool { // do not trigger the delete event for the replicated policies @@ -68,14 +67,14 @@ func AddPolicyController(mgr ctrl.Manager, inventoryRequester transport.Requeste return ctrl.NewControllerManagedBy(mgr).Named("inventory-policy-controller"). For(&policiesv1.Policy{}). WithEventFilter(policyPredicate). - Complete(&PolicyController{ + Complete(&PolicyInventorySyncer{ runtimeClient: mgr.GetClient(), - reporterInstanceId: requester.GetInventoryClientName(statusconfig.GetLeafHubName()), + reporterInstanceId: requester.GetInventoryClientName(configs.GetLeafHubName()), requester: inventoryRequester, }) } -func (p *PolicyController) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { +func (p *PolicyInventorySyncer) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { policy := &policiesv1.Policy{} err := p.runtimeClient.Get(ctx, types.NamespacedName{Namespace: req.Namespace, Name: req.Name}, policy) if err != nil { @@ -162,7 +161,7 @@ func updateK8SPolicyIsPropagatedToK8SCluster(subjectId, objectId, status, report ReporterData: &kesselv1betarelations.ReporterData{ ReporterType: kesselv1betarelations.ReporterData_ACM, ReporterInstanceId: reporterInstanceId, - ReporterVersion: config.GetMCHVersion(), + ReporterVersion: configs.GetMCHVersion(), SubjectLocalResourceId: subjectId, ObjectLocalResourceId: objectId, }, @@ -179,7 +178,7 @@ func deleteK8SPolicyIsPropagatedToK8SCluster(subjectId, objectId, reporterInstan ReporterData: &kesselv1betarelations.ReporterData{ ReporterType: kesselv1betarelations.ReporterData_ACM, ReporterInstanceId: reporterInstanceId, - ReporterVersion: config.GetMCHVersion(), + ReporterVersion: configs.GetMCHVersion(), SubjectLocalResourceId: subjectId, ObjectLocalResourceId: objectId, }, @@ -203,7 +202,7 @@ func createK8SClusterPolicy(policy policiesv1.Policy, reporterInstanceId string) ReporterData: &kessel.ReporterData{ ReporterType: kessel.ReporterData_ACM, ReporterInstanceId: reporterInstanceId, - ReporterVersion: config.GetMCHVersion(), + ReporterVersion: configs.GetMCHVersion(), LocalResourceId: policy.Namespace + "/" + policy.Name, }, ResourceData: &kessel.K8SPolicyDetail{ @@ -231,7 +230,7 @@ func updateK8SClusterPolicy(policy policiesv1.Policy, reporterInstanceId string) ReporterData: &kessel.ReporterData{ ReporterType: kessel.ReporterData_ACM, ReporterInstanceId: reporterInstanceId, - ReporterVersion: config.GetMCHVersion(), + ReporterVersion: configs.GetMCHVersion(), LocalResourceId: policy.Namespace + "/" + policy.Name, }, ResourceData: &kessel.K8SPolicyDetail{ @@ -247,7 +246,7 @@ func deleteK8SClusterPolicy(policy policiesv1.Policy, reporterInstanceId string) ReporterData: &kessel.ReporterData{ ReporterType: kessel.ReporterData_ACM, ReporterInstanceId: reporterInstanceId, - ReporterVersion: config.GetMCHVersion(), + ReporterVersion: configs.GetMCHVersion(), LocalResourceId: policy.Namespace + "/" + policy.Name, }, } diff --git a/agent/pkg/status/controller/policies/policy_controller_test.go b/agent/pkg/controllers/inventory/policy/policy_controller_test.go similarity index 96% rename from agent/pkg/status/controller/policies/policy_controller_test.go rename to agent/pkg/controllers/inventory/policy/policy_controller_test.go index 5e5cf7693..a362e8010 100644 --- a/agent/pkg/status/controller/policies/policy_controller_test.go +++ b/agent/pkg/controllers/inventory/policy/policy_controller_test.go @@ -1,4 +1,4 @@ -package policies +package policy import ( "context" @@ -20,6 +20,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/reconcile" + "github.com/stolostron/multicluster-global-hub/agent/pkg/configs" "github.com/stolostron/multicluster-global-hub/pkg/constants" "github.com/stolostron/multicluster-global-hub/pkg/transport" ) @@ -29,6 +30,7 @@ func TestPolicyControllerReconcile(t *testing.T) { scheme := runtime.NewScheme() _ = policiesv1.AddToScheme(scheme) mockRequester := &MockRequest{} + configs.SetAgentConfig(&configs.AgentConfig{LeafHubName: "test"}) // Define test cases deletintTime := metav1.NewTime(time.Now().Time) @@ -116,7 +118,7 @@ func TestPolicyControllerReconcile(t *testing.T) { Scheme: scheme, }) assert.NoError(t, err) - assert.NoError(t, AddPolicyController(mgr, nil)) + assert.NoError(t, AddPolicyInventorySyncer(mgr, nil)) for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -124,7 +126,7 @@ func TestPolicyControllerReconcile(t *testing.T) { fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(tt.policy).Build() // Create the controller with the mock requester and fake client - r := &PolicyController{ + r := &PolicyInventorySyncer{ runtimeClient: fakeClient, requester: mockRequester, reporterInstanceId: "test-clientCN", diff --git a/agent/pkg/config/lease.go b/agent/pkg/controllers/lease_controller.go similarity index 96% rename from agent/pkg/config/lease.go rename to agent/pkg/controllers/lease_controller.go index a3969ead5..757453261 100644 --- a/agent/pkg/config/lease.go +++ b/agent/pkg/controllers/lease_controller.go @@ -1,4 +1,4 @@ -package config +package controllers import ( "context" @@ -36,8 +36,8 @@ type leaseUpdater struct { healthCheckFuncs []func() bool } -// AddHoHLeaseUpdater creates a new LeaseUpdater instance aand add it to given manager -func AddHoHLeaseUpdater(mgr ctrl.Manager, addonNamespace, addonName string) error { +// AddLeaseController creates a new LeaseUpdater instance aand add it to given manager +func AddLeaseController(mgr ctrl.Manager, addonNamespace, addonName string) error { if leaseCtrlStarted { return nil } diff --git a/agent/pkg/spec/controller/syncers/dispatcher.go b/agent/pkg/spec/dispatcher.go similarity index 86% rename from agent/pkg/spec/controller/syncers/dispatcher.go rename to agent/pkg/spec/dispatcher.go index 2397e19ba..7ed5c9777 100644 --- a/agent/pkg/spec/controller/syncers/dispatcher.go +++ b/agent/pkg/spec/dispatcher.go @@ -1,4 +1,4 @@ -package syncers +package spec import ( "context" @@ -6,25 +6,37 @@ import ( "github.com/go-logr/logr" ctrl "sigs.k8s.io/controller-runtime" - "github.com/stolostron/multicluster-global-hub/agent/pkg/config" + "github.com/stolostron/multicluster-global-hub/agent/pkg/configs" "github.com/stolostron/multicluster-global-hub/pkg/constants" "github.com/stolostron/multicluster-global-hub/pkg/transport" ) +var addToMgr = false + type genericDispatcher struct { log logr.Logger consumer transport.Consumer - agentConfig config.AgentConfig + agentConfig configs.AgentConfig syncers map[string]Syncer } -func NewGenericDispatcher(consumer transport.Consumer, config config.AgentConfig) *genericDispatcher { - return &genericDispatcher{ +func AddGenericDispatcher(mgr ctrl.Manager, consumer transport.Consumer, config configs.AgentConfig, +) (*genericDispatcher, error) { + if addToMgr { + return nil, nil + } + dispatcher := &genericDispatcher{ log: ctrl.Log.WithName("spec-bundle-dispatcher"), consumer: consumer, agentConfig: config, syncers: make(map[string]Syncer), } + if err := mgr.Add(dispatcher); err != nil { + return nil, err + } + + addToMgr = true + return dispatcher, nil } func (d *genericDispatcher) RegisterSyncer(messageID string, syncer Syncer) { diff --git a/agent/pkg/spec/controller/syncers/interface.go b/agent/pkg/spec/interface.go similarity index 93% rename from agent/pkg/spec/controller/syncers/interface.go rename to agent/pkg/spec/interface.go index 84cf3d335..8d3ea9b3a 100644 --- a/agent/pkg/spec/controller/syncers/interface.go +++ b/agent/pkg/spec/interface.go @@ -1,4 +1,4 @@ -package syncers +package spec import ( "context" diff --git a/agent/pkg/spec/controller/rbac/impersonation_manager.go b/agent/pkg/spec/rbac/impersonation_manager.go similarity index 100% rename from agent/pkg/spec/controller/rbac/impersonation_manager.go rename to agent/pkg/spec/rbac/impersonation_manager.go diff --git a/agent/pkg/spec/controller/controller.go b/agent/pkg/spec/spec.go similarity index 69% rename from agent/pkg/spec/controller/controller.go rename to agent/pkg/spec/spec.go index a5493453a..58718c6f0 100644 --- a/agent/pkg/spec/controller/controller.go +++ b/agent/pkg/spec/spec.go @@ -1,4 +1,4 @@ -package controller +package spec import ( "context" @@ -7,34 +7,30 @@ import ( "k8s.io/klog/v2" ctrl "sigs.k8s.io/controller-runtime" - "github.com/stolostron/multicluster-global-hub/agent/pkg/config" - "github.com/stolostron/multicluster-global-hub/agent/pkg/spec/controller/syncers" - "github.com/stolostron/multicluster-global-hub/agent/pkg/spec/controller/workers" + "github.com/stolostron/multicluster-global-hub/agent/pkg/configs" + "github.com/stolostron/multicluster-global-hub/agent/pkg/spec/syncers" + "github.com/stolostron/multicluster-global-hub/agent/pkg/spec/workers" "github.com/stolostron/multicluster-global-hub/pkg/constants" "github.com/stolostron/multicluster-global-hub/pkg/transport" ) -var specCtrlStarted = false - -func AddToManager(context context.Context, mgr ctrl.Manager, consumer transport.Consumer, agentConfig *config.AgentConfig) error { - if specCtrlStarted { - return nil - } - +func AddToManager(context context.Context, mgr ctrl.Manager, consumer transport.Consumer, + agentConfig *configs.AgentConfig, +) error { if consumer == nil { klog.Info("the consumer is not initialized for the spec controller") return nil } // add worker pool to manager - workers := workers.NewWorkerPool(agentConfig.SpecWorkPoolSize, mgr.GetConfig()) - if err := mgr.Add(workers); err != nil { + workers, err := workers.AddWorkerPoolToMgr(mgr, agentConfig.SpecWorkPoolSize, mgr.GetConfig()) + if err != nil { return fmt.Errorf("failed to add k8s workers pool to runtime manager: %w", err) } // add bundle dispatcher to manager - dispatcher := syncers.NewGenericDispatcher(consumer, *agentConfig) - if err := mgr.Add(dispatcher); err != nil { + dispatcher, err := AddGenericDispatcher(mgr, consumer, *agentConfig) + if err != nil { return fmt.Errorf("failed to add bundle dispatcher to runtime manager: %w", err) } @@ -52,6 +48,5 @@ func AddToManager(context context.Context, mgr ctrl.Manager, consumer transport. syncers.NewManagedClusterMigrationToSyncer(mgr.GetClient())) dispatcher.RegisterSyncer(constants.ResyncMsgKey, syncers.NewResyncSyncer()) - specCtrlStarted = true return nil } diff --git a/agent/pkg/spec/controller/syncers/clusterlabel_syncer.go b/agent/pkg/spec/syncers/clusterlabel_syncer.go similarity index 98% rename from agent/pkg/spec/controller/syncers/clusterlabel_syncer.go rename to agent/pkg/spec/syncers/clusterlabel_syncer.go index 85049e939..4453255b0 100644 --- a/agent/pkg/spec/controller/syncers/clusterlabel_syncer.go +++ b/agent/pkg/spec/syncers/clusterlabel_syncer.go @@ -15,7 +15,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - "github.com/stolostron/multicluster-global-hub/agent/pkg/spec/controller/workers" + "github.com/stolostron/multicluster-global-hub/agent/pkg/spec/workers" specbundle "github.com/stolostron/multicluster-global-hub/pkg/bundle/spec" "github.com/stolostron/multicluster-global-hub/pkg/utils" ) diff --git a/agent/pkg/spec/controller/syncers/generic_syncer.go b/agent/pkg/spec/syncers/generic_syncer.go similarity index 96% rename from agent/pkg/spec/controller/syncers/generic_syncer.go rename to agent/pkg/spec/syncers/generic_syncer.go index d99a37490..912bfb2af 100644 --- a/agent/pkg/spec/controller/syncers/generic_syncer.go +++ b/agent/pkg/spec/syncers/generic_syncer.go @@ -10,9 +10,9 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - "github.com/stolostron/multicluster-global-hub/agent/pkg/config" - "github.com/stolostron/multicluster-global-hub/agent/pkg/spec/controller/rbac" - "github.com/stolostron/multicluster-global-hub/agent/pkg/spec/controller/workers" + "github.com/stolostron/multicluster-global-hub/agent/pkg/configs" + "github.com/stolostron/multicluster-global-hub/agent/pkg/spec/rbac" + "github.com/stolostron/multicluster-global-hub/agent/pkg/spec/workers" "github.com/stolostron/multicluster-global-hub/pkg/bundle/spec" "github.com/stolostron/multicluster-global-hub/pkg/utils" ) @@ -25,7 +25,7 @@ type genericBundleSyncer struct { enforceHohRbac bool } -func NewGenericSyncer(workerPool *workers.WorkerPool, config *config.AgentConfig) *genericBundleSyncer { +func NewGenericSyncer(workerPool *workers.WorkerPool, config *configs.AgentConfig) *genericBundleSyncer { return &genericBundleSyncer{ log: ctrl.Log.WithName("generic-bundle-syncer"), workerPool: workerPool, diff --git a/agent/pkg/spec/controller/syncers/migration_from_syncer.go b/agent/pkg/spec/syncers/migration_from_syncer.go similarity index 100% rename from agent/pkg/spec/controller/syncers/migration_from_syncer.go rename to agent/pkg/spec/syncers/migration_from_syncer.go diff --git a/agent/pkg/spec/controller/syncers/migration_from_syncer_test.go b/agent/pkg/spec/syncers/migration_from_syncer_test.go similarity index 100% rename from agent/pkg/spec/controller/syncers/migration_from_syncer_test.go rename to agent/pkg/spec/syncers/migration_from_syncer_test.go diff --git a/agent/pkg/spec/controller/syncers/migration_to_syncer.go b/agent/pkg/spec/syncers/migration_to_syncer.go similarity index 100% rename from agent/pkg/spec/controller/syncers/migration_to_syncer.go rename to agent/pkg/spec/syncers/migration_to_syncer.go diff --git a/agent/pkg/spec/controller/syncers/migration_to_syncer_test.go b/agent/pkg/spec/syncers/migration_to_syncer_test.go similarity index 100% rename from agent/pkg/spec/controller/syncers/migration_to_syncer_test.go rename to agent/pkg/spec/syncers/migration_to_syncer_test.go diff --git a/agent/pkg/spec/controller/syncers/resync_syncer.go b/agent/pkg/spec/syncers/resync_syncer.go similarity index 100% rename from agent/pkg/spec/controller/syncers/resync_syncer.go rename to agent/pkg/spec/syncers/resync_syncer.go diff --git a/agent/pkg/spec/controller/workers/job.go b/agent/pkg/spec/workers/job.go similarity index 100% rename from agent/pkg/spec/controller/workers/job.go rename to agent/pkg/spec/workers/job.go diff --git a/agent/pkg/spec/controller/workers/worker.go b/agent/pkg/spec/workers/worker.go similarity index 97% rename from agent/pkg/spec/controller/workers/worker.go rename to agent/pkg/spec/workers/worker.go index 628f99237..54e366286 100644 --- a/agent/pkg/spec/controller/workers/worker.go +++ b/agent/pkg/spec/workers/worker.go @@ -8,7 +8,7 @@ import ( "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" - "github.com/stolostron/multicluster-global-hub/agent/pkg/config" + "github.com/stolostron/multicluster-global-hub/agent/pkg/configs" ) // worker within the WorkerPool. runs as a goroutine and invokes Jobs. @@ -20,7 +20,7 @@ type Worker struct { } func newWorker(log logr.Logger, id int, kubeConfig *rest.Config, jobsQueue chan *Job) (*Worker, error) { - client, err := client.New(kubeConfig, client.Options{Scheme: config.GetRuntimeScheme()}) + client, err := client.New(kubeConfig, client.Options{Scheme: configs.GetRuntimeScheme()}) if err != nil { return nil, fmt.Errorf("failed to initialize worker - %w", err) } diff --git a/agent/pkg/spec/controller/workers/worker_pool.go b/agent/pkg/spec/workers/worker_pool.go similarity index 92% rename from agent/pkg/spec/controller/workers/worker_pool.go rename to agent/pkg/spec/workers/worker_pool.go index 37332ed84..8789b9f92 100644 --- a/agent/pkg/spec/controller/workers/worker_pool.go +++ b/agent/pkg/spec/workers/worker_pool.go @@ -9,9 +9,11 @@ import ( "k8s.io/client-go/rest" ctrl "sigs.k8s.io/controller-runtime" - "github.com/stolostron/multicluster-global-hub/agent/pkg/spec/controller/rbac" + "github.com/stolostron/multicluster-global-hub/agent/pkg/spec/rbac" ) +var addToMgr = false + // Workpool pool that creates all k8s workers and the assigns k8s jobs to available workers. type WorkerPool struct { ctx context.Context @@ -26,7 +28,11 @@ type WorkerPool struct { } // AddK8sWorkerPool adds k8s workers pool to the manager and returns it. -func NewWorkerPool(size int, config *rest.Config) *WorkerPool { +func AddWorkerPoolToMgr(mgr ctrl.Manager, size int, config *rest.Config) (*WorkerPool, error) { + if addToMgr { + return nil, nil + } + // for impersonation workers we have additional workers, one per impersonated user. workerPool := &WorkerPool{ log: ctrl.Log.WithName("workers-pool"), @@ -38,9 +44,13 @@ func NewWorkerPool(size int, config *rest.Config) *WorkerPool { impersonationWorkersQueues: make(map[string]chan *Job), impersonationWorkersLock: sync.Mutex{}, } - workerPool.initializationWaitingGroup.Add(1) - return workerPool + + if err := mgr.Add(workerPool); err != nil { + return nil, err + } + addToMgr = true + return workerPool, nil } // Start function starts the k8s workers pool. diff --git a/agent/pkg/status/controller/event/event_syncer.go b/agent/pkg/status/controller/event/event_syncer.go deleted file mode 100644 index fbb6a95df..000000000 --- a/agent/pkg/status/controller/event/event_syncer.go +++ /dev/null @@ -1,57 +0,0 @@ -package event - -import ( - "context" - "regexp" - - corev1 "k8s.io/api/core/v1" - policiesv1 "open-cluster-management.io/governance-policy-propagator/api/v1" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/predicate" - - "github.com/stolostron/multicluster-global-hub/agent/pkg/config" - statusconfig "github.com/stolostron/multicluster-global-hub/agent/pkg/status/controller/config" - "github.com/stolostron/multicluster-global-hub/agent/pkg/status/controller/generic" - "github.com/stolostron/multicluster-global-hub/pkg/constants" - "github.com/stolostron/multicluster-global-hub/pkg/transport" -) - -const ( - UnknownComplianceState = "Unknown" -) - -var PolicyMessageStatusRe = regexp. - MustCompile(`Policy (.+) status was updated to (.+) in cluster namespace (.+)`) - -var eventPredicateFunc = predicate.NewPredicateFuncs(func(obj client.Object) bool { - event, ok := obj.(*corev1.Event) - if !ok { - return false - } - // only sync the policy event || extend other InvolvedObject kind - return event.InvolvedObject.Kind == policiesv1.Kind || - event.InvolvedObject.Kind == constants.ManagedClusterKind -}) - -func LaunchEventSyncer(ctx context.Context, mgr ctrl.Manager, - agentConfig *config.AgentConfig, producer transport.Producer, -) error { - eventTopic := agentConfig.TransportConfig.KafkaCredential.StatusTopic - - instance := func() client.Object { - return &corev1.Event{} - } - - return generic.LaunchGenericObjectSyncer( - "status.event", - mgr, - generic.NewGenericController(instance, eventPredicateFunc), - producer, - statusconfig.GetEventDuration, - []generic.ObjectEmitter{ - NewLocalRootPolicyEmitter(ctx, mgr.GetClient(), eventTopic), - // NewLocalReplicatedPolicyEmitter(ctx, mgr.GetClient(), eventTopic), - NewManagedClusterEventEmitter(ctx, mgr.GetClient(), eventTopic), - }) -} diff --git a/agent/pkg/status/controller/event/event_test.go b/agent/pkg/status/controller/event/event_test.go deleted file mode 100644 index 59675135b..000000000 --- a/agent/pkg/status/controller/event/event_test.go +++ /dev/null @@ -1,52 +0,0 @@ -package event - -import ( - "context" - "fmt" - "testing" - - lru "github.com/hashicorp/golang-lru" - "github.com/stretchr/testify/assert" -) - -func TestVersion(t *testing.T) { - emitter := NewLocalRootPolicyEmitter(context.TODO(), nil, "event") - assert.Equal(t, "0.0", emitter.currentVersion.String()) - assert.Equal(t, "0.0", emitter.lastSentVersion.String()) - assert.False(t, emitter.ShouldSend()) - - emitter.currentVersion.Incr() - assert.Equal(t, "0.1", emitter.currentVersion.String()) - assert.Equal(t, "0.0", emitter.lastSentVersion.String()) - assert.True(t, emitter.ShouldSend()) - - emitter.PostSend() - assert.Equal(t, "1.1", emitter.currentVersion.String()) - assert.Equal(t, "1.1", emitter.lastSentVersion.String()) - assert.False(t, emitter.ShouldSend()) - - emitter.currentVersion.Incr() - assert.Equal(t, "1.2", emitter.currentVersion.String()) - assert.Equal(t, "1.1", emitter.lastSentVersion.String()) - assert.True(t, emitter.ShouldSend()) -} - -func TestLRU(t *testing.T) { - l, _ := lru.New(5) - for i := 0; i < 10; i++ { - l.Add(fmt.Sprint(i), nil) - } - assert.Equal(t, 5, l.Len()) - - ok := l.Contains("1") - assert.False(t, ok) - ok = l.Contains("4") - assert.False(t, ok) - - ok = l.Contains("5") - assert.True(t, ok) - ok = l.Contains("6") - assert.True(t, ok) - ok = l.Contains("9") - assert.True(t, ok) -} diff --git a/agent/pkg/status/controller/event/managedcluster_emitter.go b/agent/pkg/status/controller/event/managedcluster_emitter.go deleted file mode 100644 index 4b2594f31..000000000 --- a/agent/pkg/status/controller/event/managedcluster_emitter.go +++ /dev/null @@ -1,163 +0,0 @@ -package event - -import ( - "context" - "fmt" - "strings" - - cloudevents "github.com/cloudevents/sdk-go/v2" - "github.com/go-logr/logr" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - clusterv1 "open-cluster-management.io/api/cluster/v1" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - - "github.com/stolostron/multicluster-global-hub/agent/pkg/status/controller/config" - "github.com/stolostron/multicluster-global-hub/agent/pkg/status/controller/filter" - "github.com/stolostron/multicluster-global-hub/agent/pkg/status/controller/generic" - "github.com/stolostron/multicluster-global-hub/pkg/bundle/event" - "github.com/stolostron/multicluster-global-hub/pkg/bundle/version" - "github.com/stolostron/multicluster-global-hub/pkg/constants" - "github.com/stolostron/multicluster-global-hub/pkg/database/models" - "github.com/stolostron/multicluster-global-hub/pkg/enum" - "github.com/stolostron/multicluster-global-hub/pkg/utils" -) - -var _ generic.ObjectEmitter = &managedClusterEmitter{} - -type managedClusterEmitter struct { - ctx context.Context - name string - log logr.Logger - runtimeClient client.Client - eventType string - topic string - currentVersion *version.Version - lastSentVersion version.Version - payload event.ManagedClusterEventBundle -} - -func NewManagedClusterEventEmitter(ctx context.Context, c client.Client, topic string) *managedClusterEmitter { - name := strings.Replace(string(enum.ManagedClusterEventType), enum.EventTypePrefix, "", -1) - filter.RegisterTimeFilter(name) - return &managedClusterEmitter{ - ctx: ctx, - name: name, - log: ctrl.Log.WithName(name), - eventType: string(enum.ManagedClusterEventType), - topic: topic, - runtimeClient: c, - currentVersion: version.NewVersion(), - lastSentVersion: *version.NewVersion(), - payload: make([]models.ManagedClusterEvent, 0), - } -} - -func (h *managedClusterEmitter) PostUpdate() { - h.currentVersion.Incr() -} - -func (h *managedClusterEmitter) ShouldUpdate(obj client.Object) bool { - evt, ok := obj.(*corev1.Event) - if !ok { - return false - } - - if evt.InvolvedObject.Kind != constants.ManagedClusterKind { - return false - } - - // if it's a older event, then return false - if !filter.Newer(h.name, getEventLastTime(evt).Time) { - return false - } - - return true -} - -func (h *managedClusterEmitter) Update(obj client.Object) bool { - evt, ok := obj.(*corev1.Event) - if !ok { - return false - } - - cluster, err := getInvolveCluster(h.ctx, h.runtimeClient, evt) - if err != nil { - h.log.Error(err, "failed to get involved cluster", "event", evt.Namespace+"/"+evt.Name, "cluster", cluster.Name) - return false - } - - clusterId, err := utils.GetClusterId(h.ctx, h.runtimeClient, cluster.Name) - if err != nil { - h.log.Error(err, "failed to get involved clusterId", "event", evt.Namespace+"/"+evt.Name) - return false - } - - clusterEvent := models.ManagedClusterEvent{ - EventName: evt.Name, - EventNamespace: evt.Namespace, - Message: evt.Message, - Reason: evt.Reason, - ClusterName: cluster.Name, - ClusterID: clusterId, - LeafHubName: config.GetLeafHubName(), - ReportingController: evt.ReportingController, - ReportingInstance: evt.ReportingInstance, - EventType: evt.Type, - CreatedAt: getEventLastTime(evt).Time, - } - - h.payload = append(h.payload, clusterEvent) - return true -} - -func (*managedClusterEmitter) Delete(client.Object) bool { - // do nothing - return false -} - -func (h *managedClusterEmitter) ToCloudEvent() (*cloudevents.Event, error) { - if len(h.payload) < 1 { - return nil, fmt.Errorf("the cloudevent instance shouldn't be nil") - } - e := cloudevents.NewEvent() - e.SetType(h.eventType) - e.SetSource(config.GetLeafHubName()) - e.SetExtension(version.ExtVersion, h.currentVersion.String()) - err := e.SetData(cloudevents.ApplicationJSON, h.payload) - return &e, err -} - -// to assert whether emit the current cloudevent -func (h *managedClusterEmitter) ShouldSend() bool { - return h.currentVersion.NewerThan(&h.lastSentVersion) -} - -func (h *managedClusterEmitter) Topic() string { - return h.topic -} - -func (h *managedClusterEmitter) PostSend() { - // update the time filter: with latest event - for _, evt := range h.payload { - filter.CacheTime(h.name, evt.CreatedAt) - } - // update version and clean the cache - h.payload = make([]models.ManagedClusterEvent, 0) - // 1. the version get into the next generation - // 2. set the lastSenteVersion to current version - h.currentVersion.Next() - h.lastSentVersion = *h.currentVersion -} - -func getInvolveCluster(ctx context.Context, c client.Client, evt *corev1.Event) (*clusterv1.ManagedCluster, error) { - cluster := &clusterv1.ManagedCluster{ - ObjectMeta: metav1.ObjectMeta{ - Name: evt.InvolvedObject.Name, - Namespace: evt.InvolvedObject.Namespace, - }, - } - err := c.Get(ctx, client.ObjectKeyFromObject(cluster), cluster) - return cluster, err -} diff --git a/agent/pkg/status/controller/generic/generic_event_controller.go b/agent/pkg/status/controller/generic/generic_event_controller.go deleted file mode 100644 index 1e5e191a7..000000000 --- a/agent/pkg/status/controller/generic/generic_event_controller.go +++ /dev/null @@ -1,160 +0,0 @@ -package generic - -import ( - "context" - "fmt" - "strconv" - "strings" - "sync" - "time" - - "github.com/go-logr/logr" - apierrors "k8s.io/apimachinery/pkg/api/errors" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - - "github.com/stolostron/multicluster-global-hub/pkg/constants" - "github.com/stolostron/multicluster-global-hub/pkg/utils" -) - -const REQUEUE_PERIOD = 5 * time.Second - -type genericEventController struct { - log logr.Logger - client client.Client - emitter Emitter - eventController EventController - lock *sync.Mutex -} - -func AddEventController(mgr ctrl.Manager, eventController EventController, emitter Emitter, - lock *sync.Mutex, -) error { - obj := eventController.Instance() - statusSyncCtrl := &genericEventController{ - log: ctrl.Log.WithName(fmt.Sprintf("status.%s", obj.GetObjectKind())), - client: mgr.GetClient(), - emitter: emitter, - eventController: eventController, - lock: lock, - } - - controllerBuilder := ctrl.NewControllerManagedBy(mgr).For(obj) - if eventController.Predicate() != nil { - controllerBuilder = controllerBuilder.WithEventFilter(eventController.Predicate()) - } - return controllerBuilder.Complete(statusSyncCtrl) -} - -func (c *genericEventController) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.Result, error) { - reqLogger := c.log.WithValues("Namespace", request.Namespace, "Name", request.Name) - object := c.eventController.Instance() - - if err := c.client.Get(ctx, request.NamespacedName, object); apierrors.IsNotFound(err) { - // the instance was deleted and it had no finalizer on it. - // for the local resources, there is no finalizer so we need to delete the object from the bundle - object.SetNamespace(request.Namespace) - object.SetName(request.Name) - if e := c.deleteObjectAndFinalizer(ctx, object); e != nil { - return ctrl.Result{Requeue: true, RequeueAfter: REQUEUE_PERIOD}, e - } - return ctrl.Result{}, nil - } else if err != nil { - return ctrl.Result{Requeue: true, RequeueAfter: REQUEUE_PERIOD}, - fmt.Errorf("reconciliation failed: %w", err) - } - - if !object.GetDeletionTimestamp().IsZero() { - if err := c.deleteObjectAndFinalizer(ctx, object); err != nil { - return ctrl.Result{Requeue: true, RequeueAfter: REQUEUE_PERIOD}, err - } - } else { // otherwise, the object was not deleted and no error occurred - if err := c.updateObjectAndFinalizer(ctx, object); err != nil { - return ctrl.Result{Requeue: true, RequeueAfter: REQUEUE_PERIOD}, err - } - } - reqLogger.V(2).Info("Reconciliation complete.") - return ctrl.Result{}, nil -} - -func (c *genericEventController) updateObjectAndFinalizer(ctx context.Context, object client.Object) error { - // only add finalizer for the global resources - if enableCleanUpFinalizer(object) { - err := addFinalizer(ctx, c.client, object, FinalizerName) - if err != nil { - return err - } - } - - c.lock.Lock() // make sure bundles are not updated if we're during bundles sync - defer c.lock.Unlock() - if c.emitter.ShouldUpdate(object) && c.eventController.Update(object) { - c.emitter.PostUpdate() - } - return nil -} - -func (c *genericEventController) deleteObjectAndFinalizer(ctx context.Context, object client.Object) error { - c.lock.Lock() // make sure bundles are not updated if we're during bundles sync - if c.emitter.ShouldUpdate(object) && c.eventController.Delete(object) { - c.emitter.PostUpdate() - } - c.lock.Unlock() - - if enableCleanUpFinalizer(object) { - err := removeFinalizer(ctx, c.client, object, FinalizerName) - if err != nil { - return err - } - } - return nil -} - -func enableCleanUpFinalizer(obj client.Object) bool { - return utils.HasLabel(obj, constants.GlobalHubGlobalResourceLabel) || - utils.HasAnnotation(obj, constants.OriginOwnerReferenceAnnotation) -} - -func cleanObject(object client.Object) { - object.SetManagedFields(nil) - object.SetFinalizers(nil) - object.SetOwnerReferences(nil) - object.SetSelfLink("") - // object.SetClusterName("") -} - -func addFinalizer(ctx context.Context, c client.Client, obj client.Object, finalizer string) error { - // if the removing finalizer label hasn't expired, then skip the adding finalizer action - if val, found := obj.GetLabels()[constants.GlobalHubFinalizerRemovingDeadline]; found { - deadline, err := strconv.ParseInt(val, 10, 64) - if err != nil { - return err - } - if time.Now().Unix() < deadline { - return nil - } else { - delete(obj.GetLabels(), constants.GlobalHubFinalizerRemovingDeadline) - } - } - - if controllerutil.ContainsFinalizer(obj, finalizer) { - return nil - } - - controllerutil.AddFinalizer(obj, finalizer) - - if err := c.Update(ctx, obj); err != nil && !strings.Contains(err.Error(), "the object has been modified") { - return err - } - return nil -} - -func removeFinalizer(ctx context.Context, c client.Client, obj client.Object, finalizer string) error { - if !controllerutil.ContainsFinalizer(obj, finalizer) { - return nil // if finalizer is not there, do nothing. - } - controllerutil.RemoveFinalizer(obj, finalizer) - - return c.Update(ctx, obj) -} diff --git a/agent/pkg/status/controller/generic/generic_event_syncer.go b/agent/pkg/status/controller/generic/generic_event_syncer.go deleted file mode 100644 index 8698c58a0..000000000 --- a/agent/pkg/status/controller/generic/generic_event_syncer.go +++ /dev/null @@ -1,105 +0,0 @@ -package generic - -import ( - "context" - "fmt" - "sync" - "time" - - cecontext "github.com/cloudevents/sdk-go/v2/context" - "github.com/go-logr/logr" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - - "github.com/stolostron/multicluster-global-hub/agent/pkg/status/controller/config" - "github.com/stolostron/multicluster-global-hub/pkg/transport" -) - -type genericEventSyncer struct { - log logr.Logger - runtimeClient client.Client - producer transport.Producer - eventControllers []EventController - emitter Emitter - leafHubName string - syncIntervalFunc func() time.Duration - startOnce sync.Once - - // share properties with multi-object-controller - lock *sync.Mutex -} - -// LaunchGenericObjectSyncer is used to send multi event(by the eventEmitter) by a specific client.Object -func LaunchGenericEventSyncer(name string, mgr ctrl.Manager, eventControllers []EventController, - producer transport.Producer, intervalFunc func() time.Duration, emitter Emitter, -) error { - syncer := &genericEventSyncer{ - log: ctrl.Log.WithName(name), - leafHubName: config.GetLeafHubName(), - runtimeClient: mgr.GetClient(), - - eventControllers: eventControllers, - emitter: emitter, - producer: producer, - syncIntervalFunc: intervalFunc, - lock: &sync.Mutex{}, - } - - // start the periodic syncer - syncer.startOnce.Do(func() { - go syncer.periodicSync() - }) - - // start all the controllers to update the payload - for _, eventController := range eventControllers { - err := AddEventController(mgr, eventController, emitter, syncer.lock) - if err != nil { - return err - } - } - return nil -} - -func (s *genericEventSyncer) periodicSync() { - currentSyncInterval := s.syncIntervalFunc() - s.log.Info(fmt.Sprintf("sync interval has been reset to %s", currentSyncInterval.String())) - ticker := time.NewTicker(currentSyncInterval) - defer ticker.Stop() - - for { - s.syncEvent() - <-ticker.C // wait for next time interval - - resolvedInterval := s.syncIntervalFunc() - - // reset ticker if sync interval has changed - if resolvedInterval != currentSyncInterval { - currentSyncInterval = resolvedInterval - ticker.Reset(currentSyncInterval) - s.log.Info(fmt.Sprintf("sync interval has been reset to %s", currentSyncInterval.String())) - } - } -} - -func (s *genericEventSyncer) syncEvent() { - s.lock.Lock() // make sure bundles are not updated if we're during bundles sync - defer s.lock.Unlock() - - if s.emitter.ShouldSend() { - evt, err := s.emitter.ToCloudEvent() - if err != nil { - s.log.Error(err, "failed to get CloudEvent instance", "evt", evt) - return - } - - ctx := context.TODO() - if s.emitter.Topic() != "" { - ctx = cecontext.WithTopic(ctx, s.emitter.Topic()) - } - if err := s.producer.SendEvent(ctx, *evt); err != nil { - s.log.Error(err, "failed to send event", "evt", evt) - return - } - s.emitter.PostSend() - } -} diff --git a/agent/pkg/status/controller/generic/generic_object_emitter.go b/agent/pkg/status/controller/generic/generic_object_emitter.go deleted file mode 100644 index 8135461f7..000000000 --- a/agent/pkg/status/controller/generic/generic_object_emitter.go +++ /dev/null @@ -1,47 +0,0 @@ -package generic - -import ( - "sigs.k8s.io/controller-runtime/pkg/client" - - genericpayload "github.com/stolostron/multicluster-global-hub/pkg/bundle/generic" - "github.com/stolostron/multicluster-global-hub/pkg/enum" -) - -var _ ObjectEmitter = &genericObjectEmitter{} - -type genericObjectEmitter struct { - *genericEmitter - Handler -} - -func NewGenericObjectEmitter(eventType enum.EventType, eventData interface{}, - handler Handler, opts ...EmitterOption, -) ObjectEmitter { - return &genericObjectEmitter{ - NewGenericEmitter(eventType, eventData, opts...), - handler, - } -} - -func (e *genericObjectEmitter) Update(object client.Object) bool { - return e.Handler.Update(object) -} - -func (e *genericObjectEmitter) Delete(object client.Object) bool { - return e.Handler.Delete(object) -} - -func ObjectEmitterWrapper(eventType enum.EventType, - shouldUpdate func(client.Object) bool, - tweakFunc func(client.Object), - isSpecHandler bool, -) ObjectEmitter { - eventData := genericpayload.GenericObjectBundle{} - return NewGenericObjectEmitter( - eventType, - &eventData, - NewGenericObjectHandler(&eventData, isSpecHandler), - WithShouldUpdate(shouldUpdate), - WithTweakFunc(tweakFunc), - ) -} diff --git a/agent/pkg/status/controller/hubcluster/info_clusterclaim_controller.go b/agent/pkg/status/controller/hubcluster/info_clusterclaim_controller.go deleted file mode 100644 index e543a1d57..000000000 --- a/agent/pkg/status/controller/hubcluster/info_clusterclaim_controller.go +++ /dev/null @@ -1,52 +0,0 @@ -package hubcluster - -import ( - clustersv1alpha1 "open-cluster-management.io/api/cluster/v1alpha1" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/predicate" - - "github.com/stolostron/multicluster-global-hub/agent/pkg/status/controller/generic" - "github.com/stolostron/multicluster-global-hub/pkg/bundle/cluster" -) - -var _ generic.EventController = &infoClusterClaimController{} - -type infoClusterClaimController struct { - generic.Controller - evtData cluster.HubClusterInfoBundle -} - -func NewInfoClusterClaimController(eventData cluster.HubClusterInfoBundle) generic.EventController { - instance := func() client.Object { return &clustersv1alpha1.ClusterClaim{} } - clusterClaimPredicate := predicate.NewPredicateFuncs(func(object client.Object) bool { - return object.GetName() == "id.k8s.io" - }) - return &infoClusterClaimController{ - Controller: generic.NewGenericController(instance, clusterClaimPredicate), - evtData: eventData, - } -} - -func (p *infoClusterClaimController) Update(obj client.Object) bool { - clusterClaim, ok := obj.(*clustersv1alpha1.ClusterClaim) - if !ok { - return false - } - - oldClusterID := p.evtData.ClusterId - - if clusterClaim.Name == "id.k8s.io" { - p.evtData.ClusterId = clusterClaim.Spec.Value - } - // If no ClusterId, do not send the bundle - if p.evtData.ClusterId == "" { - return false - } - - return oldClusterID != p.evtData.ClusterId -} - -func (p *infoClusterClaimController) Delete(obj client.Object) bool { - // do nothing - return false -} diff --git a/agent/pkg/status/controller/hubcluster/info_route_controller.go b/agent/pkg/status/controller/hubcluster/info_route_controller.go deleted file mode 100644 index a9449f9df..000000000 --- a/agent/pkg/status/controller/hubcluster/info_route_controller.go +++ /dev/null @@ -1,73 +0,0 @@ -package hubcluster - -import ( - routev1 "github.com/openshift/api/route/v1" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/predicate" - - "github.com/stolostron/multicluster-global-hub/agent/pkg/status/controller/generic" - "github.com/stolostron/multicluster-global-hub/pkg/bundle/cluster" - "github.com/stolostron/multicluster-global-hub/pkg/constants" -) - -var _ generic.EventController = &infoRouteController{} - -type infoRouteController struct { - generic.Controller - evtData cluster.HubClusterInfoBundle -} - -func NewInfoRouteController(eventData cluster.HubClusterInfoBundle) generic.EventController { - instance := func() client.Object { return &routev1.Route{} } - predicate := predicate.NewPredicateFuncs(func(object client.Object) bool { - if object.GetNamespace() == constants.OpenShiftConsoleNamespace && - object.GetName() == constants.OpenShiftConsoleRouteName { - return true - } - if object.GetNamespace() == constants.ObservabilityNamespace && - object.GetName() == constants.ObservabilityGrafanaRouteName { - return true - } - return false - }) - - return &infoRouteController{ - Controller: generic.NewGenericController(instance, predicate), - evtData: eventData, - } -} - -func (p *infoRouteController) Update(obj client.Object) bool { - route, ok := obj.(*routev1.Route) - if !ok { - return false - } - - var newURL string - updated := false - if len(route.Spec.Host) != 0 { - newURL = "https://" + route.Spec.Host - } - if route.GetName() == constants.OpenShiftConsoleRouteName && p.evtData.ConsoleURL != newURL { - p.evtData.ConsoleURL = newURL - updated = true - } - if route.GetName() == constants.ObservabilityGrafanaRouteName && p.evtData.GrafanaURL != newURL { - p.evtData.GrafanaURL = newURL - updated = true - } - return updated -} - -func (p *infoRouteController) Delete(obj client.Object) bool { - updated := false - if obj.GetName() == constants.OpenShiftConsoleRouteName && p.evtData.ConsoleURL != "" { - p.evtData.ConsoleURL = "" - updated = true - } - if obj.GetName() == constants.ObservabilityGrafanaRouteName && p.evtData.GrafanaURL != "" { - p.evtData.GrafanaURL = "" - updated = true - } - return updated -} diff --git a/agent/pkg/status/controller/hubcluster/info_syncer.go b/agent/pkg/status/controller/hubcluster/info_syncer.go deleted file mode 100644 index e64af6058..000000000 --- a/agent/pkg/status/controller/hubcluster/info_syncer.go +++ /dev/null @@ -1,26 +0,0 @@ -package hubcluster - -import ( - ctrl "sigs.k8s.io/controller-runtime" - - "github.com/stolostron/multicluster-global-hub/agent/pkg/status/controller/config" - "github.com/stolostron/multicluster-global-hub/agent/pkg/status/controller/generic" - "github.com/stolostron/multicluster-global-hub/pkg/bundle/cluster" - "github.com/stolostron/multicluster-global-hub/pkg/enum" - "github.com/stolostron/multicluster-global-hub/pkg/transport" -) - -func LaunchHubClusterInfoSyncer(mgr ctrl.Manager, producer transport.Producer) error { - eventData := &cluster.HubClusterInfo{} - return generic.LaunchGenericEventSyncer( - "status.hub_cluster_info", - mgr, - []generic.EventController{ - NewInfoClusterClaimController(eventData), - NewInfoRouteController(eventData), - }, - producer, - config.GetHubClusterInfoDuration, - generic.NewGenericEmitter(enum.HubClusterInfoType, eventData), - ) -} diff --git a/agent/pkg/status/controller/managedclusters/managed_cluster_info_syncer_test.go b/agent/pkg/status/controller/managedclusters/managed_cluster_info_syncer_test.go deleted file mode 100644 index 196701289..000000000 --- a/agent/pkg/status/controller/managedclusters/managed_cluster_info_syncer_test.go +++ /dev/null @@ -1,138 +0,0 @@ -package managedclusters - -import ( - "context" - "testing" - - http "github.com/go-kratos/kratos/v2/transport/http" - kessel "github.com/project-kessel/inventory-api/api/kessel/inventory/v1beta1/resources" - "github.com/project-kessel/inventory-client-go/v1beta1" - clusterinfov1beta1 "github.com/stolostron/cluster-lifecycle-api/clusterinfo/v1beta1" - "github.com/stretchr/testify/assert" - "helm.sh/helm/v3/pkg/time" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client/fake" - "sigs.k8s.io/controller-runtime/pkg/reconcile" - - "github.com/stolostron/multicluster-global-hub/pkg/constants" - "github.com/stolostron/multicluster-global-hub/pkg/transport" -) - -func TestManagedClusterInfoCtrlReconcile(t *testing.T) { - // Setup scheme and mock requester - scheme := runtime.NewScheme() - _ = clusterinfov1beta1.AddToScheme(scheme) - mockRequester := &MockRequest{} - - // Define test cases - creatingTime := metav1.Now() - deletintTime := metav1.NewTime(time.Now().Time) - tests := []struct { - name string - clusterInfo *clusterinfov1beta1.ManagedClusterInfo - expectedResult reconcile.Result - expectedError bool - }{ - { - name: "Creating new cluster", - clusterInfo: &clusterinfov1beta1.ManagedClusterInfo{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-cluster", - Namespace: "default", - Annotations: map[string]string{ - constants.InventoryResourceCreatingAnnotationlKey: "", - }, - }, - }, - expectedResult: reconcile.Result{}, - expectedError: false, - }, - { - name: "Updating existing cluster", - clusterInfo: &clusterinfov1beta1.ManagedClusterInfo{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-cluster", - Namespace: "default", - CreationTimestamp: creatingTime, - }, - }, - expectedResult: reconcile.Result{}, - expectedError: false, - }, - { - name: "Deleting cluster", - clusterInfo: &clusterinfov1beta1.ManagedClusterInfo{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-cluster", - Namespace: "default", - CreationTimestamp: creatingTime, - DeletionTimestamp: &deletintTime, - Finalizers: []string{constants.InventoryResourceFinalizer}, - }, - }, - expectedResult: reconcile.Result{}, - expectedError: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - // Setup the fake Kubernetes client with test objects - fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(tt.clusterInfo).Build() - - // Create the controller with the mock requester and fake client - r := &ManagedClusterInfoCtrl{ - runtimeClient: fakeClient, - requester: mockRequester, - clientCN: "test-clientCN", - } - - // Call the Reconcile method - result, err := r.Reconcile(context.TODO(), reconcile.Request{ - NamespacedName: types.NamespacedName{Name: "test-cluster", Namespace: "default"}, - }) - - // Assert results - if tt.expectedError { - assert.Error(t, err) - } else { - assert.NoError(t, err) - } - assert.Equal(t, tt.expectedResult, result) - }) - } -} - -type MockRequest struct{} - -func (c *MockRequest) RefreshClient(ctx context.Context, restfulConn *transport.RestfulConfig) error { - return nil -} - -func (c *MockRequest) GetHttpClient() *v1beta1.InventoryHttpClient { - return &v1beta1.InventoryHttpClient{ - K8sClusterService: &ClusterServiceClient{}, - } -} - -type ClusterServiceClient struct{} - -func (c *ClusterServiceClient) CreateK8SCluster(ctx context.Context, in *kessel.CreateK8SClusterRequest, - opts ...http.CallOption, -) (*kessel.CreateK8SClusterResponse, error) { - return nil, nil -} - -func (c *ClusterServiceClient) UpdateK8SCluster(ctx context.Context, in *kessel.UpdateK8SClusterRequest, - opts ...http.CallOption, -) (*kessel.UpdateK8SClusterResponse, error) { - return nil, nil -} - -func (c *ClusterServiceClient) DeleteK8SCluster(ctx context.Context, in *kessel.DeleteK8SClusterRequest, - opts ...http.CallOption, -) (*kessel.DeleteK8SClusterResponse, error) { - return nil, nil -} diff --git a/agent/pkg/status/controller/managedclusters/managed_cluster_syncer.go b/agent/pkg/status/controller/managedclusters/managed_cluster_syncer.go deleted file mode 100644 index 4fa1e7af7..000000000 --- a/agent/pkg/status/controller/managedclusters/managed_cluster_syncer.go +++ /dev/null @@ -1,46 +0,0 @@ -package managedclusters - -import ( - "context" - - clusterv1 "open-cluster-management.io/api/cluster/v1" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/predicate" - - "github.com/stolostron/multicluster-global-hub/agent/pkg/config" - statusconfig "github.com/stolostron/multicluster-global-hub/agent/pkg/status/controller/config" - "github.com/stolostron/multicluster-global-hub/agent/pkg/status/controller/generic" - "github.com/stolostron/multicluster-global-hub/pkg/constants" - "github.com/stolostron/multicluster-global-hub/pkg/enum" - "github.com/stolostron/multicluster-global-hub/pkg/transport" - "github.com/stolostron/multicluster-global-hub/pkg/utils" -) - -func LaunchManagedClusterSyncer(ctx context.Context, mgr ctrl.Manager, agentConfig *config.AgentConfig, - producer transport.Producer, -) error { - // controller config - instance := func() client.Object { return &clusterv1.ManagedCluster{} } - predicate := predicate.NewPredicateFuncs(func(object client.Object) bool { return true }) - - // emitter config - tweakFunc := func(object client.Object) { - utils.MergeAnnotations(object, map[string]string{ - constants.ManagedClusterManagedByAnnotation: statusconfig.GetLeafHubName(), - }) - } - emitter := generic.ObjectEmitterWrapper(enum.ManagedClusterType, func(obj client.Object) bool { - return !utils.HasAnnotation(obj, constants.ManagedClusterMigrating) - }, tweakFunc, false) - - return generic.LaunchGenericObjectSyncer( - "status.managed_cluster", - mgr, - generic.NewGenericController(instance, predicate), - producer, - statusconfig.GetManagerClusterDuration, - []generic.ObjectEmitter{ - emitter, - }) -} diff --git a/agent/pkg/status/controller/placement/placement_decision_syncer.go b/agent/pkg/status/controller/placement/placement_decision_syncer.go deleted file mode 100644 index 3e769f8f8..000000000 --- a/agent/pkg/status/controller/placement/placement_decision_syncer.go +++ /dev/null @@ -1,48 +0,0 @@ -package placement - -import ( - "context" - - clustersv1beta1 "open-cluster-management.io/api/cluster/v1beta1" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/predicate" - - "github.com/stolostron/multicluster-global-hub/agent/pkg/config" - statusconfig "github.com/stolostron/multicluster-global-hub/agent/pkg/status/controller/config" - "github.com/stolostron/multicluster-global-hub/agent/pkg/status/controller/generic" - "github.com/stolostron/multicluster-global-hub/pkg/constants" - "github.com/stolostron/multicluster-global-hub/pkg/enum" - "github.com/stolostron/multicluster-global-hub/pkg/transport" - "github.com/stolostron/multicluster-global-hub/pkg/utils" -) - -func LaunchPlacementDecisionSyncer(ctx context.Context, mgr ctrl.Manager, agentConfig *config.AgentConfig, - producer transport.Producer, -) error { - // controller config - instance := func() client.Object { return &clustersv1beta1.PlacementDecision{} } - predicate := predicate.NewPredicateFuncs(func(object client.Object) bool { return true }) - - // emitter config - placementDecisionEmitter := generic.ObjectEmitterWrapper(enum.PlacementDecisionType, - func(obj client.Object) bool { - return utils.HasAnnotation(obj, constants.OriginOwnerReferenceAnnotation) // global resource - }, func(obj client.Object) { - obj.SetManagedFields(nil) - }, false) - - // syncer - name := "status.placement_decision" - syncInterval := statusconfig.GetPolicyDuration - - return generic.LaunchGenericObjectSyncer( - name, - mgr, - generic.NewGenericController(instance, predicate), - producer, - syncInterval, - []generic.ObjectEmitter{ - placementDecisionEmitter, - }) -} diff --git a/agent/pkg/status/controller/placement/placement_rule_syncer.go b/agent/pkg/status/controller/placement/placement_rule_syncer.go deleted file mode 100644 index ab6fb306e..000000000 --- a/agent/pkg/status/controller/placement/placement_rule_syncer.go +++ /dev/null @@ -1,58 +0,0 @@ -package placement - -import ( - "context" - - placementrulesv1 "open-cluster-management.io/multicloud-operators-subscription/pkg/apis/apps/placementrule/v1" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/predicate" - - "github.com/stolostron/multicluster-global-hub/agent/pkg/config" - statusconfig "github.com/stolostron/multicluster-global-hub/agent/pkg/status/controller/config" - "github.com/stolostron/multicluster-global-hub/agent/pkg/status/controller/generic" - "github.com/stolostron/multicluster-global-hub/pkg/constants" - "github.com/stolostron/multicluster-global-hub/pkg/enum" - "github.com/stolostron/multicluster-global-hub/pkg/transport" - "github.com/stolostron/multicluster-global-hub/pkg/utils" -) - -func LaunchPlacementRuleSyncer(ctx context.Context, mgr ctrl.Manager, agentConfig *config.AgentConfig, - producer transport.Producer, -) error { - // controller config - instance := func() client.Object { return &placementrulesv1.PlacementRule{} } - predicate := predicate.NewPredicateFuncs(func(object client.Object) bool { return true }) - controller := generic.NewGenericController(instance, predicate) - - // emitter config - tweakFunc := func(obj client.Object) { - obj.SetManagedFields(nil) - } - globalPlacementRuleEmitter := generic.ObjectEmitterWrapper(enum.PlacementRuleSpecType, - func(obj client.Object) bool { - return utils.HasAnnotation(obj, constants.OriginOwnerReferenceAnnotation) // global resource - }, tweakFunc, false) - - localPlacementRuleEmitter := generic.ObjectEmitterWrapper(enum.LocalPlacementRuleSpecType, - func(obj client.Object) bool { - // return statusconfig.GetEnableLocalPolicy() == statusconfig.EnableLocalPolicyTrue && - // !utils.HasAnnotation(obj, constants.OriginOwnerReferenceAnnotation) // local resource - return false // disable the placementrule now - }, tweakFunc, false) - - // syncer - name := "status.placement_rule" - syncInterval := statusconfig.GetPolicyDuration - - return generic.LaunchGenericObjectSyncer( - name, - mgr, - controller, - producer, - syncInterval, - []generic.ObjectEmitter{ - globalPlacementRuleEmitter, - localPlacementRuleEmitter, - }) -} diff --git a/agent/pkg/status/controller/placement/placement_syncer.go b/agent/pkg/status/controller/placement/placement_syncer.go deleted file mode 100644 index 62335048a..000000000 --- a/agent/pkg/status/controller/placement/placement_syncer.go +++ /dev/null @@ -1,50 +0,0 @@ -package placement - -import ( - "context" - - clustersv1beta1 "open-cluster-management.io/api/cluster/v1beta1" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/predicate" - - "github.com/stolostron/multicluster-global-hub/agent/pkg/config" - statusconfig "github.com/stolostron/multicluster-global-hub/agent/pkg/status/controller/config" - "github.com/stolostron/multicluster-global-hub/agent/pkg/status/controller/generic" - "github.com/stolostron/multicluster-global-hub/pkg/constants" - "github.com/stolostron/multicluster-global-hub/pkg/enum" - "github.com/stolostron/multicluster-global-hub/pkg/transport" - "github.com/stolostron/multicluster-global-hub/pkg/utils" -) - -func LaunchPlacementSyncer(ctx context.Context, mgr ctrl.Manager, agentConfig *config.AgentConfig, - producer transport.Producer, -) error { - // controller config - instance := func() client.Object { return &clustersv1beta1.Placement{} } - predicate := predicate.NewPredicateFuncs(func(object client.Object) bool { return true }) - - // emitter config - globalPlacementEmitter := generic.ObjectEmitterWrapper(enum.PlacementSpecType, - func(obj client.Object) bool { - return utils.HasAnnotation(obj, constants.OriginOwnerReferenceAnnotation) // global resource - }, - func(obj client.Object) { - obj.SetManagedFields(nil) - }, false, - ) - - // syncer - name := "status.placement" - syncInterval := statusconfig.GetPolicyDuration - - return generic.LaunchGenericObjectSyncer( - name, - mgr, - generic.NewGenericController(instance, predicate), - producer, - syncInterval, - []generic.ObjectEmitter{ - globalPlacementEmitter, - }) -} diff --git a/agent/pkg/status/controller/policies/policy_syncer.go b/agent/pkg/status/controller/policies/policy_syncer.go deleted file mode 100644 index c0b8b31bb..000000000 --- a/agent/pkg/status/controller/policies/policy_syncer.go +++ /dev/null @@ -1,117 +0,0 @@ -package policies - -import ( - "context" - - policiesv1 "open-cluster-management.io/governance-policy-propagator/api/v1" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/predicate" - - "github.com/stolostron/multicluster-global-hub/agent/pkg/config" - statusconfig "github.com/stolostron/multicluster-global-hub/agent/pkg/status/controller/config" - "github.com/stolostron/multicluster-global-hub/agent/pkg/status/controller/generic" - eventversion "github.com/stolostron/multicluster-global-hub/pkg/bundle/version" - "github.com/stolostron/multicluster-global-hub/pkg/constants" - "github.com/stolostron/multicluster-global-hub/pkg/enum" - "github.com/stolostron/multicluster-global-hub/pkg/transport" - "github.com/stolostron/multicluster-global-hub/pkg/utils" -) - -func LaunchPolicySyncer(ctx context.Context, mgr ctrl.Manager, agentConfig *config.AgentConfig, - producer transport.Producer, -) error { - // controller config - instance := func() client.Object { return &policiesv1.Policy{} } - predicate := predicate.NewPredicateFuncs(func(object client.Object) bool { return true }) - controller := generic.NewGenericController(instance, predicate) - - // emitters - // 1. local compliance - localComplianceVersion := eventversion.NewVersion() - localComplianceShouldUpdate := func(obj client.Object) bool { - return statusconfig.GetAggregationLevel() == statusconfig.AggregationFull && // full level - statusconfig.GetEnableLocalPolicy() == statusconfig.EnableLocalPolicyTrue && // enable local policy - !utils.HasAnnotation(obj, constants.OriginOwnerReferenceAnnotation) && // local resource - !utils.HasLabel(obj, constants.PolicyEventRootPolicyNameLabelKey) // root policy - } - localComplianceEmitter := ComplianceEmitterWrapper( - enum.LocalComplianceType, - localComplianceVersion, - localComplianceShouldUpdate, - ) - - // 2. local complete compliance - localCompleteEmitter := CompleteComplianceEmitterWrapper( - enum.LocalCompleteComplianceType, - localComplianceVersion, - localComplianceShouldUpdate, - ) - - // 3. local policy event - localStatusEventEmitter := StatusEventEmitter(ctx, enum.LocalReplicatedPolicyEventType, - func(obj client.Object) bool { - return statusconfig.GetEnableLocalPolicy() == statusconfig.EnableLocalPolicyTrue && - !utils.HasAnnotation(obj, constants.OriginOwnerReferenceAnnotation) && // local resource - utils.HasLabel(obj, constants.PolicyEventRootPolicyNameLabelKey) // replicated policy - }, - mgr.GetClient(), - agentConfig.TransportConfig.KafkaCredential.StatusTopic, - ) - - // 4. local policy spec - localPolicySpecEmitter := generic.ObjectEmitterWrapper(enum.LocalPolicySpecType, - func(obj client.Object) bool { - return statusconfig.GetEnableLocalPolicy() == statusconfig.EnableLocalPolicyTrue && // enable local policy - !utils.HasAnnotation(obj, constants.OriginOwnerReferenceAnnotation) && // local resource - !utils.HasLabel(obj, constants.PolicyEventRootPolicyNameLabelKey) // root policy - }, - cleanPolicy, - true, - ) - - // global policy emitters - // 5. global compliance - complianceVersion := eventversion.NewVersion() - compliancePredicate := func(obj client.Object) bool { - return statusconfig.GetAggregationLevel() == statusconfig.AggregationFull && // full level - utils.HasAnnotation(obj, constants.OriginOwnerReferenceAnnotation) && // global resource - !utils.HasLabel(obj, constants.PolicyEventRootPolicyNameLabelKey) // root policy - } - complianceEmitter := ComplianceEmitterWrapper( - enum.ComplianceType, - complianceVersion, - compliancePredicate, - ) - - // 6. global complete compliance - completeEmitter := CompleteComplianceEmitterWrapper( - enum.CompleteComplianceType, - complianceVersion, - compliancePredicate, - ) - - return generic.LaunchGenericObjectSyncer( - "status.policy", - mgr, - controller, - producer, - statusconfig.GetPolicyDuration, - []generic.ObjectEmitter{ - localComplianceEmitter, - localCompleteEmitter, - localStatusEventEmitter, - localPolicySpecEmitter, - // global compliance - complianceEmitter, - completeEmitter, - }) -} - -func cleanPolicy(object client.Object) { - policy, ok := object.(*policiesv1.Policy) - if !ok { - panic("Wrong instance passed to clean policy function, not a Policy") - } - policy.Status = policiesv1.PolicyStatus{} -} diff --git a/agent/pkg/status/controller/filter/time_filter.go b/agent/pkg/status/filter/time_filter.go similarity index 100% rename from agent/pkg/status/controller/filter/time_filter.go rename to agent/pkg/status/filter/time_filter.go diff --git a/agent/pkg/status/controller/filter/time_filter_test.go b/agent/pkg/status/filter/time_filter_test.go similarity index 100% rename from agent/pkg/status/controller/filter/time_filter_test.go rename to agent/pkg/status/filter/time_filter_test.go diff --git a/agent/pkg/status/controller/generic/generic_controller.go b/agent/pkg/status/generic/generic_controller.go similarity index 74% rename from agent/pkg/status/controller/generic/generic_controller.go rename to agent/pkg/status/generic/generic_controller.go index e6b243c2e..78d76d9a0 100644 --- a/agent/pkg/status/controller/generic/generic_controller.go +++ b/agent/pkg/status/generic/generic_controller.go @@ -3,16 +3,18 @@ package generic import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/predicate" + + "github.com/stolostron/multicluster-global-hub/agent/pkg/status/interfaces" ) -var _ Controller = &genericController{} +var _ interfaces.Controller = &genericController{} type genericController struct { instance func() client.Object predicate predicate.Predicate } -func NewGenericController(instance func() client.Object, predicate predicate.Predicate) Controller { +func NewGenericController(instance func() client.Object, predicate predicate.Predicate) interfaces.Controller { return &genericController{ instance: instance, predicate: predicate, diff --git a/agent/pkg/status/controller/generic/generic_emitter.go b/agent/pkg/status/generic/generic_emitter.go similarity index 54% rename from agent/pkg/status/controller/generic/generic_emitter.go rename to agent/pkg/status/generic/generic_emitter.go index 534833186..ef1b81e13 100644 --- a/agent/pkg/status/controller/generic/generic_emitter.go +++ b/agent/pkg/status/generic/generic_emitter.go @@ -2,74 +2,40 @@ package generic import ( cloudevents "github.com/cloudevents/sdk-go/v2" - "sigs.k8s.io/controller-runtime/pkg/client" - "github.com/stolostron/multicluster-global-hub/agent/pkg/spec/controller/syncers" - "github.com/stolostron/multicluster-global-hub/agent/pkg/status/controller/config" + "github.com/stolostron/multicluster-global-hub/agent/pkg/configs" + specsyncers "github.com/stolostron/multicluster-global-hub/agent/pkg/spec/syncers" + "github.com/stolostron/multicluster-global-hub/agent/pkg/status/interfaces" eventversion "github.com/stolostron/multicluster-global-hub/pkg/bundle/version" "github.com/stolostron/multicluster-global-hub/pkg/enum" ) -var _ Emitter = &genericEmitter{} - -type EmitterOption func(*genericEmitter) - -func WithTopic(topic string) EmitterOption { - return func(g *genericEmitter) { - g.topic = topic - } -} - -func WithTweakFunc(tweakFunc func(client.Object)) EmitterOption { - return func(g *genericEmitter) { - g.tweakFunc = tweakFunc - } -} - -func WithShouldUpdate(shouldUpdate func(client.Object) bool) EmitterOption { - return func(g *genericEmitter) { - g.shouldUpdate = shouldUpdate - } -} - -func WithDependencyVersion(version *eventversion.Version) EmitterOption { - return func(g *genericEmitter) { - g.dependencyVersion = version - } -} - -func WithVersion(version *eventversion.Version) EmitterOption { - return func(g *genericEmitter) { - g.currentVersion = version - } -} +var _ interfaces.Emitter = &genericEmitter{} type genericEmitter struct { eventType enum.EventType - payload interface{} currentVersion *eventversion.Version lastSentVersion eventversion.Version topic string dependencyVersion *eventversion.Version - tweakFunc func(client.Object) - shouldUpdate func(client.Object) bool + + postSend func(interface{}) } func NewGenericEmitter( eventType enum.EventType, - payload interface{}, opts ...EmitterOption, ) *genericEmitter { emitter := &genericEmitter{ - eventType: eventType, - payload: payload, - currentVersion: eventversion.NewVersion(), - lastSentVersion: *eventversion.NewVersion(), + eventType: eventType, + currentVersion: eventversion.NewVersion(), + // lastSentVersion: *eventversion.NewVersion(), } emitter.applyOptions(opts...) + emitter.lastSentVersion = *emitter.currentVersion // support resync - syncers.EnableResyc(string(eventType), emitter.currentVersion) + specsyncers.EnableResyc(string(eventType), emitter.currentVersion) return emitter } @@ -83,7 +49,10 @@ func (h *genericEmitter) ShouldSend() bool { return h.currentVersion.NewerThan(&h.lastSentVersion) } -func (h *genericEmitter) PostSend() { +func (h *genericEmitter) PostSend(data interface{}) { + if h.postSend != nil { + h.postSend(data) + } h.currentVersion.Next() h.lastSentVersion = *h.currentVersion } @@ -92,45 +61,45 @@ func (h *genericEmitter) Topic() string { return h.topic } -func (h *genericEmitter) ShouldUpdate(obj client.Object) bool { - toUpdate := true - if h.shouldUpdate != nil { - toUpdate = h.shouldUpdate(obj) - } - - if toUpdate && h.tweakFunc != nil { - h.tweakFunc(obj) - } - return toUpdate -} - func (h *genericEmitter) PostUpdate() { h.currentVersion.Incr() } -// func (h *genericEventEmitter) Update(obj client.Object) { -// if h.tweakFunc != nil { -// h.tweakFunc(obj) -// } -// if h.payload.Update(obj) { -// h.currentVersion.Incr() -// } -// } - -// func (h *genericEventEmitter) Delete(obj client.Object) { -// if h.payload.Delete(obj) { -// h.currentVersion.Incr() -// } -// } - -func (g *genericEmitter) ToCloudEvent() (*cloudevents.Event, error) { +func (g *genericEmitter) ToCloudEvent(payload interface{}) (*cloudevents.Event, error) { e := cloudevents.NewEvent() - e.SetSource(config.GetLeafHubName()) + e.SetSource(configs.GetLeafHubName()) e.SetType(string(g.eventType)) e.SetExtension(eventversion.ExtVersion, g.currentVersion.String()) if g.dependencyVersion != nil { e.SetExtension(eventversion.ExtDependencyVersion, g.dependencyVersion.String()) } - err := e.SetData(cloudevents.ApplicationJSON, g.payload) + err := e.SetData(cloudevents.ApplicationJSON, payload) return &e, err } + +// define the emitter options +type EmitterOption func(*genericEmitter) + +func WithTopic(topic string) EmitterOption { + return func(g *genericEmitter) { + g.topic = topic + } +} + +func WithDependencyVersion(version *eventversion.Version) EmitterOption { + return func(g *genericEmitter) { + g.dependencyVersion = version + } +} + +func WithVersion(version *eventversion.Version) EmitterOption { + return func(g *genericEmitter) { + g.currentVersion = version + } +} + +func WithPostSend(postSend func(interface{})) EmitterOption { + return func(g *genericEmitter) { + g.postSend = postSend + } +} diff --git a/agent/pkg/status/controller/generic/generic_object_handler.go b/agent/pkg/status/generic/generic_handler.go similarity index 54% rename from agent/pkg/status/controller/generic/generic_object_handler.go rename to agent/pkg/status/generic/generic_handler.go index 6e253460d..8fc06d24f 100644 --- a/agent/pkg/status/controller/generic/generic_object_handler.go +++ b/agent/pkg/status/generic/generic_handler.go @@ -4,24 +4,44 @@ import ( "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/stolostron/multicluster-global-hub/agent/pkg/status/interfaces" genericpayload "github.com/stolostron/multicluster-global-hub/pkg/bundle/generic" ) -type genericObjectHandler struct { +type genericHandler struct { eventData *genericpayload.GenericObjectBundle - // isSpec is to let the handler only update the event when spec is changed, that means whether it is a specHandler. + // isSpec is to let the handler only update the event when spec is changed. + // the current replicated policy event will also emit such message,it is true for policy, + // haven't handle the other object spec like placement, appsub... isSpec bool + + tweakFunc func(client.Object) + shouldUpdate func(client.Object) bool } -// TODO: isSpec is true for policy, haven't handle the other object spec like placement, appsub... -func NewGenericObjectHandler(eventData *genericpayload.GenericObjectBundle, isSpec bool) Handler { - return &genericObjectHandler{ +func NewGenericHandler(eventData *genericpayload.GenericObjectBundle, opts ...HandlerOption) interfaces.Handler { + h := &genericHandler{ eventData: eventData, - isSpec: isSpec, + isSpec: false, + } + + for _, fn := range opts { + fn(h) } + return h } -func (h *genericObjectHandler) Update(obj client.Object) bool { +func (h *genericHandler) Get() interface{} { + return h.eventData +} + +func (h *genericHandler) Update(obj client.Object) bool { + if h.shouldUpdate != nil { + if updated := h.shouldUpdate(obj); !updated { + return false + } + } + index := getObjectIndexByUID(obj.GetUID(), (*h.eventData)) if index == -1 { // object not found, need to add it to the bundle (*h.eventData) = append((*h.eventData), obj) @@ -39,10 +59,21 @@ func (h *genericObjectHandler) Update(obj client.Object) bool { } (*h.eventData)[index] = obj + + // tweak + if h.tweakFunc != nil { + h.tweakFunc(obj) + } return true } -func (h *genericObjectHandler) Delete(obj client.Object) bool { +func (h *genericHandler) Delete(obj client.Object) bool { + if h.shouldUpdate != nil { + if updated := h.shouldUpdate(obj); !updated { + return false + } + } + index := getObjectIndexByObj(obj, (*h.eventData)) if index == -1 { // trying to delete object which doesn't exist return false @@ -73,3 +104,24 @@ func getObjectIndexByObj(obj client.Object, objects []client.Object) int { } return -1 } + +// define the emitter options +type HandlerOption func(*genericHandler) + +func WithTweakFunc(tweakFunc func(client.Object)) HandlerOption { + return func(g *genericHandler) { + g.tweakFunc = tweakFunc + } +} + +func WithSpec(onlySpec bool) HandlerOption { + return func(g *genericHandler) { + g.isSpec = onlySpec + } +} + +func WithShouldUpdate(shouldUpdate func(client.Object) bool) HandlerOption { + return func(g *genericHandler) { + g.shouldUpdate = shouldUpdate + } +} diff --git a/agent/pkg/status/controller/generic/generic_object_syncer.go b/agent/pkg/status/generic/multi_event_syncer.go similarity index 76% rename from agent/pkg/status/controller/generic/generic_object_syncer.go rename to agent/pkg/status/generic/multi_event_syncer.go index e53b13fbb..0d5d68afc 100644 --- a/agent/pkg/status/controller/generic/generic_object_syncer.go +++ b/agent/pkg/status/generic/multi_event_syncer.go @@ -12,40 +12,47 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - "github.com/stolostron/multicluster-global-hub/agent/pkg/status/controller/config" + "github.com/stolostron/multicluster-global-hub/agent/pkg/configs" + "github.com/stolostron/multicluster-global-hub/agent/pkg/status/interfaces" "github.com/stolostron/multicluster-global-hub/pkg/constants" "github.com/stolostron/multicluster-global-hub/pkg/transport" ) const ( - ExtVersion = "extversion" - FinalizerName = constants.GlobalHubCleanupFinalizer + ExtVersion = "extversion" ) -type genericObjectSyncer struct { +type multiEventSyncer struct { log logr.Logger client client.Client producer transport.Producer - controller Controller - eventEmitters []ObjectEmitter + controller interfaces.Controller + eventEmitters []*EmitterHandler leafHubName string syncIntervalFunc func() time.Duration + finalizerName string startOnce sync.Once lock sync.Mutex } -// LaunchGenericObjectSyncer is used to send multi event(by the eventEmitter) by a specific client.Object -func LaunchGenericObjectSyncer(name string, mgr ctrl.Manager, controller Controller, - producer transport.Producer, intervalFunc func() time.Duration, eventEmitters []ObjectEmitter, +type EmitterHandler struct { + interfaces.Emitter + interfaces.Handler +} + +// LaunchMultiEventSyncer is used to send multi event(by the eventEmitter) by a specific client.Object +func LaunchMultiEventSyncer(name string, mgr ctrl.Manager, controller interfaces.Controller, + producer transport.Producer, intervalFunc func() time.Duration, eventEmitters []*EmitterHandler, ) error { - syncer := &genericObjectSyncer{ + syncer := &multiEventSyncer{ log: ctrl.Log.WithName(name), client: mgr.GetClient(), producer: producer, syncIntervalFunc: intervalFunc, controller: controller, eventEmitters: eventEmitters, - leafHubName: config.GetLeafHubName(), + leafHubName: configs.GetLeafHubName(), + finalizerName: constants.GlobalHubCleanupFinalizer, } // start the periodic syncer @@ -57,7 +64,7 @@ func LaunchGenericObjectSyncer(name string, mgr ctrl.Manager, controller Control WithEventFilter(controller.Predicate()).Complete(syncer) } -func (c *genericObjectSyncer) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.Result, error) { +func (c *multiEventSyncer) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.Result, error) { object := c.controller.Instance() if err := c.client.Get(ctx, request.NamespacedName, object); errors.IsNotFound(err) { // the instance was deleted and it had no finalizer on it. @@ -77,7 +84,7 @@ func (c *genericObjectSyncer) Reconcile(ctx context.Context, request ctrl.Reques if !enableCleanUpFinalizer(object) { return ctrl.Result{}, nil } - err := removeFinalizer(ctx, c.client, object, FinalizerName) + err := removeFinalizer(ctx, c.client, object, c.finalizerName) if err != nil { c.log.Error(err, "failed to remove cleanup fianlizer", "namespace", request.Namespace, "name", request.Name) return ctrl.Result{Requeue: true, RequeueAfter: REQUEUE_PERIOD}, err @@ -91,7 +98,7 @@ func (c *genericObjectSyncer) Reconcile(ctx context.Context, request ctrl.Reques if !enableCleanUpFinalizer(object) { return ctrl.Result{}, nil } - err := addFinalizer(ctx, c.client, object, FinalizerName) + err := addFinalizer(ctx, c.client, object, c.finalizerName) if err != nil { c.log.Error(err, "failed to add finalizer", "namespace", request.Namespace, "name", request.Name) return ctrl.Result{Requeue: true, RequeueAfter: REQUEUE_PERIOD}, err @@ -99,29 +106,29 @@ func (c *genericObjectSyncer) Reconcile(ctx context.Context, request ctrl.Reques return ctrl.Result{}, nil } -func (c *genericObjectSyncer) updateObject(object client.Object) { +func (c *multiEventSyncer) updateObject(object client.Object) { c.lock.Lock() // make sure handler are not updated if we're during bundles sync defer c.lock.Unlock() for _, eventEmitter := range c.eventEmitters { // update in each handler from the collection according to their order. - if eventEmitter.ShouldUpdate(object) && eventEmitter.Update(object) { + if eventEmitter.Update(object) { eventEmitter.PostUpdate() } } } -func (c *genericObjectSyncer) deleteObject(object client.Object) { +func (c *multiEventSyncer) deleteObject(object client.Object) { c.lock.Lock() // make sure bundles are not updated if we're during bundles sync defer c.lock.Unlock() for _, eventEmitter := range c.eventEmitters { - if eventEmitter.ShouldUpdate(object) && eventEmitter.Delete(object) { + if eventEmitter.Delete(object) { eventEmitter.PostUpdate() } } } -func (c *genericObjectSyncer) periodicSync() { +func (c *multiEventSyncer) periodicSync() { currentSyncInterval := c.syncIntervalFunc() ticker := time.NewTicker(currentSyncInterval) defer ticker.Stop() @@ -141,7 +148,7 @@ func (c *genericObjectSyncer) periodicSync() { } } -func (c *genericObjectSyncer) syncEvents() { +func (c *multiEventSyncer) syncEvents() { c.lock.Lock() // make sure bundles are not updated if we're during bundles sync defer c.lock.Unlock() @@ -149,7 +156,7 @@ func (c *genericObjectSyncer) syncEvents() { emitter := c.eventEmitters[i] if emitter.ShouldSend() { - evt, err := emitter.ToCloudEvent() + evt, err := emitter.ToCloudEvent(emitter.Handler.Get()) if err != nil { c.log.Error(err, "failed to get CloudEvent instance", "evt", evt) } @@ -163,7 +170,7 @@ func (c *genericObjectSyncer) syncEvents() { c.log.Error(err, "failed to send event", "evt", evt) continue } - emitter.PostSend() + emitter.PostSend(emitter.Handler.Get()) } } } diff --git a/agent/pkg/status/generic/multi_object_syncer.go b/agent/pkg/status/generic/multi_object_syncer.go new file mode 100644 index 000000000..4cf0e3ab0 --- /dev/null +++ b/agent/pkg/status/generic/multi_object_syncer.go @@ -0,0 +1,269 @@ +package generic + +import ( + "context" + "fmt" + "strconv" + "strings" + "sync" + "time" + + cecontext "github.com/cloudevents/sdk-go/v2/context" + "github.com/go-logr/logr" + apierrors "k8s.io/apimachinery/pkg/api/errors" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + "github.com/stolostron/multicluster-global-hub/agent/pkg/configs" + "github.com/stolostron/multicluster-global-hub/agent/pkg/status/interfaces" + "github.com/stolostron/multicluster-global-hub/pkg/constants" + "github.com/stolostron/multicluster-global-hub/pkg/transport" + "github.com/stolostron/multicluster-global-hub/pkg/utils" +) + +type multiObjectSyncer struct { + log logr.Logger + runtimeClient client.Client + producer transport.Producer + objectControllers []ControllerHandler + emitter interfaces.Emitter + leafHubName string + syncIntervalFunc func() time.Duration + startOnce sync.Once + + // share properties with multi-object-controller + lock *sync.Mutex +} + +type ControllerHandler struct { + interfaces.Controller + interfaces.Handler +} + +// LaunchMultiObjectSyncer send the event generated by multiple object +func LaunchMultiObjectSyncer(name string, mgr ctrl.Manager, objectControllers []ControllerHandler, + producer transport.Producer, intervalFunc func() time.Duration, emitter interfaces.Emitter, +) error { + syncer := &multiObjectSyncer{ + log: ctrl.Log.WithName(name), + leafHubName: configs.GetLeafHubName(), + runtimeClient: mgr.GetClient(), + + objectControllers: objectControllers, + emitter: emitter, + producer: producer, + syncIntervalFunc: intervalFunc, + lock: &sync.Mutex{}, + } + + // start the periodic syncer + syncer.startOnce.Do(func() { + go syncer.periodicSync() + }) + + // start all the controllers to update the payload + for _, eventController := range objectControllers { + err := AddObjectController(mgr, eventController, emitter, syncer.lock) + if err != nil { + return err + } + } + return nil +} + +func (s *multiObjectSyncer) periodicSync() { + currentSyncInterval := s.syncIntervalFunc() + s.log.Info(fmt.Sprintf("sync interval has been reset to %s", currentSyncInterval.String())) + ticker := time.NewTicker(currentSyncInterval) + defer ticker.Stop() + + for { + s.syncEvent() + <-ticker.C // wait for next time interval + + resolvedInterval := s.syncIntervalFunc() + + // reset ticker if sync interval has changed + if resolvedInterval != currentSyncInterval { + currentSyncInterval = resolvedInterval + ticker.Reset(currentSyncInterval) + s.log.Info(fmt.Sprintf("sync interval has been reset to %s", currentSyncInterval.String())) + } + } +} + +func (s *multiObjectSyncer) syncEvent() { + s.lock.Lock() // make sure bundles are not updated if we're during bundles sync + defer s.lock.Unlock() + + // retrieve the bundle from one of the handlers, since they share the same one + var eventData interface{} + if len(s.objectControllers) > 0 { + eventData = s.objectControllers[0].Handler.Get() + } + + if s.emitter.ShouldSend() { + evt, err := s.emitter.ToCloudEvent(eventData) + if err != nil { + s.log.Error(err, "failed to get CloudEvent instance", "evt", evt) + return + } + + ctx := context.TODO() + if s.emitter.Topic() != "" { + ctx = cecontext.WithTopic(ctx, s.emitter.Topic()) + } + if err := s.producer.SendEvent(ctx, *evt); err != nil { + s.log.Error(err, "failed to send event", "evt", evt) + return + } + s.emitter.PostSend(eventData) + } +} + +const REQUEUE_PERIOD = 5 * time.Second + +// single object controller to update the event +type objectController struct { + log logr.Logger + client client.Client + finalizerName string + emitter interfaces.Emitter + objectController ControllerHandler + + // objectController interfaces.ObjectController + lock *sync.Mutex +} + +func AddObjectController(mgr ctrl.Manager, objectCtrl ControllerHandler, emitter interfaces.Emitter, + lock *sync.Mutex, +) error { + object := objectCtrl.Instance() + controller := &objectController{ + log: ctrl.Log.WithName(fmt.Sprintf("status.%s", object.GetObjectKind())), + client: mgr.GetClient(), + finalizerName: constants.GlobalHubCleanupFinalizer, + emitter: emitter, + objectController: objectCtrl, + lock: lock, + } + + controllerBuilder := ctrl.NewControllerManagedBy(mgr).For(object) + if objectCtrl.Predicate() != nil { + controllerBuilder = controllerBuilder.WithEventFilter(objectCtrl.Predicate()) + } + return controllerBuilder.Complete(controller) +} + +func (c *objectController) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.Result, error) { + reqLogger := c.log.WithValues("Namespace", request.Namespace, "Name", request.Name) + object := c.objectController.Instance() + + if err := c.client.Get(ctx, request.NamespacedName, object); apierrors.IsNotFound(err) { + // the instance was deleted and it had no finalizer on it. + // for the local resources, there is no finalizer so we need to delete the object from the bundle + object.SetNamespace(request.Namespace) + object.SetName(request.Name) + if e := c.deleteObjectAndFinalizer(ctx, object); e != nil { + return ctrl.Result{Requeue: true, RequeueAfter: REQUEUE_PERIOD}, e + } + return ctrl.Result{}, nil + } else if err != nil { + return ctrl.Result{Requeue: true, RequeueAfter: REQUEUE_PERIOD}, + fmt.Errorf("reconciliation failed: %w", err) + } + + if !object.GetDeletionTimestamp().IsZero() { + if err := c.deleteObjectAndFinalizer(ctx, object); err != nil { + return ctrl.Result{Requeue: true, RequeueAfter: REQUEUE_PERIOD}, err + } + } else { // otherwise, the object was not deleted and no error occurred + if err := c.updateObjectAndFinalizer(ctx, object); err != nil { + return ctrl.Result{Requeue: true, RequeueAfter: REQUEUE_PERIOD}, err + } + } + reqLogger.V(2).Info("Reconciliation complete.") + return ctrl.Result{}, nil +} + +func (c *objectController) updateObjectAndFinalizer(ctx context.Context, object client.Object) error { + // only add finalizer for the global resources + if enableCleanUpFinalizer(object) { + err := addFinalizer(ctx, c.client, object, c.finalizerName) + if err != nil { + return err + } + } + + c.lock.Lock() // make sure bundles are not updated if we're during bundles sync + defer c.lock.Unlock() + if c.objectController.Update(object) { + c.emitter.PostUpdate() + } + return nil +} + +func (c *objectController) deleteObjectAndFinalizer(ctx context.Context, object client.Object) error { + c.lock.Lock() // make sure bundles are not updated if we're during bundles sync + if c.objectController.Delete(object) { + c.emitter.PostUpdate() + } + c.lock.Unlock() + + if enableCleanUpFinalizer(object) { + err := removeFinalizer(ctx, c.client, object, c.finalizerName) + if err != nil { + return err + } + } + return nil +} + +func enableCleanUpFinalizer(obj client.Object) bool { + return utils.HasLabel(obj, constants.GlobalHubGlobalResourceLabel) || + utils.HasAnnotation(obj, constants.OriginOwnerReferenceAnnotation) +} + +func cleanObject(object client.Object) { + object.SetManagedFields(nil) + object.SetFinalizers(nil) + object.SetOwnerReferences(nil) + object.SetSelfLink("") + // object.SetClusterName("") +} + +func addFinalizer(ctx context.Context, c client.Client, obj client.Object, finalizer string) error { + // if the removing finalizer label hasn't expired, then skip the adding finalizer action + if val, found := obj.GetLabels()[constants.GlobalHubFinalizerRemovingDeadline]; found { + deadline, err := strconv.ParseInt(val, 10, 64) + if err != nil { + return err + } + if time.Now().Unix() < deadline { + return nil + } else { + delete(obj.GetLabels(), constants.GlobalHubFinalizerRemovingDeadline) + } + } + + if controllerutil.ContainsFinalizer(obj, finalizer) { + return nil + } + + controllerutil.AddFinalizer(obj, finalizer) + + if err := c.Update(ctx, obj); err != nil && !strings.Contains(err.Error(), "the object has been modified") { + return err + } + return nil +} + +func removeFinalizer(ctx context.Context, c client.Client, obj client.Object, finalizer string) error { + if !controllerutil.ContainsFinalizer(obj, finalizer) { + return nil // if finalizer is not there, do nothing. + } + controllerutil.RemoveFinalizer(obj, finalizer) + + return c.Update(ctx, obj) +} diff --git a/agent/pkg/status/controller/generic/interface.go b/agent/pkg/status/interfaces/interface.go similarity index 75% rename from agent/pkg/status/controller/generic/interface.go rename to agent/pkg/status/interfaces/interface.go index c06824338..61987c2f2 100644 --- a/agent/pkg/status/controller/generic/interface.go +++ b/agent/pkg/status/interfaces/interface.go @@ -1,4 +1,4 @@ -package generic +package interfaces import ( cloudevents "github.com/cloudevents/sdk-go/v2" @@ -14,12 +14,9 @@ type Controller interface { // Use the event emitter to control the flow of the event syncer type Emitter interface { - // assert whether to update the payload by the current handler - ShouldUpdate(object client.Object) bool - PostUpdate() - ToCloudEvent() (*cloudevents.Event, error) + ToCloudEvent(data interface{}) (*cloudevents.Event, error) // topic Topic() string @@ -28,23 +25,15 @@ type Emitter interface { ShouldSend() bool // triggered after sending the event, incr generate, clean payload, ... - PostSend() + PostSend(data interface{}) } // Use this interface to update the event payload/data by the client.Object type Handler interface { + // Get the bundle as the cloudevent data + Get() interface{} // the method is for the controller to update payload by the object Update(object client.Object) bool // the method is for the controller to update payload by the object Delete(object client.Object) bool } - -type EventController interface { - Controller - Handler -} - -type ObjectEmitter interface { - Emitter - Handler -} diff --git a/agent/pkg/status/controller/controller.go b/agent/pkg/status/status.go similarity index 61% rename from agent/pkg/status/controller/controller.go rename to agent/pkg/status/status.go index 5e5c8790f..eab8d3bb0 100644 --- a/agent/pkg/status/controller/controller.go +++ b/agent/pkg/status/status.go @@ -1,7 +1,7 @@ // Copyright (c) 2020 Red Hat, Inc. // Copyright Contributors to the Open Cluster Management project -package controller +package status import ( "context" @@ -9,40 +9,33 @@ import ( ctrl "sigs.k8s.io/controller-runtime" - "github.com/stolostron/multicluster-global-hub/agent/pkg/config" - "github.com/stolostron/multicluster-global-hub/agent/pkg/status/controller/apps" - agentstatusconfig "github.com/stolostron/multicluster-global-hub/agent/pkg/status/controller/config" - "github.com/stolostron/multicluster-global-hub/agent/pkg/status/controller/event" - "github.com/stolostron/multicluster-global-hub/agent/pkg/status/controller/filter" - "github.com/stolostron/multicluster-global-hub/agent/pkg/status/controller/hubcluster" - "github.com/stolostron/multicluster-global-hub/agent/pkg/status/controller/managedclusters" - "github.com/stolostron/multicluster-global-hub/agent/pkg/status/controller/placement" - "github.com/stolostron/multicluster-global-hub/agent/pkg/status/controller/policies" + "github.com/stolostron/multicluster-global-hub/agent/pkg/configs" + "github.com/stolostron/multicluster-global-hub/agent/pkg/status/filter" + "github.com/stolostron/multicluster-global-hub/agent/pkg/status/syncers/apps" + "github.com/stolostron/multicluster-global-hub/agent/pkg/status/syncers/configmap" + "github.com/stolostron/multicluster-global-hub/agent/pkg/status/syncers/events" + "github.com/stolostron/multicluster-global-hub/agent/pkg/status/syncers/managedcluster" + "github.com/stolostron/multicluster-global-hub/agent/pkg/status/syncers/managedhub" + "github.com/stolostron/multicluster-global-hub/agent/pkg/status/syncers/placement" + "github.com/stolostron/multicluster-global-hub/agent/pkg/status/syncers/policies" "github.com/stolostron/multicluster-global-hub/pkg/transport" - "github.com/stolostron/multicluster-global-hub/pkg/utils" ) var statusCtrlStarted = false -// AddControllers adds all the controllers to the Manager. -func AddControllers(ctx context.Context, mgr ctrl.Manager, transportClient transport.TransportClient, - agentConfig *config.AgentConfig, +// AddToManager adds all the syncers to the Manager. +func AddToManager(ctx context.Context, mgr ctrl.Manager, transportClient transport.TransportClient, + agentConfig *configs.AgentConfig, ) error { if statusCtrlStarted { return nil } - if err := agentstatusconfig.AddConfigController(mgr, agentConfig); err != nil { + + if err := configmap.AddConfigMapController(mgr, agentConfig); err != nil { return fmt.Errorf("failed to add ConfigMap controller: %w", err) } - var err error - switch agentConfig.TransportConfig.TransportType { - case string(transport.Kafka): - err = addKafkaSyncer(ctx, mgr, transportClient.GetProducer(), agentConfig) - case string(transport.Rest): - err = addInventorySyncer(ctx, mgr, transportClient.GetRequester()) - } - if err != nil { + if err := addKafkaSyncer(ctx, mgr, transportClient.GetProducer(), agentConfig); err != nil { return fmt.Errorf("failed to add the syncer: %w", err) } @@ -50,32 +43,16 @@ func AddControllers(ctx context.Context, mgr ctrl.Manager, transportClient trans return nil } -func addInventorySyncer(ctx context.Context, mgr ctrl.Manager, inventoryRequester transport.Requester) error { - mch, err := utils.ListMCH(ctx, mgr.GetClient()) - if err != nil { - return err - } - config.SetMCHVersion(mch.Status.CurrentVersion) - - if err := managedclusters.AddManagedClusterInfoCtrl(mgr, inventoryRequester); err != nil { - return err - } - if err := policies.AddPolicyController(mgr, inventoryRequester); err != nil { - return err - } - return nil -} - func addKafkaSyncer(ctx context.Context, mgr ctrl.Manager, producer transport.Producer, - agentConfig *config.AgentConfig, + agentConfig *configs.AgentConfig, ) error { // managed cluster - if err := managedclusters.LaunchManagedClusterSyncer(ctx, mgr, agentConfig, producer); err != nil { + if err := managedcluster.LaunchManagedClusterSyncer(ctx, mgr, agentConfig, producer); err != nil { return fmt.Errorf("failed to launch managedcluster syncer: %w", err) } // event syncer - err := event.LaunchEventSyncer(ctx, mgr, agentConfig, producer) + err := events.LaunchEventSyncer(ctx, mgr, agentConfig, producer) if err != nil { return fmt.Errorf("failed to launch event syncer: %w", err) } @@ -87,13 +64,13 @@ func addKafkaSyncer(ctx context.Context, mgr ctrl.Manager, producer transport.Pr } // hub cluster info - err = hubcluster.LaunchHubClusterInfoSyncer(mgr, producer) + err = managedhub.LaunchHubClusterInfoSyncer(mgr, producer) if err != nil { return fmt.Errorf("failed to launch hub cluster info syncer: %w", err) } // hub cluster heartbeat - err = hubcluster.LaunchHubClusterHeartbeatSyncer(mgr, producer) + err = managedhub.LaunchHubClusterHeartbeatSyncer(mgr, producer) if err != nil { return fmt.Errorf("failed to launch hub cluster heartbeat syncer: %w", err) } diff --git a/agent/pkg/status/controller/apps/subscription_report_sync.go b/agent/pkg/status/syncers/apps/subscription_report_sync.go similarity index 61% rename from agent/pkg/status/controller/apps/subscription_report_sync.go rename to agent/pkg/status/syncers/apps/subscription_report_sync.go index 1363da579..1fda74c9f 100644 --- a/agent/pkg/status/controller/apps/subscription_report_sync.go +++ b/agent/pkg/status/syncers/apps/subscription_report_sync.go @@ -8,34 +8,34 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/predicate" - "github.com/stolostron/multicluster-global-hub/agent/pkg/config" - statusconfig "github.com/stolostron/multicluster-global-hub/agent/pkg/status/controller/config" - "github.com/stolostron/multicluster-global-hub/agent/pkg/status/controller/generic" + "github.com/stolostron/multicluster-global-hub/agent/pkg/configs" + "github.com/stolostron/multicluster-global-hub/agent/pkg/status/generic" + "github.com/stolostron/multicluster-global-hub/agent/pkg/status/syncers/configmap" + genericpayload "github.com/stolostron/multicluster-global-hub/pkg/bundle/generic" "github.com/stolostron/multicluster-global-hub/pkg/enum" "github.com/stolostron/multicluster-global-hub/pkg/transport" ) -func LaunchSubscriptionReportSyncer(ctx context.Context, mgr ctrl.Manager, agentConfig *config.AgentConfig, +func LaunchSubscriptionReportSyncer(ctx context.Context, mgr ctrl.Manager, agentConfig *configs.AgentConfig, producer transport.Producer, ) error { // controller config instance := func() client.Object { return &appsv1alpha1.SubscriptionReport{} } predicate := predicate.NewPredicateFuncs(func(object client.Object) bool { return true }) - // emitter config - emitter := generic.ObjectEmitterWrapper(enum.SubscriptionReportType, nil, nil, false) + // emitter, handler + eventData := genericpayload.GenericObjectBundle{} - // syncer - name := "status.subscription_report" - syncInterval := statusconfig.GetPolicyDuration - - return generic.LaunchGenericObjectSyncer( - name, + return generic.LaunchMultiEventSyncer( + "status.subscription_report", mgr, generic.NewGenericController(instance, predicate), producer, - syncInterval, - []generic.ObjectEmitter{ - emitter, + configmap.GetPolicyDuration, + []*generic.EmitterHandler{ + { + Handler: generic.NewGenericHandler(&eventData), + Emitter: generic.NewGenericEmitter(enum.SubscriptionReportType), + }, }) } diff --git a/agent/pkg/status/controller/apps/subscription_status_sync.go b/agent/pkg/status/syncers/apps/subscription_status_sync.go similarity index 61% rename from agent/pkg/status/controller/apps/subscription_status_sync.go rename to agent/pkg/status/syncers/apps/subscription_status_sync.go index a418721aa..2b3640548 100644 --- a/agent/pkg/status/controller/apps/subscription_status_sync.go +++ b/agent/pkg/status/syncers/apps/subscription_status_sync.go @@ -8,34 +8,34 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/predicate" - "github.com/stolostron/multicluster-global-hub/agent/pkg/config" - statusconfig "github.com/stolostron/multicluster-global-hub/agent/pkg/status/controller/config" - "github.com/stolostron/multicluster-global-hub/agent/pkg/status/controller/generic" + "github.com/stolostron/multicluster-global-hub/agent/pkg/configs" + "github.com/stolostron/multicluster-global-hub/agent/pkg/status/generic" + "github.com/stolostron/multicluster-global-hub/agent/pkg/status/syncers/configmap" + genericpayload "github.com/stolostron/multicluster-global-hub/pkg/bundle/generic" "github.com/stolostron/multicluster-global-hub/pkg/enum" "github.com/stolostron/multicluster-global-hub/pkg/transport" ) -func LaunchSubscriptionStatusSyncer(ctx context.Context, mgr ctrl.Manager, agentConfig *config.AgentConfig, +func LaunchSubscriptionStatusSyncer(ctx context.Context, mgr ctrl.Manager, agentConfig *configs.AgentConfig, producer transport.Producer, ) error { // controller config instance := func() client.Object { return &appsv1alpha1.SubscriptionStatus{} } predicate := predicate.NewPredicateFuncs(func(object client.Object) bool { return true }) - // emitter config - emitter := generic.ObjectEmitterWrapper(enum.SubscriptionStatusType, nil, nil, false) + // emitter, handler + eventData := genericpayload.GenericObjectBundle{} - // syncer - name := "status.subscription_status" - syncInterval := statusconfig.GetPolicyDuration - - return generic.LaunchGenericObjectSyncer( - name, + return generic.LaunchMultiEventSyncer( + "status.subscription_status", mgr, generic.NewGenericController(instance, predicate), producer, - syncInterval, - []generic.ObjectEmitter{ - emitter, + configmap.GetPolicyDuration, + []*generic.EmitterHandler{ + { + Handler: generic.NewGenericHandler(&eventData), + Emitter: generic.NewGenericEmitter(enum.SubscriptionStatusType), + }, }) } diff --git a/agent/pkg/status/controller/config/config_controller.go b/agent/pkg/status/syncers/configmap/config_controller.go similarity index 92% rename from agent/pkg/status/controller/config/config_controller.go rename to agent/pkg/status/syncers/configmap/config_controller.go index 392833490..74bed2ea4 100644 --- a/agent/pkg/status/controller/config/config_controller.go +++ b/agent/pkg/status/syncers/configmap/config_controller.go @@ -1,4 +1,4 @@ -package config +package configmap import ( "context" @@ -13,7 +13,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/predicate" - "github.com/stolostron/multicluster-global-hub/agent/pkg/config" + "github.com/stolostron/multicluster-global-hub/agent/pkg/configs" "github.com/stolostron/multicluster-global-hub/pkg/constants" ) @@ -26,13 +26,12 @@ type hubOfHubsConfigController struct { log logr.Logger } -// AddConfigController creates a new instance of config controller and adds it to the manager. -func AddConfigController(mgr ctrl.Manager, agentConfig *config.AgentConfig) error { +// AddConfigMapController creates a new instance of config controller and adds it to the manager. +func AddConfigMapController(mgr ctrl.Manager, agentConfig *configs.AgentConfig) error { hubOfHubsConfigCtrl := &hubOfHubsConfigController{ client: mgr.GetClient(), log: ctrl.Log.WithName("multicluster-global-hub-agent-config"), } - leafHubName = agentConfig.LeafHubName configMapPredicate := predicate.NewPredicateFuncs(func(object client.Object) bool { return object.GetNamespace() == constants.GHAgentNamespace && diff --git a/agent/pkg/status/controller/config/config_data.go b/agent/pkg/status/syncers/configmap/config_data.go similarity index 93% rename from agent/pkg/status/controller/config/config_data.go rename to agent/pkg/status/syncers/configmap/config_data.go index 665150464..738a95295 100644 --- a/agent/pkg/status/controller/config/config_data.go +++ b/agent/pkg/status/syncers/configmap/config_data.go @@ -1,11 +1,10 @@ -package config +package configmap import ( "time" ) var ( - leafHubName = "leaf-hub" syncIntervals = map[AgentConfigKey]time.Duration{ ManagedClusterIntervalKey: 5 * time.Second, PolicyIntervalKey: 5 * time.Second, @@ -67,14 +66,6 @@ func GetEventDuration() time.Duration { return syncIntervals[EventIntervalKey] } -func GetLeafHubName() string { - return leafHubName -} - -func SetLeafHubName(name string) { - leafHubName = name -} - func GetAggregationLevel() AgentConfigValue { return agentConfigs[AgentAggregationKey] } diff --git a/agent/pkg/status/syncers/events/event_syncer.go b/agent/pkg/status/syncers/events/event_syncer.go new file mode 100644 index 000000000..f0eecec80 --- /dev/null +++ b/agent/pkg/status/syncers/events/event_syncer.go @@ -0,0 +1,52 @@ +package events + +import ( + "context" + + corev1 "k8s.io/api/core/v1" + policiesv1 "open-cluster-management.io/governance-policy-propagator/api/v1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/predicate" + + "github.com/stolostron/multicluster-global-hub/agent/pkg/configs" + "github.com/stolostron/multicluster-global-hub/agent/pkg/status/generic" + "github.com/stolostron/multicluster-global-hub/agent/pkg/status/syncers/configmap" + "github.com/stolostron/multicluster-global-hub/agent/pkg/status/syncers/events/handlers" + "github.com/stolostron/multicluster-global-hub/pkg/constants" + "github.com/stolostron/multicluster-global-hub/pkg/enum" + "github.com/stolostron/multicluster-global-hub/pkg/transport" +) + +func LaunchEventSyncer(ctx context.Context, mgr ctrl.Manager, + agentConfig *configs.AgentConfig, producer transport.Producer, +) error { + // controller + instance := func() client.Object { return &corev1.Event{} } + eventPredicate := predicate.NewPredicateFuncs(func(obj client.Object) bool { + event, ok := obj.(*corev1.Event) + if !ok { + return false + } + // only sync the policy event || extend other InvolvedObject kind + return event.InvolvedObject.Kind == policiesv1.Kind || + event.InvolvedObject.Kind == constants.ManagedClusterKind + }) + + return generic.LaunchMultiEventSyncer( + "status.event", + mgr, + generic.NewGenericController(instance, eventPredicate), + producer, + configmap.GetEventDuration, + []*generic.EmitterHandler{ + { + Handler: handlers.NewLocalRootPolicyEventHandler(ctx, mgr.GetClient()), + Emitter: handlers.NewRootPolicyEventEmitter(enum.LocalRootPolicyEventType), + }, + { + Handler: handlers.NewManagedClusterEventHandler(ctx, mgr.GetClient()), + Emitter: handlers.NewManagedClusterEventEmitter(), + }, + }) +} diff --git a/agent/pkg/status/controller/event/local_replicated_policy_emitter.go b/agent/pkg/status/syncers/events/handlers/local_replicated_policy_handler.go similarity index 51% rename from agent/pkg/status/controller/event/local_replicated_policy_emitter.go rename to agent/pkg/status/syncers/events/handlers/local_replicated_policy_handler.go index 78c9dbc66..8d6d24783 100644 --- a/agent/pkg/status/controller/event/local_replicated_policy_emitter.go +++ b/agent/pkg/status/syncers/events/handlers/local_replicated_policy_handler.go @@ -1,30 +1,25 @@ -package event +package handlers import ( "context" - "fmt" "strings" - cloudevents "github.com/cloudevents/sdk-go/v2" - "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" - ctrl "sigs.k8s.io/controller-runtime" + "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/client" - "github.com/stolostron/multicluster-global-hub/agent/pkg/status/controller/config" - "github.com/stolostron/multicluster-global-hub/agent/pkg/status/controller/filter" - "github.com/stolostron/multicluster-global-hub/agent/pkg/status/controller/generic" - "github.com/stolostron/multicluster-global-hub/agent/pkg/status/controller/policies" + "github.com/stolostron/multicluster-global-hub/agent/pkg/status/filter" + "github.com/stolostron/multicluster-global-hub/agent/pkg/status/generic" + "github.com/stolostron/multicluster-global-hub/agent/pkg/status/interfaces" + "github.com/stolostron/multicluster-global-hub/agent/pkg/status/syncers/configmap" + policyhandler "github.com/stolostron/multicluster-global-hub/agent/pkg/status/syncers/policies/handlers" "github.com/stolostron/multicluster-global-hub/pkg/bundle/event" - "github.com/stolostron/multicluster-global-hub/pkg/bundle/version" "github.com/stolostron/multicluster-global-hub/pkg/constants" "github.com/stolostron/multicluster-global-hub/pkg/enum" "github.com/stolostron/multicluster-global-hub/pkg/utils" ) -var _ generic.ObjectEmitter = &localReplicatedPolicyEmitter{} - -// TODO: the current replicated policy event will also emit such message, +// the current replicated policy event will also emit such message, // it has contain concrete reason why the state of the compliance change to another. // I will disable the replicated policy event until it contain some valuable message. // disable it by setting the emit() return false @@ -59,60 +54,65 @@ var _ generic.ObjectEmitter = &localReplicatedPolicyEmitter{} // "kafkaoffset": "13" // } -type localReplicatedPolicyEmitter struct { - ctx context.Context - name string - log logr.Logger - eventType string - runtimeClient client.Client - currentVersion *version.Version - lastSentVersion version.Version - payload event.ReplicatedPolicyEventBundle - topic string +func NewReplicatedPolicyEventEmitter(eventType enum.EventType) interfaces.Emitter { + name := strings.Replace(string(eventType), enum.EventTypePrefix, "", -1) + return generic.NewGenericEmitter(eventType, generic.WithPostSend( + // After sending the event, update the filter cache and clear the bundle from the handler cache. + func(data interface{}) { + events, ok := data.(*event.ReplicatedPolicyEventBundle) + if !ok { + return + } + // update the time filter: with latest event + for _, evt := range *events { + filter.CacheTime(name, evt.CreatedAt.Time) + } + // reset the payload + *events = (*events)[:0] + }), + ) +} + +type localReplicatedPolicyEventHandler struct { + ctx context.Context + name string + eventType string + runtimeClient client.Client + payload *event.ReplicatedPolicyEventBundle } -func NewLocalReplicatedPolicyEmitter(ctx context.Context, runtimeClient client.Client, - topic string, -) generic.ObjectEmitter { +func NewLocalReplicatedPolicyEventHandler(ctx context.Context, c client.Client) interfaces.Handler { name := strings.Replace(string(enum.LocalReplicatedPolicyEventType), enum.EventTypePrefix, "", -1) filter.RegisterTimeFilter(name) - return &localReplicatedPolicyEmitter{ - ctx: ctx, - name: name, - log: ctrl.Log.WithName(name), - eventType: string(enum.LocalReplicatedPolicyEventType), - topic: topic, - runtimeClient: runtimeClient, - currentVersion: version.NewVersion(), - lastSentVersion: *version.NewVersion(), - payload: make([]event.ReplicatedPolicyEvent, 0), + return &localReplicatedPolicyEventHandler{ + ctx: ctx, + name: name, + eventType: string(enum.LocalReplicatedPolicyEventType), + runtimeClient: c, + payload: &event.ReplicatedPolicyEventBundle{}, } } -func (h *localReplicatedPolicyEmitter) PostUpdate() { - h.currentVersion.Incr() +func (h *localReplicatedPolicyEventHandler) Get() interface{} { + return h.payload } // enable local policy and is replicated policy -func (h *localReplicatedPolicyEmitter) ShouldUpdate(obj client.Object) bool { - if config.GetEnableLocalPolicy() != config.EnableLocalPolicyTrue { +func (h *localReplicatedPolicyEventHandler) shouldUpdate(obj client.Object) bool { + if configmap.GetEnableLocalPolicy() != configmap.EnableLocalPolicyTrue { return false } - policy, ok := policyEventPredicate(h.ctx, h.name, obj, h.runtimeClient, h.log) + policy, ok := policyEventPredicate(h.ctx, h.name, obj, h.runtimeClient) return ok && !utils.HasAnnotation(policy, constants.OriginOwnerReferenceAnnotation) && utils.HasItemKey(policy.GetLabels(), constants.PolicyEventRootPolicyNameLabelKey) } -func (h *localReplicatedPolicyEmitter) ShouldSend() bool { - return h.currentVersion.NewerThan(&h.lastSentVersion) -} - -func (h *localReplicatedPolicyEmitter) Topic() string { - return h.topic -} +func (h *localReplicatedPolicyEventHandler) Update(obj client.Object) bool { + if !h.shouldUpdate(obj) { + return false + } -func (h *localReplicatedPolicyEmitter) Update(obj client.Object) bool { evt, ok := obj.(*corev1.Event) if !ok { return false @@ -121,14 +121,14 @@ func (h *localReplicatedPolicyEmitter) Update(obj client.Object) bool { // get policy policy, err := getInvolvePolicy(h.ctx, h.runtimeClient, evt) if err != nil { - h.log.Error(err, "failed to get involved policy", "event", evt.Namespace+"/"+evt.Name, - "policy", evt.InvolvedObject.Namespace+"/"+evt.InvolvedObject.Name) + klog.Errorf("failed to get involved policy event: %s/%s, involved: %s/%s, error: %v", evt.Namespace, evt.Name, + evt.InvolvedObject.Namespace, evt.InvolvedObject.Name, err) return false } - rootPolicy, clusterID, clusterName, err := policies.GetRootPolicyAndClusterInfo(h.ctx, policy, h.runtimeClient) + rootPolicy, clusterID, clusterName, err := policyhandler.GetRootPolicyAndClusterInfo(h.ctx, policy, h.runtimeClient) if err != nil { - h.log.Error(err, "failed to get get rootPolicy/clusterID/clusterName from the replicatedPolicy") + klog.Errorf("failed to get rootPolicy/clusterID/clusterName from the replicatedPolicy: %v", err) return false } // update @@ -148,34 +148,11 @@ func (h *localReplicatedPolicyEmitter) Update(obj client.Object) bool { Compliance: policyCompliance(rootPolicy, evt), } // cache to events and update version - h.payload = append(h.payload, replicatedPolicyEvent) + *h.payload = append(*h.payload, replicatedPolicyEvent) return true } -func (*localReplicatedPolicyEmitter) Delete(client.Object) bool { +func (*localReplicatedPolicyEventHandler) Delete(client.Object) bool { // do nothing return false } - -func (h *localReplicatedPolicyEmitter) ToCloudEvent() (*cloudevents.Event, error) { - if len(h.payload) < 1 { - return nil, fmt.Errorf("the cloudevent instance shouldn't be nil") - } - e := cloudevents.NewEvent() - e.SetType(h.eventType) - e.SetSource(config.GetLeafHubName()) - e.SetExtension(version.ExtVersion, h.currentVersion.String()) - err := e.SetData(cloudevents.ApplicationJSON, h.payload) - return &e, err -} - -func (h *localReplicatedPolicyEmitter) PostSend() { - // update the time filter: with latest event - for _, evt := range h.payload { - filter.CacheTime(h.name, evt.CreatedAt.Time) - } - // update version and clean the cache - h.payload = make([]event.ReplicatedPolicyEvent, 0) - h.currentVersion.Next() - h.lastSentVersion = *h.currentVersion -} diff --git a/agent/pkg/status/controller/event/local_root_policy_emitter.go b/agent/pkg/status/syncers/events/handlers/local_root_policy_handler.go similarity index 56% rename from agent/pkg/status/controller/event/local_root_policy_emitter.go rename to agent/pkg/status/syncers/events/handlers/local_root_policy_handler.go index 49a9f2cdd..a3c2bf100 100644 --- a/agent/pkg/status/controller/event/local_root_policy_emitter.go +++ b/agent/pkg/status/syncers/events/handlers/local_root_policy_handler.go @@ -1,99 +1,83 @@ -package event +package handlers import ( "context" - "fmt" + "regexp" "strings" - cloudevents "github.com/cloudevents/sdk-go/v2" - "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/klog/v2" policiesv1 "open-cluster-management.io/governance-policy-propagator/api/v1" - ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - "github.com/stolostron/multicluster-global-hub/agent/pkg/status/controller/config" - "github.com/stolostron/multicluster-global-hub/agent/pkg/status/controller/filter" - "github.com/stolostron/multicluster-global-hub/agent/pkg/status/controller/generic" + "github.com/stolostron/multicluster-global-hub/agent/pkg/status/filter" + "github.com/stolostron/multicluster-global-hub/agent/pkg/status/generic" + "github.com/stolostron/multicluster-global-hub/agent/pkg/status/interfaces" + "github.com/stolostron/multicluster-global-hub/agent/pkg/status/syncers/configmap" "github.com/stolostron/multicluster-global-hub/pkg/bundle/event" - "github.com/stolostron/multicluster-global-hub/pkg/bundle/version" "github.com/stolostron/multicluster-global-hub/pkg/constants" "github.com/stolostron/multicluster-global-hub/pkg/enum" "github.com/stolostron/multicluster-global-hub/pkg/utils" ) -var _ generic.ObjectEmitter = &localRootPolicyEmitter{} - -type localRootPolicyEmitter struct { - ctx context.Context - name string - log logr.Logger - runtimeClient client.Client - eventType string - topic string - currentVersion *version.Version - lastSentVersion version.Version - payload event.RootPolicyEventBundle +const ( + UnknownComplianceState = "Unknown" +) + +var PolicyMessageStatusRe = regexp. + MustCompile(`Policy (.+) status was updated to (.+) in cluster namespace (.+)`) + +func NewRootPolicyEventEmitter(eventType enum.EventType) interfaces.Emitter { + name := strings.Replace(string(eventType), enum.EventTypePrefix, "", -1) + return generic.NewGenericEmitter(eventType, generic.WithPostSend( + // After sending the event, update the filter cache and clear the bundle from the handler cache. + func(data interface{}) { + events, ok := data.(*event.RootPolicyEventBundle) + if !ok { + return + } + // update the time filter: with latest event + for _, evt := range *events { + filter.CacheTime(name, evt.CreatedAt.Time) + } + // reset the payload + *events = (*events)[:0] + }), + ) +} + +var _ interfaces.Handler = &localRootPolicyHandler{} + +type localRootPolicyHandler struct { + ctx context.Context + name string + runtimeClient client.Client + eventType string + payload *event.RootPolicyEventBundle } -func NewLocalRootPolicyEmitter(ctx context.Context, c client.Client, topic string) *localRootPolicyEmitter { +func NewLocalRootPolicyEventHandler(ctx context.Context, c client.Client) *localRootPolicyHandler { name := strings.Replace(string(enum.LocalRootPolicyEventType), enum.EventTypePrefix, "", -1) filter.RegisterTimeFilter(name) - return &localRootPolicyEmitter{ - ctx: ctx, - name: name, - log: ctrl.Log.WithName(name), - eventType: string(enum.LocalRootPolicyEventType), - topic: topic, - runtimeClient: c, - currentVersion: version.NewVersion(), - lastSentVersion: *version.NewVersion(), - payload: make([]event.RootPolicyEvent, 0), + return &localRootPolicyHandler{ + ctx: ctx, + name: name, + eventType: string(enum.LocalRootPolicyEventType), + runtimeClient: c, + payload: &event.RootPolicyEventBundle{}, } } -func (h *localRootPolicyEmitter) PostUpdate() { - h.currentVersion.Incr() +func (h *localRootPolicyHandler) Get() interface{} { + return h.payload } -func (h *localRootPolicyEmitter) ShouldUpdate(obj client.Object) bool { - if config.GetEnableLocalPolicy() != config.EnableLocalPolicyTrue { +func (h *localRootPolicyHandler) Update(obj client.Object) bool { + if !h.shouldUpdate(obj) { return false } - policy, ok := policyEventPredicate(h.ctx, h.name, obj, h.runtimeClient, h.log) - - return ok && !utils.HasAnnotation(policy, constants.OriginOwnerReferenceAnnotation) && - !utils.HasLabel(policy, constants.PolicyEventRootPolicyNameLabelKey) -} - -func policyEventPredicate(ctx context.Context, name string, obj client.Object, c client.Client, log logr.Logger) ( - *policiesv1.Policy, bool, -) { - evt, ok := obj.(*corev1.Event) - if !ok { - return nil, false - } - - if !filter.Newer(name, getEventLastTime(evt).Time) { - return nil, false - } - - if evt.InvolvedObject.Kind != policiesv1.Kind { - return nil, false - } - - // get policy - policy, err := getInvolvePolicy(ctx, c, evt) - if err != nil { - log.Info("failed to get involved policy", "event", evt.Namespace+"/"+evt.Name, "error", err.Error()) - return nil, false - } - return policy, true -} - -func (h *localRootPolicyEmitter) Update(obj client.Object) bool { evt, ok := obj.(*corev1.Event) if !ok { return false @@ -102,7 +86,7 @@ func (h *localRootPolicyEmitter) Update(obj client.Object) bool { // get policy policy, err := getInvolvePolicy(h.ctx, h.runtimeClient, evt) if err != nil { - h.log.Error(err, "failed to get involved policy", "event", evt.Namespace+"/"+evt.Name, + klog.Error(err, "failed to get involved policy", "event", evt.Namespace+"/"+evt.Name, "policy", evt.InvolvedObject.Namespace+"/"+evt.InvolvedObject.Name) return false } @@ -121,47 +105,49 @@ func (h *localRootPolicyEmitter) Update(obj client.Object) bool { PolicyID: string(policy.GetUID()), Compliance: policyCompliance(policy, evt), } - h.payload = append(h.payload, rootPolicyEvent) + *h.payload = append(*h.payload, rootPolicyEvent) return true } -func (*localRootPolicyEmitter) Delete(client.Object) bool { +func (*localRootPolicyHandler) Delete(client.Object) bool { // do nothing return false } -func (h *localRootPolicyEmitter) ToCloudEvent() (*cloudevents.Event, error) { - if len(h.payload) < 1 { - return nil, fmt.Errorf("the cloudevent instance shouldn't be nil") +func (h *localRootPolicyHandler) shouldUpdate(obj client.Object) bool { + if configmap.GetEnableLocalPolicy() != configmap.EnableLocalPolicyTrue { + return false } - e := cloudevents.NewEvent() - e.SetType(h.eventType) - e.SetSource(config.GetLeafHubName()) - e.SetExtension(version.ExtVersion, h.currentVersion.String()) - err := e.SetData(cloudevents.ApplicationJSON, h.payload) - return &e, err -} -// to assert whether emit the current cloudevent -func (h *localRootPolicyEmitter) ShouldSend() bool { - return h.currentVersion.NewerThan(&h.lastSentVersion) -} + policy, ok := policyEventPredicate(h.ctx, h.name, obj, h.runtimeClient) -func (h *localRootPolicyEmitter) Topic() string { - return h.topic + return ok && !utils.HasAnnotation(policy, constants.OriginOwnerReferenceAnnotation) && + !utils.HasLabel(policy, constants.PolicyEventRootPolicyNameLabelKey) } -func (h *localRootPolicyEmitter) PostSend() { - // update the time filter: with latest event - for _, evt := range h.payload { - filter.CacheTime(h.name, evt.CreatedAt.Time) - } - // update version and clean the cache - h.payload = make([]event.RootPolicyEvent, 0) - // 1. the version get into the next generation - // 2. set the lastSenteVersion to current version - h.currentVersion.Next() - h.lastSentVersion = *h.currentVersion +func policyEventPredicate(ctx context.Context, name string, obj client.Object, c client.Client) ( + *policiesv1.Policy, bool, +) { + evt, ok := obj.(*corev1.Event) + if !ok { + return nil, false + } + + if !filter.Newer(name, getEventLastTime(evt).Time) { + return nil, false + } + + if evt.InvolvedObject.Kind != policiesv1.Kind { + return nil, false + } + + // get policy + policy, err := getInvolvePolicy(ctx, c, evt) + if err != nil { + klog.Errorf("failed to get involved policy event: %s/%s, error: %v", evt.Namespace, evt.Name, err) + return nil, false + } + return policy, true } func policyCompliance(policy *policiesv1.Policy, evt *corev1.Event) string { diff --git a/agent/pkg/status/syncers/events/handlers/managedcluster_handler.go b/agent/pkg/status/syncers/events/handlers/managedcluster_handler.go new file mode 100644 index 000000000..ea3468669 --- /dev/null +++ b/agent/pkg/status/syncers/events/handlers/managedcluster_handler.go @@ -0,0 +1,140 @@ +package handlers + +import ( + "context" + "strings" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/klog/v2" + clusterv1 "open-cluster-management.io/api/cluster/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/stolostron/multicluster-global-hub/agent/pkg/configs" + "github.com/stolostron/multicluster-global-hub/agent/pkg/status/filter" + "github.com/stolostron/multicluster-global-hub/agent/pkg/status/generic" + "github.com/stolostron/multicluster-global-hub/agent/pkg/status/interfaces" + "github.com/stolostron/multicluster-global-hub/pkg/bundle/event" + "github.com/stolostron/multicluster-global-hub/pkg/constants" + "github.com/stolostron/multicluster-global-hub/pkg/database/models" + "github.com/stolostron/multicluster-global-hub/pkg/enum" + "github.com/stolostron/multicluster-global-hub/pkg/utils" +) + +func NewManagedClusterEventEmitter() interfaces.Emitter { + name := strings.Replace(string(enum.ManagedClusterEventType), enum.EventTypePrefix, "", -1) + return generic.NewGenericEmitter(enum.ManagedClusterEventType, generic.WithPostSend( + // After sending the event, update the filter cache and clear the bundle from the handler cache. + func(data interface{}) { + events, ok := data.(*event.ManagedClusterEventBundle) + if !ok { + return + } + // update the time filter: with latest event + for _, evt := range *events { + filter.CacheTime(name, evt.CreatedAt) + } + // reset the payload + *events = (*events)[:0] + }), + ) +} + +type managedClusterEventHandler struct { + ctx context.Context + name string + runtimeClient client.Client + eventType string + payload *event.ManagedClusterEventBundle +} + +func NewManagedClusterEventHandler(ctx context.Context, c client.Client) *managedClusterEventHandler { + name := strings.Replace(string(enum.ManagedClusterEventType), enum.EventTypePrefix, "", -1) + filter.RegisterTimeFilter(name) + return &managedClusterEventHandler{ + ctx: ctx, + name: name, + eventType: string(enum.ManagedClusterEventType), + runtimeClient: c, + payload: &event.ManagedClusterEventBundle{}, + } +} + +func (h *managedClusterEventHandler) Get() interface{} { + return h.payload +} + +func (h *managedClusterEventHandler) shouldUpdate(obj client.Object) bool { + evt, ok := obj.(*corev1.Event) + if !ok { + return false + } + + if evt.InvolvedObject.Kind != constants.ManagedClusterKind { + return false + } + + // if it's a older event, then return false + if !filter.Newer(h.name, getEventLastTime(evt).Time) { + return false + } + + return true +} + +func (h *managedClusterEventHandler) Update(obj client.Object) bool { + if !h.shouldUpdate(obj) { + return false + } + + evt, ok := obj.(*corev1.Event) + if !ok { + return false + } + + cluster, err := getInvolveCluster(h.ctx, h.runtimeClient, evt) + if err != nil { + + klog.Errorf("failed to get cluster: %s, event: %s/%s, error: %v", cluster.Name, evt.Namespace, evt.Name, err) + return false + } + + clusterId, err := utils.GetClusterId(h.ctx, h.runtimeClient, cluster.Name) + if err != nil { + klog.Errorf("failed to get clusterId: %s, event: %s/%s, error: %v", cluster.Name, evt.Namespace, evt.Name, err) + return false + } + + clusterEvent := models.ManagedClusterEvent{ + EventName: evt.Name, + EventNamespace: evt.Namespace, + Message: evt.Message, + Reason: evt.Reason, + ClusterName: cluster.Name, + ClusterID: clusterId, + LeafHubName: configs.GetLeafHubName(), + ReportingController: evt.ReportingController, + ReportingInstance: evt.ReportingInstance, + EventType: evt.Type, + CreatedAt: getEventLastTime(evt).Time, + } + + *h.payload = append(*h.payload, clusterEvent) + return true +} + +func (*managedClusterEventHandler) Delete(client.Object) bool { + // do nothing + return false +} + +func getInvolveCluster(ctx context.Context, c client.Client, evt *corev1.Event) (*clusterv1.ManagedCluster, error) { + cluster := &clusterv1.ManagedCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: evt.InvolvedObject.Name, + Namespace: evt.InvolvedObject.Namespace, + }, + } + err := c.Get(ctx, client.ObjectKeyFromObject(cluster), cluster) + return cluster, err +} diff --git a/agent/pkg/status/syncers/managedcluster/managed_cluster_syncer.go b/agent/pkg/status/syncers/managedcluster/managed_cluster_syncer.go new file mode 100644 index 000000000..d1c942f01 --- /dev/null +++ b/agent/pkg/status/syncers/managedcluster/managed_cluster_syncer.go @@ -0,0 +1,48 @@ +package managedcluster + +import ( + "context" + + clusterv1 "open-cluster-management.io/api/cluster/v1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/predicate" + + "github.com/stolostron/multicluster-global-hub/agent/pkg/configs" + "github.com/stolostron/multicluster-global-hub/agent/pkg/status/generic" + "github.com/stolostron/multicluster-global-hub/agent/pkg/status/syncers/configmap" + genericpayload "github.com/stolostron/multicluster-global-hub/pkg/bundle/generic" + "github.com/stolostron/multicluster-global-hub/pkg/constants" + "github.com/stolostron/multicluster-global-hub/pkg/enum" + "github.com/stolostron/multicluster-global-hub/pkg/transport" + "github.com/stolostron/multicluster-global-hub/pkg/utils" +) + +func LaunchManagedClusterSyncer(ctx context.Context, mgr ctrl.Manager, agentConfig *configs.AgentConfig, + producer transport.Producer, +) error { + // emitter handler config + eventData := genericpayload.GenericObjectBundle{} + tweakFunc := func(object client.Object) { + utils.MergeAnnotations(object, map[string]string{ + constants.ManagedClusterManagedByAnnotation: configs.GetLeafHubName(), + }) + } + shouldUpdate := func(obj client.Object) bool { return !utils.HasAnnotation(obj, constants.ManagedClusterMigrating) } + + return generic.LaunchMultiEventSyncer( + "status.managed_cluster", + mgr, + generic.NewGenericController( + func() client.Object { return &clusterv1.ManagedCluster{} }, + predicate.NewPredicateFuncs(func(object client.Object) bool { return true })), + producer, + configmap.GetManagerClusterDuration, + []*generic.EmitterHandler{ + { + Handler: generic.NewGenericHandler(&eventData, generic.WithTweakFunc(tweakFunc), + generic.WithShouldUpdate(shouldUpdate)), + Emitter: generic.NewGenericEmitter(enum.ManagedClusterType), + }, + }) +} diff --git a/agent/pkg/status/syncers/managedhub/cluster_info_syncer.go b/agent/pkg/status/syncers/managedhub/cluster_info_syncer.go new file mode 100644 index 000000000..6dfac71ba --- /dev/null +++ b/agent/pkg/status/syncers/managedhub/cluster_info_syncer.go @@ -0,0 +1,130 @@ +package managedhub + +import ( + routev1 "github.com/openshift/api/route/v1" + clustersv1alpha1 "open-cluster-management.io/api/cluster/v1alpha1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/predicate" + + "github.com/stolostron/multicluster-global-hub/agent/pkg/status/generic" + "github.com/stolostron/multicluster-global-hub/agent/pkg/status/syncers/configmap" + "github.com/stolostron/multicluster-global-hub/pkg/bundle/cluster" + "github.com/stolostron/multicluster-global-hub/pkg/constants" + "github.com/stolostron/multicluster-global-hub/pkg/enum" + "github.com/stolostron/multicluster-global-hub/pkg/transport" +) + +func LaunchHubClusterInfoSyncer(mgr ctrl.Manager, producer transport.Producer) error { + eventData := &cluster.HubClusterInfo{} + return generic.LaunchMultiObjectSyncer( + "status.hub_cluster_info", + mgr, + []generic.ControllerHandler{ + { + Controller: generic.NewGenericController( + func() client.Object { return &clustersv1alpha1.ClusterClaim{} }, + predicate.NewPredicateFuncs(func(object client.Object) bool { + return object.GetName() == "id.k8s.io" + })), + Handler: &infoClusterClaimHandler{eventData}, + }, + { + Controller: generic.NewGenericController( + func() client.Object { return &routev1.Route{} }, + predicate.NewPredicateFuncs(func(object client.Object) bool { + if object.GetNamespace() == constants.OpenShiftConsoleNamespace && + object.GetName() == constants.OpenShiftConsoleRouteName { + return true + } + if object.GetNamespace() == constants.ObservabilityNamespace && + object.GetName() == constants.ObservabilityGrafanaRouteName { + return true + } + return false + })), + Handler: &infoRouteHandler{eventData}, + }, + }, + producer, + configmap.GetHubClusterInfoDuration, + generic.NewGenericEmitter(enum.HubClusterInfoType), + ) +} + +// 1. Use ClusterClaim to update the HubClusterInfo +type infoClusterClaimHandler struct { + evtData cluster.HubClusterInfoBundle +} + +func (p *infoClusterClaimHandler) Get() interface{} { + return p.evtData +} + +func (p *infoClusterClaimHandler) Update(obj client.Object) bool { + clusterClaim, ok := obj.(*clustersv1alpha1.ClusterClaim) + if !ok { + return false + } + + oldClusterID := p.evtData.ClusterId + + if clusterClaim.Name == "id.k8s.io" { + p.evtData.ClusterId = clusterClaim.Spec.Value + } + // If no ClusterId, do not send the bundle + if p.evtData.ClusterId == "" { + return false + } + + return oldClusterID != p.evtData.ClusterId +} + +func (p *infoClusterClaimHandler) Delete(obj client.Object) bool { + // do nothing + return false +} + +// 2. Use Route to update the HubClusterInfo +type infoRouteHandler struct { + evtData cluster.HubClusterInfoBundle +} + +func (p *infoRouteHandler) Get() interface{} { + return p.evtData +} + +func (p *infoRouteHandler) Update(obj client.Object) bool { + route, ok := obj.(*routev1.Route) + if !ok { + return false + } + + var newURL string + updated := false + if len(route.Spec.Host) != 0 { + newURL = "https://" + route.Spec.Host + } + if route.GetName() == constants.OpenShiftConsoleRouteName && p.evtData.ConsoleURL != newURL { + p.evtData.ConsoleURL = newURL + updated = true + } + if route.GetName() == constants.ObservabilityGrafanaRouteName && p.evtData.GrafanaURL != newURL { + p.evtData.GrafanaURL = newURL + updated = true + } + return updated +} + +func (p *infoRouteHandler) Delete(obj client.Object) bool { + updated := false + if obj.GetName() == constants.OpenShiftConsoleRouteName && p.evtData.ConsoleURL != "" { + p.evtData.ConsoleURL = "" + updated = true + } + if obj.GetName() == constants.ObservabilityGrafanaRouteName && p.evtData.GrafanaURL != "" { + p.evtData.GrafanaURL = "" + updated = true + } + return updated +} diff --git a/agent/pkg/status/controller/hubcluster/heartbeat_syncer.go b/agent/pkg/status/syncers/managedhub/heartbeat_syncer.go similarity index 75% rename from agent/pkg/status/controller/hubcluster/heartbeat_syncer.go rename to agent/pkg/status/syncers/managedhub/heartbeat_syncer.go index fa2ef1a4b..76750e99a 100644 --- a/agent/pkg/status/controller/hubcluster/heartbeat_syncer.go +++ b/agent/pkg/status/syncers/managedhub/heartbeat_syncer.go @@ -1,12 +1,13 @@ -package hubcluster +package managedhub import ( cloudevents "github.com/cloudevents/sdk-go/v2" ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "github.com/stolostron/multicluster-global-hub/agent/pkg/status/controller/config" - "github.com/stolostron/multicluster-global-hub/agent/pkg/status/controller/generic" + "github.com/stolostron/multicluster-global-hub/agent/pkg/configs" + "github.com/stolostron/multicluster-global-hub/agent/pkg/status/generic" + "github.com/stolostron/multicluster-global-hub/agent/pkg/status/interfaces" + "github.com/stolostron/multicluster-global-hub/agent/pkg/status/syncers/configmap" genericdata "github.com/stolostron/multicluster-global-hub/pkg/bundle/generic" eventversion "github.com/stolostron/multicluster-global-hub/pkg/bundle/version" "github.com/stolostron/multicluster-global-hub/pkg/enum" @@ -14,17 +15,17 @@ import ( ) func LaunchHubClusterHeartbeatSyncer(mgr ctrl.Manager, producer transport.Producer) error { - return generic.LaunchGenericEventSyncer( + return generic.LaunchMultiObjectSyncer( "status.hub_cluster_heartbeat", mgr, nil, producer, - config.GetHeartbeatDuration, + configmap.GetHeartbeatDuration, NewHeartbeatEmitter(), ) } -var _ generic.Emitter = &heartbeatEmitter{} +var _ interfaces.Emitter = &heartbeatEmitter{} func NewHeartbeatEmitter() *heartbeatEmitter { emitter := &heartbeatEmitter{ @@ -41,16 +42,13 @@ type heartbeatEmitter struct { lastSentVersion eventversion.Version } -// assert whether to update the payload by the current handler -func (s *heartbeatEmitter) ShouldUpdate(object client.Object) bool { return true } - func (s *heartbeatEmitter) PostUpdate() { s.currentVersion.Incr() } -func (s *heartbeatEmitter) ToCloudEvent() (*cloudevents.Event, error) { +func (s *heartbeatEmitter) ToCloudEvent(data interface{}) (*cloudevents.Event, error) { e := cloudevents.NewEvent() - e.SetSource(config.GetLeafHubName()) + e.SetSource(configs.GetLeafHubName()) e.SetType(string(s.eventType)) e.SetExtension(eventversion.ExtVersion, s.currentVersion.String()) err := e.SetData(cloudevents.ApplicationJSON, genericdata.GenericObjectBundle{}) @@ -59,7 +57,7 @@ func (s *heartbeatEmitter) ToCloudEvent() (*cloudevents.Event, error) { func (s *heartbeatEmitter) Topic() string { return "" } func (s *heartbeatEmitter) ShouldSend() bool { return true } -func (s *heartbeatEmitter) PostSend() { +func (s *heartbeatEmitter) PostSend(data interface{}) { s.currentVersion.Next() s.lastSentVersion = *s.currentVersion } diff --git a/agent/pkg/status/syncers/placement/placement_decision_syncer.go b/agent/pkg/status/syncers/placement/placement_decision_syncer.go new file mode 100644 index 000000000..1393131ed --- /dev/null +++ b/agent/pkg/status/syncers/placement/placement_decision_syncer.go @@ -0,0 +1,51 @@ +package placement + +import ( + "context" + + clustersv1beta1 "open-cluster-management.io/api/cluster/v1beta1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/predicate" + + "github.com/stolostron/multicluster-global-hub/agent/pkg/configs" + "github.com/stolostron/multicluster-global-hub/agent/pkg/status/generic" + "github.com/stolostron/multicluster-global-hub/agent/pkg/status/syncers/configmap" + genericpayload "github.com/stolostron/multicluster-global-hub/pkg/bundle/generic" + "github.com/stolostron/multicluster-global-hub/pkg/constants" + "github.com/stolostron/multicluster-global-hub/pkg/enum" + "github.com/stolostron/multicluster-global-hub/pkg/transport" + "github.com/stolostron/multicluster-global-hub/pkg/utils" +) + +func LaunchPlacementDecisionSyncer(ctx context.Context, mgr ctrl.Manager, agentConfig *configs.AgentConfig, + producer transport.Producer, +) error { + return generic.LaunchMultiEventSyncer( + "status.placement_decision", + mgr, + generic.NewGenericController( + func() client.Object { return &clustersv1beta1.PlacementDecision{} }, // instance + predicate.NewPredicateFuncs(func(object client.Object) bool { return true }), // predicate + ), + producer, + configmap.GetPolicyDuration, + []*generic.EmitterHandler{ + { + Handler: generic.NewGenericHandler( + &genericpayload.GenericObjectBundle{}, + generic.WithTweakFunc(cleanupManagedFields), + generic.WithShouldUpdate(globalResource), + ), + Emitter: generic.NewGenericEmitter(enum.PlacementDecisionType), + }, + }) +} + +func cleanupManagedFields(obj client.Object) { + obj.SetManagedFields(nil) +} + +func globalResource(obj client.Object) bool { + return utils.HasAnnotation(obj, constants.OriginOwnerReferenceAnnotation) +} diff --git a/agent/pkg/status/syncers/placement/placement_rule_syncer.go b/agent/pkg/status/syncers/placement/placement_rule_syncer.go new file mode 100644 index 000000000..9b33da108 --- /dev/null +++ b/agent/pkg/status/syncers/placement/placement_rule_syncer.go @@ -0,0 +1,58 @@ +package placement + +import ( + "context" + + placementrulesv1 "open-cluster-management.io/multicloud-operators-subscription/pkg/apis/apps/placementrule/v1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/predicate" + + "github.com/stolostron/multicluster-global-hub/agent/pkg/configs" + "github.com/stolostron/multicluster-global-hub/agent/pkg/status/generic" + "github.com/stolostron/multicluster-global-hub/agent/pkg/status/syncers/configmap" + genericpayload "github.com/stolostron/multicluster-global-hub/pkg/bundle/generic" + "github.com/stolostron/multicluster-global-hub/pkg/enum" + "github.com/stolostron/multicluster-global-hub/pkg/transport" +) + +func LaunchPlacementRuleSyncer(ctx context.Context, mgr ctrl.Manager, agentConfig *configs.AgentConfig, + producer transport.Producer, +) error { + // controller config + instance := func() client.Object { return &placementrulesv1.PlacementRule{} } + predicate := predicate.NewPredicateFuncs(func(object client.Object) bool { return true }) + + // local placementrule + localShouldUpdate := func(obj client.Object) bool { + // return statusconfig.GetEnableLocalPolicy() == statusconfig.EnableLocalPolicyTrue && + // !utils.HasAnnotation(obj, constants.OriginOwnerReferenceAnnotation) // local resource + return false // disable the placementrule now + } + + // syncer + name := "status.placement_rule" + syncInterval := configmap.GetPolicyDuration + + return generic.LaunchMultiEventSyncer( + name, + mgr, + generic.NewGenericController(instance, predicate), + producer, + syncInterval, + []*generic.EmitterHandler{ + { + Handler: generic.NewGenericHandler(&genericpayload.GenericObjectBundle{}, + generic.WithTweakFunc(cleanupManagedFields), + generic.WithShouldUpdate(globalResource)), + Emitter: generic.NewGenericEmitter(enum.PlacementRuleSpecType), + }, + + { + Handler: generic.NewGenericHandler(&genericpayload.GenericObjectBundle{}, + generic.WithTweakFunc(cleanupManagedFields), + generic.WithShouldUpdate(localShouldUpdate)), + Emitter: generic.NewGenericEmitter(enum.LocalPlacementRuleSpecType), + }, + }) +} diff --git a/agent/pkg/status/syncers/placement/placement_syncer.go b/agent/pkg/status/syncers/placement/placement_syncer.go new file mode 100644 index 000000000..0565ecf32 --- /dev/null +++ b/agent/pkg/status/syncers/placement/placement_syncer.go @@ -0,0 +1,38 @@ +package placement + +import ( + "context" + + clustersv1beta1 "open-cluster-management.io/api/cluster/v1beta1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/predicate" + + "github.com/stolostron/multicluster-global-hub/agent/pkg/configs" + "github.com/stolostron/multicluster-global-hub/agent/pkg/status/generic" + "github.com/stolostron/multicluster-global-hub/agent/pkg/status/syncers/configmap" + genericpayload "github.com/stolostron/multicluster-global-hub/pkg/bundle/generic" + "github.com/stolostron/multicluster-global-hub/pkg/enum" + "github.com/stolostron/multicluster-global-hub/pkg/transport" +) + +func LaunchPlacementSyncer(ctx context.Context, mgr ctrl.Manager, agentConfig *configs.AgentConfig, + producer transport.Producer, +) error { + return generic.LaunchMultiEventSyncer( + "status.placement", + mgr, + generic.NewGenericController( + func() client.Object { return &clustersv1beta1.Placement{} }, // instance + predicate.NewPredicateFuncs(func(object client.Object) bool { return true })), // predicate + producer, + configmap.GetPolicyDuration, + []*generic.EmitterHandler{ + { + Handler: generic.NewGenericHandler(&genericpayload.GenericObjectBundle{}, + generic.WithTweakFunc(cleanupManagedFields), + generic.WithShouldUpdate(globalResource)), + Emitter: generic.NewGenericEmitter(enum.PlacementSpecType), + }, + }) +} diff --git a/agent/pkg/status/controller/policies/complete_compliance_emitter.go b/agent/pkg/status/syncers/policies/handlers/complete_compliance_handler.go similarity index 87% rename from agent/pkg/status/controller/policies/complete_compliance_emitter.go rename to agent/pkg/status/syncers/policies/handlers/complete_compliance_handler.go index 94cd6d1f4..5e252c18c 100644 --- a/agent/pkg/status/controller/policies/complete_compliance_emitter.go +++ b/agent/pkg/status/syncers/policies/handlers/complete_compliance_handler.go @@ -1,4 +1,4 @@ -package policies +package handlers import ( "fmt" @@ -6,43 +6,37 @@ import ( policiesv1 "open-cluster-management.io/governance-policy-propagator/api/v1" "sigs.k8s.io/controller-runtime/pkg/client" - "github.com/stolostron/multicluster-global-hub/agent/pkg/status/controller/generic" + "github.com/stolostron/multicluster-global-hub/agent/pkg/status/interfaces" "github.com/stolostron/multicluster-global-hub/pkg/bundle/grc" - "github.com/stolostron/multicluster-global-hub/pkg/bundle/version" - "github.com/stolostron/multicluster-global-hub/pkg/enum" "github.com/stolostron/multicluster-global-hub/pkg/utils" ) -func CompleteComplianceEmitterWrapper( - eventType enum.EventType, - dependencyVersion *version.Version, - predicate func(client.Object) bool, -) generic.ObjectEmitter { - eventData := grc.CompleteComplianceBundle{} - return generic.NewGenericObjectEmitter( - eventType, - &eventData, - NewCompleteComplianceHandler(&eventData), - generic.WithShouldUpdate(predicate), - generic.WithDependencyVersion(dependencyVersion), - ) -} - type completeComplianceHandler struct { - eventData *grc.CompleteComplianceBundle + eventData *grc.CompleteComplianceBundle + shouldUpdate func(client.Object) bool } -func NewCompleteComplianceHandler(evtData *grc.CompleteComplianceBundle) generic.Handler { +func NewCompleteComplianceHandler(evtData *grc.CompleteComplianceBundle, + shouldUpdate func(client.Object) bool, +) interfaces.Handler { return &completeComplianceHandler{ - eventData: evtData, + eventData: evtData, + shouldUpdate: shouldUpdate, } } +func (h *completeComplianceHandler) Get() interface{} { + return h.eventData +} + func (h *completeComplianceHandler) Update(obj client.Object) bool { policy, isPolicy := obj.(*policiesv1.Policy) if !isPolicy { return false // do not handle objects other than policy } + if !h.shouldUpdate(obj) { + return false + } if policy.Status.Status == nil { return false } @@ -89,6 +83,10 @@ func (h *completeComplianceHandler) Delete(obj client.Object) bool { return false // don't handle objects other than policy } + if !h.shouldUpdate(obj) { + return false + } + index := getPayloadIndexByObj(obj, *(h.eventData)) if index == -1 { // trying to delete object which doesn't exist return false diff --git a/agent/pkg/status/controller/policies/compliance_emitter.go b/agent/pkg/status/syncers/policies/handlers/compliance_handler.go similarity index 90% rename from agent/pkg/status/controller/policies/compliance_emitter.go rename to agent/pkg/status/syncers/policies/handlers/compliance_handler.go index 8da55304a..d366d6029 100644 --- a/agent/pkg/status/controller/policies/compliance_emitter.go +++ b/agent/pkg/status/syncers/policies/handlers/compliance_handler.go @@ -1,4 +1,4 @@ -package policies +package handlers import ( "fmt" @@ -6,40 +6,33 @@ import ( policiesv1 "open-cluster-management.io/governance-policy-propagator/api/v1" "sigs.k8s.io/controller-runtime/pkg/client" - "github.com/stolostron/multicluster-global-hub/agent/pkg/status/controller/generic" + "github.com/stolostron/multicluster-global-hub/agent/pkg/status/interfaces" "github.com/stolostron/multicluster-global-hub/pkg/bundle/grc" - eventversion "github.com/stolostron/multicluster-global-hub/pkg/bundle/version" "github.com/stolostron/multicluster-global-hub/pkg/constants" - "github.com/stolostron/multicluster-global-hub/pkg/enum" "github.com/stolostron/multicluster-global-hub/pkg/utils" ) -func ComplianceEmitterWrapper( - eventType enum.EventType, - version *eventversion.Version, - predicate func(client.Object) bool, -) generic.ObjectEmitter { - eventData := grc.ComplianceBundle{} - return generic.NewGenericObjectEmitter( - eventType, - &eventData, - NewComplianceHandler(&eventData), - generic.WithShouldUpdate(predicate), - generic.WithVersion(version), - ) -} - type complianceHandler struct { - eventData *grc.ComplianceBundle + eventData *grc.ComplianceBundle + shouldUpdate func(client.Object) bool } -func NewComplianceHandler(eventData *grc.ComplianceBundle) generic.Handler { +func NewComplianceHandler(eventData *grc.ComplianceBundle, shouldUpdate func(client.Object) bool) interfaces.Handler { return &complianceHandler{ - eventData: eventData, + eventData: eventData, + shouldUpdate: shouldUpdate, } } +func (h *complianceHandler) Get() interface{} { + return h.eventData +} + func (h *complianceHandler) Update(obj client.Object) bool { + if !h.shouldUpdate(obj) { + return false + } + policy, isPolicy := obj.(*policiesv1.Policy) if !isPolicy { return false // do not handle objects other than policy @@ -97,6 +90,10 @@ func (h *complianceHandler) updatePayloadIfChanged(objectIndex int, policy *poli } func (h *complianceHandler) Delete(obj client.Object) bool { + if !h.shouldUpdate(obj) { + return false + } + index := getCompliancesIndexByObj(obj, *h.eventData) if index == -1 { // trying to delete object which doesn't exist return false diff --git a/agent/pkg/status/controller/policies/status_event_emitter.go b/agent/pkg/status/syncers/policies/handlers/status_event_handler.go similarity index 54% rename from agent/pkg/status/controller/policies/status_event_emitter.go rename to agent/pkg/status/syncers/policies/handlers/status_event_handler.go index 4a7011127..6d4901593 100644 --- a/agent/pkg/status/controller/policies/status_event_emitter.go +++ b/agent/pkg/status/syncers/policies/handlers/status_event_handler.go @@ -1,4 +1,4 @@ -package policies +package handlers import ( "context" @@ -6,94 +6,73 @@ import ( "regexp" "strings" - cloudevents "github.com/cloudevents/sdk-go/v2" - "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" "k8s.io/klog" policiesv1 "open-cluster-management.io/governance-policy-propagator/api/v1" - ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - "github.com/stolostron/multicluster-global-hub/agent/pkg/status/controller/config" - "github.com/stolostron/multicluster-global-hub/agent/pkg/status/controller/filter" - "github.com/stolostron/multicluster-global-hub/agent/pkg/status/controller/generic" + "github.com/stolostron/multicluster-global-hub/agent/pkg/status/filter" + "github.com/stolostron/multicluster-global-hub/agent/pkg/status/generic" + "github.com/stolostron/multicluster-global-hub/agent/pkg/status/interfaces" "github.com/stolostron/multicluster-global-hub/pkg/bundle/event" - eventversion "github.com/stolostron/multicluster-global-hub/pkg/bundle/version" "github.com/stolostron/multicluster-global-hub/pkg/constants" "github.com/stolostron/multicluster-global-hub/pkg/enum" "github.com/stolostron/multicluster-global-hub/pkg/utils" ) -var ( - _ generic.ObjectEmitter = &statusEventEmitter{} - MessageCompliaceStateRegex = regexp.MustCompile(`(\w+);`) -) +var MessageCompliaceStateRegex = regexp.MustCompile(`(\w+);`) -type statusEventEmitter struct { - ctx context.Context - name string - log logr.Logger - eventType string - runtimeClient client.Client - currentVersion *eventversion.Version - lastSentVersion eventversion.Version - payload event.ReplicatedPolicyEventBundle - topic string - predicate func(client.Object) bool +type policyStatusEventHandler struct { + ctx context.Context + name string + runtimeClient client.Client + payload *event.ReplicatedPolicyEventBundle + shouldUpdate func(client.Object) bool } -func StatusEventEmitter( +func NewPolicyStatusEventHandler( ctx context.Context, eventType enum.EventType, - predicate func(client.Object) bool, + shouldUpdate func(client.Object) bool, c client.Client, - topic string, -) generic.ObjectEmitter { +) *policyStatusEventHandler { name := strings.Replace(string(eventType), enum.EventTypePrefix, "", -1) filter.RegisterTimeFilter(name) - return &statusEventEmitter{ - ctx: ctx, - name: name, - log: ctrl.Log.WithName(name), - eventType: string(eventType), - topic: topic, - runtimeClient: c, - currentVersion: eventversion.NewVersion(), - lastSentVersion: *eventversion.NewVersion(), - payload: make([]event.ReplicatedPolicyEvent, 0), - predicate: predicate, + return &policyStatusEventHandler{ + ctx: ctx, + name: name, + runtimeClient: c, + payload: &event.ReplicatedPolicyEventBundle{}, + shouldUpdate: shouldUpdate, } } -// replicated policy -func (h *statusEventEmitter) ShouldUpdate(obj client.Object) bool { - return h.predicate(obj) -} - -func (h *statusEventEmitter) PostUpdate() { - h.currentVersion.Incr() +func (h *policyStatusEventHandler) Get() interface{} { + return h.payload } -func (h *statusEventEmitter) ShouldSend() bool { - return h.currentVersion.NewerThan(&h.lastSentVersion) +// only for test +func (h *policyStatusEventHandler) Append(evt event.ReplicatedPolicyEvent) { + *h.payload = append(*h.payload, evt) } -func (h *statusEventEmitter) Topic() string { - return h.topic -} +func (h *policyStatusEventHandler) Update(obj client.Object) bool { + if !h.shouldUpdate(obj) { + return false + } -func (h *statusEventEmitter) Update(obj client.Object) bool { policy, ok := obj.(*policiesv1.Policy) if !ok { return false // do not handle objects other than policy } + if policy.Status.Details == nil { return false // no status to update } rootPolicy, clusterID, clusterName, err := GetRootPolicyAndClusterInfo(h.ctx, policy, h.runtimeClient) if err != nil { - h.log.Error(err, "failed to get get rootPolicy/clusterID by replicatedPolicy") + klog.Errorf("failed to get get rootPolicy/clusterID by replicatedPolicy: %v", err) return false } @@ -107,7 +86,7 @@ func (h *statusEventEmitter) Update(obj client.Object) bool { continue } - h.payload = append(h.payload, event.ReplicatedPolicyEvent{ + *h.payload = append(*h.payload, event.ReplicatedPolicyEvent{ BaseEvent: event.BaseEvent{ EventName: evt.EventName, EventNamespace: policy.Namespace, @@ -131,32 +110,28 @@ func (h *statusEventEmitter) Update(obj client.Object) bool { return updated } -func (*statusEventEmitter) Delete(client.Object) bool { - // do nothing +func (*policyStatusEventHandler) Delete(client.Object) bool { return false } -func (h *statusEventEmitter) ToCloudEvent() (*cloudevents.Event, error) { - if len(h.payload) < 1 { - return nil, fmt.Errorf("the payload shouldn't be nil") - } - e := cloudevents.NewEvent() - e.SetSource(config.GetLeafHubName()) - e.SetType(h.eventType) - e.SetExtension(eventversion.ExtVersion, h.currentVersion.String()) - err := e.SetData(cloudevents.ApplicationJSON, h.payload) - return &e, err -} - -func (h *statusEventEmitter) PostSend() { - // update the time filter: with latest event - for _, evt := range h.payload { - filter.CacheTime(h.name, evt.CreatedAt.Time) - } - // update version and clean the cache - h.payload = make([]event.ReplicatedPolicyEvent, 0) - h.currentVersion.Next() - h.lastSentVersion = *h.currentVersion +func NewPolicyStatusEventEmitter(eventType enum.EventType) interfaces.Emitter { + name := strings.Replace(string(eventType), enum.EventTypePrefix, "", -1) + return generic.NewGenericEmitter(eventType, generic.WithPostSend( + // After sending the event, update the filter cache and clear the bundle from the handler cache. + func(data interface{}) { + events, ok := data.(*event.ReplicatedPolicyEventBundle) + if !ok { + return + } + // policyEvents, ok := data.([]event.ReplicatedPolicyEvent) + // update the time filter: with latest event + for _, evt := range *events { + filter.CacheTime(name, evt.CreatedAt.Time) + } + // reset the payload + *events = (*events)[:0] + }), + ) } func GetComplianceState(regex *regexp.Regexp, message, defaultVal string) string { diff --git a/agent/pkg/status/syncers/policies/handlers/status_event_handler_test.go b/agent/pkg/status/syncers/policies/handlers/status_event_handler_test.go new file mode 100644 index 000000000..7ca604fd2 --- /dev/null +++ b/agent/pkg/status/syncers/policies/handlers/status_event_handler_test.go @@ -0,0 +1,60 @@ +package handlers + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/stolostron/multicluster-global-hub/agent/pkg/configs" + "github.com/stolostron/multicluster-global-hub/pkg/bundle/event" + "github.com/stolostron/multicluster-global-hub/pkg/enum" +) + +func TestPostSendFunc(t *testing.T) { + configs.SetAgentConfig(&configs.AgentConfig{LeafHubName: "leaf-hub-name"}) + localStatusEventHandler := NewPolicyStatusEventHandler(context.TODO(), enum.LocalReplicatedPolicyEventType, + func(obj client.Object) bool { + return true + }, + nil, + ) + localStatusEventEmitter := NewPolicyStatusEventEmitter(enum.LocalReplicatedPolicyEventType) + + obj := localStatusEventHandler.Get() + assertBundle(t, obj, 0) + + // mock the handler update + localStatusEventHandler.Append(event.ReplicatedPolicyEvent{ + BaseEvent: event.BaseEvent{ + EventName: "name", + Reason: "PolicyStatusSync", + Count: 1, + Source: corev1.EventSource{ + Component: "policy-status-history-sync", + }, + }, + ClusterName: "hello", + }) + + // mock cloudevents to send + obj = localStatusEventHandler.Get() + assertBundle(t, obj, 1) + + _, err := localStatusEventEmitter.ToCloudEvent(obj) + assert.Nil(t, err) + + // post send -> the payload is cleanup + localStatusEventEmitter.PostSend(localStatusEventHandler.Get()) + obj = localStatusEventHandler.Get() + assertBundle(t, obj, 0) +} + +func assertBundle(t *testing.T, obj interface{}, size int) { + bundle, ok := obj.(*event.ReplicatedPolicyEventBundle) + require.Equal(t, ok, true) + require.Equal(t, size, len(*bundle)) +} diff --git a/agent/pkg/status/syncers/policies/policy_syncer.go b/agent/pkg/status/syncers/policies/policy_syncer.go new file mode 100644 index 000000000..c9cc8678e --- /dev/null +++ b/agent/pkg/status/syncers/policies/policy_syncer.go @@ -0,0 +1,133 @@ +package policies + +import ( + "context" + + policiesv1 "open-cluster-management.io/governance-policy-propagator/api/v1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/predicate" + + "github.com/stolostron/multicluster-global-hub/agent/pkg/configs" + "github.com/stolostron/multicluster-global-hub/agent/pkg/status/generic" + "github.com/stolostron/multicluster-global-hub/agent/pkg/status/syncers/configmap" + "github.com/stolostron/multicluster-global-hub/agent/pkg/status/syncers/policies/handlers" + genericpayload "github.com/stolostron/multicluster-global-hub/pkg/bundle/generic" + "github.com/stolostron/multicluster-global-hub/pkg/bundle/grc" + eventversion "github.com/stolostron/multicluster-global-hub/pkg/bundle/version" + "github.com/stolostron/multicluster-global-hub/pkg/constants" + "github.com/stolostron/multicluster-global-hub/pkg/enum" + "github.com/stolostron/multicluster-global-hub/pkg/transport" + "github.com/stolostron/multicluster-global-hub/pkg/utils" +) + +func LaunchPolicySyncer(ctx context.Context, mgr ctrl.Manager, agentConfig *configs.AgentConfig, + producer transport.Producer, +) error { + // controller config + instance := func() client.Object { return &policiesv1.Policy{} } + predicate := predicate.NewPredicateFuncs(func(object client.Object) bool { return true }) + + // 1. local compliance + localComplianceVersion := eventversion.NewVersion() + localComplianceShouldUpdate := func(obj client.Object) bool { + return configmap.GetAggregationLevel() == configmap.AggregationFull && // full level + configmap.GetEnableLocalPolicy() == configmap.EnableLocalPolicyTrue && // enable local policy + !utils.HasAnnotation(obj, constants.OriginOwnerReferenceAnnotation) && // local resource + !utils.HasLabel(obj, constants.PolicyEventRootPolicyNameLabelKey) // root policy + } + localComplianceHandler := handlers.NewComplianceHandler(&grc.ComplianceBundle{}, localComplianceShouldUpdate) + localComplianceEmitter := generic.NewGenericEmitter(enum.LocalComplianceType, + generic.WithVersion(localComplianceVersion)) + + // 2. local complete compliance + localCompleteHandler := handlers.NewCompleteComplianceHandler(&grc.CompleteComplianceBundle{}, + localComplianceShouldUpdate) + localCompleteEmitter := generic.NewGenericEmitter(enum.LocalCompleteComplianceType, + generic.WithDependencyVersion(localComplianceVersion)) + + // 3. local policy event + localStatusEventHandler := handlers.NewPolicyStatusEventHandler(ctx, enum.LocalReplicatedPolicyEventType, + // should update + func(obj client.Object) bool { + return configmap.GetEnableLocalPolicy() == configmap.EnableLocalPolicyTrue && + !utils.HasAnnotation(obj, constants.OriginOwnerReferenceAnnotation) && // local resource + utils.HasLabel(obj, constants.PolicyEventRootPolicyNameLabelKey) // replicated policy + }, + mgr.GetClient(), + ) + localStatusEventEmitter := handlers.NewPolicyStatusEventEmitter(enum.LocalReplicatedPolicyEventType) + + // 4. local policy spec + localPolicySpecHandler := generic.NewGenericHandler(&genericpayload.GenericObjectBundle{}, + generic.WithShouldUpdate( + func(obj client.Object) bool { + return configmap.GetEnableLocalPolicy() == configmap.EnableLocalPolicyTrue && // enable local policy + !utils.HasAnnotation(obj, constants.OriginOwnerReferenceAnnotation) && // local resource + !utils.HasLabel(obj, constants.PolicyEventRootPolicyNameLabelKey) // root policy + }), + generic.WithSpec(true), + generic.WithTweakFunc(cleanPolicy), + ) + localPolicySpecEmitter := generic.NewGenericEmitter(enum.LocalPolicySpecType) + + // 5. global policy compliance + complianceVersion := eventversion.NewVersion() + complianceShouldUpdate := func(obj client.Object) bool { + return configmap.GetAggregationLevel() == configmap.AggregationFull && // full level + utils.HasAnnotation(obj, constants.OriginOwnerReferenceAnnotation) && // global resource + !utils.HasLabel(obj, constants.PolicyEventRootPolicyNameLabelKey) // root policy + } + globalComplianceHandler := handlers.NewComplianceHandler(&grc.ComplianceBundle{}, complianceShouldUpdate) + globalComplianceEmitter := generic.NewGenericEmitter(enum.ComplianceType, generic.WithVersion(complianceVersion)) + + // 6. global complete compliance + globalCompleteHandler := handlers.NewCompleteComplianceHandler(&grc.CompleteComplianceBundle{}, + complianceShouldUpdate) + globalCompleteEmitter := generic.NewGenericEmitter(enum.CompleteComplianceType, + generic.WithDependencyVersion(complianceVersion)) + + return generic.LaunchMultiEventSyncer( + "status.policy", + mgr, + generic.NewGenericController(instance, predicate), + producer, + configmap.GetPolicyDuration, + []*generic.EmitterHandler{ + { + Handler: localPolicySpecHandler, + Emitter: localPolicySpecEmitter, + }, + { + Handler: localComplianceHandler, + Emitter: localComplianceEmitter, + }, + { + Handler: localCompleteHandler, + Emitter: localCompleteEmitter, + }, + + { + Handler: localStatusEventHandler, + Emitter: localStatusEventEmitter, + }, + + // global + { + Handler: globalComplianceHandler, + Emitter: globalComplianceEmitter, + }, + { + Handler: globalCompleteHandler, + Emitter: globalCompleteEmitter, + }, + }) +} + +func cleanPolicy(object client.Object) { + policy, ok := object.(*policiesv1.Policy) + if !ok { + panic("Wrong instance passed to clean policy function, not a Policy") + } + policy.Status = policiesv1.PolicyStatus{} +} diff --git a/agent/pkg/clients/stackrox_client.go b/agent/pkg/status/syncers/security/clients/stackrox_client.go similarity index 100% rename from agent/pkg/clients/stackrox_client.go rename to agent/pkg/status/syncers/security/clients/stackrox_client.go diff --git a/agent/pkg/clients/stackrox_client_test.go b/agent/pkg/status/syncers/security/clients/stackrox_client_test.go similarity index 100% rename from agent/pkg/clients/stackrox_client_test.go rename to agent/pkg/status/syncers/security/clients/stackrox_client_test.go diff --git a/agent/pkg/clients/suite_test.go b/agent/pkg/status/syncers/security/clients/suite_test.go similarity index 100% rename from agent/pkg/clients/suite_test.go rename to agent/pkg/status/syncers/security/clients/suite_test.go diff --git a/agent/pkg/status/controller/security/common.go b/agent/pkg/status/syncers/security/common.go similarity index 100% rename from agent/pkg/status/controller/security/common.go rename to agent/pkg/status/syncers/security/common.go diff --git a/agent/pkg/status/controller/security/stackrox_alert_count_summary.go b/agent/pkg/status/syncers/security/stackrox_alert_count_summary.go similarity index 100% rename from agent/pkg/status/controller/security/stackrox_alert_count_summary.go rename to agent/pkg/status/syncers/security/stackrox_alert_count_summary.go diff --git a/agent/pkg/status/controller/security/stackrox_controller.go b/agent/pkg/status/syncers/security/stackrox_controller.go similarity index 100% rename from agent/pkg/status/controller/security/stackrox_controller.go rename to agent/pkg/status/syncers/security/stackrox_controller.go diff --git a/agent/pkg/status/controller/security/stackrox_requests.go b/agent/pkg/status/syncers/security/stackrox_requests.go similarity index 100% rename from agent/pkg/status/controller/security/stackrox_requests.go rename to agent/pkg/status/syncers/security/stackrox_requests.go diff --git a/agent/pkg/status/controller/security/stackrox_syncer.go b/agent/pkg/status/syncers/security/stackrox_syncer.go similarity index 98% rename from agent/pkg/status/controller/security/stackrox_syncer.go rename to agent/pkg/status/syncers/security/stackrox_syncer.go index 645b565fd..703c50a62 100644 --- a/agent/pkg/status/controller/security/stackrox_syncer.go +++ b/agent/pkg/status/syncers/security/stackrox_syncer.go @@ -22,8 +22,8 @@ import ( crclient "sigs.k8s.io/controller-runtime/pkg/client" crmanager "sigs.k8s.io/controller-runtime/pkg/manager" - "github.com/stolostron/multicluster-global-hub/agent/pkg/clients" - "github.com/stolostron/multicluster-global-hub/agent/pkg/status/controller/config" + "github.com/stolostron/multicluster-global-hub/agent/pkg/configs" + "github.com/stolostron/multicluster-global-hub/agent/pkg/status/syncers/security/clients" "github.com/stolostron/multicluster-global-hub/pkg/bundle/version" eventversion "github.com/stolostron/multicluster-global-hub/pkg/bundle/version" "github.com/stolostron/multicluster-global-hub/pkg/enum" @@ -427,7 +427,7 @@ func (s *StackRoxSyncer) sync(ctx context.Context, data *stackRoxData) error { return fmt.Errorf("failed to generate struct for kafka message: %v", err) } - // TODO: If the payload not updated since the previous one, then don't need sync it again + // If the payload not updated since the previous one, then don't need sync it again // if equality.Semantic.DeepEqual(messageStruct, s.lastSentData) { // return nil // } @@ -446,7 +446,7 @@ func (s *StackRoxSyncer) produce(ctx context.Context, messageStruct any) error { s.dataLock.Lock() defer s.dataLock.Unlock() - evt := ToEvent(config.GetLeafHubName(), string(enum.SecurityAlertCountsType), s.currentVersion.String()) + evt := ToEvent(configs.GetLeafHubName(), string(enum.SecurityAlertCountsType), s.currentVersion.String()) err := evt.SetData(cloudevents.ApplicationJSON, messageStruct) if err != nil { return fmt.Errorf("failed to get CloudEvent instance from event %s: %v", *evt, err) diff --git a/agent/pkg/status/controller/security/stackrox_syncer_test.go b/agent/pkg/status/syncers/security/stackrox_syncer_test.go similarity index 100% rename from agent/pkg/status/controller/security/stackrox_syncer_test.go rename to agent/pkg/status/syncers/security/stackrox_syncer_test.go diff --git a/agent/pkg/status/controller/security/suite_test.go b/agent/pkg/status/syncers/security/suite_test.go similarity index 67% rename from agent/pkg/status/controller/security/suite_test.go rename to agent/pkg/status/syncers/security/suite_test.go index dc0b92a09..79a617841 100644 --- a/agent/pkg/status/controller/security/suite_test.go +++ b/agent/pkg/status/syncers/security/suite_test.go @@ -7,6 +7,8 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "sigs.k8s.io/controller-runtime/pkg/log/zap" + + "github.com/stolostron/multicluster-global-hub/agent/pkg/configs" ) func TestSecurity(t *testing.T) { @@ -17,6 +19,11 @@ func TestSecurity(t *testing.T) { var logger logr.Logger var _ = BeforeSuite(func() { + agentConfig := &configs.AgentConfig{ + LeafHubName: "leafHubName", + EnableGlobalResource: true, + } + configs.SetAgentConfig(agentConfig) // Configure logging to write to the Ginkgo writer: logger = zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true)) }) diff --git a/go.mod b/go.mod index af0079ee4..a916f2256 100644 --- a/go.mod +++ b/go.mod @@ -149,7 +149,6 @@ require ( github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-uuid v1.0.3 // indirect - github.com/hashicorp/golang-lru v0.5.4 github.com/huandu/xstrings v1.4.0 // indirect github.com/imdario/mergo v0.3.16 // indirect github.com/jackc/chunkreader/v2 v2.0.1 // indirect diff --git a/go.sum b/go.sum index 11c411ca0..60d6ef1b8 100644 --- a/go.sum +++ b/go.sum @@ -693,7 +693,6 @@ github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09 github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= diff --git a/samples/inventory/requester/global_hub.go b/samples/inventory/requester/global_hub.go index 4460aa108..07370707f 100644 --- a/samples/inventory/requester/global_hub.go +++ b/samples/inventory/requester/global_hub.go @@ -6,7 +6,7 @@ import ( kessel "github.com/project-kessel/inventory-api/api/kessel/inventory/v1beta1/resources" clusterinfov1beta1 "github.com/stolostron/cluster-lifecycle-api/clusterinfo/v1beta1" - "github.com/stolostron/multicluster-global-hub/agent/pkg/status/controller/managedclusters" + "github.com/stolostron/multicluster-global-hub/agent/pkg/controllers/inventory/managedclusterinfo" transportconfig "github.com/stolostron/multicluster-global-hub/pkg/transport/config" "github.com/stolostron/multicluster-global-hub/pkg/transport/requester" "github.com/stolostron/multicluster-global-hub/samples/config" @@ -41,7 +41,7 @@ func globalHub(ctx context.Context) error { } clusterInfo := createMockClusterInfo("local-cluster") - k8sCluster := managedclusters.GetK8SCluster(clusterInfo, "guest") + k8sCluster := managedclusterinfo.GetK8SCluster(clusterInfo, "guest") createResp, err := requesterClient.GetHttpClient().K8sClusterService.CreateK8SCluster(ctx, &kessel.CreateK8SClusterRequest{K8SCluster: k8sCluster}) if err != nil { @@ -50,7 +50,7 @@ func globalHub(ctx context.Context) error { fmt.Println("creating response", createResp) clusterInfo = createMockClusterInfo("local-cluster") - k8sCluster = managedclusters.GetK8SCluster(clusterInfo, "guest") + k8sCluster = managedclusterinfo.GetK8SCluster(clusterInfo, "guest") updatingResponse, err := requesterClient.GetHttpClient().K8sClusterService.UpdateK8SCluster(ctx, &kessel.UpdateK8SClusterRequest{K8SCluster: k8sCluster}) if err != nil { @@ -59,7 +59,7 @@ func globalHub(ctx context.Context) error { fmt.Println("updating response", updatingResponse) clusterInfo = createMockClusterInfo("local-cluster") - k8sCluster = managedclusters.GetK8SCluster(clusterInfo, "guest") + k8sCluster = managedclusterinfo.GetK8SCluster(clusterInfo, "guest") deletingResponse, err := requesterClient.GetHttpClient().K8sClusterService.DeleteK8SCluster(ctx, &kessel.DeleteK8SClusterRequest{ReporterData: k8sCluster.ReporterData}) if err != nil { diff --git a/samples/inventory/requester/managed_hub.go b/samples/inventory/requester/managed_hub.go index bf0fc6910..785a43165 100644 --- a/samples/inventory/requester/managed_hub.go +++ b/samples/inventory/requester/managed_hub.go @@ -6,8 +6,8 @@ import ( kessel "github.com/project-kessel/inventory-api/api/kessel/inventory/v1beta1/resources" clusterinfov1beta1 "github.com/stolostron/cluster-lifecycle-api/clusterinfo/v1beta1" - agentconfig "github.com/stolostron/multicluster-global-hub/agent/pkg/config" - "github.com/stolostron/multicluster-global-hub/agent/pkg/status/controller/managedclusters" + "github.com/stolostron/multicluster-global-hub/agent/pkg/configs" + "github.com/stolostron/multicluster-global-hub/agent/pkg/controllers/inventory/managedclusterinfo" transportconfig "github.com/stolostron/multicluster-global-hub/pkg/transport/config" "github.com/stolostron/multicluster-global-hub/pkg/transport/requester" "github.com/stolostron/multicluster-global-hub/samples/config" @@ -40,7 +40,7 @@ func managedHub(ctx context.Context, leafHubName string) error { return err } - k8sCluster := managedclusters.GetK8SCluster(&clusterInfoList[0], requester.GetInventoryClientName(leafHubName)) + k8sCluster := managedclusterinfo.GetK8SCluster(&clusterInfoList[0], requester.GetInventoryClientName(leafHubName)) resp, err := requesterClient.GetHttpClient().K8sClusterService.CreateK8SCluster(ctx, &kessel.CreateK8SClusterRequest{K8SCluster: k8sCluster}, @@ -70,7 +70,7 @@ func getRuntimeClient() (runtimeclient.Client, error) { if err != nil { return nil, err } - c, err := runtimeclient.New(kubeconfig, runtimeclient.Options{Scheme: agentconfig.GetRuntimeScheme()}) + c, err := runtimeclient.New(kubeconfig, runtimeclient.Options{Scheme: configs.GetRuntimeScheme()}) if err != nil { return nil, err } diff --git a/test/e2e/suite_test.go b/test/e2e/suite_test.go index f04b17d14..9fc6589f2 100644 --- a/test/e2e/suite_test.go +++ b/test/e2e/suite_test.go @@ -29,7 +29,7 @@ import ( logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" - agentconfig "github.com/stolostron/multicluster-global-hub/agent/pkg/config" + agentconfig "github.com/stolostron/multicluster-global-hub/agent/pkg/configs" managerconfig "github.com/stolostron/multicluster-global-hub/manager/pkg/config" "github.com/stolostron/multicluster-global-hub/operator/api/operator/v1alpha4" operatorconfig "github.com/stolostron/multicluster-global-hub/operator/pkg/config" diff --git a/test/integration/agent/controller/suite_test.go b/test/integration/agent/controller/suite_test.go index d03b09ef0..30b1dc8bb 100644 --- a/test/integration/agent/controller/suite_test.go +++ b/test/integration/agent/controller/suite_test.go @@ -20,7 +20,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log/zap" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" - "github.com/stolostron/multicluster-global-hub/agent/pkg/config" + "github.com/stolostron/multicluster-global-hub/agent/pkg/configs" "github.com/stolostron/multicluster-global-hub/agent/pkg/controllers" "github.com/stolostron/multicluster-global-hub/pkg/constants" ) @@ -61,7 +61,7 @@ var _ = BeforeSuite(func() { Metrics: metricsserver.Options{ BindAddress: "0", // disable the metrics serving }, - Scheme: config.GetRuntimeScheme(), + Scheme: configs.GetRuntimeScheme(), }) Expect(err).NotTo(HaveOccurred()) diff --git a/test/integration/agent/controller/version_clusterclaim_test.go b/test/integration/agent/controller/version_clusterclaim_test.go index 3020e0e12..96a8c847c 100644 --- a/test/integration/agent/controller/version_clusterclaim_test.go +++ b/test/integration/agent/controller/version_clusterclaim_test.go @@ -56,7 +56,7 @@ var _ = Describe("claim controllers", Ordered, func() { return mgr.GetClient().Get(ctx, types.NamespacedName{ Name: constants.HubClusterClaimName, }, clusterClaim) - }, 1*time.Second, 100*time.Millisecond).ShouldNot(HaveOccurred()) + }, 3*time.Second, 100*time.Millisecond).ShouldNot(HaveOccurred()) Expect(clusterClaim.Spec.Value).Should(Equal( constants.HubInstalledByUser)) }) @@ -73,11 +73,18 @@ var _ = Describe("claim controllers", Ordered, func() { clusterClaim := &clustersv1alpha1.ClusterClaim{} Eventually(func() error { - return mgr.GetClient().Get(ctx, types.NamespacedName{ + err := mgr.GetClient().Get(ctx, types.NamespacedName{ Name: constants.HubClusterClaimName, }, clusterClaim) - }, 1*time.Second, 100*time.Millisecond).ShouldNot(HaveOccurred()) - Expect(clusterClaim.Spec.Value).Should(Equal(constants.HubNotInstalled)) + if err != nil { + return err + } + if clusterClaim.Spec.Value == constants.HubNotInstalled { + return nil + } + return fmt.Errorf("the claim(%s) expect %s, but got %s", constants.HubClusterClaimName, + constants.HubNotInstalled, clusterClaim.Spec.Value) + }, 3*time.Second, 100*time.Millisecond).ShouldNot(HaveOccurred()) }) It("clusterclaim testing", func() { diff --git a/test/integration/agent/spec/resync_syncer_test.go b/test/integration/agent/spec/resync_syncer_test.go index c435026e5..37ce1d677 100644 --- a/test/integration/agent/spec/resync_syncer_test.go +++ b/test/integration/agent/spec/resync_syncer_test.go @@ -8,7 +8,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "github.com/stolostron/multicluster-global-hub/agent/pkg/spec/controller/syncers" + "github.com/stolostron/multicluster-global-hub/agent/pkg/spec/syncers" "github.com/stolostron/multicluster-global-hub/pkg/bundle/version" "github.com/stolostron/multicluster-global-hub/pkg/constants" "github.com/stolostron/multicluster-global-hub/pkg/enum" diff --git a/test/integration/agent/spec/suite_test.go b/test/integration/agent/spec/suite_test.go index 92cf949fa..fa2593367 100644 --- a/test/integration/agent/spec/suite_test.go +++ b/test/integration/agent/spec/suite_test.go @@ -15,8 +15,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log/zap" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" - "github.com/stolostron/multicluster-global-hub/agent/pkg/config" - speccontroller "github.com/stolostron/multicluster-global-hub/agent/pkg/spec/controller" + "github.com/stolostron/multicluster-global-hub/agent/pkg/configs" + speccontroller "github.com/stolostron/multicluster-global-hub/agent/pkg/spec" "github.com/stolostron/multicluster-global-hub/pkg/transport" genericconsumer "github.com/stolostron/multicluster-global-hub/pkg/transport/consumer" genericproducer "github.com/stolostron/multicluster-global-hub/pkg/transport/producer" @@ -30,7 +30,7 @@ func TestSyncers(t *testing.T) { var ( testenv *envtest.Environment leafHubName string - agentConfig *config.AgentConfig + agentConfig *configs.AgentConfig ctx context.Context cancel context.CancelFunc runtimeClient runtimeclient.Client @@ -44,7 +44,7 @@ var _ = BeforeSuite(func() { ctx, cancel = context.WithCancel(context.Background()) leafHubName = "spec-hub" - agentConfig = &config.AgentConfig{ + agentConfig = &configs.AgentConfig{ TransportConfig: &transport.TransportInternalConfig{ TransportType: string(transport.Chan), IsManager: false, @@ -73,7 +73,7 @@ var _ = BeforeSuite(func() { Metrics: metricsserver.Options{ BindAddress: "0", // disable the metrics serving }, - Scheme: config.GetRuntimeScheme(), + Scheme: configs.GetRuntimeScheme(), }) Expect(err).NotTo(HaveOccurred()) runtimeClient = mgr.GetClient() diff --git a/test/integration/agent/status/localpolicy_event_test.go b/test/integration/agent/status/localpolicy_event_test.go index f2f609453..47be1c7f6 100644 --- a/test/integration/agent/status/localpolicy_event_test.go +++ b/test/integration/agent/status/localpolicy_event_test.go @@ -14,7 +14,7 @@ import ( policyv1 "open-cluster-management.io/governance-policy-propagator/api/v1" "sigs.k8s.io/controller-runtime/pkg/client" - "github.com/stolostron/multicluster-global-hub/agent/pkg/status/controller/filter" + "github.com/stolostron/multicluster-global-hub/agent/pkg/status/filter" "github.com/stolostron/multicluster-global-hub/pkg/bundle/event" "github.com/stolostron/multicluster-global-hub/pkg/constants" "github.com/stolostron/multicluster-global-hub/pkg/enum" diff --git a/test/integration/agent/status/suite_test.go b/test/integration/agent/status/suite_test.go index b98f76474..13b7f32bc 100644 --- a/test/integration/agent/status/suite_test.go +++ b/test/integration/agent/status/suite_test.go @@ -19,14 +19,14 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log/zap" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" - "github.com/stolostron/multicluster-global-hub/agent/pkg/config" - "github.com/stolostron/multicluster-global-hub/agent/pkg/status/controller/apps" - statusconfig "github.com/stolostron/multicluster-global-hub/agent/pkg/status/controller/config" - "github.com/stolostron/multicluster-global-hub/agent/pkg/status/controller/event" - "github.com/stolostron/multicluster-global-hub/agent/pkg/status/controller/hubcluster" - "github.com/stolostron/multicluster-global-hub/agent/pkg/status/controller/managedclusters" - "github.com/stolostron/multicluster-global-hub/agent/pkg/status/controller/placement" - "github.com/stolostron/multicluster-global-hub/agent/pkg/status/controller/policies" + "github.com/stolostron/multicluster-global-hub/agent/pkg/configs" + "github.com/stolostron/multicluster-global-hub/agent/pkg/status/syncers/apps" + "github.com/stolostron/multicluster-global-hub/agent/pkg/status/syncers/configmap" + "github.com/stolostron/multicluster-global-hub/agent/pkg/status/syncers/events" + "github.com/stolostron/multicluster-global-hub/agent/pkg/status/syncers/managedcluster" + "github.com/stolostron/multicluster-global-hub/agent/pkg/status/syncers/managedhub" + "github.com/stolostron/multicluster-global-hub/agent/pkg/status/syncers/placement" + "github.com/stolostron/multicluster-global-hub/agent/pkg/status/syncers/policies" "github.com/stolostron/multicluster-global-hub/pkg/constants" "github.com/stolostron/multicluster-global-hub/pkg/transport" genericconsumer "github.com/stolostron/multicluster-global-hub/pkg/transport/consumer" @@ -73,7 +73,7 @@ var _ = BeforeSuite(func() { Expect(err).NotTo(HaveOccurred()) Expect(cfg).NotTo(BeNil()) - agentConfig := &config.AgentConfig{ + agentConfig := &configs.AgentConfig{ LeafHubName: leafHubName, TransportConfig: &transport.TransportInternalConfig{ CommitterInterval: 1 * time.Second, @@ -86,20 +86,21 @@ var _ = BeforeSuite(func() { }, EnableGlobalResource: true, } - statusconfig.SetInterval(statusconfig.HubClusterHeartBeatIntervalKey, 2*time.Second) - statusconfig.SetInterval(statusconfig.HubClusterInfoIntervalKey, 2*time.Second) + configs.SetAgentConfig(agentConfig) + configmap.SetInterval(configmap.HubClusterHeartBeatIntervalKey, 2*time.Second) + configmap.SetInterval(configmap.HubClusterInfoIntervalKey, 2*time.Second) By("Create controller-runtime manager") mgr, err := ctrl.NewManager(cfg, ctrl.Options{ Metrics: metricsserver.Options{ BindAddress: "0", // disable the metrics serving - }, Scheme: config.GetRuntimeScheme(), + }, Scheme: configs.GetRuntimeScheme(), }) Expect(err).NotTo(HaveOccurred()) Expect(mgr).NotTo(BeNil()) By("Create the configmap to disable the heartbeat on the suite test") - runtimeClient, err = client.New(cfg, client.Options{Scheme: config.GetRuntimeScheme()}) + runtimeClient, err = client.New(cfg, client.Options{Scheme: configs.GetRuntimeScheme()}) Expect(err).NotTo(HaveOccurred()) mghSystemNamespace := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: constants.GHAgentNamespace}} Expect(runtimeClient.Create(ctx, mghSystemNamespace)).Should(Succeed()) @@ -142,15 +143,15 @@ var _ = BeforeSuite(func() { Expect(err).To(Succeed()) // hubcluster info - err = statusconfig.AddConfigController(mgr, agentConfig) + err = configmap.AddConfigMapController(mgr, agentConfig) Expect(err).Should(Succeed()) - err = hubcluster.LaunchHubClusterHeartbeatSyncer(mgr, chanTransport.Producer(HeartBeatTopic)) + err = managedhub.LaunchHubClusterHeartbeatSyncer(mgr, chanTransport.Producer(HeartBeatTopic)) Expect(err).Should(Succeed()) - err = hubcluster.LaunchHubClusterInfoSyncer(mgr, chanTransport.Producer(HubClusterInfoTopic)) + err = managedhub.LaunchHubClusterInfoSyncer(mgr, chanTransport.Producer(HubClusterInfoTopic)) Expect(err).Should(Succeed()) // managed cluster - err = managedclusters.LaunchManagedClusterSyncer(ctx, mgr, agentConfig, chanTransport.Producer(ManagedClusterTopic)) + err = managedcluster.LaunchManagedClusterSyncer(ctx, mgr, agentConfig, chanTransport.Producer(ManagedClusterTopic)) Expect(err).To(Succeed()) // application @@ -158,7 +159,7 @@ var _ = BeforeSuite(func() { Expect(err).To(Succeed()) // event - err = event.LaunchEventSyncer(ctx, mgr, agentConfig, chanTransport.Producer(EventTopic)) + err = events.LaunchEventSyncer(ctx, mgr, agentConfig, chanTransport.Producer(EventTopic)) Expect(err).To(Succeed()) receivedEvents = make(map[string]*cloudevents.Event) go func() {