diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 601662f8..315f7607 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -78,7 +78,7 @@ jobs: - name: Extract docker image build version from GoReleaser id: image run: | - echo "version=$( echo '${{ steps.goreleaser.outputs.artifacts }}' | jq '.[] | select(.type == "Published Docker Image" and (.name | contains("ghcr.io")) and (.name | contains("latest") | not)) | .name | split(":") | .[1]' )" >> $GITHUB_OUTPUT + echo "version=$( echo '${{ steps.goreleaser.outputs.artifacts }}' | jq -r 'limit(1; .[] | select(.type == "Published Docker Image" and (.name | contains("ghcr.io")) and (.name | contains("latest") | not))) | .name | split(":") | .[1]' )" >> $GITHUB_OUTPUT latest-versions: name: Fetch latest versions from GH API diff --git a/.goreleaser.nightlies.yml b/.goreleaser.nightlies.yml index 720d2dbe..9aed4339 100644 --- a/.goreleaser.nightlies.yml +++ b/.goreleaser.nightlies.yml @@ -1,29 +1,57 @@ version: 2 project_name: config-server builds: - - id: config-server - binary: config-server - main: ./main.go + - id: api-server + binary: api-server + main: ./cmd/api-server/main.go env: - CGO_ENABLED=0 goos: - linux goarch: - amd64 + - arm64 + - id: controller + binary: controller + main: ./cmd/controller/main.go + env: + - CGO_ENABLED=0 + goos: + - linux + goarch: + - amd64 + - arm64 dockers: - - goos: linux + - id: api-server-docker + goos: linux + goarch: amd64 + ids: + - api-server + image_templates: + - "ghcr.io/sdcio/{{ .ProjectName }}-api-server:v0.0.0-{{ if index .Env \"PR\" }}PR{{ .Env.PR }}-{{ .ShortCommit }}{{ else }}{{ .ShortCommit }}{{ end }}" + dockerfile: goreleaser.api-server.dockerfile + skip_push: false + build_flag_templates: + - "--pull" + - "--build-arg=USERID=10000" + - "--label=org.opencontainers.image.created={{.Date}}" + - "--label=org.opencontainers.image.title={{.ProjectName}}-api-server" + - "--label=org.opencontainers.image.source=https://github.com/sdcio/{{.ProjectName}}" + - "--label=org.opencontainers.image.version=v{{.Version}}" + - id: controller-docker + goos: linux goarch: amd64 ids: - - config-server + - controller image_templates: - - "ghcr.io/sdcio/config-server:v0.0.0-{{ if index .Env \"PR\" }}PR{{ .Env.PR }}-{{ .ShortCommit }}{{ else }}{{ .ShortCommit }}{{ end }}" - dockerfile: goreleaser.dockerfile + - "ghcr.io/sdcio/{{ .ProjectName }}-controller:v0.0.0-{{ if index .Env \"PR\" }}PR{{ .Env.PR }}-{{ .ShortCommit }}{{ else }}{{ .ShortCommit }}{{ end }}" + dockerfile: goreleaser.controller.dockerfile skip_push: false build_flag_templates: - "--pull" - "--build-arg=USERID=10000" - "--label=org.opencontainers.image.created={{.Date}}" - - "--label=org.opencontainers.image.title={{.ProjectName}}" + - "--label=org.opencontainers.image.title={{.ProjectName}}-controller" - "--label=org.opencontainers.image.source=https://github.com/sdcio/{{.ProjectName}}" - "--label=org.opencontainers.image.version=v{{.Version}}" release: diff --git a/.goreleaser.yml b/.goreleaser.yml index 4c7fe363..bcf4a079 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -1,30 +1,59 @@ version: 2 project_name: config-server builds: - - id: config-server - binary: config-server - main: ./main.go + - id: api-server + binary: api-server + main: ./cmd/api-server/main.go env: - CGO_ENABLED=0 goos: - linux goarch: - amd64 + - arm64 + - id: controller + binary: controller + main: ./cmd/controller/main.go + env: + - CGO_ENABLED=0 + goos: + - linux + goarch: + - amd64 + - arm64 dockers: - - goos: linux + - id: api-server-docker + goos: linux + goarch: amd64 + ids: + - api-server + image_templates: + - "ghcr.io/sdcio/{{ .ProjectName }}-api-server:v{{ .Version }}" + - "ghcr.io/sdcio/{{ .ProjectName }}-api-server:latest" + dockerfile: goreleaser.api-server.dockerfile + skip_push: false + build_flag_templates: + - "--pull" + - "--build-arg=USERID=10000" + - "--label=org.opencontainers.image.created={{.Date}}" + - "--label=org.opencontainers.image.title={{.ProjectName}}-api-server" + - "--label=org.opencontainers.image.source=https://github.com/sdcio/{{.ProjectName}}" + - "--label=org.opencontainers.image.version=v{{.Version}}" + - id: controller-docker + goos: linux goarch: amd64 ids: - - config-server + - controller image_templates: - - "ghcr.io/sdcio/{{ .ProjectName }}:v{{ .Version }}" - - "ghcr.io/sdcio/{{ .ProjectName }}:latest" - dockerfile: goreleaser.dockerfile + - "ghcr.io/sdcio/{{ .ProjectName }}-controller:v{{ .Version }}" + - "ghcr.io/sdcio/{{ .ProjectName }}-controller:latest" + dockerfile: goreleaser.controller.dockerfile skip_push: false build_flag_templates: - "--pull" - "--build-arg=USERID=10000" - "--label=org.opencontainers.image.created={{.Date}}" - - "--label=org.opencontainers.image.title={{.ProjectName}}" + - "--label=org.opencontainers.image.title={{.ProjectName}}-controller" - "--label=org.opencontainers.image.source=https://github.com/sdcio/{{.ProjectName}}" - "--label=org.opencontainers.image.version=v{{.Version}}" archives: diff --git a/DockerfileAPIServer b/DockerfileAPIServer new file mode 100644 index 00000000..43281a62 --- /dev/null +++ b/DockerfileAPIServer @@ -0,0 +1,35 @@ +# Copyright 2024 Nokia +# Licensed under the Apache License 2.0 +# SPDX-License-Identifier: Apache-2.0 +# +FROM golang:1.24 AS builder +ARG USERID=10000 +# no need to include cgo bindings +ENV CGO_ENABLED=0 GOOS=linux GOARCH=amd64 + +# add ca certificates and timezone data files +RUN apt-get install --yes --no-install-recommends ca-certificates tzdata + +# add unprivileged user +RUN adduser --shell /bin/false --uid $USERID --disabled-login --home /app/ --no-create-home --gecos '' app \ + && sed -i -r "/^(app|root)/!d" /etc/group /etc/passwd \ + && sed -i -r 's#^(.*):[^:]*$#\1:/bin/false#' /etc/passwd + +# +#FROM scratch +FROM alpine:latest +ARG USERID=10000 +# add-in our timezone data file +COPY --from=builder /usr/share/zoneinfo /usr/share/zoneinfo +# add-in our unprivileged user +COPY --from=builder /etc/passwd /etc/group /etc/shadow /etc/ +# add-in our ca certificates +COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ + +COPY --chown=$USERID:$USERID api-server /app/ +WORKDIR /app + +# from now on, run as the unprivileged user +USER $USERID + +ENTRYPOINT [ "/app/api-server" ] diff --git a/Dockerfile b/DockerfileController similarity index 93% rename from Dockerfile rename to DockerfileController index 69ed50e6..1c04a712 100644 --- a/Dockerfile +++ b/DockerfileController @@ -28,7 +28,7 @@ RUN --mount=type=cache,target=/root/.cache/go-build \ go mod download # Copy the go source -COPY main.go main.go +COPY cmd/ cmd/ COPY apis/ apis/ COPY pkg/ pkg/ @@ -39,7 +39,7 @@ COPY pkg/ pkg/ # by leaving it empty we can ensure that the container and binary shipped on it will have the same platform. RUN --mount=type=cache,target=/root/.cache/go-build \ --mount=type=ssh \ - CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o config-server main.go + CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o controller cmd/controller/main.go # Use distroless as minimal base image to package the manager binary # Refer to https://github.com/GoogleContainerTools/distroless for more details @@ -56,8 +56,8 @@ COPY --from=builder /etc/passwd /etc/group /etc/shadow /etc/ # add-in our ca certificates COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ # -COPY --from=builder --chown=$USERID:$USERID /workspace/config-server /app/ +COPY --from=builder --chown=$USERID:$USERID /workspace/controller /app/ WORKDIR /app # from now on, run as the unprivileged user USER $USERID -ENTRYPOINT ["/app/config-server"] +ENTRYPOINT ["/app/controller"] diff --git a/Makefile b/Makefile index c1098f5c..f33f0884 100644 --- a/Makefile +++ b/Makefile @@ -4,8 +4,8 @@ VERSION ?= latest REGISTRY ?= europe-docker.pkg.dev/srlinux/eu.gcr.io -PROJECT ?= config-server -IMG ?= $(REGISTRY)/${PROJECT}:$(VERSION) +IMG_SERVER ?= $(REGISTRY)/sdc-apiserver:$(VERSION) +IMG_CONTROLLER ?= $(REGISTRY)/sdc-controller:$(VERSION) REPO = github.com/sdcio/config-server USERID := 10000 @@ -31,24 +31,23 @@ docker: .PHONY: docker-build docker-build: ## Build docker image with the manager. ssh-add ./keys/id_rsa 2>/dev/null; true - docker build --build-arg USERID="$(USERID)" . -t ${IMG} --ssh default="$(SSH_AUTH_SOCK)" + docker build --ssh default="$(SSH_AUTH_SOCK)" --build-arg USERID="$(USERID)" \ + -f DockerfileAPIServer -t ${IMG_SERVER} . + docker build --ssh default="$(SSH_AUTH_SOCK)" --build-arg USERID="$(USERID)" \ + -f DockerfileController -t ${IMG_CONTROLLER} . .PHONY: docker-push docker-push: docker-build ## Push docker image with the manager. - docker push ${IMG} + docker push ${IMG_SERVER} + docker push ${IMG_CONTROLLER} .PHONY: install -install: docker - kustomize build install | kubectl apply -f - +install: artifacts + kubectl apply -f artifacts/out .PHONY: reinstall -reinstall: docker - kustomize build install | kubectl apply -f - - kubectl delete pods -n config-system --all - -.PHONY: apiserver-logs -apiserver-logs: - kubectl logs -l apiserver=true --container apiserver -n config-system -f --tail 1000 +reinstall: docker-push artifacts + kubectl apply -f artifacts/out .PHONY: codegen codegen: diff --git a/apis/config/config_helpers.go b/apis/config/config_helpers.go index ac3707d7..fef7fbde 100644 --- a/apis/config/config_helpers.go +++ b/apis/config/config_helpers.go @@ -59,14 +59,18 @@ func (r *Config) IsRevertive() bool { return true } -func (r *Config) IsRecoverable() bool { +func (r *Config) IsRecoverable(ctx context.Context) bool { + logger := log.FromContext(ctx) c := r.GetCondition(condition.ConditionTypeReady) if c.Reason == string(condition.ConditionReasonUnrecoverable) { unrecoverableMessage := &condition.UnrecoverableMessage{} if err := json.Unmarshal([]byte(c.Message), unrecoverableMessage); err != nil { + logger.Error("is recoverable json unmarchal failed", "error", err) return true } if unrecoverableMessage.ResourceVersion != r.GetResourceVersion() { + logger.Info("is recoverable resource version changed", "old/new", + fmt.Sprintf("%s/%s", unrecoverableMessage.ResourceVersion, r.GetResourceVersion())) return true } return false diff --git a/apis/config/handlers/confighandler.go b/apis/config/handlers/confighandler.go index 9d973bc9..9d00d268 100644 --- a/apis/config/handlers/confighandler.go +++ b/apis/config/handlers/confighandler.go @@ -4,75 +4,110 @@ import ( "context" "fmt" + "github.com/henderiw/apiserver-store/pkg/storebackend" "github.com/sdcio/config-server/apis/config" - "github.com/sdcio/config-server/pkg/target" + invv1alpha1 "github.com/sdcio/config-server/apis/inv/v1alpha1" + sdctarget "github.com/sdcio/config-server/pkg/sdc/target" + sdcpb "github.com/sdcio/sdc-protos/sdcpb" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" - "github.com/sdcio/config-server/apis/condition" + "sigs.k8s.io/controller-runtime/pkg/client" ) + type ConfigStoreHandler struct { - Handler target.TargetHandler + Client client.Client } func (r *ConfigStoreHandler) DryRunCreateFn(ctx context.Context, key types.NamespacedName, obj runtime.Object, dryrun bool) (runtime.Object, error) { - accessor, err := meta.Accessor(obj) + c, target, err := r.prepareConfigAndTarget(ctx, key, obj) if err != nil { return obj, err } - targetKey, err := config.GetTargetKey(accessor.GetLabels()) + + updates, err := sdctarget.GetIntentUpdate(ctx, storebackend.KeyFromNSN(key), c, true) if err != nil { - return obj, err + return nil, err } - cfg := obj.(*config.Config) - schema, warnings, err := r.Handler.SetIntent(ctx, targetKey, cfg, config.Deviation{}, dryrun) - if err != nil { - msg := fmt.Sprintf("%s err %s", warnings, err.Error()) - cfg.SetConditions(condition.Failed(msg)) - return cfg, err + + intents := []*sdcpb.TransactionIntent{ + { + Intent: sdctarget.GetGVKNSN(c), + Priority: int32(c.Spec.Priority), + Update: updates, + }, } - cfg.SetConditions(condition.ReadyWithMsg(warnings)) - cfg.Status.LastKnownGoodSchema = schema - cfg.Status.AppliedConfig = &cfg.Spec - return cfg, nil + + return sdctarget.RunDryRunTransaction(ctx, key, c, target, intents, dryrun) } func (r *ConfigStoreHandler) DryRunUpdateFn(ctx context.Context, key types.NamespacedName, obj, old runtime.Object, dryrun bool) (runtime.Object, error) { - accessor, err := meta.Accessor(obj) + c, target, err := r.prepareConfigAndTarget(ctx, key, obj) if err != nil { return obj, err } - targetKey, err := config.GetTargetKey(accessor.GetLabels()) + + updates, err := sdctarget.GetIntentUpdate(ctx, storebackend.KeyFromNSN(key), c, true) if err != nil { - return obj, err + return nil, err } - cfg := obj.(*config.Config) - schema, warnings, err := r.Handler.SetIntent(ctx, targetKey, cfg, config.Deviation{}, dryrun) - if err != nil { - msg := fmt.Sprintf("%s err %s", warnings, err.Error()) - cfg.SetConditions(condition.Failed(msg)) - return cfg, err + + intents := []*sdcpb.TransactionIntent{ + { + Intent: sdctarget.GetGVKNSN(c), + Priority: int32(c.Spec.Priority), + Update: updates, + }, } - cfg.SetConditions(condition.ReadyWithMsg(warnings)) - cfg.Status.LastKnownGoodSchema = schema - cfg.Status.AppliedConfig = &cfg.Spec - return cfg, nil + + return sdctarget.RunDryRunTransaction(ctx, key, c, target, intents, dryrun) } func (r *ConfigStoreHandler) DryRunDeleteFn(ctx context.Context, key types.NamespacedName, obj runtime.Object, dryrun bool) (runtime.Object, error) { - accessor, err := meta.Accessor(obj) + c, target, err := r.prepareConfigAndTarget(ctx, key, obj) if err != nil { return obj, err } - targetKey, err := config.GetTargetKey(accessor.GetLabels()) - if err != nil { - return obj, err + + intents := []*sdcpb.TransactionIntent{ + { + Intent: sdctarget.GetGVKNSN(c), + Delete: true, + }, } - cfg := obj.(*config.Config) - warnings, err := r.Handler.DeleteIntent(ctx, targetKey, cfg, dryrun) + + return sdctarget.RunDryRunTransaction(ctx, key, c, target, intents, dryrun) +} + + +// prepareConfigAndTarget validates labels, casts the object, fetches the Target +// and ensures it's ready. +func (r *ConfigStoreHandler) prepareConfigAndTarget( + ctx context.Context, + key types.NamespacedName, + obj runtime.Object, +) (*config.Config, *invv1alpha1.Target, error) { + accessor, err := meta.Accessor(obj) if err != nil { - msg := fmt.Sprintf("%s err %s", warnings, err.Error()) - cfg.SetConditions(condition.Failed(msg)) - return cfg, err + return nil, nil, err } - return cfg, nil -} + + if _, err := config.GetTargetKey(accessor.GetLabels()); err != nil { + return nil, nil, err + } + + c, ok := obj.(*config.Config) + if !ok { + return nil, nil, fmt.Errorf("expected *config.Config, got %T", obj) + } + + target := &invv1alpha1.Target{} + if err := r.Client.Get(ctx, key, target); err != nil { + return nil, nil, err + } + + if !target.IsReady() { + return nil, nil, fmt.Errorf("target not ready %s", key) + } + + return c, target, nil +} \ No newline at end of file diff --git a/artifacts/allow-apiserver-extension-traffic.yaml b/artifacts/allow-apiserver-extension-traffic.yaml index df22000a..2012be33 100644 --- a/artifacts/allow-apiserver-extension-traffic.yaml +++ b/artifacts/allow-apiserver-extension-traffic.yaml @@ -2,9 +2,9 @@ apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: labels: - app.kubernetes.io/name: config-server + app.kubernetes.io/name: sdc-api-server name: allow-traffic-to-apiserver-extension - namespace: network-system + namespace: sdc-system spec: ingress: - from: @@ -15,6 +15,6 @@ spec: protocol: TCP podSelector: matchLabels: - app.kubernetes.io/name: config-server + app.kubernetes.io/name: sdc-api-server policyTypes: - Ingress diff --git a/artifacts/allow-metrics-traffic.yaml b/artifacts/allow-metrics-traffic.yaml index 569112dd..2104b429 100644 --- a/artifacts/allow-metrics-traffic.yaml +++ b/artifacts/allow-metrics-traffic.yaml @@ -4,11 +4,11 @@ apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: allow-metrics-traffic - namespace: network-system + namespace: sdc-system spec: podSelector: matchLabels: - app.kubernetes.io/name: config-server + app.kubernetes.io/name: sdc-controller policyTypes: - Ingress ingress: diff --git a/artifacts/apiservice.yaml b/artifacts/apiservice.yaml index 48abdb4b..74408b24 100644 --- a/artifacts/apiservice.yaml +++ b/artifacts/apiservice.yaml @@ -8,8 +8,8 @@ spec: groupPriorityMinimum: 1000 versionPriority: 15 service: - name: config-server - namespace: network-system + name: api-server + namespace: sdc-system port: 6443 version: v1alpha1 #caBundle: "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURZekNDQWt1Z0F3SUJBZ0lKQUtKY0FOZ3htSG1TTUEwR0NTcUdTSWIzRFFFQkN3VUFNR1V4Q3pBSkJnTlYKQkFZVEFuVnVNUXN3Q1FZRFZRUUlEQUp6ZERFS01BZ0dBMVVFQnd3QmJERUtNQWdHQTFVRUNnd0JiekVMTUFrRwpBMVVFQ3d3Q2IzVXhKREFpQmdOVkJBTU1HMkpoYzJsakxXTmxjblJwWm1sallYUmxMV0YxZEdodmNtbDBlVEFlCkZ3MHlNakF6TXpFd09URTNOVE5hRncweU16QXpNekV3T1RFM05UTmFNR1V4Q3pBSkJnTlZCQVlUQW5WdU1Rc3cKQ1FZRFZRUUlEQUp6ZERFS01BZ0dBMVVFQnd3QmJERUtNQWdHQTFVRUNnd0JiekVMTUFrR0ExVUVDd3dDYjNVeApKREFpQmdOVkJBTU1HMkpoYzJsakxXTmxjblJwWm1sallYUmxMV0YxZEdodmNtbDBlVENDQVNJd0RRWUpLb1pJCmh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTUJwMHRhNU92Vy9VcVlsR1RMZnVsam9HYkFwVnB4MW1CbUwKR0dHOUhOZmJkVWxoQ1FtMVNYK3V6dllyQ05EZEJRMWdBYWVEa1lxOWNMbnN4YU92R25peHJ1WllUV1gyaU9maQpjRTZxRUVRTm05MmxRRnBvbnBneXI2dFc3dDhkMGRNcEVVNTlYUlQzdXRGZGhHRVJUYi94clR0c1RpaUp4Vk1jCmxFSzh3ajZjLytONitHNHZEcVBydkF5cFBJaUJtbkhwVE9tbmhOdjhSeXVXc3VXVEJwb0JTMUVjbTg1VlY3MEUKUGFpYSs3bDczLzArWmFzcTBHeklCdkx4S0ZiVHVYZHh2a0REY1M5c0FuTytVcHg1YUxhbjgrR25UTWd6NzR6Vgp3WDRuSFU1blFxYkZSSC9TQzVXeGNYczJXL0JNZllBRUk2ckhnUTBKNjJiMTBSZk0vQmNDQXdFQUFhTVdNQlF3CkVnWURWUjBUQVFIL0JBZ3dCZ0VCL3dJQkFUQU5CZ2txaGtpRzl3MEJBUXNGQUFPQ0FRRUFxUXM5eUdsalBFcUgKVHF6REpEa29DbXlTWmQ0S3VVSEpPcjY1QmhmYmppKzBsSC9Rbk9mdHdpd1FvajhwSFlnejVmZGZJa3JMTFU1KwpwMzh0cSs5QllsOFNudXd6U2EzQ2VpYUlncHUvL05xaCtieDRad1liNFJmVnZmSU5NdUZJaUhLUFBJUm1QRmlECjZJQjl0WFNrSmNmanhHd1NLRmhLSGszYU9EbmsxNUlyTDA0U040S0ZER1dncnI0WkJoL1RYT25XVmRpMHRBN3kKT2lmRkpRdWt1anhNVDRUU3ZtcmtjZW5Ubk84VEZTMk03SGVPZDRLYm14QUJFR3ZzaGZ0V2tXUGh0ZW1IYVJXYwpVOEh2SG8xS1M4cGdyYVdxMU5jMjErdHJoQS9uaGRtaWRDbW1DOHZSQ1MwU1cxZE90Q3ZJRzhpVUpIeDVKMklGCnhjUGt3aHYrN1E9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg==" diff --git a/artifacts/configmap-data-server.yaml b/artifacts/configmap-data-server.yaml index b50ffa48..759ea9e2 100644 --- a/artifacts/configmap-data-server.yaml +++ b/artifacts/configmap-data-server.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: ConfigMap metadata: name: data-server - namespace: network-system + namespace: sdc-system data: data-server.yaml: | grpc-server: diff --git a/artifacts/configmap-input-vars.yaml b/artifacts/configmap-input-vars.yaml index 883a3a31..c05f46e2 100644 --- a/artifacts/configmap-input-vars.yaml +++ b/artifacts/configmap-input-vars.yaml @@ -2,11 +2,8 @@ apiVersion: v1 kind: ConfigMap metadata: name: context - namespace: network-system - annotations: - kform.dev/block-type: input - kform.dev/resource-id: context ## this serves as a way to add default and manage the merge - kform.dev/default: "true" data: - configServerImage: ghcr.io/sdcio/config-server:latest - dataServerImage: ghcr.io/sdcio/data-server:latest \ No newline at end of file + sdcApiServerImage: ghcr.io/sdcio/sdc-apiserver:latest + sdcControllerImage: ghcr.io/sdcio/sdc-controller:latest + dataServerImage: ghcr.io/sdcio/data-server:v0.0.66 + diff --git a/artifacts/deployment-apiserver.yaml b/artifacts/deployment-apiserver.yaml new file mode 100644 index 00000000..678ef130 --- /dev/null +++ b/artifacts/deployment-apiserver.yaml @@ -0,0 +1,77 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: api-server + namespace: sdc-system + labels: + app.kubernetes.io/name: sdc-api-server +spec: + strategy: + type: Recreate + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: sdc-api-server + template: + metadata: + labels: + app.kubernetes.io/name: sdc-api-server + spec: + securityContext: + runAsUser: 10000 + runAsGroup: 10000 + fsGroup: 10000 + serviceAccountName: api-server + containers: + - name: api-server + image: input.context[0].data.sdcApiServerImage + imagePullPolicy: Always + command: + - /app/api-server + args: + - "--tls-cert-file=/apiserver.local.config/certificates/tls.crt" + - "--tls-private-key-file=/apiserver.local.config/certificates/tls.key" + #- "--feature-gates=APIPriorityAndFairness=false" + - "--audit-log-path=-" + - "--audit-log-maxage=0" + - "--audit-log-maxbackup=0" + - "--secure-port=6443" + ports: + - name: api-service + containerPort: 6443 + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: "NODE_NAME" + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: "NODE_IP" + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.hostIP + - name: METRIC_PORT + value: "8443" + volumeMounts: + - name: api-server-certs + mountPath: /apiserver.local.config/certificates + readOnly: true + - name: config-store + mountPath: /config + volumes: + - name: api-server-certs + secret: + secretName: api-server-cert + - name: config-store + persistentVolumeClaim: + claimName: pvc-config-store + diff --git a/artifacts/deployment-controller.yaml b/artifacts/deployment-controller.yaml new file mode 100644 index 00000000..07ca39fa --- /dev/null +++ b/artifacts/deployment-controller.yaml @@ -0,0 +1,90 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller + namespace: sdc-system + labels: + app.kubernetes.io/name: sdc-controller +spec: + strategy: + type: Recreate + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: sdc-controller + template: + metadata: + labels: + app.kubernetes.io/name: sdc-controller + spec: + securityContext: + runAsUser: 10000 + runAsGroup: 10000 + fsGroup: 10000 + serviceAccountName: controller + containers: + - name: controller + image: input.context[0].data.sdcControllerImage + imagePullPolicy: Always + command: + - /app/controller + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: "NODE_NAME" + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: "NODE_IP" + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.hostIP + - name: METRIC_PORT + value: "8443" + - name: PPROF_PORT + value: "8081" + #- name: REVERTIVE + # value: "true" + #- name: ENABLE_SUBSCRIPTION + # value: "true" + #- name: ENABLE_TARGET + # value: "true" + #- name: ENABLE_TARGETDATASTORE + # value: "true" + #- name: ENABLE_TARGETCONFIGSERVER + # value: "true" + #- name: ENABLE_TARGETRECOVERYSERVER + # value: "true" + - name: ENABLE_DISCOVERYRULE + value: "true" + #- name: ENABLE_SCHEMA + # value: "true" + #- name: ENABLE_CONFIG + # value: "true" + - name: ENABLE_CONFIGSET + value: "true" + - name: ENABLE_WORKSPACE + value: "true" + - name: ENABLE_ROLLOUT + value: "true" + volumeMounts: + #- name: schema-store + # mountPath: /schemas + - name: workspace-store + mountPath: /workspace + volumes: + #- name: schema-store # yang file from git + # persistentVolumeClaim: + # claimName: pvc-schema-store + - name: workspace-store # git workspace + persistentVolumeClaim: + claimName: pvc-workspace-store diff --git a/artifacts/in/configmap-input-vars.yaml b/artifacts/in/configmap-input-vars.yaml index 529f4eed..fef462d7 100644 --- a/artifacts/in/configmap-input-vars.yaml +++ b/artifacts/in/configmap-input-vars.yaml @@ -3,5 +3,7 @@ kind: ConfigMap metadata: name: context data: - configServerImage: europe-docker.pkg.dev/srlinux/eu.gcr.io/config-server:latest - dataServerImage: ghcr.io/sdcio/data-server:v0.0.61 + sdcApiServerImage: europe-docker.pkg.dev/srlinux/eu.gcr.io/sdc-apiserver:latest + sdcControllerImage: europe-docker.pkg.dev/srlinux/eu.gcr.io/sdc-controller:latest + dataServerImage: ghcr.io/sdcio/data-server:v0.0.66 + diff --git a/artifacts/ns.yaml b/artifacts/ns.yaml index 4bd7dc1d..6ec09054 100644 --- a/artifacts/ns.yaml +++ b/artifacts/ns.yaml @@ -1,4 +1,4 @@ apiVersion: v1 kind: Namespace metadata: - name: network-system + name: sdc-system diff --git a/artifacts/pv-config-server-store.yaml b/artifacts/pv-config-server-store.yaml index fc268f50..46ed636e 100644 --- a/artifacts/pv-config-server-store.yaml +++ b/artifacts/pv-config-server-store.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: PersistentVolumeClaim metadata: name: pvc-config-store - namespace: network-system + namespace: sdc-system spec: accessModes: - ReadWriteOnce diff --git a/artifacts/pv-data-server-schemadb.yaml b/artifacts/pv-data-server-schemadb.yaml index 31a15902..3f97a602 100644 --- a/artifacts/pv-data-server-schemadb.yaml +++ b/artifacts/pv-data-server-schemadb.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: PersistentVolumeClaim metadata: name: pvc-schema-db - namespace: network-system + namespace: sdc-system spec: accessModes: - ReadWriteOnce diff --git a/artifacts/pv-schema-server-schema.yaml b/artifacts/pv-schema-server-schema.yaml index 941439a5..7a61d08e 100644 --- a/artifacts/pv-schema-server-schema.yaml +++ b/artifacts/pv-schema-server-schema.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: PersistentVolumeClaim metadata: name: pvc-schema-store - namespace: network-system + namespace: sdc-system spec: accessModes: - ReadWriteOnce diff --git a/artifacts/pv-workspace-store.yaml b/artifacts/pv-workspace-store.yaml index 5fb1f36c..bdc0f44f 100644 --- a/artifacts/pv-workspace-store.yaml +++ b/artifacts/pv-workspace-store.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: PersistentVolumeClaim metadata: name: pvc-workspace-store - namespace: network-system + namespace: sdc-system spec: accessModes: - ReadWriteOnce diff --git a/artifacts/rbac-cluster-role-api-server.yaml b/artifacts/rbac-cluster-role-api-server.yaml new file mode 100644 index 00000000..e43d38da --- /dev/null +++ b/artifacts/rbac-cluster-role-api-server.yaml @@ -0,0 +1,20 @@ +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: sdc-api-server-clusterrole +rules: +- apiGroups: [""] + resources: ["namespaces", "secrets", "configmaps", "services", "pods"] + verbs: ["get", "watch", "list"] +- apiGroups: [""] + resources: ["events"] + verbs: ["create", "patch"] +- apiGroups: ["admissionregistration.k8s.io"] + resources: ["mutatingwebhookconfigurations", "validatingwebhookconfigurations", "validatingadmissionpolicies", "validatingadmissionpolicybindings"] + verbs: ["get", "watch", "list"] +- apiGroups: ["flowcontrol.apiserver.k8s.io"] + resources: ["flowschemas", "prioritylevelconfigurations"] + verbs: ["get", "watch", "list"] +- apiGroups: ["inv.sdcio.dev"] + resources: ["targets", "targets/status"] + verbs: ["get", "watch", "list"] diff --git a/artifacts/rbac-cluster-role-binding-api-server.yaml b/artifacts/rbac-cluster-role-binding-api-server.yaml new file mode 100644 index 00000000..7831367a --- /dev/null +++ b/artifacts/rbac-cluster-role-binding-api-server.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: sdc-api-server-clusterrolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: sdc-api-server-clusterrole +subjects: +- kind: ServiceAccount + name: api-server + namespace: sdc-system \ No newline at end of file diff --git a/artifacts/rbac-cluster-role-binding-auth-delegator.yaml b/artifacts/rbac-cluster-role-binding-auth-delegator.yaml index 26fef741..35393e14 100644 --- a/artifacts/rbac-cluster-role-binding-auth-delegator.yaml +++ b/artifacts/rbac-cluster-role-binding-auth-delegator.yaml @@ -1,12 +1,12 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - name: config:system:auth-delegator + name: sdc:system:auth-delegator roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: system:auth-delegator subjects: - kind: ServiceAccount - name: config-server - namespace: network-system \ No newline at end of file + name: api-server + namespace: sdc-system \ No newline at end of file diff --git a/artifacts/rbac-cluster-role-binding-controller.yaml b/artifacts/rbac-cluster-role-binding-controller.yaml index a7d25043..c7d8f426 100644 --- a/artifacts/rbac-cluster-role-binding-controller.yaml +++ b/artifacts/rbac-cluster-role-binding-controller.yaml @@ -1,12 +1,12 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - name: config-server-clusterrolebinding + name: sdc-controller-clusterrolebinding roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: config-server-clusterrole + name: sdc-controller-clusterrole subjects: - kind: ServiceAccount - name: config-server - namespace: network-system \ No newline at end of file + name: controller + namespace: sdc-system \ No newline at end of file diff --git a/artifacts/rbac-cluster-role-binding-data-server-controller.yaml b/artifacts/rbac-cluster-role-binding-data-server-controller.yaml new file mode 100644 index 00000000..aac42002 --- /dev/null +++ b/artifacts/rbac-cluster-role-binding-data-server-controller.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: sdc-data-server-controller-clusterrolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: sdc-data-server-controller-clusterrole +subjects: +- kind: ServiceAccount + name: data-server-controller + namespace: sdc-system \ No newline at end of file diff --git a/artifacts/rbac-cluster-role-binding-metrics.yaml b/artifacts/rbac-cluster-role-binding-metrics.yaml index cfdf2dac..4115228e 100644 --- a/artifacts/rbac-cluster-role-binding-metrics.yaml +++ b/artifacts/rbac-cluster-role-binding-metrics.yaml @@ -1,12 +1,12 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - name: config-server-metrics-auth-rolebinding + name: sdc-controller-metrics-auth-rolebinding roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: config-server-metrics-auth-role + name: sdc-controller-metrics-auth-role subjects: - kind: ServiceAccount - name: controller-manager - namespace: network-system + name: controller + namespace: sdc-system diff --git a/artifacts/rbac-cluster-role-controller.yaml b/artifacts/rbac-cluster-role-controller.yaml index 2020ee26..09fdb9e2 100644 --- a/artifacts/rbac-cluster-role-controller.yaml +++ b/artifacts/rbac-cluster-role-controller.yaml @@ -1,7 +1,7 @@ kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: config-server-clusterrole + name: sdc-controller-clusterrole rules: - apiGroups: [""] resources: ["namespaces", "secrets", "services", "pods"] diff --git a/artifacts/rbac-cluster-role-data-server-controller.yaml b/artifacts/rbac-cluster-role-data-server-controller.yaml new file mode 100644 index 00000000..5a1bfc5b --- /dev/null +++ b/artifacts/rbac-cluster-role-data-server-controller.yaml @@ -0,0 +1,53 @@ +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: sdc-data-server-controller-clusterrole +rules: +- apiGroups: [""] + resources: ["namespaces", "secrets", "services", "pods"] + verbs: ["get", "watch", "list"] +- apiGroups: [""] + resources: ["events"] + verbs: ["create", "patch"] +- apiGroups: ["admissionregistration.k8s.io"] + resources: ["mutatingwebhookconfigurations", "validatingwebhookconfigurations", "validatingadmissionpolicies", "validatingadmissionpolicybindings"] + verbs: ["get", "watch", "list"] +- apiGroups: ["flowcontrol.apiserver.k8s.io"] + resources: ["flowschemas", "prioritylevelconfigurations"] + verbs: ["get", "watch", "list"] +- apiGroups: ["config.sdcio.dev"] + resources: ["configs", "configs/status"] + verbs: ["get", "watch", "list", "create", "update", "patch", "delete"] +- apiGroups: ["config.sdcio.dev"] + resources: ["configsets", "configsets/status"] + verbs: ["get", "watch", "list", "create", "update", "patch", "delete"] +- apiGroups: ["config.sdcio.dev"] + resources: ["deviations", "deviations/status"] + verbs: ["get", "watch", "list", "create", "update", "patch", "delete"] +- apiGroups: ["inv.sdcio.dev"] + resources: ["targets", "targets/status"] + verbs: ["get", "watch", "list", "create", "update", "patch", "delete"] +- apiGroups: ["inv.sdcio.dev"] + resources: ["targetconnectionprofiles", "targetsyncprofiles"] + verbs: ["get", "watch", "list"] +- apiGroups: ["inv.sdcio.dev"] + resources: ["discoveryrules", "discoveryrules/status"] + verbs: ["get", "watch", "list", "create", "update", "patch", "delete"] +- apiGroups: ["inv.sdcio.dev"] + resources: ["discoveryvendorprofiles"] + verbs: ["get", "watch", "list", "create", "update", "patch", "delete"] +- apiGroups: ["inv.sdcio.dev"] + resources: ["schemas", "schemas/status"] + verbs: ["get", "watch", "list", "create", "update", "patch", "delete"] +- apiGroups: ["inv.sdcio.dev"] + resources: ["subscriptions", "subscriptions/status"] + verbs: ["get", "watch", "list", "create", "update", "patch", "delete"] +- apiGroups: ["inv.sdcio.dev"] + resources: ["workspaces", "workspaces/status"] + verbs: ["get", "watch", "list", "create", "update", "patch", "delete"] +- apiGroups: ["inv.sdcio.dev"] + resources: ["rollouts", "rollouts/status"] + verbs: ["get", "watch", "list", "create", "update", "patch", "delete"] +- apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "create", "update", "patch", "delete"] diff --git a/artifacts/rbac-cluster-role-metrics.yaml b/artifacts/rbac-cluster-role-metrics.yaml index e016304f..2c4bd8f7 100644 --- a/artifacts/rbac-cluster-role-metrics.yaml +++ b/artifacts/rbac-cluster-role-metrics.yaml @@ -1,7 +1,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: config-server-metrics-auth-role + name: sdc-controller-metrics-auth-role rules: - apiGroups: ["authentication.k8s.io"] resources: ["tokenreviews"] diff --git a/artifacts/rbac-cluster-role-metrics_reader.yaml b/artifacts/rbac-cluster-role-metrics_reader.yaml index 4f2879ac..efc84d2d 100644 --- a/artifacts/rbac-cluster-role-metrics_reader.yaml +++ b/artifacts/rbac-cluster-role-metrics_reader.yaml @@ -1,7 +1,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: config-server-metrics-reader + name: sdc-controller-metrics-reader rules: - nonResourceURLs: - "/metrics" diff --git a/artifacts/rbac-role-binding-auth-reader.yaml b/artifacts/rbac-role-binding-auth-reader.yaml index 9111a7ec..f81c31c1 100644 --- a/artifacts/rbac-role-binding-auth-reader.yaml +++ b/artifacts/rbac-role-binding-auth-reader.yaml @@ -1,7 +1,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: - name: config-auth-reader + name: sdc-controller-auth-reader namespace: kube-system roleRef: apiGroup: rbac.authorization.k8s.io @@ -9,5 +9,5 @@ roleRef: name: extension-apiserver-authentication-reader subjects: - kind: ServiceAccount - name: config-server - namespace: network-system + name: controller + namespace: sdc-system diff --git a/artifacts/rbac-role.yaml b/artifacts/rbac-role.yaml index dcc134c6..2db69c37 100644 --- a/artifacts/rbac-role.yaml +++ b/artifacts/rbac-role.yaml @@ -2,7 +2,7 @@ kind: Role apiVersion: rbac.authorization.k8s.io/v1 metadata: name: aggregated-apiserver-role - namespace: network-system + namespace: sdc-system rules: - apiGroups: [""] resources: ["serviceaccounts"] diff --git a/artifacts/sa-apiserver.yaml b/artifacts/sa-apiserver.yaml new file mode 100644 index 00000000..f639fc8e --- /dev/null +++ b/artifacts/sa-apiserver.yaml @@ -0,0 +1,5 @@ +kind: ServiceAccount +apiVersion: v1 +metadata: + name: api-server + namespace: sdc-system diff --git a/artifacts/sa-controller.yaml b/artifacts/sa-controller.yaml new file mode 100644 index 00000000..57870f5e --- /dev/null +++ b/artifacts/sa-controller.yaml @@ -0,0 +1,5 @@ +kind: ServiceAccount +apiVersion: v1 +metadata: + name: controller + namespace: sdc-system diff --git a/artifacts/sa-dataserver.yaml b/artifacts/sa-dataserver.yaml new file mode 100644 index 00000000..d5448c92 --- /dev/null +++ b/artifacts/sa-dataserver.yaml @@ -0,0 +1,5 @@ +kind: ServiceAccount +apiVersion: v1 +metadata: + name: data-server-controller + namespace: sdc-system diff --git a/artifacts/sa.yaml b/artifacts/sa.yaml deleted file mode 100644 index 600e33cc..00000000 --- a/artifacts/sa.yaml +++ /dev/null @@ -1,5 +0,0 @@ -kind: ServiceAccount -apiVersion: v1 -metadata: - name: config-server - namespace: network-system diff --git a/artifacts/secret.yaml b/artifacts/secret.yaml index 4def60f7..ec394b86 100644 --- a/artifacts/secret.yaml +++ b/artifacts/secret.yaml @@ -2,10 +2,10 @@ apiVersion: v1 kind: Secret type: kubernetes.io/tls metadata: - name: config-server-cert - namespace: network-system + name: api-server-cert + namespace: sdc-system labels: - app.kubernetes.io/name: config-server + app.kubernetes.io/name: sdc-api-server data: tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURZekNDQWt1Z0F3SUJBZ0lJSC9mSFZHenZ3cnN3RFFZSktvWklodmNOQVFFTEJRQXdaVEVMTUFrR0ExVUUKQmhNQ2RXNHhDekFKQmdOVkJBZ01Bbk4wTVFvd0NBWURWUVFIREFGc01Rb3dDQVlEVlFRS0RBRnZNUXN3Q1FZRApWUVFMREFKdmRURWtNQ0lHQTFVRUF3d2JZbUZ6YVdNdFkyVnlkR2xtYVdOaGRHVXRZWFYwYUc5eWFYUjVNQjRYCkRUSXlNRE16TVRBNU1UYzFNMW9YRFRNeU1ETXlPREE1TVRjMU5Gb3dIREVhTUJnR0ExVUVBeE1SWW1GemFXTXUKWkdWbVlYVnNkQzV6ZG1Nd2dnRWlNQTBHQ1NxR1NJYjNEUUVCQVFVQUE0SUJEd0F3Z2dFS0FvSUJBUUN0TUt0eApjc3Rjdk8rdDVMazZRQkRBZ3g1akZCL2F1dStVb3BDR2Z6VitaRW5obldpaC8xMVZ2ek44cjhmdGZuUkZGTVZ6CmJqYlVhSXNDOFc1eGJDNXNpc2VrdnVBWDlpanUzMlFybEU0RTR1UzNYREdVZkhGSFhMcWxBRU9RclUvRzQ0RGgKa0I3ajJOcDRzbk9IckF0aDA3TStvbXBmVklhSTlkQmdYY3hsUE5QRkNNamlOb1VweVM4eXNha3RQRXFjZTBpawpmNDBYVERmN1YwekFFelI0QkE4Yzh0b05UMVNnSXFIV0xueERKcnZRempDaTVFN2NMNkpmTmhlZDQ5MUVNWlEwCmVnbkV5bXd6d1Jya3BYTkZ4RHJzSXpOZmhHelB6RGJLdmFIUHh5NUwvM3h3clZ3VHllbklaOVExK0tjemtCSksKRXZIaVVKL1BML0VYZkloakFnTUJBQUdqWURCZU1BNEdBMVVkRHdFQi93UUVBd0lGb0RBZEJnTlZIU1VFRmpBVQpCZ2dyQmdFRkJRY0RBUVlJS3dZQkJRVUhBd0l3TFFZRFZSMFJCQ1l3SklJSmJHOWpZV3hvYjNOMGdoRmlZWE5wCll5NWtaV1poZFd4MExuTjJZNGNFZndBQUFUQU5CZ2txaGtpRzl3MEJBUXNGQUFPQ0FRRUFEa1hsbGZMTlpzWDEKYmp1b0h4RXVUWitaODlMWUxPUDBMM0dHMFgwdVdkZzJFcXY1bmZNRHVRVmJIRmt5dVo3ZDlDY01QYk12MTdDWgoxZGwwQk1GQTJkTkJzK3V1UXFIUFh3RkI4SFdPSDhBc1pMMnYvbG91T3g2dU1QQk9uWUhuQ3pFY21FQXZoR2dLCkpXMDNkd2QwNlJPeUdLT29qSklFTlRnd0xnQ1dZSytPWmIzQklyMUJqS012Q2dHN3pJVDFUUVNna3hGN1NGNzUKYk5BaEdOa0NWMGVrSnNXQWk1UGhzVS9IdWthdGVHUGNMS3hia0RGdHpSV2tRNmdKUXhkZmVuOVBKTjVJVCt4RQpFci8wYUkrOFM5Y1FPUnk0VTNDSFRodmlnOGFyZ3FucmFWMU92OXZNTWxzZ3pnYXc3SjdaeGtkWWwrSkMyWUcvCjJrUThVd1IzQnc9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== tls.key: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb3dJQkFBS0NBUUVBclRDcmNYTExYTHp2cmVTNU9rQVF3SU1lWXhRZjJycnZsS0tRaG44MWZtUko0WjFvCm9mOWRWYjh6ZksvSDdYNTBSUlRGYzI0MjFHaUxBdkZ1Y1d3dWJJckhwTDdnRi9Zbzd0OWtLNVJPQk9Ma3Qxd3gKbEh4eFIxeTZwUUJEa0sxUHh1T0E0WkFlNDlqYWVMSnpoNndMWWRPelBxSnFYMVNHaVBYUVlGM01aVHpUeFFqSQo0amFGS2Nrdk1yR3BMVHhLbkh0SXBIK05GMHczKzFkTXdCTTBlQVFQSFBMYURVOVVvQ0toMWk1OFF5YTcwTTR3Cm91Uk8zQytpWHpZWG5lUGRSREdVTkhvSnhNcHNNOEVhNUtWelJjUTY3Q016WDRSc3o4dzJ5cjJoejhjdVMvOTgKY0sxY0U4bnB5R2ZVTmZpbk01QVNTaEx4NGxDZnp5L3hGM3lJWXdJREFRQUJBb0lCQUJWWU16ajNLZU1URWdMLwpkbWljYnJRYk5NcUhOMm5Rc2loQ1pNZCt0QXdRdGg1Tk5SRUtGT20xZDlYOUlBbkFGUHBTbGdjazVUTUdjMk40CmQrRVlzUndGZXBkdVF0WVJLM2hOSmQ1TkY5UjRWakhXOWZGVDZPNGZtbzB0WENaZmhiNkFXV2p6Unl0VGxaRmMKaE9xS3BKaDQ2OVZqVlBMTXl3dmtKN3RJdENFaHl4b0t0VVhwcm45SXBLNnNUa051OTFmMVA4czJNbDd1RlVqYwpJdGhMb3JnMEYyU3RaeEJmVDJGaFRYaFZxRlRJS1pmazFGbnRpbUwyWlQrRXZzQlpnZHYwa2Z1Q2hFdE5jRW1PCnRZc2dKT3ExTWF5M2d0dlk3VDB6WkRtTTIrOVpKQ0JLcm8yV2IxdGw0RHNnaWNkR0I2SlhnTi81aklSMTNmbDUKMTRJd1hza0NnWUVBemtQb1MrTko0QkJkR3RYem5tZWVhRFFQVVU0dkF1R3YyU2VtajR3RG1KRXB6aDdoMWlQZAprVWxmYjcxZ1VMbmk0SDVkVFlyVFpwOElUaXZvM3A1bUNrV3lFV09wMmx4VUZoM3JnVWN6NWt0RUhkejl1bjNoCnFYNVJpTWlkM0Y3dWRIODdqYTdJVi9mUEFGSnlremQrWHNaZGFuT0tPK1UvV0t2ek0rSFEzUThDZ1lFQTF2TWoKdml3dnFxM0FBa0VpN2RlOUxLUE1uS1N5VE9BdHQzS2dqV1RLNU5aQUdqeWpoSGxEbjRCempSS25DWk8xY0lJZwo0Wnl1VzQrUlB5aGQreEFubzVoMVh0Ny9LYzNFaW1ucjBLU0ZmRWVza2NORFIyVHNTdCtjYTl6aFFPTFJ0TWRCCnE5OWZDeFprK1pmcEhpSzJCK0pHVExNdVJRY0tDYU43RldKTkIyMENnWUJhc2k5bGx3WjMySm9uMzZYa3BDbGEKSm5JSnpUZ01xMUlZU1VBSzVJVDhRL0ErNndOZ2xwcXBkTHJiTmtrd2xkdjEzSHFJU3gvVGd1QXpCMG01QWF0YQpudlRDZ3JGQUM5TUplcFNBWHQrcVJyUW44WEU3M0hncWdCbTM3SWJGVEpUTGN0cXIzUXZJNm5VQjdqN2xEc1NwClJjM3pyZVE5bS9yenNZQVo4eFJVN3dLQmdRQ0JYTjg4Q3JlOVRzaHFFdTJFbXZ4ZEswOXZUcWVJSUxzaTFyZk4Kb01XREozWjQwOW5OVm5YZVBwNU1YdGRzcWhyZVZWS1l0WVV4MFp1bW1STEdrSmhxbXN5NGhoaW0vaEcxQTc1SwpXVm1FekZZTmU2aTRCUU00cEk4dFUwZTFsMHlDTWhGUjhTTHdOMUFaN3RUN3NBUkJobXFzcW9IRVJWSkRMc0phCndraDltUUtCZ0NYR2xoZzY4aVMzMldmSWVtYUFRMTJpNFRUUk1FNWppTFl0ZlkyREJTMDBWV3NxY0l1OEFUWm0KVHVoZHBRVG9mKzE3LzFyU0cyYnFaWFA2L0h3ak14OTVIdWlXbjVKSjA3RTduOUVCUDlkQTY0K0lHdWlvd0h5RAo2a3g3VVhuTUtTYXdiV2JxZ1JGZTFOZEdLbkh0ZE5GOGxndEdjdytxUTk3YkIreXFreXMxCi0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg== diff --git a/artifacts/service-api-server.yaml b/artifacts/service-apiserver.yaml similarity index 51% rename from artifacts/service-api-server.yaml rename to artifacts/service-apiserver.yaml index d3d128eb..9a41015e 100644 --- a/artifacts/service-api-server.yaml +++ b/artifacts/service-apiserver.yaml @@ -1,14 +1,14 @@ apiVersion: v1 kind: Service metadata: - name: config-server - namespace: network-system + name: api-server + namespace: sdc-system labels: - app.kubernetes.io/name: config-server + app.kubernetes.io/name: sdc-api-server spec: ports: - name: api-service port: 6443 targetPort: api-service selector: - app.kubernetes.io/name: config-server + app.kubernetes.io/name: sdc-api-server diff --git a/artifacts/service-data-server.yaml b/artifacts/service-data-server.yaml index 9dfc4801..c69cb864 100644 --- a/artifacts/service-data-server.yaml +++ b/artifacts/service-data-server.yaml @@ -2,14 +2,17 @@ apiVersion: v1 kind: Service metadata: name: data-server - namespace: network-system + namespace: sdc-system labels: - app.kubernetes.io/name: config-server + app.kubernetes.io/name: sdc-data-server-controller spec: ports: - name: data-service port: 56000 protocol: TCP targetPort: data-service + clusterIP: None selector: sdcio.dev/data-server: "true" + +#...svc.cluster.local \ No newline at end of file diff --git a/artifacts/service-metrics.yaml b/artifacts/service-metrics.yaml index 722bb3f9..a604e0cb 100644 --- a/artifacts/service-metrics.yaml +++ b/artifacts/service-metrics.yaml @@ -1,10 +1,10 @@ apiVersion: v1 kind: Service metadata: - name: config-server-metrics - namespace: network-system + name: sdc-controller-server-metrics + namespace: sdc-system labels: - app.kubernetes.io/name: config-server + app.kubernetes.io/name: sdc-controller spec: ports: - name: metrics @@ -12,4 +12,4 @@ spec: protocol: TCP targetPort: 8443 selector: - app.kubernetes.io/name: config-server + app.kubernetes.io/name: sdc-controller diff --git a/artifacts/service-schema-server.yaml b/artifacts/service-schema-server.yaml new file mode 100644 index 00000000..3fdc5db0 --- /dev/null +++ b/artifacts/service-schema-server.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: schema-server + namespace: sdc-system + labels: + app.kubernetes.io/name: sdc-data-server-controller +spec: + ports: + - name: data-service + port: 56000 + protocol: TCP + targetPort: data-service + clusterIP: None + selector: + sdcio.dev/schema-server: "true" + +#...svc.cluster.local \ No newline at end of file diff --git a/artifacts/service-target-metrics.yaml b/artifacts/service-target-metrics.yaml index 8f7d66ec..e3053a03 100644 --- a/artifacts/service-target-metrics.yaml +++ b/artifacts/service-target-metrics.yaml @@ -1,10 +1,10 @@ apiVersion: v1 kind: Service metadata: - name: config-server-target-metrics - namespace: network-system + name: sdc-controller-target-metrics + namespace: sdc-system labels: - app.kubernetes.io/name: config-server + app.kubernetes.io/name: sdc-controller spec: ports: - name: targetmetrics @@ -12,4 +12,4 @@ spec: protocol: TCP targetPort: 9443 selector: - app.kubernetes.io/name: config-server + app.kubernetes.io/name: sdc-controller diff --git a/artifacts/deployment.yaml b/artifacts/statefulset-data-server.yaml similarity index 61% rename from artifacts/deployment.yaml rename to artifacts/statefulset-data-server.yaml index 5d6ee322..b002f9d1 100644 --- a/artifacts/deployment.yaml +++ b/artifacts/statefulset-data-server.yaml @@ -1,45 +1,33 @@ apiVersion: apps/v1 -kind: Deployment +kind: StatefulSet metadata: - name: config-server - namespace: network-system + name: data-server-controller + namespace: sdc-system labels: - app.kubernetes.io/name: config-server + app.kubernetes.io/name: sdc-data-server-controller spec: - strategy: - type: Recreate replicas: 1 selector: matchLabels: - app.kubernetes.io/name: config-server + app.kubernetes.io/name: sdc-data-server-controller template: metadata: labels: - app.kubernetes.io/name: config-server + app.kubernetes.io/name: sdc-data-server-controller sdcio.dev/data-server: "true" + sdcio.dev/schema-server: "true" spec: securityContext: runAsUser: 10000 runAsGroup: 10000 fsGroup: 10000 - serviceAccountName: config-server + serviceAccountName: data-server-controller containers: - - name: config-server - image: input.context[0].data.configServerImage + - name: controller + image: input.context[0].data.sdcControllerImage imagePullPolicy: Always command: - - /app/config-server - args: - - "--tls-cert-file=/apiserver.local.config/certificates/tls.crt" - - "--tls-private-key-file=/apiserver.local.config/certificates/tls.key" - #- "--feature-gates=APIPriorityAndFairness=false" - - "--audit-log-path=-" - - "--audit-log-maxage=0" - - "--audit-log-maxbackup=0" - - "--secure-port=6443" - ports: - - name: api-service - containerPort: 6443 + - /app/controller env: - name: POD_IP valueFrom: @@ -60,6 +48,10 @@ spec: fieldRef: apiVersion: v1 fieldPath: status.hostIP + - name: LOCAL_DATASERVER + value: "true" + - name: METRIC_PORT + value: "8443" - name: PPROF_PORT value: "8081" - name: REVERTIVE @@ -74,28 +66,21 @@ spec: value: "true" - name: ENABLE_TARGETRECOVERYSERVER value: "true" - - name: ENABLE_DISCOVERYRULE - value: "true" + #- name: ENABLE_DISCOVERYRULE + # value: "true" - name: ENABLE_SCHEMA value: "true" #- name: ENABLE_CONFIG # value: "true" - - name: ENABLE_CONFIGSET - value: "true" - - name: ENABLE_WORKSPACE - value: "true" - - name: ENABLE_ROLLOUT - value: "true" + #- name: ENABLE_CONFIGSET + # value: "true" + #- name: ENABLE_WORKSPACE + # value: "true" + #- name: ENABLE_ROLLOUT + # value: "true" volumeMounts: - - name: apiserver-certs - mountPath: /apiserver.local.config/certificates - readOnly: true - - name: config-store - mountPath: /config - name: schema-store mountPath: /schemas - - name: workspace-store - mountPath: /workspace - name: data-server image: input.context[0].data.dataServerImage imagePullPolicy: Always @@ -111,29 +96,21 @@ spec: mountPath: /config - name: cache mountPath: /cached/caches - - name: schema-store - mountPath: /schemas - name: schema-db mountPath: /schemadb + - name: schema-store + mountPath: /schemas volumes: - name: data-server-config configMap: name: data-server - - name: apiserver-certs - secret: - secretName: config-server-cert - name: cache emptyDir: sizeLimit: 10Gi - - name: config-store - persistentVolumeClaim: - claimName: pvc-config-store - - name: schema-store # yang file from git - persistentVolumeClaim: - claimName: pvc-schema-store - name: schema-db # persistent schema obj from the parsed yang files persistentVolumeClaim: claimName: pvc-schema-db - - name: workspace-store # git workspace + - name: schema-store # yang file from git persistentVolumeClaim: - claimName: pvc-workspace-store + claimName: pvc-schema-store + diff --git a/artifacts/token.yaml b/artifacts/token.yaml index d87dc4c5..34b2d5db 100644 --- a/artifacts/token.yaml +++ b/artifacts/token.yaml @@ -1,8 +1,8 @@ apiVersion: v1 kind: Secret metadata: - name: config-server-token - namespace: network-system + name: sdc-controller-token + namespace: sdc-system annotations: - kubernetes.io/service-account.name: config-server + kubernetes.io/service-account.name: sdc-controller type: kubernetes.io/service-account-token \ No newline at end of file diff --git a/main.go b/cmd/api-server/main.go similarity index 69% rename from main.go rename to cmd/api-server/main.go index 027a5f27..27f53591 100644 --- a/main.go +++ b/cmd/api-server/main.go @@ -1,5 +1,5 @@ /* -Copyright 2024 Nokia. +Copyright 2025 Nokia. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -24,28 +24,20 @@ import ( "log/slog" "os" "strconv" - "strings" - "time" "github.com/henderiw/apiserver-builder/pkg/builder" "github.com/henderiw/apiserver-store/pkg/db/badgerdb" - memstore "github.com/henderiw/apiserver-store/pkg/storebackend/memory" "github.com/henderiw/logger/log" sdcconfig "github.com/sdcio/config-server/apis/config" "github.com/sdcio/config-server/apis/config/handlers" configv1alpha1 "github.com/sdcio/config-server/apis/config/v1alpha1" invv1alpha1 "github.com/sdcio/config-server/apis/inv/v1alpha1" "github.com/sdcio/config-server/pkg/generated/openapi" - "github.com/sdcio/config-server/pkg/output/prometheusserver" - "github.com/sdcio/config-server/pkg/reconcilers" _ "github.com/sdcio/config-server/pkg/reconcilers/all" - "github.com/sdcio/config-server/pkg/reconcilers/ctrlconfig" configblameregistry "github.com/sdcio/config-server/pkg/registry/configblame" genericregistry "github.com/sdcio/config-server/pkg/registry/generic" "github.com/sdcio/config-server/pkg/registry/options" runningconfigregistry "github.com/sdcio/config-server/pkg/registry/runningconfig" - sdcctx "github.com/sdcio/config-server/pkg/sdc/ctx" - "github.com/sdcio/config-server/pkg/target" "go.uber.org/zap/zapcore" "k8s.io/apimachinery/pkg/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" @@ -53,24 +45,21 @@ import ( "k8s.io/component-base/logs" "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/config" - "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/metrics/filters" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" ) var ( - schemaBaseDir = "/schemas" configDir = "/config" - workspaceDir = "/workspace" ) func main() { logs.InitLogs() defer logs.FlushLogs() - l := log.NewLogger(&log.HandlerOptions{Name: "config-server-logger", AddSource: false}) + l := log.NewLogger(&log.HandlerOptions{Name: "sdc-api-server-logger", AddSource: false}) slog.SetDefault(l) ctx := log.IntoContext(context.Background(), l) log := log.FromContext(ctx) @@ -95,9 +84,10 @@ func main() { } } + var tlsOpts []func(*tls.Config) metricsServerOptions := metricsserver.Options{ - BindAddress: ":8443", + BindAddress: MetricBindAddress(), SecureServing: true, // FilterProvider is used to protect the metrics endpoint with authn/authz. // These configurations ensure that only authorized users and service accounts @@ -109,13 +99,11 @@ func main() { // this setup is not recommended for production. TLSOpts: tlsOpts, } + mgr_options := ctrl.Options{ Scheme: runScheme, Metrics: metricsServerOptions, - Controller: config.Controller{ - MaxConcurrentReconciles: 16, - }, } if port := IsPProfEnabled(); port != nil { mgr_options.PprofBindAddress = *port @@ -127,67 +115,11 @@ func main() { os.Exit(1) } - targetStore := memstore.NewStore[*target.Context]() - // TODO dataServer/schemaServer -> this should be decoupled in a scaled out environment - time.Sleep(5 * time.Second) - dataServerStore := memstore.NewStore[sdcctx.DSContext]() - if err := sdcctx.CreateDataServerClient(ctx, dataServerStore, mgr.GetClient()); err != nil { - log.Error("cannot create data server", "error", err.Error()) - os.Exit(1) - } - schemaServerStore := memstore.NewStore[sdcctx.SSContext]() - if err := sdcctx.CreateSchemaServerClient(ctx, schemaServerStore, mgr.GetClient()); err != nil { - log.Error("cannot create schema server", "error", err.Error()) - os.Exit(1) - } - - // SchemaServerBaseDir is overwritable via Environment var - if envDir, found := os.LookupEnv("SDC_SCHEMA_SERVER_BASE_DIR"); found { - schemaBaseDir = envDir - } - - // SchemaServerBaseDir is overwritable via Environment var + // ConfigServerBaseDir is overwritable via Environment var if envDir, found := os.LookupEnv("SDC_CONFIG_DIR"); found { configDir = envDir } - // SchemaServerBaseDir is overwritable via Environment var - if envDir, found := os.LookupEnv("SDC_WORKSPACE_DIR"); found { - workspaceDir = envDir - } - - targetHandler := target.NewTargetHandler(mgr.GetClient(), targetStore) - - ctrlCfg := &ctrlconfig.ControllerConfig{ - TargetStore: targetStore, - DataServerStore: dataServerStore, - SchemaServerStore: schemaServerStore, - SchemaDir: schemaBaseDir, - TargetHandler: targetHandler, - WorkspaceDir: workspaceDir, - } - for name, reconciler := range reconcilers.Reconcilers { - log.Info("reconciler", "name", name, "enabled", IsReconcilerEnabled(name)) - if IsReconcilerEnabled(name) { - _, err := reconciler.SetupWithManager(ctx, mgr, ctrlCfg) - if err != nil { - log.Error("cannot add controllers to manager", "err", err.Error()) - os.Exit(1) - } - } - } - - promserver := prometheusserver.NewServer(&prometheusserver.Config{ - Address: ":9443", - TargetStore: targetStore, - }) - go func() { - if err := promserver.Start(ctx); err != nil { - log.Error("cannot start promerver", "err", err.Error()) - os.Exit(1) - } - }() - db, err := badgerdb.OpenDB(ctx, configDir) if err != nil { log.Error("cannot open db", "err", err.Error()) @@ -200,7 +132,9 @@ func main() { DB: db, } - configHandler := handlers.ConfigStoreHandler{Handler: targetHandler} + configHandler := handlers.ConfigStoreHandler{ + Client: mgr.GetClient(), + } configregistryOptions := *registryOptions configregistryOptions.DryRunCreateFn = configHandler.DryRunCreateFn @@ -216,14 +150,14 @@ func main() { // no storage required since the targetStore is acting as the storage for the running config resource runningConfigStorageProvider := runningconfigregistry.NewStorageProvider(ctx, sdcconfig.BuildEmptyRunningConfig(), &options.Options{ Client: mgr.GetClient(), - TargetStore: targetStore, + //TargetStore: targetStore, }) // no storage required since the targetStore is acting as the storage for the running config resource configBlameStorageProvider := configblameregistry.NewStorageProvider(ctx, sdcconfig.BuildEmptyConfigBlame(), &options.Options{ Client: mgr.GetClient(), - TargetStore: targetStore, + //TargetStore: targetStore, }) - + go func() { if err := builder.APIServer. WithServerName("config-server"). @@ -273,13 +207,10 @@ func IsPProfEnabled() *string { return nil } -// IsReconcilerEnabled checks if an environment variable `ENABLE_` exists -// return "true" if the var is set and is not equal to "false". -func IsReconcilerEnabled(reconcilerName string) bool { - if val, found := os.LookupEnv(fmt.Sprintf("ENABLE_%s", strings.ToUpper(reconcilerName))); found { - if strings.ToLower(val) != "false" { - return true - } + +func MetricBindAddress() string { + if val, found := os.LookupEnv("METRIC_PORT"); found { + return fmt.Sprintf(":%s", val) } - return false -} + return ":8443" +} \ No newline at end of file diff --git a/cmd/controller/main.go b/cmd/controller/main.go new file mode 100644 index 00000000..61e9ba85 --- /dev/null +++ b/cmd/controller/main.go @@ -0,0 +1,222 @@ +/* +Copyright 2024 Nokia. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +//go:generate apiserver-runtime-gen +package main + +import ( + "context" + "crypto/tls" + "fmt" + "log/slog" + "os" + "strconv" + "strings" + "time" + + memstore "github.com/henderiw/apiserver-store/pkg/storebackend/memory" + "github.com/henderiw/logger/log" + configv1alpha1 "github.com/sdcio/config-server/apis/config/v1alpha1" + invv1alpha1 "github.com/sdcio/config-server/apis/inv/v1alpha1" + "github.com/sdcio/config-server/pkg/output/prometheusserver" + "github.com/sdcio/config-server/pkg/reconcilers" + _ "github.com/sdcio/config-server/pkg/reconcilers/all" + "github.com/sdcio/config-server/pkg/reconcilers/ctrlconfig" + sdcctx "github.com/sdcio/config-server/pkg/sdc/ctx" + sdctarget "github.com/sdcio/config-server/pkg/sdc/target" + "go.uber.org/zap/zapcore" + "k8s.io/apimachinery/pkg/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + _ "k8s.io/client-go/plugin/pkg/client/auth" // register auth plugins + "k8s.io/component-base/logs" + "k8s.io/utils/ptr" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/config" + "sigs.k8s.io/controller-runtime/pkg/healthz" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/metrics/filters" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" +) + +var ( + schemaBaseDir = "/schemas" + workspaceDir = "/workspace" +) + +func main() { + logs.InitLogs() + defer logs.FlushLogs() + + l := log.NewLogger(&log.HandlerOptions{Name: "sdc-controller-logger", AddSource: false}) + slog.SetDefault(l) + ctx := log.IntoContext(context.Background(), l) + log := log.FromContext(ctx) + + opts := zap.Options{ + TimeEncoder: zapcore.RFC3339NanoTimeEncoder, + } + + ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) + + // setup controllers + runScheme := runtime.NewScheme() + // add the core object to the scheme + for _, api := range (runtime.SchemeBuilder{ + clientgoscheme.AddToScheme, + configv1alpha1.AddToScheme, + invv1alpha1.AddToScheme, + }) { + if err := api(runScheme); err != nil { + log.Error("cannot add scheme", "err", err) + os.Exit(1) + } + } + + var tlsOpts []func(*tls.Config) + metricsServerOptions := metricsserver.Options{ + BindAddress: MetricBindAddress(), + SecureServing: true, + // FilterProvider is used to protect the metrics endpoint with authn/authz. + // These configurations ensure that only authorized users and service accounts + // can access the metrics endpoint. The RBAC are configured in 'config/rbac/kustomization.yaml'. More info: + // https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/filters#WithAuthenticat + FilterProvider: filters.WithAuthenticationAndAuthorization, + // If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically + // generate self-signed certificates for the metrics server. While convenient for development and testing, + // this setup is not recommended for production. + TLSOpts: tlsOpts, + } + + mgr_options := ctrl.Options{ + Scheme: runScheme, + Metrics: metricsServerOptions, + Controller: config.Controller{ + MaxConcurrentReconciles: 16, + }, + } + if port := IsPProfEnabled(); port != nil { + mgr_options.PprofBindAddress = *port + } + + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), mgr_options) + if err != nil { + log.Error("cannot start manager", "err", err) + os.Exit(1) + } + + // SchemaServerBaseDir is overwritable via Environment var + if envDir, found := os.LookupEnv("SDC_SCHEMA_SERVER_BASE_DIR"); found { + schemaBaseDir = envDir + } + + // WorkspaceBaseDir is overwritable via Environment var + if envDir, found := os.LookupEnv("SDC_WORKSPACE_DIR"); found { + workspaceDir = envDir + } + + targetStore := memstore.NewStore[*sdctarget.Context]() + + ctrlCfg := &ctrlconfig.ControllerConfig{ + TargetStore: targetStore, + SchemaDir: schemaBaseDir, + WorkspaceDir: workspaceDir, + } + if IsLocalDataServerEnabled() { + time.Sleep(5 * time.Second) + dataServerStore := memstore.NewStore[sdcctx.DSContext]() + if err := sdcctx.CreateDataServerClient(ctx, dataServerStore, mgr.GetClient()); err != nil { + log.Error("cannot create data server", "error", err.Error()) + os.Exit(1) + } + ctrlCfg.DataServerStore = dataServerStore + } + + for name, reconciler := range reconcilers.Reconcilers { + log.Info("reconciler", "name", name, "enabled", IsReconcilerEnabled(name)) + if IsReconcilerEnabled(name) { + _, err := reconciler.SetupWithManager(ctx, mgr, ctrlCfg) + if err != nil { + log.Error("cannot add controllers to manager", "err", err.Error()) + os.Exit(1) + } + } + } + + promserver := prometheusserver.NewServer(&prometheusserver.Config{ + Address: ":9443", + TargetStore: targetStore, + }) + go func() { + if err := promserver.Start(ctx); err != nil { + log.Error("cannot start promerver", "err", err.Error()) + os.Exit(1) + } + }() + + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { + log.Error("unable to set up health check", "error", err.Error()) + os.Exit(1) + } + if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { + log.Error("unable to set up ready check", "error", err.Error()) + os.Exit(1) + } + + log.Info("starting manager") + if err := mgr.Start(ctx); err != nil { + log.Error("problem running manager", "error", err.Error()) + os.Exit(1) + } +} + +func IsPProfEnabled() *string { + if val, found := os.LookupEnv("PPROF_PORT"); found { + port, err := strconv.Atoi(val) + if err != nil { + return nil + } + return ptr.To(fmt.Sprintf("127.0.0.1:%d", port)) + } + return nil +} + +// IsReconcilerEnabled checks if an environment variable `ENABLE_` exists +// return "true" if the var is set and is not equal to "false". +func IsReconcilerEnabled(reconcilerName string) bool { + if val, found := os.LookupEnv(fmt.Sprintf("ENABLE_%s", strings.ToUpper(reconcilerName))); found { + if strings.ToLower(val) != "false" { + return true + } + } + return false +} + + +func IsLocalDataServerEnabled() bool { + if val, found := os.LookupEnv("LOCAL_DATASERVER"); found { + if strings.ToLower(val) == "true" { + return true + } + } + return false +} + +func MetricBindAddress() string { + if val, found := os.LookupEnv("METRIC_PORT"); found { + return fmt.Sprintf(":%s", val) + } + return ":8443" +} \ No newline at end of file diff --git a/crds/config.sdcio.dev_deviations.yaml b/crds/config.sdcio.dev_deviations.yaml index fe04e11b..431b6e4a 100644 --- a/crds/config.sdcio.dev_deviations.yaml +++ b/crds/config.sdcio.dev_deviations.yaml @@ -67,7 +67,7 @@ spec: type: array type: object status: - description: DeviationStatus defines the observed state of Deviation + description: DeviationStatus defines the observed state of Deviationgit properties: conditions: description: Conditions of the resource. diff --git a/crds/config.sdcio.dev_sensitiveconfigs.yaml b/crds/config.sdcio.dev_sensitiveconfigs.yaml new file mode 100644 index 00000000..4d5341f8 --- /dev/null +++ b/crds/config.sdcio.dev_sensitiveconfigs.yaml @@ -0,0 +1,431 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + name: sensitiveconfigs.config.sdcio.dev +spec: + group: config.sdcio.dev + names: + categories: + - sdc + kind: SensitiveConfig + listKind: SensitiveConfigList + plural: sensitiveconfigs + singular: sensitiveconfig + scope: Namespaced + versions: + - name: config + schema: + openAPIV3Schema: + description: "\tSensitiveConfig defines the Schema for the SensitiveConfig + API" + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: SensitiveConfigSpec defines the desired state of SensitiveConfig + properties: + config: + description: SensitiveConfig defines the SensitiveConfiguration to + be applied to a target device + items: + properties: + path: + description: Path defines the path relative to which the value + is applicable + type: string + secretKeyRef: + $ref: '#/definitions/k8s.io~1api~1core~1v1~0SecretKeySelector' + description: SecretKeyRef refers to a secret in the same namesapce + as the config + required: + - path + - secretKeyRef + type: object + type: array + lifecycle: + description: |- + Lifecycle determines the lifecycle policies the resource e.g. delete is orphan or delete + will follow + properties: + deletionPolicy: + description: |- + DeletionPolicy specifies what will happen to the underlying resource + when this resource is deleted - either "delete" or "orphan" the + resource. + type: string + type: object + priority: + description: Priority defines the priority of this SensitiveConfig + format: int64 + type: integer + revertive: + description: Revertive defines if this CR is enabled for revertive + or non revertve operation + type: boolean + required: + - config + type: object + status: + description: SensitiveConfigStatus defines the observed state of SensitiveConfig + properties: + appliedSensitiveConfig: + description: AppliedSensitiveConfig defines the SensitiveConfig applied + to the target + properties: + config: + description: SensitiveConfig defines the SensitiveConfiguration + to be applied to a target device + items: + properties: + path: + description: Path defines the path relative to which the + value is applicable + type: string + secretKeyRef: + $ref: '#/definitions/k8s.io~1api~1core~1v1~0SecretKeySelector' + description: SecretKeyRef refers to a secret in the same + namesapce as the config + required: + - path + - secretKeyRef + type: object + type: array + lifecycle: + description: |- + Lifecycle determines the lifecycle policies the resource e.g. delete is orphan or delete + will follow + properties: + deletionPolicy: + description: |- + DeletionPolicy specifies what will happen to the underlying resource + when this resource is deleted - either "delete" or "orphan" the + resource. + type: string + type: object + priority: + description: Priority defines the priority of this SensitiveConfig + format: int64 + type: integer + revertive: + description: Revertive defines if this CR is enabled for revertive + or non revertve operation + type: boolean + required: + - config + type: object + conditions: + description: Conditions of the resource. + items: + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: |- + type of condition in CamelCase or in foo.example.com/CamelCase. + --- + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be + useful (see .node.status.conditions), the ability to deconflict is important. + The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + deviationGeneration: + description: Deviations generation used for the latest SensitiveConfig + apply + format: int64 + type: integer + lastKnownGoodSchema: + description: LastKnownGoodSchema identifies the last known good schema + used to apply the SensitiveConfig successfully + properties: + type: + description: Schema Type + type: string + vendor: + description: Schema Vendor + type: string + version: + description: Schema Version + type: string + type: object + type: object + type: object + served: true + storage: true + subresources: + status: {} + - name: v1alpha1 + schema: + openAPIV3Schema: + description: SensitiveConfig defines the Schema for the SensitiveConfig API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: SensitiveConfigSpec defines the desired state of SensitiveConfig + properties: + config: + description: SensitiveConfig defines the SensitiveConfiguration to + be applied to a target device + items: + properties: + path: + description: Path defines the path relative to which the value + is applicable + type: string + secretKeyRef: + $ref: '#/definitions/k8s.io~1api~1core~1v1~0SecretKeySelector' + description: SecretKeyRef refers to a secret in the same namesapce + as the config + required: + - path + - secretKeyRef + type: object + type: array + lifecycle: + description: |- + Lifecycle determines the lifecycle policies the resource e.g. delete is orphan or delete + will follow + properties: + deletionPolicy: + default: delete + description: |- + DeletionPolicy specifies what will happen to the underlying resource + when this resource is deleted - either "delete" or "orphan" the + resource. + enum: + - delete + - orphan + type: string + type: object + priority: + description: Priority defines the priority of this SensitiveConfig + format: int64 + type: integer + revertive: + description: Revertive defines if this CR is enabled for revertive + or non revertve operation + type: boolean + required: + - config + type: object + status: + description: SensitiveConfigStatus defines the observed state of SensitiveConfig + properties: + appliedSensitiveConfig: + description: AppliedSensitiveConfig defines the SensitiveConfig applied + to the target + properties: + config: + description: SensitiveConfig defines the SensitiveConfiguration + to be applied to a target device + items: + properties: + path: + description: Path defines the path relative to which the + value is applicable + type: string + secretKeyRef: + $ref: '#/definitions/k8s.io~1api~1core~1v1~0SecretKeySelector' + description: SecretKeyRef refers to a secret in the same + namesapce as the config + required: + - path + - secretKeyRef + type: object + type: array + lifecycle: + description: |- + Lifecycle determines the lifecycle policies the resource e.g. delete is orphan or delete + will follow + properties: + deletionPolicy: + default: delete + description: |- + DeletionPolicy specifies what will happen to the underlying resource + when this resource is deleted - either "delete" or "orphan" the + resource. + enum: + - delete + - orphan + type: string + type: object + priority: + description: Priority defines the priority of this SensitiveConfig + format: int64 + type: integer + revertive: + description: Revertive defines if this CR is enabled for revertive + or non revertve operation + type: boolean + required: + - config + type: object + conditions: + description: Conditions of the resource. + items: + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: |- + type of condition in CamelCase or in foo.example.com/CamelCase. + --- + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be + useful (see .node.status.conditions), the ability to deconflict is important. + The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + deviationGeneration: + description: Deviations generation used for the latest SensitiveConfig + apply + format: int64 + type: integer + lastKnownGoodSchema: + description: LastKnownGoodSchema identifies the last known good schema + used to apply the SensitiveConfig successfully + properties: + type: + description: Schema Type + type: string + vendor: + description: Schema Vendor + type: string + version: + description: Schema Version + type: string + type: object + type: object + type: object + served: true + storage: false + subresources: + status: {} diff --git a/docs/0.0.54.md b/docs/0.0.54.md new file mode 100644 index 00000000..200db617 --- /dev/null +++ b/docs/0.0.54.md @@ -0,0 +1,7 @@ +# Release 0.0.54 + +[ChangeLog](https://github.com/sdcio/config-server/releases) + +## sensitive data + +introduced a new API for sensitive data diff --git a/docs/0.0.55.md b/docs/0.0.55.md new file mode 100644 index 00000000..423ef564 --- /dev/null +++ b/docs/0.0.55.md @@ -0,0 +1,7 @@ +# Release 0.0.55 + +[ChangeLog](https://github.com/sdcio/config-server/releases) + +## split + +splitted configserver in apiserver, configserver (discovery, schema, configset, etc) and configserver/dataserver for target handling diff --git a/goreleaser.dockerfile b/goreleaser.api-server.dockerfile similarity index 92% rename from goreleaser.dockerfile rename to goreleaser.api-server.dockerfile index 559c4cb1..0b3878d6 100644 --- a/goreleaser.dockerfile +++ b/goreleaser.api-server.dockerfile @@ -26,10 +26,10 @@ COPY --from=builder /etc/passwd /etc/group /etc/shadow /etc/ # add-in our ca certificates COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ -COPY --chown=$USERID:$USERID config-server /app/ +COPY --chown=$USERID:$USERID api-server /app/ WORKDIR /app # from now on, run as the unprivileged user USER $USERID -ENTRYPOINT [ "/app/config-server" ] +ENTRYPOINT [ "/app/api-server" ] diff --git a/goreleaser.controller.dockerfile b/goreleaser.controller.dockerfile new file mode 100644 index 00000000..d224a4ee --- /dev/null +++ b/goreleaser.controller.dockerfile @@ -0,0 +1,35 @@ +# Copyright 2024 Nokia +# Licensed under the Apache License 2.0 +# SPDX-License-Identifier: Apache-2.0 +# +FROM golang:1.21 AS builder +ARG USERID=10000 +# no need to include cgo bindings +ENV CGO_ENABLED=0 GOOS=linux GOARCH=amd64 + +# add ca certificates and timezone data files +RUN apt-get install --yes --no-install-recommends ca-certificates tzdata + +# add unprivileged user +RUN adduser --shell /bin/false --uid $USERID --disabled-login --home /app/ --no-create-home --gecos '' app \ + && sed -i -r "/^(app|root)/!d" /etc/group /etc/passwd \ + && sed -i -r 's#^(.*):[^:]*$#\1:/bin/false#' /etc/passwd + +# +#FROM scratch +FROM alpine:latest +ARG USERID=10000 +# add-in our timezone data file +COPY --from=builder /usr/share/zoneinfo /usr/share/zoneinfo +# add-in our unprivileged user +COPY --from=builder /etc/passwd /etc/group /etc/shadow /etc/ +# add-in our ca certificates +COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ + +COPY --chown=$USERID:$USERID controller /app/ +WORKDIR /app + +# from now on, run as the unprivileged user +USER $USERID + +ENTRYPOINT [ "/app/controller" ] diff --git a/pkg/discovery/discoveryrule/rule.go b/pkg/discovery/discoveryrule/rule.go index 7fa5dff6..19cf22e4 100644 --- a/pkg/discovery/discoveryrule/rule.go +++ b/pkg/discovery/discoveryrule/rule.go @@ -26,7 +26,7 @@ import ( "github.com/henderiw/apiserver-store/pkg/storebackend" "github.com/henderiw/apiserver-store/pkg/storebackend/memory" "github.com/henderiw/logger/log" - "github.com/sdcio/config-server/pkg/target" + sdctarget "github.com/sdcio/config-server/pkg/sdc/target" "golang.org/x/sync/semaphore" "sigs.k8s.io/controller-runtime/pkg/client" invv1alpha1 "github.com/sdcio/config-server/apis/inv/v1alpha1" @@ -38,7 +38,7 @@ type DiscoveryRule interface { GetDiscoveryRulConfig() *DiscoveryRuleConfig } -func New(client client.Client, cfg *DiscoveryRuleConfig, targetStore storebackend.Storer[*target.Context]) DiscoveryRule { +func New(client client.Client, cfg *DiscoveryRuleConfig, targetStore storebackend.Storer[*sdctarget.Context]) DiscoveryRule { r := &dr{} r.client = client r.cfg = cfg @@ -51,7 +51,7 @@ type dr struct { client client.Client cfg *DiscoveryRuleConfig protocols *protocols - targetStore storebackend.Storer[*target.Context] + targetStore storebackend.Storer[*sdctarget.Context] children storebackend.Storer[string] diff --git a/pkg/output/prometheusserver/handler_prometheus.go b/pkg/output/prometheusserver/handler_prometheus.go index 44be995c..c7257feb 100644 --- a/pkg/output/prometheusserver/handler_prometheus.go +++ b/pkg/output/prometheusserver/handler_prometheus.go @@ -26,7 +26,7 @@ import ( "github.com/openconfig/gnmi/proto/gnmi" "github.com/openconfig/gnmic/pkg/api/path" "github.com/prometheus/client_golang/prometheus" - "github.com/sdcio/config-server/pkg/target" + sdctarget "github.com/sdcio/config-server/pkg/sdc/target" ) // Describe implements prometheus.Collector @@ -37,7 +37,7 @@ func (r *PrometheusServer) Collect(ch chan<- prometheus.Metric) { log:= log.FromContext(ctx) keys := []storebackend.Key{} - if err := r.targetStore.List(ctx, func(ctx1 context.Context, k storebackend.Key, tctx *target.Context) { + if err := r.targetStore.List(ctx, func(ctx1 context.Context, k storebackend.Key, tctx *sdctarget.Context) { keys = append(keys, k) }); err != nil { log.Error("target list failed", "err", err) diff --git a/pkg/output/prometheusserver/prommetric.go b/pkg/output/prometheusserver/prommetric.go index d8b039b6..733f781a 100644 --- a/pkg/output/prometheusserver/prommetric.go +++ b/pkg/output/prometheusserver/prommetric.go @@ -33,7 +33,7 @@ import ( "github.com/prometheus/client_golang/prometheus" dto "github.com/prometheus/client_model/go" "github.com/prometheus/prometheus/prompb" - "github.com/sdcio/config-server/pkg/target" + sdctarget "github.com/sdcio/config-server/pkg/sdc/target" ) const ( @@ -57,7 +57,7 @@ type PromMetric struct { } // TODO get user input -func NewPromMetric(subName string, tctx *target.Context, update *gnmi.Update) (*PromMetric, error) { +func NewPromMetric(subName string, tctx *sdctarget.Context, update *gnmi.Update) (*PromMetric, error) { val, err := getValue(update.GetVal()) if err != nil { return nil, fmt.Errorf("prometheus metric cannot get typed value, err: %s", err) diff --git a/pkg/output/prometheusserver/server.go b/pkg/output/prometheusserver/server.go index 81eb6e01..efa1d34d 100644 --- a/pkg/output/prometheusserver/server.go +++ b/pkg/output/prometheusserver/server.go @@ -28,13 +28,13 @@ import ( "github.com/henderiw/logger/log" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" - "github.com/sdcio/config-server/pkg/target" + sdctarget "github.com/sdcio/config-server/pkg/sdc/target" certutil "k8s.io/client-go/util/cert" ) type Config struct { Address string - TargetStore storebackend.Storer[*target.Context] + TargetStore storebackend.Storer[*sdctarget.Context] } func NewServer(c *Config) *PrometheusServer { @@ -47,7 +47,7 @@ func NewServer(c *Config) *PrometheusServer { type PrometheusServer struct { address string - targetStore storebackend.Storer[*target.Context] + targetStore storebackend.Storer[*sdctarget.Context] regex *regexp.Regexp // dynamic cancel func() diff --git a/pkg/reconcilers/all/all.go b/pkg/reconcilers/all/all.go index a511b029..ed606c30 100644 --- a/pkg/reconcilers/all/all.go +++ b/pkg/reconcilers/all/all.go @@ -17,7 +17,6 @@ limitations under the License. package all import ( - //_ "github.com/sdcio/config-server/pkg/reconcilers/config" _ "github.com/sdcio/config-server/pkg/reconcilers/configset" _ "github.com/sdcio/config-server/pkg/reconcilers/discoveryrule" _ "github.com/sdcio/config-server/pkg/reconcilers/schema" diff --git a/pkg/reconcilers/config/reconciler.go b/pkg/reconcilers/config/reconciler.go deleted file mode 100644 index 7399a97f..00000000 --- a/pkg/reconcilers/config/reconciler.go +++ /dev/null @@ -1,395 +0,0 @@ -/* -Copyright 2024 Nokia. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package config - -import ( - "context" - "encoding/json" - "fmt" - "reflect" - "time" - - "github.com/henderiw/logger/log" - "github.com/pkg/errors" - condv1alpha1 "github.com/sdcio/config-server/apis/condition/v1alpha1" - "github.com/sdcio/config-server/apis/config" - configv1alpha1 "github.com/sdcio/config-server/apis/config/v1alpha1" - invv1alpha1 "github.com/sdcio/config-server/apis/inv/v1alpha1" - "github.com/sdcio/config-server/pkg/reconcilers" - "github.com/sdcio/config-server/pkg/reconcilers/ctrlconfig" - "github.com/sdcio/config-server/pkg/reconcilers/eventhandler" - "github.com/sdcio/config-server/pkg/reconcilers/resource" - "github.com/sdcio/config-server/pkg/target" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/equality" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/tools/record" - "k8s.io/utils/ptr" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/event" -) - -func init() { - reconcilers.Register(crName, &reconciler{}) -} - -const ( - crName = "config" - reconcilerName = "ConfigController" - finalizer = "config.config.sdcio.dev/finalizer" - // errors - errGetCr = "cannot get cr" - errUpdateDataStore = "cannot update datastore" - errUpdateStatus = "cannot update status" -) - -// SetupWithManager sets up the controller with the Manager. -func (r *reconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, c interface{}) (map[schema.GroupVersionKind]chan event.GenericEvent, error) { - cfg, ok := c.(*ctrlconfig.ControllerConfig) - if !ok { - return nil, fmt.Errorf("cannot initialize, expecting controllerConfig, got: %s", reflect.TypeOf(c).Name()) - } - - r.client = mgr.GetClient() - r.finalizer = resource.NewAPIFinalizer(mgr.GetClient(), finalizer, reconcilerName) - r.targetHandler = cfg.TargetHandler - r.recorder = mgr.GetEventRecorderFor(reconcilerName) - - return nil, ctrl.NewControllerManagedBy(mgr). - Named(reconcilerName). - For(&configv1alpha1.Config{}). - Watches(&invv1alpha1.Target{}, &eventhandler.TargetForConfigEventHandler{Client: mgr.GetClient(), ControllerName: reconcilerName}). - Watches(&configv1alpha1.Deviation{}, &eventhandler.DeviationForConfigEventHandler{Client: mgr.GetClient(), ControllerName: reconcilerName}). - Complete(r) -} - -type reconciler struct { - client client.Client - finalizer *resource.APIFinalizer - targetHandler target.TargetHandler - recorder record.EventRecorder -} - -func (r *reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - ctx = ctrlconfig.InitContext(ctx, reconcilerName, req.NamespacedName) - log := log.FromContext(ctx) - log.Info("reconcile") - - cfg := &configv1alpha1.Config{} - if err := r.client.Get(ctx, req.NamespacedName, cfg); err != nil { - // if the resource no longer exists the reconcile loop is done - if resource.IgnoreNotFound(err) != nil { - log.Error(errGetCr, "error", err) - return ctrl.Result{}, errors.Wrap(resource.IgnoreNotFound(err), errGetCr) - } - return ctrl.Result{}, nil - } - cfgOrig := cfg.DeepCopy() - internalcfg := &config.Config{} - if err := configv1alpha1.Convert_v1alpha1_Config_To_config_Config(cfg, internalcfg, nil); err != nil { - if err := r.handleError(ctx, cfgOrig, cfg, "cannot convert config", err, true); err != nil { - return ctrl.Result{}, err - } - return ctrl.Result{}, errors.Wrap(r.client.Status().Update(ctx, cfg), errUpdateStatus) - } - - targetKey, err := config.GetTargetKey(cfg.GetLabels()) - if err != nil { - // this should never fail since validation covered this - return ctrl.Result{}, nil - } - - if !cfg.GetDeletionTimestamp().IsZero() { - if _, err := r.targetHandler.DeleteIntent(ctx, targetKey, internalcfg, false); err != nil { - if errors.Is(err, target.ErrLookup) { - log.Warn("deleted config, target unavailable", "config", req, "err", err) - // Since the target is not available we delete the resource - // The target config might not be deleted - if err := r.finalizer.RemoveFinalizer(ctx, cfg); err != nil { - return ctrl.Result{Requeue: true}, - errors.Wrap(r.handleError(ctx, cfgOrig, cfg, "cannot delete finalizer", err, true), errUpdateStatus) - } - return ctrl.Result{}, nil - } - // all grpc errors except resource exhausted will not retry - // and a human need to intervene - var txErr *target.TransactionError - if errors.As(err, &txErr) && txErr.Recoverable { - // Retry logic for recoverable errors - return ctrl.Result{Requeue: true, RequeueAfter: 5 * time.Second}, - errors.Wrap(r.handleError(ctx, cfgOrig, cfg, "set intent failed (recoverable)", err, true), errUpdateStatus) - } - - return ctrl.Result{}, errors.Wrap(r.handleError(ctx, cfgOrig, cfg, "delete intent failed", err, false), errUpdateStatus) - - } - - // delete the deviation -> should happen using owner references - - if err := r.finalizer.RemoveFinalizer(ctx, cfg); err != nil { - return ctrl.Result{Requeue: true}, - errors.Wrap(r.handleError(ctx, cfgOrig, cfg, "cannot delete finalizer", err, true), errUpdateStatus) - } - return ctrl.Result{}, nil - } - - if err := r.finalizer.AddFinalizer(ctx, cfg); err != nil { - return ctrl.Result{Requeue: true}, - errors.Wrap(r.handleError(ctx, cfgOrig, cfg, "cannot add finalizer", err, true), errUpdateStatus) - } - - // apply an empty deviation or fetch the deviation from the apiserver if it exists - deviation, err := r.applyDeviation(ctx, cfg) - if err != nil { - log.Info("applying deviation failed") - return ctrl.Result{}, errors.Wrap(r.handleError(ctx, cfgOrig, cfg, "cannot apply deviation", err, true), errUpdateStatus) - } - - if _, _, err := r.targetHandler.GetTargetContext(ctx, targetKey); err != nil { - log.Info("applying config -> target not ready") - return ctrl.Result{}, errors.Wrap(r.handleError(ctx, cfgOrig, cfg, "target not ready", err, true), errUpdateStatus) - } - - log.Info("applying config -> target ready", "revertive", cfg.IsRevertive(), "changed deviation", cfg.HashDeviationGenerationChanged(deviation)) - - // check if we have to reapply the config - // if condition is false -> reapply the config - // if the applied Config is not set -> reapply the config - // if the applied Config is different than the spec -> reapply the config - // if the deviation is having the reason xx -> reapply the config - if cfg.IsConditionReady() && - cfg.Status.AppliedConfig != nil && - cfg.Spec.GetShaSum(ctx) == cfg.Status.AppliedConfig.GetShaSum(ctx) && - !cfg.HashDeviationGenerationChanged(deviation) { - log.Info("not reapplying the config since nothing changed") - return ctrl.Result{}, nil - } - - // Check if we got an unrecoverable error and if the resourceVersion has not changed we can stop here - if !cfg.IsRecoverable() { - return ctrl.Result{}, nil - } - - // in non revertive mode we dont include the deviation in the set intent - // in revertive mode we include it is it exists. - internalDeviation := &config.Deviation{} - if err := configv1alpha1.Convert_v1alpha1_Deviation_To_config_Deviation(&deviation, internalDeviation, nil); err != nil { - if err := r.handleError(ctx, cfgOrig, cfg, "cannot convert deviation", err, true); err != nil { - return ctrl.Result{}, err - } - return ctrl.Result{}, errors.Wrap(r.client.Status().Update(ctx, cfg), errUpdateStatus) - } - - - internalSchema, warnings, err := r.targetHandler.SetIntent(ctx, targetKey, internalcfg, *internalDeviation, false) - if err != nil { - // TODO distinguish between recoeverable and non recoverable - var txErr *target.TransactionError - if errors.As(err, &txErr) && txErr.Recoverable { - // Retry logic for recoverable errors - return ctrl.Result{Requeue: true, RequeueAfter: 5 * time.Second}, - errors.Wrap(r.handleError(ctx, cfgOrig, cfg, processMessageWithWarning("set intent failed (recoverable)", warnings), err, true), errUpdateStatus) - } - return ctrl.Result{}, errors.Wrap(r.handleError(ctx, cfgOrig, cfg, processMessageWithWarning("set intent failed", warnings), err, true), errUpdateStatus) - } - - schema := &configv1alpha1.ConfigStatusLastKnownGoodSchema{} - if err := configv1alpha1.Convert_config_ConfigStatusLastKnownGoodSchema_To_v1alpha1_ConfigStatusLastKnownGoodSchema(internalSchema, schema, nil); err != nil { - return ctrl.Result{Requeue: true}, - errors.Wrap(r.handleError(ctx, cfgOrig, cfg, processMessageWithWarning("cannot convert schema", warnings), err, true), errUpdateStatus) - } - - // in the revertive case we can delete the deviation - if cfg.IsRevertive() { - if err := r.clearDeviation(ctx, &deviation); err != nil { - return ctrl.Result{Requeue: true}, - errors.Wrap(r.handleError(ctx, cfgOrig, cfg, processMessageWithWarning("cannot delete deviation", warnings), err, true), errUpdateStatus) - } - } - - return ctrl.Result{}, errors.Wrap(r.handleSuccess(ctx, cfgOrig, schema, deviation, warnings), errUpdateStatus) -} - -func (r *reconciler) handleSuccess(ctx context.Context, cfg *configv1alpha1.Config, schema *configv1alpha1.ConfigStatusLastKnownGoodSchema, deviation configv1alpha1.Deviation, msg string) error { - log := log.FromContext(ctx) - log.Debug("handleSuccess", "key", cfg.GetNamespacedName(), "status old", cfg.DeepCopy().Status) - // take a snapshot of the current object - //patch := client.MergeFrom(cfg.DeepCopy()) - // update status - newConfig := configv1alpha1.BuildConfig( - metav1.ObjectMeta{ - Namespace: cfg.Namespace, - Name: cfg.Name, - }, - configv1alpha1.ConfigSpec{}, - configv1alpha1.ConfigStatus{}, - ) - - newConfig.SetConditions(cfg.GetCondition(condv1alpha1.ConditionTypeReady)) - newConfig.SetConditions(condv1alpha1.ReadyWithMsg(msg)) - newConfig.Status.LastKnownGoodSchema = schema - newConfig.Status.AppliedConfig = &cfg.Spec - - if cfg.IsRevertive() { - newConfig.Status.DeviationGeneration = nil - } else { - newConfig.Status.DeviationGeneration = ptr.To(deviation.GetGeneration()) - } - - deviationChange := cfg.HashDeviationGenerationChanged(deviation) - - if newConfig.GetCondition(condv1alpha1.ConditionTypeReady).Equal(cfg.GetCondition(condv1alpha1.ConditionTypeReady)) && - equalSchema(newConfig.Status.LastKnownGoodSchema, cfg.Status.LastKnownGoodSchema) && - deviationChange && - equalAppliedConfig(newConfig.Status.AppliedConfig, cfg.Status.AppliedConfig) { - log.Info("handleSuccess -> no change") - return nil - } - log.Info("handleSuccess -> changes") - - if !newConfig.GetCondition(condv1alpha1.ConditionTypeReady).Equal(cfg.GetCondition(condv1alpha1.ConditionTypeReady)) { - log.Info("handleSuccess -> condition changed") - } - if !equalSchema(newConfig.Status.LastKnownGoodSchema, cfg.Status.LastKnownGoodSchema) { - log.Info("handleSuccess -> LastKnownGoodSchema changed", "schema-a", newConfig.Status.LastKnownGoodSchema, "schema-b", cfg.Status.LastKnownGoodSchema) - } - if deviationChange { - log.Info("handleSuccess -> Deviations changed", "dev-a", newConfig.Status.DeviationGeneration, "dev-b", cfg.Status.DeviationGeneration) - } - if !equalAppliedConfig(newConfig.Status.AppliedConfig, cfg.Status.AppliedConfig) { - log.Info("handleSuccess -> AppliedConfig changed") - } - - log.Debug("handleSuccess", "key", cfg.GetNamespacedName(), "status new", cfg.Status) - - r.recorder.Eventf(newConfig, corev1.EventTypeNormal, configv1alpha1.ConfigKind, "ready") - - return r.client.Status().Patch(ctx, newConfig, client.Apply, &client.SubResourcePatchOptions{ - PatchOptions: client.PatchOptions{ - FieldManager: reconcilerName, - Force: ptr.To(true), - }, - }) -} - -func equalAppliedConfig(a, b *configv1alpha1.ConfigSpec) bool { - if a == nil && b == nil { - return true - } - if a == nil || b == nil { - return false - } - return equality.Semantic.DeepEqual(*a, *b) -} - -func equalSchema(a, b *configv1alpha1.ConfigStatusLastKnownGoodSchema) bool { - if a == nil && b == nil { - return true - } - if a == nil || b == nil { - return false - } - return equality.Semantic.DeepEqual(*a, *b) -} - -func (r *reconciler) handleError(ctx context.Context, configOrig, config *configv1alpha1.Config, msg string, err error, recoverable bool) error { - log := log.FromContext(ctx) - // take a snapshot of the current object - patch := client.MergeFrom(configOrig) - - if err != nil { - msg = fmt.Sprintf("%s err %s", msg, err.Error()) - } - - if recoverable { - config.SetConditions(condv1alpha1.Failed(msg)) - } else { - newMessage := condv1alpha1.UnrecoverableMessage{ - ResourceVersion: config.GetResourceVersion(), - Message: msg, - } - newmsg, err := json.Marshal(newMessage) - if err != nil { - return err - } - config.SetConditions(condv1alpha1.FailedUnRecoverable(string(newmsg))) - } - - log.Error(msg) - r.recorder.Eventf(config, corev1.EventTypeWarning, configv1alpha1.ConfigKind, msg) - - return r.client.Status().Patch(ctx, config, patch, &client.SubResourcePatchOptions{ - PatchOptions: client.PatchOptions{ - FieldManager: reconcilerName, - }, - }) -} - -func processMessageWithWarning(msg, warnings string) string { - if warnings != "" { - return fmt.Sprintf("%s warnings: %s", msg, warnings) - } - return msg -} - -func (r *reconciler) applyDeviation(ctx context.Context, config *configv1alpha1.Config) (configv1alpha1.Deviation, error) { - key := types.NamespacedName{ - Name: config.Name, - Namespace: config.Namespace, - } - - deviation := &configv1alpha1.Deviation{} - if err := r.client.Get(ctx, key, deviation); err != nil { - if resource.IgnoreNotFound(err) != nil { - return configv1alpha1.Deviation{}, err - } - // Not found: create new deviation - newDeviation := configv1alpha1.BuildDeviation(metav1.ObjectMeta{ - Name: config.Name, - Namespace: config.Namespace, - OwnerReferences: []metav1.OwnerReference{config.GetOwnerReference()}, - Labels: config.Labels, - }, &configv1alpha1.DeviationSpec{ - DeviationType: ptr.To(configv1alpha1.DeviationType_CONFIG), - }, nil) - - if err := r.client.Create(ctx, newDeviation); err != nil { - return configv1alpha1.Deviation{}, err - } - return *newDeviation, nil - } - return *deviation, nil -} - -func (r *reconciler) clearDeviation(ctx context.Context, deviation *configv1alpha1.Deviation) error { - log := log.FromContext(ctx) - patch := client.MergeFrom(deviation.DeepObjectCopy()) - - deviation.Spec.Deviations = []configv1alpha1.ConfigDeviation{} - - if err := r.client.Patch(ctx, deviation, patch, &client.SubResourcePatchOptions{ - PatchOptions: client.PatchOptions{ - FieldManager: reconcilerName, - }, - }); err != nil { - log.Error("cannot clear deviation", "deviation", deviation.GetNamespacedName()) - } - return nil -} diff --git a/pkg/reconcilers/ctrlconfig/config.go b/pkg/reconcilers/ctrlconfig/config.go index 8074bb06..6c85eb46 100644 --- a/pkg/reconcilers/ctrlconfig/config.go +++ b/pkg/reconcilers/ctrlconfig/config.go @@ -22,19 +22,17 @@ import ( "github.com/henderiw/apiserver-store/pkg/storebackend" "github.com/henderiw/logger/log" sdcctx "github.com/sdcio/config-server/pkg/sdc/ctx" - "github.com/sdcio/config-server/pkg/target" + sdctarget "github.com/sdcio/config-server/pkg/sdc/target" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/discovery" "sigs.k8s.io/controller-runtime/pkg/manager" ) type ControllerConfig struct { - TargetStore storebackend.Storer[*target.Context] + TargetStore storebackend.Storer[*sdctarget.Context] DataServerStore storebackend.Storer[sdcctx.DSContext] - SchemaServerStore storebackend.Storer[sdcctx.SSContext] SchemaDir string WorkspaceDir string - TargetHandler target.TargetHandler } func InitContext(ctx context.Context, controllerName string, req types.NamespacedName) context.Context { diff --git a/pkg/reconcilers/dataserver/reconciler.go b/pkg/reconcilers/dataserver/reconciler.go new file mode 100644 index 00000000..30fe5eb8 --- /dev/null +++ b/pkg/reconcilers/dataserver/reconciler.go @@ -0,0 +1,254 @@ +/* +Copyright 2025 Nokia. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dataserver + +import ( + "context" + "errors" + "fmt" + "reflect" + "time" + + "github.com/henderiw/apiserver-store/pkg/storebackend" + "github.com/henderiw/logger/log" + pkgerrors "github.com/pkg/errors" + invv1alpha1 "github.com/sdcio/config-server/apis/inv/v1alpha1" + "github.com/sdcio/config-server/pkg/reconcilers" + "github.com/sdcio/config-server/pkg/reconcilers/ctrlconfig" + "github.com/sdcio/config-server/pkg/reconcilers/resource" + sdctarget "github.com/sdcio/config-server/pkg/sdc/target" + sdcpb "github.com/sdcio/sdc-protos/sdcpb" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/tools/record" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + condv1alpha1 "github.com/sdcio/config-server/apis/condition/v1alpha1" +) + +func init() { + reconcilers.Register(crName, &reconciler{}) +} + +const ( + crName = "dataserver" + reconcilerName = "DataServerController" + finalizer = "dataserver.inv.sdcio.dev/finalizer" + // errors + errGetCr = "cannot get cr" + errUpdateDataStore = "cannot update datastore" + errUpdateStatus = "cannot update status" +) + +// SetupWithManager sets up the controller with the Manager. +func (r *reconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, c interface{}) (map[schema.GroupVersionKind]chan event.GenericEvent, error) { + cfg, ok := c.(*ctrlconfig.ControllerConfig) + if !ok { + return nil, fmt.Errorf("cannot initialize, expecting controllerConfig, got: %s", reflect.TypeOf(c).Name()) + } + + r.client = mgr.GetClient() + r.finalizer = resource.NewAPIFinalizer(mgr.GetClient(), finalizer, reconcilerName) + r.targetStore = cfg.TargetStore + r.recorder = mgr.GetEventRecorderFor(reconcilerName) + + return nil, ctrl.NewControllerManagedBy(mgr). + Named(reconcilerName). + For(&invv1alpha1.Target{}). + Complete(r) +} + +type reconciler struct { + client client.Client + finalizer *resource.APIFinalizer + targetStore storebackend.Storer[*sdctarget.Context] + recorder record.EventRecorder +} + +func (r *reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + ctx = ctrlconfig.InitContext(ctx, reconcilerName, req.NamespacedName) + log := log.FromContext(ctx) + log.Info("reconcile") + + targetKey := storebackend.KeyFromNSN(req.NamespacedName) + + target := &invv1alpha1.Target{} + if err := r.client.Get(ctx, req.NamespacedName, target); err != nil { + // if the resource no longer exists the reconcile loop is done + if resource.IgnoreNotFound(err) != nil { + log.Error(errGetCr, "error", err) + return ctrl.Result{}, pkgerrors.Wrap(resource.IgnoreNotFound(err), errGetCr) + } + return ctrl.Result{}, nil + } + + targetOrig := target.DeepCopy() + + if !target.GetDeletionTimestamp().IsZero() { + return ctrl.Result{}, nil + } + + if target.Status.GetCondition(invv1alpha1.ConditionTypeDatastoreReady).Status != metav1.ConditionTrue { + // target not ready so we can wait till the target goes to ready state + return ctrl.Result{}, + pkgerrors.Wrap(r.handleError(ctx, targetOrig, "datastore not ready", nil), errUpdateStatus) + } + + tctx, err := r.targetStore.Get(ctx, targetKey) + if err != nil { + return ctrl.Result{Requeue: true, RequeueAfter: 1 * time.Second}, + pkgerrors.Wrap(r.handleError(ctx, targetOrig, "tcxt does not exist", err), errUpdateStatus) + } + + resp, err := tctx.GetDataStore(ctx, &sdcpb.GetDataStoreRequest{DatastoreName: targetKey.String()}) + if err != nil { + if errs := r.targetStore.UpdateWithKeyFn(ctx, targetKey, func(ctx context.Context, tctx *sdctarget.Context) *sdctarget.Context { + if tctx != nil { + tctx.SetNotReady(ctx) + } + return tctx + }); errs != nil { + errs = errors.Join(errs, err) + return ctrl.Result{Requeue: true}, + pkgerrors.Wrap(r.handleError(ctx, targetOrig, "target datastore rsp error and update targetstore failed", errs), errUpdateStatus) + } + return ctrl.Result{RequeueAfter: 5 * time.Second}, + pkgerrors.Wrap(r.handleError(ctx, targetOrig, "target datastore rsp error", err), errUpdateStatus) + } + if resp.Target.Status != sdcpb.TargetStatus_CONNECTED { + if errs := r.targetStore.UpdateWithKeyFn(ctx, targetKey, func(ctx context.Context, tctx *sdctarget.Context) *sdctarget.Context { + if tctx != nil { + tctx.SetNotReady(ctx) + } + return tctx + }); errs != nil { + errs = errors.Join(errs, err) + return ctrl.Result{Requeue: true}, + pkgerrors.Wrap(r.handleError(ctx, targetOrig, "target datastore not connected and update targetstore failed", errs), errUpdateStatus) + } + return ctrl.Result{RequeueAfter: 10 * time.Second}, // requeue will happen automatically when target gets updated + pkgerrors.Wrap(r.handleError(ctx, targetOrig, "target datastore not connected", err), errUpdateStatus) + } + + if errs := r.targetStore.UpdateWithKeyFn(ctx, targetKey, func(ctx context.Context, tctx *sdctarget.Context) *sdctarget.Context { + if tctx != nil { + tctx.SetReady(ctx) + } + return tctx + }); errs != nil { + errs = errors.Join(errs, err) + return ctrl.Result{Requeue: true}, + pkgerrors.Wrap(r.handleError(ctx, targetOrig, "update targetstore failed", errs), errUpdateStatus) + } + + + return r.handleSuccess(ctx, targetOrig) +} + +func (r *reconciler) handleSuccess(ctx context.Context, target *invv1alpha1.Target) (ctrl.Result, error) { + log := log.FromContext(ctx) + log.Debug("handleSuccess", "key", target.GetNamespacedName(), "status old", target.DeepCopy().Status) + // take a snapshot of the current object + //patch := client.MergeFrom(target.DeepCopy()) + // update status + newTarget := invv1alpha1.BuildTarget( + metav1.ObjectMeta{ + Namespace: target.Namespace, + Name: target.Name, + }, + invv1alpha1.TargetSpec{}, + invv1alpha1.TargetStatus{}, + ) + // set old condition to avoid updating the new status if not changed + newTarget.SetConditions(target.GetCondition(invv1alpha1.ConditionTypeTargetConnectionReady)) + newTarget.SetConditions(target.GetCondition(condv1alpha1.ConditionTypeReady)) + // set new conditions + newTarget.SetConditions(invv1alpha1.TargetConnectionReady()) + newTarget.SetOverallStatus(target) + + result := ctrl.Result{RequeueAfter: 5 * time.Minute} + if !target.IsReady() { + result= ctrl.Result{Requeue: true} + } + + // we don't update the resource if no condition changed + if newTarget.GetCondition(invv1alpha1.ConditionTypeTargetConnectionReady).Equal(target.GetCondition(invv1alpha1.ConditionTypeTargetConnectionReady)) && + newTarget.GetCondition(condv1alpha1.ConditionTypeReady).Equal(target.GetCondition(condv1alpha1.ConditionTypeReady)) { + log.Info("handleSuccess -> no change") + return result, nil + } + log.Info("handleSuccess -> change", + "connReady condition change", newTarget.GetCondition(invv1alpha1.ConditionTypeTargetConnectionReady).Equal(target.GetCondition(invv1alpha1.ConditionTypeTargetConnectionReady)), + "Ready condition change", newTarget.GetCondition(condv1alpha1.ConditionTypeReady).Equal(target.GetCondition(condv1alpha1.ConditionTypeReady)), + ) + + r.recorder.Eventf(newTarget, corev1.EventTypeNormal, invv1alpha1.TargetKind, "ready") + + log.Debug("handleSuccess", "key", newTarget.GetNamespacedName(), "status new", target.Status) + + return result, r.client.Status().Patch(ctx, newTarget, client.Apply, &client.SubResourcePatchOptions{ + PatchOptions: client.PatchOptions{ + FieldManager: reconcilerName, + }, + }) +} + +func (r *reconciler) handleError(ctx context.Context, target *invv1alpha1.Target, msg string, err error) error { + log := log.FromContext(ctx) + // take a snapshot of the current object + //patch := client.MergeFrom(target.DeepCopy()) + + if err != nil { + msg = fmt.Sprintf("%s err %s", msg, err.Error()) + } + + newTarget := invv1alpha1.BuildTarget( + metav1.ObjectMeta{ + Namespace: target.Namespace, + Name: target.Name, + }, + invv1alpha1.TargetSpec{}, + invv1alpha1.TargetStatus{}, + ) + // set old condition to avoid updating the new status if not changed + newTarget.SetConditions(target.GetCondition(invv1alpha1.ConditionTypeTargetConnectionReady)) + newTarget.SetConditions(target.GetCondition(condv1alpha1.ConditionTypeReady)) + // set new conditions + newTarget.SetConditions(invv1alpha1.TargetConnectionFailed(msg)) + newTarget.SetOverallStatus(target) + + if newTarget.GetCondition(invv1alpha1.ConditionTypeTargetConnectionReady).Equal(target.GetCondition(invv1alpha1.ConditionTypeTargetConnectionReady)) && + newTarget.GetCondition(condv1alpha1.ConditionTypeReady).Equal(target.GetCondition(condv1alpha1.ConditionTypeReady)) { + log.Info("handleSuccess -> no change") + return nil + } + log.Info("handleSuccess -> change", + "connReady condition change", newTarget.GetCondition(invv1alpha1.ConditionTypeTargetConnectionReady).Equal(target.GetCondition(invv1alpha1.ConditionTypeTargetConnectionReady)), + "Ready condition change", newTarget.GetCondition(condv1alpha1.ConditionTypeReady).Equal(target.GetCondition(condv1alpha1.ConditionTypeReady)), + ) + + log.Error(msg, "error", err) + r.recorder.Eventf(newTarget, corev1.EventTypeWarning, invv1alpha1.TargetKind, msg) + + return r.client.Status().Patch(ctx, newTarget, client.Apply, &client.SubResourcePatchOptions{ + PatchOptions: client.PatchOptions{ + FieldManager: reconcilerName, + }, + }) +} diff --git a/pkg/reconcilers/discoveryrule/reconciler.go b/pkg/reconcilers/discoveryrule/reconciler.go index 441dd323..d715a3f2 100644 --- a/pkg/reconcilers/discoveryrule/reconciler.go +++ b/pkg/reconcilers/discoveryrule/reconciler.go @@ -32,7 +32,7 @@ import ( "github.com/sdcio/config-server/pkg/reconcilers/ctrlconfig" "github.com/sdcio/config-server/pkg/reconcilers/eventhandler" "github.com/sdcio/config-server/pkg/reconcilers/resource" - "github.com/sdcio/config-server/pkg/target" + sdctarget "github.com/sdcio/config-server/pkg/sdc/target" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" @@ -83,7 +83,7 @@ type reconciler struct { finalizer *resource.APIFinalizer discoveryStore storebackend.Storer[discoveryrule.DiscoveryRule] - targetStore storebackend.Storer[*target.Context] + targetStore storebackend.Storer[*sdctarget.Context] recorder record.EventRecorder } diff --git a/pkg/reconcilers/rollout/reconciler.go b/pkg/reconcilers/rollout/reconciler.go index 29ad483c..bd2e9fc5 100644 --- a/pkg/reconcilers/rollout/reconciler.go +++ b/pkg/reconcilers/rollout/reconciler.go @@ -36,7 +36,6 @@ import ( "github.com/sdcio/config-server/pkg/reconcilers" "github.com/sdcio/config-server/pkg/reconcilers/ctrlconfig" "github.com/sdcio/config-server/pkg/reconcilers/resource" - "github.com/sdcio/config-server/pkg/target" workspacereader "github.com/sdcio/config-server/pkg/workspace" corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" @@ -60,7 +59,6 @@ const ( finalizer = "rollout.inv.sdcio.dev/finalizer" // errors errGetCr = "cannot get cr" - errUpdateDataStore = "cannot update datastore" errUpdateStatus = "cannot update status" ) @@ -74,7 +72,6 @@ func (r *reconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, c i r.client = mgr.GetClient() r.finalizer = resource.NewAPIFinalizer(mgr.GetClient(), finalizer, reconcilerName) - r.targetHandler = cfg.TargetHandler // initializes the directory r.workspaceReader, err = workspacereader.NewReader( cfg.WorkspaceDir, @@ -96,7 +93,6 @@ func (r *reconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, c i type reconciler struct { client client.Client finalizer *resource.APIFinalizer - targetHandler target.TargetHandler workspaceReader *workspacereader.Reader recorder record.EventRecorder } @@ -134,7 +130,7 @@ func (r *reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu tm := NewTransactionManager( newTargetUpdateConfigStore, newTargetDeleteConfigStore, - r.targetHandler, + r.client, 1*time.Minute, 30*time.Second, rollout.GetSkipUnavailableTarget(), @@ -182,7 +178,7 @@ func (r *reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu tm := NewTransactionManager( newTargetUpdateConfigStore, newTargetDeleteConfigStore, - r.targetHandler, + r.client, 1*time.Minute, 30*time.Second, rollout.GetSkipUnavailableTarget(), diff --git a/pkg/reconcilers/rollout/reconciler_test.go b/pkg/reconcilers/rollout/reconciler_test.go index c94a6ae1..02f6d65d 100644 --- a/pkg/reconcilers/rollout/reconciler_test.go +++ b/pkg/reconcilers/rollout/reconciler_test.go @@ -31,6 +31,10 @@ import ( "k8s.io/apimachinery/pkg/types" ) +const ( + namespace = "default" +) + func TestGetToBeDeletedConfigsNew(t *testing.T) { cases := map[string]struct { existingConfigs map[string][]string diff --git a/pkg/reconcilers/rollout/transaction.go b/pkg/reconcilers/rollout/transaction.go index 3bdd1523..dea142cb 100644 --- a/pkg/reconcilers/rollout/transaction.go +++ b/pkg/reconcilers/rollout/transaction.go @@ -28,16 +28,19 @@ import ( "github.com/henderiw/logger/log" "github.com/sdcio/config-server/apis/config" invv1alpha1 "github.com/sdcio/config-server/apis/inv/v1alpha1" - "github.com/sdcio/config-server/pkg/target" v1 "k8s.io/api/flowcontrol/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" + "sigs.k8s.io/controller-runtime/pkg/client" + dsclient "github.com/sdcio/config-server/pkg/sdc/dataserver/client" + sdcpb "github.com/sdcio/sdc-protos/sdcpb" ) type transactionManager struct { newTargetUpdateConfigStore storebackend.Storer[storebackend.Storer[*config.Config]] newTargetDeleteConfigStore storebackend.Storer[storebackend.Storer[*config.Config]] - targetHandler target.TargetHandler + //targetHandler target.TargetHandler + client client.Client globalTimeout time.Duration targetTimeout time.Duration skipUnavailableTarget bool @@ -49,14 +52,14 @@ type transactionManager struct { func NewTransactionManager( newTargetUpdateConfigStore storebackend.Storer[storebackend.Storer[*config.Config]], newTargetDeleteConfigStore storebackend.Storer[storebackend.Storer[*config.Config]], - targetHandler target.TargetHandler, + client client.Client, globalTimeout, targetTimeout time.Duration, skipUnavailableTarget bool, ) *transactionManager { tm := &transactionManager{ newTargetUpdateConfigStore: newTargetUpdateConfigStore, newTargetDeleteConfigStore: newTargetDeleteConfigStore, - targetHandler: targetHandler, + client: client, globalTimeout: globalTimeout, targetTimeout: targetTimeout, targetStatus: memstore.NewStore[invv1alpha1.RolloutTargetStatus](), @@ -93,8 +96,11 @@ func (r *transactionManager) TransactToAllTargets(ctx context.Context, transacti errChan := make(chan error, r.targets.Len()) done := make(chan struct{}) + + for _, targetKey := range r.targets.UnsortedList() { - if _, _, err := r.targetHandler.GetTargetContext(ctx, targetKey); err != nil { + t := &invv1alpha1.Target{} + if err := r.client.Get(ctx, targetKey, t); err != nil { // target unavailable -> we continue for now targetStatus := invv1alpha1.RolloutTargetStatus{Name: targetKey.String()} targetStatus.SetConditions(invv1alpha1.ConfigApplyUnavailable(fmt.Sprintf("target unavailable %s", err.Error()))) @@ -105,6 +111,18 @@ func (r *transactionManager) TransactToAllTargets(ctx context.Context, transacti } continue } + if !t.IsReady() { + // target unavailable -> we continue for now + targetStatus := invv1alpha1.RolloutTargetStatus{Name: targetKey.String()} + targetStatus.SetConditions(invv1alpha1.ConfigApplyUnavailable("target not ready")) + _ = r.targetStatus.Update(ctx, storebackend.KeyFromNSN(targetKey), targetStatus) + if r.skipUnavailableTarget { + globalCancel() + return r.targetStatus, fmt.Errorf("transaction aborted: target %s is not ready", targetKey.String()) + } + continue + } + wg.Add(1) go func(targetKey types.NamespacedName) { @@ -241,7 +259,21 @@ func (r *transactionManager) cancel(ctx context.Context, targetKey types.Namespa done := make(chan error, 1) go func() { - done <- r.targetHandler.Cancel(ctx, targetKey, transactionID) + cfg := &dsclient.Config{ + Address: dsclient.GetDataServerAddress(), + Insecure: true, + } + + done <- dsclient.OneShot(ctx, cfg, func(ctx context.Context, c sdcpb.DataServerClient) error { + _, err := c.TransactionCancel(ctx, &sdcpb.TransactionCancelRequest{ + TransactionId: transactionID, + DatastoreName: storebackend.KeyFromNSN(targetKey).String(), + }) + if err != nil { + return err + } + return nil + }) close(done) }() @@ -269,8 +301,21 @@ func (r *transactionManager) confirm(ctx context.Context, targetKey types.Namesp done := make(chan error, 1) go func() { - err := r.targetHandler.Confirm(ctx, targetKey, transactionID) - done <- err + cfg := &dsclient.Config{ + Address: dsclient.GetDataServerAddress(), + Insecure: true, + } + + done <- dsclient.OneShot(ctx, cfg, func(ctx context.Context, c sdcpb.DataServerClient) error { + _, err := c.TransactionConfirm(ctx, &sdcpb.TransactionConfirmRequest{ + TransactionId: transactionID, + DatastoreName: storebackend.KeyFromNSN(targetKey).String(), + }) + if err != nil { + return err + } + return nil + }) close(done) }() @@ -315,12 +360,27 @@ func (r *transactionManager) applyConfigToTarget(ctx context.Context, targetKey done := make(chan error, 1) - deviations := map[string]*config.Deviation{} + //deviations := map[string]*config.Deviation{} go func() { - _, _, err := r.targetHandler.SetIntents(ctx, targetKey, transactionID, configUpdates, configDeletes, deviations, deviations, false) - done <- err + cfg := &dsclient.Config{ + Address: dsclient.GetDataServerAddress(), + Insecure: true, + } + + // ToBeUpdated + done <- dsclient.OneShot(ctx, cfg, func(ctx context.Context, c sdcpb.DataServerClient) error { + _, err := c.TransactionSet(ctx, &sdcpb.TransactionSetRequest{ + TransactionId: transactionID, + DatastoreName: storebackend.KeyFromNSN(targetKey).String(), + }) + if err != nil { + return err + } + return nil + }) close(done) + }() select { diff --git a/pkg/reconcilers/rollout/transaction_test.go b/pkg/reconcilers/rollout/transaction_test.go index 2f95d3db..0024e5f4 100644 --- a/pkg/reconcilers/rollout/transaction_test.go +++ b/pkg/reconcilers/rollout/transaction_test.go @@ -16,6 +16,8 @@ limitations under the License. package rollout +/* + import ( "context" "fmt" @@ -163,7 +165,7 @@ func getTransctionManager(targets map[string]*target.MockContext, updates, delet log.Error("cannot create target", "err", err) } } - mockHandler := target.NewMockTargetHandler(targetStore) + //mockHandler := target.NewMockTargetHandler(targetStore) updateStore := memstore.NewStore[storebackend.Storer[*config.Config]]() deleteStore := memstore.NewStore[storebackend.Storer[*config.Config]]() @@ -191,5 +193,6 @@ func getTransctionManager(targets map[string]*target.MockContext, updates, delet } } - return NewTransactionManager(updateStore, deleteStore, mockHandler, 5*time.Second, 2*time.Second, true) + return NewTransactionManager(updateStore, deleteStore, nil, 5*time.Second, 2*time.Second, true) } +*/ \ No newline at end of file diff --git a/pkg/reconcilers/schema/reconciler.go b/pkg/reconcilers/schema/reconciler.go index f4ecb5c1..1ef34652 100644 --- a/pkg/reconcilers/schema/reconciler.go +++ b/pkg/reconcilers/schema/reconciler.go @@ -23,7 +23,6 @@ import ( "path/filepath" "reflect" - "github.com/henderiw/apiserver-store/pkg/storebackend" "github.com/henderiw/logger/log" pkgerrors "github.com/pkg/errors" condv1alpha1 "github.com/sdcio/config-server/apis/condition/v1alpha1" @@ -35,7 +34,6 @@ import ( "github.com/sdcio/config-server/pkg/reconcilers/eventhandler" "github.com/sdcio/config-server/pkg/reconcilers/resource" schemaloader "github.com/sdcio/config-server/pkg/schema" - sdcctx "github.com/sdcio/config-server/pkg/sdc/ctx" ssclient "github.com/sdcio/config-server/pkg/sdc/schemaserver/client" sdcpb "github.com/sdcio/sdc-protos/sdcpb" corev1 "k8s.io/api/core/v1" @@ -59,10 +57,11 @@ const ( finalizer = "schema.inv.sdcio.dev/finalizer" // errors errGetCr = "cannot get cr" - errUpdateDataStore = "cannot update datastore" errUpdateStatus = "cannot update status" ) + + // SetupWithManager sets up the controller with the Manager. func (r *reconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, c interface{}) (map[schema.GroupVersionKind]chan event.GenericEvent, error) { var err error @@ -71,16 +70,6 @@ func (r *reconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, c i return nil, fmt.Errorf("cannot initialize, expecting controllerConfig, got: %s", reflect.TypeOf(c).Name()) } - err = cfg.SchemaServerStore.List(ctx, func(ctx context.Context, key storebackend.Key, dsCtx sdcctx.SSContext) { - r.schemaclient = dsCtx.SSClient - }) - if err != nil { - return nil, err - } - if r.schemaclient == nil { - return nil, fmt.Errorf("cannot get schema client") - } - r.client = mgr.GetClient() r.finalizer = resource.NewAPIFinalizer(mgr.GetClient(), finalizer, reconcilerName) // initializes the directory @@ -109,7 +98,6 @@ type reconciler struct { finalizer *resource.APIFinalizer schemaLoader *schemaloader.Loader - schemaclient ssclient.Client schemaBasePath string recorder record.EventRecorder } @@ -133,11 +121,29 @@ func (r *reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu status := &schema.Status if !schema.GetDeletionTimestamp().IsZero() { + + cfg := &ssclient.Config{ + Address: ssclient.GetSchemaServerAddress(), + Insecure: true, + } + + schemaclient, closeFn, err := ssclient.NewEphemeral(ctx, cfg) + if err != nil { + return r.handleError(ctx, schema, "cannot delete schema from schemaserver", err) + } + defer func() { + if err := closeFn(); err != nil { + // You can use your preferred logging framework here + fmt.Printf("failed to close connection: %v\n", err) + } + }() + + // check if the schema exists; this is == nil check; in case of err it does not exist - if _, err := r.schemaclient.GetSchemaDetails(ctx, &sdcpb.GetSchemaDetailsRequest{ + if _, err := schemaclient.GetSchemaDetails(ctx, &sdcpb.GetSchemaDetailsRequest{ Schema: spec.GetSchema(), }); err == nil { - if _, err := r.schemaclient.DeleteSchema(ctx, &sdcpb.DeleteSchemaRequest{ + if _, err := schemaclient.DeleteSchema(ctx, &sdcpb.DeleteSchemaRequest{ Schema: spec.GetSchema(), }); err != nil { return r.handleError(ctx, schema, "cannot delete schema from schemaserver", err) @@ -189,13 +195,30 @@ func (r *reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu } status.Repositories = repoStatuses } + + cfg := &ssclient.Config{ + Address: ssclient.GetSchemaServerAddress(), + Insecure: true, + } + + schemaclient, closeFn, err := ssclient.NewEphemeral(ctx, cfg) + if err != nil { + return r.handleError(ctx, schema, "cannot delete schema from schemaserver", err) + } + defer func() { + if err := closeFn(); err != nil { + // You can use your preferred logging framework here + fmt.Printf("failed to close connection: %v\n", err) + } + }() + // check if the schema exists - rsp, err := r.schemaclient.GetSchemaDetails(ctx, &sdcpb.GetSchemaDetailsRequest{ + rsp, err := schemaclient.GetSchemaDetails(ctx, &sdcpb.GetSchemaDetailsRequest{ Schema: spec.GetSchema(), }) if err != nil { // schema does not exists in schema-server -> create it - if _, err := r.schemaclient.CreateSchema(ctx, &sdcpb.CreateSchemaRequest{ + if _, err := schemaclient.CreateSchema(ctx, &sdcpb.CreateSchemaRequest{ Schema: spec.GetSchema(), File: spec.GetNewSchemaBase(r.schemaBasePath).Models, Directory: spec.GetNewSchemaBase(r.schemaBasePath).Includes, @@ -211,7 +234,7 @@ func (r *reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu switch rsp.Schema.Status { case sdcpb.SchemaStatus_FAILED: - if _, err := r.schemaclient.CreateSchema(ctx, &sdcpb.CreateSchemaRequest{ + if _, err := schemaclient.CreateSchema(ctx, &sdcpb.CreateSchemaRequest{ Schema: spec.GetSchema(), File: spec.GetNewSchemaBase(r.schemaBasePath).Models, Directory: spec.GetNewSchemaBase(r.schemaBasePath).Includes, diff --git a/pkg/reconcilers/subscription/reconciler.go b/pkg/reconcilers/subscription/reconciler.go index 994aeb48..f08fb11d 100644 --- a/pkg/reconcilers/subscription/reconciler.go +++ b/pkg/reconcilers/subscription/reconciler.go @@ -32,7 +32,7 @@ import ( "github.com/sdcio/config-server/pkg/reconcilers/ctrlconfig" "github.com/sdcio/config-server/pkg/reconcilers/eventhandler" "github.com/sdcio/config-server/pkg/reconcilers/resource" - sdctarget "github.com/sdcio/config-server/pkg/target" + sdctarget "github.com/sdcio/config-server/pkg/sdc/target" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" diff --git a/pkg/reconcilers/target/reconciler.go b/pkg/reconcilers/target/reconciler.go index 64889df5..4a561bcd 100644 --- a/pkg/reconcilers/target/reconciler.go +++ b/pkg/reconcilers/target/reconciler.go @@ -30,7 +30,7 @@ import ( "github.com/sdcio/config-server/pkg/reconcilers" "github.com/sdcio/config-server/pkg/reconcilers/ctrlconfig" "github.com/sdcio/config-server/pkg/reconcilers/resource" - sdctarget "github.com/sdcio/config-server/pkg/target" + sdctarget "github.com/sdcio/config-server/pkg/sdc/target" sdcpb "github.com/sdcio/sdc-protos/sdcpb" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/pkg/reconcilers/targetconfig/reconciler.go b/pkg/reconcilers/targetconfig/reconciler.go index 2fbd3735..e89e2cf5 100644 --- a/pkg/reconcilers/targetconfig/reconciler.go +++ b/pkg/reconcilers/targetconfig/reconciler.go @@ -31,8 +31,7 @@ import ( "github.com/sdcio/config-server/pkg/reconcilers/ctrlconfig" "github.com/sdcio/config-server/pkg/reconcilers/eventhandler" "github.com/sdcio/config-server/pkg/reconcilers/resource" - "github.com/sdcio/config-server/pkg/target" - "github.com/sdcio/config-server/pkg/transactor" + sdctarget "github.com/sdcio/config-server/pkg/sdc/target" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/discovery" "k8s.io/client-go/tools/record" @@ -72,7 +71,7 @@ func (r *reconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, c i r.finalizer = resource.NewAPIFinalizer(mgr.GetClient(), finalizer, reconcilerName) r.targetStore = cfg.TargetStore r.recorder = mgr.GetEventRecorderFor(reconcilerName) - r.transactor = transactor.New(r.client, crName, fieldmanagerfinalizer) + r.transactor = sdctarget.NewTransactor(r.client, crName, fieldmanagerfinalizer) return nil, ctrl.NewControllerManagedBy(mgr). Named(reconcilerName). @@ -86,9 +85,9 @@ type reconciler struct { client client.Client discoveryClient *discovery.DiscoveryClient finalizer *resource.APIFinalizer - targetStore storebackend.Storer[*target.Context] + targetStore storebackend.Storer[*sdctarget.Context] recorder record.EventRecorder - transactor *transactor.Transactor + transactor *sdctarget.Transactor } func (r *reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { @@ -201,7 +200,7 @@ func (r *reconciler) handleSuccess(ctx context.Context, target *invv1alpha1.Targ func (r *reconciler) handleError(ctx context.Context, target *invv1alpha1.Target, err error) error { log := log.FromContext(ctx) - log.Error("config config transaction failed", "err", err) + log.Error("config transaction failed", "err", err) return nil // take a snapshot of the current object //patch := client.MergeFrom(target.DeepCopy()) @@ -240,7 +239,7 @@ func (r *reconciler) handleError(ctx context.Context, target *invv1alpha1.Target */ } -func (r *reconciler) IsTargetDataStoreReady(ctx context.Context, key storebackend.Key, target *invv1alpha1.Target) (*target.Context, error) { +func (r *reconciler) IsTargetDataStoreReady(ctx context.Context, key storebackend.Key, target *invv1alpha1.Target) (*sdctarget.Context, error) { log := log.FromContext(ctx) // we do not find the target Context -> target is not ready tctx, err := r.targetStore.Get(ctx, key) diff --git a/pkg/reconcilers/targetdatastore/reconciler.go b/pkg/reconcilers/targetdatastore/reconciler.go index 2e41a371..37f7e1d4 100644 --- a/pkg/reconcilers/targetdatastore/reconciler.go +++ b/pkg/reconcilers/targetdatastore/reconciler.go @@ -34,7 +34,7 @@ import ( "github.com/sdcio/config-server/pkg/reconcilers/resource" sdcctx "github.com/sdcio/config-server/pkg/sdc/ctx" dsclient "github.com/sdcio/config-server/pkg/sdc/dataserver/client" - sdctarget "github.com/sdcio/config-server/pkg/target" + sdctarget "github.com/sdcio/config-server/pkg/sdc/target" sdcpb "github.com/sdcio/sdc-protos/sdcpb" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -194,7 +194,8 @@ func (r *reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu if err != nil || !curtctx.IsReady() { if curtctx != nil && curtctx.GetDSClient() != nil && !curtctx.GetDSClient().IsConnectionReady() { // The dataserver connection exists - return ctrl.Result{}, + // TO BE REVERTED ONCE WE HAVE THE DATASERVER WATCH + return ctrl.Result{RequeueAfter: 10 * time.Second}, errors.Wrap(r.handleError(ctx, targetOrig, "dataserver down", nil, true), errUpdateStatus) } @@ -235,7 +236,8 @@ func (r *reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu // check if the data server is down or not if curtctx != nil && curtctx.GetDSClient() != nil && !curtctx.GetDSClient().IsConnectionReady() { // The dataserver connection exists - return ctrl.Result{}, + // TO BE REVERTED ONCE WE HAVE THE DATASERVER WATCH + return ctrl.Result{RequeueAfter: 10 * time.Second}, errors.Wrap(r.handleError(ctx, targetOrig, "dataserver down", nil, true), errUpdateStatus) } diff --git a/pkg/reconcilers/targetrecovery/reconciler.go b/pkg/reconcilers/targetrecovery/reconciler.go index 7b853728..44944856 100644 --- a/pkg/reconcilers/targetrecovery/reconciler.go +++ b/pkg/reconcilers/targetrecovery/reconciler.go @@ -30,8 +30,7 @@ import ( "github.com/sdcio/config-server/pkg/reconcilers" "github.com/sdcio/config-server/pkg/reconcilers/ctrlconfig" "github.com/sdcio/config-server/pkg/reconcilers/resource" - "github.com/sdcio/config-server/pkg/target" - "github.com/sdcio/config-server/pkg/transactor" + sdctarget "github.com/sdcio/config-server/pkg/sdc/target" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" @@ -73,7 +72,7 @@ func (r *reconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, c i r.finalizer = resource.NewAPIFinalizer(mgr.GetClient(), finalizer, reconcilerName) r.targetStore = cfg.TargetStore r.recorder = mgr.GetEventRecorderFor(reconcilerName) - r.transactor = transactor.New(r.client, crName, fieldmanagerfinalizer) + r.transactor = sdctarget.NewTransactor(r.client, crName, fieldmanagerfinalizer) return nil, ctrl.NewControllerManagedBy(mgr). Named(reconcilerName). @@ -85,9 +84,9 @@ type reconciler struct { client client.Client discoveryClient *discovery.DiscoveryClient finalizer *resource.APIFinalizer - targetStore storebackend.Storer[*target.Context] + targetStore storebackend.Storer[*sdctarget.Context] recorder record.EventRecorder - transactor *transactor.Transactor + transactor *sdctarget.Transactor } func (r *reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { @@ -135,34 +134,6 @@ func (r *reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu return ctrl.Result{}, errors.Wrap(r.handleError(ctx, targetOrig, "setIntent failed", err), errUpdateStatus) } return ctrl.Result{}, errors.Wrap(r.handleSuccess(ctx, targetOrig, msg), errUpdateStatus) - - /* - // we split the config in config that were successfully applied and config that was not yet - recoveryConfigs, deviations, err := r.getRecoveryConfigsAndDeviations(ctx, target) - if err != nil { - return ctrl.Result{}, errors.Wrap(r.handleError(ctx, targetOrig, "reapply config failed", err), errUpdateStatus) - } - - if len(recoveryConfigs) == 0 && len(deviations) == 0 { - tctx.SetRecoveredConfigsState(ctx) - log.Info("config recovery done -> no configs to recover") - return ctrl.Result{}, errors.Wrap(r.handleSuccess(ctx, targetOrig, ""), errUpdateStatus) - } - - log.Info("config recovery new ....") - - // We need to restore the config on the target - msg, err := tctx.RecoverIntents(ctx, targetKey, recoveryConfigs, deviations) - if err != nil { - // This is bad since this means we cannot recover the applied config - // on a target. We set the target config status to Failed. - // Most likely a human intervention is needed - return ctrl.Result{}, errors.Wrap(r.handleError(ctx, targetOrig, "setIntent failed", err), errUpdateStatus) - } - tctx.SetRecoveredConfigsState(ctx) - log.Info("config recovery done -> configs recovered") - return ctrl.Result{}, errors.Wrap(r.handleSuccess(ctx, targetOrig, msg), errUpdateStatus) - */ } func (r *reconciler) handleSuccess(ctx context.Context, target *invv1alpha1.Target, msg *string) error { @@ -247,7 +218,7 @@ func (r *reconciler) handleError(ctx context.Context, target *invv1alpha1.Target }) } -func (r *reconciler) IsTargetDataStoreReady(ctx context.Context, key storebackend.Key, target *invv1alpha1.Target) (*target.Context, error) { +func (r *reconciler) IsTargetDataStoreReady(ctx context.Context, key storebackend.Key, target *invv1alpha1.Target) (*sdctarget.Context, error) { log := log.FromContext(ctx) // we do not find the target Context -> target is not ready tctx, err := r.targetStore.Get(ctx, key) diff --git a/pkg/registry/configblame/strategy_resource.go b/pkg/registry/configblame/strategy_resource.go index c9376db4..ac54c1c2 100644 --- a/pkg/registry/configblame/strategy_resource.go +++ b/pkg/registry/configblame/strategy_resource.go @@ -18,7 +18,7 @@ package configblame import ( "context" - "errors" + "fmt" "github.com/henderiw/apiserver-builder/pkg/builder/resource" "github.com/henderiw/apiserver-builder/pkg/builder/utils" @@ -29,10 +29,8 @@ import ( "github.com/sdcio/config-server/apis/config" invv1alpha1 "github.com/sdcio/config-server/apis/inv/v1alpha1" "github.com/sdcio/config-server/pkg/registry/options" - "github.com/sdcio/config-server/pkg/target" apierrors "k8s.io/apimachinery/pkg/api/errors" metainternalversion "k8s.io/apimachinery/pkg/apis/meta/internalversion" - "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" @@ -42,6 +40,10 @@ import ( "k8s.io/apiserver/pkg/storage/names" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/structured-merge-diff/v4/fieldpath" + dsclient "github.com/sdcio/config-server/pkg/sdc/dataserver/client" + sdcpb "github.com/sdcio/sdc-protos/sdcpb" + "google.golang.org/protobuf/encoding/protojson" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // NewStrategy creates and returns a strategy instance @@ -61,7 +63,6 @@ func NewStrategy( storage: storage, watcherManager: watcherManager, client: opts.Client, - targetStore: opts.TargetStore, } } @@ -81,7 +82,6 @@ type strategy struct { storage storebackend.Storer[runtime.Object] watcherManager watchermanager.WatcherManager client client.Client - targetStore storebackend.Storer[*target.Context] } func (r *strategy) NamespaceScoped() bool { return r.obj.NamespaceScoped() } @@ -150,23 +150,89 @@ func (r *strategy) Delete(ctx context.Context, key types.NamespacedName, obj run return obj, nil } -func (r *strategy) Get(ctx context.Context, key types.NamespacedName) (runtime.Object, error) { - target, tctx, err := r.getTargetContext(ctx, key) - if err != nil { +func (r *strategy) getConfigBlame(ctx context.Context, target *invv1alpha1.Target, key types.NamespacedName) (*config.ConfigBlame, error) { + if !target.IsReady() { return nil, apierrors.NewNotFound(r.gr, key.Name) } - rc, err := tctx.GetBlameConfig(ctx, storebackend.KeyFromNSN(key)) + cfg := &dsclient.Config{ + Address: dsclient.GetDataServerAddress(), + Insecure: true, + } + + dsclient, closeFn, err := dsclient.NewEphemeral(ctx, cfg) if err != nil { - return nil, apierrors.NewInternalError(err) + return nil, err } - rc.SetCreationTimestamp(target.CreationTimestamp) - rc.SetResourceVersion(target.ResourceVersion) - rc.SetAnnotations(target.Annotations) - rc.SetLabels(target.Labels) - obj := rc + defer func() { + if err := closeFn(); err != nil { + // You can use your preferred logging framework here + fmt.Printf("failed to close connection: %v\n", err) + } + }() - return obj, nil + + // check if the schema exists; this is == nil check; in case of err it does not exist + rsp, err := dsclient.BlameConfig(ctx, &sdcpb.BlameConfigRequest{ + DatastoreName: storebackend.KeyFromNSN(key).String(), + IncludeDefaults: true, + }) + if err != nil { + return nil, err + } + + if rsp == nil || rsp.ConfigTree == nil { + cb := config.BuildConfigBlame( + metav1.ObjectMeta{ + Name: key.Name, + Namespace: key.Namespace, + }, + config.ConfigBlameSpec{}, + config.ConfigBlameStatus{ + Value: runtime.RawExtension{ + Raw: nil, + }, + }, + ) + + cb.SetCreationTimestamp(target.CreationTimestamp) + cb.SetResourceVersion(target.ResourceVersion) + cb.SetAnnotations(target.Annotations) + cb.SetLabels(target.Labels) + return cb, nil + } + + json, err := protojson.Marshal(rsp.ConfigTree) + if err != nil { + return nil, err + } + + cb := config.BuildConfigBlame( + metav1.ObjectMeta{ + Name: key.Name, + Namespace: key.Namespace, + }, + config.ConfigBlameSpec{}, + config.ConfigBlameStatus{ + Value: runtime.RawExtension{ + Raw: json, + }, + }, + ) + + cb.SetCreationTimestamp(target.CreationTimestamp) + cb.SetResourceVersion(target.ResourceVersion) + cb.SetAnnotations(target.Annotations) + cb.SetLabels(target.Labels) + return cb, nil +} + +func (r *strategy) Get(ctx context.Context, key types.NamespacedName) (runtime.Object, error) { + target := &invv1alpha1.Target{} + if err := r.client.Get(ctx, key, target); err != nil { + return nil, apierrors.NewNotFound(r.gr, key.Name) + } + return r.getConfigBlame(ctx, target, key) } func (r *strategy) List(ctx context.Context, options *metainternalversion.ListOptions) (runtime.Object, error) { @@ -194,69 +260,21 @@ func (r *strategy) List(ctx context.Context, options *metainternalversion.ListOp return nil, err } - configBlameListFunc := func(ctx context.Context, key storebackend.Key, tctx *target.Context) { - target := &invv1alpha1.Target{} - if err := r.client.Get(ctx, key.NamespacedName, target); err != nil { - log.Error("cannot get target", "key", key.String(), "error", err.Error()) - return - } + targets := &invv1alpha1.TargetList{} + if err := r.client.List(ctx, targets); err != nil { + return nil, err + } - if options.LabelSelector != nil || filter != nil { - f := true - if options.LabelSelector != nil { - if options.LabelSelector.Matches(labels.Set(target.GetLabels())) { - f = false - } - } else { - // if not labels selector is present don't filter - f = false - } - // if filtered we dont have to run this section since the label requirement was not met - if filter != nil && !f { - if filter.Name != "" { - if target.GetName() == filter.Name { - f = false - } else { - f = true - } - } - if filter.Namespace != "" { - if target.GetNamespace() == filter.Namespace { - f = false - } else { - f = true - } - } - } - if !f { - obj, err := tctx.GetBlameConfig(ctx, key) - if err != nil { - log.Error("cannot get configblame", "key", key.String(), "error", err.Error()) - return - } - obj.SetCreationTimestamp(target.CreationTimestamp) - obj.SetResourceVersion(target.ResourceVersion) - obj.SetAnnotations(target.Annotations) - obj.SetLabels(target.Labels) - utils.AppendItem(v, obj) - } - } else { - obj, err := tctx.GetBlameConfig(ctx, key) - if err != nil { - log.Error("cannot get configblame", "key", key.String(), "error", err.Error()) - return - } - obj.SetCreationTimestamp(target.CreationTimestamp) - obj.SetResourceVersion(target.ResourceVersion) - obj.SetAnnotations(target.Annotations) - obj.SetLabels(target.Labels) - utils.AppendItem(v, obj) + for _, target := range targets.Items { + key := target.GetNamespacedName() + obj, err := r.getConfigBlame(ctx, &target, key) + if err != nil { + log.Error("cannot get configblame", "key", key.String(), "error", err.Error()) + continue } + utils.AppendItem(v, obj) } - if err := r.targetStore.List(ctx, configBlameListFunc); err != nil { - log.Error("list failed", "err", err) - } return newListObj, nil } @@ -287,18 +305,3 @@ func (r *strategy) GetResetFields() map[fieldpath.APIVersion]*fieldpath.Set { return fields } - -func (r *strategy) getTargetContext(ctx context.Context, targetKey types.NamespacedName) (*invv1alpha1.Target, *target.Context, error) { - target := &invv1alpha1.Target{} - if err := r.client.Get(ctx, targetKey, target); err != nil { - return nil, nil, err - } - if !target.IsReady() { - return nil, nil, errors.New(string(config.ConditionReasonTargetNotReady)) - } - tctx, err := r.targetStore.Get(ctx, storebackend.Key{NamespacedName: targetKey}) - if err != nil { - return nil, nil, errors.New(string(config.ConditionReasonTargetNotFound)) - } - return target, tctx, nil -} diff --git a/pkg/registry/options/options.go b/pkg/registry/options/options.go index fab6d0b1..b878c04d 100644 --- a/pkg/registry/options/options.go +++ b/pkg/registry/options/options.go @@ -21,7 +21,7 @@ import ( "github.com/dgraph-io/badger/v4" "github.com/henderiw/apiserver-store/pkg/storebackend" - "github.com/sdcio/config-server/pkg/target" + "github.com/sdcio/config-server/pkg/sdc/target" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" diff --git a/pkg/registry/runningconfig/strategy_resource.go b/pkg/registry/runningconfig/strategy_resource.go index edd6e3fa..8851f76f 100644 --- a/pkg/registry/runningconfig/strategy_resource.go +++ b/pkg/registry/runningconfig/strategy_resource.go @@ -18,7 +18,7 @@ package runningconfig import ( "context" - "errors" + "fmt" "github.com/henderiw/apiserver-builder/pkg/builder/resource" "github.com/henderiw/apiserver-builder/pkg/builder/utils" @@ -29,10 +29,11 @@ import ( "github.com/sdcio/config-server/apis/config" invv1alpha1 "github.com/sdcio/config-server/apis/inv/v1alpha1" "github.com/sdcio/config-server/pkg/registry/options" - "github.com/sdcio/config-server/pkg/target" + dsclient "github.com/sdcio/config-server/pkg/sdc/dataserver/client" + sdcpb "github.com/sdcio/sdc-protos/sdcpb" apierrors "k8s.io/apimachinery/pkg/api/errors" metainternalversion "k8s.io/apimachinery/pkg/apis/meta/internalversion" - "k8s.io/apimachinery/pkg/labels" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" @@ -61,7 +62,6 @@ func NewStrategy( storage: storage, watcherManager: watcherManager, client: opts.Client, - targetStore: opts.TargetStore, } } @@ -81,7 +81,6 @@ type strategy struct { storage storebackend.Storer[runtime.Object] watcherManager watchermanager.WatcherManager client client.Client - targetStore storebackend.Storer[*target.Context] } func (r *strategy) NamespaceScoped() bool { return r.obj.NamespaceScoped() } @@ -150,23 +149,63 @@ func (r *strategy) Delete(ctx context.Context, key types.NamespacedName, obj run return obj, nil } -func (r *strategy) Get(ctx context.Context, key types.NamespacedName) (runtime.Object, error) { - target, tctx, err := r.getTargetContext(ctx, key) - if err != nil { +func (r *strategy) getRunningConfig(ctx context.Context, target *invv1alpha1.Target, key types.NamespacedName) (*config.RunningConfig, error) { + if !target.IsReady() { return nil, apierrors.NewNotFound(r.gr, key.Name) } - rc, err := tctx.GetData(ctx, storebackend.KeyFromNSN(key)) + cfg := &dsclient.Config{ + Address: dsclient.GetDataServerAddress(), + Insecure: true, + } + + dsclient, closeFn, err := dsclient.NewEphemeral(ctx, cfg) + if err != nil { + return nil, err + } + defer func() { + if err := closeFn(); err != nil { + // You can use your preferred logging framework here + fmt.Printf("failed to close connection: %v\n", err) + } + }() + + // check if the schema exists; this is == nil check; in case of err it does not exist + rsp, err := dsclient.GetIntent(ctx, &sdcpb.GetIntentRequest{ + DatastoreName: storebackend.KeyFromNSN(key).String(), + Intent: "running", + Format: sdcpb.Format_Intent_Format_JSON, + }) if err != nil { - return nil, apierrors.NewInternalError(err) + return nil, err } + + rc := config.BuildRunningConfig( + metav1.ObjectMeta{ + Name: key.Name, + Namespace: key.Namespace, + }, + config.RunningConfigSpec{}, + config.RunningConfigStatus{ + Value: runtime.RawExtension{ + Raw: rsp.GetBlob(), + }, + }, + ) + rc.SetCreationTimestamp(target.CreationTimestamp) rc.SetResourceVersion(target.ResourceVersion) rc.SetAnnotations(target.Annotations) rc.SetLabels(target.Labels) - obj := rc + return rc, nil +} - return obj, nil +func (r *strategy) Get(ctx context.Context, key types.NamespacedName) (runtime.Object, error) { + target := &invv1alpha1.Target{} + if err := r.client.Get(ctx, key, target); err != nil { + return nil, apierrors.NewNotFound(r.gr, key.Name) + } + return r.getRunningConfig(ctx, target, key) } func (r *strategy) List(ctx context.Context, options *metainternalversion.ListOptions) (runtime.Object, error) { @@ -194,69 +233,21 @@ func (r *strategy) List(ctx context.Context, options *metainternalversion.ListOp return nil, err } - runningConfigListFunc := func(ctx context.Context, key storebackend.Key, tctx *target.Context) { - target := &invv1alpha1.Target{} - if err := r.client.Get(ctx, key.NamespacedName, target); err != nil { - log.Error("cannot get target", "key", key.String(), "error", err.Error()) - return - } + targets := &invv1alpha1.TargetList{} + if err := r.client.List(ctx, targets); err != nil { + return nil, err + } - if options.LabelSelector != nil || filter != nil { - f := true - if options.LabelSelector != nil { - if options.LabelSelector.Matches(labels.Set(target.GetLabels())) { - f = false - } - } else { - // if not labels selector is present don't filter - f = false - } - // if filtered we dont have to run this section since the label requirement was not met - if filter != nil && !f { - if filter.Name != "" { - if target.GetName() == filter.Name { - f = false - } else { - f = true - } - } - if filter.Namespace != "" { - if target.GetNamespace() == filter.Namespace { - f = false - } else { - f = true - } - } - } - if !f { - obj, err := tctx.GetData(ctx, key) - if err != nil { - log.Error("cannot get running config", "key", key.String(), "error", err.Error()) - return - } - obj.SetCreationTimestamp(target.CreationTimestamp) - obj.SetResourceVersion(target.ResourceVersion) - obj.SetAnnotations(target.Annotations) - obj.SetLabels(target.Labels) - utils.AppendItem(v, obj) - } - } else { - obj, err := tctx.GetData(ctx, key) - if err != nil { - log.Error("cannot get running config", "key", key.String(), "error", err.Error()) - return - } - obj.SetCreationTimestamp(target.CreationTimestamp) - obj.SetResourceVersion(target.ResourceVersion) - obj.SetAnnotations(target.Annotations) - obj.SetLabels(target.Labels) - utils.AppendItem(v, obj) + for _, target := range targets.Items { + key := target.GetNamespacedName() + obj, err := r.getRunningConfig(ctx, &target, key) + if err != nil { + log.Error("cannot get configblame", "key", key.String(), "error", err.Error()) + continue } + utils.AppendItem(v, obj) } - if err := r.targetStore.List(ctx, runningConfigListFunc); err != nil { - log.Error("list failed", "err", err) - } return newListObj, nil } @@ -287,18 +278,3 @@ func (r *strategy) GetResetFields() map[fieldpath.APIVersion]*fieldpath.Set { return fields } - -func (r *strategy) getTargetContext(ctx context.Context, targetKey types.NamespacedName) (*invv1alpha1.Target, *target.Context, error) { - target := &invv1alpha1.Target{} - if err := r.client.Get(ctx, targetKey, target); err != nil { - return nil, nil, err - } - if !target.IsReady() { - return nil, nil, errors.New(string(config.ConditionReasonTargetNotReady)) - } - tctx, err := r.targetStore.Get(ctx, storebackend.Key{NamespacedName: targetKey}) - if err != nil { - return nil, nil, errors.New(string(config.ConditionReasonTargetNotFound)) - } - return target, tctx, nil -} diff --git a/pkg/sdc/ctx/dataserver_context.go b/pkg/sdc/ctx/dataserver_context.go index 33dc9088..926be3e2 100644 --- a/pkg/sdc/ctx/dataserver_context.go +++ b/pkg/sdc/ctx/dataserver_context.go @@ -18,7 +18,6 @@ package sdcctx import ( "context" - "os" "github.com/henderiw/apiserver-store/pkg/storebackend" "github.com/henderiw/logger/log" @@ -37,15 +36,9 @@ type DSContext struct { func CreateDataServerClient(ctx context.Context, dataServerStore storebackend.Storer[DSContext], client client.Client) error { log := log.FromContext(ctx) - dataServerAddress := localDataServerAddress - if address, found := os.LookupEnv("SDC_DATA_SERVER"); found { - dataServerAddress = address - } - - // TODO the population of the dataservers in the store should become dynamic, through a controller - // right now it is static since all of this happens in the same pod and we dont have scaled out dataservers + // This is for the local connection from the controller to the dataserver on the same local pod dsConfig := &dsclient.Config{ - Address: dataServerAddress, + Address: dsclient.GetLocalDataServerAddress(), } dsClient, err := dsclient.New(dsConfig) if err != nil { @@ -61,7 +54,7 @@ func CreateDataServerClient(ctx context.Context, dataServerStore storebackend.St Targets: sets.New[string](), DSClient: dsClient, } - if err := dataServerStore.Create(ctx, storebackend.ToKey(dataServerAddress), dsCtx); err != nil { + if err := dataServerStore.Create(ctx, storebackend.ToKey(dsclient.GetLocalDataServerAddress()), dsCtx); err != nil { log.Error("cannot store datastore context in dataserver", "err", err) return err } diff --git a/pkg/sdc/ctx/schemaserver_context.go b/pkg/sdc/ctx/schemaserver_context.go index 98387f5d..6bce8f9a 100644 --- a/pkg/sdc/ctx/schemaserver_context.go +++ b/pkg/sdc/ctx/schemaserver_context.go @@ -16,6 +16,8 @@ limitations under the License. package sdcctx +/* + import ( "context" "os" @@ -37,6 +39,7 @@ type SSContext struct { const ( localDataServerAddress = "localhost:56000" + SchemaServerAddress = "data-server-0.schema-server.sdc-system.svc.cluster.local:56000" ) func CreateSchemaServerClient(ctx context.Context, schemaServerStore storebackend.Storer[SSContext], client client.Client) error { @@ -44,7 +47,7 @@ func CreateSchemaServerClient(ctx context.Context, schemaServerStore storebacken // For the schema server we first check if the SDC_SCHEMA_SERVER was et if not we could also use // the SDC_DATA_SERVER as fallback. If none are set it is the default address (localhost) - schemaServerAddress := localDataServerAddress + schemaServerAddress := SchemaServerAddress if address, found := os.LookupEnv("SDC_SCHEMA_SERVER"); found { schemaServerAddress = address } else { @@ -146,3 +149,4 @@ func DeleteSchemaServerClient(ctx context.Context, schemaServerStore storebacken log.Info("schemaserver client deleted") return nil } +*/ \ No newline at end of file diff --git a/pkg/sdc/dataserver/client/client.go b/pkg/sdc/dataserver/client/client.go index ce80c671..90d8916a 100644 --- a/pkg/sdc/dataserver/client/client.go +++ b/pkg/sdc/dataserver/client/client.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "time" + "os" "github.com/henderiw/logger/log" sdcpb "github.com/sdcio/sdc-protos/sdcpb" @@ -29,13 +30,21 @@ import ( "google.golang.org/grpc/keepalive" ) -type Client interface { - Start(ctx context.Context) error - Stop(ctx context.Context) - GetAddress() string - IsConnectionReady() bool - IsConnected() bool - sdcpb.DataServerClient +const dataServerAddress = "data-server.sdc-system.svc.cluster.local:56000" +const localDataServerAddress = "localhost:56000" + +func GetDataServerAddress() string { + if address, found := os.LookupEnv("SDC_DATA_SERVER"); found { + return address + } + return dataServerAddress +} + +func GetLocalDataServerAddress() string { + if address, found := os.LookupEnv("SDC_DATA_SERVER"); found { + return address + } + return localDataServerAddress } type Config struct { @@ -52,17 +61,117 @@ type Config struct { MaxMsgSize int } -func New(cfg *Config) (Client, error) { +func defaukltConfig(cfg *Config) { if cfg == nil { - return nil, fmt.Errorf("cannot create client with empty config") + cfg = &Config{Address: GetDataServerAddress()} } if cfg.Address == "" { - return nil, fmt.Errorf("cannot create client with an empty address") + cfg.Address = GetDataServerAddress() } +} + +/* +Example: One-shot use + + cfg := &client.Config{ + Address: "data-server.sdc-system.svc.cluster.local:50052", + Insecure: true, + } + + err := client.OneShot(ctx, cfg, func(ctx context.Context, c sdcpb.DataServerClient) error { + resp, err := c.ListDataStore(ctx, &sdcpb.ListDataStoreRequest{}) + if err != nil { + return err + } + fmt.Printf("Got %d datastores\n", len(resp.GetDatastores())) + return nil + }) + if err != nil { + // handle error + } +*/ + +// OneShot runs a function with a short-lived DataServerClient. +// It dials, runs fn, and always closes the connection. +func OneShot( + ctx context.Context, + cfg *Config, + fn func(ctx context.Context, c sdcpb.DataServerClient) error, +) error { + defaukltConfig(cfg) + conn, err := dial(ctx, cfg) + if err != nil { + return err + } + + defer func() { + if err := conn.Close(); err != nil { + // You can use your preferred logging framework here + fmt.Printf("failed to close connection: %v\n", err) + } + }() + + c := sdcpb.NewDataServerClient(conn) + return fn(ctx, c) +} + +/* +Example: Ephemeral client + + cfg := &client.Config{ + Address: "data-server.sdc-system.svc.cluster.local:50052", + Insecure: true, + } + + c, closeFn, err := client.NewEphemeral(ctx, cfg) + if err != nil { + // handle + return + } + defer closeFn() + + resp, err := c.ListIntent(ctx, &sdcpb.ListIntentRequest{}) + if err != nil { + // handle + } + _ = resp +*/ + +// NewEphemeral returns a short-lived client and a close function. +// Call closeFn() when you're done. +func NewEphemeral( + ctx context.Context, + cfg *Config, +) (sdcpb.DataServerClient, func() error, error) { + defaukltConfig(cfg) + conn, err := dial(ctx, cfg) + if err != nil { + return nil, nil, err + } + + c := sdcpb.NewDataServerClient(conn) + closeFn := func() error { + return conn.Close() + } + return c, closeFn, nil +} + +// Client is a long-lived DataServer client (for controllers, daemons, etc.). +type Client interface { + Start(ctx context.Context) error + Stop(ctx context.Context) + GetAddress() string + IsConnectionReady() bool + IsConnected() bool + sdcpb.DataServerClient +} + - return &client{ - cfg: cfg, - }, nil + +func New(cfg *Config) (Client, error) { + defaukltConfig(cfg) + // default cancel is a no-op so Stop() is always safe + return &client{cfg: cfg, cancel: func() {}}, nil } type client struct { @@ -74,24 +183,58 @@ type client struct { const DSConnectionStatusNotConnected = "DATASERVER_NOT_CONNECTED" -func (r *client) IsConnectionReady() bool { - if r.conn == nil { - return false +// dial creates a gRPC connection with a timeout and proper options, +// using the modern grpc.NewClient API. +func dial(ctx context.Context, cfg *Config) (*grpc.ClientConn, error) { + defaukltConfig(cfg) + + dialCtx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + + opts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + // TODO: add TLS / max msg size based on cfg here. + } + + conn, err := grpc.NewClient(cfg.Address, opts...) + if err != nil { + return nil, fmt.Errorf("create data-server client %q: %w", cfg.Address, err) + } + + // Proactively start connecting and wait for Ready (or timeout/cancel). + conn.Connect() + for { + state := conn.GetState() + if state == connectivity.Ready { + return conn, nil + } + if !conn.WaitForStateChange(dialCtx, state) { + _ = conn.Close() + return nil, fmt.Errorf("gRPC connect timeout to %q; last state: %s", cfg.Address, state.String()) + } } - return r.conn.GetState() == connectivity.Ready +} + +func (r *client) IsConnectionReady() bool { + return r.conn != nil && r.conn.GetState() == connectivity.Ready } func (r *client) IsConnected() bool { - return r.conn != nil + return r.conn != nil && r.conn.GetState() != connectivity.Shutdown } func (r *client) Stop(ctx context.Context) { log := log.FromContext(ctx).With("address", r.cfg.Address) log.Info("stopping...") - if err := r.conn.Close(); err != nil { - log.Error("close error", "err", err) + + if r.conn != nil { + if err := r.conn.Close(); err != nil { + log.Error("close error", "err", err) + } + } + if r.cancel != nil { + r.cancel() } - r.cancel() } func (r *client) Start(ctx context.Context) error { @@ -113,8 +256,16 @@ func (r *client) Start(ctx context.Context) error { return err } - //defer conn.Close() r.dsclient = sdcpb.NewDataServerClient(r.conn) + + // Long-lived cancel for Stop() + runCtx, cancel := context.WithCancel(context.Background()) + r.cancel = cancel + go func() { + <-runCtx.Done() + log.Info("stopped...") + }() + log.Info("started...") return nil } @@ -123,6 +274,8 @@ func (r *client) GetAddress() string { return r.cfg.Address } +// ---------- RPC wrappers for sdcpb.DataServerClient ---------- + func (r *client) ListDataStore(ctx context.Context, in *sdcpb.ListDataStoreRequest, opts ...grpc.CallOption) (*sdcpb.ListDataStoreResponse, error) { return r.dsclient.ListDataStore(ctx, in, opts...) } @@ -169,57 +322,4 @@ func (r *client) TransactionCancel(ctx context.Context, in *sdcpb.TransactionCan func (r *client) BlameConfig(ctx context.Context, in *sdcpb.BlameConfigRequest, opts ...grpc.CallOption) (*sdcpb.BlameConfigResponse, error) { return r.dsclient.BlameConfig(ctx, in, opts...) -} - -/* -func (r *client) getGRPCOpts() ([]grpc.DialOption, error) { - var opts []grpc.DialOption - fmt.Printf("grpc client config: %v\n", r.cfg) - if r.cfg.Insecure { - //opts = append(opts, grpc.WithInsecure()) - opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials())) - } else { - tlsConfig, err := r.newTLS() - if err != nil { - return nil, err - } - opts = append(opts, grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig))) - } - return opts, nil -} - -func (r *client) newTLS() (*tls.Config, error) { - tlsConfig := &tls.Config{ - Renegotiation: tls.RenegotiateNever, - InsecureSkipVerify: r.cfg.SkipVerify, - } - //err := loadCerts(tlsConfig) - //if err != nil { - // return nil, err - //} - return tlsConfig, nil -} - -func loadCerts(tlscfg *tls.Config) error { - if c.TLSCert != "" && c.TLSKey != "" { - certificate, err := tls.LoadX509KeyPair(*c.TLSCert, *c.TLSKey) - if err != nil { - return err - } - tlscfg.Certificates = []tls.Certificate{certificate} - tlscfg.BuildNameToCertificate() - } - if c.TLSCA != nil && *c.TLSCA != "" { - certPool := x509.NewCertPool() - caFile, err := ioutil.ReadFile(*c.TLSCA) - if err != nil { - return err - } - if ok := certPool.AppendCertsFromPEM(caFile); !ok { - return errors.New("failed to append certificate") - } - tlscfg.RootCAs = certPool - } - return nil -} -*/ +} \ No newline at end of file diff --git a/pkg/sdc/schemaserver/client/client.go b/pkg/sdc/schemaserver/client/client.go index 591e204e..c1402a94 100644 --- a/pkg/sdc/schemaserver/client/client.go +++ b/pkg/sdc/schemaserver/client/client.go @@ -1,5 +1,5 @@ /* -Copyright 2024 Nokia. +Copyright 2025 Nokia. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -20,21 +20,25 @@ import ( "context" "fmt" "time" + "os" "github.com/henderiw/logger/log" sdcpb "github.com/sdcio/sdc-protos/sdcpb" "google.golang.org/grpc" - "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials/insecure" ) -type Client interface { - Start(ctx context.Context) error - Stop(ctx context.Context) - GetAddress() string - IsConnectionReady() bool - IsConnected() bool - sdcpb.SchemaServerClient +const schemServerAddress = "schema-server.sdc-system.svc.cluster.local:56000" + +func GetSchemaServerAddress() string { + if address, found := os.LookupEnv("SDC_SCHEMA_SERVER"); found { + return address + } + if address, found := os.LookupEnv("SDC_DATA_SERVER"); found { + return address + } + return schemServerAddress } type Config struct { @@ -51,17 +55,115 @@ type Config struct { MaxMsgSize int } -func New(cfg *Config) (Client, error) { +func defaultConfig(cfg *Config) { if cfg == nil { - return nil, fmt.Errorf("cannot create client with empty config") + cfg = &Config{Address: GetSchemaServerAddress()} } if cfg.Address == "" { - return nil, fmt.Errorf("cannot create client with an empty address") + cfg.Address = GetSchemaServerAddress() } +} + +/* +Example: One-shot use - return &client{ - cfg: cfg, - }, nil + cfg := &client.Config{ + Address: "schema-server.sdc-system.svc.cluster.local:50051", + Insecure: true, + } + + err := client.OneShot(ctx, cfg, func(ctx context.Context, c sdcpb.SchemaServerClient) error { + resp, err := c.ListSchema(ctx, &sdcpb.ListSchemaRequest{}) + if err != nil { + return err + } + fmt.Printf("Got %d schemas\n", len(resp.GetSchemas())) + return nil + }) + if err != nil { + // handle error + } +*/ + +// OneShot runs a function with a short-lived SchemaServerClient. +// It dials, runs fn, and always closes the connection. +func OneShot( + ctx context.Context, + cfg *Config, + fn func(ctx context.Context, c sdcpb.SchemaServerClient) error, +) error { + defaultConfig(cfg) + conn, err := dial(ctx, cfg) + if err != nil { + return err + } + defer func() { + if err := conn.Close(); err != nil { + // You can use your preferred logging framework here + fmt.Printf("failed to close connection: %v\n", err) + } + }() + + c := sdcpb.NewSchemaServerClient(conn) + return fn(ctx, c) +} + +/* +Example: Ephemeral client + + cfg := &client.Config{ + Address: "schema-server.sdc-system.svc.cluster.local:50051", + Insecure: true, + } + + c, closeFn, err := client.NewEphemeral(ctx, cfg) + if err != nil { + // handle + return + } + defer closeFn() + + resp, err := c.GetSchemaDetails(ctx, &sdcpb.GetSchemaDetailsRequest{ }) + if err != nil { + // handle + } + _ = resp +*/ + +// NewEphemeral returns a short-lived client and a close function. +// Call closeFn() when you're done. +func NewEphemeral( + ctx context.Context, + cfg *Config, +) (sdcpb.SchemaServerClient, func() error, error) { + defaultConfig(cfg) + conn, err := dial(ctx, cfg) + if err != nil { + return nil, nil, err + } + + c := sdcpb.NewSchemaServerClient(conn) + closeFn := func() error { + return conn.Close() + } + return c, closeFn, nil +} + +// Client is a long-lived SchemaServer client (for controllers, daemons, etc.). +type Client interface { + Start(ctx context.Context) error + Stop(ctx context.Context) + GetAddress() string + IsConnectionReady() bool + IsConnected() bool + sdcpb.SchemaServerClient +} + + +func New(cfg *Config) (Client, error) { + defaultConfig(cfg) + // default cancel is a no-op so Stop() is always safe + return &client{cfg: cfg, cancel: func() {}}, nil } type client struct { @@ -71,6 +173,41 @@ type client struct { schemaclient sdcpb.SchemaServerClient } +const DSConnectionStatusNotConnected = "DATASERVER_NOT_CONNECTED" + +// dial creates a gRPC connection with a timeout and proper options, +// using the modern grpc.NewClient API. +func dial(ctx context.Context, cfg *Config) (*grpc.ClientConn, error) { + defaultConfig(cfg) + + dialCtx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + + opts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + // TODO: add TLS / max msg size based on cfg here. + } + + conn, err := grpc.NewClient(cfg.Address, opts...) + if err != nil { + return nil, fmt.Errorf("create schema-server client %q: %w", cfg.Address, err) + } + + // Proactively start connecting and wait for Ready (or timeout/cancel). + conn.Connect() + for { + state := conn.GetState() + if state == connectivity.Ready { + return conn, nil + } + if !conn.WaitForStateChange(dialCtx, state) { + // context expired or canceled + _ = conn.Close() + return nil, fmt.Errorf("gRPC connect timeout to %q; last state: %s", cfg.Address, state.String()) + } + } +} + func (r *client) IsConnectionReady() bool { return r.conn != nil && r.conn.GetState() == connectivity.Ready } @@ -82,54 +219,37 @@ func (r *client) IsConnected() bool { func (r *client) Stop(ctx context.Context) { log := log.FromContext(ctx).With("address", r.cfg.Address) log.Info("stopping...") - if err := r.conn.Close(); err != nil { - log.Error("close error", "err", err) + + if r.conn != nil { + if err := r.conn.Close(); err != nil { + log.Error("close error", "err", err) + } + } + if r.cancel != nil { + r.cancel() } - r.cancel() } func (r *client) Start(ctx context.Context) error { log := log.FromContext(ctx).With("address", r.cfg.Address) log.Info("starting...") - dialCtx, cancel := context.WithTimeout(ctx, 10*time.Second) - defer cancel() - - // Create the ClientConn - conn, err := grpc.NewClient( - r.cfg.Address, - grpc.WithTransportCredentials(insecure.NewCredentials()), - ) - if err != nil { - return err - } - - // Proactively start connecting and wait for Ready (or timeout/cancel) - conn.Connect() - for { - s := conn.GetState() - if s == connectivity.Ready { - break - } - if !conn.WaitForStateChange(dialCtx, s) { - // context expired or canceled - if err := r.conn.Close(); err != nil { - log.Error("close error", "err", err) - } - return fmt.Errorf("gRPC connect timeout; last state: %s", s.String()) - } - } + conn, err := dial(ctx, r.cfg) + if err != nil { + return err + } r.conn = conn r.schemaclient = sdcpb.NewSchemaServerClient(r.conn) - // Create a long-lived cancel just for Stop() - runCtx, stop := context.WithCancel(context.Background()) - r.cancel = stop + // Long-lived cancel for Stop() + runCtx, cancel := context.WithCancel(context.Background()) + r.cancel = cancel go func() { <-runCtx.Done() - log.Info("stopped...") + log.Info("stopped...") }() + log.Info("started...") return nil } @@ -138,6 +258,8 @@ func (r *client) GetAddress() string { return r.cfg.Address } +// ---------- RPC wrappers for sdcpb.SchemaServerClient ---------- + // returns schema name, vendor, version, and files path(s) func (r *client) GetSchemaDetails(ctx context.Context, in *sdcpb.GetSchemaDetailsRequest, opts ...grpc.CallOption) (*sdcpb.GetSchemaDetailsResponse, error) { return r.schemaclient.GetSchemaDetails(ctx, in, opts...) @@ -189,57 +311,4 @@ func (r *client) ExpandPath(ctx context.Context, in *sdcpb.ExpandPathRequest, op // GetSchemaElements returns the schema of each path element func (r *client) GetSchemaElements(ctx context.Context, in *sdcpb.GetSchemaRequest, opts ...grpc.CallOption) (sdcpb.SchemaServer_GetSchemaElementsClient, error) { return r.schemaclient.GetSchemaElements(ctx, in, opts...) -} - -/* -func (r *client) getGRPCOpts() ([]grpc.DialOption, error) { - var opts []grpc.DialOption - fmt.Printf("grpc client config: %v\n", r.cfg) - if r.cfg.Insecure { - //opts = append(opts, grpc.WithInsecure()) - opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials())) - } else { - tlsConfig, err := r.newTLS() - if err != nil { - return nil, err - } - opts = append(opts, grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig))) - } - return opts, nil -} - -func (r *client) newTLS() (*tls.Config, error) { - tlsConfig := &tls.Config{ - Renegotiation: tls.RenegotiateNever, - InsecureSkipVerify: r.cfg.SkipVerify, - } - //err := loadCerts(tlsConfig) - //if err != nil { - // return nil, err - //} - return tlsConfig, nil -} - -func loadCerts(tlscfg *tls.Config) error { - if c.TLSCert != "" && c.TLSKey != "" { - certificate, err := tls.LoadX509KeyPair(*c.TLSCert, *c.TLSKey) - if err != nil { - return err - } - tlscfg.Certificates = []tls.Certificate{certificate} - tlscfg.BuildNameToCertificate() - } - if c.TLSCA != nil && *c.TLSCA != "" { - certPool := x509.NewCertPool() - caFile, err := ioutil.ReadFile(*c.TLSCA) - if err != nil { - return err - } - if ok := certPool.AppendCertsFromPEM(caFile); !ok { - return errors.New("failed to append certificate") - } - tlscfg.RootCAs = certPool - } - return nil -} -*/ +} \ No newline at end of file diff --git a/pkg/target/collector.go b/pkg/sdc/target/collector.go similarity index 100% rename from pkg/target/collector.go rename to pkg/sdc/target/collector.go diff --git a/pkg/target/context.go b/pkg/sdc/target/context.go similarity index 50% rename from pkg/target/context.go rename to pkg/sdc/target/context.go index b2b76a60..3097e9c3 100644 --- a/pkg/target/context.go +++ b/pkg/sdc/target/context.go @@ -18,10 +18,9 @@ package target import ( "context" - errors "errors" "fmt" - "strings" "sync" + "log/slog" "strconv" @@ -34,13 +33,8 @@ import ( dsclient "github.com/sdcio/config-server/pkg/sdc/dataserver/client" sdcpb "github.com/sdcio/sdc-protos/sdcpb" "google.golang.org/grpc" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" - "google.golang.org/protobuf/encoding/protojson" "google.golang.org/protobuf/encoding/prototext" "google.golang.org/protobuf/proto" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" @@ -169,14 +163,18 @@ func (r *Context) CreateDS(ctx context.Context, datastoreReq *sdcpb.CreateDataSt return nil } +func (r *Context) ds() (dsclient.Client, error) { + if r == nil || r.dsclient == nil { + return nil, fmt.Errorf("datastore client not initialized") + } + return r.dsclient, nil +} + func (r *Context) IsReady() bool { if r == nil { return false } - if r.client != nil && r.getDatastoreReq() != nil && r.getReady() { - return true - } - return false + return r.client != nil && r.getDatastoreReq() != nil && r.getReady() } func (r *Context) SetNotReady(ctx context.Context) { @@ -195,22 +193,29 @@ func (r *Context) SetReady(ctx context.Context) { log := log.FromContext(ctx) log.Info("SetReady", "ready", r.getReady(), "recoveredConfigs", r.getRecoveredTargetConfigsState()) - alreadyReady := r.getReady() - if !alreadyReady { + if !r.getReady() { r.setCtxReady(true) } - // if we are not ready we set the status to ready and start the deviation watcher and subscription handler if r.deviationWatcher != nil { r.deviationWatcher.Start(ctx) } - if r.subscriptions.HasSubscriptions() && r.collector != nil && !r.collector.IsRunning() { - req := r.getDatastoreReq() - if r.IsReady() && req != nil && req.Target != nil { - if err := r.collector.Start(ctx, r.getDatastoreReq()); err != nil { - log.Error("setready starting collector failed", "err", err) - } - } + + if r.collector == nil || r.collector.IsRunning() { + return + } + + if !r.subscriptions.HasSubscriptions() { + return + } + + req := r.getDatastoreReq() + if req == nil || req.Target == nil { + return + } + + if err := r.collector.Start(ctx, req); err != nil { + log.Error("setready starting collector failed", "err", err) } } @@ -229,159 +234,33 @@ func (r *Context) IsTargetConfigRecovered(ctx context.Context) bool { } func (r *Context) deleteDataStore(ctx context.Context, in *sdcpb.DeleteDataStoreRequest, opts ...grpc.CallOption) (*sdcpb.DeleteDataStoreResponse, error) { - if r == nil { - return nil, fmt.Errorf("datastore client not initialized") - } - if r.dsclient == nil { - return nil, fmt.Errorf("datastore client not initialized") + ds, err := r.ds() + if err != nil { + return nil, err } - return r.dsclient.DeleteDataStore(ctx, in, opts...) + return ds.DeleteDataStore(ctx, in, opts...) } func (r *Context) createDataStore(ctx context.Context, in *sdcpb.CreateDataStoreRequest, opts ...grpc.CallOption) (*sdcpb.CreateDataStoreResponse, error) { - if r == nil { - return nil, fmt.Errorf("datastore client not initialized") - } - if r.dsclient == nil { - return nil, fmt.Errorf("datastore client not initialized") + ds, err := r.ds() + if err != nil { + return nil, err } - return r.dsclient.CreateDataStore(ctx, in, opts...) + return ds.CreateDataStore(ctx, in, opts...) } func (r *Context) GetDataStore(ctx context.Context, in *sdcpb.GetDataStoreRequest, opts ...grpc.CallOption) (*sdcpb.GetDataStoreResponse, error) { - if r == nil { - return nil, fmt.Errorf("datastore client not initialized") - } - if r.dsclient == nil { - return nil, fmt.Errorf("datastore client not initialized") - } - return r.dsclient.GetDataStore(ctx, in, opts...) -} - -// useSpec indicates to use the spec as the confifSpec, typically set to true; when set to false it means we are recovering -// the config -func (r *Context) getIntentUpdate(ctx context.Context, key storebackend.Key, config *config.Config, useSpec bool) ([]*sdcpb.Update, error) { - log := log.FromContext(ctx) - update := make([]*sdcpb.Update, 0, len(config.Spec.Config)) - configSpec := config.Spec.Config - if !useSpec && config.Status.AppliedConfig != nil { - update = make([]*sdcpb.Update, 0, len(config.Status.AppliedConfig.Config)) - configSpec = config.Status.AppliedConfig.Config - } - - for _, config := range configSpec { - path, err := sdcpb.ParsePath(config.Path) - if err != nil { - return nil, fmt.Errorf("create data failed for target %s, path %s invalid", key.String(), config.Path) - } - log.Debug("setIntent", "configSpec", string(config.Value.Raw)) - update = append(update, &sdcpb.Update{ - Path: path, - Value: &sdcpb.TypedValue{ - Value: &sdcpb.TypedValue_JsonVal{ - JsonVal: config.Value.Raw, - }, - }, - }) - } - return update, nil -} - -func (r *Context) getDeviationUpdate(ctx context.Context, targetKey storebackend.Key, deviation *config.Deviation) ([]*sdcpb.Update, []*sdcpb.Path, error) { - log := log.FromContext(ctx) - updates := make([]*sdcpb.Update, 0) - deletes := make([]*sdcpb.Path, 0) - - for _, deviation := range deviation.Spec.Deviations { - if deviation.Reason == "NOT_APPLIED" { - path, err := sdcpb.ParsePath(deviation.Path) - if err != nil { - return nil, nil, fmt.Errorf("deviation path parsing failed forr target %s, path %s invalid", targetKey.String(), deviation.Path) - } - if deviation.CurrentValue == nil { - deletes = append(deletes, path) - } else { - val := &sdcpb.TypedValue{} - if err := prototext.Unmarshal([]byte(*deviation.CurrentValue), val); err != nil { - log.Error("deviation proto unmarshal failed", "key", targetKey.String(), "path", deviation.Path, "currentValue", deviation.CurrentValue) - continue - } - updates = append(updates, &sdcpb.Update{ - Path: path, - Value: val, - }) - } - } + ds, err := r.ds() + if err != nil { + return nil, err } - return updates, deletes, nil + return ds.GetDataStore(ctx, in, opts...) } -/* -func parse_value(value string) (*sdcpb.TypedValue, error) { - // Remove any outer quotes or trimming whitespace - value = strings.TrimSpace(value) - - // Split on first `:` only to separate type and value - parts := strings.SplitN(value, ":", 2) - if len(parts) != 2 { - return nil, fmt.Errorf("invalid format: %s", value) - } - key := strings.TrimSpace(parts[0]) - val := strings.TrimSpace(parts[1]) - - switch key { - case "string_val": - return &sdcpb.TypedValue{ - Value: &sdcpb.TypedValue_StringVal{ - StringVal: strings.Trim(val, `"`), - }, - }, nil - - case "uint_val": - num, err := strconv.ParseUint(val, 10, 64) - if err != nil { - return nil, fmt.Errorf("invalid uint_val: %w", err) - } - return &sdcpb.TypedValue{ - Value: &sdcpb.TypedValue_UintVal{ - UintVal: num, - }, - }, nil - - case "int_val": - num, err := strconv.ParseInt(val, 10, 64) - if err != nil { - return nil, fmt.Errorf("invalid int_val: %w", err) - } - return &sdcpb.TypedValue{ - Value: &sdcpb.TypedValue_IntVal{ - IntVal: num, - }, - }, nil - - case "bool_val": - b, err := strconv.ParseBool(val) - if err != nil { - return nil, fmt.Errorf("invalid bool_val: %w", err) - } - return &sdcpb.TypedValue{ - Value: &sdcpb.TypedValue_BoolVal{ - BoolVal: b, - }, - }, nil - - // Optional: Add ascii_val, float_val, json_val, identityref etc. as needed - - default: - return nil, fmt.Errorf("unsupported value type: %s", key) - } - -} -*/ func (r *Context) TransactionSet(ctx context.Context, req *sdcpb.TransactionSetRequest) (string, error) { rsp, err := r.dsclient.TransactionSet(ctx, req) - msg, err := r.processTransactionResponse(ctx, rsp, err) + msg, err := processTransactionResponse(ctx, rsp, err) if err != nil { return msg, err } @@ -399,107 +278,9 @@ func (r *Context) TransactionSet(ctx context.Context, req *sdcpb.TransactionSetR return msg, nil } -func (r *Context) TransactionConfirm(ctx context.Context, datastoreName, transactionID string) error { - _, err := r.dsclient.TransactionConfirm(ctx, &sdcpb.TransactionConfirmRequest{ - DatastoreName: datastoreName, - TransactionId: transactionID, - }) - return err -} - -func (r *Context) SetIntent(ctx context.Context, key storebackend.Key, config *config.Config, deviation config.Deviation, dryRun bool) (string, error) { - log := log.FromContext(ctx).With("target", key.String(), "intent", GetGVKNSN(config)) - if !r.IsReady() { - return "", fmt.Errorf("target context not ready") - } - intent_summary := map[string]bool{} - intents := []*sdcpb.TransactionIntent{} - update, err := r.getIntentUpdate(ctx, key, config, true) - if err != nil { - return "", err - } - intent_summary[GetGVKNSN(config)] = true - intents = append(intents, &sdcpb.TransactionIntent{ - Intent: GetGVKNSN(config), - Priority: int32(config.Spec.Priority), - Update: update, - }) - - if !config.IsRevertive() { - updates, deletes, err := r.getDeviationUpdate(ctx, key, &deviation) - if err != nil { - return "", err - } - newPriority := config.Spec.Priority - if newPriority > 0 { - newPriority-- - } - log.Debug("SetIntent", "priority", newPriority, "deviation update", update) - - if len(deviation.Spec.Deviations) == 0 { - // delete - intent_summary[fmt.Sprintf("deviation:%s", GetGVKNSN(config))] = true - intents = append(intents, &sdcpb.TransactionIntent{ - Deviation: true, - Intent: fmt.Sprintf("deviation:%s", GetGVKNSN(config)), - Priority: int32(newPriority), - //Update: update, - Delete: true, - DeleteIgnoreNoExist: true, - }) - } else { - // update - intent_summary[fmt.Sprintf("deviation:%s", GetGVKNSN(config))] = false - intents = append(intents, &sdcpb.TransactionIntent{ - Deviation: true, - Intent: fmt.Sprintf("deviation:%s", GetGVKNSN(config)), - Priority: int32(newPriority), - Update: updates, - Deletes: deletes, - }) - } - } - - log.Info("TransactionSet", "intent summary", intent_summary) - - return r.TransactionSet(ctx, &sdcpb.TransactionSetRequest{ - TransactionId: GetGVKNSN(config), - DatastoreName: key.String(), - DryRun: dryRun, - Timeout: ptr.To(int32(60)), - Intents: intents, - }) -} - -func (r *Context) DeleteIntent(ctx context.Context, key storebackend.Key, config *config.Config, dryRun bool) (string, error) { - log := log.FromContext(ctx).With("target", key.String(), "intent", GetGVKNSN(config)) - if !r.IsReady() { - return "", fmt.Errorf("target context not ready") - } - - if config.Status.AppliedConfig == nil { - log.Debug("delete intent was never applied") - return "", nil - } - - return r.TransactionSet(ctx, &sdcpb.TransactionSetRequest{ - TransactionId: GetGVKNSN(config), - DatastoreName: key.String(), - DryRun: dryRun, - Timeout: ptr.To(int32(60)), - Intents: []*sdcpb.TransactionIntent{ - { - Intent: GetGVKNSN(config), - Priority: int32(config.Spec.Priority), - Delete: true, - DeleteIgnoreNoExist: true, - Orphan: config.Orphan(), - }, - }, - }) -} - -func (r *Context) RecoverIntents(ctx context.Context, key storebackend.Key, configs []*config.Config, deviations []*config.Deviation) (string, error) { +func (r *Context) RecoverIntents( + ctx context.Context, + key storebackend.Key, configs []*config.Config, deviations []*config.Deviation) (string, error) { log := log.FromContext(ctx).With("target", key.String()) if !r.IsReady() { return "", fmt.Errorf("target context not ready") @@ -512,31 +293,16 @@ func (r *Context) RecoverIntents(ctx context.Context, key storebackend.Key, conf intents := make([]*sdcpb.TransactionIntent, 0, len(configs)+len(deviations)) // we only include deviations that have for _, deviation := range deviations { - updates, deletes, err := r.getDeviationUpdate(ctx, key, deviation) + intent, err := buildDeviationIntent(ctx, log, key, deviation) if err != nil { return "", err } - - newPriority, err := strconv.Atoi(deviation.GetLabels()["priority"]) - if err != nil { - return "", fmt.Errorf("cannot convert priroity to int %s", err) - } - if newPriority > 0 { - newPriority-- - } - // only include items for which deviations exist - if len(updates) != 0 || len(deletes) != 0 { - intents = append(intents, &sdcpb.TransactionIntent{ - Deviation: true, - Intent: fmt.Sprintf("deviation:%s", GetGVKNSN(deviation)), - Priority: int32(newPriority), - Update: updates, - Deletes: deletes, - }) + if intent != nil { + intents = append(intents, intent) } } for _, config := range configs { - update, err := r.getIntentUpdate(ctx, key, config, false) + update, err := GetIntentUpdate(ctx, key, config, false) if err != nil { return "", err } @@ -580,30 +346,13 @@ func (r *Context) SetIntents( intents := make([]*sdcpb.TransactionIntent, 0) for key, deviation := range deviationsToUpdate { deviationsToUpdateSet.Insert(key) - - updates, deletes, err := r.getDeviationUpdate(ctx, targetKey, deviation) + intent, err := buildDeviationIntent(ctx, log, targetKey, deviation) if err != nil { log.Error("Transaction getDeviationUpdate deviation", "error", err.Error()) return nil, err } - - newPriority, err := strconv.Atoi(deviation.GetLabels()["priority"]) - if err != nil { - log.Error("Transaction cannot convert priority to int", "error", err.Error()) - return nil, fmt.Errorf("cannot convert priority to int %s", err) - } - if newPriority > 0 { - newPriority-- - } - // only include items for which deviations exist - if len(updates) != 0 || len(deletes) != 0 { - intents = append(intents, &sdcpb.TransactionIntent{ - Deviation: true, - Intent: fmt.Sprintf("deviation:%s", GetGVKNSN(deviation)), - Priority: int32(newPriority), - Update: updates, - Deletes: deletes, - }) + if intent != nil { + intents = append(intents, intent) } } @@ -619,7 +368,7 @@ func (r *Context) SetIntents( } for key, config := range configsToUpdate { configsToUpdateSet.Insert(key) - update, err := r.getIntentUpdate(ctx, targetKey, config, true) + update, err := GetIntentUpdate(ctx, targetKey, config, true) if err != nil { log.Error("Transaction getIntentUpdate config", "error", err) return nil, err @@ -644,7 +393,7 @@ func (r *Context) SetIntents( log.Info("Transaction", "configsToUpdate total", len(configsToUpdate), "configsToUpdate names", configsToUpdateSet.UnsortedList(), - "configsToDelete totall", len(configsToDelete), + "configsToDelete total", len(configsToDelete), "configsToDelete names", configsToDeleteSet.UnsortedList(), "deviationsToUpdate total", len(deviationsToUpdate), "deviationsToUpdate names", deviationsToUpdateSet.UnsortedList(), @@ -665,15 +414,57 @@ func (r *Context) SetIntents( return rsp, err } +func buildDeviationIntent( + ctx context.Context, + log *slog.Logger, + key storebackend.Key, + deviation *config.Deviation, +) (*sdcpb.TransactionIntent, error) { + updates, deletes, err := getDeviationUpdate(ctx, key, deviation) + if err != nil { + return nil, err + } + + priorityStr := deviation.GetLabels()["priority"] + if priorityStr == "" { + return nil, fmt.Errorf("deviation %s has no priority label", GetGVKNSN(deviation)) + } + + newPriority, err := strconv.Atoi(priorityStr) + if err != nil { + return nil, fmt.Errorf("cannot convert priority to int: %w", err) + } + if newPriority > 0 { + newPriority-- + } + + // Only include items for which deviations exist + if len(updates) == 0 && len(deletes) == 0 { + return nil, nil + } + + return &sdcpb.TransactionIntent{ + Deviation: true, + Intent: fmt.Sprintf("deviation:%s", GetGVKNSN(deviation)), + Priority: int32(newPriority), + Update: updates, + Deletes: deletes, + }, nil +} + func (r *Context) Confirm(ctx context.Context, key storebackend.Key, transactionID string) error { log := log.FromContext(ctx).With("target", key.String(), "transactionID", transactionID) if !r.IsReady() { return fmt.Errorf("target context not ready") } log.Debug("cancel transaction") + return r.TransactionConfirm(ctx, key.String(), transactionID) +} + +func (r *Context) TransactionConfirm(ctx context.Context, datastoreName, transactionID string) error { _, err := r.dsclient.TransactionConfirm(ctx, &sdcpb.TransactionConfirmRequest{ + DatastoreName: datastoreName, TransactionId: transactionID, - DatastoreName: key.String(), }) return err } @@ -691,114 +482,6 @@ func (r *Context) Cancel(ctx context.Context, key storebackend.Key, transactionI return err } -// processTransactionResponse returns the warnings as a string and aggregates the errors in a single error and classifies them -// as recoverable or non recoverable. -func (r *Context) processTransactionResponse(ctx context.Context, rsp *sdcpb.TransactionSetResponse, rsperr error) (string, error) { - log := log.FromContext(ctx) - var errs error - var collectedWarnings []string - var recoverable bool - if rsperr != nil { - errs = errors.Join(errs, fmt.Errorf("error: %s", rsperr.Error())) - if er, ok := status.FromError(rsperr); ok { - switch er.Code() { - // Aborted is the refering to a lock in the dataserver - case codes.Aborted, codes.ResourceExhausted: - recoverable = true - default: - recoverable = false - } - } - } - if rsp != nil { - for _, warning := range rsp.Warnings { - collectedWarnings = append(collectedWarnings, fmt.Sprintf("global warning: %q", warning)) - } - for key, intent := range rsp.Intents { - for _, intentError := range intent.Errors { - errs = errors.Join(errs, fmt.Errorf("intent %q error: %q", key, intentError)) - } - for _, intentWarning := range intent.Warnings { - collectedWarnings = append(collectedWarnings, fmt.Sprintf("intent %q warning: %q", key, intentWarning)) - } - } - } - var err error - var msg string - if errs != nil { - err = NewTransactionError(errs, recoverable) - } - if len(collectedWarnings) > 0 { - msg = strings.Join(collectedWarnings, "; ") - } - log.Debug("transaction response", "rsp", prototext.Format(rsp), "msg", msg, "error", err) - return msg, err -} - -func (r *Context) GetData(ctx context.Context, key storebackend.Key) (*config.RunningConfig, error) { - log := log.FromContext(ctx).With("target", key.String()) - if !r.IsReady() { - return nil, fmt.Errorf("target context not ready") - } - - rsp, err := r.dsclient.GetIntent(ctx, &sdcpb.GetIntentRequest{ - DatastoreName: key.String(), - Intent: "running", - Format: sdcpb.Format_Intent_Format_JSON, - }) - if err != nil { - log.Error("get data failed", "error", err.Error()) - return nil, err - } - - return config.BuildRunningConfig( - metav1.ObjectMeta{ - Name: key.Name, - Namespace: key.Namespace, - }, - config.RunningConfigSpec{}, - config.RunningConfigStatus{ - Value: runtime.RawExtension{ - Raw: rsp.GetBlob(), - }, - }, - ), nil -} - -func (r *Context) GetBlameConfig(ctx context.Context, key storebackend.Key) (*config.ConfigBlame, error) { - log := log.FromContext(ctx).With("target", key.String()) - if !r.IsReady() { - return nil, fmt.Errorf("target context not ready") - } - - rsp, err := r.dsclient.BlameConfig(ctx, &sdcpb.BlameConfigRequest{ - DatastoreName: key.String(), - IncludeDefaults: true, - }) - if err != nil { - log.Error("get blame config failed", "error", err.Error()) - return nil, err - } - - json, err := protojson.Marshal(rsp.ConfigTree) - if err != nil { - return nil, fmt.Errorf("invalid json %s", err.Error()) - } - - return config.BuildConfigBlame( - metav1.ObjectMeta{ - Name: key.Name, - Namespace: key.Namespace, - }, - config.ConfigBlameSpec{}, - config.ConfigBlameStatus{ - Value: runtime.RawExtension{ - Raw: json, - }, - }, - ), nil -} - func (r *Context) DeleteSubscription(ctx context.Context, sub *invv1alpha1.Subscription) error { log := log.FromContext(ctx).With("targetKey", r.targetKey.String()) if err := r.subscriptions.DelSubscription(sub); err != nil { @@ -853,14 +536,15 @@ func (r *Context) GetCache() cache.Cache { } func (r *Context) GetPrombLabels() []prompb.Label { - labels := make([]prompb.Label, 0) - labels = append(labels, prompb.Label{Name: "target_name", Value: r.targetKey.String()}) - - req := r.getDatastoreReq() - if req != nil { - labels = append(labels, prompb.Label{Name: "vendor", Value: req.Schema.Vendor}) - labels = append(labels, prompb.Label{Name: "version", Value: req.Schema.Version}) - labels = append(labels, prompb.Label{Name: "address", Value: req.Target.Address}) - } - return labels + labels := make([]prompb.Label, 0, 4) + labels = append(labels, prompb.Label{Name: "target_name", Value: r.targetKey.String()}) + + if req := r.getDatastoreReq(); req != nil && req.Target != nil { + labels = append(labels, + prompb.Label{Name: "vendor", Value: req.Schema.Vendor}, + prompb.Label{Name: "version", Value: req.Schema.Version}, + prompb.Label{Name: "address", Value: req.Target.Address}, + ) + } + return labels } diff --git a/pkg/target/context_test.go b/pkg/sdc/target/context_test.go similarity index 97% rename from pkg/target/context_test.go rename to pkg/sdc/target/context_test.go index 12ba38d4..34e9d9dd 100644 --- a/pkg/target/context_test.go +++ b/pkg/sdc/target/context_test.go @@ -97,11 +97,10 @@ func TestProcessTransactionResponse(t *testing.T) { } ctx := context.Background() - mockCtx := &Context{} //key := storebackend.ToKey("dummy") for name, tc := range cases { t.Run(name, func(t *testing.T) { - warning, err := mockCtx.processTransactionResponse(ctx, tc.rsp, tc.err) + warning, err := processTransactionResponse(ctx, tc.rsp, tc.err) if tc.expectErr { assert.Error(t, err, "expected an error but got none") diff --git a/pkg/target/deviation_watcher.go b/pkg/sdc/target/deviation_watcher.go similarity index 97% rename from pkg/target/deviation_watcher.go rename to pkg/sdc/target/deviation_watcher.go index 931daa3e..b66659dc 100644 --- a/pkg/target/deviation_watcher.go +++ b/pkg/sdc/target/deviation_watcher.go @@ -34,9 +34,6 @@ import ( "google.golang.org/protobuf/encoding/prototext" "k8s.io/apimachinery/pkg/types" "k8s.io/utils/ptr" - - //"k8s.io/utils/ptr" - "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -130,11 +127,14 @@ func (r *DeviationWatcher) start(ctx context.Context) { switch resp.Event { case sdcpb.DeviationEvent_START: if started { + log.Error("protocol violation start on start") + /* if err := stream.CloseSend(); err != nil { log.Error("stream failed", "err", err) } // to check if this works on the client side to inform the server to stop sending stream = nil time.Sleep(time.Second * 1) //- resilience for server crash + */ continue } deviations = make(map[string][]*sdcpb.WatchDeviationResponse, 0) @@ -144,11 +144,14 @@ func (r *DeviationWatcher) start(ctx context.Context) { started = true case sdcpb.DeviationEvent_UPDATE: if !started { + log.Error("protocol violation update without start") + /* if err := stream.CloseSend(); err != nil { log.Error("stream failed", "err", err) } // to check if this works on the client side to inform the server to stop sending stream = nil time.Sleep(time.Second * 1) //- resilience for server crash + */ continue } intent := resp.GetIntent() @@ -168,11 +171,15 @@ func (r *DeviationWatcher) start(ctx context.Context) { deviations[intent] = append(deviations[intent], resp) case sdcpb.DeviationEvent_END: if !started { + log.Error("protocol violation end without start") + + /* if err := stream.CloseSend(); err != nil { log.Error("stream failed", "err", err) } // to check if this works on the client side to inform the server to stop sending stream = nil time.Sleep(time.Second * 1) //- resilience for server crash + */ continue } started = false diff --git a/pkg/sdc/target/dryrun.go b/pkg/sdc/target/dryrun.go new file mode 100644 index 00000000..e7d6c7f8 --- /dev/null +++ b/pkg/sdc/target/dryrun.go @@ -0,0 +1,68 @@ +package target + +import ( + "context" + "fmt" + + "github.com/sdcio/config-server/apis/condition" + "github.com/sdcio/config-server/apis/config" + invv1alpha1 "github.com/sdcio/config-server/apis/inv/v1alpha1" + dsclient "github.com/sdcio/config-server/pkg/sdc/dataserver/client" + sdcpb "github.com/sdcio/sdc-protos/sdcpb" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" +) + +// runDryRunTransaction opens a short-lived data-server client, executes a +// TransactionSet dry-run with the provided intents, processes the result, +// and updates the Config status/conditions. +func RunDryRunTransaction( + ctx context.Context, + key types.NamespacedName, + c *config.Config, + target *invv1alpha1.Target, + intents []*sdcpb.TransactionIntent, + dryrun bool, +) (runtime.Object, error) { + cfg := &dsclient.Config{ + Address: dsclient.GetDataServerAddress(), + Insecure: true, + } + + dsClient, closeFn, err := dsclient.NewEphemeral(ctx, cfg) + if err != nil { + return nil, err + } + defer func() { + if err := closeFn(); err != nil { + // You can use your preferred logging framework here + fmt.Printf("failed to close connection: %v\n", err) + } + }() + + req := &sdcpb.TransactionSetRequest{ + TransactionId: GetGVKNSN(c), + DatastoreName: key.String(), + DryRun: dryrun, + Timeout: ptr.To(int32(60)), + Intents: intents, + } + + rsp, rpcErr := dsClient.TransactionSet(ctx, req) + + warnings, aggErr := processTransactionResponse(ctx, rsp, rpcErr) + if aggErr != nil { + msg := fmt.Sprintf("%s err %s", warnings, aggErr.Error()) + c.SetConditions(condition.Failed(msg)) + return c, aggErr + } + + c.SetConditions(condition.ReadyWithMsg(warnings)) + c.Status.LastKnownGoodSchema = &config.ConfigStatusLastKnownGoodSchema{ + Vendor: target.Status.DiscoveryInfo.Provider, + Version: target.Status.DiscoveryInfo.Version, + } + c.Status.AppliedConfig = &c.Spec + return c, nil +} diff --git a/pkg/target/error.go b/pkg/sdc/target/error.go similarity index 100% rename from pkg/target/error.go rename to pkg/sdc/target/error.go diff --git a/pkg/target/subscription.go b/pkg/sdc/target/subscription.go similarity index 100% rename from pkg/target/subscription.go rename to pkg/sdc/target/subscription.go diff --git a/pkg/target/subscription_test.go b/pkg/sdc/target/subscription_test.go similarity index 100% rename from pkg/target/subscription_test.go rename to pkg/sdc/target/subscription_test.go diff --git a/pkg/transactor/transactor.go b/pkg/sdc/target/transactor.go similarity index 75% rename from pkg/transactor/transactor.go rename to pkg/sdc/target/transactor.go index e44d8b35..16acc1d4 100644 --- a/pkg/transactor/transactor.go +++ b/pkg/sdc/target/transactor.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package transactor +package target import ( "context" @@ -23,6 +23,7 @@ import ( "fmt" "strconv" "strings" + "log/slog" "github.com/google/uuid" "github.com/henderiw/apiserver-store/pkg/storebackend" @@ -32,14 +33,12 @@ import ( configv1alpha1 "github.com/sdcio/config-server/apis/config/v1alpha1" invv1alpha1 "github.com/sdcio/config-server/apis/inv/v1alpha1" "github.com/sdcio/config-server/pkg/reconcilers/resource" - "github.com/sdcio/config-server/pkg/target" sdcpb "github.com/sdcio/sdc-protos/sdcpb" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "google.golang.org/protobuf/encoding/prototext" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/sets" genericapirequest "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" @@ -56,7 +55,7 @@ type Transactor struct { fieldManagerFinalizer string } -func New(client client.Client, fieldManager, fieldManagerFinalizer string) *Transactor { +func NewTransactor(client client.Client, fieldManager, fieldManagerFinalizer string) *Transactor { return &Transactor{ client: client, fieldManager: fieldManager, @@ -64,7 +63,7 @@ func New(client client.Client, fieldManager, fieldManagerFinalizer string) *Tran } } -func (r *Transactor) RecoverConfigs(ctx context.Context, target *invv1alpha1.Target, tctx *target.Context) (*string, error) { +func (r *Transactor) RecoverConfigs(ctx context.Context, target *invv1alpha1.Target, tctx *Context) (*string, error) { log := log.FromContext(ctx) log.Info("RecoverConfigs") configList, err := r.listConfigsPerTarget(ctx, target) @@ -103,9 +102,7 @@ func (r *Transactor) RecoverConfigs(ctx context.Context, target *invv1alpha1.Tar } } } - //sort.Slice(configs, func(i, j int) bool { - // return configs[i].CreationTimestamp.Before(&configs[j].CreationTimestamp) - //}) + if len(configs) == 0 && len(deviations) == 0 { tctx.SetRecoveredConfigsState(ctx) log.Info("recovered configs, nothing to recover", "count", len(configs), "deviations", len(deviations)) @@ -125,7 +122,7 @@ func (r *Transactor) RecoverConfigs(ctx context.Context, target *invv1alpha1.Tar return nil, nil } -func (r *Transactor) Transact(ctx context.Context, target *invv1alpha1.Target, tctx *target.Context) (bool, error) { +func (r *Transactor) Transact(ctx context.Context, target *invv1alpha1.Target, tctx *Context) (bool, error) { log := log.FromContext(ctx) log.Info("Transact") // get all configs for the target @@ -481,7 +478,12 @@ func getConfigsAndDeviationsToTransact( ctx context.Context, configList *config.ConfigList, deviationMap map[string]*config.Deviation, -) (map[string]*config.Config, map[string]*config.Config, map[string]*config.Deviation, map[string]*config.Deviation) { +) ( + map[string]*config.Config, + map[string]*config.Config, + map[string]*config.Deviation, + map[string]*config.Deviation, +) { log := log.FromContext(ctx) configsToUpdate := make(map[string]*config.Config) @@ -490,182 +492,178 @@ func getConfigsAndDeviationsToTransact( deviationsToUpdate := make(map[string]*config.Deviation) deviationsToDelete := make(map[string]*config.Deviation) - configsToUpdateSet := sets.New[string]() - configsToDeleteSet := sets.New[string]() - nonRecoverableSet := sets.New[string]() - deviationsToUpdateSet := sets.New[string]() - deviationsToDeleteSet := sets.New[string]() - - // collect configs to apply + // Classify configs: update / delete / non-recoverable / noop for i := range configList.Items { - config := &configList.Items[i] - key := GetGVKNSN(config) + cfg := &configList.Items[i] + key := GetGVKNSN(cfg) switch { - case !config.IsRecoverable(): - nonRecoverable[key] = config - nonRecoverableSet.Insert(key) - continue + case !cfg.IsRecoverable(ctx): + nonRecoverable[key] = cfg - case config.GetDeletionTimestamp() != nil: - configsToDelete[key] = config - configsToDeleteSet.Insert(key) + case cfg.GetDeletionTimestamp() != nil: + configsToDelete[key] = cfg - case config.Status.AppliedConfig != nil && - config.Spec.GetShaSum(ctx) == config.Status.AppliedConfig.GetShaSum(ctx): + case cfg.Status.AppliedConfig != nil && + cfg.Spec.GetShaSum(ctx) == cfg.Status.AppliedConfig.GetShaSum(ctx): + // no change, skip continue default: - configsToUpdate[key] = config - configsToUpdateSet.Insert(key) + configsToUpdate[key] = cfg } } - // Collect deviations to apply - // handle deviations + // Initial deviation classification per config for i := range configList.Items { cfg := &configList.Items[i] key := GetGVKNSN(cfg) - deviation, ok := deviationMap[key] - if !ok { - log.Warn("deviation missing for config", "config", key) - deviation = config.BuildDeviation(metav1.ObjectMeta{Name: cfg.GetName(), Namespace: cfg.GetNamespace()}, nil, nil) - } + deviation := ensureDeviationForConfig(log, cfg, deviationMap[key]) switch { - case !cfg.IsRecoverable(): + case !cfg.IsRecoverable(ctx): continue case cfg.GetDeletionTimestamp() != nil: if !cfg.IsRevertive() { - labels := safeCopyLabels(deviation.GetLabels()) - labels["orphan"] = strconv.FormatBool(cfg.Orphan()) - deviation.SetLabels(labels) + labelOrphan(deviation, cfg.Orphan()) deviationsToDelete[key] = deviation - deviationsToDeleteSet.Insert(key) } continue + default: if cfg.IsRevertive() { if deviation.HasNotAppliedDeviation() { - log.Info("config included due to non revertive deviations", "key", key, "revertive", true) + log.Info("config included due to non revertive deviations", + "key", key, "revertive", true, + ) configsToUpdate[key] = cfg - configsToUpdateSet.Insert(key) } - } else { - // check for change of deviation - if cfg.HashDeviationGenerationChanged(*deviation) { - if deviation.HasNotAppliedDeviation() { - //change - // safe copy of labels - labels := safeCopyLabels(deviation.GetLabels()) - labels["priority"] = strconv.Itoa(int(cfg.Spec.Priority)) - deviation.SetLabels(labels) - deviationsToUpdate[key] = deviation - deviationsToUpdateSet.Insert(key) - } else { - labels := safeCopyLabels(deviation.GetLabels()) - labels["orphan"] = strconv.FormatBool(cfg.Orphan()) - deviation.SetLabels(labels) - deviationsToDelete[key] = deviation - deviationsToDeleteSet.Insert(key) - } - - } - if len(deviation.Spec.Deviations) == 0 { - labels := safeCopyLabels(deviation.GetLabels()) - labels["orphan"] = strconv.FormatBool(cfg.Orphan()) + continue + } + + // Non-revertive: check deviation generation changes & applied state + if cfg.HashDeviationGenerationChanged(*deviation) { + if deviation.HasNotAppliedDeviation() { + labelPriority(deviation, int(cfg.Spec.Priority)) + deviationsToUpdate[key] = deviation + } else { + labelOrphan(deviation, cfg.Orphan()) deviationsToDelete[key] = deviation - deviationsToDeleteSet.Insert(key) - } + } } - } + + if len(deviation.Spec.Deviations) == 0 { + labelOrphan(deviation, cfg.Orphan()) + deviationsToDelete[key] = deviation + } + } } - // for every config we create and delete we will include the deviations for non revertive configs. + // For every config we create/update, ensure deviation behavior for non-revertive for key, cfg := range configsToUpdate { - deviation, ok := deviationMap[key] - if !ok { - log.Warn("deviation missing for config", "config", key) - deviation = config.BuildDeviation(metav1.ObjectMeta{Name: cfg.GetName(), Namespace: cfg.GetNamespace()}, nil, nil) + deviation := ensureDeviationForConfig(log, cfg, deviationMap[key]) + + if cfg.IsRevertive() { + continue } - if !cfg.IsRevertive() { - if len(deviation.Spec.Deviations) != 0 { - if deviation.HasNotAppliedDeviation() { - //change - // safe copy of labels - labels := safeCopyLabels(deviation.GetLabels()) - labels["priority"] = strconv.Itoa(int(cfg.Spec.Priority)) - deviation.SetLabels(labels) - deviationsToUpdate[key] = deviation - deviationsToUpdateSet.Insert(key) - } else { - labels := safeCopyLabels(deviation.GetLabels()) - labels["orphan"] = strconv.FormatBool(cfg.Orphan()) - deviation.SetLabels(labels) - deviationsToDelete[key] = deviation - deviationsToDeleteSet.Insert(key) - } + if len(deviation.Spec.Deviations) != 0 { + if deviation.HasNotAppliedDeviation() { + labelPriority(deviation, int(cfg.Spec.Priority)) + deviationsToUpdate[key] = deviation } else { - labels := safeCopyLabels(deviation.GetLabels()) - labels["orphan"] = strconv.FormatBool(cfg.Orphan()) - deviation.SetLabels(labels) + labelOrphan(deviation, cfg.Orphan()) deviationsToDelete[key] = deviation - deviationsToDeleteSet.Insert(key) } + } else { + labelOrphan(deviation, cfg.Orphan()) + deviationsToDelete[key] = deviation } } + // For every config we delete, add deviations-to-delete for non-revertive for key, cfg := range configsToDelete { - deviation, ok := deviationMap[key] - if !ok { - log.Warn("deviation missing for config", "config", key) - deviation = config.BuildDeviation(metav1.ObjectMeta{Name: cfg.GetName(), Namespace: cfg.GetNamespace()}, nil, nil) - } + deviation := ensureDeviationForConfig(log, cfg, deviationMap[key]) if !cfg.IsRevertive() { - labels := safeCopyLabels(deviation.GetLabels()) - labels["orphan"] = strconv.FormatBool(cfg.Orphan()) - deviation.SetLabels(labels) + labelOrphan(deviation, cfg.Orphan()) deviationsToDelete[key] = deviation - deviationsToDeleteSet.Insert(key) } } log.Info("getConfigsAndDeviationsToTransact classification start", - "configsToUpdate", configsToUpdateSet.UnsortedList(), - "configsToDelete", configsToDeleteSet.UnsortedList(), - "nonRecoverable", nonRecoverableSet.UnsortedList(), - "deviationsToUpdate", deviationsToUpdateSet.UnsortedList(), - "deviationsToDelete", deviationsToDeleteSet.UnsortedList(), + "configsToUpdate", mapKeys(configsToUpdate), + "configsToDelete", mapKeys(configsToDelete), + "nonRecoverable", mapKeys(nonRecoverable), + "deviationsToUpdate", mapKeys(deviationsToUpdate), + "deviationsToDelete", mapKeys(deviationsToDelete), ) - // If we have changes, retry non-recoverables + // --- 5) If we have changes, retry non-recoverables + if len(configsToUpdate) > 0 || len(configsToDelete) > 0 { for key, cfg := range nonRecoverable { if cfg.GetDeletionTimestamp() != nil { configsToDelete[key] = cfg - configsToDeleteSet.Insert(key) } else { configsToUpdate[key] = cfg - configsToUpdateSet.Insert(key) } } } log.Info("getConfigsAndDeviationsToTransact classification after change", - "configsToUpdate", configsToUpdateSet.UnsortedList(), - "configsToDelete", configsToDeleteSet.UnsortedList(), - "nonRecoverable", nonRecoverableSet.UnsortedList(), - "deviationsToUpdate", deviationsToUpdateSet.UnsortedList(), - "deviationsToDelete", deviationsToDeleteSet.UnsortedList(), + "configsToUpdate", mapKeys(configsToUpdate), + "configsToDelete", mapKeys(configsToDelete), + "nonRecoverable", mapKeys(nonRecoverable), + "deviationsToUpdate", mapKeys(deviationsToUpdate), + "deviationsToDelete", mapKeys(deviationsToDelete), ) - + return configsToUpdate, configsToDelete, deviationsToUpdate, deviationsToDelete } + +func ensureDeviationForConfig( + log *slog.Logger, + cfg *config.Config, + dev *config.Deviation, +) *config.Deviation { + if dev != nil { + return dev + } + log.Warn("deviation missing for config", "config", GetGVKNSN(cfg)) + return config.BuildDeviation( + metav1.ObjectMeta{ + Name: cfg.GetName(), + Namespace: cfg.GetNamespace(), + }, + nil, + nil, + ) +} + +func labelOrphan(dev *config.Deviation, orphan bool) { + labels := safeCopyLabels(dev.GetLabels()) + labels["orphan"] = strconv.FormatBool(orphan) + dev.SetLabels(labels) +} + +func labelPriority(dev *config.Deviation, priority int) { + labels := safeCopyLabels(dev.GetLabels()) + labels["priority"] = strconv.Itoa(priority) + dev.SetLabels(labels) +} + +func mapKeys[T any](m map[string]T) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + return out +} + func (r *Transactor) handleTransactionErrors( ctx context.Context, rsp *sdcpb.TransactionSetResponse, @@ -676,21 +674,24 @@ func (r *Transactor) handleTransactionErrors( log := log.FromContext(ctx) log.Info("handling transaction errors", "recoverable", recoverable) + // If no response at all → apply same error to all configs. if rsp == nil { - for _, configOrig := range configsToTransact { - if err := r.processFailedConfig(ctx, configOrig, "", globalErr, recoverable); err != nil { + for _, cfg := range configsToTransact { + if err := r.processFailedConfig(ctx, cfg, "", globalErr, recoverable); err != nil { return true, err } } - for _, configOrig := range deletedConfigsToTransact { - if err := r.processFailedConfig(ctx, configOrig, "", globalErr, false); err != nil { + for _, cfg := range deletedConfigsToTransact { + if err := r.processFailedConfig(ctx, cfg, "", globalErr, false); err != nil { return true, err } } return recoverable, globalErr } + // Response present: handle per-intent dataServerError := false + for intentName, intent := range rsp.Intents { log.Info("intent failed", "name", intentName, "errors", intent.Errors) @@ -705,35 +706,38 @@ func (r *Transactor) handleTransactionErrors( msg = strings.Join(warnings, "; ") } - if configOrig, ok := configsToTransact[intentName]; ok { - if err := r.processFailedConfig(ctx, configOrig, msg, errs, false); err != nil { + if cfg, ok := configsToTransact[intentName]; ok { + if err := r.processFailedConfig(ctx, cfg, msg, errs, false); err != nil { return true, err } continue - } else if configOrig, ok := deletedConfigsToTransact[intentName]; ok { - if err := r.processFailedConfig(ctx, configOrig, msg, errs, false); err != nil { + } + if cfg, ok := deletedConfigsToTransact[intentName]; ok { + if err := r.processFailedConfig(ctx, cfg, msg, errs, false); err != nil { return true, err } continue } - dataServerError =true + + // Dataserver reported an intent we don't know → treat as global error + dataServerError = true recoverable = false globalErr = errors.Join( - errs, - fmt.Errorf("dataserver reported an error in an intent %s that does not exists", intentName), + errs, + fmt.Errorf("dataserver reported an error in an intent %s that does not exist", intentName), ) - break } + if dataServerError { log.Error("transact dataserver error", "err", globalErr) - for _, configOrig := range configsToTransact { - if err := r.processFailedConfig(ctx, configOrig, "", globalErr, recoverable); err != nil { + for _, cfg := range configsToTransact { + if err := r.processFailedConfig(ctx, cfg, "", globalErr, recoverable); err != nil { return true, err } } - for _, configOrig := range deletedConfigsToTransact { - if err := r.processFailedConfig(ctx, configOrig, "", globalErr, false); err != nil { + for _, cfg := range deletedConfigsToTransact { + if err := r.processFailedConfig(ctx, cfg, "", globalErr, false); err != nil { return true, err } } @@ -759,9 +763,9 @@ func (r *Transactor) processFailedConfig( return r.updateConfigWithError(ctx, config, msg, origErr, recoverable) } -func collectWarnings(errors []string) []string { - warnings := make([]string, 0, len(errors)) - for _, err := range errors { +func collectWarnings(errorsOrMsgs []string) []string { + warnings := make([]string, 0, len(errorsOrMsgs)) + for _, err := range errorsOrMsgs { warnings = append(warnings, fmt.Sprintf("warning: %q", err)) } return warnings @@ -804,3 +808,5 @@ func analyzeIntentResponse(err error, rsp *sdcpb.TransactionSetResponse) Transac return result } + + diff --git a/pkg/sdc/target/utils.go b/pkg/sdc/target/utils.go new file mode 100644 index 00000000..391512ab --- /dev/null +++ b/pkg/sdc/target/utils.go @@ -0,0 +1,123 @@ +package target + +import ( + "context" + "fmt" + "errors" + "strings" + + "github.com/henderiw/apiserver-store/pkg/storebackend" + "github.com/henderiw/logger/log" + "github.com/sdcio/config-server/apis/config" + sdcpb "github.com/sdcio/sdc-protos/sdcpb" + "sigs.k8s.io/controller-runtime/pkg/client" + "google.golang.org/protobuf/encoding/prototext" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func GetGVKNSN(obj client.Object) string { + return fmt.Sprintf("%s.%s", obj.GetNamespace(), obj.GetName()) +} + +// useSpec indicates to use the spec as the confifSpec, typically set to true; when set to false it means we are recovering +// the config +func GetIntentUpdate(ctx context.Context, key storebackend.Key, config *config.Config, useSpec bool) ([]*sdcpb.Update, error) { + logger := log.FromContext(ctx) + update := make([]*sdcpb.Update, 0, len(config.Spec.Config)) + configSpec := config.Spec.Config + if !useSpec && config.Status.AppliedConfig != nil { + update = make([]*sdcpb.Update, 0, len(config.Status.AppliedConfig.Config)) + configSpec = config.Status.AppliedConfig.Config + } + + for _, config := range configSpec { + path, err := sdcpb.ParsePath(config.Path) + if err != nil { + return nil, fmt.Errorf("create data failed for target %s, path %s invalid", key.String(), config.Path) + } + logger.Debug("setIntent", "configSpec", string(config.Value.Raw)) + update = append(update, &sdcpb.Update{ + Path: path, + Value: &sdcpb.TypedValue{ + Value: &sdcpb.TypedValue_JsonVal{ + JsonVal: config.Value.Raw, + }, + }, + }) + } + return update, nil +} + +func getDeviationUpdate(ctx context.Context, targetKey storebackend.Key, deviation *config.Deviation) ([]*sdcpb.Update, []*sdcpb.Path, error) { + logger := log.FromContext(ctx) + updates := make([]*sdcpb.Update, 0) + deletes := make([]*sdcpb.Path, 0) + + for _, deviation := range deviation.Spec.Deviations { + if deviation.Reason == "NOT_APPLIED" { + path, err := sdcpb.ParsePath(deviation.Path) + if err != nil { + return nil, nil, fmt.Errorf("deviation path parsing failed forr target %s, path %s invalid", targetKey.String(), deviation.Path) + } + if deviation.CurrentValue == nil { + deletes = append(deletes, path) + } else { + val := &sdcpb.TypedValue{} + if err := prototext.Unmarshal([]byte(*deviation.CurrentValue), val); err != nil { + logger.Error("deviation proto unmarshal failed", "key", targetKey.String(), "path", deviation.Path, "currentValue", deviation.CurrentValue) + continue + } + updates = append(updates, &sdcpb.Update{ + Path: path, + Value: val, + }) + } + } + } + return updates, deletes, nil +} + +// processTransactionResponse returns the warnings as a string and aggregates the errors in a single error and classifies them +// as recoverable or non recoverable. +func processTransactionResponse(ctx context.Context, rsp *sdcpb.TransactionSetResponse, rsperr error) (string, error) { + log := log.FromContext(ctx) + var errs error + var collectedWarnings []string + var recoverable bool + if rsperr != nil { + errs = errors.Join(errs, fmt.Errorf("error: %s", rsperr.Error())) + if er, ok := status.FromError(rsperr); ok { + switch er.Code() { + // Aborted is the refering to a lock in the dataserver + case codes.Aborted, codes.ResourceExhausted: + recoverable = true + default: + recoverable = false + } + } + } + if rsp != nil { + for _, warning := range rsp.Warnings { + collectedWarnings = append(collectedWarnings, fmt.Sprintf("global warning: %q", warning)) + } + for key, intent := range rsp.Intents { + for _, intentError := range intent.Errors { + errs = errors.Join(errs, fmt.Errorf("intent %q error: %q", key, intentError)) + } + for _, intentWarning := range intent.Warnings { + collectedWarnings = append(collectedWarnings, fmt.Sprintf("intent %q warning: %q", key, intentWarning)) + } + } + } + var err error + var msg string + if errs != nil { + err = NewTransactionError(errs, recoverable) + } + if len(collectedWarnings) > 0 { + msg = strings.Join(collectedWarnings, "; ") + } + log.Debug("transaction response", "rsp", prototext.Format(rsp), "msg", msg, "error", err) + return msg, err +} \ No newline at end of file diff --git a/pkg/target/handler.go b/pkg/target/handler.go deleted file mode 100644 index baa0ba2c..00000000 --- a/pkg/target/handler.go +++ /dev/null @@ -1,152 +0,0 @@ -/* -Copyright 2024 Nokia. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package target - -import ( - "context" - errors "errors" - "fmt" - - "github.com/henderiw/apiserver-store/pkg/storebackend" - pkgerrors "github.com/pkg/errors" - "github.com/sdcio/config-server/apis/config" - invv1alpha1 "github.com/sdcio/config-server/apis/inv/v1alpha1" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -type TargetHandler interface { - GetTargetContext(ctx context.Context, targetKey types.NamespacedName) (*invv1alpha1.Target, *Context, error) - SetIntent(ctx context.Context, targetKey types.NamespacedName, config *config.Config, deviation config.Deviation, dryRun bool) (*config.ConfigStatusLastKnownGoodSchema, string, error) - DeleteIntent(ctx context.Context, targetKey types.NamespacedName, config *config.Config, dryRun bool) (string, error) - GetData(ctx context.Context, targetKey types.NamespacedName) (*config.RunningConfig, error) - RecoverIntents(ctx context.Context, targetKey types.NamespacedName, configs []*config.Config, deviations []*config.Deviation) (*config.ConfigStatusLastKnownGoodSchema, string, error) - SetIntents(ctx context.Context, targetKey types.NamespacedName, transactionID string, configs, deleteConfigs map[string]*config.Config, deviations, deleteDeviations map[string]*config.Deviation, dryRun bool) (*config.ConfigStatusLastKnownGoodSchema, string, error) - Confirm(ctx context.Context, targetKey types.NamespacedName, transactionID string) error - Cancel(ctx context.Context, targetKey types.NamespacedName, transactionID string) error - GetBlameConfig(ctx context.Context, targetKey types.NamespacedName) (*config.ConfigBlame, error) -} - -func NewTargetHandler(client client.Client, targetStore storebackend.Storer[*Context]) TargetHandler { - return &targetHandler{ - client: client, - targetStore: targetStore, - } -} - -type targetHandler struct { - client client.Client - targetStore storebackend.Storer[*Context] -} - -// GetTargetContext returns a invTarget and targetContext when the target is ready and the ctx is found -// Used by the config controller and should only act when the Config is ready -func (r *targetHandler) GetTargetContext(ctx context.Context, targetKey types.NamespacedName) (*invv1alpha1.Target, *Context, error) { - target := &invv1alpha1.Target{} - if err := r.client.Get(ctx, targetKey, target); err != nil { - return nil, nil, &LookupError{ - Message: fmt.Sprintf("target %s get failed to k8s apiserver ", targetKey.String()), - WrappedError: errors.Join(ErrLookup, err), - } - } - // A config snippet should only be applied if the Target is in ready state - if !target.IsReady() { - return nil, nil, &LookupError{ - Message: fmt.Sprintf("target %s not ready ", targetKey.String()), - WrappedError: pkgerrors.Wrap(ErrLookup, string(config.ConditionReasonTargetNotReady)), - } - } - tctx, err := r.targetStore.Get(ctx, storebackend.Key{NamespacedName: targetKey}) - if err != nil { - return nil, nil, &LookupError{ - Message: fmt.Sprintf("target %s not found in target datastore", targetKey.String()), - WrappedError: pkgerrors.Wrap(ErrLookup, string(config.ConditionReasonTargetNotFound)), - } - } - return target, tctx, nil -} - -func (r *targetHandler) SetIntent(ctx context.Context, targetKey types.NamespacedName, config *config.Config, deviation config.Deviation, dryRun bool) (*config.ConfigStatusLastKnownGoodSchema, string, error) { - _, tctx, err := r.GetTargetContext(ctx, targetKey) - if err != nil { - return nil, "", err - } - schema := tctx.GetSchema() - msg, err := tctx.SetIntent(ctx, storebackend.Key{NamespacedName: targetKey}, config, deviation, dryRun) - return schema, msg, err -} - -func (r *targetHandler) DeleteIntent(ctx context.Context, targetKey types.NamespacedName, config *config.Config, dryRun bool) (string, error) { - _, tctx, err := r.GetTargetContext(ctx, targetKey) - if err != nil { - return "", err - } - return tctx.DeleteIntent(ctx, storebackend.Key{NamespacedName: targetKey}, config, dryRun) -} - -func (r *targetHandler) GetData(ctx context.Context, targetKey types.NamespacedName) (*config.RunningConfig, error) { - _, tctx, err := r.GetTargetContext(ctx, targetKey) - if err != nil { - return nil, err - } - return tctx.GetData(ctx, storebackend.Key{NamespacedName: targetKey}) -} - - -func (r *targetHandler) GetBlameConfig(ctx context.Context, targetKey types.NamespacedName) (*config.ConfigBlame, error) { - _, tctx, err := r.GetTargetContext(ctx, targetKey) - if err != nil { - return nil, err - } - return tctx.GetBlameConfig(ctx, storebackend.Key{NamespacedName: targetKey}) -} - -func (r *targetHandler) RecoverIntents(ctx context.Context, targetKey types.NamespacedName, configs []*config.Config, deviations []*config.Deviation) (*config.ConfigStatusLastKnownGoodSchema, string, error) { - _, tctx, err := r.GetTargetContext(ctx, targetKey) - if err != nil { - return nil, "", err - } - schema := tctx.GetSchema() - msg, err := tctx.RecoverIntents(ctx, storebackend.Key{NamespacedName: targetKey}, configs, deviations) - return schema, msg, err -} - -func (r *targetHandler) SetIntents(ctx context.Context, targetKey types.NamespacedName, transactionID string, configsToUpdate, configsToDelete map[string]*config.Config, deviationsToUpdate, deviationsToDelete map[string]*config.Deviation, dryRun bool) (*config.ConfigStatusLastKnownGoodSchema, string, error) { - _, tctx, err := r.GetTargetContext(ctx, targetKey) - if err != nil { - return nil, "", err - } - schema := tctx.GetSchema() - _, err = tctx.SetIntents(ctx, storebackend.Key{NamespacedName: targetKey}, transactionID, configsToUpdate, configsToDelete, deviationsToUpdate, deviationsToDelete, dryRun) - return schema, "", err -} - -func (r *targetHandler) Confirm(ctx context.Context, targetKey types.NamespacedName, transactionID string) error { - _, tctx, err := r.GetTargetContext(ctx, targetKey) - if err != nil { - return err - } - return tctx.Confirm(ctx, storebackend.Key{NamespacedName: targetKey}, transactionID) -} - -func (r *targetHandler) Cancel(ctx context.Context, targetKey types.NamespacedName, transactionID string) error { - _, tctx, err := r.GetTargetContext(ctx, targetKey) - if err != nil { - return err - } - return tctx.Cancel(ctx, storebackend.Key{NamespacedName: targetKey}, transactionID) -} diff --git a/pkg/target/handler_mock.go b/pkg/target/handler_mock.go deleted file mode 100644 index 47264f2c..00000000 --- a/pkg/target/handler_mock.go +++ /dev/null @@ -1,173 +0,0 @@ -/* -Copyright 2024 Nokia. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package target - -import ( - "context" - "sync" - "time" - - "github.com/henderiw/apiserver-store/pkg/storebackend" - "github.com/henderiw/logger/log" - pkgerrors "github.com/pkg/errors" - configapi "github.com/sdcio/config-server/apis/config" - invv1alpha1 "github.com/sdcio/config-server/apis/inv/v1alpha1" - sdcerrors "github.com/sdcio/config-server/pkg/errors" - "k8s.io/apimachinery/pkg/types" -) - -type MockContext struct { - Ready bool - Busy *time.Duration - Message string - SetIntentError error - DeleteIntentError error - RecoverIntentError error - CancelError error - ConfirmError error -} - -func NewMockTargetHandler(targetStore storebackend.Storer[*MockContext]) TargetHandler { - return &mockTargetHandler{ - targetStore: targetStore, - } -} - -type mockTargetHandler struct { - targetStore storebackend.Storer[*MockContext] - mu sync.RWMutex -} - -func (r *mockTargetHandler) getMockContext(ctx context.Context, targetKey types.NamespacedName) (*MockContext, error) { - r.mu.RLock() - defer r.mu.RUnlock() - - mtctx, err := r.targetStore.Get(ctx, storebackend.Key{NamespacedName: targetKey}) - if err != nil { - return nil, &sdcerrors.RecoverableError{ - Message: "target not found", - WrappedError: pkgerrors.Wrap(ErrLookup, string(configapi.ConditionReasonTargetNotFound)), - } - } - if !mtctx.Ready { - return mtctx, &sdcerrors.RecoverableError{ - Message: "target not ready", - WrappedError: pkgerrors.Wrap(ErrLookup, string(configapi.ConditionReasonTargetNotReady)), - } - } - return mtctx, nil -} - -func (r *mockTargetHandler) GetTargetContext(ctx context.Context, targetKey types.NamespacedName) (*invv1alpha1.Target, *Context, error) { - if _, err := r.getMockContext(ctx, targetKey); err != nil { - return nil, nil, err - } - return nil, nil, nil -} - -func (r *mockTargetHandler) SetIntent(ctx context.Context, targetKey types.NamespacedName, config *configapi.Config, deviation configapi.Deviation, dryRun bool) (*configapi.ConfigStatusLastKnownGoodSchema, string, error) { - log := log.FromContext(ctx).With("target", targetKey.String(), "intent", GetGVKNSN(config)) - log.Info("SetIntent") - mctx, err := r.getMockContext(ctx, targetKey) - if err != nil { - return nil, mctx.Message, err - } - if mctx.Busy != nil { - log.Info("setIntents", "sleep", mctx.Busy.String()) - time.Sleep(*mctx.Busy) - return nil, mctx.Message, err - } - return nil, mctx.Message, mctx.SetIntentError -} - -func (r *mockTargetHandler) DeleteIntent(ctx context.Context, targetKey types.NamespacedName, config *configapi.Config, dryRun bool) (string, error) { - log := log.FromContext(ctx).With("target", targetKey.String(), "intent", GetGVKNSN(config)) - log.Info("DeleteIntent") - mctx, err := r.getMockContext(ctx, targetKey) - if err != nil { - return mctx.Message, err - } - if mctx.Busy != nil { - log.Info("setIntents", "sleep", mctx.Busy.String()) - time.Sleep(*mctx.Busy) - return mctx.Message, err - } - return mctx.Message, mctx.DeleteIntentError -} - -func (r *mockTargetHandler) GetData(ctx context.Context, targetKey types.NamespacedName) (*configapi.RunningConfig, error) { - return nil, &sdcerrors.RecoverableError{ - Message: "GetData not implemented", - } -} - -func (r *mockTargetHandler) GetBlameConfig(ctx context.Context, targetKey types.NamespacedName) (*configapi.ConfigBlame, error) { - return nil, &sdcerrors.RecoverableError{ - Message: "GetData not implemented", - } -} - -func (r *mockTargetHandler) RecoverIntents(ctx context.Context, targetKey types.NamespacedName, configs []*configapi.Config, deviations []*configapi.Deviation) (*configapi.ConfigStatusLastKnownGoodSchema, string, error) { - log := log.FromContext(ctx).With("target", targetKey.String()) - log.Info("RecoverIntents") - mctx, err := r.getMockContext(ctx, targetKey) - if err != nil { - return nil, mctx.Message, err - } - if mctx.Busy != nil { - log.Info("setIntents", "sleep", mctx.Busy.String()) - time.Sleep(*mctx.Busy) - return nil, mctx.Message, err - } - return nil, mctx.Message, mctx.RecoverIntentError -} - -func (r *mockTargetHandler) SetIntents(ctx context.Context, targetKey types.NamespacedName, transactionID string, configsToUpdate, configsToDelete map[string]*configapi.Config, deviationsToUpdate, deviationsToDelete map[string]*configapi.Deviation, dryRun bool) (*configapi.ConfigStatusLastKnownGoodSchema, string, error) { - log := log.FromContext(ctx).With("target", targetKey.String(), "transactionID", transactionID) - log.Info("setIntents") - mctx, err := r.getMockContext(ctx, targetKey) - if err != nil { - return nil, mctx.Message, err - } - if mctx.Busy != nil { - log.Info("setIntents", "sleep", mctx.Busy.String()) - time.Sleep(*mctx.Busy) - return nil, mctx.Message, err - } - return nil, mctx.Message, mctx.SetIntentError -} - -func (r *mockTargetHandler) Cancel(ctx context.Context, targetKey types.NamespacedName, transactionID string) error { - log := log.FromContext(ctx).With("target", targetKey.String(), "transactionID", transactionID) - log.Info("cancel") - mctx, err := r.getMockContext(ctx, targetKey) - if err != nil { - return err - } - return mctx.CancelError -} - -func (r *mockTargetHandler) Confirm(ctx context.Context, targetKey types.NamespacedName, transactionID string) error { - log := log.FromContext(ctx).With("target", targetKey.String(), "transactionID", transactionID) - log.Info("confirm") - mctx, err := r.getMockContext(ctx, targetKey) - if err != nil { - return err - } - - return mctx.ConfirmError -} diff --git a/pkg/target/util.go b/pkg/target/util.go deleted file mode 100644 index 0a066288..00000000 --- a/pkg/target/util.go +++ /dev/null @@ -1,27 +0,0 @@ -/* -Copyright 2024 Nokia. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package target - -import ( - "fmt" - - "sigs.k8s.io/controller-runtime/pkg/client" -) - -func GetGVKNSN(obj client.Object) string { - return fmt.Sprintf("%s.%s", obj.GetNamespace(), obj.GetName()) -} diff --git a/pkg/transactor/util.go b/pkg/transactor/util.go deleted file mode 100644 index 2f34fa03..00000000 --- a/pkg/transactor/util.go +++ /dev/null @@ -1,27 +0,0 @@ -/* -Copyright 2024 Nokia. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package transactor - -import ( - "fmt" - - "sigs.k8s.io/controller-runtime/pkg/client" -) - -func GetGVKNSN(obj client.Object) string { - return fmt.Sprintf("%s.%s", obj.GetNamespace(), obj.GetName()) -}