From c9c2e02bd8b2d32f69289dfe70c528c3a0dd5ecc Mon Sep 17 00:00:00 2001 From: Silenio Quarti Date: Mon, 13 Jan 2025 22:10:12 -0500 Subject: [PATCH] CAA: add support to look up imagePullSecrets for pods This PR initializes auth.json with the imagePullSecrets listed on the pod and service account. Fixes: #2231 Signed-off-by: Silenio Quarti --- .../install/overlays/aws/kustomization.yaml | 4 - .../install/overlays/azure/kustomization.yaml | 4 - .../overlays/docker/kustomization.yaml | 4 - .../install/overlays/gcp/kustomization.yaml | 4 - .../ibmcloud-powervs/kustomization.yaml | 4 - .../overlays/ibmcloud/kustomization.yaml | 4 - .../overlays/libvirt/kustomization.yaml | 4 - .../overlays/vsphere/kustomization.yaml | 4 - .../install/rbac/peer-pod.yaml | 6 ++ .../install/yamls/caa-pod.yaml | 7 -- .../pkg/adaptor/cloud/cloud.go | 22 ++--- .../pkg/adaptor/k8sops/node.go | 91 +++++++++++++++++++ .../test/e2e/assessment_runner.go | 17 ++-- .../provisioner/docker/provision_common.go | 4 - .../ibmcloud/provision_kustomize.go | 4 - .../test/provisioner/kustomize.go | 22 ----- .../provisioner/libvirt/provision_common.go | 4 - 17 files changed, 115 insertions(+), 94 deletions(-) diff --git a/src/cloud-api-adaptor/install/overlays/aws/kustomization.yaml b/src/cloud-api-adaptor/install/overlays/aws/kustomization.yaml index f42fc09ca..8f2a652ae 100644 --- a/src/cloud-api-adaptor/install/overlays/aws/kustomization.yaml +++ b/src/cloud-api-adaptor/install/overlays/aws/kustomization.yaml @@ -45,10 +45,6 @@ configMapGenerator: ##TLS_SETTINGS secretGenerator: -- name: auth-json-secret - namespace: confidential-containers-system - files: - #- auth.json # set - path to auth.json pull credentials file - name: peer-pods-secret namespace: confidential-containers-system # This file should look like this (w/o quotes!): diff --git a/src/cloud-api-adaptor/install/overlays/azure/kustomization.yaml b/src/cloud-api-adaptor/install/overlays/azure/kustomization.yaml index 9692e6302..8d8216a10 100644 --- a/src/cloud-api-adaptor/install/overlays/azure/kustomization.yaml +++ b/src/cloud-api-adaptor/install/overlays/azure/kustomization.yaml @@ -51,10 +51,6 @@ configMapGenerator: ##TLS_SETTINGS secretGenerator: -- name: auth-json-secret - namespace: confidential-containers-system - files: - #- auth.json # set - path to auth.json pull credentials file - name: peer-pods-secret namespace: confidential-containers-system # This file should look like this (w/o quotes!): diff --git a/src/cloud-api-adaptor/install/overlays/docker/kustomization.yaml b/src/cloud-api-adaptor/install/overlays/docker/kustomization.yaml index 1c8392a35..44b0bcd11 100644 --- a/src/cloud-api-adaptor/install/overlays/docker/kustomization.yaml +++ b/src/cloud-api-adaptor/install/overlays/docker/kustomization.yaml @@ -36,10 +36,6 @@ configMapGenerator: ##TLS_SETTINGS secretGenerator: - - name: auth-json-secret - namespace: confidential-containers-system - # files: - #- auth.json # set - path to auth.json pull credentials file - name: peer-pods-secret namespace: confidential-containers-system literals: diff --git a/src/cloud-api-adaptor/install/overlays/gcp/kustomization.yaml b/src/cloud-api-adaptor/install/overlays/gcp/kustomization.yaml index 77823af10..d44e71334 100644 --- a/src/cloud-api-adaptor/install/overlays/gcp/kustomization.yaml +++ b/src/cloud-api-adaptor/install/overlays/gcp/kustomization.yaml @@ -33,10 +33,6 @@ configMapGenerator: ##TLS_SETTINGS secretGenerator: -- name: auth-json-secret - namespace: confidential-containers-system - files: - #- auth.json # set - path to auth.json pull credentials file - name: peer-pods-secret namespace: confidential-containers-system files: diff --git a/src/cloud-api-adaptor/install/overlays/ibmcloud-powervs/kustomization.yaml b/src/cloud-api-adaptor/install/overlays/ibmcloud-powervs/kustomization.yaml index 452b4be15..77c875282 100644 --- a/src/cloud-api-adaptor/install/overlays/ibmcloud-powervs/kustomization.yaml +++ b/src/cloud-api-adaptor/install/overlays/ibmcloud-powervs/kustomization.yaml @@ -44,10 +44,6 @@ configMapGenerator: ##TLS_SETTINGS secretGenerator: -- name: auth-json-secret - namespace: confidential-containers-system - files: - #- auth.json # set - path to auth.json pull credentials file - name: peer-pods-secret namespace: confidential-containers-system # the below file should be in this format diff --git a/src/cloud-api-adaptor/install/overlays/ibmcloud/kustomization.yaml b/src/cloud-api-adaptor/install/overlays/ibmcloud/kustomization.yaml index 341271f9e..f232f21b6 100644 --- a/src/cloud-api-adaptor/install/overlays/ibmcloud/kustomization.yaml +++ b/src/cloud-api-adaptor/install/overlays/ibmcloud/kustomization.yaml @@ -42,10 +42,6 @@ configMapGenerator: ##TLS_SETTINGS secretGenerator: -- name: auth-json-secret - namespace: confidential-containers-system - files: - #- auth.json # set - path to auth.json pull credentials file - name: peer-pods-secret namespace: confidential-containers-system literals: diff --git a/src/cloud-api-adaptor/install/overlays/libvirt/kustomization.yaml b/src/cloud-api-adaptor/install/overlays/libvirt/kustomization.yaml index 5de1893ac..f6db97ee3 100644 --- a/src/cloud-api-adaptor/install/overlays/libvirt/kustomization.yaml +++ b/src/cloud-api-adaptor/install/overlays/libvirt/kustomization.yaml @@ -40,10 +40,6 @@ configMapGenerator: ##TLS_SETTINGS secretGenerator: -- name: auth-json-secret - namespace: confidential-containers-system - files: - #- auth.json # set - path to auth.json pull credentials file - name: ssh-key-secret namespace: confidential-containers-system files: # key generation example: ssh-keygen -f ./id_rsa -N "" && sudo cat id_rsa.pub >> /root/.ssh/authorized_keys diff --git a/src/cloud-api-adaptor/install/overlays/vsphere/kustomization.yaml b/src/cloud-api-adaptor/install/overlays/vsphere/kustomization.yaml index 14f5d1ffa..90e4b6b21 100644 --- a/src/cloud-api-adaptor/install/overlays/vsphere/kustomization.yaml +++ b/src/cloud-api-adaptor/install/overlays/vsphere/kustomization.yaml @@ -57,10 +57,6 @@ configMapGenerator: ##TLS_SETTINGS secretGenerator: -- name: auth-json-secret - namespace: confidential-containers-system - files: - #- auth.json # set - path to auth.json pull credentials file - name: peer-pods-secret namespace: confidential-containers-system literals: diff --git a/src/cloud-api-adaptor/install/rbac/peer-pod.yaml b/src/cloud-api-adaptor/install/rbac/peer-pod.yaml index 79bf50c39..f033c4b21 100644 --- a/src/cloud-api-adaptor/install/rbac/peer-pod.yaml +++ b/src/cloud-api-adaptor/install/rbac/peer-pod.yaml @@ -14,6 +14,12 @@ rules: - apiGroups: [""] resources: ["pods"] verbs: ["get", "list"] +- apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] +- apiGroups: [""] + resources: ["serviceaccounts"] + verbs: ["get", "list"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding diff --git a/src/cloud-api-adaptor/install/yamls/caa-pod.yaml b/src/cloud-api-adaptor/install/yamls/caa-pod.yaml index d66371d4d..b15d1e371 100644 --- a/src/cloud-api-adaptor/install/yamls/caa-pod.yaml +++ b/src/cloud-api-adaptor/install/yamls/caa-pod.yaml @@ -48,9 +48,6 @@ spec: periodSeconds: 20 initialDelaySeconds: 20 volumeMounts: - - name: auth-json - mountPath: "/root/containers/" # hardcoded - readOnly: true - mountPath: /root/.ssh/ name: ssh readOnly: true @@ -73,10 +70,6 @@ spec: nodeSelector: node.kubernetes.io/worker: "" volumes: - - name: auth-json - secret: - secretName: auth-json-secret - optional: true # failing? - name: ssh secret: defaultMode: 384 diff --git a/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go b/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go index 8a88eb9cd..200e057c1 100644 --- a/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go +++ b/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go @@ -36,8 +36,7 @@ import ( ) const ( - SrcAuthfilePath = "/root/containers/auth.json" - Version = "0.0.0" + Version = "0.0.0" ) type InitData struct { @@ -253,18 +252,6 @@ func (s *cloudService) CreateVM(ctx context.Context, req *pb.CreateVMRequest) (r agentProxy := s.proxyFactory.New(serverName, socketPath) - var authJSON []byte - _, err = os.Stat(SrcAuthfilePath) - if err != nil { - logger.Printf("credential file %s is not present, skipping image auth config", SrcAuthfilePath) - } else { - authJSON, err = os.ReadFile(SrcAuthfilePath) - if err != nil { - return nil, fmt.Errorf("error reading %s: %v", SrcAuthfilePath, err) - } - logger.Printf("configure agent to use credentials file %s", SrcAuthfilePath) - } - daemonConfig := forwarder.Config{ PodNamespace: namespace, PodName: pod, @@ -320,7 +307,14 @@ func (s *cloudService) CreateVM(ctx context.Context, req *pb.CreateVMRequest) (r }, } + // Look up image pull secrets for the pod + authJSON, err := k8sops.GetImagePullSecrets(pod, namespace) + if err != nil { + // Ignore errors + logger.Printf("error reading image pull secrets: %v", err) + } if authJSON != nil { + logger.Printf("successfully retrieved pod image pull secrets for %s/%s", namespace, pod) if len(authJSON) > cloudinit.DefaultAuthfileLimit { logger.Printf("Credentials file is too large to be included in cloud-config") } else { diff --git a/src/cloud-api-adaptor/pkg/adaptor/k8sops/node.go b/src/cloud-api-adaptor/pkg/adaptor/k8sops/node.go index c55282aee..3538ccdd9 100644 --- a/src/cloud-api-adaptor/pkg/adaptor/k8sops/node.go +++ b/src/cloud-api-adaptor/pkg/adaptor/k8sops/node.go @@ -81,6 +81,97 @@ func RemoveExtendedResources() error { return nil } +// Auths contains Registries with credentials +type Auths struct { + Registries Registries `json:"auths"` +} + +// Registries contains credentials for hosts +type Registries map[string]Auth + +// Auth contains credentials for a given host +type Auth struct { + Auth string `json:"auth"` +} + +// GetImagePullSecrets gets image pull secrets for the specified pod +func GetImagePullSecrets(podName string, namespace string) ([]byte, error) { + + config, err := getKubeConfig() + if err != nil { + return nil, fmt.Errorf("failed to get k8s config: %v", err) + } + + cli, err := getClient(config) + if err != nil { + return nil, fmt.Errorf("failed to get k8s client: %v", err) + } + + pod, err := cli.CoreV1().Pods(namespace).Get(context.TODO(), podName, metav1.GetOptions{}) + if err != nil { + return nil, err + } + + accountName := pod.Spec.ServiceAccountName + if accountName == "" { + accountName = "default" + } + serviceaAccount, err := cli.CoreV1().ServiceAccounts(namespace).Get(context.TODO(), accountName, metav1.GetOptions{}) + if err != nil { + return nil, err + } + + auths := Auths{} + auths.Registries = make(map[string]Auth) + for _, secret := range serviceaAccount.ImagePullSecrets { + err := getAuths(cli, namespace, secret.Name, &auths) + if err != nil { + return nil, err + } + } + for _, secret := range pod.Spec.ImagePullSecrets { + err := getAuths(cli, namespace, secret.Name, &auths) + if err != nil { + return nil, err + } + } + + if len(auths.Registries) > 0 { + authJSON, err := json.Marshal(auths) + if err != nil { + return nil, err + } + return authJSON, nil + } + return nil, nil +} + +// getAuths get auth credentials from specified docker secret +func getAuths(cli *k8sclient.Clientset, namespace string, secretName string, auths *Auths) error { + secret, err := cli.CoreV1().Secrets(namespace).Get(context.TODO(), secretName, metav1.GetOptions{}) + if err != nil { + return err + } + registries := Registries{} + if secretData, ok := secret.Data[".dockerconfigjson"]; ok { + auths := Auths{} + err := json.Unmarshal(secretData, &auths) + if err != nil { + return err + } + registries = auths.Registries + } else if secretData, ok := secret.Data[".dockercfg"]; ok { + err = json.Unmarshal(secretData, ®istries) + if err != nil { + return err + } + } + for registry, creds := range registries { + auths.Registries[registry] = creds + } + return nil +} + // patchNodeStatus patches the status of a node func patchNodeStatus(c *k8sclient.Clientset, nodeName string, patches []jsonPatch) error { if len(patches) > 0 { diff --git a/src/cloud-api-adaptor/test/e2e/assessment_runner.go b/src/cloud-api-adaptor/test/e2e/assessment_runner.go index d2cf345b3..5d3e82c0a 100644 --- a/src/cloud-api-adaptor/test/e2e/assessment_runner.go +++ b/src/cloud-api-adaptor/test/e2e/assessment_runner.go @@ -186,13 +186,16 @@ func writeAuthSecret(client klient.Client, ctx context.Context) error { return nil } - providerName := os.Getenv("CLOUD_PROVIDER") - // this path is relative to ./test/e2e - authFilePath := "../../install/overlays/" + providerName + "/auth.json" - authfile, err := os.ReadFile(authFilePath) - if err != nil { - return err - } + template := `{ + "auths": { + "%s": { + "auth": "%s" + } + } + }` + cred := os.Getenv("REGISTRY_CREDENTIAL_ENCODED") + registryName := strings.Split(os.Getenv("AUTHENTICATED_REGISTRY_IMAGE"), "/")[0] + authfile := []byte(fmt.Sprintf(template, registryName, cred)) secretData := map[string][]byte{v1.DockerConfigJsonKey: authfile} secret = NewSecret(E2eNamespace, DEFAULT_AUTH_SECRET, secretData, v1.SecretTypeDockerConfigJson) diff --git a/src/cloud-api-adaptor/test/provisioner/docker/provision_common.go b/src/cloud-api-adaptor/test/provisioner/docker/provision_common.go index 67656e1ae..374c2df96 100644 --- a/src/cloud-api-adaptor/test/provisioner/docker/provision_common.go +++ b/src/cloud-api-adaptor/test/provisioner/docker/provision_common.go @@ -257,10 +257,6 @@ func (lio *DockerInstallOverlay) Edit(ctx context.Context, cfg *envconf.Config, } } - if err := lio.Overlay.SetAuthJsonSecretIfApplicable(); err != nil { - return err - } - if err := lio.Overlay.YamlReload(); err != nil { return err } diff --git a/src/cloud-api-adaptor/test/provisioner/ibmcloud/provision_kustomize.go b/src/cloud-api-adaptor/test/provisioner/ibmcloud/provision_kustomize.go index 7b2f5e370..a301e1fe3 100644 --- a/src/cloud-api-adaptor/test/provisioner/ibmcloud/provision_kustomize.go +++ b/src/cloud-api-adaptor/test/provisioner/ibmcloud/provision_kustomize.go @@ -168,10 +168,6 @@ func (lio *IBMCloudInstallOverlay) Edit(ctx context.Context, cfg *envconf.Config } } - if err = lio.Overlay.SetAuthJsonSecretIfApplicable(); err != nil { - return err - } - if err = lio.Overlay.YamlReload(); err != nil { return err } diff --git a/src/cloud-api-adaptor/test/provisioner/kustomize.go b/src/cloud-api-adaptor/test/provisioner/kustomize.go index 84412365d..3fd46b012 100644 --- a/src/cloud-api-adaptor/test/provisioner/kustomize.go +++ b/src/cloud-api-adaptor/test/provisioner/kustomize.go @@ -5,7 +5,6 @@ import ( "context" "fmt" "os" - "path/filepath" "strings" "golang.org/x/exp/slices" @@ -430,24 +429,3 @@ func setSecretGeneratorLiteral(k *ktypes.Kustomization, secretName string, key s return nil } - -func (kh *KustomizeOverlay) SetAuthJsonSecretIfApplicable() error { - if cred := os.Getenv("REGISTRY_CREDENTIAL_ENCODED"); cred != "" { - registryName := strings.Split(os.Getenv("AUTHENTICATED_REGISTRY_IMAGE"), "/")[0] - template := `{ - "auths": { - "%s": { - "auth": "%s" - } - } - }` - authJSON := fmt.Sprintf(template, registryName, cred) - if err := os.WriteFile(filepath.Join(kh.ConfigDir, "auth.json"), []byte(authJSON), 0644); err != nil { - return err - } - if err := kh.SetKustomizeSecretGeneratorFile("auth-json-secret", "auth.json"); err != nil { - return err - } - } - return nil -} diff --git a/src/cloud-api-adaptor/test/provisioner/libvirt/provision_common.go b/src/cloud-api-adaptor/test/provisioner/libvirt/provision_common.go index eed55669e..02925b91e 100644 --- a/src/cloud-api-adaptor/test/provisioner/libvirt/provision_common.go +++ b/src/cloud-api-adaptor/test/provisioner/libvirt/provision_common.go @@ -387,10 +387,6 @@ func (lio *LibvirtInstallOverlay) Edit(ctx context.Context, cfg *envconf.Config, } } - if err = lio.Overlay.SetAuthJsonSecretIfApplicable(); err != nil { - return err - } - if err = lio.Overlay.YamlReload(); err != nil { return err }