Skip to content

Commit e9eac4c

Browse files
adleongadleong
authored
Add prune command to linkerd and to extensions (#10303)
Fixes: #10262 When a resource is removed from the Linkerd manifests from one version to the next, we would like that resource to be removed from the user's cluster as part of the upgrade process. Our current recommendation is to use the `linkerd upgrade` command in conjunction with the `kubectl apply` command and the `--prune` flag to remove resources which are no longer part of the manifest. However, `--prune` has many shortcomings and does not detect resources kinds which are not part of the input manifest, nor does it detect cluster scoped resources. See https://linkerd.io/2.12/tasks/upgrade/#with-the-linkerd-cli We add a `linkerd prune` command which locates all Linkerd resources on the cluster which are not part of the Linkerd manifest and prints their metadata so that users can delete them. The recommended upgrade procedure would then be: ``` > linkerd upgrade | kubectl apply -f - > linkerd prune | kubectl delete -f - ``` User must take special care to use the desired version of the CLI to run the prune command since running this command will print all resources on the cluster which are not included in that version. We also add similar prune commands to each of the `viz`, `multicluster`, and `jaeger` extensions for deleting extension resources which are not in the extension manifest. Signed-off-by: Alex Leong <[email protected]>
1 parent f211080 commit e9eac4c

File tree

10 files changed

+391
-0
lines changed

10 files changed

+391
-0
lines changed

‎cli/cmd/prune.go‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
package cmd
2+
3+
import (
4+
"errors"
5+
"fmt"
6+
"os"
7+
"strings"
8+
"time"
9+
10+
pkgCmd "github.com/linkerd/linkerd2/pkg/cmd"
11+
"github.com/linkerd/linkerd2/pkg/k8s"
12+
"github.com/spf13/cobra"
13+
valuespkg "helm.sh/helm/v3/pkg/cli/values"
14+
)
15+
16+
func newCmdPrune() *cobra.Command {
17+
18+
cmd := &cobra.Command{
19+
Use: "prune [flags]",
20+
Args: cobra.NoArgs,
21+
Short: "Output extraneous Kubernetes resources in the linkerd control plane",
22+
Long: `Output extraneous Kubernetes resources in the linkerd control plane.`,
23+
Example: ` # Prune extraneous resources.
24+
linkerd prune | kubectl delete -f -
25+
`,
26+
RunE: func(cmd *cobra.Command, _ []string) error {
27+
28+
k8sAPI, err := k8s.NewAPI(kubeconfigPath, kubeContext, impersonate, impersonateGroup, 30*time.Second)
29+
if err != nil {
30+
return err
31+
}
32+
33+
values, err := loadStoredValues(cmd.Context(), k8sAPI)
34+
if err != nil {
35+
fmt.Fprintf(os.Stderr, "failed to load stored values: %s\n", err)
36+
os.Exit(1)
37+
}
38+
39+
if values == nil {
40+
return errors.New(
41+
`Could not find the linkerd-config-overrides secret.
42+
Please note this command is only intended for instances of Linkerd that were installed via the CLI`)
43+
}
44+
45+
err = validateValues(cmd.Context(), k8sAPI, values)
46+
if err != nil {
47+
return err
48+
}
49+
50+
manifests := strings.Builder{}
51+
52+
if err = renderControlPlane(&manifests, values, make(map[string]interface{})); err != nil {
53+
return err
54+
}
55+
if err = renderCRDs(&manifests, valuespkg.Options{}); err != nil {
56+
return err
57+
}
58+
59+
return pkgCmd.Prune(cmd.Context(), k8sAPI, manifests.String(), k8s.ControllerNSLabel)
60+
},
61+
}
62+
63+
return cmd
64+
}

‎cli/cmd/root.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,7 @@ func init() {
119119
RootCmd.AddCommand(newCmdUpgrade())
120120
RootCmd.AddCommand(newCmdVersion())
121121
RootCmd.AddCommand(newCmdUninstall())
122+
RootCmd.AddCommand(newCmdPrune())
122123

123124
// Extension Sub Commands
124125
RootCmd.AddCommand(jaeger.NewCmdJaeger())

‎jaeger/cmd/prune.go‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
package cmd
2+
3+
import (
4+
"fmt"
5+
"strings"
6+
"time"
7+
8+
pkgCmd "github.com/linkerd/linkerd2/pkg/cmd"
9+
"github.com/linkerd/linkerd2/pkg/flags"
10+
"github.com/linkerd/linkerd2/pkg/healthcheck"
11+
"github.com/linkerd/linkerd2/pkg/k8s"
12+
"github.com/spf13/cobra"
13+
"helm.sh/helm/v3/pkg/cli/values"
14+
)
15+
16+
func newCmdPrune() *cobra.Command {
17+
var cniEnabled bool
18+
var wait time.Duration
19+
var options values.Options
20+
21+
cmd := &cobra.Command{
22+
Use: "prune [flags]",
23+
Args: cobra.NoArgs,
24+
Short: "Output extraneous Kubernetes resources in the linkerd-jaeger extension",
25+
Long: `Output extraneous Kubernetes resources in the linkerd-jaeger extension.`,
26+
Example: ` # Prune extraneous resources.
27+
linkerd jaeger prune | kubectl delete -f -
28+
`,
29+
RunE: func(cmd *cobra.Command, _ []string) error {
30+
hc := healthcheck.NewWithCoreChecks(&healthcheck.Options{
31+
ControlPlaneNamespace: controlPlaneNamespace,
32+
KubeConfig: kubeconfigPath,
33+
KubeContext: kubeContext,
34+
Impersonate: impersonate,
35+
ImpersonateGroup: impersonateGroup,
36+
APIAddr: apiAddr,
37+
RetryDeadline: time.Now().Add(wait),
38+
})
39+
hc.RunWithExitOnError()
40+
cniEnabled = hc.CNIEnabled
41+
42+
manifests := strings.Builder{}
43+
44+
err := install(&manifests, options, "", cniEnabled)
45+
if err != nil {
46+
return err
47+
}
48+
49+
label := fmt.Sprintf("%s=%s", k8s.LinkerdExtensionLabel, JaegerExtensionName)
50+
return pkgCmd.Prune(cmd.Context(), hc.KubeAPIClient(), manifests.String(), label)
51+
},
52+
}
53+
54+
cmd.Flags().DurationVar(&wait, "wait", 300*time.Second, "Wait for extension components to be available")
55+
56+
flags.AddValueOptionsFlags(cmd.Flags(), &options)
57+
58+
return cmd
59+
}

‎jaeger/cmd/root.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,7 @@ func NewCmdJaeger() *cobra.Command {
7070
jaegerCmd.AddCommand(newCmdInstall())
7171
jaegerCmd.AddCommand(newCmdList())
7272
jaegerCmd.AddCommand(newCmdUninstall())
73+
jaegerCmd.AddCommand(newCmdPrune())
7374

7475
// resource-aware completion flag configurations
7576
pkgcmd.ConfigureNamespaceFlagCompletion(

‎multicluster/cmd/prune.go‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
package cmd
2+
3+
import (
4+
"fmt"
5+
"os"
6+
"strings"
7+
"time"
8+
9+
pkgCmd "github.com/linkerd/linkerd2/pkg/cmd"
10+
"github.com/linkerd/linkerd2/pkg/flags"
11+
"github.com/linkerd/linkerd2/pkg/healthcheck"
12+
"github.com/linkerd/linkerd2/pkg/k8s"
13+
"github.com/spf13/cobra"
14+
valuespkg "helm.sh/helm/v3/pkg/cli/values"
15+
)
16+
17+
func newCmdPrune() *cobra.Command {
18+
var ha bool
19+
var cniEnabled bool
20+
var wait time.Duration
21+
options, err := newMulticlusterInstallOptionsWithDefault()
22+
if err != nil {
23+
fmt.Fprintln(os.Stderr, err)
24+
os.Exit(1)
25+
}
26+
var valuesOptions valuespkg.Options
27+
28+
cmd := &cobra.Command{
29+
Use: "prune [flags]",
30+
Args: cobra.NoArgs,
31+
Short: "Output extraneous Kubernetes resources in the linkerd-multicluster extension",
32+
Long: `Output extraneous Kubernetes resources in the linkerd-multicluster extension.`,
33+
Example: ` # Prune extraneous resources.
34+
linkerd multicluster prune | kubectl delete -f -
35+
`,
36+
RunE: func(cmd *cobra.Command, _ []string) error {
37+
hc := healthcheck.NewWithCoreChecks(&healthcheck.Options{
38+
ControlPlaneNamespace: controlPlaneNamespace,
39+
KubeConfig: kubeconfigPath,
40+
KubeContext: kubeContext,
41+
Impersonate: impersonate,
42+
ImpersonateGroup: impersonateGroup,
43+
APIAddr: apiAddr,
44+
RetryDeadline: time.Now().Add(wait),
45+
})
46+
hc.RunWithExitOnError()
47+
cniEnabled = hc.CNIEnabled
48+
49+
manifests := strings.Builder{}
50+
51+
err := install(cmd.Context(), &manifests, options, valuesOptions, ha, false, cniEnabled)
52+
if err != nil {
53+
return err
54+
}
55+
56+
label := fmt.Sprintf("%s=%s", k8s.LinkerdExtensionLabel, MulticlusterExtensionName)
57+
return pkgCmd.Prune(cmd.Context(), hc.KubeAPIClient(), manifests.String(), label)
58+
},
59+
}
60+
61+
cmd.Flags().BoolVar(&ha, "ha", false, `Set if Linkerd Multicluster Extension is installed in High Availability mode.`)
62+
cmd.Flags().DurationVar(&wait, "wait", 300*time.Second, "Wait for extension components to be available")
63+
64+
flags.AddValueOptionsFlags(cmd.Flags(), &valuesOptions)
65+
66+
return cmd
67+
}

‎multicluster/cmd/root.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,7 @@ components on a cluster, manage credentials and link clusters together.`,
8787
multiclusterCmd.AddCommand(newMulticlusterUninstallCommand())
8888
multiclusterCmd.AddCommand(newGatewaysCommand())
8989
multiclusterCmd.AddCommand(newAllowCommand())
90+
multiclusterCmd.AddCommand(newCmdPrune())
9091

9192
// resource-aware completion flag configurations
9293
pkgcmd.ConfigureNamespaceFlagCompletion(

‎pkg/cmd/cmd.go‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
11
package cmd
22

33
import (
4+
"bufio"
45
"context"
56
"errors"
67
"fmt"
8+
"io"
79
"os"
810
"strings"
911

@@ -15,7 +17,9 @@ import (
1517
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1618
"k8s.io/apimachinery/pkg/labels"
1719
"k8s.io/apimachinery/pkg/selection"
20+
yamlDecoder "k8s.io/apimachinery/pkg/util/yaml"
1821
"k8s.io/client-go/tools/clientcmd"
22+
"sigs.k8s.io/yaml"
1923
)
2024

2125
// GetDefaultNamespace fetches the default namespace
@@ -61,6 +65,66 @@ func Uninstall(ctx context.Context, k8sAPI *k8s.KubernetesAPI, selector string)
6165
return nil
6266
}
6367

68+
// Prune takes an install manifest and prints all resources on the cluster which
69+
// match the given label selector but are not in the given manifest. Users are
70+
// expected to pipe these resources to `kubectl delete` to clean up resources
71+
// left on the cluster which are no longer part of the install manifest.
72+
func Prune(ctx context.Context, k8sAPI *k8s.KubernetesAPI, expectedManifests string, selector string) error {
73+
expectedResources := []resource.Kubernetes{}
74+
reader := yamlDecoder.NewYAMLReader(bufio.NewReaderSize(strings.NewReader(expectedManifests), 4096))
75+
for {
76+
manifest, err := reader.Read()
77+
if err != nil {
78+
if errors.Is(err, io.EOF) {
79+
break
80+
}
81+
return err
82+
}
83+
resource := resource.Kubernetes{}
84+
err = yaml.Unmarshal(manifest, &resource)
85+
if err != nil {
86+
fmt.Fprintf(os.Stderr, "error parsing manifest: %s", manifest)
87+
os.Exit(1)
88+
}
89+
expectedResources = append(expectedResources, resource)
90+
}
91+
92+
listOptions := metav1.ListOptions{
93+
LabelSelector: selector,
94+
}
95+
resources, err := resource.FetchPrunableResources(ctx, k8sAPI, metav1.NamespaceAll, listOptions)
96+
if err != nil {
97+
fmt.Fprintf(os.Stderr, "error fetching resources: %s\n", err)
98+
os.Exit(1)
99+
}
100+
101+
for _, resource := range resources {
102+
// If the resource is not in the expected resource list, render it for
103+
// pruning.
104+
if !resourceListContains(expectedResources, resource) {
105+
if err = resource.RenderResource(os.Stdout); err != nil {
106+
return fmt.Errorf("error rendering Kubernetes resource: %w\n", err)
107+
}
108+
}
109+
}
110+
return nil
111+
}
112+
113+
func resourceListContains(list []resource.Kubernetes, a resource.Kubernetes) bool {
114+
for _, r := range list {
115+
if resourceEquals(a, r) {
116+
return true
117+
}
118+
}
119+
return false
120+
}
121+
122+
func resourceEquals(a resource.Kubernetes, b resource.Kubernetes) bool {
123+
return a.GroupVersionKind().GroupKind() == b.GroupVersionKind().GroupKind() &&
124+
a.GetName() == b.GetName() &&
125+
a.GetNamespace() == b.GetNamespace()
126+
}
127+
64128
// ConfigureNamespaceFlagCompletion sets up resource-aware completion for command
65129
// flags that accept a namespace name
66130
func ConfigureNamespaceFlagCompletion(

0 commit comments

Comments
 (0)