provider: enable k3d cluster log export helper #485
Conversation
|
Skipping CI for Draft Pull Request. |
k8s-ci-robot
added
kind/feature
|
[APPROVALNOTIFIER] This PR is APPROVED This pull-request has been approved by: harshanarayana The full list of commands accepted by this bot can be found here. The pull request process is described here
Needs approval from an approver in each of these files:
Approvers can indicate their approval by writing |
k8s-ci-robot
added
cncf-cla: yes
harshanarayana
force-pushed
the
feature/enable/log-export-for-k3d-provider
branch
from
71cdeda to
edabcee
Compare
|
/assign @cpanato PTAL when you can. @vladimirvivien @cpanato |
k8s-ci-robot
added
the
do-not-merge/hold
k8s-ci-robot
added
the
lgtm
| } | ||
| return nil | ||
| } else { | ||
| log.Warning("ExportLogs not implemented for k3d. Please use regular kubectl like commands to extract the logs from the cluster") |
There was a problem hiding this comment.
Do we know what that kubectl command would be to extract logs? Could we just use it as a fallback instead of just reporting a warning?
There was a problem hiding this comment.
@vladimirvivien Actually, That kubectl like is probably a mistake. Just kubectl won't be enough. We will also have to get a few more logs. And the problem is, we have to find a way to group them in a proper format. it can be tricky. (Also a duplicate work to replicate the logic).
If we want to add an export mechanism, we might be better off writing some mechanism that can do this for any provider that doesn't have a log export support.
vcluster doesn't support. So is the case with kwok.
For kwok there is an adhoc kubectl logs equivalent of export but it does only for the kwok's own components.
How do we go about doing this ?
There was a problem hiding this comment.
We also have to worry about fetching the journalctl logs + the dmesg + any other applicable logs. Might not be a bad idea to implement something generic that can be reused for any provider in the future that is lacking the log export capabilities
There was a problem hiding this comment.
package utils
import (
"bytes"
"context"
"fmt"
"os"
"path/filepath"
v1 "k8s.io/api/core/v1"
log "k8s.io/klog/v2"
"sigs.k8s.io/e2e-framework/klient/k8s/resources"
"sigs.k8s.io/e2e-framework/pkg/types"
)
type LogCollector struct {
resourceFetcher *resources.Resources
baseDir string
}
func NewLogCollector(provider types.E2EClusterProvider, clusterName, destination string) (*LogCollector, error) {
baseDir := filepath.Join(destination, fmt.Sprintf("debug-logs-%s", clusterName))
if err := os.MkdirAll(baseDir, os.FileMode(0755)); err != nil {
log.ErrorS(err, "failed to create base dir required to collect the logs", "dir", destination)
return nil, err
}
resourceFetcher, err := resources.New(provider.KubernetesRestConfig())
if err != nil {
log.ErrorS(err, "failed to create resource fetcher")
return nil, err
}
return &LogCollector{
resourceFetcher: resourceFetcher,
baseDir: baseDir,
}, nil
}
func (lc *LogCollector) CollectLogs(ctx context.Context) error {
var namespaces v1.NamespaceList
if err := lc.resourceFetcher.List(ctx, &namespaces); err != nil {
log.ErrorS(err, "failed to list namespaces in the cluster")
return err
}
for _, ns := range namespaces.Items {
if err := lc.collectNamespaceLogs(ctx, ns.Name); err != nil {
return err
}
}
return nil
}
func (lc *LogCollector) collectNamespaceLogs(ctx context.Context, namespace string) error {
log.V(3).InfoS("Collecting POD information for namespace", "namespace", namespace)
var pods v1.PodList
if err := lc.resourceFetcher.WithNamespace(namespace).List(ctx, &pods); err != nil {
log.ErrorS(err, "failed to list pods in the namespace", "namespace", namespace)
return err
}
for _, pod := range pods.Items {
if err := lc.collectPodLogs(ctx, namespace, pod); err != nil {
return err
}
}
return nil
}
func (lc *LogCollector) collectPodLogs(ctx context.Context, namespace string, pod v1.Pod) error {
uid := fmt.Sprintf("%s", pod.GetUID())
if hash, ok := pod.GetAnnotations()["kubernetes.io/config.hash"]; ok {
uid = hash
}
podBaseDir := filepath.Join(lc.baseDir, fmt.Sprintf("%s_%s_%s", namespace, pod.Name, uid))
if err := os.MkdirAll(podBaseDir, os.FileMode(0755)); err != nil {
return err
}
containers := append(pod.Spec.Containers, pod.Spec.InitContainers...)
containerStatus := append(pod.Status.ContainerStatuses, pod.Status.InitContainerStatuses...)
for _, container := range containers {
if err := lc.collectContainerLogs(ctx, namespace, pod.Name, container, containerStatus, podBaseDir); err != nil {
return err
}
}
return nil
}
func (lc *LogCollector) collectContainerLogs(ctx context.Context, namespace, podName string, container v1.Container, containerStatus []v1.ContainerStatus, podBaseDir string) error {
containerBaseDir := filepath.Join(podBaseDir, container.Name)
if err := os.MkdirAll(containerBaseDir, os.FileMode(0755)); err != nil {
return err
}
log.V(3).InfoS("Collecting logs for pod", "namespace", namespace, "pod", podName, "container", container.Name)
var podLog bytes.Buffer
if err := lc.resourceFetcher.GetPodLog(ctx, namespace, podName, container.Name, &podLog); err != nil {
return err
}
restartCount := 0
for _, cs := range containerStatus {
if cs.Name == container.Name {
restartCount = int(cs.RestartCount)
break
}
}
if err := os.WriteFile(filepath.Join(containerBaseDir, fmt.Sprintf("%d.log", restartCount)), podLog.Bytes(), os.FileMode(0644)); err != nil {
return err
}
return nil
}@cpanato @vladimirvivien Would something like this help ? We can also use the NodeLogQuery feature for fetching most of the node logs that we might want o collect.
There was a problem hiding this comment.
@vladimirvivien ptal.. I would love tp get this one merged.
|
The Kubernetes project currently lacks enough contributors to adequately respond to all PRs. This bot triages PRs according to the following rules:
You can:
Please send feedback to sig-contributor-experience at kubernetes/community. /lifecycle stale |
k8s-ci-robot
added
the
lifecycle/stale
|
/remove-lifecycle stale |
k8s-ci-robot
removed
the
lifecycle/stale
|
@harshanarayana are you still working on this ? |
|
The Kubernetes project currently lacks enough contributors to adequately respond to all PRs. This bot triages PRs according to the following rules:
You can:
Please send feedback to sig-contributor-experience at kubernetes/community. /lifecycle stale |
k8s-ci-robot
added
the
lifecycle/stale
|
The Kubernetes project currently lacks enough active contributors to adequately respond to all PRs. This bot triages PRs according to the following rules:
You can:
Please send feedback to sig-contributor-experience at kubernetes/community. /lifecycle rotten |
k8s-ci-robot
added
lifecycle/rotten
|
PR needs rebase. Instructions for interacting with me using PR comments are available here. If you have questions or suggestions related to my behavior, please file an issue against the kubernetes-sigs/prow repository. |
k8s-ci-robot
added
the
needs-rebase
|
The Kubernetes project currently lacks enough active contributors to adequately respond to all issues and PRs. This bot triages PRs according to the following rules:
You can:
Please send feedback to sig-contributor-experience at kubernetes/community. /close |
|
@k8s-triage-robot: Closed this PR. In response to this:
Instructions for interacting with me using PR comments are available here. If you have questions or suggestions related to my behavior, please file an issue against the kubernetes-sigs/prow repository. |
|
/reopen |
k8s-ci-robot
removed
the
lifecycle/rotten
6 participants
What type of PR is this?
/kind feature
What this PR does / why we need it:
Enable
k3dprovider with Ability to export cluster logs. Changes required to enable this was added tok3dwith k3d-io/k3d#1471.With that changes being merged, we can now add support for exporting the logs.
Which issue(s) this PR fixes:
NA
Special notes for your reviewer:
Does this PR introduce a user-facing change?
Additional documentation e.g., Usage docs, etc.: