Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

when subscriber can not get argo pod logs, make pod log message intuitive #4926

Open
wants to merge 13 commits into
base: master
Choose a base branch
from
Open
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -2,7 +2,7 @@ extend type Query {
"""
Returns experiment run based on experiment run ID
"""
getExperimentRun(projectID: ID!, experimentRunID: ID, notifyID: ID): ExperimentRun!
getExperimentRun(projectID: ID!, experimentRunID: ID, notifyID: ID, infraID: InfraIdentity): ExperimentRun!

"""
Returns the list of experiment run based on various filter parameters
Original file line number Diff line number Diff line change
@@ -288,6 +288,10 @@ input PodLogRequest {
"""
infraID: ID!
"""

"""
projectID: ID!
"""
ID of a experiment run
"""
experimentRunID: ID!
28 changes: 22 additions & 6 deletions chaoscenter/graphql/server/graph/chaos_experiment_run.resolvers.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

40 changes: 34 additions & 6 deletions chaoscenter/graphql/server/graph/generated/generated.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 2 additions & 1 deletion chaoscenter/graphql/server/graph/model/models_gen.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

6 changes: 6 additions & 0 deletions chaoscenter/subscriber/pkg/graphql/definations.go
Original file line number Diff line number Diff line change
@@ -1,8 +1,14 @@
package graphql

import (
"subscriber/pkg/types"
)

type SubscriberGql interface {
SendRequest(server string, payload []byte) (string, error)
MarshalGQLData(gqlData interface{}) (string, error)
SendExperimentRunRuquest(infraData map[string]string, podLog types.PodLogRequest) (types.ExperimentRunResponse, error)
GenerateExperimentRunPayload(cid, accessKey, version string, podLog types.PodLogRequest) ([]byte, error)
}

type subscriberGql struct {
30 changes: 30 additions & 0 deletions chaoscenter/subscriber/pkg/graphql/operations.go
Original file line number Diff line number Diff line change
@@ -7,6 +7,7 @@ import (
"net/http"
"strconv"
"strings"
"subscriber/pkg/types"

log "github.com/sirupsen/logrus"
)
@@ -48,3 +49,32 @@ func (gql *subscriberGql) MarshalGQLData(gqlData interface{}) (string, error) {
processed = strings.Replace(processed, `\"`, `\\\"`, -1)
return processed, nil
}

// Get an experiment run on the GraphQL server
func (gql *subscriberGql) SendExperimentRunRuquest(infraData map[string]string, podLog types.PodLogRequest) (types.ExperimentRunResponse, error) {

payload, _ := gql.GenerateExperimentRunPayload(infraData["INFRA_ID"], infraData["ACCESS_KEY"], infraData["VERSION"], podLog)

body, err := gql.SendRequest(infraData["SERVER_ADDR"], payload)
if err != nil {
log.WithError(err).Print("Failed to send experiment run request")
}

var respsone types.ExperimentRunResponse
err = json.Unmarshal([]byte(body), &respsone)
if err != nil {
log.WithError(err).WithField("data", string(body)).Fatal("Failed to parse ExperimentRun data")
}

log.Print("Response from the server: ", body)

return respsone, nil
}

func (gql *subscriberGql) GenerateExperimentRunPayload(cid, accessKey, version string, podLog types.PodLogRequest) ([]byte, error) {
infraID := `infraID: {infraID: \"` + cid + `\", version: \"` + version + `\", accessKey: \"` + accessKey + `\"}`
query := infraID + `, experimentRunID: \"` + podLog.ExperimentRunID + `\", projectID: \"` + podLog.ProjectID + `\", notifyID: \"\"`

var payload = []byte(`{"query": "query { getExperimentRun(` + query + `) { phase } }"}`)
return payload, nil
}
6 changes: 4 additions & 2 deletions chaoscenter/subscriber/pkg/k8s/defination.go
Original file line number Diff line number Diff line change
@@ -15,9 +15,11 @@ import (

type SubscriberK8s interface {
GetLogs(podName, namespace, container string) (string, error)
CreatePodLog(podLog types.PodLogRequest) (types.PodLog, error)
GetPodLogs(infraData map[string]string, podLog types.PodLogRequest) (types.PodLog, error)
categorizeLogByPhase(phase string) string
CreatePodLog(infraData map[string]string, logDetails types.PodLog, podLog types.PodLogRequest) (types.PodLog, error)
SendPodLogs(infraData map[string]string, podLog types.PodLogRequest)
GenerateLogPayload(cid, accessKey, version string, podLog types.PodLogRequest) ([]byte, error)
GenerateLogPayload(cid, accessKey, version, processed string, podLog types.PodLogRequest) ([]byte, error)
GetKubernetesNamespaces(request types.KubeNamespaceRequest) ([]*types.KubeNamespace, error)
GetKubernetesObjects(request types.KubeObjRequest) (*types.KubeObject, error)
GetObjectDataByNamespace(namespace string, dynamicClient dynamic.Interface, resourceType schema.GroupVersionResource) ([]types.ObjectData, error)
88 changes: 65 additions & 23 deletions chaoscenter/subscriber/pkg/k8s/log.go
Original file line number Diff line number Diff line change
@@ -53,17 +53,7 @@ func (k8s *k8sSubscriber) GetLogs(podName, namespace, container string) (string,
}

// create pod log for normal pods and chaos-engine pods
func (k8s *k8sSubscriber) CreatePodLog(podLog types.PodLogRequest) (types.PodLog, error) {
logDetails := types.PodLog{}
mainLog, err := k8s.GetLogs(podLog.PodName, podLog.PodNamespace, "main")
// try getting argo pod logs
if err != nil {
logrus.Errorf("Failed to get argo pod %v logs, err: %v", podLog.PodName, err)
logDetails.MainPod = "Failed to get argo pod logs"
} else {
logDetails.MainPod = strconv.Quote(strings.Replace(mainLog, `"`, `'`, -1))
logDetails.MainPod = logDetails.MainPod[1 : len(logDetails.MainPod)-1]
}
func (k8s *k8sSubscriber) CreatePodLog(infraData map[string]string, logDetails types.PodLog, podLog types.PodLogRequest) (types.PodLog, error) {
// try getting experiment pod logs if requested
if strings.ToLower(podLog.PodType) == "chaosengine" && podLog.ChaosNamespace != nil {
chaosLog := make(map[string]string)
@@ -96,32 +86,84 @@ func (k8s *k8sSubscriber) CreatePodLog(podLog types.PodLogRequest) (types.PodLog

// SendPodLogs generates graphql mutation to send events updates to graphql server
func (k8s *k8sSubscriber) SendPodLogs(infraData map[string]string, podLog types.PodLogRequest) {
// generate graphql payload
payload, err := k8s.GenerateLogPayload(infraData["INFRA_ID"], infraData["ACCESS_KEY"], infraData["VERSION"], podLog)

logDetails, err := k8s.GetPodLogs(infraData, podLog)
if err != nil {
logrus.WithError(err).Print("Error while retrieving the workflow logs")
logrus.WithError(err).Print("failed to get pod logs")
}

// MarashalGQLData
processed, err := k8s.gqlSubscriberServer.MarshalGQLData(logDetails)
if err != nil {
processed = " Could not get logs "
}

// Generate graphql payload
payload, _ := k8s.GenerateLogPayload(infraData["INFRA_ID"], infraData["ACCESS_KEY"], infraData["VERSION"], processed, podLog)

body, err := k8s.gqlSubscriberServer.SendRequest(infraData["SERVER_ADDR"], payload)
if err != nil {
logrus.Print(err.Error())
}
logrus.Print("Response from the server: ", body)
}

func (k8s *k8sSubscriber) GenerateLogPayload(cid, accessKey, version string, podLog types.PodLogRequest) ([]byte, error) {
infraID := `{infraID: \"` + cid + `\", version: \"` + version + `\", accessKey: \"` + accessKey + `\"}`
processed := " Could not get logs "
func (k8s *k8sSubscriber) GetPodLogs(infraData map[string]string, podLog types.PodLogRequest) (types.PodLog, error) {
logDetails := types.PodLog{}
mainLog, err := k8s.GetLogs(podLog.PodName, podLog.PodNamespace, "main")

// try getting argo pod logs
if err != nil {

// fetch ExperimentRun Phase from graphql
experimentRun, _ := k8s.gqlSubscriberServer.SendExperimentRunRuquest(infraData, podLog)
logDetails.MainPod = k8s.categorizeLogByPhase(experimentRun.Data.ExperimentRun.Phase) + " Pod name: " + podLog.PodName

logrus.WithError(err).Print("Failed to get pod logs for attempting to fetch experiment run phase")

} else {

logDetails.MainPod = strconv.Quote(strings.Replace(mainLog, `"`, `'`, -1))
logDetails.MainPod = logDetails.MainPod[1 : len(logDetails.MainPod)-1]

logDetails, err = k8s.CreatePodLog(infraData, logDetails, podLog)

// get the logs
logDetails, err := k8s.CreatePodLog(podLog)
if err == nil {
// process log data
processed, err = k8s.gqlSubscriberServer.MarshalGQLData(logDetails)
if err != nil {
processed = " Could not get logs "
return logDetails, err
}
}

return logDetails, nil
}

// Categorizing log messages by experiment phase
func (k8s *k8sSubscriber) categorizeLogByPhase(phase string) string {
switch phase {
case "Completed":
return "Experiment pod was deleted."
case "Stopped":
return "Experiment is stopped."
case "Running":
return "Experiment pod is initializing."
case "Queue":
return "Queue."
case "NA":
return "NA."
case "Terminated":
return "Terminated."
case "Completed_With_Error":
return "Completed_With_Error."
case "Timeout":
return "Timeout."
case "Error":
return "Experiment could not start."
default:
return "Experiment pod is initializing."
}
}

func (k8s *k8sSubscriber) GenerateLogPayload(cid, accessKey, version, processed string, podLog types.PodLogRequest) ([]byte, error) {
infraID := `{infraID: \"` + cid + `\", version: \"` + version + `\", accessKey: \"` + accessKey + `\"}`
mutation := `{ infraID: ` + infraID + `, requestID:\"` + podLog.RequestID + `\", experimentRunID: \"` + podLog.ExperimentRunID + `\", podName: \"` + podLog.PodName + `\", podType: \"` + podLog.PodType + `\", log:\"` + processed[1:len(processed)-1] + `\"}`
var payload = []byte(`{"query":"mutation { podLog(request:` + mutation + ` )}"}`)

13 changes: 13 additions & 0 deletions chaoscenter/subscriber/pkg/types/graphql.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
package types

type ExperimentRunResponse struct {
Data ExperimentRun `json:"data"`
}

type ExperimentRun struct {
ExperimentRun Phase `json:"getExperimentRun"`
}

type Phase struct {
Phase string `json:"phase"`
}
1 change: 1 addition & 0 deletions chaoscenter/subscriber/pkg/types/log.go
Original file line number Diff line number Diff line change
@@ -2,6 +2,7 @@ package types

type PodLogRequest struct {
RequestID string `json:"requestID"`
ProjectID string `json:"projectID"`
InfraID string `json:"infraID"`
ExperimentRunID string `json:"experimentRunID"`
PodName string `json:"podName"`
2 changes: 2 additions & 0 deletions chaoscenter/web/src/api/core/log/getPodLog.ts
Original file line number Diff line number Diff line change
@@ -14,6 +14,7 @@ interface GetPodLogsRequest {
request: {
// requestID: string;
infraID: string;
projectID: string;
experimentRunID?: string;
podName: string;
podNamespace: string;
@@ -43,6 +44,7 @@ export function getPodLogsSubscription({
variables: {
request: {
infraID: request.infraID,
projectID: request.projectID,
experimentRunID: request.experimentRunID,
podName: request.podName,
podNamespace: request.podNamespace,
Original file line number Diff line number Diff line change
@@ -11,6 +11,7 @@ interface CustomStepLogControllerProps {
namespace: string | undefined;
workflowRunID: string | undefined;
infraID: string | undefined;
projectID: string | undefined;
podName: string;
requestID: string;
phase: ExperimentRunStatus | undefined;
@@ -19,6 +20,7 @@ interface CustomStepLogControllerProps {
export default function CustomStepLogController({
workflowRunID,
infraID,
projectID,
podName,
chaosData,
nodeType,
@@ -36,6 +38,7 @@ export default function CustomStepLogController({
...scope,
infraID: infraID ?? '',
// requestID: requestID,
projectID: projectID ?? '',
experimentRunID: workflowRunID,
podName: podName,
podNamespace: namespace ?? '',
1 change: 1 addition & 0 deletions chaoscenter/web/src/controllers/LogsTab/LogsTab.tsx
Original file line number Diff line number Diff line change
@@ -7,6 +7,7 @@ interface LogsTabControllerProps {
nodeType: string | undefined;
chaosData: ChaosData | undefined;
infraID: string | undefined;
projectID: string | undefined;
workflowRunID: string | undefined;
podID: string;
experimentPod?: string;
Original file line number Diff line number Diff line change
@@ -198,6 +198,7 @@ export default function ExperimentRunDetailsView({
namespace={experimentExecutionDetails?.namespace}
probeData={probeData}
infraID={infra?.infraID}
projectID={scope.projectID}
setSelectedNodeID={setSelectedNodeID}
experimentRunID={experimentRunID}
podID={selectedNodeID}
Original file line number Diff line number Diff line change
@@ -31,6 +31,7 @@ interface DetailsTabProps {
probeData: ProbeInRuns[] | undefined;
experimentRunID: string | undefined;
infraID: string | undefined;
projectID: string | undefined;
namespace: string | undefined;
phase: ExperimentRunStatus | undefined;
podID: string;
@@ -44,6 +45,7 @@ const DetailsTabs = ({
phase,
experimentRunID,
infraID,
projectID,
podID,
manifest,
probeData,
@@ -88,6 +90,7 @@ const DetailsTabs = ({
phase={phase}
workflowRunID={experimentRunID}
infraID={infraID}
projectID={projectID}
podID={podID}
/>
)
@@ -110,6 +113,7 @@ const ExperimentRunDetailsPanel = ({
phase,
namespace,
infraID,
projectID,
probeData,
podID,
manifest,
@@ -261,6 +265,7 @@ const ExperimentRunDetailsPanel = ({
probeData={probeData}
experimentRunID={experimentRunID}
infraID={infraID}
projectID={projectID}
loading={loading}
phase={phase}
namespace={namespace}
Original file line number Diff line number Diff line change
@@ -10,6 +10,7 @@ interface LogsTabViewProps {
nodeType: string | undefined;
chaosData: ChaosData | undefined;
infraID: string | undefined;
projectID: string | undefined;
workflowRunID: string | undefined;
podID: string;
requestID: string;
@@ -22,6 +23,7 @@ export default function LogsTabView({
nodeType,
chaosData,
infraID,
projectID,
workflowRunID,
podID,
requestID,
@@ -37,6 +39,7 @@ export default function LogsTabView({
nodeType={nodeType}
namespace={namespace}
infraID={infraID}
projectID={projectID}
requestID={requestID}
workflowRunID={workflowRunID}
podName={podID}

Unchanged files with check annotations Beta

const DEFAULT_ICON: IconName = 'play';
function StartNodeStage(props: any): React.ReactElement {

Check warning on line 10 in chaoscenter/web/src/components/PipelineDiagram/Nodes/StartNode/StartNodeStage.tsx

GitHub Actions / frontend-checks

Unexpected any. Specify a different type
return (
<div id={props?.id} className={cx({ [props.className]: props.className }, css.stageNode)}>
<div className={cx(css.nodeStart)} style={{ backgroundColor: '#f3f3fa', border: '1px solid #b0b1c4' }}>
const DEFAULT_ICON: IconName = 'play';
function StartNodeStep(props: any): React.ReactElement {

Check warning on line 10 in chaoscenter/web/src/components/PipelineDiagram/Nodes/StartNode/StartNodeStep.tsx

GitHub Actions / frontend-checks

Unexpected any. Specify a different type
return (
<div id={props?.id} className={cx({ [props.className]: props.className }, css.stepNode)}>
<div className={cx(css.nodeStart)} style={{ backgroundColor: '#f3f3fa', border: '1px solid #b0b1c4' }}>
import { ExecutionPipelineNodeType, ExecutionStatus, ExecutionStatusEnum } from '../types';
import css from './ExecutionStageDiagram.module.scss';
export const getPositionOfAddIcon = (props: any, isRightNode?: boolean): string => {

Check warning on line 9 in chaoscenter/web/src/components/PipelineDiagram/Nodes/utils.tsx

GitHub Actions / frontend-checks

Unexpected any. Specify a different type
if (isRightNode) {
return '-40px';
}
const DEFAULT_POSITION: ControlPosition = { x: 30, y: 60 };
export interface PipelineGraphProps {
data: PipelineGraphState[];
fireEvent: (event: any) => void;

Check warning on line 31 in chaoscenter/web/src/components/PipelineDiagram/PipelineGraph/PipelineGraph.tsx

GitHub Actions / frontend-checks

Unexpected any. Specify a different type
getNode: GetNodeMethod;
getDefaultNode: GetNodeMethod;
selectedNodeId?: string;
useLayoutEffect(() => {
shouldDelayRender ? renderSVGLinksDelayed() : redrawSVGLinks();
}, [state, graphScale]);

Check warning on line 101 in chaoscenter/web/src/components/PipelineDiagram/PipelineGraph/PipelineGraph.tsx

GitHub Actions / frontend-checks

React Hook useLayoutEffect has missing dependencies: 'redrawSVGLinks', 'renderSVGLinksDelayed', and 'shouldDelayRender'. Either include them or remove the dependency array
const redrawSVGLinks = (): void => {
setSVGLinks();
setupDragEventListeners(draggableParent, overlay);
panZoom && scrollZoom(overlay, 40, 0.01, updateGraphScale);
}
}, []);

Check warning on line 141 in chaoscenter/web/src/components/PipelineDiagram/PipelineGraph/PipelineGraph.tsx

GitHub Actions / frontend-checks

React Hook useEffect has a missing dependency: 'panZoom'. Either include it or remove the dependency array
const handleScaleToFit = (): void => {
setPosition(DEFAULT_POSITION);
container.style.transformOrigin = '0 0';
container.onwheel = scrolled;
function scrolled(e: any): void {

Check warning on line 180 in chaoscenter/web/src/components/PipelineDiagram/PipelineGraph/PipelineGraphUtils.ts

GitHub Actions / frontend-checks

Unexpected any. Specify a different type
if (!e.ctrlKey) return;
e.preventDefault();
let delta = e.delta || e.wheelDelta;
return () => {
cleanOb();
};
}, [element, options]);

Check warning on line 39 in chaoscenter/web/src/components/PipelineDiagram/hooks/useIntersection.ts

GitHub Actions / frontend-checks

React Hook useEffect has a missing dependency: 'compareFn'. Either include it or remove the dependency array. If 'compareFn' changes too often, find the parent component that defines it and wrap that definition in useCallback
return isIntersecting;
};
}, [elementToCompare]);
useEffect(() => {
setElement(document.querySelector(parentSelector!) as HTMLElement);

Check warning on line 31 in chaoscenter/web/src/components/PipelineDiagram/hooks/useResizeObserver.ts

GitHub Actions / frontend-checks

Forbidden non-null assertion
}, [parentSelector]);
const onResize = debounce(([entry]) => {
return () => {
cleanup();
};
}, [element, refElement]);

Check warning on line 56 in chaoscenter/web/src/components/PipelineDiagram/hooks/useResizeObserver.ts

GitHub Actions / frontend-checks

React Hook useEffect has missing dependencies: 'onResize' and 'parentSelector'. Either include them or remove the dependency array
return state;
};
if err != nil {
return nil, err
}
result := collection.FindOne(ctx, query)

Check failure

Code scanning / CodeQL

Database query built from user-controlled sources High

This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
return result, nil
}
return nil, err
}
result, err := collection.Aggregate(ctx, pipeline, opts...)

Check failure

Code scanning / CodeQL

Database query built from user-controlled sources High

This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
This query depends on a
user-provided value
.
if err != nil {
return nil, err
}