diff --git a/backend/apps/common/routes/get.py b/backend/apps/common/routes/get.py index 1e338ea6..155c1c8a 100644 --- a/backend/apps/common/routes/get.py +++ b/backend/apps/common/routes/get.py @@ -1,6 +1,3 @@ -"""GET request handlers.""" -from flask import request - from kubeflow.kubeflow.crud_backend import api, logging from .. import utils, versions @@ -8,6 +5,8 @@ log = logging.getLogger(__name__) +KSERVE_CONTAINER = "kserve-container" + @bp.route("/api/namespaces//inferenceservices") def get_inference_services(namespace): @@ -24,40 +23,77 @@ def get_inference_service(namespace, name): """Return an InferenceService CR as a json object.""" inference_service = api.get_custom_rsrc(**versions.inference_service_gvk(), namespace=namespace, name=name) - if request.args.get("logs", "false") == "true": - # find the logs - return api.success_response( - "serviceLogs", get_inference_service_logs(inference_service), - ) return api.success_response("inferenceService", inference_service) -def get_inference_service_logs(svc): - """Return all logs for all isvc component pods.""" - namespace = svc["metadata"]["namespace"] - components = request.args.getlist("component") +@bp.route("/api/namespaces//inferenceservices//components//pods/containers") # noqa: E501 +def get_inference_service_containers( + namespace: str, name: str, component: str): + """Get all containers and init-containers for the latest pod of + the given component. + + The kserve-container will always be the first in the list if it exists + + Return: + { + "containers": ["kserve-container", "container2", ...] + } + """ + inference_service = api.get_custom_rsrc(**versions.inference_service_gvk(), + namespace=namespace, name=name) + + latest_pod = utils.get_component_latest_pod(inference_service, component) + + if latest_pod is None: + return api.failed_response( + f"couldn't find latest pod for component: {component}", 404) - log.info(components) + containers = [] + for container in latest_pod.spec.init_containers: + containers.append(container.name) - # dictionary{component: [pod-names]} - component_pods_dict = utils.get_inference_service_pods(svc, components) + for container in latest_pod.spec.containers: + containers.append(container.name) + + # Make kserve-container always the first container in the list if it exists + try: + idx = containers.index(KSERVE_CONTAINER) + containers.insert(0, containers.pop(idx)) + except ValueError: + # kserve-container not found in list + pass + + return api.success_response("containers", containers) + + +@bp.route("/api/namespaces//inferenceservices//components//pods/containers//logs") # noqa: E501 +def get_container_logs(namespace: str, name: str, + component: str, container: str): + """Get logs for a particular container inside the latest pod of + the given component + + Logs are split on newline and returned as an array of lines + + Return: + { + "logs": ["log\n", "text\n", ...] + } + """ + inference_service = api.get_custom_rsrc(**versions.inference_service_gvk(), + namespace=namespace, name=name) + namespace = inference_service["metadata"]["namespace"] - if len(component_pods_dict.keys()) == 0: - return {} + latest_pod = utils.get_component_latest_pod(inference_service, component) + if latest_pod is None: + return api.failed_response( + f"couldn't find latest pod for component: {component}", 404) - resp = {} - logging.info("Component pods: %s", component_pods_dict) - for component, pods in component_pods_dict.items(): - if component not in resp: - resp[component] = [] + logs = api.get_pod_logs( + namespace, latest_pod.metadata.name, container, auth=False) + logs = logs.split("\n") - for pod in pods: - logs = api.get_pod_logs(namespace, pod, "kserve-container", - auth=False) - resp[component].append({"podName": pod, - "logs": logs.split("\n")}) - return resp + return api.success_response("logs", logs) @bp.route("/api/namespaces//knativeServices/") diff --git a/backend/apps/common/utils.py b/backend/apps/common/utils.py index 1131de0d..60898a75 100644 --- a/backend/apps/common/utils.py +++ b/backend/apps/common/utils.py @@ -1,11 +1,13 @@ """Common utils for parsing and handling InferenceServices.""" import os +from typing import Dict, Union from kubeflow.kubeflow.crud_backend import api, helpers, logging log = logging.getLogger(__name__) KNATIVE_REVISION_LABEL = "serving.knative.dev/revision" +LATEST_CREATED_REVISION = "latestCreatedRevision" FILE_ABS_PATH = os.path.abspath(os.path.dirname(__file__)) INFERENCESERVICE_TEMPLATE_YAML = os.path.join( @@ -24,71 +26,57 @@ def load_inference_service_template(**kwargs): return helpers.load_param_yaml(INFERENCESERVICE_TEMPLATE_YAML, **kwargs) -# helper functions for accessing the logs of an InferenceService -def get_inference_service_pods(svc, components=[]): - """ - Return the Pod names for the different isvc components. +def get_component_latest_pod(svc: Dict, + component: str) -> Union[api.client.V1Pod, None]: + """Get pod of the latest Knative revision for the given component. - Return a dictionary with (endpoint, component) keys, - i.e. ("default", "predictor") and a list of pod names as values + Return: + Latest pod: k8s V1Pod """ namespace = svc["metadata"]["namespace"] - # dictionary{revisionName: (endpoint, component)} - revisions_dict = get_components_revisions_dict(components, svc) + latest_revision = get_component_latest_revision(svc, component) - if len(revisions_dict.keys()) == 0: - return {} + if latest_revision is None: + return None pods = api.list_pods(namespace, auth=False).items - component_pods_dict = {} + for pod in pods: - for revision in revisions_dict: - if KNATIVE_REVISION_LABEL not in pod.metadata.labels: - continue + if KNATIVE_REVISION_LABEL not in pod.metadata.labels: + continue - if pod.metadata.labels[KNATIVE_REVISION_LABEL] != revision: - continue + if pod.metadata.labels[KNATIVE_REVISION_LABEL] != latest_revision: + continue - component = revisions_dict[revision] - curr_pod_names = component_pods_dict.get(component, []) - curr_pod_names.append(pod.metadata.name) - component_pods_dict[component] = curr_pod_names + return pod - if len(component_pods_dict.keys()) == 0: - log.info("No pods are found for inference service: %s", - svc["metadata"]["name"]) + log.info( + f"No pods are found for inference service: {svc['metadata']['name']}") - return component_pods_dict + return None -# FIXME(elikatsis,kimwnasptd): Change the logic of this function according to -# https://github.com/arrikto/dev/issues/867 -def get_components_revisions_dict(components, svc): - """Return a dictionary{revisionId: component}.""" - status = svc["status"] - revisions_dict = {} +def get_component_latest_revision(svc: Dict, + component: str) -> Union[str, None]: + """Get the name of the latest created knative revision for the given component. - for component in components: - if "components" not in status: - log.info("Component '%s' not in inference service '%s'", - component, svc["metadata"]["name"]) - continue + Return: + Latest Created Knative Revision: str + """ + status = svc["status"] - if component not in status["components"]: - log.info("Component '%s' not in inference service '%s'", - component, svc["metadata"]["name"]) - continue + if "components" not in status: + log.info(f"Components field not found in status object of {svc['metadata']['name']}") # noqa: E501 + return None - if "latestReadyRevision" in status["components"][component]: - revision = status["components"][component]["latestReadyRevision"] + if component not in status["components"]: + log.info(f"Component {component} not found in inference service {svc['metadata']['name']}") # noqa: E501 + return None - revisions_dict[revision] = component + if LATEST_CREATED_REVISION in status["components"][component]: + return status["components"][component][LATEST_CREATED_REVISION] - if len(revisions_dict.keys()) == 0: - log.info( - "No revisions found for the inference service's components: %s", - svc["metadata"]["name"], - ) + log.info(f"No {LATEST_CREATED_REVISION} found for the {component} in {svc['metadata']['name']}") # noqa: E501 - return revisions_dict + return None diff --git a/frontend/src/app/pages/server-info/logs/logs-viewer/logs-viewer.component.html b/frontend/src/app/pages/server-info/logs/logs-viewer/logs-viewer.component.html index 746c2613..e22f3956 100644 --- a/frontend/src/app/pages/server-info/logs/logs-viewer/logs-viewer.component.html +++ b/frontend/src/app/pages/server-info/logs/logs-viewer/logs-viewer.component.html @@ -1,6 +1,3 @@ - - -
{{ index }} diff --git a/frontend/src/app/pages/server-info/logs/logs-viewer/logs-viewer.component.ts b/frontend/src/app/pages/server-info/logs/logs-viewer/logs-viewer.component.ts index c5d6d1f8..27ca8dda 100644 --- a/frontend/src/app/pages/server-info/logs/logs-viewer/logs-viewer.component.ts +++ b/frontend/src/app/pages/server-info/logs/logs-viewer/logs-viewer.component.ts @@ -2,15 +2,10 @@ import { Component, Input, ViewChild, - NgZone, - SimpleChanges, - OnChanges, HostBinding, - ElementRef, AfterViewInit, } from '@angular/core'; import { CdkVirtualScrollViewport } from '@angular/cdk/scrolling'; -import { take } from 'rxjs/operators'; @Component({ selector: 'app-logs-viewer', @@ -22,8 +17,6 @@ export class LogsViewerComponent implements AfterViewInit { @ViewChild(CdkVirtualScrollViewport, { static: true }) viewPort: CdkVirtualScrollViewport; - @Input() heading = 'Logs'; - @Input() subHeading = 'tit'; @Input() height = '400px'; @Input() set logs(newLogs: string[]) { diff --git a/frontend/src/app/pages/server-info/logs/logs-viewer/logs-viewer.module.ts b/frontend/src/app/pages/server-info/logs/logs-viewer/logs-viewer.module.ts index e4f2d13b..7f2e9e36 100644 --- a/frontend/src/app/pages/server-info/logs/logs-viewer/logs-viewer.module.ts +++ b/frontend/src/app/pages/server-info/logs/logs-viewer/logs-viewer.module.ts @@ -1,12 +1,18 @@ import { NgModule } from '@angular/core'; import { CommonModule } from '@angular/common'; import { ScrollingModule } from '@angular/cdk/scrolling'; +import { MatTabsModule } from '@angular/material/tabs'; import { LogsViewerComponent } from './logs-viewer.component'; import { HeadingSubheadingRowModule } from 'kubeflow'; @NgModule({ declarations: [LogsViewerComponent], - imports: [CommonModule, ScrollingModule, HeadingSubheadingRowModule], + imports: [ + CommonModule, + MatTabsModule, + ScrollingModule, + HeadingSubheadingRowModule, + ], exports: [LogsViewerComponent], }) export class LogsViewerModule {} diff --git a/frontend/src/app/pages/server-info/logs/logs.component.html b/frontend/src/app/pages/server-info/logs/logs.component.html index 3d2ed747..fe70f5a3 100644 --- a/frontend/src/app/pages/server-info/logs/logs.component.html +++ b/frontend/src/app/pages/server-info/logs/logs.component.html @@ -1,50 +1,41 @@ - - - - + - No logs were found for this InferenceService. + Logs are shown for the latest created revision - - - + + + + + + + + + + + + + + {{ loadErrorMsg }} -
- -
- -
- -
- -
- -
+
diff --git a/frontend/src/app/pages/server-info/logs/logs.component.ts b/frontend/src/app/pages/server-info/logs/logs.component.ts index 2fe73c09..dc35529f 100644 --- a/frontend/src/app/pages/server-info/logs/logs.component.ts +++ b/frontend/src/app/pages/server-info/logs/logs.component.ts @@ -1,10 +1,18 @@ -import { Component, Input, OnDestroy } from '@angular/core'; +import { Component, Input, OnDestroy, ViewChild } from '@angular/core'; import { MWABackendService } from 'src/app/services/backend.service'; import { ExponentialBackoff } from 'kubeflow'; import { Subscription } from 'rxjs'; -import { InferenceServiceLogs } from 'src/app/types/backend'; import { InferenceServiceK8s } from 'src/app/types/kfserving/v1beta1'; -import { dictIsEmpty } from 'src/app/shared/utils'; + +enum IsvcComponent { + predictor = 'predictor', + transformer = 'transformer', + explainer = 'explainer', +} + +interface IsvcComponents { + IsvcComponent?: { containers: string[] }; +} @Component({ selector: 'app-logs', @@ -12,11 +20,22 @@ import { dictIsEmpty } from 'src/app/shared/utils'; styleUrls: ['./logs.component.scss'], }) export class LogsComponent implements OnDestroy { - public goToBottom = true; - public currLogs: InferenceServiceLogs = {}; + public currLogs: string[] = []; public logsRequestCompleted = false; public loadErrorMsg = ''; + public isvcComponents: IsvcComponents = {}; + private currentComponent: string; + private currentContainer: string; + private hasLoadedContainers = false; + + @ViewChild('componentTabGroup', {static: false}) componentTabGroup; + @ViewChild('containerTabGroup', {static: false}) containerTabGroup; + + get components() { + return Object.keys(this.isvcComponents); + } + @Input() set svc(s: InferenceServiceK8s) { this.svcPrv = s; @@ -29,14 +48,41 @@ export class LogsComponent implements OnDestroy { this.pollingSub.unsubscribe(); } + for (const component of Object.keys(IsvcComponent)) { + if (!(component in this.svcPrv.spec)) { + continue; + } + + this.isvcComponents[component] = {containers: []}; + + this.backend.getInferenceServiceContainers(this.svcPrv, component).subscribe( + containers => { + if (!this.hasLoadedContainers) { + this.currentComponent = component; + this.currentContainer = containers[0]; + this.hasLoadedContainers = true; + } + this.isvcComponents[component].containers = containers; + }, + error => { + console.log(`error getting ${component} containers'`, error); + }, + ); + } + this.pollingSub = this.poller.start().subscribe(() => { - this.backend.getInferenceServiceLogs(s).subscribe( + if (!this.currentComponent || !this.currentContainer) { + return; + } + + this.backend.getInferenceServiceLogs(this.svcPrv, this.currentComponent, this.currentContainer).subscribe( logs => { this.currLogs = logs; this.logsRequestCompleted = true; this.loadErrorMsg = ''; }, error => { + this.currLogs = []; this.logsRequestCompleted = true; this.loadErrorMsg = error; }, @@ -44,26 +90,39 @@ export class LogsComponent implements OnDestroy { }); } - get logsNotEmpty(): boolean { - return !dictIsEmpty(this.currLogs); - } - private svcPrv: InferenceServiceK8s; - private components: [string, string][] = []; private pollingSub: Subscription; private poller = new ExponentialBackoff({ - interval: 3000, + interval: 5000, retries: 1, - maxInterval: 3001, + maxInterval: 5001, }); constructor(public backend: MWABackendService) {} - ngOnDestroy() { - this.pollingSub.unsubscribe(); + resetLogDisplay() { + this.logsRequestCompleted = false; + this.currLogs = []; + this.loadErrorMsg = ''; + this.poller.reset(); } - logsTrackFn(i: number, podLogs: any) { - return podLogs.podName; + componentTabChange(index: number) { + this.currentComponent = Object.keys(this.isvcComponents)[index]; + + if (!(this.currentContainer in this.isvcComponents[this.currentComponent].containers)) { + this.currentContainer = this.isvcComponents[this.currentComponent].containers[0]; + } + + this.resetLogDisplay(); + } + + containerTabChange(index: number) { + this.currentContainer = this.isvcComponents[this.currentComponent].containers[index]; + this.resetLogDisplay(); + } + + ngOnDestroy() { + this.pollingSub.unsubscribe(); } } diff --git a/frontend/src/app/pages/server-info/logs/logs.module.ts b/frontend/src/app/pages/server-info/logs/logs.module.ts index 0f35911a..53ea266c 100644 --- a/frontend/src/app/pages/server-info/logs/logs.module.ts +++ b/frontend/src/app/pages/server-info/logs/logs.module.ts @@ -1,5 +1,6 @@ import { NgModule } from '@angular/core'; import { CommonModule } from '@angular/common'; +import { MatTabsModule } from '@angular/material/tabs'; import { LogsComponent } from './logs.component'; import { KubeflowModule, PanelModule, LoadingSpinnerModule } from 'kubeflow'; import { LogsViewerModule } from './logs-viewer/logs-viewer.module'; @@ -10,6 +11,7 @@ import { LogsViewerModule } from './logs-viewer/logs-viewer.module'; CommonModule, KubeflowModule, LogsViewerModule, + MatTabsModule, LoadingSpinnerModule, PanelModule, ], diff --git a/frontend/src/app/services/backend.service.ts b/frontend/src/app/services/backend.service.ts index b6ff7f96..96ed552a 100644 --- a/frontend/src/app/services/backend.service.ts +++ b/frontend/src/app/services/backend.service.ts @@ -5,7 +5,7 @@ import { Observable } from 'rxjs'; import { catchError, map } from 'rxjs/operators'; import { svcHasComponent, getSvcComponents } from '../shared/utils'; import { InferenceServiceK8s } from '../types/kfserving/v1beta1'; -import { MWABackendResponse, InferenceServiceLogs } from '../types/backend'; +import { MWABackendResponse } from '../types/backend'; @Injectable({ providedIn: 'root', @@ -101,24 +101,37 @@ export class MWABackendService extends BackendService { ); } + public getInferenceServiceContainers( + svc: InferenceServiceK8s, + component: string, + ): Observable { + const name = svc.metadata.name; + const namespace = svc.metadata.namespace; + + const url = `api/namespaces/${namespace}/inferenceservices/${name}/components/${component}/pods/containers`; + + return this.http.get(url).pipe( + catchError(error => this.handleError(error, false)), + map((resp: MWABackendResponse) => { + return resp.containers; + }), + ); + } + public getInferenceServiceLogs( svc: InferenceServiceK8s, - components: string[] = [], - ): Observable { + component: string, + container: string, + ): Observable { const name = svc.metadata.name; const namespace = svc.metadata.namespace; - let url = `api/namespaces/${namespace}/inferenceservices/${name}?logs=true`; - ['predictor', 'explainer', 'transformer'].forEach(component => { - if (component in svc.spec) { - url += `&component=${component}`; - } - }); + const url = `api/namespaces/${namespace}/inferenceservices/${name}/components/${component}/pods/containers/${container}/logs`; return this.http.get(url).pipe( catchError(error => this.handleError(error, false)), map((resp: MWABackendResponse) => { - return resp.serviceLogs; + return resp.logs; }), ); } diff --git a/frontend/src/app/types/backend.ts b/frontend/src/app/types/backend.ts index dc24748a..817d2bd3 100644 --- a/frontend/src/app/types/backend.ts +++ b/frontend/src/app/types/backend.ts @@ -8,13 +8,8 @@ export interface MWABackendResponse extends BackendResponse { knativeConfiguration?: K8sObject; knativeRevision?: K8sObject; knativeRoute?: K8sObject; - serviceLogs?: InferenceServiceLogs; -} - -export interface InferenceServiceLogs { - predictor?: { podName: string; logs: string[] }[]; - transformer?: { podName: string; logs: string[] }[]; - explainer?: { podName: string; logs: string[] }[]; + logs?: string[]; + containers?: string[]; } // types presenting the InferenceService dependent k8s objects