diff --git a/pkg/apis/serving/v1beta1/inference_service_status.go b/pkg/apis/serving/v1beta1/inference_service_status.go index 8b581fa9a2d..19b3e73276d 100644 --- a/pkg/apis/serving/v1beta1/inference_service_status.go +++ b/pkg/apis/serving/v1beta1/inference_service_status.go @@ -293,6 +293,7 @@ func (ss *InferenceServiceStatus) PropagateRawStatus( readyCondition := conditionsMap[component] ss.SetCondition(readyCondition, condition) ss.Components[component] = statusSpec + ss.ObservedGeneration = deployment.Status.ObservedGeneration } func getDeploymentCondition(deployment *appsv1.Deployment, conditionType appsv1.DeploymentConditionType) *apis.Condition { @@ -383,6 +384,7 @@ func (ss *InferenceServiceStatus) PropagateStatus(component ComponentType, servi ss.ClearCondition(TransformerConfigurationeReady) ss.Components[component] = statusSpec + ss.ObservedGeneration = serviceStatus.ObservedGeneration } func (ss *InferenceServiceStatus) SetCondition(conditionType apis.ConditionType, condition *apis.Condition) { diff --git a/python/kserve/kserve/api/kserve_client.py b/python/kserve/kserve/api/kserve_client.py index 64f6527afaf..75756eb4346 100644 --- a/python/kserve/kserve/api/kserve_client.py +++ b/python/kserve/kserve/api/kserve_client.py @@ -251,7 +251,8 @@ def replace(self, name, inferenceservice, namespace=None, watch=False, isvc_watch( name=outputs['metadata']['name'], namespace=namespace, - timeout_seconds=timeout_seconds) + timeout_seconds=timeout_seconds, + generation=outputs['metadata']['generation']) else: return outputs diff --git a/python/kserve/kserve/api/watch.py b/python/kserve/kserve/api/watch.py index c28d6bac7f9..aa3d395cbea 100644 --- a/python/kserve/kserve/api/watch.py +++ b/python/kserve/kserve/api/watch.py @@ -21,7 +21,7 @@ from ..utils import utils -def isvc_watch(name=None, namespace=None, timeout_seconds=600): +def isvc_watch(name=None, namespace=None, timeout_seconds=600, generation=0): """Watch the created or patched InferenceService in the specified namespace""" if namespace is None: @@ -46,24 +46,29 @@ def isvc_watch(name=None, namespace=None, timeout_seconds=600): if name and name != isvc_name: continue else: + status = 'Unknown' if isvc.get('status', ''): url = isvc['status'].get('url', '') traffic = isvc['status'].get('components', {}).get( 'predictor', {}).get('traffic', []) traffic_percent = 100 - for t in traffic: - if t["latestRevision"]: - traffic_percent = t["percent"] - status = 'Unknown' - for condition in isvc['status'].get('conditions', {}): - if condition.get('type', '') == 'Ready': - status = condition.get('status', 'Unknown') - tbl(isvc_name, status, 100-traffic_percent, traffic_percent, url) + if constants.OBSERVED_GENERATION in isvc['status']: + observed_generation = isvc['status'][constants.OBSERVED_GENERATION] + for t in traffic: + if t["latestRevision"]: + traffic_percent = t["percent"] + + if generation != 0 and observed_generation != generation: + continue + for condition in isvc['status'].get('conditions', {}): + if condition.get('type', '') == 'Ready': + status = condition.get('status', 'Unknown') + tbl(isvc_name, status, 100-traffic_percent, traffic_percent, url) + if status == 'True': + break + else: - tbl(isvc_name, 'Unknown', '', '', '') + tbl(isvc_name, status, '', '', '') # Sleep 2 to avoid status section is not generated within a very short time. time.sleep(2) continue - - if name == isvc_name and status == 'True': - break diff --git a/python/kserve/kserve/constants/constants.py b/python/kserve/kserve/constants/constants.py index feeab43079e..6d952221bc4 100644 --- a/python/kserve/kserve/constants/constants.py +++ b/python/kserve/kserve/constants/constants.py @@ -66,3 +66,8 @@ "FP64": "fp64_contents", "BYTES": "bytes_contents" } +# K8S status key constants +OBSERVED_GENERATION = 'observedGeneration' + +# K8S metadata key constants +GENERATION = 'generation'