Change to fix sdk replace does not wait for new isvc to be ready (kse…

…rve#1925) * Intial commit for sdk replace fix Made changes to wait for the new replaced version to be ready Signed-off-by: Andrews Arokiam <[email protected]> * Fixed linting errors Signed-off-by: Andrews Arokiam <[email protected]> * Removed print statement Signed-off-by: Andrews Arokiam <[email protected]> * Made changes to use observed generation Signed-off-by: Andrews Arokiam <[email protected]> * Fixed issues with missing generation / observed generation Signed-off-by: Andrews Arokiam <[email protected]> * Remove unnecessary sleep * Fix indentation Signed-off-by: Andrews Arokiam <[email protected]> Co-authored-by: Dan Sun <[email protected]>
zillow · Dec 12, 2022 · 47a4e36 · 47a4e36
1 parent f5b5d00
commit 47a4e36
Show file tree

Hide file tree

Showing 4 changed files with 27 additions and 14 deletions.
diff --git a/pkg/apis/serving/v1beta1/inference_service_status.go b/pkg/apis/serving/v1beta1/inference_service_status.go
@@ -293,6 +293,7 @@ func (ss *InferenceServiceStatus) PropagateRawStatus(
 	readyCondition := conditionsMap[component]
 	ss.SetCondition(readyCondition, condition)
 	ss.Components[component] = statusSpec
+	ss.ObservedGeneration = deployment.Status.ObservedGeneration
 }
 
 func getDeploymentCondition(deployment *appsv1.Deployment, conditionType appsv1.DeploymentConditionType) *apis.Condition {
@@ -383,6 +384,7 @@ func (ss *InferenceServiceStatus) PropagateStatus(component ComponentType, servi
 	ss.ClearCondition(TransformerConfigurationeReady)
 
 	ss.Components[component] = statusSpec
+	ss.ObservedGeneration = serviceStatus.ObservedGeneration
 }
 
 func (ss *InferenceServiceStatus) SetCondition(conditionType apis.ConditionType, condition *apis.Condition) {

diff --git a/python/kserve/kserve/api/kserve_client.py b/python/kserve/kserve/api/kserve_client.py
@@ -251,7 +251,8 @@ def replace(self, name, inferenceservice, namespace=None, watch=False,
             isvc_watch(
                 name=outputs['metadata']['name'],
                 namespace=namespace,
-                timeout_seconds=timeout_seconds)
+                timeout_seconds=timeout_seconds,
+                generation=outputs['metadata']['generation'])
         else:
             return outputs
 

diff --git a/python/kserve/kserve/api/watch.py b/python/kserve/kserve/api/watch.py
@@ -21,7 +21,7 @@
 from ..utils import utils
 
 
-def isvc_watch(name=None, namespace=None, timeout_seconds=600):
+def isvc_watch(name=None, namespace=None, timeout_seconds=600, generation=0):
     """Watch the created or patched InferenceService in the specified namespace"""
 
     if namespace is None:
@@ -46,24 +46,29 @@ def isvc_watch(name=None, namespace=None, timeout_seconds=600):
         if name and name != isvc_name:
             continue
         else:
+            status = 'Unknown'
             if isvc.get('status', ''):
                 url = isvc['status'].get('url', '')
                 traffic = isvc['status'].get('components', {}).get(
                     'predictor', {}).get('traffic', [])
                 traffic_percent = 100
-                for t in traffic:
-                    if t["latestRevision"]:
-                        traffic_percent = t["percent"]
-                status = 'Unknown'
-                for condition in isvc['status'].get('conditions', {}):
-                    if condition.get('type', '') == 'Ready':
-                        status = condition.get('status', 'Unknown')
-                tbl(isvc_name, status, 100-traffic_percent, traffic_percent, url)
+                if constants.OBSERVED_GENERATION in isvc['status']:
+                    observed_generation = isvc['status'][constants.OBSERVED_GENERATION]
+                    for t in traffic:
+                        if t["latestRevision"]:
+                            traffic_percent = t["percent"]
+
+                    if generation != 0 and observed_generation != generation:
+                        continue
+                    for condition in isvc['status'].get('conditions', {}):
+                        if condition.get('type', '') == 'Ready':
+                            status = condition.get('status', 'Unknown')
+                    tbl(isvc_name, status, 100-traffic_percent, traffic_percent, url)
+                    if status == 'True':
+                        break
+
             else:
-                tbl(isvc_name, 'Unknown', '', '', '')
+                tbl(isvc_name, status, '', '', '')
                 # Sleep 2 to avoid status section is not generated within a very short time.
                 time.sleep(2)
                 continue
-
-            if name == isvc_name and status == 'True':
-                break
diff --git a/python/kserve/kserve/constants/constants.py b/python/kserve/kserve/constants/constants.py
@@ -66,3 +66,8 @@
     "FP64": "fp64_contents",
     "BYTES": "bytes_contents"
 }
+# K8S status key constants
+OBSERVED_GENERATION = 'observedGeneration'
+
+# K8S metadata key constants
+GENERATION = 'generation'