testing-Pre-compiled end-to-end gpu driver validation

Signed-off-by: shiva kumar <[email protected]>

.github/workflows/ci-precompiled.yaml

@@ -128,7 +128,6 @@ jobs:
         cmd: yq '.status.properties[] | select(.name == "instance-id") | .value' /github/workspace/.cache/holodeck.yaml
 
     - name: print holodeck cache file
-      uses: mikefarah/yq@master
       run: |
         cat /github/workspace/.cache/holodeck.yaml
 
@@ -178,6 +177,7 @@ jobs:
               rc=$status
               exit 1
           fi
+          # sleep 120
           ./tests/ci-run-e2e.sh "${TEST_CASE}" "${DRIVER_VERSION}" "${OPERATOR_OPTIONS}" || status=$?
           if [ $status -eq 1 ]; then
             echo "e2e validation failed for driver version $driver_version with status $status"

tests/scripts/.definitions.sh

@@ -25,6 +25,5 @@ CASES_DIR="$( cd "${TEST_DIR}/cases" && pwd )"
 
 : ${OPERATOR_OPTIONS:="--set driver.repository=ghcr.io/nvidia"}
 : ${AWS_SYSTEM_ONLINE_CHECK_TIMEOUT:="900"}
-: ${AWS_SYSTEM_RUNNING_STATUS_TIMEOUT:="60"}
 
 : ${BASE_TARGET:="jammy"}

tests/scripts/remote_retry.sh

@@ -1,17 +1,33 @@
 #!/bin/bash
 
 source ${SCRIPT_DIR}/.definitions.sh
-
-echo "SHIVA  ${AWS_SYSTEM_ONLINE_CHECK_TIMEOUT} ${AWS_SYSTEM_RUNNING_STATUS_TIMEOUT}" 
+START_TIME=$(date +%s)
+
 timeout "${AWS_SYSTEM_ONLINE_CHECK_TIMEOUT}" aws ec2 wait instance-status-ok --instance-ids "${instance_id}"
 if [ $? -ne 0 ]; then
     echo "AWS Instance did not become available within ${AWS_SYSTEM_ONLINE_CHECK_TIMEOUT} seconds."
     exit 1
 fi
-timeout "${AWS_SYSTEM_RUNNING_STATUS_TIMEOUT}" aws ec2 describe-instances --instance-ids "${instance_id}" --query 'Reservations[0].Instances[0].State.Name' --output text
+timeout "${AWS_SYSTEM_ONLINE_CHECK_TIMEOUT}" aws ec2 describe-instances --instance-ids "${instance_id}" --query 'Reservations[0].Instances[0].State.Name' --output text
 STATE=$(aws ec2 describe-instances --instance-ids "${instance_id}" --query 'Reservations[0].Instances[0].State.Name' --output text)
 if [ "$STATE" != "running" ]; then
-    echo "AWS Instance is not in running condition"
+    echo "AWS Instance is not in running condition within ${AWS_SYSTEM_ONLINE_CHECK_TIMEOUT} seconds."
     exit 1
 fi
-exit 0
+
+while true; do
+    INSTANCE_STATUS=$(aws ec2 describe-instance-status --instance-ids "${instance_id}" --query 'InstanceStatuses[0].InstanceStatus.Status' --output text)
+    SYSTEM_STATUS=$(aws ec2 describe-instance-status --instance-ids "${instance_id}" --query 'InstanceStatuses[0].SystemStatus.Status' --output text)
+    if [ "$INSTANCE_STATUS" == "ok" ] && [ "$SYSTEM_STATUS" == "ok" ]; then
+        echo "AWS Instance and System Status: ok"
+        exit 0
+    fi
+    ELAPSED_TIME=$(($(date +%s) - START_TIME))
+    if [ "$ELAPSED_TIME" -ge "$AWS_SYSTEM_ONLINE_CHECK_TIMEOUT" ]; then
+        echo "Failed to connect to aws within ${AWS_SYSTEM_ONLINE_CHECK_TIMEOUT} seconds after reboot."
+        exit 1
+    fi
+    sleep 10
+done
+
+exit 1