Merge pull request #52 from linuxserver/51-feat-fail-tests-if-screens…

…hot-fails

If amd64 tag fails the screenshot test, mark CI.report_status with "FAIL".

ci/ci.py

@@ -326,7 +326,11 @@ def container_test(self, tag: str) -> None:
             return
 
         # Screenshot the web interface and check connectivity
-        self.take_screenshot(container, tag)
+        screenshot: bool = self.take_screenshot(container, tag)
+        if not screenshot and self.get_platform(tag) == "amd64": # Allow ARM tags to fail the screenshot test
+            self.logger.error("Test of %s FAILED after %.2f seconds", tag, time.time() - start_time)
+            self._endtest(container, tag, build_info, sbom, False, start_time)
+            return
 
         self._endtest(container, tag, build_info, sbom, True, start_time)
         self.logger.success("Test of %s PASSED after %.2f seconds", tag, time.time() - start_time)
@@ -723,17 +727,20 @@ def _add_test_result(self, tag:str, test:str, status:str, message:str, start_tim
             "message":message,
             "runtime": runtime}.items())))
 
-    def take_screenshot(self, container: Container, tag:str) -> None:
+    def take_screenshot(self, container: Container, tag:str) -> bool:
         """Take a screenshot and save it to self.outdir if self.screenshot is True
 
         Takes a screenshot using a ChromiumDriver instance.
 
         Args:
             container (Container): Container object
             tag (str): The container tag we are testing.
+        
+        Returns:
+            bool: Return True if the screenshot was successful, otherwise False.
         """
         if not self.screenshot:
-            return
+            return True
         proto: Literal["https", "http"] = "https" if self.ssl.upper() == "TRUE" else "http"
         screenshot_timeout = time.time() + self.screenshot_timeout
         test = "Get screenshot"
@@ -757,7 +764,7 @@ def take_screenshot(self, container: Container, tag:str) -> None:
                         raise FileNotFoundError(f"Screenshot '{self.outdir}/{tag}.png' not found")
                     self._add_test_result(tag, test, "PASS", "-", start_time)
                     self.logger.success("Screenshot %s: PASSED after %.2f seconds", tag, time.time() - start_time)
-                    return
+                    return True
                 except Exception as error:
                     logger.debug("Failed to take screenshot of %s at %s, trying again in 3 seconds", tag, endpoint, exc_info=error)
                     time.sleep(3)
@@ -768,12 +775,18 @@ def take_screenshot(self, container: Container, tag:str) -> None:
         except (requests.Timeout, requests.ConnectionError, KeyError) as error:
             self._add_test_result(tag, test, "FAIL", f"CONNECTION ERROR: {str(error)}", start_time)
             self.logger.exception("Screenshot %s FAIL CONNECTION ERROR", tag)
+            self.report_status = "FAIL"
+            return False
         except TimeoutException as error:
             self._add_test_result(tag, test, "FAIL", f"TIMEOUT: {str(error)}", start_time)
             self.logger.exception("Screenshot %s FAIL TIMEOUT", tag)
+            self.report_status = "FAIL"
+            return False
         except (WebDriverException, Exception) as error:
             self._add_test_result(tag, test, "FAIL", f"UNKNOWN: {str(error)}", start_time)
             self.logger.exception("Screenshot %s FAIL UNKNOWN", tag)
+            self.report_status = "FAIL"
+            return False
         finally:
             try:
                 driver.quit()

readme-vars.yml

@@ -53,7 +53,8 @@ full_custom_readme: |
   -e CI_S6_VERBOSITY=<optional, Updates the S6_VERBOSITY env. Defaults to '2'> \
   -e CI_LOG_LEVEL=<optional, Sets the ci logging level. Defaults to 'INFO'> \
   -e DOCKER_LOGS_TIMEOUT=<optional, How long to wait in seconds while tailing the container logs before timing out. Defaults to '120'> \
-  -e DRY_RUN=<optional, Set to 'true' when you don't want to upload files to S3 when testing>
+  -e DRY_RUN=<optional, Set to 'true' when you don't want to upload files to S3 when testing> \
+  -e NODE_NAME=<optional, Name of the builder that runs the CI test.> \
   -t lsiodev/ci:latest \
   python3 test_build.py
   ```