Add smoketest for bpt secrets and gracefully shutdown devboxes (#656)

* Add smoketest for bpt secrets and gracefully shutdown devboxes when test fails

* format

* Disable secrets test if local dev

* fix tests

* extend test timeouts

Author: alb-rl

tests/smoketests/test_blueprints.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import os
 from typing import Iterator
 
 import pytest
@@ -31,7 +32,7 @@ def _cleanup(client: Runloop) -> Iterator[None]:  # pyright: ignore[reportUnused
 _blueprint_name = unique_name("bp")
 
 
-@pytest.mark.timeout(30)
+@pytest.mark.timeout(120)  # 2 minutes
 def test_create_blueprint_and_await_build(client: Runloop) -> None:
     global _blueprint_id
     created = client.blueprints.create_and_await_build_complete(
@@ -42,24 +43,65 @@ def test_create_blueprint_and_await_build(client: Runloop) -> None:
     _blueprint_id = created.id
 
 
-@pytest.mark.timeout(30)
+@pytest.mark.timeout(120)
 def test_start_devbox_from_base_blueprint_by_id(client: Runloop) -> None:
     assert _blueprint_id
-    devbox = client.devboxes.create_and_await_running(
-        blueprint_id=_blueprint_id,
-        polling_config=PollingConfig(max_attempts=120, interval_seconds=5.0, timeout_seconds=20 * 60),
-    )
-    assert devbox.blueprint_id == _blueprint_id
-    assert devbox.status == "running"
-    client.devboxes.shutdown(devbox.id)
+    devbox = None
+    try:
+        devbox = client.devboxes.create_and_await_running(
+            blueprint_id=_blueprint_id,
+            polling_config=PollingConfig(max_attempts=120, interval_seconds=5.0, timeout_seconds=20 * 60),
+        )
+        assert devbox.blueprint_id == _blueprint_id
+        assert devbox.status == "running"
+    finally:
+        if devbox:
+            client.devboxes.shutdown(devbox.id)
 
 
-@pytest.mark.timeout(30)
+@pytest.mark.timeout(120)
 def test_start_devbox_from_base_blueprint_by_name(client: Runloop) -> None:
-    devbox = client.devboxes.create_and_await_running(
-        blueprint_name=_blueprint_name,
-        polling_config=PollingConfig(max_attempts=120, interval_seconds=5.0, timeout_seconds=20 * 60),
-    )
-    assert devbox.blueprint_id
-    assert devbox.status == "running"
-    client.devboxes.shutdown(devbox.id)
+    devbox = None
+    try:
+        devbox = client.devboxes.create_and_await_running(
+            blueprint_name=_blueprint_name,
+            polling_config=PollingConfig(max_attempts=120, interval_seconds=5.0, timeout_seconds=20 * 60),
+        )
+        assert devbox.blueprint_id
+        assert devbox.status == "running"
+    finally:
+        if devbox:
+            client.devboxes.shutdown(devbox.id)
+
+
+@pytest.mark.timeout(120)
+@pytest.mark.skipif(
+    os.getenv("RUN_SMOKETESTS") != "1",
+    reason="Skip blueprint secrets test in local testing (requires RUN_SMOKETESTS=1)",
+)
+def test_create_blueprint_with_secret_and_await_build(client: Runloop) -> None:
+    bpt = None
+    try:
+        bpt = client.blueprints.create(
+            name=unique_name("bp-secrets"),
+            dockerfile=(
+                "FROM runloop:runloop/starter-arm64\n"
+                "ARG GITHUB_TOKEN\n"
+                'RUN git config --global credential.helper \'!f() { echo "username=x-access-token"; echo "password=$GITHUB_TOKEN"; }; f\' '
+                "&& git clone https://github.com/runloopai/runloop-fe.git /workspace/runloop-fe "
+                "&& git config --global --unset credential.helper\n"
+                "WORKDIR /workspace/runloop-fe"
+            ),
+            secrets={"GITHUB_TOKEN": "GITHUB_TOKEN_FOR_SMOKETESTS"},
+        )
+
+        completed = client.blueprints.await_build_complete(
+            bpt.id,
+            polling_config=PollingConfig(max_attempts=180, interval_seconds=5.0, timeout_seconds=30 * 60),
+        )
+        assert completed.status == "build_complete"
+        assert completed.parameters.secrets is not None
+        assert completed.parameters.secrets.get("GITHUB_TOKEN") == "GITHUB_TOKEN_FOR_SMOKETESTS"
+    finally:
+        if bpt:
+            client.blueprints.delete(bpt.id)

tests/smoketests/test_devboxes.py

@@ -37,7 +37,7 @@ def test_create_devbox(client: Runloop) -> None:
     client.devboxes.shutdown(created.id)
 
 
-@pytest.mark.timeout(30)
+@pytest.mark.timeout(120)
 def test_await_running_create_and_await_running(client: Runloop) -> None:
     global _devbox_id
     created = client.devboxes.create_and_await_running(
@@ -67,7 +67,7 @@ def test_shutdown_devbox(client: Runloop) -> None:
     assert view.status == "shutdown"
 
 
-@pytest.mark.timeout(90)
+@pytest.mark.timeout(120)
 def test_create_and_await_running_long_set_up(client: Runloop) -> None:
     created = client.devboxes.create_and_await_running(
         name=unique_name("smoketest-devbox-await-running-long-set-up"),

tests/smoketests/test_snapshots.py

@@ -45,12 +45,16 @@ def test_snapshot_devbox(client: Runloop) -> None:
     _snapshot_id = snap.id
 
 
-@pytest.mark.timeout(30)
+@pytest.mark.timeout(120)
 def test_launch_devbox_from_snapshot(client: Runloop) -> None:
     assert _snapshot_id
-    launched = client.devboxes.create_and_await_running(
-        snapshot_id=_snapshot_id,
-        polling_config=PollingConfig(max_attempts=120, interval_seconds=5.0, timeout_seconds=20 * 60),
-    )
-    assert launched.snapshot_id == _snapshot_id
-    client.devboxes.shutdown(launched.id)
+    launched = None
+    try:
+        launched = client.devboxes.create_and_await_running(
+            snapshot_id=_snapshot_id,
+            polling_config=PollingConfig(max_attempts=120, interval_seconds=5.0, timeout_seconds=20 * 60),
+        )
+        assert launched.snapshot_id == _snapshot_id
+    finally:
+        if launched:
+            client.devboxes.shutdown(launched.id)