Update browser to allow recording session videos

Author: ajhai

llmstack/processors/providers/promptly/static_web_browser.py

@@ -83,6 +83,10 @@ class StaticWebBrowserConfiguration(ApiProcessorSchema):
         description="Capture screenshot",
         default=True,
     )
+    capture_session_video: bool = Field(
+        description="Capture session video",
+        default=False,
+    )
 
 
 class StaticWebBrowserInput(ApiProcessorSchema):
@@ -138,6 +142,10 @@ class StaticWebBrowserOutput(ApiProcessorSchema):
         default=[],
         description="Steps taken to complete the task",
     )
+    videos: Optional[List[StaticWebBrowserFile]] = Field(
+        default=None,
+        description="Videos from the browser session. Make sure to TERMINATE the browser to capture the video",
+    )
 
 
 class StaticWebBrowser(
@@ -167,8 +175,20 @@ def provider_slug() -> str:
     def get_output_template(cls) -> Optional[OutputTemplate]:
         return OutputTemplate(
             markdown="""
+{% if session %}
+### Live Browser Session
 <promptly-web-browser-embed wsUrl="{{session.ws_url}}"></promptly-web-browser-embed>
+{% endif %}
+{% if videos %}
+### Videos
+{% for video in videos %}
+<pa-asset url="{{video.data}}" type="{{video.mime_type}}"></pa-asset>
+{% endfor %}
+{% endif %}
+{% if content.screenshot %}
+### Screenshot
 <pa-asset url="{{content.screenshot}}" type="image/png"></pa-asset>
+{% endif %}
 {{text}}
 """,
             jsonpath="$.text",
@@ -204,7 +224,7 @@ def _web_browser_instruction_to_command(self, instruction: BrowserInstruction) -
     def process(self) -> dict:
         output_stream = self._output_stream
         browser_response = None
-
+        session_videos = []
         with WebBrowser(
             f"{settings.RUNNER_HOST}:{settings.RUNNER_PORT}",
             interactive=self._config.stream_video,
@@ -216,6 +236,7 @@ def process(self) -> dict:
                 if self._config.connection_id
                 else ""
             ),
+            record_video=self._config.capture_session_video,
         ) as web_browser:
             if self._config.stream_video and web_browser.get_wss_url():
                 async_to_sync(
@@ -237,6 +258,10 @@ def process(self) -> dict:
                 ]
                 + list(map(self._web_browser_instruction_to_command, self._input.instructions))
             )
+
+            if self._config.capture_session_video:
+                session_videos = web_browser.get_videos()
+
         if browser_response:
             output_text = browser_response.text or "\n".join(
                 list(map(lambda entry: entry.output, browser_response.command_outputs))
@@ -273,6 +298,21 @@ def process(self) -> dict:
                 browser_content.downloads = swb_downloads
             async_to_sync(self._output_stream.write)(StaticWebBrowserOutput(text=output_text, content=browser_content))
 
+        if session_videos:
+            encoded_videos = []
+            for video in session_videos:
+                encoded_videos.append(
+                    StaticWebBrowserFile(
+                        name=video.name,
+                        data=self._upload_asset_from_url(
+                            f"data:{video.mime_type};name={video.name};base64,{base64.b64encode(video.data).decode('utf-8')}",
+                            mime_type=video.mime_type,
+                        ).objref,
+                        mime_type=video.mime_type,
+                    )
+                )
+            async_to_sync(self._output_stream.write)(StaticWebBrowserOutput(videos=encoded_videos))
+
         output = output_stream.finalize()
 
         return output

llmstack/processors/providers/promptly/web_browser.py

@@ -264,8 +264,8 @@ class WebBrowserConfiguration(ApiProcessorSchema):
         description="Stream video of the browser",
         default=True,
     )
-    stream_text: bool = Field(
-        description="Stream output text from the browser",
+    record_session_video: bool = Field(
+        description="Record session video",
         default=False,
     )
     timeout: int = Field(
@@ -289,9 +289,7 @@ class WebBrowserConfiguration(ApiProcessorSchema):
         default=None,
         description="Seed to use for random number generator",
     )
-    tags_to_extract: List[str] = Field(
-        description="Tags to extract", default=["a", "button", "input", "textarea", "select"]
-    )
+    tags_to_extract: List[str] = Field(description="Tags to extract", default=[])
 
 
 class WebBrowserOutput(ApiProcessorSchema):
@@ -312,6 +310,10 @@ class WebBrowserOutput(ApiProcessorSchema):
         default=[],
         description="Steps taken to complete the task",
     )
+    videos: List[StaticWebBrowserFile] = Field(
+        default=[],
+        description="Videos from the session",
+    )
 
 
 class WebBrowserInput(ApiProcessorSchema):
@@ -351,8 +353,18 @@ def provider_slug() -> str:
     @classmethod
     def get_output_template(cls) -> Optional[OutputTemplate]:
         return OutputTemplate(
-            markdown="""<promptly-web-browser-embed wsUrl="{{session.ws_url}}"></promptly-web-browser-embed>
-        {{text}}""",
+            markdown="""
+{% if session %}
+### Live Browser Session
+<promptly-web-browser-embed wsUrl="{{session.ws_url}}"></promptly-web-browser-embed>
+{% endif %}
+{{text}}
+{% if videos %}
+### Session Videos
+{% for video in videos %}
+<pa-asset url="{{video.data}}" type="{{video.mime_type}}"></pa-asset>
+{% endfor %}
+{% endif %}""",
             jsonpath="$.text",
         )
 
@@ -428,6 +440,8 @@ def _process_anthropic(self) -> dict:
         commands_executed = 0
         browser_response = None
         browser_downloads = []
+        session_videos = []
+
         with WebBrowserClient(
             f"{settings.RUNNER_HOST}:{settings.RUNNER_PORT}",
             interactive=self._config.stream_video,
@@ -439,6 +453,7 @@ def _process_anthropic(self) -> dict:
                 if self._config.connection_id
                 else ""
             ),
+            record_video=self._config.record_session_video,
         ) as web_browser:
             browser_response = web_browser.run_commands(
                 commands=[
@@ -583,6 +598,8 @@ def _process_anthropic(self) -> dict:
                         WebBrowserOutput(text=choice.message.content[0].get("text", "")),
                     )
                     break
+            if self._config.record_session_video:
+                session_videos = web_browser.get_videos()
 
         if browser_response:
             output_text = "\n".join(list(map(lambda entry: entry.output, browser_response.command_outputs)))
@@ -616,6 +633,21 @@ def _process_anthropic(self) -> dict:
                 browser_content.downloads = swb_downloads
             async_to_sync(self._output_stream.write)(WebBrowserOutput(text=output_text, content=browser_content))
 
+        if session_videos:
+            encoded_videos = []
+            for video in session_videos:
+                encoded_videos.append(
+                    StaticWebBrowserFile(
+                        name=video.name,
+                        data=self._upload_asset_from_url(
+                            f"data:{video.mime_type};name={video.name};base64,{base64.b64encode(video.data).decode('utf-8')}",
+                            mime_type=video.mime_type,
+                        ).objref,
+                        mime_type=video.mime_type,
+                    )
+                )
+            async_to_sync(self._output_stream.write)(WebBrowserOutput(videos=encoded_videos))
+
         output = self._output_stream.finalize()
         return output
 

poetry.lock

@@ -22,7 +22,6 @@ python-on-whales = "^0.71.0"
 channels-redis = "^4.2.0"
 jsonpath-ng = "^1.6.1"
 cryptography = "^43.0.1"
-langrocks = "^0.1.5"
 grpcio = "1.65.1"
 
 [tool.poetry.extras]
@@ -69,7 +68,7 @@ google-auth = {version = "^2.22.0" }
 pydantic = "^2.7.4"
 daphne = "^4.1.2"
 striprtf = "^0.0.26"
-langrocks = "0.1.9"
+langrocks = "0.2.0"
 diff-match-patch = "^20230430"
 rq = "^2.0.0"
 django-rq = "^3.0.0"