upload method for stagehand

examples/upload_test.ts

@@ -0,0 +1,103 @@
+// Import directly from local dist to ensure latest build is used
+import { Stagehand } from "../dist";
+import type { Page as PlaywrightPage } from "playwright";
+import StagehandConfig from "../stagehand.config";
+
+// Load environment variables
+import dotenv from "dotenv";
+dotenv.config({ path: "../.env" });
+
+async function main() {
+  // Accept file URL as command line argument or use default
+  const fileUrl = process.argv[2] || "https://www.orimi.com/pdf-test.pdf";
+  const targetPage =
+    process.argv[3] || "https://ps.uci.edu/~franklin/doc/file_upload.html";
+
+  console.log(`File URL: ${fileUrl}`);
+  console.log(`Target page: ${targetPage}`);
+
+  const stagehand = new Stagehand({
+    ...StagehandConfig,
+    env: "BROWSERBASE",
+    verbose: 1,
+    modelName: "openai/gpt-4o-mini",
+  });
+  await stagehand.init();
+  const page = stagehand.page;
+
+  try {
+    // Navigate to the target page
+    await page.goto(targetPage, {
+      waitUntil: "domcontentloaded",
+    });
+
+    // Debug: check presence of file inputs before calling upload
+    const count = await page.locator('input[type="file"]').count();
+    console.log("file input count:", count);
+
+    // Debug: log accessibility tree (full)
+    try {
+      const ax = await page.evaluate(() => {
+        if (typeof window.getComputedStyle !== 'undefined') {
+          return document.querySelector('body')?.innerHTML || 'No body content';
+        }
+        return 'Accessibility snapshot not available';
+      });
+      console.log("Page content:");
+      console.log(ax);
+    } catch (e) {
+      console.log("Failed to get page content:", e);
+    }
+
+    // Upload using the new helper - let observe find the right input
+    // Now we can pass the URL directly since upload() handles URLs
+    const result = await stagehand.upload("Upload this file", fileUrl);
+    console.log("upload result:", result);
+
+    // Try to submit the form using observe to find the submit button
+    try {
+      const [submitAction] = await page.observe(
+        "Find and click the submit or send button",
+      );
+      if (submitAction?.selector) {
+        console.log(
+          `Found submit button with selector: ${submitAction.selector}`,
+        );
+
+        // Avoid mixed-content warning by upgrading http action → https when possible
+        try {
+          await page.evaluate(() => {
+            const form = document.querySelector(
+              "form",
+            ) as HTMLFormElement | null;
+            if (
+              form &&
+              typeof form.action === "string" &&
+              form.action.startsWith("http://")
+            ) {
+              form.action = form.action.replace("http://", "https://");
+            }
+          });
+        } catch {
+          // ignore non-fatal submit upgrade errors
+        }
+
+        await page.act(submitAction);
+        console.log("Form submitted successfully");
+      } else {
+        console.log("No submit button found via observe");
+      }
+    } catch (e) {
+      console.log("Failed to find submit button via observe:", e);
+    }
+
+    await page.waitForTimeout(1500);
+  } finally {
+    await stagehand.close();
+  }
+}
+
+main().catch((err) => {
+  console.error(err);
+  process.exit(1);
+});

lib/index.ts

@@ -1,5 +1,5 @@
 import { Browserbase } from "@browserbasehq/sdk";
-import { Browser, chromium } from "playwright";
+import { Browser, chromium, FileChooser } from "playwright";
 import dotenv from "dotenv";
 import fs from "fs";
 import os from "os";
@@ -20,6 +20,8 @@ import {
   ActOptions,
   ExtractOptions,
   ObserveOptions,
+  FileSpec,
+  UploadResult,
 } from "../types/stagehand";
 import { StagehandContext } from "./StagehandContext";
 import { StagehandPage } from "./StagehandPage";
@@ -34,6 +36,10 @@ import { AgentExecuteOptions, AgentResult } from "../types/agent";
 import { StagehandAgentHandler } from "./handlers/agentHandler";
 import { StagehandOperatorHandler } from "./handlers/operatorHandler";
 import { StagehandLogger } from "./logger";
+import {
+  deepLocator,
+  deepLocatorWithShadow,
+} from "./handlers/handlerUtils/actHandlerUtils";
 
 import {
   StagehandError,
@@ -892,6 +898,7 @@ export class Stagehand {
       | ExtractOptions<z.AnyZodObject>
       | ObserveOptions
       | { url: string; options: GotoOptions }
+      | { hint: string; file: FileSpec }
       | string,
     result?: unknown,
   ): void {
@@ -903,6 +910,218 @@ export class Stagehand {
     });
   }
 
+  /**
+   * Upload a file to an upload control identified by a natural language hint.
+   * This method will attempt, in order:
+   *  1) Directly set files on a matching <input type="file"> (visible or hidden).
+   *  2) Trigger a file chooser by clicking the hinted control, then set files.
+   *  3) Heuristically locate an associated file input near/within the hinted element.
+   */
+  public async upload(hint: string, file: FileSpec): Promise<UploadResult> {
+    const page = this.stagehandPage.page;
+
+    const toSetInputArg = async (f: FileSpec) => {
+      if (typeof f === "string") {
+        // If it's a URL, download and return a Buffer payload
+        if (/^https?:\/\//i.test(f)) {
+          const res = await fetch(f);
+          if (!res.ok) {
+            throw new StagehandError(
+              `Failed to download file: ${res.status} ${res.statusText}`,
+            );
+          }
+          const mimeType =
+            res.headers.get("content-type") || "application/octet-stream";
+          const urlPath = new URL(f).pathname;
+          const name = urlPath.split("/").pop() || "uploaded_file";
+          const arrayBuf = await res.arrayBuffer();
+          return {
+            name,
+            mimeType,
+            buffer: Buffer.from(arrayBuf),
+          } as { name: string; mimeType: string; buffer: Buffer };
+        }
+        // Otherwise treat as a local path if provided as string (kept for compatibility)
+        return f;
+      }
+      if (f?.path) return f.path;
+      if (f?.buffer && f?.name && f?.mimeType) {
+        return { name: f.name, mimeType: f.mimeType, buffer: f.buffer } as {
+          name: string;
+          mimeType: string;
+          buffer: Buffer;
+        };
+      }
+      throw new StagehandError(
+        "Invalid FileSpec. Provide an http(s) URL, a path, or { buffer, name, mimeType }",
+      );
+    };
+
+    const filesArg = await toSetInputArg(file);
+
+    const finish = async (result: UploadResult): Promise<UploadResult> => {
+      this.addToHistory("upload", { hint, file }, result);
+      return result;
+    };
+
+    // Use NL→selector to locate the upload control strictly
+    try {
+      const [candidate] = await page.observe(
+        "Find the file upload control or input for: " + String(hint),
+      );
+      if (candidate?.selector) {
+        const raw = candidate.selector.replace(/^xpath=/i, "").trim();
+        const locator = this.experimental
+          ? await deepLocatorWithShadow(page, raw)
+          : deepLocator(page, raw);
+
+        // If this is a file input → set directly
+        const isFileInput = await locator
+          .evaluate(
+            (el): boolean => {
+              const tagName = el.tagName.toLowerCase();
+              const type = (el as HTMLInputElement).type;
+              console.log(`DEBUG: Element tagName=${tagName}, type=${type}`);
+              return tagName === "input" && type === "file";
+            },
+          )
+          .catch((e) => {
+            console.log(`DEBUG: evaluate failed:`, e);
+            return false;
+          });
+
+        if (isFileInput) {
+          await locator.setInputFiles(filesArg);
+          await this.stagehandPage._waitForSettledDom();
+          return finish({
+            success: true,
+            method: "input",
+            hint,
+            selector: candidate.selector,
+            fileName:
+              typeof file === "string" && /^https?:/i.test(file)
+                ? new URL(file).pathname.split("/").pop() || undefined
+                : typeof file === "string"
+                  ? file.split("/").pop()
+                  : file?.name,
+            message: "File attached via direct input",
+          });
+        }
+
+        // Otherwise attempt to trigger a file chooser via the hinted control
+        try {
+          const chooserPromise = page.waitForEvent("filechooser", {
+            timeout: 8000,
+          });
+          await locator.click({ timeout: 3000 }).catch((): void => {});
+          const chooser = await chooserPromise.catch(
+            (): FileChooser | undefined => undefined,
+          );
+          if (chooser) {
+            await chooser.setFiles(filesArg);
+            await this.stagehandPage._waitForSettledDom();
+            return finish({
+              success: true,
+              method: "chooser",
+              hint,
+              selector: candidate.selector,
+              fileName:
+                typeof file === "string" && /^https?:/i.test(file)
+                  ? new URL(file).pathname.split("/").pop() || undefined
+                  : typeof file === "string"
+                    ? file.split("/").pop()
+                    : file?.name,
+              message: "File attached via file chooser",
+            });
+          }
+        } catch {
+          void 0;
+        }
+
+        // Heuristic: find an associated file input near/within the hinted control
+        try {
+          const elementHandle = await locator.elementHandle();
+          if (elementHandle) {
+            const inputHandle = await elementHandle.evaluateHandle(
+              (el: Element): HTMLInputElement | null => {
+                const doc: Document = el.ownerDocument || document;
+                const asInput = (n: Element | null): HTMLInputElement | null =>
+                  n &&
+                  n.tagName.toLowerCase() === "input" &&
+                  (n as HTMLInputElement).type === "file"
+                    ? (n as HTMLInputElement)
+                    : null;
+
+                // 1) Self
+                const self = asInput(el as Element);
+                if (self) return self;
+
+                // 2) Label association
+                const label = (el as HTMLElement).closest("label");
+                if (label?.htmlFor) {
+                  const byId = doc.getElementById(label.htmlFor);
+                  const a = asInput(byId);
+                  if (a) return a;
+                }
+
+                // 3) Descendants
+                const desc = (el as Element).querySelector(
+                  'input[type="file"]',
+                );
+                const aDesc = asInput(desc);
+                if (aDesc) return aDesc;
+
+                // 4) Up to 5 ancestors, then search within each ancestor subtree
+                let cur: Element | null = (el as Element).parentElement;
+                for (let i = 0; i < 5 && cur; i++) {
+                  const within = cur.querySelector('input[type="file"]');
+                  const a = asInput(within);
+                  if (a) return a;
+                  cur = cur.parentElement;
+                }
+                return null;
+              },
+              undefined as unknown,
+            );
+
+            const el = inputHandle.asElement?.();
+            if (el) {
+              const fileEl = el as unknown as {
+                setInputFiles: (files: unknown) => Promise<void>;
+              };
+              await fileEl.setInputFiles(filesArg);
+              await this.stagehandPage._waitForSettledDom();
+              return finish({
+                success: true,
+                method: "fallback",
+                hint,
+                selector: candidate.selector,
+                fileName:
+                  typeof file === "string" && /^https?:/i.test(file)
+                    ? new URL(file).pathname.split("/").pop() || undefined
+                    : typeof file === "string"
+                      ? file.split("/").pop()
+                      : file?.name,
+                message: "File attached via heuristic input lookup",
+              });
+            }
+          }
+        } catch {
+          void 0;
+        }
+      }
+    } catch {
+      void 0;
+    }
+
+    return finish({
+      success: false,
+      method: "fallback",
+      hint,
+      message: "Could not locate a file input or trigger a chooser via observe",
+    });
+  }
+
   /**
    * Create an agent instance that can be executed with different instructions
    * @returns An agent instance with execute() method
@@ -1002,5 +1221,6 @@ export * from "../types/stagehand";
 export * from "../types/operator";
 export * from "../types/agent";
 export * from "./llm/LLMClient";
+export * from "./llm/LLMProvider";
 export * from "../types/stagehandErrors";
 export * from "../types/stagehandApiErrors";

lib/package.json

@@ -3,9 +3,9 @@
   "version": "2.4.1",
   "private": true,
   "description": "Core Stagehand library sources",
-  "main": "../dist/index.js",
-  "module": "../dist/index.js",
-  "types": "../dist/index.d.ts",
+  "main": "./index.js",
+  "module": "./index.js",
+  "types": "./index.d.ts",
   "scripts": {
     "build-dom-scripts": "tsx dom/genDomScripts.ts",
     "build-js": "tsup index.ts --dts",