From 617df3cd7f4a6be6877b5ae987a3c92f06dd78c4 Mon Sep 17 00:00:00 2001 From: Inziuhmci Date: Wed, 13 May 2026 17:10:00 +0800 Subject: [PATCH 1/4] Support coordinate click actions --- server.js | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/server.js b/server.js index 4083899..b4fbd57 100644 --- a/server.js +++ b/server.js @@ -3194,7 +3194,7 @@ app.post('/tabs/:tabId/click', async (req, res) => { const tabId = req.params.tabId; try { - const { userId, ref, selector } = req.body; + const { userId, ref, selector, coordinates } = req.body; if (!userId) return res.status(400).json({ error: 'userId required' }); const session = sessions.get(normalizeUserId(userId)); const found = session && findTab(session, tabId); @@ -3203,7 +3203,7 @@ app.post('/tabs/:tabId/click', async (req, res) => { const { tabState } = found; tabState.toolCalls++; tabState.consecutiveTimeouts = 0; tabState.consecutiveFailures = 0; - if (!ref && !selector) { + if (!ref && !selector && !coordinates) { return res.status(400).json({ error: 'ref or selector required' }); } @@ -3272,7 +3272,19 @@ app.post('/tabs/:tabId/click', async (req, res) => { } }; - if (ref) { + if (coordinates) { + const x = Number(coordinates.x); + const y = Number(coordinates.y); + if (!Number.isFinite(x) || !Number.isFinite(y)) { + return res.status(400).json({ error: 'valid coordinates required' }); + } + await tabState.page.mouse.move(x, y); + await tabState.page.waitForTimeout(50); + await tabState.page.mouse.down(); + await tabState.page.waitForTimeout(50); + await tabState.page.mouse.up(); + log('info', 'coordinate mouse sequence dispatched', { x: x.toFixed(0), y: y.toFixed(0) }); + } else if (ref) { let locator = refToLocator(tabState.page, ref, tabState.refs); if (!locator) { // Use tight timeout (4s max) to leave budget for click + post-click buildRefs From 3871fd939e99b9a31b5672cde700003a3cb7d54a Mon Sep 17 00:00:00 2001 From: Inziuhmci Date: Wed, 13 May 2026 17:51:10 +0800 Subject: [PATCH 2/4] Add Camofox file upload action --- server.js | 78 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/server.js b/server.js index b4fbd57..d02f999 100644 --- a/server.js +++ b/server.js @@ -3373,6 +3373,84 @@ app.post('/tabs/:tabId/click', async (req, res) => { } }); +// Upload file into an input[type=file] or trigger a file chooser from a ref/selector. +app.post('/tabs/:tabId/upload', async (req, res) => { + const tabId = req.params.tabId; + + try { + const { userId, ref, selector, filePath, files } = req.body; + if (!userId) return res.status(400).json({ error: 'userId required' }); + const fileList = Array.isArray(files) ? files : filePath ? [filePath] : []; + if (fileList.length === 0) return res.status(400).json({ error: 'filePath or files required' }); + const session = sessions.get(normalizeUserId(userId)); + const found = session && findTab(session, tabId); + if (!found) return tabNotFoundResponse(res, req.params.tabId || req.body?.tabId); + + const { tabState } = found; + tabState.toolCalls++; tabState.consecutiveTimeouts = 0; tabState.consecutiveFailures = 0; + + const result = await withUserLimit(userId, () => withTabLock(tabId, async () => { + let fileSet = false; + + if (ref || selector) { + let locator = null; + if (ref) { + locator = refToLocator(tabState.page, ref, tabState.refs); + if (!locator) { + tabState.refs = await refreshTabRefs(tabState, { reason: 'pre_upload' }); + locator = refToLocator(tabState.page, ref, tabState.refs); + } + if (!locator) { + const maxRef = tabState.refs.size > 0 ? `e${tabState.refs.size}` : 'none'; + throw new StaleRefsError(ref, maxRef, tabState.refs.size); + } + } else { + locator = tabState.page.locator(selector); + } + + const chooserPromise = tabState.page.waitForEvent('filechooser', { timeout: 5000 }).catch(() => null); + await locator.click({ timeout: 5000 }).catch(async (err) => { + if (err.message?.includes('intercepts pointer events') || err.message?.toLowerCase().includes('timeout')) { + await locator.click({ timeout: 5000, force: true }); + } else { + throw err; + } + }); + const chooser = await chooserPromise; + if (chooser) { + await chooser.setFiles(fileList); + fileSet = true; + } + } + + if (!fileSet) { + const inputs = tabState.page.locator('input[type="file"]'); + const count = await inputs.count().catch(() => 0); + if (count > 0) { + await inputs.nth(count - 1).setInputFiles(fileList); + fileSet = true; + } + } + + if (!fileSet) { + throw new Error('file input or file chooser not available'); + } + + await tabState.page.waitForTimeout(700); + tabState.lastSnapshot = null; + tabState.refs = new Map(); + return { ok: true, files: fileList.length, url: tabState.page.url() }; + })); + + log('info', 'uploaded file', { reqId: req.reqId, tabId, files: result.files, url: result.url }); + pluginEvents.emit('tab:upload', { userId: req.body.userId, tabId, ref: req.body.ref, selector: req.body.selector, files: result.files }); + res.json(result); + } catch (err) { + log('error', 'upload failed', { reqId: req.reqId, tabId, error: err.message }); + handleRouteError(err, req, res); + } +}); + // Type /** * @openapi From 7c2b477f8366b452b84628a3ef71e11e0f0854f2 Mon Sep 17 00:00:00 2001 From: Inziuhmci Date: Wed, 13 May 2026 17:54:33 +0800 Subject: [PATCH 3/4] Allow coordinate-triggered file uploads --- server.js | 47 +++++++++++++++++++++++++++++++++-------------- 1 file changed, 33 insertions(+), 14 deletions(-) diff --git a/server.js b/server.js index d02f999..961a0da 100644 --- a/server.js +++ b/server.js @@ -3378,7 +3378,7 @@ app.post('/tabs/:tabId/upload', async (req, res) => { const tabId = req.params.tabId; try { - const { userId, ref, selector, filePath, files } = req.body; + const { userId, ref, selector, coordinates, filePath, files } = req.body; if (!userId) return res.status(400).json({ error: 'userId required' }); const fileList = Array.isArray(files) ? files : filePath ? [filePath] : []; if (fileList.length === 0) return res.status(400).json({ error: 'filePath or files required' }); @@ -3392,9 +3392,26 @@ app.post('/tabs/:tabId/upload', async (req, res) => { const result = await withUserLimit(userId, () => withTabLock(tabId, async () => { let fileSet = false; - if (ref || selector) { + if (ref || selector || coordinates) { let locator = null; - if (ref) { + if (coordinates) { + const x = Number(coordinates.x); + const y = Number(coordinates.y); + if (!Number.isFinite(x) || !Number.isFinite(y)) { + return res.status(400).json({ error: 'valid coordinates required' }); + } + const chooserPromise = tabState.page.waitForEvent('filechooser', { timeout: 5000 }).catch(() => null); + await tabState.page.mouse.move(x, y); + await tabState.page.waitForTimeout(50); + await tabState.page.mouse.down(); + await tabState.page.waitForTimeout(50); + await tabState.page.mouse.up(); + const chooser = await chooserPromise; + if (chooser) { + await chooser.setFiles(fileList); + fileSet = true; + } + } else if (ref) { locator = refToLocator(tabState.page, ref, tabState.refs); if (!locator) { tabState.refs = await refreshTabRefs(tabState, { reason: 'pre_upload' }); @@ -3408,18 +3425,20 @@ app.post('/tabs/:tabId/upload', async (req, res) => { locator = tabState.page.locator(selector); } - const chooserPromise = tabState.page.waitForEvent('filechooser', { timeout: 5000 }).catch(() => null); - await locator.click({ timeout: 5000 }).catch(async (err) => { - if (err.message?.includes('intercepts pointer events') || err.message?.toLowerCase().includes('timeout')) { - await locator.click({ timeout: 5000, force: true }); - } else { - throw err; + if (locator) { + const chooserPromise = tabState.page.waitForEvent('filechooser', { timeout: 5000 }).catch(() => null); + await locator.click({ timeout: 5000 }).catch(async (err) => { + if (err.message?.includes('intercepts pointer events') || err.message?.toLowerCase().includes('timeout')) { + await locator.click({ timeout: 5000, force: true }); + } else { + throw err; + } + }); + const chooser = await chooserPromise; + if (chooser) { + await chooser.setFiles(fileList); + fileSet = true; } - }); - const chooser = await chooserPromise; - if (chooser) { - await chooser.setFiles(fileList); - fileSet = true; } } From cc225bdd6feed53239cdb028b0814088ad1517bc Mon Sep 17 00:00:00 2001 From: Inziuhmci Date: Wed, 13 May 2026 19:21:16 +0800 Subject: [PATCH 4/4] Document upload endpoint in OpenAPI --- openapi.json | 99 ++++++++++++++++++++++++++++++++++++++++++++++++++++ server.js | 64 +++++++++++++++++++++++++++++++++ 2 files changed, 163 insertions(+) diff --git a/openapi.json b/openapi.json index 9960486..e326b63 100644 --- a/openapi.json +++ b/openapi.json @@ -902,6 +902,105 @@ } } }, + "/tabs/{tabId}/upload": { + "post": { + "tags": [ + "Interaction" + ], + "summary": "Upload files through a file input or file chooser", + "description": "Sets files on an input[type=file] or triggers a file chooser from an element ref, CSS selector, or coordinates.", + "parameters": [ + { + "name": "tabId", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "userId" + ], + "properties": { + "userId": { + "type": "string" + }, + "filePath": { + "type": "string", + "description": "Single local file path to upload." + }, + "files": { + "type": "array", + "items": { + "type": "string" + }, + "description": "One or more local file paths to upload." + }, + "ref": { + "type": "string", + "description": "Element ref ID that opens a file chooser or identifies a file input." + }, + "selector": { + "type": "string", + "description": "CSS selector that opens a file chooser or identifies a file input." + }, + "coordinates": { + "type": "object", + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + } + } + } + } + }, + "responses": { + "200": { + "description": "Upload result.", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + }, + "400": { + "description": "Bad request.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + }, + "404": { + "description": "Tab not found.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + } + } + } + }, "/tabs/{tabId}/type": { "post": { "tags": [ diff --git a/server.js b/server.js index 961a0da..6f6b4dc 100644 --- a/server.js +++ b/server.js @@ -3374,6 +3374,70 @@ app.post('/tabs/:tabId/click', async (req, res) => { }); // Upload file into an input[type=file] or trigger a file chooser from a ref/selector. +/** + * @openapi + * /tabs/{tabId}/upload: + * post: + * tags: [Interaction] + * summary: Upload files through a file input or file chooser + * description: Sets files on an input[type=file] or triggers a file chooser from an element ref, CSS selector, or coordinates. + * parameters: + * - name: tabId + * in: path + * required: true + * schema: + * type: string + * requestBody: + * required: true + * content: + * application/json: + * schema: + * type: object + * required: [userId] + * properties: + * userId: + * type: string + * filePath: + * type: string + * description: Single local file path to upload. + * files: + * type: array + * items: + * type: string + * description: One or more local file paths to upload. + * ref: + * type: string + * description: Element ref ID that opens a file chooser or identifies a file input. + * selector: + * type: string + * description: CSS selector that opens a file chooser or identifies a file input. + * coordinates: + * type: object + * properties: + * x: + * type: number + * y: + * type: number + * responses: + * 200: + * description: Upload result. + * content: + * application/json: + * schema: + * type: object + * 400: + * description: Bad request. + * content: + * application/json: + * schema: + * $ref: '#/components/schemas/Error' + * 404: + * description: Tab not found. + * content: + * application/json: + * schema: + * $ref: '#/components/schemas/Error' + */ app.post('/tabs/:tabId/upload', async (req, res) => { const tabId = req.params.tabId;