diff --git a/lib/plugins.js b/lib/plugins.js index 0a8f2bd..c8416c3 100644 --- a/lib/plugins.js +++ b/lib/plugins.js @@ -60,7 +60,7 @@ import { EventEmitter } from 'events'; import fs from 'fs'; import path from 'path'; -import { fileURLToPath } from 'url'; +import { fileURLToPath, pathToFileURL } from 'url'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); const ROOT_DIR = path.join(__dirname, '..'); @@ -155,7 +155,7 @@ export async function loadPlugins(app, ctx) { } try { - const mod = await import(indexPath); + const mod = await import(pathToFileURL(indexPath).href); const register = mod.default || mod.register; if (typeof register !== 'function') { ctx.log('warn', `plugin "${name}" does not export a register function, skipping`); diff --git a/openapi.json b/openapi.json index a40b78d..4fe62a5 100644 --- a/openapi.json +++ b/openapi.json @@ -849,11 +849,20 @@ "type": "string", "description": "CSS selector fallback." }, + "x": { + "type": "number", + "description": "Viewport x coordinate for raw mouse clicks." + }, + "y": { + "type": "number", + "description": "Viewport y coordinate for raw mouse clicks." + }, "doubleClick": { "type": "boolean" }, "coordinates": { "type": "object", + "description": "Viewport coordinates for raw mouse clicks.", "properties": { "x": { "type": "number" diff --git a/plugin.js b/plugin.js index 57bfdad..5001ace 100644 --- a/plugin.js +++ b/plugin.js @@ -165,13 +165,16 @@ export default function register(api) { })); api.registerTool((ctx) => ({ name: "camofox_click", - description: "Click an element in a Camoufox tab by ref (e.g., e1) or CSS selector.", + description: "Click an element in a Camoufox tab by ref, CSS selector, or viewport coordinates.", parameters: { type: "object", properties: { tabId: { type: "string", description: "Tab identifier" }, ref: { type: "string", description: "Element ref from snapshot (e.g., e1)" }, selector: { type: "string", description: "CSS selector (alternative to ref)" }, + x: { type: "number", description: "Viewport x coordinate for raw mouse clicks" }, + y: { type: "number", description: "Viewport y coordinate for raw mouse clicks" }, + doubleClick: { type: "boolean", description: "Click twice instead of once" }, }, required: ["tabId"], }, diff --git a/plugin.ts b/plugin.ts index bd081d4..3eee1a4 100644 --- a/plugin.ts +++ b/plugin.ts @@ -268,13 +268,16 @@ export default function register(api: PluginApi) { api.registerTool((ctx: ToolContext) => ({ name: "camofox_click", - description: "Click an element in a Camoufox tab by ref (e.g., e1) or CSS selector.", + description: "Click an element in a Camoufox tab by ref, CSS selector, or viewport coordinates.", parameters: { type: "object", properties: { tabId: { type: "string", description: "Tab identifier" }, ref: { type: "string", description: "Element ref from snapshot (e.g., e1)" }, selector: { type: "string", description: "CSS selector (alternative to ref)" }, + x: { type: "number", description: "Viewport x coordinate for raw mouse clicks" }, + y: { type: "number", description: "Viewport y coordinate for raw mouse clicks" }, + doubleClick: { type: "boolean", description: "Click twice instead of once" }, }, required: ["tabId"], }, diff --git a/server.js b/server.js index 4083899..66de18a 100644 --- a/server.js +++ b/server.js @@ -3161,10 +3161,17 @@ app.post('/tabs/:tabId/wait', async (req, res) => { * selector: * type: string * description: CSS selector fallback. + * x: + * type: number + * description: Viewport x coordinate for raw mouse clicks. + * y: + * type: number + * description: Viewport y coordinate for raw mouse clicks. * doubleClick: * type: boolean * coordinates: * type: object + * description: Viewport coordinates for raw mouse clicks. * properties: * x: * type: number @@ -3194,8 +3201,18 @@ app.post('/tabs/:tabId/click', async (req, res) => { const tabId = req.params.tabId; try { - const { userId, ref, selector } = req.body; + const { userId, ref, selector, doubleClick } = req.body; if (!userId) return res.status(400).json({ error: 'userId required' }); + const rawCoordinates = req.body.coordinates ?? ((req.body.x !== undefined || req.body.y !== undefined) ? { x: req.body.x, y: req.body.y } : null); + const useCoordinates = !ref && !selector && rawCoordinates; + let coordinates = null; + if (useCoordinates) { + const { x, y } = rawCoordinates; + if (typeof x !== 'number' || typeof y !== 'number' || !Number.isFinite(x) || !Number.isFinite(y)) { + return res.status(400).json({ error: 'coordinates require finite numeric x and y' }); + } + coordinates = { x, y }; + } const session = sessions.get(normalizeUserId(userId)); const found = session && findTab(session, tabId); if (!found) return tabNotFoundResponse(res, req.params.tabId || req.body?.tabId); @@ -3203,8 +3220,8 @@ app.post('/tabs/:tabId/click', async (req, res) => { const { tabState } = found; tabState.toolCalls++; tabState.consecutiveTimeouts = 0; tabState.consecutiveFailures = 0; - if (!ref && !selector) { - return res.status(400).json({ error: 'ref or selector required' }); + if (!ref && !selector && !coordinates) { + return res.status(400).json({ error: 'ref, selector, or coordinates required' }); } const result = await withUserLimit(userId, () => withTabLock(tabId, async () => { @@ -3294,8 +3311,11 @@ app.post('/tabs/:tabId/click', async (req, res) => { throw new StaleRefsError(ref, maxRef, tabState.refs.size); } await doClick(locator, true); - } else { + } else if (selector) { await doClick(selector, false); + } else { + await tabState.page.mouse.click(coordinates.x, coordinates.y, { clickCount: doubleClick ? 2 : 1 }); + log('info', 'coordinate click dispatched', { x: coordinates.x.toFixed(0), y: coordinates.y.toFixed(0), doubleClick: !!doubleClick }); } // If clicking on a Google SERP, wait for potential navigation to complete @@ -3335,7 +3355,7 @@ app.post('/tabs/:tabId/click', async (req, res) => { })); log('info', 'clicked', { reqId: req.reqId, tabId, url: result.url }); - pluginEvents.emit('tab:click', { userId: req.body.userId, tabId, ref: req.body.ref, selector: req.body.selector }); + pluginEvents.emit('tab:click', { userId: req.body.userId, tabId, ref: req.body.ref, selector: req.body.selector, coordinates: req.body.coordinates }); res.json(result); } catch (err) { log('error', 'click failed', { reqId: req.reqId, tabId, error: err.message }); diff --git a/tests/e2e/formSubmission.test.js b/tests/e2e/formSubmission.test.js index 762d4d0..970f7dd 100644 --- a/tests/e2e/formSubmission.test.js +++ b/tests/e2e/formSubmission.test.js @@ -73,6 +73,28 @@ describe('Form Submission', () => { await client.cleanup(); } }); + + test('click button by viewport coordinates', async () => { + const client = createClient(serverUrl); + + try { + const { tabId } = await client.createTab(`${testSiteUrl}/click`); + const { result } = await client.request('POST', `/tabs/${tabId}/evaluate`, { + userId: client.userId, + expression: `(() => { + const rect = document.getElementById('clickMe').getBoundingClientRect(); + return { x: rect.left + rect.width / 2, y: rect.top + rect.height / 2 }; + })()`, + }); + + await client.click(tabId, { x: result.x, y: result.y }); + + const snapshot = await client.waitForSnapshotContains(tabId, 'Button was clicked!'); + expect(snapshot.snapshot).toContain('Button was clicked!'); + } finally { + await client.cleanup(); + } + }); test('click using ref', async () => { const client = createClient(serverUrl); diff --git a/tests/unit/openapi.test.js b/tests/unit/openapi.test.js index b12828f..31a21fd 100644 --- a/tests/unit/openapi.test.js +++ b/tests/unit/openapi.test.js @@ -137,6 +137,16 @@ describe('OpenAPI spec', () => { expect(createTab.requestBody.content['application/json']).toBeDefined(); }); + test('click route documents viewport coordinate clicks', () => { + const click = spec.paths['/tabs/{tabId}/click']?.post; + const properties = click.requestBody.content['application/json'].schema.properties; + + expect(properties.x.type).toBe('number'); + expect(properties.y.type).toBe('number'); + expect(properties.coordinates.properties.x.type).toBe('number'); + expect(properties.coordinates.properties.y.type).toBe('number'); + }); + test('legacy routes are marked deprecated', () => { const legacyPaths = { '/act': 'post',