From 140466e94fc8be6c7f1e5d3d04cbdf1f9150b35b Mon Sep 17 00:00:00 2001 From: David Tejada Date: Tue, 23 Apr 2024 17:50:34 +0200 Subject: [PATCH 1/7] Add command for clicking an element by fuzzy searching its text --- .../messaging/sendRequestToContent.ts | 42 ++++++++++++++ src/content/actions/clickElement.ts | 58 +++++++++++++++++++ .../actions/runRangoActionWithoutTarget.ts | 10 +++- src/content/wrappers/ElementWrapperClass.ts | 26 ++++++++- src/typings/ElementWrapper.ts | 1 + src/typings/RangoAction.ts | 10 +++- 6 files changed, 144 insertions(+), 3 deletions(-) diff --git a/src/background/messaging/sendRequestToContent.ts b/src/background/messaging/sendRequestToContent.ts index 23f9ce81..d9b47028 100644 --- a/src/background/messaging/sendRequestToContent.ts +++ b/src/background/messaging/sendRequestToContent.ts @@ -60,6 +60,46 @@ async function handleActionOnReference( } } +async function clickElementByText(text: string) { + const tabId = await getCurrentTabId(); + const allFrames = await browser.webNavigation.getAllFrames({ + tabId, + }); + + const bestScoreByFramePromise = allFrames.map(async (frame) => ({ + frameId: frame.frameId, + score: (await browser.tabs.sendMessage( + tabId, + { type: "matchElementByText", arg: text }, + { + frameId: frame.frameId, + } + )) as number | undefined, + })); + + const results = await Promise.allSettled(bestScoreByFramePromise); + const matches = results + .filter(isPromiseFulfilledResult) + .map((result) => result.value) + .filter((value) => typeof value.score === "number"); + + const sorted = matches.sort((a, b) => (a.score ?? 0) - (b.score ?? 0)); + + if (sorted[0]) { + await browser.tabs.sendMessage( + tabId, + { type: "clickTextMatchedElement" }, + { + frameId: sorted[0].frameId, + } + ); + } else { + await notify("Unable to find element with matching text", { + type: "warning", + }); + } +} + // Sends a request to the content script. If tabId is not specified it will // send it to the current tab. If frameId is not specified it will send it to // the main frame (frameId 0). @@ -133,6 +173,8 @@ export async function sendRequestToContent( request.type === "runActionOnReference" ) { return handleActionOnReference(request, targetTabId); + } else if (request.type === "clickElementByText") { + return clickElementByText(request.arg); } frameId = frameId ?? toAllFrames.has(request.type) ? undefined : 0; diff --git a/src/content/actions/clickElement.ts b/src/content/actions/clickElement.ts index 699df4a7..0949275a 100644 --- a/src/content/actions/clickElement.ts +++ b/src/content/actions/clickElement.ts @@ -1,5 +1,10 @@ +import Fuse from "fuse.js"; import { ElementWrapper } from "../../typings/ElementWrapper"; import { TalonAction } from "../../typings/RequestFromTalon"; +import { notify } from "../notify/notify"; +import { getCachedSetting } from "../settings/cacheSettings"; +import { getToggles } from "../settings/toggles"; +import { getAllWrappers } from "../wrappers/wrappers"; import { openInBackgroundTab } from "./openInNewTab"; export async function clickElement( @@ -43,3 +48,56 @@ export async function clickElement( return undefined; } + +let textMatchedWrapper: ElementWrapper | undefined; + +export async function matchElementByText(text: string) { + if (!getToggles().computed && !getCachedSetting("alwaysComputeHintables")) { + await notify( + 'Enable the setting "Always compute hintable elements" if you want this command to work while the hints are off.', + { type: "warning" } + ); + return; + } + + const hintablesWithTextContent = getAllWrappers() + .filter((w) => w.isHintable) + .map((wrapper) => ({ + wrapper, + textContent: wrapper.element.textContent?.trim(), + })); + + const fuse = new Fuse(hintablesWithTextContent, { + keys: ["textContent"], + ignoreLocation: true, + includeScore: true, + threshold: 0.3, + }); + + const sortedMatches = fuse.search(text).sort((a, b) => { + if ( + a.item.wrapper.isIntersectingViewport && + !b.item.wrapper.isIntersectingViewport + ) { + return -1; + } + + if ( + !a.item.wrapper.isIntersectingViewport && + b.item.wrapper.isIntersectingViewport + ) { + return 1; + } + + return a.score! - b.score!; + }); + + const bestMatch = sortedMatches[0]; + textMatchedWrapper = bestMatch?.item.wrapper; + + return bestMatch?.score; +} + +export function clickTextMatchedElement() { + textMatchedWrapper?.click(); +} diff --git a/src/content/actions/runRangoActionWithoutTarget.ts b/src/content/actions/runRangoActionWithoutTarget.ts index 7ec2fd23..5b3a0d24 100644 --- a/src/content/actions/runRangoActionWithoutTarget.ts +++ b/src/content/actions/runRangoActionWithoutTarget.ts @@ -18,10 +18,11 @@ import { removeReference, showReferences } from "./references"; import { refreshHints } from "./refreshHints"; import { runActionOnReference } from "./runActionOnReference"; import { scroll } from "./scroll"; +import { clickTextMatchedElement, matchElementByText } from "./clickElement"; export async function runRangoActionWithoutTarget( request: RangoActionWithoutTarget -): Promise { +): Promise { switch (request.type) { case "historyGoBack": window.history.back(); @@ -167,6 +168,13 @@ export async function runRangoActionWithoutTarget( case "removeReference": return removeReference(request.arg); + case "matchElementByText": + return matchElementByText(request.arg); + + case "clickTextMatchedElement": + clickTextMatchedElement(); + break; + default: await notify(`Invalid action "${request.type}"`, { type: "error", diff --git a/src/content/wrappers/ElementWrapperClass.ts b/src/content/wrappers/ElementWrapperClass.ts index 87b878a5..2c337d50 100644 --- a/src/content/wrappers/ElementWrapperClass.ts +++ b/src/content/wrappers/ElementWrapperClass.ts @@ -20,6 +20,7 @@ import { getUserScrollableContainer } from "../utils/getUserScrollableContainer" import { isDisabled } from "../utils/isDisabled"; import { isHintable } from "../utils/isHintable"; import { isVisible } from "../utils/isVisible"; +import { setStyleProperties } from "../hints/setStyleProperties"; import { refresh } from "./refresh"; import { addWrapper, @@ -452,7 +453,11 @@ class ElementWrapperClass implements ElementWrapper { click(): boolean { const pointerTarget = getPointerTarget(this.element); - this.hint?.flash(); + if (this.hint?.inner.isConnected) { + this.hint.flash(); + } else { + this.flashElement(); + } if (this.element instanceof HTMLAnchorElement) { const closestContentEditable = this.element.closest("[contenteditable]"); @@ -489,6 +494,25 @@ class ElementWrapperClass implements ElementWrapper { return dispatchClick(pointerTarget); } + flashElement() { + const element = this.element; + + if (!(element instanceof HTMLElement)) { + return; + } + + const previousOutline = element.style.outline; + + setStyleProperties(element, { + outline: "3px solid #0891b2", + }); + setTimeout(() => { + setStyleProperties(element, { + outline: previousOutline, + }); + }, 300); + } + hover() { const pointerTarget = getPointerTarget(this.element); this.hint?.flash(); diff --git a/src/typings/ElementWrapper.ts b/src/typings/ElementWrapper.ts index 4e83e77f..2a7083a9 100644 --- a/src/typings/ElementWrapper.ts +++ b/src/typings/ElementWrapper.ts @@ -32,6 +32,7 @@ export interface ElementWrapper { intersect(isIntersecting: boolean): void; intersectViewport(isIntersecting: boolean): void; click(): boolean; + flashElement(): void; hover(): void; unhover(): void; diff --git a/src/typings/RangoAction.ts b/src/typings/RangoAction.ts index 8c19a18f..9fca6715 100644 --- a/src/typings/RangoAction.ts +++ b/src/typings/RangoAction.ts @@ -173,6 +173,13 @@ interface RangoActionFocusTabByText { arg: string; } +type RangoActionClickElementByText = + | { + type: "clickElementByText" | "matchElementByText"; + arg: string; + } + | { type: "clickTextMatchedElement" }; + export type RangoActionWithTarget = | RangoActionWithTargets | RangoActionWithTargetsWithOptionalNumberArg @@ -192,7 +199,8 @@ export type RangoActionWithoutTarget = | RangoActionRemoveReference | RangoActionScrollPosition | RangoActionfocusOrCreateTabByUrl - | RangoActionFocusTabByText; + | RangoActionFocusTabByText + | RangoActionClickElementByText; export type RangoAction = RangoActionWithTarget | RangoActionWithoutTarget; From 8c7791343610c55e66ecc3d35834b7db0b8be2d2 Mon Sep 17 00:00:00 2001 From: David Tejada Date: Wed, 24 Apr 2024 11:15:27 +0200 Subject: [PATCH 2/7] Add the ability to run any action to a text matched element and option to prioritize viewport --- .../messaging/sendRequestToContent.ts | 22 ++++-- src/content/actions/clickElement.ts | 58 --------------- .../actions/runActionOnTextMatchedElement.ts | 72 +++++++++++++++++++ .../actions/runRangoActionWithoutTarget.ts | 11 +-- src/typings/RangoAction.ts | 20 ++++-- 5 files changed, 111 insertions(+), 72 deletions(-) create mode 100644 src/content/actions/runActionOnTextMatchedElement.ts diff --git a/src/background/messaging/sendRequestToContent.ts b/src/background/messaging/sendRequestToContent.ts index d9b47028..ab1048d8 100644 --- a/src/background/messaging/sendRequestToContent.ts +++ b/src/background/messaging/sendRequestToContent.ts @@ -7,6 +7,7 @@ import { splitRequestsByFrame } from "../utils/splitRequestsByFrame"; import { RangoActionRemoveReference, RangoActionRunActionOnReference, + RangoActionWithTargets, } from "../../typings/RangoAction"; import { notify } from "../utils/notify"; @@ -60,7 +61,11 @@ async function handleActionOnReference( } } -async function clickElementByText(text: string) { +async function runActionOnTextMatchedElement( + actionType: RangoActionWithTargets["type"], + text: string, + prioritizeViewport: boolean +) { const tabId = await getCurrentTabId(); const allFrames = await browser.webNavigation.getAllFrames({ tabId, @@ -70,7 +75,7 @@ async function clickElementByText(text: string) { frameId: frame.frameId, score: (await browser.tabs.sendMessage( tabId, - { type: "matchElementByText", arg: text }, + { type: "matchElementByText", text, prioritizeViewport }, { frameId: frame.frameId, } @@ -88,7 +93,10 @@ async function clickElementByText(text: string) { if (sorted[0]) { await browser.tabs.sendMessage( tabId, - { type: "clickTextMatchedElement" }, + { + type: "executeActionOnTextMatchedElement", + actionType, + }, { frameId: sorted[0].frameId, } @@ -173,8 +181,12 @@ export async function sendRequestToContent( request.type === "runActionOnReference" ) { return handleActionOnReference(request, targetTabId); - } else if (request.type === "clickElementByText") { - return clickElementByText(request.arg); + } else if (request.type === "runActionOnTextMatchedElement") { + return runActionOnTextMatchedElement( + request.arg, + request.arg2, + request.arg3 + ); } frameId = frameId ?? toAllFrames.has(request.type) ? undefined : 0; diff --git a/src/content/actions/clickElement.ts b/src/content/actions/clickElement.ts index 0949275a..699df4a7 100644 --- a/src/content/actions/clickElement.ts +++ b/src/content/actions/clickElement.ts @@ -1,10 +1,5 @@ -import Fuse from "fuse.js"; import { ElementWrapper } from "../../typings/ElementWrapper"; import { TalonAction } from "../../typings/RequestFromTalon"; -import { notify } from "../notify/notify"; -import { getCachedSetting } from "../settings/cacheSettings"; -import { getToggles } from "../settings/toggles"; -import { getAllWrappers } from "../wrappers/wrappers"; import { openInBackgroundTab } from "./openInNewTab"; export async function clickElement( @@ -48,56 +43,3 @@ export async function clickElement( return undefined; } - -let textMatchedWrapper: ElementWrapper | undefined; - -export async function matchElementByText(text: string) { - if (!getToggles().computed && !getCachedSetting("alwaysComputeHintables")) { - await notify( - 'Enable the setting "Always compute hintable elements" if you want this command to work while the hints are off.', - { type: "warning" } - ); - return; - } - - const hintablesWithTextContent = getAllWrappers() - .filter((w) => w.isHintable) - .map((wrapper) => ({ - wrapper, - textContent: wrapper.element.textContent?.trim(), - })); - - const fuse = new Fuse(hintablesWithTextContent, { - keys: ["textContent"], - ignoreLocation: true, - includeScore: true, - threshold: 0.3, - }); - - const sortedMatches = fuse.search(text).sort((a, b) => { - if ( - a.item.wrapper.isIntersectingViewport && - !b.item.wrapper.isIntersectingViewport - ) { - return -1; - } - - if ( - !a.item.wrapper.isIntersectingViewport && - b.item.wrapper.isIntersectingViewport - ) { - return 1; - } - - return a.score! - b.score!; - }); - - const bestMatch = sortedMatches[0]; - textMatchedWrapper = bestMatch?.item.wrapper; - - return bestMatch?.score; -} - -export function clickTextMatchedElement() { - textMatchedWrapper?.click(); -} diff --git a/src/content/actions/runActionOnTextMatchedElement.ts b/src/content/actions/runActionOnTextMatchedElement.ts new file mode 100644 index 00000000..e05f2a64 --- /dev/null +++ b/src/content/actions/runActionOnTextMatchedElement.ts @@ -0,0 +1,72 @@ +import Fuse from "fuse.js"; +import { ElementWrapper } from "../../typings/ElementWrapper"; +import { notify } from "../notify/notify"; +import { getCachedSetting } from "../settings/cacheSettings"; +import { getToggles } from "../settings/toggles"; +import { getAllWrappers } from "../wrappers/wrappers"; +import { RangoActionWithTargets } from "../../typings/RangoAction"; +import { runRangoActionWithTarget } from "./runRangoActionWithTarget"; + +let textMatchedWrapper: ElementWrapper | undefined; + +export async function matchElementByText( + text: string, + prioritizeViewport: boolean +) { + if (!getToggles().computed && !getCachedSetting("alwaysComputeHintables")) { + await notify( + 'Enable the setting "Always compute hintable elements" if you want this command to work while the hints are off.', + { type: "warning" } + ); + return; + } + + const hintablesWithTextContent = getAllWrappers() + .filter((w) => w.isHintable) + .map((wrapper) => ({ + wrapper, + textContent: wrapper.element.textContent?.trim(), + })); + + const fuse = new Fuse(hintablesWithTextContent, { + keys: ["textContent"], + ignoreLocation: true, + includeScore: true, + threshold: 0.3, + }); + + const sortedMatches = fuse.search(text).sort((a, b) => { + if (prioritizeViewport) { + if ( + a.item.wrapper.isIntersectingViewport && + !b.item.wrapper.isIntersectingViewport + ) { + return -1; + } + + if ( + !a.item.wrapper.isIntersectingViewport && + b.item.wrapper.isIntersectingViewport + ) { + return 1; + } + } + + return a.score! - b.score!; + }); + + const bestMatch = sortedMatches[0]; + textMatchedWrapper = bestMatch?.item.wrapper; + + return bestMatch?.score; +} + +export async function executeActionOnTextMatchedElement( + actionType: RangoActionWithTargets["type"] +) { + if (textMatchedWrapper) { + await runRangoActionWithTarget({ type: actionType, target: [] }, [ + textMatchedWrapper, + ]); + } +} diff --git a/src/content/actions/runRangoActionWithoutTarget.ts b/src/content/actions/runRangoActionWithoutTarget.ts index 5b3a0d24..7b1f18ff 100644 --- a/src/content/actions/runRangoActionWithoutTarget.ts +++ b/src/content/actions/runRangoActionWithoutTarget.ts @@ -18,7 +18,10 @@ import { removeReference, showReferences } from "./references"; import { refreshHints } from "./refreshHints"; import { runActionOnReference } from "./runActionOnReference"; import { scroll } from "./scroll"; -import { clickTextMatchedElement, matchElementByText } from "./clickElement"; +import { + executeActionOnTextMatchedElement, + matchElementByText, +} from "./runActionOnTextMatchedElement"; export async function runRangoActionWithoutTarget( request: RangoActionWithoutTarget @@ -169,10 +172,10 @@ export async function runRangoActionWithoutTarget( return removeReference(request.arg); case "matchElementByText": - return matchElementByText(request.arg); + return matchElementByText(request.text, request.prioritizeViewport); - case "clickTextMatchedElement": - clickTextMatchedElement(); + case "executeActionOnTextMatchedElement": + await executeActionOnTextMatchedElement(request.actionType); break; default: diff --git a/src/typings/RangoAction.ts b/src/typings/RangoAction.ts index 9fca6715..994c4624 100644 --- a/src/typings/RangoAction.ts +++ b/src/typings/RangoAction.ts @@ -173,12 +173,22 @@ interface RangoActionFocusTabByText { arg: string; } -type RangoActionClickElementByText = +type RangoActionRunActionOnTextMatchedElement = | { - type: "clickElementByText" | "matchElementByText"; - arg: string; + type: "runActionOnTextMatchedElement"; + arg: RangoActionWithTargets["type"]; + arg2: string; + arg3: boolean; } - | { type: "clickTextMatchedElement" }; + | { + type: "matchElementByText"; + text: string; + prioritizeViewport: boolean; + } + | { + type: "executeActionOnTextMatchedElement"; + actionType: RangoActionWithTargets["type"]; + }; export type RangoActionWithTarget = | RangoActionWithTargets @@ -200,7 +210,7 @@ export type RangoActionWithoutTarget = | RangoActionScrollPosition | RangoActionfocusOrCreateTabByUrl | RangoActionFocusTabByText - | RangoActionClickElementByText; + | RangoActionRunActionOnTextMatchedElement; export type RangoAction = RangoActionWithTarget | RangoActionWithoutTarget; From cf36dadab38cbe0f6009d6f15f645a40f65fa3b9 Mon Sep 17 00:00:00 2001 From: David Tejada Date: Wed, 24 Apr 2024 12:14:38 +0200 Subject: [PATCH 3/7] Only take into account visible elements --- src/content/actions/runActionOnTextMatchedElement.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/content/actions/runActionOnTextMatchedElement.ts b/src/content/actions/runActionOnTextMatchedElement.ts index e05f2a64..d4588f16 100644 --- a/src/content/actions/runActionOnTextMatchedElement.ts +++ b/src/content/actions/runActionOnTextMatchedElement.ts @@ -5,6 +5,7 @@ import { getCachedSetting } from "../settings/cacheSettings"; import { getToggles } from "../settings/toggles"; import { getAllWrappers } from "../wrappers/wrappers"; import { RangoActionWithTargets } from "../../typings/RangoAction"; +import { isVisible } from "../utils/isVisible"; import { runRangoActionWithTarget } from "./runRangoActionWithTarget"; let textMatchedWrapper: ElementWrapper | undefined; @@ -22,7 +23,7 @@ export async function matchElementByText( } const hintablesWithTextContent = getAllWrappers() - .filter((w) => w.isHintable) + .filter((w) => w.isHintable && isVisible(w.element)) .map((wrapper) => ({ wrapper, textContent: wrapper.element.textContent?.trim(), From b733a3531d394061add9ca2c6dc38a696190c384 Mon Sep 17 00:00:00 2001 From: David Tejada Date: Wed, 24 Apr 2024 12:14:55 +0200 Subject: [PATCH 4/7] Ignore numbers when matching text for better matching ui items with number labels --- src/content/actions/runActionOnTextMatchedElement.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/content/actions/runActionOnTextMatchedElement.ts b/src/content/actions/runActionOnTextMatchedElement.ts index d4588f16..eecf6f74 100644 --- a/src/content/actions/runActionOnTextMatchedElement.ts +++ b/src/content/actions/runActionOnTextMatchedElement.ts @@ -26,7 +26,7 @@ export async function matchElementByText( .filter((w) => w.isHintable && isVisible(w.element)) .map((wrapper) => ({ wrapper, - textContent: wrapper.element.textContent?.trim(), + textContent: wrapper.element.textContent?.replace(/\d/g, "").trim(), })); const fuse = new Fuse(hintablesWithTextContent, { From 6ca150a2c0db79293357f213906c3e7ed8e36b71 Mon Sep 17 00:00:00 2001 From: David Tejada Date: Thu, 25 Apr 2024 11:22:36 +0200 Subject: [PATCH 5/7] Remove the need to enable the setting "Always compute hintables" for using fuzzy text clicking --- .../actions/runActionOnTextMatchedElement.ts | 138 ++++++++++++------ 1 file changed, 95 insertions(+), 43 deletions(-) diff --git a/src/content/actions/runActionOnTextMatchedElement.ts b/src/content/actions/runActionOnTextMatchedElement.ts index eecf6f74..67c9db0c 100644 --- a/src/content/actions/runActionOnTextMatchedElement.ts +++ b/src/content/actions/runActionOnTextMatchedElement.ts @@ -1,63 +1,104 @@ import Fuse from "fuse.js"; -import { ElementWrapper } from "../../typings/ElementWrapper"; -import { notify } from "../notify/notify"; +import { RangoActionWithTargets } from "../../typings/RangoAction"; import { getCachedSetting } from "../settings/cacheSettings"; import { getToggles } from "../settings/toggles"; -import { getAllWrappers } from "../wrappers/wrappers"; -import { RangoActionWithTargets } from "../../typings/RangoAction"; +import { deepGetElements } from "../utils/deepGetElements"; +import { isHintable } from "../utils/isHintable"; import { isVisible } from "../utils/isVisible"; +import { getOrCreateWrapper } from "../wrappers/ElementWrapperClass"; +import { getAllWrappers } from "../wrappers/wrappers"; import { runRangoActionWithTarget } from "./runRangoActionWithTarget"; -let textMatchedWrapper: ElementWrapper | undefined; +let textMatchedElement: Element | undefined; +let shouldScrollIntoView = false; + +type Hintable = { + element: Element; + isIntersectingViewport: boolean; + trimmedTextContent: string; +}; + +function trimTextContent(element: Element) { + return element.textContent?.replace(/\d/g, "").trim() ?? ""; +} + +async function getHintables() { + // Hints are on, there will be wrappers for the hintable elements + if (getToggles().computed || getCachedSetting("alwaysComputeHintables")) { + return getAllWrappers() + .filter((wrapper) => wrapper.isHintable && isVisible(wrapper.element)) + .map((wrapper) => ({ + element: wrapper.element, + isIntersectingViewport: wrapper.isIntersectingViewport, + trimmedTextContent: trimTextContent(wrapper.element), + })); + } + + const hintables: Hintable[] = []; + const elements = deepGetElements(document.body, true); + + let resolveIntersection: (value: unknown) => void | undefined; + + const intersectionPromise = new Promise((resolve) => { + resolveIntersection = resolve; + }); + + const intersectionObserver = new IntersectionObserver((entries) => { + for (const entry of entries) { + if (isHintable(entry.target) && isVisible(entry.target)) { + hintables.push({ + element: entry.target, + isIntersectingViewport: entry.isIntersecting, + trimmedTextContent: trimTextContent(entry.target), + }); + } + } + + if (resolveIntersection) resolveIntersection(true); + }); + + for (const element of elements) { + intersectionObserver.observe(element); + } + + await intersectionPromise; + return hintables; +} export async function matchElementByText( text: string, prioritizeViewport: boolean ) { - if (!getToggles().computed && !getCachedSetting("alwaysComputeHintables")) { - await notify( - 'Enable the setting "Always compute hintable elements" if you want this command to work while the hints are off.', - { type: "warning" } - ); - return; - } + const hintables = await getHintables(); - const hintablesWithTextContent = getAllWrappers() - .filter((w) => w.isHintable && isVisible(w.element)) - .map((wrapper) => ({ - wrapper, - textContent: wrapper.element.textContent?.replace(/\d/g, "").trim(), - })); - - const fuse = new Fuse(hintablesWithTextContent, { - keys: ["textContent"], + const fuse = new Fuse(hintables, { + keys: ["trimmedTextContent"], ignoreLocation: true, includeScore: true, threshold: 0.3, }); - const sortedMatches = fuse.search(text).sort((a, b) => { - if (prioritizeViewport) { - if ( - a.item.wrapper.isIntersectingViewport && - !b.item.wrapper.isIntersectingViewport - ) { - return -1; - } + const matches = fuse.search(text); - if ( - !a.item.wrapper.isIntersectingViewport && - b.item.wrapper.isIntersectingViewport - ) { - return 1; - } - } + const sortedMatches = prioritizeViewport + ? matches.sort((a, b) => { + if (a.item.isIntersectingViewport && !b.item.isIntersectingViewport) { + return -1; + } - return a.score! - b.score!; - }); + if (!a.item.isIntersectingViewport && b.item.isIntersectingViewport) { + return 1; + } + + return a.score! - b.score!; + }) + : matches; const bestMatch = sortedMatches[0]; - textMatchedWrapper = bestMatch?.item.wrapper; + if (bestMatch?.item) { + textMatchedElement = bestMatch.item.element; + shouldScrollIntoView = !bestMatch.item.isIntersectingViewport; + } return bestMatch?.score; } @@ -65,9 +106,20 @@ export async function matchElementByText( export async function executeActionOnTextMatchedElement( actionType: RangoActionWithTargets["type"] ) { - if (textMatchedWrapper) { - await runRangoActionWithTarget({ type: actionType, target: [] }, [ - textMatchedWrapper, - ]); + if (textMatchedElement) { + const wrapper = getOrCreateWrapper(textMatchedElement, false); + + // If the element is outside of the viewport we need to scroll the element + // into view in order to get the topmost element, which is the one we need + // to click on. + if (shouldScrollIntoView) { + textMatchedElement.scrollIntoView({ + behavior: "instant", + block: "center", + inline: "center", + }); + } + + await runRangoActionWithTarget({ type: actionType, target: [] }, [wrapper]); } } From a97e3f8ad508d84c59f3f2154043342f30a5d38c Mon Sep 17 00:00:00 2001 From: David Tejada Date: Fri, 26 Apr 2024 10:30:48 +0200 Subject: [PATCH 6/7] Slight modifications to flashing outline --- src/content/wrappers/ElementWrapperClass.ts | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/content/wrappers/ElementWrapperClass.ts b/src/content/wrappers/ElementWrapperClass.ts index 2c337d50..bd6435a4 100644 --- a/src/content/wrappers/ElementWrapperClass.ts +++ b/src/content/wrappers/ElementWrapperClass.ts @@ -504,13 +504,11 @@ class ElementWrapperClass implements ElementWrapper { const previousOutline = element.style.outline; setStyleProperties(element, { - outline: "3px solid #0891b2", + outline: "2px solid #0891b2", }); setTimeout(() => { - setStyleProperties(element, { - outline: previousOutline, - }); - }, 300); + element.style.outline = previousOutline; + }, 150); } hover() { From 47e56b426ac46530e2d8ba3d2957ee970610613c Mon Sep 17 00:00:00 2001 From: David Tejada Date: Fri, 26 Apr 2024 10:31:51 +0200 Subject: [PATCH 7/7] Increase fuzzy search threshold for clickable elements --- src/content/actions/runActionOnTextMatchedElement.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/content/actions/runActionOnTextMatchedElement.ts b/src/content/actions/runActionOnTextMatchedElement.ts index 67c9db0c..2d80f3cf 100644 --- a/src/content/actions/runActionOnTextMatchedElement.ts +++ b/src/content/actions/runActionOnTextMatchedElement.ts @@ -75,7 +75,7 @@ export async function matchElementByText( keys: ["trimmedTextContent"], ignoreLocation: true, includeScore: true, - threshold: 0.3, + threshold: 0.4, }); const matches = fuse.search(text);