diff --git a/server/src/browser-management/inputHandlers.ts b/server/src/browser-management/inputHandlers.ts index 6b5016f3..b0be33e6 100644 --- a/server/src/browser-management/inputHandlers.ts +++ b/server/src/browser-management/inputHandlers.ts @@ -11,6 +11,8 @@ import { WorkflowGenerator } from "../workflow-management/classes/Generator"; import { Page } from "playwright"; import { throttle } from "../../../src/helpers/inputHelpers"; import { CustomActions } from "../../../src/shared/types"; +import { WhereWhatPair } from "maxun-core"; +import { RemoteBrowser } from './classes/RemoteBrowser'; /** * A wrapper function for handling user input. @@ -27,7 +29,7 @@ import { CustomActions } from "../../../src/shared/types"; */ const handleWrapper = async ( handleCallback: ( - generator: WorkflowGenerator, + activeBrowser: RemoteBrowser, page: Page, args?: any ) => Promise, @@ -44,9 +46,9 @@ const handleWrapper = async ( const currentPage = activeBrowser?.getCurrentPage(); if (currentPage && activeBrowser) { if (args) { - await handleCallback(activeBrowser.generator, currentPage, args); + await handleCallback(activeBrowser, currentPage, args); } else { - await handleCallback(activeBrowser.generator, currentPage); + await handleCallback(activeBrowser, currentPage); } } else { logger.log('warn', `No active page for browser ${id}`); @@ -85,8 +87,19 @@ const onGenerateAction = async (customActionEventData: CustomActionEventData, us * @category BrowserManagement */ const handleGenerateAction = - async (generator: WorkflowGenerator, page: Page, { action, settings }: CustomActionEventData) => { - await generator.customAction(action, settings, page); + async (activeBrowser: RemoteBrowser, page: Page, { action, settings }: CustomActionEventData) => { + try { + if (page.isClosed()) { + logger.log("debug", `Ignoring generate action event: page is closed`); + return; + } + + const generator = activeBrowser.generator; + await generator.customAction(action, settings, page); + } catch (e) { + const { message } = e as Error; + logger.log("warn", `Error handling generate action event: ${message}`); + } } /** @@ -104,40 +117,51 @@ const onMousedown = async (coordinates: Coordinates, userId: string) => { * A mousedown event handler. * Reproduces the click on the remote browser instance * and generates pair data for the recorded workflow. - * @param generator - the workflow generator {@link Generator} + * @param activeBrowser - the active remote browser {@link RemoteBrowser} * @param page - the active page of the remote browser * @param x - the x coordinate of the mousedown event * @param y - the y coordinate of the mousedown event * @category BrowserManagement */ -const handleMousedown = async (generator: WorkflowGenerator, page: Page, { x, y }: Coordinates) => { +const handleMousedown = async (activeBrowser: RemoteBrowser, page: Page, { x, y }: Coordinates) => { + try { + if (page.isClosed()) { + logger.log("debug", `Ignoring mousedown event: page is closed`); + return; + } + + const generator = activeBrowser.generator; await generator.onClick({ x, y }, page); const previousUrl = page.url(); const tabsBeforeClick = page.context().pages().length; await page.mouse.click(x, y); // try if the click caused a navigation to a new url try { - await page.waitForNavigation({ timeout: 2000 }); - const currentUrl = page.url(); - if (currentUrl !== previousUrl) { - generator.notifyUrlChange(currentUrl); - } + await page.waitForNavigation({ timeout: 2000 }); + const currentUrl = page.url(); + if (currentUrl !== previousUrl) { + generator.notifyUrlChange(currentUrl); + } } catch (e) { - const { message } = e as Error; + const { message } = e as Error; } //ignore possible timeouts // check if any new page was opened by the click const tabsAfterClick = page.context().pages().length; const numOfNewPages = tabsAfterClick - tabsBeforeClick; if (numOfNewPages > 0) { - for (let i = 1; i <= numOfNewPages; i++) { - const newPage = page.context().pages()[tabsAfterClick - i]; - if (newPage) { - generator.notifyOnNewTab(newPage, tabsAfterClick - i); - } + for (let i = 1; i <= numOfNewPages; i++) { + const newPage = page.context().pages()[tabsAfterClick - i]; + if (newPage) { + generator.notifyOnNewTab(newPage, tabsAfterClick - i); } + } } - logger.log('debug', `Clicked on position x:${x}, y:${y}`); + logger.log("debug", `Clicked on position x:${x}, y:${y}`); + } catch (e) { + const { message } = e as Error; + logger.log("warn", `Error handling mousedown event: ${message}`); + } }; /** @@ -156,15 +180,16 @@ const onWheel = async (scrollDeltas: ScrollDeltas, userId: string) => { * Reproduces the wheel event on the remote browser instance. * Scroll is not generated for the workflow pair. This is because * Playwright scrolls elements into focus on any action. - * @param generator - the workflow generator {@link Generator} + * @param activeBrowser - the active remote browser {@link RemoteBrowser} * @param page - the active page of the remote browser * @param deltaX - the delta x of the wheel event * @param deltaY - the delta y of the wheel event * @category BrowserManagement */ -const handleWheel = async (generator: WorkflowGenerator, page: Page, { deltaX, deltaY }: ScrollDeltas) => { +const handleWheel = async (activeBrowser: RemoteBrowser, page: Page, { deltaX, deltaY }: ScrollDeltas) => { try { if (page.isClosed()) { + logger.log("debug", `Ignoring wheel event: page is closed`); return; } @@ -194,28 +219,30 @@ const onMousemove = async (coordinates: Coordinates, userId: string) => { * Reproduces the mousemove event on the remote browser instance * and generates data for the client's highlighter. * Mousemove is also not reflected in the workflow. - * @param generator - the workflow generator {@link Generator} + * @param activeBrowser - the active remote browser {@link RemoteBrowser} * @param page - the active page of the remote browser * @param x - the x coordinate of the mousemove event * @param y - the y coordinate of the mousemove event * @category BrowserManagement */ -const handleMousemove = async (generator: WorkflowGenerator, page: Page, { x, y }: Coordinates) => { +const handleMousemove = async (activeBrowser: RemoteBrowser, page: Page, { x, y }: Coordinates) => { try { if (page.isClosed()) { - logger.log('debug', `Ignoring mousemove event: page is closed`); + logger.log("debug", `Ignoring mousemove event: page is closed`); return; } + + const generator = activeBrowser.generator; await page.mouse.move(x, y); throttle(async () => { if (!page.isClosed()) { await generator.generateDataForHighlighter(page, { x, y }); } }, 100)(); - logger.log('debug', `Moved over position x:${x}, y:${y}`); + logger.log("debug", `Moved over position x:${x}, y:${y}`); } catch (e) { const { message } = e as Error; - logger.log('error', message); + logger.log("error", message); } } @@ -234,28 +261,50 @@ const onKeydown = async (keyboardInput: KeyboardInput, userId: string) => { * A keydown event handler. * Reproduces the keydown event on the remote browser instance * and generates the workflow pair data. - * @param generator - the workflow generator {@link Generator} + * @param activeBrowser - the active remote browser {@link RemoteBrowser} * @param page - the active page of the remote browser * @param key - the pressed key * @param coordinates - the coordinates, where the keydown event happened * @category BrowserManagement */ -const handleKeydown = async (generator: WorkflowGenerator, page: Page, { key, coordinates }: KeyboardInput) => { - await page.keyboard.down(key); - await generator.onKeyboardInput(key, coordinates, page); - logger.log('debug', `Key ${key} pressed`); +const handleKeydown = async (activeBrowser: RemoteBrowser, page: Page, { key, coordinates }: KeyboardInput) => { + try { + if (page.isClosed()) { + logger.log("debug", `Ignoring keydown event: page is closed`); + return; + } + + const generator = activeBrowser.generator; + await page.keyboard.down(key); + await generator.onKeyboardInput(key, coordinates, page); + logger.log("debug", `Key ${key} pressed`); + } catch (e) { + const { message } = e as Error; + logger.log("warn", `Error handling keydown event: ${message}`); + } }; /** * Handles the date selection event. - * @param generator - the workflow generator {@link Generator} + * @param activeBrowser - the active remote browser {@link RemoteBrowser} * @param page - the active page of the remote browser * @param data - the data of the date selection event {@link DatePickerEventData} * @category BrowserManagement */ -const handleDateSelection = async (generator: WorkflowGenerator, page: Page, data: DatePickerEventData) => { - await generator.onDateSelection(page, data); - logger.log('debug', `Date ${data.value} selected`); +const handleDateSelection = async (activeBrowser: RemoteBrowser, page: Page, data: DatePickerEventData) => { + try { + if (page.isClosed()) { + logger.log("debug", `Ignoring date selection event: page is closed`); + return; + } + + const generator = activeBrowser.generator; + await generator.onDateSelection(page, data); + logger.log("debug", `Date ${data.value} selected`); + } catch (e) { + const { message } = e as Error; + logger.log("warn", `Error handling date selection event: ${message}`); + } } /** @@ -271,14 +320,25 @@ const onDateSelection = async (data: DatePickerEventData, userId: string) => { /** * Handles the dropdown selection event. - * @param generator - the workflow generator {@link Generator} + * @param activeBrowser - the active remote browser {@link RemoteBrowser} * @param page - the active page of the remote browser * @param data - the data of the dropdown selection event * @category BrowserManagement */ -const handleDropdownSelection = async (generator: WorkflowGenerator, page: Page, data: { selector: string, value: string }) => { - await generator.onDropdownSelection(page, data); - logger.log('debug', `Dropdown value ${data.value} selected`); +const handleDropdownSelection = async (activeBrowser: RemoteBrowser, page: Page, data: { selector: string, value: string }) => { + try { + if (page.isClosed()) { + logger.log("debug", `Ignoring dropdown selection event: page is closed`); + return; + } + + const generator = activeBrowser.generator; + await generator.onDropdownSelection(page, data); + logger.log("debug", `Dropdown value ${data.value} selected`); + } catch (e) { + const { message } = e as Error; + logger.log("warn", `Error handling dropdown selection event: ${message}`); + } } /** @@ -294,14 +354,25 @@ const onDropdownSelection = async (data: { selector: string, value: string }, us /** * Handles the time selection event. - * @param generator - the workflow generator {@link Generator} + * @param activeBrowser - the active remote browser {@link RemoteBrowser} * @param page - the active page of the remote browser * @param data - the data of the time selection event * @category BrowserManagement */ -const handleTimeSelection = async (generator: WorkflowGenerator, page: Page, data: { selector: string, value: string }) => { - await generator.onTimeSelection(page, data); - logger.log('debug', `Time value ${data.value} selected`); +const handleTimeSelection = async (activeBrowser: RemoteBrowser, page: Page, data: { selector: string, value: string }) => { + try { + if (page.isClosed()) { + logger.log("debug", `Ignoring time selection event: page is closed`); + return; + } + + const generator = activeBrowser.generator; + await generator.onTimeSelection(page, data); + logger.log("debug", `Time value ${data.value} selected`); + } catch (e) { + const { message } = e as Error; + logger.log("warn", `Error handling time selection event: ${message}`); + } } /** @@ -317,14 +388,31 @@ const onTimeSelection = async (data: { selector: string, value: string }, userId /** * Handles the datetime-local selection event. - * @param generator - the workflow generator {@link Generator} + * @param activeBrowser - the active remote browser {@link RemoteBrowser} * @param page - the active page of the remote browser * @param data - the data of the datetime-local selection event * @category BrowserManagement */ -const handleDateTimeLocalSelection = async (generator: WorkflowGenerator, page: Page, data: { selector: string, value: string }) => { - await generator.onDateTimeLocalSelection(page, data); - logger.log('debug', `DateTime Local value ${data.value} selected`); +const handleDateTimeLocalSelection = async (activeBrowser: RemoteBrowser, page: Page, data: { selector: string, value: string }) => { + try { + if (page.isClosed()) { + logger.log( + "debug", + `Ignoring datetime-local selection event: page is closed` + ); + return; + } + + const generator = activeBrowser.generator; + await generator.onDateTimeLocalSelection(page, data); + logger.log("debug", `DateTime Local value ${data.value} selected`); + } catch (e) { + const { message } = e as Error; + logger.log( + "warn", + `Error handling datetime-local selection event: ${message}` + ); + } } /** @@ -353,14 +441,24 @@ const onKeyup = async (keyboardInput: KeyboardInput, userId: string) => { * A keyup event handler. * Reproduces the keyup event on the remote browser instance. * Does not generate any data - keyup is not reflected in the workflow. - * @param generator - the workflow generator {@link Generator} + * @param activeBrowser - the active remote browser {@link RemoteBrowser} * @param page - the active page of the remote browser * @param key - the released key * @category BrowserManagement */ -const handleKeyup = async (generator: WorkflowGenerator, page: Page, key: string) => { - await page.keyboard.up(key); - logger.log('debug', `Key ${key} unpressed`); +const handleKeyup = async (activeBrowser: RemoteBrowser, page: Page, key: string) => { + try { + if (page.isClosed()) { + logger.log("debug", `Ignoring keyup event: page is closed`); + return; + } + + await page.keyboard.up(key); + logger.log("debug", `Key ${key} unpressed`); + } catch (e) { + const { message } = e as Error; + logger.log("warn", `Error handling keyup event: ${message}`); + } }; /** @@ -377,23 +475,35 @@ const onChangeUrl = async (url: string, userId: string) => { /** * An url change event handler. * Navigates the page to the given url and generates data for the workflow. - * @param generator - the workflow generator {@link Generator} + * @param activeBrowser - the active remote browser {@link RemoteBrowser} * @param page - the active page of the remote browser * @param url - the new url of the page * @category BrowserManagement */ -const handleChangeUrl = async (generator: WorkflowGenerator, page: Page, url: string) => { - if (url) { - await generator.onChangeUrl(url, page); - try { - await page.goto(url, { waitUntil: 'networkidle', timeout: 100000 }); - logger.log('debug', `Went to ${url}`); - } catch (e) { - const { message } = e as Error; - logger.log('error', message); +const handleChangeUrl = async (activeBrowser: RemoteBrowser, page: Page, url: string) => { + try { + if (page.isClosed()) { + logger.log("debug", `Ignoring change url event: page is closed`); + return; } - } else { - logger.log('warn', `No url provided`); + + if (url) { + const generator = activeBrowser.generator; + await generator.onChangeUrl(url, page); + + try { + await page.goto(url, { waitUntil: "networkidle", timeout: 100000 }); + logger.log("debug", `Went to ${url}`); + } catch (e) { + const { message } = e as Error; + logger.log("error", message); + } + } else { + logger.log("warn", `No url provided`); + } + } catch (e) { + const { message } = e as Error; + logger.log("warn", `Error handling change url event: ${message}`); } }; @@ -410,13 +520,23 @@ const onRefresh = async (userId: string) => { /** * A refresh event handler. * Refreshes the page. This is not reflected in the workflow. - * @param generator - the workflow generator {@link Generator} + * @param activeBrowser - the active remote browser {@link RemoteBrowser} * @param page - the active page of the remote browser * @category BrowserManagement */ -const handleRefresh = async (generator: WorkflowGenerator, page: Page) => { - await page.reload(); - logger.log('debug', `Page refreshed.`); +const handleRefresh = async (activeBrowser: RemoteBrowser, page: Page) => { + try { + if (page.isClosed()) { + logger.log("debug", `Ignoring refresh event: page is closed`); + return; + } + + await page.reload(); + logger.log("debug", `Page refreshed.`); + } catch (e) { + const { message } = e as Error; + logger.log("warn", `Error handling refresh event: ${message}`); + } }; /** @@ -432,14 +552,25 @@ const onGoBack = async (userId: string) => { /** * A go back event handler. * Navigates the page back and generates data for the workflow. - * @param generator - the workflow generator {@link Generator} + * @param activeBrowser - the active remote browser {@link RemoteBrowser} * @param page - the active page of the remote browser * @category BrowserManagement */ -const handleGoBack = async (generator: WorkflowGenerator, page: Page) => { - await page.goBack({ waitUntil: 'commit' }); - generator.onGoBack(page.url()); - logger.log('debug', 'Page went back') +const handleGoBack = async (activeBrowser: RemoteBrowser, page: Page) => { + try { + if (page.isClosed()) { + logger.log("debug", `Ignoring go back event: page is closed`); + return; + } + + const generator = activeBrowser.generator; + await page.goBack({ waitUntil: "commit" }); + generator.onGoBack(page.url()); + logger.log("debug", "Page went back"); + } catch (e) { + const { message } = e as Error; + logger.log("warn", `Error handling go back event: ${message}`); + } }; /** @@ -455,14 +586,207 @@ const onGoForward = async (userId: string) => { /** * A go forward event handler. * Navigates the page forward and generates data for the workflow. - * @param generator - the workflow generator {@link Generator} + * @param activeBrowser - the active remote browser {@link RemoteBrowser} * @param page - the active page of the remote browser * @category BrowserManagement */ -const handleGoForward = async (generator: WorkflowGenerator, page: Page) => { - await page.goForward({ waitUntil: 'commit' }); - generator.onGoForward(page.url()); - logger.log('debug', 'Page went forward'); +const handleGoForward = async (activeBrowser: RemoteBrowser, page: Page) => { + try { + if (page.isClosed()) { + logger.log("debug", `Ignoring go forward event: page is closed`); + return; + } + + const generator = activeBrowser.generator; + await page.goForward({ waitUntil: "commit" }); + generator.onGoForward(page.url()); + logger.log("debug", "Page went forward"); + } catch (e) { + const { message } = e as Error; + logger.log("warn", `Error handling go forward event: ${message}`); + } +}; + +/** + * Handles the click action event. + * @param activeBrowser - the active remote browser {@link RemoteBrowser} + * @param page - the active page of the remote browser + * @param data - the data of the click action event + * @category BrowserManagement + */ +const handleClickAction = async ( + activeBrowser: RemoteBrowser, + page: Page, + data: { + selector: string; + url: string; + userId: string; + elementInfo?: any; + coordinates?: { x: number; y: number }; + isSPA?: boolean; + } +) => { + try { + if (page.isClosed()) { + logger.log("debug", `Ignoring click action event: page is closed`); + return; + } + + const { selector, url, elementInfo, coordinates, isSPA = false } = data; + const currentUrl = page.url(); + + await page.click(selector); + + const generator = activeBrowser.generator; + await generator.onDOMClickAction(page, data); + + logger.log("debug", `Click action processed: ${selector}`); + + if (isSPA) { + logger.log("debug", `SPA interaction detected for selector: ${selector}`); + + await new Promise((resolve) => setTimeout(resolve, 1500)); + } else { + const newUrl = page.url(); + const hasNavigated = newUrl !== currentUrl && !newUrl.endsWith("/#"); + + if (hasNavigated) { + logger.log("debug", `Navigation detected: ${currentUrl} -> ${newUrl}`); + + await generator.onDOMNavigation(page, { + url: newUrl, + currentUrl: currentUrl, + userId: data.userId, + }); + } + } + + await new Promise((resolve) => setTimeout(resolve, 2000)); + await activeBrowser.makeAndEmitDOMSnapshot(); + } catch (e) { + const { message } = e as Error; + logger.log( + "warn", + `Error handling enhanced click action event: ${message}` + ); + } +}; + +/** + * A wrapper function for handling the click action event. + * @param socket The socket connection + * @param data - the data of the click action event + * @category HelperFunctions + */ +const onDOMClickAction = async ( + data: { + selector: string; + url: string; + userId: string; + elementInfo?: any; + coordinates?: { x: number; y: number }; + }, + userId: string +) => { + logger.log("debug", "Handling click action event emitted from client"); + await handleWrapper(handleClickAction, userId, data); +}; + +/** + * Handles the keyboard action event. + * @param activeBrowser - the active remote browser {@link RemoteBrowser} + * @param page - the active page of the remote browser + * @param data - the data of the keyboard action event + * @category BrowserManagement + */ +const handleKeyboardAction = async ( + activeBrowser: RemoteBrowser, + page: Page, + data: { + selector: string; + key: string; + url: string; + userId: string; + inputType?: string; + } +) => { + try { + if (page.isClosed()) { + logger.log("debug", `Ignoring keyboard action event: page is closed`); + return; + } + + const generator = activeBrowser.generator; + await generator.onDOMKeyboardAction(page, data); + logger.log( + "debug", + `Keyboard action processed: ${data.key} on ${data.selector}` + ); + } catch (e) { + const { message } = e as Error; + logger.log("warn", `Error handling keyboard action event: ${message}`); + } +}; + +/** + * A wrapper function for handling the keyboard action event. + * @param socket The socket connection + * @param data - the data of the keyboard action event + * @category HelperFunctions + */ +const onDOMKeyboardAction = async ( + data: { + selector: string; + key: string; + url: string; + userId: string; + inputType?: string; + }, + userId: string +) => { + logger.log("debug", "Handling keyboard action event emitted from client"); + await handleWrapper(handleKeyboardAction, userId, data); +}; + +/** + * Handles the workflow pair event. + * @param activeBrowser - the active remote browser {@link RemoteBrowser} + * @param page - the active page of the remote browser + * @param data - the data of the workflow pair event + * @category BrowserManagement + */ +const handleWorkflowPair = async ( + activeBrowser: RemoteBrowser, + page: Page, + data: { pair: WhereWhatPair; userId: string } +) => { + try { + if (page.isClosed()) { + logger.log("debug", `Ignoring workflow pair event: page is closed`); + return; + } + + const generator = activeBrowser.generator; + await generator.onDOMWorkflowPair(page, data); + logger.log("debug", `Workflow pair processed from frontend`); + } catch (e) { + const { message } = e as Error; + logger.log("warn", `Error handling workflow pair event: ${message}`); + } +}; + +/** + * A wrapper function for handling the workflow pair event. + * @param socket The socket connection + * @param data - the data of the workflow pair event + * @category HelperFunctions + */ +const onDOMWorkflowPair = async ( + data: { pair: WhereWhatPair; userId: string }, + userId: string +) => { + logger.log("debug", "Handling workflow pair event emitted from client"); + await handleWrapper(handleWorkflowPair, userId, data); }; /** @@ -493,6 +817,10 @@ const registerInputHandlers = (socket: Socket, userId: string) => { socket.on("input:time", (data) => onTimeSelection(data, userId)); socket.on("input:datetime-local", (data) => onDateTimeLocalSelection(data, userId)); socket.on("action", (data) => onGenerateAction(data, userId)); + + socket.on("dom:click", (data) => onDOMClickAction(data, userId)); + socket.on("dom:keypress", (data) => onDOMKeyboardAction(data, userId)); + socket.on("dom:addpair", (data) => onDOMWorkflowPair(data, userId)); }; export default registerInputHandlers;