From 78cd3a38d8eee7a72b7d72b02c2d69e46e05b90a Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 22:40:37 +0530 Subject: [PATCH 01/42] feat: init user input handling --- server/src/browser-management/inputHandlers.ts | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 server/src/browser-management/inputHandlers.ts diff --git a/server/src/browser-management/inputHandlers.ts b/server/src/browser-management/inputHandlers.ts new file mode 100644 index 00000000..f1faa628 --- /dev/null +++ b/server/src/browser-management/inputHandlers.ts @@ -0,0 +1,15 @@ +/** + * A set of functions handling reproduction of user input + * on the remote browser instance as well as the generation of workflow pairs. + * These functions are called by the client through socket communication. + */ +import { Socket } from 'socket.io'; + +import logger from "../logger"; +import { Coordinates, ScrollDeltas, KeyboardInput } from '../types'; +import { browserPool } from "../server"; +import { WorkflowGenerator } from "../workflow-management/classes/Generator"; +import { Page } from "playwright"; +import { throttle } from "../../../src/helpers/inputHelpers"; +import { CustomActions } from "../../../src/shared/types"; + From 7274753728077c9db1900b28072f254c16d9bb9f Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 22:50:12 +0530 Subject: [PATCH 02/42] feat: helper fxn for user input --- .../src/browser-management/inputHandlers.ts | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/server/src/browser-management/inputHandlers.ts b/server/src/browser-management/inputHandlers.ts index f1faa628..49a3bb11 100644 --- a/server/src/browser-management/inputHandlers.ts +++ b/server/src/browser-management/inputHandlers.ts @@ -13,3 +13,34 @@ import { Page } from "playwright"; import { throttle } from "../../../src/helpers/inputHelpers"; import { CustomActions } from "../../../src/shared/types"; + +const handleWrapper = async ( + handleCallback: ( + generator: WorkflowGenerator, + page: Page, + args?: any + ) => Promise, + args?: any +) => { + const id = browserPool.getActiveBrowserId(); + if (id) { + const activeBrowser = browserPool.getRemoteBrowser(id); + if (activeBrowser?.interpreter.interpretationInProgress() && !activeBrowser.interpreter.interpretationIsPaused) { + logger.log('debug', `Ignoring input, while interpretation is in progress`); + return; + } + const currentPage = activeBrowser?.getCurrentPage(); + if (currentPage && activeBrowser) { + if (args) { + await handleCallback(activeBrowser.generator, currentPage, args); + } else { + await handleCallback(activeBrowser.generator, currentPage); + } + } else { + logger.log('warn', `No active page for browser ${id}`); + } + } else { + logger.log('warn', `No active browser for id ${id}`); + } +} + From 23fdf8110c2e841fc20a4e3a7a4060b17dec253a Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 22:50:26 +0530 Subject: [PATCH 03/42] docs: helper fxn for user input --- server/src/browser-management/inputHandlers.ts | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/server/src/browser-management/inputHandlers.ts b/server/src/browser-management/inputHandlers.ts index 49a3bb11..ea28cb88 100644 --- a/server/src/browser-management/inputHandlers.ts +++ b/server/src/browser-management/inputHandlers.ts @@ -13,7 +13,18 @@ import { Page } from "playwright"; import { throttle } from "../../../src/helpers/inputHelpers"; import { CustomActions } from "../../../src/shared/types"; - +/** + * A wrapper function for handling user input. + * This function gets the active browser instance from the browser pool + * and passes necessary arguments to the appropriate handlers. + * e.g. {@link Generator}, {@link RemoteBrowser.currentPage} + * + * Also ignores any user input while interpretation is in progress. + * + * @param handleCallback The callback handler to be called + * @param args - arguments to be passed to the handler + * @category HelperFunctions + */ const handleWrapper = async ( handleCallback: ( generator: WorkflowGenerator, From 2c8d17f5e8bbaeeceaf9f3c536a801b971991494 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 22:51:37 +0530 Subject: [PATCH 04/42] feat: interface for custom actions --- server/src/browser-management/inputHandlers.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/server/src/browser-management/inputHandlers.ts b/server/src/browser-management/inputHandlers.ts index ea28cb88..655ff39b 100644 --- a/server/src/browser-management/inputHandlers.ts +++ b/server/src/browser-management/inputHandlers.ts @@ -55,3 +55,8 @@ const handleWrapper = async ( } } +interface CustomActionEventData { + action: CustomActions; + settings: any; +} + From 94e090ba027f27dd8d24189de8be859ff5cd35ed Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 22:51:51 +0530 Subject: [PATCH 05/42] docs: interface for custom actions --- server/src/browser-management/inputHandlers.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/src/browser-management/inputHandlers.ts b/server/src/browser-management/inputHandlers.ts index 655ff39b..977f7d20 100644 --- a/server/src/browser-management/inputHandlers.ts +++ b/server/src/browser-management/inputHandlers.ts @@ -55,6 +55,10 @@ const handleWrapper = async ( } } +/** + * An interface for custom action description. + * @category Types + */ interface CustomActionEventData { action: CustomActions; settings: any; From 3cdd3bf53407e7ade90d032dfa0a84ca9f4013d5 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 22:52:51 +0530 Subject: [PATCH 06/42] feat: wrapper for custom actions --- server/src/browser-management/inputHandlers.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/server/src/browser-management/inputHandlers.ts b/server/src/browser-management/inputHandlers.ts index 977f7d20..0ea938f0 100644 --- a/server/src/browser-management/inputHandlers.ts +++ b/server/src/browser-management/inputHandlers.ts @@ -64,3 +64,8 @@ interface CustomActionEventData { settings: any; } +const onGenerateAction = async (customActionEventData: CustomActionEventData) => { + logger.log('debug', `Generating ${customActionEventData.action} action emitted from client`); + await handleWrapper(handleGenerateAction, customActionEventData); +} + From 35876960fdc563c631a612f3ee41d945388600dc Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 22:53:09 +0530 Subject: [PATCH 07/42] docs: desc wrapper for custom actions --- server/src/browser-management/inputHandlers.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/server/src/browser-management/inputHandlers.ts b/server/src/browser-management/inputHandlers.ts index 0ea938f0..e653c7d0 100644 --- a/server/src/browser-management/inputHandlers.ts +++ b/server/src/browser-management/inputHandlers.ts @@ -64,6 +64,11 @@ interface CustomActionEventData { settings: any; } +/** + * A wrapper function for handling custom actions. + * @param customActionEventData The custom action event data + * @category HelperFunctions + */ const onGenerateAction = async (customActionEventData: CustomActionEventData) => { logger.log('debug', `Generating ${customActionEventData.action} action emitted from client`); await handleWrapper(handleGenerateAction, customActionEventData); From 6175d2a3a9fa14fe78f5caec218e4dba30e13427 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 22:54:15 +0530 Subject: [PATCH 08/42] feat: handle generation of custom action sin the workflow pair --- server/src/browser-management/inputHandlers.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/server/src/browser-management/inputHandlers.ts b/server/src/browser-management/inputHandlers.ts index e653c7d0..f29df05f 100644 --- a/server/src/browser-management/inputHandlers.ts +++ b/server/src/browser-management/inputHandlers.ts @@ -74,3 +74,8 @@ const onGenerateAction = async (customActionEventData: CustomActionEventData) => await handleWrapper(handleGenerateAction, customActionEventData); } +const handleGenerateAction = + async (generator: WorkflowGenerator, page: Page, {action, settings}: CustomActionEventData) => { + await generator.customAction(action, settings, page); +} + From 990f8441848ef1f48605329c7915ec275e0630c2 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 22:54:29 +0530 Subject: [PATCH 09/42] docs: handle generation of custom action sin the workflow pair --- server/src/browser-management/inputHandlers.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/server/src/browser-management/inputHandlers.ts b/server/src/browser-management/inputHandlers.ts index f29df05f..7c2dcc1a 100644 --- a/server/src/browser-management/inputHandlers.ts +++ b/server/src/browser-management/inputHandlers.ts @@ -74,6 +74,14 @@ const onGenerateAction = async (customActionEventData: CustomActionEventData) => await handleWrapper(handleGenerateAction, customActionEventData); } +/** + * Handles the generation of a cutom action workflow pair. + * @param generator The workflow generator + * @param page The active page + * @param action The custom action + * @param settings The custom action settings + * @category BrowserManagement + */ const handleGenerateAction = async (generator: WorkflowGenerator, page: Page, {action, settings}: CustomActionEventData) => { await generator.customAction(action, settings, page); From 15ffc2984b9e2d18e56598eb49971a0c68cc303a Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 22:55:12 +0530 Subject: [PATCH 10/42] feat: wrapper to handle mousedown event --- server/src/browser-management/inputHandlers.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/server/src/browser-management/inputHandlers.ts b/server/src/browser-management/inputHandlers.ts index 7c2dcc1a..8a4759c5 100644 --- a/server/src/browser-management/inputHandlers.ts +++ b/server/src/browser-management/inputHandlers.ts @@ -87,3 +87,13 @@ const handleGenerateAction = await generator.customAction(action, settings, page); } +/** + * A wrapper function for handling mousedown event. + * @param coordinates - coordinates of the mouse click + * @category HelperFunctions + */ +const onMousedown = async (coordinates: Coordinates) => { + logger.log('debug', 'Handling mousedown event emitted from client'); + await handleWrapper(handleMousedown, coordinates); +} + From c9c81f392e63d9ed6ac669d142b7844db50f5765 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 22:56:55 +0530 Subject: [PATCH 11/42] feat: handle mouse down --- server/src/browser-management/inputHandlers.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/server/src/browser-management/inputHandlers.ts b/server/src/browser-management/inputHandlers.ts index 8a4759c5..cfaa15c5 100644 --- a/server/src/browser-management/inputHandlers.ts +++ b/server/src/browser-management/inputHandlers.ts @@ -97,3 +97,13 @@ const onMousedown = async (coordinates: Coordinates) => { await handleWrapper(handleMousedown, coordinates); } + +const handleMousedown = async (generator: WorkflowGenerator, page: Page, { x, y }: Coordinates) => { + await generator.onClick({ x, y }, page); + const previousUrl = page.url(); + const tabsBeforeClick = page.context().pages().length; + await page.mouse.click(x, y); + + logger.log('debug', `Clicked on position x:${x}, y:${y}`); +}; + From 5815ca008999a2a39f82b97947b4aae27c2f893b Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 22:57:19 +0530 Subject: [PATCH 12/42] feat: check if any new page opened by click --- server/src/browser-management/inputHandlers.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/server/src/browser-management/inputHandlers.ts b/server/src/browser-management/inputHandlers.ts index cfaa15c5..c17af514 100644 --- a/server/src/browser-management/inputHandlers.ts +++ b/server/src/browser-management/inputHandlers.ts @@ -104,6 +104,17 @@ const handleMousedown = async (generator: WorkflowGenerator, page: Page, { x, y const tabsBeforeClick = page.context().pages().length; await page.mouse.click(x, y); + // check if any new page was opened by the click + const tabsAfterClick = page.context().pages().length; + const numOfNewPages = tabsAfterClick - tabsBeforeClick; + if (numOfNewPages > 0) { + for (let i = 1; i <= numOfNewPages; i++) { + const newPage = page.context().pages()[tabsAfterClick - i]; + if (newPage) { + generator.notifyOnNewTab(newPage, tabsAfterClick - i); + } + } + } logger.log('debug', `Clicked on position x:${x}, y:${y}`); }; From 9efe5b1c6b448d2eb75ec0fc457ef25addedc42c Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 22:57:47 +0530 Subject: [PATCH 13/42] feat: try if click caused a navigation to new url --- server/src/browser-management/inputHandlers.ts | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/server/src/browser-management/inputHandlers.ts b/server/src/browser-management/inputHandlers.ts index c17af514..dc845a20 100644 --- a/server/src/browser-management/inputHandlers.ts +++ b/server/src/browser-management/inputHandlers.ts @@ -103,7 +103,17 @@ const handleMousedown = async (generator: WorkflowGenerator, page: Page, { x, y const previousUrl = page.url(); const tabsBeforeClick = page.context().pages().length; await page.mouse.click(x, y); - + // try if the click caused a navigation to a new url + try { + await page.waitForNavigation({ timeout: 2000 }); + const currentUrl = page.url(); + if (currentUrl !== previousUrl) { + generator.notifyUrlChange(currentUrl); + } + } catch (e) { + const {message} = e as Error; + } //ignore possible timeouts + // check if any new page was opened by the click const tabsAfterClick = page.context().pages().length; const numOfNewPages = tabsAfterClick - tabsBeforeClick; From 528774e09c132fbd5d859fb98899c1ec4be89ae5 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 22:58:19 +0530 Subject: [PATCH 14/42] docs: handle mouse down --- server/src/browser-management/inputHandlers.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/server/src/browser-management/inputHandlers.ts b/server/src/browser-management/inputHandlers.ts index dc845a20..03fe515d 100644 --- a/server/src/browser-management/inputHandlers.ts +++ b/server/src/browser-management/inputHandlers.ts @@ -97,7 +97,16 @@ const onMousedown = async (coordinates: Coordinates) => { await handleWrapper(handleMousedown, coordinates); } - +/** + * A mousedown event handler. + * Reproduces the click on the remote browser instance + * and generates pair data for the recorded workflow. + * @param generator - the workflow generator {@link Generator} + * @param page - the active page of the remote browser + * @param x - the x coordinate of the mousedown event + * @param y - the y coordinate of the mousedown event + * @category BrowserManagement + */ const handleMousedown = async (generator: WorkflowGenerator, page: Page, { x, y }: Coordinates) => { await generator.onClick({ x, y }, page); const previousUrl = page.url(); From d042a646120b47c348f6d471f33fd238d145b57d Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 22:58:51 +0530 Subject: [PATCH 15/42] feat: wrapper to handle wheel event --- server/src/browser-management/inputHandlers.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/server/src/browser-management/inputHandlers.ts b/server/src/browser-management/inputHandlers.ts index 03fe515d..94e2ec0c 100644 --- a/server/src/browser-management/inputHandlers.ts +++ b/server/src/browser-management/inputHandlers.ts @@ -137,3 +137,13 @@ const handleMousedown = async (generator: WorkflowGenerator, page: Page, { x, y logger.log('debug', `Clicked on position x:${x}, y:${y}`); }; +/** + * A wrapper function for handling the wheel event. + * @param scrollDeltas - the scroll deltas of the wheel event + * @category HelperFunctions + */ +const onWheel = async (scrollDeltas: ScrollDeltas) => { + logger.log('debug', 'Handling scroll event emitted from client'); + await handleWrapper(handleWheel, scrollDeltas); +}; + From e61990bca35641de03c824872d51a10f859ef910 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 22:59:21 +0530 Subject: [PATCH 16/42] feat: wheel handler --- server/src/browser-management/inputHandlers.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/server/src/browser-management/inputHandlers.ts b/server/src/browser-management/inputHandlers.ts index 94e2ec0c..000244da 100644 --- a/server/src/browser-management/inputHandlers.ts +++ b/server/src/browser-management/inputHandlers.ts @@ -147,3 +147,8 @@ const onWheel = async (scrollDeltas: ScrollDeltas) => { await handleWrapper(handleWheel, scrollDeltas); }; +const handleWheel = async (generator: WorkflowGenerator, page: Page, { deltaX, deltaY }: ScrollDeltas) => { + await page.mouse.wheel(deltaX, deltaY); + logger.log('debug', `Scrolled horizontally ${deltaX} pixels and vertically ${deltaY} pixels`); +}; + From d99a29658bb7c5f34ce1966d7b2de78e09348e6c Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 22:59:31 +0530 Subject: [PATCH 17/42] docs: wheel event handler --- server/src/browser-management/inputHandlers.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/server/src/browser-management/inputHandlers.ts b/server/src/browser-management/inputHandlers.ts index 000244da..10d5f1ab 100644 --- a/server/src/browser-management/inputHandlers.ts +++ b/server/src/browser-management/inputHandlers.ts @@ -147,6 +147,17 @@ const onWheel = async (scrollDeltas: ScrollDeltas) => { await handleWrapper(handleWheel, scrollDeltas); }; +/** + * A wheel event handler. + * Reproduces the wheel event on the remote browser instance. + * Scroll is not generated for the workflow pair. This is because + * Playwright scrolls elements into focus on any action. + * @param generator - the workflow generator {@link Generator} + * @param page - the active page of the remote browser + * @param deltaX - the delta x of the wheel event + * @param deltaY - the delta y of the wheel event + * @category BrowserManagement + */ const handleWheel = async (generator: WorkflowGenerator, page: Page, { deltaX, deltaY }: ScrollDeltas) => { await page.mouse.wheel(deltaX, deltaY); logger.log('debug', `Scrolled horizontally ${deltaX} pixels and vertically ${deltaY} pixels`); From 071f835a2ef4ce53f6af955a9eb33753bb5f5a60 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 23:00:04 +0530 Subject: [PATCH 18/42] feat: wrapper to handle mouse move --- server/src/browser-management/inputHandlers.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/server/src/browser-management/inputHandlers.ts b/server/src/browser-management/inputHandlers.ts index 10d5f1ab..2f68c864 100644 --- a/server/src/browser-management/inputHandlers.ts +++ b/server/src/browser-management/inputHandlers.ts @@ -163,3 +163,13 @@ const handleWheel = async (generator: WorkflowGenerator, page: Page, { deltaX, d logger.log('debug', `Scrolled horizontally ${deltaX} pixels and vertically ${deltaY} pixels`); }; +/** + * A wrapper function for handling the mousemove event. + * @param coordinates - the coordinates of the mousemove event + * @category HelperFunctions + */ +const onMousemove = async (coordinates: Coordinates) => { + logger.log('debug', 'Handling mousemove event emitted from client'); + await handleWrapper(handleMousemove, coordinates); +} + From 9361ad916fc03d18e04b5c84220dec3dc9da0483 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 23:00:27 +0530 Subject: [PATCH 19/42] feat: handle mouse move --- server/src/browser-management/inputHandlers.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/server/src/browser-management/inputHandlers.ts b/server/src/browser-management/inputHandlers.ts index 2f68c864..0142d956 100644 --- a/server/src/browser-management/inputHandlers.ts +++ b/server/src/browser-management/inputHandlers.ts @@ -173,3 +173,17 @@ const onMousemove = async (coordinates: Coordinates) => { await handleWrapper(handleMousemove, coordinates); } + +const handleMousemove = async (generator: WorkflowGenerator, page: Page, { x, y }: Coordinates) => { + try { + await page.mouse.move(x, y); + throttle(async () => { + await generator.generateDataForHighlighter(page, { x, y }); + }, 100)(); + logger.log('debug', `Moved over position x:${x}, y:${y}`); + } catch (e) { + const { message } = e as Error; + logger.log('error', message); + } +} + From 95b59c3e1f7e36d7162d8071ccc393d87d4e672a Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 23:00:35 +0530 Subject: [PATCH 20/42] docs: handle mouse move --- server/src/browser-management/inputHandlers.ts | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/server/src/browser-management/inputHandlers.ts b/server/src/browser-management/inputHandlers.ts index 0142d956..d3efd223 100644 --- a/server/src/browser-management/inputHandlers.ts +++ b/server/src/browser-management/inputHandlers.ts @@ -173,7 +173,17 @@ const onMousemove = async (coordinates: Coordinates) => { await handleWrapper(handleMousemove, coordinates); } - +/** + * A mousemove event handler. + * Reproduces the mousemove event on the remote browser instance + * and generates data for the client's highlighter. + * Mousemove is also not reflected in the workflow. + * @param generator - the workflow generator {@link Generator} + * @param page - the active page of the remote browser + * @param x - the x coordinate of the mousemove event + * @param y - the y coordinate of the mousemove event + * @category BrowserManagement + */ const handleMousemove = async (generator: WorkflowGenerator, page: Page, { x, y }: Coordinates) => { try { await page.mouse.move(x, y); From 67e093604c590c354be7788f4e872a87be796a34 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 23:00:59 +0530 Subject: [PATCH 21/42] feat: wrapper to handle key down --- server/src/browser-management/inputHandlers.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/server/src/browser-management/inputHandlers.ts b/server/src/browser-management/inputHandlers.ts index d3efd223..5920a5f3 100644 --- a/server/src/browser-management/inputHandlers.ts +++ b/server/src/browser-management/inputHandlers.ts @@ -197,3 +197,13 @@ const handleMousemove = async (generator: WorkflowGenerator, page: Page, { x, y } } +/** + * A wrapper function for handling the keydown event. + * @param keyboardInput - the keyboard input of the keydown event + * @category HelperFunctions + */ +const onKeydown = async (keyboardInput: KeyboardInput) => { + logger.log('debug', 'Handling keydown event emitted from client'); + await handleWrapper(handleKeydown, keyboardInput); +} + From 5cf36dd81b6693215fb2aba98d9241ab87001f95 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 23:01:30 +0530 Subject: [PATCH 22/42] feat: keydown event handler --- server/src/browser-management/inputHandlers.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/server/src/browser-management/inputHandlers.ts b/server/src/browser-management/inputHandlers.ts index 5920a5f3..29eae6ae 100644 --- a/server/src/browser-management/inputHandlers.ts +++ b/server/src/browser-management/inputHandlers.ts @@ -207,3 +207,10 @@ const onKeydown = async (keyboardInput: KeyboardInput) => { await handleWrapper(handleKeydown, keyboardInput); } + +const handleKeydown = async (generator: WorkflowGenerator, page: Page, { key, coordinates }: KeyboardInput) => { + await page.keyboard.down(key); + await generator.onKeyboardInput(key, coordinates, page); + logger.log('debug', `Key ${key} pressed`); +}; + From c6a84f2b142297d6115c290bbb04d403d60e3809 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 23:01:38 +0530 Subject: [PATCH 23/42] docs: keydown event handler --- server/src/browser-management/inputHandlers.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/server/src/browser-management/inputHandlers.ts b/server/src/browser-management/inputHandlers.ts index 29eae6ae..94e35eae 100644 --- a/server/src/browser-management/inputHandlers.ts +++ b/server/src/browser-management/inputHandlers.ts @@ -207,7 +207,16 @@ const onKeydown = async (keyboardInput: KeyboardInput) => { await handleWrapper(handleKeydown, keyboardInput); } - +/** + * A keydown event handler. + * Reproduces the keydown event on the remote browser instance + * and generates the workflow pair data. + * @param generator - the workflow generator {@link Generator} + * @param page - the active page of the remote browser + * @param key - the pressed key + * @param coordinates - the coordinates, where the keydown event happened + * @category BrowserManagement + */ const handleKeydown = async (generator: WorkflowGenerator, page: Page, { key, coordinates }: KeyboardInput) => { await page.keyboard.down(key); await generator.onKeyboardInput(key, coordinates, page); From 4514a3a4af8326dcf5c517ae1d2618b88ef23231 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 23:01:58 +0530 Subject: [PATCH 24/42] feat: wrapper to handle key up --- server/src/browser-management/inputHandlers.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/server/src/browser-management/inputHandlers.ts b/server/src/browser-management/inputHandlers.ts index 94e35eae..0d7c4abc 100644 --- a/server/src/browser-management/inputHandlers.ts +++ b/server/src/browser-management/inputHandlers.ts @@ -223,3 +223,13 @@ const handleKeydown = async (generator: WorkflowGenerator, page: Page, { key, co logger.log('debug', `Key ${key} pressed`); }; +/** + * A wrapper function for handling the keyup event. + * @param keyboardInput - the keyboard input of the keyup event + * @category HelperFunctions + */ +const onKeyup = async (keyboardInput: KeyboardInput) => { + logger.log('debug', 'Handling keyup event emitted from client'); + await handleWrapper(handleKeyup, keyboardInput); +} + From cb5407179e9c888ecf8371f7b22cfb26c958e167 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 23:02:28 +0530 Subject: [PATCH 25/42] feat: keydown up handler --- server/src/browser-management/inputHandlers.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/server/src/browser-management/inputHandlers.ts b/server/src/browser-management/inputHandlers.ts index 0d7c4abc..793a81dc 100644 --- a/server/src/browser-management/inputHandlers.ts +++ b/server/src/browser-management/inputHandlers.ts @@ -233,3 +233,9 @@ const onKeyup = async (keyboardInput: KeyboardInput) => { await handleWrapper(handleKeyup, keyboardInput); } + +const handleKeyup = async (generator: WorkflowGenerator, page: Page, key: string) => { + await page.keyboard.up(key); + logger.log('debug', `Key ${key} unpressed`); +}; + From 8adbdd86b8c37307bec8114893d07835d9e43a32 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 23:02:37 +0530 Subject: [PATCH 26/42] docs: keydown up handler --- server/src/browser-management/inputHandlers.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/server/src/browser-management/inputHandlers.ts b/server/src/browser-management/inputHandlers.ts index 793a81dc..cb435765 100644 --- a/server/src/browser-management/inputHandlers.ts +++ b/server/src/browser-management/inputHandlers.ts @@ -233,7 +233,15 @@ const onKeyup = async (keyboardInput: KeyboardInput) => { await handleWrapper(handleKeyup, keyboardInput); } - +/** + * A keyup event handler. + * Reproduces the keyup event on the remote browser instance. + * Does not generate any data - keyup is not reflected in the workflow. + * @param generator - the workflow generator {@link Generator} + * @param page - the active page of the remote browser + * @param key - the released key + * @category BrowserManagement + */ const handleKeyup = async (generator: WorkflowGenerator, page: Page, key: string) => { await page.keyboard.up(key); logger.log('debug', `Key ${key} unpressed`); From ee3cd42bd1f474b7de3d487ea20c59405eb5fd6b Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 23:02:59 +0530 Subject: [PATCH 27/42] feat: wrapper for url change --- server/src/browser-management/inputHandlers.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/server/src/browser-management/inputHandlers.ts b/server/src/browser-management/inputHandlers.ts index cb435765..41c04262 100644 --- a/server/src/browser-management/inputHandlers.ts +++ b/server/src/browser-management/inputHandlers.ts @@ -247,3 +247,13 @@ const handleKeyup = async (generator: WorkflowGenerator, page: Page, key: string logger.log('debug', `Key ${key} unpressed`); }; +/** + * A wrapper function for handling the url change event. + * @param url - the new url of the page + * @category HelperFunctions + */ +const onChangeUrl = async (url: string) => { + logger.log('debug', 'Handling change url event emitted from client'); + await handleWrapper(handleChangeUrl, url); +} + From 0c0698ebb5ff19f6cdf16533c6310b3209ef66b3 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 23:03:18 +0530 Subject: [PATCH 28/42] feat: handle url change --- server/src/browser-management/inputHandlers.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/server/src/browser-management/inputHandlers.ts b/server/src/browser-management/inputHandlers.ts index 41c04262..62670030 100644 --- a/server/src/browser-management/inputHandlers.ts +++ b/server/src/browser-management/inputHandlers.ts @@ -257,3 +257,19 @@ const onChangeUrl = async (url: string) => { await handleWrapper(handleChangeUrl, url); } + +const handleChangeUrl = async (generator: WorkflowGenerator, page: Page, url: string) => { + if (url) { + await generator.onChangeUrl(url, page); + try { + await page.goto(url); + logger.log('debug', `Went to ${url}`); + } catch (e) { + const {message} = e as Error; + logger.log('error', message); + } + } else { + logger.log('warn', `No url provided`); + } +}; + From ed586fd3a79cc42b9ad3955f54566aecfbd42f1e Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 23:03:26 +0530 Subject: [PATCH 29/42] docs: handle url change --- server/src/browser-management/inputHandlers.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/server/src/browser-management/inputHandlers.ts b/server/src/browser-management/inputHandlers.ts index 62670030..3bb1bc71 100644 --- a/server/src/browser-management/inputHandlers.ts +++ b/server/src/browser-management/inputHandlers.ts @@ -257,7 +257,14 @@ const onChangeUrl = async (url: string) => { await handleWrapper(handleChangeUrl, url); } - +/** + * An url change event handler. + * Navigates the page to the given url and generates data for the workflow. + * @param generator - the workflow generator {@link Generator} + * @param page - the active page of the remote browser + * @param url - the new url of the page + * @category BrowserManagement + */ const handleChangeUrl = async (generator: WorkflowGenerator, page: Page, url: string) => { if (url) { await generator.onChangeUrl(url, page); From 06c564e757eaf08081018926ba50948f79340822 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 23:03:46 +0530 Subject: [PATCH 30/42] feat: refresh event wrapper --- server/src/browser-management/inputHandlers.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/server/src/browser-management/inputHandlers.ts b/server/src/browser-management/inputHandlers.ts index 3bb1bc71..037c1d98 100644 --- a/server/src/browser-management/inputHandlers.ts +++ b/server/src/browser-management/inputHandlers.ts @@ -280,3 +280,12 @@ const handleChangeUrl = async (generator: WorkflowGenerator, page: Page, url: st } }; +/** + * A wrapper function for handling the refresh event. + * @category HelperFunctions + */ +const onRefresh = async () => { + logger.log('debug', 'Handling refresh event emitted from client'); + await handleWrapper(handleRefresh); +} + From 17b89abe1ec370144c2e335785332b66c7d8032c Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 23:04:04 +0530 Subject: [PATCH 31/42] feat: refresh event handler --- server/src/browser-management/inputHandlers.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/server/src/browser-management/inputHandlers.ts b/server/src/browser-management/inputHandlers.ts index 037c1d98..4259923f 100644 --- a/server/src/browser-management/inputHandlers.ts +++ b/server/src/browser-management/inputHandlers.ts @@ -289,3 +289,9 @@ const onRefresh = async () => { await handleWrapper(handleRefresh); } + +const handleRefresh = async (generator: WorkflowGenerator, page: Page) => { + await page.reload(); + logger.log('debug', `Page refreshed.`); +}; + From 9e46ede9a9afad061c072e7f4fcc9cc520d6fea4 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 23:04:12 +0530 Subject: [PATCH 32/42] docs: refresh event handler --- server/src/browser-management/inputHandlers.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/server/src/browser-management/inputHandlers.ts b/server/src/browser-management/inputHandlers.ts index 4259923f..1b9e4390 100644 --- a/server/src/browser-management/inputHandlers.ts +++ b/server/src/browser-management/inputHandlers.ts @@ -289,7 +289,13 @@ const onRefresh = async () => { await handleWrapper(handleRefresh); } - +/** + * A refresh event handler. + * Refreshes the page. This is not reflected in the workflow. + * @param generator - the workflow generator {@link Generator} + * @param page - the active page of the remote browser + * @category BrowserManagement + */ const handleRefresh = async (generator: WorkflowGenerator, page: Page) => { await page.reload(); logger.log('debug', `Page refreshed.`); From 406d1d40bbef931bf0db1fee719057f513e1a62c Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 23:04:38 +0530 Subject: [PATCH 33/42] feat: go back event wrapper --- server/src/browser-management/inputHandlers.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/server/src/browser-management/inputHandlers.ts b/server/src/browser-management/inputHandlers.ts index 1b9e4390..028dacb9 100644 --- a/server/src/browser-management/inputHandlers.ts +++ b/server/src/browser-management/inputHandlers.ts @@ -301,3 +301,14 @@ const handleRefresh = async (generator: WorkflowGenerator, page: Page) => { logger.log('debug', `Page refreshed.`); }; +/** + * A wrapper function for handling the go back event. + * @category HelperFunctions + */ +const onGoBack = async () => { + logger.log('debug', 'Handling refresh event emitted from client'); + await handleWrapper(handleGoBack); +} + + + From 97174efac0afbf4d7956ca6c22a733ae2297d40b Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 23:04:52 +0530 Subject: [PATCH 34/42] feat: handle go back event --- server/src/browser-management/inputHandlers.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/server/src/browser-management/inputHandlers.ts b/server/src/browser-management/inputHandlers.ts index 028dacb9..c640fe97 100644 --- a/server/src/browser-management/inputHandlers.ts +++ b/server/src/browser-management/inputHandlers.ts @@ -311,4 +311,9 @@ const onGoBack = async () => { } +const handleGoBack = async (generator: WorkflowGenerator, page: Page) => { + await page.goBack({waitUntil: 'commit'}); + generator.onGoBack(page.url()); + logger.log('debug', 'Page went back') +}; From d0a2f4fe73f08b1e9f5a4379713c54d220e86487 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 23:05:00 +0530 Subject: [PATCH 35/42] docs: handle go back event --- server/src/browser-management/inputHandlers.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/server/src/browser-management/inputHandlers.ts b/server/src/browser-management/inputHandlers.ts index c640fe97..3d3b6937 100644 --- a/server/src/browser-management/inputHandlers.ts +++ b/server/src/browser-management/inputHandlers.ts @@ -310,7 +310,13 @@ const onGoBack = async () => { await handleWrapper(handleGoBack); } - +/** + * A go back event handler. + * Navigates the page back and generates data for the workflow. + * @param generator - the workflow generator {@link Generator} + * @param page - the active page of the remote browser + * @category BrowserManagement + */ const handleGoBack = async (generator: WorkflowGenerator, page: Page) => { await page.goBack({waitUntil: 'commit'}); generator.onGoBack(page.url()); From f62c1c12b6f8a3282a681c9fb50a074e3f375b8c Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 23:05:19 +0530 Subject: [PATCH 36/42] feat: go forward event wrapper --- server/src/browser-management/inputHandlers.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/server/src/browser-management/inputHandlers.ts b/server/src/browser-management/inputHandlers.ts index 3d3b6937..713d9ccf 100644 --- a/server/src/browser-management/inputHandlers.ts +++ b/server/src/browser-management/inputHandlers.ts @@ -323,3 +323,13 @@ const handleGoBack = async (generator: WorkflowGenerator, page: Page) => { logger.log('debug', 'Page went back') }; +/** + * A wrapper function for handling the go forward event. + * @category HelperFunctions + */ +const onGoForward = async () => { + logger.log('debug', 'Handling refresh event emitted from client'); + await handleWrapper(handleGoForward); +} + + From 2d3bb0c3f42367c9e248f2c44e9dd22281676e39 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 23:05:38 +0530 Subject: [PATCH 37/42] feat: go forward event handler --- server/src/browser-management/inputHandlers.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/server/src/browser-management/inputHandlers.ts b/server/src/browser-management/inputHandlers.ts index 713d9ccf..f6bd327d 100644 --- a/server/src/browser-management/inputHandlers.ts +++ b/server/src/browser-management/inputHandlers.ts @@ -333,3 +333,9 @@ const onGoForward = async () => { } +const handleGoForward = async (generator: WorkflowGenerator, page: Page) => { + await page.goForward({waitUntil: 'commit'}); + generator.onGoForward(page.url()); + logger.log('debug', 'Page went forward'); +}; + From c3454ae98d1ee790437acd756665fa41beae3b9c Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 23:05:47 +0530 Subject: [PATCH 38/42] docs: go forward event handler --- server/src/browser-management/inputHandlers.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/server/src/browser-management/inputHandlers.ts b/server/src/browser-management/inputHandlers.ts index f6bd327d..0f17db1e 100644 --- a/server/src/browser-management/inputHandlers.ts +++ b/server/src/browser-management/inputHandlers.ts @@ -332,7 +332,13 @@ const onGoForward = async () => { await handleWrapper(handleGoForward); } - +/** + * A go forward event handler. + * Navigates the page forward and generates data for the workflow. + * @param generator - the workflow generator {@link Generator} + * @param page - the active page of the remote browser + * @category BrowserManagement + */ const handleGoForward = async (generator: WorkflowGenerator, page: Page) => { await page.goForward({waitUntil: 'commit'}); generator.onGoForward(page.url()); From 7d10e924ae2fa5f83ba951ea04ff2e51034feeb2 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 23:06:10 +0530 Subject: [PATCH 39/42] feat: register input handlers --- server/src/browser-management/inputHandlers.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/server/src/browser-management/inputHandlers.ts b/server/src/browser-management/inputHandlers.ts index 0f17db1e..93e471a3 100644 --- a/server/src/browser-management/inputHandlers.ts +++ b/server/src/browser-management/inputHandlers.ts @@ -345,3 +345,17 @@ const handleGoForward = async (generator: WorkflowGenerator, page: Page) => { logger.log('debug', 'Page went forward'); }; +const registerInputHandlers = (socket: Socket) => { + socket.on("input:mousedown", onMousedown); + socket.on("input:wheel", onWheel); + socket.on("input:mousemove", onMousemove); + socket.on("input:keydown", onKeydown); + socket.on("input:keyup", onKeyup); + socket.on("input:url", onChangeUrl); + socket.on("input:refresh", onRefresh); + socket.on("input:back", onGoBack); + socket.on("input:forward", onGoForward); + socket.on("action", onGenerateAction); +}; + +export default registerInputHandlers; From 9df4a77e250c319968aea0a365480c3eb3e416a2 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 23:06:22 +0530 Subject: [PATCH 40/42] docs: register input handlers --- .../src/browser-management/inputHandlers.ts | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/server/src/browser-management/inputHandlers.ts b/server/src/browser-management/inputHandlers.ts index 93e471a3..544f2052 100644 --- a/server/src/browser-management/inputHandlers.ts +++ b/server/src/browser-management/inputHandlers.ts @@ -345,6 +345,29 @@ const handleGoForward = async (generator: WorkflowGenerator, page: Page) => { logger.log('debug', 'Page went forward'); }; +/** + * Helper function for registering the handlers onto established websocket connection. + * Registers: + * - mousedownHandler + * - wheelHandler + * - mousemoveHandler + * - keydownHandler + * - keyupHandler + * - changeUrlHandler + * - refreshHandler + * - goBackHandler + * - goForwardHandler + * - onGenerateAction + * input handlers. + * + * All these handlers first generates the workflow pair data + * and then calls the corresponding playwright's function to emulate the input. + * They also ignore any user input while interpretation is in progress. + * + * @param socket websocket with established connection + * @returns void + * @category BrowserManagement + */ const registerInputHandlers = (socket: Socket) => { socket.on("input:mousedown", onMousedown); socket.on("input:wheel", onWheel); From 1ae23f8f650072aa0e8cf915994e58549b559fa3 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 23:06:36 +0530 Subject: [PATCH 41/42] chore: lint --- .../src/browser-management/inputHandlers.ts | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/server/src/browser-management/inputHandlers.ts b/server/src/browser-management/inputHandlers.ts index 544f2052..22d503e2 100644 --- a/server/src/browser-management/inputHandlers.ts +++ b/server/src/browser-management/inputHandlers.ts @@ -26,12 +26,12 @@ import { CustomActions } from "../../../src/shared/types"; * @category HelperFunctions */ const handleWrapper = async ( - handleCallback: ( - generator: WorkflowGenerator, - page: Page, + handleCallback: ( + generator: WorkflowGenerator, + page: Page, + args?: any + ) => Promise, args?: any - ) => Promise, - args?: any ) => { const id = browserPool.getActiveBrowserId(); if (id) { @@ -83,9 +83,9 @@ const onGenerateAction = async (customActionEventData: CustomActionEventData) => * @category BrowserManagement */ const handleGenerateAction = - async (generator: WorkflowGenerator, page: Page, {action, settings}: CustomActionEventData) => { - await generator.customAction(action, settings, page); -} + async (generator: WorkflowGenerator, page: Page, { action, settings }: CustomActionEventData) => { + await generator.customAction(action, settings, page); + } /** * A wrapper function for handling mousedown event. @@ -120,7 +120,7 @@ const handleMousedown = async (generator: WorkflowGenerator, page: Page, { x, y generator.notifyUrlChange(currentUrl); } } catch (e) { - const {message} = e as Error; + const { message } = e as Error; } //ignore possible timeouts // check if any new page was opened by the click @@ -272,7 +272,7 @@ const handleChangeUrl = async (generator: WorkflowGenerator, page: Page, url: st await page.goto(url); logger.log('debug', `Went to ${url}`); } catch (e) { - const {message} = e as Error; + const { message } = e as Error; logger.log('error', message); } } else { @@ -318,7 +318,7 @@ const onGoBack = async () => { * @category BrowserManagement */ const handleGoBack = async (generator: WorkflowGenerator, page: Page) => { - await page.goBack({waitUntil: 'commit'}); + await page.goBack({ waitUntil: 'commit' }); generator.onGoBack(page.url()); logger.log('debug', 'Page went back') }; @@ -340,7 +340,7 @@ const onGoForward = async () => { * @category BrowserManagement */ const handleGoForward = async (generator: WorkflowGenerator, page: Page) => { - await page.goForward({waitUntil: 'commit'}); + await page.goForward({ waitUntil: 'commit' }); generator.onGoForward(page.url()); logger.log('debug', 'Page went forward'); }; From 62615a19efe66688307d46e8ba4728773a49567a Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 23:07:00 +0530 Subject: [PATCH 42/42] fix: typo --- server/src/browser-management/inputHandlers.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/browser-management/inputHandlers.ts b/server/src/browser-management/inputHandlers.ts index 22d503e2..d6902b3f 100644 --- a/server/src/browser-management/inputHandlers.ts +++ b/server/src/browser-management/inputHandlers.ts @@ -75,7 +75,7 @@ const onGenerateAction = async (customActionEventData: CustomActionEventData) => } /** - * Handles the generation of a cutom action workflow pair. + * Handles the generation of a custom action workflow pair. * @param generator The workflow generator * @param page The active page * @param action The custom action