vodorod/parcer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add dom events, active browser param

Browse Source
This commit is contained in:
Rohit
2025-06-23 14:00:23 +05:30
parent 7322e1f1ec
commit 895a4f9ace
1 changed files with 405 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files

482
server/src/browser-management/inputHandlers.ts
View File

@@ -11,6 +11,8 @@ import { WorkflowGenerator } from "../workflow-management/classes/Generator";
import { Page } from "playwright";
import { throttle } from "../../../src/helpers/inputHelpers";
import { CustomActions } from "../../../src/shared/types";
import { WhereWhatPair } from "maxun-core";
import { RemoteBrowser } from './classes/RemoteBrowser';
/**
* A wrapper function for handling user input.
@@ -27,7 +29,7 @@ import { CustomActions } from "../../../src/shared/types";
*/
const handleWrapper = async (
handleCallback: (
generator: WorkflowGenerator,
activeBrowser: RemoteBrowser,
page: Page,
args?: any
) => Promise<void>,
@@ -44,9 +46,9 @@ const handleWrapper = async (
const currentPage = activeBrowser?.getCurrentPage();
if (currentPage && activeBrowser) {
if (args) {
await handleCallback(activeBrowser.generator, currentPage, args);
await handleCallback(activeBrowser, currentPage, args);
} else {
await handleCallback(activeBrowser.generator, currentPage);
await handleCallback(activeBrowser, currentPage);
}
} else {
logger.log('warn', `No active page for browser ${id}`);
@@ -85,8 +87,19 @@ const onGenerateAction = async (customActionEventData: CustomActionEventData, us
* @category BrowserManagement
*/
const handleGenerateAction =
async (generator: WorkflowGenerator, page: Page, { action, settings }: CustomActionEventData) => {
await generator.customAction(action, settings, page);
async (activeBrowser: RemoteBrowser, page: Page, { action, settings }: CustomActionEventData) => {
try {
if (page.isClosed()) {
logger.log("debug", `Ignoring generate action event: page is closed`);
return;
}
const generator = activeBrowser.generator;
await generator.customAction(action, settings, page);
} catch (e) {
const { message } = e as Error;
logger.log("warn", `Error handling generate action event: ${message}`);
}
}
/**
@@ -104,40 +117,51 @@ const onMousedown = async (coordinates: Coordinates, userId: string) => {
* A mousedown event handler.
* Reproduces the click on the remote browser instance
* and generates pair data for the recorded workflow.
* @param generator - the workflow generator {@link Generator}
* @param activeBrowser - the active remote browser {@link RemoteBrowser}
* @param page - the active page of the remote browser
* @param x - the x coordinate of the mousedown event
* @param y - the y coordinate of the mousedown event
* @category BrowserManagement
*/
const handleMousedown = async (generator: WorkflowGenerator, page: Page, { x, y }: Coordinates) => {
const handleMousedown = async (activeBrowser: RemoteBrowser, page: Page, { x, y }: Coordinates) => {
try {
if (page.isClosed()) {
logger.log("debug", `Ignoring mousedown event: page is closed`);
return;
}
const generator = activeBrowser.generator;
await generator.onClick({ x, y }, page);
const previousUrl = page.url();
const tabsBeforeClick = page.context().pages().length;
await page.mouse.click(x, y);
// try if the click caused a navigation to a new url
try {
await page.waitForNavigation({ timeout: 2000 });
const currentUrl = page.url();
if (currentUrl !== previousUrl) {
generator.notifyUrlChange(currentUrl);
}
await page.waitForNavigation({ timeout: 2000 });
const currentUrl = page.url();
if (currentUrl !== previousUrl) {
generator.notifyUrlChange(currentUrl);
}
} catch (e) {
const { message } = e as Error;
const { message } = e as Error;
} //ignore possible timeouts
// check if any new page was opened by the click
const tabsAfterClick = page.context().pages().length;
const numOfNewPages = tabsAfterClick - tabsBeforeClick;
if (numOfNewPages > 0) {
for (let i = 1; i <= numOfNewPages; i++) {
const newPage = page.context().pages()[tabsAfterClick - i];
if (newPage) {
generator.notifyOnNewTab(newPage, tabsAfterClick - i);
}
for (let i = 1; i <= numOfNewPages; i++) {
const newPage = page.context().pages()[tabsAfterClick - i];
if (newPage) {
generator.notifyOnNewTab(newPage, tabsAfterClick - i);
}
}
}
logger.log('debug', `Clicked on position x:${x}, y:${y}`);
logger.log("debug", `Clicked on position x:${x}, y:${y}`);
} catch (e) {
const { message } = e as Error;
logger.log("warn", `Error handling mousedown event: ${message}`);
}
};
/**
@@ -156,15 +180,16 @@ const onWheel = async (scrollDeltas: ScrollDeltas, userId: string) => {
* Reproduces the wheel event on the remote browser instance.
* Scroll is not generated for the workflow pair. This is because
* Playwright scrolls elements into focus on any action.
* @param generator - the workflow generator {@link Generator}
* @param activeBrowser - the active remote browser {@link RemoteBrowser}
* @param page - the active page of the remote browser
* @param deltaX - the delta x of the wheel event
* @param deltaY - the delta y of the wheel event
* @category BrowserManagement
*/
const handleWheel = async (generator: WorkflowGenerator, page: Page, { deltaX, deltaY }: ScrollDeltas) => {
const handleWheel = async (activeBrowser: RemoteBrowser, page: Page, { deltaX, deltaY }: ScrollDeltas) => {
try {
if (page.isClosed()) {
logger.log("debug", `Ignoring wheel event: page is closed`);
return;
}
@@ -194,28 +219,30 @@ const onMousemove = async (coordinates: Coordinates, userId: string) => {
* Reproduces the mousemove event on the remote browser instance
* and generates data for the client's highlighter.
* Mousemove is also not reflected in the workflow.
* @param generator - the workflow generator {@link Generator}
* @param activeBrowser - the active remote browser {@link RemoteBrowser}
* @param page - the active page of the remote browser
* @param x - the x coordinate of the mousemove event
* @param y - the y coordinate of the mousemove event
* @category BrowserManagement
*/
const handleMousemove = async (generator: WorkflowGenerator, page: Page, { x, y }: Coordinates) => {
const handleMousemove = async (activeBrowser: RemoteBrowser, page: Page, { x, y }: Coordinates) => {
try {
if (page.isClosed()) {
logger.log('debug', `Ignoring mousemove event: page is closed`);
logger.log("debug", `Ignoring mousemove event: page is closed`);
return;
}
const generator = activeBrowser.generator;
await page.mouse.move(x, y);
throttle(async () => {
if (!page.isClosed()) {
await generator.generateDataForHighlighter(page, { x, y });
}
}, 100)();
logger.log('debug', `Moved over position x:${x}, y:${y}`);
logger.log("debug", `Moved over position x:${x}, y:${y}`);
} catch (e) {
const { message } = e as Error;
logger.log('error', message);
logger.log("error", message);
}
}
@@ -234,28 +261,50 @@ const onKeydown = async (keyboardInput: KeyboardInput, userId: string) => {
* A keydown event handler.
* Reproduces the keydown event on the remote browser instance
* and generates the workflow pair data.
* @param generator - the workflow generator {@link Generator}
* @param activeBrowser - the active remote browser {@link RemoteBrowser}
* @param page - the active page of the remote browser
* @param key - the pressed key
* @param coordinates - the coordinates, where the keydown event happened
* @category BrowserManagement
*/
const handleKeydown = async (generator: WorkflowGenerator, page: Page, { key, coordinates }: KeyboardInput) => {
await page.keyboard.down(key);
await generator.onKeyboardInput(key, coordinates, page);
logger.log('debug', `Key ${key} pressed`);
const handleKeydown = async (activeBrowser: RemoteBrowser, page: Page, { key, coordinates }: KeyboardInput) => {
try {
if (page.isClosed()) {
logger.log("debug", `Ignoring keydown event: page is closed`);
return;
}
const generator = activeBrowser.generator;
await page.keyboard.down(key);
await generator.onKeyboardInput(key, coordinates, page);
logger.log("debug", `Key ${key} pressed`);
} catch (e) {
const { message } = e as Error;
logger.log("warn", `Error handling keydown event: ${message}`);
}
};
/**
* Handles the date selection event.
* @param generator - the workflow generator {@link Generator}
* @param activeBrowser - the active remote browser {@link RemoteBrowser}
* @param page - the active page of the remote browser
* @param data - the data of the date selection event {@link DatePickerEventData}
* @category BrowserManagement
*/
const handleDateSelection = async (generator: WorkflowGenerator, page: Page, data: DatePickerEventData) => {
await generator.onDateSelection(page, data);
logger.log('debug', `Date ${data.value} selected`);
const handleDateSelection = async (activeBrowser: RemoteBrowser, page: Page, data: DatePickerEventData) => {
try {
if (page.isClosed()) {
logger.log("debug", `Ignoring date selection event: page is closed`);
return;
}
const generator = activeBrowser.generator;
await generator.onDateSelection(page, data);
logger.log("debug", `Date ${data.value} selected`);
} catch (e) {
const { message } = e as Error;
logger.log("warn", `Error handling date selection event: ${message}`);
}
}
/**
@@ -271,14 +320,25 @@ const onDateSelection = async (data: DatePickerEventData, userId: string) => {
/**
* Handles the dropdown selection event.
* @param generator - the workflow generator {@link Generator}
* @param activeBrowser - the active remote browser {@link RemoteBrowser}
* @param page - the active page of the remote browser
* @param data - the data of the dropdown selection event
* @category BrowserManagement
*/
const handleDropdownSelection = async (generator: WorkflowGenerator, page: Page, data: { selector: string, value: string }) => {
await generator.onDropdownSelection(page, data);
logger.log('debug', `Dropdown value ${data.value} selected`);
const handleDropdownSelection = async (activeBrowser: RemoteBrowser, page: Page, data: { selector: string, value: string }) => {
try {
if (page.isClosed()) {
logger.log("debug", `Ignoring dropdown selection event: page is closed`);
return;
}
const generator = activeBrowser.generator;
await generator.onDropdownSelection(page, data);
logger.log("debug", `Dropdown value ${data.value} selected`);
} catch (e) {
const { message } = e as Error;
logger.log("warn", `Error handling dropdown selection event: ${message}`);
}
}
/**
@@ -294,14 +354,25 @@ const onDropdownSelection = async (data: { selector: string, value: string }, us
/**
* Handles the time selection event.
* @param generator - the workflow generator {@link Generator}
* @param activeBrowser - the active remote browser {@link RemoteBrowser}
* @param page - the active page of the remote browser
* @param data - the data of the time selection event
* @category BrowserManagement
*/
const handleTimeSelection = async (generator: WorkflowGenerator, page: Page, data: { selector: string, value: string }) => {
await generator.onTimeSelection(page, data);
logger.log('debug', `Time value ${data.value} selected`);
const handleTimeSelection = async (activeBrowser: RemoteBrowser, page: Page, data: { selector: string, value: string }) => {
try {
if (page.isClosed()) {
logger.log("debug", `Ignoring time selection event: page is closed`);
return;
}
const generator = activeBrowser.generator;
await generator.onTimeSelection(page, data);
logger.log("debug", `Time value ${data.value} selected`);
} catch (e) {
const { message } = e as Error;
logger.log("warn", `Error handling time selection event: ${message}`);
}
}
/**
@@ -317,14 +388,31 @@ const onTimeSelection = async (data: { selector: string, value: string }, userId
/**
* Handles the datetime-local selection event.
* @param generator - the workflow generator {@link Generator}
* @param activeBrowser - the active remote browser {@link RemoteBrowser}
* @param page - the active page of the remote browser
* @param data - the data of the datetime-local selection event
* @category BrowserManagement
*/
const handleDateTimeLocalSelection = async (generator: WorkflowGenerator, page: Page, data: { selector: string, value: string }) => {
await generator.onDateTimeLocalSelection(page, data);
logger.log('debug', `DateTime Local value ${data.value} selected`);
const handleDateTimeLocalSelection = async (activeBrowser: RemoteBrowser, page: Page, data: { selector: string, value: string }) => {
try {
if (page.isClosed()) {
logger.log(
"debug",
`Ignoring datetime-local selection event: page is closed`
);
return;
}
const generator = activeBrowser.generator;
await generator.onDateTimeLocalSelection(page, data);
logger.log("debug", `DateTime Local value ${data.value} selected`);
} catch (e) {
const { message } = e as Error;
logger.log(
"warn",
`Error handling datetime-local selection event: ${message}`
);
}
}
/**
@@ -353,14 +441,24 @@ const onKeyup = async (keyboardInput: KeyboardInput, userId: string) => {
* A keyup event handler.
* Reproduces the keyup event on the remote browser instance.
* Does not generate any data - keyup is not reflected in the workflow.
* @param generator - the workflow generator {@link Generator}
* @param activeBrowser - the active remote browser {@link RemoteBrowser}
* @param page - the active page of the remote browser
* @param key - the released key
* @category BrowserManagement
*/
const handleKeyup = async (generator: WorkflowGenerator, page: Page, key: string) => {
await page.keyboard.up(key);
logger.log('debug', `Key ${key} unpressed`);
const handleKeyup = async (activeBrowser: RemoteBrowser, page: Page, key: string) => {
try {
if (page.isClosed()) {
logger.log("debug", `Ignoring keyup event: page is closed`);
return;
}
await page.keyboard.up(key);
logger.log("debug", `Key ${key} unpressed`);
} catch (e) {
const { message } = e as Error;
logger.log("warn", `Error handling keyup event: ${message}`);
}
};
/**
@@ -377,23 +475,35 @@ const onChangeUrl = async (url: string, userId: string) => {
/**
* An url change event handler.
* Navigates the page to the given url and generates data for the workflow.
* @param generator - the workflow generator {@link Generator}
* @param activeBrowser - the active remote browser {@link RemoteBrowser}
* @param page - the active page of the remote browser
* @param url - the new url of the page
* @category BrowserManagement
*/
const handleChangeUrl = async (generator: WorkflowGenerator, page: Page, url: string) => {
if (url) {
await generator.onChangeUrl(url, page);
try {
await page.goto(url, { waitUntil: 'networkidle', timeout: 100000 });
logger.log('debug', `Went to ${url}`);
} catch (e) {
const { message } = e as Error;
logger.log('error', message);
const handleChangeUrl = async (activeBrowser: RemoteBrowser, page: Page, url: string) => {
try {
if (page.isClosed()) {
logger.log("debug", `Ignoring change url event: page is closed`);
return;
}
} else {
logger.log('warn', `No url provided`);
if (url) {
const generator = activeBrowser.generator;
await generator.onChangeUrl(url, page);
try {
await page.goto(url, { waitUntil: "networkidle", timeout: 100000 });
logger.log("debug", `Went to ${url}`);
} catch (e) {
const { message } = e as Error;
logger.log("error", message);
}
} else {
logger.log("warn", `No url provided`);
}
} catch (e) {
const { message } = e as Error;
logger.log("warn", `Error handling change url event: ${message}`);
}
};
@@ -410,13 +520,23 @@ const onRefresh = async (userId: string) => {
/**
* A refresh event handler.
* Refreshes the page. This is not reflected in the workflow.
* @param generator - the workflow generator {@link Generator}
* @param activeBrowser - the active remote browser {@link RemoteBrowser}
* @param page - the active page of the remote browser
* @category BrowserManagement
*/
const handleRefresh = async (generator: WorkflowGenerator, page: Page) => {
await page.reload();
logger.log('debug', `Page refreshed.`);
const handleRefresh = async (activeBrowser: RemoteBrowser, page: Page) => {
try {
if (page.isClosed()) {
logger.log("debug", `Ignoring refresh event: page is closed`);
return;
}
await page.reload();
logger.log("debug", `Page refreshed.`);
} catch (e) {
const { message } = e as Error;
logger.log("warn", `Error handling refresh event: ${message}`);
}
};
/**
@@ -432,14 +552,25 @@ const onGoBack = async (userId: string) => {
/**
* A go back event handler.
* Navigates the page back and generates data for the workflow.
* @param generator - the workflow generator {@link Generator}
* @param activeBrowser - the active remote browser {@link RemoteBrowser}
* @param page - the active page of the remote browser
* @category BrowserManagement
*/
const handleGoBack = async (generator: WorkflowGenerator, page: Page) => {
await page.goBack({ waitUntil: 'commit' });
generator.onGoBack(page.url());
logger.log('debug', 'Page went back')
const handleGoBack = async (activeBrowser: RemoteBrowser, page: Page) => {
try {
if (page.isClosed()) {
logger.log("debug", `Ignoring go back event: page is closed`);
return;
}
const generator = activeBrowser.generator;
await page.goBack({ waitUntil: "commit" });
generator.onGoBack(page.url());
logger.log("debug", "Page went back");
} catch (e) {
const { message } = e as Error;
logger.log("warn", `Error handling go back event: ${message}`);
}
};
/**
@@ -455,14 +586,207 @@ const onGoForward = async (userId: string) => {
/**
* A go forward event handler.
* Navigates the page forward and generates data for the workflow.
* @param generator - the workflow generator {@link Generator}
* @param activeBrowser - the active remote browser {@link RemoteBrowser}
* @param page - the active page of the remote browser
* @category BrowserManagement
*/
const handleGoForward = async (generator: WorkflowGenerator, page: Page) => {
await page.goForward({ waitUntil: 'commit' });
generator.onGoForward(page.url());
logger.log('debug', 'Page went forward');
const handleGoForward = async (activeBrowser: RemoteBrowser, page: Page) => {
try {
if (page.isClosed()) {
logger.log("debug", `Ignoring go forward event: page is closed`);
return;
}
const generator = activeBrowser.generator;
await page.goForward({ waitUntil: "commit" });
generator.onGoForward(page.url());
logger.log("debug", "Page went forward");
} catch (e) {
const { message } = e as Error;
logger.log("warn", `Error handling go forward event: ${message}`);
}
};
/**
* Handles the click action event.
* @param activeBrowser - the active remote browser {@link RemoteBrowser}
* @param page - the active page of the remote browser
* @param data - the data of the click action event
* @category BrowserManagement
*/
const handleClickAction = async (
activeBrowser: RemoteBrowser,
page: Page,
data: {
selector: string;
url: string;
userId: string;
elementInfo?: any;
coordinates?: { x: number; y: number };
isSPA?: boolean;
}
) => {
try {
if (page.isClosed()) {
logger.log("debug", `Ignoring click action event: page is closed`);
return;
}
const { selector, url, elementInfo, coordinates, isSPA = false } = data;
const currentUrl = page.url();
await page.click(selector);
const generator = activeBrowser.generator;
await generator.onDOMClickAction(page, data);
logger.log("debug", `Click action processed: ${selector}`);
if (isSPA) {
logger.log("debug", `SPA interaction detected for selector: ${selector}`);
await new Promise((resolve) => setTimeout(resolve, 1500));
} else {
const newUrl = page.url();
const hasNavigated = newUrl !== currentUrl && !newUrl.endsWith("/#");
if (hasNavigated) {
logger.log("debug", `Navigation detected: ${currentUrl} -> ${newUrl}`);
await generator.onDOMNavigation(page, {
url: newUrl,
currentUrl: currentUrl,
userId: data.userId,
});
}
}
await new Promise((resolve) => setTimeout(resolve, 2000));
await activeBrowser.makeAndEmitDOMSnapshot();
} catch (e) {
const { message } = e as Error;
logger.log(
"warn",
`Error handling enhanced click action event: ${message}`
);
}
};
/**
* A wrapper function for handling the click action event.
* @param socket The socket connection
* @param data - the data of the click action event
* @category HelperFunctions
*/
const onDOMClickAction = async (
data: {
selector: string;
url: string;
userId: string;
elementInfo?: any;
coordinates?: { x: number; y: number };
},
userId: string
) => {
logger.log("debug", "Handling click action event emitted from client");
await handleWrapper(handleClickAction, userId, data);
};
/**
* Handles the keyboard action event.
* @param activeBrowser - the active remote browser {@link RemoteBrowser}
* @param page - the active page of the remote browser
* @param data - the data of the keyboard action event
* @category BrowserManagement
*/
const handleKeyboardAction = async (
activeBrowser: RemoteBrowser,
page: Page,
data: {
selector: string;
key: string;
url: string;
userId: string;
inputType?: string;
}
) => {
try {
if (page.isClosed()) {
logger.log("debug", `Ignoring keyboard action event: page is closed`);
return;
}
const generator = activeBrowser.generator;
await generator.onDOMKeyboardAction(page, data);
logger.log(
"debug",
`Keyboard action processed: ${data.key} on ${data.selector}`
);
} catch (e) {
const { message } = e as Error;
logger.log("warn", `Error handling keyboard action event: ${message}`);
}
};
/**
* A wrapper function for handling the keyboard action event.
* @param socket The socket connection
* @param data - the data of the keyboard action event
* @category HelperFunctions
*/
const onDOMKeyboardAction = async (
data: {
selector: string;
key: string;
url: string;
userId: string;
inputType?: string;
},
userId: string
) => {
logger.log("debug", "Handling keyboard action event emitted from client");
await handleWrapper(handleKeyboardAction, userId, data);
};
/**
* Handles the workflow pair event.
* @param activeBrowser - the active remote browser {@link RemoteBrowser}
* @param page - the active page of the remote browser
* @param data - the data of the workflow pair event
* @category BrowserManagement
*/
const handleWorkflowPair = async (
activeBrowser: RemoteBrowser,
page: Page,
data: { pair: WhereWhatPair; userId: string }
) => {
try {
if (page.isClosed()) {
logger.log("debug", `Ignoring workflow pair event: page is closed`);
return;
}
const generator = activeBrowser.generator;
await generator.onDOMWorkflowPair(page, data);
logger.log("debug", `Workflow pair processed from frontend`);
} catch (e) {
const { message } = e as Error;
logger.log("warn", `Error handling workflow pair event: ${message}`);
}
};
/**
* A wrapper function for handling the workflow pair event.
* @param socket The socket connection
* @param data - the data of the workflow pair event
* @category HelperFunctions
*/
const onDOMWorkflowPair = async (
data: { pair: WhereWhatPair; userId: string },
userId: string
) => {
logger.log("debug", "Handling workflow pair event emitted from client");
await handleWrapper(handleWorkflowPair, userId, data);
};
/**
@@ -493,6 +817,10 @@ const registerInputHandlers = (socket: Socket, userId: string) => {
socket.on("input:time", (data) => onTimeSelection(data, userId));
socket.on("input:datetime-local", (data) => onDateTimeLocalSelection(data, userId));
socket.on("action", (data) => onGenerateAction(data, userId));
socket.on("dom:click", (data) => onDOMClickAction(data, userId));
socket.on("dom:keypress", (data) => onDOMKeyboardAction(data, userId));
socket.on("dom:addpair", (data) => onDOMWorkflowPair(data, userId));
};
export default registerInputHandlers;