vodorod/parcer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'develop' into iframe

Browse Source
This commit is contained in:
Rohit
2025-01-04 15:51:51 +05:30
committed by GitHub
parent 7b08471ba1 170e4f19a0
commit 5e775fa83f
56 changed files with 3858 additions and 657 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
server/src/browser-management/classes/RemoteBrowser.ts
View File

@@ -245,10 +245,17 @@ export class RemoteBrowser {
await this.setupPageEventListeners(this.currentPage);
const blocker = await PlaywrightBlocker.fromLists(fetch, ['https://easylist.to/easylist/easylist.txt']);
await blocker.enableBlockingInPage(this.currentPage);
this.client = await this.currentPage.context().newCDPSession(this.currentPage);
await blocker.disableBlockingInPage(this.currentPage);
try {
const blocker = await PlaywrightBlocker.fromLists(fetch, ['https://easylist.to/easylist/easylist.txt']);
await blocker.enableBlockingInPage(this.currentPage);
this.client = await this.currentPage.context().newCDPSession(this.currentPage);
await blocker.disableBlockingInPage(this.currentPage);
console.log('Adblocker initialized');
} catch (error: any) {
console.warn('Failed to initialize adblocker, continuing without it:', error.message);
// Still need to set up the CDP session even if blocker fails
this.client = await this.currentPage.context().newCDPSession(this.currentPage);
}
};
/**

11
server/src/browser-management/inputHandlers.ts
View File

@@ -260,6 +260,16 @@ const onTimeSelection = async (data: { selector: string, value: string }) => {
await handleWrapper(handleTimeSelection, data);
}
const handleDateTimeLocalSelection = async (generator: WorkflowGenerator, page: Page, data: { selector: string, value: string }) => {
await generator.onDateTimeLocalSelection(page, data);
logger.log('debug', `DateTime Local value ${data.value} selected`);
}
const onDateTimeLocalSelection = async (data: { selector: string, value: string }) => {
logger.log('debug', 'Handling datetime-local selection event emitted from client');
await handleWrapper(handleDateTimeLocalSelection, data);
}
/**
* A wrapper function for handling the keyup event.
* @param keyboardInput - the keyboard input of the keyup event
@@ -418,6 +428,7 @@ const registerInputHandlers = (socket: Socket) => {
socket.on("input:date", onDateSelection);
socket.on("input:dropdown", onDropdownSelection);
socket.on("input:time", onTimeSelection);
socket.on("input:datetime-local", onDateTimeLocalSelection);
socket.on("action", onGenerateAction);
};

2
server/src/routes/auth.ts
View File

@@ -52,7 +52,7 @@ router.post("/register", async (req, res) => {
userId: user.id,
registeredAt: new Date().toISOString(),
});
console.log(`User registered - ${user.email}`);
console.log(`User registered`);
res.json(user);
} catch (error: any) {
console.log(`Could not register user - ${error}`);

30
server/src/workflow-management/classes/Generator.ts
View File

@@ -315,6 +315,26 @@ export class WorkflowGenerator {
await this.addPairToWorkflowAndNotifyClient(pair, page);
};
public onDateTimeLocalSelection = async (page: Page, data: { selector: string, value: string }) => {
const { selector, value } = data;
try {
await page.fill(selector, value);
} catch (error) {
console.error("Failed to fill datetime-local value:", error);
}
const pair: WhereWhatPair = {
where: { url: this.getBestUrl(page.url()) },
what: [{
action: 'fill',
args: [selector, value],
}],
};
await this.addPairToWorkflowAndNotifyClient(pair, page);
};
/**
* Generates a pair for the click event.
* @param coordinates The coordinates of the click event.
@@ -390,6 +410,16 @@ export class WorkflowGenerator {
return;
}
const isDateTimeLocal = elementInfo?.tagName === 'INPUT' && elementInfo?.attributes?.type === 'datetime-local';
if (isDateTimeLocal) {
this.socket.emit('showDateTimePicker', {
coordinates,
selector
});
return;
}
//const element = await getElementMouseIsOver(page, coordinates);
//logger.log('debug', `Element: ${JSON.stringify(element, null, 2)}`);
if (selector) {

2
server/src/workflow-management/scheduler/index.ts
View File

@@ -73,7 +73,7 @@ async function createWorkflowAndStoreMetadata(id: string, userId: string) {
} catch (e) {
const { message } = e as Error;
logger.log('info', `Error while scheduling a run with id: ${id}`);
console.log(message);
console.log(`Error while scheduling a run with id: ${id}:`, message);
return {
success: false,
error: message,

170
server/src/workflow-management/selector.ts
View File

@@ -223,24 +223,44 @@ export const getElementInformation = async (
if (originalEl) {
let element = originalEl;
while (element.parentElement) {
const parentRect = element.parentElement.getBoundingClientRect();
const childRect = element.getBoundingClientRect();
if (element.tagName === 'TD' || element.tagName === 'TH') {
const tableParent = element.closest('table');
if (tableParent) {
element = tableParent;
}
}
const fullyContained =
parentRect.left <= childRect.left &&
parentRect.right >= childRect.right &&
parentRect.top <= childRect.top &&
parentRect.bottom >= childRect.bottom;
if (element.tagName !== 'TABLE') {
while (element.parentElement) {
if (element.tagName.toLowerCase() === 'body' ||
element.tagName.toLowerCase() === 'html') {
break;
}
const significantOverlap =
(childRect.width * childRect.height) /
(parentRect.width * parentRect.height) > 0.5;
const parentRect = element.parentElement.getBoundingClientRect();
const childRect = element.getBoundingClientRect();
if (fullyContained && significantOverlap) {
element = element.parentElement;
} else {
break;
const fullyContained =
parentRect.left <= childRect.left &&
parentRect.right >= childRect.right &&
parentRect.top <= childRect.top &&
parentRect.bottom >= childRect.bottom;
const significantOverlap =
(childRect.width * childRect.height) /
(parentRect.width * parentRect.height) > 0.5;
if (fullyContained && significantOverlap) {
const nextParent = element.parentElement;
if (nextParent.tagName.toLowerCase() !== 'body' &&
nextParent.tagName.toLowerCase() !== 'html') {
element = nextParent;
} else {
break;
}
} else {
break;
}
}
}
@@ -512,24 +532,44 @@ export const getRect = async (page: Page, coordinates: Coordinates, listSelector
if (originalEl) {
let element = originalEl;
while (element.parentElement) {
const parentRect = element.parentElement.getBoundingClientRect();
const childRect = element.getBoundingClientRect();
if (element.tagName === 'TD' || element.tagName === 'TH') {
const tableParent = element.closest('table');
if (tableParent) {
element = tableParent;
}
}
const fullyContained =
parentRect.left <= childRect.left &&
parentRect.right >= childRect.right &&
parentRect.top <= childRect.top &&
parentRect.bottom >= childRect.bottom;
if (element.tagName !== 'TABLE') {
while (element.parentElement) {
if (element.tagName.toLowerCase() === 'body' ||
element.tagName.toLowerCase() === 'html') {
break;
}
const significantOverlap =
(childRect.width * childRect.height) /
(parentRect.width * parentRect.height) > 0.5;
const parentRect = element.parentElement.getBoundingClientRect();
const childRect = element.getBoundingClientRect();
if (fullyContained && significantOverlap) {
element = element.parentElement;
} else {
break;
const fullyContained =
parentRect.left <= childRect.left &&
parentRect.right >= childRect.right &&
parentRect.top <= childRect.top &&
parentRect.bottom >= childRect.bottom;
const significantOverlap =
(childRect.width * childRect.height) /
(parentRect.width * parentRect.height) > 0.5;
if (fullyContained && significantOverlap) {
const nextParent = element.parentElement;
if (nextParent.tagName.toLowerCase() !== 'body' &&
nextParent.tagName.toLowerCase() !== 'html') {
element = nextParent;
} else {
break;
}
} else {
break;
}
}
}
@@ -1386,6 +1426,13 @@ export const getNonUniqueSelectors = async (page: Page, coordinates: Coordinates
function getNonUniqueSelector(element: HTMLElement): string {
let selector = element.tagName.toLowerCase();
if (selector === 'td' && element.parentElement) {
// Find position among td siblings
const siblings = Array.from(element.parentElement.children);
const position = siblings.indexOf(element) + 1;
return `${selector}:nth-child(${position})`;
}
if (element.className) {
const classes = element.className.split(/\s+/).filter((cls: string) => Boolean(cls));
if (classes.length > 0) {
@@ -1474,25 +1521,45 @@ export const getNonUniqueSelectors = async (page: Page, coordinates: Coordinates
let element = originalEl;
if (element.tagName === 'TD' || element.tagName === 'TH') {
const tableParent = element.closest('table');
if (tableParent) {
element = tableParent;
}
}
// if (listSelector === '') {
while (element.parentElement) {
const parentRect = element.parentElement.getBoundingClientRect();
const childRect = element.getBoundingClientRect();
if (element.tagName !== 'TABLE') {
while (element.parentElement) {
if (element.tagName.toLowerCase() === 'body' ||
element.tagName.toLowerCase() === 'html') {
break;
}
const fullyContained =
parentRect.left <= childRect.left &&
parentRect.right >= childRect.right &&
parentRect.top <= childRect.top &&
parentRect.bottom >= childRect.bottom;
const parentRect = element.parentElement.getBoundingClientRect();
const childRect = element.getBoundingClientRect();
const significantOverlap =
(childRect.width * childRect.height) /
(parentRect.width * parentRect.height) > 0.5;
const fullyContained =
parentRect.left <= childRect.left &&
parentRect.right >= childRect.right &&
parentRect.top <= childRect.top &&
parentRect.bottom >= childRect.bottom;
if (fullyContained && significantOverlap) {
element = element.parentElement;
} else {
break;
const significantOverlap =
(childRect.width * childRect.height) /
(parentRect.width * parentRect.height) > 0.5;
if (fullyContained && significantOverlap) {
const nextParent = element.parentElement;
if (nextParent.tagName.toLowerCase() !== 'body' &&
nextParent.tagName.toLowerCase() !== 'html') {
element = nextParent;
} else {
break;
}
} else {
break;
}
}
}
// }
@@ -1504,7 +1571,6 @@ export const getNonUniqueSelectors = async (page: Page, coordinates: Coordinates
}, coordinates);
return selectors || { generalSelector: '' };
} else {
console.log(`NON UNIQUE: MODE 2`)
const selectors = await page.evaluate(({ x, y }: { x: number, y: number }) => {
const getDeepestElementFromPoint = (x: number, y: number): HTMLElement | null => {
// First, get the element at the specified coordinates in the main document
@@ -1555,6 +1621,12 @@ export const getNonUniqueSelectors = async (page: Page, coordinates: Coordinates
function getNonUniqueSelector(element: HTMLElement): string {
let selector = element.tagName.toLowerCase();
if (selector === 'td' && element.parentElement) {
const siblings = Array.from(element.parentElement.children);
const position = siblings.indexOf(element) + 1;
return `${selector}:nth-child(${position})`;
}
if (element.className) {
const classes = element.className.split(/\s+/).filter((cls: string) => Boolean(cls));
if (classes.length > 0) {
@@ -1664,6 +1736,12 @@ export const getChildSelectors = async (page: Page, parentSelector: string): Pro
function getNonUniqueSelector(element: HTMLElement): string {
let selector = element.tagName.toLowerCase();
if (selector === 'td' && element.parentElement) {
const siblings = Array.from(element.parentElement.children);
const position = siblings.indexOf(element) + 1;
return `${selector}:nth-child(${position})`;
}
const className = typeof element.className === 'string' ? element.className : '';
if (className) {
const classes = className.split(/\s+/).filter((cls: string) => Boolean(cls));