diff --git a/mx-interpreter/browserSide/scraper.js b/mx-interpreter/browserSide/scraper.js new file mode 100644 index 00000000..c411f642 --- /dev/null +++ b/mx-interpreter/browserSide/scraper.js @@ -0,0 +1,226 @@ +/* eslint-disable @typescript-eslint/no-unused-vars */ + +const area = (element) => element.offsetHeight * element.offsetWidth; + +function getBiggestElement(selector) { + const elements = Array.from(document.querySelectorAll(selector)); + const biggest = elements.reduce( + (max, elem) => ( + area(elem) > area(max) ? elem : max), + { offsetHeight: 0, offsetWidth: 0 }, + ); + return biggest; +} + +/** + * Generates structural selector (describing element by its DOM tree location). + * + * **The generated selector is not guaranteed to be unique!** (In fact, this is + * the desired behaviour in here.) + * @param {HTMLElement} element Element being described. + * @returns {string} CSS-compliant selector describing the element's location in the DOM tree. + */ +function GetSelectorStructural(element) { + // Base conditions for the recursive approach. + if (element.tagName === 'BODY') { + return 'BODY'; + } + const selector = element.tagName; + if (element.parentElement) { + return `${GetSelectorStructural(element.parentElement)} > ${selector}`; + } + + return selector; +} + +/** + * Heuristic method to find collections of "interesting" items on the page. + * @returns {Array} A collection of interesting DOM nodes + * (online store products, plane tickets, list items... and many more?) + */ +function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3, metricType = 'size_deviation') { + const restoreScroll = (() => { + const { scrollX, scrollY } = window; + return () => { + window.scrollTo(scrollX, scrollY); + }; + })(); + + /** +* @typedef {Array<{x: number, y: number}>} Grid +*/ + + /** + * Returns an array of grid-aligned {x,y} points. + * @param {number} [granularity=0.005] sets the number of generated points + * (the higher the granularity, the more points). + * @returns {Grid} Array of {x, y} objects. + */ + function getGrid(startX = 0, startY = 0, granularity = 0.005) { + const width = window.innerWidth; + const height = window.innerHeight; + + const out = []; + for (let x = 0; x < width; x += 1 / granularity) { + for (let y = 0; y < height; y += 1 / granularity) { + out.push({ x: startX + x, y: startY + y }); + } + } + return out; + } + + let maxSelector = { selector: 'body', metric: 0 }; + + const updateMaximumWithPoint = (point) => { + const currentElement = document.elementFromPoint(point.x, point.y); + const selector = GetSelectorStructural(currentElement); + + const elements = Array.from(document.querySelectorAll(selector)) + .filter((element) => area(element) > minArea); + + // If the current selector targets less than three elements, + // we consider it not interesting (would be a very underwhelming scraper) + if (elements.length < 3) { + return; + } + + let metric = null; + + if (metricType === 'total_area') { + metric = elements + .reduce((p, x) => p + area(x), 0); + } else if (metricType === 'size_deviation') { + // This could use a proper "statistics" approach... but meh, so far so good! + const sizes = elements + .map((element) => area(element)); + + metric = (1 - (Math.max(...sizes) - Math.min(...sizes)) / Math.max(...sizes)); + } + + // console.debug(`Total ${metricType} is ${metric}.`) + if (metric > maxSelector.metric && elements.length < maxCountPerPage) { + maxSelector = { selector, metric }; + } + }; + + for (let scroll = 0; scroll < scrolls; scroll += 1) { + window.scrollTo(0, scroll * window.innerHeight); + + const grid = getGrid(); + + grid.forEach(updateMaximumWithPoint); + } + + restoreScroll(); + + let out = Array.from(document.querySelectorAll(maxSelector.selector)); + + const different = (x, i, a) => a.findIndex((e) => e === x) === i; + // as long as we don't merge any two elements by substituing them for their parents, + // we substitute. + while (out.map((x) => x.parentElement).every(different) + && out.forEach((x) => x.parentElement !== null)) { + out = out.map((x) => x.parentElement ?? x); + } + + return out; +} + +/** + * Returns a "scrape" result from the current page. + * @returns {Array} *Curated* array of scraped information (with sparse rows removed) + */ +function scrape(selector = null) { + /** + * **crudeRecords** contains uncurated rundowns of "scrapable" elements + * @type {Array} + */ + const crudeRecords = (selector + ? Array.from(document.querySelectorAll(selector)) + : scrapableHeuristics()) + .map((record) => ({ + ...Array.from(record.querySelectorAll('img')) + .reduce((p, x, i) => { + let url = null; + if (x.srcset) { + const urls = x.srcset.split(', '); + [url] = urls[urls.length - 1].split(' '); + } + + /** + * Contains the largest elements from `srcset` - if `srcset` is not present, contains + * URL from the `src` attribute + * + * If the `src` attribute contains a data url, imgUrl contains `undefined`. + */ + let imgUrl; + if (x.srcset) { + imgUrl = url; + } else if (x.src.indexOf('data:') === -1) { + imgUrl = x.src; + } + + return ({ + ...p, + ...(imgUrl ? { [`img_${i}`]: imgUrl } : {}), + }); + }, {}), + ...record.innerText.split('\n') + .reduce((p, x, i) => ({ + ...p, + [`record_${String(i).padStart(4, '0')}`]: x.trim(), + }), {}), + })); + + return crudeRecords; +} + +/** + * Given an object with named lists of elements, + * groups the elements by their distance in the DOM tree. + * @param {Object.} lists The named lists of HTML elements. + * @returns {Array.>} + */ +function scrapeSchema(lists) { + function omap(object, f, kf = (x) => x) { + return Object.fromEntries( + Object.entries(object) + .map(([k, v]) => [kf(k), f(v)]), + ); + } + + function ofilter(object, f) { + return Object.fromEntries( + Object.entries(object) + .filter(([k, v]) => f(k, v)), + ); + } + + function getSeedKey(listObj) { + const maxLength = Math.max(...Object.values(omap(listObj, (x) => x.length))); + return Object.keys(ofilter(listObj, (_, v) => v.length === maxLength))[0]; + } + + function getMBEs(elements) { + return elements.map((element) => { + let candidate = element; + const isUniqueChild = (e) => elements + .filter((elem) => e.parentNode?.contains(elem)) + .length === 1; + + while (candidate && isUniqueChild(candidate)) { + candidate = candidate.parentNode; + } + + return candidate; + }); + } + + const seedName = getSeedKey(lists); + const MBEs = getMBEs(lists[seedName]); + + return MBEs.map((mbe) => omap( + lists, + (listOfElements) => listOfElements.find((elem) => mbe.contains(elem))?.innerText, + )); +} \ No newline at end of file diff --git a/mx-interpreter/types/logic.ts b/mx-interpreter/types/logic.ts new file mode 100644 index 00000000..5d06abbe --- /dev/null +++ b/mx-interpreter/types/logic.ts @@ -0,0 +1,5 @@ +export const unaryOperators = ['$not'] as const; +export const naryOperators = ['$and', '$or'] as const; + +export const operators = [...unaryOperators, ...naryOperators] as const; +export const meta = ['$before', '$after'] as const; \ No newline at end of file diff --git a/mx-interpreter/types/workflow.ts b/mx-interpreter/types/workflow.ts new file mode 100644 index 00000000..36c6d14d --- /dev/null +++ b/mx-interpreter/types/workflow.ts @@ -0,0 +1,58 @@ +import { Page } from 'playwright'; +import { + naryOperators, unaryOperators, operators, meta, +} from './logic'; + +export type Operator = typeof operators[number]; +export type UnaryOperator = typeof unaryOperators[number]; +export type NAryOperator = typeof naryOperators[number]; + +export type Meta = typeof meta[number]; + +export type SelectorArray = string[]; + +type RegexableString = string | { '$regex': string }; + +type BaseConditions = { + 'url': RegexableString, + 'cookies': Record, + 'selectors': SelectorArray, // (CSS/Playwright) selectors use their own logic, there is no reason (and several technical difficulties) to allow regular expression notation +} & Record; + +export type Where = + Partial<{ [key in NAryOperator]: Where[] }> & // either a logic operator (arity N) + Partial<{ [key in UnaryOperator]: Where }> & // or an unary operator + Partial; // or one of the base conditions + +type MethodNames = { + [K in keyof T]: T[K] extends Function ? K : never; +}[keyof T]; + +export type CustomFunctions = 'scrape' | 'scrapeSchema' | 'scroll' | 'screenshot' | 'script' | 'enqueueLinks' | 'flag'; + +export type What = { + action: MethodNames | CustomFunctions, + args?: any[] +}; + +export type PageState = Partial; + +export type ParamType = Record; + +export type MetaData = { + name?: string, + desc?: string, +}; + +export interface WhereWhatPair { + id?: string + where: Where + what: What[] +} + +export type Workflow = WhereWhatPair[]; + +export type WorkflowFile = { + meta?: MetaData, + workflow: Workflow +}; \ No newline at end of file diff --git a/package.json b/package.json index 5a901005..3a30a35c 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,7 @@ "dotenv": "^16.0.0", "express": "^4.17.2", "fortawesome": "^0.0.1-security", + "html2canvas-pro": "^1.5.3", "joi": "^17.6.0", "loglevel": "^1.8.0", "loglevel-plugin-remote": "^0.6.8", diff --git a/server/src/browser-management/classes/RemoteBrowser.ts b/server/src/browser-management/classes/RemoteBrowser.ts index f5527581..d7376537 100644 --- a/server/src/browser-management/classes/RemoteBrowser.ts +++ b/server/src/browser-management/classes/RemoteBrowser.ts @@ -143,7 +143,7 @@ export class RemoteBrowser { return; } this.client.on('Page.screencastFrame', ({ data: base64, sessionId }) => { - this.emitScreenshot(base64); + this.emitScreenshot(base64) setTimeout(async () => { try { if (!this.client) { @@ -180,7 +180,9 @@ export class RemoteBrowser { */ public makeAndEmitScreenshot = async (): Promise => { try { - const screenshot = await this.currentPage?.screenshot(); + const screenshot = await this.currentPage?.screenshot( + { type: 'jpeg', quality: 90, fullPage: true } + ); if (screenshot) { this.emitScreenshot(screenshot.toString('base64')); } @@ -313,7 +315,7 @@ export class RemoteBrowser { logger.log('warn', 'client is not initialized'); return; } - await this.client.send('Page.startScreencast', { format: 'jpeg', quality: 75 }); + await this.client.send('Page.startScreencast', { format: 'jpeg', quality: 90 }); logger.log('info', `Browser started with screencasting a page.`); }; diff --git a/server/src/routes/record.ts b/server/src/routes/record.ts index da62bf8e..0a20f66c 100644 --- a/server/src/routes/record.ts +++ b/server/src/routes/record.ts @@ -99,8 +99,7 @@ router.get('/interpret', async (req, res) => { await interpretWholeWorkflow(); return res.send('interpretation done'); } catch (e) { - return res.send('interpretation done'); - return res.status(400); + return res.send('interpretation failed'); } }); diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index 2634635d..234d7014 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -484,6 +484,7 @@ export class WorkflowGenerator { public generateDataForHighlighter = async (page: Page, coordinates: Coordinates) => { const rect = await getRect(page, coordinates); const displaySelector = await this.generateSelector(page, coordinates, ActionType.Click); + //console.log('Backend Rectangle:', rect) if (rect) { this.socket.emit('highlighter', { rect, selector: displaySelector }); } diff --git a/server/src/workflow-management/selector.ts b/server/src/workflow-management/selector.ts index 957cb3b9..0290ee9f 100644 --- a/server/src/workflow-management/selector.ts +++ b/server/src/workflow-management/selector.ts @@ -68,24 +68,58 @@ export const getElementInformation = async ( const el = document.elementFromPoint(x, y) as HTMLElement; if (el) { const { parentElement } = el; - // Match the logic in recorder.ts for link clicks const element = parentElement?.tagName === 'A' ? parentElement : el; - return { + + let info: { + tagName: string; + hasOnlyText?: boolean; + innerText?: string; + url?: string; + imageUrl?: string; + } = { tagName: element?.tagName ?? '', - hasOnlyText: element?.children?.length === 0 && - element?.innerText?.length > 0, + }; + + if (element?.tagName === 'A') { + info.url = (element as HTMLAnchorElement).href; + info.innerText = element.innerText ?? ''; + } else if (element?.tagName === 'IMG') { + info.imageUrl = (element as HTMLImageElement).src; + } else { + info.hasOnlyText = element?.children?.length === 0 && + element?.innerText?.length > 0; + info.innerText = element?.innerText ?? ''; } + + return info; } + return null; }, { x: coordinates.x, y: coordinates.y }, ); + + if (elementInfo) { + if (elementInfo.tagName === 'A') { + if (elementInfo.innerText) { + console.log(`Link text: ${elementInfo.innerText}, URL: ${elementInfo.url}`); + } else { + console.log(`URL: ${elementInfo.url}`); + } + } else if (elementInfo.tagName === 'IMG') { + console.log(`Image URL: ${elementInfo.imageUrl}`); + } else { + console.log(`Element innerText: ${elementInfo.innerText}`); + } + } + return elementInfo; } catch (error) { const { message, stack } = error as Error; - logger.log('error', `Error while retrieving selector: ${message}`); - logger.log('error', `Stack: ${stack}`); + console.error('Error while retrieving selector:', message); + console.error('Stack:', stack); } -} +}; + /** * Returns the best and unique css {@link Selectors} for the element on the page. diff --git a/src/components/atoms/Highlighter.tsx b/src/components/atoms/Highlighter.tsx index 3199b083..4949277d 100644 --- a/src/components/atoms/Highlighter.tsx +++ b/src/components/atoms/Highlighter.tsx @@ -14,22 +14,9 @@ export const Highlighter = ({ unmodifiedRect, displayedSelector = '', width, hei if (!unmodifiedRect) { return null; } else { - // const unshiftedRect = mapRect(unmodifiedRect, width, height); - // console.log('unshiftedRect', unshiftedRect) - // const rect = { - // bottom: unshiftedRect.bottom + canvasRect.top, - // top: unshiftedRect.top + canvasRect.top, - // left: unshiftedRect.left + canvasRect.left, - // right: unshiftedRect.right + canvasRect.left, - // x: unshiftedRect.x + canvasRect.left, - // y: unshiftedRect.y + canvasRect.top, - // width: unshiftedRect.width, - // height: unshiftedRect.height, - // } - const rect = { - top: unmodifiedRect.top + canvasRect.top, - left: unmodifiedRect.left + canvasRect.left, + top: unmodifiedRect.top + canvasRect.top + window.scrollY, + left: unmodifiedRect.left + canvasRect.left + window.scrollX, right: unmodifiedRect.right + canvasRect.left, bottom: unmodifiedRect.bottom + canvasRect.top, width: unmodifiedRect.width, @@ -41,26 +28,6 @@ export const Highlighter = ({ unmodifiedRect, displayedSelector = '', width, hei console.log('rectangle:', rect) console.log('canvas rectangle:', canvasRect) - // make the highlighting rectangle stay in browser window boundaries - // if (rect.bottom > canvasRect.bottom) { - // rect.height = height - unshiftedRect.top; - // } - - // if (rect.top < canvasRect.top) { - // rect.height = rect.height - (canvasRect.top - rect.top); - // rect.top = canvasRect.top; - // } - - // if (rect.right > canvasRect.right) { - // rect.width = width - unshiftedRect.left; - // } - - // if (rect.left < canvasRect.left) { - // rect.width = rect.width - (canvasRect.left - rect.left); - // rect.left = canvasRect.left; - // } - - return (
` position: fixed !important; background: #ff5d5b26 !important; outline: 4px solid pink !important; - // border: 4px solid #ff5d5b !important; + //border: 4px solid #ff5d5b !important; z-index: 2147483647 !important; - // border-radius: 5px; + //border-radius: 5px; top: ${(p: HighlighterOutlineProps) => p.top}px; left: ${(p: HighlighterOutlineProps) => p.left}px; width: ${(p: HighlighterOutlineProps) => p.width}px; diff --git a/src/components/atoms/canvas.tsx b/src/components/atoms/canvas.tsx index 5152cf5e..9a879106 100644 --- a/src/components/atoms/canvas.tsx +++ b/src/components/atoms/canvas.tsx @@ -123,17 +123,15 @@ const Canvas = ({ width, height, onCreateRef }: CanvasProps) => { }, [onMouseEvent]); return ( - // ); }; -export default Canvas; +export default Canvas; \ No newline at end of file diff --git a/src/components/molecules/NavBar.tsx b/src/components/molecules/NavBar.tsx index 011277c1..640f09cb 100644 --- a/src/components/molecules/NavBar.tsx +++ b/src/components/molecules/NavBar.tsx @@ -44,7 +44,7 @@ export const NavBar = ({newRecording, recordingName, isRecording}:NavBarProps) = justifyContent: 'flex-start', }}> -
Browser Recorder
+
Maxun
{ return ( <> {(highlighterData?.rect != null && highlighterData?.rect.top != null) && canvasRef?.current ? - < Highlighter + { img.src = image; img.onload = () => { URL.revokeObjectURL(img.src); - //ctx?.clearRect(0, 0, canvas?.width || 0, VIEWPORT_H || 0); - // ctx?.drawImage(img, 0, 0, canvas.width , canvas.height); - ctx?.drawImage(img, 0, 0, 1280, 720); // Explicitly draw image at 1280 x 720 + ctx?.drawImage(img, 0, 0, 1280, 720); console.log('Image drawn on canvas:', img.width, img.height); console.log('Image drawn on canvas:', canvas.width, canvas.height); }; -}; +}; \ No newline at end of file diff --git a/src/components/organisms/RightSidePanel.tsx b/src/components/organisms/RightSidePanel.tsx index 3ad8dff5..2769c0f1 100644 --- a/src/components/organisms/RightSidePanel.tsx +++ b/src/components/organisms/RightSidePanel.tsx @@ -7,15 +7,13 @@ import { SelectChangeEvent } from "@mui/material/Select/Select"; import { SimpleBox } from "../atoms/Box"; import Typography from "@mui/material/Typography"; import { useGlobalInfoStore } from "../../context/globalInfo"; -import { PairDetail } from "../molecules/PairDetail"; import { PairForEdit } from "../../pages/RecordingPage"; interface RightSidePanelProps { pairForEdit: PairForEdit; - changeBrowserDimensions: () => void; } -export const RightSidePanel = ({pairForEdit, changeBrowserDimensions}: RightSidePanelProps) => { +export const RightSidePanel = ({pairForEdit}: RightSidePanelProps) => { const [content, setContent] = useState('action'); const [action, setAction] = React.useState(''); @@ -48,9 +46,6 @@ export const RightSidePanel = ({pairForEdit, changeBrowserDimensions}: RightSide backgroundColor: 'white', alignItems: "center", }}> - Last action: @@ -58,11 +53,6 @@ export const RightSidePanel = ({pairForEdit, changeBrowserDimensions}: RightSide - - - - - {content === 'action' ? ( Type of action: @@ -87,7 +77,7 @@ export const RightSidePanel = ({pairForEdit, changeBrowserDimensions}: RightSide } ) - : + : null } ); diff --git a/src/pages/RecordingPage.tsx b/src/pages/RecordingPage.tsx index 7a2f03a8..b3bcedea 100644 --- a/src/pages/RecordingPage.tsx +++ b/src/pages/RecordingPage.tsx @@ -10,6 +10,7 @@ import { useBrowserDimensionsStore } from "../context/browserDimensions"; import { useGlobalInfoStore } from "../context/globalInfo"; import { editRecordingFromStorage } from "../api/storage"; import { WhereWhatPair } from "@wbr-project/wbr-interpret"; +import styled from "styled-components"; interface RecordingPageProps { recordingName?: string; @@ -106,22 +107,29 @@ export const RecordingPage = ({ recordingName }: RecordingPageProps) => {
{isLoaded ? - {/* + - */} + - {/* - - */} + + + : }
); }; + +const RecordingPageWrapper = styled.div` + position: relative; + width: 100vw; + height: 100vh; + overflow: hidden; +`; \ No newline at end of file