Compare commits
20 Commits
4dc9f3a130
...
7fa1901e5f
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7fa1901e5f | ||
|
|
550261e158 | ||
|
|
ca94044628 | ||
|
|
d7940235cb | ||
|
|
ae4f9d800b | ||
|
|
0b9f593597 | ||
|
|
8f88b109b9 | ||
|
|
7b60cc2672 | ||
|
|
6e90b02195 | ||
|
|
cbcb3d59de | ||
|
|
9d52302c1d | ||
|
|
fd4044d727 | ||
|
|
8acc3dfcca | ||
|
|
7cf8b789ff | ||
|
|
1a2f9bdd09 | ||
|
|
03204a7851 | ||
|
|
51156f5880 | ||
|
|
9f1b8ae6e3 | ||
|
|
d127f0c8f0 | ||
|
|
e413c1a4c3 |
110
README.md
110
README.md
@@ -1,9 +1,9 @@
|
||||
<h2 align="center">
|
||||
<h2 align="center">
|
||||
<div>
|
||||
<a href="https://www.maxun.dev/?ref=ghread">
|
||||
<img src="/src/assets/maxunlogo.png" width="70" />
|
||||
<a href="https://www.Dorod Parser.dev/?ref=ghread">
|
||||
<img src="/src/assets/Dorod Parserlogo.png" width="70" />
|
||||
<br>
|
||||
Maxun
|
||||
Dorod Parser
|
||||
</a>
|
||||
</div>
|
||||
Turn Any Website Into A Structured API
|
||||
@@ -11,44 +11,44 @@
|
||||
</h2>
|
||||
|
||||
<p align="center">
|
||||
✨ The unified open-source no-code platform for real-time web scraping, crawling, search and AI data extraction ✨
|
||||
вњЁ The unified open-source no-code platform for real-time web scraping, crawling, search and AI data extraction вњЁ
|
||||
|
||||
<p align="center">
|
||||
<a href="https://app.maxun.dev/?ref=ghread"><b>Go To App</b></a> •
|
||||
<a href="https://docs.maxun.dev/?ref=ghread"><b>Documentation</b></a> •
|
||||
<a href="https://www.maxun.dev/?ref=ghread"><b>Website</b></a> •
|
||||
<a href="https://discord.gg/5GbPjBUkws"><b>Discord</b></a> •
|
||||
<a href="https://www.youtube.com/@MaxunOSS?ref=ghread"><b>Watch Tutorials</b></a>
|
||||
<a href="https://app.Dorod Parser.dev/?ref=ghread"><b>Go To App</b></a> •
|
||||
<a href="https://docs.Dorod Parser.dev/?ref=ghread"><b>Documentation</b></a> •
|
||||
<a href="https://www.Dorod Parser.dev/?ref=ghread"><b>Website</b></a> •
|
||||
<a href="https://discord.gg/5GbPjBUkws"><b>Discord</b></a> •
|
||||
<a href="https://www.youtube.com/@Dorod ParserOSS?ref=ghread"><b>Watch Tutorials</b></a>
|
||||
<br />
|
||||
<br />
|
||||
<a href="https://trendshift.io/repositories/12113" target="_blank"><img src="https://trendshift.io/api/badge/repositories/12113" alt="getmaxun%2Fmaxun | Trendshift" style="width: 250px; height: 55px; margin-top: 10px;" width="250" height="55"/></a>
|
||||
<a href="https://trendshift.io/repositories/12113" target="_blank"><img src="https://trendshift.io/api/badge/repositories/12113" alt="getDorod Parser%2FDorod Parser | Trendshift" style="width: 250px; height: 55px; margin-top: 10px;" width="250" height="55"/></a>
|
||||
</p>
|
||||
|
||||
## What is Maxun?
|
||||
## What is Dorod Parser?
|
||||
|
||||
Maxun is an open-source no-code web data platform for turning the web into structured, reliable data.
|
||||
It supports extraction, crawling, scraping, and search — designed to scale from simple use cases to complex, automated workflows.
|
||||
Dorod Parser is an open-source no-code web data platform for turning the web into structured, reliable data.
|
||||
It supports extraction, crawling, scraping, and search — designed to scale from simple use cases to complex, automated workflows.
|
||||
|
||||
### Ecosystem
|
||||
|
||||
1. **[Extract](https://docs.maxun.dev/category/extract)** – Emulate real user behavior and collect structured data from any website.
|
||||
* **[Recorder Mode](https://docs.maxun.dev/robot/extract/robot-actions)** - Record your actions as you browse; Maxun turns them into a reusable extraction robot.
|
||||
* **[AI Mode](https://docs.maxun.dev/robot/extract/llm-extraction)** - Describe what you want in natural language and let LLM-powered extraction do the rest.
|
||||
1. **[Extract](https://docs.Dorod Parser.dev/category/extract)** – Emulate real user behavior and collect structured data from any website.
|
||||
* **[Recorder Mode](https://docs.Dorod Parser.dev/robot/extract/robot-actions)** - Record your actions as you browse; Dorod Parser turns them into a reusable extraction robot.
|
||||
* **[AI Mode](https://docs.Dorod Parser.dev/robot/extract/llm-extraction)** - Describe what you want in natural language and let LLM-powered extraction do the rest.
|
||||
|
||||
2. **[Scrape](https://docs.maxun.dev/robot/scrape/scrape-robots)** – Convert full webpages into clean Markdown or HTML and capture screenshots.
|
||||
3. **[Crawl](https://docs.maxun.dev/robot/crawl/crawl-introduction)** - Crawl entire websites and extract content from every relevant page, with full control over scope and discovery.
|
||||
4. **[Search](https://docs.maxun.dev/robot/search/search-introduction)** - Run automated web searches to discover or scrape results, with support for time-based filters.
|
||||
5. **[SDK](https://docs.maxun.dev/sdk/sdk-overview)** – A complete developer toolkit for scraping, extraction, scheduling, and end-to-end data automation.
|
||||
2. **[Scrape](https://docs.Dorod Parser.dev/robot/scrape/scrape-robots)** – Convert full webpages into clean Markdown or HTML and capture screenshots.
|
||||
3. **[Crawl](https://docs.Dorod Parser.dev/robot/crawl/crawl-introduction)** - Crawl entire websites and extract content from every relevant page, with full control over scope and discovery.
|
||||
4. **[Search](https://docs.Dorod Parser.dev/robot/search/search-introduction)** - Run automated web searches to discover or scrape results, with support for time-based filters.
|
||||
5. **[SDK](https://docs.Dorod Parser.dev/sdk/sdk-overview)** – A complete developer toolkit for scraping, extraction, scheduling, and end-to-end data automation.
|
||||
|
||||
## How Does It Work?
|
||||
|
||||
Maxun robots are automated tools that help you collect data from websites without writing any code. Think of them as your personal web assistants that can navigate websites, extract information, and organize data just like you would manually - but faster and more efficiently.
|
||||
Dorod Parser robots are automated tools that help you collect data from websites without writing any code. Think of them as your personal web assistants that can navigate websites, extract information, and organize data just like you would manually - but faster and more efficiently.
|
||||
|
||||
There are four types of robots, each designed for a different job.
|
||||
|
||||
### 1. Extract
|
||||
Extract emulates real user behavior and captures structured data.
|
||||
- <a href="/robot/extract/robot-actions">Recorder Mode</a> - Record your actions as you browse; Maxun turns them into a reusable extraction robot.
|
||||
- <a href="/robot/extract/robot-actions">Recorder Mode</a> - Record your actions as you browse; Dorod Parser turns them into a reusable extraction robot.
|
||||
### Example: Extract 10 Property Listings from Airbnb
|
||||
|
||||
[https://github.com/user-attachments/assets/recorder-mode-demo-video](https://github.com/user-attachments/assets/c6baa75f-b950-482c-8d26-8a8b6c5382c3)
|
||||
@@ -62,41 +62,41 @@ Learn more <a href="/category/extract">here</a>.
|
||||
### 2. Scrape
|
||||
Scrape converts full webpages into clean Markdown, HTML and can capture screenshots. Ideal for AI workflows, agents, and document processing.
|
||||
|
||||
Learn more <a href="https://docs.maxun.dev/robot/scrape/scrape-robots">here</a>.
|
||||
Learn more <a href="https://docs.Dorod Parser.dev/robot/scrape/scrape-robots">here</a>.
|
||||
|
||||
### 3. Crawl
|
||||
Crawl entire websites and extract content from every relevant page, with full control over scope and discovery.
|
||||
|
||||
Learn more <a href="https://docs.maxun.dev/robot/crawl/crawl-introduction">here</a>.
|
||||
Learn more <a href="https://docs.Dorod Parser.dev/robot/crawl/crawl-introduction">here</a>.
|
||||
|
||||
### 4. Search
|
||||
Run automated web searches to discover or scrape results, with support for time-based filters.
|
||||
|
||||
Learn more <a href="https://docs.maxun.dev/robot/search/search-introduction">here</a>.
|
||||
Learn more <a href="https://docs.Dorod Parser.dev/robot/search/search-introduction">here</a>.
|
||||
|
||||
## Quick Start
|
||||
|
||||
### Getting Started
|
||||
The simplest & fastest way to get started is to use the hosted version: https://app.maxun.dev. You can self-host if you prefer!
|
||||
The simplest & fastest way to get started is to use the hosted version: https://app.Dorod Parser.dev. You can self-host if you prefer!
|
||||
|
||||
### Installation
|
||||
Maxun can run locally with or without Docker
|
||||
1. [Setup with Docker Compose](https://docs.maxun.dev/installation/docker)
|
||||
2. [Setup without Docker](https://docs.maxun.dev/installation/local)
|
||||
3. [Environment Variables](https://docs.maxun.dev/installation/environment_variables)
|
||||
4. [SDK](https://github.com/getmaxun/node-sdk)
|
||||
Dorod Parser can run locally with or without Docker
|
||||
1. [Setup with Docker Compose](https://docs.Dorod Parser.dev/installation/docker)
|
||||
2. [Setup without Docker](https://docs.Dorod Parser.dev/installation/local)
|
||||
3. [Environment Variables](https://docs.Dorod Parser.dev/installation/environment_variables)
|
||||
4. [SDK](https://github.com/getDorod Parser/node-sdk)
|
||||
|
||||
### Upgrading & Self Hosting
|
||||
1. [Self Host Maxun With Docker & Portainer](https://docs.maxun.dev/self-host)
|
||||
2. [Upgrade Maxun With Docker Compose Setup](https://docs.maxun.dev/installation/upgrade#upgrading-with-docker-compose)
|
||||
3. [Upgrade Maxun Without Docker Compose Setup](https://docs.maxun.dev/installation/upgrade#upgrading-with-local-setup)
|
||||
1. [Self Host Dorod Parser With Docker & Portainer](https://docs.Dorod Parser.dev/self-host)
|
||||
2. [Upgrade Dorod Parser With Docker Compose Setup](https://docs.Dorod Parser.dev/installation/upgrade#upgrading-with-docker-compose)
|
||||
3. [Upgrade Dorod Parser Without Docker Compose Setup](https://docs.Dorod Parser.dev/installation/upgrade#upgrading-with-local-setup)
|
||||
|
||||
## Sponsors
|
||||
<table>
|
||||
<tr>
|
||||
<td width="229">
|
||||
<br/>
|
||||
<a href="https://www.testmu.ai/?utm_source=maxun&utm_medium=sponsor" target="_blank">
|
||||
<a href="https://www.testmuai.com/?utm_medium=sponsor&utm_source=Dorod Parser" target="_blank">
|
||||
<img src="https://github.com/user-attachments/assets/6c96005b-85df-43e0-9b63-96aaca676c11" /><br/><br/>
|
||||
<b>TestMu AI</b>
|
||||
</a>
|
||||
@@ -109,24 +109,24 @@ Maxun can run locally with or without Docker
|
||||
|
||||
## Features
|
||||
|
||||
- ✨ **Extract Data With No-Code** – Point and click interface
|
||||
- ✨ **LLM-Powered Extraction** – Describe what you want; use LLMs to scrape structured data
|
||||
- ✨ **Developer SDK** – Programmatic extraction, scheduling, and robot management
|
||||
- ✨ **Handle Pagination & Scrolling** – Automatic navigation
|
||||
- ✨ **Run Robots On Schedules** – Set it and forget it
|
||||
- ✨ **Turn Websites to APIs** – RESTful endpoints from any site
|
||||
- ✨ **Turn Websites to Spreadsheets** – Direct data export to Google Sheets & Airtable
|
||||
- ✨ **Adapt To Website Layout Changes** – Auto-recovery from site updates
|
||||
- ✨ **Extract Behind Login** – Handle authentication seamlessly
|
||||
- ✨ **Integrations** – Connect with your favorite tools
|
||||
- ✨ **MCP Support** – Model Context Protocol integration
|
||||
- ✨ **LLM-Ready Data** – Clean Markdown for AI applications
|
||||
- ✨ **Self-Hostable** – Full control over your infrastructure
|
||||
- ✨ **Open Source** – Transparent and community-driven
|
||||
- ✨ **Extract Data With No-Code** – Point and click interface
|
||||
- ✨ **LLM-Powered Extraction** – Describe what you want; use LLMs to scrape structured data
|
||||
- ✨ **Developer SDK** – Programmatic extraction, scheduling, and robot management
|
||||
- ✨ **Handle Pagination & Scrolling** – Automatic navigation
|
||||
- ✨ **Run Robots On Schedules** – Set it and forget it
|
||||
- ✨ **Turn Websites to APIs** – RESTful endpoints from any site
|
||||
- ✨ **Turn Websites to Spreadsheets** – Direct data export to Google Sheets & Airtable
|
||||
- ✨ **Adapt To Website Layout Changes** – Auto-recovery from site updates
|
||||
- ✨ **Extract Behind Login** – Handle authentication seamlessly
|
||||
- ✨ **Integrations** – Connect with your favorite tools
|
||||
- ✨ **MCP Support** – Model Context Protocol integration
|
||||
- ✨ **LLM-Ready Data** – Clean Markdown for AI applications
|
||||
- ✨ **Self-Hostable** – Full control over your infrastructure
|
||||
- ✨ **Open Source** – Transparent and community-driven
|
||||
|
||||
## Demos
|
||||
Maxun can be used for various use-cases, including lead generation, market research, content aggregation and more.
|
||||
View demos here: https://www.maxun.dev/usecases
|
||||
Dorod Parser can be used for various use-cases, including lead generation, market research, content aggregation and more.
|
||||
View demos here: https://www.Dorod Parser.dev/usecases
|
||||
|
||||
## Note
|
||||
This project is in early stages of development. Your feedback is very important for us - we're actively working on improvements. </a>
|
||||
@@ -143,11 +143,11 @@ If you rely on this project commercially, please consider contributing back
|
||||
or supporting its development.
|
||||
|
||||
## Support Us
|
||||
Star the repository, contribute if you love what we’re building, or [sponsor us](https://github.com/sponsors/amhsirak).
|
||||
Star the repository, contribute if you love what we’re building, or [sponsor us](https://github.com/sponsors/amhsirak).
|
||||
|
||||
## Contributors
|
||||
Thank you to the combined efforts of everyone who contributes!
|
||||
|
||||
<a href="https://github.com/getmaxun/maxun/graphs/contributors">
|
||||
<img src="https://contrib.rocks/image?repo=getmaxun/maxun" />
|
||||
<a href="https://github.com/getDorod Parser/Dorod Parser/graphs/contributors">
|
||||
<img src="https://contrib.rocks/image?repo=getDorod Parser/Dorod Parser" />
|
||||
</a>
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "maxun-core",
|
||||
"version": "0.0.30",
|
||||
"version": "0.0.31",
|
||||
"description": "Core package for Maxun, responsible for data extraction",
|
||||
"main": "build/index.js",
|
||||
"typings": "build/index.d.ts",
|
||||
|
||||
10
package.json
10
package.json
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "maxun",
|
||||
"version": "0.0.32",
|
||||
"author": "Maxun",
|
||||
{
|
||||
"name": "dorod-parser",
|
||||
"version": "0.0.33",
|
||||
"author": "Dorod Parser",
|
||||
"license": "AGPL-3.0-or-later",
|
||||
"dependencies": {
|
||||
"@anthropic-ai/sdk": "^0.71.2",
|
||||
@@ -54,7 +54,7 @@
|
||||
"lodash": "^4.17.21",
|
||||
"loglevel": "^1.8.0",
|
||||
"loglevel-plugin-remote": "^0.6.8",
|
||||
"maxun-core": "^0.0.30",
|
||||
"Dorod Parser-core": "^0.0.31",
|
||||
"minio": "^8.0.1",
|
||||
"moment-timezone": "^0.5.45",
|
||||
"node-cron": "^3.0.3",
|
||||
|
||||
@@ -101,6 +101,13 @@ export class RemoteBrowser {
|
||||
private lastScrollPosition = { x: 0, y: 0 };
|
||||
private scrollThreshold = 200;
|
||||
|
||||
/**
|
||||
* Flag to indicate if this is a recording session (requires rrweb for real-time DOM streaming)
|
||||
* When false (robot run mode), rrweb is skipped to improve performance
|
||||
* @private
|
||||
*/
|
||||
private isRecordingMode: boolean = false;
|
||||
|
||||
// private memoryCleanupInterval: NodeJS.Timeout | null = null;
|
||||
// private memoryManagementInterval: NodeJS.Timeout | null = null;
|
||||
|
||||
@@ -110,11 +117,12 @@ export class RemoteBrowser {
|
||||
* @param socket socket.io socket instance used to communicate with the client side
|
||||
* @constructor
|
||||
*/
|
||||
public constructor(socket: Socket, userId: string, poolId: string) {
|
||||
public constructor(socket: Socket, userId: string, poolId: string, isRecordingMode: boolean = false) {
|
||||
this.socket = socket;
|
||||
this.userId = userId;
|
||||
this.interpreter = new WorkflowInterpreter(socket);
|
||||
this.generator = new WorkflowGenerator(socket, poolId);
|
||||
this.isRecordingMode = isRecordingMode;
|
||||
}
|
||||
|
||||
// private initializeMemoryManagement(): void {
|
||||
@@ -264,12 +272,14 @@ export class RemoteBrowser {
|
||||
}
|
||||
});
|
||||
|
||||
if (this.isRecordingMode) {
|
||||
await page.waitForLoadState('networkidle', { timeout: 10000 }).catch(() => {
|
||||
logger.warn('[rrweb] Network idle timeout on navigation, proceeding with rrweb initialization');
|
||||
});
|
||||
|
||||
await this.initializeRRWebRecording(page);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
page.on('load', async () => {
|
||||
@@ -298,8 +308,14 @@ export class RemoteBrowser {
|
||||
/**
|
||||
* Initialize rrweb recording for real-time DOM streaming
|
||||
* This replaces the snapshot-based approach with live event streaming
|
||||
* Only runs in recording mode - skipped for robot runs to improve performance
|
||||
*/
|
||||
private async initializeRRWebRecording(page: Page): Promise<void> {
|
||||
if (!this.isRecordingMode) {
|
||||
logger.debug('[rrweb] Skipping initialization - not in recording mode (robot run)');
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const rrwebJsPath = require.resolve('rrweb/dist/rrweb.min.js');
|
||||
const rrwebScriptContent = readFileSync(rrwebJsPath, 'utf8');
|
||||
@@ -529,11 +545,13 @@ export class RemoteBrowser {
|
||||
|
||||
await this.setupPageEventListeners(this.currentPage);
|
||||
|
||||
if (this.isRecordingMode) {
|
||||
await this.currentPage.waitForLoadState('networkidle', { timeout: 10000 }).catch(() => {
|
||||
logger.warn('[rrweb] Network idle timeout, proceeding with rrweb initialization');
|
||||
});
|
||||
|
||||
await this.initializeRRWebRecording(this.currentPage);
|
||||
}
|
||||
|
||||
try {
|
||||
const blocker = await PlaywrightBlocker.fromLists(fetch, ['https://easylist.to/easylist/easylist.txt']);
|
||||
|
||||
@@ -32,7 +32,7 @@ export const initializeRemoteBrowserForRecording = (userId: string, mode: string
|
||||
const remoteBrowser = browserPool.getRemoteBrowser(activeId);
|
||||
remoteBrowser?.updateSocket(socket);
|
||||
} else {
|
||||
const browserSession = new RemoteBrowser(socket, userId, id);
|
||||
const browserSession = new RemoteBrowser(socket, userId, id, true);
|
||||
browserSession.interpreter.subscribeToPausing();
|
||||
|
||||
try {
|
||||
|
||||
@@ -452,9 +452,42 @@ const handleClickAction = async (
|
||||
}
|
||||
|
||||
const { selector, url, elementInfo, coordinates, isSPA = false } = data;
|
||||
|
||||
if (page.isClosed()) {
|
||||
logger.log("debug", "Page is closed, cannot remove target attribute");
|
||||
return;
|
||||
}
|
||||
|
||||
const anchorInfo = await page.evaluate(({ sel }) => {
|
||||
try {
|
||||
const element = document.querySelector(sel);
|
||||
if (element) {
|
||||
if (element.getAttribute('target') === '_blank') {
|
||||
element.removeAttribute('target');
|
||||
}
|
||||
|
||||
const parentAnchor = element.closest('a[target="_blank"]') as HTMLAnchorElement;
|
||||
if (parentAnchor) {
|
||||
parentAnchor.removeAttribute('target');
|
||||
}
|
||||
|
||||
const anchor = element.tagName === 'A' ? element as HTMLAnchorElement : element.closest('a') as HTMLAnchorElement;
|
||||
if (anchor && anchor.href) {
|
||||
return { hasAnchor: true, href: anchor.href };
|
||||
}
|
||||
}
|
||||
return { hasAnchor: false, href: null };
|
||||
} catch (e) {
|
||||
console.error('Error removing target attribute:', e);
|
||||
return { hasAnchor: false, href: null };
|
||||
}
|
||||
}, { sel: selector });
|
||||
|
||||
const currentUrl = page.url();
|
||||
|
||||
if (elementInfo && coordinates && (elementInfo.tagName === 'INPUT' || elementInfo.tagName === 'TEXTAREA')) {
|
||||
const isInputElement = elementInfo && (elementInfo.tagName === 'INPUT' || elementInfo.tagName === 'TEXTAREA');
|
||||
|
||||
if (isInputElement && coordinates) {
|
||||
try {
|
||||
const elementHandle = await page.$(selector);
|
||||
if (elementHandle) {
|
||||
@@ -483,7 +516,7 @@ const handleClickAction = async (
|
||||
|
||||
logger.log("debug", `Click action processed: ${selector}`);
|
||||
|
||||
if (elementInfo && (elementInfo.tagName === 'INPUT' || elementInfo.tagName === 'TEXTAREA')) {
|
||||
if (isInputElement) {
|
||||
logger.log("debug", `Input field click - skipping DOM snapshot for smooth typing`);
|
||||
return;
|
||||
}
|
||||
@@ -493,10 +526,35 @@ const handleClickAction = async (
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, 1500));
|
||||
} else {
|
||||
const newUrl = page.url();
|
||||
const hasNavigated = newUrl !== currentUrl && !newUrl.endsWith("/#");
|
||||
try {
|
||||
await page.waitForNavigation({ timeout: 1500 });
|
||||
} catch (e) {
|
||||
}
|
||||
|
||||
if (hasNavigated) {
|
||||
let newUrl = page.url();
|
||||
|
||||
if (anchorInfo.hasAnchor && anchorInfo.href) {
|
||||
try {
|
||||
const expectedUrl = new URL(anchorInfo.href);
|
||||
const actualUrl = new URL(newUrl);
|
||||
|
||||
const navigatedToExpectedPage =
|
||||
expectedUrl.origin === actualUrl.origin &&
|
||||
expectedUrl.pathname === actualUrl.pathname;
|
||||
|
||||
if (!navigatedToExpectedPage) {
|
||||
logger.log("debug", `Click did not navigate to expected URL, using page.goto as fallback`);
|
||||
await page.goto(anchorInfo.href, { waitUntil: "domcontentloaded", timeout: 30000 });
|
||||
newUrl = page.url();
|
||||
}
|
||||
} catch (urlError: any) {
|
||||
logger.log("debug", `Error comparing URLs: ${urlError.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
const finalNavigated = newUrl !== currentUrl && !newUrl.endsWith("/#");
|
||||
|
||||
if (finalNavigated) {
|
||||
logger.log("debug", `Navigation detected: ${currentUrl} -> ${newUrl}`);
|
||||
|
||||
await generator.onDOMNavigation(page, {
|
||||
|
||||
@@ -235,8 +235,8 @@ export const NavBar: React.FC<NavBarProps> = ({
|
||||
padding: '8px',
|
||||
marginRight: '20px',
|
||||
'&:hover': {
|
||||
background: 'none',
|
||||
color: 'inherit'
|
||||
background: 'inherit',
|
||||
color: '#0000008A'
|
||||
}
|
||||
}}>
|
||||
<Typography variant="body1">Browse Auto Robots</Typography>
|
||||
|
||||
@@ -648,7 +648,6 @@ export const DOMBrowserRenderer: React.FC<RRWebDOMBrowserRendererProps> = ({
|
||||
return;
|
||||
}
|
||||
|
||||
if (!isInCaptureMode) {
|
||||
const wheelEvent = e as WheelEvent;
|
||||
const deltaX = Math.round(wheelEvent.deltaX / 10) * 10;
|
||||
const deltaY = Math.round(wheelEvent.deltaY / 10) * 10;
|
||||
@@ -662,7 +661,6 @@ export const DOMBrowserRenderer: React.FC<RRWebDOMBrowserRendererProps> = ({
|
||||
}
|
||||
notifyLastAction("scroll");
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const clickHandler: EventListener = (e: Event) => {
|
||||
|
||||
@@ -28,7 +28,7 @@ const fetchWorkflow = (id: string, callback: (response: WorkflowFile) => void) =
|
||||
throw new Error("No workflow found");
|
||||
}
|
||||
}
|
||||
).catch((error) => { console.log(`Failed to fetch workflow:`,error.message) })
|
||||
).catch((error) => { console.log(`Failed to fetch workflow:`, error.message) })
|
||||
};
|
||||
|
||||
interface RightSidePanelProps {
|
||||
@@ -920,14 +920,14 @@ export const RightSidePanel: React.FC<RightSidePanelProps> = ({ onFinishCapture
|
||||
p: 2,
|
||||
mb: 1,
|
||||
borderRadius: '8px',
|
||||
backgroundColor: isDarkMode ? '#1a3a1a' : '#e8f5e9',
|
||||
border: `1px solid ${isDarkMode ? '#2e7d32' : '#4caf50'}`,
|
||||
color: '#1E2124',
|
||||
backgroundColor: isDarkMode ? '#f4f6f4' : '#f4f6f4',
|
||||
border: `1px solid ${isDarkMode ? '#f4f6f4' : '#f4f6f4'}`,
|
||||
}}
|
||||
>
|
||||
<Typography
|
||||
variant="body2"
|
||||
sx={{
|
||||
color: isDarkMode ? '#81c784' : '#2e7d32',
|
||||
fontWeight: 'bold',
|
||||
mb: 0.5
|
||||
}}
|
||||
@@ -943,7 +943,6 @@ export const RightSidePanel: React.FC<RightSidePanelProps> = ({ onFinishCapture
|
||||
<Typography
|
||||
variant="caption"
|
||||
sx={{
|
||||
color: isDarkMode ? '#a5d6a7' : '#388e3c',
|
||||
display: 'block',
|
||||
mb: 1
|
||||
}}
|
||||
@@ -1008,11 +1007,11 @@ export const RightSidePanel: React.FC<RightSidePanelProps> = ({ onFinishCapture
|
||||
}
|
||||
}}
|
||||
sx={{
|
||||
color: isDarkMode ? '#81c784' : '#2e7d32',
|
||||
borderColor: isDarkMode ? '#81c784' : '#2e7d32',
|
||||
color: '#ff00c3 !important',
|
||||
borderColor: '#ff00c3 !important',
|
||||
'&:hover': {
|
||||
borderColor: isDarkMode ? '#a5d6a7' : '#4caf50',
|
||||
backgroundColor: isDarkMode ? '#1a3a1a' : '#f1f8f4',
|
||||
borderColor: '#ff00c3 !important',
|
||||
backgroundColor: '#f4f6f4 !important',
|
||||
}
|
||||
}}
|
||||
>
|
||||
|
||||
@@ -483,31 +483,51 @@ class ClientSelectorGenerator {
|
||||
let grouped = false;
|
||||
|
||||
for (let level = 1; level <= this.groupingConfig.maxParentLevels && !grouped; level++) {
|
||||
let ancestor: HTMLElement | null = currentGroup[0];
|
||||
for (let i = 0; i < level && ancestor; i++) {
|
||||
ancestor = ancestor.parentElement;
|
||||
}
|
||||
const ancestorBuckets = new Map<HTMLElement, HTMLElement[]>();
|
||||
|
||||
if (!ancestor) break;
|
||||
|
||||
const allShareAncestor = currentGroup.every(el => {
|
||||
for (const el of currentGroup) {
|
||||
let elAncestor: HTMLElement | null = el;
|
||||
for (let i = 0; i < level && elAncestor; i++) {
|
||||
elAncestor = elAncestor.parentElement;
|
||||
}
|
||||
return elAncestor === ancestor;
|
||||
});
|
||||
if (elAncestor) {
|
||||
const bucket = ancestorBuckets.get(elAncestor) || [];
|
||||
bucket.push(el);
|
||||
ancestorBuckets.set(elAncestor, bucket);
|
||||
}
|
||||
}
|
||||
|
||||
if (allShareAncestor) {
|
||||
let bestBucket: HTMLElement[] | null = null;
|
||||
for (const bucket of ancestorBuckets.values()) {
|
||||
if (bucket.length >= this.groupingConfig.minGroupSize) {
|
||||
const containsPivot = bucket.includes(element);
|
||||
const bestContainsPivot = bestBucket ? bestBucket.includes(element) : false;
|
||||
|
||||
if (!bestBucket) {
|
||||
bestBucket = bucket;
|
||||
} else if (containsPivot && !bestContainsPivot) {
|
||||
bestBucket = bucket;
|
||||
} else if (containsPivot === bestContainsPivot && bucket.length > bestBucket.length) {
|
||||
bestBucket = bucket;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (bestBucket) {
|
||||
const group: ElementGroup = {
|
||||
elements: currentGroup,
|
||||
elements: bestBucket,
|
||||
fingerprint,
|
||||
representative: element,
|
||||
};
|
||||
currentGroup.forEach((el) => {
|
||||
bestBucket.forEach((el) => {
|
||||
this.elementGroups.set(el, group);
|
||||
this.groupedElements.add(el);
|
||||
});
|
||||
for (const el of currentGroup) {
|
||||
if (!bestBucket.includes(el)) {
|
||||
processedElements.delete(el);
|
||||
}
|
||||
}
|
||||
grouped = true;
|
||||
}
|
||||
}
|
||||
@@ -1008,11 +1028,11 @@ class ClientSelectorGenerator {
|
||||
const result: HTMLElement[] = [];
|
||||
|
||||
for (const element of groupedElements) {
|
||||
const hasGroupedChild = groupedElements.some(
|
||||
const containsGroupedChild = groupedElements.some(
|
||||
(other) => other !== element && element.contains(other)
|
||||
);
|
||||
|
||||
if (hasGroupedChild) {
|
||||
if (!containsGroupedChild) {
|
||||
result.push(element);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user