Merge pull request #257 from getmaxun/item0-fix

feat: add check for selector visibility, revert domcontentloaded to networkidle state

README.md

@@ -29,12 +29,9 @@ Maxun lets you train a robot in 2 minutes and scrape the web on auto-pilot. Web
 
 <img src="https://static.scarf.sh/a.png?x-pxid=c12a77cc-855e-4602-8a0f-614b2d0da56a" />
 
-> Note: Maxun is in its early stages of development and currently does not support self-hosting. However, you can run Maxun locally. Self-hosting capabilities are planned for a future release and will be available soon.
-
-# Local Installation
+# Installation
 ### Docker Compose
 ```
-git clone https://github.com/getmaxun/maxun
 docker-compose up -d
 ```
 You can access the frontend at http://localhost:5173/ and backend at http://localhost:8080/

docker-compose.yml

@@ -64,8 +64,6 @@ services:
       - redis
       - minio
     volumes:
-      - ./server:/app/server      # Mount server source code for hot reloading
-      - ./maxun-core:/app/maxun-core  # Mount maxun-core for any shared code updates
       - /var/run/dbus:/var/run/dbus
 
   frontend:
@@ -79,13 +77,10 @@ services:
     environment:
       PUBLIC_URL: ${PUBLIC_URL}
       BACKEND_URL: ${BACKEND_URL}
-    volumes:
-      - ./:/app             # Mount entire frontend app directory for hot reloading
-      - /app/node_modules   # Anonymous volume to prevent overwriting node_modules
     depends_on:
       - backend
 
 volumes:
   postgres_data:
   minio_data:
-  redis_data:
+  redis_data:

maxun-core/src/interpret.ts

@@ -192,8 +192,8 @@ export default class Interpreter extends EventEmitter {
     // const actionable = async (selector: string): Promise<boolean> => {
     //   try {
     //     const proms = [
-    //       page.isEnabled(selector, { timeout: 5000 }),
-    //       page.isVisible(selector, { timeout: 5000 }),
+    //       page.isEnabled(selector, { timeout: 10000 }),
+    //       page.isVisible(selector, { timeout: 10000 }),
     //     ];
 
     //     return await Promise.all(proms).then((bools) => bools.every((x) => x));
@@ -214,6 +214,17 @@ export default class Interpreter extends EventEmitter {
     //     return [];
     //   }),
     // ).then((x) => x.flat());
+
+    const presentSelectors: SelectorArray = await Promise.all(
+        selectors.map(async (selector) => {
+            try {
+                await page.waitForSelector(selector, { state: 'attached' });
+                return [selector];
+            } catch (e) {
+                return [];
+            }
+        }),
+    ).then((x) => x.flat());
     
     const action = workflowCopy[workflowCopy.length - 1];
 
@@ -233,7 +244,7 @@ export default class Interpreter extends EventEmitter {
             ...p,
             [cookie.name]: cookie.value,
           }), {}),
-      selectors,
+      selectors: presentSelectors,
     };
   }
 
@@ -767,6 +778,8 @@ export default class Interpreter extends EventEmitter {
   public async run(page: Page, params?: ParamType): Promise<void> {
     this.log('Starting the workflow.', Level.LOG);
     const context = page.context();
+
+    page.setDefaultNavigationTimeout(100000);
     
     // Check proxy settings from context options
     const contextOptions = (context as any)._options;