From e04d6e6330c8b9fbaad5db4c54f749caf525330d Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Thu, 18 Jul 2024 23:23:31 +0530 Subject: [PATCH 01/15] chore: test --- src/App.test.tsx | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 src/App.test.tsx diff --git a/src/App.test.tsx b/src/App.test.tsx new file mode 100644 index 00000000..2a68616d --- /dev/null +++ b/src/App.test.tsx @@ -0,0 +1,9 @@ +import React from 'react'; +import { render, screen } from '@testing-library/react'; +import App from './App'; + +test('renders learn react link', () => { + render(); + const linkElement = screen.getByText(/learn react/i); + expect(linkElement).toBeInTheDocument(); +}); From 2b40ca3437c533b2439a41e0e9c4faf42adbed61 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Thu, 18 Jul 2024 23:26:55 +0530 Subject: [PATCH 02/15] chore: lint --- server/src/workflow-management/utils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/workflow-management/utils.ts b/server/src/workflow-management/utils.ts index 775a0b55..c10d2c13 100644 --- a/server/src/workflow-management/utils.ts +++ b/server/src/workflow-management/utils.ts @@ -89,4 +89,4 @@ export const getBestSelectorForAction = (action: Action) => { break; } return null; -} +} \ No newline at end of file From 074b972c100d65e342c6851ae9051eda90c59f8a Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Fri, 19 Jul 2024 13:17:10 +0530 Subject: [PATCH 03/15] chore(deps): install playwright-extra --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index 3a30a35c..641b8768 100644 --- a/package.json +++ b/package.json @@ -29,6 +29,7 @@ "loglevel": "^1.8.0", "loglevel-plugin-remote": "^0.6.8", "playwright": "^1.18.1", + "playwright-extra": "^4.3.6", "prismjs": "^1.28.0", "react": "^18.0.0", "react-dom": "^18.0.0", From 100c7aab1eda82e6ce8b15ced1bd2c33b26c4529 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Fri, 19 Jul 2024 13:22:29 +0530 Subject: [PATCH 04/15] chore(deps): install puppeteer-extra-plugin-stealth --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index 641b8768..744dc65c 100644 --- a/package.json +++ b/package.json @@ -31,6 +31,7 @@ "playwright": "^1.18.1", "playwright-extra": "^4.3.6", "prismjs": "^1.28.0", + "puppeteer-extra-plugin-stealth": "^2.11.2", "react": "^18.0.0", "react-dom": "^18.0.0", "react-highlight": "^0.14.0", From e355b856cf60470e84d4287be7add3e47298a2c9 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Fri, 19 Jul 2024 13:23:46 +0530 Subject: [PATCH 05/15] chore: lint --- server/src/browser-management/classes/BrowserPool.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/server/src/browser-management/classes/BrowserPool.ts b/server/src/browser-management/classes/BrowserPool.ts index 736189db..cd4962a1 100644 --- a/server/src/browser-management/classes/BrowserPool.ts +++ b/server/src/browser-management/classes/BrowserPool.ts @@ -38,7 +38,7 @@ export class BrowserPool { /** * Holds all the instances of remote browsers. */ - private pool : PoolDictionary = {}; + private pool: PoolDictionary = {}; /** * Adds a remote browser instance to the pool indexed by the id. @@ -62,12 +62,12 @@ export class BrowserPool { * @param id remote browser instance's id * @returns true if the browser was removed successfully, false otherwise */ - public deleteRemoteBrowser = (id: string) : boolean => { + public deleteRemoteBrowser = (id: string): boolean => { if (!this.pool[id]) { logger.log('warn', `Remote browser with id: ${id} does not exist in the pool`); return false; } - delete(this.pool[id]); + delete (this.pool[id]); logger.log('debug', `Remote browser with id: ${id} deleted from the pool`); return true; }; @@ -77,7 +77,7 @@ export class BrowserPool { * @param id remote browser instance's id * @returns remote browser instance or undefined if it does not exist in the pool */ - public getRemoteBrowser = (id: string) : RemoteBrowser | undefined => { + public getRemoteBrowser = (id: string): RemoteBrowser | undefined => { logger.log('debug', `Remote browser with id: ${id} retrieved from the pool`); return this.pool[id]?.browser; }; @@ -88,7 +88,7 @@ export class BrowserPool { * If there are multiple active browsers, it returns the first one. * @returns the first remote active browser instance's id from the pool */ - public getActiveBrowserId = () : string | null => { + public getActiveBrowserId = (): string | null => { for (const id of Object.keys(this.pool)) { if (this.pool[id].active) { return id; From 845730e95d643ed25c37187bccd49c222847e546 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Fri, 19 Jul 2024 13:27:17 +0530 Subject: [PATCH 06/15] feat: use chromium from playwright-extra --- server/src/routes/record.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/routes/record.ts b/server/src/routes/record.ts index 0a20f66c..9a1d28e4 100644 --- a/server/src/routes/record.ts +++ b/server/src/routes/record.ts @@ -11,7 +11,7 @@ import { stopRunningInterpretation, getRemoteBrowserCurrentUrl, getRemoteBrowserCurrentTabs, } from '../browser-management/controller' -import { chromium } from "playwright"; +import { chromium } from 'playwright-extra'; import logger from "../logger"; export const router = Router(); From 5700224f6b29182952c48584be63f33dcf3eeed3 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Fri, 19 Jul 2024 13:38:59 +0530 Subject: [PATCH 07/15] feat: use stealth plugin --- server/src/routes/record.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/src/routes/record.ts b/server/src/routes/record.ts index 9a1d28e4..0f12f5c9 100644 --- a/server/src/routes/record.ts +++ b/server/src/routes/record.ts @@ -12,9 +12,11 @@ import { getRemoteBrowserCurrentUrl, getRemoteBrowserCurrentTabs, } from '../browser-management/controller' import { chromium } from 'playwright-extra'; +import stealthPlugin from 'puppeteer-extra-plugin-stealth'; import logger from "../logger"; export const router = Router(); +chromium.use(stealthPlugin()); /** * Logs information about remote browser recording session. From 68d24ab11474091ba29d4e8e9cf2b1ff6a1bfd9e Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Fri, 19 Jul 2024 14:10:41 +0530 Subject: [PATCH 08/15] chore(deps): install @cliqz/adblocker-playwright --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index 744dc65c..cc75e23f 100644 --- a/package.json +++ b/package.json @@ -4,6 +4,7 @@ "author": "Karishma Shukla", "license": "", "dependencies": { + "@cliqz/adblocker-playwright": "^1.30.0", "@emotion/react": "^11.9.0", "@emotion/styled": "^11.8.1", "@mui/icons-material": "^5.5.1", From d4ae958954287b93cd4d90b073dd63bb5dfe9e12 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 20 Jul 2024 05:28:42 +0530 Subject: [PATCH 09/15] chore(deps): install cross-fetch --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index cc75e23f..ba5ca0e8 100644 --- a/package.json +++ b/package.json @@ -22,6 +22,7 @@ "axios": "^0.26.0", "buffer": "^6.0.3", "cors": "^2.8.5", + "cross-fetch": "^4.0.0", "dotenv": "^16.0.0", "express": "^4.17.2", "fortawesome": "^0.0.1-security", From b1c9caa2a1a4c5e2a5c8f20f460c422a44515fc4 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 20 Jul 2024 05:57:15 +0530 Subject: [PATCH 10/15] feat: use cross-fetch & adblocker --- server/src/browser-management/classes/RemoteBrowser.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/src/browser-management/classes/RemoteBrowser.ts b/server/src/browser-management/classes/RemoteBrowser.ts index d7376537..d8afc163 100644 --- a/server/src/browser-management/classes/RemoteBrowser.ts +++ b/server/src/browser-management/classes/RemoteBrowser.ts @@ -5,6 +5,8 @@ import { BrowserContext, } from 'playwright'; import { Socket } from "socket.io"; +import { fullLists, PlaywrightBlocker, Request } from '@cliqz/adblocker-playwright'; +import fetch from 'cross-fetch'; import logger from '../../logger'; import { InterpreterSettings, RemoteBrowserOptions } from "../../types"; From 00872522709972ffd5f2b41f2900e471ef7bd021 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 20 Jul 2024 06:01:35 +0530 Subject: [PATCH 11/15] feat: playwright adblocker --- server/src/browser-management/classes/RemoteBrowser.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server/src/browser-management/classes/RemoteBrowser.ts b/server/src/browser-management/classes/RemoteBrowser.ts index d8afc163..2dcf6669 100644 --- a/server/src/browser-management/classes/RemoteBrowser.ts +++ b/server/src/browser-management/classes/RemoteBrowser.ts @@ -13,6 +13,8 @@ import { InterpreterSettings, RemoteBrowserOptions } from "../../types"; import { WorkflowGenerator } from "../../workflow-management/classes/Generator"; import { WorkflowInterpreter } from "../../workflow-management/classes/Interpreter"; + + /** * This class represents a remote browser instance. * It is used to allow a variety of interaction with the Playwright's browser instance. @@ -90,6 +92,7 @@ export class RemoteBrowser { this.browser = (await options.browser.launch(options.launchOptions)); const context = await this.browser.newContext(); this.currentPage = await context.newPage(); + const blocker = await PlaywrightBlocker.fromPrebuiltAdsAndTracking(fetch); this.client = await this.currentPage.context().newCDPSession(this.currentPage); }; From c58d816a590a7e5817896375e79d3e2d13d94bdb Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 20 Jul 2024 06:02:50 +0530 Subject: [PATCH 12/15] feat: enable blocking in current page --- server/src/browser-management/classes/RemoteBrowser.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/browser-management/classes/RemoteBrowser.ts b/server/src/browser-management/classes/RemoteBrowser.ts index 2dcf6669..96960fe6 100644 --- a/server/src/browser-management/classes/RemoteBrowser.ts +++ b/server/src/browser-management/classes/RemoteBrowser.ts @@ -93,6 +93,7 @@ export class RemoteBrowser { const context = await this.browser.newContext(); this.currentPage = await context.newPage(); const blocker = await PlaywrightBlocker.fromPrebuiltAdsAndTracking(fetch); + blocker.enableBlockingInPage(this.currentPage); this.client = await this.currentPage.context().newCDPSession(this.currentPage); }; From 4a35dc7663a4343510c04dc013a2d9820c9ded11 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 20 Jul 2024 06:09:57 +0530 Subject: [PATCH 13/15] feat: disable blocking in current page --- server/src/browser-management/classes/RemoteBrowser.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/browser-management/classes/RemoteBrowser.ts b/server/src/browser-management/classes/RemoteBrowser.ts index 96960fe6..e5ee9c93 100644 --- a/server/src/browser-management/classes/RemoteBrowser.ts +++ b/server/src/browser-management/classes/RemoteBrowser.ts @@ -95,6 +95,7 @@ export class RemoteBrowser { const blocker = await PlaywrightBlocker.fromPrebuiltAdsAndTracking(fetch); blocker.enableBlockingInPage(this.currentPage); this.client = await this.currentPage.context().newCDPSession(this.currentPage); + await blocker.disableBlockingInPage(this.currentPage); }; /** From f7b19ee2cf0f57564f8fa165e0f2bec0979eb144 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 20 Jul 2024 06:10:37 +0530 Subject: [PATCH 14/15] fix: add await to enable blocker --- server/src/browser-management/classes/RemoteBrowser.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/browser-management/classes/RemoteBrowser.ts b/server/src/browser-management/classes/RemoteBrowser.ts index e5ee9c93..04facda6 100644 --- a/server/src/browser-management/classes/RemoteBrowser.ts +++ b/server/src/browser-management/classes/RemoteBrowser.ts @@ -93,7 +93,7 @@ export class RemoteBrowser { const context = await this.browser.newContext(); this.currentPage = await context.newPage(); const blocker = await PlaywrightBlocker.fromPrebuiltAdsAndTracking(fetch); - blocker.enableBlockingInPage(this.currentPage); + await blocker.enableBlockingInPage(this.currentPage); this.client = await this.currentPage.context().newCDPSession(this.currentPage); await blocker.disableBlockingInPage(this.currentPage); }; From bf75569c0382ccebd79d2efe7f40a238ad19c276 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 20 Jul 2024 18:16:27 +0530 Subject: [PATCH 15/15] fix: -rm html2canvas-pro --- package.json | 1 - 1 file changed, 1 deletion(-) diff --git a/package.json b/package.json index ba5ca0e8..3bd306a2 100644 --- a/package.json +++ b/package.json @@ -26,7 +26,6 @@ "dotenv": "^16.0.0", "express": "^4.17.2", "fortawesome": "^0.0.1-security", - "html2canvas-pro": "^1.5.3", "joi": "^17.6.0", "loglevel": "^1.8.0", "loglevel-plugin-remote": "^0.6.8",