diff --git a/maxun-core/src/interpret.ts b/maxun-core/src/interpret.ts index d1cc8318..e11ae255 100644 --- a/maxun-core/src/interpret.ts +++ b/maxun-core/src/interpret.ts @@ -192,8 +192,8 @@ export default class Interpreter extends EventEmitter { // const actionable = async (selector: string): Promise => { // try { // const proms = [ - // page.isEnabled(selector, { timeout: 5000 }), - // page.isVisible(selector, { timeout: 5000 }), + // page.isEnabled(selector, { timeout: 10000 }), + // page.isVisible(selector, { timeout: 10000 }), // ]; // return await Promise.all(proms).then((bools) => bools.every((x) => x)); @@ -214,6 +214,17 @@ export default class Interpreter extends EventEmitter { // return []; // }), // ).then((x) => x.flat()); + + const presentSelectors: SelectorArray = await Promise.all( + selectors.map(async (selector) => { + try { + await page.waitForSelector(selector, { state: 'attached' }); + return [selector]; + } catch (e) { + return []; + } + }), + ).then((x) => x.flat()); const action = workflowCopy[workflowCopy.length - 1]; @@ -233,7 +244,7 @@ export default class Interpreter extends EventEmitter { ...p, [cookie.name]: cookie.value, }), {}), - selectors, + selectors: presentSelectors, }; } @@ -767,6 +778,8 @@ export default class Interpreter extends EventEmitter { public async run(page: Page, params?: ParamType): Promise { this.log('Starting the workflow.', Level.LOG); const context = page.context(); + + page.setDefaultNavigationTimeout(100000); // Check proxy settings from context options const contextOptions = (context as any)._options; diff --git a/server/src/browser-management/classes/RemoteBrowser.ts b/server/src/browser-management/classes/RemoteBrowser.ts index 04fb59b3..31aceada 100644 --- a/server/src/browser-management/classes/RemoteBrowser.ts +++ b/server/src/browser-management/classes/RemoteBrowser.ts @@ -104,7 +104,7 @@ export class RemoteBrowser { } catch { return url; } - } + } /** * Determines if a URL change is significant enough to emit @@ -130,11 +130,11 @@ export class RemoteBrowser { }); // Handle page load events with retry mechanism - page.on('load', async () => { + page.on('load', async () => { const injectScript = async (): Promise => { try { await page.waitForLoadState('networkidle', { timeout: 5000 }); - + await page.evaluate(getInjectableScript()); return true; } catch (error: any) { @@ -148,6 +148,19 @@ export class RemoteBrowser { }); } + private getUserAgent() { + const userAgents = [ + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.5845.140 Safari/537.36', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:117.0) Gecko/20100101 Firefox/117.0', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.1938.81 Safari/537.36 Edg/116.0.1938.81', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.5845.96 Safari/537.36 OPR/101.0.4843.25', + 'Mozilla/5.0 (Windows NT 11.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.5938.62 Safari/537.36', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:118.0) Gecko/20100101 Firefox/118.0', + ]; + + return userAgents[Math.floor(Math.random() * userAgents.length)]; + } + /** * An asynchronous constructor for asynchronously initialized properties. * Must be called right after creating an instance of RemoteBrowser class. @@ -155,37 +168,17 @@ export class RemoteBrowser { * @returns {Promise} */ public initialize = async (userId: string): Promise => { - // const launchOptions = { - // headless: true, - // proxy: options.launchOptions?.proxy, - // chromiumSandbox: false, - // args: [ - // '--no-sandbox', - // '--disable-setuid-sandbox', - // '--headless=new', - // '--disable-gpu', - // '--disable-dev-shm-usage', - // '--disable-software-rasterizer', - // '--in-process-gpu', - // '--disable-infobars', - // '--single-process', - // '--no-zygote', - // '--disable-notifications', - // '--disable-extensions', - // '--disable-background-timer-throttling', - // ...(options.launchOptions?.args || []) - // ], - // env: { - // ...process.env, - // CHROMIUM_FLAGS: '--disable-gpu --no-sandbox --headless=new' - // } - // }; - // console.log('Launch options before:', options.launchOptions); - // this.browser = (await options.browser.launch(launchOptions)); - - // console.log('Launch options after:', options.launchOptions) this.browser = (await chromium.launch({ headless: true, + args: [ + "--disable-blink-features=AutomationControlled", + "--disable-web-security", + "--disable-features=IsolateOrigins,site-per-process", + "--disable-site-isolation-trials", + "--disable-extensions", + "--no-sandbox", + "--disable-dev-shm-usage", + ], })); const proxyConfig = await getDecryptedProxyConfig(userId); let proxyOptions: { server: string, username?: string, password?: string } = { server: '' }; @@ -201,7 +194,7 @@ export class RemoteBrowser { const contextOptions: any = { viewport: { height: 400, width: 900 }, // recordVideo: { dir: 'videos/' } - // Force reduced motion to prevent animation issues + // Force reduced motion to prevent animation issues reducedMotion: 'reduce', // Force JavaScript to be enabled javaScriptEnabled: true, @@ -210,7 +203,8 @@ export class RemoteBrowser { // Disable hardware acceleration forcedColors: 'none', isMobile: false, - hasTouch: false + hasTouch: false, + userAgent: this.getUserAgent(), }; if (proxyOptions.server) { @@ -220,18 +214,37 @@ export class RemoteBrowser { password: proxyOptions.password ? proxyOptions.password : undefined, }; } - const browserUserAgent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.5481.38 Safari/537.36"; - - contextOptions.userAgent = browserUserAgent; this.context = await this.browser.newContext(contextOptions); + await this.context.addInitScript( + `const defaultGetter = Object.getOwnPropertyDescriptor( + Navigator.prototype, + "webdriver" + ).get; + defaultGetter.apply(navigator); + defaultGetter.toString(); + Object.defineProperty(Navigator.prototype, "webdriver", { + set: undefined, + enumerable: true, + configurable: true, + get: new Proxy(defaultGetter, { + apply: (target, thisArg, args) => { + Reflect.apply(target, thisArg, args); + return false; + }, + }), + }); + const patchedGetter = Object.getOwnPropertyDescriptor( + Navigator.prototype, + "webdriver" + ).get; + patchedGetter.apply(navigator); + patchedGetter.toString();` + ); this.currentPage = await this.context.newPage(); await this.setupPageEventListeners(this.currentPage); - // await this.currentPage.setExtraHTTPHeaders({ - // 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3' - // }); const blocker = await PlaywrightBlocker.fromLists(fetch, ['https://easylist.to/easylist/easylist.txt']); await blocker.enableBlockingInPage(this.currentPage); this.client = await this.currentPage.context().newCDPSession(this.currentPage); @@ -456,7 +469,7 @@ export class RemoteBrowser { this.currentPage = newPage; if (this.currentPage) { await this.setupPageEventListeners(this.currentPage); - + this.client = await this.currentPage.context().newCDPSession(this.currentPage); await this.subscribeToScreencast(); } else {