From ac20d7cbf7fd9e20860622375a57d1dd3d4fb250 Mon Sep 17 00:00:00 2001 From: RohitR311 Date: Sat, 23 Nov 2024 14:58:45 +0530 Subject: [PATCH 01/88] fix: min edit robot limit set to be 1 --- src/components/molecules/RobotEdit.tsx | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/components/molecules/RobotEdit.tsx b/src/components/molecules/RobotEdit.tsx index 74b50f62..9441ecef 100644 --- a/src/components/molecules/RobotEdit.tsx +++ b/src/components/molecules/RobotEdit.tsx @@ -155,9 +155,13 @@ export const RobotEditModal = ({ isOpen, handleStart, handleClose, initialSettin label="Robot Limit" type="number" value={robot.recording.workflow[0].what[0].args[0].limit || ''} - onChange={(e) => - handleLimitChange(parseInt(e.target.value, 10) || 0) - } + onChange={(e) =>{ + const value = parseInt(e.target.value, 10); + if (value >= 1) { + handleLimitChange(value); + } + }} + inputProps={{ min: 1 }} style={{ marginBottom: '20px' }} /> )} From a83e5be2ecae3ff26cc58419498c0a576cb60e92 Mon Sep 17 00:00:00 2001 From: RohitR311 Date: Mon, 25 Nov 2024 20:57:38 +0530 Subject: [PATCH 02/88] feat: add decryptWorkflow to decrypt user inputs --- .../classes/Interpreter.ts | 42 ++++++++++++++++++- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/server/src/workflow-management/classes/Interpreter.ts b/server/src/workflow-management/classes/Interpreter.ts index fa5e9332..b29575a4 100644 --- a/server/src/workflow-management/classes/Interpreter.ts +++ b/server/src/workflow-management/classes/Interpreter.ts @@ -3,6 +3,38 @@ import logger from "../../logger"; import { Socket } from "socket.io"; import { Page } from "playwright"; import { InterpreterSettings } from "../../types"; +import { decrypt } from "../../utils/auth"; + +/** + * Decrypts any encrypted inputs in the workflow. + * @param workflow The workflow to decrypt. + */ +function decryptWorkflow(workflow: WorkflowFile): WorkflowFile { + const decryptedWorkflow = JSON.parse(JSON.stringify(workflow)) as WorkflowFile; + + decryptedWorkflow.workflow.forEach((pair) => { + pair.what.forEach((action) => { + if (action.action === 'type' && Array.isArray(action.args) && action.args.length > 1) { + try { + const encryptedValue = action.args[1]; + if (typeof encryptedValue === 'string') { + const decryptedValue = decrypt(encryptedValue); + action.args[1] = decryptedValue; + } else { + logger.log('error', 'Encrypted value is not a string'); + action.args[1] = ''; + } + } catch (error: unknown) { + const errorMessage = error instanceof Error ? error.message : String(error); + logger.log('error', `Failed to decrypt input value: ${errorMessage}`); + action.args[1] = ''; + } + } + }); + }); + + return decryptedWorkflow; +} /** * This class implements the main interpretation functions. @@ -123,6 +155,9 @@ export class WorkflowInterpreter { ) => { const params = settings.params ? settings.params : null; delete settings.params; + + const decryptedWorkflow = decryptWorkflow(workflow); + const options = { ...settings, debugChannel: { @@ -143,7 +178,7 @@ export class WorkflowInterpreter { } } - const interpreter = new Interpreter(workflow, options); + const interpreter = new Interpreter(decryptedWorkflow, options); this.interpreter = interpreter; interpreter.on('flag', async (page, resume) => { @@ -212,6 +247,9 @@ export class WorkflowInterpreter { public InterpretRecording = async (workflow: WorkflowFile, page: Page, settings: InterpreterSettings) => { const params = settings.params ? settings.params : null; delete settings.params; + + const decryptedWorkflow = decryptWorkflow(workflow); + const options = { ...settings, debugChannel: { @@ -234,7 +272,7 @@ export class WorkflowInterpreter { } } - const interpreter = new Interpreter(workflow, options); + const interpreter = new Interpreter(decryptedWorkflow, options); this.interpreter = interpreter; const status = await interpreter.run(page, params); From 0ecde828db33691711916bb01f4c8e7cad18d45b Mon Sep 17 00:00:00 2001 From: RohitR311 Date: Mon, 25 Nov 2024 20:58:56 +0530 Subject: [PATCH 03/88] feat: add encryption for user input values --- server/src/workflow-management/classes/Generator.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index 7801a20e..4390573e 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -22,6 +22,7 @@ import { getBestSelectorForAction } from "../utils"; import { browserPool } from "../../server"; import { uuid } from "uuidv4"; import { capture } from "../../utils/analytics" +import { encrypt } from "../../utils/auth"; interface PersistedGeneratedData { lastUsedSelector: string; @@ -797,7 +798,7 @@ export class WorkflowGenerator { // when more than one press action is present, add a type action pair.what.splice(index - input.actionCounter, input.actionCounter, { action: 'type', - args: [input.selector, input.value], + args: [input.selector, encrypt(input.value)], }, { action: 'waitForLoadState', args: ['networkidle'], From aa1e70fe2d4cbe0870e5872ee4a225daebe27c6a Mon Sep 17 00:00:00 2001 From: Karishma Shukla Date: Thu, 28 Nov 2024 23:59:48 +0530 Subject: [PATCH 04/88] fix: remove --build from command --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index dd562422..c8c573a9 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ Maxun lets you train a robot in 2 minutes and scrape the web on auto-pilot. Web ### Docker Compose ``` git clone https://github.com/getmaxun/maxun -docker-compose up -d --build +docker-compose up -d ``` ### Without Docker From bf2a52be3d9f12f5eec98a206f963b2cd854112f Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Fri, 29 Nov 2024 16:50:06 +0530 Subject: [PATCH 05/88] chore: -rm esbuild.config.js --- esbuild.config.js | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 esbuild.config.js diff --git a/esbuild.config.js b/esbuild.config.js deleted file mode 100644 index e69de29b..00000000 From e80a8be9888852449857d1dc0e39443c53034a8a Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Fri, 29 Nov 2024 16:56:48 +0530 Subject: [PATCH 06/88] feat: run finished notify --- src/components/molecules/InterpretationButtons.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/molecules/InterpretationButtons.tsx b/src/components/molecules/InterpretationButtons.tsx index 0723bac2..c4e68789 100644 --- a/src/components/molecules/InterpretationButtons.tsx +++ b/src/components/molecules/InterpretationButtons.tsx @@ -105,7 +105,7 @@ export const InterpretationButtons = ({ enableStepping }: InterpretationButtonsP const finished = await interpretCurrentRecording(); setInfo({ ...info, running: false }); if (finished) { - notify('info', 'Interpretation finished'); + notify('info', 'Run finished'); } else { notify('error', 'Interpretation failed to start'); } From 07a14f5dac6085789932dbf01f76145dda9b9f03 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Fri, 29 Nov 2024 16:57:21 +0530 Subject: [PATCH 07/88] feat: run failed to start notify --- src/components/molecules/InterpretationButtons.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/molecules/InterpretationButtons.tsx b/src/components/molecules/InterpretationButtons.tsx index c4e68789..bb83cdb4 100644 --- a/src/components/molecules/InterpretationButtons.tsx +++ b/src/components/molecules/InterpretationButtons.tsx @@ -107,7 +107,7 @@ export const InterpretationButtons = ({ enableStepping }: InterpretationButtonsP if (finished) { notify('info', 'Run finished'); } else { - notify('error', 'Interpretation failed to start'); + notify('error', 'Run failed to start'); } } }; From db2bb6f5217351f7e97b6d7e2d32815f74d57e0a Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Fri, 29 Nov 2024 16:58:36 +0530 Subject: [PATCH 08/88] feat: cannot delete robots notify --- src/components/molecules/RecordingsTable.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/molecules/RecordingsTable.tsx b/src/components/molecules/RecordingsTable.tsx index e9f0aebc..48ce48ec 100644 --- a/src/components/molecules/RecordingsTable.tsx +++ b/src/components/molecules/RecordingsTable.tsx @@ -253,7 +253,7 @@ export const RecordingsTable = ({ handleEditRecording, handleRunRecording, handl checkRunsForRecording(row.id).then((result: boolean) => { if (result) { - notify('warning', 'Cannot delete recording as it has active runs'); + notify('warning', 'Cannot delete robot as it has active runs'); } }) From 4ce3bd640857d083bd41e593c4fda0f0ce7d577c Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Fri, 29 Nov 2024 16:59:01 +0530 Subject: [PATCH 09/88] feat: cannot delete robots notify --- src/components/molecules/RecordingsTable.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/molecules/RecordingsTable.tsx b/src/components/molecules/RecordingsTable.tsx index 48ce48ec..24b7d5d0 100644 --- a/src/components/molecules/RecordingsTable.tsx +++ b/src/components/molecules/RecordingsTable.tsx @@ -253,7 +253,7 @@ export const RecordingsTable = ({ handleEditRecording, handleRunRecording, handl checkRunsForRecording(row.id).then((result: boolean) => { if (result) { - notify('warning', 'Cannot delete robot as it has active runs'); + notify('warning', 'Cannot delete robot as it has associated runs'); } }) From a1ec35e8f2b8841668b8ee83c3d17ba12521a0d0 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Fri, 29 Nov 2024 16:59:21 +0530 Subject: [PATCH 10/88] feat: delete robot notify --- src/components/molecules/RecordingsTable.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/molecules/RecordingsTable.tsx b/src/components/molecules/RecordingsTable.tsx index 24b7d5d0..66888c96 100644 --- a/src/components/molecules/RecordingsTable.tsx +++ b/src/components/molecules/RecordingsTable.tsx @@ -260,7 +260,7 @@ export const RecordingsTable = ({ handleEditRecording, handleRunRecording, handl deleteRecordingFromStorage(row.id).then((result: boolean) => { if (result) { setRows([]); - notify('success', 'Recording deleted successfully'); + notify('success', 'Robot deleted successfully'); fetchRecordings(); } }) From 5c1666cd2416b0f222d22606b81d435cad357e0d Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Fri, 29 Nov 2024 17:00:31 +0530 Subject: [PATCH 11/88] feat: save robot notify --- src/components/molecules/SaveRecording.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/molecules/SaveRecording.tsx b/src/components/molecules/SaveRecording.tsx index 60ef3fa6..cfebc867 100644 --- a/src/components/molecules/SaveRecording.tsx +++ b/src/components/molecules/SaveRecording.tsx @@ -46,7 +46,7 @@ export const SaveRecording = ({ fileName }: SaveRecordingProps) => { }; const exitRecording = useCallback(async () => { - notify('success', 'Recording saved successfully'); + notify('success', 'Robot saved successfully'); if (browserId) { await stopRecording(browserId); } From 834e87c413246ec64d176827826e330c31ad4ed1 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Fri, 29 Nov 2024 17:01:35 +0530 Subject: [PATCH 12/88] feat: api key copy notify --- src/components/organisms/ApiKey.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/organisms/ApiKey.tsx b/src/components/organisms/ApiKey.tsx index 675edb72..0b0b4083 100644 --- a/src/components/organisms/ApiKey.tsx +++ b/src/components/organisms/ApiKey.tsx @@ -88,7 +88,7 @@ const ApiKeyManager = () => { navigator.clipboard.writeText(apiKey); setCopySuccess(true); setTimeout(() => setCopySuccess(false), 2000); - notify('info', 'Copied to clipboard'); + notify('info', 'Copied API Key successfully'); } }; From 46641a285a37b8f9d4a8ea501def491425d5dc81 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Fri, 29 Nov 2024 17:03:10 +0530 Subject: [PATCH 13/88] feat: robot interpretation notify --- src/pages/MainPage.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pages/MainPage.tsx b/src/pages/MainPage.tsx index 6ce7efe8..55647950 100644 --- a/src/pages/MainPage.tsx +++ b/src/pages/MainPage.tsx @@ -49,10 +49,10 @@ export const MainPage = ({ handleEditRecording }: MainPageProps) => { aborted = true; notifyAboutAbort(runId).then(async (response) => { if (response) { - notify('success', `Interpretation of ${runningRecordingName} aborted successfully`); + notify('success', `Interpretation of robot ${runningRecordingName} aborted successfully`); await stopRecording(ids.browserId); } else { - notify('error', `Failed to abort the interpretation ${runningRecordingName} recording`); + notify('error', `Failed to abort the interpretation of ${runningRecordingName} robot`); } }) } From 74c4abee21c73aa703ecddb4e13906150cbc4de2 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Fri, 29 Nov 2024 17:03:43 +0530 Subject: [PATCH 14/88] feat: robot interpretation notify --- src/pages/MainPage.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pages/MainPage.tsx b/src/pages/MainPage.tsx index 55647950..abec5929 100644 --- a/src/pages/MainPage.tsx +++ b/src/pages/MainPage.tsx @@ -67,9 +67,9 @@ export const MainPage = ({ handleEditRecording }: MainPageProps) => { interpretStoredRecording(runId).then(async (interpretation: boolean) => { if (!aborted) { if (interpretation) { - notify('success', `Interpretation of ${runningRecordingName} succeeded`); + notify('success', `Interpretation of robot ${runningRecordingName} succeeded`); } else { - notify('success', `Failed to interpret ${runningRecordingName} recording`); + notify('success', `Failed to interpret ${runningRecordingName} robot`); // destroy the created browser await stopRecording(browserId); } From 428034f48cb41a235c41826d5048af501b86212e Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Fri, 29 Nov 2024 17:04:09 +0530 Subject: [PATCH 15/88] feat: robot run notify --- src/pages/MainPage.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pages/MainPage.tsx b/src/pages/MainPage.tsx index abec5929..752a17ea 100644 --- a/src/pages/MainPage.tsx +++ b/src/pages/MainPage.tsx @@ -98,9 +98,9 @@ export const MainPage = ({ handleEditRecording }: MainPageProps) => { socket.on('debugMessage', debugMessageHandler); setContent('runs'); if (browserId) { - notify('info', `Running recording: ${runningRecordingName}`); + notify('info', `Running robot: ${runningRecordingName}`); } else { - notify('error', `Failed to run recording: ${runningRecordingName}`); + notify('error', `Failed to run robot: ${runningRecordingName}`); } }) return (socket: Socket, browserId: string, runId: string) => { From e195f9e7da2b05581f2b1660a11bae7ec0548553 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Fri, 29 Nov 2024 17:04:34 +0530 Subject: [PATCH 16/88] feat: robot schedule notify --- src/pages/MainPage.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pages/MainPage.tsx b/src/pages/MainPage.tsx index 752a17ea..8af3d3c5 100644 --- a/src/pages/MainPage.tsx +++ b/src/pages/MainPage.tsx @@ -113,9 +113,9 @@ export const MainPage = ({ handleEditRecording }: MainPageProps) => { scheduleStoredRecording(runningRecordingId, settings) .then(({ message, runId }: ScheduleRunResponse) => { if (message === 'success') { - notify('success', `Recording ${runningRecordingName} scheduled successfully`); + notify('success', `Robot ${runningRecordingName} scheduled successfully`); } else { - notify('error', `Failed to schedule recording ${runningRecordingName}`); + notify('error', `Failed to schedule robot ${runningRecordingName}`); } }); } From 3e1794bf49505896259b89051d3bbde3485aebf7 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Fri, 29 Nov 2024 17:05:35 +0530 Subject: [PATCH 17/88] feat: show error if regristration fails --- src/pages/Register.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pages/Register.tsx b/src/pages/Register.tsx index b2a3eebf..c64de4ae 100644 --- a/src/pages/Register.tsx +++ b/src/pages/Register.tsx @@ -44,7 +44,7 @@ const Register = () => { window.localStorage.setItem("user", JSON.stringify(data)); navigate("/"); } catch (error:any) { - notify("error", error.response.data || "Registration Failed. Please try again."); + notify("error", `Registration Failed. Please try again. ${error.response.data}`); setLoading(false); } }; From be55f8ee6573225d08942dde2173df2e4f66f68f Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Fri, 29 Nov 2024 22:07:26 +0530 Subject: [PATCH 18/88] feat: log robot run error --- server/src/api/record.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/api/record.ts b/server/src/api/record.ts index fef0be9a..b9d78a41 100644 --- a/server/src/api/record.ts +++ b/server/src/api/record.ts @@ -621,7 +621,7 @@ async function executeRun(id: string) { }; } catch (error: any) { - logger.log('info', `Error while running a recording with id: ${id} - ${error.message}`); + logger.log('info', `Error while running a robot with id: ${id} - ${error.message}`); const run = await Run.findOne({ where: { runId: id } }); if (run) { await run.update({ From 8cce8f41de437fad5a319d8e80cde28a8ed4a044 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Fri, 29 Nov 2024 22:07:51 +0530 Subject: [PATCH 19/88] feat: log robot run --- server/src/api/record.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/api/record.ts b/server/src/api/record.ts index b9d78a41..c377caa5 100644 --- a/server/src/api/record.ts +++ b/server/src/api/record.ts @@ -660,7 +660,7 @@ export async function handleRunRecording(id: string, userId: string) { socket.on('ready-for-run', () => readyForRunHandler(browserId, newRunId)); - logger.log('info', `Running recording: ${id}`); + logger.log('info', `Running Robot: ${id}`); socket.on('disconnect', () => { cleanupSocketListeners(socket, browserId, newRunId); From 14d9679324f71aa4452af0772391f9d98b72b808 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Fri, 29 Nov 2024 22:08:03 +0530 Subject: [PATCH 20/88] feat: log robot run error --- server/src/api/record.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/api/record.ts b/server/src/api/record.ts index c377caa5..5b33b12f 100644 --- a/server/src/api/record.ts +++ b/server/src/api/record.ts @@ -670,7 +670,7 @@ export async function handleRunRecording(id: string, userId: string) { return newRunId; } catch (error: any) { - logger.error('Error running recording:', error); + logger.error('Error running robot:', error); } } From 0c4f860b14b5490e4decebd0b05ed9fe293853c6 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Fri, 29 Nov 2024 22:10:10 +0530 Subject: [PATCH 21/88] feat: log screencast error --- server/src/browser-management/classes/RemoteBrowser.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/browser-management/classes/RemoteBrowser.ts b/server/src/browser-management/classes/RemoteBrowser.ts index 07ea8780..30541dd7 100644 --- a/server/src/browser-management/classes/RemoteBrowser.ts +++ b/server/src/browser-management/classes/RemoteBrowser.ts @@ -244,7 +244,7 @@ export class RemoteBrowser { } await this.client.send('Page.screencastFrameAck', { sessionId: sessionId }); } catch (e) { - logger.log('error', e); + logger.log('error', `Screencast error: ${e.message}`); } }, 100); }); From d503dc3fccda961794ec77a0b074b02c7a05fdee Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Fri, 29 Nov 2024 22:10:48 +0530 Subject: [PATCH 22/88] feat(ts): set type as any --- server/src/browser-management/classes/RemoteBrowser.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/browser-management/classes/RemoteBrowser.ts b/server/src/browser-management/classes/RemoteBrowser.ts index 30541dd7..5455a05d 100644 --- a/server/src/browser-management/classes/RemoteBrowser.ts +++ b/server/src/browser-management/classes/RemoteBrowser.ts @@ -243,8 +243,8 @@ export class RemoteBrowser { return; } await this.client.send('Page.screencastFrameAck', { sessionId: sessionId }); - } catch (e) { - logger.log('error', `Screencast error: ${e.message}`); + } catch (e: any) { + logger.log('error', `Screencast error: ${e}`); } }, 100); }); From 9ac4677999eea6bfda7247bcd624f3e7f8b34928 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Fri, 29 Nov 2024 22:11:16 +0530 Subject: [PATCH 23/88] feat: log screenshot error --- server/src/browser-management/classes/RemoteBrowser.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/browser-management/classes/RemoteBrowser.ts b/server/src/browser-management/classes/RemoteBrowser.ts index 5455a05d..eb4051c6 100644 --- a/server/src/browser-management/classes/RemoteBrowser.ts +++ b/server/src/browser-management/classes/RemoteBrowser.ts @@ -278,7 +278,7 @@ export class RemoteBrowser { } } catch (e) { const { message } = e as Error; - logger.log('error', message); + logger.log('error', `Screenshot error: ${message}`); } }; From 4521708db87d56deee8eac9cd56a3cb04af8366b Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Fri, 29 Nov 2024 22:12:16 +0530 Subject: [PATCH 24/88] feat: log robot read error --- server/src/routes/storage.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/routes/storage.ts b/server/src/routes/storage.ts index f84583d0..9151e533 100644 --- a/server/src/routes/storage.ts +++ b/server/src/routes/storage.ts @@ -38,7 +38,7 @@ router.get('/recordings', requireSignIn, async (req, res) => { const data = await Robot.findAll(); return res.send(data); } catch (e) { - logger.log('info', 'Error while reading recordings'); + logger.log('info', 'Error while reading robots'); return res.send(null); } }); @@ -55,7 +55,7 @@ router.get('/recordings/:id', requireSignIn, async (req, res) => { ); return res.send(data); } catch (e) { - logger.log('info', 'Error while reading recordings'); + logger.log('info', 'Error while reading robots'); return res.send(null); } }) From 5ff6dcddc1c58b3826845258bd1b9d6d88197c96 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Fri, 29 Nov 2024 22:13:03 +0530 Subject: [PATCH 25/88] feat: log robot run error --- server/src/routes/storage.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/src/routes/storage.ts b/server/src/routes/storage.ts index 9151e533..d1f648f8 100644 --- a/server/src/routes/storage.ts +++ b/server/src/routes/storage.ts @@ -400,7 +400,7 @@ router.put('/runs/:id', requireSignIn, async (req: AuthenticatedRequest, res) => }); } catch (e) { const { message } = e as Error; - logger.log('info', `Error while creating a run with recording id: ${req.params.id} - ${message}`); + logger.log('info', `Error while creating a run with robot id: ${req.params.id} - ${message}`); return res.send(''); } }); @@ -518,7 +518,7 @@ router.post('/runs/run/:id', requireSignIn, async (req: AuthenticatedRequest, re finishedAt: new Date().toLocaleString(), }); } - logger.log('info', `Error while running a recording with id: ${req.params.id} - ${message}`); + logger.log('info', `Error while running a robot with id: ${req.params.id} - ${message}`); capture( 'maxun-oss-run-created-manual', { @@ -757,7 +757,7 @@ router.post('/runs/abort/:id', requireSignIn, async (req, res) => { return res.send(true); } catch (e) { const { message } = e as Error; - logger.log('info', `Error while running a recording with name: ${req.params.fileName}_${req.params.runId}.json`); + logger.log('info', `Error while running a robot with name: ${req.params.fileName}_${req.params.runId}.json`); return res.send(false); } }); \ No newline at end of file From 3f5c30bae9344b07d368ab6fba7ceb51ef512d06 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Fri, 29 Nov 2024 22:13:45 +0530 Subject: [PATCH 26/88] feat: log robot scheduled run error --- server/src/workflow-management/scheduler/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/workflow-management/scheduler/index.ts b/server/src/workflow-management/scheduler/index.ts index 082fcf2e..02ca905f 100644 --- a/server/src/workflow-management/scheduler/index.ts +++ b/server/src/workflow-management/scheduler/index.ts @@ -171,7 +171,7 @@ async function executeRun(id: string) { processGoogleSheetUpdates(); return true; } catch (error: any) { - logger.log('info', `Error while running a recording with id: ${id} - ${error.message}`); + logger.log('info', `Error while running a robot with id: ${id} - ${error.message}`); console.log(error.message); const run = await Run.findOne({ where: { runId: id } }); if (run) { @@ -232,7 +232,7 @@ export async function handleRunRecording(id: string, userId: string) { socket.on('ready-for-run', () => readyForRunHandler(browserId, newRunId)); - logger.log('info', `Running recording: ${id}`); + logger.log('info', `Running robot: ${id}`); socket.on('disconnect', () => { cleanupSocketListeners(socket, browserId, newRunId); From a171dcc605bb23d4e01649746d31d5cb77879a93 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Fri, 29 Nov 2024 22:22:41 +0530 Subject: [PATCH 27/88] feat: remove beta tag --- src/components/molecules/NavBar.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/src/components/molecules/NavBar.tsx b/src/components/molecules/NavBar.tsx index 4c0b7296..ee8c80e8 100644 --- a/src/components/molecules/NavBar.tsx +++ b/src/components/molecules/NavBar.tsx @@ -58,7 +58,6 @@ export const NavBar: React.FC = ({ recordingName, isRecording }) => }}>
Maxun
- { user ? ( From 38516c0d8405fe111dbeef0e606ec56bb29c27dd Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Fri, 29 Nov 2024 22:32:30 +0530 Subject: [PATCH 28/88] feat: center loader --- src/components/organisms/ApiKey.tsx | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/components/organisms/ApiKey.tsx b/src/components/organisms/ApiKey.tsx index 675edb72..201947a9 100644 --- a/src/components/organisms/ApiKey.tsx +++ b/src/components/organisms/ApiKey.tsx @@ -92,7 +92,20 @@ const ApiKeyManager = () => { } }; - if (loading) return ; + if (loading) { + return ( + + + + ); + } return ( From a3c494ea28228c119928efbfcfe0f395058d334d Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Fri, 29 Nov 2024 22:33:02 +0530 Subject: [PATCH 29/88] feat: center loader vertically & horizontally --- src/components/organisms/ApiKey.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/src/components/organisms/ApiKey.tsx b/src/components/organisms/ApiKey.tsx index 201947a9..5b029d78 100644 --- a/src/components/organisms/ApiKey.tsx +++ b/src/components/organisms/ApiKey.tsx @@ -100,6 +100,7 @@ const ApiKeyManager = () => { justifyContent: 'center', alignItems: 'center', height: '100vh', + width: '100vw', }} > From dcacfbc47443593159bb4eb6adc00b895e876c21 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Fri, 29 Nov 2024 22:33:23 +0530 Subject: [PATCH 30/88] chore: lint --- src/components/organisms/ApiKey.tsx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/components/organisms/ApiKey.tsx b/src/components/organisms/ApiKey.tsx index 5b029d78..2729581e 100644 --- a/src/components/organisms/ApiKey.tsx +++ b/src/components/organisms/ApiKey.tsx @@ -36,9 +36,9 @@ const ApiKeyManager = () => { const [copySuccess, setCopySuccess] = useState(false); const { notify } = useGlobalInfoStore(); - - + + useEffect(() => { const fetchApiKey = async () => { @@ -53,7 +53,7 @@ const ApiKeyManager = () => { }; fetchApiKey(); - + }, []); const generateApiKey = async () => { @@ -61,7 +61,7 @@ const ApiKeyManager = () => { try { const { data } = await axios.post(`${apiUrl}/auth/generate-api-key`); setApiKey(data.api_key); - + notify('success', `Generated API Key successfully`); } catch (error: any) { notify('error', `Failed to generate API Key - ${error.message}`); From 58f6aa46e716fc3a66730dc4bc7d7f994fa7ed21 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 30 Nov 2024 20:25:04 +0530 Subject: [PATCH 31/88] chore: -rm whitespace --- src/components/molecules/RecordingsTable.tsx | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/components/molecules/RecordingsTable.tsx b/src/components/molecules/RecordingsTable.tsx index 66888c96..632cb92f 100644 --- a/src/components/molecules/RecordingsTable.tsx +++ b/src/components/molecules/RecordingsTable.tsx @@ -151,9 +151,6 @@ export const RecordingsTable = ({ handleEditRecording, handleRunRecording, handl row.name.toLowerCase().includes(searchTerm.toLowerCase()) ); - - - return ( From 63caff80654b381cb154fd7f4efc98f651748b1e Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 30 Nov 2024 20:26:09 +0530 Subject: [PATCH 32/88] feat: move duplicate above delete --- src/components/molecules/RecordingsTable.tsx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/components/molecules/RecordingsTable.tsx b/src/components/molecules/RecordingsTable.tsx index 632cb92f..43fc54f6 100644 --- a/src/components/molecules/RecordingsTable.tsx +++ b/src/components/molecules/RecordingsTable.tsx @@ -246,6 +246,9 @@ export const RecordingsTable = ({ handleEditRecording, handleRunRecording, handl handleEditRobot(row.id, row.name, row.params || [])} + handleDuplicate={() => { + handleDuplicateRobot(row.id, row.name, row.params || []); + }} handleDelete={() => { checkRunsForRecording(row.id).then((result: boolean) => { @@ -262,9 +265,6 @@ export const RecordingsTable = ({ handleEditRecording, handleRunRecording, handl } }) }} - handleDuplicate={() => { - handleDuplicateRobot(row.id, row.name, row.params || []); - }} /> ); From 226c22b0e2f3f7a384751e992c46040a4ec21cb7 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 30 Nov 2024 20:28:23 +0530 Subject: [PATCH 33/88] feat: move duplicate above delete --- src/components/molecules/RecordingsTable.tsx | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/components/molecules/RecordingsTable.tsx b/src/components/molecules/RecordingsTable.tsx index 43fc54f6..651d3677 100644 --- a/src/components/molecules/RecordingsTable.tsx +++ b/src/components/molecules/RecordingsTable.tsx @@ -417,18 +417,18 @@ const OptionsButton = ({ handleEdit, handleDelete, handleDuplicate }: OptionsBut Edit - { handleDelete(); handleClose(); }}> - - - - Delete - { handleDuplicate(); handleClose(); }}> Duplicate + { handleDelete(); handleClose(); }}> + + + + Delete + ); From c704d4e410d97534db7ac8261853f69508a6358d Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 30 Nov 2024 21:32:20 +0530 Subject: [PATCH 34/88] feat: show loading duration --- src/components/molecules/InterpretationButtons.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/molecules/InterpretationButtons.tsx b/src/components/molecules/InterpretationButtons.tsx index bb83cdb4..7e06b59f 100644 --- a/src/components/molecules/InterpretationButtons.tsx +++ b/src/components/molecules/InterpretationButtons.tsx @@ -139,7 +139,7 @@ export const InterpretationButtons = ({ enableStepping }: InterpretationButtonsP disabled={info.running} sx={{ display: 'grid' }} > - {info.running ? 'Extracting data...please wait' : 'Get Preview of Output Data'} + {info.running ? 'Extracting data...please wait for 10secs to 1min' : 'Get Preview of Output Data'} { }} From 91dbf7b7056a98df6ad07d6c1a2d041e5bcbcee3 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 30 Nov 2024 21:37:20 +0530 Subject: [PATCH 35/88] feat: show circular progress --- src/components/molecules/InterpretationButtons.tsx | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/components/molecules/InterpretationButtons.tsx b/src/components/molecules/InterpretationButtons.tsx index 7e06b59f..b959fd92 100644 --- a/src/components/molecules/InterpretationButtons.tsx +++ b/src/components/molecules/InterpretationButtons.tsx @@ -141,6 +141,11 @@ export const InterpretationButtons = ({ enableStepping }: InterpretationButtonsP > {info.running ? 'Extracting data...please wait for 10secs to 1min' : 'Get Preview of Output Data'} + {info.running && ( + + + + )} { }} isOpen={decisionModal.open} From 2c00ee001bdbac214650b650776ba576b6bfaab0 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 30 Nov 2024 21:37:41 +0530 Subject: [PATCH 36/88] fix: import circular progress --- src/components/molecules/InterpretationButtons.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/molecules/InterpretationButtons.tsx b/src/components/molecules/InterpretationButtons.tsx index b959fd92..c21d4518 100644 --- a/src/components/molecules/InterpretationButtons.tsx +++ b/src/components/molecules/InterpretationButtons.tsx @@ -1,4 +1,4 @@ -import { Box, Button, Stack, Typography } from "@mui/material"; +import { Box, Button, Stack, Typography, CircularProgress } from "@mui/material"; import { PlayCircle } from "@mui/icons-material"; import React, { useCallback, useEffect, useState } from "react"; import { interpretCurrentRecording, stopCurrentInterpretation } from "../../api/recording"; From 1d09f009731da5cbe971ed1d2e81fa751cd489c1 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 30 Nov 2024 21:41:42 +0530 Subject: [PATCH 37/88] feat: align progress with text --- src/components/molecules/InterpretationButtons.tsx | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/components/molecules/InterpretationButtons.tsx b/src/components/molecules/InterpretationButtons.tsx index c21d4518..3407b94a 100644 --- a/src/components/molecules/InterpretationButtons.tsx +++ b/src/components/molecules/InterpretationButtons.tsx @@ -139,13 +139,10 @@ export const InterpretationButtons = ({ enableStepping }: InterpretationButtonsP disabled={info.running} sx={{ display: 'grid' }} > - {info.running ? 'Extracting data...please wait for 10secs to 1min' : 'Get Preview of Output Data'} + {info.running ? + Extracting data...please wait for 10secs to 1min + : 'Get Preview of Output Data'} - {info.running && ( - - - - )} { }} isOpen={decisionModal.open} From a6681bfda713e3f48251612dee9a970da9be11e9 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 30 Nov 2024 21:43:20 +0530 Subject: [PATCH 38/88] feat: set margin right to 10px --- src/components/molecules/InterpretationButtons.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/molecules/InterpretationButtons.tsx b/src/components/molecules/InterpretationButtons.tsx index 3407b94a..e6edb66e 100644 --- a/src/components/molecules/InterpretationButtons.tsx +++ b/src/components/molecules/InterpretationButtons.tsx @@ -140,7 +140,7 @@ export const InterpretationButtons = ({ enableStepping }: InterpretationButtonsP sx={{ display: 'grid' }} > {info.running ? - Extracting data...please wait for 10secs to 1min + Extracting data...please wait for 10secs to 1min : 'Get Preview of Output Data'} Date: Sat, 30 Nov 2024 21:43:52 +0530 Subject: [PATCH 39/88] feat: set color to inherit --- src/components/molecules/InterpretationButtons.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/molecules/InterpretationButtons.tsx b/src/components/molecules/InterpretationButtons.tsx index e6edb66e..c52d8f6c 100644 --- a/src/components/molecules/InterpretationButtons.tsx +++ b/src/components/molecules/InterpretationButtons.tsx @@ -140,7 +140,7 @@ export const InterpretationButtons = ({ enableStepping }: InterpretationButtonsP sx={{ display: 'grid' }} > {info.running ? - Extracting data...please wait for 10secs to 1min + Extracting data...please wait for 10secs to 1min : 'Get Preview of Output Data'} Date: Sat, 30 Nov 2024 21:50:58 +0530 Subject: [PATCH 40/88] feat: remove margin left --- src/components/molecules/InterpretationButtons.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/molecules/InterpretationButtons.tsx b/src/components/molecules/InterpretationButtons.tsx index c52d8f6c..2072cfd6 100644 --- a/src/components/molecules/InterpretationButtons.tsx +++ b/src/components/molecules/InterpretationButtons.tsx @@ -140,7 +140,7 @@ export const InterpretationButtons = ({ enableStepping }: InterpretationButtonsP sx={{ display: 'grid' }} > {info.running ? - Extracting data...please wait for 10secs to 1min + Extracting data...please wait for 10secs to 1min : 'Get Preview of Output Data'} Date: Sat, 30 Nov 2024 21:52:56 +0530 Subject: [PATCH 41/88] feat: set progress size to 22 --- src/components/molecules/InterpretationButtons.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/molecules/InterpretationButtons.tsx b/src/components/molecules/InterpretationButtons.tsx index 2072cfd6..9d983761 100644 --- a/src/components/molecules/InterpretationButtons.tsx +++ b/src/components/molecules/InterpretationButtons.tsx @@ -140,7 +140,7 @@ export const InterpretationButtons = ({ enableStepping }: InterpretationButtonsP sx={{ display: 'grid' }} > {info.running ? - Extracting data...please wait for 10secs to 1min + Extracting data...please wait for 10secs to 1min : 'Get Preview of Output Data'} Date: Sun, 1 Dec 2024 22:31:11 +0530 Subject: [PATCH 42/88] docs: mention Maxun --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c8c573a9..0eec8261 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ Maxun lets you train a robot in 2 minutes and scrape the web on auto-pilot. Web -> Note: We are in early stages of development and do not support self hosting yet. You can run Maxun locally. +> Note: Maxun is in early stages of development and does not support self hosting yet. You can run Maxun locally. # Local Setup ### Docker Compose From dc9bdec79586b2a48097ee10cacf3330598121dc Mon Sep 17 00:00:00 2001 From: KaranRudra311 Date: Sun, 1 Dec 2024 22:32:22 +0530 Subject: [PATCH 43/88] docs: clearer note --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0eec8261..9306f619 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ Maxun lets you train a robot in 2 minutes and scrape the web on auto-pilot. Web -> Note: Maxun is in early stages of development and does not support self hosting yet. You can run Maxun locally. +> Note: Maxun is in its early stages of development and currently does not support self-hosting. However, you can run Maxun locally. Self-hosting capabilities are planned for a future release and will be available soon. # Local Setup ### Docker Compose From 97249c183e3394cf5bc7b38fa1e9224d5d4ca8e1 Mon Sep 17 00:00:00 2001 From: KaranRudra311 Date: Sun, 1 Dec 2024 22:34:37 +0530 Subject: [PATCH 44/88] docs: fe be url repeat for clairty --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 9306f619..0c82c7f4 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,7 @@ Maxun lets you train a robot in 2 minutes and scrape the web on auto-pilot. Web git clone https://github.com/getmaxun/maxun docker-compose up -d ``` +You can access the frontend at http://localhost:5173/ and backend at http://localhost:8080/ ### Without Docker 1. Ensure you have Node.js, PostgreSQL, MinIO and Redis installed on your system. From 838e483d1b193b0a99120e66cbc1774e920989ea Mon Sep 17 00:00:00 2001 From: Karan Date: Sun, 1 Dec 2024 22:50:30 +0530 Subject: [PATCH 45/88] feat: create CODE_OF_CONDUCT.md --- .github/CODE_OF_CONDUCT.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 .github/CODE_OF_CONDUCT.md diff --git a/.github/CODE_OF_CONDUCT.md b/.github/CODE_OF_CONDUCT.md new file mode 100644 index 00000000..58134216 --- /dev/null +++ b/.github/CODE_OF_CONDUCT.md @@ -0,0 +1,12 @@ +# Contributor Code of Conduct +As contributors and maintainers of this project, we pledge to respect all people who contribute through reporting issues, posting feature requests, updating documentation, submitting pull requests or patches, and other activities. + +We are committed to making participation in this project a harassment-free experience for everyone, regardless of the level of experience, gender, gender identity and expression, sexual orientation, disability, personal appearance, body size, race, age, or religion. + +Examples of unacceptable behavior by participants include the use of sexual language or imagery, derogatory comments or personal attacks, trolling, public or private harassment, insults, or other unprofessional conduct. + +Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct. Project maintainers who do not follow the Code of Conduct may be removed from the project team. + +Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by opening an issue or contacting one or more of the project maintainers. + +This Code of Conduct is adapted from the Contributor Covenant, version 1.0.0, available at http://contributor-covenant.org/version/1/0/0/ From 9c186e1e707a18cd116fd37f27701f5bbe282e0f Mon Sep 17 00:00:00 2001 From: Karan Date: Sun, 1 Dec 2024 22:55:51 +0530 Subject: [PATCH 46/88] feat: create COMMIT_CONVENTION.md --- .github/.github/COMMIT_CONVENTION.md | 130 +++++++++++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100644 .github/.github/COMMIT_CONVENTION.md diff --git a/.github/.github/COMMIT_CONVENTION.md b/.github/.github/COMMIT_CONVENTION.md new file mode 100644 index 00000000..076ff9cf --- /dev/null +++ b/.github/.github/COMMIT_CONVENTION.md @@ -0,0 +1,130 @@ +## Git Commit Message Convention + +> This is adapted from [Conventional Commits 1.0.0](https://www.conventionalcommits.org/en/v1.0.0/). + +## Summary + +The Conventional Commits specification is a lightweight convention on top of commit messages. +It provides an easy set of rules for creating an explicit commit history; +which makes it easier to write automated tools on top of. +This convention dovetails with [SemVer](http://semver.org), +by describing the features, fixes, and breaking changes made in commit messages. + +The commit message should be structured as follows: + +--- + +``` +[optional scope]: +[optional body] +[optional footer(s)] +``` +--- + +
+The commit contains the following structural elements, to communicate intent to the +consumers of your library: + +1. **fix:** a commit of the _type_ `fix` patches a bug in your codebase (this correlates with [`PATCH`](http://semver.org/#summary) in Semantic Versioning). +1. **feat:** a commit of the _type_ `feat` introduces a new feature to the codebase (this correlates with [`MINOR`](http://semver.org/#summary) in Semantic Versioning). +1. **BREAKING CHANGE:** a commit that has a footer `BREAKING CHANGE:`, or appends a `!` after the type/scope, introduces a breaking API change (correlating with [`MAJOR`](http://semver.org/#summary) in Semantic Versioning). +A BREAKING CHANGE can be part of commits of any _type_. +1. _types_ other than `fix:` and `feat:` are allowed, for example [@commitlint/config-conventional](https://github.com/conventional-changelog/commitlint/tree/master/%40commitlint/config-conventional) (based on the [the Angular convention](https://github.com/angular/angular/blob/22b96b9/CONTRIBUTING.md#-commit-message-guidelines)) recommends `build:`, `chore:`, + `ci:`, `docs:`, `style:`, `refactor:`, `perf:`, `test:`, and others. +1. _footers_ other than `BREAKING CHANGE: ` may be provided and follow a convention similar to + [git trailer format](https://git-scm.com/docs/git-interpret-trailers). + +Additional types are not mandated by the Conventional Commits specification, and have no implicit effect in Semantic Versioning (unless they include a BREAKING CHANGE). +

+A scope may be provided to a commit's type, to provide additional contextual information and is contained within parenthesis, e.g., `feat(parser): add ability to parse arrays`. + +## Examples + +### Commit message with description and breaking change footer +``` +feat: allow provided config object to extend other configs + +BREAKING CHANGE: `extends` key in config file is now used for extending other config files +``` + +### Commit message with `!` to draw attention to breaking change +``` +feat!: send an email to the customer when a product is shipped +``` + +### Commit message with scope and `!` to draw attention to breaking change +``` +feat(api)!: send an email to the customer when a product is shipped +``` + +### Commit message with both `!` and BREAKING CHANGE footer +``` +chore!: drop support for Node 6 + +BREAKING CHANGE: use JavaScript features not available in Node 6. +``` + +### Commit message with no body +``` +docs: correct spelling of CHANGELOG +``` + +### Commit message with scope +``` +feat(lang): add polish language +``` + +### Commit message with multi-paragraph body and multiple footers +``` +fix: prevent racing of requests + +Introduce a request id and a reference to latest request. Dismiss +incoming responses other than from latest request. + +Remove timeouts which were used to mitigate the racing issue but are +obsolete now. + +Reviewed-by: Z +Refs: #123 +``` + +## Specification + +The key words “MUST”, “MUST NOT”, “REQUIRED”, “SHALL”, “SHALL NOT”, “SHOULD”, “SHOULD NOT”, “RECOMMENDED”, “MAY”, and “OPTIONAL” in this document are to be interpreted as described in [RFC 2119](https://www.ietf.org/rfc/rfc2119.txt). + +1. Commits MUST be prefixed with a type, which consists of a noun, `feat`, `fix`, etc., followed + by the OPTIONAL scope, OPTIONAL `!`, and REQUIRED terminal colon and space. +1. The type `feat` MUST be used when a commit adds a new feature to your application or library. +1. The type `fix` MUST be used when a commit represents a bug fix for your application. +1. A scope MAY be provided after a type. A scope MUST consist of a noun describing a + section of the codebase surrounded by parenthesis, e.g., `fix(parser):` +1. A description MUST immediately follow the colon and space after the type/scope prefix. +The description is a short summary of the code changes, e.g., _fix: array parsing issue when multiple spaces were contained in string_. +1. A longer commit body MAY be provided after the short description, providing additional contextual information about the code changes. The body MUST begin one blank line after the description. +1. A commit body is free-form and MAY consist of any number of newline separated paragraphs. +1. One or more footers MAY be provided one blank line after the body. Each footer MUST consist of + a word token, followed by either a `:` or `#` separator, followed by a string value (this is inspired by the + [git trailer convention](https://git-scm.com/docs/git-interpret-trailers)). +1. A footer's token MUST use `-` in place of whitespace characters, e.g., `Acked-by` (this helps differentiate + the footer section from a multi-paragraph body). An exception is made for `BREAKING CHANGE`, which MAY also be used as a token. +1. A footer's value MAY contain spaces and newlines, and parsing MUST terminate when the next valid footer + token/separator pair is observed. +1. Breaking changes MUST be indicated in the type/scope prefix of a commit, or as an entry in the + footer. +1. If included as a footer, a breaking change MUST consist of the uppercase text BREAKING CHANGE, followed by a colon, space, and description, e.g., +_BREAKING CHANGE: environment variables now take precedence over config files_. +1. If included in the type/scope prefix, breaking changes MUST be indicated by a + `!` immediately before the `:`. If `!` is used, `BREAKING CHANGE:` MAY be omitted from the footer section, + and the commit description SHALL be used to describe the breaking change. +1. Types other than `feat` and `fix` MAY be used in your commit messages, e.g., _docs: updated ref docs._ +1. The units of information that make up Conventional Commits MUST NOT be treated as case sensitive by implementors, with the exception of BREAKING CHANGE which MUST be uppercase. +1. BREAKING-CHANGE MUST be synonymous with BREAKING CHANGE, when used as a token in a footer. + +## Why Use Conventional Commits + +* Automatically generating CHANGELOGs. +* Automatically determining a semantic version bump (based on the types of commits landed). +* Communicating the nature of changes to teammates, the public, and other stakeholders. +* Triggering build and publish processes. +* Making it easier for people to contribute to your projects, by allowing them to explore + a more structured commit history. From 1952f2edfbbd0709584cc6f2489255ddd20e2615 Mon Sep 17 00:00:00 2001 From: Karan Date: Sun, 1 Dec 2024 22:57:13 +0530 Subject: [PATCH 47/88] feat: rename setup to installation --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0c82c7f4..368753c0 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ Maxun lets you train a robot in 2 minutes and scrape the web on auto-pilot. Web > Note: Maxun is in its early stages of development and currently does not support self-hosting. However, you can run Maxun locally. Self-hosting capabilities are planned for a future release and will be available soon. -# Local Setup +# Local Installation ### Docker Compose ``` git clone https://github.com/getmaxun/maxun From 6a2755e4ae5dd09a3ad1c1d24215e616bc7b7357 Mon Sep 17 00:00:00 2001 From: RohitR311 Date: Mon, 2 Dec 2024 17:05:19 +0530 Subject: [PATCH 48/88] fix: emit socket urlChanged on frame navigation --- .../classes/RemoteBrowser.ts | 25 ++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/server/src/browser-management/classes/RemoteBrowser.ts b/server/src/browser-management/classes/RemoteBrowser.ts index eb4051c6..769787da 100644 --- a/server/src/browser-management/classes/RemoteBrowser.ts +++ b/server/src/browser-management/classes/RemoteBrowser.ts @@ -165,6 +165,13 @@ export class RemoteBrowser { contextOptions.userAgent = browserUserAgent; this.context = await this.browser.newContext(contextOptions); this.currentPage = await this.context.newPage(); + + this.currentPage.on('framenavigated', (frame) => { + if (frame === this.currentPage?.mainFrame()) { + this.socket.emit('urlChanged', this.currentPage.url()); + } + }); + // await this.currentPage.setExtraHTTPHeaders({ // 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3' // }); @@ -362,6 +369,13 @@ export class RemoteBrowser { if (page) { await this.stopScreencast(); this.currentPage = page; + + this.currentPage.on('framenavigated', (frame) => { + if (frame === this.currentPage?.mainFrame()) { + this.socket.emit('urlChanged', this.currentPage.url()); + } + }); + //await this.currentPage.setViewportSize({ height: 400, width: 900 }) this.client = await this.currentPage.context().newCDPSession(this.currentPage); this.socket.emit('urlChanged', this.currentPage.url()); @@ -388,9 +402,14 @@ export class RemoteBrowser { await this.currentPage?.close(); this.currentPage = newPage; if (this.currentPage) { - this.currentPage.on('load', (page) => { - this.socket.emit('urlChanged', page.url()); - }) + this.currentPage.on('framenavigated', (frame) => { + if (frame === this.currentPage?.mainFrame()) { + this.socket.emit('urlChanged', this.currentPage.url()); + } + }); + // this.currentPage.on('load', (page) => { + // this.socket.emit('urlChanged', page.url()); + // }) this.client = await this.currentPage.context().newCDPSession(this.currentPage); await this.subscribeToScreencast(); } else { From c1af321d8b9bf8af2c2f6214fefcc8c2e09f9263 Mon Sep 17 00:00:00 2001 From: RohitR311 Date: Tue, 3 Dec 2024 17:51:06 +0530 Subject: [PATCH 49/88] feat: add scrapeSchema data accumulation logic --- maxun-core/src/interpret.ts | 79 ++++++++++++++++++++++++++++++------- 1 file changed, 65 insertions(+), 14 deletions(-) diff --git a/maxun-core/src/interpret.ts b/maxun-core/src/interpret.ts index ecef02db..114605e2 100644 --- a/maxun-core/src/interpret.ts +++ b/maxun-core/src/interpret.ts @@ -16,6 +16,23 @@ import Concurrency from './utils/concurrency'; import Preprocessor from './preprocessor'; import log, { Level } from './utils/logger'; +/** + * Extending the Window interface for custom scraping functions. + */ +declare global { + interface Window { + scrape: (selector: string | null) => Record[]; + scrapeSchema: ( + schema: Record + ) => Record; + scrapeList: (config: { listSelector: string; fields: any; limit?: number; pagination: any }) => Record[]; + scrapeListAuto: (listSelector: string) => { selector: string; innerText: string }[]; + scrollDown: (pages?: number) => void; + scrollUp: (pages?: number) => void; + } +} + + /** * Defines optional intepreter options (passed in constructor) */ @@ -31,7 +48,6 @@ interface InterpreterOptions { }> } - /** * Class for running the Smart Workflows. */ @@ -50,6 +66,8 @@ export default class Interpreter extends EventEmitter { private blocker: PlaywrightBlocker | null = null; + private cumulativeResults: Record[] = []; + constructor(workflow: WorkflowFile, options?: Partial) { super(); this.workflow = workflow.workflow; @@ -57,7 +75,9 @@ export default class Interpreter extends EventEmitter { this.options = { maxRepeats: 5, maxConcurrency: 5, - serializableCallback: (data) => { log(JSON.stringify(data), Level.WARN); }, + serializableCallback: (data) => { + log(JSON.stringify(data), Level.WARN); + }, binaryCallback: () => { log('Received binary data, thrashing them.', Level.WARN); }, debug: false, debugChannel: {}, @@ -214,11 +234,11 @@ export default class Interpreter extends EventEmitter { // every condition is treated as a single context switch (key as keyof typeof operators) { - case '$and': + case '$and' as keyof typeof operators: return array?.every((x) => this.applicable(x, context)); - case '$or': + case '$or' as keyof typeof operators: return array?.some((x) => this.applicable(x, context)); - case '$not': + case '$not' as keyof typeof operators: return !this.applicable(value, context); // $not should be a unary operator default: throw new Error('Undefined logic operator.'); @@ -233,9 +253,9 @@ export default class Interpreter extends EventEmitter { }; switch (key as keyof typeof meta) { - case '$before': + case '$before' as keyof typeof meta: return !usedActions.find(testRegexString); - case '$after': + case '$after' as keyof typeof meta: return !!usedActions.find(testRegexString); default: throw new Error('Undefined meta operator.'); @@ -308,9 +328,35 @@ export default class Interpreter extends EventEmitter { scrapeSchema: async (schema: Record) => { await this.ensureScriptsLoaded(page); - + + // Scrape data using the schema const scrapeResult = await page.evaluate((schemaObj) => window.scrapeSchema(schemaObj), schema); - await this.options.serializableCallback(scrapeResult); + + // Log result and accumulate it + console.log("Scrape result:", scrapeResult); + this.cumulativeResults.push(...(Array.isArray(scrapeResult) ? scrapeResult : [scrapeResult])); + + const mergedResult: Record[] = [ + Object.fromEntries( + Object.entries( + this.cumulativeResults.reduce((acc, curr) => { + Object.entries(curr).forEach(([key, value]) => { + // If the key doesn't exist or the current value is not undefined, add/update it + if (value !== undefined) { + acc[key] = value; + } + }); + return acc; + }, {}) + ) + ) + ]; + + // Log cumulative results after each action + console.log("CUMULATIVE results:", this.cumulativeResults); + console.log("MERGED results:", mergedResult); + + await this.options.serializableCallback(mergedResult); }, scrapeList: async (config: { listSelector: string, fields: any, limit?: number, pagination: any }) => { @@ -357,7 +403,7 @@ export default class Interpreter extends EventEmitter { }; for (const step of steps) { - this.log(`Launching ${step.action}`, Level.LOG); + this.log(`Launching ${String(step.action)}`, Level.LOG); if (step.action in wawActions) { // "Arrayifying" here should not be needed (TS + syntax checker - only arrays; but why not) @@ -365,7 +411,7 @@ export default class Interpreter extends EventEmitter { await wawActions[step.action as CustomFunctions](...(params ?? [])); } else { // Implements the dot notation for the "method name" in the workflow - const levels = step.action.split('.'); + const levels = String(step.action).split('.'); const methodName = levels[levels.length - 1]; let invokee: any = page; @@ -534,9 +580,14 @@ export default class Interpreter extends EventEmitter { if (this.options.debug) { this.log(`Current state is: \n${JSON.stringify(pageState, null, 2)}`, Level.WARN); } - const actionId = workflow.findIndex( - (step) => this.applicable(step.where, pageState, usedActions), - ); + + const actionId = workflow.findIndex((step) => { + const isApplicable = this.applicable(step.where, pageState, usedActions); + console.log(`Where:`, step.where); + console.log(`Page state:`, pageState); + console.log(`Match result: ${isApplicable}`); + return isApplicable; + }); const action = workflow[actionId]; From de52038773b0a35cf7ccb60a6df30523aef04ef6 Mon Sep 17 00:00:00 2001 From: RohitR311 Date: Tue, 3 Dec 2024 17:53:10 +0530 Subject: [PATCH 50/88] feat: add conditio to get last value of serializedData --- server/src/workflow-management/classes/Interpreter.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/server/src/workflow-management/classes/Interpreter.ts b/server/src/workflow-management/classes/Interpreter.ts index b29575a4..d81a6b21 100644 --- a/server/src/workflow-management/classes/Interpreter.ts +++ b/server/src/workflow-management/classes/Interpreter.ts @@ -277,10 +277,14 @@ export class WorkflowInterpreter { const status = await interpreter.run(page, params); + const lastArray = this.serializableData.length > 1 + ? [this.serializableData[this.serializableData.length - 1]] + : this.serializableData; + const result = { log: this.debugMessages, result: status, - serializableOutput: this.serializableData.reduce((reducedObject, item, index) => { + serializableOutput: lastArray.reduce((reducedObject, item, index) => { return { [`item-${index}`]: item, ...reducedObject, From 7cc535adc1f9f86883df92e61515487bf009eb31 Mon Sep 17 00:00:00 2001 From: RohitR311 Date: Tue, 3 Dec 2024 17:54:24 +0530 Subject: [PATCH 51/88] feat: add selectors to scrapeSchema action where pair --- .../workflow-management/classes/Generator.ts | 48 +++++++++++++------ 1 file changed, 33 insertions(+), 15 deletions(-) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index 4390573e..f13d36cd 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -160,6 +160,20 @@ export class WorkflowGenerator { }) }; + private async getSelectorsForSchema(page: Page, schema: Record): Promise { + const selectors = Object.values(schema).map((field) => field.selector); + + // Verify if the selectors are present and actionable on the current page + const actionableSelectors: string[] = []; + for (const selector of selectors) { + const isActionable = await page.isVisible(selector).catch(() => false); + if (isActionable) { + actionableSelectors.push(selector); + } + } + return actionableSelectors; + } + /** * Adds a newly generated pair to the workflow and notifies the client about it by * sending the updated workflow through socket. @@ -185,55 +199,59 @@ export class WorkflowGenerator { */ private addPairToWorkflowAndNotifyClient = async (pair: WhereWhatPair, page: Page) => { let matched = false; - // validate if a pair with the same where conditions is already present in the workflow + + // Check for scrapeSchema actions and enhance the where condition + if (pair.what[0].action === 'scrapeSchema') { + const schema = pair.what[0]?.args?.[0]; + if (schema) { + const additionalSelectors = await this.getSelectorsForSchema(page, schema); + pair.where.selectors = [...(pair.where.selectors || []), ...additionalSelectors]; + } + } + + // Validate if the pair is already in the workflow if (pair.where.selectors && pair.where.selectors[0]) { const match = selectorAlreadyInWorkflow(pair.where.selectors[0], this.workflowRecord.workflow); if (match) { - // if a match of where conditions is found, the new action is added into the matched rule const matchedIndex = this.workflowRecord.workflow.indexOf(match); if (pair.what[0].action !== 'waitForLoadState' && pair.what[0].action !== 'press') { pair.what.push({ action: 'waitForLoadState', args: ['networkidle'], - }) + }); } this.workflowRecord.workflow[matchedIndex].what = this.workflowRecord.workflow[matchedIndex].what.concat(pair.what); - logger.log('info', `Pushed ${JSON.stringify(this.workflowRecord.workflow[matchedIndex])} to workflow pair`); matched = true; } } - // is the where conditions of the pair are not already in the workflow, we need to validate the where conditions - // for possible overshadowing of different rules and handle cases according to the recording logic + + // Handle cases where the where condition isn't already present if (!matched) { const handled = await this.handleOverShadowing(pair, page, this.generatedData.lastIndex || 0); if (!handled) { - //adding waitForLoadState with networkidle, for better success rate of automatically recorded workflows if (pair.what[0].action !== 'waitForLoadState' && pair.what[0].action !== 'press') { pair.what.push({ action: 'waitForLoadState', args: ['networkidle'], - }) + }); } if (this.generatedData.lastIndex === 0) { this.generatedData.lastIndex = null; - // we want to have the most specific selectors at the beginning of the workflow this.workflowRecord.workflow.unshift(pair); } else { this.workflowRecord.workflow.splice(this.generatedData.lastIndex || 0, 0, pair); if (this.generatedData.lastIndex) { - this.generatedData.lastIndex = this.generatedData.lastIndex - 1; + this.generatedData.lastIndex -= 1; } } - logger.log('info', - `${JSON.stringify(pair)}: Added to workflow file on index: ${this.generatedData.lastIndex || 0}`); - } else { - logger.log('debug', - ` ${JSON.stringify(this.workflowRecord.workflow[this.generatedData.lastIndex || 0])} added action to workflow pair`); } } + + // Emit the updated workflow to the client this.socket.emit('workflow', this.workflowRecord); logger.log('info', `Workflow emitted`); }; + /** * Generates a pair for the click event. From 1af752cdcb7c5e634874041a3d55b72c6cf6af7e Mon Sep 17 00:00:00 2001 From: amhsirak Date: Tue, 3 Dec 2024 21:09:19 +0530 Subject: [PATCH 52/88] refactor: move isProduction variable --- server/src/server.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/server.ts b/server/src/server.ts index e6fee5f2..87f334d2 100644 --- a/server/src/server.ts +++ b/server/src/server.ts @@ -18,6 +18,7 @@ import { fork } from 'child_process'; import { capture } from "./utils/analytics"; import swaggerUi from 'swagger-ui-express'; import swaggerSpec from './swagger/config'; +const isProduction = process.env.NODE_ENV === 'production'; const app = express(); app.use(cors({ @@ -62,7 +63,6 @@ readdirSync(path.join(__dirname, 'api')).forEach((r) => { } }); -const isProduction = process.env.NODE_ENV === 'production'; const workerPath = path.resolve(__dirname, isProduction ? './worker.js' : './worker.ts'); let workerProcess: any; From 45c34196533a508e0170a6e1c05282b59a093a76 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Tue, 3 Dec 2024 21:10:32 +0530 Subject: [PATCH 53/88] feat: allow origins based on node env --- server/src/server.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/server.ts b/server/src/server.ts index 87f334d2..26263f3a 100644 --- a/server/src/server.ts +++ b/server/src/server.ts @@ -19,6 +19,7 @@ import { capture } from "./utils/analytics"; import swaggerUi from 'swagger-ui-express'; import swaggerSpec from './swagger/config'; const isProduction = process.env.NODE_ENV === 'production'; +const allowedOrigin = isProduction ? process.env.ALLOWED_ORIGIN : '*'; const app = express(); app.use(cors({ From 52d9869b84bfde48cc2f62d31036249e11f53fd0 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Tue, 3 Dec 2024 21:14:38 +0530 Subject: [PATCH 54/88] feat: rename to ALLOWED_PUBLIC_URL --- server/src/server.ts | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/server/src/server.ts b/server/src/server.ts index 26263f3a..e69a9b1f 100644 --- a/server/src/server.ts +++ b/server/src/server.ts @@ -19,12 +19,18 @@ import { capture } from "./utils/analytics"; import swaggerUi from 'swagger-ui-express'; import swaggerSpec from './swagger/config'; const isProduction = process.env.NODE_ENV === 'production'; -const allowedOrigin = isProduction ? process.env.ALLOWED_ORIGIN : '*'; +const allowedOrigin = isProduction ? process.env.ALLOWED_PUBLIC_URL : '*'; const app = express(); app.use(cors({ - origin: 'http://localhost:5173', - credentials: true, + origin: (origin, callback) => { + if (!isProduction || origin === allowedOrigin || allowedOrigin === '*') { + callback(null, true); // Allow all in development or match production origin + } else { + callback(new Error('Not allowed by CORS')); // Block unexpected origins in production + } + }, + credentials: true, // Include credentials if needed })); app.use(express.json()); From ed7690cb290a591749b05e0c65f7232454afdb4f Mon Sep 17 00:00:00 2001 From: amhsirak Date: Tue, 3 Dec 2024 21:24:50 +0530 Subject: [PATCH 55/88] feat: dynamically set origin --- server/src/server.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/server.ts b/server/src/server.ts index e69a9b1f..832b84da 100644 --- a/server/src/server.ts +++ b/server/src/server.ts @@ -99,7 +99,7 @@ app.get('/', function (req, res) { // Add CORS headers app.use((req, res, next) => { - res.header('Access-Control-Allow-Origin', '*'); + res.header('Access-Control-Allow-Origin', allowedOrigin); res.header('Access-Control-Allow-Methods', 'GET,PUT,POST,DELETE,OPTIONS'); res.header('Access-Control-Allow-Headers', 'Content-Type, Authorization'); if (req.method === 'OPTIONS') { From c60612124915038bb83b32dba0a10f44ecfd9746 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Tue, 3 Dec 2024 21:25:25 +0530 Subject: [PATCH 56/88] feat: include credentials if needed --- server/src/server.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/server.ts b/server/src/server.ts index 832b84da..135542bd 100644 --- a/server/src/server.ts +++ b/server/src/server.ts @@ -102,6 +102,7 @@ app.use((req, res, next) => { res.header('Access-Control-Allow-Origin', allowedOrigin); res.header('Access-Control-Allow-Methods', 'GET,PUT,POST,DELETE,OPTIONS'); res.header('Access-Control-Allow-Headers', 'Content-Type, Authorization'); + res.header('Access-Control-Allow-Credentials', 'true'); if (req.method === 'OPTIONS') { return res.sendStatus(200); } From 011eee4593411da89a799dd0c4b4c89e11eb10da Mon Sep 17 00:00:00 2001 From: amhsirak Date: Tue, 3 Dec 2024 21:27:16 +0530 Subject: [PATCH 57/88] feat: handle only preflight & additional headers --- server/src/server.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/server/src/server.ts b/server/src/server.ts index 135542bd..1467e0c7 100644 --- a/server/src/server.ts +++ b/server/src/server.ts @@ -99,10 +99,8 @@ app.get('/', function (req, res) { // Add CORS headers app.use((req, res, next) => { - res.header('Access-Control-Allow-Origin', allowedOrigin); res.header('Access-Control-Allow-Methods', 'GET,PUT,POST,DELETE,OPTIONS'); res.header('Access-Control-Allow-Headers', 'Content-Type, Authorization'); - res.header('Access-Control-Allow-Credentials', 'true'); if (req.method === 'OPTIONS') { return res.sendStatus(200); } From accd0332a354ab9d6226ddec78a403b6d4384c1f Mon Sep 17 00:00:00 2001 From: amhsirak Date: Tue, 3 Dec 2024 21:39:56 +0530 Subject: [PATCH 58/88] feat: simplify to use process.env.PUBLIC_URL --- server/src/server.ts | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/server/src/server.ts b/server/src/server.ts index 1467e0c7..7965d6bd 100644 --- a/server/src/server.ts +++ b/server/src/server.ts @@ -18,19 +18,11 @@ import { fork } from 'child_process'; import { capture } from "./utils/analytics"; import swaggerUi from 'swagger-ui-express'; import swaggerSpec from './swagger/config'; -const isProduction = process.env.NODE_ENV === 'production'; -const allowedOrigin = isProduction ? process.env.ALLOWED_PUBLIC_URL : '*'; const app = express(); app.use(cors({ - origin: (origin, callback) => { - if (!isProduction || origin === allowedOrigin || allowedOrigin === '*') { - callback(null, true); // Allow all in development or match production origin - } else { - callback(new Error('Not allowed by CORS')); // Block unexpected origins in production - } - }, - credentials: true, // Include credentials if needed + origin: process.env.PUBLIC_URL ? process.env.PUBLIC_URL : 'http://localhost:5173', + credentials: true, })); app.use(express.json()); @@ -70,6 +62,7 @@ readdirSync(path.join(__dirname, 'api')).forEach((r) => { } }); +const isProduction = process.env.NODE_ENV === 'production'; const workerPath = path.resolve(__dirname, isProduction ? './worker.js' : './worker.ts'); let workerProcess: any; @@ -99,6 +92,7 @@ app.get('/', function (req, res) { // Add CORS headers app.use((req, res, next) => { + res.header('Access-Control-Allow-Origin', '*'); res.header('Access-Control-Allow-Methods', 'GET,PUT,POST,DELETE,OPTIONS'); res.header('Access-Control-Allow-Headers', 'Content-Type, Authorization'); if (req.method === 'OPTIONS') { From b97df3522af773a76b689aab9865aa9977eb92bb Mon Sep 17 00:00:00 2001 From: amhsirak Date: Tue, 3 Dec 2024 21:40:36 +0530 Subject: [PATCH 59/88] feat: ensure access-control-allow-origin matches public url --- server/src/server.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/server.ts b/server/src/server.ts index 7965d6bd..20ef14e7 100644 --- a/server/src/server.ts +++ b/server/src/server.ts @@ -92,7 +92,7 @@ app.get('/', function (req, res) { // Add CORS headers app.use((req, res, next) => { - res.header('Access-Control-Allow-Origin', '*'); + res.header('Access-Control-Allow-Origin', process.env.PUBLIC_URL || 'http://localhost:5173'); res.header('Access-Control-Allow-Methods', 'GET,PUT,POST,DELETE,OPTIONS'); res.header('Access-Control-Allow-Headers', 'Content-Type, Authorization'); if (req.method === 'OPTIONS') { From 27148072dcf1b25627b6691768eea60e55675c1c Mon Sep 17 00:00:00 2001 From: amhsirak Date: Tue, 3 Dec 2024 21:41:21 +0530 Subject: [PATCH 60/88] feat: set credentials to true --- server/src/server.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/server.ts b/server/src/server.ts index 20ef14e7..8c28c2d2 100644 --- a/server/src/server.ts +++ b/server/src/server.ts @@ -95,6 +95,7 @@ app.use((req, res, next) => { res.header('Access-Control-Allow-Origin', process.env.PUBLIC_URL || 'http://localhost:5173'); res.header('Access-Control-Allow-Methods', 'GET,PUT,POST,DELETE,OPTIONS'); res.header('Access-Control-Allow-Headers', 'Content-Type, Authorization'); + res.header('Access-Control-Allow-Credentials', 'true'); if (req.method === 'OPTIONS') { return res.sendStatus(200); } From 6b664a00d1a801cd6e83173b1763e284dfb3b35f Mon Sep 17 00:00:00 2001 From: Karishma Shukla Date: Tue, 3 Dec 2024 21:49:43 +0530 Subject: [PATCH 61/88] feat: add PUBLIC_URL variable --- ENVEXAMPLE | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ENVEXAMPLE b/ENVEXAMPLE index a387785f..0153f820 100644 --- a/ENVEXAMPLE +++ b/ENVEXAMPLE @@ -14,9 +14,10 @@ MINIO_SECRET_KEY=minio_secret_key # MinIO secret key REDIS_HOST=redis # Redis host in Docker REDIS_PORT=6379 # Redis port (default: 6379) -# Backend URLs -BACKEND_URL=http://localhost:8080 # Internal URL for backend service -VITE_BACKEND_URL=http://localhost:8080 # URL used by frontend to connect to backend +# Backend and Frontend URLs +BACKEND_URL=http://localhost:8080 # URL on which the backend runs. You can change it based on your needs. +VITE_BACKEND_URL=http://localhost:8080 # URL used by frontend to connect to backend. It should always have the same value as BACKEND_URL +PUBLIC_URL=http://localhost:5173 # URL used by backend to connect to frontend. You can change it based on your needs. # Optional Google OAuth settings for Google Sheet Integration GOOGLE_CLIENT_ID=your_google_client_id From 064318ea92196d4ef6f87a3f363c049fd1976eb9 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Tue, 3 Dec 2024 21:55:02 +0530 Subject: [PATCH 62/88] feat: redirect based on PUBLIC_URL --- server/src/routes/auth.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/routes/auth.ts b/server/src/routes/auth.ts index 692add99..cc3d879b 100644 --- a/server/src/routes/auth.ts +++ b/server/src/routes/auth.ts @@ -384,7 +384,7 @@ router.get( httpOnly: false, maxAge: 60000, }); - res.redirect(`http://localhost:5173`); + res.redirect(process.env.PUBLIC_URL as string || "http://localhost:5173"); } catch (error: any) { res.status(500).json({ message: `Google OAuth error: ${error.message}` }); } From 40e0c7e8393bfba796131883424d68a59d488c35 Mon Sep 17 00:00:00 2001 From: RohitR311 Date: Tue, 3 Dec 2024 22:21:26 +0530 Subject: [PATCH 63/88] feat: add scrapeSchema data accumulation logic --- maxun-core/src/interpret.ts | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/maxun-core/src/interpret.ts b/maxun-core/src/interpret.ts index 114605e2..a7a5de47 100644 --- a/maxun-core/src/interpret.ts +++ b/maxun-core/src/interpret.ts @@ -329,15 +329,23 @@ export default class Interpreter extends EventEmitter { scrapeSchema: async (schema: Record) => { await this.ensureScriptsLoaded(page); - // Scrape data using the schema const scrapeResult = await page.evaluate((schemaObj) => window.scrapeSchema(schemaObj), schema); - // Log result and accumulate it - console.log("Scrape result:", scrapeResult); - this.cumulativeResults.push(...(Array.isArray(scrapeResult) ? scrapeResult : [scrapeResult])); + const newResults = Array.isArray(scrapeResult) ? scrapeResult : [scrapeResult]; + newResults.forEach((result) => { + Object.entries(result).forEach(([key, value]) => { + const keyExists = this.cumulativeResults.some( + (item) => key in item && item[key] !== undefined + ); + + if (!keyExists) { + this.cumulativeResults.push({ [key]: value }); + } + }); + }); const mergedResult: Record[] = [ - Object.fromEntries( + Object.fromEntries( Object.entries( this.cumulativeResults.reduce((acc, curr) => { Object.entries(curr).forEach(([key, value]) => { From 942981e1ea14a5be5d49dd6be4adc2c5abb9c163 Mon Sep 17 00:00:00 2001 From: RohitR311 Date: Tue, 3 Dec 2024 22:44:45 +0530 Subject: [PATCH 64/88] feat: add or array empty condition for scrapeSchema --- maxun-core/src/browserSide/scraper.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maxun-core/src/browserSide/scraper.js b/maxun-core/src/browserSide/scraper.js index 828a4f84..369a08be 100644 --- a/maxun-core/src/browserSide/scraper.js +++ b/maxun-core/src/browserSide/scraper.js @@ -249,7 +249,7 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3, } }, (key) => key // Use the original key in the output - )); + )) || []; } /** From bf80c41e19efe7f23fad91ed7ea42b88a8887103 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Tue, 3 Dec 2024 23:52:55 +0530 Subject: [PATCH 65/88] feat: handle VITE_PUBLIC_URL --- vite.config.js | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/vite.config.js b/vite.config.js index 59f495a1..9ca574eb 100644 --- a/vite.config.js +++ b/vite.config.js @@ -1,11 +1,20 @@ import { defineConfig } from 'vite'; import react from '@vitejs/plugin-react'; +import dotenv from 'dotenv'; +dotenv.config(); export default defineConfig(() => { + const publicUrl = process.env.VITE_PUBLIC_URL || 'http://localhost:5173'; + return { define: { 'import.meta.env.VITE_BACKEND_URL': JSON.stringify(process.env.VITE_BACKEND_URL), + 'import.meta.env.VITE_PUBLIC_URL': JSON.stringify(publicUrl), }, + server: { + host: new URL(publicUrl).hostname, + port: parseInt(new URL(publicUrl).port), + }, build: { outDir: 'build', manifest: true, From 0299d90657939cebc8328e82ade9d3a3bab5405d Mon Sep 17 00:00:00 2001 From: Karishma Shukla Date: Tue, 3 Dec 2024 23:56:09 +0530 Subject: [PATCH 66/88] feat: add VITE_PUBLIC_URL variable --- ENVEXAMPLE | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ENVEXAMPLE b/ENVEXAMPLE index 0153f820..3de361d6 100644 --- a/ENVEXAMPLE +++ b/ENVEXAMPLE @@ -17,7 +17,9 @@ REDIS_PORT=6379 # Redis port (default: 6379) # Backend and Frontend URLs BACKEND_URL=http://localhost:8080 # URL on which the backend runs. You can change it based on your needs. VITE_BACKEND_URL=http://localhost:8080 # URL used by frontend to connect to backend. It should always have the same value as BACKEND_URL -PUBLIC_URL=http://localhost:5173 # URL used by backend to connect to frontend. You can change it based on your needs. + +PUBLIC_URL=http://localhost:5173 # URL on which the frontend runs. You can change it based on your needs. +VITE_PUBLIC_URL=http://localhost:5173 # URL used by backend to connect to frontend. It should always have the same value as PUBLIC_URL # Optional Google OAuth settings for Google Sheet Integration GOOGLE_CLIENT_ID=your_google_client_id From 4ba9a08973684432e34728d2d200f579a502eec5 Mon Sep 17 00:00:00 2001 From: Karishma Shukla Date: Tue, 3 Dec 2024 23:58:09 +0530 Subject: [PATCH 67/88] docs: VITE_PUBLIC_URL & PUBLIC_URL --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index dd562422..b0f44e08 100644 --- a/README.md +++ b/README.md @@ -78,6 +78,8 @@ You can access the frontend at http://localhost:5173/ and backend at http://loca |-----------------------|-----------|----------------------------------------------------------------------------------------------|--------------------------------------------------------------| | `BACKEND_URL` | Yes | URL to run backend on. | Default value: http://localhost:8080 | | `VITE_BACKEND_URL` | Yes | URL used by frontend to connect to backend | Default value: http://localhost:8080 | +| `PUBLIC_URL` | Yes | URL to run frontend on. | Default value: http://localhost:5173 | +| `VITE_PUBLIC_URL` | Yes | URL used by backend to connect to frontend | Default value: http://localhost:5173 | | `JWT_SECRET` | Yes | Secret key used to sign and verify JSON Web Tokens (JWTs) for authentication. | JWT authentication will not work. | | `DB_NAME` | Yes | Name of the Postgres database to connect to. | Database connection will fail. | | `DB_USER` | Yes | Username for Postgres database authentication. | Database connection will fail. | From 3f5b67af685a5f0a6017147b8b37e6dd0dc52613 Mon Sep 17 00:00:00 2001 From: RohitR311 Date: Wed, 4 Dec 2024 14:44:45 +0530 Subject: [PATCH 68/88] feat: add check for browser step id and gather settings --- src/components/organisms/RightSidePanel.tsx | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/components/organisms/RightSidePanel.tsx b/src/components/organisms/RightSidePanel.tsx index a11989bd..4aaf7b21 100644 --- a/src/components/organisms/RightSidePanel.tsx +++ b/src/components/organisms/RightSidePanel.tsx @@ -54,6 +54,7 @@ export const RightSidePanel: React.FC = ({ onFinishCapture const [showCaptureScreenshot, setShowCaptureScreenshot] = useState(true); const [showCaptureText, setShowCaptureText] = useState(true); const [hoverStates, setHoverStates] = useState<{ [id: string]: boolean }>({}); + const [browserStepIdList, setBrowserStepIdList] = useState([]); const { lastAction, notify, currentWorkflowActionsState, setCurrentWorkflowActionsState } = useGlobalInfoStore(); const { getText, startGetText, stopGetText, getScreenshot, startGetScreenshot, stopGetScreenshot, getList, startGetList, stopGetList, startPaginationMode, stopPaginationMode, paginationType, updatePaginationType, limitType, customLimit, updateLimitType, updateCustomLimit, stopLimitMode, startLimitMode, captureStage, setCaptureStage } = useActionContext(); @@ -195,12 +196,18 @@ export const RightSidePanel: React.FC = ({ onFinishCapture const getTextSettingsObject = useCallback(() => { const settings: Record = {}; browserSteps.forEach(step => { + if (browserStepIdList.includes(step.id)) { + return; + } + if (step.type === 'text' && step.label && step.selectorObj?.selector) { settings[step.label] = step.selectorObj; } + setBrowserStepIdList(prevList => [...prevList, step.id]); }); + return settings; - }, [browserSteps]); + }, [browserSteps, browserStepIdList]); const stopCaptureAndEmitGetTextSettings = useCallback(() => { @@ -211,6 +218,7 @@ export const RightSidePanel: React.FC = ({ onFinishCapture } stopGetText(); const settings = getTextSettingsObject(); + console.log("SETTINGS", settings); const hasTextSteps = browserSteps.some(step => step.type === 'text'); if (hasTextSteps) { socket?.emit('action', { action: 'scrapeSchema', settings }); From e4abddc9db94a690809f70c031002f561cca650d Mon Sep 17 00:00:00 2001 From: RohitR311 Date: Wed, 4 Dec 2024 16:55:08 +0530 Subject: [PATCH 69/88] feat: add encryption to key press action --- server/src/workflow-management/classes/Generator.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index f13d36cd..312f8a2a 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -319,7 +319,7 @@ export class WorkflowGenerator { where, what: [{ action: 'press', - args: [selector, key], + args: [selector, encrypt(key)], }], } if (selector) { From f9b8f36ff74239ef819a021c7a40213060fd5e8c Mon Sep 17 00:00:00 2001 From: RohitR311 Date: Wed, 4 Dec 2024 16:55:42 +0530 Subject: [PATCH 70/88] feat: add decryption to key press action --- server/src/workflow-management/classes/Interpreter.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/workflow-management/classes/Interpreter.ts b/server/src/workflow-management/classes/Interpreter.ts index d81a6b21..d53259b7 100644 --- a/server/src/workflow-management/classes/Interpreter.ts +++ b/server/src/workflow-management/classes/Interpreter.ts @@ -14,7 +14,7 @@ function decryptWorkflow(workflow: WorkflowFile): WorkflowFile { decryptedWorkflow.workflow.forEach((pair) => { pair.what.forEach((action) => { - if (action.action === 'type' && Array.isArray(action.args) && action.args.length > 1) { + if ((action.action === 'type' || action.action === 'press') && Array.isArray(action.args) && action.args.length > 1) { try { const encryptedValue = action.args[1]; if (typeof encryptedValue === 'string') { From 597ac392643a27469cf00a5ff321618f15514cdb Mon Sep 17 00:00:00 2001 From: amhsirak Date: Wed, 4 Dec 2024 18:08:18 +0530 Subject: [PATCH 71/88] feat: use DB_PORT --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 8b26973b..6b38f634 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,7 +8,7 @@ services: POSTGRES_PASSWORD: ${DB_PASSWORD} POSTGRES_DB: ${DB_NAME} ports: - - "5432:5432" + - "${DB_PORT}:${DB_PORT}" volumes: - postgres_data:/var/lib/postgresql/data healthcheck: From 76b2d5ad2d4683107954347cff6c561d754f0f75 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Wed, 4 Dec 2024 18:09:04 +0530 Subject: [PATCH 72/88] feat: use REDIS_PORT --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 6b38f634..805473c2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -23,7 +23,7 @@ services: REDIS_HOST: ${REDIS_HOST} REDIS_PORT: ${REDIS_PORT} ports: - - "6379:6379" + - "${REDIS_PORT}:${REDIS_PORT}" volumes: - redis_data:/data From 72fbc483f07a188ad565fbe5b99dd04a0e379559 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Wed, 4 Dec 2024 18:14:24 +0530 Subject: [PATCH 73/88] feat: use MINIO_PORT --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 805473c2..6d221ce8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -34,7 +34,7 @@ services: MINIO_ROOT_PASSWORD: ${MINIO_SECRET_KEY} command: server /data --console-address :9001 ports: - - "9000:9000" # API port + - "${MINIO_PORT}:${MINIO_PORT}" # API port - "9001:9001" # WebUI port volumes: - minio_data:/data From 3bacc7aad51229283e776a3c166ef7da5ed6d7f4 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Wed, 4 Dec 2024 19:51:29 +0530 Subject: [PATCH 74/88] feat: use FRONTEND_PORT and BACKEND_PORT --- docker-compose.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 6d221ce8..4b0d59f1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -45,9 +45,10 @@ services: #dockerfile: server/Dockerfile image: getmaxun/maxun-backend:v0.0.2 ports: - - "8080:8080" + - "${BACKEND_PORT}:${BACKEND_PORT}" env_file: .env environment: + BACKEND_URL: ${BACKEND_URL} # to ensure Playwright works in Docker PLAYWRIGHT_BROWSERS_PATH: /ms-playwright PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 0 @@ -74,8 +75,11 @@ services: #dockerfile: Dockerfile image: getmaxun/maxun-frontend:v0.0.1 ports: - - "5173:5173" + - "${FRONTEND_PORT}:${FRONTEND_PORT}" env_file: .env + environment: + PUBLIC_URL: ${PUBLIC_URL} + BACKEND_URL: ${BACKEND_URL} volumes: - ./:/app # Mount entire frontend app directory for hot reloading - /app/node_modules # Anonymous volume to prevent overwriting node_modules From abdcb36c44bfa081a28d4a625abfa052e45973df Mon Sep 17 00:00:00 2001 From: amhsirak Date: Wed, 4 Dec 2024 19:52:17 +0530 Subject: [PATCH 75/88] chore: format --- docker-compose.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 4b0d59f1..41372cf2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -57,9 +57,8 @@ services: CHROMIUM_FLAGS: '--disable-gpu --no-sandbox --headless=new' security_opt: - seccomp=unconfined # This might help with browser sandbox issues - # Increase shared memory size for Chromium - shm_size: '2gb' - mem_limit: 2g # Set a 2GB memory limit + shm_size: '2gb' # Increase shared memory size for Chromium + mem_limit: 2g # Set a 2GB memory limit depends_on: - postgres - redis @@ -82,7 +81,7 @@ services: BACKEND_URL: ${BACKEND_URL} volumes: - ./:/app # Mount entire frontend app directory for hot reloading - - /app/node_modules # Anonymous volume to prevent overwriting node_modules + - /app/node_modules # Anonymous volume to prevent overwriting node_modules depends_on: - backend From e9517d67d4d7631b38394890e8716baa084cf2e5 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Wed, 4 Dec 2024 20:17:12 +0530 Subject: [PATCH 76/88] fix: use process.env.BACKEND_PORT --- server/src/constants/config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/constants/config.ts b/server/src/constants/config.ts index 74d9de4c..1943fbe4 100644 --- a/server/src/constants/config.ts +++ b/server/src/constants/config.ts @@ -1,4 +1,4 @@ -export const SERVER_PORT = process.env.SERVER_PORT ? Number(process.env.SERVER_PORT) : 8080 +export const SERVER_PORT = process.env.BACKEND_PORT ? Number(process.env.BACKEND_PORT) : 8080 export const DEBUG = process.env.DEBUG === 'true' export const LOGS_PATH = process.env.LOGS_PATH ?? 'server/logs' export const ANALYTICS_ID = 'oss' \ No newline at end of file From 7f7221e4e41f268ebc6e48bc1d4ebfc915fd8986 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Wed, 4 Dec 2024 20:20:42 +0530 Subject: [PATCH 77/88] feat: use MINIO_CONSOLE_PORT --- docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 41372cf2..9308a563 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -32,10 +32,10 @@ services: environment: MINIO_ROOT_USER: ${MINIO_ACCESS_KEY} MINIO_ROOT_PASSWORD: ${MINIO_SECRET_KEY} - command: server /data --console-address :9001 + command: server /data --console-address :${MINIO_CONSOLE_PORT} ports: - "${MINIO_PORT}:${MINIO_PORT}" # API port - - "9001:9001" # WebUI port + - "${MINIO_CONSOLE_PORT}:${MINIO_CONSOLE_PORT}" # WebUI port volumes: - minio_data:/data From 6f8d976c9fd9687c07bb2aa0f5a8411711e77356 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Wed, 4 Dec 2024 20:24:37 +0530 Subject: [PATCH 78/88] feat: set default ports --- docker-compose.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 9308a563..a80b7328 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,7 +8,7 @@ services: POSTGRES_PASSWORD: ${DB_PASSWORD} POSTGRES_DB: ${DB_NAME} ports: - - "${DB_PORT}:${DB_PORT}" + - "${DB_PORT:-5432}:${DB_PORT:-5432}" volumes: - postgres_data:/var/lib/postgresql/data healthcheck: @@ -23,7 +23,7 @@ services: REDIS_HOST: ${REDIS_HOST} REDIS_PORT: ${REDIS_PORT} ports: - - "${REDIS_PORT}:${REDIS_PORT}" + - "${REDIS_PORT:-6379}:${REDIS_PORT:-6379}" volumes: - redis_data:/data @@ -34,8 +34,8 @@ services: MINIO_ROOT_PASSWORD: ${MINIO_SECRET_KEY} command: server /data --console-address :${MINIO_CONSOLE_PORT} ports: - - "${MINIO_PORT}:${MINIO_PORT}" # API port - - "${MINIO_CONSOLE_PORT}:${MINIO_CONSOLE_PORT}" # WebUI port + - "${MINIO_PORT:-9000}:${MINIO_PORT:-9000}" # API port + - "${MINIO_CONSOLE_PORT:-9001}:${MINIO_CONSOLE_PORT:-9001}" # WebUI port volumes: - minio_data:/data @@ -45,7 +45,7 @@ services: #dockerfile: server/Dockerfile image: getmaxun/maxun-backend:v0.0.2 ports: - - "${BACKEND_PORT}:${BACKEND_PORT}" + - "${BACKEND_PORT:-8080}:${BACKEND_PORT:-8080}" env_file: .env environment: BACKEND_URL: ${BACKEND_URL} @@ -74,7 +74,7 @@ services: #dockerfile: Dockerfile image: getmaxun/maxun-frontend:v0.0.1 ports: - - "${FRONTEND_PORT}:${FRONTEND_PORT}" + - "${FRONTEND_PORT:-5173}:${FRONTEND_PORT:-5173}" env_file: .env environment: PUBLIC_URL: ${PUBLIC_URL} From e2bd961d5faff4bf59a66ed1cdc1754a3ce29e6c Mon Sep 17 00:00:00 2001 From: RohitR311 Date: Wed, 4 Dec 2024 20:26:39 +0530 Subject: [PATCH 79/88] feat: add selectors in where pair for scrapeList action --- .../workflow-management/classes/Generator.ts | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index 312f8a2a..cfef4a30 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -160,6 +160,41 @@ export class WorkflowGenerator { }) }; + /** + * New function to handle actionable check for scrapeList + * @param page The current Playwright Page object. + * @param config The scrapeList configuration object. + * @returns {Promise} Array of actionable selectors. + */ + private async getSelectorsForScrapeList(page: Page, config: { + listSelector: string; + fields: any; + limit?: number; + pagination: any; + }): Promise { + const { listSelector } = config; + + // Verify if the selectors are present and actionable on the current page + const actionableSelectors: string[] = []; + if (listSelector) { + const isActionable = await page.isVisible(listSelector).catch(() => false); + if (isActionable) { + actionableSelectors.push(listSelector); + logger.log('debug', `List selector ${listSelector} is actionable.`); + } else { + logger.log('warn', `List selector ${listSelector} is not visible on the page.`); + } + } + + return actionableSelectors; + } + + /** + * New function to handle actionable check for scrapeList + * @param page The current Playwright Page object. + * @param schema The scrapeSchema configuration object. + * @returns {Promise} Array of actionable selectors. + */ private async getSelectorsForSchema(page: Page, schema: Record): Promise { const selectors = Object.values(schema).map((field) => field.selector); @@ -208,6 +243,14 @@ export class WorkflowGenerator { pair.where.selectors = [...(pair.where.selectors || []), ...additionalSelectors]; } } + + if (pair.what[0].action === 'scrapeList') { + const config = pair.what[0]?.args?.[0]; + if (config) { + const actionableSelectors = await this.getSelectorsForScrapeList(page, config); + pair.where.selectors = [...(pair.where.selectors || []), ...actionableSelectors]; + } + } // Validate if the pair is already in the workflow if (pair.where.selectors && pair.where.selectors[0]) { From d41481b0b73be2428b32ef73306d91eb41d78828 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Wed, 4 Dec 2024 20:34:01 +0530 Subject: [PATCH 80/88] fix: pass default minio port --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index a80b7328..dcab91a1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -32,7 +32,7 @@ services: environment: MINIO_ROOT_USER: ${MINIO_ACCESS_KEY} MINIO_ROOT_PASSWORD: ${MINIO_SECRET_KEY} - command: server /data --console-address :${MINIO_CONSOLE_PORT} + command: server /data --console-address :${MINIO_CONSOLE_PORT:-9001} ports: - "${MINIO_PORT:-9000}:${MINIO_PORT:-9000}" # API port - "${MINIO_CONSOLE_PORT:-9001}:${MINIO_CONSOLE_PORT:-9001}" # WebUI port From ca194cd9de50f157faf6a5520a731af8d925b4e8 Mon Sep 17 00:00:00 2001 From: Karishma Shukla Date: Wed, 4 Dec 2024 20:58:31 +0530 Subject: [PATCH 81/88] feat: add v0.0.3 env variables --- ENVEXAMPLE | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/ENVEXAMPLE b/ENVEXAMPLE index 3de361d6..650c7027 100644 --- a/ENVEXAMPLE +++ b/ENVEXAMPLE @@ -9,16 +9,18 @@ DB_PORT=5432 # Port for PostgreSQL (default: 5432) ENCRYPTION_KEY=f4d5e6a7b8c9d0e1f23456789abcdef01234567890abcdef123456789abcdef0 # Key for encrypting sensitive data (passwords and proxies) MINIO_ENDPOINT=minio # MinIO endpoint in Docker MINIO_PORT=9000 # Port for MinIO (default: 9000) +MINIO_CONSOLE_PORT=9001 # Web UI Port for MinIO (default: 9001) MINIO_ACCESS_KEY=minio_access_key # MinIO access key MINIO_SECRET_KEY=minio_secret_key # MinIO secret key REDIS_HOST=redis # Redis host in Docker REDIS_PORT=6379 # Redis port (default: 6379) -# Backend and Frontend URLs +# Backend and Frontend URLs and Ports +BACKEND_PORT=8080 # Port to run backend on. Needed for Docker setup +FRONTEND_PORT=5173 # Port to run frontend on. Needed for Docker setup BACKEND_URL=http://localhost:8080 # URL on which the backend runs. You can change it based on your needs. -VITE_BACKEND_URL=http://localhost:8080 # URL used by frontend to connect to backend. It should always have the same value as BACKEND_URL - PUBLIC_URL=http://localhost:5173 # URL on which the frontend runs. You can change it based on your needs. +VITE_BACKEND_URL=http://localhost:8080 # URL used by frontend to connect to backend. It should always have the same value as BACKEND_URL VITE_PUBLIC_URL=http://localhost:5173 # URL used by backend to connect to frontend. It should always have the same value as PUBLIC_URL # Optional Google OAuth settings for Google Sheet Integration From e62239f5245dc53a7e11b2edd87b92984d105640 Mon Sep 17 00:00:00 2001 From: Karishma Shukla Date: Wed, 4 Dec 2024 21:01:37 +0530 Subject: [PATCH 82/88] docs: BACKEND_PORT and FRONTEND_PORT --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index b0f44e08..f7b5977b 100644 --- a/README.md +++ b/README.md @@ -76,6 +76,8 @@ You can access the frontend at http://localhost:5173/ and backend at http://loca | Variable | Mandatory | Description | If Not Set | |-----------------------|-----------|----------------------------------------------------------------------------------------------|--------------------------------------------------------------| +| `BACKEND_PORT` | Yes | Port to run backend on. Needed for Docker setup | Default value: 8080 | +| `FRONTEND_PORT` | Yes | Port to run frontend on. Needed for Docker setup | Default value: 5173 | | `BACKEND_URL` | Yes | URL to run backend on. | Default value: http://localhost:8080 | | `VITE_BACKEND_URL` | Yes | URL used by frontend to connect to backend | Default value: http://localhost:8080 | | `PUBLIC_URL` | Yes | URL to run frontend on. | Default value: http://localhost:5173 | From 6e3266ea045e07784bfece857f54b6827af8025d Mon Sep 17 00:00:00 2001 From: Karishma Shukla Date: Wed, 4 Dec 2024 21:03:37 +0530 Subject: [PATCH 83/88] docs: add MINIO_CONSOLE_PORT --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index f7b5977b..fb2c4845 100644 --- a/README.md +++ b/README.md @@ -91,6 +91,7 @@ You can access the frontend at http://localhost:5173/ and backend at http://loca | `ENCRYPTION_KEY` | Yes | Key used for encrypting sensitive data (proxies, passwords). | Encryption functionality will not work. | | `MINIO_ENDPOINT` | Yes | Endpoint URL for MinIO, to store Robot Run Screenshots. | Connection to MinIO storage will fail. | | `MINIO_PORT` | Yes | Port number for MinIO service. | Connection to MinIO storage will fail. | +| `MINIO_CONSOLE_PORT` | No | Port number for MinIO WebUI service. Needed for Docker setup. | Cannot access MinIO Web UI. | | `MINIO_ACCESS_KEY` | Yes | Access key for authenticating with MinIO. | MinIO authentication will fail. | | `GOOGLE_CLIENT_ID` | No | Client ID for Google OAuth, used for Google Sheet integration authentication. | Google login will not work. | | `GOOGLE_CLIENT_SECRET`| No | Client Secret for Google OAuth. | Google login will not work. | From 81126366dd454ad9a9382b6e821ac8582819cfac Mon Sep 17 00:00:00 2001 From: amhsirak Date: Wed, 4 Dec 2024 21:54:39 +0530 Subject: [PATCH 84/88] chore: upgrade frontend image to v2 & backend image to v3 --- docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index dcab91a1..92b69c14 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -43,7 +43,7 @@ services: #build: #context: . #dockerfile: server/Dockerfile - image: getmaxun/maxun-backend:v0.0.2 + image: getmaxun/maxun-backend:v0.0.3 ports: - "${BACKEND_PORT:-8080}:${BACKEND_PORT:-8080}" env_file: .env @@ -72,7 +72,7 @@ services: #build: #context: . #dockerfile: Dockerfile - image: getmaxun/maxun-frontend:v0.0.1 + image: getmaxun/maxun-frontend:v0.0.2 ports: - "${FRONTEND_PORT:-5173}:${FRONTEND_PORT:-5173}" env_file: .env From 31388bd196445895882b26c1500ae1d59c3a5303 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Wed, 4 Dec 2024 23:48:39 +0530 Subject: [PATCH 85/88] feat: upgrade backend image to v0.0.4 --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 92b69c14..ca64f644 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -43,7 +43,7 @@ services: #build: #context: . #dockerfile: server/Dockerfile - image: getmaxun/maxun-backend:v0.0.3 + image: getmaxun/maxun-backend:v0.0.4 ports: - "${BACKEND_PORT:-8080}:${BACKEND_PORT:-8080}" env_file: .env From fcad9570c98bcf31852c5976af39db228cfcf331 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Thu, 5 Dec 2024 21:06:21 +0530 Subject: [PATCH 86/88] hotfix: remove scrapeList action --- .../workflow-management/classes/Generator.ts | 45 +------------------ 1 file changed, 1 insertion(+), 44 deletions(-) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index cfef4a30..c9dc3385 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -160,41 +160,6 @@ export class WorkflowGenerator { }) }; - /** - * New function to handle actionable check for scrapeList - * @param page The current Playwright Page object. - * @param config The scrapeList configuration object. - * @returns {Promise} Array of actionable selectors. - */ - private async getSelectorsForScrapeList(page: Page, config: { - listSelector: string; - fields: any; - limit?: number; - pagination: any; - }): Promise { - const { listSelector } = config; - - // Verify if the selectors are present and actionable on the current page - const actionableSelectors: string[] = []; - if (listSelector) { - const isActionable = await page.isVisible(listSelector).catch(() => false); - if (isActionable) { - actionableSelectors.push(listSelector); - logger.log('debug', `List selector ${listSelector} is actionable.`); - } else { - logger.log('warn', `List selector ${listSelector} is not visible on the page.`); - } - } - - return actionableSelectors; - } - - /** - * New function to handle actionable check for scrapeList - * @param page The current Playwright Page object. - * @param schema The scrapeSchema configuration object. - * @returns {Promise} Array of actionable selectors. - */ private async getSelectorsForSchema(page: Page, schema: Record): Promise { const selectors = Object.values(schema).map((field) => field.selector); @@ -243,14 +208,6 @@ export class WorkflowGenerator { pair.where.selectors = [...(pair.where.selectors || []), ...additionalSelectors]; } } - - if (pair.what[0].action === 'scrapeList') { - const config = pair.what[0]?.args?.[0]; - if (config) { - const actionableSelectors = await this.getSelectorsForScrapeList(page, config); - pair.where.selectors = [...(pair.where.selectors || []), ...actionableSelectors]; - } - } // Validate if the pair is already in the workflow if (pair.where.selectors && pair.where.selectors[0]) { @@ -923,4 +880,4 @@ export class WorkflowGenerator { public clearLastIndex = () => { this.generatedData.lastIndex = null; } -} +} \ No newline at end of file From cc2b1a37274013d5f3299dcc128c5df6205efbcb Mon Sep 17 00:00:00 2001 From: amhsirak Date: Thu, 5 Dec 2024 21:06:58 +0530 Subject: [PATCH 87/88] fix: upgrade v0.0.3 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index b6b73537..38104584 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "maxun", - "version": "0.0.2", + "version": "0.0.3", "author": "Maxun", "license": "AGPL-3.0-or-later", "dependencies": { From b3a973fe59b66b419e645c8d7b13bd5bf3bae460 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Thu, 5 Dec 2024 21:09:24 +0530 Subject: [PATCH 88/88] hotfix: upgrade backend image to v0.0.5 --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index ca64f644..46cc72c4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -43,7 +43,7 @@ services: #build: #context: . #dockerfile: server/Dockerfile - image: getmaxun/maxun-backend:v0.0.4 + image: getmaxun/maxun-backend:v0.0.5 ports: - "${BACKEND_PORT:-8080}:${BACKEND_PORT:-8080}" env_file: .env