From b99e1ecb08d1e8b3643c4f9785da9c5120759026 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Fri, 7 Jun 2024 23:19:43 +0530 Subject: [PATCH 01/72] feat: metadata interface --- .../workflow-management/classes/Generator.ts | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 server/src/workflow-management/classes/Generator.ts diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts new file mode 100644 index 00000000..26174b4a --- /dev/null +++ b/server/src/workflow-management/classes/Generator.ts @@ -0,0 +1,29 @@ +import { Action, ActionType, Coordinates, TagName } from "../../types"; +import { WhereWhatPair, WorkflowFile } from '@wbr-project/wbr-interpret'; +import logger from "../../logger"; +import { Socket } from "socket.io"; +import { Page } from "playwright"; +import { + getElementInformation, + getRect, + getSelectors, + isRuleOvershadowing, + selectorAlreadyInWorkflow +} from "../selector"; +import { CustomActions } from "../../../../src/shared/types"; +import { workflow } from "../../routes"; +import { saveFile } from "../storage"; +import fs from "fs"; +import { getBestSelectorForAction } from "../utils"; +import { browserPool } from "../../server"; + + +interface MetaData { + name: string; + create_date: string; + pairs: number; + update_date: string; + params: string[], +} + + From 571683a2851c9e54aa284ed0096bbe3906246374 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Fri, 7 Jun 2024 23:24:18 +0530 Subject: [PATCH 02/72] feat: data persistence interface --- server/src/workflow-management/classes/Generator.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index 26174b4a..f6a8f8bc 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -17,6 +17,11 @@ import fs from "fs"; import { getBestSelectorForAction } from "../utils"; import { browserPool } from "../../server"; +interface PersistedGeneratedData { + lastUsedSelector: string; + lastIndex: number|null; + lastAction: string; +} interface MetaData { name: string; @@ -26,4 +31,3 @@ interface MetaData { params: string[], } - From 27b4cf9474cd1941bb5b6797e62b77790a2c34aa Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Fri, 7 Jun 2024 23:26:08 +0530 Subject: [PATCH 03/72] feat: init WorkflowGenerator class --- server/src/workflow-management/classes/Generator.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index f6a8f8bc..2830027a 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -31,3 +31,14 @@ interface MetaData { params: string[], } +export class WorkflowGenerator { + + private socket : Socket; + + public constructor(socket: Socket) { + this.socket = socket; + this.registerEventHandlers(socket); + } + + +} From 71c5f3922ba8ee12cb146c3ac6b0ebdb08e2fd34 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Fri, 7 Jun 2024 23:26:57 +0530 Subject: [PATCH 04/72] chore: generator description --- server/src/workflow-management/classes/Generator.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index 2830027a..cd48f96f 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -31,6 +31,13 @@ interface MetaData { params: string[], } +/** + * Workflow generator is used to transform the user's interactions into an automatically + * generated correct workflows, using the ability of internal state persistence and + * heuristic generative algorithms. + * This class also takes care of the selector generation. + * @category WorkflowManagement + */ export class WorkflowGenerator { private socket : Socket; From 968c50445623e080f490d66da91e9e74e358c05b Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Fri, 7 Jun 2024 23:27:11 +0530 Subject: [PATCH 05/72] chore: comments --- server/src/workflow-management/classes/Generator.ts | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index cd48f96f..617da3e2 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -39,9 +39,19 @@ interface MetaData { * @category WorkflowManagement */ export class WorkflowGenerator { - + + /** + * The socket used to communicate with the client. + * @private + */ private socket : Socket; + /** + * The public constructor of the WorkflowGenerator. + * Takes socket for communication as a parameter and registers some important events on it. + * @param socket The socket used to communicate with the client. + * @constructor + */ public constructor(socket: Socket) { this.socket = socket; this.registerEventHandlers(socket); From e3582da825d2345b3b24dc8ed7377ecace0aff8c Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Fri, 7 Jun 2024 23:28:05 +0530 Subject: [PATCH 06/72] feat: private vars for workflow --- .../src/workflow-management/classes/Generator.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index 617da3e2..07c8dac5 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -58,4 +58,18 @@ export class WorkflowGenerator { } + private workflowRecord: WorkflowFile = { + workflow: [], + }; + + + private recordingMeta: MetaData = { + name: '', + create_date: '', + pairs: 0, + update_date: '', + params: [], + } + + } From 2230eb323d6f5b6e197752911243f6c82bb9368f Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Fri, 7 Jun 2024 23:28:15 +0530 Subject: [PATCH 07/72] chore: comments --- server/src/workflow-management/classes/Generator.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index 07c8dac5..01da64fc 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -57,12 +57,18 @@ export class WorkflowGenerator { this.registerEventHandlers(socket); } - + /** + * The current workflow being recorded. + * @private + */ private workflowRecord: WorkflowFile = { workflow: [], }; - + /** + * Metadata of the currently recorded workflow. + * @private + */ private recordingMeta: MetaData = { name: '', create_date: '', From 35e1250d3b7936a67a6b269abf2a924d0ab96c27 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Fri, 7 Jun 2024 23:44:04 +0530 Subject: [PATCH 08/72] chore: data persistence var --- server/src/workflow-management/classes/Generator.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index 01da64fc..8e9aaa27 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -77,5 +77,16 @@ export class WorkflowGenerator { params: [], } + /** + * The persistent data from the whole workflow generation process. + * Used for correct generation of other user inputs. + * @private + */ + private generatedData: PersistedGeneratedData = { + lastUsedSelector: '', + lastIndex: null, + lastAction: '', + } + } From 65f901cc269745b7e258ec28750116f18cecace5 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Fri, 7 Jun 2024 23:45:50 +0530 Subject: [PATCH 09/72] feat: save worflow --- server/src/workflow-management/classes/Generator.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index 8e9aaa27..30dd00bd 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -88,5 +88,13 @@ export class WorkflowGenerator { lastAction: '', } + private registerEventHandlers = (socket: Socket) => { + socket.on('save', async (fileName: string) => { + logger.log('debug', `Saving workflow ${fileName}`); + await this.saveNewWorkflow(fileName) + }); + + }; + } From e4b61768e6c86a7dc43f663aba4d65a24665e302 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Fri, 7 Jun 2024 23:48:08 +0530 Subject: [PATCH 10/72] feat: new recording --- server/src/workflow-management/classes/Generator.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index 30dd00bd..e5ad7a40 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -93,6 +93,9 @@ export class WorkflowGenerator { logger.log('debug', `Saving workflow ${fileName}`); await this.saveNewWorkflow(fileName) }); + socket.on('new-recording', () => this.workflowRecord = { + workflow: [], + } ); }; From 4b06903c88eea740afb8962927ba3085ed1a89f1 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Fri, 7 Jun 2024 23:48:30 +0530 Subject: [PATCH 11/72] feat: active recording --- server/src/workflow-management/classes/Generator.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index e5ad7a40..6d11b9ec 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -96,6 +96,7 @@ export class WorkflowGenerator { socket.on('new-recording', () => this.workflowRecord = { workflow: [], } ); + socket.on('activeIndex', (data) => this.generatedData.lastIndex = parseInt(data)); }; From f62191a56edef6f321b7f300755a68e94640e832 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Fri, 7 Jun 2024 23:49:38 +0530 Subject: [PATCH 12/72] feat: get current active browser id from pool --- server/src/workflow-management/classes/Generator.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index 6d11b9ec..ffed090c 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -97,6 +97,9 @@ export class WorkflowGenerator { workflow: [], } ); socket.on('activeIndex', (data) => this.generatedData.lastIndex = parseInt(data)); + socket.on('decision', async ({pair, actionType, decision}) => { + const id = browserPool.getActiveBrowserId(); + }) }; From 971b8c0abc1293d1ef8b36658e26cf10281ee29c Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Fri, 7 Jun 2024 23:50:11 +0530 Subject: [PATCH 13/72] feat: custom action decision --- server/src/workflow-management/classes/Generator.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index ffed090c..2bef39ea 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -99,6 +99,19 @@ export class WorkflowGenerator { socket.on('activeIndex', (data) => this.generatedData.lastIndex = parseInt(data)); socket.on('decision', async ({pair, actionType, decision}) => { const id = browserPool.getActiveBrowserId(); + if (id) { + const activeBrowser = browserPool.getRemoteBrowser(id); + const currentPage = activeBrowser?.getCurrentPage(); + if (decision) { + switch (actionType) { + case 'customAction': + pair.where.selectors = [this.generatedData.lastUsedSelector]; + break; + + } + } + + } }) }; From 65d63c3888beac1b01ff2ea59da80a0e1b077115 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Fri, 7 Jun 2024 23:50:26 +0530 Subject: [PATCH 14/72] feat: default switch case --- server/src/workflow-management/classes/Generator.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index 2bef39ea..3a866780 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -107,7 +107,7 @@ export class WorkflowGenerator { case 'customAction': pair.where.selectors = [this.generatedData.lastUsedSelector]; break; - + default: break; } } From 6df8c77886c7d0cc1d95fb0db9df1ddc2dc13b2c Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Fri, 7 Jun 2024 23:50:50 +0530 Subject: [PATCH 15/72] feat: add pair to workflow if current page --- server/src/workflow-management/classes/Generator.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index 3a866780..f94fd5bd 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -110,7 +110,9 @@ export class WorkflowGenerator { default: break; } } - + if (currentPage) { + await this.addPairToWorkflowAndNotifyClient(pair, currentPage); + } } }) From 3d2cff905cd4a9ef14f3a1dded4ad25385e29fce Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Fri, 7 Jun 2024 23:51:02 +0530 Subject: [PATCH 16/72] feat: update pair --- server/src/workflow-management/classes/Generator.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index f94fd5bd..56cfd6ea 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -115,7 +115,9 @@ export class WorkflowGenerator { } } }) - + socket.on('updatePair', (data) => { + this.updatePairInWorkflow(data.index, data.pair); + }) }; From 06bd0bfc39f61acc8350a7e3b6bf25419b5c8db9 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Fri, 7 Jun 2024 23:53:17 +0530 Subject: [PATCH 17/72] chore: register handler description --- server/src/workflow-management/classes/Generator.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index 56cfd6ea..889c5d04 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -88,6 +88,11 @@ export class WorkflowGenerator { lastAction: '', } + /** + * Registers the event handlers for all generator-related events on the socket. + * @param socket The socket used to communicate with the client. + * @private + */ private registerEventHandlers = (socket: Socket) => { socket.on('save', async (fileName: string) => { logger.log('debug', `Saving workflow ${fileName}`); From 7b977804ef3fe33dcdc9aaec5eb46390804d4511 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Fri, 7 Jun 2024 23:56:12 +0530 Subject: [PATCH 18/72] feat: validate if pair w same where conditions is already present in workflow --- server/src/workflow-management/classes/Generator.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index 889c5d04..51b13942 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -126,4 +126,14 @@ export class WorkflowGenerator { }; + private addPairToWorkflowAndNotifyClient = async(pair: WhereWhatPair, page: Page) => { + let matched = false; + // validate if a pair with the same where conditions is already present in the workflow + if (pair.where.selectors && pair.where.selectors[0]) { + + } + + }; + + } From e57b88cdfdeb6959935f61101a53893cb15c5b42 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Fri, 7 Jun 2024 23:57:51 +0530 Subject: [PATCH 19/72] feat: if match found, add new action into matched rule --- .../src/workflow-management/classes/Generator.ts | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index 51b13942..27429755 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -130,7 +130,20 @@ export class WorkflowGenerator { let matched = false; // validate if a pair with the same where conditions is already present in the workflow if (pair.where.selectors && pair.where.selectors[0]) { - + const match = selectorAlreadyInWorkflow(pair.where.selectors[0], this.workflowRecord.workflow); + if (match) { + // if a match of where conditions is found, the new action is added into the matched rule + const matchedIndex = this.workflowRecord.workflow.indexOf(match); + if (pair.what[0].action !== 'waitForLoadState' && pair.what[0].action !== 'press') { + pair.what.push({ + action: 'waitForLoadState', + args: ['networkidle'], + }) + } + this.workflowRecord.workflow[matchedIndex].what = this.workflowRecord.workflow[matchedIndex].what.concat(pair.what); + logger.log('info', `Pushed ${JSON.stringify(this.workflowRecord.workflow[matchedIndex])} to workflow pair`); + matched = true; + } } }; From 8955cfdee73474013ee76a5daa3f9e2bccefe183 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Fri, 7 Jun 2024 23:58:52 +0530 Subject: [PATCH 20/72] feat: if match !found, validate where conditions --- .../workflow-management/classes/Generator.ts | 31 ++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index 27429755..b8648e69 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -145,7 +145,36 @@ export class WorkflowGenerator { matched = true; } } - + // is the where conditions of the pair are not already in the workflow, we need to validate the where conditions + // for possible overshadowing of different rules and handle cases according to the recording logic + if (!matched) { + const handled = await this.handleOverShadowing(pair, page, this.generatedData.lastIndex || 0); + if (!handled) { + //adding waitForLoadState with networkidle, for better success rate of automatically recorded workflows + if (pair.what[0].action !== 'waitForLoadState' && pair.what[0].action !== 'press') { + pair.what.push({ + action: 'waitForLoadState', + args: ['networkidle'], + }) + } + if (this.generatedData.lastIndex === 0) { + this.generatedData.lastIndex = null; + // we want to have the most specific selectors at the beginning of the workflow + this.workflowRecord.workflow.unshift(pair); + } else { + this.workflowRecord.workflow.splice(this.generatedData.lastIndex || 0, 0, pair); + if (this.generatedData.lastIndex) { + this.generatedData.lastIndex = this.generatedData.lastIndex - 1; + } + } + logger.log('info', + `${JSON.stringify(pair)}: Added to workflow file on index: ${this.generatedData.lastIndex || 0}`); + } else { + logger.log('debug', + ` ${JSON.stringify(this.workflowRecord.workflow[this.generatedData.lastIndex || 0])} added action to workflow pair`); + } + } + }; From dceec0535269b74fdeeaf2ce2d4c568412158886 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Fri, 7 Jun 2024 23:59:10 +0530 Subject: [PATCH 21/72] feat:socket emit workflow --- server/src/workflow-management/classes/Generator.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index b8648e69..a290b837 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -174,7 +174,8 @@ export class WorkflowGenerator { ` ${JSON.stringify(this.workflowRecord.workflow[this.generatedData.lastIndex || 0])} added action to workflow pair`); } } - + this.socket.emit('workflow', this.workflowRecord); + logger.log('info',`Workflow emitted`); }; From bf8b8ead3d875d56bdfd5a46ef74d00d8289ce83 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Fri, 7 Jun 2024 23:59:53 +0530 Subject: [PATCH 22/72] chore: brief description of addPairToWorkflowAndNotifyClient --- .../workflow-management/classes/Generator.ts | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index a290b837..c51b5ef1 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -125,7 +125,29 @@ export class WorkflowGenerator { }) }; - + /** + * Adds a newly generated pair to the workflow and notifies the client about it by + * sending the updated workflow through socket. + * + * Checks some conditions for the correct addition of the pair. + * 1. The pair's action selector is already in the workflow as a different pair's where selector + * If so, the what part of the pair is added to the pair with the same where selector. + * 2. The pair's where selector is located on the page at the same time as another pair's where selector, + * having the same url. This state is called over-shadowing an already existing pair. + * If so, the pair is merged with the previous over-shadowed pair - what part is attached and + * new selector added to the where selectors. In case the over-shadowed pair is further down the + * workflow array, the new pair is added to the beginning of the workflow array. + * + * This function also makes sure to add a waitForLoadState and a generated flag + * action after every new action or pair added. The [waitForLoadState](https://playwright.dev/docs/api/class-frame#frame-wait-for-load-state) + * action waits for the networkidle event to be fired, + * and the generated flag action is used for making pausing the interpretation possible. + * + * @param pair The pair to add to the workflow. + * @param page The page to use for the state checking. + * @private + * @returns {Promise} + */ private addPairToWorkflowAndNotifyClient = async(pair: WhereWhatPair, page: Page) => { let matched = false; // validate if a pair with the same where conditions is already present in the workflow From e1e62038f911082fb71b72452c6be830a56d2aa4 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 00:04:54 +0530 Subject: [PATCH 23/72] feat: generate on click pair --- server/src/workflow-management/classes/Generator.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index c51b5ef1..ca479f09 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -201,4 +201,10 @@ export class WorkflowGenerator { }; + public onClick = async (coordinates: Coordinates, page: Page) => { + let where: WhereWhatPair["where"] = { url: this.getBestUrl(page.url()) }; + + }; + + } From ba655f0914b2eb7506764f037c817115038fdb91 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 00:05:12 +0530 Subject: [PATCH 24/72] feat: generate selector for on click pair --- server/src/workflow-management/classes/Generator.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index ca479f09..fcda6b2e 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -203,6 +203,7 @@ export class WorkflowGenerator { public onClick = async (coordinates: Coordinates, page: Page) => { let where: WhereWhatPair["where"] = { url: this.getBestUrl(page.url()) }; + const selector = await this.generateSelector(page, coordinates, ActionType.Click); }; From ea7491e6dae275d661165c819879417049ac6f4c Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 00:05:37 +0530 Subject: [PATCH 25/72] feat: get selectors & create pair --- .../workflow-management/classes/Generator.ts | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index fcda6b2e..5ff990fe 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -204,7 +204,23 @@ export class WorkflowGenerator { public onClick = async (coordinates: Coordinates, page: Page) => { let where: WhereWhatPair["where"] = { url: this.getBestUrl(page.url()) }; const selector = await this.generateSelector(page, coordinates, ActionType.Click); - + logger.log('debug', `Element's selector: ${selector}`); + //const element = await getElementMouseIsOver(page, coordinates); + //logger.log('debug', `Element: ${JSON.stringify(element, null, 2)}`); + if (selector) { + where.selectors = [selector]; + } + const pair: WhereWhatPair = { + where, + what: [{ + action: 'click', + args: [selector], + }], + } + if (selector) { + this.generatedData.lastUsedSelector = selector; + this.generatedData.lastAction = 'click'; + } }; From ec7a0936febcba9d3bac66e84f6343ac43eb2849 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 00:06:00 +0530 Subject: [PATCH 26/72] feat: add pair to worflow & notify client --- server/src/workflow-management/classes/Generator.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index 5ff990fe..e33fe0d6 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -221,6 +221,7 @@ export class WorkflowGenerator { this.generatedData.lastUsedSelector = selector; this.generatedData.lastAction = 'click'; } + await this.addPairToWorkflowAndNotifyClient(pair, page); }; From 11afd5c230d92f4cf74e5d5b662b6a385e7e4b8e Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 00:08:51 +0530 Subject: [PATCH 27/72] chore: onClick description --- server/src/workflow-management/classes/Generator.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index e33fe0d6..bf20825d 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -200,7 +200,12 @@ export class WorkflowGenerator { logger.log('info',`Workflow emitted`); }; - + /** + * Generates a pair for the click event. + * @param coordinates The coordinates of the click event. + * @param page The page to use for obtaining the needed data. + * @returns {Promise} + */ public onClick = async (coordinates: Coordinates, page: Page) => { let where: WhereWhatPair["where"] = { url: this.getBestUrl(page.url()) }; const selector = await this.generateSelector(page, coordinates, ActionType.Click); From d03d7752c2d5578e26499ec6ca6087244d64c03e Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 00:09:34 +0530 Subject: [PATCH 28/72] chore: generate pair for url change event --- .../workflow-management/classes/Generator.ts | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index bf20825d..19749134 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -229,5 +229,20 @@ export class WorkflowGenerator { await this.addPairToWorkflowAndNotifyClient(pair, page); }; - + + public onChangeUrl = async(newUrl: string, page: Page) => { + this.generatedData.lastUsedSelector = ''; + const pair: WhereWhatPair = { + where: { url: this.getBestUrl(page.url()) }, + what: [ + { + action: 'goto', + args: [newUrl], + } + ], + } + await this.addPairToWorkflowAndNotifyClient(pair, page); + }; + + } From f97b5f282f2f39114f744d78c1799b4c237f72e9 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 00:09:49 +0530 Subject: [PATCH 29/72] chore: onChangeUrl description --- server/src/workflow-management/classes/Generator.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index 19749134..41e25028 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -229,7 +229,12 @@ export class WorkflowGenerator { await this.addPairToWorkflowAndNotifyClient(pair, page); }; - + /** + * Generates a pair for the change url event. + * @param newUrl The new url to be changed to. + * @param page The page to use for obtaining the needed data. + * @returns {Promise} + */ public onChangeUrl = async(newUrl: string, page: Page) => { this.generatedData.lastUsedSelector = ''; const pair: WhereWhatPair = { From 3e240cacd6e904701bb145d7a4247336a8af627f Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 00:11:15 +0530 Subject: [PATCH 30/72] feat: generate pair for on key press event --- .../workflow-management/classes/Generator.ts | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index 41e25028..762cb865 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -249,5 +249,26 @@ export class WorkflowGenerator { await this.addPairToWorkflowAndNotifyClient(pair, page); }; - + + public onKeyboardInput = async (key: string, coordinates: Coordinates, page: Page) => { + let where: WhereWhatPair["where"] = { url: this.getBestUrl(page.url()) }; + const selector = await this.generateSelector(page, coordinates, ActionType.Keydown); + if (selector) { + where.selectors = [selector]; + } + const pair: WhereWhatPair = { + where, + what: [{ + action: 'press', + args: [selector, key], + }], + } + if (selector) { + this.generatedData.lastUsedSelector = selector; + this.generatedData.lastAction = 'press'; + } + await this.addPairToWorkflowAndNotifyClient(pair, page); + }; + + } From 03e5a1ed97f634503b00c743812feeae20d7bbf9 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 00:11:36 +0530 Subject: [PATCH 31/72] chore: onKyeboardInput description --- server/src/workflow-management/classes/Generator.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index 762cb865..f69f3f99 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -249,7 +249,13 @@ export class WorkflowGenerator { await this.addPairToWorkflowAndNotifyClient(pair, page); }; - + /** + * Generates a pair for the keypress event. + * @param key The key to be pressed. + * @param coordinates The coordinates of the keypress event. + * @param page The page to use for obtaining the needed data. + * @returns {Promise} + */ public onKeyboardInput = async (key: string, coordinates: Coordinates, page: Page) => { let where: WhereWhatPair["where"] = { url: this.getBestUrl(page.url()) }; const selector = await this.generateSelector(page, coordinates, ActionType.Keydown); From 13c9aeb89db5a609f10bb37fd537bb3d1d3d7ee8 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 00:12:41 +0530 Subject: [PATCH 32/72] feat: generate pair for custom action event --- .../workflow-management/classes/Generator.ts | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index f69f3f99..4a40eaef 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -276,5 +276,27 @@ export class WorkflowGenerator { await this.addPairToWorkflowAndNotifyClient(pair, page); }; - + + public customAction = async (action: CustomActions, settings: any, page: Page) => { + const pair: WhereWhatPair = { + where: { url: this.getBestUrl(page.url())}, + what: [{ + action, + args: settings ? Array.isArray(settings) ? settings : [settings] : [], + }], + } + + if (this.generatedData.lastUsedSelector) { + this.socket.emit('decision', { + pair, actionType: 'customAction', + lastData: { + selector: this.generatedData.lastUsedSelector, + action: this.generatedData.lastAction, + } }); + } else { + await this.addPairToWorkflowAndNotifyClient(pair, page); + } + }; + + } From 5f4cbe1e935d59d01a046c1803753eeb38b12fe5 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 00:12:59 +0530 Subject: [PATCH 33/72] chore: customAction description --- server/src/workflow-management/classes/Generator.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index 4a40eaef..36f354cc 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -276,7 +276,12 @@ export class WorkflowGenerator { await this.addPairToWorkflowAndNotifyClient(pair, page); }; - + /** + * Generates a pair for the custom action event. + * @param action The type of the custom action. + * @param settings The settings of the custom action. + * @param page The page to use for obtaining the needed data. + */ public customAction = async (action: CustomActions, settings: any, page: Page) => { const pair: WhereWhatPair = { where: { url: this.getBestUrl(page.url())}, From 4f5cb446112f307d4a58791149b0e59f07a4087b Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 00:13:54 +0530 Subject: [PATCH 34/72] feat: get workflow --- server/src/workflow-management/classes/Generator.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index 36f354cc..68e81e54 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -303,5 +303,10 @@ export class WorkflowGenerator { } }; + + public getWorkflowFile = () => { + return this.workflowRecord; + }; + } From 8d287dd5d4243063c505c014138aeb6c84d3f7a1 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 00:14:16 +0530 Subject: [PATCH 35/72] chore: comments --- server/src/workflow-management/classes/Generator.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index 68e81e54..19302c2d 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -303,10 +303,13 @@ export class WorkflowGenerator { } }; - + /** + * Returns the currently generated workflow. + * @returns {WorkflowFile} + */ public getWorkflowFile = () => { return this.workflowRecord; }; - + } From 7d9ecdd3abbf3b8d449dbce9f97a7481ab54f869 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 00:14:54 +0530 Subject: [PATCH 36/72] feat: remove pair from worflow --- server/src/workflow-management/classes/Generator.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index 19302c2d..1ec23d04 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -312,4 +312,14 @@ export class WorkflowGenerator { }; + public removePairFromWorkflow = (index: number) => { + if (index <= this.workflowRecord.workflow.length && index >= 0) { + this.workflowRecord.workflow.splice(this.workflowRecord.workflow.length - (index + 1), 1); + logger.log('debug', `pair ${index}: Removed from workflow file.`); + } else { + logger.log('error', `Delete pair ${index}: Index out of range.`); + } + }; + + } From 54dc526abecbc132cf018e9247b484fad6aa9a9d Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 00:15:11 +0530 Subject: [PATCH 37/72] chore: comments --- server/src/workflow-management/classes/Generator.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index 1ec23d04..ffb226eb 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -311,7 +311,11 @@ export class WorkflowGenerator { return this.workflowRecord; }; - + /** + * Removes a pair from the currently generated workflow. + * @param index The index of the pair to be removed. + * @returns void + */ public removePairFromWorkflow = (index: number) => { if (index <= this.workflowRecord.workflow.length && index >= 0) { this.workflowRecord.workflow.splice(this.workflowRecord.workflow.length - (index + 1), 1); From 665c5a2cf1184987a7a1ecd41194351be5f51f3d Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 00:15:52 +0530 Subject: [PATCH 38/72] feat: add pair to worflow --- server/src/workflow-management/classes/Generator.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index ffb226eb..21a8ea76 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -325,5 +325,18 @@ export class WorkflowGenerator { } }; + + public addPairToWorkflow = (index: number, pair: WhereWhatPair) => { + if (index === this.workflowRecord.workflow.length) { + this.workflowRecord.workflow.unshift(pair); + logger.log('debug', `pair ${index}: Added to workflow file.`); + } else if (index < this.workflowRecord.workflow.length && index >= 0) { + this.workflowRecord.workflow.splice( + this.workflowRecord.workflow.length - index , 0, pair); + } else { + logger.log('error', `Add pair ${index}: Index out of range.`); + } + }; + } From 5ab1f0ce278cca8bd3f69d4aeab4ef7b16d80111 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 00:16:11 +0530 Subject: [PATCH 39/72] chore: describe addPairToWorkflow --- server/src/workflow-management/classes/Generator.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index 21a8ea76..2fc603d7 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -325,7 +325,12 @@ export class WorkflowGenerator { } }; - + /** + * Adds a new pair to the currently generated workflow. + * @param index The index on which the pair should be added. + * @param pair The pair to be added. + * @returns void + */ public addPairToWorkflow = (index: number, pair: WhereWhatPair) => { if (index === this.workflowRecord.workflow.length) { this.workflowRecord.workflow.unshift(pair); From 86ffa782a17226c5affbde22eb9e59b82d0e4c34 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 00:16:51 +0530 Subject: [PATCH 40/72] feat: update pairin worflow --- server/src/workflow-management/classes/Generator.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index 2fc603d7..3d941815 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -343,5 +343,14 @@ export class WorkflowGenerator { } }; + + public updatePairInWorkflow = (index: number, pair: WhereWhatPair) => { + if (index <= this.workflowRecord.workflow.length && index >= 0) { + this.workflowRecord.workflow[this.workflowRecord.workflow.length - (index + 1)] = pair; + } else { + logger.log('error', `Update pair ${index}: Index out of range.`); + } + }; + } From 4e175de94a90a0cff8e70269d0cca8f9aea85d4e Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 00:17:10 +0530 Subject: [PATCH 41/72] chore: describe updatePairInWokflow --- server/src/workflow-management/classes/Generator.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index 3d941815..e2faf92c 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -343,7 +343,12 @@ export class WorkflowGenerator { } }; - + /** + * Updates a pair in the currently generated workflow. + * @param index The index of the pair to be updated. + * @param pair The pair to be used as a replacement. + * @returns void + */ public updatePairInWorkflow = (index: number, pair: WhereWhatPair) => { if (index <= this.workflowRecord.workflow.length && index >= 0) { this.workflowRecord.workflow[this.workflowRecord.workflow.length - (index + 1)] = pair; From 8d6d939f75ad0407e58d7dd2697d1062797c39d3 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 00:18:06 +0530 Subject: [PATCH 42/72] feat: update socket --- server/src/workflow-management/classes/Generator.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index e2faf92c..9368b2f8 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -357,5 +357,15 @@ export class WorkflowGenerator { } }; + /** + * Updates the socket used for communication with the client. + * @param socket The socket to be used for communication. + * @returns void + */ + public updateSocket = (socket: Socket) : void => { + this.socket = socket; + this.registerEventHandlers(socket); + }; + } From 5866a6ae77087bbd49e78c9129c056ae248a0e25 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 00:20:27 +0530 Subject: [PATCH 43/72] feat: return currently generated workflow w/o all generated flag actions --- server/src/workflow-management/classes/Generator.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index 9368b2f8..b8a3b583 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -368,4 +368,17 @@ export class WorkflowGenerator { }; + private removeAllGeneratedFlags = (workflow: WorkflowFile): WorkflowFile => { + for (let i = 0; i < workflow.workflow.length; i++) { + if ( + workflow.workflow[i].what[0] && + workflow.workflow[i].what[0].action === 'flag' && + workflow.workflow[i].what[0].args?.includes('generated')) { + workflow.workflow[i].what.splice(0, 1); + } + } + return workflow; + }; + + } From ed84b8b90c6332e5506e41a50e410f456264a39f Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 00:20:49 +0530 Subject: [PATCH 44/72] chore:: describe removeALlGeneratedFlags --- server/src/workflow-management/classes/Generator.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index b8a3b583..940502bc 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -367,7 +367,12 @@ export class WorkflowGenerator { this.registerEventHandlers(socket); }; - + /** + * Returns the currently generated workflow without all the generated flag actions. + * @param workflow The workflow for removing the generated flag actions from. + * @private + * @returns {WorkflowFile} + */ private removeAllGeneratedFlags = (workflow: WorkflowFile): WorkflowFile => { for (let i = 0; i < workflow.workflow.length; i++) { if ( From 928b7dd93c33c8f702fb6a3a946715474d55d27c Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 00:21:29 +0530 Subject: [PATCH 45/72] feat: add generated flags to worflow pair --- .../src/workflow-management/classes/Generator.ts | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index 940502bc..982a48bd 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -385,5 +385,17 @@ export class WorkflowGenerator { return workflow; }; - + + public AddGeneratedFlags = (workflow: WorkflowFile): WorkflowFile => { + const copy = JSON.parse(JSON.stringify(workflow)); + for (let i = 0; i < workflow.workflow.length; i++) { + copy.workflow[i].what.unshift({ + action: 'flag', + args: ['generated'], + }); + } + return copy; + }; + + } From 91ff46d8f0577d4831b57e36b1f86cf60abd39f1 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 00:21:48 +0530 Subject: [PATCH 46/72] chore: describe AddGeneratedFLags --- server/src/workflow-management/classes/Generator.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index 982a48bd..926336c8 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -385,7 +385,12 @@ export class WorkflowGenerator { return workflow; }; - + /** + * Adds generated flag actions to the workflow's pairs' what conditions. + * @param workflow The workflow for adding the generated flag actions from. + * @private + * @returns {WorkflowFile} + */ public AddGeneratedFlags = (workflow: WorkflowFile): WorkflowFile => { const copy = JSON.parse(JSON.stringify(workflow)); for (let i = 0; i < workflow.workflow.length; i++) { From 1e84773c3bb156655dfd3e19bee9c6476c030a32 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 00:22:24 +0530 Subject: [PATCH 47/72] feat: update workflow file --- .../workflow-management/classes/Generator.ts | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index 926336c8..0fc52fdf 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -402,5 +402,21 @@ export class WorkflowGenerator { return copy; }; - + /** + * Enables to update the generated workflow file. + * Adds a generated flag action for possible pausing during the interpretation. + * Used for loading a recorded workflow to already initialized Generator. + * @param workflowFile The workflow file to be used as a replacement for the current generated workflow. + * @returns void + */ + public updateWorkflowFile = (workflowFile: WorkflowFile, meta: MetaData) => { + this.recordingMeta = meta; + const params = this.checkWorkflowForParams(workflowFile); + if (params) { + this.recordingMeta.params = params; + } + this.workflowRecord = workflowFile; + } + + } From 1b42b903aa050f4350745cf6af70a4fabee90541 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 00:23:00 +0530 Subject: [PATCH 48/72] feat: save new worflow file --- .../workflow-management/classes/Generator.ts | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index 0fc52fdf..da94fd90 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -418,5 +418,34 @@ export class WorkflowGenerator { this.workflowRecord = workflowFile; } + /** + * Creates a recording metadata and stores the curren workflow + * with the metadata to the file system. + * @param fileName The name of the file. + * @returns {Promise} + */ + public saveNewWorkflow = async (fileName: string) => { + const recording = this.optimizeWorkflow(this.workflowRecord); + try { + this.recordingMeta = { + name: fileName, + create_date: this.recordingMeta.create_date || new Date().toLocaleString(), + pairs: recording.workflow.length, + update_date: new Date().toLocaleString(), + params: this.getParams() || [], + } + fs.mkdirSync('../storage/recordings', { recursive: true }) + await saveFile( + `../storage/recordings/${fileName}.waw.json`, + JSON.stringify({ recording_meta: this.recordingMeta, recording }, null, 2) + ); + } + catch (e) { + const { message } = e as Error; + logger.log('warn', `Cannot save the file to the local file system`) + } + this.socket.emit('fileSaved'); + } + } From 3715d46f0f583033398b9b0bfded813df73a4108 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 00:26:08 +0530 Subject: [PATCH 49/72] feat: generate unique css selector based on actin --- .../workflow-management/classes/Generator.ts | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index da94fd90..b25b043b 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -447,5 +447,23 @@ export class WorkflowGenerator { this.socket.emit('fileSaved'); } - + + private generateSelector = async (page:Page, coordinates:Coordinates, action: ActionType) => { + const elementInfo = await getElementInformation(page, coordinates); + const bestSelector = getBestSelectorForAction( + { + type: action, + tagName: elementInfo?.tagName as TagName || '', + inputType: undefined, + value: undefined, + selectors: await getSelectors(page, coordinates) || {}, + timestamp: 0, + isPassword: false, + hasOnlyText: elementInfo?.hasOnlyText || false, + } as Action, + ); + return bestSelector; + } + + } From 0aaf697f69cdfef3906d93d6a6c3045ebde53e34 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 00:26:29 +0530 Subject: [PATCH 50/72] docs: generateSelector descirpito --- server/src/workflow-management/classes/Generator.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index b25b043b..03757e79 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -447,7 +447,15 @@ export class WorkflowGenerator { this.socket.emit('fileSaved'); } - + /** + * Uses a system of functions to generate a correct and unique css selector + * according to the action being performed. + * @param page The page to be used for obtaining the information and selector. + * @param coordinates The coordinates of the element. + * @param action The action for which the selector is being generated. + * @private + * @returns {Promise} + */ private generateSelector = async (page:Page, coordinates:Coordinates, action: ActionType) => { const elementInfo = await getElementInformation(page, coordinates); const bestSelector = getBestSelectorForAction( From b6c4de718442d89824846d8b7b96ca42baba9c61 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 00:27:11 +0530 Subject: [PATCH 51/72] feat: generate data for highlithing element on client side --- server/src/workflow-management/classes/Generator.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index 03757e79..dd380853 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -474,4 +474,13 @@ export class WorkflowGenerator { } + public generateDataForHighlighter = async (page: Page, coordinates: Coordinates) => { + const rect = await getRect(page, coordinates); + const displaySelector = await this.generateSelector(page, coordinates, ActionType.Click); + if (rect) { + this.socket.emit('highlighter', { rect, selector: displaySelector }); + } + } + + } From 4886bbb08b7b999d1239092ddfd769df8370f3bd Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 00:27:35 +0530 Subject: [PATCH 52/72] docs: generateDataForHighlighter description --- server/src/workflow-management/classes/Generator.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index dd380853..557b37ca 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -473,7 +473,13 @@ export class WorkflowGenerator { return bestSelector; } - + /** + * Generates data for highlighting the element on client side and emits the + * highlighter event to the client. + * @param page The page to be used for obtaining data. + * @param coordinates The coordinates of the element. + * @returns {Promise} + */ public generateDataForHighlighter = async (page: Page, coordinates: Coordinates) => { const rect = await getRect(page, coordinates); const displaySelector = await this.generateSelector(page, coordinates, ActionType.Click); From 8e977900ab574f1db85db1971179ec1edd8f5454 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 00:32:49 +0530 Subject: [PATCH 53/72] feat: notify client of url change --- .../src/workflow-management/classes/Generator.ts | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index 557b37ca..87b597d5 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -488,5 +488,17 @@ export class WorkflowGenerator { } } - -} + /** + * Notifies the client about the change of the url if navigation + * happens after some performed action. + * @param url The new url. + * @param fromNavBar Whether the navigation is from the simulated browser's navbar or not. + * @returns void + */ + public notifyUrlChange = (url:string) => { + if (this.socket) { + this.socket.emit('urlChanged', url); + } + } + +} \ No newline at end of file From 7e66cd81b6dc80f725db2b4028a39fd945bb8390 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 00:33:22 +0530 Subject: [PATCH 54/72] feat: notify client on new tab --- .../src/workflow-management/classes/Generator.ts | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index 87b597d5..8762ef42 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -501,4 +501,17 @@ export class WorkflowGenerator { } } -} \ No newline at end of file + + public notifyOnNewTab = (page: Page, pageIndex: number) => { + if (this.socket) { + page.on('close', () => { + this.socket.emit('tabHasBeenClosed', pageIndex); + }) + const parsedUrl = new URL(page.url()); + const host = parsedUrl.hostname?.match(/\b(?!www\.)[a-zA-Z0-9]+/g)?.join('.'); + this.socket.emit('newTab', host ? host : 'new tab') + } + } + + +} From bb87c775bf0b0bf34e72ff283471e8564b2331f2 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 00:33:54 +0530 Subject: [PATCH 55/72] docs: notifyOnNewTab description --- server/src/workflow-management/classes/Generator.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index 8762ef42..a784185f 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -501,7 +501,12 @@ export class WorkflowGenerator { } } - + /** + * Notifies the client about the new tab if popped-up + * @param page The page to be used for obtaining data. + * @param pageIndex The index of the page. + * @returns void + */ public notifyOnNewTab = (page: Page, pageIndex: number) => { if (this.socket) { page.on('close', () => { From 42d0e4f0216a1adca3a80a94e3e03993cae52027 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 00:36:54 +0530 Subject: [PATCH 56/72] feat: navigate to prev page --- server/src/workflow-management/classes/Generator.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index a784185f..5f19e47a 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -519,4 +519,15 @@ export class WorkflowGenerator { } + public onGoBack = (newUrl: string) => { + //it's safe to always add a go back action to the first rule in the workflow + this.workflowRecord.workflow[0].what.push({ + action: 'goBack', + args: [{waitUntil: 'commit'}], + }); + this.notifyUrlChange(newUrl); + this.socket.emit('workflow', this.workflowRecord); + } + + } From d078892f71670de0ffa7abdd4e25fc7fbae48903 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 00:37:13 +0530 Subject: [PATCH 57/72] docs: onGoBack --- server/src/workflow-management/classes/Generator.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index 5f19e47a..796b64c3 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -518,7 +518,14 @@ export class WorkflowGenerator { } } - + /** + * Generates a pair for navigating to the previous page. + * This function alone adds the pair to the workflow and notifies the client. + * It's safe to always add a go back action to the first rule in the workflow and do not check + * general conditions for adding a pair to the workflow. + * @param newUrl The previous page's url. + * @returns void + */ public onGoBack = (newUrl: string) => { //it's safe to always add a go back action to the first rule in the workflow this.workflowRecord.workflow[0].what.push({ From 3a6402b2fd0bd1fe5e1cc1569d782dd5d3770332 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 00:38:00 +0530 Subject: [PATCH 58/72] feat: navigate to next page --- server/src/workflow-management/classes/Generator.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index 796b64c3..4d91462f 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -536,5 +536,16 @@ export class WorkflowGenerator { this.socket.emit('workflow', this.workflowRecord); } + + public onGoForward = (newUrl: string) => { + //it's safe to always add a go forward action to the first rule in the workflow + this.workflowRecord.workflow[0].what.push({ + action: 'goForward', + args: [{waitUntil: 'commit'}], + }); + this.notifyUrlChange(newUrl); + this.socket.emit('workflow', this.workflowRecord); + } + } From d4fb263bf5ba81c070b65d080e2c9f87db982238 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 00:38:28 +0530 Subject: [PATCH 59/72] docs: onGoForward --- server/src/workflow-management/classes/Generator.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index 4d91462f..2ae4737e 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -536,7 +536,14 @@ export class WorkflowGenerator { this.socket.emit('workflow', this.workflowRecord); } - + /** + * Generates a pair for navigating to the next page. + * This function alone adds the pair to the workflow and notifies the client. + * It's safe to always add a go forward action to the first rule in the workflow and do not check + * general conditions for adding a pair to the workflow. + * @param newUrl The next page's url. + * @returns void + */ public onGoForward = (newUrl: string) => { //it's safe to always add a go forward action to the first rule in the workflow this.workflowRecord.workflow[0].what.push({ From e5e677198514ad50ca178df87ca7745fa68c5c8d Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 00:39:59 +0530 Subject: [PATCH 60/72] feat: check if pair over-shadowed by current worflow pair --- .../workflow-management/classes/Generator.ts | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index 2ae4737e..e6e15770 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -554,5 +554,28 @@ export class WorkflowGenerator { this.socket.emit('workflow', this.workflowRecord); } - + + private IsOverShadowingAction = async (pair: WhereWhatPair, page: Page) => { + type possibleOverShadow = { + index: number; + isOverShadowing: boolean; + } + + const possibleOverShadow: possibleOverShadow[] = []; + const haveSameUrl = this.workflowRecord.workflow + .filter((p, index) => { + if (p.where.url === pair.where.url) { + possibleOverShadow.push({index: index, isOverShadowing: false}); + return true; + } else { + return false; + } + }); + + + return possibleOverShadow; + } + + + } From 1f57124672796be50d9ad2ced5333194d620954b Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 00:40:18 +0530 Subject: [PATCH 61/72] feat: if same url len !=0 --- server/src/workflow-management/classes/Generator.ts | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index e6e15770..bc5c72cd 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -572,7 +572,18 @@ export class WorkflowGenerator { } }); - + if (haveSameUrl.length !== 0) { + for (let i = 0; i < haveSameUrl.length; i++) { + //@ts-ignore + if (haveSameUrl[i].where.selectors && haveSameUrl[i].where.selectors.length > 0) { + //@ts-ignore + const isOverShadowing = await isRuleOvershadowing(haveSameUrl[i].where.selectors, page); + if (isOverShadowing) { + possibleOverShadow[i].isOverShadowing = true; + } + } + } + } return possibleOverShadow; } From 5fd176a0dea454e1e8f159a0c3505117dc415147 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 00:40:42 +0530 Subject: [PATCH 62/72] docs: isOverShadowingAction --- server/src/workflow-management/classes/Generator.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index bc5c72cd..901f7269 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -554,7 +554,14 @@ export class WorkflowGenerator { this.socket.emit('workflow', this.workflowRecord); } - + /** + * Checks and returns possible pairs that would get over-shadowed by the pair + * from the current workflow. + * @param pair The pair that could be over-shadowing. + * @param page The page to be used for checking the visibility and accessibility of the selectors. + * @private + * @returns {Promise} + */ private IsOverShadowingAction = async (pair: WhereWhatPair, page: Page) => { type possibleOverShadow = { index: number; From 4b13e1434f5a67b1f44f09b067959bde72d0d989 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 00:41:20 +0530 Subject: [PATCH 63/72] feat: overshadowing handler --- .../workflow-management/classes/Generator.ts | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index 901f7269..5ee06273 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -595,5 +595,35 @@ export class WorkflowGenerator { } + + private handleOverShadowing = async (pair: WhereWhatPair, page: Page, index: number): Promise => { + const overShadowing = (await this.IsOverShadowingAction(pair, page)) + .filter((p) => p.isOverShadowing); + if (overShadowing.length !== 0) { + for (const overShadowedAction of overShadowing) { + if (overShadowedAction.index === index) { + if (pair.where.selectors) { + for (const selector of pair.where.selectors) { + if (this.workflowRecord.workflow[index].where.selectors?.includes(selector)) { + break; + } else { + // add new selector to the where part of the overshadowing pair + this.workflowRecord.workflow[index].where.selectors?.push(selector); + } + } + } + // push the action automatically to the first/the closest rule which would be overShadowed + this.workflowRecord.workflow[index].what = + this.workflowRecord.workflow[index].what.concat(pair.what); + return true; + } else { + // notify client about overshadowing a further rule + return false; + } + } + } + return false; + } + } From ddfb31ce39363a20f8e23661b8eb3d96581fd24e Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 00:41:39 +0530 Subject: [PATCH 64/72] docs: handleOverShadowing --- server/src/workflow-management/classes/Generator.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index 5ee06273..d9e25378 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -595,7 +595,15 @@ export class WorkflowGenerator { } - + /** + * General over-shadowing handler. + * Checks for possible over-shadowed pairs and if found, + * adds the pair to the workflow in the correct way. + * @param pair The pair that could be over-shadowing. + * @param page The page to be used for checking the visibility and accessibility of the selectors. + * @private + * @returns {Promise} + */ private handleOverShadowing = async (pair: WhereWhatPair, page: Page, index: number): Promise => { const overShadowing = (await this.IsOverShadowingAction(pair, page)) .filter((p) => p.isOverShadowing); From 8cdf5ebe67b899871a76c575a6141215655aa955 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 00:42:32 +0530 Subject: [PATCH 65/72] feat: get best url representatio --- .../workflow-management/classes/Generator.ts | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index d9e25378..bade1a56 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -634,4 +634,32 @@ export class WorkflowGenerator { } + private getBestUrl = (url: string) => { + const parsedUrl = new URL(url); + const protocol = parsedUrl.protocol === 'https:' || parsedUrl.protocol === 'http:' ? `${parsedUrl.protocol}//`: parsedUrl.protocol; + const regex = new RegExp(/(?=.*[A-Z])/g) + // remove all params with uppercase letters, they are most likely dynamically generated + // also escapes all regex characters from the params + const search = parsedUrl.search + .replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + .split('&').map((param, index) => { + if (!regex.test(param)) { + return param; + } else { + return '.*'; + } + }) + .join('&'); + let bestUrl; + if (search) { + bestUrl = { + $regex: `^${protocol}${parsedUrl.host}${parsedUrl.pathname}${search}${parsedUrl.hash}` + } + } else { + bestUrl = `${protocol}${parsedUrl.host}${parsedUrl.pathname}${parsedUrl.hash}`; + } + return bestUrl; + } + + } From 702d720af2c2f8173f5e617f8719821ecd8f71b0 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 00:42:55 +0530 Subject: [PATCH 66/72] docs: getBestUrl docs --- server/src/workflow-management/classes/Generator.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index bade1a56..ae3db680 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -633,7 +633,12 @@ export class WorkflowGenerator { return false; } - + /** + * Returns the best possible url representation for a where condition according to the heuristics. + * @param url The url to be checked and possibly replaced. + * @private + * @returns {string | {$regex: string}} + */ private getBestUrl = (url: string) => { const parsedUrl = new URL(url); const protocol = parsedUrl.protocol === 'https:' || parsedUrl.protocol === 'http:' ? `${parsedUrl.protocol}//`: parsedUrl.protocol; From 4fe40759722733ba78f384fac331365a9e2a1dc9 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 00:43:34 +0530 Subject: [PATCH 67/72] feat: return worflow params if !null --- .../workflow-management/classes/Generator.ts | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index ae3db680..39977496 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -666,5 +666,30 @@ export class WorkflowGenerator { return bestUrl; } + /** + * Returns parameters if present in the workflow or null. + * @param workflow The workflow to be checked. + */ + private checkWorkflowForParams = (workflow: WorkflowFile): string[]|null => { + // for now the where condition cannot have any params, so we're checking only what part of the pair + // where only the args part of what condition can have a parameter + for (const pair of workflow.workflow) { + for (const condition of pair.what) { + if (condition.args) { + const params: any[] = []; + condition.args.forEach((arg) => { + if (arg.$param) { + params.push(arg.$param); + } + }) + if (params.length !== 0) { + return params; + } + } + } + } + return null; + } + } From 9ca5398602b2892458ebf5f27b19651d3b19125f Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 00:44:00 +0530 Subject: [PATCH 68/72] feat: optimize worfklow --- .../workflow-management/classes/Generator.ts | 75 ++++++++++++++++++- 1 file changed, 74 insertions(+), 1 deletion(-) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index 39977496..03ca6031 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -691,5 +691,78 @@ export class WorkflowGenerator { return null; } - + /** + * A function for workflow optimization once finished. + * @param workflow The workflow to be optimized. + */ + private optimizeWorkflow = (workflow: WorkflowFile) => { + + // replace a sequence of press actions by a single fill action + let input = { + selector: '', + value: '', + actionCounter: 0, + }; + + const pushTheOptimizedAction = (pair: WhereWhatPair, index: number) => { + if (input.value.length === 1) { + // when only one press action is present, keep it and add a waitForLoadState action + pair.what.splice(index + 1, 0, { + action: 'waitForLoadState', + args: ['networkidle'], + }) + } else { + // when more than one press action is present, add a type action + pair.what.splice(index - input.actionCounter, input.actionCounter, { + action: 'type', + args: [input.selector, input.value], }, { + action: 'waitForLoadState', + args: ['networkidle'], }); + } + } + + + for (const pair of workflow.workflow) { + pair.what.forEach( (condition, index) => { + if (condition.action === 'press') { + if (condition.args && condition.args[1]) { + if (!input.selector) { + input.selector = condition.args[0]; + } + if (input.selector === condition.args[0]) { + input.actionCounter++; + if (condition.args[1].length === 1) { + input.value = input.value + condition.args[1]; + } else if (condition.args[1] === 'Backspace') { + input.value = input.value.slice(0, -1); + } else if (condition.args[1] !== 'Shift') { + pushTheOptimizedAction(pair, index); + pair.what.splice(index + 1, 0, { + action: 'waitForLoadState', + args: ['networkidle'], + }) + input = {selector: '', value: '', actionCounter: 0}; + } + } else { + pushTheOptimizedAction(pair, index); + input = { + selector: condition.args[0], + value: condition.args[1], + actionCounter: 1, + }; + } + } + } else { + if (input.value.length !== 0) { + pushTheOptimizedAction(pair, index); + // clear the input + input = {selector: '', value: '', actionCounter: 0}; + } + } + }); + } + return workflow; + } + + } From 1046c020844895d1853a48b5c663f5fca8620cdc Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 00:44:22 +0530 Subject: [PATCH 69/72] feat: get worflow params from stored metadata --- server/src/workflow-management/classes/Generator.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index 03ca6031..bec48f14 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -764,5 +764,10 @@ export class WorkflowGenerator { return workflow; } + + public getParams = (): string[]|null => { + return this.checkWorkflowForParams(this.workflowRecord); + } + } From 4e531ed76091daba8b58a338956610a721151818 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 00:44:33 +0530 Subject: [PATCH 70/72] docs --- server/src/workflow-management/classes/Generator.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index bec48f14..3024b305 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -764,7 +764,9 @@ export class WorkflowGenerator { return workflow; } - + /** + * Returns workflow params from the stored metadata. + */ public getParams = (): string[]|null => { return this.checkWorkflowForParams(this.workflowRecord); } From 920ebe8bce6dd9ce8b60c88829474ff4aee88105 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 00:44:47 +0530 Subject: [PATCH 71/72] feat: clear last generated data index --- server/src/workflow-management/classes/Generator.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index 3024b305..1353ec2b 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -771,5 +771,10 @@ export class WorkflowGenerator { return this.checkWorkflowForParams(this.workflowRecord); } - + /** + * Clears the last generated data index. + */ + public clearLastIndex = () => { + this.generatedData.lastIndex = null; + } } From 7e1e8f2920001686c6d1d9c172f0b04d1b8b1074 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 8 Jun 2024 00:46:59 +0530 Subject: [PATCH 72/72] chore: lint --- .../workflow-management/classes/Generator.ts | 75 ++++++++++--------- 1 file changed, 39 insertions(+), 36 deletions(-) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index 1353ec2b..2634635d 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -19,7 +19,7 @@ import { browserPool } from "../../server"; interface PersistedGeneratedData { lastUsedSelector: string; - lastIndex: number|null; + lastIndex: number | null; lastAction: string; } @@ -44,7 +44,7 @@ export class WorkflowGenerator { * The socket used to communicate with the client. * @private */ - private socket : Socket; + private socket: Socket; /** * The public constructor of the WorkflowGenerator. @@ -100,9 +100,9 @@ export class WorkflowGenerator { }); socket.on('new-recording', () => this.workflowRecord = { workflow: [], - } ); + }); socket.on('activeIndex', (data) => this.generatedData.lastIndex = parseInt(data)); - socket.on('decision', async ({pair, actionType, decision}) => { + socket.on('decision', async ({ pair, actionType, decision }) => { const id = browserPool.getActiveBrowserId(); if (id) { const activeBrowser = browserPool.getRemoteBrowser(id); @@ -148,7 +148,7 @@ export class WorkflowGenerator { * @private * @returns {Promise} */ - private addPairToWorkflowAndNotifyClient = async(pair: WhereWhatPair, page: Page) => { + private addPairToWorkflowAndNotifyClient = async (pair: WhereWhatPair, page: Page) => { let matched = false; // validate if a pair with the same where conditions is already present in the workflow if (pair.where.selectors && pair.where.selectors[0]) { @@ -197,7 +197,7 @@ export class WorkflowGenerator { } } this.socket.emit('workflow', this.workflowRecord); - logger.log('info',`Workflow emitted`); + logger.log('info', `Workflow emitted`); }; /** @@ -235,14 +235,14 @@ export class WorkflowGenerator { * @param page The page to use for obtaining the needed data. * @returns {Promise} */ - public onChangeUrl = async(newUrl: string, page: Page) => { + public onChangeUrl = async (newUrl: string, page: Page) => { this.generatedData.lastUsedSelector = ''; const pair: WhereWhatPair = { where: { url: this.getBestUrl(page.url()) }, what: [ { - action: 'goto', - args: [newUrl], + action: 'goto', + args: [newUrl], } ], } @@ -284,7 +284,7 @@ export class WorkflowGenerator { */ public customAction = async (action: CustomActions, settings: any, page: Page) => { const pair: WhereWhatPair = { - where: { url: this.getBestUrl(page.url())}, + where: { url: this.getBestUrl(page.url()) }, what: [{ action, args: settings ? Array.isArray(settings) ? settings : [settings] : [], @@ -297,7 +297,8 @@ export class WorkflowGenerator { lastData: { selector: this.generatedData.lastUsedSelector, action: this.generatedData.lastAction, - } }); + } + }); } else { await this.addPairToWorkflowAndNotifyClient(pair, page); } @@ -337,7 +338,7 @@ export class WorkflowGenerator { logger.log('debug', `pair ${index}: Added to workflow file.`); } else if (index < this.workflowRecord.workflow.length && index >= 0) { this.workflowRecord.workflow.splice( - this.workflowRecord.workflow.length - index , 0, pair); + this.workflowRecord.workflow.length - index, 0, pair); } else { logger.log('error', `Add pair ${index}: Index out of range.`); } @@ -362,7 +363,7 @@ export class WorkflowGenerator { * @param socket The socket to be used for communication. * @returns void */ - public updateSocket = (socket: Socket) : void => { + public updateSocket = (socket: Socket): void => { this.socket = socket; this.registerEventHandlers(socket); }; @@ -374,15 +375,15 @@ export class WorkflowGenerator { * @returns {WorkflowFile} */ private removeAllGeneratedFlags = (workflow: WorkflowFile): WorkflowFile => { - for (let i = 0; i < workflow.workflow.length; i++) { - if ( - workflow.workflow[i].what[0] && - workflow.workflow[i].what[0].action === 'flag' && - workflow.workflow[i].what[0].args?.includes('generated')) { - workflow.workflow[i].what.splice(0, 1); - } + for (let i = 0; i < workflow.workflow.length; i++) { + if ( + workflow.workflow[i].what[0] && + workflow.workflow[i].what[0].action === 'flag' && + workflow.workflow[i].what[0].args?.includes('generated')) { + workflow.workflow[i].what.splice(0, 1); } - return workflow; + } + return workflow; }; /** @@ -440,7 +441,7 @@ export class WorkflowGenerator { JSON.stringify({ recording_meta: this.recordingMeta, recording }, null, 2) ); } - catch (e) { + catch (e) { const { message } = e as Error; logger.log('warn', `Cannot save the file to the local file system`) } @@ -456,7 +457,7 @@ export class WorkflowGenerator { * @private * @returns {Promise} */ - private generateSelector = async (page:Page, coordinates:Coordinates, action: ActionType) => { + private generateSelector = async (page: Page, coordinates: Coordinates, action: ActionType) => { const elementInfo = await getElementInformation(page, coordinates); const bestSelector = getBestSelectorForAction( { @@ -495,7 +496,7 @@ export class WorkflowGenerator { * @param fromNavBar Whether the navigation is from the simulated browser's navbar or not. * @returns void */ - public notifyUrlChange = (url:string) => { + public notifyUrlChange = (url: string) => { if (this.socket) { this.socket.emit('urlChanged', url); } @@ -530,7 +531,7 @@ export class WorkflowGenerator { //it's safe to always add a go back action to the first rule in the workflow this.workflowRecord.workflow[0].what.push({ action: 'goBack', - args: [{waitUntil: 'commit'}], + args: [{ waitUntil: 'commit' }], }); this.notifyUrlChange(newUrl); this.socket.emit('workflow', this.workflowRecord); @@ -548,7 +549,7 @@ export class WorkflowGenerator { //it's safe to always add a go forward action to the first rule in the workflow this.workflowRecord.workflow[0].what.push({ action: 'goForward', - args: [{waitUntil: 'commit'}], + args: [{ waitUntil: 'commit' }], }); this.notifyUrlChange(newUrl); this.socket.emit('workflow', this.workflowRecord); @@ -572,7 +573,7 @@ export class WorkflowGenerator { const haveSameUrl = this.workflowRecord.workflow .filter((p, index) => { if (p.where.url === pair.where.url) { - possibleOverShadow.push({index: index, isOverShadowing: false}); + possibleOverShadow.push({ index: index, isOverShadowing: false }); return true; } else { return false; @@ -616,7 +617,7 @@ export class WorkflowGenerator { break; } else { // add new selector to the where part of the overshadowing pair - this.workflowRecord.workflow[index].where.selectors?.push(selector); + this.workflowRecord.workflow[index].where.selectors?.push(selector); } } } @@ -641,7 +642,7 @@ export class WorkflowGenerator { */ private getBestUrl = (url: string) => { const parsedUrl = new URL(url); - const protocol = parsedUrl.protocol === 'https:' || parsedUrl.protocol === 'http:' ? `${parsedUrl.protocol}//`: parsedUrl.protocol; + const protocol = parsedUrl.protocol === 'https:' || parsedUrl.protocol === 'http:' ? `${parsedUrl.protocol}//` : parsedUrl.protocol; const regex = new RegExp(/(?=.*[A-Z])/g) // remove all params with uppercase letters, they are most likely dynamically generated // also escapes all regex characters from the params @@ -670,7 +671,7 @@ export class WorkflowGenerator { * Returns parameters if present in the workflow or null. * @param workflow The workflow to be checked. */ - private checkWorkflowForParams = (workflow: WorkflowFile): string[]|null => { + private checkWorkflowForParams = (workflow: WorkflowFile): string[] | null => { // for now the where condition cannot have any params, so we're checking only what part of the pair // where only the args part of what condition can have a parameter for (const pair of workflow.workflow) { @@ -715,15 +716,17 @@ export class WorkflowGenerator { // when more than one press action is present, add a type action pair.what.splice(index - input.actionCounter, input.actionCounter, { action: 'type', - args: [input.selector, input.value], }, { + args: [input.selector, input.value], + }, { action: 'waitForLoadState', - args: ['networkidle'], }); + args: ['networkidle'], + }); } } for (const pair of workflow.workflow) { - pair.what.forEach( (condition, index) => { + pair.what.forEach((condition, index) => { if (condition.action === 'press') { if (condition.args && condition.args[1]) { if (!input.selector) { @@ -741,7 +744,7 @@ export class WorkflowGenerator { action: 'waitForLoadState', args: ['networkidle'], }) - input = {selector: '', value: '', actionCounter: 0}; + input = { selector: '', value: '', actionCounter: 0 }; } } else { pushTheOptimizedAction(pair, index); @@ -756,7 +759,7 @@ export class WorkflowGenerator { if (input.value.length !== 0) { pushTheOptimizedAction(pair, index); // clear the input - input = {selector: '', value: '', actionCounter: 0}; + input = { selector: '', value: '', actionCounter: 0 }; } } }); @@ -767,7 +770,7 @@ export class WorkflowGenerator { /** * Returns workflow params from the stored metadata. */ - public getParams = (): string[]|null => { + public getParams = (): string[] | null => { return this.checkWorkflowForParams(this.workflowRecord); }