diff --git a/server/src/types/index.ts b/server/src/types/index.ts index 44647d9a..4fe761f1 100644 --- a/server/src/types/index.ts +++ b/server/src/types/index.ts @@ -11,6 +11,23 @@ export interface InterpreterSettings { params?: any; } +/** + * Useful coordinates interface holding the x and y coordinates of a point. + * @category Types + */ +export interface Coordinates { + x: number; + y: number; +} + +/** + * Holds the deltas of a wheel/scroll event. + * @category Types + */ +export interface ScrollDeltas { + deltaX: number; + deltaY: number; +} /** * Options for the {@link BrowserManagement.launch} method. @@ -25,3 +42,224 @@ export interface RemoteBrowserOptions { launchOptions: LaunchOptions }; +/** + * Pairs a pressed key value with the coordinates of the key press. + * @category Types + */ +export interface KeyboardInput { + key: string; + coordinates: Coordinates; +} + + +/** + * Contains index in the current workflow and result for over-shadowing check of a pair. + * @category Types + */ +export type PossibleOverShadow = { + index: number; + isOverShadowing: boolean; +} +/** + * An object representing he coordinates, width, height and corner points of the element. + * @category Types + */ +export interface Rectangle extends Coordinates { + width: number; + height: number; + top: number; + right: number; + bottom: number; + left: number; +} + +/** + * Helpful enum used for determining the type of action currently executed by the user. + * @enum {string} + * @category Types + */ +export enum ActionType { + AwaitText = 'awaitText', + Click = 'click', + DragAndDrop = 'dragAndDrop', + Screenshot = 'screenshot', + Hover = 'hover', + Input = 'input', + Keydown = 'keydown', + Load = 'load', + Navigate = 'navigate', + Scroll = 'scroll', +} + +/** + * Useful enum for determining the element's tag name. + * @enum {string} + * @category Types + */ +export enum TagName { + A = 'A', + B = 'B', + Cite = 'CITE', + EM = 'EM', + Input = 'INPUT', + Select = 'SELECT', + Span = 'SPAN', + Strong = 'STRONG', + TextArea = 'TEXTAREA', +} + +/** + * @category Types + */ +export interface BaseActionInfo { + tagName: string; + /** + * If the element only has text content inside (hint to use text selector) + */ + hasOnlyText: boolean; +} + +/** + * Holds all the possible css selectors that has been found for an element. + * @category Types + */ +export interface Selectors { + id: string|null; + generalSelector: string|null; + attrSelector: string|null; + testIdSelector: string|null; + text: string|null; + href: string|null; + hrefSelector: string|null; + accessibilitySelector: string|null; + formSelector: string|null; +} + +/** + * Base type for all actions. + * Action types are used to determine the best selector for the user action. + * They store valuable information, specific to the action. + * @category Types + */ +export interface BaseAction extends BaseActionInfo{ + type: ActionType; + associatedActions: ActionType[]; + inputType: string | undefined; + value: string | undefined; + selectors: { [key: string]: string | null }; + timestamp: number; + isPassword: boolean; + /** + * Overrides the {@link BaseActionInfo} type of tagName for the action. + */ + tagName: TagName; +} + +/** + * Action type for pressing on a keyboard. + * @category Types + */ +interface KeydownAction extends BaseAction { + type: ActionType.Keydown; + key: string; +} + +/** + * Action type for typing into an input field. + * @category Types + */ +interface InputAction extends BaseAction { + type: ActionType.Input; +} + +/** + * Action type for clicking on an element. + * @category Types + */ +interface ClickAction extends BaseAction { + type: ActionType.Click; +} + +/** + * Action type for drag and dropping an element. + * @category Types + */ +interface DragAndDropAction extends BaseAction { + type: ActionType.DragAndDrop; + sourceX: number; + sourceY: number; + targetX: number; + targetY: number; +} + +/** + * Action type for hovering over an element. + * @category Types + */ +interface HoverAction extends BaseAction { + type: ActionType.Hover; +} + +/** + * Action type for waiting on load. + * @category Types + */ +interface LoadAction extends BaseAction { + type: ActionType.Load; + url: string; +} + +/** + * Action type for page navigation. + * @category Types + */ +interface NavigateAction extends BaseAction { + type: ActionType.Navigate; + url: string; + source: string; +} + +/** + * Action type for scrolling. + * @category Types + */ +interface WheelAction extends BaseAction { + type: ActionType.Scroll; + deltaX: number; + deltaY: number; + pageXOffset: number; + pageYOffset: number; +} + +/** + * Action type for taking a screenshot. + * @category Types + */ +interface FullScreenshotAction extends BaseAction { + type: ActionType.Screenshot; +} + +/** + * Action type for waiting on the filling of text input. + * @category Types + */ +interface AwaitTextAction extends BaseAction { + type: ActionType.AwaitText; + text: string; +} + +/** + * Definition of the Action type. + * @category Types + */ +export type Action = + | KeydownAction + | InputAction + | ClickAction + | DragAndDropAction + | HoverAction + | LoadAction + | NavigateAction + | WheelAction + | FullScreenshotAction + | AwaitTextAction; diff --git a/server/src/workflow-management/classes/Interpreter.ts b/server/src/workflow-management/classes/Interpreter.ts new file mode 100644 index 00000000..ff3d8c08 --- /dev/null +++ b/server/src/workflow-management/classes/Interpreter.ts @@ -0,0 +1,281 @@ +import Interpreter, { WorkflowFile } from "@wbr-project/wbr-interpret"; +import logger from "../../logger"; +import { Socket } from "socket.io"; +import { Page } from "playwright"; +import { InterpreterSettings } from "../../types"; + +/** + * This class implements the main interpretation functions. + * It holds some information about the current interpretation process and + * registers to some events to allow the client (frontend) to interact with the interpreter. + * It uses the [@wbr-project/wbr-interpret](https://www.npmjs.com/package/@wbr-project/wbr-interpret) + * library to interpret the workflow. + * @category WorkflowManagement + */ +export class WorkflowInterpreter { + /** + * Socket.io socket instance enabling communication with the client (frontend) side. + * @private + */ + private socket: Socket; + + /** + * True if the interpretation is paused. + */ + public interpretationIsPaused: boolean = false; + + /** + * The instance of the {@link Interpreter} class used to interpret the workflow. + * From @wbr-project/wbr-interpret. + * @private + */ + private interpreter: Interpreter | null = null; + + /** + * An id of the currently interpreted pair in the workflow. + * @private + */ + private activeId: number | null = null; + + /** + * An array of debug messages emitted by the {@link Interpreter}. + */ + public debugMessages: string[] = []; + + /** + * An array of all the serializable data extracted from the run. + */ + public serializableData: string[] = []; + + /** + * An array of all the binary data extracted from the run. + */ + public binaryData: { mimetype: string, data: string }[] = []; + + /** + * An array of id's of the pairs from the workflow that are about to be paused. + * As "breakpoints". + * @private + */ + private breakpoints: boolean[] = []; + + /** + * Callback to resume the interpretation after a pause. + * @private + */ + private interpretationResume: (() => void) | null = null; + + /** + * A public constructor taking a socket instance for communication with the client. + * @param socket Socket.io socket instance enabling communication with the client (frontend) side. + * @constructor + */ + constructor(socket: Socket) { + this.socket = socket; + } + + /** + * Subscribes to the events that are used to control the interpretation. + * The events are pause, resume, step and breakpoints. + * Step is used to interpret a single pair and pause on the other matched pair. + * @returns void + */ + public subscribeToPausing = () => { + this.socket.on('pause', () => { + this.interpretationIsPaused = true; + }); + this.socket.on('resume', () => { + this.interpretationIsPaused = false; + if (this.interpretationResume) { + this.interpretationResume(); + this.socket.emit('log', '----- The interpretation has been resumed -----', false); + } else { + logger.log('debug', "Resume called but no resume function is set"); + } + }); + this.socket.on('step', () => { + if (this.interpretationResume) { + this.interpretationResume(); + } else { + logger.log('debug', "Step called but no resume function is set"); + } + }); + this.socket.on('breakpoints', (data: boolean[]) => { + logger.log('debug', "Setting breakpoints: " + data); + this.breakpoints = data + }); + } + + /** + * Sets up the instance of {@link Interpreter} and interprets + * the workflow inside the recording editor. + * Cleans up this interpreter instance after the interpretation is finished. + * @param workflow The workflow to interpret. + * @param page The page instance used to interact with the browser. + * @param updatePageOnPause A callback to update the page after a pause. + * @returns {Promise} + */ + public interpretRecordingInEditor = async ( + workflow: WorkflowFile, + page: Page, + updatePageOnPause: (page: Page) => void, + settings: InterpreterSettings, + ) => { + const params = settings.params ? settings.params : null; + delete settings.params; + const options = { + ...settings, + debugChannel: { + activeId: (id: any) => { + this.activeId = id; + this.socket.emit('activePairId', id); + }, + debugMessage: (msg: any) => { + this.debugMessages.push(`[${new Date().toLocaleString()}] ` + msg); + this.socket.emit('log', msg) + }, + }, + serializableCallback: (data: any) => { + this.socket.emit('serializableCallback', data); + }, + binaryCallback: (data: string, mimetype: string) => { + this.socket.emit('binaryCallback', { data, mimetype }); + } + } + + const interpreter = new Interpreter(workflow, options); + this.interpreter = interpreter; + + interpreter.on('flag', async (page, resume) => { + if (this.activeId !== null && this.breakpoints[this.activeId]) { + logger.log('debug', `breakpoint hit id: ${this.activeId}`); + this.socket.emit('breakpointHit'); + this.interpretationIsPaused = true; + } + + if (this.interpretationIsPaused) { + this.interpretationResume = resume; + logger.log('debug', `Paused inside of flag: ${page.url()}`); + updatePageOnPause(page); + this.socket.emit('log', '----- The interpretation has been paused -----', false); + } else { + resume(); + } + }); + + this.socket.emit('log', '----- Starting the interpretation -----', false); + + const status = await interpreter.run(page, params); + + this.socket.emit('log', `----- The interpretation finished with status: ${status} -----`, false); + + logger.log('debug', `Interpretation finished`); + this.interpreter = null; + this.socket.emit('activePairId', -1); + this.interpretationIsPaused = false; + this.interpretationResume = null; + this.socket.emit('finished'); + }; + + /** + * Stops the current process of the interpretation of the workflow. + * @returns {Promise} + */ + public stopInterpretation = async () => { + if (this.interpreter) { + logger.log('info', 'Stopping the interpretation.'); + await this.interpreter.stop(); + this.socket.emit('log', '----- The interpretation has been stopped -----', false); + this.clearState(); + } else { + logger.log('error', 'Cannot stop: No active interpretation.'); + } + }; + + private clearState = () => { + this.debugMessages = []; + this.interpretationIsPaused = false; + this.activeId = null; + this.interpreter = null; + this.breakpoints = []; + this.interpretationResume = null; + this.serializableData = []; + this.binaryData = []; + } + + /** + * Interprets the recording as a run. + * @param workflow The workflow to interpret. + * @param page The page instance used to interact with the browser. + * @param settings The settings to use for the interpretation. + */ + public InterpretRecording = async (workflow: WorkflowFile, page: Page, settings: InterpreterSettings) => { + const params = settings.params ? settings.params : null; + delete settings.params; + const options = { + ...settings, + debugChannel: { + activeId: (id: any) => { + this.activeId = id; + this.socket.emit('activePairId', id); + }, + debugMessage: (msg: any) => { + this.debugMessages.push(`[${new Date().toLocaleString()}] ` + msg); + this.socket.emit('debugMessage', msg) + }, + }, + serializableCallback: (data: string) => { + this.serializableData.push(data); + this.socket.emit('serializableCallback', data); + }, + binaryCallback: async (data: string, mimetype: string) => { + this.binaryData.push({ mimetype, data: JSON.stringify(data) }); + this.socket.emit('binaryCallback', { data, mimetype }); + } + } + + const interpreter = new Interpreter(workflow, options); + this.interpreter = interpreter; + + const status = await interpreter.run(page, params); + + const result = { + log: this.debugMessages, + result: status, + serializableOutput: this.serializableData.reduce((reducedObject, item, index) => { + return { + [`item-${index}`]: item, + ...reducedObject, + } + }, {}), + binaryOutput: this.binaryData.reduce((reducedObject, item, index) => { + return { + [`item-${index}`]: item, + ...reducedObject, + } + }, {}) + } + + logger.log('debug', `Interpretation finished`); + this.clearState(); + return result; + } + + /** + * Returns true if an interpretation is currently running. + * @returns {boolean} + */ + public interpretationInProgress = () => { + return this.interpreter !== null; + }; + + /** + * Updates the socket used for communication with the client (frontend). + * @param socket Socket.io socket instance enabling communication with the client (frontend) side. + * @returns void + */ + public updateSocket = (socket: Socket): void => { + this.socket = socket; + this.subscribeToPausing(); + }; +}