Merge pull request #5 from amhsirak/develop

feat: workflow interpreter

server/src/types/index.ts

@@ -11,6 +11,23 @@ export interface InterpreterSettings {
     params?: any;
 }
 
+/**
+ * Useful coordinates interface holding the x and y coordinates of a point.
+ * @category Types
+ */
+export interface Coordinates {
+    x: number;
+    y: number;
+}
+
+/**
+ * Holds the deltas of a wheel/scroll event.
+ * @category Types
+ */
+export interface ScrollDeltas {
+    deltaX: number;
+    deltaY: number;
+}
 
 /**
  * Options for the {@link BrowserManagement.launch} method.
@@ -25,3 +42,224 @@ export interface RemoteBrowserOptions {
     launchOptions: LaunchOptions
 };
 
+/**
+ * Pairs a pressed key value with the coordinates of the key press.
+ * @category Types
+ */
+export interface KeyboardInput {
+    key: string;
+    coordinates: Coordinates;
+}
+
+
+/**
+ * Contains index in the current workflow and result for over-shadowing check of a pair.
+ * @category Types
+ */
+export type PossibleOverShadow = {
+    index: number;
+    isOverShadowing: boolean;
+}
+/**
+ * An object representing he coordinates, width, height and corner points of the element.
+ * @category Types
+ */
+export interface Rectangle extends Coordinates {
+    width: number;
+    height: number;
+    top: number;
+    right: number;
+    bottom: number;
+    left: number;
+}
+
+/**
+ * Helpful enum used for determining the type of action currently executed by the user.
+ * @enum {string}
+ * @category Types
+ */
+export enum ActionType {
+    AwaitText = 'awaitText',
+    Click = 'click',
+    DragAndDrop = 'dragAndDrop',
+    Screenshot = 'screenshot',
+    Hover = 'hover',
+    Input = 'input',
+    Keydown = 'keydown',
+    Load = 'load',
+    Navigate = 'navigate',
+    Scroll = 'scroll',
+}
+
+/**
+ * Useful enum for determining the element's tag name.
+ * @enum {string}
+ * @category Types
+ */
+export enum TagName {
+    A = 'A',
+    B = 'B',
+    Cite = 'CITE',
+    EM = 'EM',
+    Input = 'INPUT',
+    Select = 'SELECT',
+    Span = 'SPAN',
+    Strong = 'STRONG',
+    TextArea = 'TEXTAREA',
+}
+
+/**
+ * @category Types
+ */
+export interface BaseActionInfo {
+    tagName: string;
+    /**
+     * If the element only has text content inside (hint to use text selector)
+     */
+    hasOnlyText: boolean;
+}
+
+/**
+ * Holds all the possible css selectors that has been found for an element.
+ * @category Types
+ */
+export interface Selectors {
+    id: string|null;
+    generalSelector: string|null;
+    attrSelector: string|null;
+    testIdSelector: string|null;
+    text: string|null;
+    href: string|null;
+    hrefSelector: string|null;
+    accessibilitySelector: string|null;
+    formSelector: string|null;
+}
+
+/**
+ * Base type for all actions.
+ * Action types are used to determine the best selector for the user action.
+ * They store valuable information, specific to the action.
+ * @category Types
+ */
+export interface BaseAction extends BaseActionInfo{
+    type: ActionType;
+    associatedActions: ActionType[];
+    inputType: string | undefined;
+    value: string | undefined;
+    selectors: { [key: string]: string | null };
+    timestamp: number;
+    isPassword: boolean;
+    /**
+     * Overrides the {@link BaseActionInfo} type of tagName for the action.
+     */
+    tagName: TagName;
+}
+
+/**
+ * Action type for pressing on a keyboard.
+ * @category Types
+ */
+interface KeydownAction extends BaseAction {
+    type: ActionType.Keydown;
+    key: string;
+}
+
+/**
+ * Action type for typing into an input field.
+ * @category Types
+ */
+interface InputAction extends BaseAction {
+    type: ActionType.Input;
+}
+
+/**
+ * Action type for clicking on an element.
+ * @category Types
+ */
+interface ClickAction extends BaseAction {
+    type: ActionType.Click;
+}
+
+/**
+ * Action type for drag and dropping an element.
+ * @category Types
+ */
+interface DragAndDropAction extends BaseAction {
+    type: ActionType.DragAndDrop;
+    sourceX: number;
+    sourceY: number;
+    targetX: number;
+    targetY: number;
+}
+
+/**
+ * Action type for hovering over an element.
+ * @category Types
+ */
+interface HoverAction extends BaseAction {
+    type: ActionType.Hover;
+}
+
+/**
+ * Action type for waiting on load.
+ * @category Types
+ */
+interface LoadAction extends BaseAction {
+    type: ActionType.Load;
+    url: string;
+}
+
+/**
+ * Action type for page navigation.
+ * @category Types
+ */
+interface NavigateAction extends BaseAction {
+    type: ActionType.Navigate;
+    url: string;
+    source: string;
+}
+
+/**
+ * Action type for scrolling.
+ * @category Types
+ */
+interface WheelAction extends BaseAction {
+    type: ActionType.Scroll;
+    deltaX: number;
+    deltaY: number;
+    pageXOffset: number;
+    pageYOffset: number;
+}
+
+/**
+ * Action type for taking a screenshot.
+ * @category Types
+ */
+interface FullScreenshotAction extends BaseAction {
+    type: ActionType.Screenshot;
+}
+
+/**
+ * Action type for waiting on the filling of text input.
+ * @category Types
+ */
+interface AwaitTextAction extends BaseAction {
+    type: ActionType.AwaitText;
+    text: string;
+}
+
+/**
+ * Definition of the Action type.
+ * @category Types
+ */
+export type Action =
+  | KeydownAction
+  | InputAction
+  | ClickAction
+  | DragAndDropAction
+  | HoverAction
+  | LoadAction
+  | NavigateAction
+  | WheelAction
+  | FullScreenshotAction
+  | AwaitTextAction;

server/src/workflow-management/classes/Interpreter.ts

@@ -0,0 +1,281 @@
+import Interpreter, { WorkflowFile } from "@wbr-project/wbr-interpret";
+import logger from "../../logger";
+import { Socket } from "socket.io";
+import { Page } from "playwright";
+import { InterpreterSettings } from "../../types";
+
+/**
+ * This class implements the main interpretation functions.
+ * It holds some information about the current interpretation process and
+ * registers to some events to allow the client (frontend) to interact with the interpreter.
+ * It uses the [@wbr-project/wbr-interpret](https://www.npmjs.com/package/@wbr-project/wbr-interpret)
+ * library to interpret the workflow.
+ * @category WorkflowManagement
+ */
+export class WorkflowInterpreter {
+  /**
+   * Socket.io socket instance enabling communication with the client (frontend) side.
+   * @private
+   */
+  private socket: Socket;
+
+  /**
+   * True if the interpretation is paused.
+   */
+  public interpretationIsPaused: boolean = false;
+
+  /**
+   * The instance of the {@link Interpreter} class used to interpret the workflow.
+   * From @wbr-project/wbr-interpret.
+   * @private
+   */
+  private interpreter: Interpreter | null = null;
+
+  /**
+   * An id of the currently interpreted pair in the workflow.
+   * @private
+   */
+  private activeId: number | null = null;
+
+  /**
+   * An array of debug messages emitted by the {@link Interpreter}.
+   */
+  public debugMessages: string[] = [];
+
+  /**
+   * An array of all the serializable data extracted from the run.
+   */
+  public serializableData: string[] = [];
+
+  /**
+   * An array of all the binary data extracted from the run.
+   */
+  public binaryData: { mimetype: string, data: string }[] = [];
+
+  /**
+   * An array of id's of the pairs from the workflow that are about to be paused.
+   * As "breakpoints".
+   * @private
+   */
+  private breakpoints: boolean[] = [];
+
+  /**
+   * Callback to resume the interpretation after a pause.
+   * @private
+   */
+  private interpretationResume: (() => void) | null = null;
+
+  /**
+   * A public constructor taking a socket instance for communication with the client.
+   * @param socket Socket.io socket instance enabling communication with the client (frontend) side.
+   * @constructor
+   */
+  constructor(socket: Socket) {
+    this.socket = socket;
+  }
+
+  /**
+   * Subscribes to the events that are used to control the interpretation.
+   * The events are pause, resume, step and breakpoints.
+   * Step is used to interpret a single pair and pause on the other matched pair.
+   * @returns void
+   */
+  public subscribeToPausing = () => {
+    this.socket.on('pause', () => {
+      this.interpretationIsPaused = true;
+    });
+    this.socket.on('resume', () => {
+      this.interpretationIsPaused = false;
+      if (this.interpretationResume) {
+        this.interpretationResume();
+        this.socket.emit('log', '----- The interpretation has been resumed -----', false);
+      } else {
+        logger.log('debug', "Resume called but no resume function is set");
+      }
+    });
+    this.socket.on('step', () => {
+      if (this.interpretationResume) {
+        this.interpretationResume();
+      } else {
+        logger.log('debug', "Step called but no resume function is set");
+      }
+    });
+    this.socket.on('breakpoints', (data: boolean[]) => {
+      logger.log('debug', "Setting breakpoints: " + data);
+      this.breakpoints = data
+    });
+  }
+
+  /**
+   * Sets up the instance of {@link Interpreter} and interprets
+   * the workflow inside the recording editor.
+   * Cleans up this interpreter instance after the interpretation is finished.
+   * @param workflow The workflow to interpret.
+   * @param page The page instance used to interact with the browser.
+   * @param updatePageOnPause A callback to update the page after a pause.
+   * @returns {Promise<void>}
+   */
+  public interpretRecordingInEditor = async (
+    workflow: WorkflowFile,
+    page: Page,
+    updatePageOnPause: (page: Page) => void,
+    settings: InterpreterSettings,
+  ) => {
+    const params = settings.params ? settings.params : null;
+    delete settings.params;
+    const options = {
+      ...settings,
+      debugChannel: {
+        activeId: (id: any) => {
+          this.activeId = id;
+          this.socket.emit('activePairId', id);
+        },
+        debugMessage: (msg: any) => {
+          this.debugMessages.push(`[${new Date().toLocaleString()}] ` + msg);
+          this.socket.emit('log', msg)
+        },
+      },
+      serializableCallback: (data: any) => {
+        this.socket.emit('serializableCallback', data);
+      },
+      binaryCallback: (data: string, mimetype: string) => {
+        this.socket.emit('binaryCallback', { data, mimetype });
+      }
+    }
+
+    const interpreter = new Interpreter(workflow, options);
+    this.interpreter = interpreter;
+
+    interpreter.on('flag', async (page, resume) => {
+      if (this.activeId !== null && this.breakpoints[this.activeId]) {
+        logger.log('debug', `breakpoint hit id: ${this.activeId}`);
+        this.socket.emit('breakpointHit');
+        this.interpretationIsPaused = true;
+      }
+
+      if (this.interpretationIsPaused) {
+        this.interpretationResume = resume;
+        logger.log('debug', `Paused inside of flag: ${page.url()}`);
+        updatePageOnPause(page);
+        this.socket.emit('log', '----- The interpretation has been paused -----', false);
+      } else {
+        resume();
+      }
+    });
+
+    this.socket.emit('log', '----- Starting the interpretation -----', false);
+
+    const status = await interpreter.run(page, params);
+
+    this.socket.emit('log', `----- The interpretation finished with status: ${status} -----`, false);
+
+    logger.log('debug', `Interpretation finished`);
+    this.interpreter = null;
+    this.socket.emit('activePairId', -1);
+    this.interpretationIsPaused = false;
+    this.interpretationResume = null;
+    this.socket.emit('finished');
+  };
+
+  /**
+   * Stops the current process of the interpretation of the workflow.
+   * @returns {Promise<void>}
+   */
+  public stopInterpretation = async () => {
+    if (this.interpreter) {
+      logger.log('info', 'Stopping the interpretation.');
+      await this.interpreter.stop();
+      this.socket.emit('log', '----- The interpretation has been stopped -----', false);
+      this.clearState();
+    } else {
+      logger.log('error', 'Cannot stop: No active interpretation.');
+    }
+  };
+
+  private clearState = () => {
+    this.debugMessages = [];
+    this.interpretationIsPaused = false;
+    this.activeId = null;
+    this.interpreter = null;
+    this.breakpoints = [];
+    this.interpretationResume = null;
+    this.serializableData = [];
+    this.binaryData = [];
+  }
+
+  /**
+   * Interprets the recording as a run.
+   * @param workflow The workflow to interpret.
+   * @param page The page instance used to interact with the browser.
+   * @param settings The settings to use for the interpretation.
+   */
+  public InterpretRecording = async (workflow: WorkflowFile, page: Page, settings: InterpreterSettings) => {
+    const params = settings.params ? settings.params : null;
+    delete settings.params;
+    const options = {
+      ...settings,
+      debugChannel: {
+        activeId: (id: any) => {
+          this.activeId = id;
+          this.socket.emit('activePairId', id);
+        },
+        debugMessage: (msg: any) => {
+          this.debugMessages.push(`[${new Date().toLocaleString()}] ` + msg);
+          this.socket.emit('debugMessage', msg)
+        },
+      },
+      serializableCallback: (data: string) => {
+        this.serializableData.push(data);
+        this.socket.emit('serializableCallback', data);
+      },
+      binaryCallback: async (data: string, mimetype: string) => {
+        this.binaryData.push({ mimetype, data: JSON.stringify(data) });
+        this.socket.emit('binaryCallback', { data, mimetype });
+      }
+    }
+
+    const interpreter = new Interpreter(workflow, options);
+    this.interpreter = interpreter;
+
+    const status = await interpreter.run(page, params);
+
+    const result = {
+      log: this.debugMessages,
+      result: status,
+      serializableOutput: this.serializableData.reduce((reducedObject, item, index) => {
+        return {
+          [`item-${index}`]: item,
+          ...reducedObject,
+        }
+      }, {}),
+      binaryOutput: this.binaryData.reduce((reducedObject, item, index) => {
+        return {
+          [`item-${index}`]: item,
+          ...reducedObject,
+        }
+      }, {})
+    }
+
+    logger.log('debug', `Interpretation finished`);
+    this.clearState();
+    return result;
+  }
+
+  /**
+   * Returns true if an interpretation is currently running.
+   * @returns {boolean}
+   */
+  public interpretationInProgress = () => {
+    return this.interpreter !== null;
+  };
+
+  /**
+   * Updates the socket used for communication with the client (frontend).
+   * @param socket Socket.io socket instance enabling communication with the client (frontend) side.
+   * @returns void
+   */
+  public updateSocket = (socket: Socket): void => {
+    this.socket = socket;
+    this.subscribeToPausing();
+  };
+}