Merge pull request #268 from getmaxun/handle-inputs

feat: add functionality to handle user inputs (part 1)

server/src/browser-management/inputHandlers.ts

@@ -6,7 +6,7 @@
 import { Socket } from 'socket.io';
 
 import logger from "../logger";
-import { Coordinates, ScrollDeltas, KeyboardInput } from '../types';
+import { Coordinates, ScrollDeltas, KeyboardInput, DatePickerEventData } from '../types';
 import { browserPool } from "../server";
 import { WorkflowGenerator } from "../workflow-management/classes/Generator";
 import { Page } from "playwright";
@@ -223,6 +223,43 @@ const handleKeydown = async (generator: WorkflowGenerator, page: Page, { key, co
     logger.log('debug', `Key ${key} pressed`);
 };
 
+/**
+ * Handles the date selection event.
+ * @param generator - the workflow generator {@link Generator}
+ * @param page - the active page of the remote browser
+ * @param data - the data of the date selection event {@link DatePickerEventData}
+ * @category BrowserManagement
+ */
+const handleDateSelection = async (generator: WorkflowGenerator, page: Page, data: DatePickerEventData) => {
+    await generator.onDateSelection(page, data);
+    logger.log('debug', `Date ${data.value} selected`);
+}
+
+const onDateSelection = async (data: DatePickerEventData) => {
+    logger.log('debug', 'Handling date selection event emitted from client');
+    await handleWrapper(handleDateSelection, data);
+}
+
+const handleDropdownSelection = async (generator: WorkflowGenerator, page: Page, data: { selector: string, value: string }) => {
+    await generator.onDropdownSelection(page, data);
+    logger.log('debug', `Dropdown value ${data.value} selected`);
+}
+
+const onDropdownSelection = async (data: { selector: string, value: string }) => {
+    logger.log('debug', 'Handling dropdown selection event emitted from client');
+    await handleWrapper(handleDropdownSelection, data);
+}
+
+const handleTimeSelection = async (generator: WorkflowGenerator, page: Page, data: { selector: string, value: string }) => {
+    await generator.onTimeSelection(page, data);
+    logger.log('debug', `Time value ${data.value} selected`);
+}
+
+const onTimeSelection = async (data: { selector: string, value: string }) => {
+    logger.log('debug', 'Handling time selection event emitted from client');
+    await handleWrapper(handleTimeSelection, data);
+}
+
 /**
  * A wrapper function for handling the keyup event.
  * @param keyboardInput - the keyboard input of the keyup event
@@ -378,6 +415,9 @@ const registerInputHandlers = (socket: Socket) => {
     socket.on("input:refresh", onRefresh);
     socket.on("input:back", onGoBack);
     socket.on("input:forward", onGoForward);
+    socket.on("input:date", onDateSelection);
+    socket.on("input:dropdown", onDropdownSelection);
+    socket.on("input:time", onTimeSelection);
     socket.on("action", onGenerateAction);
 };
 

server/src/types/index.ts

@@ -20,6 +20,16 @@ export interface Coordinates {
     y: number;
 }
 
+/**
+ * interface to handle date picker events.
+ * @category Types
+ */
+export interface DatePickerEventData {
+    coordinates: Coordinates;
+    selector: string;
+    value: string;
+}
+
 /**
  * Holds the deltas of a wheel/scroll event.
  * @category Types

server/src/workflow-management/classes/Generator.ts

@@ -1,4 +1,4 @@
-import { Action, ActionType, Coordinates, TagName } from "../../types";
+import { Action, ActionType, Coordinates, TagName, DatePickerEventData } from "../../types";
 import { WhereWhatPair, WorkflowFile } from 'maxun-core';
 import logger from "../../logger";
 import { Socket } from "socket.io";
@@ -255,6 +255,65 @@ export class WorkflowGenerator {
     logger.log('info', `Workflow emitted`);
   };
   
+  public onDateSelection = async (page: Page, data: DatePickerEventData) => {
+    const { selector, value } = data;
+
+    try {
+      await page.fill(selector, value);
+    } catch (error) {
+        console.error("Failed to fill date value:", error);
+    }
+    
+    const pair: WhereWhatPair = {
+        where: { url: this.getBestUrl(page.url()) },
+        what: [{
+            action: 'fill',
+            args: [selector, value],
+        }],
+    };
+
+    await this.addPairToWorkflowAndNotifyClient(pair, page);
+  };
+
+  public onDropdownSelection = async (page: Page, data: { selector: string, value: string }) => {
+    const { selector, value } = data;
+
+    try {
+      await page.selectOption(selector, value);
+    } catch (error) {
+        console.error("Failed to fill date value:", error);
+    }
+    
+    const pair: WhereWhatPair = {
+        where: { url: this.getBestUrl(page.url()) },
+        what: [{
+            action: 'selectOption',
+            args: [selector, value],
+        }],
+    };
+
+    await this.addPairToWorkflowAndNotifyClient(pair, page);
+  };
+
+  public onTimeSelection = async (page: Page, data: { selector: string, value: string }) => {
+    const { selector, value } = data;
+
+    try {
+        await page.fill(selector, value);
+    } catch (error) {
+        console.error("Failed to set time value:", error);
+    }
+    
+    const pair: WhereWhatPair = {
+        where: { url: this.getBestUrl(page.url()) },
+        what: [{
+            action: 'fill',
+            args: [selector, value],
+        }],
+    };
+
+    await this.addPairToWorkflowAndNotifyClient(pair, page);
+  };
 
   /**
    * Generates a pair for the click event.
@@ -266,6 +325,71 @@ export class WorkflowGenerator {
     let where: WhereWhatPair["where"] = { url: this.getBestUrl(page.url()) };
     const selector = await this.generateSelector(page, coordinates, ActionType.Click);
     logger.log('debug', `Element's selector: ${selector}`);
+
+    const elementInfo = await getElementInformation(page, coordinates, '', false);
+    console.log("Element info: ", elementInfo);
+
+    // Check if clicked element is a select dropdown
+    const isDropdown = elementInfo?.tagName === 'SELECT';
+    
+    if (isDropdown && elementInfo.innerHTML) {
+      // Parse options from innerHTML
+      const options = elementInfo.innerHTML
+        .split('<option')
+        .slice(1) // Remove first empty element
+        .map(optionHtml => {
+          const valueMatch = optionHtml.match(/value="([^"]*)"/);
+          const disabledMatch = optionHtml.includes('disabled="disabled"');
+          const selectedMatch = optionHtml.includes('selected="selected"');
+          
+          // Extract text content between > and </option>
+          const textMatch = optionHtml.match(/>([^<]*)</);
+          const text = textMatch 
+            ? textMatch[1]
+                .replace(/\n/g, '') // Remove all newlines
+                .replace(/\s+/g, ' ') // Replace multiple spaces with single space
+                .trim()
+            : '';
+          
+          return {
+              value: valueMatch ? valueMatch[1] : '',
+              text,
+              disabled: disabledMatch,
+              selected: selectedMatch
+          };
+        });
+
+      // Notify client to show dropdown overlay
+      this.socket.emit('showDropdown', {
+          coordinates,
+          selector,
+          options
+      });
+      return;
+    }
+
+    // Check if clicked element is a date input
+    const isDateInput = elementInfo?.tagName === 'INPUT' && elementInfo?.attributes?.type === 'date';
+
+    if (isDateInput) {
+      // Notify client to show datepicker overlay
+      this.socket.emit('showDatePicker', {
+          coordinates,
+          selector
+      });
+      return; 
+    }
+
+    const isTimeInput = elementInfo?.tagName === 'INPUT' && elementInfo?.attributes?.type === 'time';
+
+    if (isTimeInput) {
+      this.socket.emit('showTimePicker', {
+          coordinates,
+          selector
+      });
+      return;
+    }
+
     //const element = await getElementMouseIsOver(page, coordinates);
     //logger.log('debug', `Element: ${JSON.stringify(element, null, 2)}`);
     if (selector) {

src/components/atoms/DatePicker.tsx

@@ -0,0 +1,74 @@
+import React, { useState } from 'react';
+import { useSocketStore } from '../../context/socket';
+import { Coordinates } from './canvas';
+
+interface DatePickerProps {
+  coordinates: Coordinates;
+  selector: string;
+  onClose: () => void;
+}
+
+const DatePicker: React.FC<DatePickerProps> = ({ coordinates, selector, onClose }) => {
+  const { socket } = useSocketStore();
+  const [selectedDate, setSelectedDate] = useState<string>('');
+
+  const handleDateChange = (e: React.ChangeEvent<HTMLInputElement>) => {
+    setSelectedDate(e.target.value);
+  };
+
+  const handleConfirm = () => {
+    if (socket && selectedDate) {
+      socket.emit('input:date', {
+        selector,
+        value: selectedDate
+      });
+      onClose();
+    }
+  };
+
+  return (
+    <div 
+      style={{
+        position: 'absolute',
+        left: `${coordinates.x}px`,
+        top: `${coordinates.y}px`,
+        zIndex: 1000,
+        backgroundColor: 'white',
+        boxShadow: '0 2px 10px rgba(0,0,0,0.1)',
+        padding: '10px',
+        borderRadius: '4px'
+      }}
+    >
+      <div className="flex flex-col space-y-2">
+        <input
+          type="date"
+          onChange={handleDateChange}
+          value={selectedDate}
+          className="p-2 border rounded"
+          autoFocus
+        />
+        <div className="flex justify-end space-x-2">
+          <button 
+            onClick={onClose}
+            className="px-3 py-1 text-sm text-gray-600 hover:text-gray-800 border rounded"
+          >
+            Cancel
+          </button>
+          <button 
+            onClick={handleConfirm}
+            disabled={!selectedDate}
+            className={`px-3 py-1 text-sm rounded ${
+              selectedDate 
+                ? 'bg-blue-500 text-white hover:bg-blue-600' 
+                : 'bg-gray-300 text-gray-500 cursor-not-allowed'
+            }`}
+          >
+            Confirm
+          </button>
+        </div>
+      </div>
+    </div>
+  );
+};
+
+export default DatePicker;

src/components/atoms/Dropdown.tsx

@@ -0,0 +1,85 @@
+import React, { useState } from 'react';
+import { useSocketStore } from '../../context/socket';
+import { Coordinates } from './canvas';
+
+interface DropdownProps {
+    coordinates: Coordinates;
+    selector: string;
+    options: Array<{
+        value: string;
+        text: string;
+        disabled: boolean;
+        selected: boolean;
+    }>;
+    onClose: () => void;
+}
+
+const Dropdown = ({ coordinates, selector, options, onClose }: DropdownProps) => {
+    const { socket } = useSocketStore();
+    const [hoveredIndex, setHoveredIndex] = useState<number | null>(null);
+
+    const handleSelect = (value: string) => {
+        if (socket) {
+            socket.emit('input:dropdown', { selector, value });
+        }
+        onClose();
+    };
+
+    const containerStyle: React.CSSProperties = {
+        position: 'absolute',
+        left: coordinates.x,
+        top: coordinates.y,
+        zIndex: 1000,
+        width: '200px',
+        backgroundColor: 'white',
+        border: '1px solid rgb(169, 169, 169)',
+        boxShadow: '0 2px 4px rgba(0,0,0,0.15)',
+    };
+
+    const scrollContainerStyle: React.CSSProperties = {
+        maxHeight: '180px',
+        overflowY: 'auto',
+        overflowX: 'hidden',
+    };
+
+    const getOptionStyle = (option: any, index: number): React.CSSProperties => ({
+        fontSize: '13.333px',
+        lineHeight: '18px',
+        padding: '0 3px',
+        cursor: option.disabled ? 'default' : 'default',
+        backgroundColor: hoveredIndex === index ? '#0078D7' : 
+                        option.selected ? '#0078D7' : 
+                        option.disabled ? '#f8f8f8' : 'white',
+        color: (hoveredIndex === index || option.selected) ? 'white' : 
+               option.disabled ? '#a0a0a0' : 'black',
+        userSelect: 'none',
+    });
+
+    return (
+        <div 
+            className="fixed inset-0" 
+            onClick={onClose}
+        >
+            <div 
+                style={containerStyle}
+                onClick={e => e.stopPropagation()}
+            >
+                <div style={scrollContainerStyle}>
+                    {options.map((option, index) => (
+                        <div
+                            key={index}
+                            style={getOptionStyle(option, index)}
+                            onMouseEnter={() => !option.disabled && setHoveredIndex(index)}
+                            onMouseLeave={() => setHoveredIndex(null)}
+                            onClick={() => !option.disabled && handleSelect(option.value)}
+                        >
+                            {option.text}
+                        </div>
+                    ))}
+                </div>
+            </div>
+        </div>
+    );
+};
+
+export default Dropdown;

src/components/atoms/TimePicker.tsx

@@ -0,0 +1,130 @@
+import React, { useState } from 'react';
+import { useSocketStore } from '../../context/socket';
+import { Coordinates } from './canvas';
+
+interface TimePickerProps {
+    coordinates: Coordinates;
+    selector: string;
+    onClose: () => void;
+}
+
+const TimePicker = ({ coordinates, selector, onClose }: TimePickerProps) => {
+    const { socket } = useSocketStore();
+    const [hoveredHour, setHoveredHour] = useState<number | null>(null);
+    const [hoveredMinute, setHoveredMinute] = useState<number | null>(null);
+    const [selectedHour, setSelectedHour] = useState<number | null>(null);
+    const [selectedMinute, setSelectedMinute] = useState<number | null>(null);
+
+    const handleHourSelect = (hour: number) => {
+        setSelectedHour(hour);
+        // If minute is already selected, complete the selection
+        if (selectedMinute !== null) {
+            const formattedHour = hour.toString().padStart(2, '0');
+            const formattedMinute = selectedMinute.toString().padStart(2, '0');
+            if (socket) {
+                socket.emit('input:time', {
+                    selector,
+                    value: `${formattedHour}:${formattedMinute}`
+                });
+            }
+            onClose();
+        }
+    };
+
+    const handleMinuteSelect = (minute: number) => {
+        setSelectedMinute(minute);
+        // If hour is already selected, complete the selection
+        if (selectedHour !== null) {
+            const formattedHour = selectedHour.toString().padStart(2, '0');
+            const formattedMinute = minute.toString().padStart(2, '0');
+            if (socket) {
+                socket.emit('input:time', {
+                    selector,
+                    value: `${formattedHour}:${formattedMinute}`
+                });
+            }
+            onClose();
+        }
+    };
+
+    const containerStyle: React.CSSProperties = {
+        position: 'absolute',
+        left: coordinates.x,
+        top: coordinates.y,
+        zIndex: 1000,
+        display: 'flex',
+        backgroundColor: 'white',
+        border: '1px solid rgb(169, 169, 169)',
+        boxShadow: '0 2px 4px rgba(0,0,0,0.15)',
+    };
+
+    const columnStyle: React.CSSProperties = {
+        width: '60px',
+        maxHeight: '180px',
+        overflowY: 'auto',
+        overflowX: 'hidden',
+        borderRight: '1px solid rgb(169, 169, 169)',
+    };
+
+    const getOptionStyle = (value: number, isHour: boolean): React.CSSProperties => {
+        const isHovered = isHour ? hoveredHour === value : hoveredMinute === value;
+        const isSelected = isHour ? selectedHour === value : selectedMinute === value;
+        
+        return {
+            fontSize: '13.333px',
+            lineHeight: '18px',
+            padding: '0 3px',
+            cursor: 'default',
+            backgroundColor: isSelected ? '#0078D7' : isHovered ? '#0078D7' : 'white',
+            color: (isSelected || isHovered) ? 'white' : 'black',
+            userSelect: 'none',
+        };
+    };
+
+    const hours = Array.from({ length: 24 }, (_, i) => i);
+    const minutes = Array.from({ length: 60 }, (_, i) => i);
+
+    return (
+        <div 
+            className="fixed inset-0" 
+            onClick={onClose}
+        >
+            <div 
+                style={containerStyle}
+                onClick={e => e.stopPropagation()}
+            >
+                {/* Hours column */}
+                <div style={columnStyle}>
+                    {hours.map((hour) => (
+                        <div
+                            key={hour}
+                            style={getOptionStyle(hour, true)}
+                            onMouseEnter={() => setHoveredHour(hour)}
+                            onMouseLeave={() => setHoveredHour(null)}
+                            onClick={() => handleHourSelect(hour)}
+                        >
+                            {hour.toString().padStart(2, '0')}
+                        </div>
+                    ))}
+                </div>
+
+                {/* Minutes column */}
+                <div style={{...columnStyle, borderRight: 'none'}}>
+                    {minutes.map((minute) => (
+                        <div
+                            key={minute}
+                            style={getOptionStyle(minute, false)}
+                            onMouseEnter={() => setHoveredMinute(minute)}
+                            onMouseLeave={() => setHoveredMinute(null)}
+                            onClick={() => handleMinuteSelect(minute)}
+                        >
+                            {minute.toString().padStart(2, '0')}
+                        </div>
+                    ))}
+                </div>
+            </div>
+        </div>
+    );
+};
+
+export default TimePicker;

src/components/atoms/canvas.tsx

@@ -3,6 +3,9 @@ import { useSocketStore } from '../../context/socket';
 import { getMappedCoordinates } from "../../helpers/inputHelpers";
 import { useGlobalInfoStore } from "../../context/globalInfo";
 import { useActionContext } from '../../context/browserActions';
+import DatePicker from './DatePicker';
+import Dropdown from './Dropdown';
+import TimePicker from './TimePicker';
 
 interface CreateRefCallback {
     (ref: React.RefObject<HTMLCanvasElement>): void;
@@ -31,6 +34,27 @@ const Canvas = ({ width, height, onCreateRef }: CanvasProps) => {
     const getTextRef = useRef(getText);
     const getListRef = useRef(getList);
 
+    const [datePickerInfo, setDatePickerInfo] = React.useState<{
+        coordinates: Coordinates;
+        selector: string;
+    } | null>(null);
+
+    const [dropdownInfo, setDropdownInfo] = React.useState<{
+        coordinates: Coordinates;
+        selector: string;
+        options: Array<{
+            value: string;
+            text: string;
+            disabled: boolean;
+            selected: boolean;
+        }>;
+    } | null>(null);
+
+    const [timePickerInfo, setTimePickerInfo] = React.useState<{
+        coordinates: Coordinates;
+        selector: string;
+    } | null>(null);
+
     const notifyLastAction = (action: string) => {
         if (lastAction !== action) {
             setLastAction(action);
@@ -44,6 +68,36 @@ const Canvas = ({ width, height, onCreateRef }: CanvasProps) => {
         getListRef.current = getList;
     }, [getText, getList]);
 
+    useEffect(() => {
+        if (socket) {
+            socket.on('showDatePicker', (info: {coordinates: Coordinates, selector: string}) => {
+                setDatePickerInfo(info);
+            });
+
+            socket.on('showDropdown', (info: {
+                coordinates: Coordinates,
+                selector: string,
+                options: Array<{
+                    value: string;
+                    text: string;
+                    disabled: boolean;
+                    selected: boolean;
+                }>;
+            }) => {
+                setDropdownInfo(info);
+            });
+
+            socket.on('showTimePicker', (info: {coordinates: Coordinates, selector: string}) => {
+                setTimePickerInfo(info);
+            });
+
+            return () => {
+                socket.off('showDatePicker');
+                socket.off('showDropdown');
+            };
+        }
+    }, [socket]);
+
     const onMouseEvent = useCallback((event: MouseEvent) => {
         if (socket && canvasRef.current) {
             // Get the canvas bounding rectangle
@@ -146,6 +200,28 @@ const Canvas = ({ width, height, onCreateRef }: CanvasProps) => {
                 width={900}
                 style={{ display: 'block' }}
             />
+            {datePickerInfo && (
+                <DatePicker
+                    coordinates={datePickerInfo.coordinates}
+                    selector={datePickerInfo.selector}
+                    onClose={() => setDatePickerInfo(null)}
+                />
+            )}
+            {dropdownInfo && (
+                <Dropdown
+                    coordinates={dropdownInfo.coordinates}
+                    selector={dropdownInfo.selector}
+                    options={dropdownInfo.options}
+                    onClose={() => setDropdownInfo(null)}
+                />
+            )}
+            {timePickerInfo && (
+                <TimePicker
+                    coordinates={timePickerInfo.coordinates}
+                    selector={timePickerInfo.selector}
+                    onClose={() => setTimePickerInfo(null)}
+                />
+            )}
         </div>
     );