diff --git a/server/src/browser-management/inputHandlers.ts b/server/src/browser-management/inputHandlers.ts index d6902b3f..982e18de 100644 --- a/server/src/browser-management/inputHandlers.ts +++ b/server/src/browser-management/inputHandlers.ts @@ -6,7 +6,7 @@ import { Socket } from 'socket.io'; import logger from "../logger"; -import { Coordinates, ScrollDeltas, KeyboardInput } from '../types'; +import { Coordinates, ScrollDeltas, KeyboardInput, DatePickerEventData } from '../types'; import { browserPool } from "../server"; import { WorkflowGenerator } from "../workflow-management/classes/Generator"; import { Page } from "playwright"; @@ -223,6 +223,43 @@ const handleKeydown = async (generator: WorkflowGenerator, page: Page, { key, co logger.log('debug', `Key ${key} pressed`); }; +/** + * Handles the date selection event. + * @param generator - the workflow generator {@link Generator} + * @param page - the active page of the remote browser + * @param data - the data of the date selection event {@link DatePickerEventData} + * @category BrowserManagement + */ +const handleDateSelection = async (generator: WorkflowGenerator, page: Page, data: DatePickerEventData) => { + await generator.onDateSelection(page, data); + logger.log('debug', `Date ${data.value} selected`); +} + +const onDateSelection = async (data: DatePickerEventData) => { + logger.log('debug', 'Handling date selection event emitted from client'); + await handleWrapper(handleDateSelection, data); +} + +const handleDropdownSelection = async (generator: WorkflowGenerator, page: Page, data: { selector: string, value: string }) => { + await generator.onDropdownSelection(page, data); + logger.log('debug', `Dropdown value ${data.value} selected`); +} + +const onDropdownSelection = async (data: { selector: string, value: string }) => { + logger.log('debug', 'Handling dropdown selection event emitted from client'); + await handleWrapper(handleDropdownSelection, data); +} + +const handleTimeSelection = async (generator: WorkflowGenerator, page: Page, data: { selector: string, value: string }) => { + await generator.onTimeSelection(page, data); + logger.log('debug', `Time value ${data.value} selected`); +} + +const onTimeSelection = async (data: { selector: string, value: string }) => { + logger.log('debug', 'Handling time selection event emitted from client'); + await handleWrapper(handleTimeSelection, data); +} + /** * A wrapper function for handling the keyup event. * @param keyboardInput - the keyboard input of the keyup event @@ -378,6 +415,9 @@ const registerInputHandlers = (socket: Socket) => { socket.on("input:refresh", onRefresh); socket.on("input:back", onGoBack); socket.on("input:forward", onGoForward); + socket.on("input:date", onDateSelection); + socket.on("input:dropdown", onDropdownSelection); + socket.on("input:time", onTimeSelection); socket.on("action", onGenerateAction); }; diff --git a/server/src/types/index.ts b/server/src/types/index.ts index 4fe761f1..f2e327ef 100644 --- a/server/src/types/index.ts +++ b/server/src/types/index.ts @@ -20,6 +20,16 @@ export interface Coordinates { y: number; } +/** + * interface to handle date picker events. + * @category Types + */ +export interface DatePickerEventData { + coordinates: Coordinates; + selector: string; + value: string; +} + /** * Holds the deltas of a wheel/scroll event. * @category Types diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index 213a0e86..9ff4922e 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -1,4 +1,4 @@ -import { Action, ActionType, Coordinates, TagName } from "../../types"; +import { Action, ActionType, Coordinates, TagName, DatePickerEventData } from "../../types"; import { WhereWhatPair, WorkflowFile } from 'maxun-core'; import logger from "../../logger"; import { Socket } from "socket.io"; @@ -255,6 +255,65 @@ export class WorkflowGenerator { logger.log('info', `Workflow emitted`); }; + public onDateSelection = async (page: Page, data: DatePickerEventData) => { + const { selector, value } = data; + + try { + await page.fill(selector, value); + } catch (error) { + console.error("Failed to fill date value:", error); + } + + const pair: WhereWhatPair = { + where: { url: this.getBestUrl(page.url()) }, + what: [{ + action: 'fill', + args: [selector, value], + }], + }; + + await this.addPairToWorkflowAndNotifyClient(pair, page); + }; + + public onDropdownSelection = async (page: Page, data: { selector: string, value: string }) => { + const { selector, value } = data; + + try { + await page.selectOption(selector, value); + } catch (error) { + console.error("Failed to fill date value:", error); + } + + const pair: WhereWhatPair = { + where: { url: this.getBestUrl(page.url()) }, + what: [{ + action: 'selectOption', + args: [selector, value], + }], + }; + + await this.addPairToWorkflowAndNotifyClient(pair, page); + }; + + public onTimeSelection = async (page: Page, data: { selector: string, value: string }) => { + const { selector, value } = data; + + try { + await page.fill(selector, value); + } catch (error) { + console.error("Failed to set time value:", error); + } + + const pair: WhereWhatPair = { + where: { url: this.getBestUrl(page.url()) }, + what: [{ + action: 'fill', + args: [selector, value], + }], + }; + + await this.addPairToWorkflowAndNotifyClient(pair, page); + }; /** * Generates a pair for the click event. @@ -266,6 +325,71 @@ export class WorkflowGenerator { let where: WhereWhatPair["where"] = { url: this.getBestUrl(page.url()) }; const selector = await this.generateSelector(page, coordinates, ActionType.Click); logger.log('debug', `Element's selector: ${selector}`); + + const elementInfo = await getElementInformation(page, coordinates, '', false); + console.log("Element info: ", elementInfo); + + // Check if clicked element is a select dropdown + const isDropdown = elementInfo?.tagName === 'SELECT'; + + if (isDropdown && elementInfo.innerHTML) { + // Parse options from innerHTML + const options = elementInfo.innerHTML + .split(' { + const valueMatch = optionHtml.match(/value="([^"]*)"/); + const disabledMatch = optionHtml.includes('disabled="disabled"'); + const selectedMatch = optionHtml.includes('selected="selected"'); + + // Extract text content between > and + const textMatch = optionHtml.match(/>([^<]*) void; +} + +const DatePicker: React.FC = ({ coordinates, selector, onClose }) => { + const { socket } = useSocketStore(); + const [selectedDate, setSelectedDate] = useState(''); + + const handleDateChange = (e: React.ChangeEvent) => { + setSelectedDate(e.target.value); + }; + + const handleConfirm = () => { + if (socket && selectedDate) { + socket.emit('input:date', { + selector, + value: selectedDate + }); + onClose(); + } + }; + + return ( +
+
+ +
+ + +
+
+
+ ); +}; + +export default DatePicker; \ No newline at end of file diff --git a/src/components/atoms/Dropdown.tsx b/src/components/atoms/Dropdown.tsx new file mode 100644 index 00000000..c7ead64b --- /dev/null +++ b/src/components/atoms/Dropdown.tsx @@ -0,0 +1,85 @@ +import React, { useState } from 'react'; +import { useSocketStore } from '../../context/socket'; +import { Coordinates } from './canvas'; + +interface DropdownProps { + coordinates: Coordinates; + selector: string; + options: Array<{ + value: string; + text: string; + disabled: boolean; + selected: boolean; + }>; + onClose: () => void; +} + +const Dropdown = ({ coordinates, selector, options, onClose }: DropdownProps) => { + const { socket } = useSocketStore(); + const [hoveredIndex, setHoveredIndex] = useState(null); + + const handleSelect = (value: string) => { + if (socket) { + socket.emit('input:dropdown', { selector, value }); + } + onClose(); + }; + + const containerStyle: React.CSSProperties = { + position: 'absolute', + left: coordinates.x, + top: coordinates.y, + zIndex: 1000, + width: '200px', + backgroundColor: 'white', + border: '1px solid rgb(169, 169, 169)', + boxShadow: '0 2px 4px rgba(0,0,0,0.15)', + }; + + const scrollContainerStyle: React.CSSProperties = { + maxHeight: '180px', + overflowY: 'auto', + overflowX: 'hidden', + }; + + const getOptionStyle = (option: any, index: number): React.CSSProperties => ({ + fontSize: '13.333px', + lineHeight: '18px', + padding: '0 3px', + cursor: option.disabled ? 'default' : 'default', + backgroundColor: hoveredIndex === index ? '#0078D7' : + option.selected ? '#0078D7' : + option.disabled ? '#f8f8f8' : 'white', + color: (hoveredIndex === index || option.selected) ? 'white' : + option.disabled ? '#a0a0a0' : 'black', + userSelect: 'none', + }); + + return ( +
+
e.stopPropagation()} + > +
+ {options.map((option, index) => ( +
!option.disabled && setHoveredIndex(index)} + onMouseLeave={() => setHoveredIndex(null)} + onClick={() => !option.disabled && handleSelect(option.value)} + > + {option.text} +
+ ))} +
+
+
+ ); +}; + +export default Dropdown; \ No newline at end of file diff --git a/src/components/atoms/TimePicker.tsx b/src/components/atoms/TimePicker.tsx new file mode 100644 index 00000000..31353c7a --- /dev/null +++ b/src/components/atoms/TimePicker.tsx @@ -0,0 +1,130 @@ +import React, { useState } from 'react'; +import { useSocketStore } from '../../context/socket'; +import { Coordinates } from './canvas'; + +interface TimePickerProps { + coordinates: Coordinates; + selector: string; + onClose: () => void; +} + +const TimePicker = ({ coordinates, selector, onClose }: TimePickerProps) => { + const { socket } = useSocketStore(); + const [hoveredHour, setHoveredHour] = useState(null); + const [hoveredMinute, setHoveredMinute] = useState(null); + const [selectedHour, setSelectedHour] = useState(null); + const [selectedMinute, setSelectedMinute] = useState(null); + + const handleHourSelect = (hour: number) => { + setSelectedHour(hour); + // If minute is already selected, complete the selection + if (selectedMinute !== null) { + const formattedHour = hour.toString().padStart(2, '0'); + const formattedMinute = selectedMinute.toString().padStart(2, '0'); + if (socket) { + socket.emit('input:time', { + selector, + value: `${formattedHour}:${formattedMinute}` + }); + } + onClose(); + } + }; + + const handleMinuteSelect = (minute: number) => { + setSelectedMinute(minute); + // If hour is already selected, complete the selection + if (selectedHour !== null) { + const formattedHour = selectedHour.toString().padStart(2, '0'); + const formattedMinute = minute.toString().padStart(2, '0'); + if (socket) { + socket.emit('input:time', { + selector, + value: `${formattedHour}:${formattedMinute}` + }); + } + onClose(); + } + }; + + const containerStyle: React.CSSProperties = { + position: 'absolute', + left: coordinates.x, + top: coordinates.y, + zIndex: 1000, + display: 'flex', + backgroundColor: 'white', + border: '1px solid rgb(169, 169, 169)', + boxShadow: '0 2px 4px rgba(0,0,0,0.15)', + }; + + const columnStyle: React.CSSProperties = { + width: '60px', + maxHeight: '180px', + overflowY: 'auto', + overflowX: 'hidden', + borderRight: '1px solid rgb(169, 169, 169)', + }; + + const getOptionStyle = (value: number, isHour: boolean): React.CSSProperties => { + const isHovered = isHour ? hoveredHour === value : hoveredMinute === value; + const isSelected = isHour ? selectedHour === value : selectedMinute === value; + + return { + fontSize: '13.333px', + lineHeight: '18px', + padding: '0 3px', + cursor: 'default', + backgroundColor: isSelected ? '#0078D7' : isHovered ? '#0078D7' : 'white', + color: (isSelected || isHovered) ? 'white' : 'black', + userSelect: 'none', + }; + }; + + const hours = Array.from({ length: 24 }, (_, i) => i); + const minutes = Array.from({ length: 60 }, (_, i) => i); + + return ( +
+
e.stopPropagation()} + > + {/* Hours column */} +
+ {hours.map((hour) => ( +
setHoveredHour(hour)} + onMouseLeave={() => setHoveredHour(null)} + onClick={() => handleHourSelect(hour)} + > + {hour.toString().padStart(2, '0')} +
+ ))} +
+ + {/* Minutes column */} +
+ {minutes.map((minute) => ( +
setHoveredMinute(minute)} + onMouseLeave={() => setHoveredMinute(null)} + onClick={() => handleMinuteSelect(minute)} + > + {minute.toString().padStart(2, '0')} +
+ ))} +
+
+
+ ); +}; + +export default TimePicker; \ No newline at end of file diff --git a/src/components/atoms/canvas.tsx b/src/components/atoms/canvas.tsx index 1dd88e19..77128a65 100644 --- a/src/components/atoms/canvas.tsx +++ b/src/components/atoms/canvas.tsx @@ -3,6 +3,9 @@ import { useSocketStore } from '../../context/socket'; import { getMappedCoordinates } from "../../helpers/inputHelpers"; import { useGlobalInfoStore } from "../../context/globalInfo"; import { useActionContext } from '../../context/browserActions'; +import DatePicker from './DatePicker'; +import Dropdown from './Dropdown'; +import TimePicker from './TimePicker'; interface CreateRefCallback { (ref: React.RefObject): void; @@ -31,6 +34,27 @@ const Canvas = ({ width, height, onCreateRef }: CanvasProps) => { const getTextRef = useRef(getText); const getListRef = useRef(getList); + const [datePickerInfo, setDatePickerInfo] = React.useState<{ + coordinates: Coordinates; + selector: string; + } | null>(null); + + const [dropdownInfo, setDropdownInfo] = React.useState<{ + coordinates: Coordinates; + selector: string; + options: Array<{ + value: string; + text: string; + disabled: boolean; + selected: boolean; + }>; + } | null>(null); + + const [timePickerInfo, setTimePickerInfo] = React.useState<{ + coordinates: Coordinates; + selector: string; + } | null>(null); + const notifyLastAction = (action: string) => { if (lastAction !== action) { setLastAction(action); @@ -44,6 +68,36 @@ const Canvas = ({ width, height, onCreateRef }: CanvasProps) => { getListRef.current = getList; }, [getText, getList]); + useEffect(() => { + if (socket) { + socket.on('showDatePicker', (info: {coordinates: Coordinates, selector: string}) => { + setDatePickerInfo(info); + }); + + socket.on('showDropdown', (info: { + coordinates: Coordinates, + selector: string, + options: Array<{ + value: string; + text: string; + disabled: boolean; + selected: boolean; + }>; + }) => { + setDropdownInfo(info); + }); + + socket.on('showTimePicker', (info: {coordinates: Coordinates, selector: string}) => { + setTimePickerInfo(info); + }); + + return () => { + socket.off('showDatePicker'); + socket.off('showDropdown'); + }; + } + }, [socket]); + const onMouseEvent = useCallback((event: MouseEvent) => { if (socket && canvasRef.current) { // Get the canvas bounding rectangle @@ -146,6 +200,28 @@ const Canvas = ({ width, height, onCreateRef }: CanvasProps) => { width={900} style={{ display: 'block' }} /> + {datePickerInfo && ( + setDatePickerInfo(null)} + /> + )} + {dropdownInfo && ( + setDropdownInfo(null)} + /> + )} + {timePickerInfo && ( + setTimePickerInfo(null)} + /> + )} );