add CUA MoveAction (#2144)

skyvern/webeye/actions/actions.py

@@ -31,6 +31,7 @@ class ActionType(StrEnum):
     SCROLL = "scroll"
     KEYPRESS = "keypress"
     TYPE = "type"
+    MOVE = "move"
 
     def is_web_action(self) -> bool:
         return self in [
@@ -278,6 +279,12 @@ class KeypressAction(Action):
     keys: list[str] = []
 
 
+class MoveAction(Action):
+    action_type: ActionType = ActionType.MOVE
+    x: int
+    y: int
+
+
 class ScrapeResult(BaseModel):
     """
     Scraped response from a webpage, including:

skyvern/webeye/actions/handler.py

@@ -1493,6 +1493,17 @@ async def handle_keypress_action(
     return [ActionSuccess()]
 
 
+async def handle_move_action(
+    action: actions.MoveAction,
+    page: Page,
+    scraped_page: ScrapedPage,
+    task: Task,
+    step: Step,
+) -> list[ActionResult]:
+    await page.mouse.move(action.x, action.y)
+    return [ActionSuccess()]
+
+
 ActionHandler.register_action_type(ActionType.SOLVE_CAPTCHA, handle_solve_captcha_action)
 ActionHandler.register_action_type(ActionType.CLICK, handle_click_action)
 ActionHandler.register_action_type(ActionType.INPUT_TEXT, handle_input_text_action)
@@ -1506,6 +1517,7 @@ ActionHandler.register_action_type(ActionType.COMPLETE, handle_complete_action)
 ActionHandler.register_action_type(ActionType.EXTRACT, handle_extract_action)
 ActionHandler.register_action_type(ActionType.SCROLL, handle_scroll_action)
 ActionHandler.register_action_type(ActionType.KEYPRESS, handle_keypress_action)
+ActionHandler.register_action_type(ActionType.MOVE, handle_move_action)
 
 
 async def get_actual_value_of_parameter_if_secret(task: Task, parameter: str) -> Any:

skyvern/webeye/actions/parse_actions.py

@@ -18,6 +18,7 @@ from skyvern.webeye.actions.actions import (
     DownloadFileAction,
     InputTextAction,
     KeypressAction,
+    MoveAction,
     NullAction,
     ScrollAction,
     SelectOption,
@@ -281,6 +282,13 @@ async def parse_cua_actions(
                         reasoning=reasoning,
                         intention=reasoning,
                     )
+                case "move":
+                    action = MoveAction(
+                        x=cua_action.x,
+                        y=cua_action.y,
+                        reasoning=reasoning,
+                        intention=reasoning,
+                    )
                 case _:
                     raise ValueError(f"Unsupported action type: {action_type}")
             action.organization_id = task.organization_id