vodorod/Dorod-Sky
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

feat: add hover action support (#3994)

Browse Source 
Co-authored-by: LawyZheng <lawyzheng1106@gmail.com>
This commit is contained in:
Mohamed Khalil
2025-12-09 17:27:26 +02:00
committed by GitHub
parent 0e8d667959
commit f49b07f30d
22 changed files with 281 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
skyvern/forge/agent.py
View File

@@ -2284,6 +2284,33 @@ class ForgeAgent:
# If we don't have pre-scraped data, scrape normally
if scraped_page is None:
# Scroll back to last hovered element position BEFORE scraping
# This ensures the element is visible in the screenshot taken during scraping
if context and context.last_hovered_element_page_y is not None:
try:
working_page = await browser_state.must_get_working_page()
# Get viewport height to center the element
viewport_height = await working_page.evaluate("window.innerHeight")
# Calculate scroll position to center the element in viewport
target_scroll_y = context.last_hovered_element_page_y - (viewport_height / 2)
target_scroll_y = max(0, target_scroll_y) # Don't scroll to negative
LOG.info(
"Scrolling back to last hovered element position before scraping",
element_id=context.last_hovered_element_id,
absolute_page_y=context.last_hovered_element_page_y,
target_scroll_y=target_scroll_y,
)
await working_page.evaluate(f"window.scrollTo(0, {target_scroll_y})")
# Small delay to let the scroll settle
await asyncio.sleep(0.3)
# Clear the saved position after scrolling back
context.last_hovered_element_page_y = None
context.last_hovered_element_id = None
except Exception:
LOG.warning("Failed to scroll back to hovered element position before scraping", exc_info=True)
# Check PostHog for speed optimizations BEFORE scraping
# This decision will be used in both:
# 1. SVG conversion skip (in agent_functions.py cleanup)

5
skyvern/forge/prompts/skyvern/extract-action-static.j2
View File

@@ -3,9 +3,10 @@ Include only the elements that are relevant to the user goal, without altering o
Accurately interpret and understand the functional significance of SVG elements based on their shapes and context within the webpage.
Use the user details to fill in necessary values. Always satisfy required fields if the field isn't already filled in. Don't return any action for the same field, if this field is already filled in and the value is the same as the one you would have filled in.
MAKE SURE YOU OUTPUT VALID JSON. No text before or after JSON, no trailing commas, no comments (//), no unnecessary quotes, etc.
Each interactable element is tagged with an ID. Avoid taking action on a disabled element when there is an alternative action available.
Each interactable element is tagged with an ID. Avoid taking action on a disabled element when there is an alternative action available. For element-based actions (CLICK, HOVER, INPUT_TEXT, UPLOAD_FILE, SELECT_OPTION), the `id` you output MUST exactly match one of the IDs from the provided elements list—never invent IDs (e.g., `INSTER_CARD_WRAPPER`). If the user refers to an element that isn't listed, choose the closest matching real element ID or explain why no suitable element exists. Actions that do not operate on a specific element (e.g., WAIT, SOLVE_CAPTCHA, RELOAD_PAGE, OPEN_TAB) should not provide an `id`.
If you see any information in red in the page screenshot, this means a condition wasn't satisfied. prioritize actions with the red information.
If you see a popup in the page screenshot, prioritize actions on the popup.
If the UI hints that a control (like a CTA button) only appears after hovering a card, tile, or model name, always return a HOVER action on the element that needs hovering and explicitly wait for an on-hover UI update before attempting the next action. Use a WAIT action after the hover if the new element needs extra time to appear.
Reply in JSON format with the following keys:
{
@@ -18,7 +19,7 @@ Reply in JSON format with the following keys:
"user_detail_query": str, // Think of this value as a Jeopardy question and the intention behind the action. Ask the user for the details you need for executing this action. Ask the question even if the details are disclosed in user goal or user details. If it's a text field, ask for the text. If it's a file upload, ask for the file. If it's a dropdown, ask for the relevant information. If you are clicking on something specific, ask about what the intention is behind the click and what to click on. If you're downloading a file and you have multiple options, ask the user which one to download. Examples are: "What product ID should I input into the search bar?", "What file should I upload?", "What is the previous insurance provider of the user?", "Which invoice should I download?", "Does the user have any pets?". If the action doesn't require any user details, describe the intention behind the action.
"user_detail_answer": str, // The answer to the `user_detail_query`. The source of this answer can be user goal or user details.
"confidence_float": float, // The confidence of the action. Pick a number between 0.0 and 1.0. 0.0 means no confidence, 1.0 means full confidence
"action_type": str, // It's a string enum: "CLICK", "INPUT_TEXT", "UPLOAD_FILE", "SELECT_OPTION", "WAIT", "SOLVE_CAPTCHA", "COMPLETE", "TERMINATE"{{', "CLOSE_PAGE"' if has_magic_link_page else ""}}. "CLICK" is an element you'd like to click. "INPUT_TEXT" is an element you'd like to input text into. "UPLOAD_FILE" is an element you'd like to upload a file into. "SELECT_OPTION" is an element you'd like to select an option from. "WAIT" action should be used if there are no actions to take and there is some indication on screen that waiting could yield more actions. "WAIT" should not be used if there are actions to take. "SOLVE_CAPTCHA" should be used if there's a captcha to solve on the screen. "COMPLETE" is used when the {{ "complete criterion has been met" if complete_criterion else "user goal has been achieved" }} AND if there's any data extraction goal, you should be able to get data from the page. Never return a COMPLETE action unless the {{ "complete criterion is met" if complete_criterion else "user goal is achieved" }}. "TERMINATE" is used to terminate the whole task with a failure when it doesn't seem like the user goal can be achieved. Do not use "TERMINATE" if waiting could lead the user towards the goal. Only return "TERMINATE" if you are on a page where the user goal cannot be achieved. All other actions are ignored when "TERMINATE" is returned.{{' "CLOSE_PAGE" is used to close the current page when it is impossible to achieve the user goal on the current page.' if has_magic_link_page else ''}}
"action_type": str, // It's a string enum: "CLICK", "HOVER", "INPUT_TEXT", "UPLOAD_FILE", "SELECT_OPTION", "WAIT", "SOLVE_CAPTCHA", "COMPLETE", "TERMINATE"{{', "CLOSE_PAGE"' if has_magic_link_page else ""}}. "CLICK" is an element you'd like to click. "HOVER" is used to move the mouse over an element (for example to reveal hover-only menus) without clicking. "INPUT_TEXT" is an element you'd like to input text into. "UPLOAD_FILE" is an element you'd like to upload a file into. "SELECT_OPTION" is an element you'd like to select an option from. "WAIT" action should be used only if there are no actions to take and there is some indication on screen that waiting could yield more actions. "WAIT" should not be used if there are actions to take such as hovering or clicking. "SOLVE_CAPTCHA" should be used if there's a captcha to solve on the screen. "COMPLETE" is used when the user goal has been achieved AND if there's any data extraction goal, you should be able to get data from the page. Never return a COMPLETE action unless the user goal is achieved. "TERMINATE" is used to terminate the whole task with a failure when it doesn't seem like the user goal can be achieved. Do not use "TERMINATE" if waiting could lead the user towards the goal. Only return "TERMINATE" if you are on a page where the user goal cannot be achieved. All other actions are ignored when "TERMINATE" is returned.{{' "CLOSE_PAGE" is used to close the current page when it is impossible to achieve the user goal on the current page.' if has_magic_link_page else ''}}
"id": str, // The id of the element to take action on. The id has to be one from the elements list
"text": str, // Text for INPUT_TEXT action only
"file_url": str, // The url of the file to upload if applicable. This field must be present for UPLOAD_FILE but can also be present for CLICK only if the click is to upload the file. It should be null otherwise.

5
skyvern/forge/prompts/skyvern/extract-action.j2
View File

@@ -3,9 +3,10 @@ Include only the elements that are relevant to the user goal, without altering o
Accurately interpret and understand the functional significance of SVG elements based on their shapes and context within the webpage.
Use the user details to fill in necessary values. Always satisfy required fields if the field isn't already filled in. Don't return any action for the same field, if this field is already filled in and the value is the same as the one you would have filled in.
MAKE SURE YOU OUTPUT VALID JSON. No text before or after JSON, no trailing commas, no comments (//), no unnecessary quotes, etc.
Each interactable element is tagged with an ID. Avoid taking action on a disabled element when there is an alternative action available.
Each interactable element is tagged with an ID. Avoid taking action on a disabled element when there is an alternative action available. The `id` you output MUST exactly match one of the IDs from the provided elements list—never invent IDs. If the user refers to an element that isn't listed, choose the closest matching real element ID or explain why no suitable element exists.
If you see any information in red in the page screenshot, this means a condition wasn't satisfied. prioritize actions with the red information.
If you see a popup in the page screenshot, prioritize actions on the popup.
If the UI hints that a control (like a CTA button) only appears after hovering a card, tile, or model name, always return a HOVER action on the element that needs hovering and explicitly wait for an on-hover UI update before attempting the next action. Use a WAIT action after the hover if the new element needs extra time to appear.
Reply in JSON format with the following keys:
{
@@ -18,7 +19,7 @@ Reply in JSON format with the following keys:
"user_detail_query": str, // Think of this value as a Jeopardy question and the intention behind the action. Ask the user for the details you need for executing this action. Ask the question even if the details are disclosed in user goal or user details. If it's a text field, ask for the text. If it's a file upload, ask for the file. If it's a dropdown, ask for the relevant information. If you are clicking on something specific, ask about what the intention is behind the click and what to click on. If you're downloading a file and you have multiple options, ask the user which one to download. Examples are: "What product ID should I input into the search bar?", "What file should I upload?", "What is the previous insurance provider of the user?", "Which invoice should I download?", "Does the user have any pets?". If the action doesn't require any user details, describe the intention behind the action.
"user_detail_answer": str, // The answer to the `user_detail_query`. The source of this answer can be user goal or user details.
"confidence_float": float, // The confidence of the action. Pick a number between 0.0 and 1.0. 0.0 means no confidence, 1.0 means full confidence
"action_type": str, // It's a string enum: "CLICK", "INPUT_TEXT", "UPLOAD_FILE", "SELECT_OPTION", "WAIT", "SOLVE_CAPTCHA", "COMPLETE", "TERMINATE"{{', "CLOSE_PAGE"' if has_magic_link_page else ""}}. "CLICK" is an element you'd like to click. "INPUT_TEXT" is an element you'd like to input text into. "UPLOAD_FILE" is an element you'd like to upload a file into. "SELECT_OPTION" is an element you'd like to select an option from. "WAIT" action should be used if there are no actions to take and there is some indication on screen that waiting could yield more actions. "WAIT" should not be used if there are actions to take. "SOLVE_CAPTCHA" should be used if there's a captcha to solve on the screen. "COMPLETE" is used when the {{"complete criterion has been met" if complete_criterion else "user goal has been achieved"}} AND if there's any data extraction goal, you should be able to get data from the page. Never return a COMPLETE action unless the {{ "complete criterion is met" if complete_criterion else "user goal is achieved" }}. "TERMINATE" is used to terminate the whole task with a failure when it doesn't seem like the user goal can be achieved. Do not use "TERMINATE" if waiting could lead the user towards the goal. Only return "TERMINATE" if you are on a page where the user goal cannot be achieved. All other actions are ignored when "TERMINATE" is returned.{{' "CLOSE_PAGE" is used to close the current page when it is impossible to achieve the user goal on the current page.' if has_magic_link_page else ''}}
"action_type": str, // It's a string enum: "CLICK", "HOVER", "INPUT_TEXT", "UPLOAD_FILE", "SELECT_OPTION", "WAIT", "SOLVE_CAPTCHA", "COMPLETE", "TERMINATE"{{', "CLOSE_PAGE"' if has_magic_link_page else ""}}. "CLICK" is an element you'd like to click. "HOVER" is used to move the mouse over an element (for example, to reveal hover-only menus or buttons before clicking). "INPUT_TEXT" is an element you'd like to input text into. "UPLOAD_FILE" is an element you'd like to upload a file into. "SELECT_OPTION" is an element you'd like to select an option from. "WAIT" action should be used only if there are no actions to take and there is some indication on screen that waiting could yield more actions. "WAIT" should not be used if there are actions to take such as hovering or clicking. "SOLVE_CAPTCHA" should be used if there's a captcha to solve on the screen. "COMPLETE" is used when the {{"complete criterion has been met" if complete_criterion else "user goal has been achieved"}} AND if there's any data extraction goal, you should be able to get data from the page. Never return a COMPLETE action unless the {{ "complete criterion is met" if complete_criterion else "user goal is achieved" }}. "TERMINATE" is used to terminate the whole task with a failure when it doesn't seem like the user goal can be achieved. Do not use "TERMINATE" if waiting could lead the user towards the goal. Only return "TERMINATE" if you are on a page where the user goal cannot be achieved. All other actions are ignored when "TERMINATE" is returned.{{' "CLOSE_PAGE" is used to close the current page when it is impossible to achieve the user goal on the current page.' if has_magic_link_page else ''}}
"id": str, // The id of the element to take action on. The id has to be one from the elements list
"text": str, // Text for INPUT_TEXT action only
"file_url": str, // The url of the file to upload if applicable. This field must be present for UPLOAD_FILE but can also be present for CLICK only if the click is to upload the file. It should be null otherwise.

5
skyvern/forge/sdk/core/skyvern_context.py
View File

@@ -67,6 +67,11 @@ class SkyvernContext:
next_step_pre_scraped_data: dict[str, Any] | None = None
speculative_plans: dict[str, Any] = field(default_factory=dict)
# Store absolute page position of last hovered element to scroll back after re-scraping
# This is the Y position relative to the entire page (not viewport)
last_hovered_element_page_y: float | None = None
last_hovered_element_id: str | None = None
"""
Example output value:
{"loop_value": "str", "output_parameter": "the key of the parameter", "output_value": Any}

2
skyvern/forge/sdk/db/utils.py
View File

@@ -70,6 +70,7 @@ from skyvern.webeye.actions.actions import (
DragAction,
ExtractAction,
GotoUrlAction,
HoverAction,
InputTextAction,
KeypressAction,
LeftMouseAction,
@@ -138,6 +139,7 @@ ACTION_TYPE_TO_CLASS = {
ActionType.SELECT_OPTION: SelectOptionAction,
ActionType.CHECKBOX: CheckboxAction,
ActionType.WAIT: WaitAction,
ActionType.HOVER: HoverAction,
ActionType.SOLVE_CAPTCHA: SolveCaptchaAction,
ActionType.RELOAD_PAGE: ReloadPageAction,
ActionType.EXTRACT: ExtractAction,