vodorod/Dorod-Sky
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

remove screenshot when check user goal (#1147)

Browse Source
This commit is contained in:
LawyZheng
2024-11-06 23:20:45 +08:00
committed by GitHub
parent c084764373
commit b62c2caae0
3 changed files with 94 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

67
skyvern/forge/agent.py
View File

@@ -838,38 +838,26 @@ class ForgeAgent:
task_completes_on_download = task_block and task_block.complete_on_download and task.workflow_run_id
if not has_decisive_action and not task_completes_on_download:
LOG.info("Checking if user goal is achieved after re-scraping the page")
# Check if navigation goal is achieved after re-scraping the page
new_scraped_page = await self._scrape_with_type(
working_page = await browser_state.must_get_working_page()
complete_action = await self.check_user_goal_complete(
page=working_page,
scraped_page=scraped_page,
task=task,
step=step,
browser_state=browser_state,
scrape_type=ScrapeType.NORMAL,
organization=organization,
)
if new_scraped_page is None:
LOG.warning("Failed to scrape the page before checking user goal success, skipping check...")
else:
working_page = await browser_state.must_get_working_page()
complete_action = await self.check_user_goal_complete(
page=working_page,
scraped_page=new_scraped_page,
task=task,
step=step,
if complete_action is not None:
LOG.info("User goal achieved, executing complete action")
complete_action.organization_id = task.organization_id
complete_action.workflow_run_id = task.workflow_run_id
complete_action.task_id = task.task_id
complete_action.step_id = step.step_id
complete_action.step_order = step.order
complete_action.action_order = len(detailed_agent_step_output.actions_and_results)
complete_results = await ActionHandler.handle_action(
scraped_page, task, step, working_page, complete_action
)
if complete_action is not None:
LOG.info("User goal achieved, executing complete action")
complete_action.organization_id = task.organization_id
complete_action.workflow_run_id = task.workflow_run_id
complete_action.task_id = task.task_id
complete_action.step_id = step.step_id
complete_action.step_order = step.order
complete_action.action_order = len(detailed_agent_step_output.actions_and_results)
complete_results = await ActionHandler.handle_action(
scraped_page, task, step, working_page, complete_action
)
detailed_agent_step_output.actions_and_results.append((complete_action, complete_results))
await self.record_artifacts_after_action(task, step, browser_state)
detailed_agent_step_output.actions_and_results.append((complete_action, complete_results))
await self.record_artifacts_after_action(task, step, browser_state)
# If no action errors return the agent state and output
completed_step = await self.update_step(
step=step,
@@ -913,11 +901,19 @@ class ForgeAgent:
page: Page, scraped_page: ScrapedPage, task: Task, step: Step
) -> CompleteAction | None:
try:
LOG.info(
"Checking if user goal is achieved after re-scraping the page without screenshots",
task_id=task.task_id,
step_id=step.step_id,
workflow_run_id=task.workflow_run_id,
)
scraped_page_without_screenshots = await scraped_page.refresh(with_screenshot=False)
verification_prompt = prompt_engine.load_prompt(
"check-user-goal",
navigation_goal=task.navigation_goal,
navigation_payload=task.navigation_payload,
elements=scraped_page.build_element_tree(ElementTreeFormat.HTML),
elements=scraped_page_without_screenshots.build_element_tree(ElementTreeFormat.HTML),
)
# this prompt is critical to our agent so let's use the primary LLM API handler
@@ -926,6 +922,9 @@ class ForgeAgent:
LOG.error(
"Invalid LLM response for user goal success verification, skipping verification",
verification_response=verification_response,
task_id=task.task_id,
step_id=step.step_id,
workflow_run_id=task.workflow_run_id,
)
return None
@@ -940,7 +939,13 @@ class ForgeAgent:
)
except Exception:
LOG.error("LLM verification failed for complete action, skipping LLM verification", exc_info=True)
LOG.error(
"LLM verification failed for complete action, skipping LLM verification",
task_id=task.task_id,
step_id=step.step_id,
workflow_run_id=task.workflow_run_id,
exc_info=True,
)
return None
async def record_artifacts_after_action(self, task: Task, step: Step, browser_state: BrowserState) -> None:
@@ -1039,7 +1044,7 @@ class ForgeAgent:
browser_state: BrowserState,
scrape_type: ScrapeType,
organization: Organization | None = None,
) -> ScrapedPage | None:
) -> ScrapedPage:
if scrape_type == ScrapeType.NORMAL:
pass

4
skyvern/forge/app.py
View File

@@ -1,7 +1,6 @@
from typing import Awaitable, Callable
from fastapi import FastAPI
from playwright.async_api import Frame, Page
from skyvern.forge.agent import ForgeAgent
from skyvern.forge.agent_functions import AgentFunction
@@ -17,6 +16,7 @@ from skyvern.forge.sdk.settings_manager import SettingsManager
from skyvern.forge.sdk.workflow.context_manager import WorkflowContextManager
from skyvern.forge.sdk.workflow.service import WorkflowService
from skyvern.webeye.browser_manager import BrowserManager
from skyvern.webeye.scraper.scraper import ScrapeExcludeFunc
SETTINGS_MANAGER = SettingsManager.get_settings()
DATABASE = AgentDB(
@@ -37,7 +37,7 @@ SECONDARY_LLM_API_HANDLER = LLMAPIHandlerFactory.get_llm_api_handler(
WORKFLOW_CONTEXT_MANAGER = WorkflowContextManager()
WORKFLOW_SERVICE = WorkflowService()
AGENT_FUNCTION = AgentFunction()
scrape_exclude: Callable[[Page, Frame], Awaitable[bool]] | None = None
scrape_exclude: ScrapeExcludeFunc | None = None
authentication_function: Callable[[str], Awaitable[Organization]] | None = None
setup_api_app: Callable[[FastAPI], None] | None = None