pass reasoning to error code gen (#3494)
This commit is contained in:
@@ -1,4 +1,4 @@
|
|||||||
You are here to help the user use the error codes and their descriptions to surface user-defined errors based on the screenshots, user goal, user details, action history and the HTML elements.
|
You are here to help the user use the error codes and their descriptions to surface user-defined errors based on the screenshots, user goal, user details, action history{{ ", context" if reasoning else "" }} and the HTML elements.
|
||||||
Do not return any error that's not defined by the user.
|
Do not return any error that's not defined by the user.
|
||||||
|
|
||||||
Reply in JSON format with the following keys:
|
Reply in JSON format with the following keys:
|
||||||
@@ -26,6 +26,11 @@ User details:
|
|||||||
```
|
```
|
||||||
{{ navigation_payload_str }}
|
{{ navigation_payload_str }}
|
||||||
```
|
```
|
||||||
|
{% if reasoning %}
|
||||||
|
Context:
|
||||||
|
```
|
||||||
|
{{ reasoning }}
|
||||||
|
```{% endif %}
|
||||||
|
|
||||||
Consider the action history and the screenshot together.
|
Consider the action history and the screenshot together.
|
||||||
Action history from previous steps: (note: even if the action history suggests goal is achieved, check the screenshot and the DOM elements to make sure the goal is achieved)
|
Action history from previous steps: (note: even if the action history suggests goal is achieved, check the screenshot and the DOM elements to make sure the goal is achieved)
|
||||||
|
|||||||
@@ -1791,7 +1791,9 @@ async def handle_terminate_action(
|
|||||||
step: Step,
|
step: Step,
|
||||||
) -> list[ActionResult]:
|
) -> list[ActionResult]:
|
||||||
if task.error_code_mapping:
|
if task.error_code_mapping:
|
||||||
action.errors = await extract_user_defined_errors(task=task, step=step, scraped_page=scraped_page)
|
action.errors = await extract_user_defined_errors(
|
||||||
|
task=task, step=step, scraped_page=scraped_page, reasoning=action.reasoning
|
||||||
|
)
|
||||||
return [ActionSuccess()]
|
return [ActionSuccess()]
|
||||||
|
|
||||||
|
|
||||||
@@ -1833,7 +1835,9 @@ async def handle_complete_action(
|
|||||||
action.verified = True
|
action.verified = True
|
||||||
|
|
||||||
if task.error_code_mapping:
|
if task.error_code_mapping:
|
||||||
action.errors = await extract_user_defined_errors(task=task, step=step, scraped_page=scraped_page)
|
action.errors = await extract_user_defined_errors(
|
||||||
|
task=task, step=step, scraped_page=scraped_page, reasoning=action.reasoning
|
||||||
|
)
|
||||||
|
|
||||||
if not task.data_extraction_goal and verification_result.thoughts:
|
if not task.data_extraction_goal and verification_result.thoughts:
|
||||||
await app.DATABASE.update_task(
|
await app.DATABASE.update_task(
|
||||||
@@ -3729,7 +3733,9 @@ async def _get_input_or_select_context(
|
|||||||
return input_or_select_context
|
return input_or_select_context
|
||||||
|
|
||||||
|
|
||||||
async def extract_user_defined_errors(task: Task, step: Step, scraped_page: ScrapedPage) -> list[UserDefinedError]:
|
async def extract_user_defined_errors(
|
||||||
|
task: Task, step: Step, scraped_page: ScrapedPage, reasoning: str | None = None
|
||||||
|
) -> list[UserDefinedError]:
|
||||||
action_history = await get_action_history(task=task, current_step=step)
|
action_history = await get_action_history(task=task, current_step=step)
|
||||||
scraped_page_refreshed = await scraped_page.refresh(draw_boxes=False)
|
scraped_page_refreshed = await scraped_page.refresh(draw_boxes=False)
|
||||||
prompt = prompt_engine.load_prompt(
|
prompt = prompt_engine.load_prompt(
|
||||||
@@ -3741,6 +3747,7 @@ async def extract_user_defined_errors(task: Task, step: Step, scraped_page: Scra
|
|||||||
action_history=json.dumps(action_history),
|
action_history=json.dumps(action_history),
|
||||||
error_code_mapping_str=json.dumps(task.error_code_mapping) if task.error_code_mapping else "{}",
|
error_code_mapping_str=json.dumps(task.error_code_mapping) if task.error_code_mapping else "{}",
|
||||||
local_datetime=datetime.now(skyvern_context.ensure_context().tz_info).isoformat(),
|
local_datetime=datetime.now(skyvern_context.ensure_context().tz_info).isoformat(),
|
||||||
|
reasoning=reasoning,
|
||||||
)
|
)
|
||||||
json_response = await app.EXTRACTION_LLM_API_HANDLER(
|
json_response = await app.EXTRACTION_LLM_API_HANDLER(
|
||||||
prompt=prompt,
|
prompt=prompt,
|
||||||
|
|||||||
Reference in New Issue
Block a user