vodorod/Dorod-Sky
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

email relay (#598)

Browse Source 
Co-authored-by: Shuchang Zheng <wintonzheng0325@gmail.com>
This commit is contained in:
Kerem Yilmaz
2024-07-11 21:34:00 -07:00
committed by GitHub
parent f6bb4981fc
commit ea1039277f
15 changed files with 191 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
skyvern/forge/agent.py
View File

@@ -11,7 +11,12 @@ from playwright._impl._errors import TargetClosedError
from playwright.async_api import Page
from skyvern import analytics
from skyvern.constants import SCRAPE_TYPE_ORDER, ScrapeType
from skyvern.constants import (
SCRAPE_TYPE_ORDER,
SPECIAL_FIELD_VERIFICATION_CODE,
VERIFICATION_CODE_PLACEHOLDER,
ScrapeType,
)
from skyvern.exceptions import (
BrowserStateMissingPage,
EmptyScrapePage,
@@ -120,6 +125,7 @@ class ForgeAgent:
url=task_url,
title=task_block.title,
webhook_callback_url=None,
totp_verification_url=None,
navigation_goal=task_block.navigation_goal,
data_extraction_goal=task_block.data_extraction_goal,
navigation_payload=navigation_payload,
@@ -174,6 +180,7 @@ class ForgeAgent:
url=task_request.url,
title=task_request.title,
webhook_callback_url=task_request.webhook_callback_url,
totp_verification_url=task_request.totp_verification_url,
navigation_goal=task_request.navigation_goal,
data_extraction_goal=task_request.data_extraction_goal,
navigation_payload=task_request.navigation_payload,
@@ -955,10 +962,11 @@ class ForgeAgent:
)
element_tree_in_prompt: str = scraped_page.build_element_tree(element_tree_format)
final_navigation_payload = self._build_navigation_payload(task)
extract_action_prompt = prompt_engine.load_prompt(
prompt_template,
navigation_goal=navigation_goal,
navigation_payload_str=json.dumps(task.navigation_payload),
navigation_payload_str=json.dumps(final_navigation_payload),
starting_url=starting_url,
current_url=current_url,
elements=element_tree_in_prompt,
@@ -996,6 +1004,16 @@ class ForgeAgent:
return scraped_page, extract_action_prompt
def _build_navigation_payload(
self,
task: Task,
) -> dict[str, Any] | list | str | None:
final_navigation_payload = task.navigation_payload
if isinstance(final_navigation_payload, dict) and task.totp_verification_url:
if SPECIAL_FIELD_VERIFICATION_CODE not in final_navigation_payload:
final_navigation_payload[SPECIAL_FIELD_VERIFICATION_CODE] = VERIFICATION_CODE_PLACEHOLDER
return final_navigation_payload
async def _get_action_results(self, task: Task) -> str:
# Get action results from the last app.SETTINGS.PROMPT_ACTION_HISTORY_WINDOW steps
steps = await app.DATABASE.get_task_steps(task_id=task.task_id, organization_id=task.organization_id)

36
skyvern/forge/sdk/core/aiohttp_helper.py
View File

@@ -43,3 +43,39 @@ async def aiohttp_get_json(
await asyncio.sleep(retry_timeout)
count += 1
raise Exception(f"Failed to fetch data from {url}")
async def aiohttp_post(
url: str,
data: dict[str, Any] | None = None,
headers: dict[str, str] | None = None,
cookies: dict[str, str] | None = None,
retry: int = 0,
proxy: str | None = None,
timeout: int = DEFAULT_REQUEST_TIMEOUT,
raise_exception: bool = True,
retry_timeout: float = 0,
) -> dict[str, Any]:
async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=timeout)) as session:
count = 0
while count <= retry:
try:
async with session.post(
url,
# TODO: make sure to test this out
json=data,
headers=headers,
cookies=cookies,
proxy=proxy,
) as response:
if response.status == 200:
return await response.json()
if raise_exception:
raise HttpException(response.status, url)
LOG.error("None 200 async post response", url=url, status_code=response.status, method="POST")
return {}
except Exception:
if retry_timeout > 0:
await asyncio.sleep(retry_timeout)
count += 1
raise Exception(f"Failed post request url={url}")

6
skyvern/forge/sdk/db/client.py
View File

@@ -82,6 +82,7 @@ class AgentDB:
data_extraction_goal: str | None,
navigation_payload: dict[str, Any] | list | str | None,
webhook_callback_url: str | None = None,
totp_verification_url: str | None = None,
organization_id: str | None = None,
proxy_location: ProxyLocation | None = None,
extracted_information_schema: dict[str, Any] | list | str | None = None,
@@ -98,6 +99,7 @@ class AgentDB:
url=url,
title=title,
webhook_callback_url=webhook_callback_url,
totp_verification_url=totp_verification_url,
navigation_goal=navigation_goal,
data_extraction_goal=data_extraction_goal,
navigation_payload=navigation_payload,
@@ -806,6 +808,7 @@ class AgentDB:
description: str | None = None,
proxy_location: ProxyLocation | None = None,
webhook_callback_url: str | None = None,
totp_verification_url: str | None = None,
workflow_permanent_id: str | None = None,
version: int | None = None,
is_saved_task: bool = False,
@@ -818,6 +821,7 @@ class AgentDB:
workflow_definition=workflow_definition,
proxy_location=proxy_location,
webhook_callback_url=webhook_callback_url,
totp_verification_url=totp_verification_url,
is_saved_task=is_saved_task,
)
if workflow_permanent_id:
@@ -984,6 +988,7 @@ class AgentDB:
organization_id: str,
proxy_location: ProxyLocation | None = None,
webhook_callback_url: str | None = None,
totp_verification_url: str | None = None,
) -> WorkflowRun:
try:
async with self.Session() as session:
@@ -994,6 +999,7 @@ class AgentDB:
proxy_location=proxy_location,
status="created",
webhook_callback_url=webhook_callback_url,
totp_verification_url=totp_verification_url,
)
session.add(workflow_run)
await session.commit()

3
skyvern/forge/sdk/db/models.py
View File

@@ -48,6 +48,7 @@ class TaskModel(Base):
organization_id = Column(String, ForeignKey("organizations.organization_id"))
status = Column(String, index=True)
webhook_callback_url = Column(String)
totp_verification_url = Column(String)
title = Column(String)
url = Column(String)
navigation_goal = Column(String)
@@ -178,6 +179,7 @@ class WorkflowModel(Base):
workflow_definition = Column(JSON, nullable=False)
proxy_location = Column(Enum(ProxyLocation))
webhook_callback_url = Column(String)
totp_verification_url = Column(String)
created_at = Column(DateTime, default=datetime.datetime.utcnow, nullable=False)
modified_at = Column(
@@ -203,6 +205,7 @@ class WorkflowRunModel(Base):
status = Column(String, nullable=False)
proxy_location = Column(Enum(ProxyLocation))
webhook_callback_url = Column(String)
totp_verification_url = Column(String)
created_at = Column(DateTime, default=datetime.datetime.utcnow, nullable=False)
modified_at = Column(

3
skyvern/forge/sdk/db/utils.py
View File

@@ -63,6 +63,7 @@ def convert_to_task(task_obj: TaskModel, debug_enabled: bool = False) -> Task:
title=task_obj.title,
url=task_obj.url,
webhook_callback_url=task_obj.webhook_callback_url,
totp_verification_url=task_obj.totp_verification_url,
navigation_goal=task_obj.navigation_goal,
data_extraction_goal=task_obj.data_extraction_goal,
navigation_payload=task_obj.navigation_payload,
@@ -160,6 +161,7 @@ def convert_to_workflow(workflow_model: WorkflowModel, debug_enabled: bool = Fal
title=workflow_model.title,
workflow_permanent_id=workflow_model.workflow_permanent_id,
webhook_callback_url=workflow_model.webhook_callback_url,
totp_verification_url=workflow_model.totp_verification_url,
proxy_location=(ProxyLocation(workflow_model.proxy_location) if workflow_model.proxy_location else None),
version=workflow_model.version,
is_saved_task=workflow_model.is_saved_task,
@@ -188,6 +190,7 @@ def convert_to_workflow_run(workflow_run_model: WorkflowRunModel, debug_enabled:
ProxyLocation(workflow_run_model.proxy_location) if workflow_run_model.proxy_location else None
),
webhook_callback_url=workflow_run_model.webhook_callback_url,
totp_verification_url=workflow_run_model.totp_verification_url,
created_at=workflow_run_model.created_at,
modified_at=workflow_run_model.modified_at,
)

2
skyvern/forge/sdk/schemas/tasks.py
View File

@@ -32,12 +32,12 @@ class TaskRequest(BaseModel):
description="Starting URL for the task.",
examples=["https://www.geico.com"],
)
# TODO: use HttpUrl instead of str
webhook_callback_url: str | None = Field(
default=None,
description="The URL to call when the task is completed.",
examples=["https://my-webhook.com"],
)
totp_verification_url: str | None = None
navigation_goal: str | None = Field(
default=None,
description="The user's goal for the task.",

4
skyvern/forge/sdk/workflow/models/workflow.py
View File

@@ -14,6 +14,7 @@ class WorkflowRequestBody(BaseModel):
data: dict[str, Any] | None = None
proxy_location: ProxyLocation | None = None
webhook_callback_url: str | None = None
totp_verification_url: str | None = None
class RunWorkflowResponse(BaseModel):
@@ -49,6 +50,7 @@ class Workflow(BaseModel):
workflow_definition: WorkflowDefinition
proxy_location: ProxyLocation | None = None
webhook_callback_url: str | None = None
totp_verification_url: str | None = None
created_at: datetime
modified_at: datetime
@@ -71,6 +73,7 @@ class WorkflowRun(BaseModel):
status: WorkflowRunStatus
proxy_location: ProxyLocation | None = None
webhook_callback_url: str | None = None
totp_verification_url: str | None = None
created_at: datetime
modified_at: datetime
@@ -96,6 +99,7 @@ class WorkflowRunStatusResponse(BaseModel):
status: WorkflowRunStatus
proxy_location: ProxyLocation | None = None
webhook_callback_url: str | None = None
totp_verification_url: str | None = None
created_at: datetime
modified_at: datetime
parameters: dict[str, Any]

1
skyvern/forge/sdk/workflow/models/yaml.py
View File

@@ -224,5 +224,6 @@ class WorkflowCreateYAMLRequest(BaseModel):
description: str | None = None
proxy_location: ProxyLocation | None = None
webhook_callback_url: str | None = None
totp_verification_url: str | None = None
workflow_definition: WorkflowDefinitionYAML
is_saved_task: bool = False

6
skyvern/forge/sdk/workflow/service.py
View File

@@ -285,6 +285,7 @@ class WorkflowService:
description: str | None = None,
proxy_location: ProxyLocation | None = None,
webhook_callback_url: str | None = None,
totp_verification_url: str | None = None,
workflow_permanent_id: str | None = None,
version: int | None = None,
is_saved_task: bool = False,
@@ -296,6 +297,7 @@ class WorkflowService:
description=description,
proxy_location=proxy_location,
webhook_callback_url=webhook_callback_url,
totp_verification_url=totp_verification_url,
workflow_permanent_id=workflow_permanent_id,
version=version,
is_saved_task=is_saved_task,
@@ -392,6 +394,7 @@ class WorkflowService:
organization_id=organization_id,
proxy_location=workflow_request.proxy_location,
webhook_callback_url=workflow_request.webhook_callback_url,
totp_verification_url=workflow_request.totp_verification_url,
)
async def mark_workflow_run_as_completed(self, workflow_run_id: str) -> None:
@@ -634,6 +637,7 @@ class WorkflowService:
status=workflow_run.status,
proxy_location=workflow_run.proxy_location,
webhook_callback_url=workflow_run.webhook_callback_url,
totp_verification_url=workflow_run.totp_verification_url,
created_at=workflow_run.created_at,
modified_at=workflow_run.modified_at,
parameters=parameters_with_value,
@@ -821,6 +825,7 @@ class WorkflowService:
organization_id=organization_id,
proxy_location=request.proxy_location,
webhook_callback_url=request.webhook_callback_url,
totp_verification_url=request.totp_verification_url,
workflow_permanent_id=workflow_permanent_id,
version=existing_version + 1,
is_saved_task=request.is_saved_task,
@@ -833,6 +838,7 @@ class WorkflowService:
organization_id=organization_id,
proxy_location=request.proxy_location,
webhook_callback_url=request.webhook_callback_url,
totp_verification_url=request.totp_verification_url,
is_saved_task=request.is_saved_task,
)
# Create parameters from the request