import type { Page } from "playwright"; import type * as Skyvern from "../api/index.js"; import type { SkyvernBrowser } from "./SkyvernBrowser.js"; import { LOG } from "./logger.js"; export class SkyvernBrowserPageAi { private readonly _browser: SkyvernBrowser; private readonly _page: Page; constructor(browser: SkyvernBrowser, page: Page) { this._browser = browser; this._page = page; } async aiClick(options: { selector?: string; intention: string; data?: string | Record; timeout?: number; }): Promise { LOG.info("AI click", { intention: options.intention, workflow_run_id: this._browser.workflowRunId }); const response = await this._browser.skyvern.runSdkAction({ url: this._page.url(), browser_session_id: this._browser.browserSessionId, browser_address: this._browser.browserAddress, workflow_run_id: this._browser.workflowRunId, action: { type: "ai_click", selector: options.selector, intention: options.intention, data: options.data, timeout: options.timeout, } as Skyvern.RunSdkActionRequestAction, }); if (response.workflow_run_id) { this._browser.workflowRunId = response.workflow_run_id; } return response.result ? String(response.result) : options.selector || null; } async aiInputText(options: { selector?: string; value?: string; intention: string; data?: string | Record; totpIdentifier?: string; totpUrl?: string; timeout?: number; }): Promise { LOG.info("AI input text", { intention: options.intention, workflow_run_id: this._browser.workflowRunId }); const response = await this._browser.skyvern.runSdkAction({ url: this._page.url(), browser_session_id: this._browser.browserSessionId, browser_address: this._browser.browserAddress, workflow_run_id: this._browser.workflowRunId, action: { type: "ai_input_text", selector: options.selector, value: options.value, intention: options.intention, data: options.data, totp_identifier: options.totpIdentifier, totp_url: options.totpUrl, timeout: options.timeout, } as Skyvern.RunSdkActionRequestAction, }); if (response.workflow_run_id) { this._browser.workflowRunId = response.workflow_run_id; } return response.result ? String(response.result) : options.value || ""; } async aiSelectOption(options: { selector?: string; value?: string; intention: string; data?: string | Record; timeout?: number; }): Promise { LOG.info("AI select option", { intention: options.intention, workflow_run_id: this._browser.workflowRunId }); const response = await this._browser.skyvern.runSdkAction({ url: this._page.url(), browser_session_id: this._browser.browserSessionId, browser_address: this._browser.browserAddress, workflow_run_id: this._browser.workflowRunId, action: { type: "ai_select_option", selector: options.selector, value: options.value, intention: options.intention, data: options.data, timeout: options.timeout, } as Skyvern.RunSdkActionRequestAction, }); if (response.workflow_run_id) { this._browser.workflowRunId = response.workflow_run_id; } return response.result ? String(response.result) : options.value || ""; } async aiUploadFile(options: { selector?: string; fileUrl?: string; intention: string; data?: string | Record; timeout?: number; }): Promise { LOG.info("AI upload file", { intention: options.intention, workflow_run_id: this._browser.workflowRunId }); const response = await this._browser.skyvern.runSdkAction({ url: this._page.url(), browser_session_id: this._browser.browserSessionId, browser_address: this._browser.browserAddress, workflow_run_id: this._browser.workflowRunId, action: { type: "ai_upload_file", selector: options.selector, file_url: options.fileUrl, intention: options.intention, data: options.data, timeout: options.timeout, } as Skyvern.RunSdkActionRequestAction, }); if (response.workflow_run_id) { this._browser.workflowRunId = response.workflow_run_id; } return response.result ? String(response.result) : options.fileUrl || ""; } async aiExtract(options: { prompt: string; extractSchema?: Record | unknown[] | string; errorCodeMapping?: Record; intention?: string; data?: string | Record; }): Promise | unknown[] | string | null> { LOG.info("AI extract", { prompt: options.prompt, workflow_run_id: this._browser.workflowRunId }); const response = await this._browser.skyvern.runSdkAction({ url: this._page.url(), browser_session_id: this._browser.browserSessionId, browser_address: this._browser.browserAddress, workflow_run_id: this._browser.workflowRunId, action: { type: "extract", prompt: options.prompt, extract_schema: options.extractSchema, error_code_mapping: options.errorCodeMapping, intention: options.intention, data: options.data, } as Skyvern.RunSdkActionRequestAction, }); if (response.workflow_run_id) { this._browser.workflowRunId = response.workflow_run_id; } return (response.result as Record | unknown[] | string) || null; } async aiValidate(options: { prompt: string; model?: Record }): Promise { LOG.info("AI validate", { prompt: options.prompt, model: options.model, workflow_run_id: this._browser.workflowRunId }); const response = await this._browser.skyvern.runSdkAction({ url: this._page.url(), browser_session_id: this._browser.browserSessionId, browser_address: this._browser.browserAddress, workflow_run_id: this._browser.workflowRunId, action: { type: "validate", prompt: options.prompt, model: options.model, } as Skyvern.RunSdkActionRequestAction, }); if (response.workflow_run_id) { this._browser.workflowRunId = response.workflow_run_id; } return response.result != null ? Boolean(response.result) : false; } async aiAct(prompt: string): Promise { LOG.info("AI act", { prompt, workflow_run_id: this._browser.workflowRunId }); const response = await this._browser.skyvern.runSdkAction({ url: this._page.url(), browser_session_id: this._browser.browserSessionId, browser_address: this._browser.browserAddress, workflow_run_id: this._browser.workflowRunId, action: { type: "ai_act", intention: prompt, } as Skyvern.RunSdkActionRequestAction, }); if (response.workflow_run_id) { this._browser.workflowRunId = response.workflow_run_id; } } async aiLocateElement(prompt: string): Promise { LOG.info("AI locate element", { prompt, workflow_run_id: this._browser.workflowRunId }); const response = await this._browser.skyvern.runSdkAction({ url: this._page.url(), browser_session_id: this._browser.browserSessionId, browser_address: this._browser.browserAddress, workflow_run_id: this._browser.workflowRunId, action: { type: "locate_element", prompt, } as Skyvern.RunSdkActionRequestAction, }); if (response.workflow_run_id) { this._browser.workflowRunId = response.workflow_run_id; } if (response.result && typeof response.result === "string") { return response.result; } return null; } async aiPrompt(options: { prompt: string; schema?: Record; model?: Record; }): Promise | unknown[] | string | null> { LOG.info("AI prompt", { prompt: options.prompt, model: options.model, workflow_run_id: this._browser.workflowRunId }); const response = await this._browser.skyvern.runSdkAction({ url: this._page.url(), browser_session_id: this._browser.browserSessionId, browser_address: this._browser.browserAddress, workflow_run_id: this._browser.workflowRunId, action: { type: "prompt", prompt: options.prompt, schema: options.schema, model: options.model, } as Skyvern.RunSdkActionRequestAction, }); if (response.workflow_run_id) { this._browser.workflowRunId = response.workflow_run_id; } return (response.result as Record | unknown[] | string) || null; } }