Add manual SDK tests (#4555)

This commit is contained in:
Stanislav Novosad
2026-01-26 15:43:53 -07:00
committed by GitHub
parent a43b3ae3cc
commit 72b9fe960f
19 changed files with 3037 additions and 1 deletions

View File

@@ -202,4 +202,4 @@ plugins = "sqlalchemy.ext.mypy.plugin"
skyvern = "skyvern.cli.commands:cli_app" skyvern = "skyvern.cli.commands:cli_app"
[tool.pytest.ini_options] [tool.pytest.ini_options]
norecursedirs = ["eval"] norecursedirs = ["eval", "tests/sdk"]

3
tests/pytest.ini Normal file
View File

@@ -0,0 +1,3 @@
[pytest]
asyncio_mode = auto
addopts = --capture=tee-sys

59
tests/sdk/README.md Normal file
View File

@@ -0,0 +1,59 @@
# Skyvern SDK Tests
Test suite for Skyvern Python and TypeScript SDKs with shared HTML fixtures in `web/`.
## Python SDK
**Location:** `tests/sdk/python_sdk/`
### Prerequisites
- Requires `.env` with `SKYVERN_API_KEY`
- Browser fixture auto-launches on port 9222
- Web server fixture auto-starts on port 9010
### Running Tests
```bash
# Run all tests
pytest tests/sdk/python_sdk/
# Run specific test file
pytest tests/sdk/python_sdk/test_sdk_simple_actions.py
# Run specific test
pytest tests/sdk/python_sdk/test_sdk_simple_actions.py::test_clicks
```
---
## TypeScript SDK
**Location:** `tests/sdk/typescript_sdk/`
### Prerequisites
- Requires `.env` with `SKYVERN_API_KEY`
- Requires Chrome with CDP on `localhost:9222`
- Web server auto-starts via `run-test.js`
**Launch Chrome with CDP:**
```bash
/Users/stas/Library/Caches/ms-playwright/chromium-1194/chrome-mac/Chromium.app/Contents/MacOS/Chromium \
--remote-debugging-port=9222 \
--user-data-dir=~/tmp/chrome-playwright \
about:blank
```
### Running Tests
```bash
cd tests/sdk/typescript_sdk
# First time setup
npm install
# Run specific test
npm test test_simple_actions.ts testClicks
# Run all tests in a file
npm test test_simple_actions.ts all
```

2
tests/sdk/python_sdk/.gitignore vendored Normal file
View File

@@ -0,0 +1,2 @@
.env
*.png

View File

@@ -0,0 +1,99 @@
import asyncio
import socket
import threading
import time
from http.server import HTTPServer, SimpleHTTPRequestHandler
from pathlib import Path
import pytest
from skyvern import Skyvern
@pytest.fixture(scope="session")
def event_loop():
"""Create a session-scoped event loop for async session fixtures."""
loop = asyncio.get_event_loop_policy().new_event_loop()
yield loop
loop.close()
class QuietHTTPRequestHandler(SimpleHTTPRequestHandler):
"""HTTP request handler that suppresses log messages."""
def log_message(self, format, *args):
"""Override to suppress HTTP request logging."""
def _wait_for_server(host: str, port: int, timeout: float = 5.0) -> bool:
"""Wait for the server to be ready by attempting to connect."""
start_time = time.time()
deadline = start_time + timeout
while time.time() < deadline:
try:
with socket.create_connection((host, port), timeout=0.1):
return True
except (ConnectionRefusedError, OSError):
time.sleep(0.05)
return False
@pytest.fixture(scope="session")
async def web_server():
"""
Start a local HTTP server on port 9009 serving from the 'web' directory.
This is an async fixture that properly waits for the server to be ready.
"""
# Get the directory where this conftest file is located
test_dir = Path(__file__).parent
web_dir = test_dir.parent / "web"
# Create web directory if it doesn't exist
web_dir.mkdir(exist_ok=True)
# Create a handler class that serves from the specific directory
def handler_factory(*args, **kwargs):
return QuietHTTPRequestHandler(*args, directory=str(web_dir), **kwargs)
# Create and start the HTTP server
server = HTTPServer(("localhost", 9009), handler_factory)
server_thread = threading.Thread(target=server.serve_forever, daemon=True)
server_thread.start()
# Wait for the server to be ready (async-friendly)
await asyncio.sleep(0.1)
# Verify server is responding
if not _wait_for_server("localhost", 9009):
# Don't wait forever - just close the socket and let daemon thread die
server.server_close()
raise RuntimeError("Server failed to start")
base_url = "http://localhost:9009"
yield base_url
# Cleanup: Don't call shutdown() as it can block forever waiting for active connections
# Instead, just close the socket and let the daemon thread die
server.server_close()
@pytest.fixture(scope="session")
async def skyvern_browser():
"""
Launch a local browser once for the entire test session and reuse it across tests.
This ensures all tests use the same browser instance, avoiding connection issues.
"""
skyvern = Skyvern.local()
browser = await skyvern.launch_local_browser(headless=False)
yield browser
# Cleanup: close the browser after all tests complete
try:
await browser.close()
except Exception:
pass # Ignore cleanup errors

View File

@@ -0,0 +1,74 @@
import asyncio
import pytest
from skyvern.forge.sdk.schemas.credentials import NonEmptyPasswordCredential, TotpType
from skyvern.schemas.run_blocks import CredentialType
@pytest.mark.asyncio
async def test_login(web_server, skyvern_browser):
page = await skyvern_browser.get_working_page()
await page.goto(f"{web_server}/login.html")
credentials = await skyvern_browser.skyvern.get_credentials()
credential = next((item for item in credentials if item.name == "test_login"), None)
if credential is None:
print("Credentials not found. Creating new one.")
credential = await skyvern_browser.skyvern.create_credential(
name="test_login",
credential_type="password",
credential=NonEmptyPasswordCredential(
username="testlogin",
password="testpassword",
totp=None,
totp_type=TotpType.NONE,
),
)
await page.agent.login(
credential_type=CredentialType.skyvern,
credential_id=credential.credential_id,
)
await page.click("#accountBtn")
await asyncio.sleep(1)
await page.act("Click on 'Click Me' button")
assert await page.locator("#clickCounter").text_content() == "Button clicked 1 times"
await asyncio.sleep(1)
await page.screenshot(path="screenshot.png", full_page=True)
@pytest.mark.asyncio
async def test_test_finishes_login(web_server, skyvern_browser):
page = await skyvern_browser.get_working_page()
await page.goto("https://www.saucedemo.com/")
await page.fill("#user-name", "standard_user")
await page.fill("#password", "secret_sauce")
await page.agent.run_task("Click on login button", engine="skyvern-1.0")
assert await page.get_by_role("button", name="Add to cart").count() > 0
@pytest.mark.asyncio
async def test_download_file(web_server, skyvern_browser):
page = await skyvern_browser.get_working_page()
await page.goto(f"{web_server}/download_file.html")
r = await page.agent.download_files(
prompt="Click the 'Download PDF Report' button to download the sample PDF file",
download_suffix="sample_report.pdf",
)
for downloaded_file in r.downloaded_files:
print(downloaded_file)
assert len(r.downloaded_files) == 1
await asyncio.sleep(2)
await page.screenshot(path="download_test.png", full_page=True)
assert len(r.downloaded_files) == 1

View File

@@ -0,0 +1,235 @@
import pytest
@pytest.mark.asyncio
async def test_clicks(web_server, skyvern_browser):
page = await skyvern_browser.get_working_page()
await page.goto(f"{web_server}/click.html")
assert await page.locator("#counter").text_content() == "Button clicked 0 times"
await page.click("#button")
assert await page.locator("#counter").text_content() == "Button clicked 1 times"
await page.click(prompt="Click on the button")
assert await page.locator("#counter").text_content() == "Button clicked 2 times"
print("Fallback")
await page.click("#counterBroken", prompt="Click on the button")
assert await page.locator("#counter").text_content() == "Button clicked 3 times"
await page.click("#button")
assert await page.locator("#counter").text_content() == "Button clicked 4 times"
print("All tests passed")
@pytest.mark.asyncio
async def test_fill(web_server, skyvern_browser):
page = await skyvern_browser.get_working_page()
await page.goto(f"{web_server}/input.html")
assert await page.locator("#output").text_content() == ""
await page.fill("#nameInput", "Test1")
await page.click("#submitBtn")
assert await page.locator("#output").text_content() == "Hello, Test1!"
await page.fill(prompt="Type 'Test2' in the name input")
await page.click("#submitBtn")
assert await page.locator("#output").text_content() == "Hello, Test2!"
await page.fill(prompt="Type the value in the name input", value="Test3")
await page.click("#submitBtn")
assert await page.locator("#output").text_content() == "Hello, Test3!"
# fallback
await page.fill("#nameInputBroken", "TestFallback", prompt="Fill the name input")
await page.click("#submitBtn")
assert await page.locator("#output").text_content() == "Hello, TestFallback!"
@pytest.mark.asyncio
async def test_select_option(web_server, skyvern_browser):
"""Test using page.act() with natural language prompts on a combobox."""
page = await skyvern_browser.get_working_page()
await page.goto(f"{web_server}/combobox.html")
await page.select_option("#cars", "audi")
assert await page.locator("#cars").input_value() == "audi"
await page.select_option("#cars", value="opel")
assert await page.locator("#cars").input_value() == "opel"
await page.select_option("#cars", label="Saab")
assert await page.locator("#cars").input_value() == "saab"
await page.select_option(prompt="Select 'Audi' i the car combobox")
assert await page.locator("#cars").input_value() == "audi"
# fallback
await page.select_option("#cars-broken", "opel", prompt="Select 'Opel' i the car combobox")
assert await page.locator("#cars").input_value() == "opel"
await page.select_option("#cars-broken", label="Saab", prompt="Select 'Saab' i the car combobox")
assert await page.locator("#cars").input_value() == "saab"
@pytest.mark.asyncio
async def test_act_combobox(web_server, skyvern_browser):
"""Test using page.act() with natural language prompts on a combobox."""
page = await skyvern_browser.get_working_page()
await page.goto(f"{web_server}/combobox.html")
await page.act("select 'Audi' from the combobox")
assert await page.locator("#cars").input_value() == "audi"
@pytest.mark.asyncio
async def test_act_input_and_click(web_server, skyvern_browser):
page = await skyvern_browser.get_working_page()
await page.goto(f"{web_server}/input.html")
await page.act("type 'ActTest' into the input box")
await page.act("click on the button")
assert await page.locator("#output").text_content() == "Hello, ActTest!"
@pytest.mark.asyncio
async def test_upload(web_server, skyvern_browser):
page = await skyvern_browser.get_working_page()
image_url = "https://img.freepik.com/free-photo/portrait-beautiful-purebred-pussycat-with-shorthair-orange-collar-neck-sitting-floor-reacting-camera-flash-scared-looking-light-indoor_8353-12551.jpg?semt=ais_hybrid&w=740&q=80"
await page.goto(f"{web_server}/upload.html")
await page.reload()
assert (await page.locator("#uploadedImage").get_attribute("src")).__str__() == ""
await page.upload_file("#imageUpload", image_url)
assert (await page.locator("#uploadedImage").get_attribute("src")).__str__().startswith("data:image/jpeg")
await page.reload()
assert (await page.locator("#uploadedImage").get_attribute("src")).__str__() == ""
await page.upload_file(prompt="Upload this", files=image_url)
assert (await page.locator("#uploadedImage").get_attribute("src")).__str__().startswith("data:image/jpeg")
await page.reload()
assert (await page.locator("#uploadedImage").get_attribute("src")).__str__() == ""
await page.upload_file(prompt=f"Upload this file {image_url}")
assert (await page.locator("#uploadedImage").get_attribute("src")).__str__().startswith("data:image/jpeg")
print("fallback")
await page.reload()
assert (await page.locator("#uploadedImage").get_attribute("src")).__str__() == ""
await page.upload_file("#imageUploadBroken", image_url, prompt="Upload this file")
assert (await page.locator("#uploadedImage").get_attribute("src")).__str__().startswith("data:image/jpeg")
print("all done")
@pytest.mark.asyncio
async def test_extract(web_server, skyvern_browser):
page = await skyvern_browser.get_working_page()
await page.goto(f"{web_server}/click.html")
result = await page.extract("give one sentence description of this page")
print(result)
assert "click" in str(result)
result = await page.extract(
prompt="Describe this page",
schema={
"type": "object",
"properties": {
"short": {"type": "string", "description": "one sentence description of this page"},
"long": {"type": "string", "description": "two-three sentence description of this page"},
},
"required": ["short", "long"],
},
)
print(result)
assert "click" in str(result)
assert "short" in str(result)
assert "long" in str(result)
@pytest.mark.asyncio
async def test_prompt_locator(web_server, skyvern_browser):
page = await skyvern_browser.get_working_page()
await page.goto(f"{web_server}/click.html")
assert await page.locator("#counter").text_content() == "Button clicked 0 times"
await page.locator(prompt="Find the 'click me' button").click()
assert await page.locator("#counter").text_content() == "Button clicked 1 times"
await page.locator(prompt="Find the 'click me' button").nth(0).click()
assert await page.locator("#counter").text_content() == "Button clicked 2 times"
await page.locator("#bad-selector", prompt="Find the 'click me' button").click()
assert await page.locator("#counter").text_content() == "Button clicked 3 times"
await page.locator("#bad-selector", prompt="Find the 'click me' button").nth(0).click()
assert await page.locator("#counter").text_content() == "Button clicked 4 times"
await page.locator("#button").click()
assert await page.locator("#counter").text_content() == "Button clicked 5 times"
@pytest.mark.asyncio
async def test_prompt_locator_chaining(web_server, skyvern_browser):
page = await skyvern_browser.get_working_page()
await page.goto(f"{web_server}/combobox.html")
await page.locator("#cars").select_option("opel")
assert await page.locator("#cars").input_value() == "opel"
await page.locator(prompt="Find the 'cars' combobox").select_option("saab")
assert await page.locator("#cars").input_value() == "saab"
await page.locator("#bad-selector", prompt="Find the 'cars' combobox").select_option("audi")
assert await page.locator("#cars").input_value() == "audi"
@pytest.mark.asyncio
async def test_validate(web_server, skyvern_browser):
page = await skyvern_browser.get_working_page()
await page.goto(f"{web_server}/click.html")
await page.click("#button")
is_valid = await page.validate("if clicked time > 0")
assert is_valid
is_valid = await page.validate("if clicked time > 1")
assert not is_valid
# invalid prompt does not pass validation
is_valid = await page.validate("the input text is valid")
assert not is_valid
@pytest.mark.asyncio
async def test_prompt(skyvern_browser):
page = await skyvern_browser.get_working_page()
r = await page.prompt("1111+1111")
print(r)
assert "2222" in str(r)
r = await page.prompt(
"2+2",
schema={
"type": "object",
"properties": {
"result_number": {"type": "int"},
"confidence": {"type": "number", "minimum": 0, "maximum": 1},
},
},
)
print(r)
assert r["result_number"] == 4

3
tests/sdk/typescript_sdk/.gitignore vendored Normal file
View File

@@ -0,0 +1,3 @@
node_modules
.env
*.png

1279
tests/sdk/typescript_sdk/package-lock.json generated Normal file

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,23 @@
{
"name": "skyvern-ts-sdk-tests",
"version": "1.0.0",
"description": "Playground for testing Skyvern TypeScript SDK",
"type": "module",
"scripts": {
"build": "tsc",
"serve": "http-server ../web -p 9010 --silent",
"test": "node run-test.js"
},
"author": "",
"license": "ISC",
"dependencies": {
"@skyvern/client": "file:../../../skyvern-ts/client",
"dotenv": "^17.2.3"
},
"devDependencies": {
"@types/node": "^20.0.0",
"http-server": "^14.1.1",
"tsx": "^4.7.0",
"typescript": "^5.0.0"
}
}

View File

@@ -0,0 +1,91 @@
#!/usr/bin/env node
import { spawn } from "child_process";
import http from "http";
import { fileURLToPath } from "url";
import { dirname, resolve } from "path";
const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
const WEB_DIR = resolve(__dirname, "../web");
const PORT = 9010;
let server;
function startServer() {
return new Promise((resolve, reject) => {
const handler = (req, res) => {
import("http-server").then(({ default: httpServer }) => {
const serve = httpServer.createServer({ root: WEB_DIR });
serve.emit("request", req, res);
});
};
// Use http-server via spawn instead
server = spawn("npx", ["http-server", WEB_DIR, "-p", PORT, "--silent"], {
stdio: "inherit",
});
// Wait a bit for server to start
setTimeout(() => {
http
.get(`http://localhost:${PORT}`, () => {
console.log(`Web server started at http://localhost:${PORT}`);
resolve();
})
.on("error", () => {
// Retry after a short delay
setTimeout(() => {
http
.get(`http://localhost:${PORT}`, () => {
console.log(`Web server started at http://localhost:${PORT}`);
resolve();
})
.on("error", reject);
}, 500);
});
}, 500);
});
}
function runTests(args) {
return new Promise((resolve, reject) => {
const testProcess = spawn("npx", ["tsx", ...args], {
stdio: "inherit",
});
testProcess.on("close", (code) => {
if (code === 0) {
resolve();
} else {
reject(new Error(`Tests failed with exit code ${code}`));
}
});
});
}
async function main() {
const testArgs = process.argv.slice(2);
if (testArgs.length === 0) {
console.error("Usage: npm test <test-file> [test-name]");
console.error("Example: npm test test_simple_actions.ts testClicks");
process.exit(1);
}
try {
await startServer();
await runTests(testArgs);
console.log("Tests completed successfully");
} catch (error) {
console.error("Error:", error.message);
process.exit(1);
} finally {
if (server) {
server.kill();
console.log("Web server stopped");
}
}
}
main();

View File

@@ -0,0 +1,109 @@
import { Skyvern, SkyvernEnvironment } from "@skyvern/client";
import "dotenv/config";
const WEB_SERVER = process.env.WEB_SERVER || "http://localhost:9010";
const skyvern = new Skyvern({
apiKey: process.env.SKYVERN_API_KEY!,
environment: SkyvernEnvironment.Local,
});
async function testMlgameLogin() {
const browser = await skyvern.connectToBrowserOverCdp("http://localhost:9222");
const page = await browser.getWorkingPage();
await page.goto(`${WEB_SERVER}/login.html`);
const credentials = await skyvern.getCredentials();
let credential = credentials.find((item) => item.name === "test_login");
if (!credential) {
console.log("Credentials not found. Creating new one.");
credential = await skyvern.createCredential({
name: "test_login",
credential_type: "password",
credential: {
username: "testlogin",
password: "testpassword",
totp_type: "none",
},
});
}
await page.agent.login("skyvern", {
credentialId: credential.credential_id,
});
await page.click("#accountBtn");
await new Promise((resolve) => setTimeout(resolve, 1000));
await page.act("Click on 'Click Me' button");
console.assert(await page.locator("#clickCounter").textContent() == "Button clicked 1 times");
await new Promise((resolve) => setTimeout(resolve, 1000));
await page.screenshot({ path: "screenshot.png", fullPage: true });
console.log("All tests passed");
}
async function testFinishesLogin() {
const browser = await skyvern.connectToBrowserOverCdp("http://localhost:9222");
const page = await browser.getWorkingPage();
await page.goto("https://www.saucedemo.com/");
await page.fill("#user-name", "standard_user");
await page.fill("#password", "secret_sauce");
await page.agent.runTask("Click on login button", { engine: "skyvern-1.0" });
console.assert((await page.getByRole("button", { name: "Add to cart" }).count()) > 0);
console.log("All tests passed");
}
async function testDownloadFile() {
const browser = await skyvern.connectToBrowserOverCdp("http://localhost:9222");
const page = await browser.getWorkingPage();
await page.goto(`${WEB_SERVER}/download_file.html`);
const result = await page.agent.downloadFiles(
"Click the 'Download PDF Report' button to download the sample PDF file",
{ downloadSuffix: "sample_report.pdf" }
);
console.log(result.downloaded_files);
console.assert(result.downloaded_files?.length === 1);
await new Promise((resolve) => setTimeout(resolve, 2000));
await page.screenshot({ path: "download_test.png", fullPage: true });
console.log("All tests passed");
}
const tests: Record<string, () => Promise<void>> = {
testMlgameLogin,
testFinishesLogin,
testDownloadFile,
all: async () => {
await testMlgameLogin();
await testFinishesLogin();
await testDownloadFile();
},
};
const testName = process.argv[2] || "all";
if (tests[testName]) {
tests[testName]()
.catch((error) => {
console.error("Test failed:", error);
process.exit(1);
})
.finally(async () => {
await skyvern.close();
});
} else {
console.error(`Unknown test: ${testName}`);
console.error(`Available tests: ${Object.keys(tests).join(", ")}`);
process.exit(1);
}

View File

@@ -0,0 +1,208 @@
import { Skyvern, SkyvernEnvironment } from "@skyvern/client";
import "dotenv/config";
const WEB_SERVER = process.env.WEB_SERVER || "http://localhost:9010";
const skyvern = new Skyvern({
apiKey: process.env.SKYVERN_API_KEY!,
environment: SkyvernEnvironment.Local,
});
async function testClicks() {
const browser = await skyvern.connectToBrowserOverCdp("http://localhost:9222");
const page = await browser.getWorkingPage();
await page.goto(`${WEB_SERVER}/click.html`);
console.assert((await page.locator("#counter").textContent()) === "Button clicked 0 times");
await page.click("#button");
console.assert((await page.locator("#counter").textContent()) === "Button clicked 1 times");
await page.click({ prompt: "Click on the button" });
console.assert((await page.locator("#counter").textContent()) === "Button clicked 2 times");
console.log("Fallback");
await page.click("#broken-selector", { prompt: "Click on the button" });
console.assert((await page.locator("#counter").textContent()) === "Button clicked 3 times");
await page.click("#button");
console.assert((await page.locator("#counter").textContent()) === "Button clicked 4 times");
console.log("All tests passed");
}
async function testFill() {
const browser = await skyvern.connectToBrowserOverCdp("http://localhost:9222");
const page = await browser.getWorkingPage();
await page.goto(`${WEB_SERVER}/input.html`);
console.assert((await page.locator("#output").textContent()) === "");
await page.fill("#nameInput", "Test1");
await page.click("#submitBtn");
console.assert((await page.locator("#output").textContent()) === "Hello, Test1!");
await page.fill({ prompt: "Type 'Test2' in the name input" });
await page.click("#submitBtn");
console.assert((await page.locator("#output").textContent()) === "Hello, Test2!");
await page.fill({ prompt: "Type the value in the name input", value: "Test3" });
await page.click("#submitBtn");
console.assert((await page.locator("#output").textContent()) === "Hello, Test3!");
await page.fill("#nameInputBroken", "TestFallback", { prompt: "Fill the name input" })
await page.click("#submitBtn")
console.assert(await page.locator("#output").textContent() == "Hello, TestFallback!")
console.log("All tests passed");
}
async function testSelectOption() {
const browser = await skyvern.connectToBrowserOverCdp("http://localhost:9222");
const page = await browser.getWorkingPage();
await page.goto(`${WEB_SERVER}/combobox.html`);
await page.selectOption("#cars", "audi");
console.assert((await page.locator("#cars").inputValue()) === "audi");
await page.selectOption("#cars", "opel");
console.assert((await page.locator("#cars").inputValue()) === "opel");
await page.selectOption("#cars", { label: "Saab" });
console.assert((await page.locator("#cars").inputValue()) === "saab");
await page.selectOption({ prompt: "Select 'Audi' i the car combobox" });
console.assert((await page.locator("#cars").inputValue()) === "audi");
// fallback
await page.selectOption("#cars-broken", "opel", { prompt: "Select 'Opel' i the car combobox" })
console.assert(await page.locator("#cars").inputValue() == "opel")
console.log("All tests passed");
}
async function testActCombobox() {
const browser = await skyvern.connectToBrowserOverCdp("http://localhost:9222");
const page = await browser.getWorkingPage();
await page.goto(`${WEB_SERVER}/combobox.html`);
await page.act("select 'Audi' from the combobox");
console.assert((await page.locator("#cars").inputValue()) === "audi");
console.log("All tests passed");
}
async function testActInputAndClick() {
const browser = await skyvern.connectToBrowserOverCdp("http://localhost:9222");
const page = await browser.getWorkingPage();
await page.goto(`${WEB_SERVER}/input.html`);
await page.act("type 'ActTest' into the input box");
await page.act("click on the button");
console.assert((await page.locator("#output").textContent()) === "Hello, ActTest!");
console.log("All tests passed");
}
async function testExtract() {
const browser = await skyvern.connectToBrowserOverCdp("http://localhost:9222");
const page = await browser.getWorkingPage();
await page.goto(`${WEB_SERVER}/click.html`);
const r1 = await page.extract({ prompt: "give one sentence description of this page" });
console.log(r1);
const r2 = await page.extract({
prompt: "Describe this page",
schema: {
type: "object",
properties: {
short: { type: "string", description: "one sentence description of this page" },
long: { type: "string", description: "two-three sentence description of this page" },
},
required: ["short", "long"],
},
});
console.log(r2);
console.log("All tests passed");
}
async function testValidate() {
const browser = await skyvern.connectToBrowserOverCdp("http://localhost:9222");
const page = await browser.getWorkingPage();
await page.goto(`${WEB_SERVER}/click.html`);
await page.click("#button");
console.assert((await page.validate("if clicked time > 0")) === true);
console.assert((await page.validate("if clicked time > 1")) === false);
console.assert((await page.validate("the input text is valid")) === false);
console.log("All tests passed");
}
async function testPrompt() {
const browser = await skyvern.connectToBrowserOverCdp("http://localhost:9222");
const page = await browser.getWorkingPage();
const r1 = await page.prompt("1111+1111");
console.log(r1);
console.assert(String(r1).includes("2222"));
const r2 = await page.prompt("2+2", {
type: "object",
properties: {
result_number: { type: "number" },
confidence: { type: "number", minimum: 0, maximum: 1 },
},
});
console.log(r2);
console.assert((r2 as any).result_number === 4);
console.log("All tests passed");
}
const tests: Record<string, () => Promise<void>> = {
testClicks,
testFill,
testSelectOption,
testActCombobox,
testActInputAndClick,
testExtract,
testValidate,
testPrompt,
all: async () => {
await testClicks();
await testFill();
await testSelectOption();
await testActCombobox();
await testActInputAndClick();
await testExtract();
await testValidate();
await testPrompt();
},
};
const testName = process.argv[2] || "all";
if (tests[testName]) {
tests[testName]()
.catch((error) => {
console.error("Test failed:", error);
process.exit(1);
})
.finally(async () => {
await skyvern.close();
});
} else {
console.error(`Unknown test: ${testName}`);
console.error(`Available tests: ${Object.keys(tests).join(", ")}`);
process.exit(1);
}

65
tests/sdk/web/click.html Normal file
View File

@@ -0,0 +1,65 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Click Counter</title>
<style>
body {
font-family: Arial, sans-serif;
max-width: 600px;
margin: 50px auto;
padding: 20px;
text-align: center;
background-color: #121212;
color: #f5f5f5;
}
h1 {
color: #f5f5f5;
}
button {
padding: 20px 40px;
font-size: 18px;
background-color: #2563eb;
color: #f5f5f5;
border: 1px solid #1d4ed8;
border-radius: 8px;
cursor: pointer;
margin: 30px 0;
transition: background-color 0.3s, transform 0.1s;
box-shadow: 0 10px 25px rgba(37, 99, 235, 0.25);
}
button:hover {
background-color: #1d4ed8;
}
button:active {
transform: scale(0.98);
}
#counter {
margin-top: 30px;
font-size: 24px;
font-weight: bold;
color: #86efac;
min-height: 50px;
}
</style>
</head>
<body>
<h1>Click Counter</h1>
<button id="button">Click Me!</button>
<div id="counter">Button clicked 0 times</div>
<script>
let clickCount = 0;
const clickBtn = document.getElementById('button');
const counter = document.getElementById('counter');
clickBtn.addEventListener('click', function() {
clickCount++;
counter.textContent = `Button clicked ${clickCount} times`;
});
</script>
</body>
</html>

View File

@@ -0,0 +1,98 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Dark Theme Select</title>
<style>
body {
font-family: Arial, sans-serif;
max-width: 600px;
margin: 50px auto;
padding: 20px 30px;
background-color: #121212;
color: #f5f5f5;
line-height: 1.6;
}
h1 {
color: #f5f5f5;
text-align: center;
}
form {
display: flex;
flex-direction: column;
gap: 20px;
margin: 30px 0;
padding: 30px;
background-color: #1e1e1e;
border-radius: 12px;
box-shadow: 0 10px 30px rgba(0, 0, 0, 0.35);
}
label {
font-weight: 600;
color: #e0e0e0;
}
select {
padding: 12px;
font-size: 16px;
border-radius: 8px;
border: 1px solid #2d2d2d;
background-color: #1b1b1b;
color: #f5f5f5;
appearance: none;
box-shadow: 0 4px 12px rgba(0, 0, 0, 0.3);
}
select:focus {
outline: none;
border-color: #2563eb;
box-shadow: 0 0 0 3px rgba(37, 99, 235, 0.35);
}
input[type="submit"] {
align-self: flex-start;
padding: 12px 24px;
font-size: 16px;
background-color: #2563eb;
color: #f5f5f5;
border: 1px solid #1d4ed8;
border-radius: 8px;
cursor: pointer;
transition: background-color 0.3s, transform 0.1s;
box-shadow: 0 10px 25px rgba(37, 99, 235, 0.25);
}
input[type="submit"]:hover {
background-color: #1d4ed8;
}
input[type="submit"]:active {
transform: scale(0.98);
}
p {
color: #d1d1d1;
text-align: center;
}
a {
color: #93c5fd;
}
</style>
</head>
<body>
<h1>The select element</h1>
<p>The select element is used to create a drop-down list.</p>
<form action="/action_page.php">
<label for="cars">Choose a car:</label>
<select name="cars" id="cars">
<option value="volvo">Volvo</option>
<option value="saab">Saab</option>
<option value="opel">Opel</option>
<option value="audi">Audi</option>
</select>
<input type="submit" value="Submit">
</form>
<p>Click the "Submit" button and the form-data will be sent to a page on the
server called "action_page.php".</p>
</body>
</html>

View File

@@ -0,0 +1,169 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>File Download</title>
<style>
body {
font-family: Arial, sans-serif;
max-width: 600px;
margin: 50px auto;
padding: 20px;
text-align: center;
background-color: #121212;
color: #f5f5f5;
}
h1 {
color: #f5f5f5;
}
.container {
margin: 40px 0;
}
button {
padding: 20px 40px;
font-size: 18px;
background-color: #2563eb;
color: #f5f5f5;
border: 1px solid #1d4ed8;
border-radius: 8px;
cursor: pointer;
margin: 10px;
transition: background-color 0.3s, transform 0.1s;
box-shadow: 0 10px 25px rgba(37, 99, 235, 0.25);
}
button:hover {
background-color: #1d4ed8;
}
button:active {
transform: scale(0.98);
}
#status {
margin-top: 30px;
font-size: 18px;
color: #86efac;
min-height: 30px;
}
.description {
color: #9ca3af;
margin-bottom: 20px;
}
</style>
</head>
<body>
<h1>File Download Test</h1>
<p class="description">Click the button below to download a sample PDF file</p>
<div class="container">
<button id="downloadBtn">Download PDF Report</button>
</div>
<div id="status"></div>
<script>
const downloadBtn = document.getElementById('downloadBtn');
const status = document.getElementById('status');
// Generate a simple PDF content
function generatePDF() {
// Create a simple PDF structure with text content
const pdfContent = `%PDF-1.4
1 0 obj
<<
/Type /Catalog
/Pages 2 0 R
>>
endobj
2 0 obj
<<
/Type /Pages
/Kids [3 0 R]
/Count 1
>>
endobj
3 0 obj
<<
/Type /Page
/Parent 2 0 R
/Resources <<
/Font <<
/F1 <<
/Type /Font
/Subtype /Type1
/BaseFont /Helvetica
>>
>>
>>
/MediaBox [0 0 612 792]
/Contents 4 0 R
>>
endobj
4 0 obj
<<
/Length 110
>>
stream
BT
/F1 24 Tf
50 700 Td
(Sample PDF Report) Tj
0 -50 Td
/F1 12 Tf
(Generated by Skyvern Test) Tj
0 -30 Td
(This is a test PDF file for download automation.) Tj
ET
endstream
endobj
xref
0 5
0000000000 65535 f
0000000009 00000 n
0000000058 00000 n
0000000115 00000 n
0000000317 00000 n
trailer
<<
/Size 5
/Root 1 0 R
>>
startxref
476
%%EOF`;
return pdfContent;
}
downloadBtn.addEventListener('click', function() {
// Generate PDF content
const pdfContent = generatePDF();
// Create a Blob from the PDF content
const blob = new Blob([pdfContent], { type: 'application/pdf' });
// Create a download link
const url = window.URL.createObjectURL(blob);
const a = document.createElement('a');
a.href = url;
a.download = 'sample_report.pdf';
// Trigger download
document.body.appendChild(a);
a.click();
// Clean up
setTimeout(() => {
document.body.removeChild(a);
window.URL.revokeObjectURL(url);
}, 100);
// Update status
status.textContent = 'PDF downloaded successfully!';
status.style.color = '#86efac';
setTimeout(() => {
status.textContent = '';
}, 3000);
});
</script>
</body>
</html>

110
tests/sdk/web/input.html Normal file
View File

@@ -0,0 +1,110 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Text Input Demo</title>
<style>
body {
font-family: Arial, sans-serif;
max-width: 600px;
margin: 50px auto;
padding: 20px;
text-align: center;
background-color: #121212;
color: #f5f5f5;
}
h1 {
color: #f5f5f5;
}
.input-container {
margin: 30px 0;
padding: 25px;
background-color: #1e1e1e;
border-radius: 12px;
box-shadow: 0 12px 32px rgba(0, 0, 0, 0.35);
display: inline-flex;
gap: 12px;
}
input[type="text"] {
padding: 12px 14px;
font-size: 16px;
width: 300px;
border: 1px solid #2d2d2d;
border-radius: 8px;
background-color: #1b1b1b;
color: #f5f5f5;
}
input[type="text"]::placeholder {
color: #9ca3af;
}
input[type="text"]:focus {
outline: none;
border-color: #2563eb;
box-shadow: 0 0 0 3px rgba(37, 99, 235, 0.35);
}
button {
padding: 12px 24px;
font-size: 16px;
background-color: #2563eb;
color: #f5f5f5;
border: 1px solid #1d4ed8;
border-radius: 8px;
cursor: pointer;
transition: background-color 0.3s, transform 0.1s;
box-shadow: 0 10px 25px rgba(37, 99, 235, 0.25);
}
button:hover {
background-color: #1d4ed8;
}
button:active {
transform: scale(0.98);
}
#output {
margin-top: 30px;
padding: 20px;
font-size: 24px;
font-weight: bold;
color: #86efac;
min-height: 50px;
background-color: #1e1e1e;
border-radius: 12px;
box-shadow: 0 10px 25px rgba(0, 0, 0, 0.35);
}
</style>
</head>
<body>
<h1>Enter Your Name</h1>
<div class="input-container">
<input type="text" id="nameInput" placeholder="Type your name here...">
<button id="submitBtn">Submit</button>
</div>
<div id="output"></div>
<script>
const nameInput = document.getElementById('nameInput');
const submitBtn = document.getElementById('submitBtn');
const output = document.getElementById('output');
function showGreeting() {
const name = nameInput.value.trim();
if (name) {
output.textContent = `Hello, ${name}!`;
} else {
output.textContent = 'Please enter a name!';
}
}
submitBtn.addEventListener('click', showGreeting);
// Allow pressing Enter to submit
nameInput.addEventListener('keypress', function(event) {
if (event.key === 'Enter') {
showGreeting();
}
});
</script>
</body>
</html>

308
tests/sdk/web/login.html Normal file
View File

@@ -0,0 +1,308 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Login Demo</title>
<style>
body {
font-family: Arial, sans-serif;
max-width: 600px;
margin: 50px auto;
padding: 20px;
text-align: center;
background-color: #121212;
color: #f5f5f5;
}
h1 {
color: #f5f5f5;
}
.view {
display: none;
}
.view.active {
display: block;
}
.form-container {
margin: 30px 0;
padding: 30px;
background-color: #1e1e1e;
border-radius: 12px;
box-shadow: 0 12px 32px rgba(0, 0, 0, 0.35);
}
.input-group {
margin: 20px 0;
text-align: left;
}
label {
display: block;
margin-bottom: 8px;
font-size: 14px;
font-weight: 500;
color: #d1d5db;
}
input[type="text"],
input[type="password"] {
padding: 12px 14px;
font-size: 16px;
width: 100%;
box-sizing: border-box;
border: 1px solid #2d2d2d;
border-radius: 8px;
background-color: #1b1b1b;
color: #f5f5f5;
}
input[type="text"]::placeholder,
input[type="password"]::placeholder {
color: #9ca3af;
}
input[type="text"]:focus,
input[type="password"]:focus {
outline: none;
border-color: #2563eb;
box-shadow: 0 0 0 3px rgba(37, 99, 235, 0.35);
}
button {
padding: 12px 24px;
font-size: 16px;
background-color: #2563eb;
color: #f5f5f5;
border: 1px solid #1d4ed8;
border-radius: 8px;
cursor: pointer;
transition: background-color 0.3s, transform 0.1s;
box-shadow: 0 10px 25px rgba(37, 99, 235, 0.25);
margin: 10px 5px;
}
button:hover {
background-color: #1d4ed8;
}
button:active {
transform: scale(0.98);
}
button.secondary {
background-color: #374151;
border: 1px solid #4b5563;
box-shadow: 0 10px 25px rgba(55, 65, 81, 0.25);
}
button.secondary:hover {
background-color: #4b5563;
}
.error {
margin-top: 15px;
padding: 12px;
background-color: rgba(239, 68, 68, 0.15);
border: 1px solid rgba(239, 68, 68, 0.3);
border-radius: 8px;
color: #fca5a5;
font-size: 14px;
}
.welcome {
margin: 30px 0;
padding: 20px;
font-size: 20px;
color: #86efac;
background-color: #1e1e1e;
border-radius: 12px;
box-shadow: 0 10px 25px rgba(0, 0, 0, 0.35);
}
.account-info {
margin: 30px 0;
padding: 25px;
background-color: #1e1e1e;
border-radius: 12px;
box-shadow: 0 12px 32px rgba(0, 0, 0, 0.35);
text-align: left;
}
.info-row {
display: flex;
justify-content: space-between;
padding: 12px 0;
border-bottom: 1px solid #2d2d2d;
}
.info-row:last-child {
border-bottom: none;
}
.info-label {
font-weight: 500;
color: #9ca3af;
}
.info-value {
color: #f5f5f5;
}
.click-section {
margin: 30px 0;
padding: 20px;
background-color: #1e1e1e;
border-radius: 12px;
box-shadow: 0 10px 25px rgba(0, 0, 0, 0.35);
}
.click-counter {
margin-top: 15px;
font-size: 20px;
font-weight: bold;
color: #86efac;
}
</style>
</head>
<body>
<!-- Login View -->
<div id="loginView" class="view active">
<h1>Login</h1>
<div class="form-container">
<div class="input-group">
<label for="username">Username</label>
<input type="text" id="username" placeholder="Enter username">
</div>
<div class="input-group">
<label for="password">Password</label>
<input type="password" id="password" placeholder="Enter password">
</div>
<button id="loginBtn">Login</button>
<div id="errorMessage" class="error" style="display: none;"></div>
</div>
</div>
<!-- Main View -->
<div id="mainView" class="view">
<h1>Dashboard</h1>
<div class="welcome" id="welcomeMessage"></div>
<button id="accountBtn">Account</button>
<button id="logoutBtn" class="secondary">Logout</button>
</div>
<!-- Account View -->
<div id="accountView" class="view">
<h1>Account Information</h1>
<div class="account-info">
<div class="info-row">
<span class="info-label">Username:</span>
<span class="info-value" id="accountUsername"></span>
</div>
<div class="info-row">
<span class="info-label">Email:</span>
<span class="info-value" id="accountEmail"></span>
</div>
<div class="info-row">
<span class="info-label">Member Since:</span>
<span class="info-value" id="accountMemberSince"></span>
</div>
<div class="info-row">
<span class="info-label">Account Type:</span>
<span class="info-value" id="accountType"></span>
</div>
</div>
<div class="click-section">
<button id="clickMeBtn">Click Me</button>
<div id="clickCounter" class="click-counter">Button clicked 0 times</div>
</div>
<button id="backBtn" class="secondary">Back to Dashboard</button>
<button id="logoutFromAccountBtn" class="secondary">Logout</button>
</div>
<script>
// In-memory state
let currentUser = null;
let clickCount = 0;
// Views
const loginView = document.getElementById('loginView');
const mainView = document.getElementById('mainView');
const accountView = document.getElementById('accountView');
// Login elements
const usernameInput = document.getElementById('username');
const passwordInput = document.getElementById('password');
const loginBtn = document.getElementById('loginBtn');
const errorMessage = document.getElementById('errorMessage');
// Main view elements
const welcomeMessage = document.getElementById('welcomeMessage');
const accountBtn = document.getElementById('accountBtn');
const logoutBtn = document.getElementById('logoutBtn');
// Account view elements
const accountUsername = document.getElementById('accountUsername');
const accountEmail = document.getElementById('accountEmail');
const accountMemberSince = document.getElementById('accountMemberSince');
const accountType = document.getElementById('accountType');
const clickMeBtn = document.getElementById('clickMeBtn');
const clickCounter = document.getElementById('clickCounter');
const backBtn = document.getElementById('backBtn');
const logoutFromAccountBtn = document.getElementById('logoutFromAccountBtn');
// Fake user data
const validCredentials = {
username: 'testlogin',
password: 'testpassword',
email: 'testlogin@example.com',
memberSince: 'January 2024',
accountType: 'Premium'
};
function showView(view) {
loginView.classList.remove('active');
mainView.classList.remove('active');
accountView.classList.remove('active');
view.classList.add('active');
}
function login() {
const username = usernameInput.value.trim();
const password = passwordInput.value;
if (username === validCredentials.username && password === validCredentials.password) {
currentUser = validCredentials;
welcomeMessage.textContent = `Welcome back, ${currentUser.username}!`;
errorMessage.style.display = 'none';
usernameInput.value = '';
passwordInput.value = '';
showView(mainView);
} else {
errorMessage.textContent = 'Invalid username or password';
errorMessage.style.display = 'block';
}
}
function logout() {
currentUser = null;
showView(loginView);
}
function showAccount() {
if (currentUser) {
accountUsername.textContent = currentUser.username;
accountEmail.textContent = currentUser.email;
accountMemberSince.textContent = currentUser.memberSince;
accountType.textContent = currentUser.accountType;
showView(accountView);
}
}
// Event listeners
loginBtn.addEventListener('click', login);
usernameInput.addEventListener('keypress', function(event) {
if (event.key === 'Enter') {
login();
}
});
passwordInput.addEventListener('keypress', function(event) {
if (event.key === 'Enter') {
login();
}
});
accountBtn.addEventListener('click', showAccount);
backBtn.addEventListener('click', () => showView(mainView));
logoutBtn.addEventListener('click', logout);
logoutFromAccountBtn.addEventListener('click', logout);
clickMeBtn.addEventListener('click', function() {
clickCount++;
clickCounter.textContent = `Button clicked ${clickCount} times`;
});
</script>
</body>
</html>

101
tests/sdk/web/upload.html Normal file
View File

@@ -0,0 +1,101 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Image Upload</title>
<style>
body {
font-family: Arial, sans-serif;
max-width: 800px;
margin: 50px auto;
padding: 20px;
text-align: center;
background-color: #121212;
color: #f5f5f5;
}
h1 {
color: #f5f5f5;
}
.upload-container {
margin: 30px 0;
padding: 40px;
border: 2px dashed #3f3f3f;
border-radius: 12px;
background-color: #1e1e1e;
box-shadow: 0 12px 32px rgba(0, 0, 0, 0.4);
}
input[type="file"] {
padding: 10px;
font-size: 16px;
cursor: pointer;
color: #f5f5f5;
}
input[type="file"]::-webkit-file-upload-button {
padding: 10px 18px;
font-size: 14px;
background-color: #2563eb;
color: #f5f5f5;
border: 1px solid #1d4ed8;
border-radius: 8px;
cursor: pointer;
transition: background-color 0.3s, transform 0.1s;
}
input[type="file"]::-webkit-file-upload-button:hover {
background-color: #1d4ed8;
}
input[type="file"]::-webkit-file-upload-button:active {
transform: scale(0.98);
}
#preview {
margin-top: 30px;
background-color: #1e1e1e;
border-radius: 12px;
padding: 30px;
box-shadow: 0 12px 32px rgba(0, 0, 0, 0.4);
}
#preview img {
max-width: 100%;
max-height: 500px;
border-radius: 8px;
box-shadow: 0 12px 32px rgba(0, 0, 0, 0.4);
}
.hidden {
display: none;
}
</style>
</head>
<body>
<h1>Upload an Image</h1>
<div class="upload-container">
<input type="file" id="imageUpload" accept="image/*">
</div>
<div id="preview" class="hidden">
<h2>Your Image:</h2>
<img id="uploadedImage" src="" alt="Uploaded image">
</div>
<script>
const fileInput = document.getElementById('imageUpload');
const preview = document.getElementById('preview');
const uploadedImage = document.getElementById('uploadedImage');
fileInput.addEventListener('change', function(event) {
const file = event.target.files[0];
if (file && file.type.startsWith('image/')) {
const reader = new FileReader();
reader.onload = function(e) {
uploadedImage.src = e.target.result;
preview.classList.remove('hidden');
};
reader.readAsDataURL(file);
}
});
</script>
</body>
</html>