diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 24b08702..5104e06f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -51,6 +51,14 @@ repos: - id: python-check-mock-methods - id: python-no-log-warn - id: python-use-type-annotations + - repo: https://github.com/asottile/pyupgrade + rev: v3.20.0 + hooks: + - id: pyupgrade + exclude: | + (?x)( + ^skyvern/client/.* + ) - repo: https://github.com/pre-commit/mirrors-mypy rev: v1.16.0 hooks: @@ -61,7 +69,7 @@ repos: - types-requests - types-cachetools - alembic - - "sqlalchemy[mypy]" + - 'sqlalchemy[mypy]' - types-PyYAML - types-aiofiles exclude: | @@ -91,7 +99,7 @@ repos: # pass_filenames: false # always_run: true - repo: https://github.com/pre-commit/mirrors-prettier - rev: "v4.0.0-alpha.8" # Use the sha or tag you want to point at + rev: 'v4.0.0-alpha.8' # Use the sha or tag you want to point at hooks: - id: prettier types: [javascript] diff --git a/evaluation/core/utils.py b/evaluation/core/utils.py index 4e1cd936..9bc15fd8 100644 --- a/evaluation/core/utils.py +++ b/evaluation/core/utils.py @@ -21,13 +21,13 @@ class WorkflowRunResultRequest(BaseModel): def load_webvoyager_case_from_json(file_path: str, group_id: str = "") -> Iterator[WebVoyagerTestCase]: - with open("evaluation/datasets/webvoyager_reference_answer.json", "r") as answer_file: + with open("evaluation/datasets/webvoyager_reference_answer.json") as answer_file: webvoyager_answers: dict = json.load(answer_file) if not group_id: group_id = str(uuid4()) - with open(file_path, "r", encoding="utf-8") as file: + with open(file_path, encoding="utf-8") as file: for line in file: test_case: dict[str, str] = json.loads(line) web_name, id = test_case["id"].split("--") @@ -47,7 +47,7 @@ def load_webvoyager_case_from_json(file_path: str, group_id: str = "") -> Iterat def load_records_from_json(file_path: str) -> Iterator[WorkflowRunResultRequest]: - with open(file_path, "r", encoding="utf-8") as f: + with open(file_path, encoding="utf-8") as f: for line in f: item: dict[str, str] = json.loads(line) id = item["id"] diff --git a/evaluation/script/create_webvoyager_evaluation_result.py b/evaluation/script/create_webvoyager_evaluation_result.py index 506a52e0..d4b189cd 100644 --- a/evaluation/script/create_webvoyager_evaluation_result.py +++ b/evaluation/script/create_webvoyager_evaluation_result.py @@ -35,7 +35,7 @@ def main( ) -> None: client = SkyvernClient(base_url=base_url, credentials=cred) - with open(record_json_path, "r", encoding="utf-8") as file: + with open(record_json_path, encoding="utf-8") as file: with open(output_csv_path, newline="", mode="w", encoding="utf-8") as csv_file: writer = csv.DictWriter(csv_file, fieldnames=csv_headers) writer.writeheader() diff --git a/evaluation/script/eval_webvoyager_task_v2.py b/evaluation/script/eval_webvoyager_task_v2.py index f9f67e33..8bca329c 100644 --- a/evaluation/script/eval_webvoyager_task_v2.py +++ b/evaluation/script/eval_webvoyager_task_v2.py @@ -79,7 +79,7 @@ async def run_eval( ) -> None: client = SkyvernClient(base_url=base_url, credentials=cred) - with open(record_json_path, "r", encoding="utf-8") as file: + with open(record_json_path, encoding="utf-8") as file: with open(output_csv_path, newline="", mode="w", encoding="utf-8") as csv_file: writer = csv.DictWriter(csv_file, fieldnames=csv_headers) writer.writeheader() diff --git a/skyvern/analytics.py b/skyvern/analytics.py index 21d2bc0d..2294c236 100644 --- a/skyvern/analytics.py +++ b/skyvern/analytics.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import importlib.metadata import platform from typing import Any, Dict diff --git a/skyvern/cli/mcp.py b/skyvern/cli/mcp.py index 64030d46..498e6e29 100644 --- a/skyvern/cli/mcp.py +++ b/skyvern/cli/mcp.py @@ -144,7 +144,7 @@ def setup_claude_desktop_config(host_system: str, path_to_env: str) -> bool: claude_config: dict = {"mcpServers": {}} if os.path.exists(path_claude_config): try: - with open(path_claude_config, "r") as f: + with open(path_claude_config) as f: claude_config = json.load(f) claude_config["mcpServers"].pop("Skyvern", None) claude_config["mcpServers"]["Skyvern"] = { @@ -196,7 +196,7 @@ def setup_cursor_config(host_system: str, path_to_env: str) -> bool: cursor_config: dict = {"mcpServers": {}} if os.path.exists(path_cursor_config): try: - with open(path_cursor_config, "r") as f: + with open(path_cursor_config) as f: cursor_config = json.load(f) cursor_config["mcpServers"].pop("Skyvern", None) cursor_config["mcpServers"]["Skyvern"] = { @@ -245,7 +245,7 @@ def setup_windsurf_config(host_system: str, path_to_env: str) -> bool: windsurf_config: dict = {"mcpServers": {}} if os.path.exists(path_windsurf_config): try: - with open(path_windsurf_config, "r") as f: + with open(path_windsurf_config) as f: windsurf_config = json.load(f) windsurf_config["mcpServers"].pop("Skyvern", None) windsurf_config["mcpServers"]["Skyvern"] = { diff --git a/skyvern/cli/tasks.py b/skyvern/cli/tasks.py index 214cb134..823eae86 100644 --- a/skyvern/cli/tasks.py +++ b/skyvern/cli/tasks.py @@ -4,7 +4,6 @@ from __future__ import annotations import json import os -from typing import Optional import typer from dotenv import load_dotenv @@ -21,7 +20,7 @@ tasks_app = typer.Typer(help="Manage Skyvern tasks and operations.") @tasks_app.callback() def tasks_callback( ctx: typer.Context, - api_key: Optional[str] = typer.Option( + api_key: str | None = typer.Option( None, "--api-key", help="Skyvern API key", @@ -32,7 +31,7 @@ def tasks_callback( ctx.obj = {"api_key": api_key} -def _get_client(api_key: Optional[str] = None) -> Skyvern: +def _get_client(api_key: str | None = None) -> Skyvern: """Instantiate a Skyvern SDK client using environment variables.""" load_dotenv() load_dotenv(".env") diff --git a/skyvern/cli/workflow.py b/skyvern/cli/workflow.py index bc9ffb8f..f025b4aa 100644 --- a/skyvern/cli/workflow.py +++ b/skyvern/cli/workflow.py @@ -4,7 +4,6 @@ from __future__ import annotations import json import os -from typing import Optional import typer from dotenv import load_dotenv @@ -22,7 +21,7 @@ workflow_app = typer.Typer(help="Manage Skyvern workflows.") @workflow_app.callback() def workflow_callback( ctx: typer.Context, - api_key: Optional[str] = typer.Option( + api_key: str | None = typer.Option( None, "--api-key", help="Skyvern API key", @@ -33,7 +32,7 @@ def workflow_callback( ctx.obj = {"api_key": api_key} -def _get_client(api_key: Optional[str] = None) -> Skyvern: +def _get_client(api_key: str | None = None) -> Skyvern: """Instantiate a Skyvern SDK client using environment variables.""" load_dotenv() load_dotenv(".env") @@ -46,8 +45,8 @@ def start_workflow( ctx: typer.Context, workflow_id: str = typer.Argument(..., help="Workflow permanent ID"), parameters: str = typer.Option("{}", "--parameters", "-p", help="JSON parameters for the workflow"), - title: Optional[str] = typer.Option(None, "--title", help="Title for the workflow run"), - max_steps: Optional[int] = typer.Option(None, "--max-steps", help="Override the workflow max steps"), + title: str | None = typer.Option(None, "--title", help="Title for the workflow run"), + max_steps: int | None = typer.Option(None, "--max-steps", help="Override the workflow max steps"), ) -> None: """Dispatch a workflow run.""" try: diff --git a/skyvern/forge/sdk/api/aws.py b/skyvern/forge/sdk/api/aws.py index 9087f693..cab8708a 100644 --- a/skyvern/forge/sdk/api/aws.py +++ b/skyvern/forge/sdk/api/aws.py @@ -255,7 +255,7 @@ class AsyncAWSClient: return await client.deregister_task_definition(taskDefinition=task_definition) -class S3Uri(object): +class S3Uri: # From: https://stackoverflow.com/questions/42641315/s3-urls-get-bucket-name-and-path """ >>> s = S3Uri("s3://bucket/hello/world") diff --git a/skyvern/forge/sdk/artifact/storage/local.py b/skyvern/forge/sdk/artifact/storage/local.py index cfd7f764..ef94faf4 100644 --- a/skyvern/forge/sdk/artifact/storage/local.py +++ b/skyvern/forge/sdk/artifact/storage/local.py @@ -38,7 +38,7 @@ class LocalStorage(BaseStorage): if not file_path.exists(): return [] try: - with open(file_path, "r") as f: + with open(file_path) as f: return [line.strip() for line in f.readlines() if line.strip()] except Exception: return [] diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.py index a43799d1..6761f750 100644 --- a/skyvern/forge/sdk/workflow/models/block.py +++ b/skyvern/forge/sdk/workflow/models/block.py @@ -211,7 +211,7 @@ class Block(BaseModel, abc.ABC): return template.render(template_data) @classmethod - def get_subclasses(cls) -> tuple[type["Block"], ...]: + def get_subclasses(cls) -> tuple[type[Block], ...]: return tuple(cls.__subclasses__()) @staticmethod @@ -2123,7 +2123,7 @@ class FileParserBlock(Block): def validate_file_type(self, file_url_used: str, file_path: str) -> None: if self.file_type == FileType.CSV: try: - with open(file_path, "r") as file: + with open(file_path) as file: csv.Sniffer().sniff(file.read(1024)) except csv.Error as e: raise InvalidFileType(file_url=file_url_used, file_type=self.file_type, error=str(e)) @@ -2172,7 +2172,7 @@ class FileParserBlock(Block): self.validate_file_type(self.file_url, file_path) # Parse the file into a list of dictionaries where each dictionary represents a row in the file parsed_data = [] - with open(file_path, "r") as file: + with open(file_path) as file: if self.file_type == FileType.CSV: reader = csv.DictReader(file) for row in reader: diff --git a/skyvern/utils/__init__.py b/skyvern/utils/__init__.py index 72b55bad..c615c57f 100644 --- a/skyvern/utils/__init__.py +++ b/skyvern/utils/__init__.py @@ -28,7 +28,7 @@ def detect_os() -> str: system = platform.system() if system == "Linux": try: - with open("/proc/version", "r") as f: + with open("/proc/version") as f: version_info = f.read().lower() if "microsoft" in version_info: return "wsl" diff --git a/skyvern/utils/files.py b/skyvern/utils/files.py index ce614540..5190236d 100644 --- a/skyvern/utils/files.py +++ b/skyvern/utils/files.py @@ -25,5 +25,5 @@ def get_json_from_file(file_path: str) -> dict[str, str]: if not os.path.exists(file_path): return {} - with open(file_path, "r") as json_file: + with open(file_path) as json_file: return json.load(json_file) diff --git a/skyvern/webeye/browser_factory.py b/skyvern/webeye/browser_factory.py index bba67b13..d5ea7d10 100644 --- a/skyvern/webeye/browser_factory.py +++ b/skyvern/webeye/browser_factory.py @@ -162,7 +162,7 @@ class BrowserContextFactory: preference_template = f"{SKYVERN_DIR}/webeye/chromium_preferences.json" preference_file_content = "" - with open(preference_template, "r") as f: + with open(preference_template) as f: preference_file_content = f.read() preference_file_content = preference_file_content.replace("MASK_SAVEFILE_DEFAULT_DIRECTORY", download_dir) preference_file_content = preference_file_content.replace("MASK_DOWNLOAD_DEFAULT_DIRECTORY", download_dir) @@ -281,7 +281,7 @@ class BrowserContextFactory: class VideoArtifact(BaseModel): video_path: str | None = None video_artifact_id: str | None = None - video_data: bytes = bytes() + video_data: bytes = b"" class BrowserArtifacts(BaseModel): @@ -385,7 +385,7 @@ def _is_port_in_use(port: int) -> bool: try: s.bind(("localhost", port)) return False - except socket.error: + except OSError: return True diff --git a/skyvern/webeye/persistent_sessions_manager.py b/skyvern/webeye/persistent_sessions_manager.py index a85ba4ff..29c84bef 100644 --- a/skyvern/webeye/persistent_sessions_manager.py +++ b/skyvern/webeye/persistent_sessions_manager.py @@ -1,7 +1,6 @@ from __future__ import annotations from dataclasses import dataclass -from typing import Dict, Optional, Tuple import structlog from playwright._impl._errors import TargetClosedError @@ -22,7 +21,7 @@ class BrowserSession: class PersistentSessionsManager: instance: PersistentSessionsManager | None = None - _browser_sessions: Dict[str, BrowserSession] = dict() + _browser_sessions: dict[str, BrowserSession] = dict() database: AgentDB def __new__(cls, database: AgentDB) -> PersistentSessionsManager: @@ -82,7 +81,7 @@ class PersistentSessionsManager: organization_id=organization_id, ) - async def get_network_info(self, session_id: str) -> Tuple[Optional[int], Optional[str]]: + async def get_network_info(self, session_id: str) -> tuple[int | None, str | None]: """Returns cdp port and ip address of the browser session""" browser_session = self._browser_sessions.get(session_id) if browser_session: diff --git a/skyvern/webeye/scraper/scraper.py b/skyvern/webeye/scraper/scraper.py index 835bdd8d..e24b8847 100644 --- a/skyvern/webeye/scraper/scraper.py +++ b/skyvern/webeye/scraper/scraper.py @@ -75,7 +75,7 @@ def load_js_script() -> str: try: # TODO: Implement TS of domUtils.js and use the complied JS file instead of the raw JS file. # This will allow our code to be type safe. - with open(path, "r") as f: + with open(path) as f: return f.read() except FileNotFoundError as e: LOG.exception("Failed to load the JS script", path=path) diff --git a/skyvern/webeye/utils/dom.py b/skyvern/webeye/utils/dom.py index 73804f41..31b9641a 100644 --- a/skyvern/webeye/utils/dom.py +++ b/skyvern/webeye/utils/dom.py @@ -34,9 +34,7 @@ TEXT_INPUT_DELAY = 10 # 10ms between each character input TEXT_PRESS_MAX_LENGTH = 20 -async def resolve_locator( - scrape_page: ScrapedPage, page: Page, frame: str, css: str -) -> typing.Tuple[Locator, Page | Frame]: +async def resolve_locator(scrape_page: ScrapedPage, page: Page, frame: str, css: str) -> tuple[Locator, Page | Frame]: iframe_path: list[str] = [] while frame != "main.frame": @@ -335,10 +333,10 @@ class SkyvernElement: def get_frame_id(self) -> str: return self._frame_id - def get_attributes(self) -> typing.Dict: + def get_attributes(self) -> dict: return self._attributes - def get_options(self) -> typing.List[SkyvernOptionType]: + def get_options(self) -> list[SkyvernOptionType]: options = self.__static_element.get("options", None) if options is None: return [] diff --git a/skyvern/webeye/utils/page.py b/skyvern/webeye/utils/page.py index ab70d6db..d1f9546d 100644 --- a/skyvern/webeye/utils/page.py +++ b/skyvern/webeye/utils/page.py @@ -2,7 +2,7 @@ from __future__ import annotations import asyncio import time -from typing import Any, Dict, List +from typing import Any import structlog from playwright._impl._errors import TimeoutError @@ -21,7 +21,7 @@ def load_js_script() -> str: try: # TODO: Implement TS of domUtils.js and use the complied JS file instead of the raw JS file. # This will allow our code to be type safe. - with open(path, "r") as f: + with open(path) as f: return f.read() except FileNotFoundError as e: LOG.exception("Failed to load the JS script", path=path) @@ -43,7 +43,7 @@ async def _current_viewpoint_screenshot_helper( await page.wait_for_load_state(timeout=settings.BROWSER_LOADING_TIMEOUT_MS) LOG.debug("Page is fully loaded, agent is about to take screenshots") start_time = time.time() - screenshot: bytes = bytes() + screenshot: bytes = b"" if file_path: screenshot = await page.screenshot( path=file_path, @@ -77,14 +77,14 @@ async def _scrolling_screenshots_helper( url: str | None = None, draw_boxes: bool = False, max_number: int = settings.MAX_NUM_SCREENSHOTS, -) -> List[bytes]: +) -> list[bytes]: skyvern_page = await SkyvernFrame.create_instance(frame=page) # page is the main frame and the index must be 0 assert isinstance(skyvern_page.frame, Page) frame = "main.frame" frame_index = 0 - screenshots: List[bytes] = [] + screenshots: list[bytes] = [] if await skyvern_page.is_window_scrollable(): scroll_y_px_old = -30.0 scroll_y_px = await skyvern_page.scroll_to_top(draw_boxes=draw_boxes, frame=frame, frame_index=frame_index) @@ -161,7 +161,7 @@ class SkyvernFrame: draw_boxes: bool = False, max_number: int = settings.MAX_NUM_SCREENSHOTS, scroll: bool = True, - ) -> List[bytes]: + ) -> list[bytes]: if not scroll: return [await _current_viewpoint_screenshot_helper(page=page)] @@ -199,7 +199,7 @@ class SkyvernFrame: js_script = "(element) => scrollToElementTop(element)" return await self.evaluate(frame=self.frame, expression=js_script, arg=element) - async def parse_element_from_html(self, frame: str, element: ElementHandle, interactable: bool) -> Dict: + async def parse_element_from_html(self, frame: str, element: ElementHandle, interactable: bool) -> dict: js_script = "async ([frame, element, interactable]) => await buildElementObject(frame, element, interactable)" return await self.evaluate(frame=self.frame, expression=js_script, arg=[frame, element, interactable])