Issue-2756: updated alembic setup to account for asyncpg mode (#3431)

2025-09-16 00:33:51 -04:00
parent b6c1e16c96
commit 323952169e
5 changed files with 140 additions and 25 deletions
--- a/README.md
+++ b/README.md
@@ -44,6 +44,14 @@ If you'd like to try it out, navigate to [app.skyvern.com](https://app.skyvern.c

 ## Install & Run

+Dependencies needed:
+- [Python 3.11.x](https://www.python.org/downloads/), works with 3.12, not ready yet for 3.13
+- [NodeJS & NPM](https://nodejs.org/en/download/)
+
+Additionally, for Windows:
+- [Rust](https://rustup.rs/)
+- VS Code with C++ dev tools and Windows SDK
+
 ### 1. Install Skyvern

 ```bash
@@ -51,6 +59,7 @@ pip install skyvern
 ```

 ### 2. Run Skyvern
+This is most helpful for first time run (db setup, db migrations etc).

 ```bash
 skyvern quickstart
@@ -60,7 +69,7 @@ skyvern quickstart

 #### UI (Recommended)

-Start the Skyvern service and UI
+Start the Skyvern service and UI (when DB is up and running)

 ```bash
 skyvern run all
--- a/alembic/env.py
+++ b/alembic/env.py
@@ -1,6 +1,9 @@
+import asyncio
 from logging.config import fileConfig

-from sqlalchemy import engine_from_config, pool
+from sqlalchemy import pool
+from sqlalchemy.engine import Connection
+from sqlalchemy.ext.asyncio import create_async_engine

 from alembic import context

@@ -54,28 +57,49 @@ def run_migrations_offline() -> None:
        context.run_migrations()


-def run_migrations_online() -> None:
+def do_run_migrations(connection: Connection):
    """Run migrations in 'online' mode.

    In this scenario we need to create an Engine
    and associate a connection with the context.

    """
-    connectable = engine_from_config(
-        config.get_section(config.config_ini_section, {}),
-        prefix="sqlalchemy.",
+    context.configure(connection=connection, target_metadata=target_metadata)
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+async def run_migrations_online():
+    connectable = create_async_engine(
+        config.get_main_option("sqlalchemy.url"),
        poolclass=pool.NullPool,
    )

-    with connectable.connect() as connection:
-        context.configure(connection=connection, target_metadata=target_metadata)
+    async with connectable.connect() as connection:
+        await connection.run_sync(do_run_migrations)

-        with context.begin_transaction():
-            context.run_migrations()
+    await connectable.dispose()


 print("Alembic mode: ", "offline" if context.is_offline_mode() else "online")
 if context.is_offline_mode():
    run_migrations_offline()
 else:
-    run_migrations_online()
+    async def async_main():
+        await run_migrations_online()
+
+    try:
+        loop = asyncio.get_running_loop()
+    except RuntimeError:
+        # No running loop -> safe to start one
+        print("Alembic: no running loop")
+        asyncio.run(async_main())
+    else:
+        # Already running loop -> schedule task and await it
+        print("Alembic: schedule task")
+        import concurrent.futures
+
+        # Use a ThreadPoolExecutor to run the async function in a new thread
+        with concurrent.futures.ThreadPoolExecutor() as executor:
+            future = executor.submit(asyncio.run, async_main())
+            future.result()  # This blocks until completion
--- a/skyvern-frontend/package-lock.json
+++ b/skyvern-frontend/package-lock.json
@@ -4353,7 +4353,8 @@
    "node_modules/asynckit": {
      "version": "0.4.0",
      "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
-      "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="
+      "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==",
+      "license": "MIT"
    },
    "node_modules/autoprefixer": {
      "version": "10.4.18",
@@ -4393,12 +4394,13 @@
      }
    },
    "node_modules/axios": {
-      "version": "1.8.2",
-      "resolved": "https://registry.npmjs.org/axios/-/axios-1.8.2.tgz",
-      "integrity": "sha512-ls4GYBm5aig9vWx8AWDSGLpnpDQRtWAfrjU+EuytuODrFBkqesN2RkOQCBzrA1RQNHw1SmRMSDDDSwzNAYQ6Rg==",
+      "version": "1.12.1",
+      "resolved": "https://registry.npmjs.org/axios/-/axios-1.12.1.tgz",
+      "integrity": "sha512-Kn4kbSXpkFHCGE6rBFNwIv0GQs4AvDT80jlveJDKFxjbTYMUeB4QtsdPCv6H8Cm19Je7IU6VFtRl2zWZI0rudQ==",
+      "license": "MIT",
      "dependencies": {
        "follow-redirects": "^1.15.6",
-        "form-data": "^4.0.0",
+        "form-data": "^4.0.4",
        "proxy-from-env": "^1.1.0"
      }
    },
@@ -4794,6 +4796,7 @@
      "version": "1.0.8",
      "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
      "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",
+      "license": "MIT",
      "dependencies": {
        "delayed-stream": "~1.0.0"
      },
@@ -5054,6 +5057,7 @@
      "version": "1.0.0",
      "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
      "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==",
+      "license": "MIT",
      "engines": {
        "node": ">=0.4.0"
      }
@@ -5221,6 +5225,21 @@
        "node": ">= 0.4"
      }
    },
+    "node_modules/es-set-tostringtag": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz",
+      "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==",
+      "license": "MIT",
+      "dependencies": {
+        "es-errors": "^1.3.0",
+        "get-intrinsic": "^1.2.6",
+        "has-tostringtag": "^1.0.2",
+        "hasown": "^2.0.2"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      }
+    },
    "node_modules/esbuild": {
      "version": "0.21.5",
      "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.21.5.tgz",
@@ -5800,12 +5819,15 @@
      }
    },
    "node_modules/form-data": {
-      "version": "4.0.0",
-      "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz",
-      "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==",
+      "version": "4.0.4",
+      "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.4.tgz",
+      "integrity": "sha512-KrGhL9Q4zjj0kiUt5OO4Mr/A/jlI2jDYs5eHBpYHPcBEVSiipAvn2Ko2HnPe20rmcuuvMHNdZFp+4IlGTMF0Ow==",
+      "license": "MIT",
      "dependencies": {
        "asynckit": "^0.4.0",
        "combined-stream": "^1.0.8",
+        "es-set-tostringtag": "^2.1.0",
+        "hasown": "^2.0.2",
        "mime-types": "^2.1.12"
      },
      "engines": {
@@ -6065,6 +6087,21 @@
        "url": "https://github.com/sponsors/ljharb"
      }
    },
+    "node_modules/has-tostringtag": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz",
+      "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==",
+      "license": "MIT",
+      "dependencies": {
+        "has-symbols": "^1.0.3"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
    "node_modules/hasown": {
      "version": "2.0.2",
      "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz",
@@ -6748,15 +6785,16 @@
      }
    },
    "node_modules/nanoid": {
-      "version": "5.0.7",
-      "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.0.7.tgz",
-      "integrity": "sha512-oLxFY2gd2IqnjcYyOXD8XGCftpGtZP2AbHbOkthDkvRywH5ayNtPVy9YlOPcHckXzbLTCHpkb7FB+yuxKV13pQ==",
+      "version": "5.1.5",
+      "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.1.5.tgz",
+      "integrity": "sha512-Ir/+ZpE9fDsNH0hQ3C68uyThDXzYcim2EqcZ8zn8Chtt1iylPT9xXJB0kPCnqzgcEGikO9RxSrh63MsmVCU7Fw==",
      "funding": [
        {
          "type": "github",
          "url": "https://github.com/sponsors/ai"
        }
      ],
+      "license": "MIT",
      "bin": {
        "nanoid": "bin/nanoid.js"
      },
@@ -8677,10 +8715,11 @@
      }
    },
    "node_modules/vite": {
-      "version": "5.4.19",
-      "resolved": "https://registry.npmjs.org/vite/-/vite-5.4.19.tgz",
-      "integrity": "sha512-qO3aKv3HoQC8QKiNSTuUM1l9o/XX3+c+VTgLHbJWHZGeTPVAg2XwazI9UWzoxjIJCGCV2zU60uqMzjeLZuULqA==",
+      "version": "5.4.20",
+      "resolved": "https://registry.npmjs.org/vite/-/vite-5.4.20.tgz",
+      "integrity": "sha512-j3lYzGC3P+B5Yfy/pfKNgVEg4+UtcIJcVRt2cDjIOmhLourAqPqf8P7acgxeiSgUB7E3p2P8/3gNIgDLpwzs4g==",
      "dev": true,
+      "license": "MIT",
      "dependencies": {
        "esbuild": "^0.21.3",
        "postcss": "^8.4.43",
--- a/skyvern/cli/utils.py
+++ b/skyvern/cli/utils.py
@@ -1,4 +1,5 @@
 import asyncio
+import logging
 import sys

 import typer
@@ -37,4 +38,5 @@ async def start_services(server_only: bool = False) -> None:

    except Exception as e:
        console.print(f"[bold red]Error starting services: {str(e)}[/bold red]")
+        logging.error("Startup failed", exc_info=True)
        raise typer.Exit(1)
--- a/tests/unit_tests/test_alembic_loop.py
+++ b/tests/unit_tests/test_alembic_loop.py
@@ -0,0 +1,41 @@
+#!/usr/bin/env python3
+"""Test script to verify alembic works correctly with an existing event loop."""
+
+import asyncio
+import os
+import subprocess
+import sys
+
+
+async def test_alembic_with_running_loop():
+    """Test alembic migration execution within an existing event loop."""
+    print("Testing alembic migration with existing event loop...")
+
+    # Change to the project directory
+    os.chdir(os.path.dirname(__file__))
+
+    # Run alembic command in a subprocess
+    try:
+        result = subprocess.run(
+            [sys.executable, "-m", "alembic", "current"], capture_output=True, text=True, timeout=30
+        )
+
+        print(f"Return code: {result.returncode}")
+        print(f"Stdout: {result.stdout}")
+        if result.stderr:
+            print(f"Stderr: {result.stderr}")
+
+        return result.returncode == 0
+    except subprocess.TimeoutExpired:
+        print("ERROR: Alembic command timed out!")
+        return False
+    except Exception as e:
+        print(f"ERROR: {e}")
+        return False
+
+
+if __name__ == "__main__":
+    # This creates an event loop and runs alembic within it
+    success = asyncio.run(test_alembic_with_running_loop())
+    print(f"Test {'PASSED' if success else 'FAILED'}")
+    sys.exit(0 if success else 1)