Feature: Configuring OpenAI-Compatible (LiteLLM) Models (#2166)

Co-authored-by: bbeiler <bbeiler@ridgelineintl.com>

skyvern/cli/commands.py

@@ -326,6 +326,39 @@ def setup_llm_providers() -> None:
     else:
         update_or_add_env_var("ENABLE_NOVITA", "false")
 
+    # OpenAI Compatible Configuration
+    print("To enable an OpenAI-compatible provider, you must have a model name, API key, and API base URL.")
+    enable_openai_compatible = input("Do you want to enable an OpenAI-compatible provider (y/n)? ").lower() == "y"
+    if enable_openai_compatible:
+        openai_compatible_model_name = input("Enter the model name (e.g., 'yi-34b', 'mistral-large'): ")
+        openai_compatible_api_key = input("Enter your API key: ")
+        openai_compatible_api_base = input("Enter the API base URL (e.g., 'https://api.together.xyz/v1'): ")
+        openai_compatible_vision = input("Does this model support vision (y/n)? ").lower() == "y"
+
+        if not all([openai_compatible_model_name, openai_compatible_api_key, openai_compatible_api_base]):
+            print("Error: All required fields must be populated.")
+            print("OpenAI-compatible provider will not be enabled.")
+        else:
+            update_or_add_env_var("OPENAI_COMPATIBLE_MODEL_NAME", openai_compatible_model_name)
+            update_or_add_env_var("OPENAI_COMPATIBLE_API_KEY", openai_compatible_api_key)
+            update_or_add_env_var("OPENAI_COMPATIBLE_API_BASE", openai_compatible_api_base)
+
+            # Set vision support
+            if openai_compatible_vision:
+                update_or_add_env_var("OPENAI_COMPATIBLE_SUPPORTS_VISION", "true")
+            else:
+                update_or_add_env_var("OPENAI_COMPATIBLE_SUPPORTS_VISION", "false")
+
+            # Optional: Ask for API version
+            openai_compatible_api_version = input("Enter API version (optional, press enter to skip): ")
+            if openai_compatible_api_version:
+                update_or_add_env_var("OPENAI_COMPATIBLE_API_VERSION", openai_compatible_api_version)
+
+            update_or_add_env_var("ENABLE_OPENAI_COMPATIBLE", "true")
+            model_options.append("OPENAI_COMPATIBLE")
+    else:
+        update_or_add_env_var("ENABLE_OPENAI_COMPATIBLE", "false")
+
     # Model Selection
     if not model_options:
         print(

skyvern/config.py

@@ -124,10 +124,24 @@ class Settings(BaseSettings):
     ENABLE_BEDROCK: bool = False
     ENABLE_GEMINI: bool = False
     ENABLE_AZURE_CUA: bool = False
+    ENABLE_OPENAI_COMPATIBLE: bool = False
     # OPENAI
     OPENAI_API_KEY: str | None = None
     # ANTHROPIC
     ANTHROPIC_API_KEY: str | None = None
+
+    # OPENAI COMPATIBLE
+    OPENAI_COMPATIBLE_MODEL_NAME: str | None = None
+    OPENAI_COMPATIBLE_API_KEY: str | None = None
+    OPENAI_COMPATIBLE_API_BASE: str | None = None
+    OPENAI_COMPATIBLE_API_VERSION: str | None = None
+    OPENAI_COMPATIBLE_MAX_TOKENS: int | None = None
+    OPENAI_COMPATIBLE_TEMPERATURE: float | None = None
+    OPENAI_COMPATIBLE_SUPPORTS_VISION: bool = False
+    OPENAI_COMPATIBLE_ADD_ASSISTANT_PREFIX: bool = False
+    OPENAI_COMPATIBLE_MODEL_KEY: str = "OPENAI_COMPATIBLE"
+    OPENAI_COMPATIBLE_REASONING_EFFORT: str | None = None
+
     # AZURE
     AZURE_DEPLOYMENT: str | None = None
     AZURE_API_KEY: str | None = None

skyvern/forge/sdk/api/llm/config_registry.py

@@ -606,3 +606,45 @@ if settings.ENABLE_NOVITA:
             ),
         ),
     )
+
+# Add support for dynamically configuring OpenAI-compatible LLM models
+# Based on liteLLM's support for OpenAI-compatible APIs
+# See documentation: https://docs.litellm.ai/docs/providers/openai_compatible
+if settings.ENABLE_OPENAI_COMPATIBLE:
+    # Check for required model name
+    model_key = settings.OPENAI_COMPATIBLE_MODEL_KEY
+    model_name = settings.OPENAI_COMPATIBLE_MODEL_NAME
+
+    if not model_name:
+        raise InvalidLLMConfigError(
+            "OPENAI_COMPATIBLE_MODEL_NAME is required but not set. OpenAI-compatible model will not be registered."
+        )
+    else:
+        # Required environment variables to check
+        required_env_vars = ["OPENAI_COMPATIBLE_API_KEY", "OPENAI_COMPATIBLE_MODEL_NAME", "OPENAI_COMPATIBLE_API_BASE"]
+
+        # Configure litellm parameters - note the "openai/" prefix required for liteLLM routing
+        litellm_params = LiteLLMParams(
+            api_key=settings.OPENAI_COMPATIBLE_API_KEY,
+            api_base=settings.OPENAI_COMPATIBLE_API_BASE,
+            api_version=settings.OPENAI_COMPATIBLE_API_VERSION,
+            model_info={"model_name": f"openai/{model_name}"},
+        )
+
+        # Configure LLMConfig
+        LLMConfigRegistry.register_config(
+            model_key,
+            LLMConfig(
+                f"openai/{model_name}",  # Add openai/ prefix for liteLLM
+                required_env_vars,
+                supports_vision=settings.OPENAI_COMPATIBLE_SUPPORTS_VISION,
+                add_assistant_prefix=settings.OPENAI_COMPATIBLE_ADD_ASSISTANT_PREFIX,
+                max_completion_tokens=settings.OPENAI_COMPATIBLE_MAX_TOKENS or settings.LLM_CONFIG_MAX_TOKENS,
+                temperature=settings.OPENAI_COMPATIBLE_TEMPERATURE
+                if settings.OPENAI_COMPATIBLE_TEMPERATURE is not None
+                else settings.LLM_CONFIG_TEMPERATURE,
+                litellm_params=litellm_params,
+                reasoning_effort=settings.OPENAI_COMPATIBLE_REASONING_EFFORT,
+            ),
+        )
+        LOG.info(f"Registered OpenAI-compatible model with key {model_key}", model_name=model_name)