Integrate posthog feature flags + setup Claude 3 experiment (#251)

skyvern/config.py

@@ -69,6 +69,7 @@ class Settings(BaseSettings):
     ENABLE_OPENAI: bool = True
     ENABLE_ANTHROPIC: bool = False
     ENABLE_AZURE: bool = False
+    ENABLE_BEDROCK: bool = False
     # OPENAI
     OPENAI_API_KEY: str | None = None
     # ANTHROPIC

skyvern/forge/app.py

@@ -11,6 +11,7 @@ from skyvern.forge.sdk.api.llm.api_handler_factory import LLMAPIHandlerFactory
 from skyvern.forge.sdk.artifact.manager import ArtifactManager
 from skyvern.forge.sdk.artifact.storage.factory import StorageFactory
 from skyvern.forge.sdk.db.client import AgentDB
+from skyvern.forge.sdk.experimentation.providers import BaseExperimentationProvider, NoOpExperimentationProvider
 from skyvern.forge.sdk.forge_log import setup_logger
 from skyvern.forge.sdk.models import Organization
 from skyvern.forge.sdk.schemas.tasks import Task
@@ -35,6 +36,7 @@ DATABASE = AgentDB(
 STORAGE = StorageFactory.get_storage()
 ARTIFACT_MANAGER = ArtifactManager()
 BROWSER_MANAGER = BrowserManager()
+EXPERIMENTATION_PROVIDER: BaseExperimentationProvider = NoOpExperimentationProvider()
 LLM_API_HANDLER = LLMAPIHandlerFactory.get_llm_api_handler(SettingsManager.get_settings().LLM_KEY)
 WORKFLOW_CONTEXT_MANAGER = WorkflowContextManager()
 WORKFLOW_SERVICE = WorkflowService()

skyvern/forge/sdk/api/llm/config_registry.py

@@ -60,7 +60,29 @@ if SettingsManager.get_settings().ENABLE_OPENAI:
 
 if SettingsManager.get_settings().ENABLE_ANTHROPIC:
     LLMConfigRegistry.register_config(
-        "ANTHROPIC_CLAUDE3", LLMConfig("anthropic/claude-3-opus-20240229", ["ANTHROPIC_API_KEY"], True)
+        "ANTHROPIC_CLAUDE3_OPUS", LLMConfig("anthropic/claude-3-opus-20240229", ["ANTHROPIC_API_KEY"], True)
+    )
+    LLMConfigRegistry.register_config(
+        "ANTHROPIC_CLAUDE3_SONNET", LLMConfig("anthropic/claude-3-sonnet-20240229", ["ANTHROPIC_API_KEY"], True)
+    )
+
+if SettingsManager.get_settings().ENABLE_BEDROCK:
+    # Supported through AWS IAM authentication
+    LLMConfigRegistry.register_config(
+        "BEDROCK_ANTHROPIC_CLAUDE3_OPUS",
+        LLMConfig(
+            "bedrock/anthropic.claude-3-opus-20240229-v1:0",
+            ["AWS_REGION"],
+            True,
+        ),
+    )
+    LLMConfigRegistry.register_config(
+        "BEDROCK_ANTHROPIC_CLAUDE3_SONNET",
+        LLMConfig(
+            "bedrock/anthropic.claude-3-sonnet-20240229-v1:0",
+            ["AWS_REGION"],
+            True,
+        ),
     )
 
 if SettingsManager.get_settings().ENABLE_AZURE:

skyvern/forge/sdk/api/llm/models.py

@@ -46,7 +46,7 @@ class LLMRouterConfig(LLMConfig):
         "usage-based-routing-v2",
         "latency-based-routing",
     ] = "usage-based-routing"
-    num_retries: int = 2
+    num_retries: int = 1
     retry_delay_seconds: int = 15
     set_verbose: bool = False
 

skyvern/forge/sdk/experimentation/providers.py

@@ -0,0 +1,16 @@
+from abc import ABC, abstractmethod
+
+import structlog
+
+LOG = structlog.get_logger()
+
+
+class BaseExperimentationProvider(ABC):
+    @abstractmethod
+    def is_feature_enabled(self, feature_name: str, distinct_id: str) -> bool:
+        """Check if a specific feature is enabled."""
+
+
+class NoOpExperimentationProvider(BaseExperimentationProvider):
+    def is_feature_enabled(self, feature_name: str, distinct_id: str) -> bool:
+        return False