Add credentials table, CRUD endpoints, and credential parameter (#1767)

Co-authored-by: Muhammed Salih Altun <muhammedsalihaltun@gmail.com>

skyvern/forge/sdk/workflow/context_manager.py

@@ -1,10 +1,17 @@
+import json
 import uuid
 from typing import TYPE_CHECKING, Any, Self
 
 import structlog
 
 from skyvern.config import settings
-from skyvern.exceptions import BitwardenBaseError, SkyvernException, WorkflowRunContextNotInitialized
+from skyvern.exceptions import (
+    BitwardenBaseError,
+    CredentialParameterNotFoundError,
+    CredentialParameterParsingError,
+    SkyvernException,
+    WorkflowRunContextNotInitialized,
+)
 from skyvern.forge.sdk.api.aws import AsyncAWSClient
 from skyvern.forge.sdk.schemas.organizations import Organization
 from skyvern.forge.sdk.schemas.tasks import TaskStatus
@@ -17,6 +24,7 @@ from skyvern.forge.sdk.workflow.models.parameter import (
     BitwardenLoginCredentialParameter,
     BitwardenSensitiveInformationParameter,
     ContextParameter,
+    CredentialParameter,
     OutputParameter,
     Parameter,
     ParameterType,
@@ -45,6 +53,7 @@ class WorkflowRunContext:
             | BitwardenLoginCredentialParameter
             | BitwardenCreditCardDataParameter
             | BitwardenSensitiveInformationParameter
+            | CredentialParameter
         ],
     ) -> Self:
         # key is label name
@@ -68,6 +77,10 @@ class WorkflowRunContext:
         for secrete_parameter in secret_parameters:
             if isinstance(secrete_parameter, AWSSecretParameter):
                 await workflow_run_context.register_aws_secret_parameter_value(aws_client, secrete_parameter)
+            elif isinstance(secrete_parameter, CredentialParameter):
+                await workflow_run_context.register_credential_parameter_value(
+                    aws_client, secrete_parameter, organization
+                )
             elif isinstance(secrete_parameter, BitwardenLoginCredentialParameter):
                 await workflow_run_context.register_bitwarden_login_credential_parameter_value(
                     aws_client, secrete_parameter, organization
@@ -161,6 +174,37 @@ class WorkflowRunContext:
     def generate_random_secret_id() -> str:
         return f"secret_{uuid.uuid4()}"
 
+    async def register_credential_parameter_value(
+        self,
+        aws_client: AsyncAWSClient,
+        parameter: CredentialParameter,
+        organization: Organization,
+    ) -> None:
+        LOG.info(f"Fetching credential parameter value for credential: {parameter.credential_id}")
+        org_secret_values = await aws_client.get_secret(organization.organization_id)
+        if org_secret_values is None:
+            raise CredentialParameterNotFoundError(parameter.credential_id)
+        # Parse the items and extract credentials
+        try:
+            org_secret_values_json = json.loads(org_secret_values)
+
+        except json.JSONDecodeError:
+            raise CredentialParameterParsingError(
+                f"Failed to parse credential JSON. Credential ID: {parameter.credential_id}"
+            )
+
+        credentials = org_secret_values_json.get(parameter.credential_id)
+        if credentials is None:
+            raise CredentialParameterNotFoundError(parameter.credential_id)
+
+        self.parameters[parameter.key] = parameter
+        self.values[parameter.key] = {}
+        for key, value in credentials.items():
+            random_secret_id = self.generate_random_secret_id()
+            secret_id = f"{random_secret_id}_{key}"
+            self.secrets[secret_id] = value
+            self.values[parameter.key][key] = secret_id
+
     async def register_aws_secret_parameter_value(
         self,
         aws_client: AsyncAWSClient,
@@ -550,6 +594,7 @@ class WorkflowRunContext:
                     BitwardenLoginCredentialParameter,
                     BitwardenCreditCardDataParameter,
                     BitwardenSensitiveInformationParameter,
+                    CredentialParameter,
                 ),
             ):
                 LOG.error(

skyvern/forge/sdk/workflow/models/parameter.py

@@ -17,6 +17,7 @@ class ParameterType(StrEnum):
     BITWARDEN_SENSITIVE_INFORMATION = "bitwarden_sensitive_information"
     BITWARDEN_CREDIT_CARD_DATA = "bitwarden_credit_card_data"
     OUTPUT = "output"
+    CREDENTIAL = "credential"
 
 
 class Parameter(BaseModel, abc.ABC):
@@ -65,6 +66,20 @@ class BitwardenLoginCredentialParameter(Parameter):
     deleted_at: datetime | None = None
 
 
+class CredentialParameter(Parameter):
+    model_config = ConfigDict(from_attributes=True)
+    parameter_type: Literal[ParameterType.CREDENTIAL] = ParameterType.CREDENTIAL
+
+    credential_parameter_id: str
+    workflow_id: str
+
+    credential_id: str
+
+    created_at: datetime
+    modified_at: datetime
+    deleted_at: datetime | None = None
+
+
 class BitwardenSensitiveInformationParameter(Parameter):
     parameter_type: Literal[ParameterType.BITWARDEN_SENSITIVE_INFORMATION] = (
         ParameterType.BITWARDEN_SENSITIVE_INFORMATION
@@ -182,5 +197,6 @@ ParameterSubclasses = Union[
     BitwardenSensitiveInformationParameter,
     BitwardenCreditCardDataParameter,
     OutputParameter,
+    CredentialParameter,
 ]
 PARAMETER_TYPE = Annotated[ParameterSubclasses, Field(discriminator="parameter_type")]

skyvern/forge/sdk/workflow/models/yaml.py

@@ -43,6 +43,11 @@ class BitwardenLoginCredentialParameterYAML(ParameterYAML):
     bitwarden_collection_id: str | None = None
 
 
+class CredentialParameterYAML(ParameterYAML):
+    parameter_type: Literal[ParameterType.CREDENTIAL] = ParameterType.CREDENTIAL  # type: ignore
+    credential_id: str
+
+
 class BitwardenSensitiveInformationParameterYAML(ParameterYAML):
     # There is a mypy bug with Literal. Without the type: ignore, mypy will raise an error:
     # Parameter 1 of Literal[...] cannot be of type "Any"
@@ -341,6 +346,7 @@ PARAMETER_YAML_SUBCLASSES = (
     | WorkflowParameterYAML
     | ContextParameterYAML
     | OutputParameterYAML
+    | CredentialParameterYAML
 )
 PARAMETER_YAML_TYPES = Annotated[PARAMETER_YAML_SUBCLASSES, Field(discriminator="parameter_type")]
 

skyvern/forge/sdk/workflow/service.py

@@ -64,6 +64,7 @@ from skyvern.forge.sdk.workflow.models.parameter import (
     BitwardenLoginCredentialParameter,
     BitwardenSensitiveInformationParameter,
     ContextParameter,
+    CredentialParameter,
     OutputParameter,
     Parameter,
     ParameterType,
@@ -233,6 +234,7 @@ class WorkflowService:
                     BitwardenLoginCredentialParameter,
                     BitwardenCreditCardDataParameter,
                     BitwardenSensitiveInformationParameter,
+                    CredentialParameter,
                 ),
             )
         ]
@@ -817,6 +819,20 @@ class WorkflowService:
             bitwarden_collection_id=bitwarden_collection_id,
         )
 
+    async def create_credential_parameter(
+        self,
+        workflow_id: str,
+        key: str,
+        credential_id: str,
+        description: str | None = None,
+    ) -> CredentialParameter:
+        return await app.DATABASE.create_credential_parameter(
+            workflow_id=workflow_id,
+            key=key,
+            credential_id=credential_id,
+            description=description,
+        )
+
     async def create_bitwarden_sensitive_information_parameter(
         self,
         workflow_id: str,
@@ -1358,6 +1374,13 @@ class WorkflowService:
                         key=parameter.key,
                         description=parameter.description,
                     )
+                elif parameter.parameter_type == ParameterType.CREDENTIAL:
+                    parameters[parameter.key] = await self.create_credential_parameter(
+                        workflow_id=workflow.workflow_id,
+                        key=parameter.key,
+                        description=parameter.description,
+                        credential_id=parameter.credential_id,
+                    )
                 elif parameter.parameter_type == ParameterType.BITWARDEN_LOGIN_CREDENTIAL:
                     if not parameter.bitwarden_collection_id:
                         raise WorkflowParameterMissingRequiredValue(