diff --git a/pyproject.toml b/pyproject.toml index 6bb5bbbf..9391f328 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -202,6 +202,9 @@ plugins = "sqlalchemy.ext.mypy.plugin" [project.scripts] skyvern = "skyvern.cli.commands:cli_app" +[tool.pytest.ini_options] +norecursedirs = ["eval"] + [tool.codeflash] # All paths are relative to this pyproject.toml's directory. module-root = "skyvern" diff --git a/skyvern/forge/sdk/workflow/exceptions.py b/skyvern/forge/sdk/workflow/exceptions.py index 1f485af0..f02da543 100644 --- a/skyvern/forge/sdk/workflow/exceptions.py +++ b/skyvern/forge/sdk/workflow/exceptions.py @@ -121,6 +121,15 @@ class FailedToFormatJinjaStyleParameter(SkyvernException): ) +class MissingJinjaVariables(SkyvernException): + def __init__(self, template: str, variables: set[str]) -> None: + self.variables = variables + + super().__init__( + f"There are missing variables for '{template}'. Please make sure the variables are supplied. Missing variables: {variables}" + ) + + class NoIterableValueFound(SkyvernException): def __init__(self) -> None: super().__init__("No iterable value found for the loop block") diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.py index 9f90f79a..add471e1 100644 --- a/skyvern/forge/sdk/workflow/models/block.py +++ b/skyvern/forge/sdk/workflow/models/block.py @@ -23,6 +23,7 @@ import pandas as pd import pyotp import structlog from email_validator import EmailNotValidError, validate_email +from jinja2 import StrictUndefined from jinja2.sandbox import SandboxedEnvironment from playwright.async_api import Page from pydantic import BaseModel, Field @@ -74,6 +75,7 @@ from skyvern.forge.sdk.workflow.exceptions import ( InsecureCodeDetected, InvalidEmailClientConfiguration, InvalidFileType, + MissingJinjaVariables, NoIterableValueFound, NoValidEmailRecipient, ) @@ -88,12 +90,13 @@ from skyvern.forge.sdk.workflow.models.parameter import ( from skyvern.schemas.runs import RunEngine from skyvern.schemas.workflows import BlockResult, BlockStatus, BlockType, FileStorageType, FileType from skyvern.utils.strings import generate_random_string +from skyvern.utils.templating import get_missing_variables from skyvern.utils.url_validators import prepend_scheme_and_validate_url from skyvern.webeye.browser_factory import BrowserState from skyvern.webeye.utils.page import SkyvernFrame LOG = structlog.get_logger() -jinja_sandbox_env = SandboxedEnvironment() +jinja_sandbox_env = SandboxedEnvironment(undefined=StrictUndefined) # Mapping from TaskV2Status to the corresponding BlockStatus. Declared once at @@ -187,6 +190,7 @@ class Block(BaseModel, abc.ABC): ) -> str: if not potential_template: return potential_template + template = jinja_sandbox_env.from_string(potential_template) block_reference_data: dict[str, Any] = workflow_run_context.get_block_metadata(self.label) @@ -245,6 +249,12 @@ class Block(BaseModel, abc.ABC): if "workflow_run_id" not in template_data: template_data["workflow_run_id"] = workflow_run_context.workflow_run_id + if missing_variables := get_missing_variables(potential_template, template_data): + raise MissingJinjaVariables( + template=potential_template, + variables=missing_variables, + ) + return template.render(template_data) @classmethod diff --git a/skyvern/utils/templating.py b/skyvern/utils/templating.py new file mode 100644 index 00000000..d3b02c21 --- /dev/null +++ b/skyvern/utils/templating.py @@ -0,0 +1,38 @@ +import re + +from jinja2 import Environment, StrictUndefined, UndefinedError, meta + + +class Constants: + MissingVariablePattern = var_pattern = r"\{\{\s*([a-zA-Z_][a-zA-Z0-9_.\[\]'\"]*)\s*\}\}" + + +def get_missing_variables(template_source: str, template_data: dict) -> set[str]: + # quick check - catch top-level undefineds + env = Environment(undefined=StrictUndefined) + ast = env.parse(template_source) + undeclared_vars = meta.find_undeclared_variables(ast) + missing_vars = undeclared_vars - set(template_data.keys()) + + # nested undefined won't be caught; let's check for those + if not missing_vars: + # try rendering to catch nested undefineds (dotted attributes, list/dict access) + try: + template = env.from_string(template_source) + template.render(template_data) + except UndefinedError: + # matches: {{ var }}, {{ var.attr }}, {{ var[0] }}, {{ var['key'] }}, {{ var.attr[0] }} + matches = re.findall(Constants.MissingVariablePattern, template_source) + + for match in matches: + root = match.split("[")[0].split(".")[0] + + # just check if the 'root' of the variable exists in the provided data + # if it does, add the whole match as missing + if root in template_data: + missing_vars.add(match) + + if not missing_vars: + raise # re-raise if we couldn't determine missing vars + + return missing_vars