make jinja rendering in blocks more strict (#3827)

pyproject.toml

@@ -202,6 +202,9 @@ plugins = "sqlalchemy.ext.mypy.plugin"
 [project.scripts]
 skyvern = "skyvern.cli.commands:cli_app"
 
+[tool.pytest.ini_options]
+norecursedirs = ["eval"]
+
 [tool.codeflash]
 # All paths are relative to this pyproject.toml's directory.                                                                                                                                                                                                                                                                 
 module-root = "skyvern"                                                                                                                                                                                                                                                                                                      

skyvern/forge/sdk/workflow/exceptions.py

@@ -121,6 +121,15 @@ class FailedToFormatJinjaStyleParameter(SkyvernException):
         )
 
 
+class MissingJinjaVariables(SkyvernException):
+    def __init__(self, template: str, variables: set[str]) -> None:
+        self.variables = variables
+
+        super().__init__(
+            f"There are missing variables for '{template}'. Please make sure the variables are supplied. Missing variables: {variables}"
+        )
+
+
 class NoIterableValueFound(SkyvernException):
     def __init__(self) -> None:
         super().__init__("No iterable value found for the loop block")

skyvern/forge/sdk/workflow/models/block.py

@@ -23,6 +23,7 @@ import pandas as pd
 import pyotp
 import structlog
 from email_validator import EmailNotValidError, validate_email
+from jinja2 import StrictUndefined
 from jinja2.sandbox import SandboxedEnvironment
 from playwright.async_api import Page
 from pydantic import BaseModel, Field
@@ -74,6 +75,7 @@ from skyvern.forge.sdk.workflow.exceptions import (
     InsecureCodeDetected,
     InvalidEmailClientConfiguration,
     InvalidFileType,
+    MissingJinjaVariables,
     NoIterableValueFound,
     NoValidEmailRecipient,
 )
@@ -88,12 +90,13 @@ from skyvern.forge.sdk.workflow.models.parameter import (
 from skyvern.schemas.runs import RunEngine
 from skyvern.schemas.workflows import BlockResult, BlockStatus, BlockType, FileStorageType, FileType
 from skyvern.utils.strings import generate_random_string
+from skyvern.utils.templating import get_missing_variables
 from skyvern.utils.url_validators import prepend_scheme_and_validate_url
 from skyvern.webeye.browser_factory import BrowserState
 from skyvern.webeye.utils.page import SkyvernFrame
 
 LOG = structlog.get_logger()
-jinja_sandbox_env = SandboxedEnvironment()
+jinja_sandbox_env = SandboxedEnvironment(undefined=StrictUndefined)
 
 
 # Mapping from TaskV2Status to the corresponding BlockStatus. Declared once at
@@ -187,6 +190,7 @@ class Block(BaseModel, abc.ABC):
     ) -> str:
         if not potential_template:
             return potential_template
+
         template = jinja_sandbox_env.from_string(potential_template)
 
         block_reference_data: dict[str, Any] = workflow_run_context.get_block_metadata(self.label)
@@ -245,6 +249,12 @@ class Block(BaseModel, abc.ABC):
         if "workflow_run_id" not in template_data:
             template_data["workflow_run_id"] = workflow_run_context.workflow_run_id
 
+        if missing_variables := get_missing_variables(potential_template, template_data):
+            raise MissingJinjaVariables(
+                template=potential_template,
+                variables=missing_variables,
+            )
+
         return template.render(template_data)
 
     @classmethod

skyvern/utils/templating.py

@@ -0,0 +1,38 @@
+import re
+
+from jinja2 import Environment, StrictUndefined, UndefinedError, meta
+
+
+class Constants:
+    MissingVariablePattern = var_pattern = r"\{\{\s*([a-zA-Z_][a-zA-Z0-9_.\[\]'\"]*)\s*\}\}"
+
+
+def get_missing_variables(template_source: str, template_data: dict) -> set[str]:
+    # quick check - catch top-level undefineds
+    env = Environment(undefined=StrictUndefined)
+    ast = env.parse(template_source)
+    undeclared_vars = meta.find_undeclared_variables(ast)
+    missing_vars = undeclared_vars - set(template_data.keys())
+
+    # nested undefined won't be caught; let's check for those
+    if not missing_vars:
+        # try rendering to catch nested undefineds (dotted attributes, list/dict access)
+        try:
+            template = env.from_string(template_source)
+            template.render(template_data)
+        except UndefinedError:
+            # matches: {{ var }}, {{ var.attr }}, {{ var[0] }}, {{ var['key'] }}, {{ var.attr[0] }}
+            matches = re.findall(Constants.MissingVariablePattern, template_source)
+
+            for match in matches:
+                root = match.split("[")[0].split(".")[0]
+
+                # just check if the 'root' of the variable exists in the provided data
+                # if it does, add the whole match as missing
+                if root in template_data:
+                    missing_vars.add(match)
+
+            if not missing_vars:
+                raise  # re-raise if we couldn't determine missing vars
+
+    return missing_vars