added raw dada logs (#2736)

skyvern/config.py

@@ -39,6 +39,7 @@ class Settings(BaseSettings):
     ENV: str = "local"
     EXECUTE_ALL_STEPS: bool = True
     JSON_LOGGING: bool = False
+    LOG_RAW_API_REQUESTS: bool = True
     LOG_LEVEL: str = "INFO"
     PORT: int = 8000
     ALLOWED_ORIGINS: list[str] = ["*"]

skyvern/forge/api_app.py

@@ -15,6 +15,7 @@ from starlette_context.plugins.base import Plugin
 from skyvern.config import settings
 from skyvern.exceptions import SkyvernHTTPException
 from skyvern.forge import app as forge_app
+from skyvern.forge.request_logging import log_raw_request_middleware
 from skyvern.forge.sdk.core import skyvern_context
 from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
 from skyvern.forge.sdk.db.exceptions import NotFoundError
@@ -113,6 +114,10 @@ def get_agent_app() -> FastAPI:
         finally:
             skyvern_context.reset()
 
+    @app.middleware("http")
+    async def raw_request_logging(request: Request, call_next: Callable[[Request], Awaitable[Response]]) -> Response:
+        return await log_raw_request_middleware(request, call_next)
+
     if settings.ADDITIONAL_MODULES:
         for module in settings.ADDITIONAL_MODULES:
             LOG.info("Loading additional module to set up api app", module=module)

skyvern/forge/request_logging.py

@@ -0,0 +1,66 @@
+from __future__ import annotations
+
+import typing
+
+import structlog
+
+from skyvern.config import settings
+
+if typing.TYPE_CHECKING:  # pragma: no cover - import only for type hints
+    from typing import Awaitable, Callable
+
+    from fastapi import Response
+    from starlette.requests import Request
+
+LOG = structlog.get_logger()
+
+_SENSITIVE_HEADERS = {"authorization", "cookie"}
+_MAX_BODY_LENGTH = 1000
+_BINARY_PLACEHOLDER = "<binary>"
+
+
+def _sanitize_headers(headers: typing.Mapping[str, str]) -> dict[str, str]:
+    sanitized: dict[str, str] = {}
+    for key, value in headers.items():
+        if key.lower() in _SENSITIVE_HEADERS:
+            continue
+        sanitized[key] = value
+    return sanitized
+
+
+def _sanitize_body(body: bytes, content_type: str | None) -> str:
+    if not body:
+        return ""
+    if content_type and not (content_type.startswith("text/") or content_type.startswith("application/json")):
+        return _BINARY_PLACEHOLDER
+    try:
+        text = body.decode("utf-8", errors="replace")
+    except Exception:
+        return _BINARY_PLACEHOLDER
+    if len(text) > _MAX_BODY_LENGTH:
+        return text[:_MAX_BODY_LENGTH] + "...[truncated]"
+    return text
+
+
+async def log_raw_request_middleware(request: Request, call_next: Callable[[Request], Awaitable[Response]]) -> Response:
+    if not settings.LOG_RAW_API_REQUESTS:
+        return await call_next(request)
+
+    body_bytes = await request.body()
+    # ensure downstream handlers can access body again
+    try:
+        request._body = body_bytes  # type: ignore[attr-defined]
+    except Exception:
+        pass
+
+    sanitized_headers = _sanitize_headers(dict(request.headers))
+    body_text = _sanitize_body(body_bytes, request.headers.get("content-type"))
+
+    LOG.info(
+        "api.raw_request",
+        method=request.method,
+        path=request.url.path,
+        headers=sanitized_headers,
+        body=body_text,
+    )
+    return await call_next(request)