fix anthropic messages image input pattern (#2239)

skyvern/forge/sdk/api/llm/utils.py

@@ -18,6 +18,7 @@ async def llm_messages_builder(
     prompt: str,
     screenshots: list[bytes] | None = None,
     add_assistant_prefix: bool = False,
+    message_pattern: str = "openai",
 ) -> list[dict[str, Any]]:
     messages: list[dict[str, Any]] = [
         {
@@ -29,14 +30,23 @@ async def llm_messages_builder(
     if screenshots:
         for screenshot in screenshots:
             encoded_image = base64.b64encode(screenshot).decode("utf-8")
-            messages.append(
-                {
+            if message_pattern == "anthropic":
+                message = {
+                    "type": "image",
+                    "source": {
+                        "type": "base64",
+                        "media_type": "image/png",
+                        "data": encoded_image,
+                    },
+                }
+            else:
+                message = {
                     "type": "image_url",
                     "image_url": {
                         "url": f"data:image/png;base64,{encoded_image}",
                     },
                 }
-            )
+            messages.append(message)
     # Anthropic models seems to struggle to always output a valid json object so we need to prefill the response to force it:
     if add_assistant_prefix:
         return [
@@ -50,6 +60,7 @@ async def llm_messages_builder_with_history(
     prompt: str | None = None,
     screenshots: list[bytes] | None = None,
     message_history: list[dict[str, Any]] | None = None,
+    message_pattern: str = "openai",
 ) -> list[dict[str, Any]]:
     messages: list[dict[str, Any]] = []
     if message_history:
@@ -67,14 +78,23 @@ async def llm_messages_builder_with_history(
     if screenshots:
         for screenshot in screenshots:
             encoded_image = base64.b64encode(screenshot).decode("utf-8")
-            current_user_messages.append(
-                {
+            if message_pattern == "anthropic":
+                message = {
+                    "type": "image",
+                    "source": {
+                        "type": "base64",
+                        "media_type": "image/png",
+                        "data": encoded_image,
+                    },
+                }
+            else:
+                message = {
                     "type": "image_url",
                     "image_url": {
                         "url": f"data:image/png;base64,{encoded_image}",
                     },
                 }
-            )
+            current_user_messages.append(message)
     messages.append({"role": "user", "content": current_user_messages})
     return messages