Add html elements tree in prompt (#731)

skyvern-frontend/src/api/types.ts

@@ -6,6 +6,7 @@ export const ArtifactType = {
   LLMResponseParsed: "llm_response_parsed",
   VisibleElementsTree: "visible_elements_tree",
   VisibleElementsTreeTrimmed: "visible_elements_tree_trimmed",
+  VisibleElementsTreeInPrompt: "visible_elements_tree_in_prompt",
   LLMPrompt: "llm_prompt",
   LLMRequest: "llm_request",
   HTMLScrape: "html_scrape",

skyvern-frontend/src/routes/tasks/detail/HTMLArtifact.tsx

@@ -0,0 +1,66 @@
+import { artifactApiClient } from "@/api/AxiosClient";
+import { ArtifactApiResponse } from "@/api/types";
+import { Skeleton } from "@/components/ui/skeleton";
+import { Textarea } from "@/components/ui/textarea";
+import { useQuery } from "@tanstack/react-query";
+import axios from "axios";
+
+// https://stackoverflow.com/a/60338028
+function format(html: string) {
+  const tab = "\t";
+  let result = "";
+  let indent = "";
+
+  html.split(/>\s*</).forEach(function (element) {
+    if (element.match(/^\/\w/)) {
+      indent = indent.substring(tab.length);
+    }
+
+    result += indent + "<" + element + ">\r\n";
+
+    if (element.match(/^<?\w[^>]*[^/]$/) && !element.startsWith("input")) {
+      indent += tab;
+    }
+  });
+
+  return result.substring(1, result.length - 3);
+}
+
+type Props = {
+  artifact: ArtifactApiResponse;
+};
+
+function HTMLArtifact({ artifact }: Props) {
+  const { data, isFetching, isError, error } = useQuery<string>({
+    queryKey: ["artifact", artifact.artifact_id],
+    queryFn: async () => {
+      if (artifact.uri.startsWith("file://")) {
+        return artifactApiClient
+          .get(`/artifact/text`, {
+            params: {
+              path: artifact.uri.slice(7),
+            },
+          })
+          .then((response) => response.data);
+      }
+      if (artifact.uri.startsWith("s3://") && artifact.signed_url) {
+        return axios.get(artifact.signed_url).then((response) => response.data);
+      }
+    },
+  });
+
+  if (isFetching) {
+    return <Skeleton className="h-48 w-full" />;
+  }
+
+  return (
+    <Textarea
+      className="w-full"
+      rows={15}
+      value={isError ? JSON.stringify(error) : format(data ?? "")}
+      readOnly
+    />
+  );
+}
+
+export { HTMLArtifact };

skyvern-frontend/src/routes/tasks/detail/StepArtifacts.tsx

@@ -17,7 +17,7 @@ import { getImageURL } from "./artifactUtils";
 import { Input } from "@/components/ui/input";
 import { basicTimeFormat } from "@/util/timeFormat";
 import { useCredentialGetter } from "@/hooks/useCredentialGetter";
-
+import { HTMLArtifact } from "./HTMLArtifact";
 type Props = {
   id: string;
   stepProps: StepApiResponse;
@@ -61,9 +61,9 @@ function StepArtifacts({ id, stepProps }: Props) {
     (artifact) => artifact.artifact_type === ArtifactType.LLMRequest,
   );
 
-  const visibleElementsTreeTrimmed = artifacts?.filter(
+  const visibleElementsTreeInPrompt = artifacts?.find(
     (artifact) =>
-      artifact.artifact_type === ArtifactType.VisibleElementsTreeTrimmed,
+      artifact.artifact_type === ArtifactType.VisibleElementsTreeInPrompt,
   );
 
   const llmPrompt = artifacts?.find(
@@ -164,8 +164,8 @@ function StepArtifacts({ id, stepProps }: Props) {
         )}
       </TabsContent>
       <TabsContent value="element_tree_trimmed">
-        {visibleElementsTreeTrimmed ? (
-          <JSONArtifact artifacts={visibleElementsTreeTrimmed} />
+        {visibleElementsTreeInPrompt ? (
+          <HTMLArtifact artifact={visibleElementsTreeInPrompt} />
         ) : null}
       </TabsContent>
       <TabsContent value="html_element_tree">