current viewpoint screenshot and scrolling n screenshot (#2716)

Co-authored-by: lawyzheng <lawyzheng1106@gmail.com>

skyvern-frontend/src/routes/workflows/editor/FlowRenderer.tsx

@@ -289,6 +289,8 @@ function FlowRenderer({
         webhook_callback_url: data.settings.webhookCallbackUrl,
         persist_browser_session: data.settings.persistBrowserSession,
         model: data.settings.model,
+        max_screenshot_scrolling_times:
+          data.settings.maxScreenshotScrollingTimes,
         totp_verification_url: workflow.totp_verification_url,
         workflow_definition: {
           parameters: data.parameters,

skyvern-frontend/src/routes/workflows/editor/WorkflowEditor.tsx

@@ -60,6 +60,7 @@ function WorkflowEditor() {
     proxyLocation: workflow.proxy_location,
     webhookCallbackUrl: workflow.webhook_callback_url,
     model: workflow.model,
+    maxScreenshotScrollingTimes: workflow.max_screenshot_scrolling_times,
   };
 
   const elements = getElements(

skyvern-frontend/src/routes/workflows/editor/nodes/StartNode/StartNode.tsx

@@ -20,6 +20,7 @@ import { Separator } from "@/components/ui/separator";
 import { ModelsResponse } from "@/api/types";
 import { ModelSelector } from "@/components/ModelSelector";
 import { WorkflowModel } from "@/routes/workflows/types/workflowTypes";
+import { MAX_SCREENSHOT_SCROLLING_TIMES_DEFAULT } from "../Taskv2Node/types";
 
 function StartNode({ id, data }: NodeProps<StartNode>) {
   const credentialGetter = useCredentialGetter();
@@ -51,6 +52,9 @@ function StartNode({ id, data }: NodeProps<StartNode>) {
       ? data.persistBrowserSession
       : false,
     model: data.withWorkflowSettings ? data.model : workflowModel,
+    maxScreenshotScrollingTimes: data.withWorkflowSettings
+      ? data.maxScreenshotScrollingTimes
+      : null,
   });
 
   function handleChange(key: string, value: unknown) {
@@ -130,6 +134,26 @@ function StartNode({ id, data }: NodeProps<StartNode>) {
                         />
                       </div>
                     </div>
+                    <div className="space-y-2">
+                      <div className="flex items-center gap-2">
+                        <Label>Max Scrolling Screenshots</Label>
+                        <HelpTooltip
+                          content={`The maximum number of times to scroll down the page to take merged screenshots after action. Default is ${MAX_SCREENSHOT_SCROLLING_TIMES_DEFAULT}. If it's set to 0, it will take the current viewport screenshot.`}
+                        />
+                      </div>
+                      <Input
+                        value={inputs.maxScreenshotScrollingTimes ?? ""}
+                        placeholder={`Default: ${MAX_SCREENSHOT_SCROLLING_TIMES_DEFAULT}`}
+                        onChange={(event) => {
+                          const value =
+                            event.target.value === ""
+                              ? null
+                              : Number(event.target.value);
+
+                          handleChange("maxScreenshotScrollingTimes", value);
+                        }}
+                      />
+                    </div>
                   </div>
                 </AccordionContent>
               </AccordionItem>

skyvern-frontend/src/routes/workflows/editor/nodes/StartNode/types.ts

@@ -9,6 +9,7 @@ export type WorkflowStartNodeData = {
   proxyLocation: ProxyLocation;
   persistBrowserSession: boolean;
   model: WorkflowModel | null;
+  maxScreenshotScrollingTimes: number | null;
   editable: boolean;
 };
 

skyvern-frontend/src/routes/workflows/editor/nodes/Taskv2Node/types.ts

@@ -2,6 +2,7 @@ import { Node } from "@xyflow/react";
 import { NodeBaseData } from "../types";
 
 export const MAX_STEPS_DEFAULT = 25;
+export const MAX_SCREENSHOT_SCROLLING_TIMES_DEFAULT = 3;
 
 export type Taskv2NodeData = NodeBaseData & {
   prompt: string;
@@ -9,6 +10,7 @@ export type Taskv2NodeData = NodeBaseData & {
   totpVerificationUrl: string | null;
   totpIdentifier: string | null;
   maxSteps: number | null;
+  maxScreenshotScrollingTimes: number | null;
 };
 
 export type Taskv2Node = Node<Taskv2NodeData, "taskv2">;
@@ -23,6 +25,7 @@ export const taskv2NodeDefaultData: Taskv2NodeData = {
   totpVerificationUrl: null,
   maxSteps: MAX_STEPS_DEFAULT,
   model: null,
+  maxScreenshotScrollingTimes: null,
 };
 
 export function isTaskV2Node(node: Node): node is Taskv2Node {

skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.ts

@@ -251,6 +251,7 @@ function convertToNode(
           maxSteps: block.max_steps,
           totpIdentifier: block.totp_identifier,
           totpVerificationUrl: block.totp_verification_url,
+          maxScreenshotScrollingTimes: null,
         },
       };
     }
@@ -662,6 +663,7 @@ function getElements(
       proxyLocation: settings.proxyLocation ?? ProxyLocation.Residential,
       webhookCallbackUrl: settings.webhookCallbackUrl ?? "",
       model: settings.model,
+      maxScreenshotScrollingTimes: settings.maxScreenshotScrollingTimes,
       editable,
     }),
   );
@@ -1322,6 +1324,7 @@ function getWorkflowSettings(nodes: Array<AppNode>): WorkflowSettings {
     proxyLocation: ProxyLocation.Residential,
     webhookCallbackUrl: null,
     model: null,
+    maxScreenshotScrollingTimes: null,
   };
   const startNodes = nodes.filter(isStartNode);
   const startNodeWithWorkflowSettings = startNodes.find(
@@ -1337,6 +1340,7 @@ function getWorkflowSettings(nodes: Array<AppNode>): WorkflowSettings {
       proxyLocation: data.proxyLocation,
       webhookCallbackUrl: data.webhookCallbackUrl,
       model: data.model,
+      maxScreenshotScrollingTimes: data.maxScreenshotScrollingTimes,
     };
   }
   return defaultSettings;
@@ -1992,6 +1996,7 @@ function convert(workflow: WorkflowApiResponse): WorkflowCreateYAMLRequest {
     persist_browser_session: workflow.persist_browser_session,
     model: workflow.model,
     totp_verification_url: workflow.totp_verification_url,
+    max_screenshot_scrolling_times: workflow.max_screenshot_scrolling_times,
     workflow_definition: {
       parameters: convertParametersToParameterYAML(userParameters),
       blocks: convertBlocksToBlockYAML(workflow.workflow_definition.blocks),