Merge pull request #24 from amhsirak/develop

feat: handle `scrapeList` while browser recording

maxun-core/src/browserSide/scraper.js

@@ -421,7 +421,6 @@ async function clickNextPagination(selector, scrapedData, limit) {
     return results;
   };
 
-
   window.scrollDown = async function (selector, limit) {
     let previousHeight = 0;
     let itemsLoaded = 0;

server/src/workflow-management/classes/Generator.ts

@@ -7,6 +7,7 @@ import {
   getElementInformation,
   getRect,
   getSelectors,
+  getChildSelectors,
   getNonUniqueSelectors,
   isRuleOvershadowing,
   selectorAlreadyInWorkflow
@@ -53,6 +54,8 @@ export class WorkflowGenerator {
    */
   private getList: boolean = false;
 
+  private listSelector: string = '';
+
   /**
    * The public constructor of the WorkflowGenerator.
    * Takes socket for communication as a parameter and registers some important events on it.
@@ -103,6 +106,9 @@ export class WorkflowGenerator {
     this.socket.on('setGetList', (data: { getList: boolean }) => {
       this.getList = data.getList;
     });
+    this.socket.on('listSelector', (data: { selector: string }) => {
+      this.listSelector = data.selector;
+    })
   }
 
   /**
@@ -476,6 +482,11 @@ export class WorkflowGenerator {
    */
   private generateSelector = async (page: Page, coordinates: Coordinates, action: ActionType) => {
     const elementInfo = await getElementInformation(page, coordinates);
+    const generalSelector = await getNonUniqueSelectors(page, coordinates)
+    const childSelectors = await getChildSelectors(page, generalSelector.generalSelector);
+
+    console.log('Non Unique Selectors [DEBUG]:', generalSelector);
+    console.log('Child Selectors [DEBUG]:', childSelectors);
 
     const selectorBasedOnCustomAction = (this.getList === true)
       ? await getNonUniqueSelectors(page, coordinates)
@@ -507,7 +518,14 @@ export class WorkflowGenerator {
     const displaySelector = await this.generateSelector(page, coordinates, ActionType.Click);
     const elementInfo = await getElementInformation(page, coordinates);
     if (rect) {
-      this.socket.emit('highlighter', { rect, selector: displaySelector, elementInfo });
+      if (this.getList === true) {
+        if (this.listSelector !== '') {
+          const childSelectors = await getChildSelectors(page, this.listSelector || '');
+          this.socket.emit('highlighter', { rect, selector: displaySelector, elementInfo, childSelectors })
+        }
+      } else {
+        this.socket.emit('highlighter', { rect, selector: displaySelector, elementInfo });
+      }
     }
     // reset getList after usage
     this.getList = false;

server/src/workflow-management/selector.ts

@@ -4,6 +4,11 @@ import { WhereWhatPair, WorkflowFile } from "maxun-core";
 import logger from "../logger";
 import { getBestSelectorForAction } from "./utils";
 
+/*TODO:
+1. Handle TS errors (here we definetly know better)
+2. Add pending function descriptions + thought process (esp. selector generation)
+*/
+
 type Workflow = WorkflowFile["workflow"];
 
 /**
@@ -97,21 +102,6 @@ export const getElementInformation = async (
       },
       { x: coordinates.x, y: coordinates.y },
     );
-
-    // if (elementInfo) {
-    //   if (elementInfo.tagName === 'A') {
-    //     if (elementInfo.innerText) {
-    //       console.log(`Link text: ${elementInfo.innerText}, URL: ${elementInfo.url}`);
-    //     } else {
-    //       console.log(`URL: ${elementInfo.url}`);
-    //     }
-    //   } else if (elementInfo.tagName === 'IMG') {
-    //     console.log(`Image URL: ${elementInfo.imageUrl}`);
-    //   } else {
-    //     console.log(`Element innerText: ${elementInfo.innerText}`);
-    //   }
-    // }
-
     return elementInfo;
   } catch (error) {
     const { message, stack } = error as Error;
@@ -591,8 +581,6 @@ export const getSelectors = async (page: Page, coordinates: Coordinates) => {
         return output;
       }
 
-
-
       const genSelectors = (element: HTMLElement | null) => {
         if (element == null) {
           return null;
@@ -722,6 +710,10 @@ export const getSelectors = async (page: Page, coordinates: Coordinates) => {
 };
 
 
+interface SelectorResult {
+  generalSelector: string;
+}
+
 /**
  * Returns the best non-unique css {@link Selectors} for the element on the page.
  * @param page The page instance.
@@ -730,18 +722,16 @@ export const getSelectors = async (page: Page, coordinates: Coordinates) => {
  * @returns {Promise<Selectors|null|undefined>}
  */
 
-export const getNonUniqueSelectors = async (page: Page, coordinates: Coordinates) => {
+export const getNonUniqueSelectors = async (page: Page, coordinates: Coordinates): Promise<SelectorResult> => {
   try {
     const selectors = await page.evaluate(({ x, y }: { x: number, y: number }) => {
 
       function getNonUniqueSelector(element: HTMLElement): string {
         let selector = element.tagName.toLowerCase();
 
-        // Avoid using IDs to maintain non-uniqueness
         if (element.className) {
           const classes = element.className.split(/\s+/).filter((cls: string) => Boolean(cls));
           if (classes.length > 0) {
-            // Exclude utility classes and escape special characters
             const validClasses = classes.filter((cls: string) => !cls.startsWith('!') && !cls.includes(':'));
             if (validClasses.length > 0) {
               selector += '.' + validClasses.map(cls => CSS.escape(cls)).join('.');
@@ -754,11 +744,16 @@ export const getNonUniqueSelectors = async (page: Page, coordinates: Coordinates
 
       function getSelectorPath(element: HTMLElement | null): string {
         const path: string[] = [];
-        while (element && element !== document.body) {
+        let depth = 0;
+        const maxDepth = 2;
+
+        while (element && element !== document.body && depth < maxDepth) {
           const selector = getNonUniqueSelector(element);
           path.unshift(selector);
           element = element.parentElement;
+          depth++;
         }
+
         return path.join(' > ');
       }
 
@@ -771,15 +766,67 @@ export const getNonUniqueSelectors = async (page: Page, coordinates: Coordinates
       };
     }, coordinates);
 
-    return selectors || {};
+    return selectors || { generalSelector: '' };
   } catch (error) {
     console.error('Error in getNonUniqueSelectors:', error);
-    return {};
+    return { generalSelector: '' };
   }
 };
 
 
+export const getChildSelectors = async (page: Page, parentSelector: string): Promise<string[]> => {
+  try {
+    const childSelectors = await page.evaluate((parentSelector: string) => {
+      function getNonUniqueSelector(element: HTMLElement): string {
+        let selector = element.tagName.toLowerCase();
 
+        const className = typeof element.className === 'string' ? element.className : '';
+        if (className) {
+          const classes = className.split(/\s+/).filter((cls: string) => Boolean(cls));
+          if (classes.length > 0) {
+            const validClasses = classes.filter((cls: string) => !cls.startsWith('!') && !cls.includes(':'));
+            if (validClasses.length > 0) {
+              selector += '.' + validClasses.map(cls => CSS.escape(cls)).join('.');
+            }
+          }
+        }
+
+        return selector;
+      }
+
+      function getSelectorPath(element: HTMLElement | null): string {
+        if (!element || !element.parentElement) return '';
+
+        const parentSelector = getNonUniqueSelector(element.parentElement);
+        const elementSelector = getNonUniqueSelector(element);
+
+        return `${parentSelector} > ${elementSelector}`;
+      }
+
+      function getAllDescendantSelectors(element: HTMLElement, stopAtParent: HTMLElement | null): string[] {
+        let selectors: string[] = [];
+        const children = Array.from(element.children) as HTMLElement[];
+
+        for (const child of children) {
+          selectors.push(getSelectorPath(child));
+          selectors = selectors.concat(getAllDescendantSelectors(child, stopAtParent));
+        }
+
+        return selectors;
+      }
+
+      const parentElement = document.querySelector(parentSelector) as HTMLElement;
+      if (!parentElement) return [];
+
+      return getAllDescendantSelectors(parentElement, parentElement);
+    }, parentSelector);
+
+    return childSelectors || [];
+  } catch (error) {
+    console.error('Error in getChildSelectors:', error);
+    return [];
+  }
+};
 
 /**
  * Returns the first pair from the given workflow that contains the given selector

src/components/molecules/ActionSettings.tsx

@@ -1,7 +1,7 @@
 import React, { useRef } from 'react';
 import styled from "styled-components";
 import { Button } from "@mui/material";
-import { ActionDescription } from "../organisms/RightSidePanel";
+//import { ActionDescription } from "../organisms/RightSidePanel";
 import * as Settings from "./action-settings";
 import { useSocketStore } from "../../context/socket";
 
@@ -42,7 +42,7 @@ export const ActionSettings = ({ action }: ActionSettingsProps) => {
 
   return (
     <div>
-      <ActionDescription>Action settings:</ActionDescription>
+      {/* <ActionDescription>Action settings:</ActionDescription> */}
       <ActionSettingsWrapper action={action}>
         <form onSubmit={handleSubmit}>
           <DisplaySettings />

src/components/molecules/BrowserNavBar.tsx

@@ -1,5 +1,5 @@
 import type {
-    FC,
+  FC,
 } from 'react';
 import styled from 'styled-components';
 
@@ -8,9 +8,9 @@ import ArrowBackIcon from '@mui/icons-material/ArrowBack';
 import ArrowForwardIcon from '@mui/icons-material/ArrowForward';
 
 import { NavBarButton } from '../atoms/buttons/buttons';
-import { UrlForm }  from './UrlForm';
+import { UrlForm } from './UrlForm';
 import { useCallback, useEffect, useState } from "react";
-import {useSocketStore} from "../../context/socket";
+import { useSocketStore } from "../../context/socket";
 import { getCurrentUrl } from "../../api/recording";
 
 const StyledNavBar = styled.div<{ browserWidth: number }>`
@@ -21,8 +21,8 @@ const StyledNavBar = styled.div<{ browserWidth: number }>`
 `;
 
 interface NavBarProps {
-    browserWidth: number;
+  browserWidth: number;
-    handleUrlChanged: (url: string) => void;
+  handleUrlChanged: (url: string) => void;
 };
 
 const BrowserNavBar: FC<NavBarProps> = ({
@@ -30,16 +30,15 @@ const BrowserNavBar: FC<NavBarProps> = ({
   handleUrlChanged,
 }) => {
 
-  // context:
   const { socket } = useSocketStore();
 
   const [currentUrl, setCurrentUrl] = useState<string>('https://');
 
-  const handleRefresh = useCallback(() : void => {
+  const handleRefresh = useCallback((): void => {
     socket?.emit('input:refresh');
   }, [socket]);
 
-  const handleGoTo = useCallback((address: string) : void => {
+  const handleGoTo = useCallback((address: string): void => {
     socket?.emit('input:url', address);
   }, [socket]);
 
@@ -70,54 +69,54 @@ const BrowserNavBar: FC<NavBarProps> = ({
     }
   }, [socket, handleCurrentUrlChange])
 
-    const addAddress = (address: string) => {
+  const addAddress = (address: string) => {
-        if (socket) {
+    if (socket) {
-          handleUrlChanged(address);
+      handleUrlChanged(address);
-          handleGoTo(address);
+      handleGoTo(address);
-        }
+    }
-    };
+  };
 
-    return (
+  return (
-        <StyledNavBar browserWidth={browserWidth}>
+    <StyledNavBar browserWidth={browserWidth}>
-            <NavBarButton
+      <NavBarButton
-                type="button"
+        type="button"
-                onClick={() => {
+        onClick={() => {
-                    socket?.emit('input:back');
+          socket?.emit('input:back');
-                }}
+        }}
-                disabled={false}
+        disabled={false}
-            >
+      >
-                <ArrowBackIcon/>
+        <ArrowBackIcon />
-            </NavBarButton>
+      </NavBarButton>
 
-            <NavBarButton
+      <NavBarButton
-                type="button"
+        type="button"
-                onClick={()=>{
+        onClick={() => {
-                  socket?.emit('input:forward');
+          socket?.emit('input:forward');
-                }}
+        }}
-                disabled={false}
+        disabled={false}
-            >
+      >
-                <ArrowForwardIcon/>
+        <ArrowForwardIcon />
-            </NavBarButton>
+      </NavBarButton>
 
-            <NavBarButton
+      <NavBarButton
-                type="button"
+        type="button"
-                onClick={() => {
+        onClick={() => {
-                  if (socket) {
+          if (socket) {
-                    handleRefresh()
+            handleRefresh()
-                  }
+          }
-                }}
+        }}
-                disabled={false}
+        disabled={false}
-            >
+      >
-              <ReplayIcon/>
+        <ReplayIcon />
-            </NavBarButton>
+      </NavBarButton>
 
-            <UrlForm
+      <UrlForm
-                currentAddress={currentUrl}
+        currentAddress={currentUrl}
-                handleRefresh={handleRefresh}
+        handleRefresh={handleRefresh}
-                setCurrentAddress={addAddress}
+        setCurrentAddress={addAddress}
-            />
+      />
-        </StyledNavBar>
+    </StyledNavBar>
-    );
+  );
 }
 
 export default BrowserNavBar;

src/components/molecules/InterpretationButtons.tsx

@@ -29,59 +29,60 @@ export const InterpretationButtons = ({ enableStepping }: InterpretationButtonsP
     actionType: string,
     selector: string,
     action: string,
-    open:boolean
+    open: boolean
-  }>({ pair: null, actionType: '', selector: '', action: '', open: false} );
+  }>({ pair: null, actionType: '', selector: '', action: '', open: false });
 
   const { socket } = useSocketStore();
   const { notify } = useGlobalInfoStore();
 
   const finishedHandler = useCallback(() => {
-    setInfo({...info, isPaused: false});
+    setInfo({ ...info, isPaused: false });
     enableStepping(false);
   }, [info, enableStepping]);
 
   const breakpointHitHandler = useCallback(() => {
-    setInfo({running: false, isPaused: true});
+    setInfo({ running: false, isPaused: true });
     notify('warning', 'Please restart the interpretation, after updating the recording');
     enableStepping(true);
   }, [info, enableStepping]);
 
   const decisionHandler = useCallback(
-    ({pair, actionType, lastData}
+    ({ pair, actionType, lastData }
-       : {pair: WhereWhatPair | null, actionType: string, lastData: { selector: string, action: string }}) => {
+      : { pair: WhereWhatPair | null, actionType: string, lastData: { selector: string, action: string } }) => {
-      const {selector, action} = lastData;
+      const { selector, action } = lastData;
-    setDecisionModal((prevState) => {
+      setDecisionModal((prevState) => {
-      return {
+        return {
-        pair,
+          pair,
-        actionType,
+          actionType,
-        selector,
+          selector,
-        action,
+          action,
-        open: true,
+          open: true,
-      }
+        }
-    })
+      })
-  }, [decisionModal]);
+    }, [decisionModal]);
 
   const handleDecision = (decision: boolean) => {
-    const {pair, actionType} = decisionModal;
+    const { pair, actionType } = decisionModal;
-    socket?.emit('decision', {pair, actionType, decision});
+    socket?.emit('decision', { pair, actionType, decision });
-    setDecisionModal({pair: null, actionType: '', selector: '', action: '', open: false});
+    setDecisionModal({ pair: null, actionType: '', selector: '', action: '', open: false });
   }
 
   const handleDescription = () => {
-    switch (decisionModal.actionType){
+    switch (decisionModal.actionType) {
       case 'customAction':
         return (
           <React.Fragment>
-          <Typography>
+            <Typography>
-          Do you want to use the previously recorded selector
+              Do you want to use the previously recorded selector
-          as a where condition for matching the action?
+              as a where condition for matching the action?
-        </Typography>
+            </Typography>
-        <Box style={{marginTop: '4px'}}>
+            <Box style={{ marginTop: '4px' }}>
-          [previous action: <b>{decisionModal.action}</b>]
+              [previous action: <b>{decisionModal.action}</b>]
-          <pre>{decisionModal.selector}</pre>
+              <pre>{decisionModal.selector}</pre>
-        </Box>
+            </Box>
           </React.Fragment>);
-      default: return null;}
+      default: return null;
+    }
   }
 
   useEffect(() => {
@@ -100,12 +101,12 @@ export const InterpretationButtons = ({ enableStepping }: InterpretationButtonsP
   const handlePlay = async () => {
     if (info.isPaused) {
       socket?.emit("resume");
-      setInfo({running: true, isPaused: false});
+      setInfo({ running: true, isPaused: false });
       enableStepping(false);
     } else {
-      setInfo({...info, running: true});
+      setInfo({ ...info, running: true });
       const finished = await interpretCurrentRecording();
-      setInfo({...info, running: false});
+      setInfo({ ...info, running: false });
       if (finished) {
         notify('info', 'Interpretation finished');
       } else {
@@ -131,45 +132,45 @@ export const InterpretationButtons = ({ enableStepping }: InterpretationButtonsP
 
   return (
     <Stack direction="row" spacing={3}
-    sx={{ marginTop: '10px', marginBottom: '5px', justifyContent: 'space-evenly',}} >
+      sx={{ marginTop: '10px', marginBottom: '5px', justifyContent: 'space-evenly', }} >
-      <IconButton disabled={!info.running} sx={{display:'grid', '&:hover': { color: '#1976d2', backgroundColor: 'transparent' }}}
+      <IconButton disabled={!info.running} sx={{ display: 'grid', '&:hover': { color: '#1976d2', backgroundColor: 'transparent' } }}
-                  aria-label="pause" size="small" title="Pause" onClick={handlePause}>
+        aria-label="pause" size="small" title="Pause" onClick={handlePause}>
-        <PauseCircle sx={{ fontSize: 30, justifySelf:'center' }}/>
+        <PauseCircle sx={{ fontSize: 30, justifySelf: 'center' }} />
         Pause
       </IconButton>
-      <IconButton disabled={info.running} sx={{display:'grid', '&:hover': { color: '#1976d2', backgroundColor: 'transparent' }}}
+      <IconButton disabled={info.running} sx={{ display: 'grid', '&:hover': { color: '#1976d2', backgroundColor: 'transparent' } }}
-                  aria-label="play" size="small" title="Play" onClick={handlePlay}>
+        aria-label="play" size="small" title="Play" onClick={handlePlay}>
-        <PlayCircle sx={{ fontSize: 30, justifySelf:'center' }}/>
+        <PlayCircle sx={{ fontSize: 30, justifySelf: 'center' }} />
         {info.isPaused ? 'Resume' : 'Start'}
       </IconButton>
-      <IconButton disabled={!info.running && !info.isPaused} sx={{display:'grid', '&:hover': { color: '#1976d2', backgroundColor: 'transparent' }}}
+      <IconButton disabled={!info.running && !info.isPaused} sx={{ display: 'grid', '&:hover': { color: '#1976d2', backgroundColor: 'transparent' } }}
         aria-label="stop" size="small" title="Stop" onClick={handleStop}>
-        <StopCircle sx={{ fontSize: 30, justifySelf:'center' }}/>
+        <StopCircle sx={{ fontSize: 30, justifySelf: 'center' }} />
         Stop
       </IconButton>
-      <GenericModal onClose={() => {}} isOpen={decisionModal.open} canBeClosed={false}
+      <GenericModal onClose={() => { }} isOpen={decisionModal.open} canBeClosed={false}
-      modalStyle={{
+        modalStyle={{
-        position: 'absolute',
+          position: 'absolute',
-        top: '50%',
+          top: '50%',
-        left: '50%',
+          left: '50%',
-        transform: 'translate(-50%, -50%)',
+          transform: 'translate(-50%, -50%)',
-        width: 500,
+          width: 500,
-        background: 'white',
+          background: 'white',
-        border: '2px solid #000',
+          border: '2px solid #000',
-        boxShadow: '24',
+          boxShadow: '24',
-        height:'fit-content',
+          height: 'fit-content',
-        display:'block',
+          display: 'block',
-        overflow:'scroll',
+          overflow: 'scroll',
-        padding: '5px 25px 10px 25px',
+          padding: '5px 25px 10px 25px',
-      }}>
+        }}>
-        <div style={{padding: '15px'}}>
+        <div style={{ padding: '15px' }}>
-          <HelpIcon/>
+          <HelpIcon />
           {
             handleDescription()
           }
-          <div style={{float: 'right'}}>
+          <div style={{ float: 'right' }}>
-          <Button onClick={() => handleDecision(true)} color='success'>yes</Button>
+            <Button onClick={() => handleDecision(true)} color='success'>yes</Button>
-          <Button onClick={() => handleDecision(false)} color='error'>no</Button>
+            <Button onClick={() => handleDecision(false)} color='error'>no</Button>
           </div>
         </div>
       </GenericModal>

src/components/molecules/InterpretationLog.tsx

@@ -1,30 +1,57 @@
 import * as React from 'react';
-import Accordion from '@mui/material/Accordion';
+import SwipeableDrawer from '@mui/material/SwipeableDrawer';
-import AccordionDetails from '@mui/material/AccordionDetails';
-import AccordionSummary from '@mui/material/AccordionSummary';
 import Typography from '@mui/material/Typography';
-import ExpandMoreIcon from '@mui/icons-material/ExpandMore';
+import Radio from '@mui/material/Radio';
-import Highlight from 'react-highlight'
+import RadioGroup from '@mui/material/RadioGroup';
+import { Button, TextField } from '@mui/material';
+import FormControlLabel from '@mui/material/FormControlLabel';
+import FormControl from '@mui/material/FormControl';
+import FormLabel from '@mui/material/FormLabel';
+import Highlight from 'react-highlight';
 import { useCallback, useEffect, useRef, useState } from "react";
 import { useSocketStore } from "../../context/socket";
+import { useBrowserDimensionsStore } from "../../context/browserDimensions";
+import Table from '@mui/material/Table';
+import TableBody from '@mui/material/TableBody';
+import TableCell from '@mui/material/TableCell';
+import TableContainer from '@mui/material/TableContainer';
+import TableHead from '@mui/material/TableHead';
+import TableRow from '@mui/material/TableRow';
+import Paper from '@mui/material/Paper';
+import StorageIcon from '@mui/icons-material/Storage';
 
-export const InterpretationLog = () => {
+interface InterpretationLogProps {
-  const [expanded, setExpanded] = useState<boolean>(false);
+  isOpen: boolean;
+  setIsOpen: (isOpen: boolean) => void;
+}
+
+export const InterpretationLog: React.FC<InterpretationLogProps> = ({ isOpen, setIsOpen }) => {  
   const [log, setLog] = useState<string>('');
+  const [selectedOption, setSelectedOption] = useState<string>('10');
+  const [customValue, setCustomValue] = useState('');
+  const [tableData, setTableData] = useState<any[]>([]);
 
   const logEndRef = useRef<HTMLDivElement | null>(null);
 
-  const handleChange = (isExpanded: boolean) => (event: React.SyntheticEvent) => {
+  const { width } = useBrowserDimensionsStore();
-    setExpanded(isExpanded);
-  };
-
   const { socket } = useSocketStore();
 
+  const toggleDrawer = (newOpen: boolean) => (event: React.KeyboardEvent | React.MouseEvent) => {
+    if (
+      event.type === 'keydown' &&
+      ((event as React.KeyboardEvent).key === 'Tab' ||
+        (event as React.KeyboardEvent).key === 'Shift')
+    ) {
+      return;
+    }
+    setIsOpen(newOpen);
+  };
+
   const scrollLogToBottom = () => {
     if (logEndRef.current) {
-      logEndRef.current.scrollIntoView({ behavior: "smooth" })
+      logEndRef.current.scrollIntoView({ behavior: "smooth" });
     }
-  }
+  };
 
   const handleLog = useCallback((msg: string, date: boolean = true) => {
     if (!date) {
@@ -33,14 +60,20 @@ export const InterpretationLog = () => {
       setLog((prevState) => prevState + '\n' + `[${new Date().toLocaleString()}] ` + msg);
     }
     scrollLogToBottom();
-  }, [log, scrollLogToBottom])
+  }, [log, scrollLogToBottom]);
 
-  const handleSerializableCallback = useCallback((data: string) => {
+  const handleSerializableCallback = useCallback((data: any) => {
     setLog((prevState) =>
       prevState + '\n' + '---------- Serializable output data received ----------' + '\n'
       + JSON.stringify(data, null, 2) + '\n' + '--------------------------------------------------');
+    
+    // Set table data
+    if (Array.isArray(data)) {
+      setTableData(data);
+    }
+    
     scrollLogToBottom();
-  }, [log, scrollLogToBottom])
+  }, [log, scrollLogToBottom]);
 
   const handleBinaryCallback = useCallback(({ data, mimetype }: any) => {
     setLog((prevState) =>
@@ -48,7 +81,15 @@ export const InterpretationLog = () => {
       + `mimetype: ${mimetype}` + '\n' + `data: ${JSON.stringify(data)}` + '\n'
       + '------------------------------------------------');
     scrollLogToBottom();
-  }, [log, scrollLogToBottom])
+  }, [log, scrollLogToBottom]);
+
+  const handleRadioChange = (event: React.ChangeEvent<HTMLInputElement>) => {
+    setSelectedOption(event.target.value);
+  };
+
+  const handleCustomValueChange = (event: React.ChangeEvent<HTMLInputElement>) => {
+    setCustomValue(event.target.value);
+  };
 
   useEffect(() => {
     socket?.on('log', handleLog);
@@ -58,41 +99,113 @@ export const InterpretationLog = () => {
       socket?.off('log', handleLog);
       socket?.off('serializableCallback', handleSerializableCallback);
       socket?.off('binaryCallback', handleBinaryCallback);
-    }
+    };
-  }, [socket, handleLog])
+  }, [socket, handleLog, handleSerializableCallback, handleBinaryCallback]);
 
+  // Extract columns dynamically from the first item of tableData
+  const columns = tableData.length > 0 ? Object.keys(tableData[0]) : [];
+  
   return (
     <div>
-      <Accordion
+      <button
-        expanded={expanded}
+        onClick={toggleDrawer(true)}
-        onChange={handleChange(!expanded)}
+        style={{
-        style={{ background: '#3f4853', color: 'white', borderRadius: '0px' }}
+          color: 'white',
-      >
+          background: '#3f4853',
-        <AccordionSummary
+          border: 'none',
-          expandIcon={<ExpandMoreIcon sx={{ color: 'white' }} />}
+          padding: '10px 20px',
-          aria-controls="panel1bh-content"
+          width: 1280,
-          id="panel1bh-header"
+          textAlign: 'left'
-        >
-          <Typography sx={{ width: '33%', flexShrink: 0 }}>
-            Interpretation Log
-          </Typography>
-        </AccordionSummary>
-        <AccordionDetails sx={{
-          background: '#19171c',
-          overflowY: 'scroll',
-          width: '100%',
-          aspectRatio: '4/1',
-          boxSizing: 'border-box',
         }}>
-          <div>
+        Interpretation Log
-            <Highlight className="javascript">
+      </button>
-              {log}
+      <SwipeableDrawer
-            </Highlight>
+        anchor="bottom"
-            <div style={{ float: "left", clear: "both" }}
+        open={isOpen}
-              ref={logEndRef} />
+        onClose={toggleDrawer(false)}
+        onOpen={toggleDrawer(true)}
+        PaperProps={{
+          sx: {
+            background: 'white',
+            color: 'black',
+            padding: '10px',
+            height: 720,
+            width: width - 10,
+            display: 'flex'
+          }
+        }}
+      >
+        <Typography variant="h6" gutterBottom>
+          <StorageIcon /> Output Data Preview
+        </Typography>
+        <div style={{
+          height: '50vh',
+          overflow: 'none',
+          padding: '10px',
+        }}>
+          {/* <Highlight className="javascript">
+            {log}
+          </Highlight> */}
+          {tableData.length > 0 && (
+            <TableContainer component={Paper}>
+              <Table sx={{ minWidth: 650 }} stickyHeader aria-label="output data table">
+                <TableHead>
+                  <TableRow>
+                    {columns.map((column) => (
+                      <TableCell key={column}>{column}</TableCell>
+                    ))}
+                  </TableRow>
+                </TableHead>
+                <TableBody>
+                  {tableData.map((row, index) => (
+                    <TableRow key={index}>
+                      {columns.map((column) => (
+                        <TableCell key={column}>{row[column]}</TableCell>
+                      ))}
+                    </TableRow>
+                  ))}
+                </TableBody>
+              </Table>
+            </TableContainer>
+          )}
+          <div style={{ display: 'flex', alignItems: 'flex-start', gap: '200px' }}>
+            <FormControl>
+              <FormLabel>
+                <h4>What is the maximum number of rows you want to extract?</h4>
+              </FormLabel>
+              <RadioGroup row value={selectedOption} onChange={handleRadioChange} sx={{ width: '500px' }}>
+                <FormControlLabel value="10" control={<Radio />} label="10" />
+                <FormControlLabel value="100" control={<Radio />} label="100" />
+                <FormControlLabel value="custom" control={<Radio />} label="Custom" />
+                {selectedOption === 'custom' && (
+                  <TextField
+                    type="number"
+                    value={customValue}
+                    onChange={handleCustomValueChange}
+                    placeholder="Enter number"
+                    sx={{
+                      marginLeft: '10px',
+                      marginTop: '-3px',
+                      '& input': {
+                        padding: '10px',
+                      },
+                    }}
+                  />
+                )}
+              </RadioGroup>
+            </FormControl>
+            <div style={{ paddingBottom: '40px' }}>
+              <h4>How can we find the next item?</h4>
+              <p>Select and review the pagination setting this webpage is using</p>
+              <Button variant="outlined">
+                Select Pagination Setting
+              </Button>
+            </div>
           </div>
-        </AccordionDetails>
+          <div style={{ float: "left", clear: "both" }}
-      </Accordion>
+            ref={logEndRef} />
+        </div>
+      </SwipeableDrawer>
     </div>
   );
 }

src/components/organisms/BrowserContent.tsx

@@ -123,7 +123,8 @@ export const BrowserContent = () => {
         tabIndex={tabIndex}
       />
       <BrowserNavBar
-        browserWidth={width - 10}
+        // todo: use width from browser dimension once fixed
+        browserWidth={1270}
         handleUrlChanged={handleUrlChanged}
       />
       <BrowserWindow/>

src/components/organisms/BrowserWindow.tsx

@@ -50,17 +50,19 @@ const getAttributeOptions = (tagName: string, elementInfo: ElementInfo | null):
 export const BrowserWindow = () => {
     const [canvasRef, setCanvasReference] = useState<React.RefObject<HTMLCanvasElement> | undefined>(undefined);
     const [screenShot, setScreenShot] = useState<string>("");
-    const [highlighterData, setHighlighterData] = useState<{ rect: DOMRect, selector: string, elementInfo: ElementInfo | null; } | null>(null);
+    const [highlighterData, setHighlighterData] = useState<{ rect: DOMRect, selector: string, elementInfo: ElementInfo | null, childSelectors?: string[] } | null>(null);
     const [showAttributeModal, setShowAttributeModal] = useState(false);
     const [attributeOptions, setAttributeOptions] = useState<AttributeOption[]>([]);
     const [selectedElement, setSelectedElement] = useState<{ selector: string, info: ElementInfo | null } | null>(null);
+    const [currentListId, setCurrentListId] = useState<number | null>(null);
 
     const [listSelector, setListSelector] = useState<string | null>(null);
     const [fields, setFields] = useState<Record<string, TextStep>>({});
+    const [paginationSelector, setPaginationSelector] = useState<string>('');
 
     const { socket } = useSocketStore();
     const { width, height } = useBrowserDimensionsStore();
-    const { getText, getList } = useActionContext();
+    const { getText, getList, paginationMode, paginationType } = useActionContext();
     const { addTextStep, addListStep } = useBrowserSteps();
 
     const onMouseMove = (e: MouseEvent) => {
@@ -78,6 +80,18 @@ export const BrowserWindow = () => {
         }
     };
 
+    const resetListState = useCallback(() => {
+        setListSelector(null);
+        setFields({});
+        setCurrentListId(null);
+    }, []);
+
+    useEffect(() => {
+        if (!getList) {
+            resetListState();
+        }
+    }, [getList, resetListState]);
+
     const screencastHandler = useCallback((data: string) => {
         setScreenShot(data);
     }, [screenShot]);
@@ -96,12 +110,33 @@ export const BrowserWindow = () => {
         }
     }, [screenShot, canvasRef, socket, screencastHandler]);
 
-    const highlighterHandler = useCallback((data: { rect: DOMRect, selector: string, elementInfo: ElementInfo | null }) => {
+    const highlighterHandler = useCallback((data: { rect: DOMRect, selector: string, elementInfo: ElementInfo | null, childSelectors?: string[] }) => {
         if (getList === true) {
             socket?.emit('setGetList', { getList: true });
+            if (listSelector) {
+                socket?.emit('listSelector', { selector: listSelector });
+                if (paginationMode) {
+                    // Pagination mode: only set highlighterData if type is not empty, 'none', 'scrollDown', or 'scrollUp'
+                    if (paginationType !== '' && paginationType !== 'scrollDown' && paginationType !== 'scrollUp' && paginationType !== 'none') {
+                        setHighlighterData(data);
+                    } else {
+                        setHighlighterData(null);
+                    }
+                } else if (data.childSelectors && data.childSelectors.includes(data.selector)) {
+                    // !Pagination mode: highlight only valid child elements within the listSelector
+                    setHighlighterData(data);
+                } else {
+                    // If not a valid child in normal mode, clear the highlighter
+                    setHighlighterData(null);
+                }
+            } else {
+                setHighlighterData(data); // Set highlighterData for the initial listSelector selection
+            }
+        } else {
+            setHighlighterData(data); // For non-list steps
         }
-        setHighlighterData(data);
+    }, [highlighterData, getList, socket, listSelector, paginationMode, paginationType]);
-    }, [highlighterData, getList, socket]);
+
 
     useEffect(() => {
         document.addEventListener('mousemove', onMouseMove, false);
@@ -127,6 +162,7 @@ export const BrowserWindow = () => {
                 clickY >= highlightRect.top &&
                 clickY <= highlightRect.bottom
             ) {
+
                 const options = getAttributeOptions(highlighterData.elementInfo?.tagName || '', highlighterData.elementInfo);
 
                 if (getText === true) {
@@ -153,17 +189,32 @@ export const BrowserWindow = () => {
                     }
                 }
 
+                if (paginationMode && getList) {
+                    // Only allow selection in pagination mode if type is not empty, 'scrollDown', or 'scrollUp'
+                    if (paginationType !== '' && paginationType !== 'scrollDown' && paginationType !== 'scrollUp' && paginationType !== 'none') {
+                        setPaginationSelector(highlighterData.selector);
+                        addListStep(listSelector!, fields, currentListId || 0, { type: paginationType, selector: highlighterData.selector });
+                    }
+                    return;
+                }
+
                 if (getList === true && !listSelector) {
                     setListSelector(highlighterData.selector);
-                } else if (getList === true && listSelector) {
+                    setCurrentListId(Date.now());
+                    setFields({});
+                } else if (getList === true && listSelector && currentListId) {
+                    const attribute = options[0].value;
+                    const data = attribute === 'href' ? highlighterData.elementInfo?.url || '' :
+                        attribute === 'src' ? highlighterData.elementInfo?.imageUrl || '' :
+                            highlighterData.elementInfo?.innerText || '';
+                    // Add fields to the list
                     if (options.length === 1) {
-                        // Handle directly without showing the modal
                         const attribute = options[0].value;
                         const newField: TextStep = {
                             id: Date.now(),
                             type: 'text',
                             label: `Label ${Object.keys(fields).length + 1}`,
-                            data: highlighterData.elementInfo?.innerText || '',
+                            data: data,
                             selectorObj: {
                                 selector: highlighterData.selector,
                                 tag: highlighterData.elementInfo?.tagName,
@@ -176,14 +227,15 @@ export const BrowserWindow = () => {
                                 ...prevFields,
                                 [newField.label]: newField
                             };
+                            console.log(updatedFields)
                             return updatedFields;
                         });
 
                         if (listSelector) {
-                            addListStep(listSelector, { ...fields, [newField.label]: newField });
+                            addListStep(listSelector, { ...fields, [newField.label]: newField }, currentListId, { type: '', selector: paginationSelector });
                         }
+
                     } else {
-                        // Show the modal if there are multiple options
                         setAttributeOptions(options);
                         setSelectedElement({
                             selector: highlighterData.selector,
@@ -217,12 +269,12 @@ export const BrowserWindow = () => {
                         attribute: attribute
                     });
                 }
-                if (getList === true) {
+                if (getList === true && listSelector && currentListId) {
                     const newField: TextStep = {
                         id: Date.now(),
                         type: 'text',
                         label: `Label ${Object.keys(fields).length + 1}`,
-                        data: selectedElement.info?.innerText || '',
+                        data: data,
                         selectorObj: {
                             selector: selectedElement.selector,
                             tag: selectedElement.info?.tagName,
@@ -235,18 +287,32 @@ export const BrowserWindow = () => {
                             ...prevFields,
                             [newField.label]: newField
                         };
+                        console.log(updatedFields)
+
                         return updatedFields;
                     });
 
                     if (listSelector) {
-                        addListStep(listSelector, { ...fields, [newField.label]: newField });
+                        addListStep(listSelector, { ...fields, [newField.label]: newField }, currentListId, { type: '', selector: paginationSelector });
                     }
+
                 }
             }
         }
         setShowAttributeModal(false);
     };
 
+    const resetPaginationSelector = useCallback(() => {
+        setPaginationSelector('');
+    }, []);
+
+    useEffect(() => {
+        if (!paginationMode) {
+            resetPaginationSelector();
+        }
+    }, [paginationMode, resetPaginationSelector]);
+
+
     return (
         <div onClick={handleClick}>
             {

src/components/organisms/LeftSidePanel.tsx

@@ -14,21 +14,21 @@ import { RunSettings } from "../molecules/RunSettings";
 
 const fetchWorkflow = (id: string, callback: (response: WorkflowFile) => void) => {
   getActiveWorkflow(id).then(
-    (response ) => {
+    (response) => {
-      if (response){
+      if (response) {
         callback(response);
       } else {
         throw new Error("No workflow found");
       }
     }
-  ).catch((error) => {console.log(error.message)})
+  ).catch((error) => { console.log(error.message) })
 };
 
 interface LeftSidePanelProps {
   sidePanelRef: HTMLDivElement | null;
   alreadyHasScrollbar: boolean;
   recordingName: string;
-  handleSelectPairForEdit: (pair:WhereWhatPair, index:number) => void;
+  handleSelectPairForEdit: (pair: WhereWhatPair, index: number) => void;
 }
 
 export const LeftSidePanel = (
@@ -59,10 +59,11 @@ export const LeftSidePanel = (
       fetchWorkflow(id, workflowHandler);
     }
     // fetch workflow in 15min intervals
-    let interval = setInterval(() =>{
+    let interval = setInterval(() => {
-    if (id) {
+      if (id) {
-      fetchWorkflow(id, workflowHandler);
+        fetchWorkflow(id, workflowHandler);
-    }}, (1000 * 60 * 15));
+      }
+    }, (1000 * 60 * 15));
     return () => clearInterval(interval)
   }, [id]);
 
@@ -104,19 +105,19 @@ export const LeftSidePanel = (
         flexDirection: 'column',
       }}
     >
-      <SidePanelHeader/>
+      <SidePanelHeader />
       <TabContext value={tab}>
-          <Tabs value={tab} onChange={(e, newTab) => setTab(newTab)}>
+        <Tabs value={tab} onChange={(e, newTab) => setTab(newTab)}>
-            <Tab label="Recording" value='recording' />
+          <Tab label="Recording" value='recording' />
-            <Tab label="Settings" value='settings' onClick={() => {
+          <Tab label="Settings" value='settings' onClick={() => {
-              getParamsOfActiveWorkflow(id).then((response) => {
+            getParamsOfActiveWorkflow(id).then((response) => {
-                if (response) {
+              if (response) {
-                  setParams(response);
+                setParams(response);
-                }
+              }
-              })
+            })
-            }}/>
+          }} />
-          </Tabs>
+        </Tabs>
-        <TabPanel value='recording' sx={{padding: '0px'}}>
+        <TabPanel value='recording' sx={{ padding: '0px' }}>
           <LeftSidePanelContent
             workflow={workflow}
             updateWorkflow={setWorkflow}
@@ -126,7 +127,7 @@ export const LeftSidePanel = (
         </TabPanel>
         <TabPanel value='settings'>
           <LeftSidePanelSettings params={params}
-          settings={settings} setSettings={setSettings}/>
+            settings={settings} setSettings={setSettings} />
         </TabPanel>
       </TabContext>
     </Paper>

src/components/organisms/RightSidePanel.tsx

@@ -3,29 +3,40 @@ import { Button, Paper, Box, TextField } from "@mui/material";
 import EditIcon from '@mui/icons-material/Edit';
 import TextFieldsIcon from '@mui/icons-material/TextFields';
 import DocumentScannerIcon from '@mui/icons-material/DocumentScanner';
-import styled from "styled-components";
 import { SimpleBox } from "../atoms/Box";
 import Typography from "@mui/material/Typography";
 import { useGlobalInfoStore } from "../../context/globalInfo";
-import { useActionContext } from '../../context/browserActions';
+import { PaginationType, useActionContext, LimitType } from '../../context/browserActions';
-import { useBrowserSteps, ListStep, TextStep, SelectorObject } from '../../context/browserSteps';
+import { useBrowserSteps } from '../../context/browserSteps';
 import { useSocketStore } from '../../context/socket';
 import { ScreenshotSettings } from '../../shared/types';
 import InputAdornment from '@mui/material/InputAdornment';
+import { SidePanelHeader } from '../molecules/SidePanelHeader';
+import FormControlLabel from '@mui/material/FormControlLabel';
+import FormControl from '@mui/material/FormControl';
+import FormLabel from '@mui/material/FormLabel';
+import Radio from '@mui/material/Radio';
+import RadioGroup from '@mui/material/RadioGroup';
 
 // TODO: 
 // 1. Handle field label update 
 // 2. Handle field deletion | confirmation
 // 3. Add description for each browser step
 // 4. Handle non custom action steps
+interface RightSidePanelProps {
+  onFinishCapture: () => void;
+}
 
-export const RightSidePanel = () => {
+export const RightSidePanel: React.FC<RightSidePanelProps> = ({ onFinishCapture }) => {
   const [textLabels, setTextLabels] = useState<{ [id: number]: string }>({});
   const [errors, setErrors] = useState<{ [id: number]: string }>({});
   const [confirmedTextSteps, setConfirmedTextSteps] = useState<{ [id: number]: boolean }>({});
+  const [showPaginationOptions, setShowPaginationOptions] = useState(false);
+  const [showLimitOptions, setShowLimitOptions] = useState(false);
+  const [captureStage, setCaptureStage] = useState<'initial' | 'pagination' | 'limit' | 'complete'>('initial');
 
   const { lastAction, notify } = useGlobalInfoStore();
-  const { getText, startGetText, stopGetText, getScreenshot, startGetScreenshot, stopGetScreenshot, getList, startGetList, stopGetList } = useActionContext();
+  const { getText, startGetText, stopGetText, getScreenshot, startGetScreenshot, stopGetScreenshot, paginationMode, getList, startGetList, stopGetList, startPaginationMode, stopPaginationMode, paginationType, updatePaginationType, limitMode, limitType, customLimit, updateLimitType, updateCustomLimit, stopLimitMode, startLimitMode } = useActionContext();
   const { browserSteps, updateBrowserTextStepLabel, deleteBrowserStep, addScreenshotStep } = useBrowserSteps();
   const { socket } = useSocketStore();
 
@@ -83,11 +94,17 @@ export const RightSidePanel = () => {
     if (hasTextSteps) {
       socket?.emit('action', { action: 'scrapeSchema', settings });
     }
+    onFinishCapture();
   }, [stopGetText, getTextSettingsObject, socket, browserSteps, confirmedTextSteps]);
 
 
   const getListSettingsObject = useCallback(() => {
-    let settings: { listSelector?: string; fields?: Record<string, { selector: string; tag?: string;[key: string]: any }> } = {};
+    let settings: {
+      listSelector?: string;
+      fields?: Record<string, { selector: string; tag?: string;[key: string]: any }>;
+      pagination?: { type: string; selector?: string };
+      limit?: number;
+    } = {};
 
     browserSteps.forEach(step => {
       if (step.type === 'list' && step.listSelector && Object.keys(step.fields).length > 0) {
@@ -97,36 +114,101 @@ export const RightSidePanel = () => {
             fields[label] = {
               selector: field.selectorObj.selector,
               tag: field.selectorObj.tag,
-              attribute: field.selectorObj.attribute
+              attribute: field.selectorObj.attribute,
             };
           }
         });
 
         settings = {
           listSelector: step.listSelector,
-          fields: fields
+          fields: fields,
+          pagination: { type: paginationType, selector: step.pagination?.selector },
+          limit: parseInt(limitType === 'custom' ? customLimit : limitType),
         };
 
       }
     });
 
     return settings;
-  }, [browserSteps]);
+  }, [browserSteps, paginationType, limitType, customLimit]);
 
+  const resetListState = useCallback(() => {
+    setShowPaginationOptions(false);
+    updatePaginationType('');
+    setShowLimitOptions(false);
+    updateLimitType('');
+    updateCustomLimit('');
+  }, [updatePaginationType, updateLimitType, updateCustomLimit]);
+
+  const handleStopGetList = useCallback(() => {
+    stopGetList();
+    resetListState();
+  }, [stopGetList, resetListState]);
 
   const stopCaptureAndEmitGetListSettings = useCallback(() => {
-    stopGetList();
     const settings = getListSettingsObject();
     if (settings) {
       socket?.emit('action', { action: 'scrapeList', settings });
     } else {
       notify('error', 'Unable to create list settings. Make sure you have defined a field for the list.');
     }
-  }, [stopGetList, getListSettingsObject, socket, notify]);
+    handleStopGetList();
+    onFinishCapture();
+  }, [stopGetList, getListSettingsObject, socket, notify, handleStopGetList]);
 
-  // const handleListFieldChange = (stepId: number, key: 'label' | 'data', value: string) => {
+  const handleConfirmListCapture = useCallback(() => {
-  //   updateListStepField(stepId, key, value);
+    switch (captureStage) {
-  // };  
+      case 'initial':
+        startPaginationMode();
+        setShowPaginationOptions(true);
+        setCaptureStage('pagination');
+        break;
+
+      case 'pagination':
+        if (!paginationType) {
+          notify('error', 'Please select a pagination type.');
+          return;
+        }
+        const settings = getListSettingsObject();
+        const paginationSelector = settings.pagination?.selector;
+        if (['clickNext', 'clickLoadMore'].includes(paginationType) && !paginationSelector) {
+          notify('error', 'Please select the pagination element first.');
+          return;
+        }
+        stopPaginationMode();
+        setShowPaginationOptions(false);
+        startLimitMode();
+        setShowLimitOptions(true);
+        setCaptureStage('limit');
+        break;
+
+      case 'limit':
+        if (!limitType || (limitType === 'custom' && !customLimit)) {
+          notify('error', 'Please select a limit or enter a custom limit.');
+          return;
+        }
+        const limit = limitType === 'custom' ? parseInt(customLimit) : parseInt(limitType);
+        if (isNaN(limit) || limit <= 0) {
+          notify('error', 'Please enter a valid limit.');
+          return;
+        }
+        stopLimitMode();
+        setShowLimitOptions(false);
+        stopCaptureAndEmitGetListSettings();
+        setCaptureStage('complete');
+        break;
+
+      case 'complete':
+        setCaptureStage('initial');
+        break;
+    }
+  }, [captureStage, paginationType, limitType, customLimit, startPaginationMode, stopPaginationMode, startLimitMode, stopLimitMode, notify, stopCaptureAndEmitGetListSettings, getListSettingsObject]);
+
+  const handlePaginationSettingSelect = (option: PaginationType) => {
+    updatePaginationType(option);
+    if (['clickNext', 'clickLoadMore'].includes(option)) {
+    }
+  };
 
   const captureScreenshot = (fullPage: boolean) => {
     const screenshotSettings: ScreenshotSettings = {
@@ -147,28 +229,76 @@ export const RightSidePanel = () => {
       <SimpleBox height={60} width='100%' background='lightGray' radius='0%'>
         <Typography sx={{ padding: '10px' }}>Last action: {` ${lastAction}`}</Typography>
       </SimpleBox>
-
+      <SidePanelHeader />
       <Box display="flex" flexDirection="column" gap={2} style={{ margin: '15px' }}>
         {!getText && !getScreenshot && !getList && <Button variant="contained" onClick={startGetList}>Capture List</Button>}
-        {getList &&
+        {getList && (
           <>
             <Box display="flex" justifyContent="space-between" gap={2} style={{ margin: '15px' }}>
-              <Button variant="outlined" onClick={stopCaptureAndEmitGetListSettings}>Confirm</Button>
+              <Button variant="outlined" onClick={handleConfirmListCapture}>
-              <Button variant="outlined" color="error" onClick={stopGetList}>Discard</Button>
+                {captureStage === 'initial' ? 'Confirm Capture' :
+                  captureStage === 'pagination' ? 'Confirm Pagination' :
+                    captureStage === 'limit' ? 'Confirm Limit' : 'Finish Capture'}
+              </Button>
+              <Button variant="outlined" color="error" onClick={handleStopGetList}>Discard</Button>
             </Box>
           </>
-        }
+        )}
-
+        {showPaginationOptions && (
+          <Box display="flex" flexDirection="column" gap={2} style={{ margin: '15px' }}>
+            <Typography>How can we find the next list item on the page?</Typography>
+            <Button variant={paginationType === 'clickNext' ? "contained" : "outlined"} onClick={() => handlePaginationSettingSelect('clickNext')}>Click on next to navigate to the next page</Button>
+            <Button variant={paginationType === 'clickLoadMore' ? "contained" : "outlined"} onClick={() => handlePaginationSettingSelect('clickLoadMore')}>Click on load more to load more items</Button>
+            <Button variant={paginationType === 'scrollDown' ? "contained" : "outlined"} onClick={() => handlePaginationSettingSelect('scrollDown')}>Scroll down to load more items</Button>
+            <Button variant={paginationType === 'scrollUp' ? "contained" : "outlined"} onClick={() => handlePaginationSettingSelect('scrollUp')}>Scroll up to load more items</Button>
+            <Button variant={paginationType === 'none' ? "contained" : "outlined"} onClick={() => handlePaginationSettingSelect('none')}>No more items to load</Button>
+          </Box>
+        )}
+        {showLimitOptions && (
+          <FormControl>
+            <FormLabel>
+              <h4>What is the maximum number of rows you want to extract?</h4>
+            </FormLabel>
+            <RadioGroup
+              value={limitType}
+              onChange={(e) => updateLimitType(e.target.value as LimitType)}
+              sx={{
+                display: 'flex',
+                flexDirection: 'column',
+                width: '500px'
+              }}
+            >
+              <FormControlLabel value="10" control={<Radio />} label="10" />
+              <FormControlLabel value="100" control={<Radio />} label="100" />
+              <div style={{ display: 'flex', alignItems: 'center' }}>
+                <FormControlLabel value="custom" control={<Radio />} label="Custom" />
+                {limitType === 'custom' && (
+                  <TextField
+                    type="number"
+                    value={customLimit}
+                    onChange={(e) => updateCustomLimit(e.target.value)}
+                    placeholder="Enter number"
+                    sx={{
+                      marginLeft: '10px',
+                      '& input': {
+                        padding: '10px',
+                      },
+                    }}
+                  />
+                )}
+              </div>
+            </RadioGroup>
+          </FormControl>
+        )}
         {!getText && !getScreenshot && !getList && <Button variant="contained" onClick={startGetText}>Capture Text</Button>}
         {getText &&
           <>
             <Box display="flex" justifyContent="space-between" gap={2} style={{ margin: '15px' }}>
-              <Button variant="outlined" onClick={stopCaptureAndEmitGetTextSettings}>Confirm</Button>
+              <Button variant="outlined" onClick={stopCaptureAndEmitGetTextSettings} >Confirm</Button>
-              <Button variant="outlined" color="error" onClick={stopGetText}>Discard</Button>
+              <Button variant="outlined" color="error" onClick={stopGetText} >Discard</Button>
             </Box>
           </>
         }
-
         {!getText && !getScreenshot && !getList && <Button variant="contained" onClick={startGetScreenshot}>Capture Screenshot</Button>}
         {getScreenshot && (
           <Box display="flex" flexDirection="column" gap={2}>
@@ -178,7 +308,6 @@ export const RightSidePanel = () => {
           </Box>
         )}
       </Box>
-
       <Box>
         {browserSteps.map(step => (
           <Box key={step.id} sx={{ boxShadow: 5, padding: '10px', margin: '10px', borderRadius: '4px' }}>

src/context/browserActions.tsx

@@ -1,15 +1,30 @@
 import React, { createContext, useContext, useState, ReactNode } from 'react';
 
+export type PaginationType = 'scrollDown' | 'scrollUp' | 'clickNext' | 'clickLoadMore' | 'none' | '';
+export type LimitType = '10' | '100' | 'custom' | '';
+
 interface ActionContextProps {
     getText: boolean;
     getList: boolean;
     getScreenshot: boolean;
+    paginationMode: boolean;
+    limitMode: boolean;
+    paginationType: PaginationType;
+    limitType: LimitType;
+    customLimit: string;
+    startPaginationMode: () => void;
     startGetText: () => void;
     stopGetText: () => void;
     startGetList: () => void;
     stopGetList: () => void;
     startGetScreenshot: () => void;
     stopGetScreenshot: () => void;
+    stopPaginationMode: () => void;
+    updatePaginationType: (type: PaginationType) => void;
+    startLimitMode: () => void;
+    stopLimitMode: () => void;
+    updateLimitType: (type: LimitType) => void;
+    updateCustomLimit: (limit: string) => void;
 }
 
 const ActionContext = createContext<ActionContextProps | undefined>(undefined);
@@ -18,18 +33,60 @@ export const ActionProvider = ({ children }: { children: ReactNode }) => {
     const [getText, setGetText] = useState<boolean>(false);
     const [getList, setGetList] = useState<boolean>(false);
     const [getScreenshot, setGetScreenshot] = useState<boolean>(false);
+    const [paginationMode, setPaginationMode] = useState<boolean>(false);
+    const [limitMode, setLimitMode] = useState<boolean>(false);
+    const [paginationType, setPaginationType] = useState<PaginationType>('');
+    const [limitType, setLimitType] = useState<LimitType>('');
+    const [customLimit, setCustomLimit] = useState<string>('');
+
+    const updatePaginationType = (type: PaginationType) => setPaginationType(type);
+    const updateLimitType = (type: LimitType) => setLimitType(type);
+    const updateCustomLimit = (limit: string) => setCustomLimit(limit);
+
+    const startPaginationMode = () => setPaginationMode(true);
+    const stopPaginationMode = () => setPaginationMode(false);
+
+    const startLimitMode = () => setLimitMode(true);
+    const stopLimitMode = () => setLimitMode(false);
 
     const startGetText = () => setGetText(true);
     const stopGetText = () => setGetText(false);
 
     const startGetList = () => setGetList(true);
-    const stopGetList = () => setGetList(false);
+    const stopGetList = () => {
+        setGetList(false);
+        setPaginationType('');
+        setLimitType('');
+        setCustomLimit('');
+    };
 
     const startGetScreenshot = () => setGetScreenshot(true);
     const stopGetScreenshot = () => setGetScreenshot(false);
 
     return (
-        <ActionContext.Provider value={{ getText, getList, getScreenshot, startGetText, stopGetText, startGetList, stopGetList, startGetScreenshot, stopGetScreenshot }}>
+        <ActionContext.Provider value={{ 
+            getText, 
+            getList, 
+            getScreenshot, 
+            paginationMode, 
+            limitMode,
+            paginationType, 
+            limitType,
+            customLimit,
+            startGetText, 
+            stopGetText, 
+            startGetList, 
+            stopGetList, 
+            startGetScreenshot, 
+            stopGetScreenshot, 
+            startPaginationMode, 
+            stopPaginationMode,
+            startLimitMode,
+            stopLimitMode,
+            updatePaginationType,
+            updateLimitType,
+            updateCustomLimit
+        }}>
             {children}
         </ActionContext.Provider>
     );
@@ -41,4 +98,4 @@ export const useActionContext = () => {
         throw new Error('useActionContext must be used within an ActionProvider');
     }
     return context;
-};
+};

src/context/browserSteps.tsx

@@ -19,6 +19,11 @@ export interface ListStep {
     type: 'list';
     listSelector: string;
     fields: { [key: string]: TextStep };
+    pagination?: {
+        type: string;
+        selector: string;
+    };
+    limit?: number;
 }
 
 type BrowserStep = TextStep | ScreenshotStep | ListStep;
@@ -33,7 +38,7 @@ export interface SelectorObject {
 interface BrowserStepsContextType {
     browserSteps: BrowserStep[];
     addTextStep: (label: string, data: string, selectorObj: SelectorObject) => void;
-    addListStep: (listSelector: string, fields: { [key: string]: TextStep }) => void
+    addListStep: (listSelector: string, fields: { [key: string]: TextStep }, listId: number, pagination?: { type: string; selector: string }, limit?: number) => void
     addScreenshotStep: (fullPage: boolean) => void;
     deleteBrowserStep: (id: number) => void;
     updateBrowserTextStepLabel: (id: number, newLabel: string) => void;
@@ -51,10 +56,10 @@ export const BrowserStepsProvider: React.FC<{ children: React.ReactNode }> = ({
         ]);
     };
 
-    const addListStep = (listSelector: string, newFields: { [key: string]: TextStep }) => {
+    const addListStep = (listSelector: string, newFields: { [key: string]: TextStep }, listId: number, pagination?: { type: string; selector: string }, limit?: number) => {
         setBrowserSteps(prevSteps => {
             const existingListStepIndex = prevSteps.findIndex(
-                step => step.type === 'list' && step.listSelector === listSelector
+                step => step.type === 'list' && step.id === listId
             );
             if (existingListStepIndex !== -1) {
                 // Update the existing ListStep with new fields
@@ -62,20 +67,21 @@ export const BrowserStepsProvider: React.FC<{ children: React.ReactNode }> = ({
                 const existingListStep = updatedSteps[existingListStepIndex] as ListStep;
                 updatedSteps[existingListStepIndex] = {
                     ...existingListStep,
-                    fields: { ...existingListStep.fields, ...newFields }
+                    fields: { ...existingListStep.fields, ...newFields },
+                    pagination: pagination,
+                    limit: limit,
                 };
                 return updatedSteps;
             } else {
                 // Create a new ListStep
                 return [
                     ...prevSteps,
-                    { id: Date.now(), type: 'list', listSelector, fields: newFields }
+                    { id: listId, type: 'list', listSelector, fields: newFields, pagination, limit }
                 ];
             }
         });
     };
 
-
     const addScreenshotStep = (fullPage: boolean) => {
         setBrowserSteps(prevSteps => [
             ...prevSteps,

src/pages/PageWrappper.tsx

@@ -7,11 +7,8 @@ import { MainPage } from "./MainPage";
 import { useGlobalInfoStore } from "../context/globalInfo";
 import { getActiveBrowserId } from "../api/recording";
 import { AlertSnackbar } from "../components/atoms/AlertSnackbar";
-import { InterpretationLog } from "../components/molecules/InterpretationLog";
-
 
 export const PageWrapper = () => {
-
   const [recordingName, setRecordingName] = useState('');
   const [open, setOpen] = useState(false);
 
@@ -54,7 +51,6 @@ export const PageWrapper = () => {
               <BrowserDimensionsProvider>
                 <React.Fragment>
                   <RecordingPage recordingName={recordingName} />
-                  <InterpretationLog />
                 </React.Fragment>
               </BrowserDimensionsProvider>
             )

src/pages/RecordingPage.tsx

@@ -32,6 +32,7 @@ export const RecordingPage = ({ recordingName }: RecordingPageProps) => {
     pair: null,
     index: 0,
   });
+  const [showOutputData, setShowOutputData] = useState(false);
 
   const browserContentRef = React.useRef<HTMLDivElement>(null);
   const workflowListRef = React.useRef<HTMLDivElement>(null);
@@ -40,6 +41,10 @@ export const RecordingPage = ({ recordingName }: RecordingPageProps) => {
   const { setWidth } = useBrowserDimensionsStore();
   const { browserId, setBrowserId } = useGlobalInfoStore();
 
+  const handleShowOutputData = useCallback(() => {
+    setShowOutputData(true);
+  }, []);
+
   const handleSelectPairForEdit = (pair: WhereWhatPair, index: number) => {
     setPairForEdit({
       pair,
@@ -47,7 +52,6 @@ export const RecordingPage = ({ recordingName }: RecordingPageProps) => {
     });
   };
 
-  //resize browser content when loaded event is fired
   useEffect(() => changeBrowserDimensions(), [isLoaded])
 
   useEffect(() => {
@@ -122,10 +126,10 @@ export const RecordingPage = ({ recordingName }: RecordingPageProps) => {
               </Grid>
               <Grid id="browser-content" ref={browserContentRef} item xs>
                 <BrowserContent />
-                <InterpretationLog />
+                <InterpretationLog isOpen={showOutputData} setIsOpen={setShowOutputData} />
               </Grid>
               <Grid item xs={2}>
-                <RightSidePanel />
+                <RightSidePanel onFinishCapture={handleShowOutputData} />
               </Grid>
             </Grid>
             : <Loader />}