From f4a0327c9a37f90b4d9775df17c07624ee81a15b Mon Sep 17 00:00:00 2001 From: amit Date: Thu, 7 Nov 2024 00:46:47 +0530 Subject: [PATCH 001/216] dark theme added --- src/App.tsx | 84 +--------------- src/components/molecules/NavBar.tsx | 139 +++++++++++++------------- src/components/organisms/MainMenu.tsx | 50 +++------ src/context/theme-provider.tsx | 64 ++++++++++++ 4 files changed, 154 insertions(+), 183 deletions(-) create mode 100644 src/context/theme-provider.tsx diff --git a/src/App.tsx b/src/App.tsx index c37de9ea..896723a1 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -1,96 +1,22 @@ import React from 'react'; import { Routes, Route } from 'react-router-dom'; -import { ThemeProvider, createTheme } from "@mui/material/styles"; +import { createTheme } from "@mui/material/styles"; import { GlobalInfoProvider } from "./context/globalInfo"; import { PageWrapper } from "./pages/PageWrappper"; +import ThemeModeProvider from './context/theme-provider'; + -const theme = createTheme({ - palette: { - primary: { - main: "#ff00c3", - contrastText: "#ffffff", - }, - }, - components: { - MuiButton: { - styleOverrides: { - root: { - // Default styles for all buttons (optional) - textTransform: "none", - }, - containedPrimary: { - // Styles for 'contained' variant with 'primary' color - '&:hover': { - backgroundColor: "#ff66d9", - }, - }, - outlined: { - // Apply white background for all 'outlined' variant buttons - backgroundColor: "#ffffff", - '&:hover': { - backgroundColor: "#f0f0f0", // Optional lighter background on hover - }, - }, - }, - }, - MuiLink: { - styleOverrides: { - root: { - '&:hover': { - color: "#ff00c3", - }, - }, - }, - }, - MuiIconButton: { - styleOverrides: { - root: { - // '&:hover': { - // color: "#ff66d9", - // }, - }, - }, - }, - MuiTab: { - styleOverrides: { - root: { - textTransform: "none", - }, - }, - }, - MuiAlert: { - styleOverrides: { - standardInfo: { - backgroundColor: "#fce1f4", - color: "#ff00c3", - '& .MuiAlert-icon': { - color: "#ff00c3", - }, - }, - }, - }, - MuiAlertTitle: { - styleOverrides: { - root: { - '& .MuiAlert-icon': { - color: "#ffffff", - }, - }, - }, - }, - }, -}); function App() { return ( - + } /> - + ); } diff --git a/src/components/molecules/NavBar.tsx b/src/components/molecules/NavBar.tsx index 69fba352..b82f2a5d 100644 --- a/src/components/molecules/NavBar.tsx +++ b/src/components/molecules/NavBar.tsx @@ -3,14 +3,15 @@ import axios from 'axios'; import styled from "styled-components"; import { stopRecording } from "../../api/recording"; import { useGlobalInfoStore } from "../../context/globalInfo"; -import { IconButton, Menu, MenuItem, Typography, Avatar } from "@mui/material"; -import { AccountCircle, Logout, Clear } from "@mui/icons-material"; +import { IconButton, Menu, MenuItem, Typography, Avatar, Tooltip } from "@mui/material"; +import { AccountCircle, Logout, Clear, Brightness4, Brightness7 } from "@mui/icons-material"; import { useNavigate } from 'react-router-dom'; import { AuthContext } from '../../context/auth'; import { SaveRecording } from '../molecules/SaveRecording'; import DiscordIcon from '../atoms/DiscordIcon'; import { apiUrl } from '../../apiConfig'; import MaxunLogo from "../../assets/maxunlogo.png"; +import { useThemeMode } from '../../context/theme-provider'; interface NavBarProps { recordingName: string; @@ -18,10 +19,11 @@ interface NavBarProps { } export const NavBar: React.FC = ({ recordingName, isRecording }) => { - const { notify, browserId, setBrowserId, recordingUrl } = useGlobalInfoStore(); + const { notify, browserId, setBrowserId } = useGlobalInfoStore(); const { state, dispatch } = useContext(AuthContext); const { user } = state; const navigate = useNavigate(); + const { darkMode, toggleTheme } = useThemeMode(); const [anchorEl, setAnchorEl] = useState(null); @@ -51,20 +53,18 @@ export const NavBar: React.FC = ({ recordingName, isRecording }) => }; return ( - -
+ +
-
Maxun
+
+ Maxun +
- { - user ? ( -
- {!isRecording ? ( - <> - + {!isRecording ? ( + <> + = ({ recordingName, isRecording }) => padding: '8px', marginRight: '10px', }} - > + > - + - - - {user.email} + + + {user.email} + + + { handleMenuClose(); logout(); }}> + Logout + + + {/* Theme Toggle Button */} + + + {darkMode ? : } - - { handleMenuClose(); logout(); }}> - Logout - - - - ) : ( - <> - - - Discard - - - - )} -
- ) : "" - } + + + ) : ( + <> + + + Discard + + + + )} +
+ ) : null} ); }; -const NavBarWrapper = styled.div` +const NavBarWrapper = styled.div<{ mode: 'light' | 'dark' }>` grid-area: navbar; - background-color: white; - padding:5px; + background-color: ${({ mode }) => (mode === 'dark' ? '#1e2124' : '#ffffff')}; + padding: 5px; display: flex; justify-content: space-between; - border-bottom: 1px solid #e0e0e0; + border-bottom: 1px solid ${({ mode }) => (mode === 'dark' ? '#333' : '#e0e0e0')}; `; -const ProjectName = styled.b` - color: #3f4853; +const ProjectName = styled.b<{ mode: 'light' | 'dark' }>` + color: ${({ mode }) => (mode === 'dark' ? 'white' : 'black')}; font-size: 1.3em; `; diff --git a/src/components/organisms/MainMenu.tsx b/src/components/organisms/MainMenu.tsx index edb6ed29..c6bae196 100644 --- a/src/components/organisms/MainMenu.tsx +++ b/src/components/organisms/MainMenu.tsx @@ -1,10 +1,9 @@ -import * as React from 'react'; +import React from 'react'; import Tabs from '@mui/material/Tabs'; import Tab from '@mui/material/Tab'; import Box from '@mui/material/Box'; -import { Paper, Button } from "@mui/material"; -import { AutoAwesome, FormatListBulleted, VpnKey, Usb, Article, Link, CloudQueue } from "@mui/icons-material"; -import { apiUrl } from "../../apiConfig"; +import { Paper, Button, useTheme } from "@mui/material"; +import { AutoAwesome, FormatListBulleted, VpnKey, Usb, Article, CloudQueue } from "@mui/icons-material"; interface MainMenuProps { value: string; @@ -12,6 +11,7 @@ interface MainMenuProps { } export const MainMenu = ({ value = 'recordings', handleChangeContent }: MainMenuProps) => { + const theme = useTheme(); const handleChange = (event: React.SyntheticEvent, newValue: string) => { handleChangeContent(newValue); @@ -22,16 +22,14 @@ export const MainMenu = ({ value = 'recordings', handleChangeContent }: MainMenu sx={{ height: 'auto', width: '250px', - backgroundColor: 'white', + backgroundColor: theme.palette.background.paper, paddingTop: '0.5rem', + color: theme.palette.text.primary, }} variant="outlined" square > - + } iconPosition="start" /> } iconPosition="start" /> } iconPosition="start" /> } @@ -87,7 +69,7 @@ export const MainMenu = ({ value = 'recordings', handleChangeContent }: MainMenu
- } {getList && ( <> @@ -426,7 +426,7 @@ export const RightSidePanel: React.FC = ({ onFinishCapture )} {showLimitOptions && ( - +

What is the maximum number of rows you want to extract?

 = ({ onFinishCapture marginLeft: '10px', '& input': { padding: '10px', - background: 'white', + }, - width: '150px', // Ensure the text field does not go outside the panel + width: '150px', + background: isDarkMode ? "#1E2124" : 'white', + color: isDarkMode ? "white" : 'black', // Ensure the text field does not go outside the panel }} /> )} @@ -503,6 +505,7 @@ export const RightSidePanel: React.FC = ({ onFinishCapture ) }} + sx={{ background: isDarkMode ? "#1E2124" : 'white', color: isDarkMode ? "white" : 'black' }} /> = ({ onFinishCapture ) }} + sx={{ background: isDarkMode ? "#1E2124" : 'white', color: isDarkMode ? "white" : 'black' }} /> {!confirmedTextSteps[step.id] && ( @@ -553,6 +557,8 @@ export const RightSidePanel: React.FC = ({ onFinishCapture ) }} + + style={{ background: isDarkMode ? "#1E2124" : 'white' }} /> = ({ onFinishCapture ) }} + style={{ background: isDarkMode ? "#1E2124" : 'white' }} /> {!confirmedListTextFields[step.id]?.[key] && ( From f71822f844e4b88a27c47b613da6589d734dc32d Mon Sep 17 00:00:00 2001 From: AmitChauhan63390 Date: Sun, 24 Nov 2024 00:49:39 +0530 Subject: [PATCH 009/216] some fixes --- src/components/atoms/Loader.tsx | 6 +- src/components/atoms/buttons/buttons.tsx | 19 +- src/components/atoms/canvas.tsx | 2 +- .../molecules/ActionDescriptionBox.tsx | 1 + src/components/molecules/BrowserNavBar.tsx | 64 ++-- .../molecules/BrowserRecordingSave.tsx | 7 +- src/components/molecules/BrowserTabs.tsx | 71 ++++- .../molecules/InterpretationLog.tsx | 11 +- src/components/molecules/NavBar.tsx | 288 ++++++++++++------ src/components/molecules/RunContent.tsx | 49 ++- src/components/molecules/SaveRecording.tsx | 2 +- src/components/organisms/BrowserContent.tsx | 12 +- src/components/organisms/BrowserWindow.tsx | 2 +- src/components/organisms/RightSidePanel.tsx | 27 +- src/pages/RecordingPage.tsx | 2 + 15 files changed, 399 insertions(+), 164 deletions(-) diff --git a/src/components/atoms/Loader.tsx b/src/components/atoms/Loader.tsx index 35f9a506..529068a7 100644 --- a/src/components/atoms/Loader.tsx +++ b/src/components/atoms/Loader.tsx @@ -2,16 +2,13 @@ import styled from "styled-components"; import { Stack } from "@mui/material"; import { useThemeMode } from "../../context/theme-provider"; - interface LoaderProps { text: string; } export const Loader: React.FC = ({ text }) => { - const { darkMode } = useThemeMode(); - return ( @@ -29,14 +26,13 @@ interface StyledParagraphProps { darkMode: boolean; } - - const StyledParagraph = styled.p` font-size: medium; font-weight: 700; font-family: inherit; color: ${({ darkMode }) => (darkMode ? 'white' : 'black')}; margin-top: 20px; + flex-wrap: wrap; `; const DotsContainer = styled.div` diff --git a/src/components/atoms/buttons/buttons.tsx b/src/components/atoms/buttons/buttons.tsx index afc4a483..0dd72e0c 100644 --- a/src/components/atoms/buttons/buttons.tsx +++ b/src/components/atoms/buttons/buttons.tsx @@ -1,26 +1,23 @@ import styled from 'styled-components'; +import { useThemeMode } from '../../../context/theme-provider'; -export const NavBarButton = styled.button<{ disabled: boolean }>` + + +export const NavBarButton = styled.button<{ disabled: boolean, mode: 'light' | 'dark' }>` margin-left: 10px; margin-right: 5px; padding: 0; border: none; - background-color: transparent; + background-color: ${mode => mode ? '#333' : '#ffffff'}; cursor: ${({ disabled }) => disabled ? 'default' : 'pointer'}; width: 24px; height: 24px; border-radius: 12px; outline: none; - color: ${({ disabled }) => disabled ? '#999' : '#333'}; + color: ${mode => mode ? '#ffffff' : '#333333'}; + + - ${({ disabled }) => disabled ? null : ` - &:hover { - background-color: #ddd; - } - &:active { - background-color: #d0d0d0; - } - `}; `; export const UrlFormButton = styled.button` diff --git a/src/components/atoms/canvas.tsx b/src/components/atoms/canvas.tsx index 1dd88e19..e31a7094 100644 --- a/src/components/atoms/canvas.tsx +++ b/src/components/atoms/canvas.tsx @@ -142,7 +142,7 @@ const Canvas = ({ width, height, onCreateRef }: CanvasProps) => { diff --git a/src/components/molecules/ActionDescriptionBox.tsx b/src/components/molecules/ActionDescriptionBox.tsx index e064c01b..747cad18 100644 --- a/src/components/molecules/ActionDescriptionBox.tsx +++ b/src/components/molecules/ActionDescriptionBox.tsx @@ -18,6 +18,7 @@ const CustomBoxContainer = styled.div` background-color: ${({ isDarkMode }) => (isDarkMode ? '#313438' : 'white')}; color: ${({ isDarkMode }) => (isDarkMode ? 'white' : 'black')}; margin: 80px 13px 25px 13px; + box-shadow: 0px 4px 10px rgba(0, 0, 0, 0.1); `; const Triangle = styled.div` diff --git a/src/components/molecules/BrowserNavBar.tsx b/src/components/molecules/BrowserNavBar.tsx index 8fe1ba05..d3cb781f 100644 --- a/src/components/molecules/BrowserNavBar.tsx +++ b/src/components/molecules/BrowserNavBar.tsx @@ -1,6 +1,4 @@ -import type { - FC, -} from 'react'; +import type { FC } from 'react'; import styled from 'styled-components'; import ReplayIcon from '@mui/icons-material/Replay'; @@ -13,13 +11,39 @@ import { useCallback, useEffect, useState } from "react"; import { useSocketStore } from "../../context/socket"; import { getCurrentUrl } from "../../api/recording"; import { useGlobalInfoStore } from '../../context/globalInfo'; +import { useThemeMode } from '../../context/theme-provider'; -const StyledNavBar = styled.div<{ browserWidth: number }>` - display: flex; - padding: 12px 0px; - background-color: theme.palette.background.paper; - width: ${({ browserWidth }) => browserWidth}px; - border-radius: 0px 5px 0px 0px; +const StyledNavBar = styled.div<{ browserWidth: number; isDarkMode: boolean }>` + display: flex; + align-items: center; + padding: 10px 20px; + background-color: ${({ isDarkMode }) => (isDarkMode ? '#2C2F33' : '#F5F5F5')}; + width: ${({ browserWidth }) => `${browserWidth}px`}; + border-radius: 0px 0px 8px 8px; + box-shadow: ${({ isDarkMode }) => (isDarkMode ? '0px 2px 10px rgba(0, 0, 0, 0.2)' : '0px 2px 10px rgba(0, 0, 0, 0.1)')}; + transition: background-color 0.3s ease, box-shadow 0.3s ease; + margin-bottom: 15px; +`; + +const IconButton = styled(NavBarButton)<{ mode: string }>` + display: flex; + align-items: center; + justify-content: center; + padding: 8px; + margin-right: 12px; + background-color: ${({ mode }) => (mode === 'dark' ? '#40444B' : '#E0E0E0')}; + border-radius: 50%; + transition: background-color 0.3s ease, transform 0.1s ease; + color: ${({ mode }) => (mode === 'dark' ? '#FFFFFF' : '#333')}; + cursor: pointer; + + &:hover { + background-color: ${({ mode }) => (mode === 'dark' ? '#586069' : '#D0D0D0')}; + } + + &:active { + transform: scale(0.95); + } `; interface NavBarProps { @@ -31,6 +55,7 @@ const BrowserNavBar: FC = ({ browserWidth, handleUrlChanged, }) => { + const isDarkMode = useThemeMode().darkMode; const { socket } = useSocketStore(); const { recordingUrl, setRecordingUrl } = useGlobalInfoStore(); @@ -67,7 +92,7 @@ const BrowserNavBar: FC = ({ socket.off('urlChanged', handleCurrentUrlChange); } } - }, [socket, handleCurrentUrlChange]) + }, [socket, handleCurrentUrlChange]); const addAddress = (address: string) => { if (socket) { @@ -78,38 +103,41 @@ const BrowserNavBar: FC = ({ }; return ( - - + { socket?.emit('input:back'); }} disabled={false} + mode={isDarkMode ? 'dark' : 'light'} > - + - { socket?.emit('input:forward'); }} disabled={false} + mode={isDarkMode ? 'dark' : 'light'} > - + - { if (socket) { - handleRefresh() + handleRefresh(); } }} disabled={false} + mode={isDarkMode ? 'dark' : 'light'} > - + {
- ) : null} + + )} ); }; +// Styles +const styles = { + socialButton: { + display: 'flex', + alignItems: 'center', + borderRadius: '5px', + padding: '8px', + marginRight: '30px', + color: '#333333', + '&:hover': { + color: '#ff00c3' + } + }, + userButton: (darkMode: boolean) => ({ + display: 'flex', + alignItems: 'center', + borderRadius: '5px', + padding: '8px', + marginRight: '10px', + color: darkMode ? '#ffffff' : '#333333', + '&:hover': { + backgroundColor: darkMode ? '#333' : '#F5F5F5', + color: '#ff00c3' + } + }), + discardButton: { + borderRadius: '5px', + padding: '8px', + background: 'red', + color: 'white', + marginRight: '10px', + '&:hover': { + color: 'white', + backgroundColor: '#ff0000' + } + } +}; + +// Styled Components const NavBarWrapper = styled.div<{ mode: 'light' | 'dark' }>` grid-area: navbar; background-color: ${({ mode }) => (mode === 'dark' ? '#1e2124' : '#ffffff')}; @@ -159,7 +235,27 @@ const NavBarWrapper = styled.div<{ mode: 'light' | 'dark' }>` border-bottom: 1px solid ${({ mode }) => (mode === 'dark' ? '#333' : '#e0e0e0')}; `; -const ProjectName = styled.b<{ mode: 'light' | 'dark' }>` - color: ${({ mode }) => (mode === 'dark' ? 'white' : 'black')}; - font-size: 1.3em; +const BrandContainer = styled.div` + display: flex; + justify-content: flex-start; `; + +const LogoImage = styled.img.attrs({ + width: 45, + height: 40, +})` + border-radius: 5px; + margin: 5px 0px 5px 15px; +`; + +const ProjectName = styled.b<{ mode: 'light' | 'dark' }>` + color: ${({ mode }) => (mode === 'dark' ? 'white' : '#333333')}; + font-size: 1.3em; + padding: 11px; +`; + +const ControlsContainer = styled.div` + display: flex; + align-items: center; + justify-content: flex-end; +`; \ No newline at end of file diff --git a/src/components/molecules/RunContent.tsx b/src/components/molecules/RunContent.tsx index ff414628..b9c0f5fe 100644 --- a/src/components/molecules/RunContent.tsx +++ b/src/components/molecules/RunContent.tsx @@ -75,10 +75,50 @@ export const RunContent = ({ row, currentLog, interpretationInProgress, logEndRe - setTab(newTab)} aria-label="run-content-tabs"> - - - + setTab(newTab)} + aria-label="run-content-tabs" + sx={{ + // Remove the default blue indicator + '& .MuiTabs-indicator': { + backgroundColor: '#FF00C3', // Change to pink + }, + // Remove default transition effects + '& .MuiTab-root': { + '&.Mui-selected': { + color: '#FF00C3', + }, + } + }} +> + theme.palette.mode === 'dark' ? '#fff' : '#000', + '&:hover': { + color: '#FF00C3' + }, + '&.Mui-selected': { + color: '#FF00C3', + } + }} + /> + theme.palette.mode === 'dark' ? '#fff' : '#000', + '&:hover': { + color: '#FF00C3' + }, + '&.Mui-selected': { + color: '#FF00C3', + } + }} + /> + 
                     {JSON.stringify(row.serializableOutput, null, 2)}
diff --git a/src/components/molecules/SaveRecording.tsx b/src/components/molecules/SaveRecording.tsx
index 60ef3fa6..da7d3198 100644
--- a/src/components/molecules/SaveRecording.tsx
+++ b/src/components/molecules/SaveRecording.tsx
@@ -76,7 +76,7 @@ export const SaveRecording = ({ fileName }: SaveRecordingProps) => {
 
   return (
     

-      
 
diff --git a/src/components/organisms/BrowserContent.tsx b/src/components/organisms/BrowserContent.tsx
index 11af4f2f..07410bd6 100644
--- a/src/components/organisms/BrowserContent.tsx
+++ b/src/components/organisms/BrowserContent.tsx
@@ -12,6 +12,7 @@ import {
 } from "../../api/recording";
 import { Box } from "@mui/material";
 import { InterpretationLog } from "../molecules/InterpretationLog";
+import { Height } from "@mui/icons-material";
 
 // TODO: Tab !show currentUrl after recordingUrl global state
 export const BrowserContent = () => {
@@ -139,7 +140,7 @@ export const BrowserContent = () => {
   }, [handleUrlChanged]);
 
   return (
-    

+    

        {
         // todo: use width from browser dimension once fixed
         browserWidth={900}
         handleUrlChanged={handleUrlChanged}
+       
       />
-      
+      
     

   );
 };
 
-const BrowserContentWrapper = styled.div``;
+const BrowserContentWrapper = styled.div`
+  position: relative;
+  width: 100vw;
+  height: 100vh;
+  overflow: hidden;`;
diff --git a/src/components/organisms/BrowserWindow.tsx b/src/components/organisms/BrowserWindow.tsx
index 697b4adb..30e67fa4 100644
--- a/src/components/organisms/BrowserWindow.tsx
+++ b/src/components/organisms/BrowserWindow.tsx
@@ -319,7 +319,7 @@ export const BrowserWindow = () => {
 
   
     return (
-        

+        

             {
                 getText === true || getList === true ? (
                      = ({ onFinishCapture
   const theme = useThemeMode();
   const isDarkMode = theme.darkMode;
   return (
-    
+    
       {/* 
         Last action: {` ${lastAction}`}
        */}
@@ -484,7 +499,7 @@ export const RightSidePanel: React.FC = ({ onFinishCapture
       
       
         {browserSteps.map(step => (
-           handleMouseEnter(step.id)} onMouseLeave={() => handleMouseLeave(step.id)} sx={{ padding: '10px', margin: '11px', borderRadius: '5px', position: 'relative', background: 'white' }}>
+           handleMouseEnter(step.id)} onMouseLeave={() => handleMouseLeave(step.id)} sx={{ padding: '10px', margin: '11px', borderRadius: '5px', position: 'relative', background: isDarkMode ? "#1E2124" : 'white', color: isDarkMode ? "white" : 'black' }}>
             {
               step.type === 'text' && (
                 <>
@@ -520,7 +535,7 @@ export const RightSidePanel: React.FC = ({ onFinishCapture
                         
                       )
                     }}
-                    sx={{ background: isDarkMode ? "#1E2124" : 'white', color: isDarkMode ? "white" : 'black' }}
+                    
                   />
                   {!confirmedTextSteps[step.id] && (
                     
@@ -542,7 +557,7 @@ export const RightSidePanel: React.FC = ({ onFinishCapture
               <>
                 List Selected Successfully
                 {Object.entries(step.fields).map(([key, field]) => (
-                  
+                  
                      = ({ onFinishCapture
                         )
                       }}
 
-                      style={{ background: isDarkMode ? "#1E2124" : 'white' }}
+                      
                     />
                      = ({ onFinishCapture
                           
                         )
                       }}
-                      style={{ background: isDarkMode ? "#1E2124" : 'white' }}
+                      
                     />
                     {!confirmedListTextFields[step.id]?.[key] && (
                       
diff --git a/src/pages/RecordingPage.tsx b/src/pages/RecordingPage.tsx
index e82d1f74..8f9eeda0 100644
--- a/src/pages/RecordingPage.tsx
+++ b/src/pages/RecordingPage.tsx
@@ -59,7 +59,9 @@ export const RecordingPage = ({ recordingName }: RecordingPageProps) => {
 
   useEffect(() => {
     if (darkMode) {
+
       document.body.style.background = 'rgba(18,18,18,1)';
+
     } else {
       document.body.style.background = 'radial-gradient(circle, rgba(255, 255, 255, 1) 0%, rgba(232, 191, 222, 1) 100%, rgba(255, 255, 255, 1) 100%)';
     }

From e9e9070fe23b2a7887c924f516a15842c59702a3 Mon Sep 17 00:00:00 2001
From: AmitChauhan63390 
Date: Sun, 24 Nov 2024 01:10:22 +0530
Subject: [PATCH 010/216] tab indicator default color changed to ff00c3

---
 src/components/organisms/MainMenu.tsx | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/src/components/organisms/MainMenu.tsx b/src/components/organisms/MainMenu.tsx
index b8f2aaa8..27c42fe1 100644
--- a/src/components/organisms/MainMenu.tsx
+++ b/src/components/organisms/MainMenu.tsx
@@ -43,6 +43,13 @@ export const MainMenu = ({ value = 'recordings', handleChangeContent }: MainMenu
           value={value}
           onChange={handleChange}
           orientation="vertical"
+          TabIndicatorProps={{
+            style: {
+              backgroundColor: '#ff00c3', // Set the custom color for the indicator here
+              width: '2px',               // Ensure the indicator width is 2px as per your requirement
+              right: 0,                   // Position it on the right if needed
+            },
+          }}
           sx={{
             alignItems: 'flex-start',
             '& .MuiTab-root': {
@@ -54,6 +61,10 @@ export const MainMenu = ({ value = 'recordings', handleChangeContent }: MainMenu
               '&.Mui-selected': {
                 color: selectedPink, // Darker pink for selected tab
               },
+              '& .MuiTabs-indicator': {
+                backgroundColor: '#ff00c3', // Custom color for the indicator
+              },
+
             },
           }}
         >
@@ -62,24 +73,28 @@ export const MainMenu = ({ value = 'recordings', handleChangeContent }: MainMenu
             label="Robots"
             icon={}
             iconPosition="start"
+            
           />
           }
             iconPosition="start"
+           
           />
           }
             iconPosition="start"
+            
           />
           }
             iconPosition="start"
+            
           />
         
         


From 3cf0786389d00bbcaaea71f67d1ce7645c0babe4 Mon Sep 17 00:00:00 2001
From: AmitChauhan63390 
Date: Sun, 24 Nov 2024 19:36:07 +0530
Subject: [PATCH 011/216] pinkish buttons

---
 src/components/organisms/BrowserContent.tsx | 2 +-
 src/components/organisms/RightSidePanel.tsx | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/components/organisms/BrowserContent.tsx b/src/components/organisms/BrowserContent.tsx
index 07410bd6..fcb45087 100644
--- a/src/components/organisms/BrowserContent.tsx
+++ b/src/components/organisms/BrowserContent.tsx
@@ -140,7 +140,7 @@ export const BrowserContent = () => {
   }, [handleUrlChanged]);
 
   return (
-    

+    

        = ({ onFinishCapture
        */}
       
       
-        {!getText && !getScreenshot && !getList && showCaptureList && }
+        {!getText && !getScreenshot && !getList && showCaptureList && }
         {getList && (
           <>
             
@@ -479,7 +479,7 @@ export const RightSidePanel: React.FC = ({ onFinishCapture
             
           
         )}
-        {!getText && !getScreenshot && !getList && showCaptureText && }
+        {!getText && !getScreenshot && !getList && showCaptureText && }
         {getText &&
           <>
             
@@ -488,7 +488,7 @@ export const RightSidePanel: React.FC = ({ onFinishCapture
             
           
         }
-        {!getText && !getScreenshot && !getList && showCaptureScreenshot && }
+        {!getText && !getScreenshot && !getList && showCaptureScreenshot && }
         {getScreenshot && (
           
             

From 587dacdaf14cbba65c84321205d0b4c952045780 Mon Sep 17 00:00:00 2001
From: AmitChauhan63390 
Date: Sun, 8 Dec 2024 04:49:29 +0530
Subject: [PATCH 012/216] mui blue color fixes

---
 .../molecules/IntegrationSettings.tsx         | 30 +++++--
 src/components/organisms/MainMenu.tsx         |  5 ++
 src/components/organisms/ProxyForm.tsx        | 80 ++++++++++++++++---
 src/context/theme-provider.tsx                | 44 +++++++++-
 src/pages/Login.tsx                           |  2 +-
 src/pages/Register.tsx                        |  2 +
 6 files changed, 143 insertions(+), 20 deletions(-)

diff --git a/src/components/molecules/IntegrationSettings.tsx b/src/components/molecules/IntegrationSettings.tsx
index c31605de..8e38b5a0 100644
--- a/src/components/molecules/IntegrationSettings.tsx
+++ b/src/components/molecules/IntegrationSettings.tsx
@@ -14,18 +14,33 @@ import axios from "axios";
 import { useGlobalInfoStore } from "../../context/globalInfo";
 import { getStoredRecording } from "../../api/storage";
 import { apiUrl } from "../../apiConfig.js";
-import Cookies from 'js-cookie';
+
 interface IntegrationProps {
   isOpen: boolean;
   handleStart: (data: IntegrationSettings) => void;
   handleClose: () => void;
 }
+
 export interface IntegrationSettings {
   spreadsheetId: string;
   spreadsheetName: string;
   data: string;
 }
 
+// Helper functions to replace js-cookie functionality
+const getCookie = (name: string): string | null => {
+  const value = `; ${document.cookie}`;
+  const parts = value.split(`; ${name}=`);
+  if (parts.length === 2) {
+    return parts.pop()?.split(';').shift() || null;
+  }
+  return null;
+};
+
+const removeCookie = (name: string): void => {
+  document.cookie = `${name}=; expires=Thu, 01 Jan 1970 00:00:00 GMT; path=/`;
+};
+
 export const IntegrationSettingsModal = ({
   isOpen,
   handleStart,
@@ -106,7 +121,7 @@ export const IntegrationSettingsModal = ({
         },
         { withCredentials: true }
       );
-      notify(`success`, `Google Sheet selected successfully`)
+      notify(`success`, `Google Sheet selected successfully`);
       console.log("Google Sheet ID updated:", response.data);
     } catch (error: any) {
       console.error(
@@ -137,14 +152,14 @@ export const IntegrationSettingsModal = ({
 
   useEffect(() => {
     // Check if there is a success message in cookies
-    const status = Cookies.get("robot_auth_status");
-    const message = Cookies.get("robot_auth_message");
+    const status = getCookie("robot_auth_status");
+    const message = getCookie("robot_auth_message");
 
     if (status === "success" && message) {
       notify("success", message);
       // Clear the cookies after reading
-      Cookies.remove("robot_auth_status");
-      Cookies.remove("robot_auth_message");
+      removeCookie("robot_auth_status");
+      removeCookie("robot_auth_message");
     }
 
     // Check if we're on the callback URL
@@ -177,7 +192,6 @@ export const IntegrationSettingsModal = ({
       >
         
           Integrate with Google Sheet{" "}
-          {/*  */}
         
 
         {recording && recording.google_sheet_id ? (
@@ -320,4 +334,4 @@ export const modalStyle = {
   height: "fit-content",
   display: "block",
   padding: "20px",
-};
+};
\ No newline at end of file
diff --git a/src/components/organisms/MainMenu.tsx b/src/components/organisms/MainMenu.tsx
index 27c42fe1..60c9a082 100644
--- a/src/components/organisms/MainMenu.tsx
+++ b/src/components/organisms/MainMenu.tsx
@@ -124,4 +124,9 @@ const buttonStyles = {
   alignItems: 'center',
   textTransform: 'none',
   color: 'inherit',
+  backgroundColor: 'transparent',
+  '&:hover': {
+    backgroundColor: 'rgba(255, 0, 195, 0.1)',
+    color: '#ff00c3',
+  },
 };
diff --git a/src/components/organisms/ProxyForm.tsx b/src/components/organisms/ProxyForm.tsx
index a581144b..acd71e4e 100644
--- a/src/components/organisms/ProxyForm.tsx
+++ b/src/components/organisms/ProxyForm.tsx
@@ -1,8 +1,48 @@
 import React, { useState, useEffect } from 'react';
 import { styled } from '@mui/system';
-import { Alert, AlertTitle, TextField, Button, Switch, FormControlLabel, Box, Typography, Tabs, Tab, Table, TableContainer, TableHead, TableRow, TableBody, TableCell, Paper } from '@mui/material';
+import { 
+    Alert, 
+    AlertTitle, 
+    TextField, 
+    Button, 
+    Switch, 
+    FormControlLabel, 
+    Box, 
+    Typography, 
+    Tabs, 
+    Tab, 
+    Table, 
+    TableContainer, 
+    TableHead, 
+    TableRow, 
+    TableBody, 
+    TableCell, 
+    Paper 
+} from '@mui/material';
 import { sendProxyConfig, getProxyConfig, testProxyConfig, deleteProxyConfig } from '../../api/proxy';
 import { useGlobalInfoStore } from '../../context/globalInfo';
+import { useThemeMode } from '../../context/theme-provider';
+
+// Custom styled Tabs component
+const CustomTabs = styled(Tabs)(({ theme }) => ({
+    '& .MuiTabs-indicator': {
+        backgroundColor: '#ff00c3', // Pink indicator
+    },
+}));
+
+// Custom styled Tab component
+const CustomTab = styled(Tab)(({ theme }) => ({
+    '&.Mui-selected': {
+        color: '#ff00c3', // Pink for selected tab
+    },
+    '&:hover': {
+        color: '#ff00c3', // Pink on hover
+         // Subtle hover effect
+    },
+    '&.MuiTab-root': {
+        textTransform: 'none', // Removes uppercase transformation
+    },
+}));
 
 const FormContainer = styled(Box)({
     display: 'flex',
@@ -132,16 +172,37 @@ const ProxyForm: React.FC = () => {
         fetchProxyConfig();
     }, []);
 
+    const theme = useThemeMode();
+    const isDarkMode = theme.darkMode;
+
     return (
         <>
             
                 
                     Proxy Configuration
                 
-                
-                    
-                    
-                
+                
+                    
+                    
+                
                 {tabIndex === 0 && (
                     isProxyConfigured ? (
                         
@@ -236,15 +297,15 @@ const ProxyForm: React.FC = () => {
                             
                                 Coming Soon - In Open Source (Basic Rotation) & Cloud (Advanced Rotation). If you don't want to manage the infrastructure, join our cloud waitlist to get early access.
                             
-                            
                         
                     
                 )}
             
-            
-            If your proxy requires a username and password, always provide them separately from the proxy URL. 
+            
+                If your proxy requires a username and password, always provide them separately from the proxy URL. 
                 

                 The right way
                 

@@ -258,9 +319,10 @@ const ProxyForm: React.FC = () => {
                 The wrong way
                 

                 Proxy URL: http://myusername:mypassword@proxy.com:1337
+                

             
         
     );
 };
 
-export default ProxyForm;
+export default ProxyForm;
\ No newline at end of file
diff --git a/src/context/theme-provider.tsx b/src/context/theme-provider.tsx
index 02052cf5..a6097f95 100644
--- a/src/context/theme-provider.tsx
+++ b/src/context/theme-provider.tsx
@@ -6,7 +6,7 @@ const lightTheme = createTheme({
   palette: {
     mode: 'light',
     primary: {
-      main: '#1e88e5',
+      main: '#ff00c3', // Pink as the primary color
     },
     background: {
       default: '#ffffff',
@@ -16,13 +16,33 @@ const lightTheme = createTheme({
       primary: '#000000',
     },
   },
+  components: {
+    MuiTabs: {
+      styleOverrides: {
+        indicator: {
+          backgroundColor: '#ff00c3', // Pink for tab indicators
+        },
+      },
+    },
+    MuiButton: {
+      styleOverrides: {
+        root: {
+          backgroundColor: '#ff00c3', // Pink button background
+          color: '#ffffff',
+          '&:hover': {
+            backgroundColor: '#e600b3', // Slightly darker pink on hover
+          },
+        },
+      },
+    },
+  },
 });
 
 const darkTheme = createTheme({
   palette: {
     mode: 'dark',
     primary: {
-      main: '#90caf9',
+      main: '#ff00c3', // Pink as the primary color
     },
     background: {
       default: '#121212',
@@ -32,6 +52,26 @@ const darkTheme = createTheme({
       primary: '#ffffff',
     },
   },
+  components: {
+    MuiTabs: {
+      styleOverrides: {
+        indicator: {
+          backgroundColor: '#ff00c3', // Pink for tab indicators
+        },
+      },
+    },
+    MuiButton: {
+      styleOverrides: {
+        root: {
+          backgroundColor: '#ff00c3', // Pink button background
+          color: '#ffffff',
+          '&:hover': {
+            backgroundColor: '#e600b3', // Slightly darker pink on hover
+          },
+        },
+      },
+    },
+  },
 });
 
 const ThemeModeContext = createContext({
diff --git a/src/pages/Login.tsx b/src/pages/Login.tsx
index 87f90b53..de5f4838 100644
--- a/src/pages/Login.tsx
+++ b/src/pages/Login.tsx
@@ -38,7 +38,7 @@ const Login = () => {
       const { data } = await axios.post(`${apiUrl}/auth/login`, {
         email,
         password,
-      });
+      }, { withCredentials: true });
       dispatch({ type: "LOGIN", payload: data });
       notify("success", "Welcome to Maxun!");
       window.localStorage.setItem("user", JSON.stringify(data));
diff --git a/src/pages/Register.tsx b/src/pages/Register.tsx
index b2a3eebf..ad7fb89f 100644
--- a/src/pages/Register.tsx
+++ b/src/pages/Register.tsx
@@ -39,6 +39,8 @@ const Register = () => {
         email,
         password,
       });
+
+      console.log(data)
       dispatch({ type: "LOGIN", payload: data });
       notify("success", "Registration Successful!");
       window.localStorage.setItem("user", JSON.stringify(data));

From 655eadc068820b34504226ba51d32b012f5760a0 Mon Sep 17 00:00:00 2001
From: AmitChauhan63390 
Date: Sun, 8 Dec 2024 05:41:11 +0530
Subject: [PATCH 013/216] forms

---
 src/components/molecules/NavBar.tsx |   2 +-
 src/pages/Login.tsx                 | 186 +++++++++++++++++-----------
 src/pages/PageWrappper.tsx          |   3 +
 src/pages/Register.tsx              |  87 +++++++++----
 4 files changed, 175 insertions(+), 103 deletions(-)

diff --git a/src/components/molecules/NavBar.tsx b/src/components/molecules/NavBar.tsx
index d8831ea6..d7cb726c 100644
--- a/src/components/molecules/NavBar.tsx
+++ b/src/components/molecules/NavBar.tsx
@@ -258,4 +258,4 @@ const ControlsContainer = styled.div`
   display: flex;
   align-items: center;
   justify-content: flex-end;
-`;
\ No newline at end of file
+`;
diff --git a/src/pages/Login.tsx b/src/pages/Login.tsx
index de5f4838..12f49e57 100644
--- a/src/pages/Login.tsx
+++ b/src/pages/Login.tsx
@@ -1,22 +1,21 @@
 import axios from "axios";
-import { useState, useContext, useEffect, FormEvent } from "react";
+import { useState, useContext, useEffect } from "react";
 import { useNavigate, Link } from "react-router-dom";
 import { AuthContext } from "../context/auth";
-import { Box, Typography, TextField, Button, CircularProgress, Grid } from "@mui/material";
+import { Box, Typography, TextField, Button, CircularProgress } from "@mui/material";
 import { useGlobalInfoStore } from "../context/globalInfo";
 import { apiUrl } from "../apiConfig";
+import { useThemeMode } from "../context/theme-provider";
 
 const Login = () => {
-  const [form, setForm] = useState({
-    email: "",
-    password: "",
-  });
+  const [form, setForm] = useState({ email: "", password: "" });
   const [loading, setLoading] = useState(false);
   const { notify } = useGlobalInfoStore();
   const { email, password } = form;
 
   const { state, dispatch } = useContext(AuthContext);
   const { user } = state;
+  const { darkMode } = useThemeMode();
 
   const navigate = useNavigate();
 
@@ -35,10 +34,11 @@ const Login = () => {
     e.preventDefault();
     setLoading(true);
     try {
-      const { data } = await axios.post(`${apiUrl}/auth/login`, {
-        email,
-        password,
-      }, { withCredentials: true });
+      const { data } = await axios.post(
+        `${apiUrl}/auth/login`,
+        { email, password },
+        { withCredentials: true }
+      );
       dispatch({ type: "LOGIN", payload: data });
       notify("success", "Welcome to Maxun!");
       window.localStorage.setItem("user", JSON.stringify(data));
@@ -58,76 +58,112 @@ const Login = () => {
         maxHeight: "100vh",
         mt: 6,
         padding: 4,
+        backgroundColor: darkMode ? "#121212" : "#ffffff",
+        
       }}
     >
-    
-        
+        logo
+        
+          Welcome Back!
+        
+        
+        
+        
-          
-            Don’t have an account?{" "}
-            
-              Register
-            
-          
-        
-        
-     
+          {loading ? (
+            <>
+              
+              Loading
+            
+          ) : (
+            "Login"
+          )}
+        
+        
+          Don’t have an account?{" "}
+          
+            Register
+          
+        
+      
+    
   );
 };
 
diff --git a/src/pages/PageWrappper.tsx b/src/pages/PageWrappper.tsx
index 87157349..5a00cc4a 100644
--- a/src/pages/PageWrappper.tsx
+++ b/src/pages/PageWrappper.tsx
@@ -12,6 +12,7 @@ import Login from './Login';
 import Register from './Register';
 import UserRoute from '../routes/userRoute';
 import { Routes, Route, useNavigate } from 'react-router-dom';
+import { AppBar } from '@mui/material';
 
 export const PageWrapper = () => {
   const [open, setOpen] = useState(false);
@@ -50,7 +51,9 @@ export const PageWrapper = () => {
       
         
           
+         
             {!browserId && }
+           
             
               }>
                 } />
diff --git a/src/pages/Register.tsx b/src/pages/Register.tsx
index ad7fb89f..474f0182 100644
--- a/src/pages/Register.tsx
+++ b/src/pages/Register.tsx
@@ -5,18 +5,17 @@ import { AuthContext } from "../context/auth";
 import { Box, Typography, TextField, Button, CircularProgress } from "@mui/material";
 import { useGlobalInfoStore } from "../context/globalInfo";
 import { apiUrl } from "../apiConfig";
+import { useThemeMode } from "../context/theme-provider";
 
 const Register = () => {
-  const [form, setForm] = useState({
-    email: "",
-    password: "",
-  });
+  const [form, setForm] = useState({ email: "", password: "" });
   const [loading, setLoading] = useState(false);
   const { notify } = useGlobalInfoStore();
   const { email, password } = form;
 
   const { state, dispatch } = useContext(AuthContext);
   const { user } = state;
+  const { darkMode } = useThemeMode();
 
   const navigate = useNavigate();
 
@@ -35,18 +34,14 @@ const Register = () => {
     e.preventDefault();
     setLoading(true);
     try {
-      const { data } = await axios.post(`${apiUrl}/auth/register`, {
-        email,
-        password,
-      });
-
-      console.log(data)
+      const { data } = await axios.post(`${apiUrl}/auth/register`, { email, password });
+      console.log(data);
       dispatch({ type: "LOGIN", payload: data });
       notify("success", "Registration Successful!");
       window.localStorage.setItem("user", JSON.stringify(data));
       navigate("/");
-    } catch (error:any) {
-      notify("error", error.response.data || "Registration Failed. Please try again.");
+    } catch (error: any) {
+      notify("error", error.response?.data || "Registration Failed. Please try again.");
       setLoading(false);
     }
   };
@@ -60,25 +55,38 @@ const Register = () => {
         maxHeight: "100vh",
         mt: 6,
         padding: 4,
+        backgroundColor: darkMode ? "#121212" : "#ffffff",
+       
       }}
     >
       
-        logo
+        logo
         
           Create an Account
         
@@ -91,6 +99,15 @@ const Register = () => {
           margin="normal"
           variant="outlined"
           required
+          sx={{
+            input: { color: darkMode ? "#ffffff" : "#000000" },
+            label: { color: darkMode ? "#bbbbbb" : "#000000" },
+            "& .MuiOutlinedInput-root": {
+              "& fieldset": { borderColor: darkMode ? "#555555" : "#cccccc" },
+              "&:hover fieldset": { borderColor: darkMode ? "#ffffff" : "#000000" },
+              "&.Mui-focused fieldset": { borderColor: "#ff33cc" },
+            },
+          }}
         />
          {
           margin="normal"
           variant="outlined"
           required
+          sx={{
+            input: { color: darkMode ? "#ffffff" : "#000000" },
+            label: { color: darkMode ? "#bbbbbb" : "#000000" },
+            "& .MuiOutlinedInput-root": {
+              "& fieldset": { borderColor: darkMode ? "#555555" : "#cccccc" },
+              "&:hover fieldset": { borderColor: darkMode ? "#ffffff" : "#000000" },
+              "&.Mui-focused fieldset": { borderColor: "#ff33cc" },
+            },
+          }}
         />
         
-        
+        
           Already have an account?{" "}
           
             Login

From f11b36ba4ef04d691e88ef6b979bb143a00fbfa4 Mon Sep 17 00:00:00 2001
From: AmitChauhan63390 
Date: Sun, 8 Dec 2024 05:50:25 +0530
Subject: [PATCH 014/216] forms

---
 src/pages/Register.tsx | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/pages/Register.tsx b/src/pages/Register.tsx
index 626921f0..474f0182 100644
--- a/src/pages/Register.tsx
+++ b/src/pages/Register.tsx
@@ -40,10 +40,8 @@ const Register = () => {
       notify("success", "Registration Successful!");
       window.localStorage.setItem("user", JSON.stringify(data));
       navigate("/");
-
     } catch (error: any) {
       notify("error", error.response?.data || "Registration Failed. Please try again.");
-    } 
       setLoading(false);
     }
   };

From 0d763f78213671d5809102b9b501920921418ffd Mon Sep 17 00:00:00 2001
From: amhsirak 
Date: Fri, 20 Dec 2024 18:32:08 +0530
Subject: [PATCH 015/216] feat: iframe support for get element info

---
 server/src/workflow-management/selector.ts | 214 ++++++++++++++++-----
 1 file changed, 168 insertions(+), 46 deletions(-)

diff --git a/server/src/workflow-management/selector.ts b/server/src/workflow-management/selector.ts
index 240f8921..16979487 100644
--- a/server/src/workflow-management/selector.ts
+++ b/server/src/workflow-management/selector.ts
@@ -23,10 +23,8 @@ export const getElementInformation = async (
     if (!getList || listSelector !== '') {
       const elementInfo = await page.evaluate(
         async ({ x, y }) => {
-          const el = document.elementFromPoint(x, y) as HTMLElement;
-          if (el) {
-            const { parentElement } = el;
-            const element = parentElement?.tagName === 'A' ? parentElement : el;
+          // Helper function to get element info
+          const getElementInfo = (element: HTMLElement) => {
             let info: {
               tagName: string;
               hasOnlyText?: boolean;
@@ -36,9 +34,12 @@ export const getElementInformation = async (
               attributes?: Record;
               innerHTML?: string;
               outerHTML?: string;
+              fromIframe?: boolean;
+              iframePath?: string[];
             } = {
               tagName: element?.tagName ?? '',
             };
+
             if (element) {
               info.attributes = Array.from(element.attributes).reduce(
                 (acc, attr) => {
@@ -48,7 +49,7 @@ export const getElementInformation = async (
                 {} as Record
               );
             }
-            // Gather specific information based on the tag
+
             if (element?.tagName === 'A') {
               info.url = (element as HTMLAnchorElement).href;
               info.innerText = element.innerText ?? '';
@@ -61,29 +62,186 @@ export const getElementInformation = async (
                 ...info.attributes,
                 selectedValue: selectElement.value,
               };
-            } else if (element?.tagName === 'INPUT' && (element as HTMLInputElement).type === 'time' || (element as HTMLInputElement).type === 'date') {
+            } else if (element?.tagName === 'INPUT' && 
+                      ((element as HTMLInputElement).type === 'time' || 
+                       (element as HTMLInputElement).type === 'date')) {
               info.innerText = (element as HTMLInputElement).value;
             } else {
               info.hasOnlyText = element?.children?.length === 0 &&
                 element?.innerText?.length > 0;
               info.innerText = element?.innerText ?? '';
             }
+
             info.innerHTML = element.innerHTML;
             info.outerHTML = element.outerHTML;
             return info;
+          };
+
+          // Helper function to search in iframe
+          const searchInIframe = (
+            iframe: HTMLIFrameElement, 
+            relativeX: number, 
+            relativeY: number,
+            iframePath: string[]
+          ) => {
+            try {
+              if (!iframe.contentDocument) return null;
+              
+              const el = iframe.contentDocument.elementFromPoint(relativeX, relativeY) as HTMLElement;
+              if (!el) return null;
+
+              const { parentElement } = el;
+              const element = parentElement?.tagName === 'A' ? parentElement : el;
+              
+              const info = getElementInfo(element);
+              info.fromIframe = true;
+              info.iframePath = iframePath;
+
+              return info;
+            } catch (e) {
+              console.warn('Cannot access iframe content:', e);
+              return null;
+            }
+          };
+
+          const el = document.elementFromPoint(x, y) as HTMLElement;
+          if (el) {
+            // Check if the element is an iframe
+            if (el.tagName === 'IFRAME') {
+              const iframe = el as HTMLIFrameElement;
+              const rect = iframe.getBoundingClientRect();
+              const relativeX = x - rect.left;
+              const relativeY = y - rect.top;
+              
+              const iframeResult = searchInIframe(
+                iframe, 
+                relativeX, 
+                relativeY, 
+                [iframe.id || 'unnamed-iframe']
+              );
+              if (iframeResult) return iframeResult;
+            }
+
+            const { parentElement } = el;
+            const element = parentElement?.tagName === 'A' ? parentElement : el;
+            return getElementInfo(element);
           }
           return null;
         },
-        { x: coordinates.x, y: coordinates.y },
+        { x: coordinates.x, y: coordinates.y }
       );
       return elementInfo;
     } else {
       const elementInfo = await page.evaluate(
         async ({ x, y }) => {
+          // Helper function to get element info (same as above)
+          const getElementInfo = (element: HTMLElement) => {
+            let info: {
+              tagName: string;
+              hasOnlyText?: boolean;
+              innerText?: string;
+              url?: string;
+              imageUrl?: string;
+              attributes?: Record;
+              innerHTML?: string;
+              outerHTML?: string;
+              fromIframe?: boolean;
+              iframePath?: string[];
+            } = {
+              tagName: element?.tagName ?? '',
+            };
+
+            if (element) {
+              info.attributes = Array.from(element.attributes).reduce(
+                (acc, attr) => {
+                  acc[attr.name] = attr.value;
+                  return acc;
+                },
+                {} as Record
+              );
+            }
+
+            if (element?.tagName === 'A') {
+              info.url = (element as HTMLAnchorElement).href;
+              info.innerText = element.innerText ?? '';
+            } else if (element?.tagName === 'IMG') {
+              info.imageUrl = (element as HTMLImageElement).src;
+            } else {
+              info.hasOnlyText = element?.children?.length === 0 &&
+                element?.innerText?.length > 0;
+              info.innerText = element?.innerText ?? '';
+            }
+
+            info.innerHTML = element.innerHTML;
+            info.outerHTML = element.outerHTML;
+            return info;
+          };
+
+          // Helper function to search in iframe (same as above)
+          const searchInIframe = (
+            iframe: HTMLIFrameElement, 
+            relativeX: number, 
+            relativeY: number,
+            iframePath: string[]
+          ) => {
+            try {
+              if (!iframe.contentDocument) return null;
+              
+              const el = iframe.contentDocument.elementFromPoint(relativeX, relativeY) as HTMLElement;
+              if (!el) return null;
+
+              let element = el;
+              while (element.parentElement) {
+                const parentRect = element.parentElement.getBoundingClientRect();
+                const childRect = element.getBoundingClientRect();
+
+                const fullyContained =
+                  parentRect.left <= childRect.left &&
+                  parentRect.right >= childRect.right &&
+                  parentRect.top <= childRect.top &&
+                  parentRect.bottom >= childRect.bottom;
+
+                const significantOverlap =
+                  (childRect.width * childRect.height) /
+                  (parentRect.width * parentRect.height) > 0.5;
+
+                if (fullyContained && significantOverlap) {
+                  element = element.parentElement;
+                } else {
+                  break;
+                }
+              }
+
+              const info = getElementInfo(element);
+              info.fromIframe = true;
+              info.iframePath = iframePath;
+
+              return info;
+            } catch (e) {
+              console.warn('Cannot access iframe content:', e);
+              return null;
+            }
+          };
+
           const originalEl = document.elementFromPoint(x, y) as HTMLElement;
           if (originalEl) {
-            let element = originalEl;
+            // Check if the element is an iframe
+            if (originalEl.tagName === 'IFRAME') {
+              const iframe = originalEl as HTMLIFrameElement;
+              const rect = iframe.getBoundingClientRect();
+              const relativeX = x - rect.left;
+              const relativeY = y - rect.top;
+              
+              const iframeResult = searchInIframe(
+                iframe, 
+                relativeX, 
+                relativeY, 
+                [iframe.id || 'unnamed-iframe']
+              );
+              if (iframeResult) return iframeResult;
+            }
 
+            let element = originalEl;
             while (element.parentElement) {
               const parentRect = element.parentElement.getBoundingClientRect();
               const childRect = element.getBoundingClientRect();
@@ -105,47 +263,11 @@ export const getElementInformation = async (
               }
             }
 
-            let info: {
-              tagName: string;
-              hasOnlyText?: boolean;
-              innerText?: string;
-              url?: string;
-              imageUrl?: string;
-              attributes?: Record;
-              innerHTML?: string;
-              outerHTML?: string;
-            } = {
-              tagName: element?.tagName ?? '',
-            };
-
-            if (element) {
-              info.attributes = Array.from(element.attributes).reduce(
-                (acc, attr) => {
-                  acc[attr.name] = attr.value;
-                  return acc;
-                },
-                {} as Record
-              );
-            }
-
-            if (element?.tagName === 'A') {
-              info.url = (element as HTMLAnchorElement).href;
-              info.innerText = element.innerText ?? '';
-            } else if (element?.tagName === 'IMG') {
-              info.imageUrl = (element as HTMLImageElement).src;
-            } else {
-              info.hasOnlyText = element?.children?.length === 0 &&
-                element?.innerText?.length > 0;
-              info.innerText = element?.innerText ?? '';
-            }
-
-            info.innerHTML = element.innerHTML;
-            info.outerHTML = element.outerHTML;
-            return info;
+            return getElementInfo(element);
           }
           return null;
         },
-        { x: coordinates.x, y: coordinates.y },
+        { x: coordinates.x, y: coordinates.y }
       );
       return elementInfo;
     }

From 6904933036bc48bc09fc331479efbfe174181c78 Mon Sep 17 00:00:00 2001
From: amhsirak 
Date: Fri, 20 Dec 2024 20:28:11 +0530
Subject: [PATCH 016/216] feat: iframe support for getRect

---
 server/src/workflow-management/selector.ts | 189 ++++++++++++++++++---
 1 file changed, 166 insertions(+), 23 deletions(-)

diff --git a/server/src/workflow-management/selector.ts b/server/src/workflow-management/selector.ts
index 16979487..83491042 100644
--- a/server/src/workflow-management/selector.ts
+++ b/server/src/workflow-management/selector.ts
@@ -292,25 +292,90 @@ export const getRect = async (page: Page, coordinates: Coordinates, listSelector
     if (!getList || listSelector !== '') {
       const rect = await page.evaluate(
         async ({ x, y }) => {
+          // Helper function to convert rectangle to plain object
+          const getRectangleInfo = (rectangle: DOMRect) => {
+            const info = {
+              x: rectangle.x,
+              y: rectangle.y,
+              width: rectangle.width,
+              height: rectangle.height,
+              top: rectangle.top,
+              right: rectangle.right,
+              bottom: rectangle.bottom,
+              left: rectangle.left,
+              fromIframe: false,
+              iframePath: [] as string[]
+            };
+            return info;
+          };
+
+          // Helper function to search in iframe
+          const searchInIframe = (
+            iframe: HTMLIFrameElement, 
+            relativeX: number, 
+            relativeY: number,
+            iframePath: string[]
+          ) => {
+            try {
+              if (!iframe.contentDocument) return null;
+              
+              const el = iframe.contentDocument.elementFromPoint(relativeX, relativeY) as HTMLElement;
+              if (!el) return null;
+
+              const { parentElement } = el;
+              const element = parentElement?.tagName === 'A' ? parentElement : el;
+              const rectangle = element?.getBoundingClientRect();
+              
+              if (rectangle) {
+                const iframeRect = iframe.getBoundingClientRect();
+                const rectInfo = getRectangleInfo(rectangle);
+                
+                // Adjust coordinates relative to the main document
+                rectInfo.x += iframeRect.x;
+                rectInfo.y += iframeRect.y;
+                rectInfo.top += iframeRect.top;
+                rectInfo.right += iframeRect.left;
+                rectInfo.bottom += iframeRect.top;
+                rectInfo.left += iframeRect.left;
+                rectInfo.fromIframe = true;
+                rectInfo.iframePath = iframePath;
+                
+                return rectInfo;
+              }
+              return null;
+            } catch (e) {
+              console.warn('Cannot access iframe content:', e);
+              return null;
+            }
+          };
+
           const el = document.elementFromPoint(x, y) as HTMLElement;
           if (el) {
+            // Check if the element is an iframe
+            if (el.tagName === 'IFRAME') {
+              const iframe = el as HTMLIFrameElement;
+              const rect = iframe.getBoundingClientRect();
+              const relativeX = x - rect.left;
+              const relativeY = y - rect.top;
+              
+              const iframeResult = searchInIframe(
+                iframe, 
+                relativeX, 
+                relativeY, 
+                [iframe.id || 'unnamed-iframe']
+              );
+              if (iframeResult) return iframeResult;
+            }
+
             const { parentElement } = el;
-            // Match the logic in recorder.ts for link clicks
             const element = parentElement?.tagName === 'A' ? parentElement : el;
             const rectangle = element?.getBoundingClientRect();
+            
             if (rectangle) {
-              return {
-                x: rectangle.x,
-                y: rectangle.y,
-                width: rectangle.width,
-                height: rectangle.height,
-                top: rectangle.top,
-                right: rectangle.right,
-                bottom: rectangle.bottom,
-                left: rectangle.left,
-              };
+              return getRectangleInfo(rectangle);
             }
           }
+          return null;
         },
         { x: coordinates.x, y: coordinates.y },
       );
@@ -318,10 +383,98 @@ export const getRect = async (page: Page, coordinates: Coordinates, listSelector
     } else {
       const rect = await page.evaluate(
         async ({ x, y }) => {
+          // Helper function to convert rectangle to plain object (same as above)
+          const getRectangleInfo = (rectangle: DOMRect) => ({
+            x: rectangle.x,
+            y: rectangle.y,
+            width: rectangle.width,
+            height: rectangle.height,
+            top: rectangle.top,
+            right: rectangle.right,
+            bottom: rectangle.bottom,
+            left: rectangle.left,
+            fromIframe: false,
+            iframePath: [] as string[]
+          });
+
+          // Helper function to search in iframe (same as above)
+          const searchInIframe = (
+            iframe: HTMLIFrameElement, 
+            relativeX: number, 
+            relativeY: number,
+            iframePath: string[]
+          ) => {
+            try {
+              if (!iframe.contentDocument) return null;
+              
+              const el = iframe.contentDocument.elementFromPoint(relativeX, relativeY) as HTMLElement;
+              if (!el) return null;
+
+              let element = el;
+              while (element.parentElement) {
+                const parentRect = element.parentElement.getBoundingClientRect();
+                const childRect = element.getBoundingClientRect();
+
+                const fullyContained =
+                  parentRect.left <= childRect.left &&
+                  parentRect.right >= childRect.right &&
+                  parentRect.top <= childRect.top &&
+                  parentRect.bottom >= childRect.bottom;
+
+                const significantOverlap =
+                  (childRect.width * childRect.height) /
+                  (parentRect.width * parentRect.height) > 0.5;
+
+                if (fullyContained && significantOverlap) {
+                  element = element.parentElement;
+                } else {
+                  break;
+                }
+              }
+
+              const rectangle = element?.getBoundingClientRect();
+              if (rectangle) {
+                const iframeRect = iframe.getBoundingClientRect();
+                const rectInfo = getRectangleInfo(rectangle);
+                
+                // Adjust coordinates relative to the main document
+                rectInfo.x += iframeRect.x;
+                rectInfo.y += iframeRect.y;
+                rectInfo.top += iframeRect.top;
+                rectInfo.right += iframeRect.left;
+                rectInfo.bottom += iframeRect.top;
+                rectInfo.left += iframeRect.left;
+                rectInfo.fromIframe = true;
+                rectInfo.iframePath = iframePath;
+                
+                return rectInfo;
+              }
+              return null;
+            } catch (e) {
+              console.warn('Cannot access iframe content:', e);
+              return null;
+            }
+          };
+
           const originalEl = document.elementFromPoint(x, y) as HTMLElement;
           if (originalEl) {
-            let element = originalEl;
+            // Check if the element is an iframe
+            if (originalEl.tagName === 'IFRAME') {
+              const iframe = originalEl as HTMLIFrameElement;
+              const rect = iframe.getBoundingClientRect();
+              const relativeX = x - rect.left;
+              const relativeY = y - rect.top;
+              
+              const iframeResult = searchInIframe(
+                iframe, 
+                relativeX, 
+                relativeY, 
+                [iframe.id || 'unnamed-iframe']
+              );
+              if (iframeResult) return iframeResult;
+            }
 
+            let element = originalEl;
             while (element.parentElement) {
               const parentRect = element.parentElement.getBoundingClientRect();
               const childRect = element.getBoundingClientRect();
@@ -344,18 +497,8 @@ export const getRect = async (page: Page, coordinates: Coordinates, listSelector
             }
 
             const rectangle = element?.getBoundingClientRect();
-
             if (rectangle) {
-              return {
-                x: rectangle.x,
-                y: rectangle.y,
-                width: rectangle.width,
-                height: rectangle.height,
-                top: rectangle.top,
-                right: rectangle.right,
-                bottom: rectangle.bottom,
-                left: rectangle.left,
-              };
+              return getRectangleInfo(rectangle);
             }
           }
           return null;

From 8ba928dae6cac4d7e5924bcc799e792068e6734d Mon Sep 17 00:00:00 2001
From: amhsirak 
Date: Fri, 20 Dec 2024 20:28:24 +0530
Subject: [PATCH 017/216] chore: fix format

---
 server/src/workflow-management/selector.ts | 76 +++++++++++-----------
 1 file changed, 38 insertions(+), 38 deletions(-)

diff --git a/server/src/workflow-management/selector.ts b/server/src/workflow-management/selector.ts
index 83491042..dd869f3d 100644
--- a/server/src/workflow-management/selector.ts
+++ b/server/src/workflow-management/selector.ts
@@ -62,9 +62,9 @@ export const getElementInformation = async (
                 ...info.attributes,
                 selectedValue: selectElement.value,
               };
-            } else if (element?.tagName === 'INPUT' && 
-                      ((element as HTMLInputElement).type === 'time' || 
-                       (element as HTMLInputElement).type === 'date')) {
+            } else if (element?.tagName === 'INPUT' &&
+              ((element as HTMLInputElement).type === 'time' ||
+                (element as HTMLInputElement).type === 'date')) {
               info.innerText = (element as HTMLInputElement).value;
             } else {
               info.hasOnlyText = element?.children?.length === 0 &&
@@ -79,20 +79,20 @@ export const getElementInformation = async (
 
           // Helper function to search in iframe
           const searchInIframe = (
-            iframe: HTMLIFrameElement, 
-            relativeX: number, 
+            iframe: HTMLIFrameElement,
+            relativeX: number,
             relativeY: number,
             iframePath: string[]
           ) => {
             try {
               if (!iframe.contentDocument) return null;
-              
+
               const el = iframe.contentDocument.elementFromPoint(relativeX, relativeY) as HTMLElement;
               if (!el) return null;
 
               const { parentElement } = el;
               const element = parentElement?.tagName === 'A' ? parentElement : el;
-              
+
               const info = getElementInfo(element);
               info.fromIframe = true;
               info.iframePath = iframePath;
@@ -112,11 +112,11 @@ export const getElementInformation = async (
               const rect = iframe.getBoundingClientRect();
               const relativeX = x - rect.left;
               const relativeY = y - rect.top;
-              
+
               const iframeResult = searchInIframe(
-                iframe, 
-                relativeX, 
-                relativeY, 
+                iframe,
+                relativeX,
+                relativeY,
                 [iframe.id || 'unnamed-iframe']
               );
               if (iframeResult) return iframeResult;
@@ -179,14 +179,14 @@ export const getElementInformation = async (
 
           // Helper function to search in iframe (same as above)
           const searchInIframe = (
-            iframe: HTMLIFrameElement, 
-            relativeX: number, 
+            iframe: HTMLIFrameElement,
+            relativeX: number,
             relativeY: number,
             iframePath: string[]
           ) => {
             try {
               if (!iframe.contentDocument) return null;
-              
+
               const el = iframe.contentDocument.elementFromPoint(relativeX, relativeY) as HTMLElement;
               if (!el) return null;
 
@@ -231,11 +231,11 @@ export const getElementInformation = async (
               const rect = iframe.getBoundingClientRect();
               const relativeX = x - rect.left;
               const relativeY = y - rect.top;
-              
+
               const iframeResult = searchInIframe(
-                iframe, 
-                relativeX, 
-                relativeY, 
+                iframe,
+                relativeX,
+                relativeY,
                 [iframe.id || 'unnamed-iframe']
               );
               if (iframeResult) return iframeResult;
@@ -311,25 +311,25 @@ export const getRect = async (page: Page, coordinates: Coordinates, listSelector
 
           // Helper function to search in iframe
           const searchInIframe = (
-            iframe: HTMLIFrameElement, 
-            relativeX: number, 
+            iframe: HTMLIFrameElement,
+            relativeX: number,
             relativeY: number,
             iframePath: string[]
           ) => {
             try {
               if (!iframe.contentDocument) return null;
-              
+
               const el = iframe.contentDocument.elementFromPoint(relativeX, relativeY) as HTMLElement;
               if (!el) return null;
 
               const { parentElement } = el;
               const element = parentElement?.tagName === 'A' ? parentElement : el;
               const rectangle = element?.getBoundingClientRect();
-              
+
               if (rectangle) {
                 const iframeRect = iframe.getBoundingClientRect();
                 const rectInfo = getRectangleInfo(rectangle);
-                
+
                 // Adjust coordinates relative to the main document
                 rectInfo.x += iframeRect.x;
                 rectInfo.y += iframeRect.y;
@@ -339,7 +339,7 @@ export const getRect = async (page: Page, coordinates: Coordinates, listSelector
                 rectInfo.left += iframeRect.left;
                 rectInfo.fromIframe = true;
                 rectInfo.iframePath = iframePath;
-                
+
                 return rectInfo;
               }
               return null;
@@ -357,11 +357,11 @@ export const getRect = async (page: Page, coordinates: Coordinates, listSelector
               const rect = iframe.getBoundingClientRect();
               const relativeX = x - rect.left;
               const relativeY = y - rect.top;
-              
+
               const iframeResult = searchInIframe(
-                iframe, 
-                relativeX, 
-                relativeY, 
+                iframe,
+                relativeX,
+                relativeY,
                 [iframe.id || 'unnamed-iframe']
               );
               if (iframeResult) return iframeResult;
@@ -370,7 +370,7 @@ export const getRect = async (page: Page, coordinates: Coordinates, listSelector
             const { parentElement } = el;
             const element = parentElement?.tagName === 'A' ? parentElement : el;
             const rectangle = element?.getBoundingClientRect();
-            
+
             if (rectangle) {
               return getRectangleInfo(rectangle);
             }
@@ -399,14 +399,14 @@ export const getRect = async (page: Page, coordinates: Coordinates, listSelector
 
           // Helper function to search in iframe (same as above)
           const searchInIframe = (
-            iframe: HTMLIFrameElement, 
-            relativeX: number, 
+            iframe: HTMLIFrameElement,
+            relativeX: number,
             relativeY: number,
             iframePath: string[]
           ) => {
             try {
               if (!iframe.contentDocument) return null;
-              
+
               const el = iframe.contentDocument.elementFromPoint(relativeX, relativeY) as HTMLElement;
               if (!el) return null;
 
@@ -436,7 +436,7 @@ export const getRect = async (page: Page, coordinates: Coordinates, listSelector
               if (rectangle) {
                 const iframeRect = iframe.getBoundingClientRect();
                 const rectInfo = getRectangleInfo(rectangle);
-                
+
                 // Adjust coordinates relative to the main document
                 rectInfo.x += iframeRect.x;
                 rectInfo.y += iframeRect.y;
@@ -446,7 +446,7 @@ export const getRect = async (page: Page, coordinates: Coordinates, listSelector
                 rectInfo.left += iframeRect.left;
                 rectInfo.fromIframe = true;
                 rectInfo.iframePath = iframePath;
-                
+
                 return rectInfo;
               }
               return null;
@@ -464,11 +464,11 @@ export const getRect = async (page: Page, coordinates: Coordinates, listSelector
               const rect = iframe.getBoundingClientRect();
               const relativeX = x - rect.left;
               const relativeY = y - rect.top;
-              
+
               const iframeResult = searchInIframe(
-                iframe, 
-                relativeX, 
-                relativeY, 
+                iframe,
+                relativeX,
+                relativeY,
                 [iframe.id || 'unnamed-iframe']
               );
               if (iframeResult) return iframeResult;

From a131ce6c04d1d9f5b1982493f4200b972e989555 Mon Sep 17 00:00:00 2001
From: amhsirak 
Date: Tue, 24 Dec 2024 23:44:20 +0530
Subject: [PATCH 018/216] feat: shadow dom selection

---
 server/src/workflow-management/selector.ts | 105 +++++++++++++++++----
 1 file changed, 89 insertions(+), 16 deletions(-)

diff --git a/server/src/workflow-management/selector.ts b/server/src/workflow-management/selector.ts
index c0fa21f1..5a7273df 100644
--- a/server/src/workflow-management/selector.ts
+++ b/server/src/workflow-management/selector.ts
@@ -23,7 +23,28 @@ export const getElementInformation = async (
     if (!getList || listSelector !== '') {
       const elementInfo = await page.evaluate(
         async ({ x, y }) => {
-          const el = document.elementFromPoint(x, y) as HTMLElement;
+          // Helper function to get element from point including shadow DOM
+          const getDeepestElementFromPoint = (x: number, y: number): HTMLElement | null => {
+            let element = document.elementFromPoint(x, y) as HTMLElement;
+            if (!element) return null;
+
+            // Traverse through shadow roots
+            let current = element;
+            while (current) {
+              // Check if element has shadow root
+              const shadowRoot = current.shadowRoot;
+              if (!shadowRoot) break;
+
+              // Try to find deeper element in shadow DOM
+              const shadowElement = shadowRoot.elementFromPoint(x, y) as HTMLElement;
+              if (!shadowElement || shadowElement === current) break;
+
+              current = shadowElement;
+            }
+            return current;
+          };
+
+          const el = getDeepestElementFromPoint(x, y);
           if (el) {
             const { parentElement } = el;
             const element = parentElement?.tagName === 'A' ? parentElement : el;
@@ -36,9 +57,12 @@ export const getElementInformation = async (
               attributes?: Record;
               innerHTML?: string;
               outerHTML?: string;
+              isShadowRoot?: boolean;
             } = {
               tagName: element?.tagName ?? '',
+              isShadowRoot: !!element?.shadowRoot
             };
+            
             if (element) {
               info.attributes = Array.from(element.attributes).reduce(
                 (acc, attr) => {
@@ -48,6 +72,7 @@ export const getElementInformation = async (
                 {} as Record
               );
             }
+
             // Gather specific information based on the tag
             if (element?.tagName === 'A') {
               info.url = (element as HTMLAnchorElement).href;
@@ -61,7 +86,7 @@ export const getElementInformation = async (
                 ...info.attributes,
                 selectedValue: selectElement.value,
               };
-            } else if (element?.tagName === 'INPUT' && (element as HTMLInputElement).type === 'time' || (element as HTMLInputElement).type === 'date') {
+            } else if (element?.tagName === 'INPUT' && ((element as HTMLInputElement).type === 'time' || (element as HTMLInputElement).type === 'date')) {
               info.innerText = (element as HTMLInputElement).value;
             } else {
               info.hasOnlyText = element?.children?.length === 0 &&
@@ -80,7 +105,26 @@ export const getElementInformation = async (
     } else {
       const elementInfo = await page.evaluate(
         async ({ x, y }) => {
-          const originalEl = document.elementFromPoint(x, y) as HTMLElement;
+          // Helper function to get element from point including shadow DOM
+          const getDeepestElementFromPoint = (x: number, y: number): HTMLElement | null => {
+            let element = document.elementFromPoint(x, y) as HTMLElement;
+            if (!element) return null;
+
+            // Traverse through shadow roots
+            let current = element;
+            while (current) {
+              const shadowRoot = current.shadowRoot;
+              if (!shadowRoot) break;
+
+              const shadowElement = shadowRoot.elementFromPoint(x, y) as HTMLElement;
+              if (!shadowElement || shadowElement === current) break;
+
+              current = shadowElement;
+            }
+            return current;
+          };
+
+          const originalEl = getDeepestElementFromPoint(x, y);
           if (originalEl) {
             let element = originalEl;
 
@@ -114,8 +158,10 @@ export const getElementInformation = async (
               attributes?: Record;
               innerHTML?: string;
               outerHTML?: string;
+              isShadowRoot?: boolean;
             } = {
               tagName: element?.tagName ?? '',
+              isShadowRoot: !!element?.shadowRoot
             };
 
             if (element) {
@@ -156,24 +202,33 @@ export const getElementInformation = async (
   }
 };
 
-/**
- * Returns a {@link Rectangle} object representing
- * the coordinates, width, height and corner points of the element.
- * If an element is not found, returns null.
- * @param page The page instance.
- * @param coordinates Coordinates of an element.
- * @category WorkflowManagement-Selectors
- * @returns {Promise}
- */
 export const getRect = async (page: Page, coordinates: Coordinates, listSelector: string, getList: boolean) => {
   try {
     if (!getList || listSelector !== '') {
       const rect = await page.evaluate(
         async ({ x, y }) => {
-          const el = document.elementFromPoint(x, y) as HTMLElement;
+          // Helper function to get element from point including shadow DOM
+          const getDeepestElementFromPoint = (x: number, y: number): HTMLElement | null => {
+            let element = document.elementFromPoint(x, y) as HTMLElement;
+            if (!element) return null;
+
+            // Traverse through shadow roots
+            let current = element;
+            while (current) {
+              const shadowRoot = current.shadowRoot;
+              if (!shadowRoot) break;
+
+              const shadowElement = shadowRoot.elementFromPoint(x, y) as HTMLElement;
+              if (!shadowElement || shadowElement === current) break;
+
+              current = shadowElement;
+            }
+            return current;
+          };
+
+          const el = getDeepestElementFromPoint(x, y);
           if (el) {
             const { parentElement } = el;
-            // Match the logic in recorder.ts for link clicks
             const element = parentElement?.tagName === 'A' ? parentElement : el;
             const rectangle = element?.getBoundingClientRect();
             if (rectangle) {
@@ -196,7 +251,26 @@ export const getRect = async (page: Page, coordinates: Coordinates, listSelector
     } else {
       const rect = await page.evaluate(
         async ({ x, y }) => {
-          const originalEl = document.elementFromPoint(x, y) as HTMLElement;
+          // Helper function to get element from point including shadow DOM
+          const getDeepestElementFromPoint = (x: number, y: number): HTMLElement | null => {
+            let element = document.elementFromPoint(x, y) as HTMLElement;
+            if (!element) return null;
+
+            // Traverse through shadow roots
+            let current = element;
+            while (current) {
+              const shadowRoot = current.shadowRoot;
+              if (!shadowRoot) break;
+
+              const shadowElement = shadowRoot.elementFromPoint(x, y) as HTMLElement;
+              if (!shadowElement || shadowElement === current) break;
+
+              current = shadowElement;
+            }
+            return current;
+          };
+
+          const originalEl = getDeepestElementFromPoint(x, y);
           if (originalEl) {
             let element = originalEl;
 
@@ -249,7 +323,6 @@ export const getRect = async (page: Page, coordinates: Coordinates, listSelector
   }
 };
 
-
 /**
  * Returns the best and unique css {@link Selectors} for the element on the page.
  * Internally uses a finder function from https://github.com/antonmedv/finder/blob/master/finder.ts

From a5a35c1c35521d5485ebb87539e74a859c902869 Mon Sep 17 00:00:00 2001
From: amhsirak 
Date: Sat, 28 Dec 2024 22:44:21 +0530
Subject: [PATCH 019/216] feat: add language option

---
 src/components/molecules/NavBar.tsx | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/components/molecules/NavBar.tsx b/src/components/molecules/NavBar.tsx
index 142d45ab..a953b178 100644
--- a/src/components/molecules/NavBar.tsx
+++ b/src/components/molecules/NavBar.tsx
@@ -376,6 +376,14 @@ export const NavBar: React.FC = ({
                       >
                         Deutsch
                       
+                       {
+                          changeLanguage("de");
+                          handleMenuClose();
+                        }}
+                      >
+                        Add Language
+                      
                     
                   
                 

From 5419b70f616c4046bd0e65c8b917671b3ed39600 Mon Sep 17 00:00:00 2001
From: amhsirak 
Date: Sat, 28 Dec 2024 22:44:41 +0530
Subject: [PATCH 020/216] feat: add language option

---
 src/components/molecules/NavBar.tsx | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/components/molecules/NavBar.tsx b/src/components/molecules/NavBar.tsx
index a953b178..d0c084a7 100644
--- a/src/components/molecules/NavBar.tsx
+++ b/src/components/molecules/NavBar.tsx
@@ -470,6 +470,14 @@ export const NavBar: React.FC = ({
                 >
                   Deutsch
                 
+                 {
+                          changeLanguage("de");
+                          handleMenuClose();
+                        }}
+                      >
+                        Add Language
+                      
               
           )}
       

From 1630e0eb7b8cb6dfab0c9b503ca42fed845dc4ce Mon Sep 17 00:00:00 2001
From: amhsirak 
Date: Sat, 28 Dec 2024 22:44:53 +0530
Subject: [PATCH 021/216] fix: format

---
 src/components/molecules/NavBar.tsx | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/components/molecules/NavBar.tsx b/src/components/molecules/NavBar.tsx
index d0c084a7..6daf53e0 100644
--- a/src/components/molecules/NavBar.tsx
+++ b/src/components/molecules/NavBar.tsx
@@ -471,13 +471,13 @@ export const NavBar: React.FC = ({
                   Deutsch
                 
                  {
-                          changeLanguage("de");
-                          handleMenuClose();
-                        }}
-                      >
-                        Add Language
-                      
+                  onClick={() => {
+                    changeLanguage("de");
+                    handleMenuClose();
+                  }}
+                >
+                  Add Language
+                
               
           )}
       

From 6ac7068f45f7493136897ec54e0651f6f8778f81 Mon Sep 17 00:00:00 2001
From: amhsirak 
Date: Sat, 28 Dec 2024 22:54:28 +0530
Subject: [PATCH 022/216] chore: i18n docs link to do

---
 src/components/molecules/NavBar.tsx | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/components/molecules/NavBar.tsx b/src/components/molecules/NavBar.tsx
index 6daf53e0..be9f8acd 100644
--- a/src/components/molecules/NavBar.tsx
+++ b/src/components/molecules/NavBar.tsx
@@ -376,6 +376,7 @@ export const NavBar: React.FC = ({
                       >
                         Deutsch
                       
+                      {/* WIP: Replace change language with i18n docs link */}
                        {
                           changeLanguage("de");
@@ -470,6 +471,7 @@ export const NavBar: React.FC = ({
                 >
                   Deutsch
                 
+                {/* WIP: Replace change language with i18n docs link */}
                  {
                     changeLanguage("de");

From a09b03e4a75627d2adc8189f8fdd361b36b8a82b Mon Sep 17 00:00:00 2001
From: RohitR311 
Date: Sun, 29 Dec 2024 23:36:06 +0530
Subject: [PATCH 023/216] feat: get deepest shadowDOM element selector

---
 server/src/workflow-management/selector.ts | 243 +++++++++++++++------
 1 file changed, 179 insertions(+), 64 deletions(-)

diff --git a/server/src/workflow-management/selector.ts b/server/src/workflow-management/selector.ts
index 5a7273df..9b3af66e 100644
--- a/server/src/workflow-management/selector.ts
+++ b/server/src/workflow-management/selector.ts
@@ -23,31 +23,41 @@ export const getElementInformation = async (
     if (!getList || listSelector !== '') {
       const elementInfo = await page.evaluate(
         async ({ x, y }) => {
-          // Helper function to get element from point including shadow DOM
+          // Enhanced helper function to get element from point including shadow DOM
           const getDeepestElementFromPoint = (x: number, y: number): HTMLElement | null => {
             let element = document.elementFromPoint(x, y) as HTMLElement;
             if (!element) return null;
 
             // Traverse through shadow roots
             let current = element;
-            while (current) {
-              // Check if element has shadow root
-              const shadowRoot = current.shadowRoot;
-              if (!shadowRoot) break;
-
-              // Try to find deeper element in shadow DOM
+            let shadowRoot = current.shadowRoot;
+            
+            // Keep track of the deepest shadow DOM element found
+            let deepestElement = current;
+            
+            while (shadowRoot) {
+              // Try to find element at same point in shadow DOM
               const shadowElement = shadowRoot.elementFromPoint(x, y) as HTMLElement;
               if (!shadowElement || shadowElement === current) break;
-
+              
+              // Update our tracking of the deepest element
+              deepestElement = shadowElement;
               current = shadowElement;
+              shadowRoot = current.shadowRoot;
             }
-            return current;
+
+            return deepestElement;
           };
 
           const el = getDeepestElementFromPoint(x, y);
           if (el) {
             const { parentElement } = el;
             const element = parentElement?.tagName === 'A' ? parentElement : el;
+            
+            // Get the containing shadow root if any
+            const containingShadowRoot = element.getRootNode() as ShadowRoot;
+            const isShadowRoot = containingShadowRoot instanceof ShadowRoot;
+
             let info: {
               tagName: string;
               hasOnlyText?: boolean;
@@ -58,11 +68,20 @@ export const getElementInformation = async (
               innerHTML?: string;
               outerHTML?: string;
               isShadowRoot?: boolean;
+              shadowRootMode?: string;
+              shadowRootContent?: string;
             } = {
               tagName: element?.tagName ?? '',
-              isShadowRoot: !!element?.shadowRoot
+              isShadowRoot: isShadowRoot
             };
+
+            if (isShadowRoot) {
+              // Include shadow root specific information
+              info.shadowRootMode = containingShadowRoot.mode;
+              info.shadowRootContent = containingShadowRoot.innerHTML;
+            }
             
+            // Get attributes including those from shadow DOM context
             if (element) {
               info.attributes = Array.from(element.attributes).reduce(
                 (acc, attr) => {
@@ -71,84 +90,82 @@ export const getElementInformation = async (
                 },
                 {} as Record
               );
+              
+              // Get text content considering shadow DOM context
+              info.innerText = element.textContent ?? '';
+              info.innerHTML = element.innerHTML;
+              info.outerHTML = element.outerHTML;
+              info.hasOnlyText = element.children.length === 0 && 
+                   (element.textContent !== null && 
+                    element.textContent.trim().length > 0);
             }
 
-            // Gather specific information based on the tag
-            if (element?.tagName === 'A') {
-              info.url = (element as HTMLAnchorElement).href;
-              info.innerText = element.innerText ?? '';
-            } else if (element?.tagName === 'IMG') {
-              info.imageUrl = (element as HTMLImageElement).src;
-            } else if (element?.tagName === 'SELECT') {
-              const selectElement = element as HTMLSelectElement;
-              info.innerText = selectElement.options[selectElement.selectedIndex]?.text ?? '';
-              info.attributes = {
-                ...info.attributes,
-                selectedValue: selectElement.value,
-              };
-            } else if (element?.tagName === 'INPUT' && ((element as HTMLInputElement).type === 'time' || (element as HTMLInputElement).type === 'date')) {
-              info.innerText = (element as HTMLInputElement).value;
-            } else {
-              info.hasOnlyText = element?.children?.length === 0 &&
-                element?.innerText?.length > 0;
-              info.innerText = element?.innerText ?? '';
-            }
-            info.innerHTML = element.innerHTML;
-            info.outerHTML = element.outerHTML;
             return info;
           }
           return null;
         },
-        { x: coordinates.x, y: coordinates.y },
+        { x: coordinates.x, y: coordinates.y }
       );
       return elementInfo;
     } else {
       const elementInfo = await page.evaluate(
         async ({ x, y }) => {
-          // Helper function to get element from point including shadow DOM
+          // Enhanced helper function to get element from point including shadow DOM
           const getDeepestElementFromPoint = (x: number, y: number): HTMLElement | null => {
             let element = document.elementFromPoint(x, y) as HTMLElement;
             if (!element) return null;
-
+    
             // Traverse through shadow roots
             let current = element;
-            while (current) {
-              const shadowRoot = current.shadowRoot;
-              if (!shadowRoot) break;
-
+            let shadowRoot = current.shadowRoot;
+            
+            // Keep track of the deepest shadow DOM element found
+            let deepestElement = current;
+            
+            while (shadowRoot) {
+              // Try to find element at same point in shadow DOM
               const shadowElement = shadowRoot.elementFromPoint(x, y) as HTMLElement;
               if (!shadowElement || shadowElement === current) break;
-
+              
+              // Update our tracking of the deepest element
+              deepestElement = shadowElement;
               current = shadowElement;
+              shadowRoot = current.shadowRoot;
             }
-            return current;
+    
+            return deepestElement;
           };
-
+    
           const originalEl = getDeepestElementFromPoint(x, y);
           if (originalEl) {
             let element = originalEl;
-
+    
+            // Handle element hierarchy traversal for list items
             while (element.parentElement) {
               const parentRect = element.parentElement.getBoundingClientRect();
               const childRect = element.getBoundingClientRect();
-
+    
               const fullyContained =
                 parentRect.left <= childRect.left &&
                 parentRect.right >= childRect.right &&
                 parentRect.top <= childRect.top &&
                 parentRect.bottom >= childRect.bottom;
-
+    
               const significantOverlap =
                 (childRect.width * childRect.height) /
                 (parentRect.width * parentRect.height) > 0.5;
-
+    
               if (fullyContained && significantOverlap) {
                 element = element.parentElement;
               } else {
                 break;
               }
             }
-
+    
+            // Get the containing shadow root if any
+            const containingShadowRoot = element.getRootNode() as ShadowRoot;
+            const isShadowRoot = containingShadowRoot instanceof ShadowRoot;
+    
             let info: {
               tagName: string;
               hasOnlyText?: boolean;
@@ -159,12 +176,21 @@ export const getElementInformation = async (
               innerHTML?: string;
               outerHTML?: string;
               isShadowRoot?: boolean;
+              shadowRootMode?: string;
+              shadowRootContent?: string;
             } = {
               tagName: element?.tagName ?? '',
-              isShadowRoot: !!element?.shadowRoot
+              isShadowRoot: isShadowRoot
             };
-
+    
+            if (isShadowRoot) {
+              // Include shadow root specific information
+              info.shadowRootMode = containingShadowRoot.mode;
+              info.shadowRootContent = containingShadowRoot.innerHTML;
+            }
+    
             if (element) {
+              // Get attributes including those from shadow DOM context
               info.attributes = Array.from(element.attributes).reduce(
                 (acc, attr) => {
                   acc[attr.name] = attr.value;
@@ -172,21 +198,25 @@ export const getElementInformation = async (
                 },
                 {} as Record
               );
+    
+              // Handle specific element types
+              if (element.tagName === 'A') {
+                info.url = (element as HTMLAnchorElement).href;
+                info.innerText = element.textContent ?? '';
+              } else if (element.tagName === 'IMG') {
+                info.imageUrl = (element as HTMLImageElement).src;
+              } else {
+                // Handle text content with proper null checking
+                info.hasOnlyText = element.children.length === 0 && 
+                  (element.textContent !== null && 
+                   element.textContent.trim().length > 0);
+                info.innerText = element.textContent ?? '';
+              }
+    
+              info.innerHTML = element.innerHTML;
+              info.outerHTML = element.outerHTML;
             }
-
-            if (element?.tagName === 'A') {
-              info.url = (element as HTMLAnchorElement).href;
-              info.innerText = element.innerText ?? '';
-            } else if (element?.tagName === 'IMG') {
-              info.imageUrl = (element as HTMLImageElement).src;
-            } else {
-              info.hasOnlyText = element?.children?.length === 0 &&
-                element?.innerText?.length > 0;
-              info.innerText = element?.innerText ?? '';
-            }
-
-            info.innerHTML = element.innerHTML;
-            info.outerHTML = element.outerHTML;
+    
             return info;
           }
           return null;
@@ -793,6 +823,76 @@ export const getSelectors = async (page: Page, coordinates: Coordinates) => {
         return output;
       }
 
+      const MAX_DEPTH = 10;
+
+      const getDeepestElementFromPoint = (x: number, y: number): HTMLElement | null => {
+        let element = document.elementFromPoint(x, y) as HTMLElement;
+        if (!element) return null;
+
+        let current = element;
+        let deepestElement = current;
+        let depth = 0;
+
+        while (current && depth < MAX_DEPTH) {
+          const shadowRoot = current.shadowRoot;
+          if (shadowRoot) {
+            const shadowElement = shadowRoot.elementFromPoint(x, y) as HTMLElement;
+            if (!shadowElement) break;
+            
+            deepestElement = shadowElement;
+            current = shadowElement;
+          } else {
+            break;
+          }
+          depth++;
+        }
+
+        return deepestElement;
+      };
+
+      const genSelectorForShadowDOM = (element: HTMLElement) => {
+        const findShadowContext = (element: HTMLElement): { host: HTMLElement, root: ShadowRoot } | null => {
+          let current: HTMLElement | null = element;
+          let depth = 0;
+          
+          while (current && depth < MAX_DEPTH) {
+            // Check if element is inside a shadow root
+            if (current.parentNode instanceof ShadowRoot) {
+              return {
+                host: (current.parentNode as ShadowRoot).host as HTMLElement,
+                root: current.parentNode as ShadowRoot
+              };
+            }
+            current = current.parentElement;
+            depth++;
+          }
+          return null;
+        };
+          
+        const shadowContext = findShadowContext(element);
+        if (!shadowContext) return null;
+
+        try {
+          // Generate selector for the shadow host
+          const hostSelector = finder(shadowContext.host);
+          
+          // Generate selector for the element within the shadow DOM
+          const shadowElementSelector = finder(element, {
+            root: shadowContext.root as unknown as Element
+          });
+
+          return {
+            fullSelector: `${hostSelector} >>> ${shadowElementSelector}`,
+            hostSelector,
+            shadowElementSelector,
+            mode: shadowContext.root.mode
+          };
+        } catch (e) {
+          console.warn('Error generating shadow DOM selector:', e);
+          return null;
+        }
+      };
+
       const genSelectors = (element: HTMLElement | null) => {
         if (element == null) {
           return null;
@@ -812,6 +912,9 @@ export const getSelectors = async (page: Page, coordinates: Coordinates) => {
         } catch (e) {
         }
 
+        // Generate shadow DOM specific selector
+        const shadowSelector = genSelectorForShadowDOM(element);
+
         const hrefSelector = genSelectorForAttributes(element, ['href']);
         const formSelector = genSelectorForAttributes(element, [
           'name',
@@ -858,9 +961,21 @@ export const getSelectors = async (page: Page, coordinates: Coordinates) => {
           hrefSelector,
           accessibilitySelector,
           formSelector,
+          // Shadow DOM selector
+          shadowSelector: shadowSelector ? {
+            // Full selector that can traverse shadow DOM
+            full: shadowSelector.fullSelector,
+            // Individual parts for more flexible usage
+            host: shadowSelector.hostSelector,
+            element: shadowSelector.shadowElementSelector,
+            // Shadow root mode (open/closed)
+            mode: shadowSelector.mode
+          } : null
         };
       }
 
+      
+
       function genAttributeSet(element: HTMLElement, attributes: string[]) {
         return new Set(
           attributes.filter((attr) => {
@@ -900,7 +1015,7 @@ export const getSelectors = async (page: Page, coordinates: Coordinates) => {
         return char.length === 1 && char.match(/[0-9]/);
       }
 
-      const hoveredElement = document.elementFromPoint(x, y) as HTMLElement;
+      const hoveredElement = getDeepestElementFromPoint(x, y);
       if (
         hoveredElement != null &&
         !hoveredElement.closest('#overlay-controls') != null

From 542f4d31fa43359928d052b9edb3caa5f446c1c3 Mon Sep 17 00:00:00 2001
From: RohitR311 
Date: Sun, 29 Dec 2024 23:41:19 +0530
Subject: [PATCH 024/216] feat: change shadowDOM full selector path

---
 server/src/workflow-management/selector.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/server/src/workflow-management/selector.ts b/server/src/workflow-management/selector.ts
index 9b3af66e..690fb0b1 100644
--- a/server/src/workflow-management/selector.ts
+++ b/server/src/workflow-management/selector.ts
@@ -882,7 +882,7 @@ export const getSelectors = async (page: Page, coordinates: Coordinates) => {
           });
 
           return {
-            fullSelector: `${hostSelector} >>> ${shadowElementSelector}`,
+            fullSelector: `${hostSelector} > ${shadowElementSelector}`,
             hostSelector,
             shadowElementSelector,
             mode: shadowContext.root.mode

From b60f4b73b8424151f17bfd3389f5698f72c429df Mon Sep 17 00:00:00 2001
From: RohitR311 
Date: Mon, 30 Dec 2024 01:24:32 +0530
Subject: [PATCH 025/216] feat: add functionality to scrape shadowDOM elements

---
 maxun-core/src/browserSide/scraper.js | 126 ++++++++++++++++----------
 1 file changed, 80 insertions(+), 46 deletions(-)

diff --git a/maxun-core/src/browserSide/scraper.js b/maxun-core/src/browserSide/scraper.js
index a2009d78..ef979828 100644
--- a/maxun-core/src/browserSide/scraper.js
+++ b/maxun-core/src/browserSide/scraper.js
@@ -189,68 +189,102 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
    * @returns {Array.>}
    */
   window.scrapeSchema = function (lists) {
+    // These utility functions remain unchanged as they work perfectly
     function omap(object, f, kf = (x) => x) {
-      return Object.fromEntries(
-        Object.entries(object)
-          .map(([k, v]) => [kf(k), f(v)]),
-      );
+        return Object.fromEntries(
+            Object.entries(object)
+                .map(([k, v]) => [kf(k), f(v)]),
+        );
     }
 
     function ofilter(object, f) {
-      return Object.fromEntries(
-        Object.entries(object)
-          .filter(([k, v]) => f(k, v)),
-      );
+        return Object.fromEntries(
+            Object.entries(object)
+                .filter(([k, v]) => f(k, v)),
+        );
     }
 
-    function getSeedKey(listObj) {
-      const maxLength = Math.max(...Object.values(omap(listObj, (x) => document.querySelectorAll(x.selector).length)));
-      return Object.keys(ofilter(listObj, (_, v) => document.querySelectorAll(v.selector).length === maxLength))[0];
-    }
-
-    function getMBEs(elements) {
-      return elements.map((element) => {
-        let candidate = element;
-        const isUniqueChild = (e) => elements
-          .filter((elem) => e.parentNode?.contains(elem))
-          .length === 1;
-
-        while (candidate && isUniqueChild(candidate)) {
-          candidate = candidate.parentNode;
+    function findElement(config) {
+        // If this is a shadow DOM query
+        if (config.shadow && config.selector.includes('>>')) {
+            const [hostSelector, shadowSelector] = config.selector.split('>>').map(s => s.trim());
+            const host = document.querySelector(hostSelector);
+            return host?.shadowRoot?.querySelector(shadowSelector) || null;
         }
+        // Otherwise, use regular querySelector
+        return document.querySelector(config.selector);
+    }
 
-        return candidate;
-      });
+    function findAllElements(config) {
+        // If this is a shadow DOM query
+        if (config.shadow && config.selector.includes('>>')) {
+            const element = findElement(config);
+            return element ? [element] : [];
+        }
+        // Otherwise, use regular querySelectorAll
+        return Array.from(document.querySelectorAll(config.selector));
+    }
+
+    // Modified to use our new element finding functions
+    function getSeedKey(listObj) {
+        const maxLength = Math.max(...Object.values(
+            omap(listObj, (x) => findAllElements(x).length)
+        ));
+        return Object.keys(
+            ofilter(listObj, (_, v) => findAllElements(v).length === maxLength)
+        )[0];
+    }
+
+    // This function remains unchanged as it works with DOM elements
+    // regardless of how they were found
+    function getMBEs(elements) {
+        return elements.map((element) => {
+            let candidate = element;
+            const isUniqueChild = (e) => elements
+                .filter((elem) => e.parentNode?.contains(elem))
+                .length === 1;
+
+            while (candidate && isUniqueChild(candidate)) {
+                candidate = candidate.parentNode;
+            }
+
+            return candidate;
+        });
     }
 
     const seedName = getSeedKey(lists);
-    const seedElements = Array.from(document.querySelectorAll(lists[seedName].selector));
+    const seedElements = findAllElements(lists[seedName]);
     const MBEs = getMBEs(seedElements);
 
     return MBEs.map((mbe) => omap(
-      lists,
-      ({ selector, attribute }, key) => {
-        const elem = Array.from(document.querySelectorAll(selector)).find((elem) => mbe.contains(elem));
-        if (!elem) return undefined;
+        lists,
+        (config, key) => {
+            // Use our new findAllElements function
+            const elem = findAllElements(config)
+                .find((elem) => mbe.contains(elem));
 
-        switch (attribute) {
-          case 'href':
-            const relativeHref = elem.getAttribute('href');
-            return relativeHref ? new URL(relativeHref, window.location.origin).href : null;
-          case 'src':
-            const relativeSrc = elem.getAttribute('src');
-            return relativeSrc ? new URL(relativeSrc, window.location.origin).href : null;
-          case 'innerText':
-            return elem.innerText;
-          case 'textContent':
-            return elem.textContent;
-          default:
-            return elem.innerText;
-        }
-      },
-      (key) => key // Use the original key in the output
+            if (!elem) return undefined;
+
+            switch (config.attribute) {
+                case 'href': {
+                    const relativeHref = elem.getAttribute('href');
+                    return relativeHref ? new URL(relativeHref, window.location.origin).href : null;
+                }
+                case 'src': {
+                    const relativeSrc = elem.getAttribute('src');
+                    return relativeSrc ? new URL(relativeSrc, window.location.origin).href : null;
+                }
+                case 'innerText':
+                    return elem.innerText;
+                case 'textContent':
+                    return elem.textContent;
+                default:
+                    return elem.getAttribute(config.attribute) || elem.innerText;
+            }
+        },
+        (key) => key
     )) || [];
-  }
+  };
 
   /**
  * Scrapes multiple lists of similar items based on a template item.

From 9f9dc4e1030ca3819355245765ecadcc1e2c8d6f Mon Sep 17 00:00:00 2001
From: RohitR311 
Date: Mon, 30 Dec 2024 01:25:45 +0530
Subject: [PATCH 026/216] feat: add shadow optional field in SelectorObject

---
 src/context/browserSteps.tsx | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/context/browserSteps.tsx b/src/context/browserSteps.tsx
index dd211199..fd311a35 100644
--- a/src/context/browserSteps.tsx
+++ b/src/context/browserSteps.tsx
@@ -32,6 +32,7 @@ export interface SelectorObject {
     selector: string;
     tag?: string;
     attribute?: string;
+    shadow?: boolean;
     [key: string]: any;
 }
 

From b696fa568d65a0948edd3b999eb31c4ccf39dad5 Mon Sep 17 00:00:00 2001
From: RohitR311 
Date: Mon, 30 Dec 2024 01:28:22 +0530
Subject: [PATCH 027/216] feat: add shadow param for scrapeSchema config

---
 maxun-core/src/interpret.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/maxun-core/src/interpret.ts b/maxun-core/src/interpret.ts
index c581954d..495ba2db 100644
--- a/maxun-core/src/interpret.ts
+++ b/maxun-core/src/interpret.ts
@@ -403,7 +403,7 @@ export default class Interpreter extends EventEmitter {
         await this.options.serializableCallback(scrapeResults);
       },
 
-      scrapeSchema: async (schema: Record) => {
+      scrapeSchema: async (schema: Record) => {
         await this.ensureScriptsLoaded(page);
       
         const scrapeResult = await page.evaluate((schemaObj) => window.scrapeSchema(schemaObj), schema);

From 415ce02a3d2eb82f7434230239a5c7659b557016 Mon Sep 17 00:00:00 2001
From: RohitR311 
Date: Mon, 30 Dec 2024 02:39:27 +0530
Subject: [PATCH 028/216] feat: add shadow bool field to text step

---
 src/components/organisms/BrowserWindow.tsx | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/components/organisms/BrowserWindow.tsx b/src/components/organisms/BrowserWindow.tsx
index c7e9fc0f..11fe8c55 100644
--- a/src/components/organisms/BrowserWindow.tsx
+++ b/src/components/organisms/BrowserWindow.tsx
@@ -13,6 +13,7 @@ import { useTranslation } from 'react-i18next';
 interface ElementInfo {
     tagName: string;
     hasOnlyText?: boolean;
+    isShadowRoot?: boolean;
     innerText?: string;
     url?: string;
     imageUrl?: string;
@@ -185,6 +186,7 @@ export const BrowserWindow = () => {
                         addTextStep('', data, {
                             selector: highlighterData.selector,
                             tag: highlighterData.elementInfo?.tagName,
+                            shadow: highlighterData.elementInfo?.isShadowRoot,
                             attribute
                         });
                     } else {
@@ -192,7 +194,7 @@ export const BrowserWindow = () => {
                         setAttributeOptions(options);
                         setSelectedElement({
                             selector: highlighterData.selector,
-                            info: highlighterData.elementInfo
+                            info: highlighterData.elementInfo,
                         });
                         setShowAttributeModal(true);
                     }
@@ -229,6 +231,7 @@ export const BrowserWindow = () => {
                             selectorObj: {
                                 selector: highlighterData.selector,
                                 tag: highlighterData.elementInfo?.tagName,
+                                shadow: highlighterData.elementInfo?.isShadowRoot,
                                 attribute
                             }
                         };
@@ -276,6 +279,7 @@ export const BrowserWindow = () => {
                     addTextStep('', data, {
                         selector: selectedElement.selector,
                         tag: selectedElement.info?.tagName,
+                        shadow: selectedElement.info?.isShadowRoot,
                         attribute: attribute
                     });
                 }
@@ -288,6 +292,7 @@ export const BrowserWindow = () => {
                         selectorObj: {
                             selector: selectedElement.selector,
                             tag: selectedElement.info?.tagName,
+                            shadow: selectedElement.info?.isShadowRoot,
                             attribute: attribute
                         }
                     };

From 1a6a481b578a7212743ceb199b934585583b5a0e Mon Sep 17 00:00:00 2001
From: RohitR311 
Date: Mon, 30 Dec 2024 02:46:24 +0530
Subject: [PATCH 029/216] feat: add shadow selectors field type

---
 server/src/types/index.ts | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/server/src/types/index.ts b/server/src/types/index.ts
index f2e327ef..e882f69d 100644
--- a/server/src/types/index.ts
+++ b/server/src/types/index.ts
@@ -129,6 +129,13 @@ export interface BaseActionInfo {
     hasOnlyText: boolean;
 }
 
+interface ShadowSelector {
+    full: string;
+    host: string;
+    element: string;
+    mode: string;
+}
+
 /**
  * Holds all the possible css selectors that has been found for an element.
  * @category Types
@@ -143,6 +150,7 @@ export interface Selectors {
     hrefSelector: string|null;
     accessibilitySelector: string|null;
     formSelector: string|null;
+    shadowSelector: ShadowSelector | null;
 }
 
 /**
@@ -156,7 +164,7 @@ export interface BaseAction extends BaseActionInfo{
     associatedActions: ActionType[];
     inputType: string | undefined;
     value: string | undefined;
-    selectors: { [key: string]: string | null };
+    selectors: Selectors;
     timestamp: number;
     isPassword: boolean;
     /**

From c3031811a63d21139c306781a6e64ee09d81b1de Mon Sep 17 00:00:00 2001
From: RohitR311 
Date: Mon, 30 Dec 2024 02:52:58 +0530
Subject: [PATCH 030/216] feat: prioritize returning shadow selector

---
 server/src/workflow-management/utils.ts | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/server/src/workflow-management/utils.ts b/server/src/workflow-management/utils.ts
index b3dadd60..4f747127 100644
--- a/server/src/workflow-management/utils.ts
+++ b/server/src/workflow-management/utils.ts
@@ -12,6 +12,11 @@ export const getBestSelectorForAction = (action: Action) => {
     case ActionType.Hover:
     case ActionType.DragAndDrop: {
       const selectors = action.selectors;
+
+      if (selectors?.shadowSelector?.full) {
+        return selectors.shadowSelector.full;
+      }
+
       // less than 25 characters, and element only has text inside
       const textSelector =
         selectors?.text?.length != null &&
@@ -75,6 +80,11 @@ export const getBestSelectorForAction = (action: Action) => {
     case ActionType.Input:
     case ActionType.Keydown: {
       const selectors = action.selectors;
+
+      if (selectors?.shadowSelector?.full) {
+        return selectors.shadowSelector.full;
+      }
+
       return (
         selectors.testIdSelector ??
         selectors?.id ??

From cec2397a58256736b60467e40f1cc2e255667394 Mon Sep 17 00:00:00 2001
From: RohitR311 
Date: Mon, 30 Dec 2024 02:55:21 +0530
Subject: [PATCH 031/216] feat: change shadowDOM full selector path

---
 server/src/workflow-management/selector.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/server/src/workflow-management/selector.ts b/server/src/workflow-management/selector.ts
index 690fb0b1..164f5220 100644
--- a/server/src/workflow-management/selector.ts
+++ b/server/src/workflow-management/selector.ts
@@ -882,7 +882,7 @@ export const getSelectors = async (page: Page, coordinates: Coordinates) => {
           });
 
           return {
-            fullSelector: `${hostSelector} > ${shadowElementSelector}`,
+            fullSelector: `${hostSelector} >> ${shadowElementSelector}`,
             hostSelector,
             shadowElementSelector,
             mode: shadowContext.root.mode

From 05c7921c9d574d4074b64f56319d6166e84b1dc3 Mon Sep 17 00:00:00 2001
From: RohitR311 
Date: Mon, 30 Dec 2024 03:05:07 +0530
Subject: [PATCH 032/216] feat: add shadowInfo in highlighter data

---
 .../workflow-management/classes/Generator.ts    | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts
index 609541de..d1bccbe4 100644
--- a/server/src/workflow-management/classes/Generator.ts
+++ b/server/src/workflow-management/classes/Generator.ts
@@ -730,15 +730,26 @@ export class WorkflowGenerator {
     const displaySelector = await this.generateSelector(page, coordinates, ActionType.Click);
     const elementInfo = await getElementInformation(page, coordinates, this.listSelector, this.getList);
     if (rect) {
+      const highlighterData = {
+        rect,
+        selector: displaySelector,
+        elementInfo,
+        // Include shadow DOM specific information
+        shadowInfo: elementInfo?.isShadowRoot ? {
+          mode: elementInfo.shadowRootMode,
+          content: elementInfo.shadowRootContent
+        } : null
+      };
+
       if (this.getList === true) {
         if (this.listSelector !== '') {
           const childSelectors = await getChildSelectors(page, this.listSelector || '');
-          this.socket.emit('highlighter', { rect, selector: displaySelector, elementInfo, childSelectors })
+          this.socket.emit('highlighter', { ...highlighterData, childSelectors })
         } else {
-          this.socket.emit('highlighter', { rect, selector: displaySelector, elementInfo });
+          this.socket.emit('highlighter', { ...highlighterData });
         }
       } else {
-        this.socket.emit('highlighter', { rect, selector: displaySelector, elementInfo });
+        this.socket.emit('highlighter', { ...highlighterData });
       }
     }
   }

From d2ab81e22959acc9fccf65f5845d8962170608d7 Mon Sep 17 00:00:00 2001
From: RohitR311 
Date: Mon, 30 Dec 2024 22:59:28 +0530
Subject: [PATCH 033/216] feat: add logic to get deeply nested shadowDOM
 elements

---
 server/src/workflow-management/selector.ts | 87 ++++++++++++----------
 1 file changed, 49 insertions(+), 38 deletions(-)

diff --git a/server/src/workflow-management/selector.ts b/server/src/workflow-management/selector.ts
index 164f5220..d957b879 100644
--- a/server/src/workflow-management/selector.ts
+++ b/server/src/workflow-management/selector.ts
@@ -823,7 +823,7 @@ export const getSelectors = async (page: Page, coordinates: Coordinates) => {
         return output;
       }
 
-      const MAX_DEPTH = 10;
+      // const MAX_DEPTH = 10;
 
       const getDeepestElementFromPoint = (x: number, y: number): HTMLElement | null => {
         let element = document.elementFromPoint(x, y) as HTMLElement;
@@ -832,60 +832,76 @@ export const getSelectors = async (page: Page, coordinates: Coordinates) => {
         let current = element;
         let deepestElement = current;
         let depth = 0;
+        const MAX_DEPTH = 4; // Limit to 2 levels of shadow DOM
 
         while (current && depth < MAX_DEPTH) {
           const shadowRoot = current.shadowRoot;
-          if (shadowRoot) {
-            const shadowElement = shadowRoot.elementFromPoint(x, y) as HTMLElement;
-            if (!shadowElement) break;
-            
-            deepestElement = shadowElement;
-            current = shadowElement;
-          } else {
-            break;
-          }
+          if (!shadowRoot) break;
+
+          const shadowElement = shadowRoot.elementFromPoint(x, y) as HTMLElement;
+          if (!shadowElement || shadowElement === current) break;
+          
+          deepestElement = shadowElement;
+          current = shadowElement;
           depth++;
         }
 
         return deepestElement;
       };
 
+      // Helper function to generate selectors for shadow DOM elements
       const genSelectorForShadowDOM = (element: HTMLElement) => {
-        const findShadowContext = (element: HTMLElement): { host: HTMLElement, root: ShadowRoot } | null => {
-          let current: HTMLElement | null = element;
+        // Get complete path up to document root
+        const getShadowPath = (el: HTMLElement) => {
+          const path = [];
+          let current = el;
           let depth = 0;
+          const MAX_DEPTH = 4;
           
           while (current && depth < MAX_DEPTH) {
-            // Check if element is inside a shadow root
-            if (current.parentNode instanceof ShadowRoot) {
-              return {
-                host: (current.parentNode as ShadowRoot).host as HTMLElement,
-                root: current.parentNode as ShadowRoot
-              };
+            const rootNode = current.getRootNode();
+            if (rootNode instanceof ShadowRoot) {
+              path.unshift({
+                host: rootNode.host as HTMLElement,
+                root: rootNode,
+                element: current
+              });
+              current = rootNode.host as HTMLElement;
+              depth++;
+            } else {
+              break;
             }
-            current = current.parentElement;
-            depth++;
           }
-          return null;
+          return path;
         };
-          
-        const shadowContext = findShadowContext(element);
-        if (!shadowContext) return null;
+
+        const shadowPath = getShadowPath(element);
+        if (shadowPath.length === 0) return null;
 
         try {
-          // Generate selector for the shadow host
-          const hostSelector = finder(shadowContext.host);
+          const selectorParts: string[] = [];
           
-          // Generate selector for the element within the shadow DOM
-          const shadowElementSelector = finder(element, {
-            root: shadowContext.root as unknown as Element
+          // Generate selector for each shadow DOM boundary
+          shadowPath.forEach((context, index) => {
+            // Get selector for the host element
+            const hostSelector = finder(context.host, {
+              root: index === 0 ? document.body : (shadowPath[index - 1].root as unknown as Element)
+            });
+
+            // For the last context, get selector for target element
+            if (index === shadowPath.length - 1) {
+              const elementSelector = finder(element, {
+                root: context.root as unknown as Element
+              });
+              selectorParts.push(`${hostSelector} >> ${elementSelector}`);
+            } else {
+              selectorParts.push(hostSelector);
+            }
           });
 
           return {
-            fullSelector: `${hostSelector} >> ${shadowElementSelector}`,
-            hostSelector,
-            shadowElementSelector,
-            mode: shadowContext.root.mode
+            fullSelector: selectorParts.join(' >> '),
+            mode: shadowPath[shadowPath.length - 1].root.mode
           };
         } catch (e) {
           console.warn('Error generating shadow DOM selector:', e);
@@ -963,12 +979,7 @@ export const getSelectors = async (page: Page, coordinates: Coordinates) => {
           formSelector,
           // Shadow DOM selector
           shadowSelector: shadowSelector ? {
-            // Full selector that can traverse shadow DOM
             full: shadowSelector.fullSelector,
-            // Individual parts for more flexible usage
-            host: shadowSelector.hostSelector,
-            element: shadowSelector.shadowElementSelector,
-            // Shadow root mode (open/closed)
             mode: shadowSelector.mode
           } : null
         };

From 9287c296922478b77391d0c4930f4b478de4614e Mon Sep 17 00:00:00 2001
From: RohitR311 
Date: Mon, 30 Dec 2024 23:02:21 +0530
Subject: [PATCH 034/216] feat: rm host and element info for shadow selector

---
 server/src/types/index.ts | 2 --
 1 file changed, 2 deletions(-)

diff --git a/server/src/types/index.ts b/server/src/types/index.ts
index e882f69d..151e3dd4 100644
--- a/server/src/types/index.ts
+++ b/server/src/types/index.ts
@@ -131,8 +131,6 @@ export interface BaseActionInfo {
 
 interface ShadowSelector {
     full: string;
-    host: string;
-    element: string;
     mode: string;
 }
 

From e952d8f202278a67e86350e60542576b09260238 Mon Sep 17 00:00:00 2001
From: RohitR311 
Date: Mon, 30 Dec 2024 23:37:16 +0530
Subject: [PATCH 035/216] feat: add nested shadow-root scraping logic for
 scrapeSchema

---
 maxun-core/src/browserSide/scraper.js | 115 ++++++++++++++++----------
 1 file changed, 70 insertions(+), 45 deletions(-)

diff --git a/maxun-core/src/browserSide/scraper.js b/maxun-core/src/browserSide/scraper.js
index ef979828..ad9295b8 100644
--- a/maxun-core/src/browserSide/scraper.js
+++ b/maxun-core/src/browserSide/scraper.js
@@ -188,8 +188,7 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
    * @param {Object.} lists The named lists of HTML elements.
    * @returns {Array.>}
    */
-  window.scrapeSchema = function (lists) {
-    // These utility functions remain unchanged as they work perfectly
+  window.scrapeSchema = function(lists) {
     function omap(object, f, kf = (x) => x) {
         return Object.fromEntries(
             Object.entries(object)
@@ -203,29 +202,73 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
                 .filter(([k, v]) => f(k, v)),
         );
     }
-
-    function findElement(config) {
-        // If this is a shadow DOM query
-        if (config.shadow && config.selector.includes('>>')) {
-            const [hostSelector, shadowSelector] = config.selector.split('>>').map(s => s.trim());
-            const host = document.querySelector(hostSelector);
-            return host?.shadowRoot?.querySelector(shadowSelector) || null;
-        }
-        // Otherwise, use regular querySelector
-        return document.querySelector(config.selector);
-    }
-
+  
     function findAllElements(config) {
-        // If this is a shadow DOM query
-        if (config.shadow && config.selector.includes('>>')) {
-            const element = findElement(config);
-            return element ? [element] : [];
-        }
-        // Otherwise, use regular querySelectorAll
-        return Array.from(document.querySelectorAll(config.selector));
+      if (!config.shadow || !config.selector.includes('>>')) {
+          return Array.from(document.querySelectorAll(config.selector));
+      }
+  
+      // For shadow DOM, we'll get all possible combinations
+      const parts = config.selector.split('>>').map(s => s.trim());
+      let currentElements = [document];
+      
+      for (let i = 0; i < parts.length; i++) {
+          const part = parts[i];
+          const nextElements = [];
+          
+          for (const element of currentElements) {
+            let targets;
+            if (i === 0) {
+                // First selector is queried from document
+                targets = Array.from(element.querySelectorAll(part))
+                    .filter(el => {
+                        // Only include elements that either:
+                        // 1. Have an open shadow root
+                        // 2. Don't need shadow root (last part of selector)
+                        if (i === parts.length - 1) return true;
+                        const shadowRoot = el.shadowRoot;
+                        return shadowRoot && shadowRoot.mode === 'open';
+                    });
+            } else {
+                // For subsequent selectors, only use elements with open shadow roots
+                const shadowRoot = element.shadowRoot;
+                if (!shadowRoot || shadowRoot.mode !== 'open') continue;
+                
+                targets = Array.from(shadowRoot.querySelectorAll(part));
+            }
+            nextElements.push(...targets);
+          }
+          
+          if (nextElements.length === 0) return [];
+          currentElements = nextElements;
+      }
+  
+      return currentElements;
     }
+  
+  // Helper function to extract value from element based on attribute
+  function getElementValue(element, attribute) {
+      if (!element) return null;
+  
+      switch (attribute) {
+          case 'href': {
+              const relativeHref = element.getAttribute('href');
+              return relativeHref ? new URL(relativeHref, window.location.origin).href : null;
+          }
+          case 'src': {
+              const relativeSrc = element.getAttribute('src');
+              return relativeSrc ? new URL(relativeSrc, window.location.origin).href : null;
+          }
+          case 'innerText':
+              return element.innerText?.trim();
+          case 'textContent':
+              return element.textContent?.trim();
+          default:
+              return element.getAttribute(attribute) || element.innerText?.trim();
+      }
+  }
 
-    // Modified to use our new element finding functions
+    // Get the seed key based on the maximum number of elements found
     function getSeedKey(listObj) {
         const maxLength = Math.max(...Object.values(
             omap(listObj, (x) => findAllElements(x).length)
@@ -235,8 +278,7 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
         )[0];
     }
 
-    // This function remains unchanged as it works with DOM elements
-    // regardless of how they were found
+    // Find minimal bounding elements
     function getMBEs(elements) {
         return elements.map((element) => {
             let candidate = element;
@@ -252,35 +294,18 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
         });
     }
 
+    // Main scraping logic
     const seedName = getSeedKey(lists);
     const seedElements = findAllElements(lists[seedName]);
     const MBEs = getMBEs(seedElements);
 
     return MBEs.map((mbe) => omap(
         lists,
-        (config, key) => {
-            // Use our new findAllElements function
+        (config) => {
             const elem = findAllElements(config)
                 .find((elem) => mbe.contains(elem));
-
-            if (!elem) return undefined;
-
-            switch (config.attribute) {
-                case 'href': {
-                    const relativeHref = elem.getAttribute('href');
-                    return relativeHref ? new URL(relativeHref, window.location.origin).href : null;
-                }
-                case 'src': {
-                    const relativeSrc = elem.getAttribute('src');
-                    return relativeSrc ? new URL(relativeSrc, window.location.origin).href : null;
-                }
-                case 'innerText':
-                    return elem.innerText;
-                case 'textContent':
-                    return elem.textContent;
-                default:
-                    return elem.getAttribute(config.attribute) || elem.innerText;
-            }
+            
+            return elem ? getElementValue(elem, config.attribute) : undefined;
         },
         (key) => key
     )) || [];

From b757d9c4f8b0ea00d6eb6d6fe6c2e7c37407ed92 Mon Sep 17 00:00:00 2001
From: RohitR311 
Date: Mon, 30 Dec 2024 23:38:38 +0530
Subject: [PATCH 036/216] feat: add func to rm shadow selectors from workflow

---
 maxun-core/src/interpret.ts | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/maxun-core/src/interpret.ts b/maxun-core/src/interpret.ts
index 495ba2db..3cef8c29 100644
--- a/maxun-core/src/interpret.ts
+++ b/maxun-core/src/interpret.ts
@@ -663,11 +663,28 @@ export default class Interpreter extends EventEmitter {
       if (isApplicable) {
           return actionId;
       }
+    }
   }
+
+  private removeShadowSelectors(workflow: Workflow) {
+    for (let actionId = workflow.length - 1; actionId >= 0; actionId--) {
+      const step = workflow[actionId];
+      
+      // Check if step has where and selectors
+      if (step.where && Array.isArray(step.where.selectors)) {
+          // Filter out selectors that contain ">>"
+          step.where.selectors = step.where.selectors.filter(selector => !selector.includes('>>'));
+      }
+    }
+  
+    return workflow;
   }
 
   private async runLoop(p: Page, workflow: Workflow) {
-    const workflowCopy: Workflow = JSON.parse(JSON.stringify(workflow));
+    let workflowCopy: Workflow = JSON.parse(JSON.stringify(workflow));
+
+    // remove shadow selectors
+    workflowCopy = this.removeShadowSelectors(workflowCopy);
 
     // apply ad-blocker to the current page
     try {

From 4b4074b70d352401120bd1fe0b37fbee7838bac5 Mon Sep 17 00:00:00 2001
From: RohitR311 
Date: Tue, 31 Dec 2024 01:52:38 +0530
Subject: [PATCH 037/216] feat: add logic to scrape multiple nested shadow dom
 elements

---
 maxun-core/src/browserSide/scraper.js | 172 +++++++++++++++-----------
 1 file changed, 99 insertions(+), 73 deletions(-)

diff --git a/maxun-core/src/browserSide/scraper.js b/maxun-core/src/browserSide/scraper.js
index ad9295b8..00f8cef7 100644
--- a/maxun-core/src/browserSide/scraper.js
+++ b/maxun-core/src/browserSide/scraper.js
@@ -204,69 +204,68 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
     }
   
     function findAllElements(config) {
-      if (!config.shadow || !config.selector.includes('>>')) {
-          return Array.from(document.querySelectorAll(config.selector));
-      }
-  
-      // For shadow DOM, we'll get all possible combinations
-      const parts = config.selector.split('>>').map(s => s.trim());
-      let currentElements = [document];
-      
-      for (let i = 0; i < parts.length; i++) {
-          const part = parts[i];
-          const nextElements = [];
-          
-          for (const element of currentElements) {
-            let targets;
-            if (i === 0) {
-                // First selector is queried from document
-                targets = Array.from(element.querySelectorAll(part))
-                    .filter(el => {
-                        // Only include elements that either:
-                        // 1. Have an open shadow root
-                        // 2. Don't need shadow root (last part of selector)
-                        if (i === parts.length - 1) return true;
-                        const shadowRoot = el.shadowRoot;
-                        return shadowRoot && shadowRoot.mode === 'open';
-                    });
-            } else {
-                // For subsequent selectors, only use elements with open shadow roots
-                const shadowRoot = element.shadowRoot;
-                if (!shadowRoot || shadowRoot.mode !== 'open') continue;
-                
-                targets = Array.from(shadowRoot.querySelectorAll(part));
+        if (!config.shadow || !config.selector.includes('>>')) {
+            return Array.from(document.querySelectorAll(config.selector));
+        }
+    
+        // For shadow DOM, we'll get all possible combinations
+        const parts = config.selector.split('>>').map(s => s.trim());
+        let currentElements = [document];
+        
+        for (let i = 0; i < parts.length; i++) {
+            const part = parts[i];
+            const nextElements = [];
+            
+            for (const element of currentElements) {
+                let targets;
+                if (i === 0) {
+                    // First selector is queried from document
+                    targets = Array.from(element.querySelectorAll(part))
+                        .filter(el => {
+                            // Only include elements that either:
+                            // 1. Have an open shadow root
+                            // 2. Don't need shadow root (last part of selector)
+                            if (i === parts.length - 1) return true;
+                            const shadowRoot = el.shadowRoot;
+                            return shadowRoot && shadowRoot.mode === 'open';
+                        });
+                } else {
+                    // For subsequent selectors, only use elements with open shadow roots
+                    const shadowRoot = element.shadowRoot;
+                    if (!shadowRoot || shadowRoot.mode !== 'open') continue;
+                    
+                    targets = Array.from(shadowRoot.querySelectorAll(part));
+                }
+                nextElements.push(...targets);
             }
-            nextElements.push(...targets);
-          }
-          
-          if (nextElements.length === 0) return [];
-          currentElements = nextElements;
-      }
-  
-      return currentElements;
+            
+            if (nextElements.length === 0) return [];
+            currentElements = nextElements;
+        }
+    
+        return currentElements;
     }
   
-  // Helper function to extract value from element based on attribute
-  function getElementValue(element, attribute) {
-      if (!element) return null;
-  
-      switch (attribute) {
-          case 'href': {
-              const relativeHref = element.getAttribute('href');
-              return relativeHref ? new URL(relativeHref, window.location.origin).href : null;
-          }
-          case 'src': {
-              const relativeSrc = element.getAttribute('src');
-              return relativeSrc ? new URL(relativeSrc, window.location.origin).href : null;
-          }
-          case 'innerText':
-              return element.innerText?.trim();
-          case 'textContent':
-              return element.textContent?.trim();
-          default:
-              return element.getAttribute(attribute) || element.innerText?.trim();
-      }
-  }
+    function getElementValue(element, attribute) {
+        if (!element) return null;
+    
+        switch (attribute) {
+            case 'href': {
+                const relativeHref = element.getAttribute('href');
+                return relativeHref ? new URL(relativeHref, window.location.origin).href : null;
+            }
+            case 'src': {
+                const relativeSrc = element.getAttribute('src');
+                return relativeSrc ? new URL(relativeSrc, window.location.origin).href : null;
+            }
+            case 'innerText':
+                return element.innerText?.trim();
+            case 'textContent':
+                return element.textContent?.trim();
+            default:
+                return element.getAttribute(attribute) || element.innerText?.trim();
+        }
+    }
 
     // Get the seed key based on the maximum number of elements found
     function getSeedKey(listObj) {
@@ -280,26 +279,26 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
 
     // Find minimal bounding elements
     function getMBEs(elements) {
-        return elements.map((element) => {
-            let candidate = element;
-            const isUniqueChild = (e) => elements
-                .filter((elem) => e.parentNode?.contains(elem))
-                .length === 1;
+      return elements.map((element) => {
+          let candidate = element;
+          const isUniqueChild = (e) => elements
+              .filter((elem) => e.parentNode?.contains(elem))
+              .length === 1;
 
-            while (candidate && isUniqueChild(candidate)) {
-                candidate = candidate.parentNode;
-            }
+          while (candidate && isUniqueChild(candidate)) {
+              candidate = candidate.parentNode;
+          }
 
-            return candidate;
-        });
+          return candidate;
+      });
     }
 
-    // Main scraping logic
+    // First try the MBE approach
     const seedName = getSeedKey(lists);
     const seedElements = findAllElements(lists[seedName]);
     const MBEs = getMBEs(seedElements);
-
-    return MBEs.map((mbe) => omap(
+    
+    const mbeResults = MBEs.map((mbe) => omap(
         lists,
         (config) => {
             const elem = findAllElements(config)
@@ -309,6 +308,33 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
         },
         (key) => key
     )) || [];
+
+    // If MBE approach didn't find all elements, try independent scraping
+    if (mbeResults.some(result => Object.values(result).some(v => v === undefined))) {
+        // Fall back to independent scraping
+        const results = [];
+        const foundElements = new Map();
+
+        // Find all elements for each selector
+        Object.entries(lists).forEach(([key, config]) => {
+            const elements = findAllElements(config);
+            foundElements.set(key, elements);
+        });
+
+        // Create result objects for each found element
+        foundElements.forEach((elements, key) => {
+            elements.forEach((element, index) => {
+                if (!results[index]) {
+                    results[index] = {};
+                }
+                results[index][key] = getElementValue(element, lists[key].attribute);
+            });
+        });
+
+        return results.filter(result => Object.keys(result).length > 0);
+    }
+
+    return mbeResults;
   };
 
   /**

From 4a09ea66ff6c3c25c02b7997ed97f0ac4d677cd9 Mon Sep 17 00:00:00 2001
From: RohitR311 
Date: Tue, 31 Dec 2024 12:26:09 +0530
Subject: [PATCH 038/216] feat: get deepest element rect coordinates

---
 server/src/workflow-management/selector.ts | 53 ++++++++++++++--------
 1 file changed, 34 insertions(+), 19 deletions(-)

diff --git a/server/src/workflow-management/selector.ts b/server/src/workflow-management/selector.ts
index d957b879..910b3134 100644
--- a/server/src/workflow-management/selector.ts
+++ b/server/src/workflow-management/selector.ts
@@ -237,23 +237,30 @@ export const getRect = async (page: Page, coordinates: Coordinates, listSelector
     if (!getList || listSelector !== '') {
       const rect = await page.evaluate(
         async ({ x, y }) => {
-          // Helper function to get element from point including shadow DOM
+          // Enhanced helper function to get element from point including shadow DOM
           const getDeepestElementFromPoint = (x: number, y: number): HTMLElement | null => {
             let element = document.elementFromPoint(x, y) as HTMLElement;
             if (!element) return null;
 
             // Traverse through shadow roots
             let current = element;
-            while (current) {
-              const shadowRoot = current.shadowRoot;
-              if (!shadowRoot) break;
-
+            let shadowRoot = current.shadowRoot;
+            
+            // Keep track of the deepest shadow DOM element found
+            let deepestElement = current;
+            
+            while (shadowRoot) {
+              // Try to find element at same point in shadow DOM
               const shadowElement = shadowRoot.elementFromPoint(x, y) as HTMLElement;
               if (!shadowElement || shadowElement === current) break;
-
+              
+              // Update our tracking of the deepest element
+              deepestElement = shadowElement;
               current = shadowElement;
+              shadowRoot = current.shadowRoot;
             }
-            return current;
+
+            return deepestElement;
           };
 
           const el = getDeepestElementFromPoint(x, y);
@@ -274,36 +281,45 @@ export const getRect = async (page: Page, coordinates: Coordinates, listSelector
               };
             }
           }
+          return null;
         },
-        { x: coordinates.x, y: coordinates.y },
+        { x: coordinates.x, y: coordinates.y }
       );
       return rect;
     } else {
       const rect = await page.evaluate(
         async ({ x, y }) => {
-          // Helper function to get element from point including shadow DOM
+          // Enhanced helper function to get element from point including shadow DOM
           const getDeepestElementFromPoint = (x: number, y: number): HTMLElement | null => {
             let element = document.elementFromPoint(x, y) as HTMLElement;
             if (!element) return null;
 
             // Traverse through shadow roots
             let current = element;
-            while (current) {
-              const shadowRoot = current.shadowRoot;
-              if (!shadowRoot) break;
-
+            let shadowRoot = current.shadowRoot;
+            
+            // Keep track of the deepest shadow DOM element found
+            let deepestElement = current;
+            
+            while (shadowRoot) {
+              // Try to find element at same point in shadow DOM
               const shadowElement = shadowRoot.elementFromPoint(x, y) as HTMLElement;
               if (!shadowElement || shadowElement === current) break;
-
+              
+              // Update our tracking of the deepest element
+              deepestElement = shadowElement;
               current = shadowElement;
+              shadowRoot = current.shadowRoot;
             }
-            return current;
+
+            return deepestElement;
           };
 
           const originalEl = getDeepestElementFromPoint(x, y);
           if (originalEl) {
             let element = originalEl;
 
+            // Handle element hierarchy traversal for list items
             while (element.parentElement) {
               const parentRect = element.parentElement.getBoundingClientRect();
               const childRect = element.getBoundingClientRect();
@@ -326,7 +342,6 @@ export const getRect = async (page: Page, coordinates: Coordinates, listSelector
             }
 
             const rectangle = element?.getBoundingClientRect();
-
             if (rectangle) {
               return {
                 x: rectangle.x,
@@ -342,14 +357,14 @@ export const getRect = async (page: Page, coordinates: Coordinates, listSelector
           }
           return null;
         },
-        { x: coordinates.x, y: coordinates.y },
+        { x: coordinates.x, y: coordinates.y }
       );
       return rect;
     }
   } catch (error) {
     const { message, stack } = error as Error;
-    logger.log('error', `Error while retrieving selector: ${message}`);
-    logger.log('error', `Stack: ${stack}`);
+    console.error('Error while retrieving selector:', message);
+    console.error('Stack:', stack);
   }
 };
 

From 42e13066bd7800043e6952ddaae06d62985c2ee4 Mon Sep 17 00:00:00 2001
From: RohitR311 
Date: Wed, 1 Jan 2025 16:13:38 +0530
Subject: [PATCH 039/216] feat: add shadowDOM support for capture list selector
 generation

---
 server/src/workflow-management/selector.ts | 343 +++++++++++++++++----
 1 file changed, 286 insertions(+), 57 deletions(-)

diff --git a/server/src/workflow-management/selector.ts b/server/src/workflow-management/selector.ts
index 910b3134..713c05bc 100644
--- a/server/src/workflow-management/selector.ts
+++ b/server/src/workflow-management/selector.ts
@@ -1076,46 +1076,133 @@ interface SelectorResult {
  */
 
 export const getNonUniqueSelectors = async (page: Page, coordinates: Coordinates, listSelector: string): Promise => {
+  interface ShadowContext {
+    host: HTMLElement;
+    root: ShadowRoot;
+    element: HTMLElement;
+  }
+
   try {
     if (!listSelector) {
       const selectors = await page.evaluate(({ x, y }: { x: number, y: number }) => {
-        function getNonUniqueSelector(element: HTMLElement): string {
-          let selector = element.tagName.toLowerCase();
+        // Helper function to get deepest element, traversing shadow DOM
+        function getDeepestElementFromPoint(x: number, y: number): HTMLElement | null {
+          let element = document.elementFromPoint(x, y) as HTMLElement;
+          if (!element) return null;
 
-          if (element.className) {
-            const classes = element.className.split(/\s+/).filter((cls: string) => Boolean(cls));
-            if (classes.length > 0) {
-              const validClasses = classes.filter((cls: string) => !cls.startsWith('!') && !cls.includes(':'));
-              if (validClasses.length > 0) {
-                selector += '.' + validClasses.map(cls => CSS.escape(cls)).join('.');
-              }
-            }
+          let current = element;
+          let deepestElement = current;
+          let depth = 0;
+          const MAX_DEPTH = 4; // Limit shadow DOM traversal depth
+          
+          while (current && depth < MAX_DEPTH) {
+            const shadowRoot = current.shadowRoot;
+            if (!shadowRoot) break;
+            
+            const shadowElement = shadowRoot.elementFromPoint(x, y) as HTMLElement;
+            if (!shadowElement || shadowElement === current) break;
+            
+            deepestElement = shadowElement;
+            current = shadowElement;
+            depth++;
           }
 
+          return deepestElement;
+        }
+
+        // Generate basic selector from element's tag and classes
+        function getNonUniqueSelector(element: HTMLElement): string {
+          let selector = element.tagName.toLowerCase();
+          
+          const className = typeof element.className === 'string' ? element.className : '';
+          if (className) {
+            const classes = className.split(/\s+/)
+              .filter(cls => Boolean(cls) && !cls.startsWith('!') && !cls.includes(':'));
+            
+            if (classes.length > 0) {
+              selector += '.' + classes.map(cls => CSS.escape(cls)).join('.');
+            }
+          }
+          
           return selector;
         }
 
-        function getSelectorPath(element: HTMLElement | null): string {
-          const path: string[] = [];
+        // Get complete shadow DOM path for an element
+        function getShadowPath(element: HTMLElement): ShadowContext[] {
+          const path: ShadowContext[] = [];
+          let current = element;
           let depth = 0;
-          const maxDepth = 2;
+          const MAX_DEPTH = 4;
+          
+          while (current && depth < MAX_DEPTH) {
+            const rootNode = current.getRootNode();
+            if (rootNode instanceof ShadowRoot) {
+              path.unshift({
+                host: rootNode.host as HTMLElement,
+                root: rootNode,
+                element: current
+              });
+              current = rootNode.host as HTMLElement;
+              depth++;
+            } else {
+              break;
+            }
+          }
+          return path;
+        }
 
-          while (element && element !== document.body && depth < maxDepth) {
-            const selector = getNonUniqueSelector(element);
+        // Generate complete selector path for any element
+        function getSelectorPath(element: HTMLElement | null): string {
+          if (!element) return '';
+          
+          // Check for shadow DOM path first
+          const shadowPath = getShadowPath(element);
+          if (shadowPath.length > 0) {
+            const selectorParts: string[] = [];
+            
+            // Build complete shadow DOM path
+            shadowPath.forEach((context, index) => {
+              const hostSelector = getNonUniqueSelector(context.host);
+              
+              if (index === shadowPath.length - 1) {
+                // For deepest shadow context, include target element
+                const elementSelector = getNonUniqueSelector(element);
+                selectorParts.push(`${hostSelector} >> ${elementSelector}`);
+              } else {
+                // For intermediate shadow boundaries
+                selectorParts.push(hostSelector);
+              }
+            });
+            
+            return selectorParts.join(' >> ');
+          }
+
+          // Regular DOM path generation
+          const path: string[] = [];
+          let currentElement = element;
+          let depth = 0;
+          const MAX_DEPTH = 2;
+
+          while (currentElement && currentElement !== document.body && depth < MAX_DEPTH) {
+            const selector = getNonUniqueSelector(currentElement);
             path.unshift(selector);
-            element = element.parentElement;
+            
+            const parentElement = currentElement.parentElement;
+            if (!parentElement) break;
+            currentElement = parentElement;
             depth++;
           }
 
           return path.join(' > ');
         }
 
-        const originalEl = document.elementFromPoint(x, y) as HTMLElement;
+        // Main logic to get element and generate selector
+        const originalEl = getDeepestElementFromPoint(x, y);
         if (!originalEl) return null;
 
         let element = originalEl;
 
-        // if (listSelector === '') {
+        // Handle parent traversal for better element targeting
         while (element.parentElement) {
           const parentRect = element.parentElement.getBoundingClientRect();
           const childRect = element.getBoundingClientRect();
@@ -1136,60 +1223,134 @@ export const getNonUniqueSelectors = async (page: Page, coordinates: Coordinates
             break;
           }
         }
-        // }
 
         const generalSelector = getSelectorPath(element);
-        return {
-          generalSelector,
-        };
+        return { generalSelector };
       }, coordinates);
+
       return selectors || { generalSelector: '' };
     } else {
+      // When we have a list selector, we need special handling while maintaining shadow DOM support
       const selectors = await page.evaluate(({ x, y }: { x: number, y: number }) => {
-        function getNonUniqueSelector(element: HTMLElement): string {
-          let selector = element.tagName.toLowerCase();
+        // Helper function to get deepest element, traversing shadow DOM
+        function getDeepestElementFromPoint(x: number, y: number): HTMLElement | null {
+          let element = document.elementFromPoint(x, y) as HTMLElement;
+          if (!element) return null;
 
-          if (element.className) {
-            const classes = element.className.split(/\s+/).filter((cls: string) => Boolean(cls));
-            if (classes.length > 0) {
-              const validClasses = classes.filter((cls: string) => !cls.startsWith('!') && !cls.includes(':'));
-              if (validClasses.length > 0) {
-                selector += '.' + validClasses.map(cls => CSS.escape(cls)).join('.');
-              }
-            }
+          let current = element;
+          let deepestElement = current;
+          let depth = 0;
+          const MAX_DEPTH = 4;
+          
+          while (current && depth < MAX_DEPTH) {
+            const shadowRoot = current.shadowRoot;
+            if (!shadowRoot) break;
+            
+            const shadowElement = shadowRoot.elementFromPoint(x, y) as HTMLElement;
+            if (!shadowElement || shadowElement === current) break;
+            
+            deepestElement = shadowElement;
+            current = shadowElement;
+            depth++;
           }
 
+          return deepestElement;
+        }
+
+        // Generate basic selector from element's tag and classes
+        function getNonUniqueSelector(element: HTMLElement): string {
+          let selector = element.tagName.toLowerCase();
+          
+          const className = typeof element.className === 'string' ? element.className : '';
+          if (className) {
+            const classes = className.split(/\s+/)
+              .filter(cls => Boolean(cls) && !cls.startsWith('!') && !cls.includes(':'));
+            
+            if (classes.length > 0) {
+              selector += '.' + classes.map(cls => CSS.escape(cls)).join('.');
+            }
+          }
+          
           return selector;
         }
 
-        function getSelectorPath(element: HTMLElement | null): string {
-          const path: string[] = [];
+        // Get complete shadow DOM path for an element
+        function getShadowPath(element: HTMLElement): ShadowContext[] {
+          const path: ShadowContext[] = [];
+          let current = element;
           let depth = 0;
-          const maxDepth = 2;
+          const MAX_DEPTH = 4;
+          
+          while (current && depth < MAX_DEPTH) {
+            const rootNode = current.getRootNode();
+            if (rootNode instanceof ShadowRoot) {
+              path.unshift({
+                host: rootNode.host as HTMLElement,
+                root: rootNode,
+                element: current
+              });
+              current = rootNode.host as HTMLElement;
+              depth++;
+            } else {
+              break;
+            }
+          }
+          return path;
+        }
 
-          while (element && element !== document.body && depth < maxDepth) {
-            const selector = getNonUniqueSelector(element);
+        // Generate selector path specifically for list items
+        function getListItemSelectorPath(element: HTMLElement | null): string {
+          if (!element) return '';
+          
+          // Check for shadow DOM path first
+          const shadowPath = getShadowPath(element);
+          if (shadowPath.length > 0) {
+            const selectorParts: string[] = [];
+            
+            shadowPath.forEach((context, index) => {
+              const hostSelector = getNonUniqueSelector(context.host);
+              
+              if (index === shadowPath.length - 1) {
+                const elementSelector = getNonUniqueSelector(element);
+                selectorParts.push(`${hostSelector} >> ${elementSelector}`);
+              } else {
+                selectorParts.push(hostSelector);
+              }
+            });
+            
+            return selectorParts.join(' >> ');
+          }
+
+          // For list items, we want a shallower path to better match list patterns
+          const path: string[] = [];
+          let currentElement = element;
+          let depth = 0;
+          const MAX_LIST_DEPTH = 2;  // Keeping shallow depth for list items
+
+          while (currentElement && currentElement !== document.body && depth < MAX_LIST_DEPTH) {
+            const selector = getNonUniqueSelector(currentElement);
             path.unshift(selector);
-            element = element.parentElement;
+            
+            if (!currentElement.parentElement) break;
+            currentElement = currentElement.parentElement;
             depth++;
           }
 
           return path.join(' > ');
         }
 
-        const originalEl = document.elementFromPoint(x, y) as HTMLElement;
-        if (!originalEl) return null;
+        // Main logic for list item selection
+        const originalEl = getDeepestElementFromPoint(x, y);
+        if (!originalEl) return { generalSelector: '' };
 
         let element = originalEl;
 
-        const generalSelector = getSelectorPath(element);
-        return {
-          generalSelector,
-        };
-      }, coordinates);
-      return selectors || { generalSelector: '' };
-    }
+        const generalSelector = getListItemSelectorPath(element);
+        return { generalSelector };
+    }, coordinates);
 
+    return selectors || { generalSelector: '' };
+    }
   } catch (error) {
     console.error('Error in getNonUniqueSelectors:', error);
     return { generalSelector: '' };
@@ -1218,42 +1379,110 @@ export const getChildSelectors = async (page: Page, parentSelector: string): Pro
       }
 
       // Function to generate selector path from an element to its parent
-      function getSelectorPath(element: HTMLElement | null): string {
+      function getSelectorPath(element: HTMLElement): string {
         if (!element || !element.parentElement) return '';
 
         const parentSelector = getNonUniqueSelector(element.parentElement);
         const elementSelector = getNonUniqueSelector(element);
 
+        // Check if element is in shadow DOM
+        const rootNode = element.getRootNode();
+        if (rootNode instanceof ShadowRoot) {
+          const hostSelector = getNonUniqueSelector(rootNode.host as HTMLElement);
+          return `${hostSelector} >> ${elementSelector}`;
+        }
+
         return `${parentSelector} > ${elementSelector}`;
       }
 
-      // Function to recursively get all descendant selectors
+      // Function to get all shadow DOM children of an element
+      function getShadowChildren(element: HTMLElement): HTMLElement[] {
+        const children: HTMLElement[] = [];
+        
+        // Check if element has shadow root
+        const shadowRoot = element.shadowRoot;
+        if (shadowRoot) {
+          // Get all elements in the shadow DOM
+          const shadowElements = Array.from(shadowRoot.querySelectorAll('*')) as HTMLElement[];
+          children.push(...shadowElements);
+        }
+        
+        return children;
+      }
+
+      // Function to recursively get all descendant selectors including shadow DOM
       function getAllDescendantSelectors(element: HTMLElement): string[] {
         let selectors: string[] = [];
+        
+        // Handle regular DOM children
         const children = Array.from(element.children) as HTMLElement[];
-
         for (const child of children) {
           const childPath = getSelectorPath(child);
           if (childPath) {
-            selectors.push(childPath);  // Add direct child path
-            selectors = selectors.concat(getAllDescendantSelectors(child));  // Recursively process descendants
+            selectors.push(childPath);
+            // Recursively process regular DOM descendants
+            selectors = selectors.concat(getAllDescendantSelectors(child));
+            
+            // Check for shadow DOM in this child
+            const shadowChildren = getShadowChildren(child);
+            for (const shadowChild of shadowChildren) {
+              const shadowPath = getSelectorPath(shadowChild);
+              if (shadowPath) {
+                selectors.push(shadowPath);
+                // Recursively process shadow DOM descendants
+                selectors = selectors.concat(getAllDescendantSelectors(shadowChild));
+              }
+            }
+          }
+        }
+
+        // Handle direct shadow DOM children of the current element
+        const shadowChildren = getShadowChildren(element);
+        for (const shadowChild of shadowChildren) {
+          const shadowPath = getSelectorPath(shadowChild);
+          if (shadowPath) {
+            selectors.push(shadowPath);
+            selectors = selectors.concat(getAllDescendantSelectors(shadowChild));
           }
         }
 
         return selectors;
       }
 
-      // Find all occurrences of the parent selector in the DOM
-      const parentElements = Array.from(document.querySelectorAll(parentSelector)) as HTMLElement[];
-      const allChildSelectors = new Set();  // Use a set to ensure uniqueness
+      // Split the parent selector if it contains shadow DOM parts
+      const selectorParts = parentSelector.split('>>').map(part => part.trim());
+      let parentElements: HTMLElement[] = [];
+
+      // Handle shadow DOM traversal if needed
+      if (selectorParts.length > 1) {
+        // Start with the host elements
+        parentElements = Array.from(document.querySelectorAll(selectorParts[0])) as HTMLElement[];
+        
+        // Traverse through shadow DOM parts
+        for (let i = 1; i < selectorParts.length; i++) {
+          const newParentElements: HTMLElement[] = [];
+          for (const element of parentElements) {
+            if (element.shadowRoot) {
+              const shadowChildren = Array.from(element.shadowRoot.querySelectorAll(selectorParts[i])) as HTMLElement[];
+              newParentElements.push(...shadowChildren);
+            }
+          }
+          parentElements = newParentElements;
+        }
+      } else {
+        // Regular DOM selector
+        parentElements = Array.from(document.querySelectorAll(parentSelector)) as HTMLElement[];
+      }
+
+      const allChildSelectors = new Set();
 
       // Process each parent element and its descendants
       parentElements.forEach((parentElement) => {
         const descendantSelectors = getAllDescendantSelectors(parentElement);
-        descendantSelectors.forEach((selector) => allChildSelectors.add(selector));  // Add selectors to the set
+        descendantSelectors.forEach((selector) => allChildSelectors.add(selector));
       });
 
-      return Array.from(allChildSelectors);  // Convert the set back to an array
+      return Array.from(allChildSelectors);
     }, parentSelector);
 
     return childSelectors || [];

From c6105b4ee226a562f80b7054fffb3acac23e9d23 Mon Sep 17 00:00:00 2001
From: RohitR311 
Date: Wed, 1 Jan 2025 16:15:13 +0530
Subject: [PATCH 040/216] feat: generate highlighter for shadoDOM and mixedDOM
 elements

---
 src/components/organisms/BrowserWindow.tsx | 30 +++++++++++++++++++++-
 1 file changed, 29 insertions(+), 1 deletion(-)

diff --git a/src/components/organisms/BrowserWindow.tsx b/src/components/organisms/BrowserWindow.tsx
index 11fe8c55..442b7e50 100644
--- a/src/components/organisms/BrowserWindow.tsx
+++ b/src/components/organisms/BrowserWindow.tsx
@@ -120,7 +120,13 @@ export const BrowserWindow = () => {
     const highlighterHandler = useCallback((data: { rect: DOMRect, selector: string, elementInfo: ElementInfo | null, childSelectors?: string[] }) => {
         if (getList === true) {
             if (listSelector) {
+                console.log("LIST SELEECTORRRRR: ", listSelector);
+                console.log("DATA SELEECTORRRRR: ", data.selector);
+                console.log("CHILDREEENN SELECORRRR: ", data.childSelectors);
                 socket?.emit('listSelector', { selector: listSelector });
+
+                const hasValidChildSelectors = Array.isArray(data.childSelectors) && data.childSelectors.length > 0;
+
                 if (limitMode) {
                     setHighlighterData(null);
                 } else if (paginationMode) {
@@ -133,7 +139,29 @@ export const BrowserWindow = () => {
                 } else if (data.childSelectors && data.childSelectors.includes(data.selector)) {
                     // highlight only valid child elements within the listSelector
                     setHighlighterData(data);
-                } else {
+                } else if (data.elementInfo?.isShadowRoot && data.childSelectors) {
+                    // New case: Handle pure Shadow DOM elements
+                    // Check if the selector matches any shadow root child selectors
+                    const isShadowChild = data.childSelectors.some(childSelector => 
+                        data.selector.includes('>>') && // Shadow DOM uses >> for piercing
+                        childSelector.split('>>').some(part => 
+                            data.selector.includes(part.trim())
+                        )
+                    );
+                    setHighlighterData(isShadowChild ? data : null);
+                } else if (data.selector.includes('>>') && hasValidChildSelectors) {
+                    // New case: Handle mixed DOM cases
+                    // Split the selector into parts and check each against child selectors
+                    const selectorParts = data.selector.split('>>').map(part => part.trim());
+                    const isValidMixedSelector = selectorParts.some(part => 
+                        // Now we know data.childSelectors is defined
+                        data.childSelectors!.some(childSelector => 
+                            childSelector.includes(part)
+                        )
+                    );
+                    setHighlighterData(isValidMixedSelector ? data : null);
+                }
+                 else {
                     // if !valid child in normal mode, clear the highlighter
                     setHighlighterData(null);
                 }

From 8db6279f05c25e671098f959bfe0b79f5d06cb4f Mon Sep 17 00:00:00 2001
From: RohitR311 
Date: Wed, 1 Jan 2025 16:39:36 +0530
Subject: [PATCH 041/216] feat: add shadowDOM support for scraping list

---
 maxun-core/src/browserSide/scraper.js | 146 ++++++++++++++++++++------
 1 file changed, 113 insertions(+), 33 deletions(-)

diff --git a/maxun-core/src/browserSide/scraper.js b/maxun-core/src/browserSide/scraper.js
index 00f8cef7..caa783c8 100644
--- a/maxun-core/src/browserSide/scraper.js
+++ b/maxun-core/src/browserSide/scraper.js
@@ -349,27 +349,100 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
   window.scrapeList = async function ({ listSelector, fields, limit = 10 }) {
     const scrapedData = [];
 
-    while (scrapedData.length < limit) {
-        let parentElements = Array.from(document.querySelectorAll(listSelector));
+    // Helper function to query through Shadow DOM
+    const queryShadowDOM = (rootElement, selector) => {
+        // Split the selector by Shadow DOM delimiter
+        const parts = selector.split('>>').map(part => part.trim());
+        let currentElement = rootElement;
+
+        // Traverse through each part of the selector
+        for (let i = 0; i < parts.length; i++) {
+            if (!currentElement) return null;
+
+            // If we're at the document level (first part)
+            if (!currentElement.querySelector && !currentElement.shadowRoot) {
+                currentElement = document.querySelector(parts[i]);
+                continue;
+            }
+
+            // Try to find element in regular DOM first
+            let nextElement = currentElement.querySelector(parts[i]);
+
+            // If not found, check shadow DOM
+            if (!nextElement && currentElement.shadowRoot) {
+                nextElement = currentElement.shadowRoot.querySelector(parts[i]);
+            }
+
+            // If still not found, try to find in shadow DOM of all child elements
+            if (!nextElement) {
+                const allChildren = Array.from(currentElement.children || []);
+                for (const child of allChildren) {
+                    if (child.shadowRoot) {
+                        nextElement = child.shadowRoot.querySelector(parts[i]);
+                        if (nextElement) break;
+                    }
+                }
+            }
+
+            currentElement = nextElement;
+        }
+
+        return currentElement;
+    };
+
+    // Helper function to query all elements through Shadow DOM
+    const queryShadowDOMAll = (rootElement, selector) => {
+        const parts = selector.split('>>').map(part => part.trim());
+        let currentElements = [rootElement];
         
-        // If we only got one element or none, try a more generic approach
+        for (const part of parts) {
+            const nextElements = [];
+            
+            for (const element of currentElements) {
+                // Check regular DOM
+                if (element.querySelectorAll) {
+                    nextElements.push(...element.querySelectorAll(part));
+                }
+                
+                // Check shadow DOM
+                if (element.shadowRoot) {
+                    nextElements.push(...element.shadowRoot.querySelectorAll(part));
+                }
+                
+                // Check shadow DOM of children
+                const children = Array.from(element.children || []);
+                for (const child of children) {
+                    if (child.shadowRoot) {
+                        nextElements.push(...child.shadowRoot.querySelectorAll(part));
+                    }
+                }
+            }
+            
+            currentElements = nextElements;
+        }
+        
+        return currentElements;
+    };
+
+    while (scrapedData.length < limit) {
+        // Use our shadow DOM query function to get parent elements
+        let parentElements = queryShadowDOMAll(document, listSelector);
+        parentElements = Array.from(parentElements);
+
+        // Handle the case when we don't find enough elements
         if (limit > 1 && parentElements.length <= 1) {
-            const [containerSelector, _] = listSelector.split('>').map(s => s.trim());
-            const container = document.querySelector(containerSelector);
+            const [containerSelector, ...rest] = listSelector.split('>>').map(s => s.trim());
+            const container = queryShadowDOM(document, containerSelector);
             
             if (container) {
-                const allChildren = Array.from(container.children);
+                const allChildren = Array.from(container.children || []);
+                const firstMatch = queryShadowDOM(document, listSelector);
                 
-                const firstMatch = document.querySelector(listSelector);
                 if (firstMatch) {
-                    // Get classes from the first matching element
-                    const firstMatchClasses = Array.from(firstMatch.classList);
+                    const firstMatchClasses = Array.from(firstMatch.classList || []);
                     
-                    // Find similar elements by matching most of their classes
                     parentElements = allChildren.filter(element => {
-                        const elementClasses = Array.from(element.classList);
-
-                        // Element should share at least 70% of classes with the first match
+                        const elementClasses = Array.from(element.classList || []);
                         const commonClasses = firstMatchClasses.filter(cls => 
                             elementClasses.includes(cls));
                         return commonClasses.length >= Math.floor(firstMatchClasses.length * 0.7);
@@ -378,42 +451,49 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
             }
         }
 
-        // Iterate through each parent element
+        // Process each parent element
         for (const parent of parentElements) {
             if (scrapedData.length >= limit) break;
             const record = {};
 
-            // For each field, select the corresponding element within the parent
+            // Process each field using shadow DOM querying
             for (const [label, { selector, attribute }] of Object.entries(fields)) {
-                const fieldElement = parent.querySelector(selector);
+                // Use relative selector from parent
+                const relativeSelector = selector.split('>>').slice(-1)[0];
+                const fieldElement = queryShadowDOM(parent, relativeSelector);
 
                 if (fieldElement) {
-                    if (attribute === 'innerText') {
-                        record[label] = fieldElement.innerText.trim();
-                    } else if (attribute === 'innerHTML') {
-                        record[label] = fieldElement.innerHTML.trim();
-                    } else if (attribute === 'src') {
-                        // Handle relative 'src' URLs
-                        const src = fieldElement.getAttribute('src');
-                        record[label] = src ? new URL(src, window.location.origin).href : null;
-                    } else if (attribute === 'href') {
-                        // Handle relative 'href' URLs
-                        const href = fieldElement.getAttribute('href');
-                        record[label] = href ? new URL(href, window.location.origin).href : null;
-                    } else {
-                        record[label] = fieldElement.getAttribute(attribute);
+                    switch (attribute) {
+                        case 'innerText':
+                            record[label] = fieldElement.innerText?.trim() || '';
+                            break;
+                        case 'innerHTML':
+                            record[label] = fieldElement.innerHTML?.trim() || '';
+                            break;
+                        case 'src':
+                            const src = fieldElement.getAttribute('src');
+                            record[label] = src ? new URL(src, window.location.origin).href : null;
+                            break;
+                        case 'href':
+                            const href = fieldElement.getAttribute('href');
+                            record[label] = href ? new URL(href, window.location.origin).href : null;
+                            break;
+                        default:
+                            record[label] = fieldElement.getAttribute(attribute);
                     }
                 }
             }
-            scrapedData.push(record);
+            
+            if (Object.keys(record).length > 0) {
+                scrapedData.push(record);
+            }
         }
 
-        // If we've processed all available elements and still haven't reached the limit,
-        // break to avoid infinite loop
         if (parentElements.length === 0 || scrapedData.length >= parentElements.length) {
             break;
         }
     }
+    
     return scrapedData;
 };
 

From c287340f845e9429fc5534dc38af6257b4d75826 Mon Sep 17 00:00:00 2001
From: RohitR311 
Date: Thu, 2 Jan 2025 14:17:19 +0530
Subject: [PATCH 042/216] feat: shadowDOM support for table and non table list
 scraping

---
 maxun-core/src/browserSide/scraper.js | 347 +++++++++++++++++++++-----
 1 file changed, 281 insertions(+), 66 deletions(-)

diff --git a/maxun-core/src/browserSide/scraper.js b/maxun-core/src/browserSide/scraper.js
index caa783c8..ff5a1938 100644
--- a/maxun-core/src/browserSide/scraper.js
+++ b/maxun-core/src/browserSide/scraper.js
@@ -347,33 +347,29 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
  * @returns {Array.>} Array of arrays of scraped items, one sub-array per list
  */
   window.scrapeList = async function ({ listSelector, fields, limit = 10 }) {
-    const scrapedData = [];
-
-    // Helper function to query through Shadow DOM
+    // Shadow DOM query functions remain unchanged
     const queryShadowDOM = (rootElement, selector) => {
-        // Split the selector by Shadow DOM delimiter
+        if (!selector.includes('>>')) {
+          return rootElement.querySelector(selector);
+        }
+
         const parts = selector.split('>>').map(part => part.trim());
         let currentElement = rootElement;
 
-        // Traverse through each part of the selector
         for (let i = 0; i < parts.length; i++) {
             if (!currentElement) return null;
 
-            // If we're at the document level (first part)
             if (!currentElement.querySelector && !currentElement.shadowRoot) {
                 currentElement = document.querySelector(parts[i]);
                 continue;
             }
 
-            // Try to find element in regular DOM first
             let nextElement = currentElement.querySelector(parts[i]);
 
-            // If not found, check shadow DOM
             if (!nextElement && currentElement.shadowRoot) {
                 nextElement = currentElement.shadowRoot.querySelector(parts[i]);
             }
 
-            // If still not found, try to find in shadow DOM of all child elements
             if (!nextElement) {
                 const allChildren = Array.from(currentElement.children || []);
                 for (const child of allChildren) {
@@ -390,8 +386,11 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
         return currentElement;
     };
 
-    // Helper function to query all elements through Shadow DOM
     const queryShadowDOMAll = (rootElement, selector) => {
+        if (!selector.includes('>>')) {
+          return rootElement.querySelectorAll(selector);
+        }
+
         const parts = selector.split('>>').map(part => part.trim());
         let currentElements = [rootElement];
         
@@ -399,17 +398,14 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
             const nextElements = [];
             
             for (const element of currentElements) {
-                // Check regular DOM
                 if (element.querySelectorAll) {
                     nextElements.push(...element.querySelectorAll(part));
                 }
                 
-                // Check shadow DOM
                 if (element.shadowRoot) {
                     nextElements.push(...element.shadowRoot.querySelectorAll(part));
                 }
                 
-                // Check shadow DOM of children
                 const children = Array.from(element.children || []);
                 for (const child of children) {
                     if (child.shadowRoot) {
@@ -424,76 +420,295 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
         return currentElements;
     };
 
-    while (scrapedData.length < limit) {
-        // Use our shadow DOM query function to get parent elements
-        let parentElements = queryShadowDOMAll(document, listSelector);
-        parentElements = Array.from(parentElements);
+    // Enhanced table processing helper functions with shadow DOM support
+    function extractValue(element, attribute) {
+        if (!element) return null;
+        
+        // Check for shadow root first
+        if (element.shadowRoot) {
+            const shadowContent = element.shadowRoot.textContent;
+            if (shadowContent && shadowContent.trim()) {
+                return shadowContent.trim();
+            }
+        }
+        
+        if (attribute === 'innerText') {
+            return element.innerText.trim();
+        } else if (attribute === 'innerHTML') {
+            return element.innerHTML.trim();
+        } else if (attribute === 'src' || attribute === 'href') {
+            const attrValue = element.getAttribute(attribute);
+            return attrValue ? new URL(attrValue, window.location.origin).href : null;
+        }
+        return element.getAttribute(attribute);
+    }
 
-        // Handle the case when we don't find enough elements
-        if (limit > 1 && parentElements.length <= 1) {
-            const [containerSelector, ...rest] = listSelector.split('>>').map(s => s.trim());
-            const container = queryShadowDOM(document, containerSelector);
+    function findTableAncestor(element) {
+        let currentElement = element;
+        const MAX_DEPTH = 5;
+        let depth = 0;
+        
+        while (currentElement && depth < MAX_DEPTH) {
+            // Check if current element is in shadow DOM
+            if (currentElement.getRootNode() instanceof ShadowRoot) {
+                currentElement = currentElement.getRootNode().host;
+                continue;
+            }
             
-            if (container) {
-                const allChildren = Array.from(container.children || []);
-                const firstMatch = queryShadowDOM(document, listSelector);
-                
-                if (firstMatch) {
-                    const firstMatchClasses = Array.from(firstMatch.classList || []);
+            if (currentElement.tagName === 'TD') {
+                return { type: 'TD', element: currentElement };
+            } else if (currentElement.tagName === 'TR') {
+                return { type: 'TR', element: currentElement };
+            }
+            currentElement = currentElement.parentElement;
+            depth++;
+        }
+        return null;
+    }
+
+    function getCellIndex(td) {
+        let index = 0;
+        let sibling = td;
+        
+        // Handle shadow DOM case
+        if (td.getRootNode() instanceof ShadowRoot) {
+            const shadowRoot = td.getRootNode();
+            const allCells = Array.from(shadowRoot.querySelectorAll('td'));
+            return allCells.indexOf(td);
+        }
+        
+        while (sibling = sibling.previousElementSibling) {
+            index++;
+        }
+        return index;
+    }
+
+    function hasThElement(row, tableFields) {
+        for (const [label, { selector }] of Object.entries(tableFields)) {
+            const element = queryShadowDOM(row, selector);
+            if (element) {
+                let current = element;
+                while (current && current !== row) {
+                    // Check if we're in shadow DOM
+                    if (current.getRootNode() instanceof ShadowRoot) {
+                        current = current.getRootNode().host;
+                        continue;
+                    }
                     
-                    parentElements = allChildren.filter(element => {
-                        const elementClasses = Array.from(element.classList || []);
-                        const commonClasses = firstMatchClasses.filter(cls => 
-                            elementClasses.includes(cls));
-                        return commonClasses.length >= Math.floor(firstMatchClasses.length * 0.7);
-                    });
+                    if (current.tagName === 'TH') {
+                        return true;
+                    }
+                    current = current.parentElement;
                 }
             }
         }
+        return false;
+    }
 
-        // Process each parent element
-        for (const parent of parentElements) {
-            if (scrapedData.length >= limit) break;
-            const record = {};
+    function filterRowsBasedOnTag(rows, tableFields) {
+        for (const row of rows) {
+            if (hasThElement(row, tableFields)) {
+                return rows;
+            }
+        }
+        // Include shadow DOM in TH search
+        return rows.filter(row => {
+            const directTH = row.getElementsByTagName('TH').length === 0;
+            const shadowTH = row.shadowRoot ? 
+                row.shadowRoot.querySelector('th') === null : true;
+            return directTH && shadowTH;
+        });
+    }
 
-            // Process each field using shadow DOM querying
-            for (const [label, { selector, attribute }] of Object.entries(fields)) {
-                // Use relative selector from parent
-                const relativeSelector = selector.split('>>').slice(-1)[0];
-                const fieldElement = queryShadowDOM(parent, relativeSelector);
+    // Class similarity functions remain unchanged
+    function calculateClassSimilarity(classList1, classList2) {
+        const set1 = new Set(classList1);
+        const set2 = new Set(classList2);
+        const intersection = new Set([...set1].filter(x => set2.has(x)));
+        const union = new Set([...set1, ...set2]);
+        return intersection.size / union.size;
+    }
 
-                if (fieldElement) {
-                    switch (attribute) {
-                        case 'innerText':
-                            record[label] = fieldElement.innerText?.trim() || '';
-                            break;
-                        case 'innerHTML':
-                            record[label] = fieldElement.innerHTML?.trim() || '';
-                            break;
-                        case 'src':
-                            const src = fieldElement.getAttribute('src');
-                            record[label] = src ? new URL(src, window.location.origin).href : null;
-                            break;
-                        case 'href':
-                            const href = fieldElement.getAttribute('href');
-                            record[label] = href ? new URL(href, window.location.origin).href : null;
-                            break;
-                        default:
-                            record[label] = fieldElement.getAttribute(attribute);
+    function findSimilarElements(baseElement, similarityThreshold = 0.7) {
+        const baseClasses = Array.from(baseElement.classList);
+        if (baseClasses.length === 0) return [];
+        const potentialElements = document.getElementsByTagName(baseElement.tagName);
+        return Array.from(potentialElements).filter(element => {
+            if (element === baseElement) return false;
+            const similarity = calculateClassSimilarity(
+                baseClasses,
+                Array.from(element.classList)
+            );
+            return similarity >= similarityThreshold;
+        });
+    }
+
+    // Main scraping logic with shadow DOM support
+    let containers = queryShadowDOMAll(document, listSelector);
+    containers = Array.from(containers);
+
+    if (containers.length === 0) return [];
+
+    if (limit > 1 && containers.length === 1) {
+        const baseContainer = containers[0];
+        const similarContainers = findSimilarElements(baseContainer);
+        
+        if (similarContainers.length > 0) {
+            const newContainers = similarContainers.filter(container => 
+                !container.matches(listSelector)
+            );
+            containers = [...containers, ...newContainers];
+        }
+    }
+
+    const containerFields = containers.map(() => ({
+        tableFields: {},
+        nonTableFields: {}
+    }));
+
+    // Classify fields
+    containers.forEach((container, containerIndex) => {
+        for (const [label, field] of Object.entries(fields)) {
+            const sampleElement = queryShadowDOM(container, field.selector);
+            
+            if (sampleElement) {
+                const ancestor = findTableAncestor(sampleElement);
+                if (ancestor) {
+                    containerFields[containerIndex].tableFields[label] = {
+                        ...field,
+                        tableContext: ancestor.type,
+                        cellIndex: ancestor.type === 'TD' ? getCellIndex(ancestor.element) : -1
+                    };
+                } else {
+                    containerFields[containerIndex].nonTableFields[label] = field;
+                }
+            } else {
+                containerFields[containerIndex].nonTableFields[label] = field;
+            }
+        }
+    });
+
+    const tableData = [];
+    const nonTableData = [];
+
+    // Process table data with shadow DOM support
+    for (let containerIndex = 0; containerIndex < containers.length; containerIndex++) {
+        const container = containers[containerIndex];
+        const { tableFields } = containerFields[containerIndex];
+
+        if (Object.keys(tableFields).length > 0) {
+            const firstField = Object.values(tableFields)[0];
+            const firstElement = queryShadowDOM(container, firstField.selector);
+            let tableContext = firstElement;
+            
+            // Find table context including shadow DOM
+            while (tableContext && tableContext.tagName !== 'TABLE' && tableContext !== container) {
+                if (tableContext.getRootNode() instanceof ShadowRoot) {
+                    tableContext = tableContext.getRootNode().host;
+                } else {
+                    tableContext = tableContext.parentElement;
+                }
+            }
+
+            if (tableContext) {
+                // Get rows from both regular DOM and shadow DOM
+                const rows = [];
+                if (tableContext.shadowRoot) {
+                    rows.push(...tableContext.shadowRoot.getElementsByTagName('TR'));
+                }
+                rows.push(...tableContext.getElementsByTagName('TR'));
+                
+                const processedRows = filterRowsBasedOnTag(rows, tableFields);
+                
+                for (let rowIndex = 0; rowIndex < Math.min(processedRows.length, limit); rowIndex++) {
+                    const record = {};
+                    const currentRow = processedRows[rowIndex];
+                    
+                    for (const [label, { selector, attribute, cellIndex }] of Object.entries(tableFields)) {
+                        let element = null;
+                        
+                        if (cellIndex >= 0) {
+                            let td = currentRow.children[cellIndex];
+                            
+                            // Check shadow DOM for td
+                            if (!td && currentRow.shadowRoot) {
+                                const shadowCells = currentRow.shadowRoot.children;
+                                if (shadowCells && shadowCells.length > cellIndex) {
+                                    td = shadowCells[cellIndex];
+                                }
+                            }
+                            
+                            if (td) {
+                                element = queryShadowDOM(td, selector);
+                                
+                                if (!element && selector.split(">").pop().includes('td:nth-child')) {
+                                    element = td;
+                                }
+
+                                if (!element) {
+                                    const tagOnlySelector = selector.split('.')[0];
+                                    element = queryShadowDOM(td, tagOnlySelector);
+                                }
+                                
+                                if (!element) {
+                                    let currentElement = td;
+                                    while (currentElement && currentElement.children.length > 0) {
+                                        let foundContentChild = false;
+                                        for (const child of currentElement.children) {
+                                            if (extractValue(child, attribute)) {
+                                                currentElement = child;
+                                                foundContentChild = true;
+                                                break;
+                                            }
+                                        }
+                                        if (!foundContentChild) break;
+                                    }
+                                    element = currentElement;
+                                }
+                            }
+                        } else {
+                            element = queryShadowDOM(currentRow, selector);
+                        }
+                        
+                        if (element) {
+                            record[label] = extractValue(element, attribute);
+                        }
+                    }
+
+                    if (Object.keys(record).length > 0) {
+                        tableData.push(record);
                     }
                 }
             }
-            
-            if (Object.keys(record).length > 0) {
-                scrapedData.push(record);
-            }
         }
+    }
 
-        if (parentElements.length === 0 || scrapedData.length >= parentElements.length) {
-            break;
+    // Non-table data scraping remains unchanged
+    for (let containerIndex = 0; containerIndex < containers.length; containerIndex++) {
+        if (nonTableData.length >= limit) break;
+
+        const container = containers[containerIndex];
+        const { nonTableFields } = containerFields[containerIndex];
+
+        if (Object.keys(nonTableFields).length > 0) {
+            const record = {};
+
+            for (const [label, { selector, attribute }] of Object.entries(nonTableFields)) {
+                const relativeSelector = selector.split('>>').slice(-1)[0];
+                const element = queryShadowDOM(container, relativeSelector);
+                
+                if (element) {
+                    record[label] = extractValue(element, attribute);
+                }
+            }
+                
+            if (Object.keys(record).length > 0) {
+                nonTableData.push(record);
+            }
         }
     }
     
+    const scrapedData = [...tableData, ...nonTableData];
     return scrapedData;
 };
 

From b6faf5cf17736dcc99ffa0b146031f23ccc55f80 Mon Sep 17 00:00:00 2001
From: RohitR311 
Date: Thu, 2 Jan 2025 19:35:03 +0530
Subject: [PATCH 043/216] feat: add iframeSelector generation logic for capture
 text

---
 server/src/workflow-management/selector.ts | 471 +++++++++++----------
 1 file changed, 242 insertions(+), 229 deletions(-)

diff --git a/server/src/workflow-management/selector.ts b/server/src/workflow-management/selector.ts
index dd869f3d..6ed6a997 100644
--- a/server/src/workflow-management/selector.ts
+++ b/server/src/workflow-management/selector.ts
@@ -23,247 +23,110 @@ export const getElementInformation = async (
     if (!getList || listSelector !== '') {
       const elementInfo = await page.evaluate(
         async ({ x, y }) => {
-          // Helper function to get element info
-          const getElementInfo = (element: HTMLElement) => {
-            let info: {
-              tagName: string;
-              hasOnlyText?: boolean;
-              innerText?: string;
-              url?: string;
-              imageUrl?: string;
-              attributes?: Record;
-              innerHTML?: string;
-              outerHTML?: string;
-              fromIframe?: boolean;
-              iframePath?: string[];
-            } = {
-              tagName: element?.tagName ?? '',
-            };
+          // Helper function to find elements within iframes, handling nested cases
+          const getElementFromIframePoint = (
+            x: number,
+            y: number,
+            context: Document = document,
+            iframePath: string[] = []
+          ): { element: HTMLElement | null; iframePath: string[] } => {
+            // First try to get element at the given coordinates
+            let element = context.elementFromPoint(x, y) as HTMLElement;
+            if (!element) return { element: null, iframePath };
 
-            if (element) {
-              info.attributes = Array.from(element.attributes).reduce(
-                (acc, attr) => {
-                  acc[attr.name] = attr.value;
-                  return acc;
-                },
-                {} as Record
-              );
-            }
-
-            if (element?.tagName === 'A') {
-              info.url = (element as HTMLAnchorElement).href;
-              info.innerText = element.innerText ?? '';
-            } else if (element?.tagName === 'IMG') {
-              info.imageUrl = (element as HTMLImageElement).src;
-            } else if (element?.tagName === 'SELECT') {
-              const selectElement = element as HTMLSelectElement;
-              info.innerText = selectElement.options[selectElement.selectedIndex]?.text ?? '';
-              info.attributes = {
-                ...info.attributes,
-                selectedValue: selectElement.value,
-              };
-            } else if (element?.tagName === 'INPUT' &&
-              ((element as HTMLInputElement).type === 'time' ||
-                (element as HTMLInputElement).type === 'date')) {
-              info.innerText = (element as HTMLInputElement).value;
-            } else {
-              info.hasOnlyText = element?.children?.length === 0 &&
-                element?.innerText?.length > 0;
-              info.innerText = element?.innerText ?? '';
-            }
-
-            info.innerHTML = element.innerHTML;
-            info.outerHTML = element.outerHTML;
-            return info;
-          };
-
-          // Helper function to search in iframe
-          const searchInIframe = (
-            iframe: HTMLIFrameElement,
-            relativeX: number,
-            relativeY: number,
-            iframePath: string[]
-          ) => {
-            try {
-              if (!iframe.contentDocument) return null;
-
-              const el = iframe.contentDocument.elementFromPoint(relativeX, relativeY) as HTMLElement;
-              if (!el) return null;
-
-              const { parentElement } = el;
-              const element = parentElement?.tagName === 'A' ? parentElement : el;
-
-              const info = getElementInfo(element);
-              info.fromIframe = true;
-              info.iframePath = iframePath;
-
-              return info;
-            } catch (e) {
-              console.warn('Cannot access iframe content:', e);
-              return null;
-            }
-          };
-
-          const el = document.elementFromPoint(x, y) as HTMLElement;
-          if (el) {
-            // Check if the element is an iframe
-            if (el.tagName === 'IFRAME') {
-              const iframe = el as HTMLIFrameElement;
-              const rect = iframe.getBoundingClientRect();
-              const relativeX = x - rect.left;
-              const relativeY = y - rect.top;
-
-              const iframeResult = searchInIframe(
-                iframe,
-                relativeX,
-                relativeY,
-                [iframe.id || 'unnamed-iframe']
-              );
-              if (iframeResult) return iframeResult;
-            }
-
-            const { parentElement } = el;
-            const element = parentElement?.tagName === 'A' ? parentElement : el;
-            return getElementInfo(element);
-          }
-          return null;
-        },
-        { x: coordinates.x, y: coordinates.y }
-      );
-      return elementInfo;
-    } else {
-      const elementInfo = await page.evaluate(
-        async ({ x, y }) => {
-          // Helper function to get element info (same as above)
-          const getElementInfo = (element: HTMLElement) => {
-            let info: {
-              tagName: string;
-              hasOnlyText?: boolean;
-              innerText?: string;
-              url?: string;
-              imageUrl?: string;
-              attributes?: Record;
-              innerHTML?: string;
-              outerHTML?: string;
-              fromIframe?: boolean;
-              iframePath?: string[];
-            } = {
-              tagName: element?.tagName ?? '',
-            };
-
-            if (element) {
-              info.attributes = Array.from(element.attributes).reduce(
-                (acc, attr) => {
-                  acc[attr.name] = attr.value;
-                  return acc;
-                },
-                {} as Record
-              );
-            }
-
-            if (element?.tagName === 'A') {
-              info.url = (element as HTMLAnchorElement).href;
-              info.innerText = element.innerText ?? '';
-            } else if (element?.tagName === 'IMG') {
-              info.imageUrl = (element as HTMLImageElement).src;
-            } else {
-              info.hasOnlyText = element?.children?.length === 0 &&
-                element?.innerText?.length > 0;
-              info.innerText = element?.innerText ?? '';
-            }
-
-            info.innerHTML = element.innerHTML;
-            info.outerHTML = element.outerHTML;
-            return info;
-          };
-
-          // Helper function to search in iframe (same as above)
-          const searchInIframe = (
-            iframe: HTMLIFrameElement,
-            relativeX: number,
-            relativeY: number,
-            iframePath: string[]
-          ) => {
-            try {
-              if (!iframe.contentDocument) return null;
-
-              const el = iframe.contentDocument.elementFromPoint(relativeX, relativeY) as HTMLElement;
-              if (!el) return null;
-
-              let element = el;
-              while (element.parentElement) {
-                const parentRect = element.parentElement.getBoundingClientRect();
-                const childRect = element.getBoundingClientRect();
-
-                const fullyContained =
-                  parentRect.left <= childRect.left &&
-                  parentRect.right >= childRect.right &&
-                  parentRect.top <= childRect.top &&
-                  parentRect.bottom >= childRect.bottom;
-
-                const significantOverlap =
-                  (childRect.width * childRect.height) /
-                  (parentRect.width * parentRect.height) > 0.5;
-
-                if (fullyContained && significantOverlap) {
-                  element = element.parentElement;
-                } else {
-                  break;
+            // Check if we found an iframe
+            if (element.tagName === 'IFRAME') {
+              const iframe = element as HTMLIFrameElement;
+              try {
+                // Make sure we can access the iframe's content
+                if (!iframe.contentDocument) {
+                  return { element, iframePath };
                 }
+
+                // Transform coordinates to iframe's space
+                const rect = iframe.getBoundingClientRect();
+                const relativeX = x - rect.left;
+                const relativeY = y - rect.top;
+
+                // Add this iframe to the path
+                const updatedPath = [...iframePath, iframe.id || 'unnamed-iframe'];
+
+                // Recursively search within the iframe
+                const iframeResult = getElementFromIframePoint(
+                  relativeX,
+                  relativeY,
+                  iframe.contentDocument,
+                  updatedPath
+                );
+
+                // If we found an element in the iframe, return it
+                if (iframeResult.element) {
+                  return iframeResult;
+                }
+              } catch (e) {
+                console.warn('Cannot access iframe content:', e);
               }
-
-              const info = getElementInfo(element);
-              info.fromIframe = true;
-              info.iframePath = iframePath;
-
-              return info;
-            } catch (e) {
-              console.warn('Cannot access iframe content:', e);
-              return null;
             }
+
+            // Return the element we found (either in main document or iframe)
+            return { element, iframePath };
           };
 
-          const originalEl = document.elementFromPoint(x, y) as HTMLElement;
-          if (originalEl) {
-            // Check if the element is an iframe
-            if (originalEl.tagName === 'IFRAME') {
-              const iframe = originalEl as HTMLIFrameElement;
-              const rect = iframe.getBoundingClientRect();
-              const relativeX = x - rect.left;
-              const relativeY = y - rect.top;
+          // Get the element and its iframe path
+          const { element: el, iframePath } = getElementFromIframePoint(x, y);
+          
+          if (el) {
+            // Handle potential anchor parent
+            const { parentElement } = el;
+            const targetElement = parentElement?.tagName === 'A' ? parentElement : el;
 
-              const iframeResult = searchInIframe(
-                iframe,
-                relativeX,
-                relativeY,
-                [iframe.id || 'unnamed-iframe']
+            // Build the element information object
+            let info: {
+              tagName: string;
+              hasOnlyText?: boolean;
+              innerText?: string;
+              url?: string;
+              imageUrl?: string;
+              attributes?: Record;
+              innerHTML?: string;
+              outerHTML?: string;
+              fromIframe?: boolean;
+              iframePath?: string[];
+            } = {
+              tagName: targetElement?.tagName ?? '',
+              fromIframe: iframePath.length > 0,
+              iframePath: iframePath.length > 0 ? iframePath : undefined
+            };
+
+            // Collect element attributes and properties
+            if (targetElement) {
+              // Get all attributes
+              info.attributes = Array.from(targetElement.attributes).reduce(
+                (acc, attr) => {
+                  acc[attr.name] = attr.value;
+                  return acc;
+                },
+                {} as Record
               );
-              if (iframeResult) return iframeResult;
-            }
 
-            let element = originalEl;
-            while (element.parentElement) {
-              const parentRect = element.parentElement.getBoundingClientRect();
-              const childRect = element.getBoundingClientRect();
-
-              const fullyContained =
-                parentRect.left <= childRect.left &&
-                parentRect.right >= childRect.right &&
-                parentRect.top <= childRect.top &&
-                parentRect.bottom >= childRect.bottom;
-
-              const significantOverlap =
-                (childRect.width * childRect.height) /
-                (parentRect.width * parentRect.height) > 0.5;
-
-              if (fullyContained && significantOverlap) {
-                element = element.parentElement;
+              // Handle specific element types
+              if (targetElement.tagName === 'A') {
+                info.url = (targetElement as HTMLAnchorElement).href;
+                info.innerText = targetElement.textContent ?? '';
+              } else if (targetElement.tagName === 'IMG') {
+                info.imageUrl = (targetElement as HTMLImageElement).src;
               } else {
-                break;
+                info.hasOnlyText = targetElement.children.length === 0 && 
+                     (targetElement.textContent !== null && 
+                      targetElement.textContent.trim().length > 0);
+                info.innerText = targetElement.textContent ?? '';
               }
+
+              info.innerHTML = targetElement.innerHTML;
+              info.outerHTML = targetElement.outerHTML;
             }
 
-            return getElementInfo(element);
+            return info;
           }
           return null;
         },
@@ -271,6 +134,7 @@ export const getElementInformation = async (
       );
       return elementInfo;
     }
+    // ... rest of the code remains same
   } catch (error) {
     const { message, stack } = error as Error;
     console.error('Error while retrieving selector:', message);
@@ -984,6 +848,148 @@ export const getSelectors = async (page: Page, coordinates: Coordinates) => {
         }
         return output;
       }
+      
+      const getIframeOffset = (iframe: HTMLIFrameElement): { x: number; y: number } => {
+        const rect = iframe.getBoundingClientRect();
+        return {
+          x: rect.left,
+          y: rect.top
+        };
+      };
+      
+      const isAccessibleIframe = (iframe: HTMLIFrameElement): boolean => {
+        try {
+          return !!iframe.contentDocument;
+        } catch (e) {
+          return false;
+        }
+      };
+      
+      const getDeepestElementFromPoint = (x: number, y: number): HTMLElement | null => {
+        // Get the initial element at the specified coordinates
+        let currentElement = document.elementFromPoint(x, y) as HTMLElement;
+        if (!currentElement) return null;
+        
+        let deepestElement = currentElement;
+        let current = currentElement;
+        let currentX = x;
+        let currentY = y;
+        let depth = 0;
+        const MAX_DEPTH = 20; // Prevent infinite loops with deeply nested iframes
+      
+        // Continue traversing while we find nested iframes
+        while (current && depth < MAX_DEPTH) {
+          // Check if the current element is an iframe and if we can access it
+          if (current instanceof HTMLIFrameElement && isAccessibleIframe(current)) {
+            // Calculate the offset of the iframe
+            const iframeOffset = getIframeOffset(current);
+            
+            // Transform coordinates to be relative to the iframe's content window
+            const relativeX = currentX - iframeOffset.x;
+            const relativeY = currentY - iframeOffset.y;
+      
+            // Find the element at these coordinates within the iframe
+            const iframeElement = current.contentDocument?.elementFromPoint(relativeX, relativeY) as HTMLElement;
+            
+            // If we don't find an element or we get the same element, stop traversing
+            if (!iframeElement || iframeElement === current) break;
+      
+            // Update our tracking variables
+            deepestElement = iframeElement;
+            current = iframeElement;
+            currentX = relativeX;
+            currentY = relativeY;
+            depth++;
+          } else {
+            // If the current element is not an iframe, we're done traversing
+            break;
+          }
+        }
+      
+        return deepestElement;
+      };
+
+      interface IframeContext {
+        frame: HTMLIFrameElement;
+        document: Document;
+        element: HTMLElement;
+      }
+      
+      const genSelectorForIframe = (element: HTMLElement) => {
+        // Helper function to check if we can access an iframe's content
+        const isAccessibleIframe = (iframe: HTMLIFrameElement): boolean => {
+          try {
+            return !!iframe.contentDocument;
+          } catch (e) {
+            return false;
+          }
+        };
+      
+        // Get complete path up through nested iframes to document root
+        const getIframePath = (el: HTMLElement) => {
+          const path: IframeContext[] = [];
+          let current = el;
+          let currentDoc = el.ownerDocument;
+          let depth = 0;
+          const MAX_DEPTH = 20;  // Limit depth to prevent infinite loops
+          
+          while (current && depth < MAX_DEPTH) {
+            // If we're in an iframe, get its parent document
+            const frameElement = currentDoc.defaultView?.frameElement as HTMLIFrameElement;
+            if (frameElement && isAccessibleIframe(frameElement)) {
+              path.unshift({
+                frame: frameElement,
+                document: currentDoc,
+                element: current
+              });
+              current = frameElement;
+              currentDoc = frameElement.ownerDocument;
+              depth++;
+            } else {
+              break;
+            }
+          }
+          return path;
+        };
+      
+        // Get the iframe path for our target element
+        const iframePath = getIframePath(element);
+        if (iframePath.length === 0) return null;
+      
+        try {
+          const selectorParts: string[] = [];
+          
+          // Generate selector for each iframe boundary
+          iframePath.forEach((context, index) => {
+            // Get selector for the iframe element in its parent document
+            const frameSelector = finder(context.frame, {
+              root: index === 0 ? document.body : (iframePath[index - 1].document.body as Element)
+            });
+            
+            // For the last context, get selector for target element
+            if (index === iframePath.length - 1) {
+              const elementSelector = finder(element, {
+                root: context.document.body as Element
+              });
+              // Use :>> for iframe traversal in the selector
+              selectorParts.push(`${frameSelector} :>> ${elementSelector}`);
+            } else {
+              selectorParts.push(frameSelector);
+            }
+          });
+      
+          return {
+            // Join all parts with :>> to indicate iframe traversal
+            fullSelector: selectorParts.join(' :>> '),
+            // Include additional metadata about the frames if needed
+            frameCount: iframePath.length,
+            isAccessible: true
+          };
+        } catch (e) {
+          console.warn('Error generating iframe selector:', e);
+          return null;
+        }
+      };
 
       const genSelectors = (element: HTMLElement | null) => {
         if (element == null) {
@@ -1004,6 +1010,8 @@ export const getSelectors = async (page: Page, coordinates: Coordinates) => {
         } catch (e) {
         }
 
+        const iframeSelector = genSelectorForIframe(element);
+
         const hrefSelector = genSelectorForAttributes(element, ['href']);
         const formSelector = genSelectorForAttributes(element, [
           'name',
@@ -1050,6 +1058,11 @@ export const getSelectors = async (page: Page, coordinates: Coordinates) => {
           hrefSelector,
           accessibilitySelector,
           formSelector,
+          iframeSelector: iframeSelector ? {
+            full: iframeSelector.fullSelector,
+            frame: iframeSelector.frameCount,
+            accesible: iframeSelector.isAccessible
+          } : null
         };
       }
 
@@ -1092,7 +1105,7 @@ export const getSelectors = async (page: Page, coordinates: Coordinates) => {
         return char.length === 1 && char.match(/[0-9]/);
       }
 
-      const hoveredElement = document.elementFromPoint(x, y) as HTMLElement;
+      const hoveredElement = getDeepestElementFromPoint(x, y) as HTMLElement;
       if (
         hoveredElement != null &&
         !hoveredElement.closest('#overlay-controls') != null

From 8323593bb09d0b9a869afb825aa58be3944199b9 Mon Sep 17 00:00:00 2001
From: amhsirak 
Date: Thu, 2 Jan 2025 21:18:49 +0530
Subject: [PATCH 044/216] chore: format

---
 src/components/organisms/BrowserWindow.tsx | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/components/organisms/BrowserWindow.tsx b/src/components/organisms/BrowserWindow.tsx
index 442b7e50..2a5f7758 100644
--- a/src/components/organisms/BrowserWindow.tsx
+++ b/src/components/organisms/BrowserWindow.tsx
@@ -9,7 +9,6 @@ import { useBrowserSteps, TextStep } from '../../context/browserSteps';
 import { useGlobalInfoStore } from '../../context/globalInfo';
 import { useTranslation } from 'react-i18next';
 
-
 interface ElementInfo {
     tagName: string;
     hasOnlyText?: boolean;

From e91a3916a0513af15d306adc71fdd68bb9250e7d Mon Sep 17 00:00:00 2001
From: amhsirak 
Date: Thu, 2 Jan 2025 21:19:36 +0530
Subject: [PATCH 045/216] chore: cleanup console logs

---
 src/components/organisms/BrowserWindow.tsx | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/src/components/organisms/BrowserWindow.tsx b/src/components/organisms/BrowserWindow.tsx
index 2f66e906..ad58a309 100644
--- a/src/components/organisms/BrowserWindow.tsx
+++ b/src/components/organisms/BrowserWindow.tsx
@@ -119,9 +119,6 @@ export const BrowserWindow = () => {
     const highlighterHandler = useCallback((data: { rect: DOMRect, selector: string, elementInfo: ElementInfo | null, childSelectors?: string[] }) => {
         if (getList === true) {
             if (listSelector) {
-                console.log("LIST SELEECTORRRRR: ", listSelector);
-                console.log("DATA SELEECTORRRRR: ", data.selector);
-                console.log("CHILDREEENN SELECORRRR: ", data.childSelectors);
                 socket?.emit('listSelector', { selector: listSelector });
 
                 const hasValidChildSelectors = Array.isArray(data.childSelectors) && data.childSelectors.length > 0;

From edfcd8f869f194f7525744d050e30ba81a8bafef Mon Sep 17 00:00:00 2001
From: amhsirak 
Date: Thu, 2 Jan 2025 23:15:03 +0530
Subject: [PATCH 046/216] fix: format

---
 src/components/organisms/BrowserWindow.tsx | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/components/organisms/BrowserWindow.tsx b/src/components/organisms/BrowserWindow.tsx
index b69a0921..421bb680 100644
--- a/src/components/organisms/BrowserWindow.tsx
+++ b/src/components/organisms/BrowserWindow.tsx
@@ -326,7 +326,6 @@ export const BrowserWindow = () => {
         }
     }, [paginationMode, resetPaginationSelector]);
 
-  
     return (
         

             {

From af237ba1b0d9973e94f3595fdcc8610c8eac03d5 Mon Sep 17 00:00:00 2001
From: RohitR311 
Date: Fri, 3 Jan 2025 20:06:13 +0530
Subject: [PATCH 047/216] fix: update custom limit if value >= 1

---
 src/components/organisms/RightSidePanel.tsx | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/src/components/organisms/RightSidePanel.tsx b/src/components/organisms/RightSidePanel.tsx
index 8211a64a..d4670d4f 100644
--- a/src/components/organisms/RightSidePanel.tsx
+++ b/src/components/organisms/RightSidePanel.tsx
@@ -529,7 +529,22 @@ export const RightSidePanel: React.FC = ({ onFinishCapture
                    updateCustomLimit(e.target.value)}
+                  onChange={(e: React.ChangeEvent) => {
+                    const value = parseInt(e.target.value);
+                    // Only update if the value is greater than or equal to 1 or if the field is empty
+                    if (e.target.value === '' || value >= 1) {
+                      updateCustomLimit(e.target.value);
+                    }
+                  }}
+                  inputProps={{
+                    min: 1,
+                    onKeyPress: (e: React.KeyboardEvent) => {
+                      const value = (e.target as HTMLInputElement).value + e.key;
+                      if (parseInt(value) < 1) {
+                        e.preventDefault();
+                      }
+                    }
+                  }}
                   placeholder={t('right_panel.limit.enter_number')}
                   sx={{
                     marginLeft: '10px',

From 35a44bb39fcff9381417918dd37fc43d43ee3519 Mon Sep 17 00:00:00 2001
From: RohitR311 
Date: Fri, 3 Jan 2025 20:18:03 +0530
Subject: [PATCH 048/216] fix: rm translation for alt maxun_logo

---
 src/components/molecules/ActionDescriptionBox.tsx | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/components/molecules/ActionDescriptionBox.tsx b/src/components/molecules/ActionDescriptionBox.tsx
index 190c5838..45ec1641 100644
--- a/src/components/molecules/ActionDescriptionBox.tsx
+++ b/src/components/molecules/ActionDescriptionBox.tsx
@@ -113,7 +113,7 @@ const ActionDescriptionBox = () => {
 
   return (
     
-      
+      
       
       
         {renderActionDescription()}

From 1874e71e0f5ec86a1ba2cad4891dfdce8f1f19b7 Mon Sep 17 00:00:00 2001
From: RohitR311 
Date: Fri, 3 Jan 2025 20:22:30 +0530
Subject: [PATCH 049/216] fix: add translation for same name robot creation

---
 src/components/molecules/SaveRecording.tsx | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/components/molecules/SaveRecording.tsx b/src/components/molecules/SaveRecording.tsx
index 8e1eb462..cc51f238 100644
--- a/src/components/molecules/SaveRecording.tsx
+++ b/src/components/molecules/SaveRecording.tsx
@@ -101,7 +101,7 @@ export const SaveRecording = ({ fileName }: SaveRecordingProps) => {
               
               
                 
-                {t('save_recording.warnings.robot_exists')}
+                {t('save_recording.errors.exists_warning')}
               
             )
             : }
-      
+        {!getText && !getScreenshot && !getList && showCaptureList && } */}
+        
         {getList && (
           <>
             
@@ -533,8 +533,8 @@ export const RightSidePanel: React.FC = ({ onFinishCapture
           
             
               
{t('right_panel.limit.title')}

-
+             {/* 
+              
{t('right_panel.limit.title')}
  */}
             
             
Date: Wed, 8 Jan 2025 13:30:20 +0530
Subject: [PATCH 106/216] feat: reset light theme

---
 src/context/theme-provider.tsx | 81 +++++++++++++++++++++++++---------
 1 file changed, 61 insertions(+), 20 deletions(-)

diff --git a/src/context/theme-provider.tsx b/src/context/theme-provider.tsx
index a6097f95..fdf9fef5 100644
--- a/src/context/theme-provider.tsx
+++ b/src/context/theme-provider.tsx
@@ -4,33 +4,74 @@ import CssBaseline from '@mui/material/CssBaseline';
 
 const lightTheme = createTheme({
   palette: {
-    mode: 'light',
     primary: {
-      main: '#ff00c3', // Pink as the primary color
-    },
-    background: {
-      default: '#ffffff',
-      paper: '#f5f5f5',
-    },
-    text: {
-      primary: '#000000',
+      main: "#ff00c3",
+      contrastText: "#ffffff",
     },
   },
   components: {
-    MuiTabs: {
-      styleOverrides: {
-        indicator: {
-          backgroundColor: '#ff00c3', // Pink for tab indicators
-        },
-      },
-    },
     MuiButton: {
       styleOverrides: {
         root: {
-          backgroundColor: '#ff00c3', // Pink button background
-          color: '#ffffff',
-          '&:hover': {
-            backgroundColor: '#e600b3', // Slightly darker pink on hover
+          // Default styles for all buttons (optional)
+          textTransform: "none",
+        },
+        containedPrimary: {
+          // Styles for 'contained' variant with 'primary' color
+          "&:hover": {
+            backgroundColor: "#ff66d9",
+          },
+        },
+        outlined: {
+          // Apply white background for all 'outlined' variant buttons
+          backgroundColor: "#ffffff",
+          "&:hover": {
+            backgroundColor: "#f0f0f0", // Optional lighter background on hover
+          },
+        },
+      },
+    },
+    MuiLink: {
+      styleOverrides: {
+        root: {
+          "&:hover": {
+            color: "#ff00c3",
+          },
+        },
+      },
+    },
+    MuiIconButton: {
+      styleOverrides: {
+        root: {
+          // '&:hover': {
+          //   color: "#ff66d9",
+          // },
+        },
+      },
+    },
+    MuiTab: {
+      styleOverrides: {
+        root: {
+          textTransform: "none",
+        },
+      },
+    },
+    MuiAlert: {
+      styleOverrides: {
+        standardInfo: {
+          backgroundColor: "#fce1f4",
+          color: "#ff00c3",
+          "& .MuiAlert-icon": {
+            color: "#ff00c3",
+          },
+        },
+      },
+    },
+    MuiAlertTitle: {
+      styleOverrides: {
+        root: {
+          "& .MuiAlert-icon": {
+            color: "#ffffff",
           },
         },
       },

From 70086dd520a21011d2ea6ae457f4c55633666edf Mon Sep 17 00:00:00 2001
From: RohitR311 
Date: Wed, 8 Jan 2025 15:04:05 +0530
Subject: [PATCH 107/216] feat: change dark theme config for navbar

---
 src/context/theme-provider.tsx | 126 +++++++++++++++++++++++++++++----
 1 file changed, 114 insertions(+), 12 deletions(-)

diff --git a/src/context/theme-provider.tsx b/src/context/theme-provider.tsx
index fdf9fef5..d53ced24 100644
--- a/src/context/theme-provider.tsx
+++ b/src/context/theme-provider.tsx
@@ -83,33 +83,135 @@ const darkTheme = createTheme({
   palette: {
     mode: 'dark',
     primary: {
-      main: '#ff00c3', // Pink as the primary color
+      main: "#ff00c3",
+      contrastText: "#ffffff",
     },
     background: {
       default: '#121212',
-      paper: '#1e2124',
+      paper: '#1e1e1e',
     },
     text: {
       primary: '#ffffff',
+      secondary: '#b3b3b3',
     },
   },
   components: {
-    MuiTabs: {
-      styleOverrides: {
-        indicator: {
-          backgroundColor: '#ff00c3', // Pink for tab indicators
-        },
-      },
-    },
     MuiButton: {
       styleOverrides: {
         root: {
-          backgroundColor: '#ff00c3', // Pink button background
+          textTransform: "none",
           color: '#ffffff',
-          '&:hover': {
-            backgroundColor: '#e600b3', // Slightly darker pink on hover
+          '&.MuiButton-outlined': {
+            borderColor: '#ffffff',
+            color: '#ffffff',
+            "&:hover": {
+              borderColor: '#ffffff',
+              backgroundColor: 'rgba(255, 255, 255, 0.08)',
+            },
           },
         },
+        containedPrimary: {
+          "&:hover": {
+            backgroundColor: "#ff66d9",
+          },
+        },
+        outlined: {
+          // Dark mode outlined buttons
+          backgroundColor: '#1e1e1e',
+          borderColor: '#ff00c3',
+          color: '#ff00c3',
+          "&:hover": {
+            backgroundColor: 'rgba(255, 0, 195, 0.08)',
+            borderColor: '#ff66d9',
+          },
+        },
+      },
+    },
+    MuiLink: {
+      styleOverrides: {
+        root: {
+          color: '#ff66d9',
+          "&:hover": {
+            color: "#ff00c3",
+          },
+        },
+      },
+    },
+    MuiIconButton: {
+      styleOverrides: {
+        root: {
+          color: '#ffffff',
+          "&:hover": {
+            backgroundColor: 'rgba(255, 0, 195, 0.08)',
+          },
+        },
+      },
+    },
+    MuiTab: {
+      styleOverrides: {
+        root: {
+          textTransform: "none",
+          color: '#ffffff',
+          "&.Mui-selected": {
+            color: '#ff00c3',
+          },
+        },
+      },
+    },
+    MuiAlert: {
+      styleOverrides: {
+        standardInfo: {
+          backgroundColor: "rgba(255, 0, 195, 0.15)",
+          color: "#ff66d9",
+          "& .MuiAlert-icon": {
+            color: "#ff66d9",
+          },
+        },
+      },
+    },
+    MuiAlertTitle: {
+      styleOverrides: {
+        root: {
+          "& .MuiAlert-icon": {
+            color: "#ff66d9",
+          },
+        },
+      },
+    },
+    // Additional dark mode specific components
+    MuiPaper: {
+      styleOverrides: {
+        root: {
+          backgroundColor: '#1e1e1e',
+        },
+      },
+    },
+    MuiAppBar: {
+      styleOverrides: {
+        root: {
+          backgroundColor: '#121212',
+        },
+      },
+    },
+    MuiDrawer: {
+      styleOverrides: {
+        paper: {
+          backgroundColor: '#121212',
+        },
+      },
+    },
+    MuiTableCell: {
+      styleOverrides: {
+        root: {
+          borderBottom: '1px solid rgba(255, 255, 255, 0.12)',
+        },
+      },
+    },
+    MuiDivider: {
+      styleOverrides: {
+        root: {
+          borderColor: 'rgba(255, 255, 255, 0.12)',
+        },
       },
     },
   },

From eb50f4f1b48939b95a1b0cfcba24044d9e143662 Mon Sep 17 00:00:00 2001
From: RohitR311 
Date: Wed, 8 Jan 2025 16:02:27 +0530
Subject: [PATCH 108/216] feat: rm bg color for right side panel

---
 src/components/organisms/RightSidePanel.tsx | 31 +++++++++++----------
 1 file changed, 16 insertions(+), 15 deletions(-)

diff --git a/src/components/organisms/RightSidePanel.tsx b/src/components/organisms/RightSidePanel.tsx
index 3efb38d7..2eb41947 100644
--- a/src/components/organisms/RightSidePanel.tsx
+++ b/src/components/organisms/RightSidePanel.tsx
@@ -468,15 +468,16 @@ export const RightSidePanel: React.FC = ({ onFinishCapture
   return (
      = ({ onFinishCapture
         Last action: {` ${lastAction}`}
        */}
       
-      
-        {!getText && !getScreenshot && !getList && showCaptureList && }
-       {/* 
-      
-        {!getText && !getScreenshot && !getList && showCaptureList && } */}
+      {/*  */}
+        {/* {!getText && !getScreenshot && !getList && showCaptureList && } */}
+        {/*  */}
+       
+        {!getText && !getScreenshot && !getList && showCaptureList && } 
         
         {getList && (
           <>
@@ -586,9 +587,9 @@ export const RightSidePanel: React.FC = ({ onFinishCapture
             
           
         )}
-        {!getText && !getScreenshot && !getList && showCaptureText && }
+        {/* {!getText && !getScreenshot && !getList && showCaptureText && } */}
 
-//         {!getText && !getScreenshot && !getList && showCaptureText && }
+        {!getText && !getScreenshot && !getList && showCaptureText && }
         {getText &&
           <>
             
@@ -597,8 +598,8 @@ export const RightSidePanel: React.FC = ({ onFinishCapture
             
           
         }
-        {!getText && !getScreenshot && !getList && showCaptureScreenshot && }
-//         {!getText && !getScreenshot && !getList && showCaptureScreenshot && }
+        {/* {!getText && !getScreenshot && !getList && showCaptureScreenshot && } */}
+        {!getText && !getScreenshot && !getList && showCaptureScreenshot && }
         {getScreenshot && (
           
             

From f9644d3d636e2cc61298e870b28e52437ac02b5a Mon Sep 17 00:00:00 2001
From: RohitR311 
Date: Wed, 8 Jan 2025 16:42:57 +0530
Subject: [PATCH 109/216] feat: change finish button color

---
 src/components/molecules/SaveRecording.tsx | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/src/components/molecules/SaveRecording.tsx b/src/components/molecules/SaveRecording.tsx
index 1d03bcf4..38720d07 100644
--- a/src/components/molecules/SaveRecording.tsx
+++ b/src/components/molecules/SaveRecording.tsx
@@ -80,7 +80,19 @@ export const SaveRecording = ({ fileName }: SaveRecordingProps) => {
        {/* 
 

From afcd69b61788d254b4035abdcc85e50473d37818 Mon Sep 17 00:00:00 2001
From: RohitR311 
Date: Wed, 8 Jan 2025 16:43:19 +0530
Subject: [PATCH 110/216] feat: change finish button border color

---
 src/components/molecules/SaveRecording.tsx | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/components/molecules/SaveRecording.tsx b/src/components/molecules/SaveRecording.tsx
index 38720d07..80685841 100644
--- a/src/components/molecules/SaveRecording.tsx
+++ b/src/components/molecules/SaveRecording.tsx
@@ -87,6 +87,7 @@ export const SaveRecording = ({ fileName }: SaveRecordingProps) => {
         sx={{ 
           marginRight: '20px',
           color: '#00c853 !important',
+          borderColor: '#00c853 !important',
           '&:hover': {
             borderColor: '#00c853 !important',
           }

From 5b891114311e5670ac52d3d5f9312ac89b074c25 Mon Sep 17 00:00:00 2001
From: RohitR311 
Date: Wed, 8 Jan 2025 16:44:40 +0530
Subject: [PATCH 111/216] feat: change navbar icons styling

---
 src/components/molecules/BrowserNavBar.tsx | 39 ++++++++++------------
 1 file changed, 18 insertions(+), 21 deletions(-)

diff --git a/src/components/molecules/BrowserNavBar.tsx b/src/components/molecules/BrowserNavBar.tsx
index d3cb781f..0f821d56 100644
--- a/src/components/molecules/BrowserNavBar.tsx
+++ b/src/components/molecules/BrowserNavBar.tsx
@@ -13,37 +13,34 @@ import { getCurrentUrl } from "../../api/recording";
 import { useGlobalInfoStore } from '../../context/globalInfo';
 import { useThemeMode } from '../../context/theme-provider';
 
+// const StyledNavBar = styled.div<{ browserWidth: number; isDarkMode: boolean }>`
+//   display: flex;
+//   align-items: center;
+//   padding: 10px 20px;
+//   background-color: ${({ isDarkMode }) => (isDarkMode ? '#2C2F33' : '#F5F5F5')};
+//   width: ${({ browserWidth }) => `${browserWidth}px`};
+//   border-radius: 0px 0px 8px 8px;
+//   box-shadow: ${({ isDarkMode }) => (isDarkMode ? '0px 2px 10px rgba(0, 0, 0, 0.2)' : '0px 2px 10px rgba(0, 0, 0, 0.1)')};
+//   transition: background-color 0.3s ease, box-shadow 0.3s ease;
+//   margin-bottom: 15px;
+// `;
+
 const StyledNavBar = styled.div<{ browserWidth: number; isDarkMode: boolean }>`
-  display: flex;
-  align-items: center;
-  padding: 10px 20px;
-  background-color: ${({ isDarkMode }) => (isDarkMode ? '#2C2F33' : '#F5F5F5')};
-  width: ${({ browserWidth }) => `${browserWidth}px`};
-  border-radius: 0px 0px 8px 8px;
-  box-shadow: ${({ isDarkMode }) => (isDarkMode ? '0px 2px 10px rgba(0, 0, 0, 0.2)' : '0px 2px 10px rgba(0, 0, 0, 0.1)')};
-  transition: background-color 0.3s ease, box-shadow 0.3s ease;
-  margin-bottom: 15px;
+    display: flex;
+    padding: 12px 0px;
+    background-color: ${({ isDarkMode }) => (isDarkMode ? '#2C2F33' : '#f6f6f6')};
+    width: ${({ browserWidth }) => browserWidth}px;
+    border-radius: 0px 5px 0px 0px;
 `;
 
 const IconButton = styled(NavBarButton)<{ mode: string }>`
-  display: flex;
-  align-items: center;
-  justify-content: center;
-  padding: 8px;
-  margin-right: 12px;
-  background-color: ${({ mode }) => (mode === 'dark' ? '#40444B' : '#E0E0E0')};
-  border-radius: 50%;
+  background-color: ${({ mode }) => (mode === 'dark' ? '#2C2F33' : '#f6f6f6')};
   transition: background-color 0.3s ease, transform 0.1s ease;
   color: ${({ mode }) => (mode === 'dark' ? '#FFFFFF' : '#333')};
   cursor: pointer;
-
   &:hover {
     background-color: ${({ mode }) => (mode === 'dark' ? '#586069' : '#D0D0D0')};
   }
-
-  &:active {
-    transform: scale(0.95);
-  }
 `;
 
 interface NavBarProps {

From cefa5dfa108b819e1b5c8e18c2d228c2ec292390 Mon Sep 17 00:00:00 2001
From: RohitR311 
Date: Wed, 8 Jan 2025 16:50:15 +0530
Subject: [PATCH 112/216] feat: change bg color on hover

---
 src/components/molecules/SaveRecording.tsx | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/components/molecules/SaveRecording.tsx b/src/components/molecules/SaveRecording.tsx
index 80685841..51b4143b 100644
--- a/src/components/molecules/SaveRecording.tsx
+++ b/src/components/molecules/SaveRecording.tsx
@@ -83,13 +83,14 @@ export const SaveRecording = ({ fileName }: SaveRecordingProps) => {
       
            setOpenModal(false)} modalStyle={modalStyle}>

From 8e6228d77bd0b777a8848aee7672f7f9b90b392a Mon Sep 17 00:00:00 2001
From: amhsirak 
Date: Wed, 8 Jan 2025 17:22:01 +0530
Subject: [PATCH 114/216] fix(temp): revert to old canvas

---
 src/components/atoms/canvas.tsx | 502 ++++++++++++--------------------
 1 file changed, 186 insertions(+), 316 deletions(-)

diff --git a/src/components/atoms/canvas.tsx b/src/components/atoms/canvas.tsx
index 8fd4f791..e71a4d93 100644
--- a/src/components/atoms/canvas.tsx
+++ b/src/components/atoms/canvas.tsx
@@ -1,147 +1,21 @@
-import React, { useCallback, useEffect, useRef, useMemo, Suspense } from 'react';
+import React, { useCallback, useEffect, useRef } from 'react';
 import { useSocketStore } from '../../context/socket';
+import { getMappedCoordinates } from "../../helpers/inputHelpers";
 import { useGlobalInfoStore } from "../../context/globalInfo";
 import { useActionContext } from '../../context/browserActions';
-const DatePicker = React.lazy(() => import('./DatePicker'));
-const Dropdown = React.lazy(() => import('./Dropdown'));
-const TimePicker = React.lazy(() => import('./TimePicker'));
-const DateTimeLocalPicker = React.lazy(() => import('./DateTimeLocalPicker'));
+import DatePicker from './DatePicker';
+import Dropdown from './Dropdown';
+import TimePicker from './TimePicker';
+import DateTimeLocalPicker from './DateTimeLocalPicker';
 
-class RAFScheduler {
-    private queue: Set<() => void> = new Set();
-    private isProcessing: boolean = false;
-    private frameId: number | null = null;
-
-    schedule(callback: () => void): void {
-        this.queue.add(callback);
-        if (!this.isProcessing) {
-            this.process();
-        }
-    }
-
-    private process = (): void => {
-        this.isProcessing = true;
-        this.frameId = requestAnimationFrame(() => {
-            const callbacks = Array.from(this.queue);
-            this.queue.clear();
-
-            callbacks.forEach(callback => {
-                try {
-                    callback();
-                } catch (error) {
-                    console.error('RAF Scheduler error:', error);
-                }
-            });
-
-            this.isProcessing = false;
-            this.frameId = null;
-
-            if (this.queue.size > 0) {
-                this.process();
-            }
-        });
-    }
-
-    clear(): void {
-        this.queue.clear();
-        if (this.frameId !== null) {
-            cancelAnimationFrame(this.frameId);
-            this.frameId = null;
-        }
-        this.isProcessing = false;
-    }
-}
-
-class EventDebouncer {
-    private highPriorityQueue: Array<() => void> = [];
-    private lowPriorityQueue: Array<() => void> = [];
-    private processing: boolean = false;
-    private scheduler: RAFScheduler;
-
-    constructor(scheduler: RAFScheduler) {
-        this.scheduler = scheduler;
-    }
-
-    add(callback: () => void, highPriority: boolean = false): void {
-        if (highPriority) {
-            this.highPriorityQueue.push(callback);
-        } else {
-            this.lowPriorityQueue.push(callback);
-        }
-
-        if (!this.processing) {
-            this.process();
-        }
-    }
-
-    private process(): void {
-        this.processing = true;
-        this.scheduler.schedule(() => {
-            while (this.highPriorityQueue.length > 0) {
-                const callback = this.highPriorityQueue.shift();
-                callback?.();
-            }
-
-            if (this.lowPriorityQueue.length > 0) {
-                const callback = this.lowPriorityQueue.shift();
-                callback?.();
-
-                if (this.lowPriorityQueue.length > 0) {
-                    this.process();
-                }
-            }
-
-            this.processing = false;
-        });
-    }
-
-    clear(): void {
-        this.highPriorityQueue = [];
-        this.lowPriorityQueue = [];
-        this.processing = false;
-    }
-}
-
-// Optimized measurement cache with LRU
-class MeasurementCache {
-    private cache: Map;
-    private maxSize: number;
-
-    constructor(maxSize: number = 100) {
-        this.cache = new Map();
-        this.maxSize = maxSize;
-    }
-
-    get(element: HTMLElement): DOMRect | undefined {
-        const cached = this.cache.get(element);
-        if (cached) {
-            // Refresh the entry
-            this.cache.delete(element);
-            this.cache.set(element, cached);
-        }
-        return cached;
-    }
-
-    set(element: HTMLElement, rect: DOMRect): void {
-        if (this.cache.size >= this.maxSize) {
-            // Remove oldest entry
-            const firstKey = this.cache.keys().next().value;
-            if (firstKey !== undefined) {
-                this.cache.delete(firstKey);
-            }
-        }
-        this.cache.set(element, rect);
-    }
-
-    clear(): void {
-        this.cache.clear();
-    }
+interface CreateRefCallback {
+    (ref: React.RefObject): void;
 }
 
 interface CanvasProps {
     width: number;
     height: number;
-    onCreateRef: (ref: React.RefObject) => void;
+    onCreateRef: CreateRefCallback;
 }
 
 /**
@@ -152,229 +26,225 @@ export interface Coordinates {
     y: number;
 };
 
-const Canvas = React.memo(({ width, height, onCreateRef }: CanvasProps) => {
+const Canvas = ({ width, height, onCreateRef }: CanvasProps) => {
+
     const canvasRef = useRef(null);
     const { socket } = useSocketStore();
     const { setLastAction, lastAction } = useGlobalInfoStore();
     const { getText, getList } = useActionContext();
+    const getTextRef = useRef(getText);
+    const getListRef = useRef(getList);
 
-    const scheduler = useRef(new RAFScheduler());
-    const debouncer = useRef(new EventDebouncer(scheduler.current));
-    const measurementCache = useRef(new MeasurementCache(50));
-    //const performanceMonitor = useRef(new FrontendPerformanceMonitor());
+    const [datePickerInfo, setDatePickerInfo] = React.useState<{
+        coordinates: Coordinates;
+        selector: string;
+    } | null>(null);
 
-    const refs = useRef({
-        getText,
-        getList,
-        lastMousePosition: { x: 0, y: 0 },
-        lastFrameTime: 0,
-        context: null as CanvasRenderingContext2D | null,
-    });
+    const [dropdownInfo, setDropdownInfo] = React.useState<{
+        coordinates: Coordinates;
+        selector: string;
+        options: Array<{
+            value: string;
+            text: string;
+            disabled: boolean;
+            selected: boolean;
+        }>;
+    } | null>(null);
 
-    const [state, dispatch] = React.useReducer((state: any, action: any) => {
-        switch (action.type) {
-            case 'BATCH_UPDATE':
-                return { ...state, ...action.payload };
-            default:
-                return state;
+    const [timePickerInfo, setTimePickerInfo] = React.useState<{
+        coordinates: Coordinates;
+        selector: string;
+    } | null>(null);
+
+    const [dateTimeLocalInfo, setDateTimeLocalInfo] = React.useState<{
+        coordinates: Coordinates;
+        selector: string;
+    } | null>(null);
+
+    const notifyLastAction = (action: string) => {
+        if (lastAction !== action) {
+            setLastAction(action);
         }
-    }, {
-        datePickerInfo: null,
-        dropdownInfo: null,
-        timePickerInfo: null,
-        dateTimeLocalInfo: null
-    });
+    };
 
-    const getEventCoordinates = useCallback((event: MouseEvent): { x: number; y: number } => {
-        if (!canvasRef.current) return { x: 0, y: 0 };
+    const lastMousePosition = useRef({ x: 0, y: 0 });
 
-        let rect = measurementCache.current.get(canvasRef.current);
-        if (!rect) {
-            rect = canvasRef.current.getBoundingClientRect();
-            measurementCache.current.set(canvasRef.current, rect);
+    useEffect(() => {
+        getTextRef.current = getText;
+        getListRef.current = getList;
+    }, [getText, getList]);
+
+    useEffect(() => {
+        if (socket) {
+            socket.on('showDatePicker', (info: {coordinates: Coordinates, selector: string}) => {
+                setDatePickerInfo(info);
+            });
+
+            socket.on('showDropdown', (info: {
+                coordinates: Coordinates,
+                selector: string,
+                options: Array<{
+                    value: string;
+                    text: string;
+                    disabled: boolean;
+                    selected: boolean;
+                }>;
+            }) => {
+                setDropdownInfo(info);
+            });
+
+            socket.on('showTimePicker', (info: {coordinates: Coordinates, selector: string}) => {
+                setTimePickerInfo(info);
+            });
+
+            socket.on('showDateTimePicker', (info: {coordinates: Coordinates, selector: string}) => {
+                setDateTimeLocalInfo(info);
+            });
+
+            return () => {
+                socket.off('showDatePicker');
+                socket.off('showDropdown');
+                socket.off('showTimePicker');
+                socket.off('showDateTimePicker');
+            };
         }
+    }, [socket]);
 
-        return {
-            x: event.clientX - rect.left,
-            y: event.clientY - rect.top
-        };
-    }, []);
+    const onMouseEvent = useCallback((event: MouseEvent) => {
+        if (socket && canvasRef.current) {
+            // Get the canvas bounding rectangle
+            const rect = canvasRef.current.getBoundingClientRect();
+            const clickCoordinates = {
+                x: event.clientX - rect.left, // Use relative x coordinate
+                y: event.clientY - rect.top, // Use relative y coordinate
+            };
 
-    const handleMouseEvent = useCallback((event: MouseEvent) => {
-        if (!socket || !canvasRef.current) return;
-
-        //performanceMonitor.current.measureEventLatency(event);
-        const coordinates = getEventCoordinates(event);
-
-        switch (event.type) {
-            case 'mousedown':
-                debouncer.current.add(() => {
-                    if (refs.current.getText) {
+            switch (event.type) {
+                case 'mousedown':
+                    if (getTextRef.current === true) {
                         console.log('Capturing Text...');
-                    } else if (refs.current.getList) {
+                    } else if (getListRef.current === true) {
                         console.log('Capturing List...');
                     } else {
-                        socket.emit('input:mousedown', coordinates);
+                        socket.emit('input:mousedown', clickCoordinates);
                     }
-                    setLastAction('click');
-                }, true); // High priority
-                break;
-
-            case 'mousemove':
-                if (refs.current.lastMousePosition.x !== coordinates.x ||
-                    refs.current.lastMousePosition.y !== coordinates.y) {
-                    debouncer.current.add(() => {
-                        refs.current.lastMousePosition = coordinates;
-                        socket.emit('input:mousemove', coordinates);
-                        setLastAction('move');
-                    });
-                }
-                break;
-
-            case 'wheel':
-                const wheelEvent = event as WheelEvent;
-                debouncer.current.add(() => {
-                    socket.emit('input:wheel', {
+                    notifyLastAction('click');
+                    break;
+                case 'mousemove':
+                    if (lastMousePosition.current.x !== clickCoordinates.x ||
+                        lastMousePosition.current.y !== clickCoordinates.y) {
+                        lastMousePosition.current = {
+                            x: clickCoordinates.x,
+                            y: clickCoordinates.y,
+                        };
+                        socket.emit('input:mousemove', {
+                            x: clickCoordinates.x,
+                            y: clickCoordinates.y,
+                        });
+                        notifyLastAction('move');
+                    }
+                    break;
+                case 'wheel':
+                    const wheelEvent = event as WheelEvent;
+                    const deltas = {
                         deltaX: Math.round(wheelEvent.deltaX),
-                        deltaY: Math.round(wheelEvent.deltaY)
-                    });
-                    setLastAction('scroll');
-                });
-                break;
+                        deltaY: Math.round(wheelEvent.deltaY),
+                    };
+                    socket.emit('input:wheel', deltas);
+                    notifyLastAction('scroll');
+                    break;
+                default:
+                    console.log('Default mouseEvent registered');
+                    return;
+            }
         }
-    }, [socket, getEventCoordinates]);
+    }, [socket]);
 
-    const handleKeyboardEvent = useCallback((event: KeyboardEvent) => {
-        if (!socket) return;
-
-        debouncer.current.add(() => {
+    const onKeyboardEvent = useCallback((event: KeyboardEvent) => {
+        if (socket) {
             switch (event.type) {
                 case 'keydown':
-                    socket.emit('input:keydown', {
-                        key: event.key,
-                        coordinates: refs.current.lastMousePosition
-                    });
-                    setLastAction(`${event.key} pressed`);
+                    socket.emit('input:keydown', { key: event.key, coordinates: lastMousePosition.current });
+                    notifyLastAction(`${event.key} pressed`);
                     break;
                 case 'keyup':
                     socket.emit('input:keyup', event.key);
                     break;
+                default:
+                    console.log('Default keyEvent registered');
+                    return;
             }
-        }, event.type === 'keydown'); // High priority for keydown
+        }
     }, [socket]);
 
-    // Setup and cleanup
-    useEffect(() => {
-        if (!canvasRef.current) return;
-
-        const canvas = canvasRef.current;
-        refs.current.context = canvas.getContext('2d', {
-            alpha: false,
-            desynchronized: true
-        });
-
-        onCreateRef(canvasRef);
-
-        const options = { passive: true };
-        canvas.addEventListener('mousedown', handleMouseEvent, options);
-        canvas.addEventListener('mousemove', handleMouseEvent, options);
-        canvas.addEventListener('wheel', handleMouseEvent, options);
-        canvas.addEventListener('keydown', handleKeyboardEvent, options);
-        canvas.addEventListener('keyup', handleKeyboardEvent, options);
-
-        return () => {
-            canvas.removeEventListener('mousedown', handleMouseEvent);
-            canvas.removeEventListener('mousemove', handleMouseEvent);
-            canvas.removeEventListener('wheel', handleMouseEvent);
-            canvas.removeEventListener('keydown', handleKeyboardEvent);
-            canvas.removeEventListener('keyup', handleKeyboardEvent);
-
-            scheduler.current.clear();
-            debouncer.current.clear();
-            measurementCache.current.clear();
-        };
-    }, [handleMouseEvent, handleKeyboardEvent, onCreateRef]);
-
-    // Performance monitoring
-    // useEffect(() => {
-    //     const intervalId = setInterval(() => {
-    //         console.log('Performance Report:', performanceMonitor.current.getPerformanceReport());
-    //     }, 20000);
-
-    //     return () => clearInterval(intervalId);
-    // }, []);
 
     useEffect(() => {
-        if (!socket) return;
+        if (canvasRef.current) {
+            onCreateRef(canvasRef);
+            canvasRef.current.addEventListener('mousedown', onMouseEvent);
+            canvasRef.current.addEventListener('mousemove', onMouseEvent);
+            canvasRef.current.addEventListener('wheel', onMouseEvent, { passive: true });
+            canvasRef.current.addEventListener('keydown', onKeyboardEvent);
+            canvasRef.current.addEventListener('keyup', onKeyboardEvent);
 
-        const handlers = {
-            showDatePicker: (info: any) => dispatch({ type: 'BATCH_UPDATE', payload: { datePickerInfo: info } }),
-            showDropdown: (info: any) => dispatch({ type: 'BATCH_UPDATE', payload: { dropdownInfo: info } }),
-            showTimePicker: (info: any) => dispatch({ type: 'BATCH_UPDATE', payload: { timePickerInfo: info } }),
-            showDateTimePicker: (info: any) => dispatch({ type: 'BATCH_UPDATE', payload: { dateTimeLocalInfo: info } })
-        };
+            return () => {
+                if (canvasRef.current) {
+                    canvasRef.current.removeEventListener('mousedown', onMouseEvent);
+                    canvasRef.current.removeEventListener('mousemove', onMouseEvent);
+                    canvasRef.current.removeEventListener('wheel', onMouseEvent);
+                    canvasRef.current.removeEventListener('keydown', onKeyboardEvent);
+                    canvasRef.current.removeEventListener('keyup', onKeyboardEvent);
+                }
 
-        Object.entries(handlers).forEach(([event, handler]) => socket.on(event, handler));
-        return () => {
-            Object.keys(handlers).forEach(event => socket.off(event));
-        };
-    }, [socket]);
+            };
+        } else {
+            console.log('Canvas not initialized');
+        }
 
-    const memoizedDimensions = useMemo(() => ({
-        width: width || 900,
-        height: height || 400
-    }), [width, height]);
+    }, [onMouseEvent]);
 
     return (
-        

+        

             
-            
-                {state.datePickerInfo && (
-                     dispatch({
-                            type: 'BATCH_UPDATE',
-                            payload: { datePickerInfo: null }
-                        })}
-                    />
-                )}
-                {state.dropdownInfo && (
-                     dispatch({
-                            type: 'BATCH_UPDATE',
-                            payload: { dropdownInfo: null }
-                        })}
-                    />
-                )}
-                {state.timePickerInfo && (
-                     dispatch({ type: 'SET_TIME_PICKER', payload: null })}
-                    />
-                )}
-                {state.dateTimeLocalInfo && (
-                     dispatch({ type: 'SET_DATETIME_PICKER', payload: null })}
-                    />
-                )}
-            
+            {datePickerInfo && (
+                 setDatePickerInfo(null)}
+                />
+            )}
+            {dropdownInfo && (
+                 setDropdownInfo(null)}
+                />
+            )}
+            {timePickerInfo && (
+                 setTimePickerInfo(null)}
+                />
+            )}
+            {dateTimeLocalInfo && (
+                 setDateTimeLocalInfo(null)}
+                />
+            )}
         

     );
-});
 
-Canvas.displayName = 'Canvas';
+};
+
 
 export default Canvas;
\ No newline at end of file

From c46ef9fda7c8acc581b8cb7300cdb22dca4848f7 Mon Sep 17 00:00:00 2001
From: amhsirak 
Date: Wed, 8 Jan 2025 17:22:35 +0530
Subject: [PATCH 115/216] chore: FE 0.0.7

---
 docker-compose.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker-compose.yml b/docker-compose.yml
index 6506a4c5..91b72428 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -70,7 +70,7 @@ services:
       #build:
       #context: .
       #dockerfile: Dockerfile
-    image: getmaxun/maxun-frontend:v0.0.6
+    image: getmaxun/maxun-frontend:v0.0.7
     ports:
       - "${FRONTEND_PORT:-5173}:${FRONTEND_PORT:-5173}"
     env_file: .env

From 843db677731e74611c448833e3f90f0c7ea2e6a2 Mon Sep 17 00:00:00 2001
From: RohitR311 
Date: Wed, 8 Jan 2025 17:44:17 +0530
Subject: [PATCH 116/216] feat: add styling to right side panel buttons

---
 src/components/organisms/RightSidePanel.tsx | 103 ++++++++++++++++++--
 1 file changed, 97 insertions(+), 6 deletions(-)

diff --git a/src/components/organisms/RightSidePanel.tsx b/src/components/organisms/RightSidePanel.tsx
index 2eb41947..30d976d9 100644
--- a/src/components/organisms/RightSidePanel.tsx
+++ b/src/components/organisms/RightSidePanel.tsx
@@ -500,6 +500,14 @@ export const RightSidePanel: React.FC = ({ onFinishCapture
                 
@@ -508,13 +516,32 @@ export const RightSidePanel: React.FC = ({ onFinishCapture
                 variant="outlined"
                 onClick={handleConfirmListCapture}
                 disabled={captureStage === 'initial' ? isConfirmCaptureDisabled : hasUnconfirmedListTextFields}
+                sx={{ 
+                  color: '#ff00c3 !important',
+                  borderColor: '#ff00c3 !important',
+                  '&:hover': {
+                    borderColor: '#ff00c3 !important',
+                    backgroundColor: 'whitesmoke !important',
+                  }
+                }} 
               >
                 {captureStage === 'initial' ? t('right_panel.buttons.confirm_capture') :
                 captureStage === 'pagination' ? t('right_panel.buttons.confirm_pagination') :
                 captureStage === 'limit' ? t('right_panel.buttons.confirm_limit') : 
                 t('right_panel.buttons.finish_capture')}
               
-              
             
@@ -523,11 +550,75 @@ export const RightSidePanel: React.FC = ({ onFinishCapture
         {showPaginationOptions && (
           
             {t('right_panel.pagination.title')}
-            
-            
-            
-            
-            
+            
+            
+            
+            
+            
           
         )}
         {showLimitOptions && (

From 8ea90a7c2a2a3ac5d4d29a061eac2efdf882e7c4 Mon Sep 17 00:00:00 2001
From: RohitR311 
Date: Wed, 8 Jan 2025 17:47:11 +0530
Subject: [PATCH 117/216] feat: add style to select attribute button

---
 src/components/organisms/BrowserWindow.tsx | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/components/organisms/BrowserWindow.tsx b/src/components/organisms/BrowserWindow.tsx
index 409079a9..9e8902ba 100644
--- a/src/components/organisms/BrowserWindow.tsx
+++ b/src/components/organisms/BrowserWindow.tsx
@@ -403,6 +403,14 @@ export const BrowserWindow = () => {
                                             overflow: 'hidden',
                                             padding: '5px 10px',
                                         }}
+                                        sx={{ 
+                                            color: '#ff00c3 !important',
+                                            borderColor: '#ff00c3 !important',
+                                            '&:hover': {
+                                                borderColor: '#ff00c3 !important',
+                                                backgroundColor: 'whitesmoke !important',
+                                            }
+                                        }}
                                     >
                                         
Date: Wed, 8 Jan 2025 17:49:38 +0530
Subject: [PATCH 118/216] feat: latest compose changes for upgrade

---
 src/components/molecules/NavBar.tsx | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/components/molecules/NavBar.tsx b/src/components/molecules/NavBar.tsx
index 8aeeb05d..2a38ad39 100644
--- a/src/components/molecules/NavBar.tsx
+++ b/src/components/molecules/NavBar.tsx
@@ -261,6 +261,11 @@ export const NavBar: React.FC = ({
                                 docker-compose down
                                 

                                 

+                                # replace existing docker-compose file with new one by copy pasting the code from
+                                

+                                Latest Docker Compose
+                                

+                                

                                 # pull latest docker images
                                 

                                 docker-compose pull

From 07f1aba701ac3e81d506e802ab360158cced8067 Mon Sep 17 00:00:00 2001
From: RohitR311 
Date: Wed, 8 Jan 2025 17:59:36 +0530
Subject: [PATCH 119/216] feat: add default bg color on selection

---
 src/components/organisms/RightSidePanel.tsx | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/components/organisms/RightSidePanel.tsx b/src/components/organisms/RightSidePanel.tsx
index 30d976d9..430377d8 100644
--- a/src/components/organisms/RightSidePanel.tsx
+++ b/src/components/organisms/RightSidePanel.tsx
@@ -556,7 +556,7 @@ export const RightSidePanel: React.FC = ({ onFinishCapture
               sx={{ 
                 color: '#ff00c3 !important',
                 borderColor: '#ff00c3 !important',
-                backgroundColor: paginationType === 'clickNext' && isDarkMode ? 'whitesmoke !important' : 'transparent !important',
+                backgroundColor: paginationType === 'clickNext' ? 'whitesmoke !important' : 'transparent !important',
                 '&:hover': {
                   borderColor: '#ff00c3 !important',
                   backgroundColor: 'whitesmoke !important',
@@ -570,7 +570,7 @@ export const RightSidePanel: React.FC = ({ onFinishCapture
               sx={{ 
                 color: '#ff00c3 !important',
                 borderColor: '#ff00c3 !important',
-                backgroundColor: paginationType === 'clickLoadMore' && isDarkMode ? 'whitesmoke !important' : 'transparent !important',
+                backgroundColor: paginationType === 'clickLoadMore' ? 'whitesmoke !important' : 'transparent !important',
                 '&:hover': {
                   borderColor: '#ff00c3 !important',
                   backgroundColor: 'whitesmoke !important',
@@ -584,7 +584,7 @@ export const RightSidePanel: React.FC = ({ onFinishCapture
               sx={{ 
                 color: '#ff00c3 !important',
                 borderColor: '#ff00c3 !important',
-                backgroundColor: paginationType === 'scrollDown' && isDarkMode ? 'whitesmoke !important' : 'transparent !important',
+                backgroundColor: paginationType === 'scrollDown' ? 'whitesmoke !important' : 'transparent !important',
                 '&:hover': {
                   borderColor: '#ff00c3 !important',
                   backgroundColor: 'whitesmoke !important',
@@ -598,7 +598,7 @@ export const RightSidePanel: React.FC = ({ onFinishCapture
               sx={{ 
                 color: '#ff00c3 !important',
                 borderColor: '#ff00c3 !important',
-                backgroundColor: paginationType === 'scrollUp' && isDarkMode ? 'whitesmoke !important' : 'transparent !important',
+                backgroundColor: paginationType === 'scrollUp' ? 'whitesmoke !important' : 'transparent !important',
                 '&:hover': {
                   borderColor: '#ff00c3 !important',
                   backgroundColor: 'whitesmoke !important',
@@ -612,7 +612,7 @@ export const RightSidePanel: React.FC = ({ onFinishCapture
               sx={{ 
                 color: '#ff00c3 !important',
                 borderColor: '#ff00c3 !important',
-                backgroundColor: paginationType === 'none' && isDarkMode ? 'whitesmoke !important' : 'transparent !important',
+                backgroundColor: paginationType === 'none' ? 'whitesmoke !important' : 'transparent !important',
                 '&:hover': {
                   borderColor: '#ff00c3 !important',
                   backgroundColor: 'whitesmoke !important',

From 8cb72691d1b688e743c290bd764b4278c1aab38a Mon Sep 17 00:00:00 2001
From: RohitR311 
Date: Wed, 8 Jan 2025 18:06:18 +0530
Subject: [PATCH 120/216] feat: add styling for cancel button

---
 src/components/molecules/BrowserRecordingSave.tsx | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/src/components/molecules/BrowserRecordingSave.tsx b/src/components/molecules/BrowserRecordingSave.tsx
index f2ededea..4eda4ec4 100644
--- a/src/components/molecules/BrowserRecordingSave.tsx
+++ b/src/components/molecules/BrowserRecordingSave.tsx
@@ -63,7 +63,17 @@ const BrowserRecordingSave = () => {
                 
-                
               

From e0c3cf7f55970098378d2d85aedd03b113ad357e Mon Sep 17 00:00:00 2001
From: RohitR311 
Date: Wed, 8 Jan 2025 18:11:23 +0530
Subject: [PATCH 121/216] feat: add styling for cancel button

---
 src/components/molecules/ScheduleSettings.tsx | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/src/components/molecules/ScheduleSettings.tsx b/src/components/molecules/ScheduleSettings.tsx
index 917696c9..95282163 100644
--- a/src/components/molecules/ScheduleSettings.tsx
+++ b/src/components/molecules/ScheduleSettings.tsx
@@ -273,7 +273,19 @@ export const ScheduleSettingsModal = ({ isOpen, handleStart, handleClose, initia
                 
-                
               

From baf0845f67bb9fa1bd6d01c0831889b4092dec87 Mon Sep 17 00:00:00 2001
From: RohitR311 
Date: Wed, 8 Jan 2025 18:11:43 +0530
Subject: [PATCH 122/216] feat: add styling for cancel button

---
 src/components/molecules/RobotEdit.tsx | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/components/molecules/RobotEdit.tsx b/src/components/molecules/RobotEdit.tsx
index 6547d93b..30878f77 100644
--- a/src/components/molecules/RobotEdit.tsx
+++ b/src/components/molecules/RobotEdit.tsx
@@ -179,6 +179,14 @@ export const RobotEditModal = ({ isOpen, handleStart, handleClose, initialSettin
                                         color="primary" 
                                         variant="outlined" 
                                         style={{ marginLeft: '10px' }}
+                                        sx={{ 
+                                            color: '#ff00c3 !important',
+                                            borderColor: '#ff00c3 !important',
+                                            '&:hover': {
+                                              borderColor: '#ff00c3 !important',
+                                              backgroundColor: 'whitesmoke !important',
+                                            }
+                                        }} 
                                     >
                                         {t('robot_edit.cancel')}
                                     

From 9210d2f9ee579b78f871eb699cf2c21c2c240586 Mon Sep 17 00:00:00 2001
From: RohitR311 
Date: Wed, 8 Jan 2025 18:12:00 +0530
Subject: [PATCH 123/216] feat: add styling for cancel button

---
 src/components/molecules/RobotDuplicate.tsx | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/src/components/molecules/RobotDuplicate.tsx b/src/components/molecules/RobotDuplicate.tsx
index ce3ee5ca..68711aa0 100644
--- a/src/components/molecules/RobotDuplicate.tsx
+++ b/src/components/molecules/RobotDuplicate.tsx
@@ -152,7 +152,19 @@ export const RobotDuplicationModal = ({ isOpen, handleStart, handleClose, initia
                                     
-                                    
                                 

From 94bb35cc8c0282adb5f5a061e909b4f91db8ed12 Mon Sep 17 00:00:00 2001
From: RohitR311 
Date: Wed, 8 Jan 2025 19:58:50 +0530
Subject: [PATCH 124/216] feat: add theme icon and change colors

---
 src/components/molecules/NavBar.tsx | 174 ++--------------------------
 1 file changed, 10 insertions(+), 164 deletions(-)

diff --git a/src/components/molecules/NavBar.tsx b/src/components/molecules/NavBar.tsx
index 01fa01b7..c68873ff 100644
--- a/src/components/molecules/NavBar.tsx
+++ b/src/components/molecules/NavBar.tsx
@@ -103,79 +103,6 @@ export const NavBar: React.FC = ({
     localStorage.setItem("language", lang);
   };
 
-//   const renderBrandSection = () => (
-//     
-//       
-//       Maxun
-//       
-//     
-//   );
-
-//   const renderSocialButtons = () => (
-//     <>
-//       
-//         
-//       
-//