diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..b675b276 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,11 @@ +node_modules +npm-debug.log +dist +.git +.gitignore +.env +.md +.vscode +coverage +docker-compose.yml +Dockerfile \ No newline at end of file diff --git a/ENVEXAMPLE b/ENVEXAMPLE index d17b1fc5..a387785f 100644 --- a/ENVEXAMPLE +++ b/ENVEXAMPLE @@ -1,12 +1,12 @@ # App Setup NODE_ENV=production # Set to 'development' or 'production' as required -JWT_SECRET=your_jwt_secret_key # Replace with a secure JWT secret key +JWT_SECRET=a9Z$kLq7^f03GzNw!bP9dH4xV6sT2yXl3O8vR@uYq3 # Replace with a secure JWT secret key DB_NAME=maxun # Your PostgreSQL database name DB_USER=postgres # PostgreSQL username DB_PASSWORD=postgres # PostgreSQL password DB_HOST=postgres # Host for PostgreSQL in Docker DB_PORT=5432 # Port for PostgreSQL (default: 5432) -ENCRYPTION_KEY=your_encryption_key # Key for encrypting sensitive data (passwords and proxies) +ENCRYPTION_KEY=f4d5e6a7b8c9d0e1f23456789abcdef01234567890abcdef123456789abcdef0 # Key for encrypting sensitive data (passwords and proxies) MINIO_ENDPOINT=minio # MinIO endpoint in Docker MINIO_PORT=9000 # Port for MinIO (default: 9000) MINIO_ACCESS_KEY=minio_access_key # MinIO access key diff --git a/README.md b/README.md index 602cb05c..dd562422 100644 --- a/README.md +++ b/README.md @@ -20,12 +20,17 @@ Maxun lets you train a robot in 2 minutes and scrape the web on auto-pilot. Web Twitter | Join Maxun Cloud | Watch Tutorials +
+
+getmaxun%2Fmaxun | Trendshift

![maxun_demo](https://github.com/user-attachments/assets/a61ba670-e56a-4ae1-9681-0b4bd6ba9cdc) +> Note: We are in early stages of development and do not support self hosting yet. You can run Maxun locally. + # Local Setup ### Docker Compose ``` @@ -49,6 +54,16 @@ npm install cd maxun-core npm install +# get back to the root directory +cd .. + +# make sure playwright is properly initialized +npx playwright install +npx playwright install-deps + +# get back to the root directory +cd .. + # start frontend and backend together npm run start ``` @@ -61,8 +76,8 @@ You can access the frontend at http://localhost:5173/ and backend at http://loca | Variable | Mandatory | Description | If Not Set | |-----------------------|-----------|----------------------------------------------------------------------------------------------|--------------------------------------------------------------| -| `BACKEND_URL` | Yes | URL to run backend on. | Backend won't start. If not sure, set to http://localhost:8080 | -| `VITE_BACKEND_URL` | Yes | URL to run backend on. | Backend won't start. If not sure, set to http://localhost:8080 | +| `BACKEND_URL` | Yes | URL to run backend on. | Default value: http://localhost:8080 | +| `VITE_BACKEND_URL` | Yes | URL used by frontend to connect to backend | Default value: http://localhost:8080 | | `JWT_SECRET` | Yes | Secret key used to sign and verify JSON Web Tokens (JWTs) for authentication. | JWT authentication will not work. | | `DB_NAME` | Yes | Name of the Postgres database to connect to. | Database connection will fail. | | `DB_USER` | Yes | Username for Postgres database authentication. | Database connection will fail. | diff --git a/docker-compose.yml b/docker-compose.yml index 1aeefd26..8b26973b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -32,16 +32,18 @@ services: environment: MINIO_ROOT_USER: ${MINIO_ACCESS_KEY} MINIO_ROOT_PASSWORD: ${MINIO_SECRET_KEY} - command: server /data + command: server /data --console-address :9001 ports: - - "9000:9000" + - "9000:9000" # API port + - "9001:9001" # WebUI port volumes: - minio_data:/data backend: - build: - context: . - dockerfile: server/Dockerfile + #build: + #context: . + #dockerfile: server/Dockerfile + image: getmaxun/maxun-backend:v0.0.2 ports: - "8080:8080" env_file: .env @@ -56,6 +58,7 @@ services: - seccomp=unconfined # This might help with browser sandbox issues # Increase shared memory size for Chromium shm_size: '2gb' + mem_limit: 2g # Set a 2GB memory limit depends_on: - postgres - redis @@ -66,9 +69,10 @@ services: - /var/run/dbus:/var/run/dbus frontend: - build: - context: . - dockerfile: Dockerfile + #build: + #context: . + #dockerfile: Dockerfile + image: getmaxun/maxun-frontend:v0.0.1 ports: - "5173:5173" env_file: .env diff --git a/esbuild.config.js b/esbuild.config.js new file mode 100644 index 00000000..e69de29b diff --git a/maxun-core/package.json b/maxun-core/package.json index 45c69ffe..faa133ec 100644 --- a/maxun-core/package.json +++ b/maxun-core/package.json @@ -1,6 +1,6 @@ { "name": "maxun-core", - "version": "0.0.3", + "version": "0.0.4", "description": "Core package for Maxun, responsible for data extraction", "main": "build/index.js", "typings": "build/index.d.ts", @@ -20,7 +20,12 @@ "automation", "workflow", "data extraction", - "scraping" + "scraping", + "web scraper", + "web scraping", + "data scraping", + "no-code web scraper", + "no-code web scraping" ], "author": "Maxun", "license": "AGPL-3.0-or-later", diff --git a/maxun-core/src/browserSide/scraper.js b/maxun-core/src/browserSide/scraper.js index 79893568..828a4f84 100644 --- a/maxun-core/src/browserSide/scraper.js +++ b/maxun-core/src/browserSide/scraper.js @@ -235,9 +235,11 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3, switch (attribute) { case 'href': - return elem.getAttribute('href'); + const relativeHref = elem.getAttribute('href'); + return relativeHref ? new URL(relativeHref, window.location.origin).href : null; case 'src': - return elem.getAttribute('src'); + const relativeSrc = elem.getAttribute('src'); + return relativeSrc ? new URL(relativeSrc, window.location.origin).href : null; case 'innerText': return elem.innerText; case 'textContent': @@ -281,9 +283,13 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3, } else if (attribute === 'innerHTML') { record[label] = fieldElement.innerHTML.trim(); } else if (attribute === 'src') { - record[label] = fieldElement.src; + // Handle relative 'src' URLs + const src = fieldElement.getAttribute('src'); + record[label] = src ? new URL(src, baseUrl).href : null; } else if (attribute === 'href') { - record[label] = fieldElement.href; + // Handle relative 'href' URLs + const href = fieldElement.getAttribute('href'); + record[label] = href ? new URL(href, baseUrl).href : null; } else { record[label] = fieldElement.getAttribute(attribute); } diff --git a/package.json b/package.json index fd966458..b6b73537 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "maxun", - "version": "0.0.1", + "version": "0.0.2", "author": "Maxun", "license": "AGPL-3.0-or-later", "dependencies": { @@ -39,17 +39,19 @@ "ioredis": "^5.4.1", "joi": "^17.6.0", "jsonwebtoken": "^9.0.2", + "jwt-decode": "^4.0.0", "loglevel": "^1.8.0", "loglevel-plugin-remote": "^0.6.8", - "maxun-core": "^0.0.3", + "maxun-core": "0.0.4", "minio": "^8.0.1", "moment-timezone": "^0.5.45", "node-cron": "^3.0.3", "pg": "^8.13.0", - "playwright": "^1.20.1", + "playwright": "^1.48.2", "playwright-extra": "^4.3.6", "posthog-node": "^4.2.1", "prismjs": "^1.28.0", + "puppeteer-extra-plugin-recaptcha": "^3.6.8", "puppeteer-extra-plugin-stealth": "^2.11.2", "react": "^18.0.0", "react-dom": "^18.0.0", @@ -89,6 +91,7 @@ "devDependencies": { "@types/cookie-parser": "^1.4.7", "@types/express": "^4.17.13", + "@types/js-cookie": "^3.0.6", "@types/loglevel": "^1.6.3", "@types/node": "22.7.9", "@types/node-cron": "^3.0.11", @@ -102,8 +105,9 @@ "ajv": "^8.8.2", "concurrently": "^7.0.0", "cross-env": "^7.0.3", + "js-cookie": "^3.0.5", "nodemon": "^2.0.15", "ts-node": "^10.4.0", "vite": "^5.4.10" } -} +} \ No newline at end of file diff --git a/server/Dockerfile b/server/Dockerfile index a9904269..ae26e8eb 100644 --- a/server/Dockerfile +++ b/server/Dockerfile @@ -18,33 +18,6 @@ RUN npm install # Install Playwright browsers and dependencies RUN npx playwright install --with-deps chromium -# Install xvfb for display support -#RUN apt-get update && apt-get install -y xvfb - -# RUN apt-get update && apt-get install -y \ -# libgbm-dev \ -# libxkbcommon-x11-0 \ -# libxcomposite1 \ -# libxdamage1 \ -# libxrandr2 \ -# libxshmfence1 \ -# libxtst6 \ -# libnss3 \ -# libatk1.0-0 \ -# libatk-bridge2.0-0 \ -# libdrm2 \ -# libxcb1 \ -# libxkbcommon0 \ -# fonts-noto-color-emoji \ -# fonts-unifont \ -# libpango-1.0-0 \ -# libcairo2 \ -# libasound2 \ -# libglib2.0-0 \ -# libdbus-1-3 \ -# && rm -rf /var/lib/apt/lists/* - -# Create and set permissions for chrome directories # Create the Chromium data directory with necessary permissions RUN mkdir -p /tmp/chromium-data-dir && \ chmod -R 777 /tmp/chromium-data-dir diff --git a/server/src/api/record.ts b/server/src/api/record.ts index b55f06bc..fef0be9a 100644 --- a/server/src/api/record.ts +++ b/server/src/api/record.ts @@ -1,6 +1,7 @@ import { readFile, readFiles } from "../workflow-management/storage"; import { Router, Request, Response } from 'express'; -import { chromium } from "playwright"; +import { chromium } from "playwright-extra"; +import stealthPlugin from 'puppeteer-extra-plugin-stealth'; import { requireAPIKey } from "../middlewares/api"; import Robot from "../models/Robot"; import Run from "../models/Run"; @@ -14,6 +15,7 @@ import { io, Socket } from "socket.io-client"; import { BinaryOutputService } from "../storage/mino"; import { AuthenticatedRequest } from "../routes/record" import {capture} from "../utils/analytics"; +chromium.use(stealthPlugin()); const formatRecording = (recordingData: any) => { const recordingMeta = recordingData.recording_meta; @@ -289,7 +291,7 @@ router.get("/robots/:id", requireAPIKey, async (req: Request, res: Response) => * type: string * example: "Failed to retrieve runs" */ -router.get("/robots/:id/runs", requireAPIKey, async (req: Request, res: Response) => { +router.get("/robots/:id/runs",requireAPIKey, async (req: Request, res: Response) => { try { const runs = await Run.findAll({ where: { @@ -321,6 +323,7 @@ router.get("/robots/:id/runs", requireAPIKey, async (req: Request, res: Response } ); + function formatRunResponse(run: any) { const formattedRun = { id: run.id, @@ -494,7 +497,7 @@ async function createWorkflowAndStoreMetadata(id: string, userId: string) { } catch (e) { const { message } = e as Error; logger.log('info', `Error while scheduling a run with id: ${id}`); - console.log(message); + console.log(`Error scheduling run:`, message); return { success: false, error: message, @@ -766,7 +769,6 @@ router.post("/robots/:id/runs", requireAPIKey, async (req: AuthenticatedRequest, return res.status(401).json({ ok: false, error: 'Unauthorized' }); } const runId = await handleRunRecording(req.params.id, req.user.dataValues.id); - console.log(`Result`, runId); if (!runId) { throw new Error('Run ID is undefined'); diff --git a/server/src/browser-management/classes/RemoteBrowser.ts b/server/src/browser-management/classes/RemoteBrowser.ts index cbda3942..07ea8780 100644 --- a/server/src/browser-management/classes/RemoteBrowser.ts +++ b/server/src/browser-management/classes/RemoteBrowser.ts @@ -3,9 +3,10 @@ import { Browser, CDPSession, BrowserContext, - chromium, } from 'playwright'; import { Socket } from "socket.io"; +import { chromium } from 'playwright-extra'; +import stealthPlugin from 'puppeteer-extra-plugin-stealth'; import { PlaywrightBlocker } from '@cliqz/adblocker-playwright'; import fetch from 'cross-fetch'; @@ -14,7 +15,7 @@ import { InterpreterSettings, RemoteBrowserOptions } from "../../types"; import { WorkflowGenerator } from "../../workflow-management/classes/Generator"; import { WorkflowInterpreter } from "../../workflow-management/classes/Interpreter"; import { getDecryptedProxyConfig } from '../../routes/proxy'; - +chromium.use(stealthPlugin()); /** @@ -163,9 +164,7 @@ export class RemoteBrowser { contextOptions.userAgent = browserUserAgent; this.context = await this.browser.newContext(contextOptions); - console.log(`Context from initialize: ${JSON.stringify(this.context)}`) this.currentPage = await this.context.newPage(); - console.log(`CPage from initialize: ${JSON.stringify(this.currentPage)}`) // await this.currentPage.setExtraHTTPHeaders({ // 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3' // }); @@ -307,6 +306,7 @@ export class RemoteBrowser { const workflow = this.generator.AddGeneratedFlags(this.generator.getWorkflowFile()); await this.initializeNewPage(); if (this.currentPage) { + this.currentPage.setViewportSize({ height: 400, width: 900 }); const params = this.generator.getParams(); if (params) { this.interpreterSettings.params = params.reduce((acc, param) => { diff --git a/server/src/browser-management/inputHandlers.ts b/server/src/browser-management/inputHandlers.ts index 2e43a4c5..d6902b3f 100644 --- a/server/src/browser-management/inputHandlers.ts +++ b/server/src/browser-management/inputHandlers.ts @@ -271,7 +271,6 @@ const handleChangeUrl = async (generator: WorkflowGenerator, page: Page, url: st try { await page.goto(url); logger.log('debug', `Went to ${url}`); - console.log(`Went to ${url}`) } catch (e) { const { message } = e as Error; logger.log('error', message); diff --git a/server/src/middlewares/auth.ts b/server/src/middlewares/auth.ts index 75540ad5..74770c3a 100644 --- a/server/src/middlewares/auth.ts +++ b/server/src/middlewares/auth.ts @@ -16,12 +16,16 @@ export const requireSignIn = (req: UserRequest, res: Response, next: any) => { } verify(token, secret, (err: any, user: any) => { - console.log(err) - - if (err) return res.sendStatus(403) - + if (err) { + console.log('JWT verification error:', err); + return res.sendStatus(403); + } + // Normalize payload key + if (user.userId && !user.id) { + user.id = user.userId; + delete user.userId; // temporary: del the old key for clarity + } req.user = user; - - next() - }) + next(); + }); }; diff --git a/server/src/routes/auth.ts b/server/src/routes/auth.ts index 6766a356..692add99 100644 --- a/server/src/routes/auth.ts +++ b/server/src/routes/auth.ts @@ -1,480 +1,557 @@ -import { Router, Request, Response } from 'express'; -import User from '../models/User'; -import Robot from '../models/Robot'; -import jwt from 'jsonwebtoken'; -import { hashPassword, comparePassword } from '../utils/auth'; -import { requireSignIn } from '../middlewares/auth'; -import { genAPIKey } from '../utils/api'; -import { google } from 'googleapis'; -import { capture } from "../utils/analytics" +import { Router, Request, Response } from "express"; +import User from "../models/User"; +import Robot from "../models/Robot"; +import jwt from "jsonwebtoken"; +import { hashPassword, comparePassword } from "../utils/auth"; +import { requireSignIn } from "../middlewares/auth"; +import { genAPIKey } from "../utils/api"; +import { google } from "googleapis"; +import { capture } from "../utils/analytics"; export const router = Router(); interface AuthenticatedRequest extends Request { - user?: { id: string }; + user?: { id: string }; } -router.post('/register', async (req, res) => { - console.log('Received request at /auth/register'); - console.log('Received body:', req.body); +router.post("/register", async (req, res) => { + try { + const { email, password } = req.body; + + if (!email) return res.status(400).send("Email is required"); + if (!password || password.length < 6) + return res + .status(400) + .send("Password is required and must be at least 6 characters"); + + let userExist = await User.findOne({ raw: true, where: { email } }); + if (userExist) return res.status(400).send("User already exists"); + + const hashedPassword = await hashPassword(password); + + let user: any; + try { - const { email, password } = req.body - - if (!email) return res.status(400).send('Email is required') - if (!password || password.length < 6) return res.status(400).send('Password is required and must be at least 6 characters') - - let userExist = await User.findOne({ raw: true, where: { email } }); - if (userExist) return res.status(400).send('User already exists') - - const hashedPassword = await hashPassword(password) - - let user: any; - - try { - user = await User.create({ email, password: hashedPassword }); - } catch ( - error: any - ) { - console.log(`Could not create user - ${error}`) - return res.status(500).send(`Could not create user - ${error.message}`) - } - - if (!process.env.JWT_SECRET) { - console.log('JWT_SECRET is not defined in the environment'); - return res.status(500).send('Internal Server Error'); - } - - const token = jwt.sign({ id: user.id }, process.env.JWT_SECRET as string, { expiresIn: '12h' }); - user.password = undefined as unknown as string - res.cookie('token', token, { - httpOnly: true - }) - capture( - 'maxun-oss-user-registered', - { - email: user.email, - userId: user.id, - registeredAt: new Date().toISOString() - } - ) - console.log(`User registered - ${user.email}`) - res.json(user) + user = await User.create({ email, password: hashedPassword }); } catch (error: any) { - console.log(`Could not register user - ${error}`) - res.status(500).send(`Could not register user - ${error.message}`) + console.log(`Could not create user - ${error}`); + return res.status(500).send(`Could not create user - ${error.message}`); } -}) -router.post('/login', async (req, res) => { - try { - const { email, password } = req.body; - if (!email || !password) return res.status(400).send('Email and password are required') - if (password.length < 6) return res.status(400).send('Password must be at least 6 characters') - - let user = await User.findOne({ raw: true, where: { email } }); - if (!user) return res.status(400).send('User does not exist'); - - const match = await comparePassword(password, user.password) - if (!match) return res.status(400).send('Invalid email or password') - - const token = jwt.sign({ id: user?.id }, process.env.JWT_SECRET as string, { expiresIn: '12h' }); - - // return user and token to client, exclude hashed password - if (user) { - user.password = undefined as unknown as string; - } - res.cookie('token', token, { - httpOnly: true - }) - res.json(user) - } catch (error: any) { - res.status(400).send(`Could not login user - ${error.message}`) - console.log(`Could not login user - ${error}`) + if (!process.env.JWT_SECRET) { + console.log("JWT_SECRET is not defined in the environment"); + return res.status(500).send("Internal Server Error"); } -}) -router.get('/logout', async (req, res) => { - try { - res.clearCookie('token') - return res.json({ message: 'Logout successful' }) - } catch (error: any) { - res.status(500).send(`Could not logout user - ${error.message}`) - } -}) - -router.get('/current-user', requireSignIn, async (req: AuthenticatedRequest, res) => { - try { - if (!req.user) { - return res.status(401).json({ ok: false, error: 'Unauthorized' }); - } - const user = await User.findByPk(req.user.id, { - attributes: { exclude: ['password'] }, - }); - if (!user) { - return res.status(404).json({ ok: false, error: 'User not found' }); - } else { - return res.status(200).json({ ok: true, user: user }); - } - } catch (error: any) { - console.error('Error in current-user route:', error); - return res.status(500).json({ ok: false, error: `Could not fetch current user: ${error.message}` }); - } + const token = jwt.sign({ id: user.id }, process.env.JWT_SECRET as string); + user.password = undefined as unknown as string; + res.cookie("token", token, { + httpOnly: true, + }); + capture("maxun-oss-user-registered", { + email: user.email, + userId: user.id, + registeredAt: new Date().toISOString(), + }); + console.log(`User registered - ${user.email}`); + res.json(user); + } catch (error: any) { + console.log(`Could not register user - ${error}`); + res.status(500).send(`Could not register user - ${error.message}`); + } }); -router.get('/user/:id', requireSignIn, async (req: AuthenticatedRequest, res) => { - try { - const { id } = req.params; - if (!id) { - return res.status(400).json({ message: 'User ID is required' }); - } +router.post("/login", async (req, res) => { + try { + const { email, password } = req.body; + if (!email || !password) + return res.status(400).send("Email and password are required"); + if (password.length < 6) + return res.status(400).send("Password must be at least 6 characters"); - const user = await User.findByPk(id, { - attributes: { exclude: ['password'] }, - }); + let user = await User.findOne({ raw: true, where: { email } }); + if (!user) return res.status(400).send("User does not exist"); - if (!user) { - return res.status(404).json({ message: 'User not found' }); - } + const match = await comparePassword(password, user.password); + if (!match) return res.status(400).send("Invalid email or password"); - return res.status(200).json({ message: 'User fetched successfully', user }); - } catch (error: any) { - return res.status(500).json({ message: 'Error fetching user', error: error.message }); + const token = jwt.sign({ id: user?.id }, process.env.JWT_SECRET as string); + + // return user and token to client, exclude hashed password + if (user) { + user.password = undefined as unknown as string; } + res.cookie("token", token, { + httpOnly: true, + }); + capture("maxun-oss-user-login", { + email: user.email, + userId: user.id, + loggedInAt: new Date().toISOString(), + }); + res.json(user); + } catch (error: any) { + res.status(400).send(`Could not login user - ${error.message}`); + console.log(`Could not login user - ${error}`); + } }); -router.post('/generate-api-key', requireSignIn, async (req: AuthenticatedRequest, res) => { +router.get("/logout", async (req, res) => { + try { + res.clearCookie("token"); + return res.json({ message: "Logout successful" }); + } catch (error: any) { + res.status(500).send(`Could not logout user - ${error.message}`); + } +}); + +router.get( + "/current-user", + requireSignIn, + async (req: AuthenticatedRequest, res) => { try { - if (!req.user) { - return res.status(401).json({ ok: false, error: 'Unauthorized' }); - } - const user = await User.findByPk(req.user.id, { - attributes: { exclude: ['password'] }, + if (!req.user) { + return res.status(401).json({ ok: false, error: "Unauthorized" }); + } + const user = await User.findByPk(req.user.id, { + attributes: { exclude: ["password"] }, + }); + if (!user) { + return res.status(404).json({ ok: false, error: "User not found" }); + } else { + return res.status(200).json({ ok: true, user: user }); + } + } catch (error: any) { + console.error("Error in current-user route:", error); + return res + .status(500) + .json({ + ok: false, + error: `Could not fetch current user: ${error.message}`, }); + } + } +); - if (!user) { - return res.status(404).json({ message: 'User not found' }); - } +router.get( + "/user/:id", + requireSignIn, + async (req: AuthenticatedRequest, res) => { + try { + const { id } = req.params; + if (!id) { + return res.status(400).json({ message: "User ID is required" }); + } - if (user.api_key) { - return res.status(400).json({ message: 'API key already exists' }); - } - const apiKey = genAPIKey(); + const user = await User.findByPk(id, { + attributes: { exclude: ["password"] }, + }); - await user.update({ api_key: apiKey }); + if (!user) { + return res.status(404).json({ message: "User not found" }); + } - capture( - 'maxun-oss-api-key-created', - { - user_id: user.id, - created_at: new Date().toISOString() - } - ) + return res + .status(200) + .json({ message: "User fetched successfully", user }); + } catch (error: any) { + return res + .status(500) + .json({ message: "Error fetching user", error: error.message }); + } + } +); - return res.status(200).json({ - message: 'API key generated successfully', - api_key: apiKey, - }); +router.post( + "/generate-api-key", + requireSignIn, + async (req: AuthenticatedRequest, res) => { + try { + if (!req.user) { + return res.status(401).json({ ok: false, error: "Unauthorized" }); + } + const user = await User.findByPk(req.user.id, { + attributes: { exclude: ["password"] }, + }); + + if (!user) { + return res.status(404).json({ message: "User not found" }); + } + + if (user.api_key) { + return res.status(400).json({ message: "API key already exists" }); + } + const apiKey = genAPIKey(); + + await user.update({ api_key: apiKey }); + + capture("maxun-oss-api-key-created", { + user_id: user.id, + created_at: new Date().toISOString(), + }); + + return res.status(200).json({ + message: "API key generated successfully", + api_key: apiKey, + }); } catch (error) { - return res.status(500).json({ message: 'Error generating API key', error }); + return res + .status(500) + .json({ message: "Error generating API key", error }); } -}); + } +); -router.get('/api-key', requireSignIn, async (req: AuthenticatedRequest, res) => { +router.get( + "/api-key", + requireSignIn, + async (req: AuthenticatedRequest, res) => { try { - if (!req.user) { - return res.status(401).json({ ok: false, error: 'Unauthorized' }); - } + if (!req.user) { + return res.status(401).json({ ok: false, error: "Unauthorized" }); + } - const user = await User.findByPk(req.user.id, { - raw: true, - attributes: ['api_key'], - }); + const user = await User.findByPk(req.user.id, { + raw: true, + attributes: ["api_key"], + }); - if (!user) { - return res.status(404).json({ message: 'User not found' }); - } + if (!user) { + return res.status(404).json({ message: "User not found" }); + } - return res.status(200).json({ - message: 'API key fetched successfully', - api_key: user.api_key || null, - }); + return res.status(200).json({ + message: "API key fetched successfully", + api_key: user.api_key || null, + }); } catch (error) { - return res.status(500).json({ message: 'Error fetching API key', error }); + return res.status(500).json({ message: "Error fetching API key", error }); } -}); - -router.delete('/delete-api-key', requireSignIn, async (req: AuthenticatedRequest, res) => { + } +); +router.delete( + "/delete-api-key", + requireSignIn, + async (req: AuthenticatedRequest, res) => { if (!req.user) { - return res.status(401).send({ error: 'Unauthorized' }); + return res.status(401).send({ error: "Unauthorized" }); } try { - const user = await User.findByPk(req.user.id, { raw: true }); + const user = await User.findByPk(req.user.id, { raw: true }); - if (!user) { - return res.status(404).json({ message: 'User not found' }); - } + if (!user) { + return res.status(404).json({ message: "User not found" }); + } - if (!user.api_key) { - return res.status(404).json({ message: 'API Key not found' }); - } + if (!user.api_key) { + return res.status(404).json({ message: "API Key not found" }); + } - await User.update({ api_key: null }, { where: { id: req.user.id } }); + await User.update({ api_key: null }, { where: { id: req.user.id } }); - capture( - 'maxun-oss-api-key-deleted', - { - user_id: user.id, - deleted_at: new Date().toISOString() - } - ) + capture("maxun-oss-api-key-deleted", { + user_id: user.id, + deleted_at: new Date().toISOString(), + }); - return res.status(200).json({ message: 'API Key deleted successfully' }); + return res.status(200).json({ message: "API Key deleted successfully" }); } catch (error: any) { - return res.status(500).json({ message: 'Error deleting API key', error: error.message }); + return res + .status(500) + .json({ message: "Error deleting API key", error: error.message }); } -}); + } +); const oauth2Client = new google.auth.OAuth2( - process.env.GOOGLE_CLIENT_ID, - process.env.GOOGLE_CLIENT_SECRET, - process.env.GOOGLE_REDIRECT_URI + process.env.GOOGLE_CLIENT_ID, + process.env.GOOGLE_CLIENT_SECRET, + process.env.GOOGLE_REDIRECT_URI ); // Step 1: Redirect to Google for authentication -router.get('/google', (req, res) => { - const { robotId } = req.query; - if (!robotId) { - return res.status(400).json({ message: 'Robot ID is required' }); - } - const scopes = [ - 'https://www.googleapis.com/auth/spreadsheets', - 'https://www.googleapis.com/auth/userinfo.email', - 'https://www.googleapis.com/auth/drive.readonly', - ]; - const url = oauth2Client.generateAuthUrl({ - access_type: 'offline', - prompt: 'consent', // Ensures you get a refresh token on first login - scope: scopes, - state: robotId.toString(), - }); - res.redirect(url); +router.get("/google", (req, res) => { + const { robotId } = req.query; + if (!robotId) { + return res.status(400).json({ message: "Robot ID is required" }); + } + const scopes = [ + "https://www.googleapis.com/auth/spreadsheets", + "https://www.googleapis.com/auth/userinfo.email", + "https://www.googleapis.com/auth/drive.readonly", + ]; + const url = oauth2Client.generateAuthUrl({ + access_type: "offline", + prompt: "consent", // Ensures you get a refresh token on first login + scope: scopes, + state: robotId.toString(), + }); + res.redirect(url); }); // Step 2: Handle Google OAuth callback -router.get('/google/callback', requireSignIn, async (req: AuthenticatedRequest, res) => { +router.get( + "/google/callback", + requireSignIn, + async (req: AuthenticatedRequest, res) => { const { code, state } = req.query; try { - if (!state) { - return res.status(400).json({ message: 'Robot ID is required' }); - } + if (!state) { + return res.status(400).json({ message: "Robot ID is required" }); + } - const robotId = state + const robotId = state; - // Get access and refresh tokens - if (typeof code !== 'string') { - return res.status(400).json({ message: 'Invalid code' }); - } - const { tokens } = await oauth2Client.getToken(code); - oauth2Client.setCredentials(tokens); + // Get access and refresh tokens + if (typeof code !== "string") { + return res.status(400).json({ message: "Invalid code" }); + } + const { tokens } = await oauth2Client.getToken(code); + oauth2Client.setCredentials(tokens); - // Get user profile from Google - const oauth2 = google.oauth2({ version: 'v2', auth: oauth2Client }); - const { data: { email } } = await oauth2.userinfo.get(); + // Get user profile from Google + const oauth2 = google.oauth2({ version: "v2", auth: oauth2Client }); + const { + data: { email }, + } = await oauth2.userinfo.get(); - if (!email) { - return res.status(400).json({ message: 'Email not found' }); - } + if (!email) { + return res.status(400).json({ message: "Email not found" }); + } - if (!req.user) { - return res.status(401).send({ error: 'Unauthorized' }); - } + if (!req.user) { + return res.status(401).send({ error: "Unauthorized" }); + } - // Get the currently authenticated user (from `requireSignIn`) - let user = await User.findOne({ where: { id: req.user.id } }); + // Get the currently authenticated user (from `requireSignIn`) + let user = await User.findOne({ where: { id: req.user.id } }); - if (!user) { - return res.status(400).json({ message: 'User not found' }); - } + if (!user) { + return res.status(400).json({ message: "User not found" }); + } - let robot = await Robot.findOne({ where: { 'recording_meta.id': robotId } }); + let robot = await Robot.findOne({ + where: { "recording_meta.id": robotId }, + }); - if (!robot) { - return res.status(400).json({ message: 'Robot not found' }); - } + if (!robot) { + return res.status(400).json({ message: "Robot not found" }); + } - robot = await robot.update({ - google_sheet_email: email, - google_access_token: tokens.access_token, - google_refresh_token: tokens.refresh_token, - }); - capture( - 'maxun-oss-google-sheet-integration-created', - { - user_id: user.id, - robot_id: robot.recording_meta.id, - created_at: new Date().toISOString() - } - ) + robot = await robot.update({ + google_sheet_email: email, + google_access_token: tokens.access_token, + google_refresh_token: tokens.refresh_token, + }); + capture("maxun-oss-google-sheet-integration-created", { + user_id: user.id, + robot_id: robot.recording_meta.id, + created_at: new Date().toISOString(), + }); - // List user's Google Sheets from their Google Drive - const drive = google.drive({ version: 'v3', auth: oauth2Client }); - const response = await drive.files.list({ - q: "mimeType='application/vnd.google-apps.spreadsheet'", // List only Google Sheets files - fields: 'files(id, name)', // Retrieve the ID and name of each file - }); + // List user's Google Sheets from their Google Drive + const drive = google.drive({ version: "v3", auth: oauth2Client }); + const response = await drive.files.list({ + q: "mimeType='application/vnd.google-apps.spreadsheet'", // List only Google Sheets files + fields: "files(id, name)", // Retrieve the ID and name of each file + }); - const files = response.data.files || []; - if (files.length === 0) { - return res.status(404).json({ message: 'No spreadsheets found.' }); - } + const files = response.data.files || []; + if (files.length === 0) { + return res.status(404).json({ message: "No spreadsheets found." }); + } - // Generate JWT token for session - const jwtToken = jwt.sign({ userId: user.id }, process.env.JWT_SECRET as string, { expiresIn: '12h' }); - res.cookie('token', jwtToken, { httpOnly: true }); + // Generate JWT token for session + const jwtToken = jwt.sign( + { id: user.id }, + process.env.JWT_SECRET as string + ); + res.cookie("token", jwtToken, { httpOnly: true }); - res.json({ - message: 'Google authentication successful', - google_sheet_email: robot.google_sheet_email, - jwtToken, - files - }); + // res.json({ + // message: 'Google authentication successful', + // google_sheet_email: robot.google_sheet_email, + // jwtToken, + // files + // }); + + res.cookie("robot_auth_status", "success", { + httpOnly: false, + maxAge: 60000, + }); // 1-minute expiration + res.cookie("robot_auth_message", "Robot successfully authenticated", { + httpOnly: false, + maxAge: 60000, + }); + res.redirect(`http://localhost:5173`); } catch (error: any) { - res.status(500).json({ message: `Google OAuth error: ${error.message}` }); + res.status(500).json({ message: `Google OAuth error: ${error.message}` }); } -}); + } +); // Step 3: Get data from Google Sheets -router.post('/gsheets/data', requireSignIn, async (req: AuthenticatedRequest, res) => { +router.post( + "/gsheets/data", + requireSignIn, + async (req: AuthenticatedRequest, res) => { const { spreadsheetId, robotId } = req.body; if (!req.user) { - return res.status(401).send({ error: 'Unauthorized' }); + return res.status(401).send({ error: "Unauthorized" }); } const user = await User.findByPk(req.user.id, { raw: true }); if (!user) { - return res.status(400).json({ message: 'User not found' }); + return res.status(400).json({ message: "User not found" }); } - const robot = await Robot.findOne({ where: { 'recording_meta.id': robotId }, raw: true }); + const robot = await Robot.findOne({ + where: { "recording_meta.id": robotId }, + raw: true, + }); if (!robot) { - return res.status(400).json({ message: 'Robot not found' }); + return res.status(400).json({ message: "Robot not found" }); } // Set Google OAuth credentials oauth2Client.setCredentials({ - access_token: robot.google_access_token, - refresh_token: robot.google_refresh_token, + access_token: robot.google_access_token, + refresh_token: robot.google_refresh_token, }); - const sheets = google.sheets({ version: 'v4', auth: oauth2Client }); + const sheets = google.sheets({ version: "v4", auth: oauth2Client }); try { - // Fetch data from the spreadsheet (you can let the user choose a specific range too) - const sheetData = await sheets.spreadsheets.values.get({ - spreadsheetId, - range: 'Sheet1!A1:D5', // Default range, could be dynamic based on user input - }); - res.json(sheetData.data); + // Fetch data from the spreadsheet (you can let the user choose a specific range too) + const sheetData = await sheets.spreadsheets.values.get({ + spreadsheetId, + range: "Sheet1!A1:D5", // Default range, could be dynamic based on user input + }); + res.json(sheetData.data); } catch (error: any) { - res.status(500).json({ message: `Error accessing Google Sheets: ${error.message}` }); + res + .status(500) + .json({ message: `Error accessing Google Sheets: ${error.message}` }); } -}); + } +); // Step 4: Get user's Google Sheets files (new route) -router.get('/gsheets/files', requireSignIn, async (req, res) => { - try { - const robotId = req.query.robotId; - const robot = await Robot.findOne({ where: { 'recording_meta.id': robotId }, raw: true }); +router.get("/gsheets/files", requireSignIn, async (req, res) => { + try { + const robotId = req.query.robotId; + const robot = await Robot.findOne({ + where: { "recording_meta.id": robotId }, + raw: true, + }); - if (!robot) { - return res.status(400).json({ message: 'Robot not found' }); - } - - oauth2Client.setCredentials({ - access_token: robot.google_access_token, - refresh_token: robot.google_refresh_token, - }); - - // List user's Google Sheets files from their Google Drive - const drive = google.drive({ version: 'v3', auth: oauth2Client }); - const response = await drive.files.list({ - q: "mimeType='application/vnd.google-apps.spreadsheet'", - fields: 'files(id, name)', - }); - - const files = response.data.files || []; - if (files.length === 0) { - return res.status(404).json({ message: 'No spreadsheets found.' }); - } - - res.json(files); - } catch (error: any) { - console.log('Error fetching Google Sheets files:', error); - res.status(500).json({ message: `Error retrieving Google Sheets files: ${error.message}` }); + if (!robot) { + return res.status(400).json({ message: "Robot not found" }); } + + oauth2Client.setCredentials({ + access_token: robot.google_access_token, + refresh_token: robot.google_refresh_token, + }); + + // List user's Google Sheets files from their Google Drive + const drive = google.drive({ version: "v3", auth: oauth2Client }); + const response = await drive.files.list({ + q: "mimeType='application/vnd.google-apps.spreadsheet'", + fields: "files(id, name)", + }); + + const files = response.data.files || []; + if (files.length === 0) { + return res.status(404).json({ message: "No spreadsheets found." }); + } + + res.json(files); + } catch (error: any) { + console.log("Error fetching Google Sheets files:", error); + res + .status(500) + .json({ + message: `Error retrieving Google Sheets files: ${error.message}`, + }); + } }); // Step 5: Update robot's google_sheet_id when a Google Sheet is selected -router.post('/gsheets/update', requireSignIn, async (req, res) => { - const { spreadsheetId, spreadsheetName, robotId } = req.body; +router.post("/gsheets/update", requireSignIn, async (req, res) => { + const { spreadsheetId, spreadsheetName, robotId } = req.body; - if (!spreadsheetId || !robotId) { - return res.status(400).json({ message: 'Spreadsheet ID and Robot ID are required' }); + if (!spreadsheetId || !robotId) { + return res + .status(400) + .json({ message: "Spreadsheet ID and Robot ID are required" }); + } + + try { + let robot = await Robot.findOne({ + where: { "recording_meta.id": robotId }, + }); + + if (!robot) { + return res.status(404).json({ message: "Robot not found" }); } - try { - let robot = await Robot.findOne({ where: { 'recording_meta.id': robotId } }); + await robot.update({ + google_sheet_id: spreadsheetId, + google_sheet_name: spreadsheetName, + }); - if (!robot) { - return res.status(404).json({ message: 'Robot not found' }); - } - - await robot.update({ google_sheet_id: spreadsheetId, google_sheet_name: spreadsheetName }); - - res.json({ message: 'Robot updated with selected Google Sheet ID' }); - } catch (error: any) { - res.status(500).json({ message: `Error updating robot: ${error.message}` }); - } + res.json({ message: "Robot updated with selected Google Sheet ID" }); + } catch (error: any) { + res.status(500).json({ message: `Error updating robot: ${error.message}` }); + } }); -router.post('/gsheets/remove', requireSignIn, async (req: AuthenticatedRequest, res) => { +router.post( + "/gsheets/remove", + requireSignIn, + async (req: AuthenticatedRequest, res) => { const { robotId } = req.body; if (!robotId) { - return res.status(400).json({ message: 'Robot ID is required' }); + return res.status(400).json({ message: "Robot ID is required" }); } if (!req.user) { - return res.status(401).send({ error: 'Unauthorized' }); + return res.status(401).send({ error: "Unauthorized" }); } try { - let robot = await Robot.findOne({ where: { 'recording_meta.id': robotId } }); + let robot = await Robot.findOne({ + where: { "recording_meta.id": robotId }, + }); - if (!robot) { - return res.status(404).json({ message: 'Robot not found' }); - } + if (!robot) { + return res.status(404).json({ message: "Robot not found" }); + } - await robot.update({ - google_sheet_id: null, - google_sheet_name: null, - google_sheet_email: null, - google_access_token: null, - google_refresh_token: null - }); + await robot.update({ + google_sheet_id: null, + google_sheet_name: null, + google_sheet_email: null, + google_access_token: null, + google_refresh_token: null, + }); - capture( - 'maxun-oss-google-sheet-integration-removed', - { - user_id: req.user.id, - robot_id: robotId, - deleted_at: new Date().toISOString() - } - ) + capture("maxun-oss-google-sheet-integration-removed", { + user_id: req.user.id, + robot_id: robotId, + deleted_at: new Date().toISOString(), + }); - res.json({ message: 'Google Sheets integration removed successfully' }); + res.json({ message: "Google Sheets integration removed successfully" }); } catch (error: any) { - res.status(500).json({ message: `Error removing Google Sheets integration: ${error.message}` }); + res + .status(500) + .json({ + message: `Error removing Google Sheets integration: ${error.message}`, + }); } -}); + } +); diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.ts index 9182c44c..d68a889e 100644 --- a/server/src/routes/proxy.ts +++ b/server/src/routes/proxy.ts @@ -1,8 +1,10 @@ import { Router, Request, Response } from 'express'; -import { chromium } from "playwright"; +import { chromium } from 'playwright-extra'; +import stealthPlugin from 'puppeteer-extra-plugin-stealth'; import User from '../models/User'; import { encrypt, decrypt } from '../utils/auth'; import { requireSignIn } from '../middlewares/auth'; +chromium.use(stealthPlugin()); export const router = Router(); @@ -74,8 +76,6 @@ router.get('/test', requireSignIn, async (req: AuthenticatedRequest, res: Respon const decryptedProxyUsername = user.proxy_username ? decrypt(user.proxy_username) : null; const decryptedProxyPassword = user.proxy_password ? decrypt(user.proxy_password) : null; - console.log(`Decrypted vals: ${decryptedProxyPassword}, ${decryptedProxyUrl}, ${decryptedProxyUsername}`); - const proxyOptions: any = { server: decryptedProxyUrl, ...(decryptedProxyUsername && decryptedProxyPassword && { @@ -170,8 +170,6 @@ export const getDecryptedProxyConfig = async (userId: string) => { const decryptedProxyUsername = user.proxy_username ? decrypt(user.proxy_username) : null; const decryptedProxyPassword = user.proxy_password ? decrypt(user.proxy_password) : null; - console.log(`Decrypting ${decryptedProxyUrl}, ${decryptedProxyUsername}, ${decryptedProxyPassword}`); - return { proxy_url: decryptedProxyUrl, proxy_username: decryptedProxyUsername, diff --git a/server/src/routes/record.ts b/server/src/routes/record.ts index 56089118..51d3ff92 100644 --- a/server/src/routes/record.ts +++ b/server/src/routes/record.ts @@ -11,14 +11,14 @@ import { stopRunningInterpretation, getRemoteBrowserCurrentUrl, getRemoteBrowserCurrentTabs, } from '../browser-management/controller' -import { chromium } from 'playwright'; +import { chromium } from 'playwright-extra'; import stealthPlugin from 'puppeteer-extra-plugin-stealth'; import logger from "../logger"; import { getDecryptedProxyConfig } from './proxy'; import { requireSignIn } from '../middlewares/auth'; export const router = Router(); -// chromium.use(stealthPlugin()); +chromium.use(stealthPlugin()); export interface AuthenticatedRequest extends Request { @@ -57,7 +57,6 @@ router.get('/start', requireSignIn, async (req: AuthenticatedRequest, res: Respo } const id = initializeRemoteBrowserForRecording(req.user.id); - console.log('id start:', id); return res.send(id); }); @@ -70,7 +69,6 @@ router.post('/start', requireSignIn, (req: AuthenticatedRequest, res:Response) = return res.status(401).send('User not authenticated'); } const id = initializeRemoteBrowserForRecording(req.user.id); - console.log('id start POST:', id); return res.send(id); }); diff --git a/server/src/routes/storage.ts b/server/src/routes/storage.ts index 228f60de..f84583d0 100644 --- a/server/src/routes/storage.ts +++ b/server/src/routes/storage.ts @@ -1,7 +1,8 @@ import { Router } from 'express'; import logger from "../logger"; import { createRemoteBrowserForRun, destroyRemoteBrowser } from "../browser-management/controller"; -import { chromium } from "playwright"; +import { chromium } from 'playwright-extra'; +import stealthPlugin from 'puppeteer-extra-plugin-stealth'; import { browserPool } from "../server"; import { uuid } from "uuidv4"; import moment from 'moment-timezone'; @@ -16,6 +17,8 @@ import { workflowQueue } from '../worker'; import { AuthenticatedRequest } from './record'; import { computeNextRun } from '../utils/schedule'; import { capture } from "../utils/analytics"; +import { tryCatch } from 'bullmq'; +chromium.use(stealthPlugin()); export const router = Router(); @@ -57,6 +60,217 @@ router.get('/recordings/:id', requireSignIn, async (req, res) => { } }) +router.get(('/recordings/:id/runs'), requireSignIn, async (req, res) => { + try { + const runs = await Run.findAll({ + where: { + robotMetaId: req.params.id + }, + raw: true + }); + const formattedRuns = runs.map(formatRunResponse); + const response = { + statusCode: 200, + messageCode: "success", + runs: { + totalCount: formattedRuns.length, + items: formattedRuns, + }, + }; + + res.status(200).json(response); +} catch (error) { + console.error("Error fetching runs:", error); + res.status(500).json({ + statusCode: 500, + messageCode: "error", + message: "Failed to retrieve runs", + }); +} +}) + +function formatRunResponse(run: any) { + const formattedRun = { + id: run.id, + status: run.status, + name: run.name, + robotId: run.robotMetaId, // Renaming robotMetaId to robotId + startedAt: run.startedAt, + finishedAt: run.finishedAt, + runId: run.runId, + runByUserId: run.runByUserId, + runByScheduleId: run.runByScheduleId, + runByAPI: run.runByAPI, + data: {}, + screenshot: null, + }; + + if (run.serializableOutput && run.serializableOutput['item-0']) { + formattedRun.data = run.serializableOutput['item-0']; + } else if (run.binaryOutput && run.binaryOutput['item-0']) { + formattedRun.screenshot = run.binaryOutput['item-0']; + } + + return formattedRun; +} + +/** + * PUT endpoint to update the name and limit of a robot. + */ +router.put('/recordings/:id', requireSignIn, async (req: AuthenticatedRequest, res) => { + try { + const { id } = req.params; + const { name, limit } = req.body; + + // Validate input + if (!name && limit === undefined) { + return res.status(400).json({ error: 'Either "name" or "limit" must be provided.' }); + } + + // Fetch the robot by ID + const robot = await Robot.findOne({ where: { 'recording_meta.id': id } }); + + if (!robot) { + return res.status(404).json({ error: 'Robot not found.' }); + } + + // Update fields if provided + if (name) { + robot.set('recording_meta', { ...robot.recording_meta, name }); + } + + // Update the limit + if (limit !== undefined) { + const workflow = [...robot.recording.workflow]; // Create a copy of the workflow + + // Ensure the workflow structure is valid before updating + if ( + workflow.length > 0 && + workflow[0]?.what?.[0] + ) { + // Create a new workflow object with the updated limit + const updatedWorkflow = workflow.map((step, index) => { + if (index === 0) { // Assuming you want to update the first step + return { + ...step, + what: step.what.map((action, actionIndex) => { + if (actionIndex === 0) { // Assuming the first action needs updating + return { + ...action, + args: (action.args ?? []).map((arg, argIndex) => { + if (argIndex === 0) { // Assuming the first argument needs updating + return { ...arg, limit }; + } + return arg; + }), + }; + } + return action; + }), + }; + } + return step; + }); + + // Replace the workflow in the recording object + robot.set('recording', { ...robot.recording, workflow: updatedWorkflow }); + } else { + return res.status(400).json({ error: 'Invalid workflow structure for updating limit.' }); + } + } + + await robot.save(); + + const updatedRobot = await Robot.findOne({ where: { 'recording_meta.id': id } }); + + logger.log('info', `Robot with ID ${id} was updated successfully.`); + + return res.status(200).json({ message: 'Robot updated successfully', robot }); + } catch (error) { + // Safely handle the error type + if (error instanceof Error) { + logger.log('error', `Error updating robot with ID ${req.params.id}: ${error.message}`); + return res.status(500).json({ error: error.message }); + } else { + logger.log('error', `Unknown error updating robot with ID ${req.params.id}`); + return res.status(500).json({ error: 'An unknown error occurred.' }); + } + } +}); + + +/** + * POST endpoint to duplicate a robot and update its target URL. + */ +router.post('/recordings/:id/duplicate', requireSignIn, async (req: AuthenticatedRequest, res) => { + try { + const { id } = req.params; + const { targetUrl } = req.body; + + if (!targetUrl) { + return res.status(400).json({ error: 'The "targetUrl" field is required.' }); + } + + const originalRobot = await Robot.findOne({ where: { 'recording_meta.id': id } }); + + if (!originalRobot) { + return res.status(404).json({ error: 'Original robot not found.' }); + } + + const lastWord = targetUrl.split('/').filter(Boolean).pop() || 'Unnamed'; + + const workflow = originalRobot.recording.workflow.map((step) => { + if (step.where?.url && step.where.url !== "about:blank") { + step.where.url = targetUrl; + } + + step.what.forEach((action) => { + if (action.action === "goto" && action.args?.length) { + action.args[0] = targetUrl; + } + }); + + return step; + }); + + const currentTimestamp = new Date().toISOString(); + + const newRobot = await Robot.create({ + id: uuid(), + userId: originalRobot.userId, + recording_meta: { + ...originalRobot.recording_meta, + id: uuid(), + name: `${originalRobot.recording_meta.name} (${lastWord})`, + createdAt: currentTimestamp, + updatedAt: currentTimestamp, + }, + recording: { ...originalRobot.recording, workflow }, + google_sheet_email: null, + google_sheet_name: null, + google_sheet_id: null, + google_access_token: null, + google_refresh_token: null, + schedule: null, + }); + + logger.log('info', `Robot with ID ${id} duplicated successfully as ${newRobot.id}.`); + + return res.status(201).json({ + message: 'Robot duplicated and target URL updated successfully.', + robot: newRobot, + }); + } catch (error) { + if (error instanceof Error) { + logger.log('error', `Error duplicating robot with ID ${req.params.id}: ${error.message}`); + return res.status(500).json({ error: error.message }); + } else { + logger.log('error', `Unknown error duplicating robot with ID ${req.params.id}`); + return res.status(500).json({ error: 'An unknown error occurred.' }); + } + } +}); + /** * DELETE endpoint for deleting a recording from the storage. */ diff --git a/server/src/storage/mino.ts b/server/src/storage/mino.ts index 04efaf8c..0f29c3cd 100644 --- a/server/src/storage/mino.ts +++ b/server/src/storage/mino.ts @@ -12,47 +12,48 @@ const minioClient = new Client({ minioClient.bucketExists('maxun-test') .then((exists) => { if (exists) { - console.log('MinIO was connected successfully.'); + console.log('MinIO connected successfully.'); } else { - console.log('Bucket does not exist, but MinIO was connected.'); + console.log('MinIO connected successfully.'); } }) .catch((err) => { console.error('Error connecting to MinIO:', err); }) -async function createBucketWithPolicy(bucketName: string, policy?: 'public-read' | 'private') { +async function createBucketWithPolicy(bucketName: string, policy = 'public-read') { try { const bucketExists = await minioClient.bucketExists(bucketName); if (!bucketExists) { await minioClient.makeBucket(bucketName); console.log(`Bucket ${bucketName} created successfully.`); - - if (policy === 'public-read') { - // Define a public-read policy - const policyJSON = { - Version: "2012-10-17", - Statement: [ - { - Effect: "Allow", - Principal: "", - Action: ["s3:GetObject"], - Resource: [`arn:aws:s3:::${bucketName}/*`] - } - ] - }; - await minioClient.setBucketPolicy(bucketName, JSON.stringify(policyJSON)); - console.log(`Public-read policy applied to bucket ${bucketName}.`); - } } else { console.log(`Bucket ${bucketName} already exists.`); } + + if (policy === 'public-read') { + // Apply public-read policy after confirming the bucket exists + const policyJSON = { + Version: "2012-10-17", + Statement: [ + { + Effect: "Allow", + Principal: "*", + Action: ["s3:GetObject"], + Resource: [`arn:aws:s3:::${bucketName}/*`] + } + ] + }; + await minioClient.setBucketPolicy(bucketName, JSON.stringify(policyJSON)); + console.log(`Public-read policy applied to bucket ${bucketName}.`); + } } catch (error) { console.error('Error in bucket creation or policy application:', error); } } + class BinaryOutputService { private bucketName: string; diff --git a/server/src/utils/auth.ts b/server/src/utils/auth.ts index f8313df7..ddab4bd4 100644 --- a/server/src/utils/auth.ts +++ b/server/src/utils/auth.ts @@ -6,29 +6,37 @@ export const hashPassword = (password: string): Promise => { return new Promise((resolve, reject) => { bcrypt.genSalt(12, (err, salt) => { if (err) { - reject(err) + reject(err); } bcrypt.hash(password, salt, (err, hash) => { if (err) { - reject(err) + reject(err); } - resolve(hash) - }) - }) - }) -} + resolve(hash); + }); + }); + }); +}; // password from frontend and hash from database export const comparePassword = (password: string, hash: string): Promise => { - return bcrypt.compare(password, hash) -} + return bcrypt.compare(password, hash); +}; export const encrypt = (text: string): string => { const ivLength = 16; const iv = crypto.randomBytes(ivLength); const algorithm = 'aes-256-cbc'; - const key = Buffer.from(getEnvVariable('ENCRYPTION_KEY'), 'hex'); - const cipher = crypto.createCipheriv(algorithm, key, iv); + + // Retrieve the encryption key or generate a new one if invalid or empty + let key = getEnvVariable('ENCRYPTION_KEY'); + if (!key || key.length !== 64) { // aes-256-cbc requires a 256-bit key, which is 64 hex characters + console.warn('Invalid or missing ENCRYPTION_KEY, generating a new one.'); + key = crypto.randomBytes(32).toString('hex'); // Generate a new 256-bit (32-byte) key + } + const keyBuffer = Buffer.from(key, 'hex'); + + const cipher = crypto.createCipheriv(algorithm, keyBuffer, iv); let encrypted = cipher.update(text, 'utf8', 'hex'); encrypted += cipher.final('hex'); return `${iv.toString('hex')}:${encrypted}`; @@ -37,9 +45,17 @@ export const encrypt = (text: string): string => { export const decrypt = (encryptedText: string): string => { const [iv, encrypted] = encryptedText.split(':'); const algorithm = "aes-256-cbc"; - const key = Buffer.from(getEnvVariable('ENCRYPTION_KEY'), 'hex'); - const decipher = crypto.createDecipheriv(algorithm, key, Buffer.from(iv, 'hex')); + + // Retrieve the encryption key or generate a new one if invalid or empty + let key = getEnvVariable('ENCRYPTION_KEY'); + if (!key || key.length !== 64) { // aes-256-cbc requires a 256-bit key, which is 64 hex characters + console.warn('Invalid or missing ENCRYPTION_KEY, generating a new one.'); + key = crypto.randomBytes(32).toString('hex'); // Generate a new 256-bit (32-byte) key + } + const keyBuffer = Buffer.from(key, 'hex'); + + const decipher = crypto.createDecipheriv(algorithm, keyBuffer, Buffer.from(iv, 'hex')); let decrypted = decipher.update(encrypted, 'hex', 'utf8'); decrypted += decipher.final('utf8'); return decrypted; -}; \ No newline at end of file +}; diff --git a/server/src/worker.ts b/server/src/worker.ts index fd3470d4..3a82ee73 100644 --- a/server/src/worker.ts +++ b/server/src/worker.ts @@ -5,11 +5,6 @@ import { handleRunRecording } from "./workflow-management/scheduler"; import Robot from './models/Robot'; import { computeNextRun } from './utils/schedule'; -console.log('Environment variables:', { - REDIS_HOST: process.env.REDIS_HOST, - REDIS_PORT: process.env.REDIS_PORT, -}); - const connection = new IORedis({ host: process.env.REDIS_HOST, port: process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT, 10) : 6379, @@ -68,7 +63,6 @@ console.log('Worker is running...'); async function jobCounts() { const jobCounts = await workflowQueue.getJobCounts(); - console.log('Jobs:', jobCounts); } jobCounts(); @@ -78,8 +72,4 @@ process.on('SIGINT', () => { process.exit(); }); -export { workflowQueue, worker }; - -export const temp = () => { - console.log('temp'); -} \ No newline at end of file +export { workflowQueue, worker }; \ No newline at end of file diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index a2db820e..7801a20e 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -128,7 +128,6 @@ export class WorkflowGenerator { */ private registerEventHandlers = (socket: Socket) => { socket.on('save', (data) => { - console.log('Received data:', data); const { fileName, userId } = data; logger.log('debug', `Saving workflow ${fileName} for user ID ${userId}`); this.saveNewWorkflow(fileName, userId); @@ -560,6 +559,8 @@ export class WorkflowGenerator { if (this.listSelector !== '') { const childSelectors = await getChildSelectors(page, this.listSelector || ''); this.socket.emit('highlighter', { rect, selector: displaySelector, elementInfo, childSelectors }) + console.log(`Child Selectors: ${childSelectors}`) + console.log(`Parent Selector: ${this.listSelector}`) } else { this.socket.emit('highlighter', { rect, selector: displaySelector, elementInfo }); } diff --git a/server/src/workflow-management/scheduler/index.ts b/server/src/workflow-management/scheduler/index.ts index a94c6d16..082fcf2e 100644 --- a/server/src/workflow-management/scheduler/index.ts +++ b/server/src/workflow-management/scheduler/index.ts @@ -1,5 +1,6 @@ import { uuid } from "uuidv4"; -import { chromium } from "playwright"; +import { chromium } from 'playwright-extra'; +import stealthPlugin from 'puppeteer-extra-plugin-stealth'; import { io, Socket } from "socket.io-client"; import { createRemoteBrowserForRun, destroyRemoteBrowser } from '../../browser-management/controller'; import logger from '../../logger'; @@ -10,6 +11,7 @@ import Run from "../../models/Run"; import { getDecryptedProxyConfig } from "../../routes/proxy"; import { BinaryOutputService } from "../../storage/mino"; import { capture } from "../../utils/analytics"; +chromium.use(stealthPlugin()); async function createWorkflowAndStoreMetadata(id: string, userId: string) { try { diff --git a/server/src/workflow-management/selector.ts b/server/src/workflow-management/selector.ts index 18b878ff..193de891 100644 --- a/server/src/workflow-management/selector.ts +++ b/server/src/workflow-management/selector.ts @@ -794,6 +794,7 @@ export const getNonUniqueSelectors = async (page: Page, coordinates: Coordinates export const getChildSelectors = async (page: Page, parentSelector: string): Promise => { try { const childSelectors = await page.evaluate((parentSelector: string) => { + // Function to get a non-unique selector based on tag and class (if present) function getNonUniqueSelector(element: HTMLElement): string { let selector = element.tagName.toLowerCase(); @@ -811,6 +812,7 @@ export const getChildSelectors = async (page: Page, parentSelector: string): Pro return selector; } + // Function to generate selector path from an element to its parent function getSelectorPath(element: HTMLElement | null): string { if (!element || !element.parentElement) return ''; @@ -820,22 +822,33 @@ export const getChildSelectors = async (page: Page, parentSelector: string): Pro return `${parentSelector} > ${elementSelector}`; } - function getAllDescendantSelectors(element: HTMLElement, stopAtParent: HTMLElement | null): string[] { + // Function to recursively get all descendant selectors + function getAllDescendantSelectors(element: HTMLElement): string[] { let selectors: string[] = []; const children = Array.from(element.children) as HTMLElement[]; for (const child of children) { - selectors.push(getSelectorPath(child)); - selectors = selectors.concat(getAllDescendantSelectors(child, stopAtParent)); + const childPath = getSelectorPath(child); + if (childPath) { + selectors.push(childPath); // Add direct child path + selectors = selectors.concat(getAllDescendantSelectors(child)); // Recursively process descendants + } } return selectors; } - const parentElement = document.querySelector(parentSelector) as HTMLElement; - if (!parentElement) return []; + // Find all occurrences of the parent selector in the DOM + const parentElements = Array.from(document.querySelectorAll(parentSelector)) as HTMLElement[]; + const allChildSelectors = new Set(); // Use a set to ensure uniqueness - return getAllDescendantSelectors(parentElement, parentElement); + // Process each parent element and its descendants + parentElements.forEach((parentElement) => { + const descendantSelectors = getAllDescendantSelectors(parentElement); + descendantSelectors.forEach((selector) => allChildSelectors.add(selector)); // Add selectors to the set + }); + + return Array.from(allChildSelectors); // Convert the set back to an array }, parentSelector); return childSelectors || []; @@ -845,6 +858,7 @@ export const getChildSelectors = async (page: Page, parentSelector: string): Pro } }; + /** * Returns the first pair from the given workflow that contains the given selector * inside the where condition, and it is the only selector there. diff --git a/server/src/workflow-management/utils.ts b/server/src/workflow-management/utils.ts index c10d2c13..b3dadd60 100644 --- a/server/src/workflow-management/utils.ts +++ b/server/src/workflow-management/utils.ts @@ -17,7 +17,7 @@ export const getBestSelectorForAction = (action: Action) => { selectors?.text?.length != null && selectors?.text?.length < 25 && action.hasOnlyText - ? `text=${selectors.text}` + ? selectors.generalSelector : null; if (action.tagName === TagName.Input) { diff --git a/src/api/storage.ts b/src/api/storage.ts index 9b4b06b2..4b2f4e80 100644 --- a/src/api/storage.ts +++ b/src/api/storage.ts @@ -5,6 +5,11 @@ import { ScheduleSettings } from "../components/molecules/ScheduleSettings"; import { CreateRunResponse, ScheduleRunResponse } from "../pages/MainPage"; import { apiUrl } from "../apiConfig"; + + + + + export const getStoredRecordings = async (): Promise => { try { const response = await axios.get(`${apiUrl}/storage/recordings`); @@ -19,6 +24,36 @@ export const getStoredRecordings = async (): Promise => { } }; +export const updateRecording = async (id: string, data: { name?: string; limit?: number }): Promise => { + try { + const response = await axios.put(`${apiUrl}/storage/recordings/${id}`, data); + if (response.status === 200) { + return true; + } else { + throw new Error(`Couldn't update recording with id ${id}`); + } + } catch (error: any) { + console.error(`Error updating recording: ${error.message}`); + return false; + } +}; + +export const duplicateRecording = async (id: string, targetUrl: string): Promise => { + try { + const response = await axios.post(`${apiUrl}/storage/recordings/${id}/duplicate`, { + targetUrl, + }); + if (response.status === 201) { + return response.data; // Returns the duplicated robot details + } else { + throw new Error(`Couldn't duplicate recording with id ${id}`); + } + } catch (error: any) { + console.error(`Error duplicating recording: ${error.message}`); + return null; + } +}; + export const getStoredRuns = async (): Promise => { try { const response = await axios.get(`${apiUrl}/storage/runs`); @@ -47,18 +82,49 @@ export const getStoredRecording = async (id: string) => { } } + + +export const checkRunsForRecording = async (id: string): Promise => { + + + try { + const response = await axios.get(`${apiUrl}/storage/recordings/${id}/runs`); + + const runs = response.data; + console.log(runs.runs.totalCount) + return runs.runs.totalCount > 0; + } catch (error) { + console.error('Error checking runs for recording:', error); + return false; + } +}; + + export const deleteRecordingFromStorage = async (id: string): Promise => { + + const hasRuns = await checkRunsForRecording(id); + + if (hasRuns) { + + return false; + } try { const response = await axios.delete(`${apiUrl}/storage/recordings/${id}`); if (response.status === 200) { - return response.data; + + return true; } else { throw new Error(`Couldn't delete stored recording ${id}`); } } catch (error: any) { console.log(error); + return false; } + + + + }; export const deleteRunFromStorage = async (id: string): Promise => { @@ -93,7 +159,7 @@ export const createRunForStoredRecording = async (id: string, settings: RunSetti try { const response = await axios.put( `${apiUrl}/storage/runs/${id}`, - { ...settings }); + { ...settings }); if (response.status === 200) { return response.data; } else { diff --git a/src/components/molecules/IntegrationSettings.tsx b/src/components/molecules/IntegrationSettings.tsx index 1150dd72..c31605de 100644 --- a/src/components/molecules/IntegrationSettings.tsx +++ b/src/components/molecules/IntegrationSettings.tsx @@ -1,227 +1,323 @@ -import React, { useState, useEffect } from 'react'; +import React, { useState, useEffect } from "react"; import { GenericModal } from "../atoms/GenericModal"; -import { MenuItem, Typography, CircularProgress, Alert, AlertTitle, Chip } from "@mui/material"; +import { + MenuItem, + Typography, + CircularProgress, + Alert, + AlertTitle, + Chip, +} from "@mui/material"; import Button from "@mui/material/Button"; import TextField from "@mui/material/TextField"; -import axios from 'axios'; -import { useGlobalInfoStore } from '../../context/globalInfo'; -import { getStoredRecording } from '../../api/storage'; -import { apiUrl } from '../../apiConfig.js'; +import axios from "axios"; +import { useGlobalInfoStore } from "../../context/globalInfo"; +import { getStoredRecording } from "../../api/storage"; +import { apiUrl } from "../../apiConfig.js"; +import Cookies from 'js-cookie'; interface IntegrationProps { - isOpen: boolean; - handleStart: (data: IntegrationSettings) => void; - handleClose: () => void; + isOpen: boolean; + handleStart: (data: IntegrationSettings) => void; + handleClose: () => void; } export interface IntegrationSettings { - spreadsheetId: string; - spreadsheetName: string; - data: string; + spreadsheetId: string; + spreadsheetName: string; + data: string; } -export const IntegrationSettingsModal = ({ isOpen, handleStart, handleClose }: IntegrationProps) => { - const [settings, setSettings] = useState({ - spreadsheetId: '', - spreadsheetName: '', - data: '', - }); +export const IntegrationSettingsModal = ({ + isOpen, + handleStart, + handleClose, +}: IntegrationProps) => { + const [settings, setSettings] = useState({ + spreadsheetId: "", + spreadsheetName: "", + data: "", + }); - const [spreadsheets, setSpreadsheets] = useState<{ id: string, name: string }[]>([]); - const [loading, setLoading] = useState(false); - const [error, setError] = useState(null); + const [spreadsheets, setSpreadsheets] = useState< + { id: string; name: string }[] + >([]); + const [loading, setLoading] = useState(false); + const [error, setError] = useState(null); - const { recordingId, notify } = useGlobalInfoStore(); - const [recording, setRecording] = useState(null); + const { recordingId, notify } = useGlobalInfoStore(); + const [recording, setRecording] = useState(null); - const authenticateWithGoogle = () => { - window.location.href = `${apiUrl}/auth/google?robotId=${recordingId}`; - }; + const authenticateWithGoogle = () => { + window.location.href = `${apiUrl}/auth/google?robotId=${recordingId}`; + }; - const handleOAuthCallback = async () => { - try { - const response = await axios.get(`${apiUrl}/auth/google/callback`); - const { google_sheet_email, files } = response.data; - } catch (error) { - setError('Error authenticating with Google'); + const handleOAuthCallback = async () => { + try { + const response = await axios.get(`${apiUrl}/auth/google/callback`); + const { google_sheet_email, files } = response.data; + } catch (error) { + setError("Error authenticating with Google"); + } + }; + + const fetchSpreadsheetFiles = async () => { + try { + const response = await axios.get( + `${apiUrl}/auth/gsheets/files?robotId=${recordingId}`, + { + withCredentials: true, } - }; + ); + setSpreadsheets(response.data); + } catch (error: any) { + console.error( + "Error fetching spreadsheet files:", + error.response?.data?.message || error.message + ); + notify( + "error", + `Error fetching spreadsheet files: ${ + error.response?.data?.message || error.message + }` + ); + } + }; - const fetchSpreadsheetFiles = async () => { - try { - const response = await axios.get(`${apiUrl}/auth/gsheets/files?robotId=${recordingId}`, { - withCredentials: true, - }); - setSpreadsheets(response.data); - } catch (error: any) { - console.error('Error fetching spreadsheet files:', error.response?.data?.message || error.message); - notify('error', `Error fetching spreadsheet files: ${error.response?.data?.message || error.message}`); - } - }; - - const handleSpreadsheetSelect = (e: React.ChangeEvent) => { - const selectedSheet = spreadsheets.find(sheet => sheet.id === e.target.value); - if (selectedSheet) { - setSettings({ ...settings, spreadsheetId: selectedSheet.id, spreadsheetName: selectedSheet.name }); - } - }; - - const updateGoogleSheetId = async () => { - try { - const response = await axios.post( - `${apiUrl}/auth/gsheets/update`, - { spreadsheetId: settings.spreadsheetId, spreadsheetName: settings.spreadsheetName, robotId: recordingId }, - { withCredentials: true } - ); - console.log('Google Sheet ID updated:', response.data); - } catch (error: any) { - console.error('Error updating Google Sheet ID:', error.response?.data?.message || error.message); - } - }; - - const removeIntegration = async () => { - try { - await axios.post( - `${apiUrl}/auth/gsheets/remove`, - { robotId: recordingId }, - { withCredentials: true } - ); - - setRecording(null); - setSpreadsheets([]); - setSettings({ spreadsheetId: '', spreadsheetName: '', data: '' }); - } catch (error: any) { - console.error('Error removing Google Sheets integration:', error.response?.data?.message || error.message); - } - }; - - useEffect(() => { - // Check if we're on the callback URL - const urlParams = new URLSearchParams(window.location.search); - const code = urlParams.get('code'); - if (code) { - handleOAuthCallback(); - } - - const fetchRecordingInfo = async () => { - if (!recordingId) return; - const recording = await getStoredRecording(recordingId); - if (recording) { - setRecording(recording); - } - }; - - fetchRecordingInfo(); - }, [recordingId]); - - return ( - -
- Integrate with Google Sheet - - {recording && recording.google_sheet_id ? ( - <> - - Google Sheet Integrated Successfully. - Every time this robot creates a successful run, its captured data is appended to your {recording.google_sheet_name} Google Sheet. You can check the data updates here. -
- Note: The data extracted before integrating with Google Sheets will not be synced in the Google Sheet. Only the data extracted after the integration will be synced. - - - - ) : ( - <> - {!recording?.google_sheet_email ? ( - <> -

If you enable this option, every time this robot runs a task successfully, its captured data will be appended to your Google Sheet.

- - - ) : ( - <> - {recording.google_sheet_email && ( - - Authenticated as: {recording.google_sheet_email} - - )} - - {loading ? ( - - ) : error ? ( - {error} - ) : spreadsheets.length === 0 ? ( - <> -
- - -
- - ) : ( - <> - - {spreadsheets.map(sheet => ( - - {sheet.name} - - ))} - - - {settings.spreadsheetId && ( - - Selected Sheet: {spreadsheets.find(s => s.id === settings.spreadsheetId)?.name} (ID: {settings.spreadsheetId}) - - )} - - - - )} - - )} - - )} -
- + const handleSpreadsheetSelect = (e: React.ChangeEvent) => { + const selectedSheet = spreadsheets.find( + (sheet) => sheet.id === e.target.value ); + if (selectedSheet) { + setSettings({ + ...settings, + spreadsheetId: selectedSheet.id, + spreadsheetName: selectedSheet.name, + }); + } + }; + + const updateGoogleSheetId = async () => { + try { + const response = await axios.post( + `${apiUrl}/auth/gsheets/update`, + { + spreadsheetId: settings.spreadsheetId, + spreadsheetName: settings.spreadsheetName, + robotId: recordingId, + }, + { withCredentials: true } + ); + notify(`success`, `Google Sheet selected successfully`) + console.log("Google Sheet ID updated:", response.data); + } catch (error: any) { + console.error( + "Error updating Google Sheet ID:", + error.response?.data?.message || error.message + ); + } + }; + + const removeIntegration = async () => { + try { + await axios.post( + `${apiUrl}/auth/gsheets/remove`, + { robotId: recordingId }, + { withCredentials: true } + ); + + setRecording(null); + setSpreadsheets([]); + setSettings({ spreadsheetId: "", spreadsheetName: "", data: "" }); + } catch (error: any) { + console.error( + "Error removing Google Sheets integration:", + error.response?.data?.message || error.message + ); + } + }; + + useEffect(() => { + // Check if there is a success message in cookies + const status = Cookies.get("robot_auth_status"); + const message = Cookies.get("robot_auth_message"); + + if (status === "success" && message) { + notify("success", message); + // Clear the cookies after reading + Cookies.remove("robot_auth_status"); + Cookies.remove("robot_auth_message"); + } + + // Check if we're on the callback URL + const urlParams = new URLSearchParams(window.location.search); + const code = urlParams.get("code"); + if (code) { + handleOAuthCallback(); + } + + const fetchRecordingInfo = async () => { + if (!recordingId) return; + const recording = await getStoredRecording(recordingId); + if (recording) { + setRecording(recording); + } + }; + + fetchRecordingInfo(); + }, [recordingId]); + + return ( + +
+ + Integrate with Google Sheet{" "} + {/* */} + + + {recording && recording.google_sheet_id ? ( + <> + + Google Sheet Integrated Successfully. + Every time this robot creates a successful run, its captured data + is appended to your {recording.google_sheet_name} Google Sheet. + You can check the data updates{" "} + + here + + . +
+ Note: The data extracted before integrating with + Google Sheets will not be synced in the Google Sheet. Only the + data extracted after the integration will be synced. + + + + ) : ( + <> + {!recording?.google_sheet_email ? ( + <> +

+ If you enable this option, every time this robot runs a task + successfully, its captured data will be appended to your + Google Sheet. +

+ + + ) : ( + <> + {recording.google_sheet_email && ( + + Authenticated as: {recording.google_sheet_email} + + )} + + {loading ? ( + + ) : error ? ( + {error} + ) : spreadsheets.length === 0 ? ( + <> +
+ + +
+ + ) : ( + <> + + {spreadsheets.map((sheet) => ( + + {sheet.name} + + ))} + + + {settings.spreadsheetId && ( + + Selected Sheet:{" "} + { + spreadsheets.find( + (s) => s.id === settings.spreadsheetId + )?.name + }{" "} + (ID: {settings.spreadsheetId}) + + )} + + + + )} + + )} + + )} +
+ + ); +}; + +export const modalStyle = { + top: "40%", + left: "50%", + transform: "translate(-50%, -50%)", + width: "50%", + backgroundColor: "background.paper", + p: 4, + height: "fit-content", + display: "block", + padding: "20px", }; diff --git a/src/components/molecules/NavBar.tsx b/src/components/molecules/NavBar.tsx index 2bbaa84b..11a76115 100644 --- a/src/components/molecules/NavBar.tsx +++ b/src/components/molecules/NavBar.tsx @@ -3,8 +3,12 @@ import axios from 'axios'; import styled from "styled-components"; import { stopRecording } from "../../api/recording"; import { useGlobalInfoStore } from "../../context/globalInfo"; -import { IconButton, Menu, MenuItem, Typography, Avatar, Tooltip } from "@mui/material"; + +import { IconButton, Menu, MenuItem, Typography, Avatar, Tooltip,Chip } from "@mui/material"; import { AccountCircle, Logout, Clear, Brightness4, Brightness7 } from "@mui/icons-material"; + +import { AccountCircle, Logout, Clear } from "@mui/icons-material"; + import { useNavigate } from 'react-router-dom'; import { AuthContext } from '../../context/auth'; import { SaveRecording } from '../molecules/SaveRecording'; @@ -57,25 +61,38 @@ export const NavBar: React.FC = ({ recordingName, isRecording }) =>
-
- Maxun -
+ +
Maxun
+
- {user ? ( -
- {!isRecording ? ( - <> - + {!isRecording ? ( + <> + + + + + @@ -107,6 +124,7 @@ export const NavBar: React.FC = ({ recordingName, isRecording }) => {darkMode ? : } + diff --git a/src/components/molecules/RecordingsTable.tsx b/src/components/molecules/RecordingsTable.tsx index c5895193..e9f0aebc 100644 --- a/src/components/molecules/RecordingsTable.tsx +++ b/src/components/molecules/RecordingsTable.tsx @@ -9,24 +9,22 @@ import TablePagination from '@mui/material/TablePagination'; import TableRow from '@mui/material/TableRow'; import { useEffect } from "react"; import { WorkflowFile } from "maxun-core"; -import { IconButton, Button, Box, Typography, TextField } from "@mui/material"; -import { Schedule, DeleteForever, Edit, PlayCircle, Settings, Power } from "@mui/icons-material"; -import LinkIcon from '@mui/icons-material/Link'; +import SearchIcon from '@mui/icons-material/Search'; +import { IconButton, Button, Box, Typography, TextField, MenuItem, Menu, ListItemIcon, ListItemText } from "@mui/material"; +import { Schedule, DeleteForever, Edit, PlayCircle, Settings, Power, ContentCopy, MoreHoriz } from "@mui/icons-material"; import { useGlobalInfoStore } from "../../context/globalInfo"; -import { deleteRecordingFromStorage, getStoredRecordings } from "../../api/storage"; +import { checkRunsForRecording, deleteRecordingFromStorage, getStoredRecordings } from "../../api/storage"; import { Add } from "@mui/icons-material"; import { useNavigate } from 'react-router-dom'; import { stopRecording } from "../../api/recording"; import { GenericModal } from '../atoms/GenericModal'; - /** TODO: * 1. allow editing existing robot after persisting browser steps - * 2. show robot settings: id, url, etc. */ interface Column { - id: 'interpret' | 'name' | 'delete' | 'schedule' | 'integrate' | 'settings'; + id: 'interpret' | 'name' | 'options' | 'schedule' | 'integrate' | 'settings'; label: string; minWidth?: number; align?: 'right'; @@ -36,17 +34,6 @@ interface Column { const columns: readonly Column[] = [ { id: 'interpret', label: 'Run', minWidth: 80 }, { id: 'name', label: 'Name', minWidth: 80 }, - // { - // id: 'createdAt', - // label: 'Created at', - // minWidth: 80, - // //format: (value: string) => value.toLocaleString('en-US'), - // }, - // { - // id: 'edit', - // label: 'Edit', - // minWidth: 80, - // }, { id: 'schedule', label: 'Schedule', @@ -57,20 +44,14 @@ const columns: readonly Column[] = [ label: 'Integrate', minWidth: 80, }, - // { - // id: 'updatedAt', - // label: 'Updated at', - // minWidth: 80, - // //format: (value: string) => value.toLocaleString('en-US'), - // }, { id: 'settings', label: 'Settings', minWidth: 80, }, { - id: 'delete', - label: 'Delete', + id: 'options', + label: 'Options', minWidth: 80, }, ]; @@ -90,15 +71,16 @@ interface RecordingsTableProps { handleScheduleRecording: (id: string, fileName: string, params: string[]) => void; handleIntegrateRecording: (id: string, fileName: string, params: string[]) => void; handleSettingsRecording: (id: string, fileName: string, params: string[]) => void; + handleEditRobot: (id: string, name: string, params: string[]) => void; + handleDuplicateRobot: (id: string, name: string, params: string[]) => void; } -export const RecordingsTable = ({ handleEditRecording, handleRunRecording, handleScheduleRecording, handleIntegrateRecording, handleSettingsRecording }: RecordingsTableProps) => { +export const RecordingsTable = ({ handleEditRecording, handleRunRecording, handleScheduleRecording, handleIntegrateRecording, handleSettingsRecording, handleEditRobot, handleDuplicateRobot }: RecordingsTableProps) => { const [page, setPage] = React.useState(0); const [rowsPerPage, setRowsPerPage] = React.useState(10); const [rows, setRows] = React.useState([]); const [isModalOpen, setModalOpen] = React.useState(false); - - console.log('rows', rows); + const [searchTerm, setSearchTerm] = React.useState(''); const { notify, setRecordings, browserId, setBrowserId, recordingUrl, setRecordingUrl, recordingName, setRecordingName, recordingId, setRecordingId } = useGlobalInfoStore(); const navigate = useNavigate(); @@ -112,6 +94,11 @@ export const RecordingsTable = ({ handleEditRecording, handleRunRecording, handl setPage(0); }; + const handleSearchChange = (event: React.ChangeEvent) => { + setSearchTerm(event.target.value); + setPage(0); + }; + const fetchRecordings = async () => { const recordings = await getStoredRecordings(); if (recordings) { @@ -150,7 +137,6 @@ export const RecordingsTable = ({ handleEditRecording, handleRunRecording, handl const startRecording = () => { setModalOpen(false); handleStartRecording(); - // notify('info', 'New Recording started for ' + recordingUrl); }; useEffect(() => { @@ -159,34 +145,54 @@ export const RecordingsTable = ({ handleEditRecording, handleRunRecording, handl } }, []); + + // Filter rows based on search term + const filteredRows = rows.filter((row) => + row.name.toLowerCase().includes(searchTerm.toLowerCase()) + ); + + + + return ( My Robots - - Create Robot - + + + }} + sx={{ width: '250px' }} + /> + + Create Robot + + @@ -204,7 +210,7 @@ export const RecordingsTable = ({ handleEditRecording, handleRunRecording, handl - {rows.length !== 0 ? rows + {filteredRows.length !== 0 ? filteredRows .slice(page * rowsPerPage, page * rowsPerPage + rowsPerPage) .map((row) => { return ( @@ -226,16 +232,6 @@ export const RecordingsTable = ({ handleEditRecording, handleRunRecording, handl handleRunRecording(row.id, row.name, row.params || [])} /> ); - // case 'edit': - // return ( - // - // { - // handleEditRecording(row.id, row.name); - // }} sx={{ '&:hover': { color: '#1976d2', backgroundColor: 'transparent' } }}> - // - // - // - // ); case 'schedule': return ( @@ -248,20 +244,31 @@ export const RecordingsTable = ({ handleEditRecording, handleRunRecording, handl handleIntegrateRecording(row.id, row.name, row.params || [])} /> ); - case 'delete': + case 'options': return ( - { - deleteRecordingFromStorage(row.id).then((result: boolean) => { - if (result) { - setRows([]); - notify('success', 'Recording deleted successfully'); - fetchRecordings(); - } - }) - }}> - - + handleEditRobot(row.id, row.name, row.params || [])} + handleDelete={() => { + + checkRunsForRecording(row.id).then((result: boolean) => { + if (result) { + notify('warning', 'Cannot delete recording as it has active runs'); + } + }) + + deleteRecordingFromStorage(row.id).then((result: boolean) => { + if (result) { + setRows([]); + notify('success', 'Recording deleted successfully'); + fetchRecordings(); + } + }) + }} + handleDuplicate={() => { + handleDuplicateRobot(row.id, row.name, row.params || []); + }} + /> ); case 'settings': @@ -285,7 +292,7 @@ export const RecordingsTable = ({ handleEditRecording, handleRunRecording, handl { ) } - interface ScheduleButtonProps { handleSchedule: () => void; } @@ -377,6 +383,60 @@ const SettingsButton = ({ handleSettings }: SettingsButtonProps) => { ) } +interface OptionsButtonProps { + handleEdit: () => void; + handleDelete: () => void; + handleDuplicate: () => void; +} + +const OptionsButton = ({ handleEdit, handleDelete, handleDuplicate }: OptionsButtonProps) => { + const [anchorEl, setAnchorEl] = React.useState(null); + + const handleClick = (event: React.MouseEvent) => { + setAnchorEl(event.currentTarget); + }; + + const handleClose = () => { + setAnchorEl(null); + }; + + return ( + <> + + + + + { handleEdit(); handleClose(); }}> + + + + Edit + + { handleDelete(); handleClose(); }}> + + + + Delete + + { handleDuplicate(); handleClose(); }}> + + + + Duplicate + + + + ); +}; + const modalStyle = { top: '50%', left: '50%', diff --git a/src/components/molecules/RobotDuplicate.tsx b/src/components/molecules/RobotDuplicate.tsx new file mode 100644 index 00000000..850614b0 --- /dev/null +++ b/src/components/molecules/RobotDuplicate.tsx @@ -0,0 +1,166 @@ +import React, { useState, useEffect } from 'react'; +import { GenericModal } from "../atoms/GenericModal"; +import { TextField, Typography, Box, Button, Chip } from "@mui/material"; +import { modalStyle } from "./AddWhereCondModal"; +import { useGlobalInfoStore } from '../../context/globalInfo'; +import { duplicateRecording, getStoredRecording } from '../../api/storage'; +import { WhereWhatPair } from 'maxun-core'; +import { getUserById } from "../../api/auth"; + +interface RobotMeta { + name: string; + id: string; + createdAt: string; + pairs: number; + updatedAt: string; + params: any[]; +} + +interface RobotWorkflow { + workflow: WhereWhatPair[]; +} + +interface ScheduleConfig { + runEvery: number; + runEveryUnit: 'MINUTES' | 'HOURS' | 'DAYS' | 'WEEKS' | 'MONTHS'; + startFrom: 'SUNDAY' | 'MONDAY' | 'TUESDAY' | 'WEDNESDAY' | 'THURSDAY' | 'FRIDAY' | 'SATURDAY'; + atTimeStart?: string; + atTimeEnd?: string; + timezone: string; + lastRunAt?: Date; + nextRunAt?: Date; + cronExpression?: string; +} + +export interface RobotSettings { + id: string; + userId?: number; + recording_meta: RobotMeta; + recording: RobotWorkflow; + google_sheet_email?: string | null; + google_sheet_name?: string | null; + google_sheet_id?: string | null; + google_access_token?: string | null; + google_refresh_token?: string | null; + schedule?: ScheduleConfig | null; +} + +interface RobotSettingsProps { + isOpen: boolean; + handleStart: (settings: RobotSettings) => void; + handleClose: () => void; + initialSettings?: RobotSettings | null; + +} + +export const RobotDuplicationModal = ({ isOpen, handleStart, handleClose, initialSettings }: RobotSettingsProps) => { + const [robot, setRobot] = useState(null); + const [targetUrl, setTargetUrl] = useState(''); + const { recordingId, notify } = useGlobalInfoStore(); + + useEffect(() => { + if (isOpen) { + getRobot(); + } + }, [isOpen]); + + useEffect(() => { + // Update the targetUrl when the robot data is loaded + if (robot) { + const lastPair = robot?.recording.workflow[robot?.recording.workflow.length - 1]; + const url = lastPair?.what.find(action => action.action === "goto")?.args?.[0]; + setTargetUrl(url); + } + }, [robot]); + + const getRobot = async () => { + if (recordingId) { + const robot = await getStoredRecording(recordingId); + setRobot(robot); + } else { + notify('error', 'Could not find robot details. Please try again.'); + } + } + + // const lastPair = robot?.recording.workflow[robot?.recording.workflow.length - 1]; + + // // Find the `goto` action in `what` and retrieve its arguments + // const targetUrl = lastPair?.what.find(action => action.action === "goto")?.args?.[0]; + + const handleTargetUrlChange = (e: React.ChangeEvent) => { + setTargetUrl(e.target.value); + }; + + const handleSave = async () => { + if (!robot || !targetUrl) { + notify('error', 'Target URL is required.'); + return; + } + + console.log("handle save"); + + try { + const success = await duplicateRecording(robot.recording_meta.id, targetUrl); + + if (success) { + notify('success', 'Robot duplicated successfully.'); + handleStart(robot); // Inform parent about the updated robot + handleClose(); + + setTimeout(() => { + window.location.reload(); + }, 1000); + } else { + notify('error', 'Failed to update the Target URL. Please try again.'); + } + } catch (error) { + notify('error', 'An error occurred while updating the Target URL.'); + console.error('Error updating Target URL:', error); + } + }; + + return ( + + <> + Duplicate Robot + + { + robot && ( + <> + Robot duplication is useful to extract data from pages with the same structure. +
+ + Example: If you've created a robot for producthunt.com/topics/api, you can duplicate it to scrape similar pages + like producthunt.com/topics/database without training a robot from scratch. + +
+ + ⚠️ Ensure the new page has the same structure as the original page. + + + + + + + + ) + } + + + + ); +}; diff --git a/src/components/molecules/RobotEdit.tsx b/src/components/molecules/RobotEdit.tsx new file mode 100644 index 00000000..74b50f62 --- /dev/null +++ b/src/components/molecules/RobotEdit.tsx @@ -0,0 +1,180 @@ +import React, { useState, useEffect } from 'react'; +import { GenericModal } from "../atoms/GenericModal"; +import { TextField, Typography, Box, Button } from "@mui/material"; +import { modalStyle } from "./AddWhereCondModal"; +import { useGlobalInfoStore } from '../../context/globalInfo'; +import { getStoredRecording, updateRecording } from '../../api/storage'; +import { WhereWhatPair } from 'maxun-core'; + +interface RobotMeta { + name: string; + id: string; + createdAt: string; + pairs: number; + updatedAt: string; + params: any[]; +} + +interface RobotWorkflow { + workflow: WhereWhatPair[]; +} + +interface RobotEditOptions { + name: string; + limit?: number; +} + +interface ScheduleConfig { + runEvery: number; + runEveryUnit: 'MINUTES' | 'HOURS' | 'DAYS' | 'WEEKS' | 'MONTHS'; + startFrom: 'SUNDAY' | 'MONDAY' | 'TUESDAY' | 'WEDNESDAY' | 'THURSDAY' | 'FRIDAY' | 'SATURDAY'; + atTimeStart?: string; + atTimeEnd?: string; + timezone: string; + lastRunAt?: Date; + nextRunAt?: Date; + cronExpression?: string; +} + +export interface RobotSettings { + id: string; + userId?: number; + recording_meta: RobotMeta; + recording: RobotWorkflow; + google_sheet_email?: string | null; + google_sheet_name?: string | null; + google_sheet_id?: string | null; + google_access_token?: string | null; + google_refresh_token?: string | null; + schedule?: ScheduleConfig | null; +} + +interface RobotSettingsProps { + isOpen: boolean; + handleStart: (settings: RobotSettings) => void; + handleClose: () => void; + initialSettings?: RobotSettings | null; + +} + +export const RobotEditModal = ({ isOpen, handleStart, handleClose, initialSettings }: RobotSettingsProps) => { + const [robot, setRobot] = useState(null); + const { recordingId, notify } = useGlobalInfoStore(); + + useEffect(() => { + if (isOpen) { + getRobot(); + } + }, [isOpen]); + + const getRobot = async () => { + if (recordingId) { + const robot = await getStoredRecording(recordingId); + setRobot(robot); + } else { + notify('error', 'Could not find robot details. Please try again.'); + } + } + + const handleRobotNameChange = (newName: string) => { + setRobot((prev) => + prev ? { ...prev, recording_meta: { ...prev.recording_meta, name: newName } } : prev + ); + }; + + const handleLimitChange = (newLimit: number) => { + setRobot((prev) => { + if (!prev) return prev; + + const updatedWorkflow = [...prev.recording.workflow]; + + if ( + updatedWorkflow.length > 0 && + updatedWorkflow[0]?.what && + updatedWorkflow[0].what.length > 0 && + updatedWorkflow[0].what[0].args && + updatedWorkflow[0].what[0].args.length > 0 && + updatedWorkflow[0].what[0].args[0] + ) { + updatedWorkflow[0].what[0].args[0].limit = newLimit; + } + + return { ...prev, recording: { ...prev.recording, workflow: updatedWorkflow } }; + }); + }; + const handleSave = async () => { + if (!robot) return; + + try { + const payload = { + name: robot.recording_meta.name, + limit: robot.recording.workflow[0]?.what[0]?.args?.[0]?.limit, + }; + + const success = await updateRecording(robot.recording_meta.id, payload); + + if (success) { + notify('success', 'Robot updated successfully.'); + handleStart(robot); // Inform parent about the updated robot + handleClose(); + + setTimeout(() => { + window.location.reload(); + }, 1000); + } else { + notify('error', 'Failed to update the robot. Please try again.'); + } + } catch (error) { + notify('error', 'An error occurred while updating the robot.'); + console.error('Error updating robot:', error); + } + }; + + return ( + + <> + Edit Robot + + { + robot && ( + <> + handleRobotNameChange(e.target.value)} + style={{ marginBottom: '20px' }} + /> + {robot.recording.workflow?.[0]?.what?.[0]?.args?.[0]?.limit !== undefined && ( + + handleLimitChange(parseInt(e.target.value, 10) || 0) + } + style={{ marginBottom: '20px' }} + /> + )} + + + + + + + ) + } + + + + ); +}; diff --git a/src/components/molecules/RobotSettings.tsx b/src/components/molecules/RobotSettings.tsx index 98638120..53d21d7b 100644 --- a/src/components/molecules/RobotSettings.tsx +++ b/src/components/molecules/RobotSettings.tsx @@ -104,6 +104,7 @@ export const RobotSettingsModal = ({ isOpen, handleStart, handleClose, initialSe <> + {robot.recording.workflow?.[0]?.what?.[0]?.args?.[0]?.limit !== undefined && ( + + )} { setTab(tab); - }, [interpretationInProgress]) + }, [interpretationInProgress]); useEffect(() => { if (row.serializableOutput && Object.keys(row.serializableOutput).length > 0) { const firstKey = Object.keys(row.serializableOutput)[0]; const data = row.serializableOutput[firstKey]; if (Array.isArray(data)) { - setTableData(data); - if (data.length > 0) { - setColumns(Object.keys(data[0])); + // Filter out completely empty rows + const filteredData = data.filter(row => + Object.values(row).some(value => value !== undefined && value !== "") + ); + setTableData(filteredData); + if (filteredData.length > 0) { + setColumns(Object.keys(filteredData[0])); } } } }, [row.serializableOutput]); + + // Function to convert table data to CSV format + const convertToCSV = (data: any[], columns: string[]): string => { + const header = columns.join(','); + const rows = data.map(row => + columns.map(col => JSON.stringify(row[col], null, 2)).join(',') + ); + return [header, ...rows].join('\n'); + }; + + const downloadCSV = () => { + const csvContent = convertToCSV(tableData, columns); + const blob = new Blob([csvContent], { type: 'text/csv;charset=utf-8;' }); + const url = URL.createObjectURL(blob); + + const link = document.createElement("a"); + link.href = url; + link.setAttribute("download", "data.csv"); + document.body.appendChild(link); + link.click(); + document.body.removeChild(link); + }; + return ( @@ -54,7 +78,6 @@ export const RunContent = ({ row, currentLog, interpretationInProgress, logEndRe setTab(newTab)} aria-label="run-content-tabs"> - {/* */} @@ -94,16 +117,19 @@ export const RunContent = ({ row, currentLog, interpretationInProgress, logEndRe Captured Data - {Object.keys(row.serializableOutput).map((key) => { - return ( - - ) - })} + + + + Download as JSON + + + + Download as CSV + + {tableData.length > 0 ? (
@@ -118,7 +144,9 @@ export const RunContent = ({ row, currentLog, interpretationInProgress, logEndRe {tableData.map((row, index) => ( {columns.map((column) => ( - {row[column]} + + {row[column] === undefined || row[column] === "" ? "-" : row[column]} + ))} ))} @@ -139,12 +167,12 @@ export const RunContent = ({ row, currentLog, interpretationInProgress, logEndRe )} } - {row.binaryOutput - && Object.keys(row.binaryOutput).length !== 0 && + {row.binaryOutput && Object.keys(row.binaryOutput).length !== 0 &&
- Captured Screenshot + Captured Screenshot + {Object.keys(row.binaryOutput).map((key) => { try { const imageUrl = row.binaryOutput[key]; @@ -152,10 +180,10 @@ export const RunContent = ({ row, currentLog, interpretationInProgress, logEndRe - + Download Screenshot - {key} + {key} ) } catch (e) { @@ -171,4 +199,4 @@ export const RunContent = ({ row, currentLog, interpretationInProgress, logEndRe ); -} +}; diff --git a/src/components/molecules/RunsTable.tsx b/src/components/molecules/RunsTable.tsx index b1cb97ef..669cecd6 100644 --- a/src/components/molecules/RunsTable.tsx +++ b/src/components/molecules/RunsTable.tsx @@ -12,8 +12,9 @@ import { useGlobalInfoStore } from "../../context/globalInfo"; import { getStoredRuns } from "../../api/storage"; import { RunSettings } from "./RunSettings"; import { CollapsibleRow } from "./ColapsibleRow"; -import { Accordion, AccordionSummary, AccordionDetails, Typography } from '@mui/material'; +import { Accordion, AccordionSummary, AccordionDetails, Typography, Box, TextField } from '@mui/material'; import ExpandMoreIcon from '@mui/icons-material/ExpandMore'; +import SearchIcon from '@mui/icons-material/Search'; interface Column { id: 'runStatus' | 'name' | 'startedAt' | 'finishedAt' | 'delete' | 'settings'; @@ -28,7 +29,6 @@ export const columns: readonly Column[] = [ { id: 'name', label: 'Robot Name', minWidth: 80 }, { id: 'startedAt', label: 'Started at', minWidth: 80 }, { id: 'finishedAt', label: 'Finished at', minWidth: 80 }, - // { id: 'task', label: 'Task', minWidth: 80 }, { id: 'settings', label: 'Settings', minWidth: 80 }, { id: 'delete', label: 'Delete', minWidth: 80 }, ]; @@ -42,9 +42,10 @@ export interface Data { runByUserId?: string; runByScheduleId?: string; runByAPI?: boolean; - // task: string; log: string; runId: string; + robotId: string; + robotMetaId: string; interpreterSettings: RunSettings; serializableOutput: any; binaryOutput: any; @@ -63,7 +64,8 @@ export const RunsTable = ( const [rowsPerPage, setRowsPerPage] = useState(10); const [rows, setRows] = useState([]); - console.log(`rows runs: ${JSON.stringify(rows)}`); + const [searchTerm, setSearchTerm] = useState(''); + const { notify, rerenderRuns, setRerenderRuns } = useGlobalInfoStore(); @@ -76,6 +78,11 @@ export const RunsTable = ( setPage(0); }; + const handleSearchChange = (event: React.ChangeEvent) => { + setSearchTerm(event.target.value); + setPage(0); + }; + const fetchRuns = async () => { const runs = await getStoredRuns(); if (runs) { @@ -105,25 +112,46 @@ export const RunsTable = ( fetchRuns(); }; - // Group runs by recording name - const groupedRows = rows.reduce((acc, row) => { - if (!acc[row.name]) { - acc[row.name] = []; + + // Filter rows based on search term + const filteredRows = rows.filter((row) => + row.name.toLowerCase().includes(searchTerm.toLowerCase()) + ); + + // Group filtered rows by robot meta id + const groupedRows = filteredRows.reduce((acc, row) => { + + if (!acc[row.robotMetaId]) { + acc[row.robotMetaId] = []; } - acc[row.name].push(row); + acc[row.robotMetaId].push(row); return acc; }, {} as Record); return ( - - All Runs - + + + All Runs + + + }} + sx={{ width: '250px' }} + /> + - {Object.entries(groupedRows).map(([name, group]) => ( - + {Object.entries(groupedRows).map(([id, data]) => ( + }> - {name} + + {data[data.length - 1].name} +
@@ -142,17 +170,19 @@ export const RunsTable = ( - {group.slice(page * rowsPerPage, page * rowsPerPage + rowsPerPage).map((row) => ( - - ))} + {data + .slice(page * rowsPerPage, page * rowsPerPage + rowsPerPage) + .map((row) => ( + + ))}
@@ -162,7 +192,7 @@ export const RunsTable = ( { const [copySuccess, setCopySuccess] = useState(false); const { notify } = useGlobalInfoStore(); + + + + useEffect(() => { const fetchApiKey = async () => { try { @@ -49,6 +53,7 @@ const ApiKeyManager = () => { }; fetchApiKey(); + }, []); const generateApiKey = async () => { @@ -56,6 +61,7 @@ const ApiKeyManager = () => { try { const { data } = await axios.post(`${apiUrl}/auth/generate-api-key`); setApiKey(data.api_key); + notify('success', `Generated API Key successfully`); } catch (error: any) { notify('error', `Failed to generate API Key - ${error.message}`); diff --git a/src/components/organisms/BrowserContent.tsx b/src/components/organisms/BrowserContent.tsx index 79fe6e63..11af4f2f 100644 --- a/src/components/organisms/BrowserContent.tsx +++ b/src/components/organisms/BrowserContent.tsx @@ -1,12 +1,16 @@ -import React, { useCallback, useEffect, useState } from 'react'; +import React, { useCallback, useEffect, useState } from "react"; import styled from "styled-components"; import BrowserNavBar from "../molecules/BrowserNavBar"; import { BrowserWindow } from "./BrowserWindow"; import { useBrowserDimensionsStore } from "../../context/browserDimensions"; import { BrowserTabs } from "../molecules/BrowserTabs"; import { useSocketStore } from "../../context/socket"; -import { getCurrentTabs, getCurrentUrl, interpretCurrentRecording } from "../../api/recording"; -import { Box } from '@mui/material'; +import { + getCurrentTabs, + getCurrentUrl, + interpretCurrentRecording, +} from "../../api/recording"; +import { Box } from "@mui/material"; import { InterpretationLog } from "../molecules/InterpretationLog"; // TODO: Tab !show currentUrl after recordingUrl global state @@ -14,107 +18,125 @@ export const BrowserContent = () => { const { width } = useBrowserDimensionsStore(); const { socket } = useSocketStore(); - const [tabs, setTabs] = useState(['current']); + const [tabs, setTabs] = useState(["current"]); const [tabIndex, setTabIndex] = React.useState(0); const [showOutputData, setShowOutputData] = useState(false); - const handleChangeIndex = useCallback((index: number) => { - setTabIndex(index); - }, [tabIndex]) + const handleChangeIndex = useCallback( + (index: number) => { + setTabIndex(index); + }, + [tabIndex] + ); - const handleCloseTab = useCallback((index: number) => { - // the tab needs to be closed on the backend - socket?.emit('closeTab', { - index, - isCurrent: tabIndex === index, - }); - // change the current index as current tab gets closed - if (tabIndex === index) { - if (tabs.length > index + 1) { - handleChangeIndex(index); + const handleCloseTab = useCallback( + (index: number) => { + // the tab needs to be closed on the backend + socket?.emit("closeTab", { + index, + isCurrent: tabIndex === index, + }); + // change the current index as current tab gets closed + if (tabIndex === index) { + if (tabs.length > index + 1) { + handleChangeIndex(index); + } else { + handleChangeIndex(index - 1); + } } else { - handleChangeIndex(index - 1); + handleChangeIndex(tabIndex - 1); } - } else { - handleChangeIndex(tabIndex - 1); - } - // update client tabs - setTabs((prevState) => [ - ...prevState.slice(0, index), - ...prevState.slice(index + 1) - ]) - }, [tabs, socket, tabIndex]); + // update client tabs + setTabs((prevState) => [ + ...prevState.slice(0, index), + ...prevState.slice(index + 1), + ]); + }, + [tabs, socket, tabIndex] + ); const handleAddNewTab = useCallback(() => { // Adds new tab by pressing the plus button - socket?.emit('addTab'); + socket?.emit("addTab"); // Adds a new tab to the end of the tabs array and shifts focus - setTabs((prevState) => [...prevState, 'new tab']); + setTabs((prevState) => [...prevState, "new tab"]); handleChangeIndex(tabs.length); }, [socket, tabs]); - const handleNewTab = useCallback((tab: string) => { - // Adds a new tab to the end of the tabs array and shifts focus - setTabs((prevState) => [...prevState, tab]); - // changes focus on the new tab - same happens in the remote browser - handleChangeIndex(tabs.length); - handleTabChange(tabs.length); - }, [tabs]); + const handleNewTab = useCallback( + (tab: string) => { + // Adds a new tab to the end of the tabs array and shifts focus + setTabs((prevState) => [...prevState, tab]); + // changes focus on the new tab - same happens in the remote browser + handleChangeIndex(tabs.length); + handleTabChange(tabs.length); + }, + [tabs] + ); - const handleTabChange = useCallback((index: number) => { - // page screencast and focus needs to be changed on backend - socket?.emit('changeTab', index); - }, [socket]); + const handleTabChange = useCallback( + (index: number) => { + // page screencast and focus needs to be changed on backend + socket?.emit("changeTab", index); + }, + [socket] + ); const handleUrlChanged = (url: string) => { const parsedUrl = new URL(url); if (parsedUrl.hostname) { - const host = parsedUrl.hostname.match(/\b(?!www\.)[a-zA-Z0-9]+/g)?.join('.') + const host = parsedUrl.hostname + .match(/\b(?!www\.)[a-zA-Z0-9]+/g) + ?.join("."); if (host && host !== tabs[tabIndex]) { setTabs((prevState) => [ ...prevState.slice(0, tabIndex), host, - ...prevState.slice(tabIndex + 1) - ]) + ...prevState.slice(tabIndex + 1), + ]); } } else { - if (tabs[tabIndex] !== 'new tab') { + if (tabs[tabIndex] !== "new tab") { setTabs((prevState) => [ ...prevState.slice(0, tabIndex), - 'new tab', - ...prevState.slice(tabIndex + 1) - ]) + "new tab", + ...prevState.slice(tabIndex + 1), + ]); } } - }; - const tabHasBeenClosedHandler = useCallback((index: number) => { - handleCloseTab(index); - }, [handleCloseTab]) + const tabHasBeenClosedHandler = useCallback( + (index: number) => { + handleCloseTab(index); + }, + [handleCloseTab] + ); useEffect(() => { if (socket) { - socket.on('newTab', handleNewTab); - socket.on('tabHasBeenClosed', tabHasBeenClosedHandler); + socket.on("newTab", handleNewTab); + socket.on("tabHasBeenClosed", tabHasBeenClosedHandler); } return () => { if (socket) { - socket.off('newTab', handleNewTab); - socket.off('tabHasBeenClosed', tabHasBeenClosedHandler); + socket.off("newTab", handleNewTab); + socket.off("tabHasBeenClosed", tabHasBeenClosedHandler); } - } - }, [socket, handleNewTab]) + }; + }, [socket, handleNewTab]); useEffect(() => { - getCurrentTabs().then((response) => { - if (response) { - setTabs(response); - } - }).catch((error) => { - console.log("Fetching current url failed"); - }) - }, []) + getCurrentTabs() + .then((response) => { + if (response) { + setTabs(response); + } + }) + .catch((error) => { + console.log("Fetching current url failed"); + }); + }, [handleUrlChanged]); return (
@@ -134,7 +156,6 @@ export const BrowserContent = () => {
); -} +}; -const BrowserContentWrapper = styled.div` -`; \ No newline at end of file +const BrowserContentWrapper = styled.div``; diff --git a/src/components/organisms/BrowserWindow.tsx b/src/components/organisms/BrowserWindow.tsx index cbc46731..697b4adb 100644 --- a/src/components/organisms/BrowserWindow.tsx +++ b/src/components/organisms/BrowserWindow.tsx @@ -8,6 +8,7 @@ import { useActionContext } from '../../context/browserActions'; import { useBrowserSteps, TextStep } from '../../context/browserSteps'; import { useGlobalInfoStore } from '../../context/globalInfo'; + interface ElementInfo { tagName: string; hasOnlyText?: boolean; @@ -316,7 +317,7 @@ export const BrowserWindow = () => { } }, [paginationMode, resetPaginationSelector]); - + return (
{ diff --git a/src/components/organisms/MainMenu.tsx b/src/components/organisms/MainMenu.tsx index 03ef3f49..b8f2aaa8 100644 --- a/src/components/organisms/MainMenu.tsx +++ b/src/components/organisms/MainMenu.tsx @@ -2,8 +2,13 @@ import React from 'react'; import Tabs from '@mui/material/Tabs'; import Tab from '@mui/material/Tab'; import Box from '@mui/material/Box'; + import { Paper, Button, useTheme } from "@mui/material"; -import { AutoAwesome, FormatListBulleted, VpnKey, Usb, Article, CloudQueue } from "@mui/icons-material"; +import { AutoAwesome, FormatListBulleted, VpnKey, Usb, Article, CloudQueue,Code, } from "@mui/icons-material"; + + +import { apiUrl } from "../../apiConfig"; + interface MainMenuProps { value: string; @@ -79,8 +84,10 @@ export const MainMenu = ({ value = 'recordings', handleChangeContent }: MainMenu
+ @@ -231,18 +231,34 @@ const ProxyForm: React.FC = () => { ))} {tabIndex === 1 && ( - + <> Coming Soon - In Open Source (Basic Rotation) & Cloud (Advanced Rotation). If you don't want to manage the infrastructure, join our cloud waitlist to get early access. )} + + If your proxy requires a username and password, always provide them separately from the proxy URL. +
+ The right way +
+ Proxy URL: http://proxy.com:1337 +
+ Username: myusername +
+ Password: mypassword +
+
+ The wrong way +
+ Proxy URL: http://myusername:mypassword@proxy.com:1337 + ); }; diff --git a/src/components/organisms/Recordings.tsx b/src/components/organisms/Recordings.tsx index 053a3479..495a25f8 100644 --- a/src/components/organisms/Recordings.tsx +++ b/src/components/organisms/Recordings.tsx @@ -5,6 +5,8 @@ import { RunSettings, RunSettingsModal } from "../molecules/RunSettings"; import { ScheduleSettings, ScheduleSettingsModal } from "../molecules/ScheduleSettings"; import { IntegrationSettings, IntegrationSettingsModal } from "../molecules/IntegrationSettings"; import { RobotSettings, RobotSettingsModal } from "../molecules/RobotSettings"; +import { RobotEditModal } from '../molecules/RobotEdit'; +import { RobotDuplicationModal } from '../molecules/RobotDuplicate'; interface RecordingsProps { handleEditRecording: (id: string, fileName: string) => void; @@ -18,10 +20,14 @@ export const Recordings = ({ handleEditRecording, handleRunRecording, setRecordi const [scheduleSettingsAreOpen, setScheduleSettingsAreOpen] = useState(false); const [integrateSettingsAreOpen, setIntegrateSettingsAreOpen] = useState(false); const [robotSettingsAreOpen, setRobotSettingsAreOpen] = useState(false); + const [robotEditAreOpen, setRobotEditAreOpen] = useState(false); + const [robotDuplicateAreOpen, setRobotDuplicateAreOpen] = useState(false); const [params, setParams] = useState([]); const [selectedRecordingId, setSelectedRecordingId] = useState(''); const handleIntegrateRecording = (id: string, settings: IntegrationSettings) => {}; const handleSettingsRecording = (id: string, settings: RobotSettings) => {}; + const handleEditRobot = (id: string, settings: RobotSettings) => {}; + const handleDuplicateRobot = (id: string, settings: RobotSettings) => {}; const handleSettingsAndIntegrate = (id: string, name: string, params: string[]) => { if (params.length === 0) { @@ -75,6 +81,32 @@ export const Recordings = ({ handleEditRecording, handleRunRecording, setRecordi } } + const handleEditRobotOption = (id: string, name: string, params: string[]) => { + if (params.length === 0) { + setRobotEditAreOpen(true); + setRecordingInfo(id, name); + setSelectedRecordingId(id); + } else { + setParams(params); + setRobotEditAreOpen(true); + setRecordingInfo(id, name); + setSelectedRecordingId(id); + } + } + + const handleDuplicateRobotOption = (id: string, name: string, params: string[]) => { + if (params.length === 0) { + setRobotDuplicateAreOpen(true); + setRecordingInfo(id, name); + setSelectedRecordingId(id); + } else { + setParams(params); + setRobotDuplicateAreOpen(true); + setRecordingInfo(id, name); + setSelectedRecordingId(id); + } + } + const handleClose = () => { setParams([]); setRunSettingsAreOpen(false); @@ -103,6 +135,20 @@ export const Recordings = ({ handleEditRecording, handleRunRecording, setRecordi setSelectedRecordingId(''); } + const handleRobotEditClose = () => { + setParams([]); + setRobotEditAreOpen(false); + setRecordingInfo('', ''); + setSelectedRecordingId(''); + } + + const handleRobotDuplicateClose = () => { + setParams([]); + setRobotDuplicateAreOpen(false); + setRecordingInfo('', ''); + setSelectedRecordingId(''); + } + return ( handleSettingsRecording(selectedRecordingId, settings)} /> + handleEditRobot(selectedRecordingId,settings)} + /> + handleDuplicateRobot(selectedRecordingId, settings)} + /> diff --git a/src/context/auth.tsx b/src/context/auth.tsx index 1b99686a..5c46e4de 100644 --- a/src/context/auth.tsx +++ b/src/context/auth.tsx @@ -90,4 +90,4 @@ const AuthProvider = ({ children }: AuthProviderProps) => { ); }; -export { AuthContext, AuthProvider }; \ No newline at end of file +export { AuthContext, AuthProvider }; diff --git a/src/index.css b/src/index.css index 0d151134..252b737a 100644 --- a/src/index.css +++ b/src/index.css @@ -29,7 +29,8 @@ a { code { font-family: source-code-pro, Menlo, Monaco, Consolas, 'Courier New', - monospace; + monospace; + color: #ff00c3; } #browser-actions { @@ -55,10 +56,9 @@ code { transform-origin: top left; /* Keep the position fixed */ } + #browser { - - } #browser-window { diff --git a/src/pages/Login.tsx b/src/pages/Login.tsx index 4e4a1013..87f90b53 100644 --- a/src/pages/Login.tsx +++ b/src/pages/Login.tsx @@ -1,119 +1,134 @@ -import axios from 'axios'; -import { useState, useContext, useEffect } from 'react'; -import { useNavigate, Link } from 'react-router-dom'; -import { AuthContext } from '../context/auth'; -import { - Box, - Typography, - TextField, - Button, - CircularProgress, -} from '@mui/material'; +import axios from "axios"; +import { useState, useContext, useEffect, FormEvent } from "react"; +import { useNavigate, Link } from "react-router-dom"; +import { AuthContext } from "../context/auth"; +import { Box, Typography, TextField, Button, CircularProgress, Grid } from "@mui/material"; import { useGlobalInfoStore } from "../context/globalInfo"; import { apiUrl } from "../apiConfig"; const Login = () => { - const [form, setForm] = useState({ - email: '', - password: '', - }); - const [loading, setLoading] = useState(false); - const { notify } = useGlobalInfoStore(); - const { email, password } = form; + const [form, setForm] = useState({ + email: "", + password: "", + }); + const [loading, setLoading] = useState(false); + const { notify } = useGlobalInfoStore(); + const { email, password } = form; - const { state, dispatch } = useContext(AuthContext); - const { user } = state; + const { state, dispatch } = useContext(AuthContext); + const { user } = state; - const navigate = useNavigate(); + const navigate = useNavigate(); - useEffect(() => { - if (user) { - navigate('/'); - } - }, [user, navigate]); + useEffect(() => { + if (user) { + navigate("/"); + } + }, [user, navigate]); - const handleChange = (e: any) => { - const { name, value } = e.target; - setForm({ ...form, [name]: value }); - }; + const handleChange = (e: any) => { + const { name, value } = e.target; + setForm({ ...form, [name]: value }); + }; - const submitForm = async (e: any) => { - e.preventDefault(); - setLoading(true); - try { - const { data } = await axios.post(`${apiUrl}/auth/login`, { email, password }); - dispatch({ type: 'LOGIN', payload: data }); - notify('success', 'Welcome to Maxun!'); - window.localStorage.setItem('user', JSON.stringify(data)); - navigate('/'); - } catch (err: any) { - notify('error', err.response.data || 'Login Failed. Please try again.'); - setLoading(false); - } - }; + const submitForm = async (e: any) => { + e.preventDefault(); + setLoading(true); + try { + const { data } = await axios.post(`${apiUrl}/auth/login`, { + email, + password, + }); + dispatch({ type: "LOGIN", payload: data }); + notify("success", "Welcome to Maxun!"); + window.localStorage.setItem("user", JSON.stringify(data)); + navigate("/"); + } catch (err) { + notify("error", "Login Failed. Please try again."); + setLoading(false); + } + }; - return ( + return ( + + - - Welcome Back! - - - - - - - - - Don’t have an account?{' '} - - Register - - - + logo + + Welcome Back! + + + + + + Don’t have an account?{" "} + + Register + + - ); + + + ); }; export default Login; diff --git a/src/pages/Register.tsx b/src/pages/Register.tsx index 8c92fe7b..b2a3eebf 100644 --- a/src/pages/Register.tsx +++ b/src/pages/Register.tsx @@ -1,117 +1,132 @@ -import { useState, useContext, useEffect } from 'react'; -import { useNavigate, Link } from 'react-router-dom'; -import axios from 'axios'; -import { AuthContext } from '../context/auth'; -import { TextField, Button, CircularProgress, Typography, Box, Container } from '@mui/material'; +import axios from "axios"; +import { useState, useContext, useEffect } from "react"; +import { useNavigate, Link } from "react-router-dom"; +import { AuthContext } from "../context/auth"; +import { Box, Typography, TextField, Button, CircularProgress } from "@mui/material"; import { useGlobalInfoStore } from "../context/globalInfo"; import { apiUrl } from "../apiConfig"; const Register = () => { - const [form, setForm] = useState({ - email: '', - password: '', - }); - const [loading, setLoading] = useState(false); - const { notify } = useGlobalInfoStore(); - const { email, password } = form; + const [form, setForm] = useState({ + email: "", + password: "", + }); + const [loading, setLoading] = useState(false); + const { notify } = useGlobalInfoStore(); + const { email, password } = form; - const { state, dispatch } = useContext(AuthContext); - const { user } = state; - const navigate = useNavigate(); + const { state, dispatch } = useContext(AuthContext); + const { user } = state; - useEffect(() => { - if (user !== null) navigate('/'); - }, [user, navigate]); + const navigate = useNavigate(); - const handleChange = (e: any) => { - const { name, value } = e.target; - setForm({ ...form, [name]: value }); - }; + useEffect(() => { + if (user) { + navigate("/"); + } + }, [user, navigate]); - const submitForm = async (e: any) => { - e.preventDefault(); - setLoading(true); - try { - const { data } = await axios.post(`${apiUrl}/auth/register`, { - email, - password, - }); - dispatch({ - type: 'LOGIN', - payload: data, - }); - notify('success', 'Welcome to Maxun!'); - window.localStorage.setItem('user', JSON.stringify(data)); - navigate('/'); - } catch (err: any) { - notify('error', err.response.data || 'Registration Failed. Please try again.'); - } finally { - setLoading(false); - } - }; + const handleChange = (e: any) => { + const { name, value } = e.target; + setForm({ ...form, [name]: value }); + }; - return ( - { + e.preventDefault(); + setLoading(true); + try { + const { data } = await axios.post(`${apiUrl}/auth/register`, { + email, + password, + }); + dispatch({ type: "LOGIN", payload: data }); + notify("success", "Registration Successful!"); + window.localStorage.setItem("user", JSON.stringify(data)); + navigate("/"); + } catch (error:any) { + notify("error", error.response.data || "Registration Failed. Please try again."); + setLoading(false); + } + }; + + return ( + + + logo + + Create an Account + + + + - - Already have an account?{' '} - - Login - - - - - ); + {loading ? ( + <> + + Loading + + ) : ( + "Register" + )} + + + Already have an account?{" "} + + Login + + + + + ); }; export default Register;